From 5cf0e1a6190d5651f45cef76fd729aa8a1a0a80e Mon Sep 17 00:00:00 2001 From: tangentstorm Date: Thu, 29 Oct 2026 05:40:40 +0000 Subject: [PATCH 1/6] Add bin/deploy, rollback, and deploy workflow for implish.org (#6) bin/deploy builds web/ with vite, backs up the live docroot to a timestamped copy (keeping the last DEPLOY_KEEP), and rsyncs dist-web/ over it. Supports --dry-run, --rollback [NAME] and --list-backups. Host/user/docroot come from deploy/deploy.conf or DEPLOY_* env vars. .github/workflows/deploy.yml tests on PRs and deploys on push to main or workflow_dispatch (optional ref), with a DEPLOY_SSH_KEY secret and pinned host keys in deploy/known_hosts. test/deploy.test.sh covers dry run, failed build, rollback, retention and path guards, locally and through a fake ssh. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/deploy.yml | 70 ++++++++++++ .gitignore | 1 + CLAUDE.md | 2 +- bin/deploy | 213 +++++++++++++++++++++++++++++++++++ deploy/README.md | 117 +++++++++++++++++++ deploy/deploy.conf | 11 ++ deploy/known_hosts | 3 + package.json | 4 +- test/deploy.test.sh | 182 ++++++++++++++++++++++++++++++ 9 files changed, 601 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/deploy.yml create mode 100755 bin/deploy create mode 100644 deploy/README.md create mode 100644 deploy/deploy.conf create mode 100644 deploy/known_hosts create mode 100755 test/deploy.test.sh diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..08cc91c --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,70 @@ +name: deploy implish.org + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + inputs: + ref: + description: Git ref (branch, tag or SHA) to deploy. Empty = the selected branch. + required: false + default: '' + +# One deploy at a time; a newer push waits rather than cancelling a deploy +# that is halfway through its rsync. +concurrency: + group: ${{ github.event_name == 'pull_request' && format('test-{0}', github.ref) || 'deploy-implish-org' }} + cancel-in-progress: false + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + - uses: actions/setup-node@v4 + with: + node-version: 22 + - run: npm install --no-audit --no-fund + - run: npm run build + - run: npm run test:deploy + + deploy: + needs: test + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + DEPLOY_USER: ${{ vars.DEPLOY_USER }} + DEPLOY_PATH: ${{ vars.DEPLOY_PATH }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + - uses: actions/setup-node@v4 + with: + node-version: 22 + - run: npm install --no-audit --no-fund + - name: Install deploy key + env: + DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }} + run: | + if [ -z "$DEPLOY_SSH_KEY" ]; then + echo "::error::secret DEPLOY_SSH_KEY is not set; see deploy/README.md" + exit 1 + fi + install -m 700 -d "$RUNNER_TEMP/ssh" + printf '%s\n' "$DEPLOY_SSH_KEY" > "$RUNNER_TEMP/ssh/id" + chmod 600 "$RUNNER_TEMP/ssh/id" + echo "DEPLOY_SSH=ssh -i $RUNNER_TEMP/ssh/id -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=$GITHUB_WORKSPACE/deploy/known_hosts" >> "$GITHUB_ENV" + - run: bin/deploy + - name: Check the live site + run: curl -fsS --retry 3 https://implish.org/ -o /dev/null + - name: Remove deploy key + if: always() + run: rm -rf "$RUNNER_TEMP/ssh" diff --git a/.gitignore b/.gitignore index 6259cae..40304f4 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ package-lock.json dist/ dist-web/ plan.org +.deploy-test/ diff --git a/CLAUDE.md b/CLAUDE.md index 9cddf73..31043dd 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -213,7 +213,7 @@ npm run preview The Vite build automatically runs `npm run build` (TypeScript compilation) first via the `ensureCoreBuild` plugin, then bundles everything for the browser. -**Deployment**: `dist-web/` is the live site served at implish.org. After running `npx vite build`, deploy the contents of `dist-web/` to the server. +**Deployment**: `dist-web/` is the live site served at implish.org (nginx on 45.79.174.182). Deploy only with `bin/deploy`; never copy files by hand. Pushes to main deploy automatically via `.github/workflows/deploy.yml`. Use `bin/deploy --dry-run` to preview and `bin/deploy --rollback` to restore the previous tree. Host, user and docroot live in `deploy/deploy.conf` and the `DEPLOY_*` repo variables. See `deploy/README.md`. Tests: `npm run test:deploy`. Vite 7 needs Node >= 20.19. ### TypeScript Details diff --git a/bin/deploy b/bin/deploy new file mode 100755 index 0000000..08d8c65 --- /dev/null +++ b/bin/deploy @@ -0,0 +1,213 @@ +#!/usr/bin/env bash +# bin/deploy: build web/ and publish dist-web/ to the implish.org docroot. +# +# bin/deploy [--dry-run] [--skip-build] build, back up live docroot, rsync +# bin/deploy --rollback [NAME] [--dry-run] restore newest (or NAME) backup +# bin/deploy --list-backups list backups, oldest first +# +# Configuration comes from deploy/deploy.conf, overridden by environment: +# DEPLOY_HOST ssh host, or "local" to treat DEPLOY_PATH as a local path +# DEPLOY_USER ssh user (optional; empty = ssh default / ~/.ssh/config) +# DEPLOY_PATH absolute path of the live docroot +# DEPLOY_BACKUP_DIR where timestamped backups live (default: DEPLOY_PATH.backups) +# DEPLOY_KEEP how many backups to keep (default 5) +# DEPLOY_SSH ssh command (default: ssh -o BatchMode=yes) +# DEPLOY_BUILD_CMD build command (default: npx vite build) +# DEPLOY_SRC build output to publish (default: dist-web) +# +# Exits nonzero on any failure. Nothing on the target is touched until the +# build has succeeded and produced an index.html. +set -euo pipefail + +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +CONF=${DEPLOY_CONF:-$ROOT/deploy/deploy.conf} + +die() { echo "deploy: $*" >&2; exit 1; } +log() { echo "deploy: $*" >&2; } + +# Non-empty environment values win over the config file. Empty ones are +# ignored, because CI maps unset repository variables to empty strings. +declare -A from_env=() +for v in DEPLOY_HOST DEPLOY_USER DEPLOY_PATH DEPLOY_BACKUP_DIR DEPLOY_KEEP \ + DEPLOY_SSH DEPLOY_BUILD_CMD DEPLOY_SRC; do + [[ -n ${!v:-} ]] && from_env[$v]=${!v} +done +if [[ -f $CONF ]]; then + # shellcheck source=/dev/null + source "$CONF" +fi +for v in "${!from_env[@]}"; do printf -v "$v" '%s' "${from_env[$v]}"; done + +DEPLOY_HOST=${DEPLOY_HOST:-} +DEPLOY_USER=${DEPLOY_USER:-} +DEPLOY_PATH=${DEPLOY_PATH:-} +DEPLOY_KEEP=${DEPLOY_KEEP:-5} +DEPLOY_SSH=${DEPLOY_SSH:-ssh -o BatchMode=yes} +DEPLOY_BUILD_CMD=${DEPLOY_BUILD_CMD:-npx vite build} +DEPLOY_SRC=${DEPLOY_SRC:-dist-web} + +mode=deploy dry_run=0 skip_build=0 rollback_name= +while (($#)); do + case $1 in + --dry-run|-n) dry_run=1 ;; + --skip-build) skip_build=1 ;; + --rollback) + mode=rollback + if [[ ${2:-} && ${2:0:1} != - ]]; then rollback_name=$2; shift; fi ;; + --list-backups) mode=list ;; + -h|--help) sed -n '2,/^set -e/{/^set -e/d;s/^# \{0,1\}//;p}' "$0"; exit 0 ;; + *) die "unknown argument: $1" ;; + esac + shift +done + +# --- validate config before doing anything ------------------------------- +[[ -n $DEPLOY_HOST ]] || die "DEPLOY_HOST is not set (see deploy/deploy.conf)" +[[ -n $DEPLOY_PATH ]] || die "DEPLOY_PATH is not set (see deploy/deploy.conf)" +DEPLOY_PATH=${DEPLOY_PATH%/} +DEPLOY_BACKUP_DIR=${DEPLOY_BACKUP_DIR:-$DEPLOY_PATH.backups} +DEPLOY_BACKUP_DIR=${DEPLOY_BACKUP_DIR%/} +check_path() { # name value: refuse paths that rsync --delete could wreck + local name=$1 p=$2 + [[ $p == /* ]] || die "$name must be absolute: '$p'" + [[ $p =~ ^/+[^/]+/+[^/]+ ]] || die "$name is too shallow to be safe: '$p'" + [[ /$p/ != */../* && /$p/ != */./* ]] || die "$name must not contain . or .. : '$p'" + [[ $p != *[[:space:]\'\"\\]* ]] || die "$name must not contain spaces or quotes: '$p'" +} +check_path DEPLOY_PATH "$DEPLOY_PATH" +check_path DEPLOY_BACKUP_DIR "$DEPLOY_BACKUP_DIR" +[[ $DEPLOY_BACKUP_DIR != "$DEPLOY_PATH" && $DEPLOY_BACKUP_DIR/ != "$DEPLOY_PATH"/* ]] \ + || die "DEPLOY_BACKUP_DIR must not be inside DEPLOY_PATH (rsync --delete would remove it)" +[[ $DEPLOY_KEEP =~ ^[1-9][0-9]*$ ]] || die "DEPLOY_KEEP must be a positive integer: '$DEPLOY_KEEP'" + +if [[ $DEPLOY_HOST == local ]]; then + target=$DEPLOY_PATH +else + target=${DEPLOY_USER:+$DEPLOY_USER@}$DEPLOY_HOST:$DEPLOY_PATH +fi + +# on_target SCRIPT ARGS...: run a bash snippet where the docroot lives. +on_target() { + local script=$1; shift + if [[ $DEPLOY_HOST == local ]]; then + bash -c "$script" on_target "$@" + else + local q; q=$(printf ' %q' "$@") + # shellcheck disable=SC2086 + $DEPLOY_SSH "${DEPLOY_USER:+$DEPLOY_USER@}$DEPLOY_HOST" "bash -s --$q" <<<"$script" + fi +} + +# rsync options shared by deploy and rollback. --checksum so CI checkouts +# (fresh mtimes) only list files whose content changed. /.well-known/ is +# protected from --delete because certbot keeps its challenges there. +RSYNC_OPTS=(-rlp --checksum --delete --delay-updates --itemize-changes + --chmod=Du=rwx,Dgo=rx,Fu=rw,Fgo=r --filter='P /.well-known/') +rsync_to() { # SRC DEST [extra rsync args...] + local src=$1 dest=$2; shift 2 + if [[ $DEPLOY_HOST == local ]]; then + rsync "${RSYNC_OPTS[@]}" "$@" "$src" "$dest" + else + rsync "${RSYNC_OPTS[@]}" -e "$DEPLOY_SSH" "$@" "$src" "$dest" + fi +} + +# Runs on the target. Each step takes an flock on the backup dir so backup, +# prune and drop never race each other; whole deploys are serialized by the +# workflow's concurrency group. Backup names are UTC timestamps, so name +# order = age order. +# shellcheck disable=SC2016 +REMOTE_LIB=' +set -euo pipefail +docroot=$1 backups=$2 keep=$3 action=$4 arg=${5:-} +mkdir -p "$backups" +exec 9>"$backups/.lock" +flock -w 60 9 || { echo "deploy: another deploy holds $backups/.lock" >&2; exit 1; } +list() { find "$backups" -mindepth 1 -maxdepth 1 -type d -name "20*Z" -printf "%f\n" | LC_ALL=C sort; } +case $action in + list) list ;; + backup) + if [[ ! -d $docroot ]] || [[ -z $(ls -A "$docroot") ]]; then + echo "deploy: $docroot is empty or missing; no backup needed" >&2; exit 0 + fi + while :; do + name=$(date -u +%Y%m%dT%H%M%SZ) + [[ -e $backups/$name ]] || break + sleep 1 + done + cp -a "$docroot" "$backups/$name.tmp" + mv "$backups/$name.tmp" "$backups/$name" + echo "deploy: backed up $docroot to $backups/$name" >&2 ;; + prune) + list | head -n -"$keep" | while read -r old; do + rm -rf -- "${backups:?}/$old"; echo "deploy: pruned backup $old" >&2 + done ;; + pick) # print the backup to roll back to + if [[ -n $arg ]]; then + [[ $arg =~ ^20[0-9]{6}T[0-9]{6}Z$ && -d $backups/$arg ]] \ + || { echo "deploy: no such backup: $arg" >&2; exit 1; } + name=$arg + else + name=$(list | tail -n 1) + [[ -n $name ]] || { echo "deploy: no backups in $backups" >&2; exit 1; } + fi + [[ -f $backups/$name/index.html ]] \ + || { echo "deploy: backup $name has no index.html; refusing to restore it" >&2; exit 1; } + echo "$name" ;; + drop) rm -rf -- "${backups:?}/$arg" ;; + mkroot) mkdir -p "$docroot" ;; +esac +' +remote() { on_target "$REMOTE_LIB" "$DEPLOY_PATH" "$DEPLOY_BACKUP_DIR" "$DEPLOY_KEEP" "$@"; } + +case $mode in + list) remote list; exit 0 ;; + + rollback) + name=$(remote pick "$rollback_name") + src=$DEPLOY_BACKUP_DIR/$name/ + log "rolling back $target to backup $name" + if ((dry_run)); then + # rsync can't read a remote source into a remote dest; run it there. + on_target 'rsync -rl --checksum --delete --dry-run --itemize-changes --filter="P /.well-known/" "$1" "$2/"' \ + "$src" "$DEPLOY_PATH" + log "dry run: nothing changed" + exit 0 + fi + on_target 'rsync -rlp --checksum --delete --delay-updates --filter="P /.well-known/" "$1" "$2/"' \ + "$src" "$DEPLOY_PATH" + # The restored tree is now live, so the backup is redundant. Dropping it + # makes a second --rollback step one deploy further back. + if [[ -z $rollback_name ]]; then remote drop "$name"; fi + log "rolled back to $name" + exit 0 ;; +esac + +# --- deploy --------------------------------------------------------------- +cd "$ROOT" +if ((skip_build)); then + log "skipping build (--skip-build)" +else + log "building: $DEPLOY_BUILD_CMD" + bash -c "$DEPLOY_BUILD_CMD" || die "build failed; nothing was deployed" +fi +[[ -f $DEPLOY_SRC/index.html ]] || die "$DEPLOY_SRC/index.html missing after build; nothing was deployed" + +if ((dry_run)); then + log "dry run: changes that would be made to $target" + exists=$(on_target '[[ -d $1 ]] && echo yes || echo no' "$DEPLOY_PATH") + if [[ $exists == no ]]; then + log "(docroot does not exist yet; every file would be new)" + (cd "$DEPLOY_SRC" && find . -type f -printf '>f+++++++++ %P\n' | LC_ALL=C sort) + else + rsync_to "$DEPLOY_SRC/" "$target/" --dry-run + fi + log "dry run: nothing changed" + exit 0 +fi + +remote backup +remote mkroot +rsync_to "$DEPLOY_SRC/" "$target/" +remote prune +log "deployed $(git -C "$ROOT" rev-parse --short HEAD 2>/dev/null || echo '?') to $target" diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 0000000..cdf0178 --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,117 @@ +# Deploying implish.org + +implish.org is static: `npx vite build` turns `web/` into `dist-web/`, and +nginx on 45.79.174.182 (Ubuntu) serves those files from a docroot. + +`bin/deploy` is the only deploy path. CI runs it, and so can a human: + +```bash +bin/deploy --dry-run # build, then list what would change; touches nothing +bin/deploy # build, back up live docroot, rsync dist-web/ over it +bin/deploy --list-backups # timestamped backups, oldest first +bin/deploy --rollback # restore the newest backup (repeat to step further back) +bin/deploy --rollback 20261029T120000Z # restore a specific backup +bin/deploy --rollback --dry-run # show what a rollback would change +``` + +What a deploy does, in order (any failure exits nonzero and stops): + +1. Validates config. `DEPLOY_PATH` and `DEPLOY_BACKUP_DIR` must be absolute, + at least two levels deep, free of `.`/`..`/spaces, and the backup dir + must not be inside the docroot. +2. Builds. If the build fails or leaves no `dist-web/index.html`, it stops + here and nothing on the server has been touched. +3. Copies the live docroot to `DEPLOY_BACKUP_DIR/`. It skips + this on the first deploy, when the docroot is missing or empty. +4. Runs rsync `dist-web/` → docroot with `--delete --checksum --delay-updates`. + Files come out 644 and dirs 755. `/.well-known/` is never deleted, because + certbot uses it. +5. Prunes the backups down to the newest `DEPLOY_KEEP` (default 5). + +Rollback rsyncs a backup back over the docroot and then removes that backup, +because the same tree is now live. The next `--rollback` therefore goes one +deploy further back. A named rollback keeps its backup. + +## Configuration + +Values come from `deploy/deploy.conf`. Any **non-empty** environment variable +of the same name overrides them. See `bin/deploy --help` for the full list. + +| variable | value | +|---|---| +| `DEPLOY_HOST` | `45.79.174.182` (set in deploy.conf) | +| `DEPLOY_USER` | **unknown: fill in** (repo variable or deploy.conf) | +| `DEPLOY_PATH` | **unknown: fill in** the nginx `root` for implish.org | +| `DEPLOY_BACKUP_DIR` | default `$DEPLOY_PATH.backups` | +| `DEPLOY_KEEP` | 5 | + +The user and docroot are unknown because the machine that wrote this had no +ssh access to the server. To find the docroot on the server, run: +`grep -R 'root\|server_name' /etc/nginx/sites-enabled/`. + +`deploy/known_hosts` pins the server's host keys, so CI never trusts a key on +first use. They were collected with `ssh-keyscan` on 2026-10-29: + +``` +SHA256:crs05VrOGRUK0xAZw8uwpPmmlboM2rdq4MyaqrUbqRM (ED25519) +SHA256:5ztPwBwMsF+/xXMgJhFKhtE941Dq6nktzI6WAbxiF/M (ECDSA) +SHA256:kKeq5mlSsrOY4BA+TZnUxLuk5+kIkSnTLGqVqyPikYQ (RSA) +``` + +Check them on the server with `ssh-keygen -lf /etc/ssh/ssh_host_ed25519_key.pub` +(and the same for the other key types). + +## One-time setup (Michal) + +The server needs `rsync` and `flock`, which stock Ubuntu has. The deploy user +needs write access to the docroot and to its parent directory, or to wherever +`DEPLOY_BACKUP_DIR` points. Steps 2 and 3 need root. + +1. Generate a dedicated key on your machine. Don't commit it: + ```bash + ssh-keygen -t ed25519 -N '' -C implish-deploy@github-actions -f implish-deploy + ``` +2. On the server, give the deploy user ownership of the docroot and backup + dir. The docroot below is an example; use the real `root` from nginx: + ```bash + sudo mkdir -p /var/www/implish.org.backups + sudo chown -R DEPLOYUSER: /var/www/implish.org /var/www/implish.org.backups + ``` +3. Append one line to `~DEPLOYUSER/.ssh/authorized_keys` on the server. + `restrict` turns off pty, agent, port and X11 forwarding. The key can + still run the `bash` and `rsync` commands that the deploy needs: + ``` + restrict ssh-ed25519 AAAA...contents-of-implish-deploy.pub... implish-deploy@github-actions + ``` +4. Add the secret and variables to the repo: + ```bash + gh secret set DEPLOY_SSH_KEY --repo tangentcode/implish < implish-deploy + gh variable set DEPLOY_USER --repo tangentcode/implish --body DEPLOYUSER + gh variable set DEPLOY_PATH --repo tangentcode/implish --body /var/www/implish.org + ``` + You can commit `DEPLOY_USER`/`DEPLOY_PATH` to `deploy/deploy.conf` + instead of setting the variables. Then delete the local private key. +5. Check it: Actions → "deploy implish.org" → Run workflow. You can also + run `bin/deploy --dry-run` locally with the key loaded. + +## CI + +`.github/workflows/deploy.yml`: + +- **pull_request**: `npm run build` + `npm run test:deploy`. It never deploys. +- **push to main**: runs the tests, then `bin/deploy`, then checks + `https://implish.org/` with curl. +- **workflow_dispatch**: same as push. The optional `ref` input deploys any + branch, tag or SHA, so a PR can go live on request. Roll back by running + `bin/deploy --rollback` locally, or by dispatching an older ref. + +Deploys run one at a time (concurrency group `deploy-implish-org`, no +cancellation). Until `DEPLOY_SSH_KEY` is set, the deploy job fails with a +message that points here. + +## Tests + +`npm run test:deploy` runs `test/deploy.test.sh`. It tests against throwaway +docroots in `.deploy-test/`, both with `DEPLOY_HOST=local` and through a +fake `ssh` that runs the remote half locally, so the ssh/rsync `-e` path is +exercised too. diff --git a/deploy/deploy.conf b/deploy/deploy.conf new file mode 100644 index 0000000..b968b38 --- /dev/null +++ b/deploy/deploy.conf @@ -0,0 +1,11 @@ +# Config for bin/deploy. Environment variables of the same name override these +# (the GitHub workflow sets them from repository variables). +# +# DEPLOY_USER and DEPLOY_PATH are not known yet: the box that wrote this +# script had no ssh access to the server. Fill them in here, or set the +# repository variables of the same name (see deploy/README.md). +DEPLOY_HOST=45.79.174.182 +DEPLOY_USER= +DEPLOY_PATH= +# DEPLOY_BACKUP_DIR defaults to "$DEPLOY_PATH.backups" (sibling of the docroot) +DEPLOY_KEEP=5 diff --git a/deploy/known_hosts b/deploy/known_hosts new file mode 100644 index 0000000..caa52ff --- /dev/null +++ b/deploy/known_hosts @@ -0,0 +1,3 @@ +45.79.174.182 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMzMyCJaJrE+vZdkRGDjfUNO9ehVbvhNTR/sgu3SHuiF6AXbxxFfFrYoIcfcueVgYFiPCQ54N5EW9gWGyVOA5oY= +45.79.174.182 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIHUyCe+QSaI9dqWgcl4ZjbRnvRNw5YmI9/j7o5Ds6u9 +45.79.174.182 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDA0P/eHMuHI9CHJvBw2mbg8bUpL7A2YDUNjoCgeV/vx0rIjpQ4LM09S6MSH4RNJh7VNFA1EHKYg2jMVaDTq5J7Q10x//puQ8omzyX3Ny4jvu1h/7h8ZHsVSgYa9T5hfxofpSEfJtb3EIQZ2SpRbspOlQzggkM4cmTYuJOMn3FChP9NpJv70x0JAbqNyGAkzGrLBL9Pv8oKlrt6gPmHjHwFtBTPWk5U3d03J8ZU7a6EQ9ADsbaztIpc+Axl78d8V0stj99yr1RmUgY7CmQwuU2rELZxgmwtpP3RqYkkmeGTBI2ioyoPNwAC+2rOe/HjIZ09wr44GdTVQD1roPfGKKgr diff --git a/package.json b/package.json index afd00e5..0cd71ca 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,9 @@ "test": "echo \"Error: no test specified\" && exit 1", "dev": "vite", "preview": "vite preview", - "repl": "node dist/imp-cmd.mjs" + "repl": "node dist/imp-cmd.mjs", + "test:deploy": "bash test/deploy.test.sh", + "deploy": "bin/deploy" }, "repository": { "type": "git", diff --git a/test/deploy.test.sh b/test/deploy.test.sh new file mode 100755 index 0000000..d59f9b8 --- /dev/null +++ b/test/deploy.test.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash +# Tests for bin/deploy against throwaway docroots under .deploy-test/. +# Every case runs twice: DEPLOY_HOST=local, and through a fake ssh that runs +# the remote commands locally (exercising the ssh + rsync -e code path). +set -uo pipefail + +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +DEPLOY=$ROOT/bin/deploy +WORK=$ROOT/.deploy-test +rm -rf "$WORK"; mkdir -p "$WORK" +trap 'rm -rf "$WORK"' EXIT + +pass=0 total=0 failed=() +ok() { :; } +fail() { echo " FAIL: $*" >&2; return 1; } + +# fake ssh: skip options, drop the host, run the command string locally. +cat > "$WORK/fake-ssh" <<'EOF' +#!/usr/bin/env bash +while [[ $1 == -* ]]; do case $1 in -[ilopFJ]) shift 2 ;; *) shift ;; esac; done +shift +exec bash -c "$*" +EOF +chmod +x "$WORK/fake-ssh" + +# setup: fresh sandbox with a live docroot holding the "old" site +setup() { + T=$WORK/case$total + rm -rf "$T"; mkdir -p "$T/srv/www/site/.well-known/acme" "$T/src" + echo old > "$T/srv/www/site/index.html" + echo stale > "$T/srv/www/site/stale.js" + echo token > "$T/srv/www/site/.well-known/acme/tok" + export DEPLOY_CONF=/dev/null DEPLOY_PATH=$T/srv/www/site DEPLOY_KEEP=3 + export DEPLOY_SRC=$T/out + unset DEPLOY_BACKUP_DIR + # the "build" writes a new site whose content is $T/src/version + echo new > "$T/src/version" + export DEPLOY_BUILD_CMD="rm -rf '$T/out' && mkdir -p '$T/out/assets' && cp '$T/src/version' '$T/out/index.html' && echo js > '$T/out/assets/app.js'" +} +backups() { ls "$DEPLOY_PATH.backups" 2>/dev/null | grep -v '^\.' || true; } +tree() { (cd "$1" && find . -type f -printf '%P\t' -exec cat {} \; | LC_ALL=C sort); } + +t_dry_run_lists_files() { + out=$("$DEPLOY" --dry-run 2>&1) || fail "exit $?: $out" || return 1 + grep -q 'index.html' <<<"$out" || fail "index.html not listed: $out" || return 1 + grep -q 'assets/app.js' <<<"$out" || fail "assets/app.js not listed" || return 1 + grep -Eq 'deleting +stale.js' <<<"$out" || fail "stale.js deletion not listed" || return 1 + ! grep -q 'well-known' <<<"$out" || fail ".well-known would be touched" || return 1 + [[ $(cat "$DEPLOY_PATH/index.html") == old ]] || fail "dry run changed docroot" || return 1 + [[ -z $(backups) ]] || fail "dry run made a backup" || return 1 +} + +t_dry_run_new_docroot() { + rm -rf "$DEPLOY_PATH" + out=$("$DEPLOY" --dry-run 2>&1) || fail "exit $?: $out" || return 1 + grep -q 'assets/app.js' <<<"$out" || fail "app.js not listed: $out" || return 1 + [[ ! -e $DEPLOY_PATH ]] || fail "dry run created docroot" || return 1 +} + +t_failed_build_aborts() { + DEPLOY_BUILD_CMD='echo boom >&2; exit 3' out=$("$DEPLOY" 2>&1) && fail "exit 0" && return 1 + grep -q 'build failed' <<<"$out" || fail "no build-failed message: $out" || return 1 + [[ $(cat "$DEPLOY_PATH/index.html") == old && -f $DEPLOY_PATH/stale.js ]] || fail "docroot changed" || return 1 + [[ -z $(backups) ]] || fail "backup made" || return 1 +} + +t_empty_build_aborts() { + DEPLOY_BUILD_CMD="mkdir -p '$T/out'" out=$("$DEPLOY" 2>&1) && fail "exit 0" && return 1 + grep -q 'index.html missing' <<<"$out" || fail "$out" || return 1 + [[ -f $DEPLOY_PATH/stale.js && -z $(backups) ]] || fail "target touched" || return 1 +} + +t_deploy_replaces_tree() { + "$DEPLOY" >/dev/null 2>&1 || fail "deploy failed" || return 1 + [[ $(cat "$DEPLOY_PATH/index.html") == new ]] || fail "index not updated" || return 1 + [[ -f $DEPLOY_PATH/assets/app.js ]] || fail "app.js missing" || return 1 + [[ ! -e $DEPLOY_PATH/stale.js ]] || fail "stale.js not deleted" || return 1 + [[ -f $DEPLOY_PATH/.well-known/acme/tok ]] || fail ".well-known deleted" || return 1 + [[ $(stat -c %a "$DEPLOY_PATH/index.html") == 644 ]] || fail "index.html not world-readable" || return 1 + [[ $(backups | wc -l) == 1 ]] || fail "expected 1 backup, got: $(backups)" || return 1 + b=$DEPLOY_PATH.backups/$(backups) + [[ $(cat "$b/index.html") == old && -f $b/stale.js ]] || fail "backup is not the old tree" || return 1 +} + +t_first_deploy_no_backup() { + rm -rf "$DEPLOY_PATH" + "$DEPLOY" >/dev/null 2>&1 || fail "deploy failed" || return 1 + [[ $(cat "$DEPLOY_PATH/index.html") == new ]] || fail "not deployed" || return 1 + [[ -z $(backups) ]] || fail "backed up an empty docroot" || return 1 +} + +t_rollback_restores_previous() { + before=$(tree "$DEPLOY_PATH") + "$DEPLOY" >/dev/null 2>&1 || fail "deploy failed" || return 1 + out=$("$DEPLOY" --rollback --dry-run 2>&1) || fail "rollback dry run: $out" || return 1 + [[ $(cat "$DEPLOY_PATH/index.html") == new ]] || fail "rollback dry run changed docroot" || return 1 + "$DEPLOY" --rollback >/dev/null 2>&1 || fail "rollback failed" || return 1 + [[ $(tree "$DEPLOY_PATH") == "$before" ]] || fail "tree differs after rollback: +$(diff <(echo "$before") <(tree "$DEPLOY_PATH"))" || return 1 + [[ -z $(backups) ]] || fail "used backup not dropped" || return 1 +} + +t_rollback_steps_back() { + for v in v1 v2 v3; do + echo "$v" > "$T/src/version"; "$DEPLOY" >/dev/null 2>&1 || fail "deploy $v" || return 1 + done + "$DEPLOY" --rollback >/dev/null 2>&1 && [[ $(cat "$DEPLOY_PATH/index.html") == v2 ]] || fail "1st rollback" || return 1 + "$DEPLOY" --rollback >/dev/null 2>&1 && [[ $(cat "$DEPLOY_PATH/index.html") == v1 ]] || fail "2nd rollback" || return 1 +} + +t_rollback_named() { + "$DEPLOY" >/dev/null 2>&1 || fail "deploy" || return 1 + name=$(backups) + echo v2 > "$T/src/version"; "$DEPLOY" >/dev/null 2>&1 || fail "deploy 2" || return 1 + "$DEPLOY" --rollback "$name" >/dev/null 2>&1 || fail "named rollback" || return 1 + [[ $(cat "$DEPLOY_PATH/index.html") == old ]] || fail "wrong tree" || return 1 + out=$("$DEPLOY" --rollback 20000101T000000Z 2>&1) && fail "bogus name accepted" && return 1 + out=$("$DEPLOY" --rollback '../../etc' 2>&1) && fail "traversal name accepted" && return 1 + return 0 +} + +t_rollback_without_backups_fails() { + out=$("$DEPLOY" --rollback 2>&1) && fail "exit 0" && return 1 + grep -q 'no backups' <<<"$out" || fail "$out" || return 1 + [[ $(cat "$DEPLOY_PATH/index.html") == old ]] || fail "docroot changed" || return 1 +} + +t_retention_keeps_last_n() { + for v in v1 v2 v3 v4 v5; do + echo "$v" > "$T/src/version"; "$DEPLOY" >/dev/null 2>&1 || fail "deploy $v" || return 1 + done + mapfile -t bs < <(backups) + [[ ${#bs[@]} == 3 ]] || fail "expected 3 backups, got ${#bs[@]}: ${bs[*]}" || return 1 + # newest three hold v4, v3, v2 (the tree before deploys v5, v4, v3) + got=$(for b in "${bs[@]}"; do cat "$DEPLOY_PATH.backups/$b/index.html"; done | tr '\n' ' ') + [[ $got == "v2 v3 v4 " ]] || fail "kept wrong backups: $got" || return 1 + [[ $("$DEPLOY" --list-backups 2>/dev/null | wc -l) == 3 ]] || fail "--list-backups" || return 1 +} + +t_rejects_unsafe_paths() { + for p in '' relative/path / /var '/srv/www/../etc' '/srv/my site'; do + DEPLOY_PATH=$p "$DEPLOY" --dry-run >/dev/null 2>&1 && fail "accepted DEPLOY_PATH='$p'" && return 1 + done + DEPLOY_BACKUP_DIR=$DEPLOY_PATH/bak "$DEPLOY" --dry-run >/dev/null 2>&1 \ + && fail "accepted backup dir inside docroot" && return 1 + DEPLOY_BACKUP_DIR=/tmp "$DEPLOY" --dry-run >/dev/null 2>&1 \ + && fail "accepted shallow backup dir" && return 1 + DEPLOY_KEEP=0 "$DEPLOY" --dry-run >/dev/null 2>&1 && fail "accepted DEPLOY_KEEP=0" && return 1 + "$DEPLOY" --bogus >/dev/null 2>&1 && fail "accepted --bogus" && return 1 + [[ $(cat "$DEPLOY_PATH/index.html") == old ]] || fail "docroot changed" || return 1 +} + +t_unreachable_host_fails() { + DEPLOY_HOST=remote DEPLOY_SSH=false out=$("$DEPLOY" 2>&1) && fail "exit 0" && return 1 + [[ $(cat "$DEPLOY_PATH/index.html") == old ]] || fail "docroot changed" || return 1 +} + +t_conf_file_and_env_override() { + printf 'DEPLOY_HOST=local\nDEPLOY_PATH=%s\nDEPLOY_KEEP=7\n' "$T/elsewhere/site" > "$T/conf" + out=$(unset DEPLOY_PATH; DEPLOY_CONF=$T/conf "$DEPLOY" --dry-run 2>&1) || fail "$out" || return 1 + grep -q "$T/elsewhere/site" <<<"$out" || fail "conf not used: $out" || return 1 + out=$(DEPLOY_CONF=$T/conf DEPLOY_PATH=$DEPLOY_PATH "$DEPLOY" --dry-run 2>&1) || fail "$out" || return 1 + grep -Eq 'deleting +stale.js' <<<"$out" || fail "env did not override conf: $out" || return 1 + # empty env values (unset CI variables) must not clobber the conf file + out=$(DEPLOY_CONF=$T/conf DEPLOY_PATH= DEPLOY_KEEP= DEPLOY_USER= "$DEPLOY" --dry-run 2>&1) || fail "$out" || return 1 + grep -q "$T/elsewhere/site" <<<"$out" || fail "empty env clobbered conf: $out" || return 1 +} + +cases=$(declare -F | awk '{print $3}' | grep '^t_') +for mode in local ssh; do + for c in $cases; do + total=$((total+1)) + setup + if [[ $mode == local ]]; then export DEPLOY_HOST=local; unset DEPLOY_SSH DEPLOY_USER + else export DEPLOY_HOST=testhost DEPLOY_USER=tester DEPLOY_SSH="$WORK/fake-ssh -o BatchMode=yes"; fi + if ( $c ); then pass=$((pass+1)); echo "ok $mode $c" + else failed+=("$mode $c"); echo "FAIL $mode $c"; fi + done +done + +echo "deploy tests: $pass/$total passed" +((pass == total)) From f21cc168bac06adaedfead1e85900f9e86ea0564 Mon Sep 17 00:00:00 2001 From: tangentstorm Date: Thu, 29 Oct 2026 05:42:30 +0000 Subject: [PATCH 2/6] deploy workflow: use checkout/setup-node v5 (Node 24 runtime) Co-Authored-By: Claude Opus 5.5 --- .github/workflows/deploy.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 08cc91c..46d8bce 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -24,10 +24,10 @@ jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: ref: ${{ inputs.ref || github.sha }} - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v5 with: node-version: 22 - run: npm install --no-audit --no-fund @@ -43,10 +43,10 @@ jobs: DEPLOY_USER: ${{ vars.DEPLOY_USER }} DEPLOY_PATH: ${{ vars.DEPLOY_PATH }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: ref: ${{ inputs.ref || github.sha }} - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v5 with: node-version: 22 - run: npm install --no-audit --no-fund From e6297d892a7cf3373892bee6de881b0ecb16cdca Mon Sep 17 00:00:00 2001 From: tangentstorm Date: Thu, 29 Oct 2026 14:39:34 +0000 Subject: [PATCH 3/6] deploy: production environment, merged-only refs, whole-run lock - Deploy job runs in the `production` environment (deployment branches limited to main) and only on refs/heads/main. workflow_dispatch `ref` must already be merged into main (deploy/select-ref.sh). - Move the server-side half to deploy/remote.sh. Every rm -rf goes through a strict backup-name check. A failed backup copy removes its .tmp and aborts before rsync, and stale .tmp dirs are cleaned. - Hold the backup-dir flock for a whole deploy or rollback through a held ssh session, instead of locking per step. - Tests: fix the `VAR=x out=$(cmd)` idiom (VAR was never exported to cmd), report mode-specific skips, and add cases for all of the above. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/deploy.yml | 21 ++++++-- CLAUDE.md | 2 +- bin/deploy | 88 ++++++++++++------------------ deploy/README.md | 63 +++++++++++++++++----- deploy/remote.sh | 77 +++++++++++++++++++++++++++ deploy/select-ref.sh | 16 ++++++ test/deploy.test.sh | 100 ++++++++++++++++++++++++++++++++--- 7 files changed, 288 insertions(+), 79 deletions(-) create mode 100644 deploy/remote.sh create mode 100755 deploy/select-ref.sh diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 46d8bce..c4284ea 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -7,7 +7,7 @@ on: workflow_dispatch: inputs: ref: - description: Git ref (branch, tag or SHA) to deploy. Empty = the selected branch. + description: Commit, tag or branch to (re)deploy. It must already be merged into main. Empty = main's head. required: false default: '' @@ -34,18 +34,33 @@ jobs: - run: npm run build - run: npm run test:deploy + # The deploy key lives only in the `production` environment, whose + # deployment branches are limited to main (see deploy/README.md). So a PR + # branch that edits this file still can't read the secret. The `if:` and + # the ancestor check below are defense in depth on top of that. deploy: needs: test - if: github.event_name != 'pull_request' + if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' runs-on: ubuntu-latest timeout-minutes: 15 + environment: + name: production + url: https://implish.org/ env: DEPLOY_USER: ${{ vars.DEPLOY_USER }} DEPLOY_PATH: ${{ vars.DEPLOY_PATH }} steps: - uses: actions/checkout@v5 with: - ref: ${{ inputs.ref || github.sha }} + ref: ${{ github.sha }} + fetch-depth: 0 + - name: Select commit (only commits already merged into main) + env: + REF: ${{ inputs.ref }} + run: | + sha=$(deploy/select-ref.sh "$REF" "$GITHUB_SHA") + git checkout --detach "$sha" + echo "deploying ${REF:-main} = $sha" - uses: actions/setup-node@v5 with: node-version: 22 diff --git a/CLAUDE.md b/CLAUDE.md index 31043dd..b9a0246 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -213,7 +213,7 @@ npm run preview The Vite build automatically runs `npm run build` (TypeScript compilation) first via the `ensureCoreBuild` plugin, then bundles everything for the browser. -**Deployment**: `dist-web/` is the live site served at implish.org (nginx on 45.79.174.182). Deploy only with `bin/deploy`; never copy files by hand. Pushes to main deploy automatically via `.github/workflows/deploy.yml`. Use `bin/deploy --dry-run` to preview and `bin/deploy --rollback` to restore the previous tree. Host, user and docroot live in `deploy/deploy.conf` and the `DEPLOY_*` repo variables. See `deploy/README.md`. Tests: `npm run test:deploy`. Vite 7 needs Node >= 20.19. +**Deployment**: `dist-web/` is the live site served at implish.org (nginx on 45.79.174.182). Deploy only with `bin/deploy`; never copy files by hand. Pushes to main deploy automatically via `.github/workflows/deploy.yml`. Use `bin/deploy --dry-run` to preview and `bin/deploy --rollback` to restore the previous tree. Host, user and docroot live in `deploy/deploy.conf` and the `production` environment's `DEPLOY_*` variables; the deploy key is an environment secret released only to main. Server-side steps are in `deploy/remote.sh`. See `deploy/README.md`. Tests: `npm run test:deploy`. Vite 7 needs Node >= 20.19. ### TypeScript Details diff --git a/bin/deploy b/bin/deploy index 08d8c65..1b65d18 100755 --- a/bin/deploy +++ b/bin/deploy @@ -13,6 +13,7 @@ # DEPLOY_KEEP how many backups to keep (default 5) # DEPLOY_SSH ssh command (default: ssh -o BatchMode=yes) # DEPLOY_BUILD_CMD build command (default: npx vite build) +# DEPLOY_LOCK_WAIT seconds to wait for another deploy's lock (default 60) # DEPLOY_SRC build output to publish (default: dist-web) # # Exits nonzero on any failure. Nothing on the target is touched until the @@ -29,7 +30,7 @@ log() { echo "deploy: $*" >&2; } # ignored, because CI maps unset repository variables to empty strings. declare -A from_env=() for v in DEPLOY_HOST DEPLOY_USER DEPLOY_PATH DEPLOY_BACKUP_DIR DEPLOY_KEEP \ - DEPLOY_SSH DEPLOY_BUILD_CMD DEPLOY_SRC; do + DEPLOY_SSH DEPLOY_BUILD_CMD DEPLOY_SRC DEPLOY_LOCK_WAIT; do [[ -n ${!v:-} ]] && from_env[$v]=${!v} done if [[ -f $CONF ]]; then @@ -79,6 +80,8 @@ check_path DEPLOY_BACKUP_DIR "$DEPLOY_BACKUP_DIR" [[ $DEPLOY_BACKUP_DIR != "$DEPLOY_PATH" && $DEPLOY_BACKUP_DIR/ != "$DEPLOY_PATH"/* ]] \ || die "DEPLOY_BACKUP_DIR must not be inside DEPLOY_PATH (rsync --delete would remove it)" [[ $DEPLOY_KEEP =~ ^[1-9][0-9]*$ ]] || die "DEPLOY_KEEP must be a positive integer: '$DEPLOY_KEEP'" +DEPLOY_LOCK_WAIT=${DEPLOY_LOCK_WAIT:-60} +[[ $DEPLOY_LOCK_WAIT =~ ^[0-9]+$ ]] || die "DEPLOY_LOCK_WAIT must be a whole number of seconds: '$DEPLOY_LOCK_WAIT'" if [[ $DEPLOY_HOST == local ]]; then target=$DEPLOY_PATH @@ -112,70 +115,45 @@ rsync_to() { # SRC DEST [extra rsync args...] fi } -# Runs on the target. Each step takes an flock on the backup dir so backup, -# prune and drop never race each other; whole deploys are serialized by the -# workflow's concurrency group. Backup names are UTC timestamps, so name -# order = age order. -# shellcheck disable=SC2016 -REMOTE_LIB=' -set -euo pipefail -docroot=$1 backups=$2 keep=$3 action=$4 arg=${5:-} -mkdir -p "$backups" -exec 9>"$backups/.lock" -flock -w 60 9 || { echo "deploy: another deploy holds $backups/.lock" >&2; exit 1; } -list() { find "$backups" -mindepth 1 -maxdepth 1 -type d -name "20*Z" -printf "%f\n" | LC_ALL=C sort; } -case $action in - list) list ;; - backup) - if [[ ! -d $docroot ]] || [[ -z $(ls -A "$docroot") ]]; then - echo "deploy: $docroot is empty or missing; no backup needed" >&2; exit 0 - fi - while :; do - name=$(date -u +%Y%m%dT%H%M%SZ) - [[ -e $backups/$name ]] || break - sleep 1 - done - cp -a "$docroot" "$backups/$name.tmp" - mv "$backups/$name.tmp" "$backups/$name" - echo "deploy: backed up $docroot to $backups/$name" >&2 ;; - prune) - list | head -n -"$keep" | while read -r old; do - rm -rf -- "${backups:?}/$old"; echo "deploy: pruned backup $old" >&2 - done ;; - pick) # print the backup to roll back to - if [[ -n $arg ]]; then - [[ $arg =~ ^20[0-9]{6}T[0-9]{6}Z$ && -d $backups/$arg ]] \ - || { echo "deploy: no such backup: $arg" >&2; exit 1; } - name=$arg - else - name=$(list | tail -n 1) - [[ -n $name ]] || { echo "deploy: no backups in $backups" >&2; exit 1; } - fi - [[ -f $backups/$name/index.html ]] \ - || { echo "deploy: backup $name has no index.html; refusing to restore it" >&2; exit 1; } - echo "$name" ;; - drop) rm -rf -- "${backups:?}/$arg" ;; - mkroot) mkdir -p "$docroot" ;; -esac -' +# The target-side half lives in deploy/remote.sh and is piped to bash. +REMOTE_LIB=$(<"$ROOT/deploy/remote.sh") remote() { on_target "$REMOTE_LIB" "$DEPLOY_PATH" "$DEPLOY_BACKUP_DIR" "$DEPLOY_KEEP" "$@"; } +# hold_lock: take $DEPLOY_BACKUP_DIR/.lock on the target and keep it until +# this script exits, so a manual --rollback can't interleave with a CI deploy. +# A coprocess keeps an ssh session (or a local bash) open, which runs flock +# and then waits on stdin. When we exit, or the connection drops, its stdin +# closes and the lock is released. The script is passed as a bash -c string +# because stdin is the keepalive pipe. It must not contain single quotes. +# shellcheck disable=SC2016 +LOCK_SCRIPT='mkdir -p "$1" && exec 9>>"$1/.lock" && flock -w "$2" 9 && echo locked && exec cat >/dev/null' +hold_lock() { + local q; q=$(printf ' %q' "$DEPLOY_BACKUP_DIR" "$DEPLOY_LOCK_WAIT") + if [[ $DEPLOY_HOST == local ]]; then + coproc LOCK { bash -c "$LOCK_SCRIPT" hold_lock "$DEPLOY_BACKUP_DIR" "$DEPLOY_LOCK_WAIT"; } + else + # shellcheck disable=SC2086 + coproc LOCK { $DEPLOY_SSH "${DEPLOY_USER:+$DEPLOY_USER@}$DEPLOY_HOST" "bash -c '$LOCK_SCRIPT' hold_lock$q"; } + fi + local line= + read -r line <&"${LOCK[0]}" || true + [[ $line == locked ]] || die "could not take $DEPLOY_BACKUP_DIR/.lock (another deploy or rollback running?); nothing changed" + log "holding $DEPLOY_BACKUP_DIR/.lock" +} + case $mode in list) remote list; exit 0 ;; rollback) + ((dry_run)) || hold_lock name=$(remote pick "$rollback_name") - src=$DEPLOY_BACKUP_DIR/$name/ log "rolling back $target to backup $name" if ((dry_run)); then - # rsync can't read a remote source into a remote dest; run it there. - on_target 'rsync -rl --checksum --delete --dry-run --itemize-changes --filter="P /.well-known/" "$1" "$2/"' \ - "$src" "$DEPLOY_PATH" + remote restore "$name" dry log "dry run: nothing changed" exit 0 fi - on_target 'rsync -rlp --checksum --delete --delay-updates --filter="P /.well-known/" "$1" "$2/"' \ - "$src" "$DEPLOY_PATH" + remote restore "$name" # The restored tree is now live, so the backup is redundant. Dropping it # makes a second --rollback step one deploy further back. if [[ -z $rollback_name ]]; then remote drop "$name"; fi @@ -195,8 +173,7 @@ fi if ((dry_run)); then log "dry run: changes that would be made to $target" - exists=$(on_target '[[ -d $1 ]] && echo yes || echo no' "$DEPLOY_PATH") - if [[ $exists == no ]]; then + if [[ $(remote exists) == no ]]; then log "(docroot does not exist yet; every file would be new)" (cd "$DEPLOY_SRC" && find . -type f -printf '>f+++++++++ %P\n' | LC_ALL=C sort) else @@ -206,6 +183,7 @@ if ((dry_run)); then exit 0 fi +hold_lock remote backup remote mkroot rsync_to "$DEPLOY_SRC/" "$target/" diff --git a/deploy/README.md b/deploy/README.md index cdf0178..0b70217 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -21,12 +21,25 @@ What a deploy does, in order (any failure exits nonzero and stops): must not be inside the docroot. 2. Builds. If the build fails or leaves no `dist-web/index.html`, it stops here and nothing on the server has been touched. -3. Copies the live docroot to `DEPLOY_BACKUP_DIR/`. It skips - this on the first deploy, when the docroot is missing or empty. -4. Runs rsync `dist-web/` → docroot with `--delete --checksum --delay-updates`. +3. Takes `DEPLOY_BACKUP_DIR/.lock` on the server and holds it until the + deploy ends, waiting up to `DEPLOY_LOCK_WAIT` seconds (default 60). A + manual `--rollback` takes the same lock, so it can't interleave with a CI + deploy. A held ssh session keeps the lock; if the script dies or the + connection drops, the lock is released. +4. Copies the live docroot to `DEPLOY_BACKUP_DIR/`, via a + `.tmp` copy that is renamed when complete. It skips this on the first + deploy, when the docroot is missing or empty. If the copy fails, the + partial `.tmp` is removed and the deploy stops before rsync. Stale `.tmp` + dirs from a killed run are removed on the next backup or prune. +5. Runs rsync `dist-web/` → docroot with `--delete --checksum --delay-updates`. Files come out 644 and dirs 755. `/.well-known/` is never deleted, because certbot uses it. -5. Prunes the backups down to the newest `DEPLOY_KEEP` (default 5). +6. Prunes the backups down to the newest `DEPLOY_KEEP` (default 5). + +The server-side half (backup, prune, restore, drop) is `deploy/remote.sh`, +which `bin/deploy` pipes to `bash` over ssh. Every `rm -rf` there checks +that the name is exactly a backup timestamp (`^20[0-9]{6}T[0-9]{6}Z$`), so +an empty or malformed name can't reach the backup dir itself. Rollback rsyncs a backup back over the docroot and then removes that backup, because the same tree is now live. The next `--rollback` therefore goes one @@ -83,14 +96,25 @@ needs write access to the docroot and to its parent directory, or to wherever ``` restrict ssh-ed25519 AAAA...contents-of-implish-deploy.pub... implish-deploy@github-actions ``` -4. Add the secret and variables to the repo: +4. Create the `production` environment and limit it to main. Then store the + key and variables **in that environment, not as repository secrets**. + Branch PRs can read repository secrets, so a PR that edited the workflow + could deploy. An environment secret is only released to jobs running on + main. + - Settings → Environments → New environment → `production`. + - Deployment branches and tags → **Selected branches and tags** → add + the rule `main`. (Optional: tick "Allow administrators to bypass" off.) ```bash - gh secret set DEPLOY_SSH_KEY --repo tangentcode/implish < implish-deploy - gh variable set DEPLOY_USER --repo tangentcode/implish --body DEPLOYUSER - gh variable set DEPLOY_PATH --repo tangentcode/implish --body /var/www/implish.org + gh secret set DEPLOY_SSH_KEY --repo tangentcode/implish --env production < implish-deploy + gh variable set DEPLOY_USER --repo tangentcode/implish --env production --body DEPLOYUSER + gh variable set DEPLOY_PATH --repo tangentcode/implish --env production --body /var/www/implish.org ``` + If a repo-level `DEPLOY_SSH_KEY` secret already exists, delete it: + `gh secret delete DEPLOY_SSH_KEY --repo tangentcode/implish`. You can commit `DEPLOY_USER`/`DEPLOY_PATH` to `deploy/deploy.conf` instead of setting the variables. Then delete the local private key. + Also protect `main` (no direct pushes; PRs merged by you), since + anything that lands on main gets deployed. 5. Check it: Actions → "deploy implish.org" → Run workflow. You can also run `bin/deploy --dry-run` locally with the key loaded. @@ -99,11 +123,24 @@ needs write access to the docroot and to its parent directory, or to wherever `.github/workflows/deploy.yml`: - **pull_request**: `npm run build` + `npm run test:deploy`. It never deploys. -- **push to main**: runs the tests, then `bin/deploy`, then checks - `https://implish.org/` with curl. -- **workflow_dispatch**: same as push. The optional `ref` input deploys any - branch, tag or SHA, so a PR can go live on request. Roll back by running - `bin/deploy --rollback` locally, or by dispatching an older ref. +- **push to main**: runs the tests, then `bin/deploy` in the `production` + environment, then checks `https://implish.org/` with curl. +- **workflow_dispatch** (main only): same as push. The optional `ref` + input (SHA, tag or branch) must already be merged into main; + `deploy/select-ref.sh` checks this with `git merge-base --is-ancestor`. + So the input can redeploy or roll back to an older merged commit, but it + can't put an unmerged PR live. To get a PR live, merge it. Roll back by + dispatching an older merged SHA, or with `bin/deploy --rollback` from a + machine that has the key. + +Three layers keep the key away from unmerged code: + +1. The `production` environment only releases `DEPLOY_SSH_KEY` to jobs on + main. This is the real guarantee: a PR branch can rewrite the workflow + but still can't read the secret. +2. The deploy job has `if: github.ref == 'refs/heads/main'` and never runs + on `pull_request`. +3. `select-ref.sh` rejects refs that aren't merged into main. Deploys run one at a time (concurrency group `deploy-implish-org`, no cancellation). Until `DEPLOY_SSH_KEY` is set, the deploy job fails with a diff --git a/deploy/remote.sh b/deploy/remote.sh new file mode 100644 index 0000000..80a5cb1 --- /dev/null +++ b/deploy/remote.sh @@ -0,0 +1,77 @@ +# deploy/remote.sh: the half of bin/deploy that runs where the docroot lives. +# bin/deploy pipes this file to `bash -s` (over ssh, or locally), with args: +# +# remote.sh DOCROOT BACKUPS KEEP ACTION [ARG] +# +# The caller holds $BACKUPS/.lock for the whole deploy or rollback (see +# hold_lock in bin/deploy), so the actions here don't lock on their own. +# Backup names are UTC timestamps (20261029T120000Z), so name order is age +# order. Every rm -rf goes through valid_name, so a bad or empty name can +# never reach the backup dir itself. +set -euo pipefail +docroot=$1 backups=$2 keep=$3 action=$4 arg=${5:-} + +die() { echo "deploy: $*" >&2; exit 1; } +valid_name() { [[ $1 =~ ^20[0-9]{6}T[0-9]{6}Z$ ]]; } +rm_backup() { # NAME [SUFFIX]: remove one backup (or its .tmp) by validated name + valid_name "$1" || die "refusing to remove invalid backup name: '$1'" + rm -rf -- "${backups:?}/$1${2:-}" +} +list() { + [[ -d $backups ]] || return 0 + find "$backups" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' \ + | grep -E '^20[0-9]{6}T[0-9]{6}Z$' | LC_ALL=C sort || true +} +clean_tmp() { # leftovers of a backup whose cp failed partway + [[ -d $backups ]] || return 0 + find "$backups" -mindepth 1 -maxdepth 1 -type d -name '20*Z.tmp' -printf '%f\n' \ + | while read -r t; do + rm_backup "${t%.tmp}" .tmp; echo "deploy: removed stale partial backup $t" >&2 + done +} + +case $action in + list) list ;; + exists) [[ -d $docroot ]] && echo yes || echo no ;; + backup) + mkdir -p "$backups" + clean_tmp + if [[ ! -d $docroot ]] || [[ -z $(ls -A "$docroot") ]]; then + echo "deploy: $docroot is empty or missing; no backup needed" >&2; exit 0 + fi + while :; do + name=$(date -u +%Y%m%dT%H%M%SZ) + [[ -e $backups/$name || -e $backups/$name.tmp ]] || break + sleep 1 + done + cp -a "$docroot" "$backups/$name.tmp" || { rm_backup "$name" .tmp; die "backup of $docroot failed; nothing was deployed"; } + mv "$backups/$name.tmp" "$backups/$name" + echo "deploy: backed up $docroot to $backups/$name" >&2 ;; + prune) + clean_tmp + list | head -n -"$keep" | while read -r old; do + rm_backup "$old"; echo "deploy: pruned backup $old" >&2 + done ;; + pick) # print the backup to roll back to: ARG, or the newest + if [[ -n $arg ]]; then + valid_name "$arg" && [[ -d $backups/$arg ]] || die "no such backup: $arg" + name=$arg + else + name=$(list | tail -n 1) + [[ -n $name ]] || die "no backups in $backups" + fi + [[ -f $backups/$name/index.html ]] || die "backup $name has no index.html; refusing to restore it" + echo "$name" ;; + restore) # ARG [dry]: rsync a backup over the docroot + valid_name "$arg" && [[ -d $backups/$arg ]] || die "no such backup: $arg" + if [[ ${6:-} == dry ]]; then + rsync -rl --checksum --delete --dry-run --itemize-changes --filter='P /.well-known/' \ + "$backups/$arg/" "$docroot/" + else + rsync -rlp --checksum --delete --delay-updates --filter='P /.well-known/' \ + "$backups/$arg/" "$docroot/" + fi ;; + drop) rm_backup "$arg" ;; + mkroot) mkdir -p "$docroot" ;; + *) die "unknown remote action: $action" ;; +esac diff --git a/deploy/select-ref.sh b/deploy/select-ref.sh new file mode 100755 index 0000000..3680d01 --- /dev/null +++ b/deploy/select-ref.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# deploy/select-ref.sh REF BASE: print the commit REF names, but only if it is +# already merged into BASE (main's head). workflow_dispatch uses it so the +# `ref` input can redeploy or roll back to merged history, never to an +# unmerged branch or PR head. Empty REF prints BASE. Exits nonzero otherwise. +set -euo pipefail +ref=${1:-} base=${2:?usage: select-ref.sh REF BASE} +err() { echo "::error::$*" >&2; exit 1; } +base=$(git rev-parse --verify --quiet --end-of-options "$base^{commit}") || err "unknown base: $2" +if [[ -z $ref ]]; then echo "$base"; exit 0; fi +[[ $ref != -* ]] || err "bad ref: $ref" +sha=$(git rev-parse --verify --quiet --end-of-options "origin/$ref^{commit}" || + git rev-parse --verify --quiet --end-of-options "$ref^{commit}") || err "unknown ref: $ref" +git merge-base --is-ancestor "$sha" "$base" || + err "$ref ($sha) is not merged into main; only merged commits can be deployed" +echo "$sha" diff --git a/test/deploy.test.sh b/test/deploy.test.sh index d59f9b8..267d0f1 100755 --- a/test/deploy.test.sh +++ b/test/deploy.test.sh @@ -10,7 +10,7 @@ WORK=$ROOT/.deploy-test rm -rf "$WORK"; mkdir -p "$WORK" trap 'rm -rf "$WORK"' EXIT -pass=0 total=0 failed=() +pass=0 total=0 skipped=0 failed=() ok() { :; } fail() { echo " FAIL: $*" >&2; return 1; } @@ -37,7 +37,7 @@ setup() { echo new > "$T/src/version" export DEPLOY_BUILD_CMD="rm -rf '$T/out' && mkdir -p '$T/out/assets' && cp '$T/src/version' '$T/out/index.html' && echo js > '$T/out/assets/app.js'" } -backups() { ls "$DEPLOY_PATH.backups" 2>/dev/null | grep -v '^\.' || true; } +backups() { ls "$DEPLOY_PATH.backups" 2>/dev/null | grep -E '^20[0-9]{6}T[0-9]{6}Z$' || true; } tree() { (cd "$1" && find . -type f -printf '%P\t' -exec cat {} \; | LC_ALL=C sort); } t_dry_run_lists_files() { @@ -58,14 +58,14 @@ t_dry_run_new_docroot() { } t_failed_build_aborts() { - DEPLOY_BUILD_CMD='echo boom >&2; exit 3' out=$("$DEPLOY" 2>&1) && fail "exit 0" && return 1 + out=$(DEPLOY_BUILD_CMD='echo boom >&2; exit 3' "$DEPLOY" 2>&1) && fail "exit 0" && return 1 grep -q 'build failed' <<<"$out" || fail "no build-failed message: $out" || return 1 [[ $(cat "$DEPLOY_PATH/index.html") == old && -f $DEPLOY_PATH/stale.js ]] || fail "docroot changed" || return 1 [[ -z $(backups) ]] || fail "backup made" || return 1 } t_empty_build_aborts() { - DEPLOY_BUILD_CMD="mkdir -p '$T/out'" out=$("$DEPLOY" 2>&1) && fail "exit 0" && return 1 + out=$(DEPLOY_BUILD_CMD="mkdir -p '$T/out'" "$DEPLOY" 2>&1) && fail "exit 0" && return 1 grep -q 'index.html missing' <<<"$out" || fail "$out" || return 1 [[ -f $DEPLOY_PATH/stale.js && -z $(backups) ]] || fail "target touched" || return 1 } @@ -146,12 +146,13 @@ t_rejects_unsafe_paths() { DEPLOY_BACKUP_DIR=/tmp "$DEPLOY" --dry-run >/dev/null 2>&1 \ && fail "accepted shallow backup dir" && return 1 DEPLOY_KEEP=0 "$DEPLOY" --dry-run >/dev/null 2>&1 && fail "accepted DEPLOY_KEEP=0" && return 1 + DEPLOY_LOCK_WAIT=soon "$DEPLOY" --dry-run >/dev/null 2>&1 && fail "accepted DEPLOY_LOCK_WAIT=soon" && return 1 "$DEPLOY" --bogus >/dev/null 2>&1 && fail "accepted --bogus" && return 1 [[ $(cat "$DEPLOY_PATH/index.html") == old ]] || fail "docroot changed" || return 1 } t_unreachable_host_fails() { - DEPLOY_HOST=remote DEPLOY_SSH=false out=$("$DEPLOY" 2>&1) && fail "exit 0" && return 1 + out=$(DEPLOY_HOST=remote DEPLOY_SSH=false "$DEPLOY" 2>&1) && fail "exit 0" && return 1 [[ $(cat "$DEPLOY_PATH/index.html") == old ]] || fail "docroot changed" || return 1 } @@ -166,6 +167,89 @@ t_conf_file_and_env_override() { grep -q "$T/elsewhere/site" <<<"$out" || fail "empty env clobbered conf: $out" || return 1 } +t_stale_tmp_backup_removed() { + mkdir -p "$DEPLOY_PATH.backups/20200101T000000Z.tmp/junk" + "$DEPLOY" >/dev/null 2>&1 || fail "deploy failed" || return 1 + [[ ! -e $DEPLOY_PATH.backups/20200101T000000Z.tmp ]] || fail "stale .tmp left behind" || return 1 + [[ $(backups | wc -l) == 1 ]] || fail "expected 1 backup, got: $(backups)" || return 1 +} + +t_failed_backup_aborts_and_cleans_up() { + echo secret > "$DEPLOY_PATH/unreadable"; chmod 000 "$DEPLOY_PATH/unreadable" + out=$("$DEPLOY" 2>&1) && { chmod 644 "$DEPLOY_PATH/unreadable"; fail "exit 0"; return 1; } + chmod 644 "$DEPLOY_PATH/unreadable" + grep -q 'backup of .* failed' <<<"$out" || fail "no backup-failed message: $out" || return 1 + [[ $(cat "$DEPLOY_PATH/index.html") == old && -f $DEPLOY_PATH/stale.js ]] || fail "docroot changed" || return 1 + [[ -z $(ls -A "$DEPLOY_PATH.backups" | grep -v '^\.lock$') ]] || fail "leftovers: $(ls -A "$DEPLOY_PATH.backups")" || return 1 + "$DEPLOY" >/dev/null 2>&1 || fail "retry after fixing perms failed" || return 1 + [[ $(backups | wc -l) == 1 ]] || fail "retry: expected 1 backup" || return 1 +} + +t_remote_rm_rejects_bad_names() { + "$DEPLOY" >/dev/null 2>&1 || fail "deploy" || return 1 + b=$DEPLOY_PATH.backups; keep=$(backups) + for bad in '' . .. '*' '../site' '20261029T120000Z/..' 'x' '20261029T120000Z.tmp'; do + bash "$ROOT/deploy/remote.sh" "$DEPLOY_PATH" "$b" 3 drop "$bad" >/dev/null 2>&1 \ + && fail "drop accepted '$bad'" && return 1 + bash "$ROOT/deploy/remote.sh" "$DEPLOY_PATH" "$b" 3 restore "$bad" >/dev/null 2>&1 \ + && fail "restore accepted '$bad'" && return 1 + done + [[ $(backups) == "$keep" && -f $b/$keep/index.html && -f $DEPLOY_PATH/index.html ]] \ + || fail "backups or docroot damaged" || return 1 + # a dir that only looks like a backup (glob match, not the exact format) is never pruned or listed + mkdir -p "$b/20xxZ" + bash "$ROOT/deploy/remote.sh" "$DEPLOY_PATH" "$b" 1 prune >/dev/null 2>&1 || fail "prune failed" || return 1 + [[ -d $b/20xxZ ]] || fail "prune touched a non-backup dir" || return 1 +} + +t_lock_blocks_concurrent_runs() { + mkdir -p "$DEPLOY_PATH.backups" + "$DEPLOY" >/dev/null 2>&1 || fail "deploy" || return 1 + # exec so $holder is the process holding fd 9; killing it releases the lock + ( exec 9>>"$DEPLOY_PATH.backups/.lock"; flock 9; exec sleep 30 ) & holder=$! + sleep 0.3 + out=$(DEPLOY_LOCK_WAIT=1 "$DEPLOY" 2>&1) && { kill $holder; fail "deploy ran while locked"; return 1; } + grep -q 'could not take' <<<"$out" || { kill $holder; fail "no lock message: $out"; return 1; } + out=$(DEPLOY_LOCK_WAIT=1 "$DEPLOY" --rollback 2>&1) && { kill $holder; fail "rollback ran while locked"; return 1; } + kill $holder; wait $holder 2>/dev/null + [[ $(cat "$DEPLOY_PATH/index.html") == new && $(backups | wc -l) == 1 ]] || fail "state changed while locked" || return 1 + # released after each run: back-to-back runs don't wait on each other + DEPLOY_LOCK_WAIT=0 "$DEPLOY" >/dev/null 2>&1 || fail "lock not released after deploy" || return 1 + DEPLOY_LOCK_WAIT=0 "$DEPLOY" --rollback >/dev/null 2>&1 || fail "lock not released after rollback" || return 1 +} + +t_lock_held_during_upload() { + [[ $DEPLOY_HOST == local ]] && return 77 # skip: needs the ssh hook below + # wrap ssh: when rsync's upload connects, record whether the lock is held + cat > "$T/probe-ssh" <> "$T/probe" || echo held >> "$T/probe" +fi +exec "$WORK/fake-ssh" "\$@" +EOF + chmod +x "$T/probe-ssh" + DEPLOY_SSH=$T/probe-ssh "$DEPLOY" >/dev/null 2>&1 || fail "deploy failed" || return 1 + [[ $(cat "$T/probe") == held ]] || fail "lock not held during rsync: $(cat "$T/probe")" || return 1 +} + +t_select_ref_only_merged() { + [[ $DEPLOY_HOST == local ]] || return 77 # skip: git-only; once is enough + g=$T/repo; sel=$ROOT/deploy/select-ref.sh + git init -q -b main "$g" && cd "$g" || return 1 + c() { git -c user.name=t -c user.email=t@t commit -q --allow-empty -m "$1"; git rev-parse HEAD; } + old=$(c one); git tag v1; main=$(c two) + git checkout -q -b pr; pr=$(c unmerged); git checkout -q main + [[ $("$sel" '' "$main") == "$main" ]] || fail "empty ref" || return 1 + [[ $("$sel" "$old" "$main") == "$old" ]] || fail "merged sha" || return 1 + [[ $("$sel" v1 "$main") == "$old" ]] || fail "merged tag" || return 1 + [[ $("$sel" main "$main") == "$main" ]] || fail "main itself" || return 1 + for bad in pr "$pr" nosuchref -x --help "$main~5"; do + "$sel" "$bad" "$main" >/dev/null 2>&1 && fail "accepted '$bad'" && return 1 + done + return 0 +} + cases=$(declare -F | awk '{print $3}' | grep '^t_') for mode in local ssh; do for c in $cases; do @@ -173,10 +257,12 @@ for mode in local ssh; do setup if [[ $mode == local ]]; then export DEPLOY_HOST=local; unset DEPLOY_SSH DEPLOY_USER else export DEPLOY_HOST=testhost DEPLOY_USER=tester DEPLOY_SSH="$WORK/fake-ssh -o BatchMode=yes"; fi - if ( $c ); then pass=$((pass+1)); echo "ok $mode $c" + ( $c ); rc=$? + if ((rc == 0)); then pass=$((pass+1)); echo "ok $mode $c" + elif ((rc == 77)); then total=$((total-1)); skipped=$((skipped+1)) else failed+=("$mode $c"); echo "FAIL $mode $c"; fi done done -echo "deploy tests: $pass/$total passed" +echo "deploy tests: $pass/$total passed ($skipped mode-specific skips)" ((pass == total)) From a374a0b894f7fe16a1601f81742547f155a19b0d Mon Sep 17 00:00:00 2001 From: tangentstorm Date: Thu, 29 Oct 2026 22:38:38 +0000 Subject: [PATCH 4/6] deploy: atomic release switch; current tooling for old sources Codex P2s on #7: - Never edit the live tree in place. Each deploy uploads a complete tree into DEPLOY_RELEASES/.tmp (unchanged files hard-linked from the live release), renames it, then repoints the DEPLOY_PATH symlink with a single rename(2). Rollback is the same atomic switch to an older release, and older releases are the backups. A failed or killed upload leaves the live site untouched; the first deploy migrates a plain docroot into the oldest release. - Keep deploy tooling separate from the source being built. CI runs bin/deploy, deploy/* and the tests from github.sha, and builds the selected merged commit from a git-archive export (DEPLOY_SOURCE_DIR), so commits from before bin/deploy existed can be redeployed. - Everything created on the target uses umask 022, so nginx can read it even when the deploy shell has umask 077 (found in a local e2e run). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/deploy.yml | 21 ++- CLAUDE.md | 2 +- bin/deploy | 116 +++++++----- deploy/README.md | 132 ++++++++++---- deploy/deploy.conf | 3 +- deploy/remote.sh | 138 +++++++++----- deploy/select-ref.sh | 37 ++-- test/deploy.test.sh | 344 ++++++++++++++++++++++++----------- 8 files changed, 540 insertions(+), 253 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index c4284ea..2a81171 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -20,13 +20,19 @@ concurrency: permissions: contents: read +# Tooling vs. source: both jobs run bin/deploy, deploy/* and the tests from +# the workflow's own commit (github.sha). Only the application source that +# gets built comes from the `ref` input, exported separately by +# deploy/select-ref.sh. So redeploying an old commit (even one from before +# bin/deploy existed) still uses the current deploy tooling. jobs: test: runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 with: - ref: ${{ inputs.ref || github.sha }} + ref: ${{ github.sha }} + fetch-depth: 0 # the tests export a pre-tooling commit from history - uses: actions/setup-node@v5 with: node-version: 22 @@ -54,17 +60,20 @@ jobs: with: ref: ${{ github.sha }} fetch-depth: 0 - - name: Select commit (only commits already merged into main) + - name: Export source to build (only commits already merged into main) env: REF: ${{ inputs.ref }} run: | - sha=$(deploy/select-ref.sh "$REF" "$GITHUB_SHA") - git checkout --detach "$sha" - echo "deploying ${REF:-main} = $sha" + src=$RUNNER_TEMP/source + sha=$(deploy/select-ref.sh "$REF" "$GITHUB_SHA" "$src") + echo "DEPLOY_SOURCE_DIR=$src" >> "$GITHUB_ENV" + echo "building ${REF:-main} = $sha with tooling from $GITHUB_SHA" - uses: actions/setup-node@v5 with: node-version: 22 - - run: npm install --no-audit --no-fund + - name: Install the source's dependencies + run: npm install --no-audit --no-fund + working-directory: ${{ runner.temp }}/source - name: Install deploy key env: DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }} diff --git a/CLAUDE.md b/CLAUDE.md index b9a0246..afd110a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -213,7 +213,7 @@ npm run preview The Vite build automatically runs `npm run build` (TypeScript compilation) first via the `ensureCoreBuild` plugin, then bundles everything for the browser. -**Deployment**: `dist-web/` is the live site served at implish.org (nginx on 45.79.174.182). Deploy only with `bin/deploy`; never copy files by hand. Pushes to main deploy automatically via `.github/workflows/deploy.yml`. Use `bin/deploy --dry-run` to preview and `bin/deploy --rollback` to restore the previous tree. Host, user and docroot live in `deploy/deploy.conf` and the `production` environment's `DEPLOY_*` variables; the deploy key is an environment secret released only to main. Server-side steps are in `deploy/remote.sh`. See `deploy/README.md`. Tests: `npm run test:deploy`. Vite 7 needs Node >= 20.19. +**Deployment**: `dist-web/` is the live site served at implish.org (nginx on 45.79.174.182). Deploy only with `bin/deploy`; never copy files by hand. Pushes to main deploy automatically via `.github/workflows/deploy.yml`. Each deploy uploads a new release dir and atomically switches the docroot symlink to it. Use `bin/deploy --dry-run` to preview and `bin/deploy --rollback` to switch back to the previous release. CI runs the deploy tooling from the workflow's own commit and only builds the selected (merged) commit's source. Host, user and docroot live in `deploy/deploy.conf` and the `production` environment's `DEPLOY_*` variables; the deploy key is an environment secret released only to main. Server-side steps are in `deploy/remote.sh`. See `deploy/README.md`. Tests: `npm run test:deploy`. Vite 7 needs Node >= 20.19. ### TypeScript Details diff --git a/bin/deploy b/bin/deploy index 1b65d18..0ca837e 100755 --- a/bin/deploy +++ b/bin/deploy @@ -1,20 +1,28 @@ #!/usr/bin/env bash # bin/deploy: build web/ and publish dist-web/ to the implish.org docroot. # -# bin/deploy [--dry-run] [--skip-build] build, back up live docroot, rsync -# bin/deploy --rollback [NAME] [--dry-run] restore newest (or NAME) backup -# bin/deploy --list-backups list backups, oldest first +# bin/deploy [--dry-run] [--skip-build] build, upload a new release, switch to it +# bin/deploy --rollback [NAME] [--dry-run] switch back to the previous (or NAME) release +# bin/deploy --list-releases list releases, oldest first +# +# Every deploy uploads a complete tree to DEPLOY_RELEASES/, +# then atomically repoints the DEPLOY_PATH symlink at it (see deploy/remote.sh). +# The live site is never edited in place, so an interrupted deploy leaves +# the old site fully intact. # # Configuration comes from deploy/deploy.conf, overridden by environment: # DEPLOY_HOST ssh host, or "local" to treat DEPLOY_PATH as a local path # DEPLOY_USER ssh user (optional; empty = ssh default / ~/.ssh/config) -# DEPLOY_PATH absolute path of the live docroot -# DEPLOY_BACKUP_DIR where timestamped backups live (default: DEPLOY_PATH.backups) -# DEPLOY_KEEP how many backups to keep (default 5) +# DEPLOY_PATH absolute path of the live docroot (nginx root); becomes a symlink +# DEPLOY_RELEASES where releases live (default: DEPLOY_PATH.releases) +# DEPLOY_KEEP how many releases to keep, live one included (default 5) # DEPLOY_SSH ssh command (default: ssh -o BatchMode=yes) -# DEPLOY_BUILD_CMD build command (default: npx vite build) +# DEPLOY_SOURCE_DIR source tree to build (default: this checkout). CI points +# it at an exported older commit to redeploy that commit +# with the current tooling. +# DEPLOY_BUILD_CMD build command, run in DEPLOY_SOURCE_DIR (default: npx vite build) +# DEPLOY_SRC build output to publish, relative to DEPLOY_SOURCE_DIR (default: dist-web) # DEPLOY_LOCK_WAIT seconds to wait for another deploy's lock (default 60) -# DEPLOY_SRC build output to publish (default: dist-web) # # Exits nonzero on any failure. Nothing on the target is touched until the # build has succeeded and produced an index.html. @@ -29,8 +37,9 @@ log() { echo "deploy: $*" >&2; } # Non-empty environment values win over the config file. Empty ones are # ignored, because CI maps unset repository variables to empty strings. declare -A from_env=() -for v in DEPLOY_HOST DEPLOY_USER DEPLOY_PATH DEPLOY_BACKUP_DIR DEPLOY_KEEP \ - DEPLOY_SSH DEPLOY_BUILD_CMD DEPLOY_SRC DEPLOY_LOCK_WAIT; do +for v in DEPLOY_HOST DEPLOY_USER DEPLOY_PATH DEPLOY_RELEASES DEPLOY_KEEP \ + DEPLOY_SSH DEPLOY_SOURCE_DIR DEPLOY_BUILD_CMD DEPLOY_SRC DEPLOY_LOCK_WAIT \ + DEPLOY_RSYNC; do [[ -n ${!v:-} ]] && from_env[$v]=${!v} done if [[ -f $CONF ]]; then @@ -46,6 +55,8 @@ DEPLOY_KEEP=${DEPLOY_KEEP:-5} DEPLOY_SSH=${DEPLOY_SSH:-ssh -o BatchMode=yes} DEPLOY_BUILD_CMD=${DEPLOY_BUILD_CMD:-npx vite build} DEPLOY_SRC=${DEPLOY_SRC:-dist-web} +DEPLOY_SOURCE_DIR=${DEPLOY_SOURCE_DIR:-$ROOT} +DEPLOY_RSYNC=${DEPLOY_RSYNC:-rsync} # test hook mode=deploy dry_run=0 skip_build=0 rollback_name= while (($#)); do @@ -55,7 +66,7 @@ while (($#)); do --rollback) mode=rollback if [[ ${2:-} && ${2:0:1} != - ]]; then rollback_name=$2; shift; fi ;; - --list-backups) mode=list ;; + --list-releases|--list-backups) mode=list ;; -h|--help) sed -n '2,/^set -e/{/^set -e/d;s/^# \{0,1\}//;p}' "$0"; exit 0 ;; *) die "unknown argument: $1" ;; esac @@ -66,9 +77,9 @@ done [[ -n $DEPLOY_HOST ]] || die "DEPLOY_HOST is not set (see deploy/deploy.conf)" [[ -n $DEPLOY_PATH ]] || die "DEPLOY_PATH is not set (see deploy/deploy.conf)" DEPLOY_PATH=${DEPLOY_PATH%/} -DEPLOY_BACKUP_DIR=${DEPLOY_BACKUP_DIR:-$DEPLOY_PATH.backups} -DEPLOY_BACKUP_DIR=${DEPLOY_BACKUP_DIR%/} -check_path() { # name value: refuse paths that rsync --delete could wreck +DEPLOY_RELEASES=${DEPLOY_RELEASES:-$DEPLOY_PATH.releases} +DEPLOY_RELEASES=${DEPLOY_RELEASES%/} +check_path() { # name value: refuse paths that rsync --delete or rm -rf could wreck local name=$1 p=$2 [[ $p == /* ]] || die "$name must be absolute: '$p'" [[ $p =~ ^/+[^/]+/+[^/]+ ]] || die "$name is too shallow to be safe: '$p'" @@ -76,18 +87,20 @@ check_path() { # name value: refuse paths that rsync --delete could wreck [[ $p != *[[:space:]\'\"\\]* ]] || die "$name must not contain spaces or quotes: '$p'" } check_path DEPLOY_PATH "$DEPLOY_PATH" -check_path DEPLOY_BACKUP_DIR "$DEPLOY_BACKUP_DIR" -[[ $DEPLOY_BACKUP_DIR != "$DEPLOY_PATH" && $DEPLOY_BACKUP_DIR/ != "$DEPLOY_PATH"/* ]] \ - || die "DEPLOY_BACKUP_DIR must not be inside DEPLOY_PATH (rsync --delete would remove it)" +check_path DEPLOY_RELEASES "$DEPLOY_RELEASES" +[[ $DEPLOY_RELEASES != "$DEPLOY_PATH" && $DEPLOY_RELEASES/ != "$DEPLOY_PATH"/* ]] \ + || die "DEPLOY_RELEASES must not be inside DEPLOY_PATH" [[ $DEPLOY_KEEP =~ ^[1-9][0-9]*$ ]] || die "DEPLOY_KEEP must be a positive integer: '$DEPLOY_KEEP'" DEPLOY_LOCK_WAIT=${DEPLOY_LOCK_WAIT:-60} [[ $DEPLOY_LOCK_WAIT =~ ^[0-9]+$ ]] || die "DEPLOY_LOCK_WAIT must be a whole number of seconds: '$DEPLOY_LOCK_WAIT'" -if [[ $DEPLOY_HOST == local ]]; then - target=$DEPLOY_PATH -else - target=${DEPLOY_USER:+$DEPLOY_USER@}$DEPLOY_HOST:$DEPLOY_PATH -fi +[[ -d $DEPLOY_SOURCE_DIR ]] || die "DEPLOY_SOURCE_DIR is not a directory: '$DEPLOY_SOURCE_DIR'" + +at_target() { # PATH: rsync address of PATH on the target + if [[ $DEPLOY_HOST == local ]]; then echo "$1" + else echo "${DEPLOY_USER:+$DEPLOY_USER@}$DEPLOY_HOST:$1"; fi +} +target=$(at_target "$DEPLOY_PATH") # on_target SCRIPT ARGS...: run a bash snippet where the docroot lives. on_target() { @@ -101,44 +114,44 @@ on_target() { fi } -# rsync options shared by deploy and rollback. --checksum so CI checkouts -# (fresh mtimes) only list files whose content changed. /.well-known/ is -# protected from --delete because certbot keeps its challenges there. -RSYNC_OPTS=(-rlp --checksum --delete --delay-updates --itemize-changes +# rsync options for uploads and the dry-run listing. --checksum so CI +# checkouts (fresh mtimes) only list files whose content changed. +# /.well-known/ (certbot) is protected from --delete. +RSYNC_OPTS=(-rlp --checksum --delete --itemize-changes --chmod=Du=rwx,Dgo=rx,Fu=rw,Fgo=r --filter='P /.well-known/') rsync_to() { # SRC DEST [extra rsync args...] local src=$1 dest=$2; shift 2 if [[ $DEPLOY_HOST == local ]]; then - rsync "${RSYNC_OPTS[@]}" "$@" "$src" "$dest" + $DEPLOY_RSYNC "${RSYNC_OPTS[@]}" "$@" "$src" "$dest" else - rsync "${RSYNC_OPTS[@]}" -e "$DEPLOY_SSH" "$@" "$src" "$dest" + $DEPLOY_RSYNC "${RSYNC_OPTS[@]}" -e "$DEPLOY_SSH" "$@" "$src" "$dest" fi } # The target-side half lives in deploy/remote.sh and is piped to bash. REMOTE_LIB=$(<"$ROOT/deploy/remote.sh") -remote() { on_target "$REMOTE_LIB" "$DEPLOY_PATH" "$DEPLOY_BACKUP_DIR" "$DEPLOY_KEEP" "$@"; } +remote() { on_target "$REMOTE_LIB" "$DEPLOY_PATH" "$DEPLOY_RELEASES" "$DEPLOY_KEEP" "$@"; } -# hold_lock: take $DEPLOY_BACKUP_DIR/.lock on the target and keep it until +# hold_lock: take $DEPLOY_RELEASES/.lock on the target and keep it until # this script exits, so a manual --rollback can't interleave with a CI deploy. # A coprocess keeps an ssh session (or a local bash) open, which runs flock # and then waits on stdin. When we exit, or the connection drops, its stdin # closes and the lock is released. The script is passed as a bash -c string # because stdin is the keepalive pipe. It must not contain single quotes. # shellcheck disable=SC2016 -LOCK_SCRIPT='mkdir -p "$1" && exec 9>>"$1/.lock" && flock -w "$2" 9 && echo locked && exec cat >/dev/null' +LOCK_SCRIPT='umask 022 && mkdir -p "$1" && exec 9>>"$1/.lock" && flock -w "$2" 9 && echo locked && exec cat >/dev/null' hold_lock() { - local q; q=$(printf ' %q' "$DEPLOY_BACKUP_DIR" "$DEPLOY_LOCK_WAIT") + local q; q=$(printf ' %q' "$DEPLOY_RELEASES" "$DEPLOY_LOCK_WAIT") if [[ $DEPLOY_HOST == local ]]; then - coproc LOCK { bash -c "$LOCK_SCRIPT" hold_lock "$DEPLOY_BACKUP_DIR" "$DEPLOY_LOCK_WAIT"; } + coproc LOCK { bash -c "$LOCK_SCRIPT" hold_lock "$DEPLOY_RELEASES" "$DEPLOY_LOCK_WAIT"; } else # shellcheck disable=SC2086 coproc LOCK { $DEPLOY_SSH "${DEPLOY_USER:+$DEPLOY_USER@}$DEPLOY_HOST" "bash -c '$LOCK_SCRIPT' hold_lock$q"; } fi local line= read -r line <&"${LOCK[0]}" || true - [[ $line == locked ]] || die "could not take $DEPLOY_BACKUP_DIR/.lock (another deploy or rollback running?); nothing changed" - log "holding $DEPLOY_BACKUP_DIR/.lock" + [[ $line == locked ]] || die "could not take $DEPLOY_RELEASES/.lock (another deploy or rollback running?); nothing changed" + log "holding $DEPLOY_RELEASES/.lock" } case $mode in @@ -147,26 +160,26 @@ case $mode in rollback) ((dry_run)) || hold_lock name=$(remote pick "$rollback_name") - log "rolling back $target to backup $name" if ((dry_run)); then - remote restore "$name" dry + log "dry run: switching $target to release $name would change:" + remote diff "$name" log "dry run: nothing changed" exit 0 fi - remote restore "$name" - # The restored tree is now live, so the backup is redundant. Dropping it - # makes a second --rollback step one deploy further back. - if [[ -z $rollback_name ]]; then remote drop "$name"; fi - log "rolled back to $name" + # One atomic symlink swap. The release we leave stays on disk, so + # `--rollback NAME` can roll forward again, and a second --rollback + # steps one more release back. + remote switch "$name" + log "rolled back $target to release $name" exit 0 ;; esac # --- deploy --------------------------------------------------------------- -cd "$ROOT" +cd "$DEPLOY_SOURCE_DIR" if ((skip_build)); then log "skipping build (--skip-build)" else - log "building: $DEPLOY_BUILD_CMD" + log "building in $DEPLOY_SOURCE_DIR: $DEPLOY_BUILD_CMD" bash -c "$DEPLOY_BUILD_CMD" || die "build failed; nothing was deployed" fi [[ -f $DEPLOY_SRC/index.html ]] || die "$DEPLOY_SRC/index.html missing after build; nothing was deployed" @@ -184,8 +197,15 @@ if ((dry_run)); then fi hold_lock -remote backup -remote mkroot -rsync_to "$DEPLOY_SRC/" "$target/" +link=() +if [[ $(remote exists) == yes ]]; then link=(--link-dest="$DEPLOY_PATH/"); fi +name=$(remote prepare) +log "uploading release $name" +# Upload into RELEASES/.tmp. Unchanged files are hard-linked from the +# live tree (--link-dest), so they aren't re-sent. The live tree is only +# read here; nothing in it changes until `activate`. +rsync_to "$DEPLOY_SRC/" "$(at_target "$DEPLOY_RELEASES/$name.tmp")/" "${link[@]}" \ + || { remote discard "$name" || true; die "upload failed; live site unchanged"; } +remote activate "$name" remote prune -log "deployed $(git -C "$ROOT" rev-parse --short HEAD 2>/dev/null || echo '?') to $target" +log "deployed $(cat "$DEPLOY_SOURCE_DIR/.deploy-sha" 2>/dev/null || git -C "$DEPLOY_SOURCE_DIR" rev-parse HEAD 2>/dev/null || echo '?') as release $name" diff --git a/deploy/README.md b/deploy/README.md index 0b70217..ae5901a 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -7,43 +7,75 @@ nginx on 45.79.174.182 (Ubuntu) serves those files from a docroot. ```bash bin/deploy --dry-run # build, then list what would change; touches nothing -bin/deploy # build, back up live docroot, rsync dist-web/ over it -bin/deploy --list-backups # timestamped backups, oldest first -bin/deploy --rollback # restore the newest backup (repeat to step further back) -bin/deploy --rollback 20261029T120000Z # restore a specific backup +bin/deploy # build, upload a new release, switch the site to it +bin/deploy --list-releases # releases, oldest first, live one marked +bin/deploy --rollback # switch to the release before the live one (repeat to go further back) +bin/deploy --rollback 20261029T120000Z # switch to a specific release (also rolls forward) bin/deploy --rollback --dry-run # show what a rollback would change ``` +## Layout: releases and one symlink + +``` +/var/www/implish.org -> /var/www/implish.org.releases/20261029T120501Z (symlink: nginx root) +/var/www/implish.org.releases/ + 20261029T110000Z/ older release (rollback target) + 20261029T120501Z/ live release + .lock +``` + +Every deploy uploads a complete tree to a new release dir, then repoints the +docroot symlink with a single `rename(2)`. The live tree is never edited in +place. nginx serves the complete old site until the switch and the complete +new site after it. Vite's hashed assets therefore never go missing under a +live `index.html`, and an interrupted upload leaves the old site untouched. +Rollback is the same atomic switch to an older release. Older releases are +the backups. + What a deploy does, in order (any failure exits nonzero and stops): -1. Validates config. `DEPLOY_PATH` and `DEPLOY_BACKUP_DIR` must be absolute, - at least two levels deep, free of `.`/`..`/spaces, and the backup dir +1. Validates config. `DEPLOY_PATH` and `DEPLOY_RELEASES` must be absolute, + at least two levels deep, free of `.`/`..`/spaces, and the releases dir must not be inside the docroot. -2. Builds. If the build fails or leaves no `dist-web/index.html`, it stops - here and nothing on the server has been touched. -3. Takes `DEPLOY_BACKUP_DIR/.lock` on the server and holds it until the +2. Builds in `DEPLOY_SOURCE_DIR` (default: this checkout). If the build fails + or leaves no `dist-web/index.html`, it stops here and nothing on the + server has been touched. +3. Takes `DEPLOY_RELEASES/.lock` on the server and holds it until the deploy ends, waiting up to `DEPLOY_LOCK_WAIT` seconds (default 60). A manual `--rollback` takes the same lock, so it can't interleave with a CI deploy. A held ssh session keeps the lock; if the script dies or the connection drops, the lock is released. -4. Copies the live docroot to `DEPLOY_BACKUP_DIR/`, via a - `.tmp` copy that is renamed when complete. It skips this on the first - deploy, when the docroot is missing or empty. If the copy fails, the - partial `.tmp` is removed and the deploy stops before rsync. Stale `.tmp` - dirs from a killed run are removed on the next backup or prune. -5. Runs rsync `dist-web/` → docroot with `--delete --checksum --delay-updates`. - Files come out 644 and dirs 755. `/.well-known/` is never deleted, because - certbot uses it. -6. Prunes the backups down to the newest `DEPLOY_KEEP` (default 5). - -The server-side half (backup, prune, restore, drop) is `deploy/remote.sh`, -which `bin/deploy` pipes to `bash` over ssh. Every `rm -rf` there checks -that the name is exactly a backup timestamp (`^20[0-9]{6}T[0-9]{6}Z$`), so -an empty or malformed name can't reach the backup dir itself. - -Rollback rsyncs a backup back over the docroot and then removes that backup, -because the same tree is now live. The next `--rollback` therefore goes one -deploy further back. A named rollback keeps its backup. +4. Creates `DEPLOY_RELEASES/.tmp` and copies `/.well-known/` + (certbot) into it from the live tree. Stale `.tmp` dirs from a killed run + are removed first. +5. Uses rsync to upload `dist-web/` into the `.tmp` dir. Unchanged files are + hard-linked from the live release (`--link-dest`), so they aren't re-sent; + rsync never writes into the live tree. Files come out 644, dirs 755, and + everything on the target is created under `umask 022`, so nginx can read + it whatever umask the deploy shell has. If the upload fails, the `.tmp` + is removed and the live site is unchanged. +6. Activates: renames `.tmp` to its final name and atomically switches the + symlink. On the **first deploy**, if the docroot is still a plain + directory, it is moved into the releases dir (one second older than the + new release, so it becomes the rollback target). That migration is two + back-to-back renames, so for a moment the path is missing. This is the + one non-atomic step, and it happens once. An empty or missing docroot + just becomes the symlink. +7. Prunes down to the newest `DEPLOY_KEEP` releases (default 5). The live + release is never pruned, even after a rollback to an old one. + +The server-side half is `deploy/remote.sh`, which `bin/deploy` pipes to +`bash` over ssh. Every `rm -rf` there checks that the name is exactly a +release timestamp (`^20[0-9]{6}T[0-9]{6}Z$`), so an empty or malformed +name can't reach the releases dir itself. `switch` refuses to replace a +docroot that is a real directory; only the first deploy's migration does +that. + +Rollback switches to the newest release older than the live one, and keeps +the release it leaves. So repeated `--rollback` steps back, and +`--rollback NAME` can roll forward again. Certbot challenge files written +after a release was made aren't in older releases. That's harmless, because +certbot writes fresh ones for each renewal. ## Configuration @@ -55,7 +87,8 @@ of the same name overrides them. See `bin/deploy --help` for the full list. | `DEPLOY_HOST` | `45.79.174.182` (set in deploy.conf) | | `DEPLOY_USER` | **unknown: fill in** (repo variable or deploy.conf) | | `DEPLOY_PATH` | **unknown: fill in** the nginx `root` for implish.org | -| `DEPLOY_BACKUP_DIR` | default `$DEPLOY_PATH.backups` | +| `DEPLOY_RELEASES` | default `$DEPLOY_PATH.releases` | +| `DEPLOY_SOURCE_DIR` | default: this checkout (CI sets it to the exported source) | | `DEPLOY_KEEP` | 5 | The user and docroot are unknown because the machine that wrote this had no @@ -76,20 +109,32 @@ Check them on the server with `ssh-keygen -lf /etc/ssh/ssh_host_ed25519_key.pub` ## One-time setup (Michal) -The server needs `rsync` and `flock`, which stock Ubuntu has. The deploy user -needs write access to the docroot and to its parent directory, or to wherever -`DEPLOY_BACKUP_DIR` points. Steps 2 and 3 need root. +The server needs `rsync` and `flock`, which stock Ubuntu has, and nginx must +follow symlinks. That's the default; check that `disable_symlinks` isn't +turned on. Because the docroot becomes a symlink that the deploy swaps, the +deploy user needs write access to the docroot's **parent directory** (to +rename entries there) and to `DEPLOY_RELEASES`. Steps 2 and 3 need root. 1. Generate a dedicated key on your machine. Don't commit it: ```bash ssh-keygen -t ed25519 -N '' -C implish-deploy@github-actions -f implish-deploy ``` -2. On the server, give the deploy user ownership of the docroot and backup - dir. The docroot below is an example; use the real `root` from nginx: +2. On the server, give the deploy user a parent dir it owns. If the nginx + `root` today is directly under a root-owned dir like `/var/www`, the + least-privilege option is a dedicated parent. Move the site one level + down, and point nginx (and certbot's webroot, if it uses one) at the new + path: ```bash - sudo mkdir -p /var/www/implish.org.backups - sudo chown -R DEPLOYUSER: /var/www/implish.org /var/www/implish.org.backups + sudo mkdir -p /var/www/implish + sudo mv /var/www/implish.org /var/www/implish/site # current root -> new spot + sudo chown -R DEPLOYUSER: /var/www/implish + # nginx: root /var/www/implish/site; then: sudo nginx -t && sudo systemctl reload nginx ``` + Then set `DEPLOY_PATH=/var/www/implish/site`. Releases go to + `/var/www/implish/site.releases`. The alternative is to give the deploy + user write access to the existing parent dir, which is broader. + On the first deploy, `bin/deploy` turns `site` into a symlink and keeps + the old tree as the oldest release. 3. Append one line to `~DEPLOYUSER/.ssh/authorized_keys` on the server. `restrict` turns off pty, agent, port and X11 forwarding. The key can still run the `bash` and `rsync` commands that the deploy needs: @@ -133,6 +178,17 @@ needs write access to the docroot and to its parent directory, or to wherever dispatching an older merged SHA, or with `bin/deploy --rollback` from a machine that has the key. +**Tooling vs. source.** Both jobs always run `bin/deploy`, `deploy/*` and +the tests from the workflow's own commit (`github.sha`). Only the +application source comes from `ref`: `select-ref.sh` exports it with +`git archive` into `$RUNNER_TEMP/source`, and `bin/deploy` builds there via +`DEPLOY_SOURCE_DIR`. So you can redeploy a commit from before `bin/deploy` +existed (e.g. `6813323`), and fixes to the deploy tooling apply to +redeploys of old code too. The old commit's own `package.json` and +`vite.config.js` are used for its build. A commit too old to build with +`npx vite build` fails at the build step, before anything on the server is +touched. + Three layers keep the key away from unmerged code: 1. The `production` environment only releases `DEPLOY_SSH_KEY` to jobs on @@ -151,4 +207,8 @@ message that points here. `npm run test:deploy` runs `test/deploy.test.sh`. It tests against throwaway docroots in `.deploy-test/`, both with `DEPLOY_HOST=local` and through a fake `ssh` that runs the remote half locally, so the ssh/rsync `-e` path is -exercised too. +exercised too. An rsync wrapper (`DEPLOY_RSYNC`) checks that the live tree +stays byte-identical for the whole upload, and simulates cut-off and killed +transfers. One case exports the real pre-tooling commit `6813323` from this +repo's history and deploys it with the current `bin/deploy`; CI checks out +with `fetch-depth: 0` for that. diff --git a/deploy/deploy.conf b/deploy/deploy.conf index b968b38..8b337ba 100644 --- a/deploy/deploy.conf +++ b/deploy/deploy.conf @@ -7,5 +7,6 @@ DEPLOY_HOST=45.79.174.182 DEPLOY_USER= DEPLOY_PATH= -# DEPLOY_BACKUP_DIR defaults to "$DEPLOY_PATH.backups" (sibling of the docroot) +# DEPLOY_RELEASES defaults to "$DEPLOY_PATH.releases" (sibling of the docroot). +# DEPLOY_PATH becomes a symlink to the live release (see deploy/README.md). DEPLOY_KEEP=5 diff --git a/deploy/remote.sh b/deploy/remote.sh index 80a5cb1..db7363f 100644 --- a/deploy/remote.sh +++ b/deploy/remote.sh @@ -1,77 +1,123 @@ # deploy/remote.sh: the half of bin/deploy that runs where the docroot lives. # bin/deploy pipes this file to `bash -s` (over ssh, or locally), with args: # -# remote.sh DOCROOT BACKUPS KEEP ACTION [ARG] +# remote.sh DOCROOT RELEASES KEEP ACTION [ARG] # -# The caller holds $BACKUPS/.lock for the whole deploy or rollback (see +# Layout: every deploy is a complete tree in RELEASES/, and +# DOCROOT is a symlink to the live one. A deploy uploads into +# RELEASES/.tmp, renames it, then swaps the symlink with one rename(2). +# Rollback is the same swap to an older release. So nginx sees either the +# whole old tree or the whole new one, never a mix, even if a transfer is +# interrupted. Older releases double as the backups. +# +# The caller holds $RELEASES/.lock for the whole deploy or rollback (see # hold_lock in bin/deploy), so the actions here don't lock on their own. -# Backup names are UTC timestamps (20261029T120000Z), so name order is age -# order. Every rm -rf goes through valid_name, so a bad or empty name can -# never reach the backup dir itself. +# Release names are UTC timestamps (20261029T120000Z), so name order is age +# order. Every rm -rf goes through rm_release, which checks the name. set -euo pipefail -docroot=$1 backups=$2 keep=$3 action=$4 arg=${5:-} +# nginx (another user) must be able to traverse and read everything we +# create, whatever umask the ssh session or CI runner has. +umask 022 +docroot=$1 releases=$2 keep=$3 action=$4 arg=${5:-} die() { echo "deploy: $*" >&2; exit 1; } valid_name() { [[ $1 =~ ^20[0-9]{6}T[0-9]{6}Z$ ]]; } -rm_backup() { # NAME [SUFFIX]: remove one backup (or its .tmp) by validated name - valid_name "$1" || die "refusing to remove invalid backup name: '$1'" - rm -rf -- "${backups:?}/$1${2:-}" +rm_release() { # NAME [SUFFIX]: remove one release (or its .tmp) by validated name + valid_name "$1" || die "refusing to remove invalid release name: '$1'" + rm -rf -- "${releases:?}/$1${2:-}" } list() { - [[ -d $backups ]] || return 0 - find "$backups" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' \ + [[ -d $releases ]] || return 0 + find "$releases" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' \ | grep -E '^20[0-9]{6}T[0-9]{6}Z$' | LC_ALL=C sort || true } -clean_tmp() { # leftovers of a backup whose cp failed partway - [[ -d $backups ]] || return 0 - find "$backups" -mindepth 1 -maxdepth 1 -type d -name '20*Z.tmp' -printf '%f\n' \ +live() { # name of the release DOCROOT points at, or nothing + [[ -L $docroot ]] || return 0 + local t; t=$(readlink "$docroot") + if [[ $(dirname "$t") == "$releases" ]] && valid_name "$(basename "$t")"; then basename "$t"; fi +} +clean_tmp() { # leftovers of an upload that failed or was killed + [[ -d $releases ]] || return 0 + find "$releases" -mindepth 1 -maxdepth 1 -type d -name '20*Z.tmp' -printf '%f\n' \ | while read -r t; do - rm_backup "${t%.tmp}" .tmp; echo "deploy: removed stale partial backup $t" >&2 + rm_release "${t%.tmp}" .tmp; echo "deploy: removed stale partial upload $t" >&2 done } +new_name() { + local name + while :; do + name=$(date -u +%Y%m%dT%H%M%SZ) + [[ -e $releases/$name || -e $releases/$name.tmp ]] || break + sleep 1 + done + echo "$name" +} +check_release() { # NAME: must be an existing, complete release + valid_name "$1" && [[ -d $releases/$1 ]] || die "no such release: $1" + [[ -f $releases/$1/index.html ]] || die "release $1 has no index.html; refusing to make it live" +} +switch() { # NAME: atomically point DOCROOT at a release + check_release "$1" + [[ -L $docroot || ! -e $docroot ]] || die "$docroot is not a symlink; refusing to replace it" + mkdir -p "$(dirname "$docroot")" + ln -sfn "$releases/$1" "$docroot.switch" + mv -T "$docroot.switch" "$docroot" +} case $action in - list) list ;; + list) live=$(live); list | while read -r n; do [[ $n == "$live" ]] && echo "$n (live)" || echo "$n"; done ;; + live) live ;; exists) [[ -d $docroot ]] && echo yes || echo no ;; - backup) - mkdir -p "$backups" + prepare) # make an empty RELEASES/.tmp for the upload; print + mkdir -p "$releases" + chmod 755 "$releases" # hold_lock may have created it first clean_tmp - if [[ ! -d $docroot ]] || [[ -z $(ls -A "$docroot") ]]; then - echo "deploy: $docroot is empty or missing; no backup needed" >&2; exit 0 + name=$(new_name) + mkdir "$releases/$name.tmp" + # certbot's challenges live in the docroot; carry them into the new tree + if [[ -d $docroot/.well-known ]]; then cp -a "$docroot/.well-known" "$releases/$name.tmp/"; fi + echo "$name" ;; + discard) rm_release "$arg" .tmp ;; + activate) # ARG: finish an upload and make it live + valid_name "$arg" || die "invalid release name: '$arg'" + [[ -f $releases/$arg.tmp/index.html ]] || die "upload $arg.tmp is incomplete; live site unchanged" + mv -T "$releases/$arg.tmp" "$releases/$arg" + if [[ -d $docroot && ! -L $docroot ]]; then + # First deploy onto a plain directory: keep it as the oldest release. + # This is the one non-atomic step: two renames back to back. + if [[ -z $(ls -A "$docroot") ]]; then + rmdir "$docroot" + else + legacy=$(date -u -d "@$(( $(date -u -d "${arg:0:8} ${arg:9:2}:${arg:11:2}:${arg:13:2}" +%s) - 1 ))" +%Y%m%dT%H%M%SZ) + [[ ! -e $releases/$legacy ]] || die "cannot migrate $docroot: $releases/$legacy exists" + ln -sfn "$releases/$arg" "$docroot.switch" + mv -T "$docroot" "$releases/$legacy" + mv -T "$docroot.switch" "$docroot" + echo "deploy: moved the old docroot to release $legacy" >&2 + fi fi - while :; do - name=$(date -u +%Y%m%dT%H%M%SZ) - [[ -e $backups/$name || -e $backups/$name.tmp ]] || break - sleep 1 - done - cp -a "$docroot" "$backups/$name.tmp" || { rm_backup "$name" .tmp; die "backup of $docroot failed; nothing was deployed"; } - mv "$backups/$name.tmp" "$backups/$name" - echo "deploy: backed up $docroot to $backups/$name" >&2 ;; - prune) + switch "$arg" ;; + switch) switch "$arg" ;; + prune) # keep the newest KEEP releases, and always the live one clean_tmp + live=$(live) list | head -n -"$keep" | while read -r old; do - rm_backup "$old"; echo "deploy: pruned backup $old" >&2 + [[ $old == "$live" ]] && continue + rm_release "$old"; echo "deploy: pruned release $old" >&2 done ;; - pick) # print the backup to roll back to: ARG, or the newest + pick) # print the release to roll back to: ARG, or the newest one older than live if [[ -n $arg ]]; then - valid_name "$arg" && [[ -d $backups/$arg ]] || die "no such backup: $arg" name=$arg else - name=$(list | tail -n 1) - [[ -n $name ]] || die "no backups in $backups" + live=$(live) + [[ -n $live ]] || die "$docroot is not a release symlink yet; nothing to roll back to" + name=$(list | awk -v l="$live" '$0 < l' | tail -n 1) + [[ -n $name ]] || die "no release older than the live one ($live)" fi - [[ -f $backups/$name/index.html ]] || die "backup $name has no index.html; refusing to restore it" + check_release "$name" echo "$name" ;; - restore) # ARG [dry]: rsync a backup over the docroot - valid_name "$arg" && [[ -d $backups/$arg ]] || die "no such backup: $arg" - if [[ ${6:-} == dry ]]; then - rsync -rl --checksum --delete --dry-run --itemize-changes --filter='P /.well-known/' \ - "$backups/$arg/" "$docroot/" - else - rsync -rlp --checksum --delete --delay-updates --filter='P /.well-known/' \ - "$backups/$arg/" "$docroot/" - fi ;; - drop) rm_backup "$arg" ;; - mkroot) mkdir -p "$docroot" ;; + diff) # ARG: what switching to ARG would change + check_release "$arg" + rsync -rl --checksum --delete --dry-run --itemize-changes "$releases/$arg/" "$docroot/" ;; *) die "unknown remote action: $action" ;; esac diff --git a/deploy/select-ref.sh b/deploy/select-ref.sh index 3680d01..f4c3d31 100755 --- a/deploy/select-ref.sh +++ b/deploy/select-ref.sh @@ -1,16 +1,31 @@ #!/usr/bin/env bash -# deploy/select-ref.sh REF BASE: print the commit REF names, but only if it is -# already merged into BASE (main's head). workflow_dispatch uses it so the -# `ref` input can redeploy or roll back to merged history, never to an -# unmerged branch or PR head. Empty REF prints BASE. Exits nonzero otherwise. +# deploy/select-ref.sh REF BASE [DEST]: resolve REF to a commit, but only if +# it is already merged into BASE (main's head). Empty REF means BASE. +# Prints the SHA. With DEST, it also exports that commit's tree into DEST +# (git archive, no .git) and writes DEST/.deploy-sha. +# +# workflow_dispatch uses this so the `ref` input can redeploy or roll back to +# merged history, never to an unmerged branch or PR head. Only the +# *application source* comes from REF. The deploy tooling (bin/deploy, +# deploy/*, tests) always comes from the workflow's own checkout, so commits +# from before this tooling existed can still be deployed. set -euo pipefail -ref=${1:-} base=${2:?usage: select-ref.sh REF BASE} +ref=${1:-} base=${2:?usage: select-ref.sh REF BASE [DEST]} dest=${3:-} err() { echo "::error::$*" >&2; exit 1; } base=$(git rev-parse --verify --quiet --end-of-options "$base^{commit}") || err "unknown base: $2" -if [[ -z $ref ]]; then echo "$base"; exit 0; fi -[[ $ref != -* ]] || err "bad ref: $ref" -sha=$(git rev-parse --verify --quiet --end-of-options "origin/$ref^{commit}" || - git rev-parse --verify --quiet --end-of-options "$ref^{commit}") || err "unknown ref: $ref" -git merge-base --is-ancestor "$sha" "$base" || - err "$ref ($sha) is not merged into main; only merged commits can be deployed" +if [[ -z $ref ]]; then + sha=$base +else + [[ $ref != -* ]] || err "bad ref: $ref" + sha=$(git rev-parse --verify --quiet --end-of-options "origin/$ref^{commit}" || + git rev-parse --verify --quiet --end-of-options "$ref^{commit}") || err "unknown ref: $ref" + git merge-base --is-ancestor "$sha" "$base" || + err "$ref ($sha) is not merged into main; only merged commits can be deployed" +fi +if [[ -n $dest ]]; then + [[ ! -e $dest || -z $(ls -A "$dest") ]] || err "export dir is not empty: $dest" + mkdir -p "$dest" + git archive --format=tar "$sha" | tar -x -C "$dest" + echo "$sha" > "$dest/.deploy-sha" +fi echo "$sha" diff --git a/test/deploy.test.sh b/test/deploy.test.sh index 267d0f1..b4c45bd 100755 --- a/test/deploy.test.sh +++ b/test/deploy.test.sh @@ -2,16 +2,17 @@ # Tests for bin/deploy against throwaway docroots under .deploy-test/. # Every case runs twice: DEPLOY_HOST=local, and through a fake ssh that runs # the remote commands locally (exercising the ssh + rsync -e code path). +# A case that only makes sense in one mode returns 77 in the other (skip). set -uo pipefail ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) DEPLOY=$ROOT/bin/deploy +REMOTE=$ROOT/deploy/remote.sh WORK=$ROOT/.deploy-test rm -rf "$WORK"; mkdir -p "$WORK" trap 'rm -rf "$WORK"' EXIT pass=0 total=0 skipped=0 failed=() -ok() { :; } fail() { echo " FAIL: $*" >&2; return 1; } # fake ssh: skip options, drop the host, run the command string locally. @@ -23,137 +24,282 @@ exec bash -c "$*" EOF chmod +x "$WORK/fake-ssh" -# setup: fresh sandbox with a live docroot holding the "old" site +# setup: fresh sandbox whose live docroot is a plain directory (like the +# server today) holding the "old" site. Like vite output, the index refers to +# a content-hashed asset, so serving a mix of old and new trees breaks the page. setup() { T=$WORK/case$total - rm -rf "$T"; mkdir -p "$T/srv/www/site/.well-known/acme" "$T/src" - echo old > "$T/srv/www/site/index.html" + rm -rf "$T"; mkdir -p "$T/srv/www/site/.well-known/acme" "$T/srv/www/site/assets" "$T/src" + echo 'old assets/app-old.js' > "$T/srv/www/site/index.html" + echo old-js > "$T/srv/www/site/assets/app-old.js" echo stale > "$T/srv/www/site/stale.js" echo token > "$T/srv/www/site/.well-known/acme/tok" export DEPLOY_CONF=/dev/null DEPLOY_PATH=$T/srv/www/site DEPLOY_KEEP=3 - export DEPLOY_SRC=$T/out - unset DEPLOY_BACKUP_DIR - # the "build" writes a new site whose content is $T/src/version + export DEPLOY_SOURCE_DIR=$T/src DEPLOY_SRC=out + unset DEPLOY_RELEASES DEPLOY_RSYNC + R=$DEPLOY_PATH.releases + # the "build" makes index.html " assets/app-.js" echo new > "$T/src/version" - export DEPLOY_BUILD_CMD="rm -rf '$T/out' && mkdir -p '$T/out/assets' && cp '$T/src/version' '$T/out/index.html' && echo js > '$T/out/assets/app.js'" + export DEPLOY_BUILD_CMD='v=$(cat version); rm -rf out && mkdir -p out/assets && echo "$v assets/app-$v.js" > out/index.html && echo "$v-js" > "out/assets/app-$v.js"' } -backups() { ls "$DEPLOY_PATH.backups" 2>/dev/null | grep -E '^20[0-9]{6}T[0-9]{6}Z$' || true; } +releases() { ls "$R" 2>/dev/null | grep -E '^20[0-9]{6}T[0-9]{6}Z$' || true; } +live() { readlink "$DEPLOY_PATH"; } +site() { head -c 100 "$DEPLOY_PATH/index.html" | cut -d' ' -f1; } # version that is live tree() { (cd "$1" && find . -type f -printf '%P\t' -exec cat {} \; | LC_ALL=C sort); } +deploy_v() { echo "$1" > "$T/src/version"; "$DEPLOY" >/dev/null 2>&1; } +# the page works only if the asset its index.html names is present +page_ok() { local a; a=$(cut -d' ' -f2 "$DEPLOY_PATH/index.html"); [[ -f $DEPLOY_PATH/$a ]]; } + +# --- dry runs and aborts ---------------------------------------------------- t_dry_run_lists_files() { + before=$(tree "$DEPLOY_PATH") out=$("$DEPLOY" --dry-run 2>&1) || fail "exit $?: $out" || return 1 grep -q 'index.html' <<<"$out" || fail "index.html not listed: $out" || return 1 - grep -q 'assets/app.js' <<<"$out" || fail "assets/app.js not listed" || return 1 + grep -q 'assets/app-new.js' <<<"$out" || fail "new asset not listed" || return 1 grep -Eq 'deleting +stale.js' <<<"$out" || fail "stale.js deletion not listed" || return 1 ! grep -q 'well-known' <<<"$out" || fail ".well-known would be touched" || return 1 - [[ $(cat "$DEPLOY_PATH/index.html") == old ]] || fail "dry run changed docroot" || return 1 - [[ -z $(backups) ]] || fail "dry run made a backup" || return 1 + [[ $(tree "$DEPLOY_PATH") == "$before" && ! -L $DEPLOY_PATH ]] || fail "dry run changed docroot" || return 1 + [[ ! -e $R ]] || fail "dry run created $R" || return 1 } t_dry_run_new_docroot() { rm -rf "$DEPLOY_PATH" out=$("$DEPLOY" --dry-run 2>&1) || fail "exit $?: $out" || return 1 - grep -q 'assets/app.js' <<<"$out" || fail "app.js not listed: $out" || return 1 - [[ ! -e $DEPLOY_PATH ]] || fail "dry run created docroot" || return 1 + grep -q 'assets/app-new.js' <<<"$out" || fail "asset not listed: $out" || return 1 + [[ ! -e $DEPLOY_PATH && ! -e $R ]] || fail "dry run created something" || return 1 } t_failed_build_aborts() { + before=$(tree "$DEPLOY_PATH") out=$(DEPLOY_BUILD_CMD='echo boom >&2; exit 3' "$DEPLOY" 2>&1) && fail "exit 0" && return 1 grep -q 'build failed' <<<"$out" || fail "no build-failed message: $out" || return 1 - [[ $(cat "$DEPLOY_PATH/index.html") == old && -f $DEPLOY_PATH/stale.js ]] || fail "docroot changed" || return 1 - [[ -z $(backups) ]] || fail "backup made" || return 1 + [[ $(tree "$DEPLOY_PATH") == "$before" && ! -e $R ]] || fail "target touched" || return 1 } t_empty_build_aborts() { - out=$(DEPLOY_BUILD_CMD="mkdir -p '$T/out'" "$DEPLOY" 2>&1) && fail "exit 0" && return 1 + out=$(DEPLOY_BUILD_CMD='mkdir -p out' "$DEPLOY" 2>&1) && fail "exit 0" && return 1 grep -q 'index.html missing' <<<"$out" || fail "$out" || return 1 - [[ -f $DEPLOY_PATH/stale.js && -z $(backups) ]] || fail "target touched" || return 1 + [[ -f $DEPLOY_PATH/stale.js && ! -e $R ]] || fail "target touched" || return 1 } -t_deploy_replaces_tree() { +# --- deploy ----------------------------------------------------------------- + +t_deploy_migrates_plain_docroot() { + before=$(tree "$DEPLOY_PATH") "$DEPLOY" >/dev/null 2>&1 || fail "deploy failed" || return 1 - [[ $(cat "$DEPLOY_PATH/index.html") == new ]] || fail "index not updated" || return 1 - [[ -f $DEPLOY_PATH/assets/app.js ]] || fail "app.js missing" || return 1 - [[ ! -e $DEPLOY_PATH/stale.js ]] || fail "stale.js not deleted" || return 1 - [[ -f $DEPLOY_PATH/.well-known/acme/tok ]] || fail ".well-known deleted" || return 1 - [[ $(stat -c %a "$DEPLOY_PATH/index.html") == 644 ]] || fail "index.html not world-readable" || return 1 - [[ $(backups | wc -l) == 1 ]] || fail "expected 1 backup, got: $(backups)" || return 1 - b=$DEPLOY_PATH.backups/$(backups) - [[ $(cat "$b/index.html") == old && -f $b/stale.js ]] || fail "backup is not the old tree" || return 1 + [[ -L $DEPLOY_PATH ]] || fail "docroot is not a symlink now" || return 1 + [[ $(site) == new ]] && page_ok || fail "new site not live/complete" || return 1 + [[ ! -e $DEPLOY_PATH/stale.js && ! -e $DEPLOY_PATH/assets/app-old.js ]] || fail "old files in new release" || return 1 + [[ -f $DEPLOY_PATH/.well-known/acme/tok ]] || fail ".well-known not carried over" || return 1 + [[ $(stat -c %a "$DEPLOY_PATH/index.html") == 644 ]] || fail "index.html not 644" || return 1 + mapfile -t rs < <(releases) + [[ ${#rs[@]} == 2 ]] || fail "expected old + new release, got: ${rs[*]}" || return 1 + [[ $(live) == "$R/${rs[1]}" ]] || fail "live is not the newest release" || return 1 + [[ $(tree "$R/${rs[0]}") == "$before" ]] || fail "old docroot not kept intact as a release" || return 1 } -t_first_deploy_no_backup() { - rm -rf "$DEPLOY_PATH" +t_first_deploy_missing_docroot() { + rm -rf "$T/srv" + "$DEPLOY" >/dev/null 2>&1 || fail "deploy failed" || return 1 + [[ -L $DEPLOY_PATH && $(site) == new ]] && page_ok || fail "not deployed" || return 1 + [[ $(releases | wc -l) == 1 ]] || fail "expected 1 release: $(releases)" || return 1 +} + +# nginx runs as another user, so every dir on the path to every served file +# must be o+rx and every file o+r, even when the deploying shell has umask 077. +t_nginx_can_read_everything() { + rm -rf "$T/srv" # also covers the docroot's parent dirs being created by the deploy + ( umask 077; "$DEPLOY" >/dev/null 2>&1 ) || fail "deploy failed" || return 1 + ( umask 077; deploy_v v2 ) || fail "deploy 2 failed" || return 1 + bad=$(find "$T/srv" ! -name .lock \( -type d ! -perm -o=rx -o -type f ! -perm -o=r \) -printf '%M %p\n') + [[ -z $bad ]] || fail "not readable by nginx: +$bad" || return 1 +} + +t_first_deploy_empty_docroot() { + rm -rf "$DEPLOY_PATH"; mkdir -p "$DEPLOY_PATH" + "$DEPLOY" >/dev/null 2>&1 || fail "deploy failed" || return 1 + [[ -L $DEPLOY_PATH && $(site) == new ]] || fail "not deployed" || return 1 + [[ $(releases | wc -l) == 1 ]] || fail "empty docroot kept as a release: $(releases)" || return 1 +} + +t_redeploy_reuses_unchanged_files() { + deploy_v v1 || fail "deploy v1" || return 1 + first=$(live) + echo extra > "$T/src/extra.txt" + DEPLOY_BUILD_CMD="$DEPLOY_BUILD_CMD"' && cp extra.txt out/' "$DEPLOY" >/dev/null 2>&1 || fail "deploy 2" || return 1 + [[ $(live) != "$first" ]] || fail "no new release" || return 1 + # unchanged index.html is a hard link to the previous release's copy + [[ $(stat -c %i "$first/index.html") == $(stat -c %i "$DEPLOY_PATH/index.html") ]] \ + || fail "unchanged file was re-sent instead of linked" || return 1 + [[ $(cat "$first/index.html") == "v1 assets/app-v1.js" ]] || fail "previous release modified" || return 1 +} + +# The live tree must be untouched for the whole upload: a reader sees the +# complete old site until the switch, then the complete new one. +t_live_untouched_during_upload() { + before=$(tree "$DEPLOY_PATH") + cat > "$T/rsync-probe" <> "$T/probe"; rsync "\$@"; rc=\$?; check >> "$T/probe"; exit \$rc +EOF + chmod +x "$T/rsync-probe"; printf '%s' "$before" > "$T/before" + DEPLOY_RSYNC=$T/rsync-probe "$DEPLOY" >/dev/null 2>&1 || fail "deploy failed" || return 1 + [[ $(tr '\n' ' ' < "$T/probe") == "intact intact " ]] || fail "live changed during upload: $(cat "$T/probe")" || return 1 + [[ $(site) == new ]] && page_ok || fail "new site not live" || return 1 +} + +t_failed_upload_leaves_live_intact() { + deploy_v v1 || fail "deploy v1" || return 1 + before=$(tree "$DEPLOY_PATH"); was=$(live); nrel=$(releases | wc -l) + # partial transfer: every asset but index.html arrives, then rsync fails + printf '#!/usr/bin/env bash\nrsync --exclude=/index.html "$@"; exit 30\n' > "$T/rsync-cut"; chmod +x "$T/rsync-cut" + echo v2 > "$T/src/version" + out=$(DEPLOY_RSYNC=$T/rsync-cut "$DEPLOY" 2>&1) && fail "exit 0" && return 1 + grep -q 'upload failed; live site unchanged' <<<"$out" || fail "$out" || return 1 + [[ $(live) == "$was" && $(tree "$DEPLOY_PATH") == "$before" ]] && page_ok || fail "live site changed" || return 1 + [[ $(releases | wc -l) == "$nrel" && -z $(ls "$R" | grep tmp) ]] || fail "partial upload left behind: $(ls -a "$R")" || return 1 +} + +t_killed_deploy_recovers() { + deploy_v v1 || fail "deploy v1" || return 1 + before=$(tree "$DEPLOY_PATH") + # rsync uploads part of the tree, then the whole deploy dies (CI runner lost) + printf '#!/usr/bin/env bash\nrsync --exclude=/index.html "$@"; kill -9 $PPID; exit 1\n' > "$T/rsync-kill"; chmod +x "$T/rsync-kill" + echo v2 > "$T/src/version" + { DEPLOY_RSYNC=$T/rsync-kill "$DEPLOY" >/dev/null 2>&1; } 2>/dev/null && fail "exit 0" && return 1 + sleep 0.3 + [[ $(tree "$DEPLOY_PATH") == "$before" ]] && page_ok || fail "live site changed" || return 1 + [[ -n $(ls "$R" | grep '\.tmp$') ]] || fail "expected a stale .tmp after kill" || return 1 + # the lock was released by the dead run, and the next deploy cleans up + out=$(DEPLOY_LOCK_WAIT=2 "$DEPLOY" 2>&1) || fail "next deploy failed: $out" || return 1 + grep -q 'removed stale partial upload' <<<"$out" || fail "stale .tmp not reported" || return 1 + [[ -z $(ls "$R" | grep tmp) && $(site) == v2 ]] && page_ok || fail "not recovered" || return 1 +} + +t_stale_tmp_removed() { + mkdir -p "$R/20200101T000000Z.tmp/junk" "$DEPLOY" >/dev/null 2>&1 || fail "deploy failed" || return 1 - [[ $(cat "$DEPLOY_PATH/index.html") == new ]] || fail "not deployed" || return 1 - [[ -z $(backups) ]] || fail "backed up an empty docroot" || return 1 + [[ ! -e $R/20200101T000000Z.tmp ]] || fail "stale .tmp left behind" || return 1 } +# --- rollback --------------------------------------------------------------- + t_rollback_restores_previous() { before=$(tree "$DEPLOY_PATH") "$DEPLOY" >/dev/null 2>&1 || fail "deploy failed" || return 1 + newrel=$(live) out=$("$DEPLOY" --rollback --dry-run 2>&1) || fail "rollback dry run: $out" || return 1 - [[ $(cat "$DEPLOY_PATH/index.html") == new ]] || fail "rollback dry run changed docroot" || return 1 + grep -q 'app-old.js' <<<"$out" || fail "dry run did not list changes: $out" || return 1 + [[ $(live) == "$newrel" ]] || fail "rollback dry run switched" || return 1 "$DEPLOY" --rollback >/dev/null 2>&1 || fail "rollback failed" || return 1 [[ $(tree "$DEPLOY_PATH") == "$before" ]] || fail "tree differs after rollback: $(diff <(echo "$before") <(tree "$DEPLOY_PATH"))" || return 1 - [[ -z $(backups) ]] || fail "used backup not dropped" || return 1 + [[ -L $DEPLOY_PATH ]] && page_ok || fail "rollback broke the symlink/page" || return 1 + # the release we left is kept, so we can roll forward by name + "$DEPLOY" --rollback "$(basename "$newrel")" >/dev/null 2>&1 && [[ $(site) == new ]] || fail "roll forward" || return 1 +} + +t_rollback_is_one_atomic_switch() { + deploy_v v1 && deploy_v v2 || fail "deploys" || return 1 + v1=$(releases | tail -n 2 | head -n 1); v2=$(releases | tail -n 1) + t1=$(tree "$R/$v1"); t2=$(tree "$R/$v2") + "$DEPLOY" --rollback >/dev/null 2>&1 || fail "rollback" || return 1 + [[ $(live) == "$R/$v1" ]] || fail "live is $(live), expected $v1" || return 1 + [[ $(tree "$R/$v1") == "$t1" && $(tree "$R/$v2") == "$t2" ]] || fail "rollback modified a release" || return 1 + [[ -z $(ls -a "$(dirname "$DEPLOY_PATH")" | grep switch) ]] || fail "temp symlink left behind" || return 1 } t_rollback_steps_back() { - for v in v1 v2 v3; do - echo "$v" > "$T/src/version"; "$DEPLOY" >/dev/null 2>&1 || fail "deploy $v" || return 1 - done - "$DEPLOY" --rollback >/dev/null 2>&1 && [[ $(cat "$DEPLOY_PATH/index.html") == v2 ]] || fail "1st rollback" || return 1 - "$DEPLOY" --rollback >/dev/null 2>&1 && [[ $(cat "$DEPLOY_PATH/index.html") == v1 ]] || fail "2nd rollback" || return 1 + for v in v1 v2 v3; do deploy_v $v || fail "deploy $v" || return 1; done + "$DEPLOY" --rollback >/dev/null 2>&1 && [[ $(site) == v2 ]] || fail "1st rollback" || return 1 + "$DEPLOY" --rollback >/dev/null 2>&1 && [[ $(site) == v1 ]] || fail "2nd rollback" || return 1 + deploy_v v4 && [[ $(site) == v4 ]] && page_ok || fail "deploy after rollback" || return 1 } t_rollback_named() { - "$DEPLOY" >/dev/null 2>&1 || fail "deploy" || return 1 - name=$(backups) - echo v2 > "$T/src/version"; "$DEPLOY" >/dev/null 2>&1 || fail "deploy 2" || return 1 - "$DEPLOY" --rollback "$name" >/dev/null 2>&1 || fail "named rollback" || return 1 - [[ $(cat "$DEPLOY_PATH/index.html") == old ]] || fail "wrong tree" || return 1 - out=$("$DEPLOY" --rollback 20000101T000000Z 2>&1) && fail "bogus name accepted" && return 1 - out=$("$DEPLOY" --rollback '../../etc' 2>&1) && fail "traversal name accepted" && return 1 - return 0 + "$DEPLOY" >/dev/null 2>&1 && deploy_v v2 || fail "deploys" || return 1 + oldest=$(releases | head -n 1) + "$DEPLOY" --rollback "$oldest" >/dev/null 2>&1 || fail "named rollback" || return 1 + [[ $(site) == old ]] && page_ok || fail "wrong tree" || return 1 + "$DEPLOY" --rollback 20000101T000000Z >/dev/null 2>&1 && fail "bogus name accepted" && return 1 + "$DEPLOY" --rollback '../../etc' >/dev/null 2>&1 && fail "traversal name accepted" && return 1 + [[ $(site) == old ]] || fail "failed rollback changed the site" || return 1 } -t_rollback_without_backups_fails() { - out=$("$DEPLOY" --rollback 2>&1) && fail "exit 0" && return 1 - grep -q 'no backups' <<<"$out" || fail "$out" || return 1 - [[ $(cat "$DEPLOY_PATH/index.html") == old ]] || fail "docroot changed" || return 1 +t_rollback_without_previous_fails() { + out=$("$DEPLOY" --rollback 2>&1) && fail "exit 0 on plain docroot" && return 1 + grep -q 'nothing to roll back to' <<<"$out" || fail "$out" || return 1 + rm -rf "$DEPLOY_PATH"; "$DEPLOY" >/dev/null 2>&1 || fail "deploy" || return 1 + out=$("$DEPLOY" --rollback 2>&1) && fail "exit 0 with a single release" && return 1 + grep -q 'no release older' <<<"$out" || fail "$out" || return 1 + [[ $(site) == new ]] || fail "site changed" || return 1 } +# --- retention -------------------------------------------------------------- + t_retention_keeps_last_n() { - for v in v1 v2 v3 v4 v5; do - echo "$v" > "$T/src/version"; "$DEPLOY" >/dev/null 2>&1 || fail "deploy $v" || return 1 + for v in v1 v2 v3 v4 v5; do deploy_v $v || fail "deploy $v" || return 1; done + mapfile -t rs < <(releases) + [[ ${#rs[@]} == 3 ]] || fail "expected 3 releases, got ${#rs[@]}: ${rs[*]}" || return 1 + got=$(for r in "${rs[@]}"; do cut -d' ' -f1 "$R/$r/index.html"; done | tr '\n' ' ') + [[ $got == "v3 v4 v5 " ]] || fail "kept wrong releases: $got" || return 1 + [[ $("$DEPLOY" --list-releases 2>/dev/null | grep -c '(live)') == 1 ]] || fail "--list-releases live marker" || return 1 +} + +t_prune_never_removes_live() { + for v in v1 v2 v3; do deploy_v $v || fail "deploy $v" || return 1; done + "$DEPLOY" --rollback >/dev/null 2>&1 && "$DEPLOY" --rollback >/dev/null 2>&1 || fail "rollbacks" || return 1 + bash "$REMOTE" "$DEPLOY_PATH" "$R" 1 prune >/dev/null 2>&1 || fail "prune" || return 1 + [[ $(site) == v1 ]] && page_ok || fail "prune removed the live release" || return 1 +} + +# --- guards ----------------------------------------------------------------- + +t_remote_rejects_bad_names() { + "$DEPLOY" >/dev/null 2>&1 || fail "deploy" || return 1 + keep=$(releases); was=$(live) + for bad in '' . .. '*' '../site' '20261029T120000Z/..' 'x' '20261029T120000Z.tmp'; do + for act in discard switch activate diff; do + bash "$REMOTE" "$DEPLOY_PATH" "$R" 3 $act "$bad" >/dev/null 2>&1 \ + && fail "$act accepted '$bad'" && return 1 + done done - mapfile -t bs < <(backups) - [[ ${#bs[@]} == 3 ]] || fail "expected 3 backups, got ${#bs[@]}: ${bs[*]}" || return 1 - # newest three hold v4, v3, v2 (the tree before deploys v5, v4, v3) - got=$(for b in "${bs[@]}"; do cat "$DEPLOY_PATH.backups/$b/index.html"; done | tr '\n' ' ') - [[ $got == "v2 v3 v4 " ]] || fail "kept wrong backups: $got" || return 1 - [[ $("$DEPLOY" --list-backups 2>/dev/null | wc -l) == 3 ]] || fail "--list-backups" || return 1 + [[ $(releases) == "$keep" && $(live) == "$was" ]] && page_ok || fail "releases or live damaged" || return 1 + # a dir that only looks like a release (glob match, not the exact format) is never pruned + mkdir -p "$R/20xxZ" + bash "$REMOTE" "$DEPLOY_PATH" "$R" 1 prune >/dev/null 2>&1 || fail "prune failed" || return 1 + [[ -d $R/20xxZ ]] || fail "prune touched a non-release dir" || return 1 +} + +t_switch_refuses_plain_dir() { + deploy_v v1 || fail "deploy" || return 1 + name=$(releases | tail -n 1) + rm "$DEPLOY_PATH"; mkdir "$DEPLOY_PATH"; echo hand > "$DEPLOY_PATH/index.html" + bash "$REMOTE" "$DEPLOY_PATH" "$R" 3 switch "$name" >/dev/null 2>&1 && fail "replaced a real dir" && return 1 + [[ $(cat "$DEPLOY_PATH/index.html") == hand ]] || fail "real dir touched" || return 1 } t_rejects_unsafe_paths() { for p in '' relative/path / /var '/srv/www/../etc' '/srv/my site'; do DEPLOY_PATH=$p "$DEPLOY" --dry-run >/dev/null 2>&1 && fail "accepted DEPLOY_PATH='$p'" && return 1 done - DEPLOY_BACKUP_DIR=$DEPLOY_PATH/bak "$DEPLOY" --dry-run >/dev/null 2>&1 \ - && fail "accepted backup dir inside docroot" && return 1 - DEPLOY_BACKUP_DIR=/tmp "$DEPLOY" --dry-run >/dev/null 2>&1 \ - && fail "accepted shallow backup dir" && return 1 + DEPLOY_RELEASES=$DEPLOY_PATH/rel "$DEPLOY" --dry-run >/dev/null 2>&1 \ + && fail "accepted releases dir inside docroot" && return 1 + DEPLOY_RELEASES=/tmp "$DEPLOY" --dry-run >/dev/null 2>&1 \ + && fail "accepted shallow releases dir" && return 1 DEPLOY_KEEP=0 "$DEPLOY" --dry-run >/dev/null 2>&1 && fail "accepted DEPLOY_KEEP=0" && return 1 DEPLOY_LOCK_WAIT=soon "$DEPLOY" --dry-run >/dev/null 2>&1 && fail "accepted DEPLOY_LOCK_WAIT=soon" && return 1 + DEPLOY_SOURCE_DIR=$T/nope "$DEPLOY" --dry-run >/dev/null 2>&1 && fail "accepted missing source dir" && return 1 "$DEPLOY" --bogus >/dev/null 2>&1 && fail "accepted --bogus" && return 1 - [[ $(cat "$DEPLOY_PATH/index.html") == old ]] || fail "docroot changed" || return 1 + [[ ! -L $DEPLOY_PATH && ! -e $R ]] || fail "target touched" || return 1 } t_unreachable_host_fails() { out=$(DEPLOY_HOST=remote DEPLOY_SSH=false "$DEPLOY" 2>&1) && fail "exit 0" && return 1 - [[ $(cat "$DEPLOY_PATH/index.html") == old ]] || fail "docroot changed" || return 1 + [[ ! -L $DEPLOY_PATH && ! -e $R ]] || fail "target touched" || return 1 } t_conf_file_and_env_override() { @@ -167,55 +313,22 @@ t_conf_file_and_env_override() { grep -q "$T/elsewhere/site" <<<"$out" || fail "empty env clobbered conf: $out" || return 1 } -t_stale_tmp_backup_removed() { - mkdir -p "$DEPLOY_PATH.backups/20200101T000000Z.tmp/junk" - "$DEPLOY" >/dev/null 2>&1 || fail "deploy failed" || return 1 - [[ ! -e $DEPLOY_PATH.backups/20200101T000000Z.tmp ]] || fail "stale .tmp left behind" || return 1 - [[ $(backups | wc -l) == 1 ]] || fail "expected 1 backup, got: $(backups)" || return 1 -} - -t_failed_backup_aborts_and_cleans_up() { - echo secret > "$DEPLOY_PATH/unreadable"; chmod 000 "$DEPLOY_PATH/unreadable" - out=$("$DEPLOY" 2>&1) && { chmod 644 "$DEPLOY_PATH/unreadable"; fail "exit 0"; return 1; } - chmod 644 "$DEPLOY_PATH/unreadable" - grep -q 'backup of .* failed' <<<"$out" || fail "no backup-failed message: $out" || return 1 - [[ $(cat "$DEPLOY_PATH/index.html") == old && -f $DEPLOY_PATH/stale.js ]] || fail "docroot changed" || return 1 - [[ -z $(ls -A "$DEPLOY_PATH.backups" | grep -v '^\.lock$') ]] || fail "leftovers: $(ls -A "$DEPLOY_PATH.backups")" || return 1 - "$DEPLOY" >/dev/null 2>&1 || fail "retry after fixing perms failed" || return 1 - [[ $(backups | wc -l) == 1 ]] || fail "retry: expected 1 backup" || return 1 -} - -t_remote_rm_rejects_bad_names() { - "$DEPLOY" >/dev/null 2>&1 || fail "deploy" || return 1 - b=$DEPLOY_PATH.backups; keep=$(backups) - for bad in '' . .. '*' '../site' '20261029T120000Z/..' 'x' '20261029T120000Z.tmp'; do - bash "$ROOT/deploy/remote.sh" "$DEPLOY_PATH" "$b" 3 drop "$bad" >/dev/null 2>&1 \ - && fail "drop accepted '$bad'" && return 1 - bash "$ROOT/deploy/remote.sh" "$DEPLOY_PATH" "$b" 3 restore "$bad" >/dev/null 2>&1 \ - && fail "restore accepted '$bad'" && return 1 - done - [[ $(backups) == "$keep" && -f $b/$keep/index.html && -f $DEPLOY_PATH/index.html ]] \ - || fail "backups or docroot damaged" || return 1 - # a dir that only looks like a backup (glob match, not the exact format) is never pruned or listed - mkdir -p "$b/20xxZ" - bash "$ROOT/deploy/remote.sh" "$DEPLOY_PATH" "$b" 1 prune >/dev/null 2>&1 || fail "prune failed" || return 1 - [[ -d $b/20xxZ ]] || fail "prune touched a non-backup dir" || return 1 -} +# --- locking ---------------------------------------------------------------- t_lock_blocks_concurrent_runs() { - mkdir -p "$DEPLOY_PATH.backups" "$DEPLOY" >/dev/null 2>&1 || fail "deploy" || return 1 + was=$(live) # exec so $holder is the process holding fd 9; killing it releases the lock - ( exec 9>>"$DEPLOY_PATH.backups/.lock"; flock 9; exec sleep 30 ) & holder=$! + ( exec 9>>"$R/.lock"; flock 9; exec sleep 30 ) & holder=$! sleep 0.3 out=$(DEPLOY_LOCK_WAIT=1 "$DEPLOY" 2>&1) && { kill $holder; fail "deploy ran while locked"; return 1; } grep -q 'could not take' <<<"$out" || { kill $holder; fail "no lock message: $out"; return 1; } out=$(DEPLOY_LOCK_WAIT=1 "$DEPLOY" --rollback 2>&1) && { kill $holder; fail "rollback ran while locked"; return 1; } kill $holder; wait $holder 2>/dev/null - [[ $(cat "$DEPLOY_PATH/index.html") == new && $(backups | wc -l) == 1 ]] || fail "state changed while locked" || return 1 + [[ $(live) == "$was" && $(releases | wc -l) == 2 ]] || fail "state changed while locked" || return 1 # released after each run: back-to-back runs don't wait on each other - DEPLOY_LOCK_WAIT=0 "$DEPLOY" >/dev/null 2>&1 || fail "lock not released after deploy" || return 1 - DEPLOY_LOCK_WAIT=0 "$DEPLOY" --rollback >/dev/null 2>&1 || fail "lock not released after rollback" || return 1 + out=$(DEPLOY_LOCK_WAIT=0 "$DEPLOY" 2>&1) || fail "lock not released after deploy: $out" || return 1 + out=$(DEPLOY_LOCK_WAIT=0 "$DEPLOY" --rollback 2>&1) || fail "lock not released after rollback: $out" || return 1 } t_lock_held_during_upload() { @@ -224,7 +337,7 @@ t_lock_held_during_upload() { cat > "$T/probe-ssh" <> "$T/probe" || echo held >> "$T/probe" + flock -n "$R/.lock" true && echo free >> "$T/probe" || echo held >> "$T/probe" fi exec "$WORK/fake-ssh" "\$@" EOF @@ -233,6 +346,8 @@ EOF [[ $(cat "$T/probe") == held ]] || fail "lock not held during rsync: $(cat "$T/probe")" || return 1 } +# --- CI ref selection and old sources --------------------------------------- + t_select_ref_only_merged() { [[ $DEPLOY_HOST == local ]] || return 77 # skip: git-only; once is enough g=$T/repo; sel=$ROOT/deploy/select-ref.sh @@ -246,10 +361,31 @@ t_select_ref_only_merged() { [[ $("$sel" main "$main") == "$main" ]] || fail "main itself" || return 1 for bad in pr "$pr" nosuchref -x --help "$main~5"; do "$sel" "$bad" "$main" >/dev/null 2>&1 && fail "accepted '$bad'" && return 1 + [[ ! -e $T/x ]] || fail "exported for rejected '$bad'" || return 1 + "$sel" "$bad" "$main" "$T/x" >/dev/null 2>&1; [[ ! -e $T/x ]] || fail "exported for rejected '$bad'" || return 1 done return 0 } +# Codex P2: redeploying a commit from before bin/deploy existed. The source +# comes from that commit, and the tooling (this bin/deploy) from the current one. +PRE_TOOLING=6813323575a2f29d2a289139aad4450f77676e89 +t_redeploy_pre_tooling_commit() { + [[ $DEPLOY_HOST == local ]] || return 77 # skip: git-only; once is enough + git -C "$ROOT" cat-file -e "$PRE_TOOLING^{commit}" 2>/dev/null \ + || fail "commit $PRE_TOOLING not in history (CI must use fetch-depth: 0)" || return 1 + cd "$ROOT" && src=$T/old-source + sha=$("$ROOT/deploy/select-ref.sh" "$PRE_TOOLING" HEAD "$src" 2>&1) || fail "select: $sha" || return 1 + [[ ! -e $src/bin/deploy && ! -e $src/deploy ]] || fail "old commit unexpectedly has tooling" || return 1 + ! grep -q test:deploy "$src/package.json" || fail "old package.json has test:deploy" || return 1 + [[ $(cat "$src/.deploy-sha") == "$PRE_TOOLING" ]] || fail ".deploy-sha" || return 1 + out=$(DEPLOY_SOURCE_DIR=$src DEPLOY_BUILD_CMD='mkdir -p out && cp web/index.html out/' "$DEPLOY" 2>&1) \ + || fail "deploy of old source failed: $out" || return 1 + cmp -s "$DEPLOY_PATH/index.html" <(git -C "$ROOT" show "$PRE_TOOLING:web/index.html") \ + || fail "live index.html is not the old commit's" || return 1 + grep -q "deployed $PRE_TOOLING" <<<"$out" || fail "log does not name the source commit: $out" || return 1 +} + cases=$(declare -F | awk '{print $3}' | grep '^t_') for mode in local ssh; do for c in $cases; do From a357e30dc8c728d3e25fd665932ac78b2ed415f9 Mon Sep 17 00:00:00 2001 From: tangentstorm Date: Fri, 30 Oct 2026 00:33:41 +0000 Subject: [PATCH 5/6] deploy: keep recent releases' files reachable across switches Codex P2 on #7 (remote.sh:64): a browser that loaded index.html from release N and fetches its hashed asset after the switch to N+1 got a 404, because URLs resolve through the docroot symlink and N+1 doesn't ship that file. Before every switch (deploy, migration of a plain docroot, rollback, roll-forward), the target release now gets hard links to every file the last DEPLOY_ASSET_GRACE (default 3) live releases shipped and it lacks. Its own files are never overwritten. Shipped files are recorded per release (RELEASES/.files) and live history in RELEASES/.history, so the window is bounded and follows what browsers actually saw. Prune keeps releases in the window, and manifest entries with .. or a leading / are ignored. Tests reproduce the old-index-then-switch 404 over HTTP for deploy, rollback and roll-forward, and cover removed data files, no-overwrite, the window bound, prune vs. the window, bad manifest entries, and grace 0. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 2 +- bin/deploy | 11 ++++- deploy/README.md | 39 +++++++++++++-- deploy/remote.sh | 63 +++++++++++++++++++++--- test/deploy.test.sh | 115 ++++++++++++++++++++++++++++++++++++++++---- 5 files changed, 206 insertions(+), 24 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index afd110a..190c35a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -213,7 +213,7 @@ npm run preview The Vite build automatically runs `npm run build` (TypeScript compilation) first via the `ensureCoreBuild` plugin, then bundles everything for the browser. -**Deployment**: `dist-web/` is the live site served at implish.org (nginx on 45.79.174.182). Deploy only with `bin/deploy`; never copy files by hand. Pushes to main deploy automatically via `.github/workflows/deploy.yml`. Each deploy uploads a new release dir and atomically switches the docroot symlink to it. Use `bin/deploy --dry-run` to preview and `bin/deploy --rollback` to switch back to the previous release. CI runs the deploy tooling from the workflow's own commit and only builds the selected (merged) commit's source. Host, user and docroot live in `deploy/deploy.conf` and the `production` environment's `DEPLOY_*` variables; the deploy key is an environment secret released only to main. Server-side steps are in `deploy/remote.sh`. See `deploy/README.md`. Tests: `npm run test:deploy`. Vite 7 needs Node >= 20.19. +**Deployment**: `dist-web/` is the live site served at implish.org (nginx on 45.79.174.182). Deploy only with `bin/deploy`; never copy files by hand. Pushes to main deploy automatically via `.github/workflows/deploy.yml`. Each deploy uploads a new release dir and atomically switches the docroot symlink to it; files from the last `DEPLOY_ASSET_GRACE` live releases stay reachable so pages loaded before a switch don't 404. Use `bin/deploy --dry-run` to preview and `bin/deploy --rollback` to switch back to the previous release. CI runs the deploy tooling from the workflow's own commit and only builds the selected (merged) commit's source. Host, user and docroot live in `deploy/deploy.conf` and the `production` environment's `DEPLOY_*` variables; the deploy key is an environment secret released only to main. Server-side steps are in `deploy/remote.sh`. See `deploy/README.md`. Tests: `npm run test:deploy`. Vite 7 needs Node >= 20.19. ### TypeScript Details diff --git a/bin/deploy b/bin/deploy index 0ca837e..7772b6a 100755 --- a/bin/deploy +++ b/bin/deploy @@ -16,6 +16,9 @@ # DEPLOY_PATH absolute path of the live docroot (nginx root); becomes a symlink # DEPLOY_RELEASES where releases live (default: DEPLOY_PATH.releases) # DEPLOY_KEEP how many releases to keep, live one included (default 5) +# DEPLOY_ASSET_GRACE files of the last N live releases stay reachable after a +# switch, so pages loaded before it can still fetch their +# hashed assets (default 3; 0 disables) # DEPLOY_SSH ssh command (default: ssh -o BatchMode=yes) # DEPLOY_SOURCE_DIR source tree to build (default: this checkout). CI points # it at an exported older commit to redeploy that commit @@ -38,7 +41,7 @@ log() { echo "deploy: $*" >&2; } # ignored, because CI maps unset repository variables to empty strings. declare -A from_env=() for v in DEPLOY_HOST DEPLOY_USER DEPLOY_PATH DEPLOY_RELEASES DEPLOY_KEEP \ - DEPLOY_SSH DEPLOY_SOURCE_DIR DEPLOY_BUILD_CMD DEPLOY_SRC DEPLOY_LOCK_WAIT \ + DEPLOY_SSH DEPLOY_ASSET_GRACE DEPLOY_SOURCE_DIR DEPLOY_BUILD_CMD DEPLOY_SRC DEPLOY_LOCK_WAIT \ DEPLOY_RSYNC; do [[ -n ${!v:-} ]] && from_env[$v]=${!v} done @@ -52,6 +55,7 @@ DEPLOY_HOST=${DEPLOY_HOST:-} DEPLOY_USER=${DEPLOY_USER:-} DEPLOY_PATH=${DEPLOY_PATH:-} DEPLOY_KEEP=${DEPLOY_KEEP:-5} +DEPLOY_ASSET_GRACE=${DEPLOY_ASSET_GRACE:-3} DEPLOY_SSH=${DEPLOY_SSH:-ssh -o BatchMode=yes} DEPLOY_BUILD_CMD=${DEPLOY_BUILD_CMD:-npx vite build} DEPLOY_SRC=${DEPLOY_SRC:-dist-web} @@ -91,6 +95,7 @@ check_path DEPLOY_RELEASES "$DEPLOY_RELEASES" [[ $DEPLOY_RELEASES != "$DEPLOY_PATH" && $DEPLOY_RELEASES/ != "$DEPLOY_PATH"/* ]] \ || die "DEPLOY_RELEASES must not be inside DEPLOY_PATH" [[ $DEPLOY_KEEP =~ ^[1-9][0-9]*$ ]] || die "DEPLOY_KEEP must be a positive integer: '$DEPLOY_KEEP'" +[[ $DEPLOY_ASSET_GRACE =~ ^[0-9]+$ ]] || die "DEPLOY_ASSET_GRACE must be a whole number: '$DEPLOY_ASSET_GRACE'" DEPLOY_LOCK_WAIT=${DEPLOY_LOCK_WAIT:-60} [[ $DEPLOY_LOCK_WAIT =~ ^[0-9]+$ ]] || die "DEPLOY_LOCK_WAIT must be a whole number of seconds: '$DEPLOY_LOCK_WAIT'" @@ -130,7 +135,7 @@ rsync_to() { # SRC DEST [extra rsync args...] # The target-side half lives in deploy/remote.sh and is piped to bash. REMOTE_LIB=$(<"$ROOT/deploy/remote.sh") -remote() { on_target "$REMOTE_LIB" "$DEPLOY_PATH" "$DEPLOY_RELEASES" "$DEPLOY_KEEP" "$@"; } +remote() { on_target "$REMOTE_LIB" "$DEPLOY_PATH" "$DEPLOY_RELEASES" "$DEPLOY_KEEP" "$DEPLOY_ASSET_GRACE" "$@"; } # hold_lock: take $DEPLOY_RELEASES/.lock on the target and keep it until # this script exits, so a manual --rollback can't interleave with a CI deploy. @@ -163,6 +168,7 @@ case $mode in if ((dry_run)); then log "dry run: switching $target to release $name would change:" remote diff "$name" + ((DEPLOY_ASSET_GRACE == 0)) || log "(files listed as deleted stay reachable: they are carried over from recent releases)" log "dry run: nothing changed" exit 0 fi @@ -191,6 +197,7 @@ if ((dry_run)); then (cd "$DEPLOY_SRC" && find . -type f -printf '>f+++++++++ %P\n' | LC_ALL=C sort) else rsync_to "$DEPLOY_SRC/" "$target/" --dry-run + ((DEPLOY_ASSET_GRACE == 0)) || log "(files listed as deleted stay reachable until $DEPLOY_ASSET_GRACE more releases have gone live; see DEPLOY_ASSET_GRACE)" fi log "dry run: nothing changed" exit 0 diff --git a/deploy/README.md b/deploy/README.md index ae5901a..52fd41f 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -32,6 +32,32 @@ live `index.html`, and an interrupted upload leaves the old site untouched. Rollback is the same atomic switch to an older release. Older releases are the backups. +### Grace period: no 404s for pages loaded just before a switch + +A browser can fetch `index.html` from release N, then request the hashed +`assets/index-AbC123.js` that it names after the site has switched to N+1, +which doesn't contain that file. The same is true of any other file a page +loads later, such as fetched data like `b4-image.json`, and of rollbacks in +either direction. Keeping old releases on disk doesn't help, because URLs +resolve through the docroot symlink. + +So before **every** switch (deploy, rollback, roll-forward), the target +release gets hard links to every file that the last `DEPLOY_ASSET_GRACE` +(default 3) *live* releases shipped and the target lacks: + +- Its own files are never overwritten, so a changed `data.json` serves the + new version, while a removed one stays reachable. +- Each release's shipped files are recorded in `RELEASES/.files`. Files + that were only carried aren't recorded, so the window doesn't snowball: + a file disappears once it has been out of every one of the last N live + releases. +- `RELEASES/.history` records which release was live when, so after a + rollback the window follows what browsers actually saw, not name order. +- Prune never removes a release inside the window. Files already carried + are hard links, so they also survive their source release being pruned. +- Consequence: a file you delete stays reachable for N more deploys. Dry + runs say so. `DEPLOY_ASSET_GRACE=0` turns this off. + What a deploy does, in order (any failure exits nonzero and stops): 1. Validates config. `DEPLOY_PATH` and `DEPLOY_RELEASES` must be absolute, @@ -54,15 +80,17 @@ What a deploy does, in order (any failure exits nonzero and stops): everything on the target is created under `umask 022`, so nginx can read it whatever umask the deploy shell has. If the upload fails, the `.tmp` is removed and the live site is unchanged. -6. Activates: renames `.tmp` to its final name and atomically switches the - symlink. On the **first deploy**, if the docroot is still a plain +6. Activates: records the release's file list, renames `.tmp` to its final + name, carries in the grace-window files (see above), and atomically + switches the symlink. On the **first deploy**, if the docroot is still a plain directory, it is moved into the releases dir (one second older than the new release, so it becomes the rollback target). That migration is two back-to-back renames, so for a moment the path is missing. This is the one non-atomic step, and it happens once. An empty or missing docroot just becomes the symlink. 7. Prunes down to the newest `DEPLOY_KEEP` releases (default 5). The live - release is never pruned, even after a rollback to an old one. + release and the releases in the grace window are never pruned, even + after a rollback to an old one. The server-side half is `deploy/remote.sh`, which `bin/deploy` pipes to `bash` over ssh. Every `rm -rf` there checks that the name is exactly a @@ -88,6 +116,7 @@ of the same name overrides them. See `bin/deploy --help` for the full list. | `DEPLOY_USER` | **unknown: fill in** (repo variable or deploy.conf) | | `DEPLOY_PATH` | **unknown: fill in** the nginx `root` for implish.org | | `DEPLOY_RELEASES` | default `$DEPLOY_PATH.releases` | +| `DEPLOY_ASSET_GRACE` | 3 (live releases whose files stay reachable after a switch) | | `DEPLOY_SOURCE_DIR` | default: this checkout (CI sets it to the exported source) | | `DEPLOY_KEEP` | 5 | @@ -209,6 +238,8 @@ docroots in `.deploy-test/`, both with `DEPLOY_HOST=local` and through a fake `ssh` that runs the remote half locally, so the ssh/rsync `-e` path is exercised too. An rsync wrapper (`DEPLOY_RSYNC`) checks that the live tree stays byte-identical for the whole upload, and simulates cut-off and killed -transfers. One case exports the real pre-tooling commit `6813323` from this +transfers. The grace-period tests serve the docroot with `python3 -m +http.server`, which resolves the symlink per request like nginx. They fetch +`index.html`, switch (deploy or rollback), then fetch the asset it named. One case exports the real pre-tooling commit `6813323` from this repo's history and deploys it with the current `bin/deploy`; CI checks out with `fetch-depth: 0` for that. diff --git a/deploy/remote.sh b/deploy/remote.sh index db7363f..18d2028 100644 --- a/deploy/remote.sh +++ b/deploy/remote.sh @@ -1,7 +1,7 @@ # deploy/remote.sh: the half of bin/deploy that runs where the docroot lives. # bin/deploy pipes this file to `bash -s` (over ssh, or locally), with args: # -# remote.sh DOCROOT RELEASES KEEP ACTION [ARG] +# remote.sh DOCROOT RELEASES KEEP GRACE ACTION [ARG] # # Layout: every deploy is a complete tree in RELEASES/, and # DOCROOT is a symlink to the live one. A deploy uploads into @@ -10,6 +10,15 @@ # whole old tree or the whole new one, never a mix, even if a transfer is # interrupted. Older releases double as the backups. # +# Grace period: a browser may load index.html from one release and fetch its +# hashed assets (or any other file) after the switch. So before every switch, +# deploy or rollback, in either direction, the target release gets hard links +# to every file that the last GRACE live releases shipped and the target +# lacks. Its own files are never overwritten. Each release's shipped files +# are listed in RELEASES/.files, and RELEASES/.history records which +# release was live when, so the window follows what browsers actually saw, +# not name order. Prune never removes a release inside the window. +# # The caller holds $RELEASES/.lock for the whole deploy or rollback (see # hold_lock in bin/deploy), so the actions here don't lock on their own. # Release names are UTC timestamps (20261029T120000Z), so name order is age @@ -18,11 +27,12 @@ set -euo pipefail # nginx (another user) must be able to traverse and read everything we # create, whatever umask the ssh session or CI runner has. umask 022 -docroot=$1 releases=$2 keep=$3 action=$4 arg=${5:-} +docroot=$1 releases=$2 keep=$3 grace=$4 action=$5 arg=${6:-} +history=$releases/.history die() { echo "deploy: $*" >&2; exit 1; } valid_name() { [[ $1 =~ ^20[0-9]{6}T[0-9]{6}Z$ ]]; } -rm_release() { # NAME [SUFFIX]: remove one release (or its .tmp) by validated name +rm_release() { # NAME [SUFFIX]: remove one release (or its .tmp/.files) by validated name valid_name "$1" || die "refusing to remove invalid release name: '$1'" rm -rf -- "${releases:?}/$1${2:-}" } @@ -36,6 +46,34 @@ live() { # name of the release DOCROOT points at, or nothing local t; t=$(readlink "$docroot") if [[ $(dirname "$t") == "$releases" ]] && valid_name "$(basename "$t")"; then basename "$t"; fi } +recent() { # the last GRACE distinct releases that were live, newest first + ((grace > 0)) || return 0 + { live; [[ -f $history ]] && tac "$history"; } \ + | grep -E '^20[0-9]{6}T[0-9]{6}Z$' | awk '!seen[$0]++' | head -n "$grace" || true +} +manifest() { # DIR > list of files a release ships (not certbot's, not carried ones) + (cd "$1" && find . -type f ! -path './.well-known/*' -printf '%P\n' | LC_ALL=C sort) +} +carry() { # TARGET_DIR: hard-link in files the recent live releases shipped and TARGET lacks + local target=$1 prev f n=0 + for prev in $(recent); do + [[ -d $releases/$prev && $releases/$prev != "$target" ]] || continue + [[ -f $releases/$prev.files ]] || manifest "$releases/$prev" > "$releases/$prev.files" + while IFS= read -r f; do + # manifests are ours, but never let one point outside the release + [[ -n $f && $f != /* && /$f/ != */../* ]] || continue + [[ -e $target/$f || -L $target/$f || ! -f $releases/$prev/$f ]] && continue + mkdir -p "$(dirname "$target/$f")" + ln "$releases/$prev/$f" "$target/$f" 2>/dev/null || cp -p "$releases/$prev/$f" "$target/$f" + n=$((n+1)) + done < "$releases/$prev.files" + done + ((n == 0)) || echo "deploy: kept $n file(s) from recent releases reachable" >&2 +} +remember() { # NAME: record that NAME went live + echo "$1" >> "$history" + tail -n 100 "$history" > "$history.new" && mv "$history.new" "$history" +} clean_tmp() { # leftovers of an upload that failed or was killed [[ -d $releases ]] || return 0 find "$releases" -mindepth 1 -maxdepth 1 -type d -name '20*Z.tmp' -printf '%f\n' \ @@ -56,12 +94,15 @@ check_release() { # NAME: must be an existing, complete release valid_name "$1" && [[ -d $releases/$1 ]] || die "no such release: $1" [[ -f $releases/$1/index.html ]] || die "release $1 has no index.html; refusing to make it live" } -switch() { # NAME: atomically point DOCROOT at a release +switch() { # NAME: carry recent files into it, then atomically point DOCROOT at it check_release "$1" [[ -L $docroot || ! -e $docroot ]] || die "$docroot is not a symlink; refusing to replace it" + [[ -f $releases/$1.files ]] || manifest "$releases/$1" > "$releases/$1.files" + carry "$releases/$1" mkdir -p "$(dirname "$docroot")" ln -sfn "$releases/$1" "$docroot.switch" mv -T "$docroot.switch" "$docroot" + remember "$1" } case $action in @@ -81,6 +122,7 @@ case $action in activate) # ARG: finish an upload and make it live valid_name "$arg" || die "invalid release name: '$arg'" [[ -f $releases/$arg.tmp/index.html ]] || die "upload $arg.tmp is incomplete; live site unchanged" + manifest "$releases/$arg.tmp" > "$releases/$arg.files" mv -T "$releases/$arg.tmp" "$releases/$arg" if [[ -d $docroot && ! -L $docroot ]]; then # First deploy onto a plain directory: keep it as the oldest release. @@ -90,20 +132,25 @@ case $action in else legacy=$(date -u -d "@$(( $(date -u -d "${arg:0:8} ${arg:9:2}:${arg:11:2}:${arg:13:2}" +%s) - 1 ))" +%Y%m%dT%H%M%SZ) [[ ! -e $releases/$legacy ]] || die "cannot migrate $docroot: $releases/$legacy exists" + manifest "$docroot" > "$releases/$legacy.files" ln -sfn "$releases/$arg" "$docroot.switch" mv -T "$docroot" "$releases/$legacy" + remember "$legacy" + carry "$releases/$arg" # the old site's files, before anyone is pointed at the new tree mv -T "$docroot.switch" "$docroot" + remember "$arg" echo "deploy: moved the old docroot to release $legacy" >&2 + exit 0 fi fi switch "$arg" ;; switch) switch "$arg" ;; - prune) # keep the newest KEEP releases, and always the live one + prune) # keep the newest KEEP releases, the live one, and the grace window clean_tmp - live=$(live) + protect=" $(recent | tr '\n' ' ') $(live) " list | head -n -"$keep" | while read -r old; do - [[ $old == "$live" ]] && continue - rm_release "$old"; echo "deploy: pruned release $old" >&2 + [[ $protect == *" $old "* ]] && continue + rm_release "$old"; rm_release "$old" .files; echo "deploy: pruned release $old" >&2 done ;; pick) # print the release to roll back to: ARG, or the newest one older than live if [[ -n $arg ]]; then diff --git a/test/deploy.test.sh b/test/deploy.test.sh index b4c45bd..cbc3c12 100755 --- a/test/deploy.test.sh +++ b/test/deploy.test.sh @@ -46,6 +46,17 @@ releases() { ls "$R" 2>/dev/null | grep -E '^20[0-9]{6}T[0-9]{6}Z$' || true; } live() { readlink "$DEPLOY_PATH"; } site() { head -c 100 "$DEPLOY_PATH/index.html" | cut -d' ' -f1; } # version that is live tree() { (cd "$1" && find . -type f -printf '%P\t' -exec cat {} \; | LC_ALL=C sort); } +# a release's own files (its manifest) with contents: what it shipped, ignoring carried files +own() { (cd "$R/$1" && while IFS= read -r f; do printf '%s\t%s\n' "$f" "$(cat "$f")"; done < "$R/$1.files"); } +# serve DEPLOY_PATH over HTTP like nginx: the symlink is resolved per request +serve() { + HTTP_PORT=$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1",0)); print(s.getsockname()[1])') + python3 -m http.server --bind 127.0.0.1 --directory "$DEPLOY_PATH" "$HTTP_PORT" >/dev/null 2>&1 & HTTP_PID=$! + for _ in $(seq 50); do curl -fs "http://127.0.0.1:$HTTP_PORT/" >/dev/null && return 0; sleep 0.1; done + return 1 +} +get() { curl -s -o "$T/body" -w '%{http_code}' "http://127.0.0.1:$HTTP_PORT/$1"; } # prints status +asset_of() { cut -d' ' -f2 <<<"$1"; } # the asset an index.html body refers to deploy_v() { echo "$1" > "$T/src/version"; "$DEPLOY" >/dev/null 2>&1; } # the page works only if the asset its index.html names is present page_ok() { local a; a=$(cut -d' ' -f2 "$DEPLOY_PATH/index.html"); [[ -f $DEPLOY_PATH/$a ]]; } @@ -90,7 +101,9 @@ t_deploy_migrates_plain_docroot() { "$DEPLOY" >/dev/null 2>&1 || fail "deploy failed" || return 1 [[ -L $DEPLOY_PATH ]] || fail "docroot is not a symlink now" || return 1 [[ $(site) == new ]] && page_ok || fail "new site not live/complete" || return 1 - [[ ! -e $DEPLOY_PATH/stale.js && ! -e $DEPLOY_PATH/assets/app-old.js ]] || fail "old files in new release" || return 1 + # the old site's files stay reachable (carried) but aren't part of the new release + [[ -f $DEPLOY_PATH/stale.js && -f $DEPLOY_PATH/assets/app-old.js ]] || fail "old files not kept reachable" || return 1 + ! grep -q 'stale.js\|app-old.js' "$R/$(basename "$(live)").files" || fail "carried files in new manifest" || return 1 [[ -f $DEPLOY_PATH/.well-known/acme/tok ]] || fail ".well-known not carried over" || return 1 [[ $(stat -c %a "$DEPLOY_PATH/index.html") == 644 ]] || fail "index.html not 644" || return 1 mapfile -t rs < <(releases) @@ -192,11 +205,15 @@ t_rollback_restores_previous() { "$DEPLOY" >/dev/null 2>&1 || fail "deploy failed" || return 1 newrel=$(live) out=$("$DEPLOY" --rollback --dry-run 2>&1) || fail "rollback dry run: $out" || return 1 - grep -q 'app-old.js' <<<"$out" || fail "dry run did not list changes: $out" || return 1 + grep -q 'index.html' <<<"$out" || fail "dry run did not list changes: $out" || return 1 [[ $(live) == "$newrel" ]] || fail "rollback dry run switched" || return 1 "$DEPLOY" --rollback >/dev/null 2>&1 || fail "rollback failed" || return 1 - [[ $(tree "$DEPLOY_PATH") == "$before" ]] || fail "tree differs after rollback: + # every file of the old tree is back, byte for byte; the only extras are the + # rolled-back release's files, kept reachable for pages loaded from it + extra=$(comm -13 <(echo "$before") <(tree "$DEPLOY_PATH")) + [[ -z $(comm -23 <(echo "$before") <(tree "$DEPLOY_PATH")) ]] || fail "old tree not restored: $(diff <(echo "$before") <(tree "$DEPLOY_PATH"))" || return 1 + [[ $(cut -f1 <<<"$extra" | tr '\n' ' ') == "assets/app-new.js " ]] || fail "unexpected extras: $extra" || return 1 [[ -L $DEPLOY_PATH ]] && page_ok || fail "rollback broke the symlink/page" || return 1 # the release we left is kept, so we can roll forward by name "$DEPLOY" --rollback "$(basename "$newrel")" >/dev/null 2>&1 && [[ $(site) == new ]] || fail "roll forward" || return 1 @@ -205,10 +222,10 @@ $(diff <(echo "$before") <(tree "$DEPLOY_PATH"))" || return 1 t_rollback_is_one_atomic_switch() { deploy_v v1 && deploy_v v2 || fail "deploys" || return 1 v1=$(releases | tail -n 2 | head -n 1); v2=$(releases | tail -n 1) - t1=$(tree "$R/$v1"); t2=$(tree "$R/$v2") + t1=$(own "$v1"); t2=$(own "$v2") "$DEPLOY" --rollback >/dev/null 2>&1 || fail "rollback" || return 1 [[ $(live) == "$R/$v1" ]] || fail "live is $(live), expected $v1" || return 1 - [[ $(tree "$R/$v1") == "$t1" && $(tree "$R/$v2") == "$t2" ]] || fail "rollback modified a release" || return 1 + [[ $(own "$v1") == "$t1" && $(own "$v2") == "$t2" ]] || fail "rollback changed a release's own files" || return 1 [[ -z $(ls -a "$(dirname "$DEPLOY_PATH")" | grep switch) ]] || fail "temp symlink left behind" || return 1 } @@ -252,10 +269,90 @@ t_retention_keeps_last_n() { t_prune_never_removes_live() { for v in v1 v2 v3; do deploy_v $v || fail "deploy $v" || return 1; done "$DEPLOY" --rollback >/dev/null 2>&1 && "$DEPLOY" --rollback >/dev/null 2>&1 || fail "rollbacks" || return 1 - bash "$REMOTE" "$DEPLOY_PATH" "$R" 1 prune >/dev/null 2>&1 || fail "prune" || return 1 + bash "$REMOTE" "$DEPLOY_PATH" "$R" 1 3 prune >/dev/null 2>&1 || fail "prune" || return 1 [[ $(site) == v1 ]] && page_ok || fail "prune removed the live release" || return 1 } +# --- grace period: pages from release N can fetch N's files after a switch --- + +# Codex P2 repro: a browser loads index.html, the site switches, then the +# browser fetches the asset that index.html named. Over real HTTP. +t_old_index_then_switch_no_404() { + serve || fail "http server did not start" || return 1 + trap 'kill $HTTP_PID 2>/dev/null' RETURN + for v in v1 v2 v3; do + [[ $(get /) == 200 ]] || fail "GET / before $v" || return 1 + page=$(cat "$T/body"); asset=$(asset_of "$page") + deploy_v $v || fail "deploy $v" || return 1 + [[ $(get /) == 200 && $(cat "$T/body") == "$v assets/app-$v.js" ]] || fail "$v not live" || return 1 + code=$(get "$asset") + [[ $code == 200 ]] || fail "after switching to $v, the previous page's $asset is $code" || return 1 + [[ $(cat "$T/body") == "$(cut -d' ' -f1 <<<"$page")-js" ]] || fail "$asset has the wrong content" || return 1 + done +} + +t_old_index_then_rollback_no_404() { + deploy_v v1 && deploy_v v2 || fail "deploys" || return 1 + serve || fail "http server" || return 1 + trap 'kill $HTTP_PID 2>/dev/null' RETURN + get / >/dev/null; page2=$(cat "$T/body") + "$DEPLOY" --rollback >/dev/null 2>&1 || fail "rollback" || return 1 + [[ $(get "$(asset_of "$page2")") == 200 ]] || fail "rollback: v2 page's asset gone" || return 1 + get / >/dev/null; page1=$(cat "$T/body"); [[ $page1 == "v1 "* ]] || fail "v1 not live" || return 1 + # roll forward again: pages loaded from v1 during the rollback keep working + "$DEPLOY" --rollback "$(releases | tail -n 1)" >/dev/null 2>&1 || fail "roll forward" || return 1 + [[ $(get "$(asset_of "$page1")") == 200 ]] || fail "roll forward: v1 page's asset gone" || return 1 +} + +t_removed_data_file_stays_reachable() { + # not just hashed assets: any file a page may fetch (e.g. b4-image.json) + DEPLOY_BUILD_CMD="$DEPLOY_BUILD_CMD"' && echo "$v data" > out/data.json && echo "$v" > out/only-$v.json' \ + "$DEPLOY" >/dev/null 2>&1 || fail "deploy with data" || return 1 + echo v2 > "$T/src/version" + DEPLOY_BUILD_CMD="$DEPLOY_BUILD_CMD"' && echo "$v data" > out/data.json' "$DEPLOY" >/dev/null 2>&1 || fail "deploy 2" || return 1 + [[ $(cat "$DEPLOY_PATH/only-new.json" 2>/dev/null) == new ]] || fail "removed data file not reachable" || return 1 + # a file the new release ships itself is never overwritten by an older copy + [[ $(cat "$DEPLOY_PATH/data.json") == "v2 data" ]] || fail "carried file overwrote the new release's own" || return 1 +} + +t_grace_window_is_bounded() { + for v in v1 v2 v3 v4; do DEPLOY_ASSET_GRACE=2 deploy_v $v || fail "deploy $v" || return 1; done + # live v4; the two releases live before it (v3, v2) stay reachable, v1 has aged out + for v in v4 v3 v2; do [[ -f $DEPLOY_PATH/assets/app-$v.js ]] || fail "app-$v.js missing" || return 1; done + [[ ! -e $DEPLOY_PATH/assets/app-v1.js ]] || fail "app-v1.js still carried after the window" || return 1 + [[ ! -e $DEPLOY_PATH/assets/app-new.js && ! -e $DEPLOY_PATH/assets/app-old.js ]] || fail "older files still carried" || return 1 +} + +t_prune_keeps_grace_window() { + # KEEP=1 would keep only the live release; the grace window must win + for v in v1 v2 v3 v4; do DEPLOY_KEEP=1 DEPLOY_ASSET_GRACE=2 deploy_v $v || fail "deploy $v" || return 1; done + mapfile -t rs < <(releases) + got=$(for r in "${rs[@]}"; do cut -d' ' -f1 "$R/$r/index.html"; done | tr '\n' ' ') + [[ $got == "v3 v4 " ]] || fail "expected v3 v4 kept, got: $got" || return 1 + # v2 was pruned, but its files carried into v4 are hard links and survive + [[ $(cat "$DEPLOY_PATH/assets/app-v2.js") == v2-js ]] || fail "pruning v2 broke its carried asset" || return 1 + [[ $(ls "$R" | grep -c '\.files$') == 2 ]] || fail "pruned releases' manifests left behind: $(ls "$R")" || return 1 +} + +t_carry_ignores_bad_manifest_entries() { + deploy_v v1 || fail "deploy" || return 1 + v1=$(basename "$(live)") + # entries with .. or a leading / are skipped even when they resolve to a real file + echo planted > "$R/$v1/planted.txt"; echo planted > "$R/$v1/abs.txt" + printf '%s\n' 'assets/../planted.txt' '/abs.txt' '../../escape' >> "$R/$v1.files" + deploy_v v2 || fail "deploy 2" || return 1 + [[ ! -e $DEPLOY_PATH/planted.txt && ! -e $DEPLOY_PATH/abs.txt ]] || fail "followed a bad manifest entry" || return 1 + [[ -z $(find "$T" -name 'escape*') ]] || fail "manifest entry escaped the release" || return 1 + [[ -f $DEPLOY_PATH/assets/app-v1.js ]] || fail "good entries no longer carried" || return 1 +} + +t_grace_zero_disables_carry() { + DEPLOY_ASSET_GRACE=0 deploy_v v1 || fail "deploy" || return 1 + [[ ! -e $DEPLOY_PATH/assets/app-old.js ]] || fail "carried with grace 0" || return 1 + DEPLOY_ASSET_GRACE=x "$DEPLOY" --dry-run >/dev/null 2>&1 && fail "accepted DEPLOY_ASSET_GRACE=x" && return 1 + return 0 +} + # --- guards ----------------------------------------------------------------- t_remote_rejects_bad_names() { @@ -263,14 +360,14 @@ t_remote_rejects_bad_names() { keep=$(releases); was=$(live) for bad in '' . .. '*' '../site' '20261029T120000Z/..' 'x' '20261029T120000Z.tmp'; do for act in discard switch activate diff; do - bash "$REMOTE" "$DEPLOY_PATH" "$R" 3 $act "$bad" >/dev/null 2>&1 \ + bash "$REMOTE" "$DEPLOY_PATH" "$R" 3 3 $act "$bad" >/dev/null 2>&1 \ && fail "$act accepted '$bad'" && return 1 done done [[ $(releases) == "$keep" && $(live) == "$was" ]] && page_ok || fail "releases or live damaged" || return 1 # a dir that only looks like a release (glob match, not the exact format) is never pruned mkdir -p "$R/20xxZ" - bash "$REMOTE" "$DEPLOY_PATH" "$R" 1 prune >/dev/null 2>&1 || fail "prune failed" || return 1 + bash "$REMOTE" "$DEPLOY_PATH" "$R" 1 3 prune >/dev/null 2>&1 || fail "prune failed" || return 1 [[ -d $R/20xxZ ]] || fail "prune touched a non-release dir" || return 1 } @@ -278,7 +375,7 @@ t_switch_refuses_plain_dir() { deploy_v v1 || fail "deploy" || return 1 name=$(releases | tail -n 1) rm "$DEPLOY_PATH"; mkdir "$DEPLOY_PATH"; echo hand > "$DEPLOY_PATH/index.html" - bash "$REMOTE" "$DEPLOY_PATH" "$R" 3 switch "$name" >/dev/null 2>&1 && fail "replaced a real dir" && return 1 + bash "$REMOTE" "$DEPLOY_PATH" "$R" 3 3 switch "$name" >/dev/null 2>&1 && fail "replaced a real dir" && return 1 [[ $(cat "$DEPLOY_PATH/index.html") == hand ]] || fail "real dir touched" || return 1 } From a01ddb4dedb1b4c14f27398c86b1dbe04d22676b Mon Sep 17 00:00:00 2001 From: tangentstorm Date: Fri, 30 Oct 2026 01:09:46 +0000 Subject: [PATCH 6/6] deploy: every switch leaves the old site serving if anything fails Codex P2 on #7 (remote.sh:139): the first deploy onto a plain docroot moved the live directory away before the fallible carry/remember steps, so a failure (e.g. old downloads/old.json vs. a new regular file named downloads) left the site down. - Split every switch (deploy, migration, rollback, roll-forward) into `stage` (all fallible work, live site untouched) and `commit` (one rename). Post-switch history writes only warn; a prune failure says the new release is live. - Migration stages from the plain docroot before moving it, and restores it if the final rename fails. - Carry never fails a switch: conflicts (file vs. dir either way) and symlinked parents on either side are skipped with a warning. - An exit trap removes the staged symlink and any release that never went live, so retries don't collide and rollback never picks one. - DEPLOY_FAULT= injects failures; tests cover each point for each switch kind, plus the conflict cases. Co-Authored-By: Claude Opus 5.5 --- bin/deploy | 3 +- deploy/README.md | 33 +++++++++++ deploy/remote.sh | 140 +++++++++++++++++++++++++++++++++----------- test/deploy.test.sh | 130 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 272 insertions(+), 34 deletions(-) diff --git a/bin/deploy b/bin/deploy index 7772b6a..464c090 100755 --- a/bin/deploy +++ b/bin/deploy @@ -214,5 +214,6 @@ log "uploading release $name" rsync_to "$DEPLOY_SRC/" "$(at_target "$DEPLOY_RELEASES/$name.tmp")/" "${link[@]}" \ || { remote discard "$name" || true; die "upload failed; live site unchanged"; } remote activate "$name" -remote prune +# After the switch: a failure here leaves the new release live, so say so. +remote prune || die "release $name is live, but pruning old releases failed" log "deployed $(cat "$DEPLOY_SOURCE_DIR/.deploy-sha" 2>/dev/null || git -C "$DEPLOY_SOURCE_DIR" rev-parse HEAD 2>/dev/null || echo '?') as release $name" diff --git a/deploy/README.md b/deploy/README.md index 52fd41f..d80600e 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -32,6 +32,39 @@ live `index.html`, and an interrupted upload leaves the old site untouched. Rollback is the same atomic switch to an older release. Older releases are the backups. +### Failure safety: the old site keeps serving + +Every switch (deploy, the first-deploy migration, rollback, roll-forward) is +split into two parts in `deploy/remote.sh`: + +- **stage**: everything that can fail, with the live site untouched. This + covers the release checks, the file lists, carrying grace-period files, + and creating the new symlink next to the docroot. +- **commit**: one `rename(2)` of that symlink over the docroot. + +After the commit, only bookkeeping runs. A failed history write only +warns. A failed prune exits nonzero with "release X is live, but pruning old +releases failed". + +On any failure, an exit trap removes the staged symlink. If the failure came +before an activation's commit, the trap also removes that never-live +release, so it can't collide with a quick retry or become a rollback target. + +Carrying never fails a switch. A file that can't be kept reachable gets a +warning and is skipped. For example, the old site had `downloads/old.json` +and the new release ships a plain file called `downloads`; or a parent path +is a symlink, which carrying never follows on either side. + +The one-time migration of a plain-directory docroot stages everything first, +with the old directory still serving. Then it does two renames: the +directory into the releases dir, and the symlink into its place. If the +second rename fails, the first is undone and the old site is restored. + +`DEPLOY_FAULT=` injects a failure at a named point +(`stage`, `before-commit`, `migrate-before-move`, `migrate-between-renames`, +`prune`). The tests use it to show that each switch kind leaves the old site +serving. + ### Grace period: no 404s for pages loaded just before a switch A browser can fetch `index.html` from release N, then request the hashed diff --git a/deploy/remote.sh b/deploy/remote.sh index 18d2028..61a1546 100644 --- a/deploy/remote.sh +++ b/deploy/remote.sh @@ -19,6 +19,15 @@ # release was live when, so the window follows what browsers actually saw, # not name order. Prune never removes a release inside the window. # +# Failure safety: every step that can fail (checks, manifests, carrying +# files, creating the new symlink) runs in `stage`, before the live site is +# touched. The switch itself is one rename(2) in `commit`. Bookkeeping after +# it (history) only warns. So whatever fails, the old site keeps serving. +# The single exception is the one-time migration of a plain-directory +# docroot (see `activate`), which restores the old directory if its last +# rename fails. DEPLOY_FAULT= injects a failure at a named point; the +# tests use it to prove this for each step. +# # The caller holds $RELEASES/.lock for the whole deploy or rollback (see # hold_lock in bin/deploy), so the actions here don't lock on their own. # Release names are UTC timestamps (20261029T120000Z), so name order is age @@ -31,6 +40,10 @@ docroot=$1 releases=$2 keep=$3 grace=$4 action=$5 arg=${6:-} history=$releases/.history die() { echo "deploy: $*" >&2; exit 1; } +warn() { echo "deploy: warning: $*" >&2; } +fault() { # POINT: fail here if DEPLOY_FAULT names this point (tests only) + [[ ${DEPLOY_FAULT:-} != "$1" ]] || { echo "deploy: injected fault at $1" >&2; return 1; } +} valid_name() { [[ $1 =~ ^20[0-9]{6}T[0-9]{6}Z$ ]]; } rm_release() { # NAME [SUFFIX]: remove one release (or its .tmp/.files) by validated name valid_name "$1" || die "refusing to remove invalid release name: '$1'" @@ -54,25 +67,50 @@ recent() { # the last GRACE distinct releases that were live, newest first manifest() { # DIR > list of files a release ships (not certbot's, not carried ones) (cd "$1" && find . -type f ! -path './.well-known/*' -printf '%P\n' | LC_ALL=C sort) } -carry() { # TARGET_DIR: hard-link in files the recent live releases shipped and TARGET lacks - local target=$1 prev f n=0 +safe_path() { # ROOT REL: no parent of ROOT/REL (below ROOT) is a symlink or a non-directory + local p=$1 c parts + IFS=/ read -ra parts <<<"$(dirname "$2")" + for c in "${parts[@]}"; do + [[ -z $c || $c == . ]] && continue + p=$p/$c + [[ -L $p ]] && return 1 + [[ -e $p && ! -d $p ]] && return 1 + done + return 0 +} +carried=0 +carry_from() { # SRC_DIR MANIFEST TARGET_DIR: link in what SRC shipped and TARGET lacks + local src=$1 list=$2 target=$3 f + while IFS= read -r f; do + # manifests are ours, but never let one point outside the release + [[ -n $f && $f != /* && /$f/ != */../* ]] || continue + [[ -f $src/$f && ! -L $src/$f ]] && safe_path "$src" "$f" || continue + if [[ -e $target/$f || -L $target/$f ]]; then + [[ -f $target/$f && ! -L $target/$f ]] || warn "can't keep /$f reachable: the new release has a directory or link there" + continue + fi + if ! safe_path "$target" "$f"; then + warn "can't keep /$f reachable: a parent path in the new release is a file or link"; continue + fi + if mkdir -p "$(dirname "$target/$f")" 2>/dev/null && + { ln "$src/$f" "$target/$f" 2>/dev/null || cp -p "$src/$f" "$target/$f" 2>/dev/null; }; then + carried=$((carried+1)) + else + warn "can't keep /$f reachable: copying it failed" + fi + done < "$list" +} +carry() { # TARGET_DIR: carry from the recent live releases. Never fails the switch. + local target=$1 prev for prev in $(recent); do [[ -d $releases/$prev && $releases/$prev != "$target" ]] || continue [[ -f $releases/$prev.files ]] || manifest "$releases/$prev" > "$releases/$prev.files" - while IFS= read -r f; do - # manifests are ours, but never let one point outside the release - [[ -n $f && $f != /* && /$f/ != */../* ]] || continue - [[ -e $target/$f || -L $target/$f || ! -f $releases/$prev/$f ]] && continue - mkdir -p "$(dirname "$target/$f")" - ln "$releases/$prev/$f" "$target/$f" 2>/dev/null || cp -p "$releases/$prev/$f" "$target/$f" - n=$((n+1)) - done < "$releases/$prev.files" + carry_from "$releases/$prev" "$releases/$prev.files" "$target" done - ((n == 0)) || echo "deploy: kept $n file(s) from recent releases reachable" >&2 } -remember() { # NAME: record that NAME went live - echo "$1" >> "$history" - tail -n 100 "$history" > "$history.new" && mv "$history.new" "$history" +remember() { # NAME: record that NAME went live. Runs after the switch: warn, don't fail. + { echo "$1" >> "$history" && tail -n 100 "$history" > "$history.new" && mv "$history.new" "$history"; } \ + || warn "release $1 is live, but recording it in $history failed (grace window may be short)" } clean_tmp() { # leftovers of an upload that failed or was killed [[ -d $releases ]] || return 0 @@ -94,14 +132,39 @@ check_release() { # NAME: must be an existing, complete release valid_name "$1" && [[ -d $releases/$1 ]] || die "no such release: $1" [[ -f $releases/$1/index.html ]] || die "release $1 has no index.html; refusing to make it live" } -switch() { # NAME: carry recent files into it, then atomically point DOCROOT at it +stage() { # NAME [SRC_DIR SRC_MANIFEST]: every fallible step of a switch, live site untouched check_release "$1" - [[ -L $docroot || ! -e $docroot ]] || die "$docroot is not a symlink; refusing to replace it" + [[ -L $docroot.switch ]] && rm -f -- "$docroot.switch" # left by an earlier failed run [[ -f $releases/$1.files ]] || manifest "$releases/$1" > "$releases/$1.files" + carried=0 + if [[ -n ${2:-} ]] && ((grace > 0)); then carry_from "$2" "$3" "$releases/$1"; fi carry "$releases/$1" + ((carried == 0)) || echo "deploy: kept $carried file(s) from recent releases reachable" >&2 + fault stage mkdir -p "$(dirname "$docroot")" - ln -sfn "$releases/$1" "$docroot.switch" - mv -T "$docroot.switch" "$docroot" + ln -sfnT "$releases/$1" "$docroot.switch" + fault before-commit +} +commit() { mv -T "$docroot.switch" "$docroot"; } # the switch: one atomic rename + +# On any failure: drop the staged symlink, and if an activation never reached +# its commit, drop that release too. It was never live, and keeping it would +# collide with a quick retry or become a bogus rollback target. +pending= +on_exit() { + local rc=$? + ((rc != 0)) || return 0 + [[ ! -L $docroot.switch ]] || rm -f -- "$docroot.switch" + if [[ -n $pending ]]; then + rm_release "$pending"; rm_release "$pending" .files + echo "deploy: removed release $pending, which never went live" >&2 + fi +} +trap on_exit EXIT +switch() { # NAME: atomically point the DOCROOT symlink at a release + [[ -L $docroot || ! -e $docroot ]] || die "$docroot is not a symlink; refusing to replace it" + stage "$1" + commit remember "$1" } @@ -123,29 +186,40 @@ case $action in valid_name "$arg" || die "invalid release name: '$arg'" [[ -f $releases/$arg.tmp/index.html ]] || die "upload $arg.tmp is incomplete; live site unchanged" manifest "$releases/$arg.tmp" > "$releases/$arg.files" + pending=$arg mv -T "$releases/$arg.tmp" "$releases/$arg" if [[ -d $docroot && ! -L $docroot ]]; then - # First deploy onto a plain directory: keep it as the oldest release. - # This is the one non-atomic step: two renames back to back. + # First deploy onto a plain directory, which becomes the oldest release. + # Everything fallible is staged first, with the old site still serving. + # Then two renames replace the directory with the symlink. If the + # second one fails, the first is undone, so the old site comes back. if [[ -z $(ls -A "$docroot") ]]; then + stage "$arg" rmdir "$docroot" - else - legacy=$(date -u -d "@$(( $(date -u -d "${arg:0:8} ${arg:9:2}:${arg:11:2}:${arg:13:2}" +%s) - 1 ))" +%Y%m%dT%H%M%SZ) - [[ ! -e $releases/$legacy ]] || die "cannot migrate $docroot: $releases/$legacy exists" - manifest "$docroot" > "$releases/$legacy.files" - ln -sfn "$releases/$arg" "$docroot.switch" - mv -T "$docroot" "$releases/$legacy" - remember "$legacy" - carry "$releases/$arg" # the old site's files, before anyone is pointed at the new tree - mv -T "$docroot.switch" "$docroot" - remember "$arg" - echo "deploy: moved the old docroot to release $legacy" >&2 - exit 0 + commit || { mkdir -p "$docroot"; die "could not install the docroot symlink"; } + pending= + remember "$arg"; exit 0 + fi + legacy=$(date -u -d "@$(( $(date -u -d "${arg:0:8} ${arg:9:2}:${arg:11:2}:${arg:13:2}" +%s) - 1 ))" +%Y%m%dT%H%M%SZ) + [[ ! -e $releases/$legacy ]] || die "cannot migrate $docroot: $releases/$legacy exists" + manifest "$docroot" > "$releases/$legacy.files" + stage "$arg" "$docroot" "$releases/$legacy.files" # the old site's files, from where they are now + fault migrate-before-move + mv -T "$docroot" "$releases/$legacy" + if ! { fault migrate-between-renames && commit; }; then + mv -T "$releases/$legacy" "$docroot" || die "CRITICAL: $docroot is missing; restore it with: mv -T $releases/$legacy $docroot" + rm_release "$legacy" .files + die "could not install the docroot symlink; the old site was restored" fi + pending= + remember "$legacy"; remember "$arg" + echo "deploy: moved the old docroot to release $legacy" >&2 + exit 0 fi - switch "$arg" ;; + switch "$arg"; pending= ;; switch) switch "$arg" ;; prune) # keep the newest KEEP releases, the live one, and the grace window + fault prune clean_tmp protect=" $(recent | tr '\n' ' ') $(live) " list | head -n -"$keep" | while read -r old; do diff --git a/test/deploy.test.sh b/test/deploy.test.sh index cbc3c12..398d6b7 100755 --- a/test/deploy.test.sh +++ b/test/deploy.test.sh @@ -353,6 +353,136 @@ t_grace_zero_disables_carry() { return 0 } +# --- failure safety: whatever fails before the switch, the old site keeps serving --- + +# old site still fully served: docroot present, same tree, page loads +still_serving() { [[ -e $DEPLOY_PATH ]] && [[ $(tree "$DEPLOY_PATH") == "$1" ]] && page_ok; } +no_switch_left() { [[ -z $(ls -A "$(dirname "$DEPLOY_PATH")" | grep '\.switch$') ]]; } +# build variants: ship $1 as a directory with a file in it, or as a plain file +build_dir() { echo "$DEPLOY_BUILD_CMD"' && mkdir -p out/'"$1"' && echo "$v" > out/'"$1"'/old.json'; } +build_file() { echo "$DEPLOY_BUILD_CMD"' && echo "$v" > out/'"$1"; } + +# Codex repro: the old docroot has downloads/old.json; the new release ships a +# regular file called downloads. Carrying old.json can't work; it must not take the site down. +t_migration_file_vs_dir_conflict() { + mkdir -p "$DEPLOY_PATH/downloads"; echo old > "$DEPLOY_PATH/downloads/old.json" + before=$(tree "$DEPLOY_PATH") + out=$(DEPLOY_BUILD_CMD=$(build_file downloads) "$DEPLOY" 2>&1) || fail "deploy failed: $out" || return 1 + grep -q "can't keep /downloads/old.json reachable" <<<"$out" || fail "no conflict warning: $out" || return 1 + [[ -L $DEPLOY_PATH && $(site) == new && $(cat "$DEPLOY_PATH/downloads") == new ]] && page_ok || fail "new site not live" || return 1 + [[ $(tree "$R/$(releases | head -n 1)") == "$before" ]] || fail "old site not kept as a release" || return 1 + [[ -f $DEPLOY_PATH/assets/app-old.js ]] || fail "other old files no longer carried" || return 1 +} + +t_deploy_file_vs_dir_conflict() { + DEPLOY_BUILD_CMD=$(build_dir downloads) deploy_v v1 || fail "deploy v1" || return 1 + out=$(echo v2 > "$T/src/version"; DEPLOY_BUILD_CMD=$(build_file downloads) "$DEPLOY" 2>&1) || fail "deploy v2: $out" || return 1 + grep -q "can't keep /downloads/old.json" <<<"$out" || fail "no warning: $out" || return 1 + [[ $(site) == v2 && -f $DEPLOY_PATH/downloads ]] && page_ok || fail "v2 not live" || return 1 +} + +t_deploy_dir_vs_file_conflict() { + DEPLOY_BUILD_CMD=$(build_file downloads) deploy_v v1 || fail "deploy v1" || return 1 + out=$(echo v2 > "$T/src/version"; DEPLOY_BUILD_CMD=$(build_dir downloads) "$DEPLOY" 2>&1) || fail "deploy v2: $out" || return 1 + grep -q "can't keep /downloads reachable" <<<"$out" || fail "no warning: $out" || return 1 + [[ $(site) == v2 && -f $DEPLOY_PATH/downloads/old.json ]] && page_ok || fail "v2 not live" || return 1 +} + +t_rollback_file_vs_dir_conflict() { + DEPLOY_BUILD_CMD=$(build_file downloads) deploy_v v1 || fail "deploy v1" || return 1 + echo v2 > "$T/src/version"; DEPLOY_BUILD_CMD=$(build_dir downloads) "$DEPLOY" >/dev/null 2>&1 || fail "deploy v2" || return 1 + out=$("$DEPLOY" --rollback 2>&1) || fail "rollback failed: $out" || return 1 + grep -q "can't keep /downloads/old.json" <<<"$out" || fail "no warning: $out" || return 1 + [[ $(site) == v1 && -f $DEPLOY_PATH/downloads ]] && page_ok || fail "v1 not live" || return 1 + out=$("$DEPLOY" --rollback "$(releases | tail -n 1)" 2>&1) || fail "roll forward failed: $out" || return 1 + [[ $(site) == v2 ]] && page_ok || fail "v2 not live again" || return 1 +} + +t_carry_never_writes_through_symlinks() { + mkdir -p "$T/outside" "$T/secret"; echo secret > "$T/secret/s.txt" + DEPLOY_BUILD_CMD=$(build_dir data) deploy_v v1 || fail "deploy v1" || return 1 + v1=$(basename "$(live)") + # the next release ships `data` as a symlink out of the tree: carrying + # data/old.json must not follow it + echo v2 > "$T/src/version" + out=$(DEPLOY_BUILD_CMD="$DEPLOY_BUILD_CMD && ln -s '$T/outside' out/data" "$DEPLOY" 2>&1) || fail "deploy v2: $out" || return 1 + [[ -z $(ls -A "$T/outside") ]] || fail "carry wrote outside the release: $(ls "$T/outside")" || return 1 + # and a source path through a symlink is never carried in + ln -s "$T/secret" "$R/$v1/lnk"; echo lnk/s.txt >> "$R/$v1.files" + echo v3 > "$T/src/version"; "$DEPLOY" >/dev/null 2>&1 || fail "deploy v3" || return 1 + [[ ! -e $DEPLOY_PATH/lnk ]] || fail "carried a file through a symlinked dir" || return 1 +} + +# Inject a failure at every fallible point of every kind of switch. +t_fault_migration_keeps_old_site() { + before=$(tree "$DEPLOY_PATH") + for point in stage before-commit migrate-before-move migrate-between-renames; do + out=$(DEPLOY_FAULT=$point "$DEPLOY" 2>&1) && fail "$point: exit 0" && return 1 + grep -q "injected fault at $point" <<<"$out" || fail "$point: fault not reached: $out" || return 1 + [[ -d $DEPLOY_PATH && ! -L $DEPLOY_PATH ]] || fail "$point: docroot is gone or replaced" || return 1 + still_serving "$before" || fail "$point: old site not serving" || return 1 + [[ -z $(releases) ]] && no_switch_left || fail "$point: leftovers: $(ls -A "$R") $(ls -A "$(dirname "$DEPLOY_PATH")")" || return 1 + done + grep -q 'old site was restored' <<<"$out" || fail "no restore message: $out" || return 1 + "$DEPLOY" >/dev/null 2>&1 || fail "fault-free deploy afterwards failed" || return 1 + [[ $(site) == new ]] && page_ok && no_switch_left || fail "not recovered" || return 1 + [[ $(tree "$R/$(releases | head -n 1)") == "$before" ]] || fail "old site not kept as a release" || return 1 +} + +t_fault_empty_docroot_migration() { + rm -rf "$DEPLOY_PATH"; mkdir "$DEPLOY_PATH" + for point in stage before-commit; do + DEPLOY_FAULT=$point "$DEPLOY" >/dev/null 2>&1 && fail "$point: exit 0" && return 1 + [[ -d $DEPLOY_PATH && ! -L $DEPLOY_PATH ]] || fail "$point: docroot gone" || return 1 + done + "$DEPLOY" >/dev/null 2>&1 && [[ $(site) == new ]] || fail "fault-free deploy afterwards" || return 1 +} + +t_fault_deploy_keeps_old_site() { + deploy_v v1 || fail "deploy v1" || return 1 + before=$(tree "$DEPLOY_PATH"); was=$(live) + for point in stage before-commit; do + echo v2 > "$T/src/version" + out=$(DEPLOY_FAULT=$point "$DEPLOY" 2>&1) && fail "$point: exit 0" && return 1 + grep -q "injected fault at $point" <<<"$out" || fail "$point: fault not reached: $out" || return 1 + [[ $(live) == "$was" ]] && still_serving "$before" || fail "$point: old site not serving" || return 1 + [[ $(releases | tail -n 1) == "$(basename "$was")" ]] || fail "$point: a never-live release was kept: $(releases)" || return 1 + done + deploy_v v3 && [[ $(site) == v3 ]] && no_switch_left || fail "fault-free deploy afterwards" || return 1 +} + +t_fault_rollback_keeps_site() { + deploy_v v1 && deploy_v v2 || fail "deploys" || return 1 + before=$(tree "$DEPLOY_PATH"); was=$(live) + for point in stage before-commit; do + out=$(DEPLOY_FAULT=$point "$DEPLOY" --rollback 2>&1) && fail "rollback $point: exit 0" && return 1 + [[ $(live) == "$was" ]] && still_serving "$before" || fail "rollback $point: site changed" || return 1 + done + "$DEPLOY" --rollback >/dev/null 2>&1 && [[ $(site) == v1 ]] || fail "rollback afterwards" || return 1 + # roll-forward is the same switch + before=$(tree "$DEPLOY_PATH"); was=$(live) + for point in stage before-commit; do + DEPLOY_FAULT=$point "$DEPLOY" --rollback "$(releases | tail -n 1)" >/dev/null 2>&1 && fail "forward $point: exit 0" && return 1 + [[ $(live) == "$was" ]] && still_serving "$before" || fail "forward $point: site changed" || return 1 + done + no_switch_left || fail "temp symlink left" || return 1 +} + +t_after_switch_failures_report_live() { + deploy_v v1 || fail "deploy v1" || return 1 + # history can't be written: the switch already happened, so warn and succeed + chmod 444 "$R/.history" + out=$(echo v2 > "$T/src/version"; "$DEPLOY" 2>&1); rc=$? + chmod 644 "$R/.history" + ((rc == 0)) || fail "history failure failed the deploy: $out" || return 1 + [[ $(site) == v2 ]] || fail "v2 not live" || return 1 + if [[ $(id -u) != 0 ]]; then grep -q 'recording it in' <<<"$out" || fail "no history warning: $out" || return 1; fi + # prune fails after the switch: nonzero, saying the new release is live + out=$(echo v3 > "$T/src/version"; DEPLOY_FAULT=prune "$DEPLOY" 2>&1) && fail "prune failure exited 0" && return 1 + grep -q 'is live, but pruning old releases failed' <<<"$out" || fail "$out" || return 1 + [[ $(site) == v3 ]] && page_ok || fail "v3 not live" || return 1 +} + # --- guards ----------------------------------------------------------------- t_remote_rejects_bad_names() {