diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..2a81171 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,94 @@ +name: deploy implish.org + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + inputs: + ref: + description: Commit, tag or branch to (re)deploy. It must already be merged into main. Empty = main's head. + required: false + default: '' + +# One deploy at a time; a newer push waits rather than cancelling a deploy +# that is halfway through its rsync. +concurrency: + group: ${{ github.event_name == 'pull_request' && format('test-{0}', github.ref) || 'deploy-implish-org' }} + cancel-in-progress: false + +permissions: + contents: read + +# Tooling vs. source: both jobs run bin/deploy, deploy/* and the tests from +# the workflow's own commit (github.sha). Only the application source that +# gets built comes from the `ref` input, exported separately by +# deploy/select-ref.sh. So redeploying an old commit (even one from before +# bin/deploy existed) still uses the current deploy tooling. +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + with: + ref: ${{ github.sha }} + fetch-depth: 0 # the tests export a pre-tooling commit from history + - uses: actions/setup-node@v5 + with: + node-version: 22 + - run: npm install --no-audit --no-fund + - run: npm run build + - run: npm run test:deploy + + # The deploy key lives only in the `production` environment, whose + # deployment branches are limited to main (see deploy/README.md). So a PR + # branch that edits this file still can't read the secret. The `if:` and + # the ancestor check below are defense in depth on top of that. + deploy: + needs: test + if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 15 + environment: + name: production + url: https://implish.org/ + env: + DEPLOY_USER: ${{ vars.DEPLOY_USER }} + DEPLOY_PATH: ${{ vars.DEPLOY_PATH }} + steps: + - uses: actions/checkout@v5 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + - name: Export source to build (only commits already merged into main) + env: + REF: ${{ inputs.ref }} + run: | + src=$RUNNER_TEMP/source + sha=$(deploy/select-ref.sh "$REF" "$GITHUB_SHA" "$src") + echo "DEPLOY_SOURCE_DIR=$src" >> "$GITHUB_ENV" + echo "building ${REF:-main} = $sha with tooling from $GITHUB_SHA" + - uses: actions/setup-node@v5 + with: + node-version: 22 + - name: Install the source's dependencies + run: npm install --no-audit --no-fund + working-directory: ${{ runner.temp }}/source + - name: Install deploy key + env: + DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }} + run: | + if [ -z "$DEPLOY_SSH_KEY" ]; then + echo "::error::secret DEPLOY_SSH_KEY is not set; see deploy/README.md" + exit 1 + fi + install -m 700 -d "$RUNNER_TEMP/ssh" + printf '%s\n' "$DEPLOY_SSH_KEY" > "$RUNNER_TEMP/ssh/id" + chmod 600 "$RUNNER_TEMP/ssh/id" + echo "DEPLOY_SSH=ssh -i $RUNNER_TEMP/ssh/id -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=$GITHUB_WORKSPACE/deploy/known_hosts" >> "$GITHUB_ENV" + - run: bin/deploy + - name: Check the live site + run: curl -fsS --retry 3 https://implish.org/ -o /dev/null + - name: Remove deploy key + if: always() + run: rm -rf "$RUNNER_TEMP/ssh" diff --git a/.gitignore b/.gitignore index 6259cae..40304f4 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ package-lock.json dist/ dist-web/ plan.org +.deploy-test/ diff --git a/CLAUDE.md b/CLAUDE.md index 9cddf73..190c35a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -213,7 +213,7 @@ npm run preview The Vite build automatically runs `npm run build` (TypeScript compilation) first via the `ensureCoreBuild` plugin, then bundles everything for the browser. -**Deployment**: `dist-web/` is the live site served at implish.org. After running `npx vite build`, deploy the contents of `dist-web/` to the server. +**Deployment**: `dist-web/` is the live site served at implish.org (nginx on 45.79.174.182). Deploy only with `bin/deploy`; never copy files by hand. Pushes to main deploy automatically via `.github/workflows/deploy.yml`. Each deploy uploads a new release dir and atomically switches the docroot symlink to it; files from the last `DEPLOY_ASSET_GRACE` live releases stay reachable so pages loaded before a switch don't 404. Use `bin/deploy --dry-run` to preview and `bin/deploy --rollback` to switch back to the previous release. CI runs the deploy tooling from the workflow's own commit and only builds the selected (merged) commit's source. Host, user and docroot live in `deploy/deploy.conf` and the `production` environment's `DEPLOY_*` variables; the deploy key is an environment secret released only to main. Server-side steps are in `deploy/remote.sh`. See `deploy/README.md`. Tests: `npm run test:deploy`. Vite 7 needs Node >= 20.19. ### TypeScript Details diff --git a/bin/deploy b/bin/deploy new file mode 100755 index 0000000..464c090 --- /dev/null +++ b/bin/deploy @@ -0,0 +1,219 @@ +#!/usr/bin/env bash +# bin/deploy: build web/ and publish dist-web/ to the implish.org docroot. +# +# bin/deploy [--dry-run] [--skip-build] build, upload a new release, switch to it +# bin/deploy --rollback [NAME] [--dry-run] switch back to the previous (or NAME) release +# bin/deploy --list-releases list releases, oldest first +# +# Every deploy uploads a complete tree to DEPLOY_RELEASES/, +# then atomically repoints the DEPLOY_PATH symlink at it (see deploy/remote.sh). +# The live site is never edited in place, so an interrupted deploy leaves +# the old site fully intact. +# +# Configuration comes from deploy/deploy.conf, overridden by environment: +# DEPLOY_HOST ssh host, or "local" to treat DEPLOY_PATH as a local path +# DEPLOY_USER ssh user (optional; empty = ssh default / ~/.ssh/config) +# DEPLOY_PATH absolute path of the live docroot (nginx root); becomes a symlink +# DEPLOY_RELEASES where releases live (default: DEPLOY_PATH.releases) +# DEPLOY_KEEP how many releases to keep, live one included (default 5) +# DEPLOY_ASSET_GRACE files of the last N live releases stay reachable after a +# switch, so pages loaded before it can still fetch their +# hashed assets (default 3; 0 disables) +# DEPLOY_SSH ssh command (default: ssh -o BatchMode=yes) +# DEPLOY_SOURCE_DIR source tree to build (default: this checkout). CI points +# it at an exported older commit to redeploy that commit +# with the current tooling. +# DEPLOY_BUILD_CMD build command, run in DEPLOY_SOURCE_DIR (default: npx vite build) +# DEPLOY_SRC build output to publish, relative to DEPLOY_SOURCE_DIR (default: dist-web) +# DEPLOY_LOCK_WAIT seconds to wait for another deploy's lock (default 60) +# +# Exits nonzero on any failure. Nothing on the target is touched until the +# build has succeeded and produced an index.html. +set -euo pipefail + +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +CONF=${DEPLOY_CONF:-$ROOT/deploy/deploy.conf} + +die() { echo "deploy: $*" >&2; exit 1; } +log() { echo "deploy: $*" >&2; } + +# Non-empty environment values win over the config file. Empty ones are +# ignored, because CI maps unset repository variables to empty strings. +declare -A from_env=() +for v in DEPLOY_HOST DEPLOY_USER DEPLOY_PATH DEPLOY_RELEASES DEPLOY_KEEP \ + DEPLOY_SSH DEPLOY_ASSET_GRACE DEPLOY_SOURCE_DIR DEPLOY_BUILD_CMD DEPLOY_SRC DEPLOY_LOCK_WAIT \ + DEPLOY_RSYNC; do + [[ -n ${!v:-} ]] && from_env[$v]=${!v} +done +if [[ -f $CONF ]]; then + # shellcheck source=/dev/null + source "$CONF" +fi +for v in "${!from_env[@]}"; do printf -v "$v" '%s' "${from_env[$v]}"; done + +DEPLOY_HOST=${DEPLOY_HOST:-} +DEPLOY_USER=${DEPLOY_USER:-} +DEPLOY_PATH=${DEPLOY_PATH:-} +DEPLOY_KEEP=${DEPLOY_KEEP:-5} +DEPLOY_ASSET_GRACE=${DEPLOY_ASSET_GRACE:-3} +DEPLOY_SSH=${DEPLOY_SSH:-ssh -o BatchMode=yes} +DEPLOY_BUILD_CMD=${DEPLOY_BUILD_CMD:-npx vite build} +DEPLOY_SRC=${DEPLOY_SRC:-dist-web} +DEPLOY_SOURCE_DIR=${DEPLOY_SOURCE_DIR:-$ROOT} +DEPLOY_RSYNC=${DEPLOY_RSYNC:-rsync} # test hook + +mode=deploy dry_run=0 skip_build=0 rollback_name= +while (($#)); do + case $1 in + --dry-run|-n) dry_run=1 ;; + --skip-build) skip_build=1 ;; + --rollback) + mode=rollback + if [[ ${2:-} && ${2:0:1} != - ]]; then rollback_name=$2; shift; fi ;; + --list-releases|--list-backups) mode=list ;; + -h|--help) sed -n '2,/^set -e/{/^set -e/d;s/^# \{0,1\}//;p}' "$0"; exit 0 ;; + *) die "unknown argument: $1" ;; + esac + shift +done + +# --- validate config before doing anything ------------------------------- +[[ -n $DEPLOY_HOST ]] || die "DEPLOY_HOST is not set (see deploy/deploy.conf)" +[[ -n $DEPLOY_PATH ]] || die "DEPLOY_PATH is not set (see deploy/deploy.conf)" +DEPLOY_PATH=${DEPLOY_PATH%/} +DEPLOY_RELEASES=${DEPLOY_RELEASES:-$DEPLOY_PATH.releases} +DEPLOY_RELEASES=${DEPLOY_RELEASES%/} +check_path() { # name value: refuse paths that rsync --delete or rm -rf could wreck + local name=$1 p=$2 + [[ $p == /* ]] || die "$name must be absolute: '$p'" + [[ $p =~ ^/+[^/]+/+[^/]+ ]] || die "$name is too shallow to be safe: '$p'" + [[ /$p/ != */../* && /$p/ != */./* ]] || die "$name must not contain . or .. : '$p'" + [[ $p != *[[:space:]\'\"\\]* ]] || die "$name must not contain spaces or quotes: '$p'" +} +check_path DEPLOY_PATH "$DEPLOY_PATH" +check_path DEPLOY_RELEASES "$DEPLOY_RELEASES" +[[ $DEPLOY_RELEASES != "$DEPLOY_PATH" && $DEPLOY_RELEASES/ != "$DEPLOY_PATH"/* ]] \ + || die "DEPLOY_RELEASES must not be inside DEPLOY_PATH" +[[ $DEPLOY_KEEP =~ ^[1-9][0-9]*$ ]] || die "DEPLOY_KEEP must be a positive integer: '$DEPLOY_KEEP'" +[[ $DEPLOY_ASSET_GRACE =~ ^[0-9]+$ ]] || die "DEPLOY_ASSET_GRACE must be a whole number: '$DEPLOY_ASSET_GRACE'" +DEPLOY_LOCK_WAIT=${DEPLOY_LOCK_WAIT:-60} +[[ $DEPLOY_LOCK_WAIT =~ ^[0-9]+$ ]] || die "DEPLOY_LOCK_WAIT must be a whole number of seconds: '$DEPLOY_LOCK_WAIT'" + +[[ -d $DEPLOY_SOURCE_DIR ]] || die "DEPLOY_SOURCE_DIR is not a directory: '$DEPLOY_SOURCE_DIR'" + +at_target() { # PATH: rsync address of PATH on the target + if [[ $DEPLOY_HOST == local ]]; then echo "$1" + else echo "${DEPLOY_USER:+$DEPLOY_USER@}$DEPLOY_HOST:$1"; fi +} +target=$(at_target "$DEPLOY_PATH") + +# on_target SCRIPT ARGS...: run a bash snippet where the docroot lives. +on_target() { + local script=$1; shift + if [[ $DEPLOY_HOST == local ]]; then + bash -c "$script" on_target "$@" + else + local q; q=$(printf ' %q' "$@") + # shellcheck disable=SC2086 + $DEPLOY_SSH "${DEPLOY_USER:+$DEPLOY_USER@}$DEPLOY_HOST" "bash -s --$q" <<<"$script" + fi +} + +# rsync options for uploads and the dry-run listing. --checksum so CI +# checkouts (fresh mtimes) only list files whose content changed. +# /.well-known/ (certbot) is protected from --delete. +RSYNC_OPTS=(-rlp --checksum --delete --itemize-changes + --chmod=Du=rwx,Dgo=rx,Fu=rw,Fgo=r --filter='P /.well-known/') +rsync_to() { # SRC DEST [extra rsync args...] + local src=$1 dest=$2; shift 2 + if [[ $DEPLOY_HOST == local ]]; then + $DEPLOY_RSYNC "${RSYNC_OPTS[@]}" "$@" "$src" "$dest" + else + $DEPLOY_RSYNC "${RSYNC_OPTS[@]}" -e "$DEPLOY_SSH" "$@" "$src" "$dest" + fi +} + +# The target-side half lives in deploy/remote.sh and is piped to bash. +REMOTE_LIB=$(<"$ROOT/deploy/remote.sh") +remote() { on_target "$REMOTE_LIB" "$DEPLOY_PATH" "$DEPLOY_RELEASES" "$DEPLOY_KEEP" "$DEPLOY_ASSET_GRACE" "$@"; } + +# hold_lock: take $DEPLOY_RELEASES/.lock on the target and keep it until +# this script exits, so a manual --rollback can't interleave with a CI deploy. +# A coprocess keeps an ssh session (or a local bash) open, which runs flock +# and then waits on stdin. When we exit, or the connection drops, its stdin +# closes and the lock is released. The script is passed as a bash -c string +# because stdin is the keepalive pipe. It must not contain single quotes. +# shellcheck disable=SC2016 +LOCK_SCRIPT='umask 022 && mkdir -p "$1" && exec 9>>"$1/.lock" && flock -w "$2" 9 && echo locked && exec cat >/dev/null' +hold_lock() { + local q; q=$(printf ' %q' "$DEPLOY_RELEASES" "$DEPLOY_LOCK_WAIT") + if [[ $DEPLOY_HOST == local ]]; then + coproc LOCK { bash -c "$LOCK_SCRIPT" hold_lock "$DEPLOY_RELEASES" "$DEPLOY_LOCK_WAIT"; } + else + # shellcheck disable=SC2086 + coproc LOCK { $DEPLOY_SSH "${DEPLOY_USER:+$DEPLOY_USER@}$DEPLOY_HOST" "bash -c '$LOCK_SCRIPT' hold_lock$q"; } + fi + local line= + read -r line <&"${LOCK[0]}" || true + [[ $line == locked ]] || die "could not take $DEPLOY_RELEASES/.lock (another deploy or rollback running?); nothing changed" + log "holding $DEPLOY_RELEASES/.lock" +} + +case $mode in + list) remote list; exit 0 ;; + + rollback) + ((dry_run)) || hold_lock + name=$(remote pick "$rollback_name") + if ((dry_run)); then + log "dry run: switching $target to release $name would change:" + remote diff "$name" + ((DEPLOY_ASSET_GRACE == 0)) || log "(files listed as deleted stay reachable: they are carried over from recent releases)" + log "dry run: nothing changed" + exit 0 + fi + # One atomic symlink swap. The release we leave stays on disk, so + # `--rollback NAME` can roll forward again, and a second --rollback + # steps one more release back. + remote switch "$name" + log "rolled back $target to release $name" + exit 0 ;; +esac + +# --- deploy --------------------------------------------------------------- +cd "$DEPLOY_SOURCE_DIR" +if ((skip_build)); then + log "skipping build (--skip-build)" +else + log "building in $DEPLOY_SOURCE_DIR: $DEPLOY_BUILD_CMD" + bash -c "$DEPLOY_BUILD_CMD" || die "build failed; nothing was deployed" +fi +[[ -f $DEPLOY_SRC/index.html ]] || die "$DEPLOY_SRC/index.html missing after build; nothing was deployed" + +if ((dry_run)); then + log "dry run: changes that would be made to $target" + if [[ $(remote exists) == no ]]; then + log "(docroot does not exist yet; every file would be new)" + (cd "$DEPLOY_SRC" && find . -type f -printf '>f+++++++++ %P\n' | LC_ALL=C sort) + else + rsync_to "$DEPLOY_SRC/" "$target/" --dry-run + ((DEPLOY_ASSET_GRACE == 0)) || log "(files listed as deleted stay reachable until $DEPLOY_ASSET_GRACE more releases have gone live; see DEPLOY_ASSET_GRACE)" + fi + log "dry run: nothing changed" + exit 0 +fi + +hold_lock +link=() +if [[ $(remote exists) == yes ]]; then link=(--link-dest="$DEPLOY_PATH/"); fi +name=$(remote prepare) +log "uploading release $name" +# Upload into RELEASES/.tmp. Unchanged files are hard-linked from the +# live tree (--link-dest), so they aren't re-sent. The live tree is only +# read here; nothing in it changes until `activate`. +rsync_to "$DEPLOY_SRC/" "$(at_target "$DEPLOY_RELEASES/$name.tmp")/" "${link[@]}" \ + || { remote discard "$name" || true; die "upload failed; live site unchanged"; } +remote activate "$name" +# After the switch: a failure here leaves the new release live, so say so. +remote prune || die "release $name is live, but pruning old releases failed" +log "deployed $(cat "$DEPLOY_SOURCE_DIR/.deploy-sha" 2>/dev/null || git -C "$DEPLOY_SOURCE_DIR" rev-parse HEAD 2>/dev/null || echo '?') as release $name" diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 0000000..d80600e --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,278 @@ +# Deploying implish.org + +implish.org is static: `npx vite build` turns `web/` into `dist-web/`, and +nginx on 45.79.174.182 (Ubuntu) serves those files from a docroot. + +`bin/deploy` is the only deploy path. CI runs it, and so can a human: + +```bash +bin/deploy --dry-run # build, then list what would change; touches nothing +bin/deploy # build, upload a new release, switch the site to it +bin/deploy --list-releases # releases, oldest first, live one marked +bin/deploy --rollback # switch to the release before the live one (repeat to go further back) +bin/deploy --rollback 20261029T120000Z # switch to a specific release (also rolls forward) +bin/deploy --rollback --dry-run # show what a rollback would change +``` + +## Layout: releases and one symlink + +``` +/var/www/implish.org -> /var/www/implish.org.releases/20261029T120501Z (symlink: nginx root) +/var/www/implish.org.releases/ + 20261029T110000Z/ older release (rollback target) + 20261029T120501Z/ live release + .lock +``` + +Every deploy uploads a complete tree to a new release dir, then repoints the +docroot symlink with a single `rename(2)`. The live tree is never edited in +place. nginx serves the complete old site until the switch and the complete +new site after it. Vite's hashed assets therefore never go missing under a +live `index.html`, and an interrupted upload leaves the old site untouched. +Rollback is the same atomic switch to an older release. Older releases are +the backups. + +### Failure safety: the old site keeps serving + +Every switch (deploy, the first-deploy migration, rollback, roll-forward) is +split into two parts in `deploy/remote.sh`: + +- **stage**: everything that can fail, with the live site untouched. This + covers the release checks, the file lists, carrying grace-period files, + and creating the new symlink next to the docroot. +- **commit**: one `rename(2)` of that symlink over the docroot. + +After the commit, only bookkeeping runs. A failed history write only +warns. A failed prune exits nonzero with "release X is live, but pruning old +releases failed". + +On any failure, an exit trap removes the staged symlink. If the failure came +before an activation's commit, the trap also removes that never-live +release, so it can't collide with a quick retry or become a rollback target. + +Carrying never fails a switch. A file that can't be kept reachable gets a +warning and is skipped. For example, the old site had `downloads/old.json` +and the new release ships a plain file called `downloads`; or a parent path +is a symlink, which carrying never follows on either side. + +The one-time migration of a plain-directory docroot stages everything first, +with the old directory still serving. Then it does two renames: the +directory into the releases dir, and the symlink into its place. If the +second rename fails, the first is undone and the old site is restored. + +`DEPLOY_FAULT=` injects a failure at a named point +(`stage`, `before-commit`, `migrate-before-move`, `migrate-between-renames`, +`prune`). The tests use it to show that each switch kind leaves the old site +serving. + +### Grace period: no 404s for pages loaded just before a switch + +A browser can fetch `index.html` from release N, then request the hashed +`assets/index-AbC123.js` that it names after the site has switched to N+1, +which doesn't contain that file. The same is true of any other file a page +loads later, such as fetched data like `b4-image.json`, and of rollbacks in +either direction. Keeping old releases on disk doesn't help, because URLs +resolve through the docroot symlink. + +So before **every** switch (deploy, rollback, roll-forward), the target +release gets hard links to every file that the last `DEPLOY_ASSET_GRACE` +(default 3) *live* releases shipped and the target lacks: + +- Its own files are never overwritten, so a changed `data.json` serves the + new version, while a removed one stays reachable. +- Each release's shipped files are recorded in `RELEASES/.files`. Files + that were only carried aren't recorded, so the window doesn't snowball: + a file disappears once it has been out of every one of the last N live + releases. +- `RELEASES/.history` records which release was live when, so after a + rollback the window follows what browsers actually saw, not name order. +- Prune never removes a release inside the window. Files already carried + are hard links, so they also survive their source release being pruned. +- Consequence: a file you delete stays reachable for N more deploys. Dry + runs say so. `DEPLOY_ASSET_GRACE=0` turns this off. + +What a deploy does, in order (any failure exits nonzero and stops): + +1. Validates config. `DEPLOY_PATH` and `DEPLOY_RELEASES` must be absolute, + at least two levels deep, free of `.`/`..`/spaces, and the releases dir + must not be inside the docroot. +2. Builds in `DEPLOY_SOURCE_DIR` (default: this checkout). If the build fails + or leaves no `dist-web/index.html`, it stops here and nothing on the + server has been touched. +3. Takes `DEPLOY_RELEASES/.lock` on the server and holds it until the + deploy ends, waiting up to `DEPLOY_LOCK_WAIT` seconds (default 60). A + manual `--rollback` takes the same lock, so it can't interleave with a CI + deploy. A held ssh session keeps the lock; if the script dies or the + connection drops, the lock is released. +4. Creates `DEPLOY_RELEASES/.tmp` and copies `/.well-known/` + (certbot) into it from the live tree. Stale `.tmp` dirs from a killed run + are removed first. +5. Uses rsync to upload `dist-web/` into the `.tmp` dir. Unchanged files are + hard-linked from the live release (`--link-dest`), so they aren't re-sent; + rsync never writes into the live tree. Files come out 644, dirs 755, and + everything on the target is created under `umask 022`, so nginx can read + it whatever umask the deploy shell has. If the upload fails, the `.tmp` + is removed and the live site is unchanged. +6. Activates: records the release's file list, renames `.tmp` to its final + name, carries in the grace-window files (see above), and atomically + switches the symlink. On the **first deploy**, if the docroot is still a plain + directory, it is moved into the releases dir (one second older than the + new release, so it becomes the rollback target). That migration is two + back-to-back renames, so for a moment the path is missing. This is the + one non-atomic step, and it happens once. An empty or missing docroot + just becomes the symlink. +7. Prunes down to the newest `DEPLOY_KEEP` releases (default 5). The live + release and the releases in the grace window are never pruned, even + after a rollback to an old one. + +The server-side half is `deploy/remote.sh`, which `bin/deploy` pipes to +`bash` over ssh. Every `rm -rf` there checks that the name is exactly a +release timestamp (`^20[0-9]{6}T[0-9]{6}Z$`), so an empty or malformed +name can't reach the releases dir itself. `switch` refuses to replace a +docroot that is a real directory; only the first deploy's migration does +that. + +Rollback switches to the newest release older than the live one, and keeps +the release it leaves. So repeated `--rollback` steps back, and +`--rollback NAME` can roll forward again. Certbot challenge files written +after a release was made aren't in older releases. That's harmless, because +certbot writes fresh ones for each renewal. + +## Configuration + +Values come from `deploy/deploy.conf`. Any **non-empty** environment variable +of the same name overrides them. See `bin/deploy --help` for the full list. + +| variable | value | +|---|---| +| `DEPLOY_HOST` | `45.79.174.182` (set in deploy.conf) | +| `DEPLOY_USER` | **unknown: fill in** (repo variable or deploy.conf) | +| `DEPLOY_PATH` | **unknown: fill in** the nginx `root` for implish.org | +| `DEPLOY_RELEASES` | default `$DEPLOY_PATH.releases` | +| `DEPLOY_ASSET_GRACE` | 3 (live releases whose files stay reachable after a switch) | +| `DEPLOY_SOURCE_DIR` | default: this checkout (CI sets it to the exported source) | +| `DEPLOY_KEEP` | 5 | + +The user and docroot are unknown because the machine that wrote this had no +ssh access to the server. To find the docroot on the server, run: +`grep -R 'root\|server_name' /etc/nginx/sites-enabled/`. + +`deploy/known_hosts` pins the server's host keys, so CI never trusts a key on +first use. They were collected with `ssh-keyscan` on 2026-10-29: + +``` +SHA256:crs05VrOGRUK0xAZw8uwpPmmlboM2rdq4MyaqrUbqRM (ED25519) +SHA256:5ztPwBwMsF+/xXMgJhFKhtE941Dq6nktzI6WAbxiF/M (ECDSA) +SHA256:kKeq5mlSsrOY4BA+TZnUxLuk5+kIkSnTLGqVqyPikYQ (RSA) +``` + +Check them on the server with `ssh-keygen -lf /etc/ssh/ssh_host_ed25519_key.pub` +(and the same for the other key types). + +## One-time setup (Michal) + +The server needs `rsync` and `flock`, which stock Ubuntu has, and nginx must +follow symlinks. That's the default; check that `disable_symlinks` isn't +turned on. Because the docroot becomes a symlink that the deploy swaps, the +deploy user needs write access to the docroot's **parent directory** (to +rename entries there) and to `DEPLOY_RELEASES`. Steps 2 and 3 need root. + +1. Generate a dedicated key on your machine. Don't commit it: + ```bash + ssh-keygen -t ed25519 -N '' -C implish-deploy@github-actions -f implish-deploy + ``` +2. On the server, give the deploy user a parent dir it owns. If the nginx + `root` today is directly under a root-owned dir like `/var/www`, the + least-privilege option is a dedicated parent. Move the site one level + down, and point nginx (and certbot's webroot, if it uses one) at the new + path: + ```bash + sudo mkdir -p /var/www/implish + sudo mv /var/www/implish.org /var/www/implish/site # current root -> new spot + sudo chown -R DEPLOYUSER: /var/www/implish + # nginx: root /var/www/implish/site; then: sudo nginx -t && sudo systemctl reload nginx + ``` + Then set `DEPLOY_PATH=/var/www/implish/site`. Releases go to + `/var/www/implish/site.releases`. The alternative is to give the deploy + user write access to the existing parent dir, which is broader. + On the first deploy, `bin/deploy` turns `site` into a symlink and keeps + the old tree as the oldest release. +3. Append one line to `~DEPLOYUSER/.ssh/authorized_keys` on the server. + `restrict` turns off pty, agent, port and X11 forwarding. The key can + still run the `bash` and `rsync` commands that the deploy needs: + ``` + restrict ssh-ed25519 AAAA...contents-of-implish-deploy.pub... implish-deploy@github-actions + ``` +4. Create the `production` environment and limit it to main. Then store the + key and variables **in that environment, not as repository secrets**. + Branch PRs can read repository secrets, so a PR that edited the workflow + could deploy. An environment secret is only released to jobs running on + main. + - Settings → Environments → New environment → `production`. + - Deployment branches and tags → **Selected branches and tags** → add + the rule `main`. (Optional: tick "Allow administrators to bypass" off.) + ```bash + gh secret set DEPLOY_SSH_KEY --repo tangentcode/implish --env production < implish-deploy + gh variable set DEPLOY_USER --repo tangentcode/implish --env production --body DEPLOYUSER + gh variable set DEPLOY_PATH --repo tangentcode/implish --env production --body /var/www/implish.org + ``` + If a repo-level `DEPLOY_SSH_KEY` secret already exists, delete it: + `gh secret delete DEPLOY_SSH_KEY --repo tangentcode/implish`. + You can commit `DEPLOY_USER`/`DEPLOY_PATH` to `deploy/deploy.conf` + instead of setting the variables. Then delete the local private key. + Also protect `main` (no direct pushes; PRs merged by you), since + anything that lands on main gets deployed. +5. Check it: Actions → "deploy implish.org" → Run workflow. You can also + run `bin/deploy --dry-run` locally with the key loaded. + +## CI + +`.github/workflows/deploy.yml`: + +- **pull_request**: `npm run build` + `npm run test:deploy`. It never deploys. +- **push to main**: runs the tests, then `bin/deploy` in the `production` + environment, then checks `https://implish.org/` with curl. +- **workflow_dispatch** (main only): same as push. The optional `ref` + input (SHA, tag or branch) must already be merged into main; + `deploy/select-ref.sh` checks this with `git merge-base --is-ancestor`. + So the input can redeploy or roll back to an older merged commit, but it + can't put an unmerged PR live. To get a PR live, merge it. Roll back by + dispatching an older merged SHA, or with `bin/deploy --rollback` from a + machine that has the key. + +**Tooling vs. source.** Both jobs always run `bin/deploy`, `deploy/*` and +the tests from the workflow's own commit (`github.sha`). Only the +application source comes from `ref`: `select-ref.sh` exports it with +`git archive` into `$RUNNER_TEMP/source`, and `bin/deploy` builds there via +`DEPLOY_SOURCE_DIR`. So you can redeploy a commit from before `bin/deploy` +existed (e.g. `6813323`), and fixes to the deploy tooling apply to +redeploys of old code too. The old commit's own `package.json` and +`vite.config.js` are used for its build. A commit too old to build with +`npx vite build` fails at the build step, before anything on the server is +touched. + +Three layers keep the key away from unmerged code: + +1. The `production` environment only releases `DEPLOY_SSH_KEY` to jobs on + main. This is the real guarantee: a PR branch can rewrite the workflow + but still can't read the secret. +2. The deploy job has `if: github.ref == 'refs/heads/main'` and never runs + on `pull_request`. +3. `select-ref.sh` rejects refs that aren't merged into main. + +Deploys run one at a time (concurrency group `deploy-implish-org`, no +cancellation). Until `DEPLOY_SSH_KEY` is set, the deploy job fails with a +message that points here. + +## Tests + +`npm run test:deploy` runs `test/deploy.test.sh`. It tests against throwaway +docroots in `.deploy-test/`, both with `DEPLOY_HOST=local` and through a +fake `ssh` that runs the remote half locally, so the ssh/rsync `-e` path is +exercised too. An rsync wrapper (`DEPLOY_RSYNC`) checks that the live tree +stays byte-identical for the whole upload, and simulates cut-off and killed +transfers. The grace-period tests serve the docroot with `python3 -m +http.server`, which resolves the symlink per request like nginx. They fetch +`index.html`, switch (deploy or rollback), then fetch the asset it named. One case exports the real pre-tooling commit `6813323` from this +repo's history and deploys it with the current `bin/deploy`; CI checks out +with `fetch-depth: 0` for that. diff --git a/deploy/deploy.conf b/deploy/deploy.conf new file mode 100644 index 0000000..8b337ba --- /dev/null +++ b/deploy/deploy.conf @@ -0,0 +1,12 @@ +# Config for bin/deploy. Environment variables of the same name override these +# (the GitHub workflow sets them from repository variables). +# +# DEPLOY_USER and DEPLOY_PATH are not known yet: the box that wrote this +# script had no ssh access to the server. Fill them in here, or set the +# repository variables of the same name (see deploy/README.md). +DEPLOY_HOST=45.79.174.182 +DEPLOY_USER= +DEPLOY_PATH= +# DEPLOY_RELEASES defaults to "$DEPLOY_PATH.releases" (sibling of the docroot). +# DEPLOY_PATH becomes a symlink to the live release (see deploy/README.md). +DEPLOY_KEEP=5 diff --git a/deploy/known_hosts b/deploy/known_hosts new file mode 100644 index 0000000..caa52ff --- /dev/null +++ b/deploy/known_hosts @@ -0,0 +1,3 @@ +45.79.174.182 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMzMyCJaJrE+vZdkRGDjfUNO9ehVbvhNTR/sgu3SHuiF6AXbxxFfFrYoIcfcueVgYFiPCQ54N5EW9gWGyVOA5oY= +45.79.174.182 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIHUyCe+QSaI9dqWgcl4ZjbRnvRNw5YmI9/j7o5Ds6u9 +45.79.174.182 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDA0P/eHMuHI9CHJvBw2mbg8bUpL7A2YDUNjoCgeV/vx0rIjpQ4LM09S6MSH4RNJh7VNFA1EHKYg2jMVaDTq5J7Q10x//puQ8omzyX3Ny4jvu1h/7h8ZHsVSgYa9T5hfxofpSEfJtb3EIQZ2SpRbspOlQzggkM4cmTYuJOMn3FChP9NpJv70x0JAbqNyGAkzGrLBL9Pv8oKlrt6gPmHjHwFtBTPWk5U3d03J8ZU7a6EQ9ADsbaztIpc+Axl78d8V0stj99yr1RmUgY7CmQwuU2rELZxgmwtpP3RqYkkmeGTBI2ioyoPNwAC+2rOe/HjIZ09wr44GdTVQD1roPfGKKgr diff --git a/deploy/remote.sh b/deploy/remote.sh new file mode 100644 index 0000000..61a1546 --- /dev/null +++ b/deploy/remote.sh @@ -0,0 +1,244 @@ +# deploy/remote.sh: the half of bin/deploy that runs where the docroot lives. +# bin/deploy pipes this file to `bash -s` (over ssh, or locally), with args: +# +# remote.sh DOCROOT RELEASES KEEP GRACE ACTION [ARG] +# +# Layout: every deploy is a complete tree in RELEASES/, and +# DOCROOT is a symlink to the live one. A deploy uploads into +# RELEASES/.tmp, renames it, then swaps the symlink with one rename(2). +# Rollback is the same swap to an older release. So nginx sees either the +# whole old tree or the whole new one, never a mix, even if a transfer is +# interrupted. Older releases double as the backups. +# +# Grace period: a browser may load index.html from one release and fetch its +# hashed assets (or any other file) after the switch. So before every switch, +# deploy or rollback, in either direction, the target release gets hard links +# to every file that the last GRACE live releases shipped and the target +# lacks. Its own files are never overwritten. Each release's shipped files +# are listed in RELEASES/.files, and RELEASES/.history records which +# release was live when, so the window follows what browsers actually saw, +# not name order. Prune never removes a release inside the window. +# +# Failure safety: every step that can fail (checks, manifests, carrying +# files, creating the new symlink) runs in `stage`, before the live site is +# touched. The switch itself is one rename(2) in `commit`. Bookkeeping after +# it (history) only warns. So whatever fails, the old site keeps serving. +# The single exception is the one-time migration of a plain-directory +# docroot (see `activate`), which restores the old directory if its last +# rename fails. DEPLOY_FAULT= injects a failure at a named point; the +# tests use it to prove this for each step. +# +# The caller holds $RELEASES/.lock for the whole deploy or rollback (see +# hold_lock in bin/deploy), so the actions here don't lock on their own. +# Release names are UTC timestamps (20261029T120000Z), so name order is age +# order. Every rm -rf goes through rm_release, which checks the name. +set -euo pipefail +# nginx (another user) must be able to traverse and read everything we +# create, whatever umask the ssh session or CI runner has. +umask 022 +docroot=$1 releases=$2 keep=$3 grace=$4 action=$5 arg=${6:-} +history=$releases/.history + +die() { echo "deploy: $*" >&2; exit 1; } +warn() { echo "deploy: warning: $*" >&2; } +fault() { # POINT: fail here if DEPLOY_FAULT names this point (tests only) + [[ ${DEPLOY_FAULT:-} != "$1" ]] || { echo "deploy: injected fault at $1" >&2; return 1; } +} +valid_name() { [[ $1 =~ ^20[0-9]{6}T[0-9]{6}Z$ ]]; } +rm_release() { # NAME [SUFFIX]: remove one release (or its .tmp/.files) by validated name + valid_name "$1" || die "refusing to remove invalid release name: '$1'" + rm -rf -- "${releases:?}/$1${2:-}" +} +list() { + [[ -d $releases ]] || return 0 + find "$releases" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' \ + | grep -E '^20[0-9]{6}T[0-9]{6}Z$' | LC_ALL=C sort || true +} +live() { # name of the release DOCROOT points at, or nothing + [[ -L $docroot ]] || return 0 + local t; t=$(readlink "$docroot") + if [[ $(dirname "$t") == "$releases" ]] && valid_name "$(basename "$t")"; then basename "$t"; fi +} +recent() { # the last GRACE distinct releases that were live, newest first + ((grace > 0)) || return 0 + { live; [[ -f $history ]] && tac "$history"; } \ + | grep -E '^20[0-9]{6}T[0-9]{6}Z$' | awk '!seen[$0]++' | head -n "$grace" || true +} +manifest() { # DIR > list of files a release ships (not certbot's, not carried ones) + (cd "$1" && find . -type f ! -path './.well-known/*' -printf '%P\n' | LC_ALL=C sort) +} +safe_path() { # ROOT REL: no parent of ROOT/REL (below ROOT) is a symlink or a non-directory + local p=$1 c parts + IFS=/ read -ra parts <<<"$(dirname "$2")" + for c in "${parts[@]}"; do + [[ -z $c || $c == . ]] && continue + p=$p/$c + [[ -L $p ]] && return 1 + [[ -e $p && ! -d $p ]] && return 1 + done + return 0 +} +carried=0 +carry_from() { # SRC_DIR MANIFEST TARGET_DIR: link in what SRC shipped and TARGET lacks + local src=$1 list=$2 target=$3 f + while IFS= read -r f; do + # manifests are ours, but never let one point outside the release + [[ -n $f && $f != /* && /$f/ != */../* ]] || continue + [[ -f $src/$f && ! -L $src/$f ]] && safe_path "$src" "$f" || continue + if [[ -e $target/$f || -L $target/$f ]]; then + [[ -f $target/$f && ! -L $target/$f ]] || warn "can't keep /$f reachable: the new release has a directory or link there" + continue + fi + if ! safe_path "$target" "$f"; then + warn "can't keep /$f reachable: a parent path in the new release is a file or link"; continue + fi + if mkdir -p "$(dirname "$target/$f")" 2>/dev/null && + { ln "$src/$f" "$target/$f" 2>/dev/null || cp -p "$src/$f" "$target/$f" 2>/dev/null; }; then + carried=$((carried+1)) + else + warn "can't keep /$f reachable: copying it failed" + fi + done < "$list" +} +carry() { # TARGET_DIR: carry from the recent live releases. Never fails the switch. + local target=$1 prev + for prev in $(recent); do + [[ -d $releases/$prev && $releases/$prev != "$target" ]] || continue + [[ -f $releases/$prev.files ]] || manifest "$releases/$prev" > "$releases/$prev.files" + carry_from "$releases/$prev" "$releases/$prev.files" "$target" + done +} +remember() { # NAME: record that NAME went live. Runs after the switch: warn, don't fail. + { echo "$1" >> "$history" && tail -n 100 "$history" > "$history.new" && mv "$history.new" "$history"; } \ + || warn "release $1 is live, but recording it in $history failed (grace window may be short)" +} +clean_tmp() { # leftovers of an upload that failed or was killed + [[ -d $releases ]] || return 0 + find "$releases" -mindepth 1 -maxdepth 1 -type d -name '20*Z.tmp' -printf '%f\n' \ + | while read -r t; do + rm_release "${t%.tmp}" .tmp; echo "deploy: removed stale partial upload $t" >&2 + done +} +new_name() { + local name + while :; do + name=$(date -u +%Y%m%dT%H%M%SZ) + [[ -e $releases/$name || -e $releases/$name.tmp ]] || break + sleep 1 + done + echo "$name" +} +check_release() { # NAME: must be an existing, complete release + valid_name "$1" && [[ -d $releases/$1 ]] || die "no such release: $1" + [[ -f $releases/$1/index.html ]] || die "release $1 has no index.html; refusing to make it live" +} +stage() { # NAME [SRC_DIR SRC_MANIFEST]: every fallible step of a switch, live site untouched + check_release "$1" + [[ -L $docroot.switch ]] && rm -f -- "$docroot.switch" # left by an earlier failed run + [[ -f $releases/$1.files ]] || manifest "$releases/$1" > "$releases/$1.files" + carried=0 + if [[ -n ${2:-} ]] && ((grace > 0)); then carry_from "$2" "$3" "$releases/$1"; fi + carry "$releases/$1" + ((carried == 0)) || echo "deploy: kept $carried file(s) from recent releases reachable" >&2 + fault stage + mkdir -p "$(dirname "$docroot")" + ln -sfnT "$releases/$1" "$docroot.switch" + fault before-commit +} +commit() { mv -T "$docroot.switch" "$docroot"; } # the switch: one atomic rename + +# On any failure: drop the staged symlink, and if an activation never reached +# its commit, drop that release too. It was never live, and keeping it would +# collide with a quick retry or become a bogus rollback target. +pending= +on_exit() { + local rc=$? + ((rc != 0)) || return 0 + [[ ! -L $docroot.switch ]] || rm -f -- "$docroot.switch" + if [[ -n $pending ]]; then + rm_release "$pending"; rm_release "$pending" .files + echo "deploy: removed release $pending, which never went live" >&2 + fi +} +trap on_exit EXIT +switch() { # NAME: atomically point the DOCROOT symlink at a release + [[ -L $docroot || ! -e $docroot ]] || die "$docroot is not a symlink; refusing to replace it" + stage "$1" + commit + remember "$1" +} + +case $action in + list) live=$(live); list | while read -r n; do [[ $n == "$live" ]] && echo "$n (live)" || echo "$n"; done ;; + live) live ;; + exists) [[ -d $docroot ]] && echo yes || echo no ;; + prepare) # make an empty RELEASES/.tmp for the upload; print + mkdir -p "$releases" + chmod 755 "$releases" # hold_lock may have created it first + clean_tmp + name=$(new_name) + mkdir "$releases/$name.tmp" + # certbot's challenges live in the docroot; carry them into the new tree + if [[ -d $docroot/.well-known ]]; then cp -a "$docroot/.well-known" "$releases/$name.tmp/"; fi + echo "$name" ;; + discard) rm_release "$arg" .tmp ;; + activate) # ARG: finish an upload and make it live + valid_name "$arg" || die "invalid release name: '$arg'" + [[ -f $releases/$arg.tmp/index.html ]] || die "upload $arg.tmp is incomplete; live site unchanged" + manifest "$releases/$arg.tmp" > "$releases/$arg.files" + pending=$arg + mv -T "$releases/$arg.tmp" "$releases/$arg" + if [[ -d $docroot && ! -L $docroot ]]; then + # First deploy onto a plain directory, which becomes the oldest release. + # Everything fallible is staged first, with the old site still serving. + # Then two renames replace the directory with the symlink. If the + # second one fails, the first is undone, so the old site comes back. + if [[ -z $(ls -A "$docroot") ]]; then + stage "$arg" + rmdir "$docroot" + commit || { mkdir -p "$docroot"; die "could not install the docroot symlink"; } + pending= + remember "$arg"; exit 0 + fi + legacy=$(date -u -d "@$(( $(date -u -d "${arg:0:8} ${arg:9:2}:${arg:11:2}:${arg:13:2}" +%s) - 1 ))" +%Y%m%dT%H%M%SZ) + [[ ! -e $releases/$legacy ]] || die "cannot migrate $docroot: $releases/$legacy exists" + manifest "$docroot" > "$releases/$legacy.files" + stage "$arg" "$docroot" "$releases/$legacy.files" # the old site's files, from where they are now + fault migrate-before-move + mv -T "$docroot" "$releases/$legacy" + if ! { fault migrate-between-renames && commit; }; then + mv -T "$releases/$legacy" "$docroot" || die "CRITICAL: $docroot is missing; restore it with: mv -T $releases/$legacy $docroot" + rm_release "$legacy" .files + die "could not install the docroot symlink; the old site was restored" + fi + pending= + remember "$legacy"; remember "$arg" + echo "deploy: moved the old docroot to release $legacy" >&2 + exit 0 + fi + switch "$arg"; pending= ;; + switch) switch "$arg" ;; + prune) # keep the newest KEEP releases, the live one, and the grace window + fault prune + clean_tmp + protect=" $(recent | tr '\n' ' ') $(live) " + list | head -n -"$keep" | while read -r old; do + [[ $protect == *" $old "* ]] && continue + rm_release "$old"; rm_release "$old" .files; echo "deploy: pruned release $old" >&2 + done ;; + pick) # print the release to roll back to: ARG, or the newest one older than live + if [[ -n $arg ]]; then + name=$arg + else + live=$(live) + [[ -n $live ]] || die "$docroot is not a release symlink yet; nothing to roll back to" + name=$(list | awk -v l="$live" '$0 < l' | tail -n 1) + [[ -n $name ]] || die "no release older than the live one ($live)" + fi + check_release "$name" + echo "$name" ;; + diff) # ARG: what switching to ARG would change + check_release "$arg" + rsync -rl --checksum --delete --dry-run --itemize-changes "$releases/$arg/" "$docroot/" ;; + *) die "unknown remote action: $action" ;; +esac diff --git a/deploy/select-ref.sh b/deploy/select-ref.sh new file mode 100755 index 0000000..f4c3d31 --- /dev/null +++ b/deploy/select-ref.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# deploy/select-ref.sh REF BASE [DEST]: resolve REF to a commit, but only if +# it is already merged into BASE (main's head). Empty REF means BASE. +# Prints the SHA. With DEST, it also exports that commit's tree into DEST +# (git archive, no .git) and writes DEST/.deploy-sha. +# +# workflow_dispatch uses this so the `ref` input can redeploy or roll back to +# merged history, never to an unmerged branch or PR head. Only the +# *application source* comes from REF. The deploy tooling (bin/deploy, +# deploy/*, tests) always comes from the workflow's own checkout, so commits +# from before this tooling existed can still be deployed. +set -euo pipefail +ref=${1:-} base=${2:?usage: select-ref.sh REF BASE [DEST]} dest=${3:-} +err() { echo "::error::$*" >&2; exit 1; } +base=$(git rev-parse --verify --quiet --end-of-options "$base^{commit}") || err "unknown base: $2" +if [[ -z $ref ]]; then + sha=$base +else + [[ $ref != -* ]] || err "bad ref: $ref" + sha=$(git rev-parse --verify --quiet --end-of-options "origin/$ref^{commit}" || + git rev-parse --verify --quiet --end-of-options "$ref^{commit}") || err "unknown ref: $ref" + git merge-base --is-ancestor "$sha" "$base" || + err "$ref ($sha) is not merged into main; only merged commits can be deployed" +fi +if [[ -n $dest ]]; then + [[ ! -e $dest || -z $(ls -A "$dest") ]] || err "export dir is not empty: $dest" + mkdir -p "$dest" + git archive --format=tar "$sha" | tar -x -C "$dest" + echo "$sha" > "$dest/.deploy-sha" +fi +echo "$sha" diff --git a/package.json b/package.json index afd00e5..0cd71ca 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,9 @@ "test": "echo \"Error: no test specified\" && exit 1", "dev": "vite", "preview": "vite preview", - "repl": "node dist/imp-cmd.mjs" + "repl": "node dist/imp-cmd.mjs", + "test:deploy": "bash test/deploy.test.sh", + "deploy": "bin/deploy" }, "repository": { "type": "git", diff --git a/test/deploy.test.sh b/test/deploy.test.sh new file mode 100755 index 0000000..398d6b7 --- /dev/null +++ b/test/deploy.test.sh @@ -0,0 +1,631 @@ +#!/usr/bin/env bash +# Tests for bin/deploy against throwaway docroots under .deploy-test/. +# Every case runs twice: DEPLOY_HOST=local, and through a fake ssh that runs +# the remote commands locally (exercising the ssh + rsync -e code path). +# A case that only makes sense in one mode returns 77 in the other (skip). +set -uo pipefail + +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +DEPLOY=$ROOT/bin/deploy +REMOTE=$ROOT/deploy/remote.sh +WORK=$ROOT/.deploy-test +rm -rf "$WORK"; mkdir -p "$WORK" +trap 'rm -rf "$WORK"' EXIT + +pass=0 total=0 skipped=0 failed=() +fail() { echo " FAIL: $*" >&2; return 1; } + +# fake ssh: skip options, drop the host, run the command string locally. +cat > "$WORK/fake-ssh" <<'EOF' +#!/usr/bin/env bash +while [[ $1 == -* ]]; do case $1 in -[ilopFJ]) shift 2 ;; *) shift ;; esac; done +shift +exec bash -c "$*" +EOF +chmod +x "$WORK/fake-ssh" + +# setup: fresh sandbox whose live docroot is a plain directory (like the +# server today) holding the "old" site. Like vite output, the index refers to +# a content-hashed asset, so serving a mix of old and new trees breaks the page. +setup() { + T=$WORK/case$total + rm -rf "$T"; mkdir -p "$T/srv/www/site/.well-known/acme" "$T/srv/www/site/assets" "$T/src" + echo 'old assets/app-old.js' > "$T/srv/www/site/index.html" + echo old-js > "$T/srv/www/site/assets/app-old.js" + echo stale > "$T/srv/www/site/stale.js" + echo token > "$T/srv/www/site/.well-known/acme/tok" + export DEPLOY_CONF=/dev/null DEPLOY_PATH=$T/srv/www/site DEPLOY_KEEP=3 + export DEPLOY_SOURCE_DIR=$T/src DEPLOY_SRC=out + unset DEPLOY_RELEASES DEPLOY_RSYNC + R=$DEPLOY_PATH.releases + # the "build" makes index.html " assets/app-.js" + echo new > "$T/src/version" + export DEPLOY_BUILD_CMD='v=$(cat version); rm -rf out && mkdir -p out/assets && echo "$v assets/app-$v.js" > out/index.html && echo "$v-js" > "out/assets/app-$v.js"' +} +releases() { ls "$R" 2>/dev/null | grep -E '^20[0-9]{6}T[0-9]{6}Z$' || true; } +live() { readlink "$DEPLOY_PATH"; } +site() { head -c 100 "$DEPLOY_PATH/index.html" | cut -d' ' -f1; } # version that is live +tree() { (cd "$1" && find . -type f -printf '%P\t' -exec cat {} \; | LC_ALL=C sort); } +# a release's own files (its manifest) with contents: what it shipped, ignoring carried files +own() { (cd "$R/$1" && while IFS= read -r f; do printf '%s\t%s\n' "$f" "$(cat "$f")"; done < "$R/$1.files"); } +# serve DEPLOY_PATH over HTTP like nginx: the symlink is resolved per request +serve() { + HTTP_PORT=$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1",0)); print(s.getsockname()[1])') + python3 -m http.server --bind 127.0.0.1 --directory "$DEPLOY_PATH" "$HTTP_PORT" >/dev/null 2>&1 & HTTP_PID=$! + for _ in $(seq 50); do curl -fs "http://127.0.0.1:$HTTP_PORT/" >/dev/null && return 0; sleep 0.1; done + return 1 +} +get() { curl -s -o "$T/body" -w '%{http_code}' "http://127.0.0.1:$HTTP_PORT/$1"; } # prints status +asset_of() { cut -d' ' -f2 <<<"$1"; } # the asset an index.html body refers to +deploy_v() { echo "$1" > "$T/src/version"; "$DEPLOY" >/dev/null 2>&1; } +# the page works only if the asset its index.html names is present +page_ok() { local a; a=$(cut -d' ' -f2 "$DEPLOY_PATH/index.html"); [[ -f $DEPLOY_PATH/$a ]]; } + +# --- dry runs and aborts ---------------------------------------------------- + +t_dry_run_lists_files() { + before=$(tree "$DEPLOY_PATH") + out=$("$DEPLOY" --dry-run 2>&1) || fail "exit $?: $out" || return 1 + grep -q 'index.html' <<<"$out" || fail "index.html not listed: $out" || return 1 + grep -q 'assets/app-new.js' <<<"$out" || fail "new asset not listed" || return 1 + grep -Eq 'deleting +stale.js' <<<"$out" || fail "stale.js deletion not listed" || return 1 + ! grep -q 'well-known' <<<"$out" || fail ".well-known would be touched" || return 1 + [[ $(tree "$DEPLOY_PATH") == "$before" && ! -L $DEPLOY_PATH ]] || fail "dry run changed docroot" || return 1 + [[ ! -e $R ]] || fail "dry run created $R" || return 1 +} + +t_dry_run_new_docroot() { + rm -rf "$DEPLOY_PATH" + out=$("$DEPLOY" --dry-run 2>&1) || fail "exit $?: $out" || return 1 + grep -q 'assets/app-new.js' <<<"$out" || fail "asset not listed: $out" || return 1 + [[ ! -e $DEPLOY_PATH && ! -e $R ]] || fail "dry run created something" || return 1 +} + +t_failed_build_aborts() { + before=$(tree "$DEPLOY_PATH") + out=$(DEPLOY_BUILD_CMD='echo boom >&2; exit 3' "$DEPLOY" 2>&1) && fail "exit 0" && return 1 + grep -q 'build failed' <<<"$out" || fail "no build-failed message: $out" || return 1 + [[ $(tree "$DEPLOY_PATH") == "$before" && ! -e $R ]] || fail "target touched" || return 1 +} + +t_empty_build_aborts() { + out=$(DEPLOY_BUILD_CMD='mkdir -p out' "$DEPLOY" 2>&1) && fail "exit 0" && return 1 + grep -q 'index.html missing' <<<"$out" || fail "$out" || return 1 + [[ -f $DEPLOY_PATH/stale.js && ! -e $R ]] || fail "target touched" || return 1 +} + +# --- deploy ----------------------------------------------------------------- + +t_deploy_migrates_plain_docroot() { + before=$(tree "$DEPLOY_PATH") + "$DEPLOY" >/dev/null 2>&1 || fail "deploy failed" || return 1 + [[ -L $DEPLOY_PATH ]] || fail "docroot is not a symlink now" || return 1 + [[ $(site) == new ]] && page_ok || fail "new site not live/complete" || return 1 + # the old site's files stay reachable (carried) but aren't part of the new release + [[ -f $DEPLOY_PATH/stale.js && -f $DEPLOY_PATH/assets/app-old.js ]] || fail "old files not kept reachable" || return 1 + ! grep -q 'stale.js\|app-old.js' "$R/$(basename "$(live)").files" || fail "carried files in new manifest" || return 1 + [[ -f $DEPLOY_PATH/.well-known/acme/tok ]] || fail ".well-known not carried over" || return 1 + [[ $(stat -c %a "$DEPLOY_PATH/index.html") == 644 ]] || fail "index.html not 644" || return 1 + mapfile -t rs < <(releases) + [[ ${#rs[@]} == 2 ]] || fail "expected old + new release, got: ${rs[*]}" || return 1 + [[ $(live) == "$R/${rs[1]}" ]] || fail "live is not the newest release" || return 1 + [[ $(tree "$R/${rs[0]}") == "$before" ]] || fail "old docroot not kept intact as a release" || return 1 +} + +t_first_deploy_missing_docroot() { + rm -rf "$T/srv" + "$DEPLOY" >/dev/null 2>&1 || fail "deploy failed" || return 1 + [[ -L $DEPLOY_PATH && $(site) == new ]] && page_ok || fail "not deployed" || return 1 + [[ $(releases | wc -l) == 1 ]] || fail "expected 1 release: $(releases)" || return 1 +} + +# nginx runs as another user, so every dir on the path to every served file +# must be o+rx and every file o+r, even when the deploying shell has umask 077. +t_nginx_can_read_everything() { + rm -rf "$T/srv" # also covers the docroot's parent dirs being created by the deploy + ( umask 077; "$DEPLOY" >/dev/null 2>&1 ) || fail "deploy failed" || return 1 + ( umask 077; deploy_v v2 ) || fail "deploy 2 failed" || return 1 + bad=$(find "$T/srv" ! -name .lock \( -type d ! -perm -o=rx -o -type f ! -perm -o=r \) -printf '%M %p\n') + [[ -z $bad ]] || fail "not readable by nginx: +$bad" || return 1 +} + +t_first_deploy_empty_docroot() { + rm -rf "$DEPLOY_PATH"; mkdir -p "$DEPLOY_PATH" + "$DEPLOY" >/dev/null 2>&1 || fail "deploy failed" || return 1 + [[ -L $DEPLOY_PATH && $(site) == new ]] || fail "not deployed" || return 1 + [[ $(releases | wc -l) == 1 ]] || fail "empty docroot kept as a release: $(releases)" || return 1 +} + +t_redeploy_reuses_unchanged_files() { + deploy_v v1 || fail "deploy v1" || return 1 + first=$(live) + echo extra > "$T/src/extra.txt" + DEPLOY_BUILD_CMD="$DEPLOY_BUILD_CMD"' && cp extra.txt out/' "$DEPLOY" >/dev/null 2>&1 || fail "deploy 2" || return 1 + [[ $(live) != "$first" ]] || fail "no new release" || return 1 + # unchanged index.html is a hard link to the previous release's copy + [[ $(stat -c %i "$first/index.html") == $(stat -c %i "$DEPLOY_PATH/index.html") ]] \ + || fail "unchanged file was re-sent instead of linked" || return 1 + [[ $(cat "$first/index.html") == "v1 assets/app-v1.js" ]] || fail "previous release modified" || return 1 +} + +# The live tree must be untouched for the whole upload: a reader sees the +# complete old site until the switch, then the complete new one. +t_live_untouched_during_upload() { + before=$(tree "$DEPLOY_PATH") + cat > "$T/rsync-probe" <> "$T/probe"; rsync "\$@"; rc=\$?; check >> "$T/probe"; exit \$rc +EOF + chmod +x "$T/rsync-probe"; printf '%s' "$before" > "$T/before" + DEPLOY_RSYNC=$T/rsync-probe "$DEPLOY" >/dev/null 2>&1 || fail "deploy failed" || return 1 + [[ $(tr '\n' ' ' < "$T/probe") == "intact intact " ]] || fail "live changed during upload: $(cat "$T/probe")" || return 1 + [[ $(site) == new ]] && page_ok || fail "new site not live" || return 1 +} + +t_failed_upload_leaves_live_intact() { + deploy_v v1 || fail "deploy v1" || return 1 + before=$(tree "$DEPLOY_PATH"); was=$(live); nrel=$(releases | wc -l) + # partial transfer: every asset but index.html arrives, then rsync fails + printf '#!/usr/bin/env bash\nrsync --exclude=/index.html "$@"; exit 30\n' > "$T/rsync-cut"; chmod +x "$T/rsync-cut" + echo v2 > "$T/src/version" + out=$(DEPLOY_RSYNC=$T/rsync-cut "$DEPLOY" 2>&1) && fail "exit 0" && return 1 + grep -q 'upload failed; live site unchanged' <<<"$out" || fail "$out" || return 1 + [[ $(live) == "$was" && $(tree "$DEPLOY_PATH") == "$before" ]] && page_ok || fail "live site changed" || return 1 + [[ $(releases | wc -l) == "$nrel" && -z $(ls "$R" | grep tmp) ]] || fail "partial upload left behind: $(ls -a "$R")" || return 1 +} + +t_killed_deploy_recovers() { + deploy_v v1 || fail "deploy v1" || return 1 + before=$(tree "$DEPLOY_PATH") + # rsync uploads part of the tree, then the whole deploy dies (CI runner lost) + printf '#!/usr/bin/env bash\nrsync --exclude=/index.html "$@"; kill -9 $PPID; exit 1\n' > "$T/rsync-kill"; chmod +x "$T/rsync-kill" + echo v2 > "$T/src/version" + { DEPLOY_RSYNC=$T/rsync-kill "$DEPLOY" >/dev/null 2>&1; } 2>/dev/null && fail "exit 0" && return 1 + sleep 0.3 + [[ $(tree "$DEPLOY_PATH") == "$before" ]] && page_ok || fail "live site changed" || return 1 + [[ -n $(ls "$R" | grep '\.tmp$') ]] || fail "expected a stale .tmp after kill" || return 1 + # the lock was released by the dead run, and the next deploy cleans up + out=$(DEPLOY_LOCK_WAIT=2 "$DEPLOY" 2>&1) || fail "next deploy failed: $out" || return 1 + grep -q 'removed stale partial upload' <<<"$out" || fail "stale .tmp not reported" || return 1 + [[ -z $(ls "$R" | grep tmp) && $(site) == v2 ]] && page_ok || fail "not recovered" || return 1 +} + +t_stale_tmp_removed() { + mkdir -p "$R/20200101T000000Z.tmp/junk" + "$DEPLOY" >/dev/null 2>&1 || fail "deploy failed" || return 1 + [[ ! -e $R/20200101T000000Z.tmp ]] || fail "stale .tmp left behind" || return 1 +} + +# --- rollback --------------------------------------------------------------- + +t_rollback_restores_previous() { + before=$(tree "$DEPLOY_PATH") + "$DEPLOY" >/dev/null 2>&1 || fail "deploy failed" || return 1 + newrel=$(live) + out=$("$DEPLOY" --rollback --dry-run 2>&1) || fail "rollback dry run: $out" || return 1 + grep -q 'index.html' <<<"$out" || fail "dry run did not list changes: $out" || return 1 + [[ $(live) == "$newrel" ]] || fail "rollback dry run switched" || return 1 + "$DEPLOY" --rollback >/dev/null 2>&1 || fail "rollback failed" || return 1 + # every file of the old tree is back, byte for byte; the only extras are the + # rolled-back release's files, kept reachable for pages loaded from it + extra=$(comm -13 <(echo "$before") <(tree "$DEPLOY_PATH")) + [[ -z $(comm -23 <(echo "$before") <(tree "$DEPLOY_PATH")) ]] || fail "old tree not restored: +$(diff <(echo "$before") <(tree "$DEPLOY_PATH"))" || return 1 + [[ $(cut -f1 <<<"$extra" | tr '\n' ' ') == "assets/app-new.js " ]] || fail "unexpected extras: $extra" || return 1 + [[ -L $DEPLOY_PATH ]] && page_ok || fail "rollback broke the symlink/page" || return 1 + # the release we left is kept, so we can roll forward by name + "$DEPLOY" --rollback "$(basename "$newrel")" >/dev/null 2>&1 && [[ $(site) == new ]] || fail "roll forward" || return 1 +} + +t_rollback_is_one_atomic_switch() { + deploy_v v1 && deploy_v v2 || fail "deploys" || return 1 + v1=$(releases | tail -n 2 | head -n 1); v2=$(releases | tail -n 1) + t1=$(own "$v1"); t2=$(own "$v2") + "$DEPLOY" --rollback >/dev/null 2>&1 || fail "rollback" || return 1 + [[ $(live) == "$R/$v1" ]] || fail "live is $(live), expected $v1" || return 1 + [[ $(own "$v1") == "$t1" && $(own "$v2") == "$t2" ]] || fail "rollback changed a release's own files" || return 1 + [[ -z $(ls -a "$(dirname "$DEPLOY_PATH")" | grep switch) ]] || fail "temp symlink left behind" || return 1 +} + +t_rollback_steps_back() { + for v in v1 v2 v3; do deploy_v $v || fail "deploy $v" || return 1; done + "$DEPLOY" --rollback >/dev/null 2>&1 && [[ $(site) == v2 ]] || fail "1st rollback" || return 1 + "$DEPLOY" --rollback >/dev/null 2>&1 && [[ $(site) == v1 ]] || fail "2nd rollback" || return 1 + deploy_v v4 && [[ $(site) == v4 ]] && page_ok || fail "deploy after rollback" || return 1 +} + +t_rollback_named() { + "$DEPLOY" >/dev/null 2>&1 && deploy_v v2 || fail "deploys" || return 1 + oldest=$(releases | head -n 1) + "$DEPLOY" --rollback "$oldest" >/dev/null 2>&1 || fail "named rollback" || return 1 + [[ $(site) == old ]] && page_ok || fail "wrong tree" || return 1 + "$DEPLOY" --rollback 20000101T000000Z >/dev/null 2>&1 && fail "bogus name accepted" && return 1 + "$DEPLOY" --rollback '../../etc' >/dev/null 2>&1 && fail "traversal name accepted" && return 1 + [[ $(site) == old ]] || fail "failed rollback changed the site" || return 1 +} + +t_rollback_without_previous_fails() { + out=$("$DEPLOY" --rollback 2>&1) && fail "exit 0 on plain docroot" && return 1 + grep -q 'nothing to roll back to' <<<"$out" || fail "$out" || return 1 + rm -rf "$DEPLOY_PATH"; "$DEPLOY" >/dev/null 2>&1 || fail "deploy" || return 1 + out=$("$DEPLOY" --rollback 2>&1) && fail "exit 0 with a single release" && return 1 + grep -q 'no release older' <<<"$out" || fail "$out" || return 1 + [[ $(site) == new ]] || fail "site changed" || return 1 +} + +# --- retention -------------------------------------------------------------- + +t_retention_keeps_last_n() { + for v in v1 v2 v3 v4 v5; do deploy_v $v || fail "deploy $v" || return 1; done + mapfile -t rs < <(releases) + [[ ${#rs[@]} == 3 ]] || fail "expected 3 releases, got ${#rs[@]}: ${rs[*]}" || return 1 + got=$(for r in "${rs[@]}"; do cut -d' ' -f1 "$R/$r/index.html"; done | tr '\n' ' ') + [[ $got == "v3 v4 v5 " ]] || fail "kept wrong releases: $got" || return 1 + [[ $("$DEPLOY" --list-releases 2>/dev/null | grep -c '(live)') == 1 ]] || fail "--list-releases live marker" || return 1 +} + +t_prune_never_removes_live() { + for v in v1 v2 v3; do deploy_v $v || fail "deploy $v" || return 1; done + "$DEPLOY" --rollback >/dev/null 2>&1 && "$DEPLOY" --rollback >/dev/null 2>&1 || fail "rollbacks" || return 1 + bash "$REMOTE" "$DEPLOY_PATH" "$R" 1 3 prune >/dev/null 2>&1 || fail "prune" || return 1 + [[ $(site) == v1 ]] && page_ok || fail "prune removed the live release" || return 1 +} + +# --- grace period: pages from release N can fetch N's files after a switch --- + +# Codex P2 repro: a browser loads index.html, the site switches, then the +# browser fetches the asset that index.html named. Over real HTTP. +t_old_index_then_switch_no_404() { + serve || fail "http server did not start" || return 1 + trap 'kill $HTTP_PID 2>/dev/null' RETURN + for v in v1 v2 v3; do + [[ $(get /) == 200 ]] || fail "GET / before $v" || return 1 + page=$(cat "$T/body"); asset=$(asset_of "$page") + deploy_v $v || fail "deploy $v" || return 1 + [[ $(get /) == 200 && $(cat "$T/body") == "$v assets/app-$v.js" ]] || fail "$v not live" || return 1 + code=$(get "$asset") + [[ $code == 200 ]] || fail "after switching to $v, the previous page's $asset is $code" || return 1 + [[ $(cat "$T/body") == "$(cut -d' ' -f1 <<<"$page")-js" ]] || fail "$asset has the wrong content" || return 1 + done +} + +t_old_index_then_rollback_no_404() { + deploy_v v1 && deploy_v v2 || fail "deploys" || return 1 + serve || fail "http server" || return 1 + trap 'kill $HTTP_PID 2>/dev/null' RETURN + get / >/dev/null; page2=$(cat "$T/body") + "$DEPLOY" --rollback >/dev/null 2>&1 || fail "rollback" || return 1 + [[ $(get "$(asset_of "$page2")") == 200 ]] || fail "rollback: v2 page's asset gone" || return 1 + get / >/dev/null; page1=$(cat "$T/body"); [[ $page1 == "v1 "* ]] || fail "v1 not live" || return 1 + # roll forward again: pages loaded from v1 during the rollback keep working + "$DEPLOY" --rollback "$(releases | tail -n 1)" >/dev/null 2>&1 || fail "roll forward" || return 1 + [[ $(get "$(asset_of "$page1")") == 200 ]] || fail "roll forward: v1 page's asset gone" || return 1 +} + +t_removed_data_file_stays_reachable() { + # not just hashed assets: any file a page may fetch (e.g. b4-image.json) + DEPLOY_BUILD_CMD="$DEPLOY_BUILD_CMD"' && echo "$v data" > out/data.json && echo "$v" > out/only-$v.json' \ + "$DEPLOY" >/dev/null 2>&1 || fail "deploy with data" || return 1 + echo v2 > "$T/src/version" + DEPLOY_BUILD_CMD="$DEPLOY_BUILD_CMD"' && echo "$v data" > out/data.json' "$DEPLOY" >/dev/null 2>&1 || fail "deploy 2" || return 1 + [[ $(cat "$DEPLOY_PATH/only-new.json" 2>/dev/null) == new ]] || fail "removed data file not reachable" || return 1 + # a file the new release ships itself is never overwritten by an older copy + [[ $(cat "$DEPLOY_PATH/data.json") == "v2 data" ]] || fail "carried file overwrote the new release's own" || return 1 +} + +t_grace_window_is_bounded() { + for v in v1 v2 v3 v4; do DEPLOY_ASSET_GRACE=2 deploy_v $v || fail "deploy $v" || return 1; done + # live v4; the two releases live before it (v3, v2) stay reachable, v1 has aged out + for v in v4 v3 v2; do [[ -f $DEPLOY_PATH/assets/app-$v.js ]] || fail "app-$v.js missing" || return 1; done + [[ ! -e $DEPLOY_PATH/assets/app-v1.js ]] || fail "app-v1.js still carried after the window" || return 1 + [[ ! -e $DEPLOY_PATH/assets/app-new.js && ! -e $DEPLOY_PATH/assets/app-old.js ]] || fail "older files still carried" || return 1 +} + +t_prune_keeps_grace_window() { + # KEEP=1 would keep only the live release; the grace window must win + for v in v1 v2 v3 v4; do DEPLOY_KEEP=1 DEPLOY_ASSET_GRACE=2 deploy_v $v || fail "deploy $v" || return 1; done + mapfile -t rs < <(releases) + got=$(for r in "${rs[@]}"; do cut -d' ' -f1 "$R/$r/index.html"; done | tr '\n' ' ') + [[ $got == "v3 v4 " ]] || fail "expected v3 v4 kept, got: $got" || return 1 + # v2 was pruned, but its files carried into v4 are hard links and survive + [[ $(cat "$DEPLOY_PATH/assets/app-v2.js") == v2-js ]] || fail "pruning v2 broke its carried asset" || return 1 + [[ $(ls "$R" | grep -c '\.files$') == 2 ]] || fail "pruned releases' manifests left behind: $(ls "$R")" || return 1 +} + +t_carry_ignores_bad_manifest_entries() { + deploy_v v1 || fail "deploy" || return 1 + v1=$(basename "$(live)") + # entries with .. or a leading / are skipped even when they resolve to a real file + echo planted > "$R/$v1/planted.txt"; echo planted > "$R/$v1/abs.txt" + printf '%s\n' 'assets/../planted.txt' '/abs.txt' '../../escape' >> "$R/$v1.files" + deploy_v v2 || fail "deploy 2" || return 1 + [[ ! -e $DEPLOY_PATH/planted.txt && ! -e $DEPLOY_PATH/abs.txt ]] || fail "followed a bad manifest entry" || return 1 + [[ -z $(find "$T" -name 'escape*') ]] || fail "manifest entry escaped the release" || return 1 + [[ -f $DEPLOY_PATH/assets/app-v1.js ]] || fail "good entries no longer carried" || return 1 +} + +t_grace_zero_disables_carry() { + DEPLOY_ASSET_GRACE=0 deploy_v v1 || fail "deploy" || return 1 + [[ ! -e $DEPLOY_PATH/assets/app-old.js ]] || fail "carried with grace 0" || return 1 + DEPLOY_ASSET_GRACE=x "$DEPLOY" --dry-run >/dev/null 2>&1 && fail "accepted DEPLOY_ASSET_GRACE=x" && return 1 + return 0 +} + +# --- failure safety: whatever fails before the switch, the old site keeps serving --- + +# old site still fully served: docroot present, same tree, page loads +still_serving() { [[ -e $DEPLOY_PATH ]] && [[ $(tree "$DEPLOY_PATH") == "$1" ]] && page_ok; } +no_switch_left() { [[ -z $(ls -A "$(dirname "$DEPLOY_PATH")" | grep '\.switch$') ]]; } +# build variants: ship $1 as a directory with a file in it, or as a plain file +build_dir() { echo "$DEPLOY_BUILD_CMD"' && mkdir -p out/'"$1"' && echo "$v" > out/'"$1"'/old.json'; } +build_file() { echo "$DEPLOY_BUILD_CMD"' && echo "$v" > out/'"$1"; } + +# Codex repro: the old docroot has downloads/old.json; the new release ships a +# regular file called downloads. Carrying old.json can't work; it must not take the site down. +t_migration_file_vs_dir_conflict() { + mkdir -p "$DEPLOY_PATH/downloads"; echo old > "$DEPLOY_PATH/downloads/old.json" + before=$(tree "$DEPLOY_PATH") + out=$(DEPLOY_BUILD_CMD=$(build_file downloads) "$DEPLOY" 2>&1) || fail "deploy failed: $out" || return 1 + grep -q "can't keep /downloads/old.json reachable" <<<"$out" || fail "no conflict warning: $out" || return 1 + [[ -L $DEPLOY_PATH && $(site) == new && $(cat "$DEPLOY_PATH/downloads") == new ]] && page_ok || fail "new site not live" || return 1 + [[ $(tree "$R/$(releases | head -n 1)") == "$before" ]] || fail "old site not kept as a release" || return 1 + [[ -f $DEPLOY_PATH/assets/app-old.js ]] || fail "other old files no longer carried" || return 1 +} + +t_deploy_file_vs_dir_conflict() { + DEPLOY_BUILD_CMD=$(build_dir downloads) deploy_v v1 || fail "deploy v1" || return 1 + out=$(echo v2 > "$T/src/version"; DEPLOY_BUILD_CMD=$(build_file downloads) "$DEPLOY" 2>&1) || fail "deploy v2: $out" || return 1 + grep -q "can't keep /downloads/old.json" <<<"$out" || fail "no warning: $out" || return 1 + [[ $(site) == v2 && -f $DEPLOY_PATH/downloads ]] && page_ok || fail "v2 not live" || return 1 +} + +t_deploy_dir_vs_file_conflict() { + DEPLOY_BUILD_CMD=$(build_file downloads) deploy_v v1 || fail "deploy v1" || return 1 + out=$(echo v2 > "$T/src/version"; DEPLOY_BUILD_CMD=$(build_dir downloads) "$DEPLOY" 2>&1) || fail "deploy v2: $out" || return 1 + grep -q "can't keep /downloads reachable" <<<"$out" || fail "no warning: $out" || return 1 + [[ $(site) == v2 && -f $DEPLOY_PATH/downloads/old.json ]] && page_ok || fail "v2 not live" || return 1 +} + +t_rollback_file_vs_dir_conflict() { + DEPLOY_BUILD_CMD=$(build_file downloads) deploy_v v1 || fail "deploy v1" || return 1 + echo v2 > "$T/src/version"; DEPLOY_BUILD_CMD=$(build_dir downloads) "$DEPLOY" >/dev/null 2>&1 || fail "deploy v2" || return 1 + out=$("$DEPLOY" --rollback 2>&1) || fail "rollback failed: $out" || return 1 + grep -q "can't keep /downloads/old.json" <<<"$out" || fail "no warning: $out" || return 1 + [[ $(site) == v1 && -f $DEPLOY_PATH/downloads ]] && page_ok || fail "v1 not live" || return 1 + out=$("$DEPLOY" --rollback "$(releases | tail -n 1)" 2>&1) || fail "roll forward failed: $out" || return 1 + [[ $(site) == v2 ]] && page_ok || fail "v2 not live again" || return 1 +} + +t_carry_never_writes_through_symlinks() { + mkdir -p "$T/outside" "$T/secret"; echo secret > "$T/secret/s.txt" + DEPLOY_BUILD_CMD=$(build_dir data) deploy_v v1 || fail "deploy v1" || return 1 + v1=$(basename "$(live)") + # the next release ships `data` as a symlink out of the tree: carrying + # data/old.json must not follow it + echo v2 > "$T/src/version" + out=$(DEPLOY_BUILD_CMD="$DEPLOY_BUILD_CMD && ln -s '$T/outside' out/data" "$DEPLOY" 2>&1) || fail "deploy v2: $out" || return 1 + [[ -z $(ls -A "$T/outside") ]] || fail "carry wrote outside the release: $(ls "$T/outside")" || return 1 + # and a source path through a symlink is never carried in + ln -s "$T/secret" "$R/$v1/lnk"; echo lnk/s.txt >> "$R/$v1.files" + echo v3 > "$T/src/version"; "$DEPLOY" >/dev/null 2>&1 || fail "deploy v3" || return 1 + [[ ! -e $DEPLOY_PATH/lnk ]] || fail "carried a file through a symlinked dir" || return 1 +} + +# Inject a failure at every fallible point of every kind of switch. +t_fault_migration_keeps_old_site() { + before=$(tree "$DEPLOY_PATH") + for point in stage before-commit migrate-before-move migrate-between-renames; do + out=$(DEPLOY_FAULT=$point "$DEPLOY" 2>&1) && fail "$point: exit 0" && return 1 + grep -q "injected fault at $point" <<<"$out" || fail "$point: fault not reached: $out" || return 1 + [[ -d $DEPLOY_PATH && ! -L $DEPLOY_PATH ]] || fail "$point: docroot is gone or replaced" || return 1 + still_serving "$before" || fail "$point: old site not serving" || return 1 + [[ -z $(releases) ]] && no_switch_left || fail "$point: leftovers: $(ls -A "$R") $(ls -A "$(dirname "$DEPLOY_PATH")")" || return 1 + done + grep -q 'old site was restored' <<<"$out" || fail "no restore message: $out" || return 1 + "$DEPLOY" >/dev/null 2>&1 || fail "fault-free deploy afterwards failed" || return 1 + [[ $(site) == new ]] && page_ok && no_switch_left || fail "not recovered" || return 1 + [[ $(tree "$R/$(releases | head -n 1)") == "$before" ]] || fail "old site not kept as a release" || return 1 +} + +t_fault_empty_docroot_migration() { + rm -rf "$DEPLOY_PATH"; mkdir "$DEPLOY_PATH" + for point in stage before-commit; do + DEPLOY_FAULT=$point "$DEPLOY" >/dev/null 2>&1 && fail "$point: exit 0" && return 1 + [[ -d $DEPLOY_PATH && ! -L $DEPLOY_PATH ]] || fail "$point: docroot gone" || return 1 + done + "$DEPLOY" >/dev/null 2>&1 && [[ $(site) == new ]] || fail "fault-free deploy afterwards" || return 1 +} + +t_fault_deploy_keeps_old_site() { + deploy_v v1 || fail "deploy v1" || return 1 + before=$(tree "$DEPLOY_PATH"); was=$(live) + for point in stage before-commit; do + echo v2 > "$T/src/version" + out=$(DEPLOY_FAULT=$point "$DEPLOY" 2>&1) && fail "$point: exit 0" && return 1 + grep -q "injected fault at $point" <<<"$out" || fail "$point: fault not reached: $out" || return 1 + [[ $(live) == "$was" ]] && still_serving "$before" || fail "$point: old site not serving" || return 1 + [[ $(releases | tail -n 1) == "$(basename "$was")" ]] || fail "$point: a never-live release was kept: $(releases)" || return 1 + done + deploy_v v3 && [[ $(site) == v3 ]] && no_switch_left || fail "fault-free deploy afterwards" || return 1 +} + +t_fault_rollback_keeps_site() { + deploy_v v1 && deploy_v v2 || fail "deploys" || return 1 + before=$(tree "$DEPLOY_PATH"); was=$(live) + for point in stage before-commit; do + out=$(DEPLOY_FAULT=$point "$DEPLOY" --rollback 2>&1) && fail "rollback $point: exit 0" && return 1 + [[ $(live) == "$was" ]] && still_serving "$before" || fail "rollback $point: site changed" || return 1 + done + "$DEPLOY" --rollback >/dev/null 2>&1 && [[ $(site) == v1 ]] || fail "rollback afterwards" || return 1 + # roll-forward is the same switch + before=$(tree "$DEPLOY_PATH"); was=$(live) + for point in stage before-commit; do + DEPLOY_FAULT=$point "$DEPLOY" --rollback "$(releases | tail -n 1)" >/dev/null 2>&1 && fail "forward $point: exit 0" && return 1 + [[ $(live) == "$was" ]] && still_serving "$before" || fail "forward $point: site changed" || return 1 + done + no_switch_left || fail "temp symlink left" || return 1 +} + +t_after_switch_failures_report_live() { + deploy_v v1 || fail "deploy v1" || return 1 + # history can't be written: the switch already happened, so warn and succeed + chmod 444 "$R/.history" + out=$(echo v2 > "$T/src/version"; "$DEPLOY" 2>&1); rc=$? + chmod 644 "$R/.history" + ((rc == 0)) || fail "history failure failed the deploy: $out" || return 1 + [[ $(site) == v2 ]] || fail "v2 not live" || return 1 + if [[ $(id -u) != 0 ]]; then grep -q 'recording it in' <<<"$out" || fail "no history warning: $out" || return 1; fi + # prune fails after the switch: nonzero, saying the new release is live + out=$(echo v3 > "$T/src/version"; DEPLOY_FAULT=prune "$DEPLOY" 2>&1) && fail "prune failure exited 0" && return 1 + grep -q 'is live, but pruning old releases failed' <<<"$out" || fail "$out" || return 1 + [[ $(site) == v3 ]] && page_ok || fail "v3 not live" || return 1 +} + +# --- guards ----------------------------------------------------------------- + +t_remote_rejects_bad_names() { + "$DEPLOY" >/dev/null 2>&1 || fail "deploy" || return 1 + keep=$(releases); was=$(live) + for bad in '' . .. '*' '../site' '20261029T120000Z/..' 'x' '20261029T120000Z.tmp'; do + for act in discard switch activate diff; do + bash "$REMOTE" "$DEPLOY_PATH" "$R" 3 3 $act "$bad" >/dev/null 2>&1 \ + && fail "$act accepted '$bad'" && return 1 + done + done + [[ $(releases) == "$keep" && $(live) == "$was" ]] && page_ok || fail "releases or live damaged" || return 1 + # a dir that only looks like a release (glob match, not the exact format) is never pruned + mkdir -p "$R/20xxZ" + bash "$REMOTE" "$DEPLOY_PATH" "$R" 1 3 prune >/dev/null 2>&1 || fail "prune failed" || return 1 + [[ -d $R/20xxZ ]] || fail "prune touched a non-release dir" || return 1 +} + +t_switch_refuses_plain_dir() { + deploy_v v1 || fail "deploy" || return 1 + name=$(releases | tail -n 1) + rm "$DEPLOY_PATH"; mkdir "$DEPLOY_PATH"; echo hand > "$DEPLOY_PATH/index.html" + bash "$REMOTE" "$DEPLOY_PATH" "$R" 3 3 switch "$name" >/dev/null 2>&1 && fail "replaced a real dir" && return 1 + [[ $(cat "$DEPLOY_PATH/index.html") == hand ]] || fail "real dir touched" || return 1 +} + +t_rejects_unsafe_paths() { + for p in '' relative/path / /var '/srv/www/../etc' '/srv/my site'; do + DEPLOY_PATH=$p "$DEPLOY" --dry-run >/dev/null 2>&1 && fail "accepted DEPLOY_PATH='$p'" && return 1 + done + DEPLOY_RELEASES=$DEPLOY_PATH/rel "$DEPLOY" --dry-run >/dev/null 2>&1 \ + && fail "accepted releases dir inside docroot" && return 1 + DEPLOY_RELEASES=/tmp "$DEPLOY" --dry-run >/dev/null 2>&1 \ + && fail "accepted shallow releases dir" && return 1 + DEPLOY_KEEP=0 "$DEPLOY" --dry-run >/dev/null 2>&1 && fail "accepted DEPLOY_KEEP=0" && return 1 + DEPLOY_LOCK_WAIT=soon "$DEPLOY" --dry-run >/dev/null 2>&1 && fail "accepted DEPLOY_LOCK_WAIT=soon" && return 1 + DEPLOY_SOURCE_DIR=$T/nope "$DEPLOY" --dry-run >/dev/null 2>&1 && fail "accepted missing source dir" && return 1 + "$DEPLOY" --bogus >/dev/null 2>&1 && fail "accepted --bogus" && return 1 + [[ ! -L $DEPLOY_PATH && ! -e $R ]] || fail "target touched" || return 1 +} + +t_unreachable_host_fails() { + out=$(DEPLOY_HOST=remote DEPLOY_SSH=false "$DEPLOY" 2>&1) && fail "exit 0" && return 1 + [[ ! -L $DEPLOY_PATH && ! -e $R ]] || fail "target touched" || return 1 +} + +t_conf_file_and_env_override() { + printf 'DEPLOY_HOST=local\nDEPLOY_PATH=%s\nDEPLOY_KEEP=7\n' "$T/elsewhere/site" > "$T/conf" + out=$(unset DEPLOY_PATH; DEPLOY_CONF=$T/conf "$DEPLOY" --dry-run 2>&1) || fail "$out" || return 1 + grep -q "$T/elsewhere/site" <<<"$out" || fail "conf not used: $out" || return 1 + out=$(DEPLOY_CONF=$T/conf DEPLOY_PATH=$DEPLOY_PATH "$DEPLOY" --dry-run 2>&1) || fail "$out" || return 1 + grep -Eq 'deleting +stale.js' <<<"$out" || fail "env did not override conf: $out" || return 1 + # empty env values (unset CI variables) must not clobber the conf file + out=$(DEPLOY_CONF=$T/conf DEPLOY_PATH= DEPLOY_KEEP= DEPLOY_USER= "$DEPLOY" --dry-run 2>&1) || fail "$out" || return 1 + grep -q "$T/elsewhere/site" <<<"$out" || fail "empty env clobbered conf: $out" || return 1 +} + +# --- locking ---------------------------------------------------------------- + +t_lock_blocks_concurrent_runs() { + "$DEPLOY" >/dev/null 2>&1 || fail "deploy" || return 1 + was=$(live) + # exec so $holder is the process holding fd 9; killing it releases the lock + ( exec 9>>"$R/.lock"; flock 9; exec sleep 30 ) & holder=$! + sleep 0.3 + out=$(DEPLOY_LOCK_WAIT=1 "$DEPLOY" 2>&1) && { kill $holder; fail "deploy ran while locked"; return 1; } + grep -q 'could not take' <<<"$out" || { kill $holder; fail "no lock message: $out"; return 1; } + out=$(DEPLOY_LOCK_WAIT=1 "$DEPLOY" --rollback 2>&1) && { kill $holder; fail "rollback ran while locked"; return 1; } + kill $holder; wait $holder 2>/dev/null + [[ $(live) == "$was" && $(releases | wc -l) == 2 ]] || fail "state changed while locked" || return 1 + # released after each run: back-to-back runs don't wait on each other + out=$(DEPLOY_LOCK_WAIT=0 "$DEPLOY" 2>&1) || fail "lock not released after deploy: $out" || return 1 + out=$(DEPLOY_LOCK_WAIT=0 "$DEPLOY" --rollback 2>&1) || fail "lock not released after rollback: $out" || return 1 +} + +t_lock_held_during_upload() { + [[ $DEPLOY_HOST == local ]] && return 77 # skip: needs the ssh hook below + # wrap ssh: when rsync's upload connects, record whether the lock is held + cat > "$T/probe-ssh" <> "$T/probe" || echo held >> "$T/probe" +fi +exec "$WORK/fake-ssh" "\$@" +EOF + chmod +x "$T/probe-ssh" + DEPLOY_SSH=$T/probe-ssh "$DEPLOY" >/dev/null 2>&1 || fail "deploy failed" || return 1 + [[ $(cat "$T/probe") == held ]] || fail "lock not held during rsync: $(cat "$T/probe")" || return 1 +} + +# --- CI ref selection and old sources --------------------------------------- + +t_select_ref_only_merged() { + [[ $DEPLOY_HOST == local ]] || return 77 # skip: git-only; once is enough + g=$T/repo; sel=$ROOT/deploy/select-ref.sh + git init -q -b main "$g" && cd "$g" || return 1 + c() { git -c user.name=t -c user.email=t@t commit -q --allow-empty -m "$1"; git rev-parse HEAD; } + old=$(c one); git tag v1; main=$(c two) + git checkout -q -b pr; pr=$(c unmerged); git checkout -q main + [[ $("$sel" '' "$main") == "$main" ]] || fail "empty ref" || return 1 + [[ $("$sel" "$old" "$main") == "$old" ]] || fail "merged sha" || return 1 + [[ $("$sel" v1 "$main") == "$old" ]] || fail "merged tag" || return 1 + [[ $("$sel" main "$main") == "$main" ]] || fail "main itself" || return 1 + for bad in pr "$pr" nosuchref -x --help "$main~5"; do + "$sel" "$bad" "$main" >/dev/null 2>&1 && fail "accepted '$bad'" && return 1 + [[ ! -e $T/x ]] || fail "exported for rejected '$bad'" || return 1 + "$sel" "$bad" "$main" "$T/x" >/dev/null 2>&1; [[ ! -e $T/x ]] || fail "exported for rejected '$bad'" || return 1 + done + return 0 +} + +# Codex P2: redeploying a commit from before bin/deploy existed. The source +# comes from that commit, and the tooling (this bin/deploy) from the current one. +PRE_TOOLING=6813323575a2f29d2a289139aad4450f77676e89 +t_redeploy_pre_tooling_commit() { + [[ $DEPLOY_HOST == local ]] || return 77 # skip: git-only; once is enough + git -C "$ROOT" cat-file -e "$PRE_TOOLING^{commit}" 2>/dev/null \ + || fail "commit $PRE_TOOLING not in history (CI must use fetch-depth: 0)" || return 1 + cd "$ROOT" && src=$T/old-source + sha=$("$ROOT/deploy/select-ref.sh" "$PRE_TOOLING" HEAD "$src" 2>&1) || fail "select: $sha" || return 1 + [[ ! -e $src/bin/deploy && ! -e $src/deploy ]] || fail "old commit unexpectedly has tooling" || return 1 + ! grep -q test:deploy "$src/package.json" || fail "old package.json has test:deploy" || return 1 + [[ $(cat "$src/.deploy-sha") == "$PRE_TOOLING" ]] || fail ".deploy-sha" || return 1 + out=$(DEPLOY_SOURCE_DIR=$src DEPLOY_BUILD_CMD='mkdir -p out && cp web/index.html out/' "$DEPLOY" 2>&1) \ + || fail "deploy of old source failed: $out" || return 1 + cmp -s "$DEPLOY_PATH/index.html" <(git -C "$ROOT" show "$PRE_TOOLING:web/index.html") \ + || fail "live index.html is not the old commit's" || return 1 + grep -q "deployed $PRE_TOOLING" <<<"$out" || fail "log does not name the source commit: $out" || return 1 +} + +cases=$(declare -F | awk '{print $3}' | grep '^t_') +for mode in local ssh; do + for c in $cases; do + total=$((total+1)) + setup + if [[ $mode == local ]]; then export DEPLOY_HOST=local; unset DEPLOY_SSH DEPLOY_USER + else export DEPLOY_HOST=testhost DEPLOY_USER=tester DEPLOY_SSH="$WORK/fake-ssh -o BatchMode=yes"; fi + ( $c ); rc=$? + if ((rc == 0)); then pass=$((pass+1)); echo "ok $mode $c" + elif ((rc == 77)); then total=$((total-1)); skipped=$((skipped+1)) + else failed+=("$mode $c"); echo "FAIL $mode $c"; fi + done +done + +echo "deploy tests: $pass/$total passed ($skipped mode-specific skips)" +((pass == total))