diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b892ca0..613604c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,7 +43,7 @@ jobs: persist-credentials: false - name: Shellcheck - run: shellcheck -x scripts/*.sh tests/*.sh tests/lib/*.sh tests/lib/fake-gh + run: shellcheck -x scripts/*.sh tests/*.sh tests/lib/*.sh tests/lib/fake-gh tests/lib/fake-curl - name: Run script tests run: | diff --git a/scripts/check-image-updates.sh b/scripts/check-image-updates.sh index 1297021..23c9cd3 100755 --- a/scripts/check-image-updates.sh +++ b/scripts/check-image-updates.sh @@ -64,22 +64,35 @@ print('^' + pat + r'$') " "$1" } -# Fetch all tags from Docker Hub or GHCR. +# Fetch all tags from Docker Hub or GHCR. Fails if any request fails: the +# caller runs it inside `if`, where errexit is off, and a partial list would +# be taken as complete. fetch_tags() { local image=$1 if [[ $image == ghcr.io/* ]]; then local repo=${image#ghcr.io/} local token token=$(curl -fsSL "https://ghcr.io/token?scope=repository:${repo}:pull" \ - | python3 -c "import sys,json;print(json.load(sys.stdin)['token'])") - # n=10000: OCI tags/list defaults to ~100 in lex order which misses - # high-numbered versions (e.g., v1.51.0 sorts after v1.9.0) - curl -fsSL -H "Authorization: Bearer $token" \ - "https://ghcr.io/v2/${repo}/tags/list?n=10000" \ - | python3 -c "import sys,json;[print(t) for t in json.load(sys.stdin).get('tags',[])]" + | python3 -c "import sys,json;print(json.load(sys.stdin)['token'])") || return 1 + # GHCR returns at most 1000 tags per page whatever n asks for, in + # lexicographic order, so the newest versions are on the last page. + local path="/v2/${repo}/tags/list?n=10000" + local headers + headers=$(mktemp) + while [ -n "$path" ]; do + curl -fsSL -D "$headers" -H "Authorization: Bearer $token" \ + "https://ghcr.io${path}" \ + | python3 -c "import sys,json;[print(t) for t in json.load(sys.stdin).get('tags',[])]" \ + || { rm -f "$headers"; return 1; } + path=$(tr -d '\r' < "$headers" | grep -i '^link:' \ + | sed -nE 's/.*<([^>]*)>; *rel="next".*/\1/p') + done + rm -f "$headers" else local path=$image [[ $image != */* ]] && path="library/$image" + # One page suffices: last_updated ordering puts the newest tags first, and + # anonymous requests are refused past 1000 results. curl -fsSL \ "https://hub.docker.com/v2/repositories/${path}/tags?page_size=100&ordering=last_updated" \ | python3 -c "import sys,json;[print(r['name']) for r in json.load(sys.stdin).get('results',[])]" @@ -114,7 +127,9 @@ while IFS='|' read -r file service image current_tag; do continue fi - latest=$(echo "$matching" | sort -V | tail -1) + # The current tag takes part, so a tag list missing the newest versions + # yields no change rather than a downgrade. + latest=$(printf '%s\n%s\n' "$matching" "$current_tag" | sort -V | tail -1) echo " Latest: $latest" if [ "$latest" != "$current_tag" ]; then diff --git a/tests/check-image-updates.test.sh b/tests/check-image-updates.test.sh new file mode 100755 index 0000000..48f4c91 --- /dev/null +++ b/tests/check-image-updates.test.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# Tests for scripts/check-image-updates.sh against a fake registry. +set -euo pipefail + +# shellcheck source=tests/lib/step.sh +source "$(dirname "$0")/lib/step.sh" +SCRIPT="$ROOT/scripts/check-image-updates.sh" + +TAB=$'\t' +GHCR_TOKEN="https://ghcr.io/token?scope=repository:acme/app:pull${TAB}0${TAB}{\"token\":\"t\"}${TAB}" +GHCR_PAGE1="https://ghcr.io/v2/acme/app/tags/list?n=10000" +GHCR_PAGE2="https://ghcr.io/v2/acme/app/tags/list?last=2023.2.0&n=10000" +GHCR_NEXT="; rel=\"next\"" +HUB_PAGE1="https://hub.docker.com/v2/repositories/acme/tool/tags?page_size=100&ordering=last_updated" + +# check_image — runs the script on a compose file +# holding that image; leaves the resulting image line in IMAGE_LINE. +check_image() { + in_temp_dir + printf 'services:\n app:\n image: %s\n' "$1" > docker-compose.yml + shift + CURL_ROUTES=$(mktemp) + printf '%s\n' "$@" > "$CURL_ROUTES" + export CURL_ROUTES + COMPOSE_PATTERN=docker-compose.yml run_step "$SCRIPT" || { + cat "$STDOUT_FILE" + FAILURES=$((FAILURES + 1)) + } + IMAGE_LINE=$(grep 'image:' docker-compose.yml | sed 's/ *image: //') +} + +test_ghcr_follows_link_to_second_page() { + check_image ghcr.io/acme/app:2023.1.0 "$GHCR_TOKEN" \ + "$GHCR_PAGE1${TAB}0${TAB}{\"tags\":[\"2023.1.0\",\"2023.2.0\"]}${TAB}$GHCR_NEXT" \ + "$GHCR_PAGE2${TAB}0${TAB}{\"tags\":[\"2026.1.0\"]}${TAB}" + check "tag from the second GHCR page wins" ghcr.io/acme/app:2026.1.0 "$IMAGE_LINE" +} + +test_ghcr_single_page() { + check_image ghcr.io/acme/app:2023.1.0 "$GHCR_TOKEN" \ + "$GHCR_PAGE1${TAB}0${TAB}{\"tags\":[\"2023.1.0\",\"2023.2.0\"]}${TAB}" + check "single GHCR page without Link" ghcr.io/acme/app:2023.2.0 "$IMAGE_LINE" +} + +test_ghcr_failed_page_skips_image() { + check_image ghcr.io/acme/app:2023.1.0 "$GHCR_TOKEN" \ + "$GHCR_PAGE1${TAB}0${TAB}{\"tags\":[\"2023.1.0\",\"2023.2.0\"]}${TAB}$GHCR_NEXT" \ + "$GHCR_PAGE2${TAB}22${TAB}${TAB}" + check "failed second page leaves the image alone" ghcr.io/acme/app:2023.1.0 "$IMAGE_LINE" + check "failed second page is reported" yes \ + "$(grep -q 'Failed to fetch tags' "$STDOUT_FILE" && echo yes || echo no)" +} + +test_ghcr_failed_token_is_reported() { + check_image ghcr.io/acme/app:2023.1.0 + check "failed token request is reported" yes \ + "$(grep -q 'Failed to fetch tags' "$STDOUT_FILE" && echo yes || echo no)" +} + +test_never_proposes_an_older_tag() { + check_image ghcr.io/acme/app:2026.1.0 "$GHCR_TOKEN" \ + "$GHCR_PAGE1${TAB}0${TAB}{\"tags\":[\"2023.1.0\",\"2023.2.0\"]}${TAB}" + check "older tags only leave the image alone" ghcr.io/acme/app:2026.1.0 "$IMAGE_LINE" +} + +test_docker_hub_reads_newest_page_only() { + check_image acme/tool:1.0.0 \ + "$HUB_PAGE1${TAB}0${TAB}{\"results\":[{\"name\":\"1.2.0\"},{\"name\":\"1.0.0\"}],\"next\":\"$HUB_PAGE1&page=2\"}${TAB}" + check "Docker Hub newest tag from the first page" acme/tool:1.2.0 "$IMAGE_LINE" +} + +FAKEBIN=$(mktemp -d) +ln -s "$ROOT/tests/lib/fake-curl" "$FAKEBIN/curl" +PATH="$FAKEBIN:$PATH" +for t in $(declare -F | awk '{print $3}' | grep '^test_'); do + "$t" +done + +[ "$FAILURES" -eq 0 ] diff --git a/tests/lib/fake-curl b/tests/lib/fake-curl new file mode 100755 index 0000000..c27d995 --- /dev/null +++ b/tests/lib/fake-curl @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Fake curl for tests. Answers URLs listed in $CURL_ROUTES, one per line as +# TAB TAB TAB . +# Writes the response headers to the -D file; an unlisted URL fails like a 404. +headers="" +url="" +while [ $# -gt 0 ]; do + case $1 in + -D) headers=$2; shift 2 ;; + -H) shift 2 ;; + -*) shift ;; + *) url=$1; shift ;; + esac +done + +while IFS=$'\t' read -r route code body link; do + [ "$route" = "$url" ] || continue + if [ -n "$headers" ]; then + { + printf 'HTTP/2 200\r\n' + [ -n "$link" ] && printf 'link: %s\r\n' "$link" + printf '\r\n' + } > "$headers" + fi + [ "$code" = 0 ] || exit "$code" + printf '%s\n' "$body" + exit 0 +done < "$CURL_ROUTES" +exit 22