fix: follow GHCR pagination when fetching tags, not just the first page

.github/workflows/ci.yml

@@ -43,7 +43,7 @@ jobs:
           persist-credentials: false
 
       - name: Shellcheck
-        run: shellcheck -x scripts/*.sh tests/*.sh tests/lib/*.sh tests/lib/fake-gh
+        run: shellcheck -x scripts/*.sh tests/*.sh tests/lib/*.sh tests/lib/fake-gh tests/lib/fake-curl
 
       - name: Run script tests
         run: |

scripts/check-image-updates.sh

@@ -64,22 +64,35 @@ print('^' + pat + r'$')
 " "$1"
 }
 
-# Fetch all tags from Docker Hub or GHCR.
+# Fetch all tags from Docker Hub or GHCR. Fails if any request fails: the
+# caller runs it inside `if`, where errexit is off, and a partial list would
+# be taken as complete.
 fetch_tags() {
   local image=$1
   if [[ $image == ghcr.io/* ]]; then
     local repo=${image#ghcr.io/}
     local token
     token=$(curl -fsSL "https://ghcr.io/token?scope=repository:${repo}:pull" \
-      | python3 -c "import sys,json;print(json.load(sys.stdin)['token'])")
-    # n=10000: OCI tags/list defaults to ~100 in lex order which misses
-    # high-numbered versions (e.g., v1.51.0 sorts after v1.9.0)
-    curl -fsSL -H "Authorization: Bearer $token" \
-      "https://ghcr.io/v2/${repo}/tags/list?n=10000" \
-      | python3 -c "import sys,json;[print(t) for t in json.load(sys.stdin).get('tags',[])]"
+      | python3 -c "import sys,json;print(json.load(sys.stdin)['token'])") || return 1
+    # GHCR returns at most 1000 tags per page whatever n asks for, in
+    # lexicographic order, so the newest versions are on the last page.
+    local path="/v2/${repo}/tags/list?n=10000"
+    local headers
+    headers=$(mktemp)
+    while [ -n "$path" ]; do
+      curl -fsSL -D "$headers" -H "Authorization: Bearer $token" \
+        "https://ghcr.io${path}" \
+        | python3 -c "import sys,json;[print(t) for t in json.load(sys.stdin).get('tags',[])]" \
+        || { rm -f "$headers"; return 1; }
+      path=$(tr -d '\r' < "$headers" | grep -i '^link:' \
+        | sed -nE 's/.*<([^>]*)>; *rel="next".*/\1/p')
+    done
+    rm -f "$headers"
   else
     local path=$image
     [[ $image != */* ]] && path="library/$image"
+    # One page suffices: last_updated ordering puts the newest tags first, and
+    # anonymous requests are refused past 1000 results.
     curl -fsSL \
       "https://hub.docker.com/v2/repositories/${path}/tags?page_size=100&ordering=last_updated" \
       | python3 -c "import sys,json;[print(r['name']) for r in json.load(sys.stdin).get('results',[])]"
@@ -114,7 +127,9 @@ while IFS='|' read -r file service image current_tag; do
     continue
   fi
 
-  latest=$(echo "$matching" | sort -V | tail -1)
+  # The current tag takes part, so a tag list missing the newest versions
+  # yields no change rather than a downgrade.
+  latest=$(printf '%s\n%s\n' "$matching" "$current_tag" | sort -V | tail -1)
   echo "  Latest: $latest"
 
   if [ "$latest" != "$current_tag" ]; then

tests/check-image-updates.test.sh

@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+# Tests for scripts/check-image-updates.sh against a fake registry.
+set -euo pipefail
+
+# shellcheck source=tests/lib/step.sh
+source "$(dirname "$0")/lib/step.sh"
+SCRIPT="$ROOT/scripts/check-image-updates.sh"
+
+TAB=$'\t'
+GHCR_TOKEN="https://ghcr.io/token?scope=repository:acme/app:pull${TAB}0${TAB}{\"token\":\"t\"}${TAB}"
+GHCR_PAGE1="https://ghcr.io/v2/acme/app/tags/list?n=10000"
+GHCR_PAGE2="https://ghcr.io/v2/acme/app/tags/list?last=2023.2.0&n=10000"
+GHCR_NEXT="</v2/acme/app/tags/list?last=2023.2.0&n=10000>; rel=\"next\""
+HUB_PAGE1="https://hub.docker.com/v2/repositories/acme/tool/tags?page_size=100&ordering=last_updated"
+
+# check_image <image:tag> <routes...> — runs the script on a compose file
+# holding that image; leaves the resulting image line in IMAGE_LINE.
+check_image() {
+  in_temp_dir
+  printf 'services:\n  app:\n    image: %s\n' "$1" > docker-compose.yml
+  shift
+  CURL_ROUTES=$(mktemp)
+  printf '%s\n' "$@" > "$CURL_ROUTES"
+  export CURL_ROUTES
+  COMPOSE_PATTERN=docker-compose.yml run_step "$SCRIPT" || {
+    cat "$STDOUT_FILE"
+    FAILURES=$((FAILURES + 1))
+  }
+  IMAGE_LINE=$(grep 'image:' docker-compose.yml | sed 's/ *image: //')
+}
+
+test_ghcr_follows_link_to_second_page() {
+  check_image ghcr.io/acme/app:2023.1.0 "$GHCR_TOKEN" \
+    "$GHCR_PAGE1${TAB}0${TAB}{\"tags\":[\"2023.1.0\",\"2023.2.0\"]}${TAB}$GHCR_NEXT" \
+    "$GHCR_PAGE2${TAB}0${TAB}{\"tags\":[\"2026.1.0\"]}${TAB}"
+  check "tag from the second GHCR page wins" ghcr.io/acme/app:2026.1.0 "$IMAGE_LINE"
+}
+
+test_ghcr_single_page() {
+  check_image ghcr.io/acme/app:2023.1.0 "$GHCR_TOKEN" \
+    "$GHCR_PAGE1${TAB}0${TAB}{\"tags\":[\"2023.1.0\",\"2023.2.0\"]}${TAB}"
+  check "single GHCR page without Link" ghcr.io/acme/app:2023.2.0 "$IMAGE_LINE"
+}
+
+test_ghcr_failed_page_skips_image() {
+  check_image ghcr.io/acme/app:2023.1.0 "$GHCR_TOKEN" \
+    "$GHCR_PAGE1${TAB}0${TAB}{\"tags\":[\"2023.1.0\",\"2023.2.0\"]}${TAB}$GHCR_NEXT" \
+    "$GHCR_PAGE2${TAB}22${TAB}${TAB}"
+  check "failed second page leaves the image alone" ghcr.io/acme/app:2023.1.0 "$IMAGE_LINE"
+  check "failed second page is reported" yes \
+    "$(grep -q 'Failed to fetch tags' "$STDOUT_FILE" && echo yes || echo no)"
+}
+
+test_ghcr_failed_token_is_reported() {
+  check_image ghcr.io/acme/app:2023.1.0
+  check "failed token request is reported" yes \
+    "$(grep -q 'Failed to fetch tags' "$STDOUT_FILE" && echo yes || echo no)"
+}
+
+test_never_proposes_an_older_tag() {
+  check_image ghcr.io/acme/app:2026.1.0 "$GHCR_TOKEN" \
+    "$GHCR_PAGE1${TAB}0${TAB}{\"tags\":[\"2023.1.0\",\"2023.2.0\"]}${TAB}"
+  check "older tags only leave the image alone" ghcr.io/acme/app:2026.1.0 "$IMAGE_LINE"
+}
+
+test_docker_hub_reads_newest_page_only() {
+  check_image acme/tool:1.0.0 \
+    "$HUB_PAGE1${TAB}0${TAB}{\"results\":[{\"name\":\"1.2.0\"},{\"name\":\"1.0.0\"}],\"next\":\"$HUB_PAGE1&page=2\"}${TAB}"
+  check "Docker Hub newest tag from the first page" acme/tool:1.2.0 "$IMAGE_LINE"
+}
+
+FAKEBIN=$(mktemp -d)
+ln -s "$ROOT/tests/lib/fake-curl" "$FAKEBIN/curl"
+PATH="$FAKEBIN:$PATH"
+for t in $(declare -F | awk '{print $3}' | grep '^test_'); do
+  "$t"
+done
+
+[ "$FAILURES" -eq 0 ]

tests/lib/fake-curl

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+# Fake curl for tests. Answers URLs listed in $CURL_ROUTES, one per line as
+# <url> TAB <exit code> TAB <body> TAB <Link header value or empty>.
+# Writes the response headers to the -D file; an unlisted URL fails like a 404.
+headers=""
+url=""
+while [ $# -gt 0 ]; do
+  case $1 in
+    -D) headers=$2; shift 2 ;;
+    -H) shift 2 ;;
+    -*) shift ;;
+    *) url=$1; shift ;;
+  esac
+done
+
+while IFS=$'\t' read -r route code body link; do
+  [ "$route" = "$url" ] || continue
+  if [ -n "$headers" ]; then
+    {
+      printf 'HTTP/2 200\r\n'
+      [ -n "$link" ] && printf 'link: %s\r\n' "$link"
+      printf '\r\n'
+    } > "$headers"
+  fi
+  [ "$code" = 0 ] || exit "$code"
+  printf '%s\n' "$body"
+  exit 0
+done < "$CURL_ROUTES"
+exit 22