From 20698fe89570ddf2a6290f5382e0e371d2cb7b6e Mon Sep 17 00:00:00 2001 From: Dan Rosser Date: Fri, 16 Oct 2026 02:06:30 +1300 Subject: [PATCH 1/2] Reject failed library downloads before clearing installed libraries --- .../workflows/validate-library-downloads.yml | 29 +++++++ scripts/dev/download_libs.sh | 5 +- scripts/dev/downloader.sh | 21 +++-- scripts/dev/sha_verify.sh | 31 ++++++-- scripts/dev/tests/test-download-validation.sh | 79 +++++++++++++++++++ 5 files changed, 150 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/validate-library-downloads.yml create mode 100644 scripts/dev/tests/test-download-validation.sh diff --git a/.github/workflows/validate-library-downloads.yml b/.github/workflows/validate-library-downloads.yml new file mode 100644 index 00000000000..f3510d9a4dd --- /dev/null +++ b/.github/workflows/validate-library-downloads.yml @@ -0,0 +1,29 @@ +name: Validate library downloads +on: + pull_request: + paths: + - scripts/dev/download_libs.sh + - scripts/dev/downloader.sh + - scripts/dev/sha_verify.sh + - scripts/dev/tests/test-download-validation.sh + - .github/workflows/validate-library-downloads.yml + push: + branches: [master] + paths: + - scripts/dev/download_libs.sh + - scripts/dev/downloader.sh + - scripts/dev/sha_verify.sh + - scripts/dev/tests/test-download-validation.sh + - .github/workflows/validate-library-downloads.yml +permissions: + contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6.0.2 + - run: bash -n scripts/dev/download_libs.sh scripts/dev/downloader.sh scripts/dev/sha_verify.sh + - run: bash scripts/dev/tests/test-download-validation.sh diff --git a/scripts/dev/download_libs.sh b/scripts/dev/download_libs.sh index e1c70fb9e4f..fae90d45b51 100755 --- a/scripts/dev/download_libs.sh +++ b/scripts/dev/download_libs.sh @@ -459,6 +459,7 @@ if ! verifyPackageDigests "$PKGS"; then writeVerifyState "$(pwd)" "$REPO" "$PLATFORM" "$ARCH" exit 1 fi +verifyPackageArchives "$PKGS" writeVerifyState "$(pwd)" "$REPO" "$PLATFORM" "$ARCH" cd ../ # back to libs @@ -544,9 +545,9 @@ for PKG in $PKGS; do if [ "$PLATFORM" == "linux" ] && { [ "$ARCH" == "aarch64" ] || [ "$ARCH" == "armv7l" ] || [ "$ARCH" == "armv6l" ]; }; then echo "tar xjfv download/$PKG --strip-components=1" - tar xf download/$PKG --strip-components=1 > /dev/null 2>&1 + tar xf "download/$PKG" --strip-components=1 > /dev/null else - tar xf download/$PKG > /dev/null 2>&1 + tar xf "download/$PKG" > /dev/null fi # rm -r download/$PKG fi diff --git a/scripts/dev/downloader.sh b/scripts/dev/downloader.sh index 311e20aa096..b7b40ca34e8 100755 --- a/scripts/dev/downloader.sh +++ b/scripts/dev/downloader.sh @@ -401,16 +401,17 @@ downloader() { echo " [downloader] No URLS to download, continue..." fi else + local download_status=0 if [[ "${SILENT}" == 1 ]]; then if [[ $WGET2 == 1 ]] && [[ $WGET2_INSTALLED == 1 ]]; then echo "" - wget2 -N -nv --no-tcp-fastopen --progress=bar --tries=${RETRY_MAX} --max-redirect=${MAX_REDIRECTS} --retry-connrefused --timeout=1500 --waitretry=${RETRY_DELAY_S} ${EXTRA_ARGS} ${FINAL_EXTRA_ARGS} ${URLS_TO_DOWNLOAD} + wget2 -N -nv --no-tcp-fastopen --progress=bar --tries=${RETRY_MAX} --max-redirect=${MAX_REDIRECTS} --retry-connrefused --timeout=1500 --waitretry=${RETRY_DELAY_S} ${EXTRA_ARGS} ${FINAL_EXTRA_ARGS} ${URLS_TO_DOWNLOAD} || download_status=$? elif [[ $CURL == 1 ]] && [[ $CURL_INSTALLED == 1 ]]; then echo - curl -Z -L --silent --retry ${RETRY_MAX} --retry-delay ${RETRY_DELAY_S} --max-redirs ${MAX_REDIRECTS} --header "Connection: close" --progress-bar ${EXTRA_ARGS} ${FINAL_EXTRA_ARGS} ${URLS_TO_DOWNLOAD} + curl -Z -L --fail --fail-early --show-error --silent --retry ${RETRY_MAX} --retry-delay ${RETRY_DELAY_S} --max-redirs ${MAX_REDIRECTS} --header "Connection: close" --progress-bar ${EXTRA_ARGS} ${FINAL_EXTRA_ARGS} ${URLS_TO_DOWNLOAD} || download_status=$? elif [[ $WGET == 1 ]] && [[ $WGET_INSTALLED == 1 ]]; then echo - wget -nv -N --tries=${RETRY_MAX} --retry-connrefused --waitretry=${RETRY_DELAY_S} ${EXTRA_ARGS} ${FINAL_EXTRA_ARGS} ${URLS_TO_DOWNLOAD} + wget -nv -N --tries=${RETRY_MAX} --retry-connrefused --waitretry=${RETRY_DELAY_S} ${EXTRA_ARGS} ${FINAL_EXTRA_ARGS} ${URLS_TO_DOWNLOAD} || download_status=$? else echo $ERROR_MSG exit 1 @@ -421,25 +422,29 @@ downloader() { echo " [downloader] [wget2] urls:[$URLS_TO_DOWNLOAD] args:[$EXTRA_ARGS $FINAL_EXTRA_ARGS" fi echo "" - wget2 -N -c --no-tcp-fastopen --progress=bar --force-progress --tries=${RETRY_MAX} --max-redirect=${MAX_REDIRECTS} --retry-connrefused --waitretry=${RETRY_DELAY_S} --timeout=1500 ${EXTRA_ARGS} ${FINAL_EXTRA_ARGS} ${URLS_TO_DOWNLOAD} + wget2 -N -c --no-tcp-fastopen --progress=bar --force-progress --tries=${RETRY_MAX} --max-redirect=${MAX_REDIRECTS} --retry-connrefused --waitretry=${RETRY_DELAY_S} --timeout=1500 ${EXTRA_ARGS} ${FINAL_EXTRA_ARGS} ${URLS_TO_DOWNLOAD} || download_status=$? elif [[ $CURL == 1 ]] && [[ $CURL_INSTALLED == 1 ]]; then if [[ $VERBOSE == 1 ]]; then echo " [downloader] [cURL] urls:[$URLS_TO_DOWNLOAD] args:[$EXTRA_ARGS $FINAL_EXTRA_ARGS ${CONNECTION_EXTRA_ARGS[@]}]" fi echo - curl -Z -L --retry ${RETRY_MAX} --retry-delay ${RETRY_DELAY_S} --max-redirs ${MAX_REDIRECTS} --progress-bar --header "Connection: close" ${EXTRA_ARGS} ${FINAL_EXTRA_ARGS} ${URLS_TO_DOWNLOAD} + curl -Z -L --fail --fail-early --show-error --retry ${RETRY_MAX} --retry-delay ${RETRY_DELAY_S} --max-redirs ${MAX_REDIRECTS} --progress-bar --header "Connection: close" ${EXTRA_ARGS} ${FINAL_EXTRA_ARGS} ${URLS_TO_DOWNLOAD} || download_status=$? elif [[ $WGET == 1 ]] && [[ $WGET_INSTALLED == 1 ]]; then if [[ $VERBOSE == 1 ]]; then echo " [downloader] [wget] [$FILENAME] urls:[$URLS_TO_DOWNLOAD] args:[$EXTRA_ARGS $FINAL_EXTRA_ARGS]" fi echo - wget -nv --progress=bar -N --tries=${RETRY_MAX} --retry-connrefused --waitretry=${RETRY_DELAY_S} ${EXTRA_ARGS} ${FINAL_EXTRA_ARGS} ${URLS_TO_DOWNLOAD} + wget -nv --progress=bar -N --tries=${RETRY_MAX} --retry-connrefused --waitretry=${RETRY_DELAY_S} ${EXTRA_ARGS} ${FINAL_EXTRA_ARGS} ${URLS_TO_DOWNLOAD} || download_status=$? else echo $ERROR_MSG exit 1 fi fi + if [ "$download_status" -ne 0 ]; then + echo " [downloader] Download failed (exit $download_status); not installing libraries." >&2 + return "$download_status" + fi if [[ "$CLOSE_CONNECTION" == "1" ]]; then if [[ $CURL == 1 ]] && [[ $CURL_INSTALLED == 1 ]] || [[ $CURL == 1 && $WGET2 == 1 ]]; then if [[ ${#FORWARDED_URLS[@]} -eq 0 ]]; then @@ -452,10 +457,12 @@ downloader() { if [[ $VERBOSE == 1 ]]; then echo " [downloader] Closing the ports yarr url:[$FIRST_URL]" fi - curl -I -L --retry-connrefused --insecure --silent --head --max-time 1 --retry ${RETRY_MAX} ${CLOSE_EXTRA_ARGS} --no-keepalive --header "Connection: close" --retry-delay ${RETRY_DELAY_S} --max-redirs ${MAX_REDIRECTS} ${FIRST_URL} + curl -I -L --retry-connrefused --insecure --silent --head --max-time 1 --retry ${RETRY_MAX} ${CLOSE_EXTRA_ARGS} --no-keepalive --header "Connection: close" --retry-delay ${RETRY_DELAY_S} --max-redirs ${MAX_REDIRECTS} ${FIRST_URL} || true fi fi fi fi + return 0 + } diff --git a/scripts/dev/sha_verify.sh b/scripts/dev/sha_verify.sh index 6edbf6c6c39..21d016b2771 100644 --- a/scripts/dev/sha_verify.sh +++ b/scripts/dev/sha_verify.sh @@ -91,13 +91,10 @@ fetchGithubReleaseDigests(){ local out="$3" local url api : > "$out" - if [[ "$tag" == "latest" ]]; then - url="https://api.github.com/repos/${repo}/releases/latest" - else - url="https://api.github.com/repos/${repo}/releases/tags/${tag}" - fi + # Callers download /releases/download//..., including the literal latest tag. + url="https://api.github.com/repos/${repo}/releases/tags/${tag}" api=$(mktemp 2>/dev/null || mktemp -t ofdigest) - if ! curl -fsSL -H "Accept: application/vnd.github+json" \ + if ! curl -fsSL --retry 3 --retry-connrefused --connect-timeout 15 --max-time 60 -H "Accept: application/vnd.github+json" \ -H "User-Agent: openFrameworks-sha-verify" \ "$url" -o "$api" 2>/dev/null; then rm -f "$api" @@ -307,3 +304,25 @@ verifyPackagesAgainstMap(){ fi return 0 } + +# Validate all archives before download_libs removes any installed output. +# This also protects deployments when remote SHA metadata is unavailable. +verifyPackageArchives(){ + local packages="$1" pkg + for pkg in $packages; do + if [[ ! -s "$pkg" ]]; then + echo "ERROR: Missing or empty library archive: $pkg" >&2 + return 1 + fi + case "$pkg" in + *.zip) + if unzip -tq "$pkg" >/dev/null; then continue; fi + ;; + *) + if tar -tf "$pkg" >/dev/null; then continue; fi + ;; + esac + echo "ERROR: Invalid library archive: $pkg. Remove the cached file and retry." >&2 + return 1 + done +} diff --git a/scripts/dev/tests/test-download-validation.sh b/scripts/dev/tests/test-download-validation.sh new file mode 100644 index 00000000000..e22db38cb24 --- /dev/null +++ b/scripts/dev/tests/test-download-validation.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT=$(cd "$(dirname "$0")/../../.." && pwd) +TEMP=$(mktemp -d) +trap 'rm -rf "$TEMP"' EXIT +mkdir -p "$TEMP/data" "$TEMP/bin" +printf contents > "$TEMP/data/object.a" +tar -cjf "$TEMP/good.tar.bz2" -C "$TEMP/data" object.a +(cd "$TEMP/data" && zip -q "$TEMP/good.zip" object.a) +printf 'download failed' > "$TEMP/bad.tar.bz2" +head -c 20 "$TEMP/good.tar.bz2" > "$TEMP/truncated.tar.bz2" +: > "$TEMP/empty.tar.bz2" +source "$ROOT/scripts/dev/sha_verify.sh" +verifyPackageArchives "$TEMP/good.tar.bz2 $TEMP/good.zip" +for archive in bad.tar.bz2 truncated.tar.bz2 empty.tar.bz2 absent.tar.bz2; do + if verifyPackageArchives "$TEMP/$archive" > "$TEMP/validation.log" 2>&1; then + echo "Accepted invalid archive: $archive" >&2; exit 1 + fi +done +# The real downloader must preserve failures instead of returning the status +# of an optional connection-cleanup request. +cat > "$TEMP/bin/curl" <<'CURL' +#!/usr/bin/env bash +case "$1" in -V|--version) echo 'curl 8.0.1'; exit 0 ;; esac +case " $* " in + *' -Z '*) + for option in --fail --fail-early --show-error; do + case " $* " in *" $option "*) : ;; *) exit 99 ;; esac + done + exit 22 + ;; + *' --write-out '*) printf 'https://example.invalid/fixture.tar.bz2'; exit 0 ;; + *) exit 22 ;; +esac +CURL +cat > "$TEMP/bin/wget" <<'WGET' +#!/usr/bin/env bash +exit 4 +WGET +chmod +x "$TEMP/bin/curl" "$TEMP/bin/wget" +export PATH="$TEMP/bin:$PATH" +source "$ROOT/scripts/dev/downloader.sh" +cd "$TEMP" +for mode in normal silent; do + result=0 + if [ "$mode" = silent ]; then + downloader --curl --silent --no-compression https://example.invalid/missing.tar.bz2 > "$TEMP/curl.log" 2>&1 || result=$? + else + downloader --curl --no-compression https://example.invalid/missing.tar.bz2 > "$TEMP/curl.log" 2>&1 || result=$? + fi + [ "$result" -eq 22 ] || { cat "$TEMP/curl.log"; echo "Masked $mode HTTP failure: $result" >&2; exit 1; } +done +result=0 +downloader --wget --no-compression https://example.invalid/missing.tar.bz2 > "$TEMP/wget.log" 2>&1 || result=$? +[ "$result" -eq 4 ] || { cat "$TEMP/wget.log"; echo "Masked wget failure: $result" >&2; exit 1; } +# Exercise the actual installer with unavailable remote digests and an invalid +# third bundle. It must reject the download before deleting installed libraries. +mkdir -p "$TEMP/of/scripts/dev" "$TEMP/of/libs/zlib/lib/osx" +cp "$ROOT/scripts/dev/download_libs.sh" "$ROOT/scripts/dev/sha_verify.sh" "$TEMP/of/scripts/dev/" +printf installed > "$TEMP/of/libs/zlib/lib/osx/sentinel" +cat > "$TEMP/of/scripts/dev/downloader.sh" <<'DOWNLOAD' +downloader() { + local url + for url in "$@"; do + case "$url" in + *osx_3.tar.bz2) printf 'failed' > "${url##*/}" ;; + https://*) cp "$GOOD_ARCHIVE" "${url##*/}" ;; + esac + done +} +DOWNLOAD +export GOOD_ARCHIVE="$TEMP/good.tar.bz2" +if bash "$TEMP/of/scripts/dev/download_libs.sh" -p osx -b > "$TEMP/install.log" 2>&1; then + cat "$TEMP/install.log" + echo 'Installer accepted invalid third bundle' >&2; exit 1 +fi +grep -q 'Invalid library archive: openFrameworksLibs_latest_osx_3.tar.bz2' "$TEMP/install.log" +test -f "$TEMP/of/libs/zlib/lib/osx/sentinel" +echo 'Download validation passed: HTTP/wget failures propagate, corrupt archives rejected, installed libraries preserved' From 41d0713d8da12b2c9b8a560df0a97e7123800204 Mon Sep 17 00:00:00 2001 From: Dan Rosser Date: Sun, 25 Oct 2026 14:55:59 +1100 Subject: [PATCH 2/2] Skip the archive check for packages whose SHA-256 matched A package that matched its published digest is the release asset byte for byte, so verifyPackageArchives no longer decompresses it a second time (tar -tf on a .tar.bz2 costs about as much as extracting it). Packages without a digest, or every package when digests are unavailable or --skip-sha is given, are still opened before anything is removed. verifyPackagesAgainstMap records the matched names in SHA_VERIFIED_PKGS. verifyPackageArchives is indented with tabs, like the rest of the file. Tested: test-download-validation.sh under Bash 3.2 and 5.3; macOS download_libs: three verified bundles extract with no tar -tf, and with --skip-sha all three are checked first. --- scripts/dev/sha_verify.sh | 46 +++++++++++-------- scripts/dev/tests/test-download-validation.sh | 13 ++++++ 2 files changed, 40 insertions(+), 19 deletions(-) diff --git a/scripts/dev/sha_verify.sh b/scripts/dev/sha_verify.sh index 21d016b2771..1b310e2c2cb 100644 --- a/scripts/dev/sha_verify.sh +++ b/scripts/dev/sha_verify.sh @@ -234,6 +234,8 @@ verifyPackagesAgainstMap(){ SHA_SKIPPED=0 SHA_FAILED=0 SHA_STATUS="unchecked" + # Space-padded names of packages whose SHA-256 matched (see verifyPackageArchives) + SHA_VERIFIED_PKGS=" " if [[ "${VERIFY_SHA}" == "0" ]]; then SHA_STATUS="skipped" @@ -270,6 +272,7 @@ verifyPackagesAgainstMap(){ any_expected=1 if verifyFileSha256 "$pkg" "$expected"; then SHA_VERIFIED=$((SHA_VERIFIED+1)) + SHA_VERIFIED_PKGS="${SHA_VERIFIED_PKGS}${pkg} " actual=$(cat "${pkg}.sha256" 2>/dev/null || true) echo " ✓ $pkg" echo " sha256:${actual}" @@ -305,24 +308,29 @@ verifyPackagesAgainstMap(){ return 0 } -# Validate all archives before download_libs removes any installed output. -# This also protects deployments when remote SHA metadata is unavailable. +# Validate archives before download_libs removes any installed output, so a bad +# download is caught even when remote SHA metadata is unavailable. A package +# whose SHA-256 matched the published digest is the release asset byte for +# byte, so it isn't decompressed a second time; only the others are opened. verifyPackageArchives(){ - local packages="$1" pkg - for pkg in $packages; do - if [[ ! -s "$pkg" ]]; then - echo "ERROR: Missing or empty library archive: $pkg" >&2 - return 1 - fi - case "$pkg" in - *.zip) - if unzip -tq "$pkg" >/dev/null; then continue; fi - ;; - *) - if tar -tf "$pkg" >/dev/null; then continue; fi - ;; - esac - echo "ERROR: Invalid library archive: $pkg. Remove the cached file and retry." >&2 - return 1 - done + local packages="$1" pkg + for pkg in $packages; do + if [[ ! -s "$pkg" ]]; then + echo "ERROR: Missing or empty library archive: $pkg" >&2 + return 1 + fi + if [[ "${SHA_VERIFIED_PKGS:-}" == *" $pkg "* ]]; then + continue + fi + case "$pkg" in + *.zip) + if unzip -tq "$pkg" >/dev/null; then continue; fi + ;; + *) + if tar -tf "$pkg" >/dev/null; then continue; fi + ;; + esac + echo "ERROR: Invalid library archive: $pkg. Remove the cached file and retry." >&2 + return 1 + done } diff --git a/scripts/dev/tests/test-download-validation.sh b/scripts/dev/tests/test-download-validation.sh index e22db38cb24..0c3148b7510 100644 --- a/scripts/dev/tests/test-download-validation.sh +++ b/scripts/dev/tests/test-download-validation.sh @@ -17,6 +17,19 @@ for archive in bad.tar.bz2 truncated.tar.bz2 empty.tar.bz2 absent.tar.bz2; do echo "Accepted invalid archive: $archive" >&2; exit 1 fi done +# A package whose SHA-256 matched is skipped (not decompressed again); one +# without a matching digest is still opened. +SHA_VERIFIED_PKGS=" $TEMP/bad.tar.bz2 " +verifyPackageArchives "$TEMP/bad.tar.bz2" +SHA_VERIFIED_PKGS=" $TEMP/bad.tar.bz2.old " +if verifyPackageArchives "$TEMP/bad.tar.bz2" > "$TEMP/validation.log" 2>&1; then + echo "Skipped an archive whose digest didn't match by name" >&2; exit 1 +fi +SHA_VERIFIED_PKGS=" " +if verifyPackageArchives "$TEMP/bad.tar.bz2" > "$TEMP/validation.log" 2>&1; then + echo "Accepted invalid archive without a verified digest" >&2; exit 1 +fi +unset SHA_VERIFIED_PKGS # The real downloader must preserve failures instead of returning the status # of an optional connection-cleanup request. cat > "$TEMP/bin/curl" <<'CURL'