Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 29 additions & 0 deletions .github/workflows/validate-library-downloads.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
name: Validate library downloads
on:
pull_request:
paths:
- scripts/dev/download_libs.sh
- scripts/dev/downloader.sh
- scripts/dev/sha_verify.sh
- scripts/dev/tests/test-download-validation.sh
- .github/workflows/validate-library-downloads.yml
push:
branches: [master]
paths:
- scripts/dev/download_libs.sh
- scripts/dev/downloader.sh
- scripts/dev/sha_verify.sh
- scripts/dev/tests/test-download-validation.sh
- .github/workflows/validate-library-downloads.yml
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
validate:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6.0.2
- run: bash -n scripts/dev/download_libs.sh scripts/dev/downloader.sh scripts/dev/sha_verify.sh
- run: bash scripts/dev/tests/test-download-validation.sh
5 changes: 3 additions & 2 deletions scripts/dev/download_libs.sh
Original file line number Diff line number Diff line change
Expand Up @@ -459,6 +459,7 @@ if ! verifyPackageDigests "$PKGS"; then
writeVerifyState "$(pwd)" "$REPO" "$PLATFORM" "$ARCH"
exit 1
fi
verifyPackageArchives "$PKGS"
writeVerifyState "$(pwd)" "$REPO" "$PLATFORM" "$ARCH"

cd ../ # back to libs
Expand Down Expand Up @@ -544,9 +545,9 @@ for PKG in $PKGS; do

if [ "$PLATFORM" == "linux" ] && { [ "$ARCH" == "aarch64" ] || [ "$ARCH" == "armv7l" ] || [ "$ARCH" == "armv6l" ]; }; then
echo "tar xjfv download/$PKG --strip-components=1"
tar xf download/$PKG --strip-components=1 > /dev/null 2>&1
tar xf "download/$PKG" --strip-components=1 > /dev/null
else
tar xf download/$PKG > /dev/null 2>&1
tar xf "download/$PKG" > /dev/null
fi
# rm -r download/$PKG
fi
Expand Down
21 changes: 14 additions & 7 deletions scripts/dev/downloader.sh
Original file line number Diff line number Diff line change
Expand Up @@ -401,16 +401,17 @@ downloader() {
echo " [downloader] No URLS to download, continue..."
fi
else
local download_status=0
if [[ "${SILENT}" == 1 ]]; then
if [[ $WGET2 == 1 ]] && [[ $WGET2_INSTALLED == 1 ]]; then
echo ""
wget2 -N -nv --no-tcp-fastopen --progress=bar --tries=${RETRY_MAX} --max-redirect=${MAX_REDIRECTS} --retry-connrefused --timeout=1500 --waitretry=${RETRY_DELAY_S} ${EXTRA_ARGS} ${FINAL_EXTRA_ARGS} ${URLS_TO_DOWNLOAD}
wget2 -N -nv --no-tcp-fastopen --progress=bar --tries=${RETRY_MAX} --max-redirect=${MAX_REDIRECTS} --retry-connrefused --timeout=1500 --waitretry=${RETRY_DELAY_S} ${EXTRA_ARGS} ${FINAL_EXTRA_ARGS} ${URLS_TO_DOWNLOAD} || download_status=$?
elif [[ $CURL == 1 ]] && [[ $CURL_INSTALLED == 1 ]]; then
echo
curl -Z -L --silent --retry ${RETRY_MAX} --retry-delay ${RETRY_DELAY_S} --max-redirs ${MAX_REDIRECTS} --header "Connection: close" --progress-bar ${EXTRA_ARGS} ${FINAL_EXTRA_ARGS} ${URLS_TO_DOWNLOAD}
curl -Z -L --fail --fail-early --show-error --silent --retry ${RETRY_MAX} --retry-delay ${RETRY_DELAY_S} --max-redirs ${MAX_REDIRECTS} --header "Connection: close" --progress-bar ${EXTRA_ARGS} ${FINAL_EXTRA_ARGS} ${URLS_TO_DOWNLOAD} || download_status=$?
elif [[ $WGET == 1 ]] && [[ $WGET_INSTALLED == 1 ]]; then
echo
wget -nv -N --tries=${RETRY_MAX} --retry-connrefused --waitretry=${RETRY_DELAY_S} ${EXTRA_ARGS} ${FINAL_EXTRA_ARGS} ${URLS_TO_DOWNLOAD}
wget -nv -N --tries=${RETRY_MAX} --retry-connrefused --waitretry=${RETRY_DELAY_S} ${EXTRA_ARGS} ${FINAL_EXTRA_ARGS} ${URLS_TO_DOWNLOAD} || download_status=$?
else
echo $ERROR_MSG
exit 1
Expand All @@ -421,25 +422,29 @@ downloader() {
echo " [downloader] [wget2] urls:[$URLS_TO_DOWNLOAD] args:[$EXTRA_ARGS $FINAL_EXTRA_ARGS"
fi
echo ""
wget2 -N -c --no-tcp-fastopen --progress=bar --force-progress --tries=${RETRY_MAX} --max-redirect=${MAX_REDIRECTS} --retry-connrefused --waitretry=${RETRY_DELAY_S} --timeout=1500 ${EXTRA_ARGS} ${FINAL_EXTRA_ARGS} ${URLS_TO_DOWNLOAD}
wget2 -N -c --no-tcp-fastopen --progress=bar --force-progress --tries=${RETRY_MAX} --max-redirect=${MAX_REDIRECTS} --retry-connrefused --waitretry=${RETRY_DELAY_S} --timeout=1500 ${EXTRA_ARGS} ${FINAL_EXTRA_ARGS} ${URLS_TO_DOWNLOAD} || download_status=$?
elif [[ $CURL == 1 ]] && [[ $CURL_INSTALLED == 1 ]]; then
if [[ $VERBOSE == 1 ]]; then
echo " [downloader] [cURL] urls:[$URLS_TO_DOWNLOAD] args:[$EXTRA_ARGS $FINAL_EXTRA_ARGS ${CONNECTION_EXTRA_ARGS[@]}]"
fi
echo
curl -Z -L --retry ${RETRY_MAX} --retry-delay ${RETRY_DELAY_S} --max-redirs ${MAX_REDIRECTS} --progress-bar --header "Connection: close" ${EXTRA_ARGS} ${FINAL_EXTRA_ARGS} ${URLS_TO_DOWNLOAD}
curl -Z -L --fail --fail-early --show-error --retry ${RETRY_MAX} --retry-delay ${RETRY_DELAY_S} --max-redirs ${MAX_REDIRECTS} --progress-bar --header "Connection: close" ${EXTRA_ARGS} ${FINAL_EXTRA_ARGS} ${URLS_TO_DOWNLOAD} || download_status=$?

elif [[ $WGET == 1 ]] && [[ $WGET_INSTALLED == 1 ]]; then
if [[ $VERBOSE == 1 ]]; then
echo " [downloader] [wget] [$FILENAME] urls:[$URLS_TO_DOWNLOAD] args:[$EXTRA_ARGS $FINAL_EXTRA_ARGS]"
fi
echo
wget -nv --progress=bar -N --tries=${RETRY_MAX} --retry-connrefused --waitretry=${RETRY_DELAY_S} ${EXTRA_ARGS} ${FINAL_EXTRA_ARGS} ${URLS_TO_DOWNLOAD}
wget -nv --progress=bar -N --tries=${RETRY_MAX} --retry-connrefused --waitretry=${RETRY_DELAY_S} ${EXTRA_ARGS} ${FINAL_EXTRA_ARGS} ${URLS_TO_DOWNLOAD} || download_status=$?
else
echo $ERROR_MSG
exit 1
fi
fi
if [ "$download_status" -ne 0 ]; then
echo " [downloader] Download failed (exit $download_status); not installing libraries." >&2
return "$download_status"
fi
if [[ "$CLOSE_CONNECTION" == "1" ]]; then
if [[ $CURL == 1 ]] && [[ $CURL_INSTALLED == 1 ]] || [[ $CURL == 1 && $WGET2 == 1 ]]; then
if [[ ${#FORWARDED_URLS[@]} -eq 0 ]]; then
Expand All @@ -452,10 +457,12 @@ downloader() {
if [[ $VERBOSE == 1 ]]; then
echo " [downloader] Closing the ports yarr url:[$FIRST_URL]"
fi
curl -I -L --retry-connrefused --insecure --silent --head --max-time 1 --retry ${RETRY_MAX} ${CLOSE_EXTRA_ARGS} --no-keepalive --header "Connection: close" --retry-delay ${RETRY_DELAY_S} --max-redirs ${MAX_REDIRECTS} ${FIRST_URL}
curl -I -L --retry-connrefused --insecure --silent --head --max-time 1 --retry ${RETRY_MAX} ${CLOSE_EXTRA_ARGS} --no-keepalive --header "Connection: close" --retry-delay ${RETRY_DELAY_S} --max-redirs ${MAX_REDIRECTS} ${FIRST_URL} || true
fi
fi
fi
fi

return 0

}
39 changes: 33 additions & 6 deletions scripts/dev/sha_verify.sh
Original file line number Diff line number Diff line change
Expand Up @@ -91,13 +91,10 @@ fetchGithubReleaseDigests(){
local out="$3"
local url api
: > "$out"
if [[ "$tag" == "latest" ]]; then
url="https://api.github.com/repos/${repo}/releases/latest"
else
url="https://api.github.com/repos/${repo}/releases/tags/${tag}"
fi
# Callers download /releases/download/<tag>/..., including the literal latest tag.
url="https://api.github.com/repos/${repo}/releases/tags/${tag}"
api=$(mktemp 2>/dev/null || mktemp -t ofdigest)
if ! curl -fsSL -H "Accept: application/vnd.github+json" \
if ! curl -fsSL --retry 3 --retry-connrefused --connect-timeout 15 --max-time 60 -H "Accept: application/vnd.github+json" \
-H "User-Agent: openFrameworks-sha-verify" \
"$url" -o "$api" 2>/dev/null; then
rm -f "$api"
Expand Down Expand Up @@ -237,6 +234,8 @@ verifyPackagesAgainstMap(){
SHA_SKIPPED=0
SHA_FAILED=0
SHA_STATUS="unchecked"
# Space-padded names of packages whose SHA-256 matched (see verifyPackageArchives)
SHA_VERIFIED_PKGS=" "

if [[ "${VERIFY_SHA}" == "0" ]]; then
SHA_STATUS="skipped"
Expand Down Expand Up @@ -273,6 +272,7 @@ verifyPackagesAgainstMap(){
any_expected=1
if verifyFileSha256 "$pkg" "$expected"; then
SHA_VERIFIED=$((SHA_VERIFIED+1))
SHA_VERIFIED_PKGS="${SHA_VERIFIED_PKGS}${pkg} "
actual=$(cat "${pkg}.sha256" 2>/dev/null || true)
echo " ✓ $pkg"
echo " sha256:${actual}"
Expand Down Expand Up @@ -307,3 +307,30 @@ verifyPackagesAgainstMap(){
fi
return 0
}

# Validate archives before download_libs removes any installed output, so a bad
# download is caught even when remote SHA metadata is unavailable. A package
# whose SHA-256 matched the published digest is the release asset byte for
# byte, so it isn't decompressed a second time; only the others are opened.
verifyPackageArchives(){
local packages="$1" pkg
for pkg in $packages; do
if [[ ! -s "$pkg" ]]; then
echo "ERROR: Missing or empty library archive: $pkg" >&2
return 1
fi
if [[ "${SHA_VERIFIED_PKGS:-}" == *" $pkg "* ]]; then
continue
fi
case "$pkg" in
*.zip)
if unzip -tq "$pkg" >/dev/null; then continue; fi
;;
*)
if tar -tf "$pkg" >/dev/null; then continue; fi
;;
esac
echo "ERROR: Invalid library archive: $pkg. Remove the cached file and retry." >&2
return 1
done
}
92 changes: 92 additions & 0 deletions scripts/dev/tests/test-download-validation.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT=$(cd "$(dirname "$0")/../../.." && pwd)
TEMP=$(mktemp -d)
trap 'rm -rf "$TEMP"' EXIT
mkdir -p "$TEMP/data" "$TEMP/bin"
printf contents > "$TEMP/data/object.a"
tar -cjf "$TEMP/good.tar.bz2" -C "$TEMP/data" object.a
(cd "$TEMP/data" && zip -q "$TEMP/good.zip" object.a)
printf '<html>download failed</html>' > "$TEMP/bad.tar.bz2"
head -c 20 "$TEMP/good.tar.bz2" > "$TEMP/truncated.tar.bz2"
: > "$TEMP/empty.tar.bz2"
source "$ROOT/scripts/dev/sha_verify.sh"
verifyPackageArchives "$TEMP/good.tar.bz2 $TEMP/good.zip"
for archive in bad.tar.bz2 truncated.tar.bz2 empty.tar.bz2 absent.tar.bz2; do
if verifyPackageArchives "$TEMP/$archive" > "$TEMP/validation.log" 2>&1; then
echo "Accepted invalid archive: $archive" >&2; exit 1
fi
done
# A package whose SHA-256 matched is skipped (not decompressed again); one
# without a matching digest is still opened.
SHA_VERIFIED_PKGS=" $TEMP/bad.tar.bz2 "
verifyPackageArchives "$TEMP/bad.tar.bz2"
SHA_VERIFIED_PKGS=" $TEMP/bad.tar.bz2.old "
if verifyPackageArchives "$TEMP/bad.tar.bz2" > "$TEMP/validation.log" 2>&1; then
echo "Skipped an archive whose digest didn't match by name" >&2; exit 1
fi
SHA_VERIFIED_PKGS=" "
if verifyPackageArchives "$TEMP/bad.tar.bz2" > "$TEMP/validation.log" 2>&1; then
echo "Accepted invalid archive without a verified digest" >&2; exit 1
fi
unset SHA_VERIFIED_PKGS
# The real downloader must preserve failures instead of returning the status
# of an optional connection-cleanup request.
cat > "$TEMP/bin/curl" <<'CURL'
#!/usr/bin/env bash
case "$1" in -V|--version) echo 'curl 8.0.1'; exit 0 ;; esac
case " $* " in
*' -Z '*)
for option in --fail --fail-early --show-error; do
case " $* " in *" $option "*) : ;; *) exit 99 ;; esac
done
exit 22
;;
*' --write-out '*) printf 'https://example.invalid/fixture.tar.bz2'; exit 0 ;;
*) exit 22 ;;
esac
CURL
cat > "$TEMP/bin/wget" <<'WGET'
#!/usr/bin/env bash
exit 4
WGET
chmod +x "$TEMP/bin/curl" "$TEMP/bin/wget"
export PATH="$TEMP/bin:$PATH"
source "$ROOT/scripts/dev/downloader.sh"
cd "$TEMP"
for mode in normal silent; do
result=0
if [ "$mode" = silent ]; then
downloader --curl --silent --no-compression https://example.invalid/missing.tar.bz2 > "$TEMP/curl.log" 2>&1 || result=$?
else
downloader --curl --no-compression https://example.invalid/missing.tar.bz2 > "$TEMP/curl.log" 2>&1 || result=$?
fi
[ "$result" -eq 22 ] || { cat "$TEMP/curl.log"; echo "Masked $mode HTTP failure: $result" >&2; exit 1; }
done
result=0
downloader --wget --no-compression https://example.invalid/missing.tar.bz2 > "$TEMP/wget.log" 2>&1 || result=$?
[ "$result" -eq 4 ] || { cat "$TEMP/wget.log"; echo "Masked wget failure: $result" >&2; exit 1; }
# Exercise the actual installer with unavailable remote digests and an invalid
# third bundle. It must reject the download before deleting installed libraries.
mkdir -p "$TEMP/of/scripts/dev" "$TEMP/of/libs/zlib/lib/osx"
cp "$ROOT/scripts/dev/download_libs.sh" "$ROOT/scripts/dev/sha_verify.sh" "$TEMP/of/scripts/dev/"
printf installed > "$TEMP/of/libs/zlib/lib/osx/sentinel"
cat > "$TEMP/of/scripts/dev/downloader.sh" <<'DOWNLOAD'
downloader() {
local url
for url in "$@"; do
case "$url" in
*osx_3.tar.bz2) printf '<html>failed</html>' > "${url##*/}" ;;
https://*) cp "$GOOD_ARCHIVE" "${url##*/}" ;;
esac
done
}
DOWNLOAD
export GOOD_ARCHIVE="$TEMP/good.tar.bz2"
if bash "$TEMP/of/scripts/dev/download_libs.sh" -p osx -b > "$TEMP/install.log" 2>&1; then
cat "$TEMP/install.log"
echo 'Installer accepted invalid third bundle' >&2; exit 1
fi
grep -q 'Invalid library archive: openFrameworksLibs_latest_osx_3.tar.bz2' "$TEMP/install.log"
test -f "$TEMP/of/libs/zlib/lib/osx/sentinel"
echo 'Download validation passed: HTTP/wget failures propagate, corrupt archives rejected, installed libraries preserved'
Loading