From 9073b84842d65d2fa26dd2f81da7ca54b6a67644 Mon Sep 17 00:00:00 2001 From: FosterStack Admin <317177128+fosterstack-admin@users.noreply.github.com> Date: Thu, 22 Oct 2026 15:04:44 -0400 Subject: [PATCH 1/5] Daily live check: the live site against main and every link on it, one step in the existing scheduled run; failures go to the one tracking issue --- .github/workflows/hygiene.yml | 33 +++++++- bin/check-file-allowlist.sh | 2 + bin/live-check.py | 149 ++++++++++++++++++++++++++++++++++ bin/proof-pages.txt | 1 + bin/test-live-check.sh | 107 ++++++++++++++++++++++++ 5 files changed, 289 insertions(+), 3 deletions(-) create mode 100755 bin/live-check.py create mode 100644 bin/test-live-check.sh diff --git a/.github/workflows/hygiene.yml b/.github/workflows/hygiene.yml index 0a25e08..45bc3ef 100644 --- a/.github/workflows/hygiene.yml +++ b/.github/workflows/hygiene.yml @@ -42,6 +42,7 @@ on: - howto-pages-11 - howto-pages-12 - proof-watch + - live-check - proof-selftest default: miss-cost @@ -71,6 +72,9 @@ jobs: - name: self-test the site-rot check run: bash bin/test-check-site-rot.sh + - name: self-test the live check (a local server stands in for the live site and for GitHub) + run: bash bin/test-live-check.sh + # proof-watch: on the daily schedule it decides whether today's run proves every bench job below (a new release tag of # fosterstack/cache, or Monday for the weekly run). It reads only the releases of fosterstack/cache and only tags matching # ^v[0-9]+\.[0-9]+\.[0-9]+$ (any other tag is reported, never run); see bin/proof-watch.sh. On a manual run with @@ -98,6 +102,29 @@ jobs: GH_TOKEN: ${{ github.token }} run: bash bin/proof-watch.sh + # live-check: once a day on the schedule (and by hand with which=live-check), the LIVE site is checked, not the repository: the page + # list comes from the live sitemap.xml; every listed page must answer 200 and be byte for byte the file on main (so the live site and + # what the site-rot check reads cannot drift apart unseen); every internal link and github.com/fosterstack link on the live pages must + # answer 200. One fixed timeout and ONE retry per request. The site-rot check runs first on main with the live rules (no page hidden + # from search, none listed hidden). Read-only: no secret, no write permission, no action beyond the pinned checkout. A failure is + # reported by proof-report in the ONE tracking issue (job name live-check in bin/proof-pages.txt); see bin/live-check.py. + live-check: + if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.which == 'live-check') + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: the site-rot check on main, with the live rules + run: python3 bin/check-site-rot.py --live + + - name: the live site against main, and every link on it + run: python3 bin/live-check.py + # Timed runs for the "what does a cache miss cost" page. Manual dispatch only; never runs on push or # pull request. No secrets, no write permission, no third-party action beyond the pinned checkout above. # The script downloads the FosterStack Cache release and Gradle, and checks both against pinned sha256 values. @@ -799,8 +826,8 @@ jobs: # bin/proof-report.sh. After a fully GREEN scheduled run only, it leaves the one-line artifact "proven-release" (tag, run id) that # proof-watch reads the next day (so a red run is tried again the next day when the tag is new). Nothing is committed back and no page is edited by a workflow. proof-report: - needs: [proof-watch, bench-miss-cost, bench-server-down, bench-verify-release, bench-setup-pages, bench-howto-pages, bench-howto-pages-2, bench-howto-pages-3, bench-k8s-page, bench-ga-page, bench-howto-pages-4, bench-howto-pages-5, bench-howto-pages-6, bench-howto-pages-7, bench-howto-pages-8, bench-howto-pages-9, bench-howto-pages-10, bench-howto-pages-11, bench-howto-pages-12, bench-maclinux-linux-store, bench-maclinux-mac-restore, bench-maclinux-mac-store, bench-maclinux-linux-restore, bench-android, demo-github-actions, proof-selftest] - if: ${{ always() && ((github.event_name == 'schedule' && (needs.proof-watch.result == 'failure' || (needs.proof-watch.result == 'success' && (needs.proof-watch.outputs.run == 'true' || startsWith(needs.proof-watch.outputs.reason, 'the newest release tag') || startsWith(needs.proof-watch.outputs.reason, 'release lookup failed'))))) || (github.event_name == 'workflow_dispatch' && inputs.which == 'proof-selftest')) }} + needs: [proof-watch, live-check, bench-miss-cost, bench-server-down, bench-verify-release, bench-setup-pages, bench-howto-pages, bench-howto-pages-2, bench-howto-pages-3, bench-k8s-page, bench-ga-page, bench-howto-pages-4, bench-howto-pages-5, bench-howto-pages-6, bench-howto-pages-7, bench-howto-pages-8, bench-howto-pages-9, bench-howto-pages-10, bench-howto-pages-11, bench-howto-pages-12, bench-maclinux-linux-store, bench-maclinux-mac-restore, bench-maclinux-mac-store, bench-maclinux-linux-restore, bench-android, demo-github-actions, proof-selftest] + if: ${{ always() && ((github.event_name == 'schedule' && (needs.live-check.result == 'failure' || needs.live-check.result == 'success' || needs.proof-watch.result == 'failure' || (needs.proof-watch.result == 'success' && (needs.proof-watch.outputs.run == 'true' || startsWith(needs.proof-watch.outputs.reason, 'the newest release tag') || startsWith(needs.proof-watch.outputs.reason, 'release lookup failed'))))) || (github.event_name == 'workflow_dispatch' && inputs.which == 'proof-selftest')) }} runs-on: ubuntu-latest timeout-minutes: 15 concurrency: @@ -820,7 +847,7 @@ jobs: GH_TOKEN: ${{ github.token }} RUN_ID: ${{ github.run_id }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - TAG: ${{ needs.proof-watch.outputs.tag }} + TAG: ${{ needs.proof-watch.outputs.run == 'true' && needs.proof-watch.outputs.tag || '' }} # empty on a day nothing was proven, so a live-check-only report never writes the proven-release artifact WATCH_REASON: ${{ needs.proof-watch.outputs.reason }} WATCH_RUN: ${{ needs.proof-watch.outputs.run }} MODE: ${{ github.event_name == 'schedule' && 'schedule' || 'selftest' }} diff --git a/bin/check-file-allowlist.sh b/bin/check-file-allowlist.sh index 6a12e28..49ff931 100755 --- a/bin/check-file-allowlist.sh +++ b/bin/check-file-allowlist.sh @@ -64,6 +64,8 @@ ALLOW_PATTERNS=( '^bin/proof-watch\.sh$' '^bin/proof-report\.sh$' '^bin/proof-pages\.txt$' + '^bin/live-check\.py$' + '^bin/test-live-check\.sh$' '^\.github/workflows/[A-Za-z0-9_-][A-Za-z0-9._-]*\.ya?ml$' '^functions/(api/)?[A-Za-z0-9._-]+\.js$' '^[A-Za-z0-9._-]+/index\.html$' # pre-positioned pages, e.g. bcn-removed/index.html diff --git a/bin/live-check.py b/bin/live-check.py new file mode 100755 index 0000000..bf90438 --- /dev/null +++ b/bin/live-check.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Check the LIVE site once a day (launch week and after). Standard library only; reads the network and the checkout, writes nothing. + +What it does, in order (a problem is one line starting with "FAIL", which is what the tracking issue quotes): + 1. fetch /sitemap.xml from the live site and take the page list FROM IT (not from the repository); every entry must be + on the site's own host (https://fosterstack.com/), or the entry is refused and never fetched; + 2. fetch every listed page: it must answer 200 (a redirect is a failure) and be byte for byte the file in the checkout of main; + sitemap.xml, llms.txt and robots.txt the same (so "the live site" and "what main says" cannot drift apart unseen); + 3. from the live pages take every internal link (a path from the site root) and every github.com/fosterstack link; each must + answer 200 (the private-advisory form may redirect to a login). +Every request has one fixed timeout (LIVE_CHECK_TIMEOUT, default 20 seconds) and ONE retry after a short wait +(LIVE_CHECK_RETRY_WAIT, default 5 seconds), so a flaky answer does not raise an issue by itself. + +Usage: live-check.py [--root DIR] [--base URL] [--github URL] (defaults: the checkout, https://fosterstack.com, https://github.com) +""" +import argparse +import hashlib +import html +import os +import re +import sys +import time +import urllib.error +import urllib.request + +SITE = "https://fosterstack.com/" +TIMEOUT = float(os.environ.get("LIVE_CHECK_TIMEOUT", "20")) +RETRY_WAIT = float(os.environ.get("LIVE_CHECK_RETRY_WAIT", "5")) +MAX_BYTES = 8 * 1024 * 1024 + + +class _NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, *a, **k): + return None + + +OPENER = urllib.request.build_opener(_NoRedirect) + + +def fetch(url): + """(status, body) with one retry; status 0 means no answer. Never raises.""" + last = (0, b"") + for attempt in (1, 2): + try: + with OPENER.open(urllib.request.Request(url, headers={"User-Agent": "fosterstack-live-check"}), timeout=TIMEOUT) as r: + body = r.read(MAX_BYTES + 1) + last = (r.status, body[:MAX_BYTES]) + except urllib.error.HTTPError as e: + last = (e.code, b"") + except Exception: + last = (0, b"") + # a good answer, or a plain "no such thing", is final; a timeout, a refusal and a 5xx are tried once more + if last[0] in (200, 301, 302, 303, 307, 308, 400, 401, 403, 404, 410) : + break + if attempt == 1: + time.sleep(RETRY_WAIT) + return last + + +def clean(text): + return re.sub(r"[\x00-\x1f\x7f-\x9f]", " ", text)[:300] + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--root", default=".") + ap.add_argument("--base", default=SITE.rstrip("/")) + ap.add_argument("--github", default="https://github.com") + args = ap.parse_args() + base = args.base.rstrip("/") + problems = [] + + def bad(kind, msg): + problems.append("FAIL %s: %s" % (kind, clean(msg))) + + def live_url(site_url): + return base + "/" + site_url[len(SITE):] + + def repo_bytes(site_url): + rel = site_url[len(SITE):] + path = os.path.join(args.root, rel + ("index.html" if rel.endswith("/") or rel == "" else "")) + try: + with open(path, "rb") as f: + return f.read() + except OSError: + return None + + # 1. the page list, from the live sitemap + status, body = fetch(base + "/sitemap.xml") + if status != 200: + bad("sitemap", "sitemap.xml answered %s" % (status or "nothing")) + locs = [] + else: + locs = [html.unescape(x).strip() for x in re.findall(r"([^<]*)", body.decode("utf-8", "replace"))] + pages = [] + for u in locs: + if not u.startswith(SITE): + bad("sitemap", "the live sitemap lists %s, which is not on %s" % (u, SITE)) + else: + pages.append(u) + + # 2. every listed page, and the three files, against main + texts = {} + for u in dict.fromkeys(pages + [SITE + "llms.txt", SITE + "robots.txt", SITE + "sitemap.xml"]): + status, got = fetch(live_url(u)) + label = u[len(SITE):] or "/" + kind = "page" if u in pages else "drift" + if status != 200: + bad("page" if u in pages else "file", "%s answered %s" % (u, status or "nothing")) + continue + texts[u] = got + want = repo_bytes(u) + if want is None: + bad("drift", "%s is live but main has no such file" % u) + elif hashlib.sha256(want).digest() != hashlib.sha256(got).digest(): + bad("drift", "%s is live but differs from the file on main" % u) + + # 3. links on the live pages + internal, external = {}, {} + for u, got in texts.items(): + if not u.endswith("/"): + continue + for m in re.finditer(r'(?/dev/null; done; rm -rf "$tmp"; } +trap cleanup EXIT +fail=0 + +site="$tmp/site"; served="$tmp/served"; gh="$tmp/gh" +fresh() { rm -rf "$site" "$served"; mkdir "$site"; tar --exclude=.git --exclude=.claude -cf - . | tar -xf - -C "$site"; cp -R "$site" "$served"; } +# two throwaway servers: the served copy (live) and a fake GitHub +python3 - "$tmp" <<'PY' & +import http.server, os, socketserver, sys, threading +tmp = sys.argv[1] +class Live(http.server.SimpleHTTPRequestHandler): + def __init__(self, *a, **k): super().__init__(*a, directory=os.path.join(tmp, "served"), **k) + def log_message(self, *a): pass + def do_GET(self): + if self.path.startswith("/slow-"): + import time; time.sleep(60) + return super().do_GET() +class GH(http.server.BaseHTTPRequestHandler): + def log_message(self, *a): pass + def do_GET(self): + n = self.path.split("/")[-1] + if self.path.endswith("/security/advisories/new"): + self.send_response(302); self.send_header("Location", "/login"); self.end_headers(); return + if "/dead/" in self.path: + self.send_response(404); self.end_headers(); return + if "/flaky/" in self.path: + flag = os.path.join(tmp, "flaky-seen") + if not os.path.exists(flag): + open(flag, "w").close(); self.send_response(503); self.end_headers(); return + self.send_response(200); self.send_header("Content-Length", "2"); self.end_headers(); self.wfile.write(b"ok") +def serve(h, portfile): + s = socketserver.ThreadingTCPServer(("127.0.0.1", 0), h); s.daemon_threads = True + open(portfile, "w").write(str(s.server_address[1])); s.serve_forever() +threading.Thread(target=serve, args=(GH, os.path.join(tmp, "gh.port")), daemon=True).start() +serve(Live, os.path.join(tmp, "live.port")) +PY +pids+=($!) +for i in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20; do [ -s "$tmp/live.port" ] && [ -s "$tmp/gh.port" ] && break; sleep 0.3; done +live="http://127.0.0.1:$(cat "$tmp/live.port")"; ghurl="http://127.0.0.1:$(cat "$tmp/gh.port")" + +run() { LIVE_CHECK_TIMEOUT=${T:-3} LIVE_CHECK_RETRY_WAIT=0 python3 "$here/bin/live-check.py" --root "$site" --base "$live" --github "$ghurl" "$@" >"$tmp/out" 2>&1; } +expect_pass() { if run; then echo "ok $1"; else echo "FAIL $1 (expected a pass)"; sed 's/^/ /' "$tmp/out" | head -8; fail=1; fi; } +expect_fail() { if run; then echo "FAIL $1 (expected a failure, got a pass)"; fail=1; elif grep -q "$2" "$tmp/out"; then echo "ok $1"; else echo "FAIL $1 (failed, but without '$2')"; sed 's/^/ /' "$tmp/out" | head -8; fail=1; fi; } +pages=$(for d in [a-z]*/; do [ -f "$d/index.html" ] && echo "${d%/}"; done) +first=$(echo "$pages" | sed -n 1p) +second=$(echo "$pages" | sed -n 2p) + +fresh; expect_pass "a live copy equal to the repository passes" +fresh; rm -rf "$served/$first" +expect_fail "a page in the live sitemap that answers 404 is reported" "FAIL page: .*$first" +fresh; echo "" >> "$served/$second/index.html" +expect_fail "a live page that differs from the repository is reported" "FAIL drift: .*$second" +fresh; echo " " >> "$served/sitemap.xml" +expect_fail "a live sitemap that differs from the repository is reported" "FAIL drift: .*sitemap.xml" +fresh; rm "$served/llms.txt" +expect_fail "a missing live llms.txt is reported" "FAIL .*llms.txt" +# the sitemap list comes from the live site, not the repository: a page only the live sitemap names is fetched +fresh; python3 - "$served/sitemap.xml" <<'PY' +import sys +p = sys.argv[1]; s = open(p).read() +open(p, "w").write(s.replace("", "https://fosterstack.com/only-live-xyz/")) +PY +expect_fail "a page only the live sitemap lists is fetched and reported" "only-live-xyz" +# links on the live pages +fresh; python3 - "$served/$first/index.html" "$site/$first/index.html" <<'PY' +import sys +for p in sys.argv[1:]: + s = open(p).read(); open(p, "w").write(s.replace("", '

x

', 1)) +PY +expect_fail "a dead GitHub link on a live page is reported" "FAIL link: .*dead" +fresh; python3 - "$served/$first/index.html" "$site/$first/index.html" <<'PY' +import sys +for p in sys.argv[1:]: + s = open(p).read(); open(p, "w").write(s.replace("", '

x y

', 1)) +PY +rm -f "$tmp/flaky-seen" +expect_pass "the advisory form redirect is fine, and one flaky answer is retried once" +fresh; python3 - "$served/$first/index.html" "$site/$first/index.html" <<'PY' +import sys +for p in sys.argv[1:]: + s = open(p).read(); open(p, "w").write(s.replace("", '

x

', 1)) +PY +expect_fail "an internal link that answers 404 on the live site is reported" "FAIL link: .*no-such-live-page" +# a stuck server is cut off by the fixed timeout; the run says so and ends +fresh; python3 - "$served/sitemap.xml" "$site/sitemap.xml" <<'PY' +import sys +for p in sys.argv[1:]: + s = open(p).read(); open(p, "w").write(s.replace("", "https://fosterstack.com/slow-page/")) +PY +T=1 expect_fail "a page that never answers is cut off by the timeout" "FAIL page: .*slow-page" +# only the site's own host is fetched: a sitemap entry for another host is refused, not fetched +fresh; python3 - "$served/sitemap.xml" <<'PY' +import sys +p = sys.argv[1]; s = open(p).read() +open(p, "w").write(s.replace("", "https://example.org/x/")) +PY +expect_fail "a live sitemap entry for another host is refused" "FAIL sitemap: .*example.org" +[ "$fail" = 0 ] && echo "all live-check self-tests passed" || { echo "live-check self-test FAILED"; exit 1; } From e1cef95bdea5c539f010f0ebf6ff2e86f059379d Mon Sep 17 00:00:00 2001 From: FosterStack Admin <317177128+fosterstack-admin@users.noreply.github.com> Date: Thu, 22 Oct 2026 15:17:51 -0400 Subject: [PATCH 2/5] Live check: retry every unacceptable answer once, HTML-parsed links resolved against their page, sitemap paths kept inside the checkout, main checked out explicitly, manual runs reported; tests for each --- .github/workflows/hygiene.yml | 5 ++- bin/live-check.py | 80 +++++++++++++++++++++++++++-------- bin/test-live-check.sh | 70 +++++++++++++++++++++++++++--- 3 files changed, 130 insertions(+), 25 deletions(-) diff --git a/.github/workflows/hygiene.yml b/.github/workflows/hygiene.yml index 45bc3ef..e78cd7f 100644 --- a/.github/workflows/hygiene.yml +++ b/.github/workflows/hygiene.yml @@ -117,6 +117,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + ref: main # the live site is compared with main, whatever branch a manual run starts on persist-credentials: false - name: the site-rot check on main, with the live rules @@ -827,7 +828,7 @@ jobs: # proof-watch reads the next day (so a red run is tried again the next day when the tag is new). Nothing is committed back and no page is edited by a workflow. proof-report: needs: [proof-watch, live-check, bench-miss-cost, bench-server-down, bench-verify-release, bench-setup-pages, bench-howto-pages, bench-howto-pages-2, bench-howto-pages-3, bench-k8s-page, bench-ga-page, bench-howto-pages-4, bench-howto-pages-5, bench-howto-pages-6, bench-howto-pages-7, bench-howto-pages-8, bench-howto-pages-9, bench-howto-pages-10, bench-howto-pages-11, bench-howto-pages-12, bench-maclinux-linux-store, bench-maclinux-mac-restore, bench-maclinux-mac-store, bench-maclinux-linux-restore, bench-android, demo-github-actions, proof-selftest] - if: ${{ always() && ((github.event_name == 'schedule' && (needs.live-check.result == 'failure' || needs.live-check.result == 'success' || needs.proof-watch.result == 'failure' || (needs.proof-watch.result == 'success' && (needs.proof-watch.outputs.run == 'true' || startsWith(needs.proof-watch.outputs.reason, 'the newest release tag') || startsWith(needs.proof-watch.outputs.reason, 'release lookup failed'))))) || (github.event_name == 'workflow_dispatch' && inputs.which == 'proof-selftest')) }} + if: ${{ always() && ((github.event_name == 'schedule' && (needs.live-check.result == 'failure' || needs.live-check.result == 'success' || needs.proof-watch.result == 'failure' || (needs.proof-watch.result == 'success' && (needs.proof-watch.outputs.run == 'true' || startsWith(needs.proof-watch.outputs.reason, 'the newest release tag') || startsWith(needs.proof-watch.outputs.reason, 'release lookup failed'))))) || (github.event_name == 'workflow_dispatch' && inputs.which == 'proof-selftest') || (github.event_name == 'workflow_dispatch' && inputs.which == 'live-check' && (needs.live-check.result == 'failure' || needs.live-check.result == 'success'))) }} runs-on: ubuntu-latest timeout-minutes: 15 concurrency: @@ -850,7 +851,7 @@ jobs: TAG: ${{ needs.proof-watch.outputs.run == 'true' && needs.proof-watch.outputs.tag || '' }} # empty on a day nothing was proven, so a live-check-only report never writes the proven-release artifact WATCH_REASON: ${{ needs.proof-watch.outputs.reason }} WATCH_RUN: ${{ needs.proof-watch.outputs.run }} - MODE: ${{ github.event_name == 'schedule' && 'schedule' || 'selftest' }} + MODE: ${{ (github.event_name == 'schedule' || inputs.which == 'live-check') && 'schedule' || 'selftest' }} OUT_FILE: proven-release.txt run: bash bin/proof-report.sh diff --git a/bin/live-check.py b/bin/live-check.py index bf90438..3b0e8bf 100755 --- a/bin/live-check.py +++ b/bin/live-check.py @@ -22,6 +22,8 @@ import time import urllib.error import urllib.request +from html.parser import HTMLParser +from urllib.parse import urljoin, urlsplit SITE = "https://fosterstack.com/" TIMEOUT = float(os.environ.get("LIVE_CHECK_TIMEOUT", "20")) @@ -37,8 +39,9 @@ def redirect_request(self, *a, **k): OPENER = urllib.request.build_opener(_NoRedirect) -def fetch(url): - """(status, body) with one retry; status 0 means no answer. Never raises.""" +def fetch(url, accept=(200,)): + """(status, body). ONE retry for any answer not in accept (a timeout, a refusal, a 5xx, a 403, a 404, a redirect: all may be a + flaky moment), so only a failure seen twice is reported; status 0 means no answer. Never raises.""" last = (0, b"") for attempt in (1, 2): try: @@ -49,14 +52,46 @@ def fetch(url): last = (e.code, b"") except Exception: last = (0, b"") - # a good answer, or a plain "no such thing", is final; a timeout, a refusal and a 5xx are tried once more - if last[0] in (200, 301, 302, 303, 307, 308, 400, 401, 403, 404, 410) : + if last[0] in accept: break if attempt == 1: time.sleep(RETRY_WAIT) return last +class _Links(HTMLParser): + """Every href of every a and link tag, read the way a browser reads the page.""" + + def __init__(self): + super().__init__(convert_charrefs=True) + self.hrefs = [] + + def handle_starttag(self, tag, attrs): + if tag in ("a", "link", "area"): + for k, v in attrs: + if k.lower() == "href" and v: + self.hrefs.append(v.strip()) + + handle_startendtag = handle_starttag + + +def links_of(text): + parser = _Links() + parser.feed(text) + parser.close() + return parser.hrefs + + +def safe_suffix(suffix): + """The part of a sitemap URL after the site root, if it is a plain relative path (no leading slash, no '..', no + backslash, no query or fragment, no control characters); None otherwise.""" + if suffix.startswith("/") or "\\" in suffix or "?" in suffix or "#" in suffix or re.search(r"[\x00-\x1f\x7f-\x9f]", suffix): + return None + if any(part in ("..", ".") for part in suffix.split("/")): + return None + return suffix + + def clean(text): return re.sub(r"[\x00-\x1f\x7f-\x9f]", " ", text)[:300] @@ -77,8 +112,12 @@ def live_url(site_url): return base + "/" + site_url[len(SITE):] def repo_bytes(site_url): - rel = site_url[len(SITE):] - path = os.path.join(args.root, rel + ("index.html" if rel.endswith("/") or rel == "" else "")) + rel = safe_suffix(site_url[len(SITE):]) + if rel is None: + return None + path = os.path.realpath(os.path.join(args.root, rel + ("index.html" if rel.endswith("/") or rel == "" else ""))) + if not path.startswith(os.path.realpath(args.root) + os.sep): + return None try: with open(path, "rb") as f: return f.read() @@ -96,6 +135,8 @@ def repo_bytes(site_url): for u in locs: if not u.startswith(SITE): bad("sitemap", "the live sitemap lists %s, which is not on %s" % (u, SITE)) + elif safe_suffix(u[len(SITE):]) is None: + bad("sitemap", "the live sitemap lists %s, which is not a plain path on the site" % u) else: pages.append(u) @@ -120,22 +161,27 @@ def repo_bytes(site_url): for u, got in texts.items(): if not u.endswith("/"): continue - for m in re.finditer(r'(?/dev/null; done; rm -rf "$tmp"; } trap cleanup EXIT @@ -32,10 +33,13 @@ class GH(http.server.BaseHTTPRequestHandler): self.send_response(302); self.send_header("Location", "/login"); self.end_headers(); return if "/dead/" in self.path: self.send_response(404); self.end_headers(); return - if "/flaky/" in self.path: - flag = os.path.join(tmp, "flaky-seen") - if not os.path.exists(flag): - open(flag, "w").close(); self.send_response(503); self.end_headers(); return + for name, code in (("flaky", 503), ("flaky403", 403), ("flaky404", 404), ("flaky302", 302)): + if "/%s/" % name in self.path: + flag = os.path.join(tmp, name + "-seen") + if not os.path.exists(flag): + open(flag, "w").close(); self.send_response(code) + if code == 302: self.send_header("Location", "/elsewhere") + self.end_headers(); return self.send_response(200); self.send_header("Content-Length", "2"); self.end_headers(); self.wfile.write(b"ok") def serve(h, portfile): s = socketserver.ThreadingTCPServer(("127.0.0.1", 0), h); s.daemon_threads = True @@ -82,7 +86,7 @@ import sys for p in sys.argv[1:]: s = open(p).read(); open(p, "w").write(s.replace("", '

x y

', 1)) PY -rm -f "$tmp/flaky-seen" +rm -f "$tmp"/flaky*-seen expect_pass "the advisory form redirect is fine, and one flaky answer is retried once" fresh; python3 - "$served/$first/index.html" "$site/$first/index.html" <<'PY' import sys @@ -104,4 +108,58 @@ p = sys.argv[1]; s = open(p).read() open(p, "w").write(s.replace("", "https://example.org/x/")) PY expect_fail "a live sitemap entry for another host is refused" "FAIL sitemap: .*example.org" +# one retry for EVERY answer that is not acceptable: a flaky 403, 404 or redirect is tried again, a steady one is reported +fresh; python3 - "$served/$first/index.html" "$site/$first/index.html" <<'PY' +import sys +for p in sys.argv[1:]: + s = open(p).read(); open(p, "w").write(s.replace("", '

a b c

', 1)) +PY +rm -f "$tmp"/flaky*-seen +expect_pass "a 403, a 404 and a redirect that are fine on the second try do not fail the run" +# every way a link is written is read, and links are resolved against their page +for link in "x" "x" "x"; do + fresh; python3 - "$served/$first/index.html" "$site/$first/index.html" "$link" <<'PY' +import sys +for p in sys.argv[1:3]: + s = open(p).read(); open(p, "w").write(s.replace("", "

" + sys.argv[3] + "

", 1)) +PY + expect_fail "a link written as $link is checked" "FAIL link: .*dead" +done +fresh; python3 - "$served/$first/index.html" "$site/$first/index.html" <<'PY' +import sys +for p in sys.argv[1:]: + s = open(p).read(); open(p, "w").write(s.replace("", '

x y

', 1)) +PY +expect_fail "a relative link is resolved against its page" "FAIL link: .*no-such-relative-xyz" +expect_fail "an absolute link to the site itself is checked" "FAIL link: .*no-such-absolute-xyz" +# a sitemap entry can never make the check read a file outside the checkout +for loc in "https://fosterstack.com//etc/hosts" "https://fosterstack.com/../../etc/hosts" "https://fosterstack.com/a/../../x/"; do + fresh; python3 - "$served/sitemap.xml" "$loc" <<'PY' +import sys +p = sys.argv[1]; s = open(p).read() +open(p, "w").write(s.replace("", "" + sys.argv[2] + "")) +PY + expect_fail "a sitemap entry $loc is refused" "FAIL sitemap: .*$(printf '%s' "$loc" | sed 's/[.]/./g' | cut -c1-30)" +done +# a page that redirects, robots.txt that drifts, and control characters never reach the output +fresh; printf 'x' >> "$served/robots.txt" +expect_fail "a live robots.txt that differs from main is reported" "FAIL drift: .*robots.txt" +fresh; python3 - "$served/sitemap.xml" <<'PY' +import sys +p = sys.argv[1]; s = open(p).read() +open(p, "w").write(s.replace("", "https://fosterstack.com/bad\x9b[31mpage/")) +PY +run || true +python3 -c "import sys; d=open(sys.argv[1],encoding='utf-8',errors='replace').read(); sys.exit(0 if 'FAIL' in d and not any(0x80<=ord(c)<=0x9f or (ord(c)<32 and c!=chr(10)) for c in d) else 1)" "$tmp/out" && echo "ok control characters from the live site are not printed" || { echo "FAIL control characters reached the output"; fail=1; } +# the workflow is wired as promised (static reads of .github/workflows/hygiene.yml) +wf=.github/workflows/hygiene.yml +chk() { if eval "$2"; then echo "ok $1"; else echo "FAIL $1"; fail=1; fi; } +chk "the live-check job checks out main, whatever branch a manual run starts on" "awk '/^ live-check:/{f=1} f&&/ref: main/{r=1} /^ # Timed runs/{f=0} END{exit !r}' $wf" +chk "issues: write is on exactly one job (proof-report)" "[ \$(grep -v '^\\s*#' $wf | grep -c 'issues: write') = 1 ] && awk '/^ proof-report:/{f=1} f&&/issues: write/{r=1} END{exit !r}' $wf" +chk "the live-check job has contents: read and no write permission" "awk '/^ live-check:/{f=1} f&&/permissions:/{p=1} f&&p&&/write/{bad=1} /^ # Timed runs/{f=0} END{exit bad}' $wf" +chk "every uses: line is a full commit digest with a tag comment" "! grep -E '^\s*-?\s*uses:' $wf | grep -v -E 'uses: [^ ]+@[0-9a-f]{40} # v?[0-9]' | grep -q ." +chk "a manual live-check run is reported (the report job's condition names it)" "awk '/^ proof-report:/{f=1} f&&/which == .live-check./{r=1} END{exit !r}' $wf" +chk "the report's TAG is empty unless the proof watch decided to run" "grep -q \"TAG: \\\${{ needs.proof-watch.outputs.run == 'true' && needs.proof-watch.outputs.tag || '' }}\" $wf" +chk "no new workflow file: only hygiene.yml and the existing ones" "[ \$(ls .github/workflows | wc -l) -le \$(git ls-tree --name-only origin/main .github/workflows/ 2>/dev/null | wc -l) ]" + [ "$fail" = 0 ] && echo "all live-check self-tests passed" || { echo "live-check self-test FAILED"; exit 1; } From 173b2fcecf30926671955ca49acdf68e63c50678 Mon Sep 17 00:00:00 2001 From: FosterStack Admin <317177128+fosterstack-admin@users.noreply.github.com> Date: Thu, 22 Oct 2026 15:29:53 -0400 Subject: [PATCH 3/5] Live check: one sitemap answer, plain https links to the two hosts in any case, control characters refused before trimming, no baseline ref needed; tests for the page redirect, link forms and proof-report --- bin/live-check.py | 46 ++++++++++++++++++++------ bin/test-live-check.sh | 73 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 107 insertions(+), 12 deletions(-) diff --git a/bin/live-check.py b/bin/live-check.py index 3b0e8bf..a40a6af 100755 --- a/bin/live-check.py +++ b/bin/live-check.py @@ -82,6 +82,17 @@ def links_of(text): return parser.hrefs +def has_control(raw): + """A control character written out, or as a numeric reference ( ), which the decoder would otherwise drop.""" + if re.search(r"[\x00-\x1f\x7f-\x9f]", raw): + return True + for m in re.finditer(r"&#(?:[xX]([0-9a-fA-F]+)|([0-9]+));", raw): + n = int(m.group(1), 16) if m.group(1) else int(m.group(2)) + if n < 0x20 or 0x7f <= n <= 0x9f: + return True + return False + + def safe_suffix(suffix): """The part of a sitemap URL after the site root, if it is a plain relative path (no leading slash, no '..', no backslash, no query or fragment, no control characters); None otherwise.""" @@ -126,24 +137,36 @@ def repo_bytes(site_url): # 1. the page list, from the live sitemap status, body = fetch(base + "/sitemap.xml") + sitemap_body = body + status_sitemap_ok = status == 200 if status != 200: bad("sitemap", "sitemap.xml answered %s" % (status or "nothing")) locs = [] else: - locs = [html.unescape(x).strip() for x in re.findall(r"([^<]*)", body.decode("utf-8", "replace"))] + # only plain spaces and line breaks around an entry are ignored; any other control character (a tab, U+001F, ...) + # is looked at after the text is decoded, and the entry is refused, so it is never fetched and no file is read for it + locs = [x.strip(" \r\n") for x in re.findall(r"([^<]*)", body.decode("utf-8", "replace"))] pages = [] - for u in locs: - if not u.startswith(SITE): + for raw in locs: + u = html.unescape(raw) + if has_control(raw): + bad("sitemap", "the live sitemap lists %s, which is not a plain path on the site (it holds a control character)" % raw) + elif not u.startswith(SITE): bad("sitemap", "the live sitemap lists %s, which is not on %s" % (u, SITE)) elif safe_suffix(u[len(SITE):]) is None: bad("sitemap", "the live sitemap lists %s, which is not a plain path on the site" % u) else: pages.append(u) + if status == 200 and not pages: + bad("sitemap", "the live sitemap lists no page") # 2. every listed page, and the three files, against main texts = {} for u in dict.fromkeys(pages + [SITE + "llms.txt", SITE + "robots.txt", SITE + "sitemap.xml"]): - status, got = fetch(live_url(u)) + if u == SITE + "sitemap.xml": + status, got = (200, sitemap_body) if sitemap_body is not None and status_sitemap_ok else (0, b"") # the same answer the page list came from + else: + status, got = fetch(live_url(u)) label = u[len(SITE):] or "/" kind = "page" if u in pages else "drift" if status != 200: @@ -164,13 +187,16 @@ def repo_bytes(site_url): for href in links_of(got.decode("utf-8", "replace")): full = urljoin(u, href) # resolved against its own page, so relative and absolute links are both read parts = urlsplit(full) - if parts.scheme != "https": + host = (parts.hostname or "").lower() + if parts.scheme not in ("http", "https") or host not in ("fosterstack.com", "github.com"): + continue + if host == "github.com" and not parts.path.startswith("/fosterstack/"): + continue + if parts.scheme != "https" or parts.username or parts.password or parts.port not in (None, 443): + bad("link", "%s links %s, which is not a plain https link (http, a port or credentials)" % (u, full)) continue - clean_url = parts._replace(query="", fragment="").geturl() - if parts.netloc == "fosterstack.com": - internal.setdefault(clean_url, u) - elif parts.netloc == "github.com" and parts.path.startswith("/fosterstack/"): - external.setdefault(clean_url, u) + clean_url = "https://%s%s" % (host, parts.path) + (internal if host == "fosterstack.com" else external).setdefault(clean_url, u) for v, src in sorted(internal.items()): if v in pages or v == SITE: continue diff --git a/bin/test-live-check.sh b/bin/test-live-check.sh index c783e33..1b3174f 100644 --- a/bin/test-live-check.sh +++ b/bin/test-live-check.sh @@ -22,6 +22,8 @@ class Live(http.server.SimpleHTTPRequestHandler): def __init__(self, *a, **k): super().__init__(*a, directory=os.path.join(tmp, "served"), **k) def log_message(self, *a): pass def do_GET(self): + if self.path.startswith("/redir-"): + self.send_response(301); self.send_header("Location", "/"); self.end_headers(); return if self.path.startswith("/slow-"): import time; time.sleep(60) return super().do_GET() @@ -151,15 +153,82 @@ open(p, "w").write(s.replace("", "https://fosterstack.com/bad PY run || true python3 -c "import sys; d=open(sys.argv[1],encoding='utf-8',errors='replace').read(); sys.exit(0 if 'FAIL' in d and not any(0x80<=ord(c)<=0x9f or (ord(c)<32 and c!=chr(10)) for c in d) else 1)" "$tmp/out" && echo "ok control characters from the live site are not printed" || { echo "FAIL control characters reached the output"; fail=1; } +# a page that redirects (a steady 301) is a failure, a link with another scheme, port, credentials or an upper-case host is still read +fresh; python3 - "$served/sitemap.xml" <<'PY' +import sys +p = sys.argv[1]; s = open(p).read() +open(p, "w").write(s.replace("", "https://fosterstack.com/redir-page/")) +PY +expect_fail "a page that keeps redirecting is reported" "FAIL page: .*redir-page.* 301" +for link in 'x' 'x' 'x' 'x' 'x'; do + fresh; python3 - "$served/$first/index.html" "$site/$first/index.html" "$link" <<'PY' +import sys +for p in sys.argv[1:3]: + s = open(p).read(); open(p, "w").write(s.replace("", "

" + sys.argv[3] + "

", 1)) +PY + expect_fail "$link is not skipped" "FAIL link: " +done +# a sitemap entry with a trailing control character is refused, not trimmed into a real page +for tail in ' ' ''; do + fresh; python3 - "$served/sitemap.xml" "$first" "$tail" <<'PY' +import sys +p = sys.argv[1]; s = open(p).read() +open(p, "w").write(s.replace("", "https://fosterstack.com/" + sys.argv[2] + "/" + sys.argv[3] + "")) +PY + expect_fail "a sitemap entry ending in $tail is refused" "FAIL sitemap: .*not a plain path" +done +# an empty live sitemap is a failure, not a green run over nothing +fresh; : > "$served/sitemap.xml" +expect_fail "an empty live sitemap is refused" "FAIL sitemap: .*no page" + +# proof-report, with a stand-in gh that records what it is asked to do (no network) +mkdir "$tmp/fakebin" +cat >"$tmp/fakebin/gh" <<'GH' +#!/usr/bin/env bash +echo "gh $*" >> "$FAKE_LOG" +case "$*" in + *"/jobs?"*) cat "$FAKE_JOBS" ;; + *"/logs"*) echo "2026-10-22T05:00:00Z FAIL live: https://fosterstack.com/x/ answered 404" ;; + "issue list"*) printf '%s' "${FAKE_OPEN:-}" ;; + *) : ;; +esac +exit 0 +GH +chmod +x "$tmp/fakebin/gh" +report() { # name, jobs-json, open-issue, TAG, WATCH_RUN + rm -f "$tmp/proven.txt" "$tmp/fake.log"; printf '%s' "$2" >"$tmp/jobs.json" + PATH="$tmp/fakebin:$PATH" FAKE_LOG="$tmp/fake.log" FAKE_JOBS="$tmp/jobs.json" FAKE_OPEN="$3" GH_TOKEN=x GITHUB_REPOSITORY=o/r RUN_ID=1 RUN_URL=http://x \ + TAG="$4" WATCH_RUN="$5" WATCH_REASON="" MODE=schedule OUT_FILE="$tmp/proven.txt" bash bin/proof-report.sh >"$tmp/rep.out" 2>&1 +} +okrun='{"jobs":[{"id":1,"name":"proof-watch","conclusion":"success","status":"completed","steps":[]},{"id":3,"name":"bench-miss-cost","conclusion":"success","status":"completed","steps":[]}]}' +ok='{"jobs":[{"id":1,"name":"proof-watch","conclusion":"success","status":"completed","steps":[]},{"id":2,"name":"live-check","conclusion":"success","status":"completed","steps":[]},{"id":3,"name":"bench-miss-cost","conclusion":"skipped","status":"completed","steps":[]}]}' +bad='{"jobs":[{"id":2,"name":"live-check","conclusion":"failure","status":"completed","steps":[{"name":"the live site against main, and every link on it","conclusion":"failure"}]},{"id":3,"name":"bench-miss-cost","conclusion":"skipped","status":"completed","steps":[]}]}' +chk2() { if eval "$2"; then echo "ok $1"; else echo "FAIL $1"; sed 's/^/ /' "$tmp/rep.out" "$tmp/fake.log" 2>/dev/null | head -6; fail=1; fi; } +report a "$ok" "" "" false +chk2 "report: a day with only the live check, all green, opens nothing and writes no proven-release file" "[ ! -e $tmp/proven.txt ] && ! grep -q 'issue create' $tmp/fake.log" +report b "$bad" "" "" false +chk2 "report: a failed live-check opens the one issue, naming the job, and writes no proven-release file" "grep -q 'issue create' $tmp/fake.log && grep -q 'live-check' $tmp/fake.log && [ ! -e $tmp/proven.txt ]" +report c "$bad" "7" "" false +chk2 "report: a failed live-check with an open issue comments on it, no second issue" "grep -q 'issue comment 7' $tmp/fake.log && ! grep -q 'issue create' $tmp/fake.log" +report d "$ok" "7" "" false +chk2 "report: a green day closes the open issue" "grep -q 'issue close 7' $tmp/fake.log" +report e "$okrun" "" "v0.2.2" true +chk2 "report: a green proof run with a release writes the proven-release file" "[ -s $tmp/proven.txt ]" +report f "$ok" "" "" true +chk2 "report: a proof run that skipped a bench job proved nothing for it" "grep -q 'issue create' $tmp/fake.log && [ ! -e $tmp/proven.txt ]" + # the workflow is wired as promised (static reads of .github/workflows/hygiene.yml) wf=.github/workflows/hygiene.yml chk() { if eval "$2"; then echo "ok $1"; else echo "FAIL $1"; fail=1; fi; } chk "the live-check job checks out main, whatever branch a manual run starts on" "awk '/^ live-check:/{f=1} f&&/ref: main/{r=1} /^ # Timed runs/{f=0} END{exit !r}' $wf" chk "issues: write is on exactly one job (proof-report)" "[ \$(grep -v '^\\s*#' $wf | grep -c 'issues: write') = 1 ] && awk '/^ proof-report:/{f=1} f&&/issues: write/{r=1} END{exit !r}' $wf" -chk "the live-check job has contents: read and no write permission" "awk '/^ live-check:/{f=1} f&&/permissions:/{p=1} f&&p&&/write/{bad=1} /^ # Timed runs/{f=0} END{exit bad}' $wf" +chk "the live-check job has exactly contents: read, and no write permission" "awk '/^ live-check:/{f=1} f&&/^ permissions:/{p=1;next} f&&p&&/^ [a-z-]+:/{n++; if (\$0 !~ /contents: read/) bad=1} f&&p&&!/^ /{p=0} /^ # Timed runs/{f=0} END{exit (bad||n!=1)}' $wf" +chk "the live-check job runs only on the schedule or a manual which=live-check" "awk '/^ live-check:/{f=1;next} f&&/^ if:/{print; exit}' $wf | grep -q \"github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.which == 'live-check')\$\"" +chk "a manual live-check run is reported under the schedule mode, not the self-test mode" "grep -q \"MODE: \\\${{ (github.event_name == 'schedule' || inputs.which == 'live-check') && 'schedule' || 'selftest' }}\" $wf" chk "every uses: line is a full commit digest with a tag comment" "! grep -E '^\s*-?\s*uses:' $wf | grep -v -E 'uses: [^ ]+@[0-9a-f]{40} # v?[0-9]' | grep -q ." chk "a manual live-check run is reported (the report job's condition names it)" "awk '/^ proof-report:/{f=1} f&&/which == .live-check./{r=1} END{exit !r}' $wf" chk "the report's TAG is empty unless the proof watch decided to run" "grep -q \"TAG: \\\${{ needs.proof-watch.outputs.run == 'true' && needs.proof-watch.outputs.tag || '' }}\" $wf" -chk "no new workflow file: only hygiene.yml and the existing ones" "[ \$(ls .github/workflows | wc -l) -le \$(git ls-tree --name-only origin/main .github/workflows/ 2>/dev/null | wc -l) ]" +chk "the sitemap is fetched once: the page list and the comparison with main use the same answer" "[ \$(grep -c 'fetch(base + \"/sitemap.xml\")' bin/live-check.py) = 1 ] && grep -q 'sitemap_body' bin/live-check.py" +chk "no new workflow file: hygiene.yml is the only one" "[ \"\$(ls .github/workflows)\" = hygiene.yml ]" [ "$fail" = 0 ] && echo "all live-check self-tests passed" || { echo "live-check self-test FAILED"; exit 1; } From a7da757de1eb52296949d3c41510e36fc675c967 Mon Sep 17 00:00:00 2001 From: FosterStack Admin <317177128+fosterstack-admin@users.noreply.github.com> Date: Thu, 22 Oct 2026 15:38:32 -0400 Subject: [PATCH 4/5] Live check: nothing trimmed before a sitemap entry is judged, bare-host https links only, bad addresses reported instead of crashing; tests --- bin/live-check.py | 17 +++++++++++------ bin/test-live-check.sh | 19 +++++++++++++++++++ 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/bin/live-check.py b/bin/live-check.py index a40a6af..13f78fc 100755 --- a/bin/live-check.py +++ b/bin/live-check.py @@ -145,7 +145,7 @@ def repo_bytes(site_url): else: # only plain spaces and line breaks around an entry are ignored; any other control character (a tab, U+001F, ...) # is looked at after the text is decoded, and the entry is refused, so it is never fetched and no file is read for it - locs = [x.strip(" \r\n") for x in re.findall(r"([^<]*)", body.decode("utf-8", "replace"))] + locs = re.findall(r"([^<]*)", body.decode("utf-8", "replace")) # the untouched text: nothing is trimmed before it is judged pages = [] for raw in locs: u = html.unescape(raw) @@ -185,15 +185,20 @@ def repo_bytes(site_url): if not u.endswith("/"): continue for href in links_of(got.decode("utf-8", "replace")): - full = urljoin(u, href) # resolved against its own page, so relative and absolute links are both read - parts = urlsplit(full) - host = (parts.hostname or "").lower() + try: + full = urljoin(u, href) # resolved against its own page, so relative and absolute links are both read + parts = urlsplit(full) + host = (parts.hostname or "").lower() + except ValueError: + bad("link", "%s has a link that is not a valid address: %s" % (u, href)) + continue if parts.scheme not in ("http", "https") or host not in ("fosterstack.com", "github.com"): continue if host == "github.com" and not parts.path.startswith("/fosterstack/"): continue - if parts.scheme != "https" or parts.username or parts.password or parts.port not in (None, 443): - bad("link", "%s links %s, which is not a plain https link (http, a port or credentials)" % (u, full)) + # a plain link is https and the bare host: no port (not even :443), no user information (not even an empty one) + if parts.scheme != "https" or parts.netloc.lower() != host: + bad("link", "%s links %s, which is not a plain https link (http, a port or user information)" % (u, full)) continue clean_url = "https://%s%s" % (host, parts.path) (internal if host == "fosterstack.com" else external).setdefault(clean_url, u) diff --git a/bin/test-live-check.sh b/bin/test-live-check.sh index 1b3174f..8a7b706 100644 --- a/bin/test-live-check.sh +++ b/bin/test-live-check.sh @@ -177,6 +177,25 @@ open(p, "w").write(s.replace("", "https://fosterstack.com/" + PY expect_fail "a sitemap entry ending in $tail is refused" "FAIL sitemap: .*not a plain path" done +# a literal CR or LF around a sitemap entry is refused too (never trimmed into a real page), and neither a fetch nor a file read happens +for tail in $'\n' $'\r'; do + fresh; python3 - "$served/sitemap.xml" "$first" "$tail" <<'PY' +import sys +p = sys.argv[1]; s = open(p).read() +open(p, "w", newline="").write(s.replace("", "https://fosterstack.com/" + sys.argv[2] + "/" + sys.argv[3] + "")) +PY + expect_fail "a sitemap entry with a literal CR or LF at its end is refused" "FAIL sitemap: .*control character" +done +# the bare host only: an explicit :443, an empty user, a bad port and a bad address are failures, and the run goes on to print them +for link in 'x' 'x' 'x' 'x' 'x' 'x' 'x'; do + fresh; python3 - "$served/$first/index.html" "$site/$first/index.html" "$link" <<'PY' +import sys +for p in sys.argv[1:3]: + s = open(p).read(); open(p, "w").write(s.replace("", "

" + sys.argv[3] + "

", 1)) +PY + expect_fail "$link is a plain-link failure, not a crash" "FAIL link: " + grep -q "live pages checked" "$tmp/out" || { echo "FAIL the run ended without its summary line for $link"; fail=1; } +done # an empty live sitemap is a failure, not a green run over nothing fresh; : > "$served/sitemap.xml" expect_fail "an empty live sitemap is refused" "FAIL sitemap: .*no page" From 341d19d3173346b9cceb1ea418d752ec7e700a98 Mon Sep 17 00:00:00 2001 From: FosterStack Admin <317177128+fosterstack-admin@users.noreply.github.com> Date: Thu, 22 Oct 2026 15:51:12 -0400 Subject: [PATCH 5/5] Live check: the query string is part of a link (only the fragment is dropped); tests for both hosts and a page already in the sitemap --- bin/live-check.py | 4 ++-- bin/test-live-check.sh | 13 ++++++++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/bin/live-check.py b/bin/live-check.py index 13f78fc..2525fb6 100755 --- a/bin/live-check.py +++ b/bin/live-check.py @@ -200,7 +200,7 @@ def repo_bytes(site_url): if parts.scheme != "https" or parts.netloc.lower() != host: bad("link", "%s links %s, which is not a plain https link (http, a port or user information)" % (u, full)) continue - clean_url = "https://%s%s" % (host, parts.path) + clean_url = "https://%s%s%s" % (host, parts.path, "?" + parts.query if parts.query else "") # the query is part of the resource; only the fragment is dropped (internal if host == "fosterstack.com" else external).setdefault(clean_url, u) for v, src in sorted(internal.items()): if v in pages or v == SITE: @@ -209,7 +209,7 @@ def repo_bytes(site_url): if status != 200: bad("link", "%s links %s, which answered %s" % (src, v, status or "nothing")) for v, src in sorted(external.items()): - advisory = v.endswith("/security/advisories/new") + advisory = urlsplit(v).path.endswith("/security/advisories/new") status, _ = fetch(args.github.rstrip("/") + v[len("https://github.com"):], accept=(200, 301, 302, 303, 307, 308) if advisory else (200,)) if status in (200, 301, 302, 303, 307, 308) and (status == 200 or advisory): diff --git a/bin/test-live-check.sh b/bin/test-live-check.sh index 8a7b706..8da3444 100644 --- a/bin/test-live-check.sh +++ b/bin/test-live-check.sh @@ -22,6 +22,8 @@ class Live(http.server.SimpleHTTPRequestHandler): def __init__(self, *a, **k): super().__init__(*a, directory=os.path.join(tmp, "served"), **k) def log_message(self, *a): pass def do_GET(self): + if "bad=1" in self.path: + self.send_response(404); self.end_headers(); return if self.path.startswith("/redir-"): self.send_response(301); self.send_header("Location", "/"); self.end_headers(); return if self.path.startswith("/slow-"): @@ -33,7 +35,7 @@ class GH(http.server.BaseHTTPRequestHandler): n = self.path.split("/")[-1] if self.path.endswith("/security/advisories/new"): self.send_response(302); self.send_header("Location", "/login"); self.end_headers(); return - if "/dead/" in self.path: + if "/dead/" in self.path or "bad=1" in self.path: self.send_response(404); self.end_headers(); return for name, code in (("flaky", 503), ("flaky403", 403), ("flaky404", 404), ("flaky302", 302)): if "/%s/" % name in self.path: @@ -196,6 +198,15 @@ PY expect_fail "$link is a plain-link failure, not a crash" "FAIL link: " grep -q "live pages checked" "$tmp/out" || { echo "FAIL the run ended without its summary line for $link"; fail=1; } done +# the query string is part of the link: a dead link that differs only by its query is found, for both hosts and for a page that is already in the sitemap +for link in 'x' 'x' "x" 'x'; do + fresh; python3 - "$served/$first/index.html" "$site/$first/index.html" "$link" <<'PY' +import sys +for p in sys.argv[1:3]: + s = open(p).read(); open(p, "w").write(s.replace("", "

" + sys.argv[3] + "

", 1)) +PY + expect_fail "a dead link that differs only by its query is found: $link" "FAIL link: .*bad=1" +done # an empty live sitemap is a failure, not a green run over nothing fresh; : > "$served/sitemap.xml" expect_fail "an empty live sitemap is refused" "FAIL sitemap: .*no page"