Daily live check for launch week: the live site against main, and every link on it

.github/workflows/hygiene.yml

@@ -42,6 +42,7 @@ on:
           - howto-pages-11
           - howto-pages-12
           - proof-watch
+          - live-check
           - proof-selftest
         default: miss-cost
 
@@ -71,6 +72,9 @@ jobs:
       - name: self-test the site-rot check
         run: bash bin/test-check-site-rot.sh
 
+      - name: self-test the live check (a local server stands in for the live site and for GitHub)
+        run: bash bin/test-live-check.sh
+
   # proof-watch: on the daily schedule it decides whether today's run proves every bench job below (a new release tag of
   # fosterstack/cache, or Monday for the weekly run). It reads only the releases of fosterstack/cache and only tags matching
   # ^v[0-9]+\.[0-9]+\.[0-9]+$ (any other tag is reported, never run); see bin/proof-watch.sh. On a manual run with
@@ -98,6 +102,30 @@ jobs:
           GH_TOKEN: ${{ github.token }}
         run: bash bin/proof-watch.sh
 
+  # live-check: once a day on the schedule (and by hand with which=live-check), the LIVE site is checked, not the repository: the page
+  # list comes from the live sitemap.xml; every listed page must answer 200 and be byte for byte the file on main (so the live site and
+  # what the site-rot check reads cannot drift apart unseen); every internal link and github.com/fosterstack link on the live pages must
+  # answer 200. One fixed timeout and ONE retry per request. The site-rot check runs first on main with the live rules (no page hidden
+  # from search, none listed hidden). Read-only: no secret, no write permission, no action beyond the pinned checkout. A failure is
+  # reported by proof-report in the ONE tracking issue (job name live-check in bin/proof-pages.txt); see bin/live-check.py.
+  live-check:
+    if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.which == 'live-check')
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: main   # the live site is compared with main, whatever branch a manual run starts on
+          persist-credentials: false
+
+      - name: the site-rot check on main, with the live rules
+        run: python3 bin/check-site-rot.py --live
+
+      - name: the live site against main, and every link on it
+        run: python3 bin/live-check.py
+
   # Timed runs for the "what does a cache miss cost" page. Manual dispatch only; never runs on push or
   # pull request. No secrets, no write permission, no third-party action beyond the pinned checkout above.
   # The script downloads the FosterStack Cache release and Gradle, and checks both against pinned sha256 values.
@@ -799,8 +827,8 @@ jobs:
   # bin/proof-report.sh. After a fully GREEN scheduled run only, it leaves the one-line artifact "proven-release" (tag, run id) that
   # proof-watch reads the next day (so a red run is tried again the next day when the tag is new). Nothing is committed back and no page is edited by a workflow.
   proof-report:
-    needs: [proof-watch, bench-miss-cost, bench-server-down, bench-verify-release, bench-setup-pages, bench-howto-pages, bench-howto-pages-2, bench-howto-pages-3, bench-k8s-page, bench-ga-page, bench-howto-pages-4, bench-howto-pages-5, bench-howto-pages-6, bench-howto-pages-7, bench-howto-pages-8, bench-howto-pages-9, bench-howto-pages-10, bench-howto-pages-11, bench-howto-pages-12, bench-maclinux-linux-store, bench-maclinux-mac-restore, bench-maclinux-mac-store, bench-maclinux-linux-restore, bench-android, demo-github-actions, proof-selftest]
-    if: ${{ always() && ((github.event_name == 'schedule' && (needs.proof-watch.result == 'failure' || (needs.proof-watch.result == 'success' && (needs.proof-watch.outputs.run == 'true' || startsWith(needs.proof-watch.outputs.reason, 'the newest release tag') || startsWith(needs.proof-watch.outputs.reason, 'release lookup failed'))))) || (github.event_name == 'workflow_dispatch' && inputs.which == 'proof-selftest')) }}
+    needs: [proof-watch, live-check, bench-miss-cost, bench-server-down, bench-verify-release, bench-setup-pages, bench-howto-pages, bench-howto-pages-2, bench-howto-pages-3, bench-k8s-page, bench-ga-page, bench-howto-pages-4, bench-howto-pages-5, bench-howto-pages-6, bench-howto-pages-7, bench-howto-pages-8, bench-howto-pages-9, bench-howto-pages-10, bench-howto-pages-11, bench-howto-pages-12, bench-maclinux-linux-store, bench-maclinux-mac-restore, bench-maclinux-mac-store, bench-maclinux-linux-restore, bench-android, demo-github-actions, proof-selftest]
+    if: ${{ always() && ((github.event_name == 'schedule' && (needs.live-check.result == 'failure' || needs.live-check.result == 'success' || needs.proof-watch.result == 'failure' || (needs.proof-watch.result == 'success' && (needs.proof-watch.outputs.run == 'true' || startsWith(needs.proof-watch.outputs.reason, 'the newest release tag') || startsWith(needs.proof-watch.outputs.reason, 'release lookup failed'))))) || (github.event_name == 'workflow_dispatch' && inputs.which == 'proof-selftest') || (github.event_name == 'workflow_dispatch' && inputs.which == 'live-check' && (needs.live-check.result == 'failure' || needs.live-check.result == 'success'))) }}
     runs-on: ubuntu-latest
     timeout-minutes: 15
     concurrency:
@@ -820,10 +848,10 @@ jobs:
           GH_TOKEN: ${{ github.token }}
           RUN_ID: ${{ github.run_id }}
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          TAG: ${{ needs.proof-watch.outputs.tag }}
+          TAG: ${{ needs.proof-watch.outputs.run == 'true' && needs.proof-watch.outputs.tag || '' }}   # empty on a day nothing was proven, so a live-check-only report never writes the proven-release artifact
           WATCH_REASON: ${{ needs.proof-watch.outputs.reason }}
           WATCH_RUN: ${{ needs.proof-watch.outputs.run }}
-          MODE: ${{ github.event_name == 'schedule' && 'schedule' || 'selftest' }}
+          MODE: ${{ (github.event_name == 'schedule' || inputs.which == 'live-check') && 'schedule' || 'selftest' }}
           OUT_FILE: proven-release.txt
         run: bash bin/proof-report.sh
 

bin/check-file-allowlist.sh

@@ -64,6 +64,8 @@ ALLOW_PATTERNS=(
   '^bin/proof-watch\.sh$'
   '^bin/proof-report\.sh$'
   '^bin/proof-pages\.txt$'
+  '^bin/live-check\.py$'
+  '^bin/test-live-check\.sh$'
   '^\.github/workflows/[A-Za-z0-9_-][A-Za-z0-9._-]*\.ya?ml$'
   '^functions/(api/)?[A-Za-z0-9._-]+\.js$'
   '^[A-Za-z0-9._-]+/index\.html$'   # pre-positioned pages, e.g. bcn-removed/index.html

bin/live-check.py

@@ -0,0 +1,226 @@
+#!/usr/bin/env python3
+"""Check the LIVE site once a day (launch week and after). Standard library only; reads the network and the checkout, writes nothing.
+
+What it does, in order (a problem is one line starting with "FAIL", which is what the tracking issue quotes):
+  1. fetch /sitemap.xml from the live site and take the page list FROM IT (not from the repository); every entry must be
+     on the site's own host (https://fosterstack.com/), or the entry is refused and never fetched;
+  2. fetch every listed page: it must answer 200 (a redirect is a failure) and be byte for byte the file in the checkout of main;
+     sitemap.xml, llms.txt and robots.txt the same (so "the live site" and "what main says" cannot drift apart unseen);
+  3. from the live pages take every internal link (a path from the site root) and every github.com/fosterstack link; each must
+     answer 200 (the private-advisory form may redirect to a login).
+Every request has one fixed timeout (LIVE_CHECK_TIMEOUT, default 20 seconds) and ONE retry after a short wait
+(LIVE_CHECK_RETRY_WAIT, default 5 seconds), so a flaky answer does not raise an issue by itself.
+
+Usage: live-check.py [--root DIR] [--base URL] [--github URL]   (defaults: the checkout, https://fosterstack.com, https://github.com)
+"""
+import argparse
+import hashlib
+import html
+import os
+import re
+import sys
+import time
+import urllib.error
+import urllib.request
+from html.parser import HTMLParser
+from urllib.parse import urljoin, urlsplit
+
+SITE = "https://fosterstack.com/"
+TIMEOUT = float(os.environ.get("LIVE_CHECK_TIMEOUT", "20"))
+RETRY_WAIT = float(os.environ.get("LIVE_CHECK_RETRY_WAIT", "5"))
+MAX_BYTES = 8 * 1024 * 1024
+
+
+class _NoRedirect(urllib.request.HTTPRedirectHandler):
+    def redirect_request(self, *a, **k):
+        return None
+
+
+OPENER = urllib.request.build_opener(_NoRedirect)
+
+
+def fetch(url, accept=(200,)):
+    """(status, body). ONE retry for any answer not in accept (a timeout, a refusal, a 5xx, a 403, a 404, a redirect: all may be a
+    flaky moment), so only a failure seen twice is reported; status 0 means no answer. Never raises."""
+    last = (0, b"")
+    for attempt in (1, 2):
+        try:
+            with OPENER.open(urllib.request.Request(url, headers={"User-Agent": "fosterstack-live-check"}), timeout=TIMEOUT) as r:
+                body = r.read(MAX_BYTES + 1)
+                last = (r.status, body[:MAX_BYTES])
+        except urllib.error.HTTPError as e:
+            last = (e.code, b"")
+        except Exception:
+            last = (0, b"")
+        if last[0] in accept:
+            break
+        if attempt == 1:
+            time.sleep(RETRY_WAIT)
+    return last
+
+
+class _Links(HTMLParser):
+    """Every href of every a and link tag, read the way a browser reads the page."""
+
+    def __init__(self):
+        super().__init__(convert_charrefs=True)
+        self.hrefs = []
+
+    def handle_starttag(self, tag, attrs):
+        if tag in ("a", "link", "area"):
+            for k, v in attrs:
+                if k.lower() == "href" and v:
+                    self.hrefs.append(v.strip())
+
+    handle_startendtag = handle_starttag
+
+
+def links_of(text):
+    parser = _Links()
+    parser.feed(text)
+    parser.close()
+    return parser.hrefs
+
+
+def has_control(raw):
+    """A control character written out, or as a numeric reference (	 ), which the decoder would otherwise drop."""
+    if re.search(r"[\x00-\x1f\x7f-\x9f]", raw):
+        return True
+    for m in re.finditer(r"&#(?:[xX]([0-9a-fA-F]+)|([0-9]+));", raw):
+        n = int(m.group(1), 16) if m.group(1) else int(m.group(2))
+        if n < 0x20 or 0x7f <= n <= 0x9f:
+            return True
+    return False
+
+
+def safe_suffix(suffix):
+    """The part of a sitemap URL after the site root, if it is a plain relative path (no leading slash, no '..', no
+    backslash, no query or fragment, no control characters); None otherwise."""
+    if suffix.startswith("/") or "\\" in suffix or "?" in suffix or "#" in suffix or re.search(r"[\x00-\x1f\x7f-\x9f]", suffix):
+        return None
+    if any(part in ("..", ".") for part in suffix.split("/")):
+        return None
+    return suffix
+
+
+def clean(text):
+    return re.sub(r"[\x00-\x1f\x7f-\x9f]", " ", text)[:300]
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--root", default=".")
+    ap.add_argument("--base", default=SITE.rstrip("/"))
+    ap.add_argument("--github", default="https://github.com")
+    args = ap.parse_args()
+    base = args.base.rstrip("/")
+    problems = []
+
+    def bad(kind, msg):
+        problems.append("FAIL %s: %s" % (kind, clean(msg)))
+
+    def live_url(site_url):
+        return base + "/" + site_url[len(SITE):]
+
+    def repo_bytes(site_url):
+        rel = safe_suffix(site_url[len(SITE):])
+        if rel is None:
+            return None
+        path = os.path.realpath(os.path.join(args.root, rel + ("index.html" if rel.endswith("/") or rel == "" else "")))
+        if not path.startswith(os.path.realpath(args.root) + os.sep):
+            return None
+        try:
+            with open(path, "rb") as f:
+                return f.read()
+        except OSError:
+            return None
+
+    # 1. the page list, from the live sitemap
+    status, body = fetch(base + "/sitemap.xml")
+    sitemap_body = body
+    status_sitemap_ok = status == 200
+    if status != 200:
+        bad("sitemap", "sitemap.xml answered %s" % (status or "nothing"))
+        locs = []
+    else:
+        # only plain spaces and line breaks around an entry are ignored; any other control character (a tab, U+001F, ...)
+        # is looked at after the text is decoded, and the entry is refused, so it is never fetched and no file is read for it
+        locs = re.findall(r"<loc>([^<]*)</loc>", body.decode("utf-8", "replace"))   # the untouched text: nothing is trimmed before it is judged
+    pages = []
+    for raw in locs:
+        u = html.unescape(raw)
+        if has_control(raw):
+            bad("sitemap", "the live sitemap lists %s, which is not a plain path on the site (it holds a control character)" % raw)
+        elif not u.startswith(SITE):
+            bad("sitemap", "the live sitemap lists %s, which is not on %s" % (u, SITE))
+        elif safe_suffix(u[len(SITE):]) is None:
+            bad("sitemap", "the live sitemap lists %s, which is not a plain path on the site" % u)
+        else:
+            pages.append(u)
+    if status == 200 and not pages:
+        bad("sitemap", "the live sitemap lists no page")
+
+    # 2. every listed page, and the three files, against main
+    texts = {}
+    for u in dict.fromkeys(pages + [SITE + "llms.txt", SITE + "robots.txt", SITE + "sitemap.xml"]):
+        if u == SITE + "sitemap.xml":
+            status, got = (200, sitemap_body) if sitemap_body is not None and status_sitemap_ok else (0, b"")   # the same answer the page list came from
+        else:
+            status, got = fetch(live_url(u))
+        label = u[len(SITE):] or "/"
+        kind = "page" if u in pages else "drift"
+        if status != 200:
+            bad("page" if u in pages else "file", "%s answered %s" % (u, status or "nothing"))
+            continue
+        texts[u] = got
+        want = repo_bytes(u)
+        if want is None:
+            bad("drift", "%s is live but main has no such file" % u)
+        elif hashlib.sha256(want).digest() != hashlib.sha256(got).digest():
+            bad("drift", "%s is live but differs from the file on main" % u)
+
+    # 3. links on the live pages
+    internal, external = {}, {}
+    for u, got in texts.items():
+        if not u.endswith("/"):
+            continue
+        for href in links_of(got.decode("utf-8", "replace")):
+            try:
+                full = urljoin(u, href)           # resolved against its own page, so relative and absolute links are both read
+                parts = urlsplit(full)
+                host = (parts.hostname or "").lower()
+            except ValueError:
+                bad("link", "%s has a link that is not a valid address: %s" % (u, href))
+                continue
+            if parts.scheme not in ("http", "https") or host not in ("fosterstack.com", "github.com"):
+                continue
+            if host == "github.com" and not parts.path.startswith("/fosterstack/"):
+                continue
+            # a plain link is https and the bare host: no port (not even :443), no user information (not even an empty one)
+            if parts.scheme != "https" or parts.netloc.lower() != host:
+                bad("link", "%s links %s, which is not a plain https link (http, a port or user information)" % (u, full))
+                continue
+            clean_url = "https://%s%s%s" % (host, parts.path, "?" + parts.query if parts.query else "")   # the query is part of the resource; only the fragment is dropped
+            (internal if host == "fosterstack.com" else external).setdefault(clean_url, u)
+    for v, src in sorted(internal.items()):
+        if v in pages or v == SITE:
+            continue
+        status, _ = fetch(live_url(v))
+        if status != 200:
+            bad("link", "%s links %s, which answered %s" % (src, v, status or "nothing"))
+    for v, src in sorted(external.items()):
+        advisory = urlsplit(v).path.endswith("/security/advisories/new")
+        status, _ = fetch(args.github.rstrip("/") + v[len("https://github.com"):],
+                          accept=(200, 301, 302, 303, 307, 308) if advisory else (200,))
+        if status in (200, 301, 302, 303, 307, 308) and (status == 200 or advisory):
+            continue
+        bad("link", "%s links %s, which answered %s" % (src, v, status or "nothing"))
+
+    for line in problems:
+        print(line)
+    print("%d live pages checked, %d problem%s" % (len(pages), len(problems), "" if len(problems) == 1 else "s"))
+    return 1 if problems else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

bin/proof-pages.txt

@@ -25,3 +25,4 @@ bench-android	kotlin-android-gradle-build-cache
 demo-github-actions	github-actions-remote-build-cache
 proof-selftest	(self-test: no page)
 proof-watch	(the daily watch itself: it could not read the release or the earlier record)
+live-check	the live site: every page in the live sitemap and every link on it (see the FAIL lines)