From 3cdd8bb15f7010cbcc2606eceddcc0414412d2f3 Mon Sep 17 00:00:00 2001 From: FosterStack Admin <317177128+fosterstack-admin@users.noreply.github.com> Date: Thu, 22 Oct 2026 14:12:28 -0400 Subject: [PATCH 1/6] Site-rot check: no hidden page in the sitemap; on the live site only the 404 page may be hidden (owner's rule, Oct 22) --- bin/check-site-rot.py | 60 ++++++++++++++++++++++++++++++++++++-- bin/test-check-site-rot.sh | 40 +++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 2 deletions(-) diff --git a/bin/check-site-rot.py b/bin/check-site-rot.py index cf53eff..a5570f7 100755 --- a/bin/check-site-rot.py +++ b/bin/check-site-rot.py @@ -8,8 +8,12 @@ redirect whose target page or #anchor does not exist, or a redirect whose source is a live page 4. a mismatch between the sitemap, llms.txt and the home page's links to pages 5. a longer than 60 characters (or a page with none) + 6. hidden pages (owner ratified Oct 22 2026): a page hidden from search (a robots or googlebot noindex tag, or an + X-Robots-Tag noindex rule in _headers) that is listed in the sitemap; and, for the live site only, any hidden + page except 404.html. The second applies with --live, and by itself when GitHub says this run is for a pull + request whose base is main (GITHUB_BASE_REF) or for main itself (GITHUB_REF), so a draft branch stays testable. -Usage: check-site-rot.py [--root DIR] [--base REF] [--removed FILE...] +Usage: check-site-rot.py [--root DIR] [--base REF] [--removed FILE...] [--live] --base REF asks git which pages were removed since REF (the workflow passes HEAD^1) and requires each to have its two redirect lines; it fails if REF cannot be found, so a shallow checkout cannot hide a removal. --removed takes page paths directly (for example "old-page/index.html"), for tests. @@ -48,6 +52,35 @@ def ids_of(text): return set(a or b for a, b in re.findall(r'(?<![\w-])id\s*=\s*(?:"([^"]+)"|\'([^\']+)\')', text, re.I)) +META_RE = re.compile(r"<meta\b[^>]*>", re.I) +ATTR_RE = re.compile(r"""([a-zA-Z-]+)\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'>]+))""") + + +def hidden_by_meta(text): + """True if a robots or googlebot meta tag in the page carries noindex (or none, which means noindex and nofollow).""" + for tag in META_RE.findall(text): + attrs = {k.lower(): html.unescape(a or b or c).strip().lower() for k, a, b, c in ATTR_RE.findall(tag)} + if attrs.get("name") in ("robots", "googlebot") and \ + {x.strip() for x in attrs.get("content", "").split(",")} & {"noindex", "none"}: + return True + return False + + +def hidden_by_headers(headers_text): + """Path patterns of the _headers blocks that send an X-Robots-Tag with noindex (or none).""" + patterns, cur, hit = [], None, False + for line in headers_text.splitlines() + ["/end-of-file"]: + if not line.strip() or line.lstrip().startswith("#"): + continue + if not line[0].isspace(): + if cur is not None and hit: + patterns.append(cur) + cur, hit = line.strip(), False + elif re.match(r"\s*x-robots-tag\s*:", line, re.I): + hit = hit or bool({x.strip().lower() for x in re.split(r"[,\s]+", line.split(":", 1)[1])} & {"noindex", "none"}) + return patterns + + # href="..." / src='...' / href=bare, any case: every way a link can be written LINK_RE = re.compile(r'(?<![\w-])(href|src)\s*=\s*(?:"([^"]*)"|\'([^\']*)\'|([^\s"\'>]+))', re.I) @@ -56,10 +89,14 @@ def main(): ap = argparse.ArgumentParser() ap.add_argument("--root", default=".") ap.add_argument("--removed", nargs="*", default=[]) + ap.add_argument("--live", action="store_true", + help="the live-site rule: no page except 404.html may be hidden from search (turned on by itself for a " + "pull request based on main and for main)") ap.add_argument("--base", help="git ref of the parent commit (for example HEAD^1): pages removed since then " "must have their two 301 lines. Fails if the ref cannot be found.") args = ap.parse_args() root = args.root + live = args.live or os.environ.get("GITHUB_BASE_REF") == "main" or os.environ.get("GITHUB_REF") == "refs/heads/main" problems = [] removed_from_git = [] if args.base: @@ -110,6 +147,17 @@ def bad(msg): texts[k] = "" # a symlink was reported above; do not read through it ids = {k: ids_of(t) for k, t in texts.items()} + # 6. hidden pages: found by a meta tag in the page or an X-Robots-Tag rule in _headers + import fnmatch + hp = os.path.join(root, "_headers") + header_patterns = hidden_by_headers(read(hp)) if os.path.isfile(hp) else [] + hidden = set() + for key, t in texts.items(): + url_path = "/" if key == "" else "/404" if key == "404.html" else "/%s/" % key + if hidden_by_meta(t) or any(fnmatch.fnmatchcase(url_path, pat) or fnmatch.fnmatchcase(url_path.rstrip("/"), pat.rstrip("/")) + for pat in header_patterns): + hidden.add(key) + # 5. titles for key, t in texts.items(): m = re.search(r"<title>(.*?)", t, re.S) @@ -208,14 +256,22 @@ def resolve(path): smp = os.path.join(root, "sitemap.xml") if os.path.isfile(smp): locs = re.findall(r"([^<]*)", read(smp)) - want = {SITE} | {SITE + p + "/" for p in pages} + want = ({SITE} if "" not in hidden else set()) | {SITE + p + "/" for p in pages if p not in hidden} got = set(locs) if len(locs) != len(got): bad("sitemap: has a duplicate ") + for key in sorted(hidden): + url = SITE if key == "" else SITE + key + "/" + if key != "404.html" and url in got: + bad("hidden: %s is hidden from search but listed in the sitemap" % (key or "/")) + got.discard(url) for u in sorted(got - want): bad("sitemap: lists %s but no such page exists" % u) for u in sorted(want - got): bad("sitemap: is missing %s" % u) + if live: + for key in sorted(hidden - {"404.html"}): + bad("live: %s is hidden from search; on the live site only the 404 page may be" % (key or "/")) lp = os.path.join(root, "llms.txt") if os.path.isfile(lp): urls = set(re.findall(r"\((https://fosterstack\.com/[^)\s]*)\)", read(lp))) diff --git a/bin/test-check-site-rot.sh b/bin/test-check-site-rot.sh index 8c8fbac..5147f4a 100755 --- a/bin/test-check-site-rot.sh +++ b/bin/test-check-site-rot.sh @@ -2,6 +2,8 @@ # Self-test for bin/check-site-rot.py: the clean site passes, and each kind of rot it promises to catch # makes it fail with the matching message. Works on a throwaway copy; touches nothing in the repo. set -u +# the check reads these to know whether it runs for the live site; the tests set them on purpose, never inherit them +unset GITHUB_BASE_REF GITHUB_REF GITHUB_EVENT_NAME cd "$(dirname "$0")/.." here=$PWD tmp=$(mktemp -d) @@ -194,4 +196,42 @@ expect_fail "a 302 line is not a 301 twin" "no matching" fresh; rm -rf "$site/$second"; ln -s "$site/$first" "$site/$second" expect_fail "a symbolic link in the site is refused" "symbolic link" +# --- hidden pages and the sitemap (owner ratified Oct 22 2026, ops/docs/ratify/2026-10-22-www-noindex-and-sitemap.md) +# Sentence 1: no page that is hidden from search may be listed in the sitemap. +# Sentence 2: on the live site no page except the 404 page may be hidden from search. It applies with --live, and the +# check turns it on by itself for a pull request whose base is main (GITHUB_BASE_REF) and for main itself (GITHUB_REF). +hide() { edit "$site/$1/index.html" "" ''; } +unlist() { edit "$site/sitemap.xml" "REGEX:\s*https://fosterstack\.com/$1/.*?" ""; } + +fresh; hide "$first" +expect_fail "a hidden page listed in the sitemap is refused" "hidden: .*$first.*sitemap" +fresh; hide "$first"; unlist "$first" +expect_pass "a hidden page left out of the sitemap passes (not live)" +fresh; hide "$first"; unlist "$first" +expect_fail "--live: a hidden page on the live site is refused" "live: .*$first.*hidden" --live +fresh; expect_pass "--live: a site with no hidden page passes" --live +fresh; edit "$site/404.html" "" '' +expect_pass "--live: the 404 page may be hidden" --live + +# every way of hiding a page is read +for tag in '' "" '' '' ''; do + fresh; edit "$site/$first/index.html" "" "$tag" + expect_fail "hidden by $tag is found" "hidden: .*$first.*sitemap" +done +fresh; edit "$site/$first/index.html" "" '' +expect_pass "a robots tag that says index, and the word in another tag, hide nothing" --live +# X-Robots-Tag in _headers hides the pages its path pattern matches +fresh; append "$site/_headers" "/$first/"; append "$site/_headers" " X-Robots-Tag: noindex" +expect_fail "an X-Robots-Tag noindex rule hides a page (and it is listed)" "hidden: .*$first.*sitemap" +fresh; append "$site/_headers" "/$first/"; append "$site/_headers" " X-Robots-Tag: noindex"; unlist "$first" +expect_fail "--live: an X-Robots-Tag noindex rule is refused" "live: .*$first.*hidden" --live +fresh; append "$site/_headers" "/*"; append "$site/_headers" " X-Robots-Tag: noindex" +expect_fail "--live: a noindex rule for every path is refused" "live: " --live + +# the check switches sentence 2 on for pull requests based on main and for main itself, and not for other bases +fresh; hide "$first"; unlist "$first" +if GITHUB_BASE_REF=main python3 "$here/bin/check-site-rot.py" --root "$site" >"$tmp/out" 2>&1; then echo "FAIL a pull request based on main may not hide a page"; fail=1; else grep -q "live: " "$tmp/out" && echo "ok a pull request based on main may not hide a page" || { echo "FAIL based-on-main message"; fail=1; }; fi +if GITHUB_REF=refs/heads/main python3 "$here/bin/check-site-rot.py" --root "$site" >"$tmp/out" 2>&1; then echo "FAIL main itself may not hide a page"; fail=1; else grep -q "live: " "$tmp/out" && echo "ok main itself may not hide a page" || { echo "FAIL main message"; fail=1; }; fi +if GITHUB_BASE_REF=www/v030-revamp GITHUB_REF=refs/pull/7/merge python3 "$here/bin/check-site-rot.py" --root "$site" >"$tmp/out" 2>&1; then echo "ok a pull request based on another branch may hide an unlisted page"; else echo "FAIL other-base case"; sed 's/^/ /' "$tmp/out"; fail=1; fi + [ "$fail" = 0 ] && echo "all site-rot self-tests passed" || { echo "site-rot self-test FAILED"; exit 1; } From 1df637561d274b158256ffa2763e52c30ddc8f67 Mon Sep 17 00:00:00 2001 From: FosterStack Admin <317177128+fosterstack-admin@users.noreply.github.com> Date: Thu, 22 Oct 2026 14:19:38 -0400 Subject: [PATCH 2/6] Hidden-page check: read meta tags with an HTML parser, support absolute-URL and placeholder header rules, named bots and value-only directives; tests for each --- bin/check-site-rot.py | 60 +++++++++++++++++++++++++++++++------- bin/test-check-site-rot.sh | 22 ++++++++++++++ 2 files changed, 71 insertions(+), 11 deletions(-) diff --git a/bin/check-site-rot.py b/bin/check-site-rot.py index a5570f7..ca8d2d8 100755 --- a/bin/check-site-rot.py +++ b/bin/check-site-rot.py @@ -22,6 +22,7 @@ """ import argparse import html +from html.parser import HTMLParser import os import re import subprocess @@ -52,16 +53,41 @@ def ids_of(text): return set(a or b for a, b in re.findall(r'(?]*>", re.I) -ATTR_RE = re.compile(r"""([a-zA-Z-]+)\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'>]+))""") +VALUE_DIRECTIVES = {"max-snippet", "max-image-preview", "max-video-preview", "unavailable_after"} + + +class _RobotsMeta(HTMLParser): + """Finds robots and googlebot meta tags the way a browser reads the page: comments, script and style text are not tags, + and a quoted '>' does not end a tag.""" + + def __init__(self): + super().__init__(convert_charrefs=True) + self.hidden = False + + def handle_starttag(self, tag, attrs): + if tag != "meta": + return + a = {k.lower(): (v or "").strip().lower() for k, v in attrs} + if a.get("name") in ("robots", "googlebot") and {x.strip() for x in a.get("content", "").split(",")} & {"noindex", "none"}: + self.hidden = True + + handle_startendtag = handle_starttag def hidden_by_meta(text): """True if a robots or googlebot meta tag in the page carries noindex (or none, which means noindex and nofollow).""" - for tag in META_RE.findall(text): - attrs = {k.lower(): html.unescape(a or b or c).strip().lower() for k, a, b, c in ATTR_RE.findall(tag)} - if attrs.get("name") in ("robots", "googlebot") and \ - {x.strip() for x in attrs.get("content", "").split(",")} & {"noindex", "none"}: + parser = _RobotsMeta() + parser.feed(text) + parser.close() + return parser.hidden + + +def header_hides(value): + """True if an X-Robots-Tag value carries noindex or none as a directive, alone or for one named bot ("googlebot: noindex"). + A directive that only has the word as its value, such as "max-image-preview: none", hides nothing.""" + for part in value.split(","): + m = re.fullmatch(r"\s*(?:([A-Za-z0-9_-]+)\s*:\s*)?(noindex|none)\s*", part, re.I) + if m and (m.group(1) or "").lower() not in VALUE_DIRECTIVES: return True return False @@ -77,10 +103,24 @@ def hidden_by_headers(headers_text): patterns.append(cur) cur, hit = line.strip(), False elif re.match(r"\s*x-robots-tag\s*:", line, re.I): - hit = hit or bool({x.strip().lower() for x in re.split(r"[,\s]+", line.split(":", 1)[1])} & {"noindex", "none"}) + hit = hit or header_hides(line.split(":", 1)[1]) return patterns +def pattern_regex(pat): + """A _headers path pattern as a regex over the URL path: * matches anything, :name matches one path segment; an absolute + URL on this site is read as its path, one on another host matches nothing here.""" + m = re.match(r"https?://([^/]+)(/.*)?$", pat) + if m: + if m.group(1) not in ("fosterstack.com", "www.fosterstack.com"): + return None + pat = m.group(2) or "/" + out = "" + for tok in re.split(r"(\*|:[A-Za-z][A-Za-z0-9_]*)", pat): + out += ".*" if tok == "*" else "[^/]+" if re.fullmatch(r":[A-Za-z][A-Za-z0-9_]*", tok) else re.escape(tok) + return re.compile(out) + + # href="..." / src='...' / href=bare, any case: every way a link can be written LINK_RE = re.compile(r'(?]+))', re.I) @@ -148,14 +188,12 @@ def bad(msg): ids = {k: ids_of(t) for k, t in texts.items()} # 6. hidden pages: found by a meta tag in the page or an X-Robots-Tag rule in _headers - import fnmatch hp = os.path.join(root, "_headers") - header_patterns = hidden_by_headers(read(hp)) if os.path.isfile(hp) else [] + header_res = [r for r in (pattern_regex(x) for x in (hidden_by_headers(read(hp)) if os.path.isfile(hp) else [])) if r] hidden = set() for key, t in texts.items(): url_path = "/" if key == "" else "/404" if key == "404.html" else "/%s/" % key - if hidden_by_meta(t) or any(fnmatch.fnmatchcase(url_path, pat) or fnmatch.fnmatchcase(url_path.rstrip("/"), pat.rstrip("/")) - for pat in header_patterns): + if hidden_by_meta(t) or any(r.fullmatch(url_path) or r.fullmatch(url_path.rstrip("/") or "/") for r in header_res): hidden.add(key) # 5. titles diff --git a/bin/test-check-site-rot.sh b/bin/test-check-site-rot.sh index 5147f4a..5082c72 100755 --- a/bin/test-check-site-rot.sh +++ b/bin/test-check-site-rot.sh @@ -228,6 +228,28 @@ expect_fail "--live: an X-Robots-Tag noindex rule is refused" "live: .*$first.*h fresh; append "$site/_headers" "/*"; append "$site/_headers" " X-Robots-Tag: noindex" expect_fail "--live: a noindex rule for every path is refused" "live: " --live +# the page is read the way a browser reads it: a quoted ">" does not end a tag; a tag in a comment or a script is not a tag +fresh; edit "$site/$first/index.html" "" '' +expect_fail "a quoted > inside a meta tag does not hide the tag" "hidden: .*$first.*sitemap" +fresh; edit "$site/$first/index.html" "" '' +expect_pass "a noindex tag in a comment or a script hides nothing" --live +# the home page can be hidden too, and is then listed in the sitemap +fresh; edit "$site/index.html" "" '' +expect_fail "a hidden home page that is listed in the sitemap is refused" "hidden: /" +# _headers: absolute URL, placeholder, a named bot, none, comments and several rules in one block, and a value that only contains the word +fresh; append "$site/_headers" "https://fosterstack.com/$first/"; append "$site/_headers" " X-Robots-Tag: noindex"; unlist "$first" +expect_fail "--live: an absolute URL in _headers is read as its path" "live: .*$first" --live +fresh; append "$site/_headers" "/:slug/"; append "$site/_headers" " X-Robots-Tag: noindex"; unlist "$first" +expect_fail "--live: a :placeholder in _headers matches a page" "live: .*$first" --live +fresh; append "$site/_headers" "/$first/"; append "$site/_headers" " # a comment"; append "$site/_headers" " Cache-Control: no-store"; append "$site/_headers" " X-Robots-Tag: googlebot: noindex"; unlist "$first" +expect_fail "--live: a noindex for one named bot, after a comment and another rule, hides the page" "live: .*$first" --live +fresh; append "$site/_headers" "/$first/"; append "$site/_headers" " X-Robots-Tag: none"; unlist "$first" +expect_fail "--live: X-Robots-Tag none hides the page" "live: .*$first" --live +fresh; append "$site/_headers" "/$first/"; append "$site/_headers" " X-Robots-Tag: max-image-preview: none"; append "$site/_headers" "/$second/"; append "$site/_headers" " X-Robots-Tag: index, max-snippet: 0" +expect_pass "--live: a value that only contains the word none hides nothing" --live +fresh; append "$site/_headers" "https://example.org/$first/"; append "$site/_headers" " X-Robots-Tag: noindex" +expect_pass "--live: a _headers rule for another host hides nothing here" --live + # the check switches sentence 2 on for pull requests based on main and for main itself, and not for other bases fresh; hide "$first"; unlist "$first" if GITHUB_BASE_REF=main python3 "$here/bin/check-site-rot.py" --root "$site" >"$tmp/out" 2>&1; then echo "FAIL a pull request based on main may not hide a page"; fail=1; else grep -q "live: " "$tmp/out" && echo "ok a pull request based on main may not hide a page" || { echo "FAIL based-on-main message"; fail=1; }; fi From 0afe56cd3c036021c06046ba8b4d48eb14311857 Mon Sep 17 00:00:00 2001 From: FosterStack Admin <317177128+fosterstack-admin@users.noreply.github.com> Date: Thu, 22 Oct 2026 14:20:21 -0400 Subject: [PATCH 3/6] Problem lines: C1 control characters are replaced too (with a test) --- bin/check-site-rot.py | 2 +- bin/test-check-site-rot.sh | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/bin/check-site-rot.py b/bin/check-site-rot.py index ca8d2d8..ca09856 100755 --- a/bin/check-site-rot.py +++ b/bin/check-site-rot.py @@ -164,7 +164,7 @@ def main(): def bad(msg): # one printable line per problem: text from a pull request must never start a line with "::" # (GitHub Actions reads such a line as a workflow command) or smuggle in a newline - problems.append(re.sub(r"[\x00-\x1f\x7f]", " ", msg)) + problems.append(re.sub(r"[\x00-\x1f\x7f-\x9f]", " ", msg)) # a symlink could point a page (or its title) at a file outside the site: refuse them for name in sorted(os.listdir(root)): diff --git a/bin/test-check-site-rot.sh b/bin/test-check-site-rot.sh index 5082c72..1d8d99a 100755 --- a/bin/test-check-site-rot.sh +++ b/bin/test-check-site-rot.sh @@ -250,6 +250,14 @@ expect_pass "--live: a value that only contains the word none hides nothing" --l fresh; append "$site/_headers" "https://example.org/$first/"; append "$site/_headers" " X-Robots-Tag: noindex" expect_pass "--live: a _headers rule for another host hides nothing here" --live +# a C1 control character (U+009B is read by some terminals as an escape) never reaches the output +fresh; python3 - "$site/$first/index.html" <<'EOF2' +import re, sys +p = sys.argv[1]; s = open(p, encoding="utf-8").read() +open(p, "w", encoding="utf-8").write(re.sub(r".*?", "" + "x" * 70 + "\u009b[31m", s, count=1, flags=re.S)) +EOF2 +python3 "$here/bin/check-site-rot.py" --root "$site" >"$tmp/out" 2>&1; python3 -c "import sys; d=open(sys.argv[1],encoding='utf-8').read(); sys.exit(0 if 'title:' in d and not any(0x80<=ord(c)<=0x9f for c in d) else 1)" "$tmp/out" && echo "ok a C1 control character in a title is not printed" || { echo "FAIL a C1 control character reached the output"; fail=1; } + # the check switches sentence 2 on for pull requests based on main and for main itself, and not for other bases fresh; hide "$first"; unlist "$first" if GITHUB_BASE_REF=main python3 "$here/bin/check-site-rot.py" --root "$site" >"$tmp/out" 2>&1; then echo "FAIL a pull request based on main may not hide a page"; fail=1; else grep -q "live: " "$tmp/out" && echo "ok a pull request based on main may not hide a page" || { echo "FAIL based-on-main message"; fail=1; }; fi From 70f5f9fa864834fd621699966d1d84f310c2f52b Mon Sep 17 00:00:00 2001 From: FosterStack Admin <317177128+fosterstack-admin@users.noreply.github.com> Date: Thu, 22 Oct 2026 14:31:46 -0400 Subject: [PATCH 4/6] Hidden-page check: effective headers (hostname placeholders, removals, file order); --base diagnostics drop C1 controls; tests --- bin/check-site-rot.py | 74 ++++++++++++++++++++++++++------------ bin/test-check-site-rot.sh | 15 ++++++++ 2 files changed, 67 insertions(+), 22 deletions(-) diff --git a/bin/check-site-rot.py b/bin/check-site-rot.py index ca09856..4b3b054 100755 --- a/bin/check-site-rot.py +++ b/bin/check-site-rot.py @@ -92,33 +92,63 @@ def header_hides(value): return False -def hidden_by_headers(headers_text): - """Path patterns of the _headers blocks that send an X-Robots-Tag with noindex (or none).""" - patterns, cur, hit = [], None, False - for line in headers_text.splitlines() + ["/end-of-file"]: +def parse_headers(headers_text): + """The _headers file as [(pattern, [(op, name, value)])], in file order: op is "set" for 'Name: value' and "detach" for '! Name'.""" + blocks, cur = [], None + for line in headers_text.splitlines(): if not line.strip() or line.lstrip().startswith("#"): continue if not line[0].isspace(): - if cur is not None and hit: - patterns.append(cur) - cur, hit = line.strip(), False - elif re.match(r"\s*x-robots-tag\s*:", line, re.I): - hit = hit or header_hides(line.split(":", 1)[1]) - return patterns + cur = (line.strip(), []) + blocks.append(cur) + elif cur is not None: + t = line.strip() + if t.startswith("!"): + cur[1].append(("detach", t[1:].strip().lower(), "")) + elif ":" in t: + name, value = t.split(":", 1) + cur[1].append(("set", name.strip().lower(), value.strip())) + return blocks + + +def _glob(text, host=False): + """* matches anything, :name matches one path segment (one label in a host); everything else is literal.""" + one = r"[^./]+" if host else r"[^/]+" + out = "" + for tok in re.split(r"(\*|:[A-Za-z][A-Za-z0-9_]*)", text): + out += ".*" if tok == "*" else one if re.fullmatch(r":[A-Za-z][A-Za-z0-9_]*", tok) else re.escape(tok) + return out + +SITE_HOSTS = ("fosterstack.com", "www.fosterstack.com") -def pattern_regex(pat): - """A _headers path pattern as a regex over the URL path: * matches anything, :name matches one path segment; an absolute - URL on this site is read as its path, one on another host matches nothing here.""" + +def pattern_matches(pat, url_path): + """Does a _headers pattern match this path on the site? A pattern with a host matches if it matches one of the site's hosts.""" m = re.match(r"https?://([^/]+)(/.*)?$", pat) if m: - if m.group(1) not in ("fosterstack.com", "www.fosterstack.com"): - return None + if not any(re.fullmatch(_glob(m.group(1), host=True), h) for h in SITE_HOSTS): + return False pat = m.group(2) or "/" - out = "" - for tok in re.split(r"(\*|:[A-Za-z][A-Za-z0-9_]*)", pat): - out += ".*" if tok == "*" else "[^/]+" if re.fullmatch(r":[A-Za-z][A-Za-z0-9_]*", tok) else re.escape(tok) - return re.compile(out) + rx = re.compile(_glob(pat)) + return bool(rx.fullmatch(url_path) or rx.fullmatch(url_path.rstrip("/") or "/")) + + +def hidden_by_headers(blocks, url_path): + """True if the X-Robots-Tag the page ends up with carries noindex or none: the matching blocks are applied in file + order, values add up, and '! X-Robots-Tag' removes what came before it.""" + values = [] + for pat, rules in blocks: + if not pattern_matches(pat, url_path): + continue + for op, name, value in rules: + if name != "x-robots-tag": + continue + if op == "detach": + values = [] + else: + values.append(value) + return any(header_hides(v) for v in values) # href="..." / src='...' / href=bare, any case: every way a link can be written @@ -141,7 +171,7 @@ def main(): removed_from_git = [] if args.base: # fail closed: a missing parent must never quietly turn into "nothing was removed" - shown = re.sub(r"[\x00-\x1f\x7f]", " ", args.base) + shown = re.sub(r"[\x00-\x1f\x7f-\x9f]", " ", args.base) if not re.fullmatch(r"[A-Za-z0-9_./~^@{}][A-Za-z0-9_./~^@{}-]*", args.base): print("FAIL base: %s is not a plain git ref" % shown) return 1 @@ -189,11 +219,11 @@ def bad(msg): # 6. hidden pages: found by a meta tag in the page or an X-Robots-Tag rule in _headers hp = os.path.join(root, "_headers") - header_res = [r for r in (pattern_regex(x) for x in (hidden_by_headers(read(hp)) if os.path.isfile(hp) else [])) if r] + header_blocks = parse_headers(read(hp)) if os.path.isfile(hp) else [] hidden = set() for key, t in texts.items(): url_path = "/" if key == "" else "/404" if key == "404.html" else "/%s/" % key - if hidden_by_meta(t) or any(r.fullmatch(url_path) or r.fullmatch(url_path.rstrip("/") or "/") for r in header_res): + if hidden_by_meta(t) or hidden_by_headers(header_blocks, url_path): hidden.add(key) # 5. titles diff --git a/bin/test-check-site-rot.sh b/bin/test-check-site-rot.sh index 1d8d99a..06cf4b5 100755 --- a/bin/test-check-site-rot.sh +++ b/bin/test-check-site-rot.sh @@ -258,6 +258,21 @@ open(p, "w", encoding="utf-8").write(re.sub(r".*?", "" + " EOF2 python3 "$here/bin/check-site-rot.py" --root "$site" >"$tmp/out" 2>&1; python3 -c "import sys; d=open(sys.argv[1],encoding='utf-8').read(); sys.exit(0 if 'title:' in d and not any(0x80<=ord(c)<=0x9f for c in d) else 1)" "$tmp/out" && echo "ok a C1 control character in a title is not printed" || { echo "FAIL a C1 control character reached the output"; fail=1; } +# a hostname placeholder matches the site's host; a removal ("! X-Robots-Tag") undoes an earlier noindex for that page +fresh; append "$site/_headers" "https://:host.com/$first/"; append "$site/_headers" " X-Robots-Tag: noindex"; unlist "$first" +expect_fail "--live: a hostname placeholder in _headers matches the site" "live: .*$first" --live +fresh; append "$site/_headers" "https://*.example.org/$first/"; append "$site/_headers" " X-Robots-Tag: noindex" +expect_pass "--live: a host pattern that cannot match the site hides nothing" --live +fresh; append "$site/_headers" "/$first/*"; append "$site/_headers" " X-Robots-Tag: noindex"; append "$site/_headers" "/$first/"; append "$site/_headers" " ! X-Robots-Tag" +expect_pass "--live: a removal undoes an earlier noindex for that page, which stays listed" --live +fresh; append "$site/_headers" "/$first/*"; append "$site/_headers" " X-Robots-Tag: noindex"; append "$site/_headers" "/$first/"; append "$site/_headers" " ! X-Robots-Tag"; unlist "$first" +expect_fail "the sitemap still has to list a page that the removal un-hides" "sitemap: is missing" +fresh; append "$site/_headers" "/$first/"; append "$site/_headers" " ! X-Robots-Tag"; append "$site/_headers" "/$first/"; append "$site/_headers" " X-Robots-Tag: noindex" +expect_fail "--live: a noindex that comes after a removal still hides the page" "live: .*$first" --live +# an invalid --base is reported without control characters of any kind +python3 "$here/bin/check-site-rot.py" --root "$site" --base "bad"$'\u009b'"ref" >"$tmp/out" 2>&1 +python3 -c "import sys; d=open(sys.argv[1],encoding='utf-8').read(); sys.exit(0 if 'FAIL base' in d and not any(0x80<=ord(c)<=0x9f for c in d) else 1)" "$tmp/out" && echo "ok a C1 control character in --base is not printed" || { echo "FAIL a C1 control character in --base reached the output"; fail=1; } + # the check switches sentence 2 on for pull requests based on main and for main itself, and not for other bases fresh; hide "$first"; unlist "$first" if GITHUB_BASE_REF=main python3 "$here/bin/check-site-rot.py" --root "$site" >"$tmp/out" 2>&1; then echo "FAIL a pull request based on main may not hide a page"; fail=1; else grep -q "live: " "$tmp/out" && echo "ok a pull request based on main may not hide a page" || { echo "FAIL based-on-main message"; fail=1; }; fi From adf8f973d76a65f6f401ac83c917cd50a3fc4d14 Mon Sep 17 00:00:00 2001 From: FosterStack Admin <317177128+fosterstack-admin@users.noreply.github.com> Date: Thu, 22 Oct 2026 14:42:20 -0400 Subject: [PATCH 5/6] Hidden-page check: effective headers per request (each host, with and without the slash); tests --- bin/check-site-rot.py | 59 +++++++++++++++++++++++--------------- bin/test-check-site-rot.sh | 8 ++++++ 2 files changed, 44 insertions(+), 23 deletions(-) diff --git a/bin/check-site-rot.py b/bin/check-site-rot.py index 4b3b054..abc998a 100755 --- a/bin/check-site-rot.py +++ b/bin/check-site-rot.py @@ -123,32 +123,46 @@ def _glob(text, host=False): SITE_HOSTS = ("fosterstack.com", "www.fosterstack.com") -def pattern_matches(pat, url_path): - """Does a _headers pattern match this path on the site? A pattern with a host matches if it matches one of the site's hosts.""" +def pattern_matches(pat, host, path): + """Does a _headers pattern match a request for this host and path? Like Cloudflare, a pattern is matched against the + exact request: a pattern with a host only matches that host, and /name/ and /name are different paths.""" m = re.match(r"https?://([^/]+)(/.*)?$", pat) if m: - if not any(re.fullmatch(_glob(m.group(1), host=True), h) for h in SITE_HOSTS): + if not re.fullmatch(_glob(m.group(1), host=True), host): return False pat = m.group(2) or "/" - rx = re.compile(_glob(pat)) - return bool(rx.fullmatch(url_path) or rx.fullmatch(url_path.rstrip("/") or "/")) - - -def hidden_by_headers(blocks, url_path): - """True if the X-Robots-Tag the page ends up with carries noindex or none: the matching blocks are applied in file - order, values add up, and '! X-Robots-Tag' removes what came before it.""" - values = [] - for pat, rules in blocks: - if not pattern_matches(pat, url_path): - continue - for op, name, value in rules: - if name != "x-robots-tag": + return bool(re.fullmatch(_glob(pat), path)) + + +def request_urls(key): + """Every request the site answers for one page: both hosts, with and without the trailing slash.""" + if key == "": + paths = ["/"] + elif key == "404.html": + paths = ["/404", "/404.html"] + else: + paths = ["/%s/" % key, "/%s" % key] + return [(h, p) for h in SITE_HOSTS for p in paths] + + +def hidden_by_headers(blocks, key): + """True if, for any request the page answers, the X-Robots-Tag it ends up with carries noindex or none: the matching + blocks are applied in file order, values add up, and '! X-Robots-Tag' removes what came before it.""" + for host, path in request_urls(key): + values = [] + for pat, rules in blocks: + if not pattern_matches(pat, host, path): continue - if op == "detach": - values = [] - else: - values.append(value) - return any(header_hides(v) for v in values) + for op, name, value in rules: + if name != "x-robots-tag": + continue + if op == "detach": + values = [] + else: + values.append(value) + if any(header_hides(v) for v in values): + return True + return False # href="..." / src='...' / href=bare, any case: every way a link can be written @@ -222,8 +236,7 @@ def bad(msg): header_blocks = parse_headers(read(hp)) if os.path.isfile(hp) else [] hidden = set() for key, t in texts.items(): - url_path = "/" if key == "" else "/404" if key == "404.html" else "/%s/" % key - if hidden_by_meta(t) or hidden_by_headers(header_blocks, url_path): + if hidden_by_meta(t) or hidden_by_headers(header_blocks, key): hidden.add(key) # 5. titles diff --git a/bin/test-check-site-rot.sh b/bin/test-check-site-rot.sh index 06cf4b5..bc44059 100755 --- a/bin/test-check-site-rot.sh +++ b/bin/test-check-site-rot.sh @@ -269,6 +269,14 @@ fresh; append "$site/_headers" "/$first/*"; append "$site/_headers" " X-Robots- expect_fail "the sitemap still has to list a page that the removal un-hides" "sitemap: is missing" fresh; append "$site/_headers" "/$first/"; append "$site/_headers" " ! X-Robots-Tag"; append "$site/_headers" "/$first/"; append "$site/_headers" " X-Robots-Tag: noindex" expect_fail "--live: a noindex that comes after a removal still hides the page" "live: .*$first" --live +# a removal only undoes a noindex on the same request: another host, or the path without the slash, is another request +fresh; append "$site/_headers" "https://fosterstack.com/$first/"; append "$site/_headers" " X-Robots-Tag: noindex"; append "$site/_headers" "https://www.fosterstack.com/$first/"; append "$site/_headers" " ! X-Robots-Tag" +expect_fail "--live: a removal for the www host does not undo a noindex for the apex host" "live: .*$first" --live +fresh; append "$site/_headers" "/$first/"; append "$site/_headers" " X-Robots-Tag: noindex"; append "$site/_headers" "/$first"; append "$site/_headers" " ! X-Robots-Tag" +expect_fail "--live: a removal for the path without the slash does not undo a noindex for the path with it" "live: .*$first" --live +fresh; append "$site/_headers" "/$first/*"; append "$site/_headers" " X-Robots-Tag: noindex"; append "$site/_headers" "/$first*"; append "$site/_headers" " ! X-Robots-Tag" +expect_pass "--live: a removal that matches both the path with and without the slash undoes the noindex for both" --live + # an invalid --base is reported without control characters of any kind python3 "$here/bin/check-site-rot.py" --root "$site" --base "bad"$'\u009b'"ref" >"$tmp/out" 2>&1 python3 -c "import sys; d=open(sys.argv[1],encoding='utf-8').read(); sys.exit(0 if 'FAIL base' in d and not any(0x80<=ord(c)<=0x9f for c in d) else 1)" "$tmp/out" && echo "ok a C1 control character in --base is not printed" || { echo "FAIL a C1 control character in --base reached the output"; fail=1; } From d5ea07ef264289bafe7104ba7cbecb5901e38d63 Mon Sep 17 00:00:00 2001 From: FosterStack Admin <317177128+fosterstack-admin@users.noreply.github.com> Date: Thu, 22 Oct 2026 14:53:08 -0400 Subject: [PATCH 6/6] Hidden-page check: header values are read after :placeholder and :splat are filled in; tests for it and for added-up values --- bin/check-site-rot.py | 46 +++++++++++++++++++++++++++++--------- bin/test-check-site-rot.sh | 10 +++++++++ 2 files changed, 46 insertions(+), 10 deletions(-) diff --git a/bin/check-site-rot.py b/bin/check-site-rot.py index abc998a..54d5d21 100755 --- a/bin/check-site-rot.py +++ b/bin/check-site-rot.py @@ -111,12 +111,22 @@ def parse_headers(headers_text): return blocks -def _glob(text, host=False): - """* matches anything, :name matches one path segment (one label in a host); everything else is literal.""" +def _glob(text, host=False, names=None): + """* matches anything (it is :splat), :name matches one path segment (one label in a host); everything else is literal. + With names (a list) every placeholder becomes a capture group and its name is appended, so values can be expanded.""" one = r"[^./]+" if host else r"[^/]+" out = "" for tok in re.split(r"(\*|:[A-Za-z][A-Za-z0-9_]*)", text): - out += ".*" if tok == "*" else one if re.fullmatch(r":[A-Za-z][A-Za-z0-9_]*", tok) else re.escape(tok) + if tok == "*": + out += "(.*)" if names is not None else ".*" + if names is not None: + names.append("splat") + elif re.fullmatch(r":[A-Za-z][A-Za-z0-9_]*", tok): + out += "(%s)" % one if names is not None else one + if names is not None: + names.append(tok[1:]) + else: + out += re.escape(tok) return out @@ -124,14 +134,29 @@ def _glob(text, host=False): def pattern_matches(pat, host, path): - """Does a _headers pattern match a request for this host and path? Like Cloudflare, a pattern is matched against the - exact request: a pattern with a host only matches that host, and /name/ and /name are different paths.""" + """Match a _headers pattern against a request for this host and path. Like Cloudflare, the pattern is matched against the + exact request: a pattern with a host only matches that host, and /name/ and /name are different paths. Returns None, or + the placeholders it captured ({"splat": ..., "name": ...}) so header values can be expanded.""" + found = {} m = re.match(r"https?://([^/]+)(/.*)?$", pat) if m: - if not re.fullmatch(_glob(m.group(1), host=True), host): - return False + names = [] + hm = re.fullmatch(_glob(m.group(1), host=True, names=names), host) + if not hm: + return None + found.update(zip(names, hm.groups())) pat = m.group(2) or "/" - return bool(re.fullmatch(_glob(pat), path)) + names = [] + pm = re.fullmatch(_glob(pat, names=names), path) + if not pm: + return None + found.update(zip(names, pm.groups())) + return found + + +def expand(value, found): + """Replace :name and :splat in a header value with what the pattern captured (an unknown :word stays as written).""" + return re.sub(r":([A-Za-z][A-Za-z0-9_]*)", lambda m: found.get(m.group(1), m.group(0)), value) def request_urls(key): @@ -151,7 +176,8 @@ def hidden_by_headers(blocks, key): for host, path in request_urls(key): values = [] for pat, rules in blocks: - if not pattern_matches(pat, host, path): + found = pattern_matches(pat, host, path) + if found is None: continue for op, name, value in rules: if name != "x-robots-tag": @@ -159,7 +185,7 @@ def hidden_by_headers(blocks, key): if op == "detach": values = [] else: - values.append(value) + values.append(expand(value, found)) if any(header_hides(v) for v in values): return True return False diff --git a/bin/test-check-site-rot.sh b/bin/test-check-site-rot.sh index bc44059..75e8d40 100755 --- a/bin/test-check-site-rot.sh +++ b/bin/test-check-site-rot.sh @@ -277,6 +277,16 @@ expect_fail "--live: a removal for the path without the slash does not undo a no fresh; append "$site/_headers" "/$first/*"; append "$site/_headers" " X-Robots-Tag: noindex"; append "$site/_headers" "/$first*"; append "$site/_headers" " ! X-Robots-Tag" expect_pass "--live: a removal that matches both the path with and without the slash undoes the noindex for both" --live +# values add up, and a header value is read after its placeholders are filled in +fresh; append "$site/_headers" "/$first/"; append "$site/_headers" " X-Robots-Tag: noindex"; append "$site/_headers" " X-Robots-Tag: index" +expect_fail "a later index does not undo an earlier noindex on the same page" "hidden: .*$first.*sitemap" +fresh; append "$site/_headers" "/:directive/"; append "$site/_headers" " X-Robots-Tag: :directive"; unlist "$first" +mkdir "$site/none"; sed 's#<title>[^<]*#None page#' "$site/$first/index.html" >"$site/none/index.html" +expect_fail "--live: a :placeholder filled into the header value (a page called none) hides that page" "live: none" --live +fresh; append "$site/_headers" "/*"; append "$site/_headers" " X-Robots-Tag: :splat" +mkdir "$site/noindex"; sed 's#[^<]*#Noindex page#' "$site/$first/index.html" >"$site/noindex/index.html" +expect_fail "--live: :splat filled into the header value hides a page whose path is noindex" "live: noindex" --live + # an invalid --base is reported without control characters of any kind python3 "$here/bin/check-site-rot.py" --root "$site" --base "bad"$'\u009b'"ref" >"$tmp/out" 2>&1 python3 -c "import sys; d=open(sys.argv[1],encoding='utf-8').read(); sys.exit(0 if 'FAIL base' in d and not any(0x80<=ord(c)<=0x9f for c in d) else 1)" "$tmp/out" && echo "ok a C1 control character in --base is not printed" || { echo "FAIL a C1 control character in --base reached the output"; fail=1; }