diff --git a/.github/workflows/bot-serving-check.yml b/.github/workflows/bot-serving-check.yml
index 4eff940ee3..9b1c1d8973 100644
--- a/.github/workflows/bot-serving-check.yml
+++ b/.github/workflows/bot-serving-check.yml
@@ -28,11 +28,12 @@ permissions:
jobs:
bot-serving:
runs-on: ubuntu-latest
- # 19 checks x (--retry 2 -> up to 3 attempts x --max-time 30) can reach
- # ~28.5 min worst-case; 32 leaves room to report a clean failure rather
- # than dying to the job timeout, which reports nothing useful. Recompute
- # this when adding checks: the ceiling is checks x 90s, plus margin.
- timeout-minutes: 32
+ # 22 check() calls x (--retry 2 -> up to 3 attempts x --max-time 30) can
+ # reach ~33 min worst-case, plus one non-retried trailing-slash probe
+ # (30s); 38 leaves room to report a clean failure rather than dying to the
+ # job timeout, which reports nothing useful. Recompute this when adding
+ # checks: the ceiling is check() calls x 90s, plus margin.
+ timeout-minutes: 38
steps:
- name: Crawler UAs must get 200 + per-route titles
run: |
@@ -96,7 +97,10 @@ jobs:
"Mozilla/5.0 (compatible; MistralAI-Index/1.0; +https://docs.mistral.ai/robots)" \
"DuckAssistBot/1.2; (+http://duckduckgo.com/duckassistbot.html)" \
"Mozilla/5.0 (compatible; Amzn-SearchBot/1.0)" \
- "Mozilla/5.0 (compatible; Amzn-User/1.0)"
+ "Mozilla/5.0 (compatible; Amzn-User/1.0)" \
+ "Mozilla/5.0 (compatible; Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)" \
+ "meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)" \
+ "Grok/1.0"
do
check "$ua" "$ORIGIN/scatter-basic" "
Basic Scatter Plot | anyplot.ai"
done
@@ -107,6 +111,11 @@ jobs:
check "$GOOGLEBOT" "$ORIGIN/llms.txt" "# anyplot"
check "$CHATGPTUSER" "$ORIGIN/llms.txt" "# anyplot"
+ # llms-full.txt is proxied to the API for EVERY client (mapped or
+ # not) — before, the SPA catch-all soft-404'd it with the homepage
+ # shell. The catalogue-index line proves the API generated it.
+ check "$HUMAN" "$ORIGIN/llms-full.txt" "# anyplot — full catalogue index"
+
# A trailing slash must normalise to the canonical URL on THIS host.
# It used to 307 to http://api.anyplot.ai/seo-proxy/... — internal
# path, wrong host, plain http, and that host disallows all crawling.
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5f6f8362de..5151cc5d18 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -53,6 +53,27 @@ aggregate instead: an italic *Catalog* line at the end of the version section an
project version, injected by Vite from the `[project]` version in `pyproject.toml` — the same
field the release flow bumps — instead of a literal that nobody would think to update. A test
asserts the injected value matches `pyproject.toml`, so the two cannot drift (#10485).
+- **The crawler allowlist no longer misses documented crawlers — or Grok** — nginx's UA map
+ lacked `Amazonbot` and `meta-externalagent` (both documented crawler tokens; the existing
+ `amzn-*` / `meta-externalfetcher` entries never matched them) and matched xAI only via
+ `grokbot`/`xai-grok`/`grok-deepsearch`, so a bare `Grok` UA fell through to the empty SPA
+ shell — the exact "insufficient relevant content" a user's Grok session reported. The map now
+ matches `~*grok` (subsuming all three), Amazonbot, meta-externalagent, Diffbot and Firecrawl,
+ and the daily bot-serving monitor covers the new tokens (#10488).
+- **JS-less clients no longer see a completely empty page** — the SPA shell was
+ `