diff --git a/.github/workflows/bot-serving-check.yml b/.github/workflows/bot-serving-check.yml index 4eff940ee3..9b1c1d8973 100644 --- a/.github/workflows/bot-serving-check.yml +++ b/.github/workflows/bot-serving-check.yml @@ -28,11 +28,12 @@ permissions: jobs: bot-serving: runs-on: ubuntu-latest - # 19 checks x (--retry 2 -> up to 3 attempts x --max-time 30) can reach - # ~28.5 min worst-case; 32 leaves room to report a clean failure rather - # than dying to the job timeout, which reports nothing useful. Recompute - # this when adding checks: the ceiling is checks x 90s, plus margin. - timeout-minutes: 32 + # 22 check() calls x (--retry 2 -> up to 3 attempts x --max-time 30) can + # reach ~33 min worst-case, plus one non-retried trailing-slash probe + # (30s); 38 leaves room to report a clean failure rather than dying to the + # job timeout, which reports nothing useful. Recompute this when adding + # checks: the ceiling is check() calls x 90s, plus margin. + timeout-minutes: 38 steps: - name: Crawler UAs must get 200 + per-route titles run: | @@ -96,7 +97,10 @@ jobs: "Mozilla/5.0 (compatible; MistralAI-Index/1.0; +https://docs.mistral.ai/robots)" \ "DuckAssistBot/1.2; (+http://duckduckgo.com/duckassistbot.html)" \ "Mozilla/5.0 (compatible; Amzn-SearchBot/1.0)" \ - "Mozilla/5.0 (compatible; Amzn-User/1.0)" + "Mozilla/5.0 (compatible; Amzn-User/1.0)" \ + "Mozilla/5.0 (compatible; Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)" \ + "meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)" \ + "Grok/1.0" do check "$ua" "$ORIGIN/scatter-basic" "Basic Scatter Plot | anyplot.ai" done @@ -107,6 +111,11 @@ jobs: check "$GOOGLEBOT" "$ORIGIN/llms.txt" "# anyplot" check "$CHATGPTUSER" "$ORIGIN/llms.txt" "# anyplot" + # llms-full.txt is proxied to the API for EVERY client (mapped or + # not) — before, the SPA catch-all soft-404'd it with the homepage + # shell. The catalogue-index line proves the API generated it. + check "$HUMAN" "$ORIGIN/llms-full.txt" "# anyplot — full catalogue index" + # A trailing slash must normalise to the canonical URL on THIS host. # It used to 307 to http://api.anyplot.ai/seo-proxy/... — internal # path, wrong host, plain http, and that host disallows all crawling. diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f6f8362de..5151cc5d18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,27 @@ aggregate instead: an italic *Catalog* line at the end of the version section an project version, injected by Vite from the `[project]` version in `pyproject.toml` — the same field the release flow bumps — instead of a literal that nobody would think to update. A test asserts the injected value matches `pyproject.toml`, so the two cannot drift (#10485). +- **The crawler allowlist no longer misses documented crawlers — or Grok** — nginx's UA map + lacked `Amazonbot` and `meta-externalagent` (both documented crawler tokens; the existing + `amzn-*` / `meta-externalfetcher` entries never matched them) and matched xAI only via + `grokbot`/`xai-grok`/`grok-deepsearch`, so a bare `Grok` UA fell through to the empty SPA + shell — the exact "insufficient relevant content" a user's Grok session reported. The map now + matches `~*grok` (subsuming all three), Amazonbot, meta-externalagent, Diffbot and Firecrawl, + and the daily bot-serving monitor covers the new tokens (#10488). +- **JS-less clients no longer see a completely empty page** — the SPA shell was + `
` with no `