feat(seo): real bot content for the five doc pages

CHANGELOG.md

@@ -135,6 +135,14 @@ aggregate instead: an italic *Catalog* line at the end of the version section an
   honest note that prerendered page HTML is gated on a crawler user agent while these URLs are
   not. `anyplot.ai/llms-full.txt` is now proxied to the API's generated catalogue index instead
   of soft-404ing to the homepage shell, and the daily monitor asserts it (#10488).
+- **The five doc pages carry real content for crawlers** — the bot renderings of
+  `/libraries`, `/stats`, `/about`, `/legal` and `/palette` were 29–59-word title+description
+  stubs, so a crawler following llms.txt's own link descriptions found none of the promised
+  content (AI-access audit 2026-08-19). They now serve the full library registry (derived from
+  `core/constants.py`), the live catalogue counts (reusing the cached `/stats` data), the
+  pipeline story, the operator/privacy/transparency facts, and the actual palette hex values
+  (straight from `core/palette.py`) — each derived from its single source of truth so the pages
+  cannot drift (#10491).
 
 ## [3.1.0] — 2026-08-19 — Legible to machines
 

api/routers/seo.py

@@ -12,6 +12,9 @@
 
 from api.cache import cache_key, get_cache, get_or_set_cache, set_cache
 from api.dependencies import optional_db
+from api.routers.stats import _compute_stats, _refresh_stats
+from api.schemas import StatsResponse
+from core import palette
 from core.config import settings
 from core.constants import LANGUAGES_METADATA, LIBRARIES_METADATA
 from core.database import ImplRepository, SpecRepository
@@ -886,28 +889,78 @@ async def seo_specs(db: AsyncSession | None = Depends(optional_db)):
     )
 
 
+def _build_libraries_body() -> str:
+    """Real bot body for /libraries, derived from the canonical registry.
+
+    The page was a 29-word stub even though its entire content already sits in
+    core/constants.py (AI-access audit 2026-08-19) — a crawler following
+    llms.txt's "the fifteen supported plotting libraries" link found no such
+    list. Registry-derived, so it can never drift from the actual catalog.
+    """
+    sections = [
+        f"<h1>{_LIBRARY_COUNT} plotting libraries across {len(LANGUAGES_METADATA)} languages</h1>",
+        "<p>Every plot specification on anyplot.ai is implemented once per library below. "
+        'Browse renders per library in the <a href="https://anyplot.ai/plots">gallery</a>.</p>',
+    ]
+    for lang in LANGUAGES_METADATA:
+        libs = [lib for lib in LIBRARIES_METADATA if lib["language_id"] == lang["id"]]
+        if not libs:
+            continue
+        items = "".join(
+            f'<li><a href="{html.escape(str(lib["documentation_url"]), quote=True)}">'
+            f"{html.escape(str(lib['name']))}</a> {html.escape(str(lib['version']))} — "
+            f"{html.escape(str(lib['description']))}</li>"
+            for lib in libs
+        )
+        sections.append(f"<h2>{html.escape(str(lang['name']))}</h2><ul>{items}</ul>")
+    return "".join(sections)
+
+
+_LIBRARIES_BOT_BODY = _build_libraries_body()
+
+
 @router.get("/seo-proxy/libraries")
 async def seo_libraries():
-    """Bot-optimized libraries page with correct og:tags."""
+    """Bot-optimized libraries page: the full registry, not just og:tags."""
     return HTMLResponse(
         _render_bot_html(
             title="libraries | anyplot.ai",
             description="All supported plotting libraries across languages.",
             image=DEFAULT_PLOTS_IMAGE,
             url="https://anyplot.ai/libraries",
+            body=_LIBRARIES_BOT_BODY,
         )
     )
 
 
+# Mirrors the load-bearing facts of app/src/pages/LegalPage.tsx — keep the two
+# in sync when operator, analytics, or hosting details change.
+_LEGAL_BOT_BODY = (
+    "<h1>Legal notice</h1>"
+    "<p>Operator: Markus Neusinger "
+    '(<a href="https://github.com/MarkusNeusinger">GitHub</a>, '
+    '<a href="https://x.com/MarkusNeusinger">X</a>). Full legal notice with '
+    'contact details on the <a href="https://anyplot.ai/legal">interactive page</a>.</p>'
+    "<h2>Privacy</h2>"
+    "<p>Analytics: Plausible Analytics (EU, proxied) — no cookies, no personal data collected. "
+    "Hosting: Google Cloud Run (Netherlands).</p>"
+    "<h2>Transparency</h2>"
+    "<p>The whole stack — specs, pipeline, API and frontend — is open source at "
+    '<a href="https://github.com/MarkusNeusinger/anyplot">github.com/MarkusNeusinger/anyplot</a>; '
+    "the catalogue content is MIT-licensed.</p>"
+)
+
+
 @router.get("/seo-proxy/legal")
 async def seo_legal():
-    """Bot-optimized legal page with correct og:tags."""
+    """Bot-optimized legal page: operator, privacy and transparency facts."""
     return HTMLResponse(
         _render_bot_html(
             title="Legal | anyplot.ai",
             description="Legal notice, privacy policy, and transparency information for anyplot.ai",
             image=DEFAULT_HOME_IMAGE,
             url="https://anyplot.ai/legal",
+            body=_LEGAL_BOT_BODY,
         )
     )
 
@@ -959,28 +1012,80 @@ async def seo_mcp():
     )
 
 
+# Mirrors app/src/pages/AboutPage.tsx — the pipeline story is the page's
+# content; counts derive from the registry so they cannot drift.
+_ABOUT_BOT_BODY = (
+    "<h1>About anyplot</h1>"
+    f"<p>anyplot.ai is a catalogue of plotting examples across {_LIBRARY_COUNT} libraries "
+    f"in {_LANGUAGE_LIST}. Plot ideas come from humans; AI drafts the specification, "
+    "generates code for every library, and reviews each implementation. Humans approve "
+    "specs and tune the rules.</p>"
+    "<h2>Pipeline</h2>"
+    "<p>idea → spec (AI-drafted, human-approved) → code (AI-generated per library) → "
+    "review (AI-evaluated). When a library ships a new release the pipeline re-runs; when a "
+    "better example pattern emerges the spec is updated and every library regenerates. "
+    "Generated code is never patched by hand.</p>"
+    "<h2>Open</h2>"
+    "<p>Source, specs and pipeline live at "
+    '<a href="https://github.com/MarkusNeusinger/anyplot">github.com/MarkusNeusinger/anyplot</a> '
+    '(MIT). Machine access is documented in <a href="https://anyplot.ai/llms.txt">llms.txt</a>; '
+    "propose a new plot type via a "
+    '<a href="https://github.com/MarkusNeusinger/anyplot/issues/new?labels=spec-request">spec request</a>.</p>'
+)
+
+
 @router.get("/seo-proxy/about")
 async def seo_about():
-    """Bot-optimized about page with correct og:tags."""
+    """Bot-optimized about page: the pipeline story, not just og:tags."""
     return HTMLResponse(
         _render_bot_html(
             title="About | anyplot.ai",
             description="About anyplot.ai — library-agnostic, AI-powered plotting.",
             image=DEFAULT_HOME_IMAGE,
             url="https://anyplot.ai/about",
+            body=_ABOUT_BOT_BODY,
         )
     )
 
 
+# Hex values come straight from core/palette.py — the single source of truth —
+# so the bot page can never disagree with what the plots actually use.
+_PALETTE_SLOT_NAMES = ("green", "lavender", "blue", "ochre", "red", "cyan", "rose", "lime")
+_PALETTE_BOT_BODY = (
+    "<h1>imprint — the anyplot palette</h1>"
+    "<p>A colorblind-safe categorical palette of 8 hues plus 3 semantic anchors, tuned for "
+    "warm-paper rendering and validated against deuteranopia, protanopia and tritanopia. "
+    "Every plot in the catalogue uses it.</p>"
+    "<h2>Categorical hues (slot order)</h2>"
+    "<ol>"
+    + "".join(
+        f"<li>{name} <code>{hex_value}</code></li>"
+        for name, hex_value in zip(_PALETTE_SLOT_NAMES, palette.IMPRINT, strict=True)
+    )
+    + "</ol>"
+    "<h2>Semantic anchors (outside the categorical pool)</h2>"
+    "<ul>"
+    f"<li>amber <code>{palette.AMBER}</code> — warning / caution</li>"
+    f"<li>neutral — theme-adaptive ink: <code>{palette.neutral_for('light')}</code> light / "
+    f"<code>{palette.neutral_for('dark')}</code> dark</li>"
+    f"<li>muted — theme-adaptive soft ink: <code>{palette.muted_for('light')}</code> light / "
+    f"<code>{palette.muted_for('dark')}</code> dark</li>"
+    "</ul>"
+    "<p>Copy-paste snippets for Python, R, Julia and JavaScript are on the "
+    '<a href="https://anyplot.ai/palette">interactive page</a>.</p>'
+)
+
+
 @router.get("/seo-proxy/palette")
 async def seo_palette():
-    """Bot-optimized palette page with correct og:tags."""
+    """Bot-optimized palette page: the actual hex values, not just og:tags."""
     return HTMLResponse(
         _render_bot_html(
             title="imprint palette | anyplot.ai",
             description="Imprint — a colorblind-safe categorical palette of 8 hues plus 3 semantic anchors (amber, neutral, muted). Tuned for warm-paper rendering, validated against deuteranopia / protanopia / tritanopia. The palette every plot on anyplot.ai uses.",
             image=DEFAULT_HOME_IMAGE,
             url="https://anyplot.ai/palette",
+            body=_PALETTE_BOT_BODY,
         )
     )
 
@@ -1002,14 +1107,45 @@ async def seo_map():
 
 
 @router.get("/seo-proxy/stats")
-async def seo_stats():
-    """Bot-optimized stats page with correct og:tags."""
+async def seo_stats(db: AsyncSession | None = Depends(optional_db)):
+    """Bot-optimized stats page: the live catalogue counts, not just og:tags.
+
+    Uses the same cache key the /stats endpoint populates (and the startup
+    prewarm fills), so this adds no extra DB load. Without a DB the body
+    degrades to the registry-derived counts.
+    """
+    if db is not None:
+
+        async def _fetch() -> StatsResponse:
+            return await _compute_stats(db)
+
+        stats = await get_or_set_cache(
+            cache_key("stats"), _fetch, refresh_after=settings.cache_refresh_after, refresh_factory=_refresh_stats
+        )
+        body = (
+            "<h1>anyplot in numbers</h1>"
+            "<ul>"
+            f"<li>{stats.specs} plot specifications</li>"
+            f"<li>{stats.plots} rendered implementations</li>"
+            f"<li>{stats.libraries} libraries across {stats.languages} languages</li>"
+            "</ul>"
+        )
+    else:
+        body = (
+            f"<h1>anyplot in numbers</h1><p>{_LIBRARY_COUNT} libraries across {len(LANGUAGES_METADATA)} languages.</p>"
+        )
+    body += (
+        '<p>Live machine-readable counts: <a href="https://api.anyplot.ai/stats">api.anyplot.ai/stats</a>; '
+        "per-library quality scores and coverage on the "
+        '<a href="https://anyplot.ai/stats">interactive page</a>.</p>'
+    )
     return HTMLResponse(
         _render_bot_html(
             title="Stats | anyplot.ai",
             description="Platform statistics: library scores, coverage, tags, and top implementations.",
             image=DEFAULT_HOME_IMAGE,
             url="https://anyplot.ai/stats",
+            body=body,
         )
     )
 

tests/unit/api/test_routers.py

@@ -1048,12 +1048,36 @@ def test_seo_spec_implementation_fallback_image(self, db_client, mock_spec) -> N
             assert "<pre>" not in response.text
 
     def test_seo_about(self, client: TestClient) -> None:
-        """SEO about page should return HTML with og:tags."""
+        """SEO about page must carry the pipeline story, not just og:tags."""
         response = client.get("/seo-proxy/about")
         assert response.status_code == 200
         assert "text/html" in response.headers["content-type"]
         assert "og:title" in response.text
         assert "https://anyplot.ai/about" in response.text
+        # registry-derived count + the pipeline sequence
+        assert "15 libraries" in response.text
+        assert "AI-drafted, human-approved" in response.text
+        assert "never patched by hand" in response.text
+
+    def test_seo_libraries_lists_the_registry(self, client: TestClient) -> None:
+        """The /libraries bot page was a 29-word stub although its whole
+        content sits in core/constants.py (AI-access audit 2026-08-19)."""
+        response = client.get("/seo-proxy/libraries")
+        assert response.status_code == 200
+        # one section per language, every library named with its version
+        for heading in ("<h2>Python</h2>", "<h2>R</h2>", "<h2>Julia</h2>", "<h2>JavaScript</h2>"):
+            assert heading in response.text
+        for lib in ("Matplotlib", "ggplot2", "Makie", "ECharts", "MUI X Charts"):
+            assert lib in response.text
+        assert "https://anyplot.ai/plots" in response.text
+
+    def test_seo_legal_names_operator_and_privacy_facts(self, client: TestClient) -> None:
+        response = client.get("/seo-proxy/legal")
+        assert response.status_code == 200
+        assert "Markus Neusinger" in response.text
+        assert "Plausible Analytics" in response.text
+        assert "no cookies" in response.text
+        assert "Google Cloud Run" in response.text
 
     def test_seo_mcp_tells_agents_how_to_connect(self, client: TestClient) -> None:
         """The /mcp page's audience is AI agents — the bot body must carry the
@@ -1073,12 +1097,24 @@ def test_seo_mcp_tells_agents_how_to_connect(self, client: TestClient) -> None:
             assert tool in response.text
 
     def test_seo_palette(self, client: TestClient) -> None:
-        """SEO palette page should return HTML with og:tags."""
+        """SEO palette page must carry the actual hex values from core/palette.py."""
         response = client.get("/seo-proxy/palette")
         assert response.status_code == 200
         assert "text/html" in response.headers["content-type"]
         assert "og:title" in response.text
         assert "https://anyplot.ai/palette" in response.text
+        # all 8 categorical hues plus every semantic anchor, straight from the module
+        from core import palette
+
+        anchors = [
+            palette.AMBER,
+            palette.neutral_for("light"),
+            palette.neutral_for("dark"),
+            palette.muted_for("light"),
+            palette.muted_for("dark"),
+        ]
+        for hex_value in [*palette.IMPRINT, *anchors]:
+            assert hex_value in response.text
 
     def test_seo_map(self, client: TestClient) -> None:
         """SEO map page should return HTML with og:tags."""
@@ -1088,13 +1124,31 @@ def test_seo_map(self, client: TestClient) -> None:
         assert "og:title" in response.text
         assert "https://anyplot.ai/map" in response.text
 
-    def test_seo_stats(self, client: TestClient) -> None:
-        """SEO stats page should return HTML with og:tags."""
-        response = client.get("/seo-proxy/stats")
+    def test_seo_stats_without_db_degrades_to_registry_counts(self, client: TestClient) -> None:
+        """Without a DB the stats body still carries the registry-derived counts."""
+        with patch(DB_CONFIG_PATCH, return_value=False):
+            response = client.get("/seo-proxy/stats")
         assert response.status_code == 200
-        assert "text/html" in response.headers["content-type"]
         assert "og:title" in response.text
-        assert "https://anyplot.ai/stats" in response.text
+        assert "anyplot in numbers" in response.text
+        assert "15 libraries across 4 languages" in response.text
+        assert "https://api.anyplot.ai/stats" in response.text
+
+    def test_seo_stats_with_db_shows_live_counts(self, db_client) -> None:
+        """With a DB the body carries the cached live counts."""
+        from api.schemas import StatsResponse
+
+        client, _ = db_client
+
+        async def _cached(key, factory, **kwargs):
+            return StatsResponse(specs=324, plots=3583, libraries=15, languages=4)
+
+        with patch("api.routers.seo.get_or_set_cache", side_effect=_cached):
+            response = client.get("/seo-proxy/stats")
+        assert response.status_code == 200
+        assert "324 plot specifications" in response.text
+        assert "3583 rendered implementations" in response.text
+        assert "15 libraries across 4 languages" in response.text
 
 
 class TestOgImagesRouter: