From ee22fbf2a1caa754e1e733553c0e5769e4b17568 Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Wed, 19 Aug 2026 22:02:16 +0200 Subject: [PATCH 1/3] feat(seo): real bot content for the five doc pages /libraries, /stats, /about, /legal and /palette rendered for crawlers as 29-59-word title+description stubs, so a crawler following llms.txt's own link descriptions found none of the promised content (AI-access audit 2026-08-19, machine-files verifier). Each page now derives its body from its single source of truth: - /libraries: the full registry from core/constants.py, grouped by language, with versions, descriptions and doc links - /stats: the live catalogue counts via the same cached /stats data the startup prewarm fills (registry-derived fallback without a DB) - /about: the pipeline story, mirroring AboutPage.tsx, with registry-derived counts - /legal: operator, privacy (Plausible, no cookies) and hosting facts, mirroring LegalPage.tsx - /palette: all 8 categorical hexes plus the amber/neutral/muted anchors, straight from core/palette.py Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 8 ++ api/routers/seo.py | 147 +++++++++++++++++++++++++++++++-- tests/unit/api/test_routers.py | 61 ++++++++++++-- 3 files changed, 203 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index af376ad6fc..f0ef0475c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -135,6 +135,14 @@ aggregate instead: an italic *Catalog* line at the end of the version section an honest note that prerendered page HTML is gated on a crawler user agent while these URLs are not. `anyplot.ai/llms-full.txt` is now proxied to the API's generated catalogue index instead of soft-404ing to the homepage shell, and the daily monitor asserts it (#10488). +- **The five doc pages carry real content for crawlers** — the bot renderings of + `/libraries`, `/stats`, `/about`, `/legal` and `/palette` were 29–59-word title+description + stubs, so a crawler following llms.txt's own link descriptions found none of the promised + content (AI-access audit 2026-08-19). They now serve the full library registry (derived from + `core/constants.py`), the live catalogue counts (reusing the cached `/stats` data), the + pipeline story, the operator/privacy/transparency facts, and the actual palette hex values + (straight from `core/palette.py`) — each derived from its single source of truth so the pages + cannot drift. ## [3.1.0] — 2026-08-19 — Legible to machines diff --git a/api/routers/seo.py b/api/routers/seo.py index efc0f887ed..ba0fff2c5c 100644 --- a/api/routers/seo.py +++ b/api/routers/seo.py @@ -12,6 +12,8 @@ from api.cache import cache_key, get_cache, get_or_set_cache, set_cache from api.dependencies import optional_db +from api.routers.stats import _compute_stats, _refresh_stats +from core import palette from core.config import settings from core.constants import LANGUAGES_METADATA, LIBRARIES_METADATA from core.database import ImplRepository, SpecRepository @@ -886,28 +888,78 @@ async def seo_specs(db: AsyncSession | None = Depends(optional_db)): ) +def _build_libraries_body() -> str: + """Real bot body for /libraries, derived from the canonical registry. + + The page was a 29-word stub even though its entire content already sits in + core/constants.py (AI-access audit 2026-08-19) — a crawler following + llms.txt's "the fifteen supported plotting libraries" link found no such + list. Registry-derived, so it can never drift from the actual catalog. + """ + sections = [ + f"

{_LIBRARY_COUNT} plotting libraries across {len(LANGUAGES_METADATA)} languages

", + "

Every plot specification on anyplot.ai is implemented once per library below. " + 'Browse renders per library in the gallery.

', + ] + for lang in LANGUAGES_METADATA: + libs = [lib for lib in LIBRARIES_METADATA if lib["language_id"] == lang["id"]] + if not libs: + continue + items = "".join( + f'
  • ' + f"{html.escape(str(lib['name']))} {html.escape(str(lib['version']))} — " + f"{html.escape(str(lib['description']))}
    • " + for lib in libs + ) + sections.append(f"

    {html.escape(str(lang['name']))}

    ") + return "".join(sections) + + +_LIBRARIES_BOT_BODY = _build_libraries_body() + + @router.get("/seo-proxy/libraries") async def seo_libraries(): - """Bot-optimized libraries page with correct og:tags.""" + """Bot-optimized libraries page: the full registry, not just og:tags.""" return HTMLResponse( _render_bot_html( title="libraries | anyplot.ai", description="All supported plotting libraries across languages.", image=DEFAULT_PLOTS_IMAGE, url="https://anyplot.ai/libraries", + body=_LIBRARIES_BOT_BODY, ) ) +# Mirrors the load-bearing facts of app/src/pages/LegalPage.tsx — keep the two +# in sync when operator, analytics, or hosting details change. +_LEGAL_BOT_BODY = ( + "

    Legal notice

    " + "

    Operator: Markus Neusinger " + '(GitHub, ' + 'X). Full legal notice with ' + 'contact details on the interactive page.

    ' + "

    Privacy

    " + "

    Analytics: Plausible Analytics (EU, proxied) — no cookies, no personal data collected. " + "Hosting: Google Cloud Run (Netherlands).

    " + "

    Transparency

    " + "

    The whole stack — specs, pipeline, API and frontend — is open source at " + 'github.com/MarkusNeusinger/anyplot; ' + "the catalogue content is MIT-licensed.

    " +) + + @router.get("/seo-proxy/legal") async def seo_legal(): - """Bot-optimized legal page with correct og:tags.""" + """Bot-optimized legal page: operator, privacy and transparency facts.""" return HTMLResponse( _render_bot_html( title="Legal | anyplot.ai", description="Legal notice, privacy policy, and transparency information for anyplot.ai", image=DEFAULT_HOME_IMAGE, url="https://anyplot.ai/legal", + body=_LEGAL_BOT_BODY, ) ) @@ -959,28 +1011,80 @@ async def seo_mcp(): ) +# Mirrors app/src/pages/AboutPage.tsx — the pipeline story is the page's +# content; counts derive from the registry so they cannot drift. +_ABOUT_BOT_BODY = ( + "

    About anyplot

    " + f"

    anyplot.ai is a catalogue of plotting examples across {_LIBRARY_COUNT} libraries " + f"in {_LANGUAGE_LIST}. Plot ideas come from humans; AI drafts the specification, " + "generates code for every library, and reviews each implementation. Humans approve " + "specs and tune the rules.

    " + "

    Pipeline

    " + "

    idea → spec (AI-drafted, human-approved) → code (AI-generated per library) → " + "review (AI-evaluated). When a library ships a new release the pipeline re-runs; when a " + "better example pattern emerges the spec is updated and every library regenerates. " + "Generated code is never patched by hand.

    " + "

    Open

    " + "

    Source, specs and pipeline live at " + 'github.com/MarkusNeusinger/anyplot ' + '(MIT). Machine access is documented in llms.txt; ' + "propose a new plot type via a " + 'spec request.

    ' +) + + @router.get("/seo-proxy/about") async def seo_about(): - """Bot-optimized about page with correct og:tags.""" + """Bot-optimized about page: the pipeline story, not just og:tags.""" return HTMLResponse( _render_bot_html( title="About | anyplot.ai", description="About anyplot.ai — library-agnostic, AI-powered plotting.", image=DEFAULT_HOME_IMAGE, url="https://anyplot.ai/about", + body=_ABOUT_BOT_BODY, ) ) +# Hex values come straight from core/palette.py — the single source of truth — +# so the bot page can never disagree with what the plots actually use. +_PALETTE_SLOT_NAMES = ("green", "lavender", "blue", "ochre", "red", "cyan", "rose", "lime") +_PALETTE_BOT_BODY = ( + "

    imprint — the anyplot palette

    " + "

    A colorblind-safe categorical palette of 8 hues plus 3 semantic anchors, tuned for " + "warm-paper rendering and validated against deuteranopia, protanopia and tritanopia. " + "Every plot in the catalogue uses it.

    " + "

    Categorical hues (slot order)

    " + "
      " + + "".join( + f"
    1. {name} {hex_value}
      2. " + for name, hex_value in zip(_PALETTE_SLOT_NAMES, palette.IMPRINT, strict=True) + ) + + "
    " + "

    Semantic anchors (outside the categorical pool)

    " + "" + "

    Copy-paste snippets for Python, R, Julia and JavaScript are on the " + 'interactive page.

    ' +) + + @router.get("/seo-proxy/palette") async def seo_palette(): - """Bot-optimized palette page with correct og:tags.""" + """Bot-optimized palette page: the actual hex values, not just og:tags.""" return HTMLResponse( _render_bot_html( title="imprint palette | anyplot.ai", description="Imprint — a colorblind-safe categorical palette of 8 hues plus 3 semantic anchors (amber, neutral, muted). Tuned for warm-paper rendering, validated against deuteranopia / protanopia / tritanopia. The palette every plot on anyplot.ai uses.", image=DEFAULT_HOME_IMAGE, url="https://anyplot.ai/palette", + body=_PALETTE_BOT_BODY, ) ) @@ -1002,14 +1106,45 @@ async def seo_map(): @router.get("/seo-proxy/stats") -async def seo_stats(): - """Bot-optimized stats page with correct og:tags.""" +async def seo_stats(db: AsyncSession | None = Depends(optional_db)): + """Bot-optimized stats page: the live catalogue counts, not just og:tags. + + Uses the same cache key the /stats endpoint populates (and the startup + prewarm fills), so this adds no extra DB load. Without a DB the body + degrades to the registry-derived counts. + """ + if db is not None: + + async def _fetch(): + return await _compute_stats(db) + + stats = await get_or_set_cache( + cache_key("stats"), _fetch, refresh_after=settings.cache_refresh_after, refresh_factory=_refresh_stats + ) + body = ( + "

    anyplot in numbers

    " + "" + ) + else: + body = ( + f"

    anyplot in numbers

    {_LIBRARY_COUNT} libraries across {len(LANGUAGES_METADATA)} languages.

    " + ) + body += ( + '

    Live machine-readable counts: api.anyplot.ai/stats; ' + "per-library quality scores and coverage on the " + 'interactive page.

    ' + ) return HTMLResponse( _render_bot_html( title="Stats | anyplot.ai", description="Platform statistics: library scores, coverage, tags, and top implementations.", image=DEFAULT_HOME_IMAGE, url="https://anyplot.ai/stats", + body=body, ) ) diff --git a/tests/unit/api/test_routers.py b/tests/unit/api/test_routers.py index 2045aa6d77..018ac5b24c 100644 --- a/tests/unit/api/test_routers.py +++ b/tests/unit/api/test_routers.py @@ -1048,12 +1048,36 @@ def test_seo_spec_implementation_fallback_image(self, db_client, mock_spec) -> N assert "
    " not in response.text
 
     def test_seo_about(self, client: TestClient) -> None:
-        """SEO about page should return HTML with og:tags."""
+        """SEO about page must carry the pipeline story, not just og:tags."""
         response = client.get("/seo-proxy/about")
         assert response.status_code == 200
         assert "text/html" in response.headers["content-type"]
         assert "og:title" in response.text
         assert "https://anyplot.ai/about" in response.text
+        # registry-derived count + the pipeline sequence
+        assert "15 libraries" in response.text
+        assert "AI-drafted, human-approved" in response.text
+        assert "never patched by hand" in response.text
+
+    def test_seo_libraries_lists_the_registry(self, client: TestClient) -> None:
+        """The /libraries bot page was a 29-word stub although its whole
+        content sits in core/constants.py (AI-access audit 2026-08-19)."""
+        response = client.get("/seo-proxy/libraries")
+        assert response.status_code == 200
+        # one section per language, every library named with its version
+        for heading in ("
    Python
    ", "
    R
    ", "
    Julia
    ", "
    JavaScript
    "):
+            assert heading in response.text
+        for lib in ("Matplotlib", "ggplot2", "Makie", "ECharts", "MUI X Charts"):
+            assert lib in response.text
+        assert "https://anyplot.ai/plots" in response.text
+
+    def test_seo_legal_names_operator_and_privacy_facts(self, client: TestClient) -> None:
+        response = client.get("/seo-proxy/legal")
+        assert response.status_code == 200
+        assert "Markus Neusinger" in response.text
+        assert "Plausible Analytics" in response.text
+        assert "no cookies" in response.text
+        assert "Google Cloud Run" in response.text
 
     def test_seo_mcp_tells_agents_how_to_connect(self, client: TestClient) -> None:
         """The /mcp page's audience is AI agents — the bot body must carry the
@@ -1073,12 +1097,17 @@ def test_seo_mcp_tells_agents_how_to_connect(self, client: TestClient) -> None:
             assert tool in response.text
 
     def test_seo_palette(self, client: TestClient) -> None:
-        """SEO palette page should return HTML with og:tags."""
+        """SEO palette page must carry the actual hex values from core/palette.py."""
         response = client.get("/seo-proxy/palette")
         assert response.status_code == 200
         assert "text/html" in response.headers["content-type"]
         assert "og:title" in response.text
         assert "https://anyplot.ai/palette" in response.text
+        # all 8 categorical hues plus the amber anchor, straight from the module
+        from core import palette
+
+        for hex_value in [*palette.IMPRINT, palette.AMBER]:
+            assert hex_value in response.text
 
     def test_seo_map(self, client: TestClient) -> None:
         """SEO map page should return HTML with og:tags."""
@@ -1088,13 +1117,31 @@ def test_seo_map(self, client: TestClient) -> None:
         assert "og:title" in response.text
         assert "https://anyplot.ai/map" in response.text
 
-    def test_seo_stats(self, client: TestClient) -> None:
-        """SEO stats page should return HTML with og:tags."""
-        response = client.get("/seo-proxy/stats")
+    def test_seo_stats_without_db_degrades_to_registry_counts(self, client: TestClient) -> None:
+        """Without a DB the stats body still carries the registry-derived counts."""
+        with patch(DB_CONFIG_PATCH, return_value=False):
+            response = client.get("/seo-proxy/stats")
         assert response.status_code == 200
-        assert "text/html" in response.headers["content-type"]
         assert "og:title" in response.text
-        assert "https://anyplot.ai/stats" in response.text
+        assert "anyplot in numbers" in response.text
+        assert "15 libraries across 4 languages" in response.text
+        assert "https://api.anyplot.ai/stats" in response.text
+
+    def test_seo_stats_with_db_shows_live_counts(self, db_client) -> None:
+        """With a DB the body carries the cached live counts."""
+        from api.schemas import StatsResponse
+
+        client, _ = db_client
+
+        async def _cached(key, factory, **kwargs):
+            return StatsResponse(specs=324, plots=3583, libraries=15, languages=4)
+
+        with patch("api.routers.seo.get_or_set_cache", side_effect=_cached):
+            response = client.get("/seo-proxy/stats")
+        assert response.status_code == 200
+        assert "324 plot specifications" in response.text
+        assert "3583 rendered implementations" in response.text
+        assert "15 libraries across 4 languages" in response.text
 
 
 class TestOgImagesRouter:

From 1d78ffe5278c9fd327aac7bfed5c9c00d039c19d Mon Sep 17 00:00:00 2001
From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com>
Date: Wed, 19 Aug 2026 22:05:20 +0200
Subject: [PATCH 2/3] docs(changelog): reference #10491

Co-Authored-By: Claude Fable 5 
---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f0ef0475c0..843d4627bf 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -142,7 +142,7 @@ aggregate instead: an italic *Catalog* line at the end of the version section an
   `core/constants.py`), the live catalogue counts (reusing the cached `/stats` data), the
   pipeline story, the operator/privacy/transparency facts, and the actual palette hex values
   (straight from `core/palette.py`) — each derived from its single source of truth so the pages
-  cannot drift.
+  cannot drift (#10491).
 
 ## [3.1.0] — 2026-08-19 — Legible to machines
 

From 6e72d15117e44002d312735c61a926771d9d3633 Mon Sep 17 00:00:00 2001
From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com>
Date: Wed, 19 Aug 2026 22:14:57 +0200
Subject: [PATCH 3/3] test(seo): assert every palette anchor; type the stats
 fetch factory

Copilot review on #10491.

Co-Authored-By: Claude Fable 5 
---
 api/routers/seo.py             |  3 ++-
 tests/unit/api/test_routers.py | 11 +++++++++--
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/api/routers/seo.py b/api/routers/seo.py
index ba0fff2c5c..383d45f2f7 100644
--- a/api/routers/seo.py
+++ b/api/routers/seo.py
@@ -13,6 +13,7 @@
 from api.cache import cache_key, get_cache, get_or_set_cache, set_cache
 from api.dependencies import optional_db
 from api.routers.stats import _compute_stats, _refresh_stats
+from api.schemas import StatsResponse
 from core import palette
 from core.config import settings
 from core.constants import LANGUAGES_METADATA, LIBRARIES_METADATA
@@ -1115,7 +1116,7 @@ async def seo_stats(db: AsyncSession | None = Depends(optional_db)):
     """
     if db is not None:
 
-        async def _fetch():
+        async def _fetch() -> StatsResponse:
             return await _compute_stats(db)
 
         stats = await get_or_set_cache(
diff --git a/tests/unit/api/test_routers.py b/tests/unit/api/test_routers.py
index 018ac5b24c..cd122b1f43 100644
--- a/tests/unit/api/test_routers.py
+++ b/tests/unit/api/test_routers.py
@@ -1103,10 +1103,17 @@ def test_seo_palette(self, client: TestClient) -> None:
         assert "text/html" in response.headers["content-type"]
         assert "og:title" in response.text
         assert "https://anyplot.ai/palette" in response.text
-        # all 8 categorical hues plus the amber anchor, straight from the module
+        # all 8 categorical hues plus every semantic anchor, straight from the module
         from core import palette
 
-        for hex_value in [*palette.IMPRINT, palette.AMBER]:
+        anchors = [
+            palette.AMBER,
+            palette.neutral_for("light"),
+            palette.neutral_for("dark"),
+            palette.muted_for("light"),
+            palette.muted_for("dark"),
+        ]
+        for hex_value in [*palette.IMPRINT, *anchors]:
             assert hex_value in response.text
 
     def test_seo_map(self, client: TestClient) -> None: