diff --git a/CHANGELOG.md b/CHANGELOG.md index af376ad6fc..843d4627bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -135,6 +135,14 @@ aggregate instead: an italic *Catalog* line at the end of the version section an honest note that prerendered page HTML is gated on a crawler user agent while these URLs are not. `anyplot.ai/llms-full.txt` is now proxied to the API's generated catalogue index instead of soft-404ing to the homepage shell, and the daily monitor asserts it (#10488). +- **The five doc pages carry real content for crawlers** — the bot renderings of + `/libraries`, `/stats`, `/about`, `/legal` and `/palette` were 29–59-word title+description + stubs, so a crawler following llms.txt's own link descriptions found none of the promised + content (AI-access audit 2026-08-19). They now serve the full library registry (derived from + `core/constants.py`), the live catalogue counts (reusing the cached `/stats` data), the + pipeline story, the operator/privacy/transparency facts, and the actual palette hex values + (straight from `core/palette.py`) — each derived from its single source of truth so the pages + cannot drift (#10491). ## [3.1.0] — 2026-08-19 — Legible to machines diff --git a/api/routers/seo.py b/api/routers/seo.py index efc0f887ed..383d45f2f7 100644 --- a/api/routers/seo.py +++ b/api/routers/seo.py @@ -12,6 +12,9 @@ from api.cache import cache_key, get_cache, get_or_set_cache, set_cache from api.dependencies import optional_db +from api.routers.stats import _compute_stats, _refresh_stats +from api.schemas import StatsResponse +from core import palette from core.config import settings from core.constants import LANGUAGES_METADATA, LIBRARIES_METADATA from core.database import ImplRepository, SpecRepository @@ -886,28 +889,78 @@ async def seo_specs(db: AsyncSession | None = Depends(optional_db)): ) +def _build_libraries_body() -> str: + """Real bot body for /libraries, derived from the canonical registry. + + The page was a 29-word stub even though its entire content already sits in + core/constants.py (AI-access audit 2026-08-19) — a crawler following + llms.txt's "the fifteen supported plotting libraries" link found no such + list. Registry-derived, so it can never drift from the actual catalog. + """ + sections = [ + f"

{_LIBRARY_COUNT} plotting libraries across {len(LANGUAGES_METADATA)} languages

", + "

Every plot specification on anyplot.ai is implemented once per library below. " + 'Browse renders per library in the gallery.

', + ] + for lang in LANGUAGES_METADATA: + libs = [lib for lib in LIBRARIES_METADATA if lib["language_id"] == lang["id"]] + if not libs: + continue + items = "".join( + f'
  • ' + f"{html.escape(str(lib['name']))} {html.escape(str(lib['version']))} — " + f"{html.escape(str(lib['description']))}
  • " + for lib in libs + ) + sections.append(f"

    {html.escape(str(lang['name']))}

    ") + return "".join(sections) + + +_LIBRARIES_BOT_BODY = _build_libraries_body() + + @router.get("/seo-proxy/libraries") async def seo_libraries(): - """Bot-optimized libraries page with correct og:tags.""" + """Bot-optimized libraries page: the full registry, not just og:tags.""" return HTMLResponse( _render_bot_html( title="libraries | anyplot.ai", description="All supported plotting libraries across languages.", image=DEFAULT_PLOTS_IMAGE, url="https://anyplot.ai/libraries", + body=_LIBRARIES_BOT_BODY, ) ) +# Mirrors the load-bearing facts of app/src/pages/LegalPage.tsx — keep the two +# in sync when operator, analytics, or hosting details change. +_LEGAL_BOT_BODY = ( + "

    Legal notice

    " + "

    Operator: Markus Neusinger " + '(GitHub, ' + 'X). Full legal notice with ' + 'contact details on the interactive page.

    ' + "

    Privacy

    " + "

    Analytics: Plausible Analytics (EU, proxied) — no cookies, no personal data collected. " + "Hosting: Google Cloud Run (Netherlands).

    " + "

    Transparency

    " + "

    The whole stack — specs, pipeline, API and frontend — is open source at " + 'github.com/MarkusNeusinger/anyplot; ' + "the catalogue content is MIT-licensed.

    " +) + + @router.get("/seo-proxy/legal") async def seo_legal(): - """Bot-optimized legal page with correct og:tags.""" + """Bot-optimized legal page: operator, privacy and transparency facts.""" return HTMLResponse( _render_bot_html( title="Legal | anyplot.ai", description="Legal notice, privacy policy, and transparency information for anyplot.ai", image=DEFAULT_HOME_IMAGE, url="https://anyplot.ai/legal", + body=_LEGAL_BOT_BODY, ) ) @@ -959,28 +1012,80 @@ async def seo_mcp(): ) +# Mirrors app/src/pages/AboutPage.tsx — the pipeline story is the page's +# content; counts derive from the registry so they cannot drift. +_ABOUT_BOT_BODY = ( + "

    About anyplot

    " + f"

    anyplot.ai is a catalogue of plotting examples across {_LIBRARY_COUNT} libraries " + f"in {_LANGUAGE_LIST}. Plot ideas come from humans; AI drafts the specification, " + "generates code for every library, and reviews each implementation. Humans approve " + "specs and tune the rules.

    " + "

    Pipeline

    " + "

    idea → spec (AI-drafted, human-approved) → code (AI-generated per library) → " + "review (AI-evaluated). When a library ships a new release the pipeline re-runs; when a " + "better example pattern emerges the spec is updated and every library regenerates. " + "Generated code is never patched by hand.

    " + "

    Open

    " + "

    Source, specs and pipeline live at " + 'github.com/MarkusNeusinger/anyplot ' + '(MIT). Machine access is documented in llms.txt; ' + "propose a new plot type via a " + 'spec request.

    ' +) + + @router.get("/seo-proxy/about") async def seo_about(): - """Bot-optimized about page with correct og:tags.""" + """Bot-optimized about page: the pipeline story, not just og:tags.""" return HTMLResponse( _render_bot_html( title="About | anyplot.ai", description="About anyplot.ai — library-agnostic, AI-powered plotting.", image=DEFAULT_HOME_IMAGE, url="https://anyplot.ai/about", + body=_ABOUT_BOT_BODY, ) ) +# Hex values come straight from core/palette.py — the single source of truth — +# so the bot page can never disagree with what the plots actually use. +_PALETTE_SLOT_NAMES = ("green", "lavender", "blue", "ochre", "red", "cyan", "rose", "lime") +_PALETTE_BOT_BODY = ( + "

    imprint — the anyplot palette

    " + "

    A colorblind-safe categorical palette of 8 hues plus 3 semantic anchors, tuned for " + "warm-paper rendering and validated against deuteranopia, protanopia and tritanopia. " + "Every plot in the catalogue uses it.

    " + "

    Categorical hues (slot order)

    " + "
      " + + "".join( + f"
    1. {name} {hex_value}
    2. " + for name, hex_value in zip(_PALETTE_SLOT_NAMES, palette.IMPRINT, strict=True) + ) + + "
    " + "

    Semantic anchors (outside the categorical pool)

    " + "" + "

    Copy-paste snippets for Python, R, Julia and JavaScript are on the " + 'interactive page.

    ' +) + + @router.get("/seo-proxy/palette") async def seo_palette(): - """Bot-optimized palette page with correct og:tags.""" + """Bot-optimized palette page: the actual hex values, not just og:tags.""" return HTMLResponse( _render_bot_html( title="imprint palette | anyplot.ai", description="Imprint — a colorblind-safe categorical palette of 8 hues plus 3 semantic anchors (amber, neutral, muted). Tuned for warm-paper rendering, validated against deuteranopia / protanopia / tritanopia. The palette every plot on anyplot.ai uses.", image=DEFAULT_HOME_IMAGE, url="https://anyplot.ai/palette", + body=_PALETTE_BOT_BODY, ) ) @@ -1002,14 +1107,45 @@ async def seo_map(): @router.get("/seo-proxy/stats") -async def seo_stats(): - """Bot-optimized stats page with correct og:tags.""" +async def seo_stats(db: AsyncSession | None = Depends(optional_db)): + """Bot-optimized stats page: the live catalogue counts, not just og:tags. + + Uses the same cache key the /stats endpoint populates (and the startup + prewarm fills), so this adds no extra DB load. Without a DB the body + degrades to the registry-derived counts. + """ + if db is not None: + + async def _fetch() -> StatsResponse: + return await _compute_stats(db) + + stats = await get_or_set_cache( + cache_key("stats"), _fetch, refresh_after=settings.cache_refresh_after, refresh_factory=_refresh_stats + ) + body = ( + "

    anyplot in numbers

    " + "" + ) + else: + body = ( + f"

    anyplot in numbers

    {_LIBRARY_COUNT} libraries across {len(LANGUAGES_METADATA)} languages.

    " + ) + body += ( + '

    Live machine-readable counts: api.anyplot.ai/stats; ' + "per-library quality scores and coverage on the " + 'interactive page.

    ' + ) return HTMLResponse( _render_bot_html( title="Stats | anyplot.ai", description="Platform statistics: library scores, coverage, tags, and top implementations.", image=DEFAULT_HOME_IMAGE, url="https://anyplot.ai/stats", + body=body, ) ) diff --git a/tests/unit/api/test_routers.py b/tests/unit/api/test_routers.py index 2045aa6d77..cd122b1f43 100644 --- a/tests/unit/api/test_routers.py +++ b/tests/unit/api/test_routers.py @@ -1048,12 +1048,36 @@ def test_seo_spec_implementation_fallback_image(self, db_client, mock_spec) -> N assert "
    " not in response.text
     
         def test_seo_about(self, client: TestClient) -> None:
    -        """SEO about page should return HTML with og:tags."""
    +        """SEO about page must carry the pipeline story, not just og:tags."""
             response = client.get("/seo-proxy/about")
             assert response.status_code == 200
             assert "text/html" in response.headers["content-type"]
             assert "og:title" in response.text
             assert "https://anyplot.ai/about" in response.text
    +        # registry-derived count + the pipeline sequence
    +        assert "15 libraries" in response.text
    +        assert "AI-drafted, human-approved" in response.text
    +        assert "never patched by hand" in response.text
    +
    +    def test_seo_libraries_lists_the_registry(self, client: TestClient) -> None:
    +        """The /libraries bot page was a 29-word stub although its whole
    +        content sits in core/constants.py (AI-access audit 2026-08-19)."""
    +        response = client.get("/seo-proxy/libraries")
    +        assert response.status_code == 200
    +        # one section per language, every library named with its version
    +        for heading in ("

    Python

    ", "

    R

    ", "

    Julia

    ", "

    JavaScript

    "): + assert heading in response.text + for lib in ("Matplotlib", "ggplot2", "Makie", "ECharts", "MUI X Charts"): + assert lib in response.text + assert "https://anyplot.ai/plots" in response.text + + def test_seo_legal_names_operator_and_privacy_facts(self, client: TestClient) -> None: + response = client.get("/seo-proxy/legal") + assert response.status_code == 200 + assert "Markus Neusinger" in response.text + assert "Plausible Analytics" in response.text + assert "no cookies" in response.text + assert "Google Cloud Run" in response.text def test_seo_mcp_tells_agents_how_to_connect(self, client: TestClient) -> None: """The /mcp page's audience is AI agents — the bot body must carry the @@ -1073,12 +1097,24 @@ def test_seo_mcp_tells_agents_how_to_connect(self, client: TestClient) -> None: assert tool in response.text def test_seo_palette(self, client: TestClient) -> None: - """SEO palette page should return HTML with og:tags.""" + """SEO palette page must carry the actual hex values from core/palette.py.""" response = client.get("/seo-proxy/palette") assert response.status_code == 200 assert "text/html" in response.headers["content-type"] assert "og:title" in response.text assert "https://anyplot.ai/palette" in response.text + # all 8 categorical hues plus every semantic anchor, straight from the module + from core import palette + + anchors = [ + palette.AMBER, + palette.neutral_for("light"), + palette.neutral_for("dark"), + palette.muted_for("light"), + palette.muted_for("dark"), + ] + for hex_value in [*palette.IMPRINT, *anchors]: + assert hex_value in response.text def test_seo_map(self, client: TestClient) -> None: """SEO map page should return HTML with og:tags.""" @@ -1088,13 +1124,31 @@ def test_seo_map(self, client: TestClient) -> None: assert "og:title" in response.text assert "https://anyplot.ai/map" in response.text - def test_seo_stats(self, client: TestClient) -> None: - """SEO stats page should return HTML with og:tags.""" - response = client.get("/seo-proxy/stats") + def test_seo_stats_without_db_degrades_to_registry_counts(self, client: TestClient) -> None: + """Without a DB the stats body still carries the registry-derived counts.""" + with patch(DB_CONFIG_PATCH, return_value=False): + response = client.get("/seo-proxy/stats") assert response.status_code == 200 - assert "text/html" in response.headers["content-type"] assert "og:title" in response.text - assert "https://anyplot.ai/stats" in response.text + assert "anyplot in numbers" in response.text + assert "15 libraries across 4 languages" in response.text + assert "https://api.anyplot.ai/stats" in response.text + + def test_seo_stats_with_db_shows_live_counts(self, db_client) -> None: + """With a DB the body carries the cached live counts.""" + from api.schemas import StatsResponse + + client, _ = db_client + + async def _cached(key, factory, **kwargs): + return StatsResponse(specs=324, plots=3583, libraries=15, languages=4) + + with patch("api.routers.seo.get_or_set_cache", side_effect=_cached): + response = client.get("/seo-proxy/stats") + assert response.status_code == 200 + assert "324 plot specifications" in response.text + assert "3583 rendered implementations" in response.text + assert "15 libraries across 4 languages" in response.text class TestOgImagesRouter: