diff --git a/CHANGELOG.md b/CHANGELOG.md index af376ad6fc..843d4627bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -135,6 +135,14 @@ aggregate instead: an italic *Catalog* line at the end of the version section an honest note that prerendered page HTML is gated on a crawler user agent while these URLs are not. `anyplot.ai/llms-full.txt` is now proxied to the API's generated catalogue index instead of soft-404ing to the homepage shell, and the daily monitor asserts it (#10488). +- **The five doc pages carry real content for crawlers** — the bot renderings of + `/libraries`, `/stats`, `/about`, `/legal` and `/palette` were 29–59-word title+description + stubs, so a crawler following llms.txt's own link descriptions found none of the promised + content (AI-access audit 2026-08-19). They now serve the full library registry (derived from + `core/constants.py`), the live catalogue counts (reusing the cached `/stats` data), the + pipeline story, the operator/privacy/transparency facts, and the actual palette hex values + (straight from `core/palette.py`) — each derived from its single source of truth so the pages + cannot drift (#10491). ## [3.1.0] — 2026-08-19 — Legible to machines diff --git a/api/routers/seo.py b/api/routers/seo.py index efc0f887ed..383d45f2f7 100644 --- a/api/routers/seo.py +++ b/api/routers/seo.py @@ -12,6 +12,9 @@ from api.cache import cache_key, get_cache, get_or_set_cache, set_cache from api.dependencies import optional_db +from api.routers.stats import _compute_stats, _refresh_stats +from api.schemas import StatsResponse +from core import palette from core.config import settings from core.constants import LANGUAGES_METADATA, LIBRARIES_METADATA from core.database import ImplRepository, SpecRepository @@ -886,28 +889,78 @@ async def seo_specs(db: AsyncSession | None = Depends(optional_db)): ) +def _build_libraries_body() -> str: + """Real bot body for /libraries, derived from the canonical registry. + + The page was a 29-word stub even though its entire content already sits in + core/constants.py (AI-access audit 2026-08-19) — a crawler following + llms.txt's "the fifteen supported plotting libraries" link found no such + list. Registry-derived, so it can never drift from the actual catalog. + """ + sections = [ + f"
Every plot specification on anyplot.ai is implemented once per library below. " + 'Browse renders per library in the gallery.
', + ] + for lang in LANGUAGES_METADATA: + libs = [lib for lib in LIBRARIES_METADATA if lib["language_id"] == lang["id"]] + if not libs: + continue + items = "".join( + f'Operator: Markus Neusinger " + '(GitHub, ' + 'X). Full legal notice with ' + 'contact details on the interactive page.
' + "Analytics: Plausible Analytics (EU, proxied) — no cookies, no personal data collected. " + "Hosting: Google Cloud Run (Netherlands).
" + "The whole stack — specs, pipeline, API and frontend — is open source at " + 'github.com/MarkusNeusinger/anyplot; ' + "the catalogue content is MIT-licensed.
" +) + + @router.get("/seo-proxy/legal") async def seo_legal(): - """Bot-optimized legal page with correct og:tags.""" + """Bot-optimized legal page: operator, privacy and transparency facts.""" return HTMLResponse( _render_bot_html( title="Legal | anyplot.ai", description="Legal notice, privacy policy, and transparency information for anyplot.ai", image=DEFAULT_HOME_IMAGE, url="https://anyplot.ai/legal", + body=_LEGAL_BOT_BODY, ) ) @@ -959,28 +1012,80 @@ async def seo_mcp(): ) +# Mirrors app/src/pages/AboutPage.tsx — the pipeline story is the page's +# content; counts derive from the registry so they cannot drift. +_ABOUT_BOT_BODY = ( + "anyplot.ai is a catalogue of plotting examples across {_LIBRARY_COUNT} libraries " + f"in {_LANGUAGE_LIST}. Plot ideas come from humans; AI drafts the specification, " + "generates code for every library, and reviews each implementation. Humans approve " + "specs and tune the rules.
" + "idea → spec (AI-drafted, human-approved) → code (AI-generated per library) → " + "review (AI-evaluated). When a library ships a new release the pipeline re-runs; when a " + "better example pattern emerges the spec is updated and every library regenerates. " + "Generated code is never patched by hand.
" + "Source, specs and pipeline live at " + 'github.com/MarkusNeusinger/anyplot ' + '(MIT). Machine access is documented in llms.txt; ' + "propose a new plot type via a " + 'spec request.
' +) + + @router.get("/seo-proxy/about") async def seo_about(): - """Bot-optimized about page with correct og:tags.""" + """Bot-optimized about page: the pipeline story, not just og:tags.""" return HTMLResponse( _render_bot_html( title="About | anyplot.ai", description="About anyplot.ai — library-agnostic, AI-powered plotting.", image=DEFAULT_HOME_IMAGE, url="https://anyplot.ai/about", + body=_ABOUT_BOT_BODY, ) ) +# Hex values come straight from core/palette.py — the single source of truth — +# so the bot page can never disagree with what the plots actually use. +_PALETTE_SLOT_NAMES = ("green", "lavender", "blue", "ochre", "red", "cyan", "rose", "lime") +_PALETTE_BOT_BODY = ( + "A colorblind-safe categorical palette of 8 hues plus 3 semantic anchors, tuned for " + "warm-paper rendering and validated against deuteranopia, protanopia and tritanopia. " + "Every plot in the catalogue uses it.
" + "{hex_value}{palette.AMBER} — warning / caution{palette.neutral_for('light')} light / "
+ f"{palette.neutral_for('dark')} dark{palette.muted_for('light')} light / "
+ f"{palette.muted_for('dark')} darkCopy-paste snippets for Python, R, Julia and JavaScript are on the " + 'interactive page.
' +) + + @router.get("/seo-proxy/palette") async def seo_palette(): - """Bot-optimized palette page with correct og:tags.""" + """Bot-optimized palette page: the actual hex values, not just og:tags.""" return HTMLResponse( _render_bot_html( title="imprint palette | anyplot.ai", description="Imprint — a colorblind-safe categorical palette of 8 hues plus 3 semantic anchors (amber, neutral, muted). Tuned for warm-paper rendering, validated against deuteranopia / protanopia / tritanopia. The palette every plot on anyplot.ai uses.", image=DEFAULT_HOME_IMAGE, url="https://anyplot.ai/palette", + body=_PALETTE_BOT_BODY, ) ) @@ -1002,14 +1107,45 @@ async def seo_map(): @router.get("/seo-proxy/stats") -async def seo_stats(): - """Bot-optimized stats page with correct og:tags.""" +async def seo_stats(db: AsyncSession | None = Depends(optional_db)): + """Bot-optimized stats page: the live catalogue counts, not just og:tags. + + Uses the same cache key the /stats endpoint populates (and the startup + prewarm fills), so this adds no extra DB load. Without a DB the body + degrades to the registry-derived counts. + """ + if db is not None: + + async def _fetch() -> StatsResponse: + return await _compute_stats(db) + + stats = await get_or_set_cache( + cache_key("stats"), _fetch, refresh_after=settings.cache_refresh_after, refresh_factory=_refresh_stats + ) + body = ( + "{_LIBRARY_COUNT} libraries across {len(LANGUAGES_METADATA)} languages.
" + ) + body += ( + 'Live machine-readable counts: api.anyplot.ai/stats; ' + "per-library quality scores and coverage on the " + 'interactive page.
' + ) return HTMLResponse( _render_bot_html( title="Stats | anyplot.ai", description="Platform statistics: library scores, coverage, tags, and top implementations.", image=DEFAULT_HOME_IMAGE, url="https://anyplot.ai/stats", + body=body, ) ) diff --git a/tests/unit/api/test_routers.py b/tests/unit/api/test_routers.py index 2045aa6d77..cd122b1f43 100644 --- a/tests/unit/api/test_routers.py +++ b/tests/unit/api/test_routers.py @@ -1048,12 +1048,36 @@ def test_seo_spec_implementation_fallback_image(self, db_client, mock_spec) -> N assert "" not in response.text
def test_seo_about(self, client: TestClient) -> None:
- """SEO about page should return HTML with og:tags."""
+ """SEO about page must carry the pipeline story, not just og:tags."""
response = client.get("/seo-proxy/about")
assert response.status_code == 200
assert "text/html" in response.headers["content-type"]
assert "og:title" in response.text
assert "https://anyplot.ai/about" in response.text
+ # registry-derived count + the pipeline sequence
+ assert "15 libraries" in response.text
+ assert "AI-drafted, human-approved" in response.text
+ assert "never patched by hand" in response.text
+
+ def test_seo_libraries_lists_the_registry(self, client: TestClient) -> None:
+ """The /libraries bot page was a 29-word stub although its whole
+ content sits in core/constants.py (AI-access audit 2026-08-19)."""
+ response = client.get("/seo-proxy/libraries")
+ assert response.status_code == 200
+ # one section per language, every library named with its version
+ for heading in ("Python
", "R
", "Julia
", "JavaScript
"):
+ assert heading in response.text
+ for lib in ("Matplotlib", "ggplot2", "Makie", "ECharts", "MUI X Charts"):
+ assert lib in response.text
+ assert "https://anyplot.ai/plots" in response.text
+
+ def test_seo_legal_names_operator_and_privacy_facts(self, client: TestClient) -> None:
+ response = client.get("/seo-proxy/legal")
+ assert response.status_code == 200
+ assert "Markus Neusinger" in response.text
+ assert "Plausible Analytics" in response.text
+ assert "no cookies" in response.text
+ assert "Google Cloud Run" in response.text
def test_seo_mcp_tells_agents_how_to_connect(self, client: TestClient) -> None:
"""The /mcp page's audience is AI agents — the bot body must carry the
@@ -1073,12 +1097,24 @@ def test_seo_mcp_tells_agents_how_to_connect(self, client: TestClient) -> None:
assert tool in response.text
def test_seo_palette(self, client: TestClient) -> None:
- """SEO palette page should return HTML with og:tags."""
+ """SEO palette page must carry the actual hex values from core/palette.py."""
response = client.get("/seo-proxy/palette")
assert response.status_code == 200
assert "text/html" in response.headers["content-type"]
assert "og:title" in response.text
assert "https://anyplot.ai/palette" in response.text
+ # all 8 categorical hues plus every semantic anchor, straight from the module
+ from core import palette
+
+ anchors = [
+ palette.AMBER,
+ palette.neutral_for("light"),
+ palette.neutral_for("dark"),
+ palette.muted_for("light"),
+ palette.muted_for("dark"),
+ ]
+ for hex_value in [*palette.IMPRINT, *anchors]:
+ assert hex_value in response.text
def test_seo_map(self, client: TestClient) -> None:
"""SEO map page should return HTML with og:tags."""
@@ -1088,13 +1124,31 @@ def test_seo_map(self, client: TestClient) -> None:
assert "og:title" in response.text
assert "https://anyplot.ai/map" in response.text
- def test_seo_stats(self, client: TestClient) -> None:
- """SEO stats page should return HTML with og:tags."""
- response = client.get("/seo-proxy/stats")
+ def test_seo_stats_without_db_degrades_to_registry_counts(self, client: TestClient) -> None:
+ """Without a DB the stats body still carries the registry-derived counts."""
+ with patch(DB_CONFIG_PATCH, return_value=False):
+ response = client.get("/seo-proxy/stats")
assert response.status_code == 200
- assert "text/html" in response.headers["content-type"]
assert "og:title" in response.text
- assert "https://anyplot.ai/stats" in response.text
+ assert "anyplot in numbers" in response.text
+ assert "15 libraries across 4 languages" in response.text
+ assert "https://api.anyplot.ai/stats" in response.text
+
+ def test_seo_stats_with_db_shows_live_counts(self, db_client) -> None:
+ """With a DB the body carries the cached live counts."""
+ from api.schemas import StatsResponse
+
+ client, _ = db_client
+
+ async def _cached(key, factory, **kwargs):
+ return StatsResponse(specs=324, plots=3583, libraries=15, languages=4)
+
+ with patch("api.routers.seo.get_or_set_cache", side_effect=_cached):
+ response = client.get("/seo-proxy/stats")
+ assert response.status_code == 200
+ assert "324 plot specifications" in response.text
+ assert "3583 rendered implementations" in response.text
+ assert "15 libraries across 4 languages" in response.text
class TestOgImagesRouter: