From 81157de875addb110207b6c4bae0c4f407faab6d Mon Sep 17 00:00:00 2001
From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com>
Date: Wed, 19 Aug 2026 22:03:56 +0200
Subject: [PATCH 1/2] fix(app): declare utf-8 on text responses

llms.txt and robots.txt carry em dashes and arrows but were served as
bare text/plain (live-verification sweep 2026-08-19) - a strict client
decodes them as Latin-1 mojibake. charset utf-8 in both server blocks;
the daily bot-serving monitor asserts it on llms.txt.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 .github/workflows/bot-serving-check.yml | 19 +++++++++++++++----
 CHANGELOG.md                            |  4 ++++
 app/nginx.conf                          | 11 +++++++++++
 3 files changed, 30 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/bot-serving-check.yml b/.github/workflows/bot-serving-check.yml
index 9b1c1d8973..2a0d5eca54 100644
--- a/.github/workflows/bot-serving-check.yml
+++ b/.github/workflows/bot-serving-check.yml
@@ -29,10 +29,11 @@ jobs:
   bot-serving:
     runs-on: ubuntu-latest
     # 22 check() calls x (--retry 2 -> up to 3 attempts x --max-time 30) can
-    # reach ~33 min worst-case, plus one non-retried trailing-slash probe
-    # (30s); 38 leaves room to report a clean failure rather than dying to the
-    # job timeout, which reports nothing useful. Recompute this when adding
-    # checks: the ceiling is check() calls x 90s, plus margin.
+    # reach ~33 min worst-case, plus two non-retried probes (llms.txt charset,
+    # trailing slash — 30s each); 38 leaves room to report a clean failure
+    # rather than dying to the job timeout, which reports nothing useful.
+    # Recompute this when adding checks: the ceiling is check() calls x 90s,
+    # plus margin.
     timeout-minutes: 38
     steps:
       - name: Crawler UAs must get 200 + per-route titles
@@ -116,6 +117,16 @@ jobs:
           # shell. The catalogue-index line proves the API generated it.
           check "$HUMAN" "$ORIGIN/llms-full.txt" "# anyplot — full catalogue index"
 
+          # llms.txt carries UTF-8 punctuation (em dashes, arrows); without an
+          # explicit charset a strict client decodes it as Latin-1 mojibake.
+          ct=$(curl -sS --max-time 30 -A "$GOOGLEBOT" -o /dev/null -w '%{content_type}' "$ORIGIN/llms.txt")
+          case "$ct" in
+            *charset=utf-8*) echo "OK: llms.txt content-type: $ct" ;;
+            *)
+              echo "::error::llms.txt served without utf-8 charset: $ct"
+              fail=1 ;;
+          esac
+
           # A trailing slash must normalise to the canonical URL on THIS host.
           # It used to 307 to http://api.anyplot.ai/seo-proxy/... — internal
           # path, wrong host, plain http, and that host disallows all crawling.
diff --git a/CHANGELOG.md b/CHANGELOG.md
index af376ad6fc..66c00519a8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -125,6 +125,10 @@ aggregate instead: an italic *Catalog* line at the end of the version section an
   `<div id="root">` with no `<noscript>`, so any fetcher missing from the UA allowlist rendered
   literally nothing. A noscript block now points such clients at `llms.txt`, `llms-full.txt`,
   the JSON API and the GitHub repository (#10488).
+- **Text files declare UTF-8** — `llms.txt` and `robots.txt` carry em dashes and arrows but
+  were served as bare `text/plain`, which a strict client decodes as Latin-1 mojibake. nginx now
+  declares `charset utf-8` on text responses in both server blocks, and the daily bot-serving
+  monitor asserts it on `llms.txt`.
 
 ### Changed
 
diff --git a/app/nginx.conf b/app/nginx.conf
index 467f47f9e3..c95212b642 100644
--- a/app/nginx.conf
+++ b/app/nginx.conf
@@ -145,6 +145,13 @@ server {
     resolver 8.8.8.8 8.8.4.4 valid=300s;
     resolver_timeout 5s;
 
+    # Declare UTF-8 on text responses. llms.txt and robots.txt contain UTF-8
+    # em dashes and arrows but were served as bare `text/plain` — a strict
+    # client may decode them as Latin-1 mojibake. Applies to nginx's default
+    # charset_types (text/html, text/plain, ...); responses that already
+    # declare a charset (the proxied API pages) keep theirs.
+    charset utf-8;
+
     # Root directory for serving files
     root /usr/share/nginx/html;
     index index.html;
@@ -328,6 +335,10 @@ server {
     resolver 8.8.8.8 8.8.4.4 valid=300s;
     resolver_timeout 5s;
 
+    # Same UTF-8 declaration as the main block (llms.txt/robots.txt are
+    # served from this block too).
+    charset utf-8;
+
     root /usr/share/nginx/html;
     index index.html;
 

From e44955e101455787afce6a260355ec72c9e3c616 Mon Sep 17 00:00:00 2001
From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com>
Date: Wed, 19 Aug 2026 22:04:56 +0200
Subject: [PATCH 2/2] docs(changelog): reference #10492

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 66c00519a8..66927c106d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -128,7 +128,7 @@ aggregate instead: an italic *Catalog* line at the end of the version section an
 - **Text files declare UTF-8** — `llms.txt` and `robots.txt` carry em dashes and arrows but
   were served as bare `text/plain`, which a strict client decodes as Latin-1 mojibake. nginx now
   declares `charset utf-8` on text responses in both server blocks, and the daily bot-serving
-  monitor asserts it on `llms.txt`.
+  monitor asserts it on `llms.txt` (#10492).
 
 ### Changed