fix(app): declare utf-8 on text responses

.github/workflows/bot-serving-check.yml

@@ -29,10 +29,11 @@ jobs:
   bot-serving:
     runs-on: ubuntu-latest
     # 22 check() calls x (--retry 2 -> up to 3 attempts x --max-time 30) can
-    # reach ~33 min worst-case, plus one non-retried trailing-slash probe
-    # (30s); 38 leaves room to report a clean failure rather than dying to the
-    # job timeout, which reports nothing useful. Recompute this when adding
-    # checks: the ceiling is check() calls x 90s, plus margin.
+    # reach ~33 min worst-case, plus two non-retried probes (llms.txt charset,
+    # trailing slash — 30s each); 38 leaves room to report a clean failure
+    # rather than dying to the job timeout, which reports nothing useful.
+    # Recompute this when adding checks: the ceiling is check() calls x 90s,
+    # plus margin.
     timeout-minutes: 38
     steps:
       - name: Crawler UAs must get 200 + per-route titles
@@ -116,6 +117,16 @@ jobs:
           # shell. The catalogue-index line proves the API generated it.
           check "$HUMAN" "$ORIGIN/llms-full.txt" "# anyplot — full catalogue index"
 
+          # llms.txt carries UTF-8 punctuation (em dashes, arrows); without an
+          # explicit charset a strict client decodes it as Latin-1 mojibake.
+          ct=$(curl -sS --max-time 30 -A "$GOOGLEBOT" -o /dev/null -w '%{content_type}' "$ORIGIN/llms.txt")
+          case "$ct" in
+            *charset=utf-8*) echo "OK: llms.txt content-type: $ct" ;;
+            *)
+              echo "::error::llms.txt served without utf-8 charset: $ct"
+              fail=1 ;;
+          esac
+
           # A trailing slash must normalise to the canonical URL on THIS host.
           # It used to 307 to http://api.anyplot.ai/seo-proxy/... — internal
           # path, wrong host, plain http, and that host disallows all crawling.

CHANGELOG.md

@@ -125,6 +125,10 @@ aggregate instead: an italic *Catalog* line at the end of the version section an
   `<div id="root">` with no `<noscript>`, so any fetcher missing from the UA allowlist rendered
   literally nothing. A noscript block now points such clients at `llms.txt`, `llms-full.txt`,
   the JSON API and the GitHub repository (#10488).
+- **Text files declare UTF-8** — `llms.txt` and `robots.txt` carry em dashes and arrows but
+  were served as bare `text/plain`, which a strict client decodes as Latin-1 mojibake. nginx now
+  declares `charset utf-8` on text responses in both server blocks, and the daily bot-serving
+  monitor asserts it on `llms.txt` (#10492).
 
 ### Changed
 

app/nginx.conf

@@ -145,6 +145,13 @@ server {
     resolver 8.8.8.8 8.8.4.4 valid=300s;
     resolver_timeout 5s;
 
+    # Declare UTF-8 on text responses. llms.txt and robots.txt contain UTF-8
+    # em dashes and arrows but were served as bare `text/plain` — a strict
+    # client may decode them as Latin-1 mojibake. Applies to nginx's default
+    # charset_types (text/html, text/plain, ...); responses that already
+    # declare a charset (the proxied API pages) keep theirs.
+    charset utf-8;
+
     # Root directory for serving files
     root /usr/share/nginx/html;
     index index.html;
@@ -328,6 +335,10 @@ server {
     resolver 8.8.8.8 8.8.4.4 valid=300s;
     resolver_timeout 5s;
 
+    # Same UTF-8 declaration as the main block (llms.txt/robots.txt are
+    # served from this block too).
+    charset utf-8;
+
     root /usr/share/nginx/html;
     index index.html;