fix: stop hiding arXiv API failures in paper ref backfill

scripts/arxiv_refs.py

@@ -32,7 +32,11 @@
 from __future__ import annotations
 
 import re
+import subprocess
+import sys
+import time
 import unicodedata
+import urllib.error
 import urllib.parse
 import urllib.request
 import xml.etree.ElementTree as ET
@@ -58,6 +62,19 @@
 API_URL = "https://export.arxiv.org/api/query"
 #: arXiv asks for one request every few seconds; the backfill honours this.
 API_DELAY_SECONDS = 3.0
+USER_AGENT = (
+    "PyAutoMemory reading-queue backfill "
+    "(+https://github.com/PyAutoLabs/PyAutoMemory)"
+)
+ACCEPT = "application/atom+xml"
+RETRY_DELAYS = (5, 15, 45, 90, 180)
+RETRY_STATUSES = {406, 429, 500, 502, 503, 504}
+TRANSIENT = (urllib.error.URLError, TimeoutError, ConnectionError)
+_CURL_STATUS_MARKER = b"\n__PYAUTO_HTTP_STATUS__:"
+
+
+class ArxivAPIError(RuntimeError):
+    """The arXiv API could not answer a lookup reliably."""
 
 
 def arxiv_id(ref: str | None) -> str | None:
@@ -151,13 +168,112 @@ def normalise_title(title: str) -> str:
     return _PUNCT_RE.sub("", folded.lower())
 
 
-def _api_query(params: dict) -> bytes:
-    url = f"{API_URL}?{urllib.parse.urlencode(params)}"
+def _curl_get(url: str) -> bytes:
+    """Alternate transport for the Fastly 406 seen from urllib clients."""
+    try:
+        result = subprocess.run(
+            [
+                "curl",
+                "--silent",
+                "--show-error",
+                "--location",
+                "--connect-timeout",
+                "20",
+                "--max-time",
+                "60",
+                "--user-agent",
+                USER_AGENT,
+                "--header",
+                f"Accept: {ACCEPT}",
+                "--write-out",
+                _CURL_STATUS_MARKER.decode() + "%{http_code}",
+                url,
+            ],
+            capture_output=True,
+            check=False,
+        )
+    except FileNotFoundError as e:
+        raise urllib.error.URLError("curl is not installed") from e
+
+    if result.returncode != 0:
+        detail = result.stderr.decode("utf-8", "replace").strip()
+        raise urllib.error.URLError(detail or f"curl exited {result.returncode}")
+
+    body, marker, status_raw = result.stdout.rpartition(_CURL_STATUS_MARKER)
+    if not marker:
+        raise urllib.error.URLError("curl returned no HTTP status marker")
+    try:
+        status = int(status_raw.strip())
+    except ValueError as e:
+        raise urllib.error.URLError(
+            f"curl returned invalid HTTP status {status_raw!r}"
+        ) from e
+    if not 200 <= status < 300:
+        raise urllib.error.HTTPError(
+            url, status, f"curl HTTP {status}", {}, None
+        )
+    return body
+
+
+def _request_once(url: str) -> bytes:
     req = urllib.request.Request(
-        url, headers={"User-Agent": "PyAutoMemory reading-queue backfill "
-                                    "(+https://github.com/PyAutoLabs/PyAutoMemory)"})
-    with urllib.request.urlopen(req, timeout=60) as resp:
-        return resp.read()
+        url, headers={"User-Agent": USER_AGENT, "Accept": ACCEPT}
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=60) as resp:
+            return resp.read()
+    except urllib.error.HTTPError as e:
+        if e.code != 406:
+            raise
+        print(
+            "  arXiv API urllib HTTP 406 — retrying this request with curl",
+            file=sys.stderr,
+        )
+        return _curl_get(url)
+
+
+def _api_query(params: dict, *, delays: tuple | None = None) -> bytes:
+    """Query arXiv with retry/backoff and make transport failure explicit.
+
+    September 2026 exposed two independent failure modes: arXiv can throttle
+    shared runners with 429/406, and Fastly can reject Python's stdlib client
+    fingerprint with a persistent 406 while curl succeeds. The latter gets one
+    immediate alternate-transport attempt; real throttling still backs off.
+    """
+    if delays is None:
+        delays = RETRY_DELAYS
+    url = f"{API_URL}?{urllib.parse.urlencode(params)}"
+    attempts = len(delays) + 1
+    last_error = None
+    for attempt in range(1, attempts + 1):
+        try:
+            return _request_once(url)
+        except urllib.error.HTTPError as e:
+            last_error = e
+            if e.code not in RETRY_STATUSES or attempt == attempts:
+                break
+            wait = delays[attempt - 1]
+            retry_after = (e.headers or {}).get("Retry-After")
+            if retry_after and str(retry_after).strip().isdigit():
+                wait = max(wait, min(int(retry_after), max(delays)))
+            reason = f"HTTP {e.code}"
+        except TRANSIENT as e:
+            last_error = e
+            if attempt == attempts:
+                break
+            wait = delays[attempt - 1]
+            reason = f"{type(e).__name__}: {e}"
+
+        print(
+            f"  arXiv API {reason} (attempt {attempt}/{attempts}) — "
+            f"retrying in {wait}s",
+            file=sys.stderr,
+        )
+        time.sleep(wait)
+
+    raise ArxivAPIError(
+        f"arXiv API unavailable after {attempts} attempt(s): {last_error}"
+    ) from last_error
 
 
 def parse_api_entries(raw: bytes) -> list[dict]:
@@ -242,6 +358,12 @@ def resolve_title(title: str, max_results: int = 50,
     try:
         raw = query({"search_query": f'ti:"{phrase}"',
                      "start": 0, "max_results": max_results})
-    except Exception:
-        return None
+    except ArxivAPIError:
+        raise
+    except (urllib.error.URLError, TimeoutError, ConnectionError, OSError) as e:
+        # Injected query functions in tests and alternate callers may not use
+        # _api_query(), so normalise their transport failures to the same
+        # explicit signal. None remains reserved for a real arXiv answer that
+        # contains no unique exact title match.
+        raise ArxivAPIError(f"arXiv API lookup failed: {e}") from e
     return match_entries(title, parse_api_entries(raw))

scripts/backfill_arxiv_refs.py

@@ -122,9 +122,10 @@ def backfill(root: Path, *, write: bool, limit: int, mark_unresolved: bool = Fal
     so no later run asks about it again — a human-driven cleanup, never the
     nightly's own decision (see :func:`mark_unresolved_line`).
     """
-    stats = {"scanned": 0, "matched": 0, "unmatched": 0, "marked": 0,
-             "files": []}
+    stats = {"scanned": 0, "matched": 0, "unmatched": 0, "api_failed": 0,
+             "marked": 0, "files": []}
     budget = limit
+    api_down = False
     for name in TARGETS:
         path = root / name
         if not path.exists():
@@ -137,7 +138,17 @@ def backfill(root: Path, *, write: bool, limit: int, mark_unresolved: bool = Fal
             if budget <= 0:
                 break
             budget -= 1
-            ref = resolve(title)
+            try:
+                ref = resolve(title)
+            except arxiv_refs.ArxivAPIError as e:
+                # A transport failure is not a negative identification. Stop
+                # here: another title would hit the same dead endpoint, and
+                # --mark-unresolved must never turn an outage into permanent
+                # NOTE markers.
+                stats["api_failed"] += 1
+                api_down = True
+                log(f"  ! arXiv API failure   {title[:90]} ({e})")
+                break
             if ref:
                 lines[idx] = apply_ref(lines[idx], ref)
                 changed += 1
@@ -162,8 +173,11 @@ def backfill(root: Path, *, write: bool, limit: int, mark_unresolved: bool = Fal
             if write:
                 path.write_text("\n".join(lines) + "\n")
         marked = f", {file_marked} NOTE-marked" if file_marked else ""
+        failed = f", {stats['api_failed']} API failure" if stats["api_failed"] else ""
         log(f"{name}: {len(todo)} ref-less, {changed - file_marked} resolved"
-            f"{marked}{'' if write else ' (dry run — nothing written)'}")
+            f"{marked}{failed}{'' if write else ' (dry run — nothing written)'}")
+        if api_down:
+            break
     return stats
 
 
@@ -185,12 +199,13 @@ def main(argv: list[str] | None = None) -> int:
     stats = backfill(args.root, write=args.write, limit=args.limit,
                      mark_unresolved=args.mark_unresolved)
     marked = f", {stats['marked']} NOTE-marked" if stats["marked"] else ""
+    failed = f", {stats['api_failed']} API failure" if stats["api_failed"] else ""
     print(f"\n{stats['matched']} resolved, {stats['unmatched']} unresolved"
-          f"{marked}, {stats['scanned']} ref-less lines in total.")
-    # Not an error: an unresolved title is a normal outcome (not on arXiv, or
-    # too ambiguous to be safe), and a run that resolves nothing must not fail
-    # the nightly job.
-    return 0
+          f"{marked}{failed}, {stats['scanned']} ref-less lines in total.")
+    # A genuine no-match is normal and stays exit 0. A transport failure is a
+    # broken run: after _api_query has already exhausted its retry ladder we
+    # stop rather than misreporting the whole queue as unresolved.
+    return 1 if stats["api_failed"] else 0
 
 
 if __name__ == "__main__":

tests/test_arxiv_refs.py

@@ -11,8 +11,11 @@
 from __future__ import annotations
 
 import sys
+import urllib.error
 from pathlib import Path
 
+import pytest
+
 sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
 
 import arxiv_refs  # noqa: E402
@@ -181,7 +184,7 @@ def fake(params):
         query=fake) is None
 
 
-def test_resolve_title_is_injectable_and_survives_a_dead_api():
+def test_resolve_title_is_injectable_and_reports_a_dead_api():
     calls = []
 
     def fake(params):
@@ -195,7 +198,25 @@ def fake(params):
     def dead(params):
         raise OSError("arxiv is down")
 
-    assert arxiv_refs.resolve_title("Anything", query=dead) is None
+    with pytest.raises(arxiv_refs.ArxivAPIError):
+        arxiv_refs.resolve_title("Anything", query=dead)
+
+
+def test_api_query_falls_back_to_curl_on_urllib_406(monkeypatch):
+    def rejected(*args, **kwargs):
+        raise urllib.error.HTTPError(
+            "https://export.arxiv.org/api/query", 406, "edge refusal", {}, None
+        )
+
+    seen = []
+    monkeypatch.setattr(arxiv_refs.urllib.request, "urlopen", rejected)
+    monkeypatch.setattr(
+        arxiv_refs, "_curl_get",
+        lambda url: seen.append(url) or b"<feed/>",
+    )
+
+    assert arxiv_refs._api_query({"search_query": "all:test"}, delays=()) == b"<feed/>"
+    assert len(seen) == 1
 
 
 # --- the backfill -------------------------------------------------------------
@@ -264,6 +285,45 @@ def resolve(title):
     assert len(seen) == 2
 
 
+def test_backfill_reports_transport_failure_without_marking(tmp_path):
+    root = _tree(tmp_path)
+    before_queue = (root / "reading-queue.md").read_text()
+
+    def failed(_title):
+        raise arxiv_refs.ArxivAPIError("HTTP 406 after retries")
+
+    stats = backfill.backfill(
+        root,
+        write=True,
+        limit=10,
+        mark_unresolved=True,
+        resolve=failed,
+        sleep=lambda _: None,
+        log=lambda *a: None,
+    )
+
+    assert stats["api_failed"] == 1
+    assert stats["unmatched"] == 0
+    assert stats["marked"] == 0
+    assert (root / "reading-queue.md").read_text() == before_queue
+
+
+def test_main_exits_nonzero_when_the_api_failed(monkeypatch, tmp_path):
+    monkeypatch.setattr(
+        backfill,
+        "backfill",
+        lambda *args, **kwargs: {
+            "scanned": 3,
+            "matched": 0,
+            "unmatched": 0,
+            "api_failed": 1,
+            "marked": 0,
+            "files": [],
+        },
+    )
+    assert backfill.main(["--root", str(tmp_path)]) == 1
+
+
 def test_backfill_is_idempotent(tmp_path):
     root = _tree(tmp_path)
     kw = dict(write=True, limit=10, resolve=lambda t: "2608.11111",