diff --git a/scripts/arxiv_refs.py b/scripts/arxiv_refs.py
index a153e5c..02a781a 100644
--- a/scripts/arxiv_refs.py
+++ b/scripts/arxiv_refs.py
@@ -32,7 +32,11 @@
from __future__ import annotations
import re
+import subprocess
+import sys
+import time
import unicodedata
+import urllib.error
import urllib.parse
import urllib.request
import xml.etree.ElementTree as ET
@@ -58,6 +62,19 @@
API_URL = "https://export.arxiv.org/api/query"
#: arXiv asks for one request every few seconds; the backfill honours this.
API_DELAY_SECONDS = 3.0
+USER_AGENT = (
+ "PyAutoMemory reading-queue backfill "
+ "(+https://github.com/PyAutoLabs/PyAutoMemory)"
+)
+ACCEPT = "application/atom+xml"
+RETRY_DELAYS = (5, 15, 45, 90, 180)
+RETRY_STATUSES = {406, 429, 500, 502, 503, 504}
+TRANSIENT = (urllib.error.URLError, TimeoutError, ConnectionError)
+_CURL_STATUS_MARKER = b"\n__PYAUTO_HTTP_STATUS__:"
+
+
+class ArxivAPIError(RuntimeError):
+ """The arXiv API could not answer a lookup reliably."""
def arxiv_id(ref: str | None) -> str | None:
@@ -151,13 +168,112 @@ def normalise_title(title: str) -> str:
return _PUNCT_RE.sub("", folded.lower())
-def _api_query(params: dict) -> bytes:
- url = f"{API_URL}?{urllib.parse.urlencode(params)}"
+def _curl_get(url: str) -> bytes:
+ """Alternate transport for the Fastly 406 seen from urllib clients."""
+ try:
+ result = subprocess.run(
+ [
+ "curl",
+ "--silent",
+ "--show-error",
+ "--location",
+ "--connect-timeout",
+ "20",
+ "--max-time",
+ "60",
+ "--user-agent",
+ USER_AGENT,
+ "--header",
+ f"Accept: {ACCEPT}",
+ "--write-out",
+ _CURL_STATUS_MARKER.decode() + "%{http_code}",
+ url,
+ ],
+ capture_output=True,
+ check=False,
+ )
+ except FileNotFoundError as e:
+ raise urllib.error.URLError("curl is not installed") from e
+
+ if result.returncode != 0:
+ detail = result.stderr.decode("utf-8", "replace").strip()
+ raise urllib.error.URLError(detail or f"curl exited {result.returncode}")
+
+ body, marker, status_raw = result.stdout.rpartition(_CURL_STATUS_MARKER)
+ if not marker:
+ raise urllib.error.URLError("curl returned no HTTP status marker")
+ try:
+ status = int(status_raw.strip())
+ except ValueError as e:
+ raise urllib.error.URLError(
+ f"curl returned invalid HTTP status {status_raw!r}"
+ ) from e
+ if not 200 <= status < 300:
+ raise urllib.error.HTTPError(
+ url, status, f"curl HTTP {status}", {}, None
+ )
+ return body
+
+
+def _request_once(url: str) -> bytes:
req = urllib.request.Request(
- url, headers={"User-Agent": "PyAutoMemory reading-queue backfill "
- "(+https://github.com/PyAutoLabs/PyAutoMemory)"})
- with urllib.request.urlopen(req, timeout=60) as resp:
- return resp.read()
+ url, headers={"User-Agent": USER_AGENT, "Accept": ACCEPT}
+ )
+ try:
+ with urllib.request.urlopen(req, timeout=60) as resp:
+ return resp.read()
+ except urllib.error.HTTPError as e:
+ if e.code != 406:
+ raise
+ print(
+ " arXiv API urllib HTTP 406 — retrying this request with curl",
+ file=sys.stderr,
+ )
+ return _curl_get(url)
+
+
+def _api_query(params: dict, *, delays: tuple | None = None) -> bytes:
+ """Query arXiv with retry/backoff and make transport failure explicit.
+
+ September 2026 exposed two independent failure modes: arXiv can throttle
+ shared runners with 429/406, and Fastly can reject Python's stdlib client
+ fingerprint with a persistent 406 while curl succeeds. The latter gets one
+ immediate alternate-transport attempt; real throttling still backs off.
+ """
+ if delays is None:
+ delays = RETRY_DELAYS
+ url = f"{API_URL}?{urllib.parse.urlencode(params)}"
+ attempts = len(delays) + 1
+ last_error = None
+ for attempt in range(1, attempts + 1):
+ try:
+ return _request_once(url)
+ except urllib.error.HTTPError as e:
+ last_error = e
+ if e.code not in RETRY_STATUSES or attempt == attempts:
+ break
+ wait = delays[attempt - 1]
+ retry_after = (e.headers or {}).get("Retry-After")
+ if retry_after and str(retry_after).strip().isdigit():
+ wait = max(wait, min(int(retry_after), max(delays)))
+ reason = f"HTTP {e.code}"
+ except TRANSIENT as e:
+ last_error = e
+ if attempt == attempts:
+ break
+ wait = delays[attempt - 1]
+ reason = f"{type(e).__name__}: {e}"
+
+ print(
+ f" arXiv API {reason} (attempt {attempt}/{attempts}) — "
+ f"retrying in {wait}s",
+ file=sys.stderr,
+ )
+ time.sleep(wait)
+
+ raise ArxivAPIError(
+ f"arXiv API unavailable after {attempts} attempt(s): {last_error}"
+ ) from last_error
def parse_api_entries(raw: bytes) -> list[dict]:
@@ -242,6 +358,12 @@ def resolve_title(title: str, max_results: int = 50,
try:
raw = query({"search_query": f'ti:"{phrase}"',
"start": 0, "max_results": max_results})
- except Exception:
- return None
+ except ArxivAPIError:
+ raise
+ except (urllib.error.URLError, TimeoutError, ConnectionError, OSError) as e:
+ # Injected query functions in tests and alternate callers may not use
+ # _api_query(), so normalise their transport failures to the same
+ # explicit signal. None remains reserved for a real arXiv answer that
+ # contains no unique exact title match.
+ raise ArxivAPIError(f"arXiv API lookup failed: {e}") from e
return match_entries(title, parse_api_entries(raw))
diff --git a/scripts/backfill_arxiv_refs.py b/scripts/backfill_arxiv_refs.py
index dbdd3a9..a666dd5 100644
--- a/scripts/backfill_arxiv_refs.py
+++ b/scripts/backfill_arxiv_refs.py
@@ -122,9 +122,10 @@ def backfill(root: Path, *, write: bool, limit: int, mark_unresolved: bool = Fal
so no later run asks about it again — a human-driven cleanup, never the
nightly's own decision (see :func:`mark_unresolved_line`).
"""
- stats = {"scanned": 0, "matched": 0, "unmatched": 0, "marked": 0,
- "files": []}
+ stats = {"scanned": 0, "matched": 0, "unmatched": 0, "api_failed": 0,
+ "marked": 0, "files": []}
budget = limit
+ api_down = False
for name in TARGETS:
path = root / name
if not path.exists():
@@ -137,7 +138,17 @@ def backfill(root: Path, *, write: bool, limit: int, mark_unresolved: bool = Fal
if budget <= 0:
break
budget -= 1
- ref = resolve(title)
+ try:
+ ref = resolve(title)
+ except arxiv_refs.ArxivAPIError as e:
+ # A transport failure is not a negative identification. Stop
+ # here: another title would hit the same dead endpoint, and
+ # --mark-unresolved must never turn an outage into permanent
+ # NOTE markers.
+ stats["api_failed"] += 1
+ api_down = True
+ log(f" ! arXiv API failure {title[:90]} ({e})")
+ break
if ref:
lines[idx] = apply_ref(lines[idx], ref)
changed += 1
@@ -162,8 +173,11 @@ def backfill(root: Path, *, write: bool, limit: int, mark_unresolved: bool = Fal
if write:
path.write_text("\n".join(lines) + "\n")
marked = f", {file_marked} NOTE-marked" if file_marked else ""
+ failed = f", {stats['api_failed']} API failure" if stats["api_failed"] else ""
log(f"{name}: {len(todo)} ref-less, {changed - file_marked} resolved"
- f"{marked}{'' if write else ' (dry run — nothing written)'}")
+ f"{marked}{failed}{'' if write else ' (dry run — nothing written)'}")
+ if api_down:
+ break
return stats
@@ -185,12 +199,13 @@ def main(argv: list[str] | None = None) -> int:
stats = backfill(args.root, write=args.write, limit=args.limit,
mark_unresolved=args.mark_unresolved)
marked = f", {stats['marked']} NOTE-marked" if stats["marked"] else ""
+ failed = f", {stats['api_failed']} API failure" if stats["api_failed"] else ""
print(f"\n{stats['matched']} resolved, {stats['unmatched']} unresolved"
- f"{marked}, {stats['scanned']} ref-less lines in total.")
- # Not an error: an unresolved title is a normal outcome (not on arXiv, or
- # too ambiguous to be safe), and a run that resolves nothing must not fail
- # the nightly job.
- return 0
+ f"{marked}{failed}, {stats['scanned']} ref-less lines in total.")
+ # A genuine no-match is normal and stays exit 0. A transport failure is a
+ # broken run: after _api_query has already exhausted its retry ladder we
+ # stop rather than misreporting the whole queue as unresolved.
+ return 1 if stats["api_failed"] else 0
if __name__ == "__main__":
diff --git a/tests/test_arxiv_refs.py b/tests/test_arxiv_refs.py
index 824eebe..e8b8cd6 100644
--- a/tests/test_arxiv_refs.py
+++ b/tests/test_arxiv_refs.py
@@ -11,8 +11,11 @@
from __future__ import annotations
import sys
+import urllib.error
from pathlib import Path
+import pytest
+
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
import arxiv_refs # noqa: E402
@@ -181,7 +184,7 @@ def fake(params):
query=fake) is None
-def test_resolve_title_is_injectable_and_survives_a_dead_api():
+def test_resolve_title_is_injectable_and_reports_a_dead_api():
calls = []
def fake(params):
@@ -195,7 +198,25 @@ def fake(params):
def dead(params):
raise OSError("arxiv is down")
- assert arxiv_refs.resolve_title("Anything", query=dead) is None
+ with pytest.raises(arxiv_refs.ArxivAPIError):
+ arxiv_refs.resolve_title("Anything", query=dead)
+
+
+def test_api_query_falls_back_to_curl_on_urllib_406(monkeypatch):
+ def rejected(*args, **kwargs):
+ raise urllib.error.HTTPError(
+ "https://export.arxiv.org/api/query", 406, "edge refusal", {}, None
+ )
+
+ seen = []
+ monkeypatch.setattr(arxiv_refs.urllib.request, "urlopen", rejected)
+ monkeypatch.setattr(
+ arxiv_refs, "_curl_get",
+ lambda url: seen.append(url) or b"",
+ )
+
+ assert arxiv_refs._api_query({"search_query": "all:test"}, delays=()) == b""
+ assert len(seen) == 1
# --- the backfill -------------------------------------------------------------
@@ -264,6 +285,45 @@ def resolve(title):
assert len(seen) == 2
+def test_backfill_reports_transport_failure_without_marking(tmp_path):
+ root = _tree(tmp_path)
+ before_queue = (root / "reading-queue.md").read_text()
+
+ def failed(_title):
+ raise arxiv_refs.ArxivAPIError("HTTP 406 after retries")
+
+ stats = backfill.backfill(
+ root,
+ write=True,
+ limit=10,
+ mark_unresolved=True,
+ resolve=failed,
+ sleep=lambda _: None,
+ log=lambda *a: None,
+ )
+
+ assert stats["api_failed"] == 1
+ assert stats["unmatched"] == 0
+ assert stats["marked"] == 0
+ assert (root / "reading-queue.md").read_text() == before_queue
+
+
+def test_main_exits_nonzero_when_the_api_failed(monkeypatch, tmp_path):
+ monkeypatch.setattr(
+ backfill,
+ "backfill",
+ lambda *args, **kwargs: {
+ "scanned": 3,
+ "matched": 0,
+ "unmatched": 0,
+ "api_failed": 1,
+ "marked": 0,
+ "files": [],
+ },
+ )
+ assert backfill.main(["--root", str(tmp_path)]) == 1
+
+
def test_backfill_is_idempotent(tmp_path):
root = _tree(tmp_path)
kw = dict(write=True, limit=10, resolve=lambda t: "2608.11111",