diff --git a/scripts/arxiv_refs.py b/scripts/arxiv_refs.py index a153e5c..02a781a 100644 --- a/scripts/arxiv_refs.py +++ b/scripts/arxiv_refs.py @@ -32,7 +32,11 @@ from __future__ import annotations import re +import subprocess +import sys +import time import unicodedata +import urllib.error import urllib.parse import urllib.request import xml.etree.ElementTree as ET @@ -58,6 +62,19 @@ API_URL = "https://export.arxiv.org/api/query" #: arXiv asks for one request every few seconds; the backfill honours this. API_DELAY_SECONDS = 3.0 +USER_AGENT = ( + "PyAutoMemory reading-queue backfill " + "(+https://github.com/PyAutoLabs/PyAutoMemory)" +) +ACCEPT = "application/atom+xml" +RETRY_DELAYS = (5, 15, 45, 90, 180) +RETRY_STATUSES = {406, 429, 500, 502, 503, 504} +TRANSIENT = (urllib.error.URLError, TimeoutError, ConnectionError) +_CURL_STATUS_MARKER = b"\n__PYAUTO_HTTP_STATUS__:" + + +class ArxivAPIError(RuntimeError): + """The arXiv API could not answer a lookup reliably.""" def arxiv_id(ref: str | None) -> str | None: @@ -151,13 +168,112 @@ def normalise_title(title: str) -> str: return _PUNCT_RE.sub("", folded.lower()) -def _api_query(params: dict) -> bytes: - url = f"{API_URL}?{urllib.parse.urlencode(params)}" +def _curl_get(url: str) -> bytes: + """Alternate transport for the Fastly 406 seen from urllib clients.""" + try: + result = subprocess.run( + [ + "curl", + "--silent", + "--show-error", + "--location", + "--connect-timeout", + "20", + "--max-time", + "60", + "--user-agent", + USER_AGENT, + "--header", + f"Accept: {ACCEPT}", + "--write-out", + _CURL_STATUS_MARKER.decode() + "%{http_code}", + url, + ], + capture_output=True, + check=False, + ) + except FileNotFoundError as e: + raise urllib.error.URLError("curl is not installed") from e + + if result.returncode != 0: + detail = result.stderr.decode("utf-8", "replace").strip() + raise urllib.error.URLError(detail or f"curl exited {result.returncode}") + + body, marker, status_raw = result.stdout.rpartition(_CURL_STATUS_MARKER) + if not marker: + raise urllib.error.URLError("curl returned no HTTP status marker") + try: + status = int(status_raw.strip()) + except ValueError as e: + raise urllib.error.URLError( + f"curl returned invalid HTTP status {status_raw!r}" + ) from e + if not 200 <= status < 300: + raise urllib.error.HTTPError( + url, status, f"curl HTTP {status}", {}, None + ) + return body + + +def _request_once(url: str) -> bytes: req = urllib.request.Request( - url, headers={"User-Agent": "PyAutoMemory reading-queue backfill " - "(+https://github.com/PyAutoLabs/PyAutoMemory)"}) - with urllib.request.urlopen(req, timeout=60) as resp: - return resp.read() + url, headers={"User-Agent": USER_AGENT, "Accept": ACCEPT} + ) + try: + with urllib.request.urlopen(req, timeout=60) as resp: + return resp.read() + except urllib.error.HTTPError as e: + if e.code != 406: + raise + print( + " arXiv API urllib HTTP 406 — retrying this request with curl", + file=sys.stderr, + ) + return _curl_get(url) + + +def _api_query(params: dict, *, delays: tuple | None = None) -> bytes: + """Query arXiv with retry/backoff and make transport failure explicit. + + September 2026 exposed two independent failure modes: arXiv can throttle + shared runners with 429/406, and Fastly can reject Python's stdlib client + fingerprint with a persistent 406 while curl succeeds. The latter gets one + immediate alternate-transport attempt; real throttling still backs off. + """ + if delays is None: + delays = RETRY_DELAYS + url = f"{API_URL}?{urllib.parse.urlencode(params)}" + attempts = len(delays) + 1 + last_error = None + for attempt in range(1, attempts + 1): + try: + return _request_once(url) + except urllib.error.HTTPError as e: + last_error = e + if e.code not in RETRY_STATUSES or attempt == attempts: + break + wait = delays[attempt - 1] + retry_after = (e.headers or {}).get("Retry-After") + if retry_after and str(retry_after).strip().isdigit(): + wait = max(wait, min(int(retry_after), max(delays))) + reason = f"HTTP {e.code}" + except TRANSIENT as e: + last_error = e + if attempt == attempts: + break + wait = delays[attempt - 1] + reason = f"{type(e).__name__}: {e}" + + print( + f" arXiv API {reason} (attempt {attempt}/{attempts}) — " + f"retrying in {wait}s", + file=sys.stderr, + ) + time.sleep(wait) + + raise ArxivAPIError( + f"arXiv API unavailable after {attempts} attempt(s): {last_error}" + ) from last_error def parse_api_entries(raw: bytes) -> list[dict]: @@ -242,6 +358,12 @@ def resolve_title(title: str, max_results: int = 50, try: raw = query({"search_query": f'ti:"{phrase}"', "start": 0, "max_results": max_results}) - except Exception: - return None + except ArxivAPIError: + raise + except (urllib.error.URLError, TimeoutError, ConnectionError, OSError) as e: + # Injected query functions in tests and alternate callers may not use + # _api_query(), so normalise their transport failures to the same + # explicit signal. None remains reserved for a real arXiv answer that + # contains no unique exact title match. + raise ArxivAPIError(f"arXiv API lookup failed: {e}") from e return match_entries(title, parse_api_entries(raw)) diff --git a/scripts/backfill_arxiv_refs.py b/scripts/backfill_arxiv_refs.py index dbdd3a9..a666dd5 100644 --- a/scripts/backfill_arxiv_refs.py +++ b/scripts/backfill_arxiv_refs.py @@ -122,9 +122,10 @@ def backfill(root: Path, *, write: bool, limit: int, mark_unresolved: bool = Fal so no later run asks about it again — a human-driven cleanup, never the nightly's own decision (see :func:`mark_unresolved_line`). """ - stats = {"scanned": 0, "matched": 0, "unmatched": 0, "marked": 0, - "files": []} + stats = {"scanned": 0, "matched": 0, "unmatched": 0, "api_failed": 0, + "marked": 0, "files": []} budget = limit + api_down = False for name in TARGETS: path = root / name if not path.exists(): @@ -137,7 +138,17 @@ def backfill(root: Path, *, write: bool, limit: int, mark_unresolved: bool = Fal if budget <= 0: break budget -= 1 - ref = resolve(title) + try: + ref = resolve(title) + except arxiv_refs.ArxivAPIError as e: + # A transport failure is not a negative identification. Stop + # here: another title would hit the same dead endpoint, and + # --mark-unresolved must never turn an outage into permanent + # NOTE markers. + stats["api_failed"] += 1 + api_down = True + log(f" ! arXiv API failure {title[:90]} ({e})") + break if ref: lines[idx] = apply_ref(lines[idx], ref) changed += 1 @@ -162,8 +173,11 @@ def backfill(root: Path, *, write: bool, limit: int, mark_unresolved: bool = Fal if write: path.write_text("\n".join(lines) + "\n") marked = f", {file_marked} NOTE-marked" if file_marked else "" + failed = f", {stats['api_failed']} API failure" if stats["api_failed"] else "" log(f"{name}: {len(todo)} ref-less, {changed - file_marked} resolved" - f"{marked}{'' if write else ' (dry run — nothing written)'}") + f"{marked}{failed}{'' if write else ' (dry run — nothing written)'}") + if api_down: + break return stats @@ -185,12 +199,13 @@ def main(argv: list[str] | None = None) -> int: stats = backfill(args.root, write=args.write, limit=args.limit, mark_unresolved=args.mark_unresolved) marked = f", {stats['marked']} NOTE-marked" if stats["marked"] else "" + failed = f", {stats['api_failed']} API failure" if stats["api_failed"] else "" print(f"\n{stats['matched']} resolved, {stats['unmatched']} unresolved" - f"{marked}, {stats['scanned']} ref-less lines in total.") - # Not an error: an unresolved title is a normal outcome (not on arXiv, or - # too ambiguous to be safe), and a run that resolves nothing must not fail - # the nightly job. - return 0 + f"{marked}{failed}, {stats['scanned']} ref-less lines in total.") + # A genuine no-match is normal and stays exit 0. A transport failure is a + # broken run: after _api_query has already exhausted its retry ladder we + # stop rather than misreporting the whole queue as unresolved. + return 1 if stats["api_failed"] else 0 if __name__ == "__main__": diff --git a/tests/test_arxiv_refs.py b/tests/test_arxiv_refs.py index 824eebe..e8b8cd6 100644 --- a/tests/test_arxiv_refs.py +++ b/tests/test_arxiv_refs.py @@ -11,8 +11,11 @@ from __future__ import annotations import sys +import urllib.error from pathlib import Path +import pytest + sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) import arxiv_refs # noqa: E402 @@ -181,7 +184,7 @@ def fake(params): query=fake) is None -def test_resolve_title_is_injectable_and_survives_a_dead_api(): +def test_resolve_title_is_injectable_and_reports_a_dead_api(): calls = [] def fake(params): @@ -195,7 +198,25 @@ def fake(params): def dead(params): raise OSError("arxiv is down") - assert arxiv_refs.resolve_title("Anything", query=dead) is None + with pytest.raises(arxiv_refs.ArxivAPIError): + arxiv_refs.resolve_title("Anything", query=dead) + + +def test_api_query_falls_back_to_curl_on_urllib_406(monkeypatch): + def rejected(*args, **kwargs): + raise urllib.error.HTTPError( + "https://export.arxiv.org/api/query", 406, "edge refusal", {}, None + ) + + seen = [] + monkeypatch.setattr(arxiv_refs.urllib.request, "urlopen", rejected) + monkeypatch.setattr( + arxiv_refs, "_curl_get", + lambda url: seen.append(url) or b"", + ) + + assert arxiv_refs._api_query({"search_query": "all:test"}, delays=()) == b"" + assert len(seen) == 1 # --- the backfill ------------------------------------------------------------- @@ -264,6 +285,45 @@ def resolve(title): assert len(seen) == 2 +def test_backfill_reports_transport_failure_without_marking(tmp_path): + root = _tree(tmp_path) + before_queue = (root / "reading-queue.md").read_text() + + def failed(_title): + raise arxiv_refs.ArxivAPIError("HTTP 406 after retries") + + stats = backfill.backfill( + root, + write=True, + limit=10, + mark_unresolved=True, + resolve=failed, + sleep=lambda _: None, + log=lambda *a: None, + ) + + assert stats["api_failed"] == 1 + assert stats["unmatched"] == 0 + assert stats["marked"] == 0 + assert (root / "reading-queue.md").read_text() == before_queue + + +def test_main_exits_nonzero_when_the_api_failed(monkeypatch, tmp_path): + monkeypatch.setattr( + backfill, + "backfill", + lambda *args, **kwargs: { + "scanned": 3, + "matched": 0, + "unmatched": 0, + "api_failed": 1, + "marked": 0, + "files": [], + }, + ) + assert backfill.main(["--root", str(tmp_path)]) == 1 + + def test_backfill_is_idempotent(tmp_path): root = _tree(tmp_path) kw = dict(write=True, limit=10, resolve=lambda t: "2608.11111",