diff --git a/.github/workflows/arxiv_refs.yml b/.github/workflows/arxiv_refs.yml new file mode 100644 index 0000000..3b471b7 --- /dev/null +++ b/.github/workflows/arxiv_refs.yml @@ -0,0 +1,102 @@ +name: arXiv refs backfill + +# Gives ref-less reading-queue lines their arXiv id, so the Dashboard can offer +# the abstract page and the πŸ“„ PDF button instead of an arXiv title search. +# +# The queue predates the nightly digest: most of its lines are bare titles, and +# a bare title is only ever searchable. scripts/backfill_arxiv_refs.py resolves +# them against the arXiv API and appends `β€” ` where exactly one paper's +# title matches; anything looser is left alone, because a wrong id sends the +# reader confidently to the wrong paper. +# +# Idempotent and resumable: a line that already has a ref is skipped, so each +# run only works on what is left and the job goes quiet once the queue is fully +# resolved. --limit caps the arXiv calls per run (they are spaced ~3 s apart out +# of politeness, so the cap is really a runtime cap); the default drains the +# historical backlog over a few nights. Titles arXiv genuinely does not have are +# re-attempted each run β€” the deliberate cost of keeping no per-line failure +# state, and bounded by the same cap. + +on: + schedule: + # 04:10 UTC: after PyAutoMind's 02:00 digest has filed the night's papers, + # before the morning read. + - cron: "10 4 * * *" + workflow_dispatch: + inputs: + limit: + description: "Most arXiv lookups this run (blank = 60)" + required: false + default: "" + dry_run: + description: "Report only β€” resolve nothing into the files" + type: boolean + default: false + +# contents: write β†’ the queue commit; actions: write β†’ re-dispatch the +# Dashboard (a token push does not retrigger `on: push` workflows). +permissions: + contents: write + actions: write + +concurrency: + group: arxiv-refs + cancel-in-progress: false + +jobs: + backfill: + runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ github.token }} + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Resolve ref-less titles against arXiv + id: backfill + env: + LIMIT: ${{ inputs.limit }} + DRY_RUN: ${{ inputs.dry_run }} + run: | + set -u + args="--limit ${LIMIT:-60}" + [ "${DRY_RUN:-false}" = "true" ] || args="$args --write" + python scripts/backfill_arxiv_refs.py $args | tee /tmp/backfill.log + { + echo '## arXiv refs backfill' + echo '```' + cat /tmp/backfill.log + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + if [ -n "$(git status --porcelain -- reading-queue.md arxiv-inbox.md)" ]; then + echo "changed=yes" >> "$GITHUB_OUTPUT" + else + echo "changed=no" >> "$GITHUB_OUTPUT" + fi + + # The queue is validated like any other edit to it β€” a backfill that + # broke the line grammar must fail here, not on the board. + - name: Validate + if: steps.backfill.outputs.changed == 'yes' + run: make validate + + - name: Commit + push (explicit paths only) + if: steps.backfill.outputs.changed == 'yes' && github.ref == 'refs/heads/main' + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add reading-queue.md arxiv-inbox.md + git commit -m "queue: backfill arXiv refs for ref-less papers [skip ci]" + for attempt in 1 2 3; do + if git push; then exit 0; fi + git pull --rebase origin main || exit 1 + done + echo "::error::could not push the backfill commit after 3 attempts" + exit 1 + + - name: Refresh the dashboard + if: steps.backfill.outputs.changed == 'yes' && github.ref == 'refs/heads/main' + run: gh workflow run knowledge_board.yml --ref main diff --git a/README.md b/README.md index e4071fe..4f9fa0b 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,10 @@ for managing all of it from one page: the reading queue, the paper sections still needing a canonical citation key, and each sub-wiki's maturity β€” every work queue carrying a one-tap πŸ“‹ button that copies a paste-ready Claude prompt (file the next paper, resolve keys, -upgrade a stub, or `/memory ` to recall what's known). +upgrade a stub, or `/memory ` to recall what's known). Every queued +paper links to its arXiv abstract page and carries a πŸ“„ button onto the PDF +itself, so a phone can collect a stack of papers to read offline in one tap +each. ## Current contents diff --git a/arxiv-inbox.md b/arxiv-inbox.md index 51370e3..67e9710 100644 --- a/arxiv-inbox.md +++ b/arxiv-inbox.md @@ -9,8 +9,9 @@ Format: ` β€” ` (same title/ref convention as `reading-queue.md`). - The date is the arXiv **announcement** date the digest ran against, and is the expiry anchor β€” not the day the paper was submitted. -- The Dashboard renders each line with four one-tap actions: βž• add to the - reading queue, πŸ“₯ intake into memory, πŸ“‘ make citeable, βœ–οΈ dismiss. +- The Dashboard renders each line with a πŸ“„ button straight to the paper's PDF + (the ref above is what makes it possible) plus four one-tap actions: βž• add to + the reading queue, πŸ“₯ intake into memory, πŸ“‘ make citeable, βœ–οΈ dismiss. - Un-acted lines **lapse after 7 days** (`INBOX_WINDOW_DAYS` in `scripts/inbox_actions.py`) and are swept by the nightly job. A swept line is not lost: git history holds it. This is deliberate β€” the never-delete rule in diff --git a/reading-queue.md b/reading-queue.md index 72f7819..ac22bdc 100644 --- a/reading-queue.md +++ b/reading-queue.md @@ -5,6 +5,12 @@ Papers waiting to be read and filed. Format: - `##
` headers group papers by domain (matching `wiki//` where one exists). - One paper per line β€” a bare title, optionally followed by ` β€” `. +- **Add the ref whenever you know it.** It is what lets the Dashboard link the + paper's arXiv abstract page and put a πŸ“„ one-tap PDF button beside it; without + one the board can only offer an arXiv title *search*, which costs a results + page and a squint per paper. `scripts/backfill_arxiv_refs.py` (nightly, via + `.github/workflows/arxiv_refs.yml`) fills in the ones nobody wrote by hand, + and only when exactly one arXiv paper's title matches. - A read/consumed paper is never deleted: prefix its line with `DONE β€” ` and leave it in place (the reading history). - Filing a read paper: add its canonical entry to `bibliography/`, stub it in diff --git a/scripts/arxiv_refs.py b/scripts/arxiv_refs.py new file mode 100644 index 0000000..bfe2d91 --- /dev/null +++ b/scripts/arxiv_refs.py @@ -0,0 +1,181 @@ +"""scripts/arxiv_refs.py β€” the arXiv ref grammar, in one place. + +A reading-queue / arXiv-inbox paper line is a bare title, optionally followed +by ` β€” `. That trailing **ref** is what turns a queue line +into a one-tap paper on the Dashboard: with it the board can link the arXiv +*abstract* page and, next to it, the PDF itself β€” the whole point being that a +phone can collect the PDFs before a flight without a search-results detour. +Without a ref the board can only offer an arXiv title *search*, which costs a +page load and a tap per paper and never reaches a PDF. + +So this module holds two things, and nothing else: + +* **The grammar (pure, offline).** Every ref shape the queue has ever carried β€” + ``2103.16254``, ``arXiv:2103.16254v2``, ``https://arxiv.org/abs/2103.16254``, + ``https://arxiv.org/pdf/2103.16254.pdf``, the pre-2007 ``astro-ph/0601001`` β€” + normalises to one canonical id, from which :func:`abs_url` and :func:`pdf_url` + follow. A ref that is not an arXiv paper (a journal DOI, a PDF on a group + page) stays exactly as written: it is still a link, just not one with a + derivable PDF sibling. +* **The lookup (networked).** :func:`resolve_title` asks the arXiv API for the + id of a bare title. It is used only by ``scripts/backfill_arxiv_refs.py``, + which is what fills in the refs the historical bulk-pasted queue lines never + had. The board itself never calls it β€” the board stays fully local, rendered + from the checkout with no network (see ``scripts/board.py``). + +``scripts/board.py`` and ``scripts/inbox_actions.py`` keep their own +``…REF_RE`` line-splitting regexes: those answer "where does the title end", +which is a *line* question. This module answers "what paper is that ref", which +is an *arXiv* question. They are deliberately separate. +""" + +from __future__ import annotations + +import re +import unicodedata +import urllib.parse +import urllib.request +import xml.etree.ElementTree as ET + +ATOM = "{http://www.w3.org/2005/Atom}" + +#: Post-2007 identifiers: ``YYMM.NNNNN``, optionally versioned. +NEW_ID = r"\d{4}\.\d{4,5}" +#: Pre-2007 identifiers: ``archive[.subject]/YYMMNNN``, e.g. ``astro-ph/0601001``. +OLD_ID = r"[a-z-]+(?:\.[A-Z]{2})?/\d{7}" + +_ID_RE = re.compile(rf"^(?:arxiv:)?({NEW_ID}|{OLD_ID})(?:v\d+)?$", re.I) +_URL_RE = re.compile( + rf"^https?://(?:www\.)?arxiv\.org/(?:abs|pdf|format|ps)/" + rf"({NEW_ID}|{OLD_ID})(?:v\d+)?(?:\.pdf)?/?$", + re.I, +) + +ABS_BASE = "https://arxiv.org/abs/" +PDF_BASE = "https://arxiv.org/pdf/" +SEARCH_BASE = "https://arxiv.org/search/?searchtype=title&query=" + +API_URL = "https://export.arxiv.org/api/query" +#: arXiv asks for one request every few seconds; the backfill honours this. +API_DELAY_SECONDS = 3.0 + + +def arxiv_id(ref: str | None) -> str | None: + """The canonical, version-stripped arXiv id in `ref`, or None. + + Accepts a bare id, an ``arXiv:``-prefixed id, or any arxiv.org abs/pdf URL. + Anything else β€” a DOI, a publisher link, a PDF hosted elsewhere β€” is not an + arXiv paper as far as this repo is concerned, and returns None. + """ + if not ref: + return None + ref = ref.strip() + for pattern in (_ID_RE, _URL_RE): + m = pattern.match(ref) + if m: + return m.group(1).lower() if "/" in m.group(1) else m.group(1) + return None + + +def abs_url(ref: str | None) -> str | None: + """The arXiv abstract page for `ref`, or None when it is not an arXiv id. + + The abstract page β€” never the PDF β€” because it is the page that carries the + authors, the abstract and arXiv's own download links; the direct PDF is + offered beside it, as its own button. + """ + aid = arxiv_id(ref) + return ABS_BASE + aid if aid else None + + +def pdf_url(ref: str | None) -> str | None: + """The direct PDF for `ref`, or None when it is not an arXiv id. + + No ``.pdf`` suffix: arXiv serves the extensionless form and phones open it + in the built-in viewer, from which "save to Files" is one more tap. + """ + aid = arxiv_id(ref) + return PDF_BASE + aid if aid else None + + +def search_url(title: str) -> str: + """The arXiv title search for `title` β€” the fallback for a ref-less line. + + Strictly worse than a real ref (a results page, not a paper, and no PDF to + tap), which is why ``backfill_arxiv_refs.py`` exists to retire it. + """ + return SEARCH_BASE + urllib.parse.quote(title, safe="") + + +# --- title matching ----------------------------------------------------------- +_PUNCT_RE = re.compile(r"[^a-z0-9]+") + + +def normalise_title(title: str) -> str: + """A title reduced to its comparable core: lowercase alphanumerics only. + + arXiv, the digest and a hand-pasted queue line disagree constantly about + LaTeX (``$z=2.48$``), dashes (``-``/``–``/``β€”``), accents and double spaces, + and none of that disagreement is about *which paper it is*. Stripping it all + lets the backfill demand an exact match on what remains, rather than a fuzzy + match on what does not β€” a wrong id is worse than no id, because it links + the reader confidently to somebody else's paper. + """ + folded = unicodedata.normalize("NFKD", title) + folded = "".join(c for c in folded if not unicodedata.combining(c)) + return _PUNCT_RE.sub("", folded.lower()) + + +def _api_query(params: dict) -> bytes: + url = f"{API_URL}?{urllib.parse.urlencode(params)}" + req = urllib.request.Request( + url, headers={"User-Agent": "PyAutoMemory reading-queue backfill " + "(+https://github.com/PyAutoLabs/PyAutoMemory)"}) + with urllib.request.urlopen(req, timeout=60) as resp: + return resp.read() + + +def parse_api_entries(raw: bytes) -> list[dict]: + """``[{'id': …, 'title': …}, …]`` from an arXiv Atom response.""" + entries = [] + for entry in ET.fromstring(raw).findall(f"{ATOM}entry"): + ident = (entry.findtext(f"{ATOM}id") or "").strip() + title = " ".join((entry.findtext(f"{ATOM}title") or "").split()) + aid = arxiv_id(ident) + if aid and title: + entries.append({"id": aid, "title": title}) + return entries + + +def match_entries(title: str, entries: list[dict]) -> str | None: + """The id of the one entry whose title matches `title`, or None. + + "One" is literal: a title that normalises onto two different arXiv papers + (v1 cross-lists, a comment sharing its parent's title, two genuinely + identically-titled papers) is left for a human. Silence is the safe answer. + """ + want = normalise_title(title) + hits = {e["id"] for e in entries if normalise_title(e["title"]) == want} + return hits.pop() if len(hits) == 1 else None + + +def resolve_title(title: str, max_results: int = 20, + query: object = _api_query) -> str | None: + """Look `title` up on the arXiv API and return its id, or None. + + Networked β€” the Dashboard never calls this; only the backfill does. + `query` is injected so the matching can be tested without arXiv. + """ + # `ti:"…"` is a phrase search over titles. arXiv's search index chokes on + # unbalanced quotes and LaTeX, so the phrase is sent as plain words; the + # exactness comes from match_entries afterwards, never from the query. + words = re.sub(r'[^\w\s]', " ", title).split() + if not words: + return None + phrase = " ".join(words) + try: + raw = query({"search_query": f'ti:"{phrase}"', + "start": 0, "max_results": max_results}) + except Exception: + return None + return match_entries(title, parse_api_entries(raw)) diff --git a/scripts/backfill_arxiv_refs.py b/scripts/backfill_arxiv_refs.py new file mode 100644 index 0000000..1dcb988 --- /dev/null +++ b/scripts/backfill_arxiv_refs.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +"""scripts/backfill_arxiv_refs.py β€” give every queue line its arXiv ref. + +The Dashboard can only offer a paper's abstract page and its πŸ“„ PDF button when +the queue line carries a ref (``… β€” 2608.23534``). The nightly digest has +written refs since it existed, but the bulk of ``reading-queue.md`` predates it: +those lines are bare titles, and a bare title can only be rendered as an arXiv +*title search* β€” a results page, a squint and a tap, per paper, on a phone. + +This script retires that fallback. For every ref-less line it asks the arXiv API +for the title, and appends ``β€” `` **only** when exactly one arXiv paper's +title matches after normalisation (``arxiv_refs.match_entries``). Anything +looser is left alone: a wrong id is worse than a search box, because it sends +the reader confidently to somebody else's paper. + +It is idempotent and resumable β€” a line that already has a ref is skipped, so +successive runs only ever work on what is left. Lines arXiv genuinely does not +have (a journal-only paper, a thesis, a talk) are simply never matched and are +re-attempted each run; that is the cost of holding no per-line failure state, +and it is bounded by ``--limit``. + +Run by ``.github/workflows/arxiv_refs.yml`` (a checkout has no arXiv access +inside a sandboxed session; CI does). Locally: + + python scripts/backfill_arxiv_refs.py # report, change nothing + python scripts/backfill_arxiv_refs.py --write # apply + python scripts/backfill_arxiv_refs.py --write --limit 60 +""" + +from __future__ import annotations + +import argparse +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import arxiv_refs # noqa: E402 +import board # noqa: E402 +import inbox_actions # noqa: E402 + +MEMORY_HOME = Path(__file__).resolve().parents[1] + +#: The files whose paper lines carry refs. Both use the same title/ref grammar, +#: so both are backfilled by the same pass. +TARGETS = (inbox_actions.QUEUE_FILE, inbox_actions.INBOX_FILE) + + +def candidates(text: str, source: str) -> list[tuple[int, str]]: + """``[(line_index, title), …]`` for the lines this pass could fill in. + + Which lines *are* papers is not re-decided here: the queue's section gating + and the inbox's date-stamped line grammar are read straight off the two + parsers that render the board, so a line the board does not show as a paper + is never touched. Of those, a line qualifies when it is not already DONE + (reading history needs no new links) and carries no ref. + """ + out = [] + in_section = False + for i, raw in enumerate(text.splitlines()): + if source == inbox_actions.INBOX_FILE: + stamped = inbox_actions.parse_line(raw) + if not stamped: + continue + body = stamped[1] + else: + m = board.QUEUE_SECTION_RE.match(raw) + if m and not m.group(1).startswith("#"): + in_section = True + continue + if not in_section or not raw.strip(): + continue + body = raw.strip() + paper = board._parse_paper(body) + if paper["done"] or paper["ref"] or not paper["title"]: + continue + out.append((i, paper["title"])) + return out + + +def apply_ref(raw_line: str, ref: str) -> str: + """`raw_line` with ``β€” `` appended, preserving its exact prefix. + + Appending rather than rewriting keeps the inbox's date stamp, any stray + indentation and the title's own punctuation byte-for-byte intact β€” the queue + is a human-edited file and this script is a guest in it. + """ + return f"{raw_line.rstrip()} β€” {ref}" + + +def backfill(root: Path, *, write: bool, limit: int, + resolve=arxiv_refs.resolve_title, + delay: float = arxiv_refs.API_DELAY_SECONDS, + sleep=time.sleep, log=print) -> dict: + """Resolve up to `limit` ref-less titles across the queue and the inbox.""" + stats = {"scanned": 0, "matched": 0, "unmatched": 0, "files": []} + budget = limit + for name in TARGETS: + path = root / name + if not path.exists(): + continue + lines = path.read_text(errors="replace").splitlines() + todo = candidates("\n".join(lines), name) + stats["scanned"] += len(todo) + changed = 0 + for idx, title in todo: + if budget <= 0: + break + budget -= 1 + ref = resolve(title) + if ref: + lines[idx] = apply_ref(lines[idx], ref) + changed += 1 + stats["matched"] += 1 + log(f" βœ” {ref} {title[:90]}") + else: + stats["unmatched"] += 1 + log(f" Β· no unique match {title[:90]}") + # Politeness, not throughput: arXiv asks for a few seconds between + # calls, and this job has all night. + if budget > 0: + sleep(delay) + if changed: + stats["files"].append(name) + if write: + path.write_text("\n".join(lines) + "\n") + log(f"{name}: {len(todo)} ref-less, {changed} resolved" + f"{'' if write else ' (dry run β€” nothing written)'}") + return stats + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--write", action="store_true", + help="apply the refs (default: report only)") + ap.add_argument("--limit", type=int, default=60, + help="most arXiv lookups to make in one run (default 60)") + ap.add_argument("--root", type=Path, default=MEMORY_HOME) + args = ap.parse_args(argv) + + stats = backfill(args.root, write=args.write, limit=args.limit) + print(f"\n{stats['matched']} resolved, {stats['unmatched']} unresolved, " + f"{stats['scanned']} ref-less lines in total.") + # Not an error: an unresolved title is a normal outcome (not on arXiv, or + # too ambiguous to be safe), and a run that resolves nothing must not fail + # the nightly job. + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/board.py b/scripts/board.py index 47dead4..b4ff0f4 100644 --- a/scripts/board.py +++ b/scripts/board.py @@ -7,8 +7,10 @@ documented workflow (``bibliography/README.md`` "Adding a paper", ``wiki/CLAUDE.md``'s schema) β€” plus contents cards with ``/memory `` recall chips. Reading-queue sections expand to the individual papers: each -title links out (arXiv abstract page, or a title search when the line has no -ref) and carries three prefilled-GitHub-issue actions β€” πŸ“₯ intake-into-memory +title links out to the arXiv abstract page (or, when the line carries no ref, +a title search β€” see ``scripts/arxiv_refs.py``), carries a πŸ“„ button onto the +PDF itself so a phone can collect a stack of papers without a detour through +search, and carries three prefilled-GitHub-issue actions β€” πŸ“₯ intake-into-memory (really interesting: full filing), πŸ“‘ make-citeable (worth citing, not pivotal: bib entry + minimal sources section), both open work items carrying the human's free-text notes, and βœ… read-don't-file (processed automatically @@ -17,7 +19,8 @@ Above the queue sits the **arXiv inbox** (``arxiv-inbox.md``): what the nightly digest suggested overnight, each line showing how many days it has left before -it lapses and carrying two further actions β€” βž• add-to-the-queue and βœ–οΈ dismiss. +it lapses and carrying the same πŸ“„ PDF button plus two further actions β€” +βž• add-to-the-queue and βœ–οΈ dismiss. The inbox's format, window and transitions live in ``scripts/inbox_actions.py``; this module reads them rather than re-deriving them. @@ -62,6 +65,7 @@ # The arXiv inbox owns its own line format, window and transitions; the # board reads them rather than re-deriving them (scripts/inbox_actions.py). sys.path.insert(0, str(Path(__file__).resolve().parent)) +import arxiv_refs # noqa: E402 import inbox_actions # noqa: E402 # The family look lives once, in the Brain (``board/_theme.py``): the @@ -248,14 +252,29 @@ def _parse_paper(text: str) -> dict: def paper_url(paper: dict) -> str: - """Where to read the paper: its abstract page, or an arXiv title search.""" + """Where to read the paper: its abstract page, or an arXiv title search. + + An arXiv ref in any shape β€” bare id, ``arXiv:``-prefixed, an abs URL, a + ``/pdf/…pdf`` URL β€” resolves to the *abstract* page, so every queue line + lands on the same kind of page and the PDF is a separate, explicit button + (:func:`paper_pdf_url`). A non-arXiv ref is a link the human wrote and is + followed verbatim. A line with no ref at all can only be searched for; the + search is the fallback, not the design (``scripts/arxiv_refs.py``). + """ ref = paper.get("ref") if ref: - if ref.startswith("http"): - return ref - return "https://arxiv.org/abs/" + ref.removeprefix("arXiv:") - return ("https://arxiv.org/search/?searchtype=title&query=" - + _quote(paper.get("title", ""), safe="")) + return arxiv_refs.abs_url(ref) or ref + return arxiv_refs.search_url(paper.get("title", "")) + + +def paper_pdf_url(paper: dict) -> str: + """The paper's direct PDF, or "" when the line carries no arXiv ref. + + Empty is a rendering instruction: no ref, no πŸ“„ button β€” the board never + shows a button that cannot work. Retiring those empties is the backfill's + job (``scripts/backfill_arxiv_refs.py``), not the renderer's. + """ + return arxiv_refs.pdf_url(paper.get("ref")) or "" def _queue_issue_url(snapshot: dict, section: str, paper: dict, @@ -508,6 +527,9 @@ def _render_md(snapshot: dict) -> str: for p in inbox: label = p["title"].replace("[", "\\[").replace("]", "\\]") bits = [f"- [{label}]({paper_url(p)}) β€” _{p['days_left']}d left_"] + pdf = paper_pdf_url(p) + if pdf: + bits.append(f"[pdf πŸ“„]({pdf})") for action, chip in (("add", "queue βž•"), ("intake", "intake πŸ“₯"), ("cite", "cite πŸ“‘"), ("dismiss", "dismiss βœ–οΈ")): u = _queue_issue_url(snapshot, inbox_actions.INBOX_TARGET_SECTION, @@ -533,6 +555,9 @@ def _render_md(snapshot: dict) -> str: continue label = p["title"].replace("[", "\\[").replace("]", "\\]") bits = [f"- [{label}]({paper_url(p)})"] + pdf = paper_pdf_url(p) + if pdf: + bits.append(f"[pdf πŸ“„]({pdf})") for action, chip in (("intake", "intake πŸ“₯"), ("cite", "cite πŸ“‘"), ("read", "read βœ…")): u = _queue_issue_url(snapshot, q["section"], p, action) @@ -606,6 +631,7 @@ def _bar(statuses: dict) -> str: border:1px solid var(--line);border-radius:6px;background:var(--btn)} a.act:hover{background:var(--tint);border-color:var(--accent); text-decoration:none} +a.act.pdf{border-color:var(--accent)} details.hist{margin:.25rem 0 .25rem 1.3rem} details.hist>summary{cursor:pointer} #pfilter{width:100%;padding:.45rem .6rem;margin:.25rem 0 .5rem; @@ -667,6 +693,24 @@ def _bar(statuses: dict) -> str: """ +def _pdf_act(paper: dict) -> str: + """The πŸ“„ chip: the paper's PDF, one tap from the board. + + First in the row on purpose β€” the filing actions (πŸ“₯ πŸ“‘ βœ…) decide what to do + *after* reading, and this is the one that gets the paper onto the phone in + the first place. ``rel='noopener'`` + ``target='_blank'`` so collecting a + dozen PDFs before a flight does not keep navigating the board away. + """ + url = paper_pdf_url(paper) + if not url: + return "" + hint = ("download the PDF: opens arXiv's PDF directly β€” no search, no " + "abstract page in between") + return (f"\U0001f4c4") + + def _render_html(snapshot: dict) -> str: t_ = theme() t = _totals(snapshot) @@ -684,7 +728,7 @@ def _render_html(snapshot: dict) -> str: f"
  • DONE {_html.escape(p.get('done_date') or '?')}" f" β€” {_html.escape(p['title'])}
    • ") continue - acts = [] + acts = [_pdf_act(p)] for action, icon, hint in ( ("intake", "πŸ“₯", "intake into memory: really interesting β€” " "opens a prefilled issue (add your notes) " @@ -716,7 +760,7 @@ def _render_html(snapshot: dict) -> str: inbox = snapshot.get("inbox") or [] inbox_items = [] for p in inbox: - acts = [] + acts = [_pdf_act(p)] for action, icon, hint in ( ("add", "βž•", "add to the reading queue: worth reading β€” opens " "a prefilled issue; submitting files the line"), diff --git a/tests/test_arxiv_refs.py b/tests/test_arxiv_refs.py new file mode 100644 index 0000000..fa4d666 --- /dev/null +++ b/tests/test_arxiv_refs.py @@ -0,0 +1,179 @@ +"""tests/test_arxiv_refs.py β€” the arXiv ref grammar and the queue backfill. + +Two contracts. ``arxiv_refs``: every ref shape the queue has ever carried +normalises to one id, from which the abstract page and the PDF follow, and a +non-arXiv ref yields neither. ``backfill_arxiv_refs``: it only ever touches +lines the board renders as ref-less papers, and it only writes an id when +exactly one arXiv title matches β€” the guard that keeps a wrong id out of the +queue. Neither test touches the network: the resolver's HTTP call is injected. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +import arxiv_refs # noqa: E402 +import backfill_arxiv_refs as backfill # noqa: E402 + + +def _atom(*papers) -> bytes: + entries = "".join( + f"http://arxiv.org/abs/{i}v1{t}" + for i, t in papers) + return (f"{entries}" + f"").encode() + + +# --- the grammar -------------------------------------------------------------- +def test_every_ref_shape_normalises_to_one_id(): + for ref in ("2103.16254", "arXiv:2103.16254", "2103.16254v3", + "https://arxiv.org/abs/2103.16254", + "https://arxiv.org/pdf/2103.16254.pdf", + "https://arxiv.org/pdf/2103.16254v2", + "http://www.arxiv.org/abs/2103.16254"): + assert arxiv_refs.arxiv_id(ref) == "2103.16254", ref + + +def test_pre_2007_identifiers_still_resolve(): + assert arxiv_refs.arxiv_id("astro-ph/0601001") == "astro-ph/0601001" + assert arxiv_refs.abs_url("https://arxiv.org/abs/astro-ph/0601001") == ( + "https://arxiv.org/abs/astro-ph/0601001") + + +def test_a_non_arxiv_ref_has_no_derivable_urls(): + for ref in ("https://doi.org/10.1093/mnras/stab123", + "https://example.org/papers/thesis.pdf", "", None): + assert arxiv_refs.arxiv_id(ref) is None + assert arxiv_refs.abs_url(ref) is None + assert arxiv_refs.pdf_url(ref) is None + + +def test_abs_and_pdf_are_siblings_of_the_same_id(): + assert arxiv_refs.abs_url("arXiv:2608.23534") == \ + "https://arxiv.org/abs/2608.23534" + # extensionless: phones open it in the built-in viewer + assert arxiv_refs.pdf_url("arXiv:2608.23534") == \ + "https://arxiv.org/pdf/2608.23534" + + +def test_search_url_escapes_the_title(): + url = arxiv_refs.search_url("A Title: with / punctuation") + assert url.startswith(arxiv_refs.SEARCH_BASE) + assert " " not in url and "/" not in url.removeprefix(arxiv_refs.SEARCH_BASE) + + +# --- title matching ----------------------------------------------------------- +def test_normalise_title_ignores_the_disagreements_that_are_not_about_identity(): + same = ["VENUS: A Strongly Lensed Clumpy Galaxy at $z=2.48$", + "VENUS - A Strongly Lensed Clumpy Galaxy at z = 2.48", + "VENUS: a strongly lensed clumpy galaxy at z=2.48"] + assert len({arxiv_refs.normalise_title(t) for t in same}) == 1 + + +def test_a_match_needs_exactly_one_arxiv_paper(): + entries = arxiv_refs.parse_api_entries( + _atom(("2608.00001", "The Koi Pond"), ("2608.00002", "Another Paper"))) + assert arxiv_refs.match_entries("The Koi Pond", entries) == "2608.00001" + # near-misses never match: a wrong id is worse than a search box + assert arxiv_refs.match_entries("The Koi Ponds", entries) is None + # two papers with the same title β†’ a human decides, not this code + ambiguous = arxiv_refs.parse_api_entries( + _atom(("2608.00003", "Shared Title"), ("2608.00004", "Shared title"))) + assert arxiv_refs.match_entries("Shared Title", ambiguous) is None + + +def test_resolve_title_is_injectable_and_survives_a_dead_api(): + calls = [] + + def fake(params): + calls.append(params) + return _atom(("2608.12345", "A Lensed Quasar Survey")) + + assert arxiv_refs.resolve_title("A Lensed Quasar Survey", query=fake) == \ + "2608.12345" + assert "ti:" in calls[0]["search_query"] + + def dead(params): + raise OSError("arxiv is down") + + assert arxiv_refs.resolve_title("Anything", query=dead) is None + + +# --- the backfill ------------------------------------------------------------- +QUEUE = ("# Reading queue\n\nprose that is not a paper\n\n---\n\n" + "## Strong Lensing\n" + "Bare Title\n" + "Already Reffed β€” 2406.01234\n" + "DONE 2026-06-01 β€” Old Title\n" + "\n## Methods\n" + "Another Bare Title\n") +INBOX = ("# arXiv inbox\n\nprose\n\n---\nlast digest: 2026-08-25\n" + "2026-08-25 β€” Inbox Bare Title\n" + "2026-08-25 β€” Inbox Reffed β€” 2608.00001\n") + + +def test_candidates_are_only_the_ref_less_unread_papers(): + got = backfill.candidates(QUEUE, "reading-queue.md") + assert [t for _, t in got] == ["Bare Title", "Another Bare Title"] + assert [t for _, t in backfill.candidates(INBOX, "arxiv-inbox.md")] == \ + ["Inbox Bare Title"] + + +def _tree(tmp_path: Path) -> Path: + (tmp_path / "reading-queue.md").write_text(QUEUE) + (tmp_path / "arxiv-inbox.md").write_text(INBOX) + return tmp_path + + +def test_backfill_appends_refs_without_disturbing_the_rest_of_the_line(tmp_path): + root = _tree(tmp_path) + refs = {"Bare Title": "2608.11111", "Inbox Bare Title": "2608.22222"} + stats = backfill.backfill(root, write=True, limit=10, + resolve=refs.get, sleep=lambda _: None, + log=lambda *a: None) + assert stats["matched"] == 2 and stats["unmatched"] == 1 + queue = (root / "reading-queue.md").read_text() + assert "Bare Title β€” 2608.11111" in queue + assert "Another Bare Title\n" in queue # unresolved, untouched + assert "DONE 2026-06-01 β€” Old Title" in queue # history, untouched + assert "Already Reffed β€” 2406.01234" in queue # idempotent + assert "prose that is not a paper" in queue + # the inbox keeps its date stamp; the ref is appended after the title + assert "2026-08-25 β€” Inbox Bare Title β€” 2608.22222" in \ + (root / "arxiv-inbox.md").read_text() + + +def test_backfill_writes_nothing_without_write(tmp_path): + root = _tree(tmp_path) + before = (root / "reading-queue.md").read_text() + backfill.backfill(root, write=False, limit=10, + resolve=lambda t: "2608.11111", sleep=lambda _: None, + log=lambda *a: None) + assert (root / "reading-queue.md").read_text() == before + + +def test_backfill_limit_caps_the_arxiv_calls(tmp_path): + root = _tree(tmp_path) + seen = [] + + def resolve(title): + seen.append(title) + return None + + backfill.backfill(root, write=True, limit=2, resolve=resolve, + sleep=lambda _: None, log=lambda *a: None) + assert len(seen) == 2 + + +def test_backfill_is_idempotent(tmp_path): + root = _tree(tmp_path) + kw = dict(write=True, limit=10, resolve=lambda t: "2608.11111", + sleep=lambda _: None, log=lambda *a: None) + backfill.backfill(root, **kw) + once = (root / "reading-queue.md").read_text() + stats = backfill.backfill(root, **kw) + assert stats["scanned"] == 0 + assert (root / "reading-queue.md").read_text() == once diff --git a/tests/test_board.py b/tests/test_board.py index 9ccf48f..c6ba5c0 100644 --- a/tests/test_board.py +++ b/tests/test_board.py @@ -43,7 +43,8 @@ def _tree(tmp_path: Path) -> Path: (tmp_path / "reading-queue.md").write_text( "# Reading queue\n\nintro prose\n\n---\n\n" "## Demo Papers\n\nTitle One\nTitle Two β€” 2406.01234\n" - "DONE 2026-06-01 β€” Old Title\n\n## Other Things\n\nTitle Three\n") + "DONE 2026-06-01 β€” Old Title\n\n## Other Things\n\nTitle Three\n" + "Title Four β€” https://arxiv.org/pdf/2201.00042.pdf\n") (tmp_path / "arxiv-inbox.md").write_text( "# arXiv inbox\n\nintro prose\n\n---\n" "2026-08-24 β€” Suggested One β€” 2608.00001\n" @@ -75,9 +76,12 @@ def test_counts_from_a_synthetic_tree(tmp_path): {"title": "Old Title", "ref": None, "done": True, "done_date": "2026-06-01", "line": "DONE 2026-06-01 β€” Old Title"}]}, - {"section": "Other Things", "count": 1, "done": 0, "papers": [ + {"section": "Other Things", "count": 2, "done": 0, "papers": [ {"title": "Title Three", "ref": None, "done": False, - "done_date": None, "line": "Title Three"}]}] + "done_date": None, "line": "Title Three"}, + {"title": "Title Four", "ref": "https://arxiv.org/pdf/2201.00042.pdf", + "done": False, "done_date": None, + "line": "Title Four β€” https://arxiv.org/pdf/2201.00042.pdf"}]}] # alpha exists; beta/gamma are wanted assert snap["links"]["wanted"] == 2 @@ -110,6 +114,9 @@ def test_paper_links_and_issue_actions(tmp_path): assert "https://arxiv.org/abs/2406.01234" in html assert ("https://arxiv.org/search/?searchtype=title&query=Title%20One" in html) + # a ref written as a /pdf/ URL still titles onto the abstract page, so + # every paper row reads the same way + assert "https://arxiv.org/abs/2201.00042" in html # both prefilled-issue actions, on the repo's own issue tracker assert "https://github.com/PyAutoLabs/PyAutoMemory/issues/new?" in html assert "labels=queue-read" in html and "labels=queue-intake" in html @@ -337,3 +344,36 @@ def test_an_unstamped_inbox_hands_the_browser_nothing_to_check(tmp_path): # The attribute-with-value form: the script's own selector text mentions # the bare attribute name and is always present. assert "data-last-digest='" not in board._render_html(_inbox(tmp_path)) + + +def test_a_reffed_paper_carries_a_one_tap_pdf_button(tmp_path): + """The πŸ“„ button: the PDF itself, no search and no abstract page first. + + The reason the queue wants refs at all β€” a phone collecting a stack of + papers before a flight taps once per paper, not three times. + """ + snap = _snap_with_remote(tmp_path) + html = board.render(snap, "html") + md = board.render(snap, "md") + for fmt in (html, md): + assert "https://arxiv.org/pdf/2406.01234" in fmt # bare-id ref + assert "https://arxiv.org/pdf/2201.00042" in fmt # /pdf/ URL ref + assert "https://arxiv.org/pdf/2608.00001" in fmt # inbox paper + # it opens away from the board, so collecting a stack does not lose the page + assert "class='act pdf'" in html and "rel='noopener'" in html + + +def test_no_pdf_button_where_there_is_no_ref(tmp_path): + """A button that cannot work is never rendered β€” a search is not a PDF.""" + snap = _snap_with_remote(tmp_path) + for fmt in ("html", "md"): + out = board.render(snap, fmt) + # Title One / Title Three / Suggested Two are ref-less; the only PDF + # links on the page belong to the three papers that carry a ref. + assert out.count("arxiv.org/pdf/") == 3 + bare = {"title": "Title One", "ref": None} + assert board.paper_pdf_url(bare) == "" + # a non-arXiv ref is still a link, but there is no PDF to derive from it + assert board.paper_url({"title": "T", "ref": "https://doi.org/10.1/x"}) == \ + "https://doi.org/10.1/x" + assert board.paper_pdf_url({"title": "T", "ref": "https://doi.org/10.1/x"}) == ""