queue: line kinds — a heading is not a paper you can download

reading-queue.md

@@ -13,6 +13,20 @@ Papers waiting to be read and filed. Format:
   and only when exactly one arXiv paper's title matches.
 - A read/consumed paper is never deleted: prefix its line with
   `DONE <YYYY-MM-DD> — ` and leave it in place (the reading history).
+- **Not every line is a paper.** Two kinds are not, and marking them keeps the
+  Dashboard from offering a download button for a heading and stops the nightly
+  ref backfill spending an arXiv lookup on one:
+  - `__Bold Text__` on its own line is a **sub-heading** grouping the papers
+    below it. Rendered as a heading, never counted, never looked up.
+  - `NOTE <text>` is an **annotation** — a remark, a link that is not a paper,
+    a stray line of pasted prose. Sibling of the `DONE ` prefix: same shape,
+    same never-delete rule. A bare non-arXiv link counts as one automatically.
+    `scripts/backfill_arxiv_refs.py --mark-unresolved` applies the prefix in
+    bulk to lines it could not identify — run it after reading a dry run, since
+    an unresolved title is more often a real paper whose line lost something
+    (a LaTeX redshift, a wrapped subtitle) than a genuine non-paper.
+  - A line that is *only* an arXiv link is still a paper — the title was just
+    never written, so the board labels it `arXiv:<id>`.
 - Filing a read paper: add its canonical entry to `bibliography/`, stub it in
   the matching `wiki/<domain>/sources/*.md` page per `wiki/CLAUDE.md`, then
   mark the line DONE and run `make validate`.

scripts/backfill_arxiv_refs.py

@@ -73,12 +73,35 @@ def candidates(text: str, source: str) -> list[tuple[int, str]]:
                 continue
             body = raw.strip()
         paper = board._parse_paper(body)
+        # `kind != "paper"` is the whole point of the marker: a `__Heading__`
+        # and a `NOTE ` line are not papers, so they are never looked up — not
+        # this run and not any future one. Without that, every night spends an
+        # arXiv call on a line that can never resolve.
+        if paper["kind"] != "paper":
+            continue
         if paper["done"] or paper["ref"] or not paper["title"]:
             continue
         out.append((i, paper["title"]))
     return out
 
 
+def mark_unresolved_line(raw_line: str) -> str:
+    """`raw_line` prefixed `NOTE `, the "stop asking about this one" marker.
+
+    Written only under ``--mark-unresolved``, never by the nightly run. The
+    distinction matters: a title arXiv cannot identify is usually not a
+    non-paper but a real paper whose line lost something on the way in (a
+    LaTeX redshift, a wrapped subtitle), and those are worth re-trying after a
+    human fixes the line. So the nightly keeps asking, and a human who has
+    looked at the misses runs this once to retire the ones that are genuinely
+    prose, links or headings. The text is untouched and the line stays in
+    place — the never-delete rule covers annotations too.
+    """
+    stripped = raw_line.lstrip()
+    indent = raw_line[:len(raw_line) - len(stripped)]
+    return f"{indent}NOTE {stripped.rstrip()}"
+
+
 def apply_ref(raw_line: str, ref: str) -> str:
     """`raw_line` with ``— <ref>`` appended, preserving its exact prefix.
 
@@ -89,12 +112,18 @@ def apply_ref(raw_line: str, ref: str) -> str:
     return f"{raw_line.rstrip()} — {ref}"
 
 
-def backfill(root: Path, *, write: bool, limit: int,
+def backfill(root: Path, *, write: bool, limit: int, mark_unresolved: bool = False,
              resolve=arxiv_refs.resolve_title,
              delay: float = arxiv_refs.API_DELAY_SECONDS,
              sleep=time.sleep, log=print) -> dict:
-    """Resolve up to `limit` ref-less titles across the queue and the inbox."""
-    stats = {"scanned": 0, "matched": 0, "unmatched": 0, "files": []}
+    """Resolve up to `limit` ref-less titles across the queue and the inbox.
+
+    With `mark_unresolved`, a line that fails its lookup is also `NOTE `-marked
+    so no later run asks about it again — a human-driven cleanup, never the
+    nightly's own decision (see :func:`mark_unresolved_line`).
+    """
+    stats = {"scanned": 0, "matched": 0, "unmatched": 0, "marked": 0,
+             "files": []}
     budget = limit
     for name in TARGETS:
         path = root / name
@@ -103,7 +132,7 @@ def backfill(root: Path, *, write: bool, limit: int,
         lines = path.read_text(errors="replace").splitlines()
         todo = candidates("\n".join(lines), name)
         stats["scanned"] += len(todo)
-        changed = 0
+        changed = file_marked = 0
         for idx, title in todo:
             if budget <= 0:
                 break
@@ -116,7 +145,14 @@ def backfill(root: Path, *, write: bool, limit: int,
                 log(f"  ✔ {ref}  {title[:90]}")
             else:
                 stats["unmatched"] += 1
-                log(f"  · no unique match  {title[:90]}")
+                if mark_unresolved:
+                    lines[idx] = mark_unresolved_line(lines[idx])
+                    changed += 1
+                    file_marked += 1
+                    stats["marked"] += 1
+                    log(f"  ✖ NOTE-marked     {title[:90]}")
+                else:
+                    log(f"  · no unique match  {title[:90]}")
             # Politeness, not throughput: arXiv asks for a few seconds between
             # calls, and this job has all night.
             if budget > 0:
@@ -125,8 +161,9 @@ def backfill(root: Path, *, write: bool, limit: int,
             stats["files"].append(name)
             if write:
                 path.write_text("\n".join(lines) + "\n")
-        log(f"{name}: {len(todo)} ref-less, {changed} resolved"
-            f"{'' if write else ' (dry run — nothing written)'}")
+        marked = f", {file_marked} NOTE-marked" if file_marked else ""
+        log(f"{name}: {len(todo)} ref-less, {changed - file_marked} resolved"
+            f"{marked}{'' if write else ' (dry run — nothing written)'}")
     return stats
 
 
@@ -136,12 +173,20 @@ def main(argv: list[str] | None = None) -> int:
                     help="apply the refs (default: report only)")
     ap.add_argument("--limit", type=int, default=60,
                     help="most arXiv lookups to make in one run (default 60)")
+    ap.add_argument("--mark-unresolved", action="store_true",
+                    help="also NOTE-mark every line that fails to resolve, so "
+                         "later runs skip it. A human-reviewed cleanup for "
+                         "lines that are prose, links or headings rather than "
+                         "papers — never used by the nightly run, which keeps "
+                         "re-trying in case a line gets fixed.")
     ap.add_argument("--root", type=Path, default=MEMORY_HOME)
     args = ap.parse_args(argv)
 
-    stats = backfill(args.root, write=args.write, limit=args.limit)
-    print(f"\n{stats['matched']} resolved, {stats['unmatched']} unresolved, "
-          f"{stats['scanned']} ref-less lines in total.")
+    stats = backfill(args.root, write=args.write, limit=args.limit,
+                     mark_unresolved=args.mark_unresolved)
+    marked = f", {stats['marked']} NOTE-marked" if stats["marked"] else ""
+    print(f"\n{stats['matched']} resolved, {stats['unmatched']} unresolved"
+          f"{marked}, {stats['scanned']} ref-less lines in total.")
     # Not an error: an unresolved title is a normal outcome (not on arXiv, or
     # too ambiguous to be safe), and a run that resolves nothing must not fail
     # the nightly job.

scripts/board.py

@@ -30,6 +30,12 @@
 warning is computed in the reader's browser (see ``_EXTRA_JS``), because the
 page freezes at its last good render in exactly the failure the warning is for.
 
+A queue section is not a flat list of papers: a `__Bold__` line is a
+sub-heading the human wrote, and a `NOTE `-prefixed line (or a bare non-arXiv
+link) is an annotation. Both render as themselves — no link, no buttons, no
+slot in the "N waiting" count — because a heading is not a paper you can
+download. `_parse_paper` decides which is which; see `reading-queue.md`.
+
 **Contents-level only.** The board shows titles and counts, never claim text
 or summaries — the knowledge itself stays in the wiki pages.
 
@@ -123,6 +129,22 @@ def theme():
 # is part of the title.
 QUEUE_REF_RE = re.compile(
     r"^(.*\S)\s+—\s+((?:arXiv:)?\d{4}\.\d{4,5}(?:v\d+)?|https?://\S+)$")
+# Not every line under a `## Section` is a paper. Two kinds are not, and saying
+# so is what stops the board rendering a search button for a heading and the
+# backfill spending an arXiv lookup on one every night:
+#   `__Dark Matter__`  a bold sub-heading the human wrote to group the papers
+#                      below it — structure, rendered as structure.
+#   `NOTE <text>`      an annotation: a free-text remark, a link that is not a
+#                      paper, a stray line of pasted prose. Sibling of the
+#                      `DONE ` prefix — same shape, same never-delete rule, and
+#                      the marker `backfill_arxiv_refs.py --mark-unresolved`
+#                      writes so a line it cannot identify is asked about once
+#                      and then left alone.
+QUEUE_SUBHEAD_RE = re.compile(r"^__(.+?)__$")
+QUEUE_NOTE_RE = re.compile(r"^NOTE\b\s*(?:—\s*)?(.*)$")
+#: A line that is nothing but a URL. An arXiv one is a paper whose title was
+#: never written (the ref is the whole line); anything else is an annotation.
+QUEUE_BARE_URL_RE = re.compile(r"^https?://\S+$")
 
 
 def _owner_repo() -> tuple[str, str]:
@@ -195,7 +217,11 @@ def _flush():
             if section is not None:
                 snapshot["queue"].append({
                     "section": section,
-                    "count": sum(1 for p in papers if not p["done"]),
+                    # Headings and annotations are carried in `papers` so the
+                    # section renders in file order, but they are not work:
+                    # counting them would inflate every "N waiting" on the board.
+                    "count": sum(1 for p in papers
+                                 if p["kind"] == "paper" and not p["done"]),
                     "done": sum(1 for p in papers if p["done"]),
                     "papers": papers,
                 })
@@ -236,19 +262,59 @@ def _flush():
 
 # --- pure helpers --------------------------------------------------------------
 def _parse_paper(text: str) -> dict:
-    """One queue line → {title, ref, done, done_date, line} (line = as written)."""
+    """One queue line → {kind, title, ref, done, done_date, line}.
+
+    `line` is the line as written — the token every issue button round-trips.
+    `kind` is what the line *is*: a `paper`, a `subhead` (a `__Bold__` grouping
+    the human wrote), or a `note` (a `NOTE `-prefixed annotation, or a bare
+    non-arXiv link). Only a `paper` is counted, given buttons, or looked up on
+    arXiv; the other two are carried through and rendered as themselves, which
+    is the whole point — a heading is not a paper you can download.
+    """
     raw, done_date = text, None
     m = QUEUE_DONE_LINE_RE.match(text)
     if m:
         done_date, text = m.group(1), m.group(2)
     elif QUEUE_DONE_RE.match(text):
         done_date, text = "", text[4:].lstrip(" —-")
+
+    def _built(kind, title, ref=None):
+        return {"kind": kind, "title": title, "ref": ref,
+                "done": done_date is not None,
+                "done_date": done_date or None, "line": raw}
+
+    # Checked before the ref split: a heading or an annotation has no ref to
+    # find, and `NOTE https://…` must not be read as a titleless paper.
+    m = QUEUE_SUBHEAD_RE.match(text)
+    if m:
+        return _built("subhead", m.group(1).strip())
+    m = QUEUE_NOTE_RE.match(text)
+    if m:
+        return _built("note", m.group(1).strip())
+
     ref = None
     m = QUEUE_REF_RE.match(text)
     if m:
         text, ref = m.group(1), m.group(2)
-    return {"title": text, "ref": ref, "done": done_date is not None,
-            "done_date": done_date or None, "line": raw}
+    elif QUEUE_BARE_URL_RE.match(text):
+        # A line that is only a URL. If it points at arXiv the human did write a
+        # paper — just as a link instead of a title — so it earns its abstract
+        # page and its PDF button, labelled by the identifier the line already
+        # contains rather than by a hundred characters of raw URL. (No title is
+        # invented: the id is read out of the line, and the line itself is what
+        # every issue button still round-trips.) Any other bare link is an
+        # annotation.
+        aid = arxiv_refs.arxiv_id(text)
+        if not aid:
+            return _built("note", text)
+        return _built("paper", f"arXiv:{aid}", text)
+    return _built("paper", text, ref)
+
+
+def _url_host(url: str) -> str:
+    """`https://sub.example.org/a/b?c` → `sub.example.org` (the url if unparsable)."""
+    m = re.match(r"^https?://([^/?#]+)", url)
+    return m.group(1) if m else url
 
 
 def paper_url(paper: dict) -> str:
@@ -554,6 +620,14 @@ def _render_md(snapshot: dict) -> str:
             if p["done"]:
                 continue
             label = p["title"].replace("[", "\\[").replace("]", "\\]")
+            if p["kind"] == "subhead":
+                lines.append(f"- **{label}**")
+                continue
+            if p["kind"] == "note":
+                if QUEUE_BARE_URL_RE.match(p["title"]):
+                    label = f"[{_url_host(p['title'])} →]({p['title']})"
+                lines.append(f"- _{label}_")
+                continue
             bits = [f"- [{label}]({paper_url(p)})"]
             pdf = paper_pdf_url(p)
             if pdf:
@@ -627,6 +701,11 @@ def _bar(statuses: dict) -> str:
 ul.papers{margin:.5rem 0 .25rem;padding-left:1.3rem}
 ul.papers li{margin:.4rem 0}
 ul.papers li.done{color:var(--muted)}
+/* A heading the human wrote inside a section, and a line that is not a paper:
+   both render as themselves — no link, no buttons, nothing to tap. */
+ul.papers li.subhead{list-style:none;margin:.7rem 0 .2rem -1.3rem;
+ font-weight:600;letter-spacing:.02em}
+ul.papers li.note{color:var(--muted);font-style:italic}
 a.act{margin-left:.35rem;padding:.05rem .4rem;font-size:.85rem;
  border:1px solid var(--line);border-radius:6px;background:var(--btn)}
 a.act:hover{background:var(--tint);border-color:var(--accent);
@@ -728,6 +807,23 @@ def _render_html(snapshot: dict) -> str:
                     f"<li class='done'>DONE {_html.escape(p.get('done_date') or '?')}"
                     f" — {_html.escape(p['title'])}</li>")
                 continue
+            if p["kind"] == "subhead":
+                items.append(f"<li class='subhead'>"
+                             f"{_html.escape(p['title'])}</li>")
+                continue
+            if p["kind"] == "note":
+                text = _html.escape(p["title"])
+                if QUEUE_BARE_URL_RE.match(p["title"]):
+                    # A saved link is still a link: render it clickable, just
+                    # without the paper buttons it cannot honour. Labelled by
+                    # host — a tracking-wrapped URL is hundreds of characters
+                    # of noise on a phone, and the line itself keeps the whole
+                    # thing.
+                    text = (f"<a href=\"{_html.escape(p['title'], quote=True)}\" "
+                            f"target='_blank' rel='noopener'>"
+                            f"{_html.escape(_url_host(p['title']))} →</a>")
+                items.append(f"<li class='note'>{text}</li>")
+                continue
             acts = [_pdf_act(p)]
             for action, icon, hint in (
                     ("intake", "📥", "intake into memory: really interesting — "

tests/test_arxiv_refs.py

@@ -177,3 +177,57 @@ def test_backfill_is_idempotent(tmp_path):
     stats = backfill.backfill(root, **kw)
     assert stats["scanned"] == 0
     assert (root / "reading-queue.md").read_text() == once
+
+
+# --- line kinds: what the backfill refuses to look up -------------------------
+KINDED = ("# Reading queue\n\nprose\n\n---\n\n## Strong Lensing\n"
+          "__A Bold Heading__\n"
+          "Bare Title\n"
+          "NOTE a stray remark someone pasted\n"
+          "NOTE — https://example.org/not-a-paper\n"
+          "https://example.org/also-not-a-paper\n"
+          "https://arxiv.org/abs/2202.00099\n")
+
+
+def test_headings_notes_and_non_arxiv_links_are_never_looked_up():
+    """The marker's whole purpose: these lines cost zero arXiv calls, forever."""
+    got = [t for _, t in backfill.candidates(KINDED, "reading-queue.md")]
+    assert got == ["Bare Title"]   # the bare arXiv link already has its ref
+
+
+def test_mark_unresolved_line_prefixes_without_touching_the_text():
+    assert backfill.mark_unresolved_line("Some Prose Line") == \
+        "NOTE Some Prose Line"
+    # indentation survives; the text is never rewritten
+    assert backfill.mark_unresolved_line("   Indented  ") == "   NOTE Indented"
+
+
+def test_mark_unresolved_retires_a_line_from_all_future_runs(tmp_path):
+    (tmp_path / "reading-queue.md").write_text(
+        "# Q\n\n---\n\n## S\nResolves Fine\nNever Resolves\n")
+    (tmp_path / "arxiv-inbox.md").write_text("# I\n\n---\n")
+    kw = dict(write=True, limit=10, sleep=lambda _: None, log=lambda *a: None,
+              resolve=lambda t: "2608.11111" if t == "Resolves Fine" else None)
+
+    stats = backfill.backfill(tmp_path, mark_unresolved=True, **kw)
+    assert stats["matched"] == 1 and stats["marked"] == 1
+    text = (tmp_path / "reading-queue.md").read_text()
+    assert "Resolves Fine — 2608.11111" in text
+    assert "NOTE Never Resolves" in text
+
+    # the next run has nothing left to ask arXiv about
+    assert backfill.backfill(tmp_path, mark_unresolved=True, **kw)["scanned"] == 0
+
+
+def test_the_nightly_never_marks_on_its_own(tmp_path):
+    """Default off: an unresolved title is usually a real paper with a mangled
+    line, and retrying it after a human fixes the line is the right behaviour."""
+    (tmp_path / "reading-queue.md").write_text(
+        "# Q\n\n---\n\n## S\nNever Resolves\n")
+    (tmp_path / "arxiv-inbox.md").write_text("# I\n\n---\n")
+    stats = backfill.backfill(tmp_path, write=True, limit=10,
+                              resolve=lambda t: None, sleep=lambda _: None,
+                              log=lambda *a: None)
+    assert stats["marked"] == 0
+    assert (tmp_path / "reading-queue.md").read_text().endswith("Never Resolves\n")
+