From 1bd7d38369236828827660943fb4303750ea6eb8 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 25 Aug 2026 18:22:53 +0000 Subject: [PATCH] queue: reconcile LaTeX with rendered maths, and repair damaged lines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two cleanups on the backfill's remaining misses. **The matcher could not see past maths.** arXiv's metadata carries the LaTeX source, a title pasted off the abstract page carries the rendered glyphs, and a plain punctuation strip does not bring the two together: `$0.5\leq z<1.0$` and `0.5≤z<1.0` reduced to different strings, as did `$\Lambda$CDM`/`ΛCDM`, `$M_\star$`/`M⋆`, `$z\sim3$`/`z∼3`, `$2.2\times10^{-21}$`/`2.2×10−21`. normalise_title now erases both spellings rather than translating one into the other: LaTeX control sequences go, and so does every non-ASCII character left after accent folding. `\leq` and `≤` both become nothing and the two sides meet at `05z10`. Erasing beats a `\leq`→`≤` table, which would need every symbol astronomers use and would silently mismatch the day it missed one; the uniqueness rule in match_entries is what keeps the lossier reduction safe. Accented Latin still survives — NFKD leaves an ASCII base (`Ekström` → `ekstrom`), Greek has none. **Five lines were damaged rather than unmatched**, and no matcher could have helped: * `Chandra Observations of Spikey: … Supermassive Black Hole` / `Binary System` — one title that lost its wrap, rejoined. * `…the local MBH-M⋆ relationProj C` — junk welded onto the end by whatever pasted it, trimmed. * `__SMBH Binaries_` — a sub-heading with one trailing underscore, so it read as a paper. * `Massive Black Holes summary: https://arxiv.org/pdf/…` — a ref written after a colon instead of ` — `, so it was never recognised. Now a paper with a working PDF button, keeping the human's own label as its title. * Two paragraph fragments pasted out of a paper's introduction (13 lines) — NOTE-marked by hand rather than spending 13 arXiv lookups discovering they are prose. Live queue: 232 → 208 papers waiting (the rest were never papers), PDF coverage 133 → 134 before the matcher improvement has even run against arXiv. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PjWdVFPn2WDzcqpMXTMp7G --- reading-queue.md | 35 +++++++++++++++++------------------ scripts/arxiv_refs.py | 27 ++++++++++++++++++++++++++- tests/test_arxiv_refs.py | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 19 deletions(-) diff --git a/reading-queue.md b/reading-queue.md index 8283774..f9e615f 100644 --- a/reading-queue.md +++ b/reading-queue.md @@ -126,19 +126,19 @@ Accretion onto supermassive black hole binaries — 2405.14843 Astrometric Jitter as a Detection Diagnostic for Recoiling and Slingshot Supermassive Black Hole Candidates — 2405.11026 Detection and Prediction of Future Massive Black Hole Mergers with Machine Learning and Truncated Waveforms — 2405.11340 How do Primordial Black Holes change the Halo Mass Function and Structure? — 2405.11381 -Tip of the iceberg: overmassive black holes at 4 str: # --- title matching ----------------------------------------------------------- _PUNCT_RE = re.compile(r"[^a-z0-9]+") +#: A LaTeX control sequence: ``\leq``, ``\Lambda``, ``\rm``, ``\,``. +_LATEX_CMD_RE = re.compile(r"\\[A-Za-z]+|\\.") def normalise_title(title: str) -> str: - """A title reduced to its comparable core: lowercase alphanumerics only. + """A title reduced to its comparable core: lowercase ASCII alphanumerics. arXiv, the digest and a hand-pasted queue line disagree constantly about LaTeX (``$z=2.48$``), dashes (``-``/``–``/``—``), accents and double spaces, @@ -120,9 +122,32 @@ def normalise_title(title: str) -> str: lets the backfill demand an exact match on what remains, rather than a fuzzy match on what does not — a wrong id is worse than no id, because it links the reader confidently to somebody else's paper. + + The one disagreement that needs more than stripping is **maths**, because + the two sides spell it differently rather than merely punctuate it + differently: arXiv's metadata carries the LaTeX source (``$0.5\\leq + z<1.0$``, ``$\\Lambda$CDM``, ``$M_\\star$``) while a title pasted off the + abstract page carries the rendered glyphs (``0.5≤z<1.0``, ``ΛCDM``, + ``M⋆``). Neither survives a plain punctuation strip into the same string. + + So both are erased rather than reconciled: LaTeX control sequences go, and + so does every non-ASCII character left after accent folding. ``\\leq`` and + ``≤`` both become nothing, and the two spellings meet in the middle — + ``05z10`` from either side. Erasing is deliberately preferred to a + ``\\leq``→``≤`` translation table, which would need every symbol + astronomers use and would silently mismatch the day it missed one. The + reduction is lossier, and the uniqueness rule in :func:`match_entries` is + what keeps that safe: a title that reduces onto two arXiv papers matches + neither. + + Accented Latin is *kept* (``Ekström`` → ``ekstrom``): NFKD splits the accent + off as a combining mark and the base letter is ASCII, so it survives the + non-ASCII drop that removes Greek and maths. """ folded = unicodedata.normalize("NFKD", title) folded = "".join(c for c in folded if not unicodedata.combining(c)) + folded = _LATEX_CMD_RE.sub(" ", folded) + folded = folded.encode("ascii", "ignore").decode("ascii") return _PUNCT_RE.sub("", folded.lower()) diff --git a/tests/test_arxiv_refs.py b/tests/test_arxiv_refs.py index cf30dee..2f33b50 100644 --- a/tests/test_arxiv_refs.py +++ b/tests/test_arxiv_refs.py @@ -73,6 +73,39 @@ def test_normalise_title_ignores_the_disagreements_that_are_not_about_identity() assert len({arxiv_refs.normalise_title(t) for t in same}) == 1 +def test_latex_source_and_rendered_glyphs_reduce_to_the_same_title(): + """arXiv carries the LaTeX; a pasted line carries the rendered glyphs. + + Every pair below is one paper written two ways, and every pair failed to + match before maths was erased rather than punctuated away. + """ + for latex, rendered in [ + (r"$0.5\leq z<1.0$", "0.5≤z<1.0"), + (r"$\Lambda$CDM Model", "ΛCDM Model"), + (r"the local $M_{\rm BH}-M_\star$ relation", + "the local MBH-M⋆ relation"), + (r"$2.2\times10^{-21}$eV", "2.2×10−21eV"), + (r"galaxies at $z\sim3$", "galaxies at z∼3"), + (r"SLICE -- Combining X-ray in AC\,114", "SLICE - Combining X-ray in AC 114")]: + assert arxiv_refs.normalise_title(latex) == \ + arxiv_refs.normalise_title(rendered), latex + + +def test_accented_latin_survives_but_greek_does_not(): + """NFKD splits the accent off a Latin letter, so the ASCII base remains.""" + assert arxiv_refs.normalise_title("Ekström") == "ekstrom" + assert arxiv_refs.normalise_title("Muñoz-Darias") == "munozdarias" + # Greek has no ASCII base and is erased on both sides alike + assert arxiv_refs.normalise_title("ΛCDM") == arxiv_refs.normalise_title(r"$\Lambda$CDM") + + +def test_erasing_maths_still_keeps_different_papers_apart(): + """The reduction is lossier — it must not collapse unrelated titles.""" + a = arxiv_refs.normalise_title(r"Dark matter at $z\sim3$ in dwarf galaxies") + b = arxiv_refs.normalise_title(r"Dark energy at $z\sim3$ in dwarf galaxies") + assert a != b + + def test_a_match_needs_exactly_one_arxiv_paper(): entries = arxiv_refs.parse_api_entries( _atom(("2608.00001", "The Koi Pond"), ("2608.00002", "Another Paper")))