diff --git a/reading-queue.md b/reading-queue.md index 8283774..f9e615f 100644 --- a/reading-queue.md +++ b/reading-queue.md @@ -126,19 +126,19 @@ Accretion onto supermassive black hole binaries — 2405.14843 Astrometric Jitter as a Detection Diagnostic for Recoiling and Slingshot Supermassive Black Hole Candidates — 2405.11026 Detection and Prediction of Future Massive Black Hole Mergers with Machine Learning and Truncated Waveforms — 2405.11340 How do Primordial Black Holes change the Halo Mass Function and Structure? — 2405.11381 -Tip of the iceberg: overmassive black holes at 4 str: # --- title matching ----------------------------------------------------------- _PUNCT_RE = re.compile(r"[^a-z0-9]+") +#: A LaTeX control sequence: ``\leq``, ``\Lambda``, ``\rm``, ``\,``. +_LATEX_CMD_RE = re.compile(r"\\[A-Za-z]+|\\.") def normalise_title(title: str) -> str: - """A title reduced to its comparable core: lowercase alphanumerics only. + """A title reduced to its comparable core: lowercase ASCII alphanumerics. arXiv, the digest and a hand-pasted queue line disagree constantly about LaTeX (``$z=2.48$``), dashes (``-``/``–``/``—``), accents and double spaces, @@ -120,9 +122,32 @@ def normalise_title(title: str) -> str: lets the backfill demand an exact match on what remains, rather than a fuzzy match on what does not — a wrong id is worse than no id, because it links the reader confidently to somebody else's paper. + + The one disagreement that needs more than stripping is **maths**, because + the two sides spell it differently rather than merely punctuate it + differently: arXiv's metadata carries the LaTeX source (``$0.5\\leq + z<1.0$``, ``$\\Lambda$CDM``, ``$M_\\star$``) while a title pasted off the + abstract page carries the rendered glyphs (``0.5≤z<1.0``, ``ΛCDM``, + ``M⋆``). Neither survives a plain punctuation strip into the same string. + + So both are erased rather than reconciled: LaTeX control sequences go, and + so does every non-ASCII character left after accent folding. ``\\leq`` and + ``≤`` both become nothing, and the two spellings meet in the middle — + ``05z10`` from either side. Erasing is deliberately preferred to a + ``\\leq``→``≤`` translation table, which would need every symbol + astronomers use and would silently mismatch the day it missed one. The + reduction is lossier, and the uniqueness rule in :func:`match_entries` is + what keeps that safe: a title that reduces onto two arXiv papers matches + neither. + + Accented Latin is *kept* (``Ekström`` → ``ekstrom``): NFKD splits the accent + off as a combining mark and the base letter is ASCII, so it survives the + non-ASCII drop that removes Greek and maths. """ folded = unicodedata.normalize("NFKD", title) folded = "".join(c for c in folded if not unicodedata.combining(c)) + folded = _LATEX_CMD_RE.sub(" ", folded) + folded = folded.encode("ascii", "ignore").decode("ascii") return _PUNCT_RE.sub("", folded.lower()) diff --git a/tests/test_arxiv_refs.py b/tests/test_arxiv_refs.py index cf30dee..2f33b50 100644 --- a/tests/test_arxiv_refs.py +++ b/tests/test_arxiv_refs.py @@ -73,6 +73,39 @@ def test_normalise_title_ignores_the_disagreements_that_are_not_about_identity() assert len({arxiv_refs.normalise_title(t) for t in same}) == 1 +def test_latex_source_and_rendered_glyphs_reduce_to_the_same_title(): + """arXiv carries the LaTeX; a pasted line carries the rendered glyphs. + + Every pair below is one paper written two ways, and every pair failed to + match before maths was erased rather than punctuated away. + """ + for latex, rendered in [ + (r"$0.5\leq z<1.0$", "0.5≤z<1.0"), + (r"$\Lambda$CDM Model", "ΛCDM Model"), + (r"the local $M_{\rm BH}-M_\star$ relation", + "the local MBH-M⋆ relation"), + (r"$2.2\times10^{-21}$eV", "2.2×10−21eV"), + (r"galaxies at $z\sim3$", "galaxies at z∼3"), + (r"SLICE -- Combining X-ray in AC\,114", "SLICE - Combining X-ray in AC 114")]: + assert arxiv_refs.normalise_title(latex) == \ + arxiv_refs.normalise_title(rendered), latex + + +def test_accented_latin_survives_but_greek_does_not(): + """NFKD splits the accent off a Latin letter, so the ASCII base remains.""" + assert arxiv_refs.normalise_title("Ekström") == "ekstrom" + assert arxiv_refs.normalise_title("Muñoz-Darias") == "munozdarias" + # Greek has no ASCII base and is erased on both sides alike + assert arxiv_refs.normalise_title("ΛCDM") == arxiv_refs.normalise_title(r"$\Lambda$CDM") + + +def test_erasing_maths_still_keeps_different_papers_apart(): + """The reduction is lossier — it must not collapse unrelated titles.""" + a = arxiv_refs.normalise_title(r"Dark matter at $z\sim3$ in dwarf galaxies") + b = arxiv_refs.normalise_title(r"Dark energy at $z\sim3$ in dwarf galaxies") + assert a != b + + def test_a_match_needs_exactly_one_arxiv_paper(): entries = arxiv_refs.parse_api_entries( _atom(("2608.00001", "The Koi Pond"), ("2608.00002", "Another Paper")))