arxiv: search on the words, not on the mangled maths

.github/workflows/arxiv_refs.yml

@@ -32,6 +32,10 @@ on:
         description: "Report only — resolve nothing into the files"
         type: boolean
         default: false
+      mark_unresolved:
+        description: "Also NOTE-mark every line that fails to resolve, so later runs skip it. Human-reviewed cleanup — read a dry run first."
+        type: boolean
+        default: false
 
 # contents: write → the queue commit; actions: write → re-dispatch the
 # Dashboard (a token push does not retrigger `on: push` workflows).
@@ -60,11 +64,17 @@ jobs:
         env:
           LIMIT: ${{ inputs.limit }}
           DRY_RUN: ${{ inputs.dry_run }}
+          MARK_UNRESOLVED: ${{ inputs.mark_unresolved }}
         run: |
           set -u
           args="--limit ${LIMIT:-60}"
           [ "${DRY_RUN:-false}" = "true" ] || args="$args --write"
-          python scripts/backfill_arxiv_refs.py $args | tee /tmp/backfill.log
+          # Never on the schedule: `inputs.*` is empty on a cron fire, so this
+          # can only ever be a deliberate dispatch.
+          [ "${MARK_UNRESOLVED:-false}" != "true" ] || args="$args --mark-unresolved"
+          # `-u`: unbuffered, so the log streams while the run is in flight
+          # instead of arriving in one block at the end.
+          python -u scripts/backfill_arxiv_refs.py $args | tee /tmp/backfill.log
           {
             echo '## arXiv refs backfill'
             echo '```'
@@ -85,11 +95,18 @@ jobs:
 
       - name: Commit + push (explicit paths only)
         if: steps.backfill.outputs.changed == 'yes' && github.ref == 'refs/heads/main'
+        env:
+          MARK_UNRESOLVED: ${{ inputs.mark_unresolved }}
         run: |
           git config user.name  "github-actions[bot]"
           git config user.email "github-actions[bot]@users.noreply.github.com"
           git add reading-queue.md arxiv-inbox.md
-          git commit -m "queue: backfill arXiv refs for ref-less papers [skip ci]"
+          if [ "${MARK_UNRESOLVED:-false}" = "true" ]; then
+            msg="queue: backfill arXiv refs, NOTE-mark what cannot be identified"
+          else
+            msg="queue: backfill arXiv refs for ref-less papers"
+          fi
+          git commit -m "${msg} [skip ci]"
           for attempt in 1 2 3; do
             if git push; then exit 0; fi
             git pull --rebase origin main || exit 1

scripts/arxiv_refs.py

@@ -184,20 +184,61 @@ def match_entries(title: str, entries: list[dict]) -> str | None:
     return hits.pop() if len(hits) == 1 else None
 
 
-def resolve_title(title: str, max_results: int = 20,
+#: A phrase shorter than this is too generic to be worth searching on.
+MIN_PHRASE_WORDS = 3
+
+
+def query_phrase(title: str) -> str:
+    """The longest run of plain alphabetic words in `title`.
+
+    What gets *sent* to arXiv, as opposed to what gets compared afterwards.
+    The two are different problems and conflating them was a real bug: a title
+    carrying maths used to be flattened word-by-word into the phrase search, so
+    ``…galaxies at 0.5≤z<1.0`` was sent as ``ti:"…galaxies at 0 5 z 1 0"``.
+    arXiv indexes ``$0.5\\leq z<1.0$``, matches none of that, and returns
+    nothing — at which point no amount of clever comparison in
+    :func:`match_entries` has anything to compare.
+
+    So the query drops the maths instead of mangling it, and searches on the
+    longest stretch of ordinary words the title contains — the part both sides
+    spell the same way. That is a *broader* search, not a looser match: it only
+    decides which candidates come back, and every one of them still has to
+    survive the exact, unique-title check afterwards.
+
+    Falls back to every ASCII word when no run is long enough
+    (``MIN_PHRASE_WORDS``), since a two-word phrase would drown the result set.
+    """
+    # `flags=re.ASCII` so `\w` stops treating Greek as a word character: `Λ`
+    # became a literal `Λ` in the query string before, which arXiv cannot match
+    # against its own `$\Lambda$` source.
+    tokens = re.sub(r"[^\w\s]", " ", title, flags=re.ASCII).split()
+    best: list[str] = []
+    run: list[str] = []
+    for token in tokens:
+        if token.isalpha():
+            run.append(token)
+            if len(run) > len(best):
+                best = list(run)
+        else:
+            run = []
+    return " ".join(best if len(best) >= MIN_PHRASE_WORDS else tokens)
+
+
+def resolve_title(title: str, max_results: int = 50,
                   query: object = _api_query) -> str | None:
     """Look `title` up on the arXiv API and return its id, or None.
 
     Networked — the Dashboard never calls this; only the backfill does.
     `query` is injected so the matching can be tested without arXiv.
+
+    `max_results` is generous because :func:`query_phrase` deliberately searches
+    on part of a title rather than all of it: more candidates come back, and
+    picking among them costs nothing — the choice is made locally, by exact
+    match, and an ambiguous set still resolves to nothing.
     """
-    # `ti:"…"` is a phrase search over titles. arXiv's search index chokes on
-    # unbalanced quotes and LaTeX, so the phrase is sent as plain words; the
-    # exactness comes from match_entries afterwards, never from the query.
-    words = re.sub(r'[^\w\s]', " ", title).split()
-    if not words:
+    phrase = query_phrase(title)
+    if not phrase:
         return None
-    phrase = " ".join(words)
     try:
         raw = query({"search_query": f'ti:"{phrase}"',
                      "start": 0, "max_results": max_results})

tests/test_arxiv_refs.py

@@ -118,6 +118,69 @@ def test_a_match_needs_exactly_one_arxiv_paper():
     assert arxiv_refs.match_entries("Shared Title", ambiguous) is None
 
 
+def test_the_query_drops_maths_instead_of_mangling_it():
+    """The bug this fixes: maths was flattened into the phrase search.
+
+    `…galaxies at 0.5≤z<1.0` went out as `ti:"…galaxies at 0 5 z 1 0"`, which
+    matches nothing arXiv indexes — so no candidates came back and the exact
+    matcher had nothing to work on.
+    """
+    assert arxiv_refs.query_phrase(
+        "Photometric properties of classical bulge and pseudo-bulge "
+        "galaxies at 0.5≤z<1.0") == (
+        "Photometric properties of classical bulge and pseudo bulge "
+        "galaxies at")
+    assert arxiv_refs.query_phrase(
+        "Dwarf galaxies imply dark matter is heavier than 2.2×10−21eV") == \
+        "Dwarf galaxies imply dark matter is heavier than"
+    # Greek is not a word character to arXiv's index either — before, `Λ` went
+    # into the query string literally
+    assert "Λ" not in arxiv_refs.query_phrase(
+        "Review of solutions to the Cusp-core problem of the ΛCDM Model")
+
+
+def test_the_query_takes_the_longest_clean_run_wherever_it_falls():
+    # maths in the middle: the longer side wins, not merely the leading one
+    assert arxiv_refs.query_phrase("A z=3 study of the formation of dwarf "
+                                   "galaxies in clusters") == \
+        "study of the formation of dwarf galaxies in clusters"
+
+
+def test_a_title_with_no_usable_run_still_sends_something():
+    """Better a mangled query than no query: `match_entries` is the safety net."""
+    phrase = arxiv_refs.query_phrase("M87 z=6 3C273")
+    assert phrase and len(phrase.split()) >= 1
+
+
+def test_resolve_title_searches_on_the_phrase_not_the_raw_title():
+    sent = {}
+
+    def fake(params):
+        sent.update(params)
+        return _atom(("2401.00001", "Dwarf galaxies imply dark matter is "
+                                    r"heavier than $2.2\times10^{-21}$eV"))
+
+    got = arxiv_refs.resolve_title(
+        "Dwarf galaxies imply dark matter is heavier than 2.2×10−21eV",
+        query=fake)
+    # the phrase went out clean...
+    assert sent["search_query"] == \
+        'ti:"Dwarf galaxies imply dark matter is heavier than"'
+    # ...and the full title still had to match exactly to be accepted
+    assert got == "2401.00001"
+
+
+def test_a_broader_query_does_not_mean_a_looser_match():
+    """The phrase only chooses candidates; the title check is unchanged."""
+    def fake(params):
+        return _atom(("2401.00002", "Dwarf galaxies imply dark matter is "
+                                    "heavier than something else entirely"))
+
+    assert arxiv_refs.resolve_title(
+        "Dwarf galaxies imply dark matter is heavier than 2.2×10−21eV",
+        query=fake) is None
+
+
 def test_resolve_title_is_injectable_and_survives_a_dead_api():
     calls = []