diff --git a/.github/workflows/arxiv_refs.yml b/.github/workflows/arxiv_refs.yml index 3b471b7..e561eb8 100644 --- a/.github/workflows/arxiv_refs.yml +++ b/.github/workflows/arxiv_refs.yml @@ -32,6 +32,10 @@ on: description: "Report only — resolve nothing into the files" type: boolean default: false + mark_unresolved: + description: "Also NOTE-mark every line that fails to resolve, so later runs skip it. Human-reviewed cleanup — read a dry run first." + type: boolean + default: false # contents: write → the queue commit; actions: write → re-dispatch the # Dashboard (a token push does not retrigger `on: push` workflows). @@ -60,11 +64,17 @@ jobs: env: LIMIT: ${{ inputs.limit }} DRY_RUN: ${{ inputs.dry_run }} + MARK_UNRESOLVED: ${{ inputs.mark_unresolved }} run: | set -u args="--limit ${LIMIT:-60}" [ "${DRY_RUN:-false}" = "true" ] || args="$args --write" - python scripts/backfill_arxiv_refs.py $args | tee /tmp/backfill.log + # Never on the schedule: `inputs.*` is empty on a cron fire, so this + # can only ever be a deliberate dispatch. + [ "${MARK_UNRESOLVED:-false}" != "true" ] || args="$args --mark-unresolved" + # `-u`: unbuffered, so the log streams while the run is in flight + # instead of arriving in one block at the end. + python -u scripts/backfill_arxiv_refs.py $args | tee /tmp/backfill.log { echo '## arXiv refs backfill' echo '```' @@ -85,11 +95,18 @@ jobs: - name: Commit + push (explicit paths only) if: steps.backfill.outputs.changed == 'yes' && github.ref == 'refs/heads/main' + env: + MARK_UNRESOLVED: ${{ inputs.mark_unresolved }} run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" git add reading-queue.md arxiv-inbox.md - git commit -m "queue: backfill arXiv refs for ref-less papers [skip ci]" + if [ "${MARK_UNRESOLVED:-false}" = "true" ]; then + msg="queue: backfill arXiv refs, NOTE-mark what cannot be identified" + else + msg="queue: backfill arXiv refs for ref-less papers" + fi + git commit -m "${msg} [skip ci]" for attempt in 1 2 3; do if git push; then exit 0; fi git pull --rebase origin main || exit 1 diff --git a/scripts/arxiv_refs.py b/scripts/arxiv_refs.py index 2243bb5..a153e5c 100644 --- a/scripts/arxiv_refs.py +++ b/scripts/arxiv_refs.py @@ -184,20 +184,61 @@ def match_entries(title: str, entries: list[dict]) -> str | None: return hits.pop() if len(hits) == 1 else None -def resolve_title(title: str, max_results: int = 20, +#: A phrase shorter than this is too generic to be worth searching on. +MIN_PHRASE_WORDS = 3 + + +def query_phrase(title: str) -> str: + """The longest run of plain alphabetic words in `title`. + + What gets *sent* to arXiv, as opposed to what gets compared afterwards. + The two are different problems and conflating them was a real bug: a title + carrying maths used to be flattened word-by-word into the phrase search, so + ``…galaxies at 0.5≤z<1.0`` was sent as ``ti:"…galaxies at 0 5 z 1 0"``. + arXiv indexes ``$0.5\\leq z<1.0$``, matches none of that, and returns + nothing — at which point no amount of clever comparison in + :func:`match_entries` has anything to compare. + + So the query drops the maths instead of mangling it, and searches on the + longest stretch of ordinary words the title contains — the part both sides + spell the same way. That is a *broader* search, not a looser match: it only + decides which candidates come back, and every one of them still has to + survive the exact, unique-title check afterwards. + + Falls back to every ASCII word when no run is long enough + (``MIN_PHRASE_WORDS``), since a two-word phrase would drown the result set. + """ + # `flags=re.ASCII` so `\w` stops treating Greek as a word character: `Λ` + # became a literal `Λ` in the query string before, which arXiv cannot match + # against its own `$\Lambda$` source. + tokens = re.sub(r"[^\w\s]", " ", title, flags=re.ASCII).split() + best: list[str] = [] + run: list[str] = [] + for token in tokens: + if token.isalpha(): + run.append(token) + if len(run) > len(best): + best = list(run) + else: + run = [] + return " ".join(best if len(best) >= MIN_PHRASE_WORDS else tokens) + + +def resolve_title(title: str, max_results: int = 50, query: object = _api_query) -> str | None: """Look `title` up on the arXiv API and return its id, or None. Networked — the Dashboard never calls this; only the backfill does. `query` is injected so the matching can be tested without arXiv. + + `max_results` is generous because :func:`query_phrase` deliberately searches + on part of a title rather than all of it: more candidates come back, and + picking among them costs nothing — the choice is made locally, by exact + match, and an ambiguous set still resolves to nothing. """ - # `ti:"…"` is a phrase search over titles. arXiv's search index chokes on - # unbalanced quotes and LaTeX, so the phrase is sent as plain words; the - # exactness comes from match_entries afterwards, never from the query. - words = re.sub(r'[^\w\s]', " ", title).split() - if not words: + phrase = query_phrase(title) + if not phrase: return None - phrase = " ".join(words) try: raw = query({"search_query": f'ti:"{phrase}"', "start": 0, "max_results": max_results}) diff --git a/tests/test_arxiv_refs.py b/tests/test_arxiv_refs.py index 2f33b50..824eebe 100644 --- a/tests/test_arxiv_refs.py +++ b/tests/test_arxiv_refs.py @@ -118,6 +118,69 @@ def test_a_match_needs_exactly_one_arxiv_paper(): assert arxiv_refs.match_entries("Shared Title", ambiguous) is None +def test_the_query_drops_maths_instead_of_mangling_it(): + """The bug this fixes: maths was flattened into the phrase search. + + `…galaxies at 0.5≤z<1.0` went out as `ti:"…galaxies at 0 5 z 1 0"`, which + matches nothing arXiv indexes — so no candidates came back and the exact + matcher had nothing to work on. + """ + assert arxiv_refs.query_phrase( + "Photometric properties of classical bulge and pseudo-bulge " + "galaxies at 0.5≤z<1.0") == ( + "Photometric properties of classical bulge and pseudo bulge " + "galaxies at") + assert arxiv_refs.query_phrase( + "Dwarf galaxies imply dark matter is heavier than 2.2×10−21eV") == \ + "Dwarf galaxies imply dark matter is heavier than" + # Greek is not a word character to arXiv's index either — before, `Λ` went + # into the query string literally + assert "Λ" not in arxiv_refs.query_phrase( + "Review of solutions to the Cusp-core problem of the ΛCDM Model") + + +def test_the_query_takes_the_longest_clean_run_wherever_it_falls(): + # maths in the middle: the longer side wins, not merely the leading one + assert arxiv_refs.query_phrase("A z=3 study of the formation of dwarf " + "galaxies in clusters") == \ + "study of the formation of dwarf galaxies in clusters" + + +def test_a_title_with_no_usable_run_still_sends_something(): + """Better a mangled query than no query: `match_entries` is the safety net.""" + phrase = arxiv_refs.query_phrase("M87 z=6 3C273") + assert phrase and len(phrase.split()) >= 1 + + +def test_resolve_title_searches_on_the_phrase_not_the_raw_title(): + sent = {} + + def fake(params): + sent.update(params) + return _atom(("2401.00001", "Dwarf galaxies imply dark matter is " + r"heavier than $2.2\times10^{-21}$eV")) + + got = arxiv_refs.resolve_title( + "Dwarf galaxies imply dark matter is heavier than 2.2×10−21eV", + query=fake) + # the phrase went out clean... + assert sent["search_query"] == \ + 'ti:"Dwarf galaxies imply dark matter is heavier than"' + # ...and the full title still had to match exactly to be accepted + assert got == "2401.00001" + + +def test_a_broader_query_does_not_mean_a_looser_match(): + """The phrase only chooses candidates; the title check is unchanged.""" + def fake(params): + return _atom(("2401.00002", "Dwarf galaxies imply dark matter is " + "heavier than something else entirely")) + + assert arxiv_refs.resolve_title( + "Dwarf galaxies imply dark matter is heavier than 2.2×10−21eV", + query=fake) is None + + def test_resolve_title_is_injectable_and_survives_a_dead_api(): calls = []