diff --git a/.github/workflows/knowledge_board.yml b/.github/workflows/knowledge_board.yml index f070995..31c3d88 100644 --- a/.github/workflows/knowledge_board.yml +++ b/.github/workflows/knowledge_board.yml @@ -49,7 +49,12 @@ jobs: name: github-pages url: ${{ steps.deployment.outputs.page_url }} steps: + # Full history: the catch-up banner dates the last ingest from the + # commits that added bib entries (scripts/catch_up.py); a shallow clone + # has none of them. - uses: actions/checkout@v4 + with: + fetch-depth: 0 # A prefilled `…/issues/new?…&labels=x` URL silently drops `x` when the # label does not exist in the repo, and queue_actions.yml gates on the diff --git a/AGENTS.md b/AGENTS.md index 25a7e0d..b0e364e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -39,6 +39,14 @@ without a file extension) — read it, stub it in the right and run `make validate`. Unrecognised top-level files/folders fail the lint; the allowlist is in `scripts/validate_structure.py`. +## Skills + +Organ-owned skills live in `skills//` (`SKILL.md` + the procedure), +like Mind's. [`skills/catch_up/`](skills/catch_up/catch_up.md) is the +after-time-away door: `scripts/catch_up.py` harvests what was missed since +the last ingest, the agent curates it into intake / cite / queue / skip, the +human confirms once, and the chosen papers are filed in one PR. + ## What does NOT live here - **Operational history** — what the organism *did* (prior tasks, decisions, diff --git a/README.md b/README.md index dd4777a..54fe61f 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,12 @@ the metadata and the claim support together, then passes `make validate` (CI-enforced on every push). +Away for a while? [`/catch_up`](skills/catch_up/catch_up.md) collects every +paper missed since memory last ingested one — lapsed suggestions from git +history, the open queue, and an arXiv gap-fill over days the digest never ran +(`scripts/catch_up.py`) — for one triage and one filing PR. The dashboard +shows a banner once a week has passed without an ingest. + The wiki schema is defined in [`wiki/AGENTS.md`](wiki/AGENTS.md) and inherited by every sub-wiki. How agents should read this repo: [AGENTS.md](AGENTS.md). The diff --git a/scripts/board.py b/scripts/board.py index 7239fda..b425588 100644 --- a/scripts/board.py +++ b/scripts/board.py @@ -84,6 +84,16 @@ import arxiv_refs # noqa: E402 import inbox_actions # noqa: E402 import interests_actions # noqa: E402 +import catch_up # noqa: E402 + +#: How many days without an ingest before the board asks for a catch-up. A week: +#: the suggestion tiers lapse on the same seven days +#: (``inbox_actions.INBOX_WINDOW_DAYS``), so past it the inbox alone no longer +#: holds everything that was missed — git history does, which is exactly what +#: ``scripts/catch_up.py`` reads back. +CATCH_UP_DAYS = 7 +CATCH_UP_LYRIC = "…so… I've been lost for a while" # Fred again.. +CATCH_UP_CMD = "/catch_up lensing" # The family look lives once, in the Brain (``board/_theme.py``): the # stylesheet, the hero that redraws this organ's logo as a mark, and the @@ -233,7 +243,14 @@ def collect(root: Path | None = None) -> dict: # Filings that reached a `queue-filing/issue-` branch but not # main — the one thing on the page that is a human's merge to make. "filings": _pending_filings() if root == MEMORY_HOME else [], + # When memory last took a paper in (any scope): git + files, never the + # network. None when nothing is on record or git is unavailable. + "last_ingested": None, } + try: + snapshot["last_ingested"] = catch_up.last_ingested("all", root) + except Exception: # noqa: BLE001 — the banner is advisory, never fatal + snapshot["last_ingested"] = None stems: set[str] = set() all_slugs: list[str] = [] @@ -595,6 +612,51 @@ def _read_trend(snapshot: dict) -> dict: return trend +def _days_since_ingest(snapshot: dict) -> int | None: + """Days from ``last_ingested`` to the snapshot's ``generated`` date. + + Pure like :func:`_read_trend` — relative to the snapshot, not the wall + clock — so a fixture renders the same banner on any calendar day. None when + either date is missing or unparseable. + """ + try: + now = datetime.date.fromisoformat(str(snapshot.get("generated"))[:10]) + last = datetime.date.fromisoformat(str(snapshot.get("last_ingested"))) + except ValueError: + return None + return max(0, (now - last).days) + + +def _catch_up_due(snapshot: dict) -> int | None: + """The day count when the catch-up banner should show, else None.""" + days = _days_since_ingest(snapshot) + return days if days is not None and days >= CATCH_UP_DAYS else None + + +def _catch_up_md(snapshot: dict) -> list[str]: + days = _catch_up_due(snapshot) + if days is None: + return [] + return [f"> _{CATCH_UP_LYRIC}_", ">", + f"> **{days} days since papers were last ingested into memory** " + f"(last: {snapshot.get('last_ingested')}) — run `{CATCH_UP_CMD}`", + ""] + + +def _catch_up_html(snapshot: dict) -> str: + days = _catch_up_due(snapshot) + if days is None: + return "" + last = _html.escape(str(snapshot.get("last_ingested"))) + return (f"

" + f"{_html.escape(CATCH_UP_LYRIC)}

" + f"

{days} days since papers were last ingested into " + f"memory (last: {last}) " + f"{_html.escape(CATCH_UP_CMD)} " + f"{_copy_btn(CATCH_UP_CMD, 'copy: catch up on strong lensing')}" + f"

") + + def _totals(snapshot: dict) -> dict: wikis = snapshot.get("wikis") or [] statuses: dict[str, int] = {} @@ -717,7 +779,7 @@ def _inbox_empty_note(fresh: dict) -> str: # --- renderers ------------------------------------------------------------------ def _render_md(snapshot: dict) -> str: t = _totals(snapshot) - lines = ["# PyAutoMemory Dashboard", "", + lines = ["# PyAutoMemory Dashboard", "", *_catch_up_md(snapshot), "_Contents and work queues — the knowledge itself lives in the " "wiki pages._", "", f"**{t['pages']} pages** across {len(snapshot.get('wikis') or [])} " @@ -956,6 +1018,9 @@ def _bar(statuses: dict) -> str: color:var(--bg);opacity:0;transition:opacity .2s;pointer-events:none;z-index:9} #toast.show{opacity:1} #toast.bad{background:var(--warn)} +.catchup{border-left:3px solid var(--warn);padding:.4em .9em;margin:0 0 1em} +.catchup p{margin:.25em 0} +.catchup .lyric{color:var(--warn)} """ # The shared copy handler is delegated, so a chip inside a would @@ -1413,6 +1478,7 @@ def _fresh_span(fresh: dict, now_text: str, stale_text: str, +{_catch_up_html(snapshot)} {hero} {stats}

markdown version{github_link}{onetap}

@@ -1525,6 +1591,8 @@ def to_state(snapshot: dict) -> dict: "updated": _iso_z(snapshot.get("generated")), "pages_url": page, "items": [], + # Days since memory last ingested a paper (None: nothing on record). + "days_since_ingest": _days_since_ingest(snapshot), } if not t["pages"]: return state diff --git a/scripts/catch_up.py b/scripts/catch_up.py new file mode 100644 index 0000000..bc3f308 --- /dev/null +++ b/scripts/catch_up.py @@ -0,0 +1,719 @@ +"""scripts/catch_up.py — everything memory missed while nobody was filing. + +The suggestion tiers are built for a human who looks every day. ``arxiv-inbox.md`` +and ``arxiv-interests.md`` lapse after :data:`inbox_actions.INBOX_WINDOW_DAYS`, +the reading queue only grows, and the nightly digest itself has gaps (it never +ran 2026-09-17→22 or 09-24→28, so for most of that fortnight nothing was even +fetched). After time away there is no single place that says *what did I miss*. +This script is that place; the ``/catch_up`` skill (``skills/catch_up/``) is the +curation and filing on top of it. + +It answers two questions, both per **scope** (:data:`SCOPES`): + +1. **When did memory last take a paper in?** — :func:`last_ingested`. The + newest of (a) a ``DONE `` line in the scope's reading-queue sections + and (b) a commit that adds an ``@type{key,`` entry to + ``bibliography/pyautomemory.bib`` *and* touches the scope's + ``wiki//sources/``. Both halves, because either alone lies: a bib + commit with no sources page is a citation tidy-up, and a sources commit + with no new entry is a restructure (the 2026-09-17 seed split touched every + lensing sources page and ingested nothing). Merges are diffed against their + first parent, so a ``queue-filing/`` branch counts on the day it landed on + ``main``. The dashboard banner reads this too — git-only, never network. + +2. **What arrived since?** — three harvests, deduped by arXiv id (else a + normalised title), minus anything already in memory (id or title in the + ``.bib``, or a ``DONE`` line anywhere in the queue): + + * ``history`` — every line ever *added* to the scope's suggestion file, read + from ``git log -p``. A swept line is not lost, git holds it; this is where + it comes back from. + * ``queue`` — the open paper lines of the scope's reading-queue sections + (``DONE``/``NOTE``/``__sub-heading__`` lines are not papers; ``board.py``'s + ``_parse_paper`` decides, so there is one grammar). + * ``arxiv`` — the Mind digest's own query re-run over the whole window, which + is what fills the digest's gaps. The query and the fetch/parse are imported + from ``PyAutoMind/.github/scripts/`` rather than copied, so "strong + lensing" means the same thing here as it does at 02:00 UTC. Anything + harvested without an abstract is then looked up by id in one request, so + the curating agent reads abstracts rather than titles. + +The network half is best-effort by design: Mind missing, arXiv refusing, a +timeout — each is a ``warnings`` entry on stderr and the source is skipped. A +catch-up that crashes because arXiv 429'd would lose the git harvest too. + +**Interests volume.** The interests query is *categories* (a few hundred papers +a day), so over weeks it is thousands. It is capped at :data:`INTERESTS_ARXIV_CAP` +most-recent results, ranked by the Mind digest's own interest scorer, and the +cap is reported as ``truncated`` — interests curation leans on the interests +file's history, and ``--topic`` narrows the query to that topic's terms. + +Pure functions + a thin CLI, stdlib only. ``--json`` is the skill's input. + +Usage: + python scripts/catch_up.py --scope lensing [--json] + python scripts/catch_up.py --scope interests --topic SMBHs + python scripts/catch_up.py --scope all --since 2026-09-01 --offline +""" + +from __future__ import annotations + +import argparse +import datetime +import importlib.util +import json +import os +import re +import subprocess +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import arxiv_refs # noqa: E402 +import inbox_actions # noqa: E402 +import interests_actions # noqa: E402 + +MEMORY_HOME = Path(__file__).resolve().parents[1] +BIB_FILE = "bibliography/pyautomemory.bib" + +#: What each scope means on disk: the reading-queue sections it owns, the +#: sub-wikis whose ``sources/`` count as an ingest, the suggestion file whose +#: history is harvested, and which Mind digest's query fills the gaps. +#: Section names are ``reading-queue.md`` headers verbatim. "Interests" is the +#: interests list's fallback section (``interests_actions.FALLBACK_SECTION``), +#: so a paper ➕'d from it is caught. Project sections (SETI, Cancer, KetJU) +#: are deliberately out of every scope — they are not arXiv-fed reading. +SCOPES = { + "lensing": { + "sections": ("Strong Lensing",), + "domains": ("lensing",), + "suggestions": inbox_actions.INBOX_FILE, + "digest": "arxiv_fetch", + }, + "interests": { + "sections": ("SMBHs", "Galaxy Formation / Evolution", "Dark Matter", + "Stats", interests_actions.FALLBACK_SECTION), + "domains": ("smbh", "galaxies", "methods"), + "suggestions": interests_actions.INTERESTS_FILE, + "digest": "arxiv_interests", + }, +} +#: The sub-wiki a single interests topic files into, for ``--topic``. A topic +#: with no sub-wiki of its own (Dark Matter) is dated by its DONE lines alone. +TOPIC_DOMAINS = {"SMBHs": ("smbh",), "Galaxy Formation / Evolution": ("galaxies",), + "Stats": ("methods",), "Dark Matter": (), "Interests": ()} + +#: Submission lead before the cutoff. A paper *announced* on the cutoff day was +#: submitted one to three days earlier (a weekend band is three), so the arXiv +#: window opens that far back; anything already filed dedups away. +ANNOUNCE_LAG_DAYS = 3 +#: The lensing query is ~1.5 papers a day; pages are a runaway guard. +LENSING_PAGE_SIZE = 100 +LENSING_MAX_PAGES = 5 +#: The interests query is whole categories. Most-recent-first, so the cap keeps +#: the newest days whole and loses the oldest — reported, never silent. +INTERESTS_ARXIV_CAP = 300 +INTERESTS_PAGE_SIZE = 100 +#: arXiv's API terms ask for 3 s between requests. +ARXIV_PAUSE_S = 3.0 +#: One id_list lookup fills abstracts for harvested ids; the API pages at ~200. +ENRICH_BATCH = 100 + +SOURCE_ORDER = ("queue", "history", "arxiv") + +BIB_ENTRY_ADD_RE = re.compile(r"^\+@\w+\{") +BIB_ID_RE = re.compile( + r"^\s*(?:eprint|arxivid|url|doi)\s*=\s*[{\"](.*?)[}\"]\s*,?\s*$", + re.IGNORECASE) +BIB_TITLE_RE = re.compile(r"^\s*title\s*=\s*\{(.*)\}\s*,?\s*$", re.IGNORECASE) +ARXIV_NUM_RE = re.compile(r"(\d{4}\.\d{4,5})") +DONE_DATE_RE = re.compile(r"^DONE\s+(\d{4}-\d{2}-\d{2})\b") +SECTION_RE = re.compile(r"^##\s+(.+?)\s*$") + + +# --- keys ---------------------------------------------------------------------- +def norm_title(title: str | None) -> str: + """A title reduced to what survives BibTeX bracing, LaTeX and re-typing. + + Stronger than ``inbox_actions.identity``'s case-fold: a bib title arrives as + ``{{TDCOSMO XXVIII. … J1537$-$3010 …}}`` and a queue line as plain text, so + only letters and digits are compared. + """ + return re.sub(r"[^a-z0-9]", "", (title or "").lower()) + + +def paper_id(ref: str | None) -> str | None: + """The bare arXiv number in a queue/inbox ref, or None.""" + return arxiv_refs.arxiv_id(ref) if ref else None + + +# --- scope ---------------------------------------------------------------------- +def resolve_scope(scope: str, topic: str | None = None) -> dict: + """The sections/domains/suggestion files a scope (and topic) covers. + + ``all`` is the union. A ``topic`` narrows interests to one section; naming a + topic under ``lensing`` is an error rather than a silent no-op. + """ + if scope == "all": + parts = [SCOPES["lensing"], SCOPES["interests"]] + elif scope in SCOPES: + parts = [SCOPES[scope]] + else: + raise ValueError(f"unknown scope {scope!r} (lensing|interests|all)") + if topic: + if scope == "lensing": + raise ValueError("--topic narrows the interests scope only") + if topic not in SCOPES["interests"]["sections"]: + raise ValueError(f"unknown topic {topic!r}: one of " + + ", ".join(SCOPES["interests"]["sections"])) + sections, domains, files = [], [], [] + for p in parts: + is_interests = p is SCOPES["interests"] + sec = (topic,) if (topic and is_interests) else p["sections"] + dom = TOPIC_DOMAINS.get(topic, ()) if (topic and is_interests) else p["domains"] + sections += sec + domains += dom + files.append((p["suggestions"], p["digest"])) + return {"scope": scope, "topic": topic, "sections": tuple(sections), + "domains": tuple(domains), "files": files} + + +# --- the queue ------------------------------------------------------------------ +def queue_sections(queue_text: str) -> dict[str, list[dict]]: + """``{section: [parsed line, …]}`` via ``board._parse_paper`` — one grammar. + + Imported lazily: ``board`` imports this module for the banner, and a + module-level import in both directions is a cycle. + """ + import board # noqa: PLC0415 + out: dict[str, list[dict]] = {} + section = None + for line in queue_text.splitlines(): + m = SECTION_RE.match(line) + if m and not m.group(1).startswith("#"): + section = m.group(1) + out.setdefault(section, []) + elif section is not None and line.strip(): + out[section].append(board._parse_paper(line.strip())) + return out + + +def last_done(queue_text: str, sections) -> str | None: + """The newest ``DONE `` in the named sections (all when None).""" + best = None + for sec, lines in queue_sections(queue_text).items(): + if sections is not None and sec not in sections: + continue + for p in lines: + d = p.get("done_date") + if d and (best is None or d > best): + best = d + return best + + +# --- git ------------------------------------------------------------------------- +def _git(root: Path, *args: str) -> str | None: + """stdout of a git command in ``root``, or None when git/the repo is absent.""" + try: + got = subprocess.run(["git", "-C", str(root), *args], + capture_output=True, text=True, errors="replace", + timeout=120) + except (OSError, subprocess.SubprocessError): + return None + return got.stdout if got.returncode == 0 else None + + +def parse_ingest_log(log: str, domains) -> dict | None: + """The newest ingest commit in a ``git log -p`` of bib + sources. + + ``log`` is ``git log --format=%x00%cs %h --unified=0 -p`` output: commits + separated by NUL. A commit is an ingest when it adds a bib entry AND touches + one of ``domains``' ``sources/``. Max by date, not first-seen: merge commits + and their branch commits interleave out of date order. + """ + prefixes = tuple(f"wiki/{d}/sources/" for d in domains) + best = None + for chunk in log.split("\x00"): + if not chunk.strip(): + continue + head, _, body = chunk.partition("\n") + date, _, sha = head.strip().partition(" ") + current, adds, touched = None, False, False + for line in body.splitlines(): + if line.startswith("diff --git "): + current = line.rsplit(" b/", 1)[-1] + if prefixes and current.startswith(prefixes): + touched = True + elif current == BIB_FILE and BIB_ENTRY_ADD_RE.match(line): + adds = True + if adds and touched and (best is None or date > best["date"]): + best = {"date": date, "commit": sha} + return best + + +def last_bib_ingest(root: Path, domains) -> dict | None: + if not domains: + return None + # A shallow clone (CI's default checkout) diffs its one commit against + # nothing, so every bib entry looks added today and the banner would never + # show. No history, no evidence: the DONE half stands alone. + if (_git(root, "rev-parse", "--is-shallow-repository") or "").strip() == "true": + return None + paths = [BIB_FILE] + [f"wiki/{d}/sources" for d in domains] + log = _git(root, "log", "--diff-merges=first-parent", "--unified=0", + "--format=%x00%cs %h", "-p", "--", *paths) + return parse_ingest_log(log, domains) if log else None + + +def ingest_evidence(scope: str = "all", root: Path | None = None, + topic: str | None = None) -> dict: + """Both halves of the cutoff, for reporting: DONE date and ingest commit.""" + root = root or MEMORY_HOME + spec = resolve_scope(scope, topic) + queue = root / inbox_actions.QUEUE_FILE + text = queue.read_text(errors="replace") if queue.exists() else "" + done = last_done(text, spec["sections"]) + bib = last_bib_ingest(root, spec["domains"]) + dates = [d for d in (done, bib and bib["date"]) if d] + return {"done": done, "bib": bib, "last_ingested": max(dates) if dates else None} + + +def last_ingested(scope: str = "all", root: Path | None = None, + topic: str | None = None) -> str | None: + """The date memory last took a paper in, for ``scope``. Git + files only.""" + return ingest_evidence(scope, root, topic)["last_ingested"] + + +# --- in-memory index ----------------------------------------------------------------- +def in_memory_index(bib_text: str, queue_text: str) -> tuple[set, set]: + """``(ids, titles)`` already in memory: the bib, plus every DONE line.""" + ids, titles = set(), set() + for line in bib_text.splitlines(): + m = BIB_ID_RE.match(line) + if m: + ids |= set(ARXIV_NUM_RE.findall(m.group(1))) + continue + m = BIB_TITLE_RE.match(line) + if m: + titles.add(norm_title(m.group(1))) + for lines in queue_sections(queue_text).values(): + for p in lines: + if p.get("done") and p.get("kind") == "paper": + pid = paper_id(p.get("ref")) + if pid: + ids.add(pid) + titles.add(norm_title(p.get("title"))) + titles.discard("") + return ids, titles + + +# --- harvests -------------------------------------------------------------------- +def parse_history(log: str, suggestions_file: str, since: str, + topic: str | None = None) -> list[dict]: + """Paper lines ever added to a suggestion file, dated on or after ``since``. + + ``log`` is ``git log -p`` of the file; only ``+`` lines are read, so a line + later swept or dismissed still comes back. The line's own date is the + anchor (the announcement date), not the commit's. + """ + interests = suggestions_file == interests_actions.INTERESTS_FILE + out = [] + for raw in log.splitlines(): + if not raw.startswith("+") or raw.startswith("+++"): + continue + line = raw[1:] + if interests: + got = interests_actions.parse_line(line) + if not got: + continue + added, tag, rest = got + else: + got = inbox_actions.parse_line(line) + if not got: + continue + (added, rest), tag = got, None + if added < since: + continue + if topic and (tag or interests_actions.FALLBACK_SECTION) != topic: + continue + title, ref = inbox_actions.split_ref(rest) + out.append({"id": paper_id(ref), "title": title, "ref": ref, + "date": added, "topic": tag, "source": "history"}) + return out + + +def harvest_history(root: Path, suggestions_file: str, since: str, + topic: str | None = None) -> list[dict]: + log = _git(root, "log", "-p", "--format=", "--", suggestions_file) or "" + # The working copy too: an uncommitted local line is still a suggestion. + path = root / suggestions_file + if path.exists(): + log += "\n" + "\n".join("+" + ln for ln in + path.read_text(errors="replace").splitlines()) + return parse_history(log, suggestions_file, since, topic) + + +def harvest_queue(queue_text: str, sections) -> list[dict]: + """Open paper lines of the named sections, in file order.""" + out = [] + for sec, lines in queue_sections(queue_text).items(): + if sec not in sections: + continue + for p in lines: + if p.get("kind") != "paper" or p.get("done"): + continue + out.append({"id": paper_id(p.get("ref")), "title": p["title"], + "ref": p.get("ref"), "date": None, "section": sec, + "source": "queue"}) + return out + + +# --- arXiv (network, best-effort) ------------------------------------------------------- +def mind_scripts_dir(root: Path | None = None) -> Path | None: + """PyAutoMind's ``.github/scripts``: ``$PYAUTO_MIND``, else the Brain resolver. + + The Brain's ``agents/_repo_paths.py`` is the one place that knows how a + workspace lays repos out (flat, family dirs, task bundles); it is found by + walking up from this checkout. A sibling ``PyAutoMind`` is the last resort. + """ + root = root or MEMORY_HOME + cands = [] + if os.environ.get("PYAUTO_MIND"): + cands.append(Path(os.environ["PYAUTO_MIND"])) + for parent in (root, *root.parents): + resolver = None + for brain in (parent / "organs" / "PyAutoBrain", parent / "PyAutoBrain"): + if (brain / "agents" / "_repo_paths.py").is_file(): + resolver = brain / "agents" / "_repo_paths.py" + break + if resolver: + try: + mod = _load(resolver, "_pyauto_repo_paths") + cands.append(Path(mod.repo_path(parent, "PyAutoMind"))) + except Exception: # noqa: BLE001 — a resolver fault is a fallback + pass + break + cands.append(root.parent / "PyAutoMind") + for c in cands: + d = c / ".github" / "scripts" + if (d / "arxiv_fetch.py").is_file(): + return d + return None + + +def _load(path: Path, name: str): + spec = importlib.util.spec_from_file_location(name, path) + mod = importlib.util.module_from_spec(spec) + sys.modules[name] = mod + spec.loader.exec_module(mod) + return mod + + +def load_digests(scripts_dir: Path) -> dict: + """``{"arxiv_fetch": mod, "arxiv_interests": mod}`` from Mind, by file path. + + ``arxiv_interests`` does ``import arxiv_fetch`` itself, so the scripts dir + goes on ``sys.path`` for the duration. + """ + sys.path.insert(0, str(scripts_dir)) + try: + fetch = _load(scripts_dir / "arxiv_fetch.py", "arxiv_fetch") + interests = _load(scripts_dir / "arxiv_interests.py", "arxiv_interests") + finally: + sys.path.remove(str(scripts_dir)) + return {"arxiv_fetch": fetch, "arxiv_interests": interests} + + +def window_query(query: str, start: datetime.date, end: datetime.date) -> str: + """The digest's query restricted to a submission window (UTC, inclusive).""" + return (f"({query}) AND submittedDate:[{start:%Y%m%d}0000 TO " + f"{end:%Y%m%d}2359]") + + +def _band(start: datetime.date, end: datetime.date): + """``parse``'s half-open band, wide enough to keep the whole window.""" + utc = datetime.timezone.utc + lo = datetime.datetime(start.year, start.month, start.day, tzinfo=utc) + hi = datetime.datetime(end.year, end.month, end.day, tzinfo=utc) + return lo - datetime.timedelta(seconds=1), hi + datetime.timedelta(days=1) + + +def _to_candidate(paper: dict, source: str = "arxiv") -> dict: + aid = arxiv_refs.arxiv_id(paper["url"]) + return {"id": aid, "title": paper["title"], "ref": aid, + "date": (paper.get("published") or "")[:10] or None, + "authors": paper.get("authors") or [], + "abstract": paper.get("abstract"), + "primary_category": paper.get("primary_category"), + "topic": paper.get("topic"), "source": source} + + +def harvest_arxiv(digests: dict, digest: str, start: datetime.date, + end: datetime.date, topic: str | None = None, + pause: float = ARXIV_PAUSE_S) -> tuple[list[dict], bool]: + """One paged sequence of the digest's query over the window. + + Returns ``(candidates, truncated)``. Lensing pages to the end of the window; + interests stops at :data:`INTERESTS_ARXIV_CAP`, ranks with the digest's own + scorer, keeps the scope's topic(s), and drops what the lensing net catches + (that is the other scope's paper). + """ + fetch = digests["arxiv_fetch"] + if digest == "arxiv_fetch": + query, size, pages = fetch.QUERY, LENSING_PAGE_SIZE, LENSING_MAX_PAGES + else: + mod = digests["arxiv_interests"] + query = mod.QUERY + if topic and topic in getattr(mod, "INTERESTS", {}): + terms = mod.INTERESTS[topic] + query = (f"({query}) AND (" + + " OR ".join(f'abs:"{t}"' for t in terms) + ")") + size = INTERESTS_PAGE_SIZE + pages = -(-INTERESTS_ARXIV_CAP // size) + q = window_query(query, start, end) + lo, hi = _band(start, end) + got, seen, truncated = [], set(), True + for page in range(pages): + if page: + time.sleep(pause) + raw = fetch.fetch(q, size, start=page * size) + batch = fetch.parse(raw, lo, hi) + n_entries = raw.count(b"") if isinstance(raw, bytes) else len(batch) + for p in batch: + if p["url"] not in seen: + seen.add(p["url"]) + got.append(p) + if n_entries < size: + truncated = False + break + if digest == "arxiv_fetch": + return [_to_candidate(p) for p in got], truncated + mod = digests["arxiv_interests"] + ranked, _ = mod.rank(got, cap=len(got) or 1) + keep = SCOPES["interests"]["sections"] if not topic else (topic,) + out = [_to_candidate(p) for p in ranked + if not p.get("strong_lensing") and (p.get("topic") or "Interests") in keep] + return out, truncated + + +def enrich(digests: dict, ids: list[str], pause: float = ARXIV_PAUSE_S) -> dict: + """``{id: arXiv record}`` for ids harvested without an abstract.""" + fetch = digests["arxiv_fetch"] + lo = datetime.datetime(1991, 1, 1, tzinfo=datetime.timezone.utc) + hi = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=2) + out = {} + for i in range(0, len(ids), ENRICH_BATCH): + if i: + time.sleep(pause) + chunk = ids[i:i + ENRICH_BATCH] + raw = fetch._get({"id_list": ",".join(chunk), "max_results": len(chunk)}) + for p in fetch.parse(raw, lo, hi): + c = _to_candidate(p) + if c["id"]: + out[c["id"]] = c + return out + + +# --- merge ------------------------------------------------------------------------- +def merge(harvests: list[dict], ids_in_memory: set, titles_in_memory: set + ) -> tuple[list[dict], int]: + """Dedup across sources and drop what memory already holds. + + Keyed by arXiv id, falling back to the normalised title (and an id-less + line still meets its id'd twin through the title). The first source in + :data:`SOURCE_ORDER` owns the record; later ones land in ``also_in`` and + fill any field the owner lacked (the abstract, usually). Returns + ``(candidates, dropped_as_in_memory)``. + """ + rank = {s: i for i, s in enumerate(SOURCE_ORDER)} + by_id: dict[str, dict] = {} + by_title: dict[str, dict] = {} + records: list[dict] = [] + dropped: set = set() + for h in sorted(harvests, key=lambda h: rank.get(h["source"], 99)): + nt = norm_title(h.get("title")) + key = h.get("id") or nt + if (h.get("id") and h["id"] in ids_in_memory) or (nt and nt in titles_in_memory): + dropped.add(key) + continue + rec = (by_id.get(h["id"]) if h.get("id") else None) or (by_title.get(nt) if nt else None) + if rec is None: + rec = {"id": None, "title": None, "ref": None, "date": None, + "authors": [], "abstract": None, "section": None, + "topic": None, "primary_category": None, + "source": h["source"], "also_in": []} + records.append(rec) + elif h["source"] != rec["source"] and h["source"] not in rec["also_in"]: + rec["also_in"].append(h["source"]) + for k, v in h.items(): + if k in ("source",): + continue + if v and not rec.get(k): + rec[k] = v + if rec.get("id"): + by_id[rec["id"]] = rec + if nt: + by_title[nt] = rec + for rec in records: + rec["first_author"] = rec["authors"][0] if rec.get("authors") else None + rec["url"] = f"https://arxiv.org/abs/{rec['id']}" if rec.get("id") else None + return records, len(dropped) + + +# --- the whole run ---------------------------------------------------------------------- +def catch_up(scope: str = "lensing", topic: str | None = None, + since: str | None = None, today: datetime.date | None = None, + root: Path | None = None, offline: bool = False, + digests: dict | None = None, pause: float = ARXIV_PAUSE_S) -> dict: + """The report the CLI prints and the skill curates. ``digests`` stubs arXiv.""" + root = root or MEMORY_HOME + today = today or datetime.datetime.now(datetime.timezone.utc).date() + spec = resolve_scope(scope, topic) + evidence = ingest_evidence(scope, root, topic) + warnings: list[str] = [] + cutoff = since or evidence["last_ingested"] + cutoff_from = "--since" if since else "last_ingested" + if not cutoff: + cutoff_from = "oldest_suggestion" + # Never ingested (the interests sub-wikis were imported, not filed): + # everything the digest ever suggested is "missed", so the cutoff is + # the oldest suggestion on record — else one inbox window back. + ever = [h["date"] for fname, _ in spec["files"] + for h in harvest_history(root, fname, "0000-00-00")] + cutoff = min(ever) if ever else ( + today - datetime.timedelta(days=inbox_actions.INBOX_WINDOW_DAYS) + ).isoformat() + warnings.append(f"no ingest on record for scope {scope!r}; cutoff set " + f"to the oldest suggestion, {cutoff}") + + queue_path = root / inbox_actions.QUEUE_FILE + queue_text = queue_path.read_text(errors="replace") if queue_path.exists() else "" + bib_path = root / BIB_FILE + bib_text = bib_path.read_text(errors="replace") if bib_path.exists() else "" + + harvests: list[dict] = [] + for fname, _ in spec["files"]: + t = topic if fname == interests_actions.INTERESTS_FILE else None + harvests += harvest_history(root, fname, cutoff, t) + harvests += harvest_queue(queue_text, spec["sections"]) + + start = datetime.date.fromisoformat(cutoff) - datetime.timedelta(days=ANNOUNCE_LAG_DAYS) + truncated = {} + if not offline: + if digests is None: + sdir = mind_scripts_dir(root) + if sdir is None: + warnings.append("PyAutoMind not found (set PYAUTO_MIND); arXiv " + "gap-fill skipped") + else: + try: + digests = load_digests(sdir) + except Exception as e: # noqa: BLE001 + warnings.append(f"could not load the Mind digest scripts: {e}") + if digests is not None: + for _, digest in spec["files"]: + t = topic if digest == "arxiv_interests" else None + try: + got, cut = harvest_arxiv(digests, digest, start, today, t, pause) + harvests += got + truncated[digest] = cut + if cut: + warnings.append( + f"{digest}: window truncated at " + f"{INTERESTS_ARXIV_CAP if digest == 'arxiv_interests' else LENSING_PAGE_SIZE * LENSING_MAX_PAGES}" + " most-recent results — the oldest days are missing") + except Exception as e: # noqa: BLE001 — network is best-effort + warnings.append(f"{digest}: arXiv query failed ({e}); skipped") + + ids_mem, titles_mem = in_memory_index(bib_text, queue_text) + candidates, dropped = merge(harvests, ids_mem, titles_mem) + + if digests is not None and not offline: + need = [c["id"] for c in candidates if c["id"] and not c.get("abstract")] + if need: + try: + time.sleep(pause) + found = enrich(digests, need, pause) + for c in candidates: + f = found.get(c["id"]) if c["id"] else None + if f: + for k in ("abstract", "authors", "primary_category"): + if not c.get(k): + c[k] = f[k] + c["date"] = c["date"] or f["date"] + c["first_author"] = c["first_author"] or ( + f["authors"][0] if f["authors"] else None) + except Exception as e: # noqa: BLE001 + warnings.append(f"abstract lookup failed ({e}); titles only") + + counts = {s: sum(1 for c in candidates if c["source"] == s) for s in SOURCE_ORDER} + harvested = {s: sum(1 for h in harvests if h["source"] == s) for s in SOURCE_ORDER} + return { + "scope": scope, "topic": topic, "today": today.isoformat(), + "cutoff": cutoff, "cutoff_from": cutoff_from, + "evidence": evidence, + "days_lost": (today - datetime.date.fromisoformat(cutoff)).days, + "arxiv_window": None if offline else [start.isoformat(), today.isoformat()], + "truncated": truncated, + "harvested": harvested, "counts": counts, + "dropped_in_memory": dropped, "warnings": warnings, + "candidates": candidates, + } + + +def render_text(report: dict, abstract_chars: int = 280) -> str: + ev = report["evidence"] + bib = ev.get("bib") or {} + lines = [f"catch-up · scope {report['scope']}" + + (f" / {report['topic']}" if report["topic"] else ""), + f"cutoff {report['cutoff']} ({report['cutoff_from']}: last DONE " + f"{ev.get('done')}, last bib+sources ingest {bib.get('date')} " + f"{bib.get('commit') or ''}) → {report['days_lost']} days lost", + f"arXiv window {report['arxiv_window']} · " + f"{len(report['candidates'])} candidates " + + " · ".join(f"{s} {n}" for s, n in report["counts"].items()) + + f" · {report['dropped_in_memory']} already in memory"] + for s in SOURCE_ORDER: + group = [c for c in report["candidates"] if c["source"] == s] + if not group: + continue + lines += ["", f"## {s} ({len(group)})", ""] + for c in group: + also = f" [+{','.join(c['also_in'])}]" if c["also_in"] else "" + lines.append(f"- {c['id'] or '—':<11} {c['date'] or '':<10} " + f"{c['title']} — {c['first_author'] or '?'}{also}") + if c.get("abstract"): + a = c["abstract"] + lines.append(" " + (a[:abstract_chars] + "…" + if len(a) > abstract_chars else a)) + return "\n".join(lines) + "\n" + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(prog="scripts/catch_up.py", description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--scope", choices=("lensing", "interests", "all"), + default="lensing") + ap.add_argument("--topic", default=None, + help="narrow interests to one reading-queue section") + ap.add_argument("--since", default=None, help="override the cutoff (YYYY-MM-DD)") + ap.add_argument("--today", default=None, help="override today (YYYY-MM-DD)") + ap.add_argument("--offline", action="store_true", help="git harvests only") + ap.add_argument("--json", action="store_true") + ns = ap.parse_args(argv) + try: + report = catch_up(ns.scope, ns.topic, ns.since, + datetime.date.fromisoformat(ns.today) if ns.today else None, + offline=ns.offline) + except ValueError as e: + print(f"error: {e}", file=sys.stderr) + return 2 + for w in report["warnings"]: + print(f"warning: {w}", file=sys.stderr) + print(json.dumps(report, indent=2) if ns.json else render_text(report), end="") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/validate_structure.py b/scripts/validate_structure.py index b6c99bb..c88f004 100644 --- a/scripts/validate_structure.py +++ b/scripts/validate_structure.py @@ -22,6 +22,7 @@ ".github", "bibliography", "scripts", + "skills", # organ-owned agent skills (e.g. skills/catch_up), like Mind's skills/ "tests", "wiki", } diff --git a/skills/catch_up/SKILL.md b/skills/catch_up/SKILL.md new file mode 100644 index 0000000..ce12681 --- /dev/null +++ b/skills/catch_up/SKILL.md @@ -0,0 +1,11 @@ +--- +name: catch_up +description: Catch memory up after time away — harvest every paper missed since memory last ingested one (swept inbox/interests lines from git history, open reading-queue lines, and an arXiv gap-fill over the nightly digest's missed days) for one scope (strong lensing, the other interest topics, or all), curate them into intake / cite / queue / skip tiers for one human confirmation, then file the chosen papers in one PR. Use when the dashboard says "I've been lost for a while", or after any stretch without filing. +--- + +# Catch up + +Follow [`catch_up.md`](catch_up.md) exactly. The harvest is +`scripts/catch_up.py` in the PyAutoMemory checkout; this skill curates and +files on top of it. Stop at the tier table for the human's confirmation — +nothing is filed before it. diff --git a/skills/catch_up/catch_up.md b/skills/catch_up/catch_up.md new file mode 100644 index 0000000..4553caa --- /dev/null +++ b/skills/catch_up/catch_up.md @@ -0,0 +1,83 @@ +# /catch_up — "…so… I've been lost for a while" + +After time away the suggestion tiers have lapsed (seven days), the reading +queue has grown, and the nightly digest may not even have run. This door +collects everything missed **since memory last took a paper in**, for one +scope, lets the human triage it once, and files what they choose. A +**PyAutoMemory primitive**: the harvest is `scripts/catch_up.py` (read its +docstring for how the cutoff and the three sources work); this skill curates +and files. + +## Usage + +``` +/catch_up # strong lensing (the default) +/catch_up lensing +/catch_up interests # SMBHs, Galaxy Formation / Evolution, Dark Matter, Stats, Interests +/catch_up interests SMBHs # one interests topic (a reading-queue section, verbatim) +/catch_up all +/catch_up lensing --since 2026-09-01 # override the cutoff +``` + +## Steps + +1. **Resolve the Memory checkout.** The canonical one, never a guessed sibling: + `python3 /agents/_repo_paths.py path PyAutoMemory --required --root `. + Report its repo root and branch. Filing needs a clean worktree off + `origin/main` on a `feature/catch-up--` branch + (`git worktree add -b … origin/main`); the harvest itself is + read-only and may run from the canonical checkout. +2. **Harvest.** `python scripts/catch_up.py --scope [--topic ] [--since D] --json` + (network: the arXiv gap-fill imports the Mind digest's query; set + `PYAUTO_MIND` if Mind is not found). Report one line: + **"N days lost, M candidates since "** — plus the per-source counts + (`counts`: queue / history / arxiv), `dropped_in_memory`, and every entry of + `warnings` verbatim (a skipped arXiv source or a `truncated` interests + window is a real gap, never hide it). +3. **Curate.** Read each candidate's abstract (fetch it from arXiv for the few + with none) and rank against the human's interests — take the concepts from + the scope's `wiki//index.md` (for lensing: PyAutoLens-style + modelling methods, dark-matter substructure, interferometry, Euclid and + survey lens searches, time-delay cosmography, lensed sources/quasars, JAX / + differentiable inference). Propose four tiers, each paper with a one-line + **why**: + - **intake** — full filing: bib entry + a proper section in `sources/`. + - **cite** — bib entry + a one-line stub, worth citing, not pivotal. + - **queue** — not now; add (or keep) a `reading-queue.md` line. + - **skip** — off-topic or superseded; nothing is written. + More than ~20 candidates → a ranked table (tier · id · title · first + author · why), intake and cite first. Queue lines already in the queue that + land in **queue** or **skip** stay as they are — never delete a queue line. +4. **Stop for the human.** Present the table and wait. They move papers + between tiers; the confirmed table is the only input to step 5. This is the + one human gate — do not file anything before it. +5. **File.** In the filing worktree, delegate to **Opus subagents** in batches + (≈5–8 papers each, disjoint `sources/` pages where possible, each given a + progress file). Each paper follows the board's filing recipe + (`_read_prompt` in `scripts/board.py`): + verify it against an authoritative record (arXiv / ADS), add its canonical + entry to `bibliography/pyautomemory.bib` per `bibliography/README.md` + ("Adding a paper" — search for an existing key first), stub it in the + matching `wiki//sources/*.md` page per `wiki/AGENTS.md` (intake: a + full section; cite: a one-line stub), and mark any queue line + `DONE — ` in place. **queue** tier papers not already + queued get a line appended to the right `## <Section>` (title ` — <arXiv id>`). +6. **Validate and record.** `make validate` (and `make test` if a script was + touched) must be green. Append one entry to each touched + `wiki/<domain>/log.md`: `## <date> — Catch-up since <cutoff>` listing the + papers filed per tier. +7. **Ship.** One commit, one PR against `main` summarising the tier counts and + the cutoff; report its URL and stop. The human merges — a paper is not in + memory until the PR lands (the dashboard banner resets from the merge). + +## Notes + +- The cutoff is the newest DONE line in the scope's queue sections or the + newest commit that adds a bib entry **and** touches the scope's + `wiki/<domain>/sources/` (a restructure or a bib tidy-up does not count). A + scope never ingested (the interests sub-wikis were imported, not filed) cuts + off at its oldest suggestion instead, and says so in `warnings`. +- The interests arXiv query is whole categories; it is capped (most recent + first) and reported `truncated`. Lean on the interests-file history there, + and narrow with a topic. +- Never arm a wait for CI or the merge; end at the PR. diff --git a/tests/test_board.py b/tests/test_board.py index b0997d0..86d0a44 100644 --- a/tests/test_board.py +++ b/tests/test_board.py @@ -916,3 +916,58 @@ def test_an_empty_tree_is_grey_never_green(tmp_path): def test_render_state_is_valid_json(tmp_path): state = json.loads(board.render(_state_snap(tmp_path), "state")) _assert_state_shape(state) + + +# --- the catch-up banner ------------------------------------------------------------- +# `last_ingested` is computed by `catch_up.last_ingested("all")`; each test +# below pins both it and the snapshot's own clock, so no wall clock is read. +def _banner_snap(tmp_path, last, now="2026-09-30T09:00:00+00:00"): + snap = _state_snap(tmp_path) + snap["last_ingested"], snap["generated"] = last, now + return snap + + +def test_last_ingested_is_collected_from_a_scope_section(tmp_path): + root = _tree(tmp_path) + # `_tree`'s DONE sits in "Demo Papers", which no catch-up scope owns. + assert board.collect(root)["last_ingested"] is None + q = root / "reading-queue.md" + q.write_text(q.read_text() + "\n## Strong Lensing\nDONE 2026-09-01 — X\n") + assert board.collect(root)["last_ingested"] == "2026-09-01" + + +def test_no_banner_under_the_threshold(tmp_path): + snap = _banner_snap(tmp_path, "2026-09-24") # 6 days + assert board._days_since_ingest(snap) == 6 + assert "lost for a while" not in board.render(snap, "md") + assert "class='catchup'" not in board.render(snap, "html") + assert board.to_state(snap)["days_since_ingest"] == 6 + + +def test_banner_at_the_threshold_carries_the_day_count(tmp_path): + snap = _banner_snap(tmp_path, "2026-09-23") # exactly CATCH_UP_DAYS + assert board.CATCH_UP_DAYS == 7 + md = board.render(snap, "md") + assert "_…so… I've been lost for a while_" in md + assert "**7 days since papers were last ingested into memory**" in md + assert "`/catch_up lensing`" in md + # Top of the markdown: before the contents line. + assert md.index("lost for a while") < md.index("_Contents") + + +def test_banner_html_sits_above_the_hero_with_a_copy_button(tmp_path): + snap = _banner_snap(tmp_path, "2026-09-11") + page = board.render(snap, "html") + assert "<em>…so… I've been lost for a while</em>" in page + assert "19 days since papers were last ingested into memory" in page + assert 'data-cmd="/catch_up lensing"' in page + assert page.index("class='catchup'") < page.index('class="hero"') + assert board.to_state(snap)["days_since_ingest"] == 19 + + +def test_no_banner_when_nothing_is_on_record(tmp_path): + snap = _banner_snap(tmp_path, None) + assert board._days_since_ingest(snap) is None + assert "lost for a while" not in board.render(snap, "md") + assert "class='catchup'" not in board.render(snap, "html") + assert board.to_state(snap)["days_since_ingest"] is None diff --git a/tests/test_catch_up.py b/tests/test_catch_up.py new file mode 100644 index 0000000..8009d20 --- /dev/null +++ b/tests/test_catch_up.py @@ -0,0 +1,293 @@ +"""tests/test_catch_up.py — the catch-up harvest (scripts/catch_up.py). + +The contract: the cutoff is the newest *real* ingest (a DONE line, or a commit +that adds a bib entry AND touches the scope's sources — never either half +alone); swept suggestion lines come back from git history; whatever memory +already holds (bib id/title, DONE line) is dropped; and the arXiv source is +best-effort — stubbed here, so the suite never touches the network. +""" + +from __future__ import annotations + +import datetime +import subprocess +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +import catch_up # noqa: E402 + +QUEUE = """# Reading queue + +--- + +## Strong Lensing +DONE 2026-09-05 — An Old Filed Paper — 2608.00001 +__Sub heading__ +NOTE a remark, not a paper +An Open Queue Paper — 2609.00010 +A Queue Paper Already In The Bib — 2608.27566 +A Queue Paper Without A Ref + +## SMBHs +DONE 2026-09-20 — A Black Hole Paper — 2609.00020 +Open SMBH Paper — 2609.00021 +""" + +BIB = """@article{Williams2026, + eprint = {2608.27566}, + title = {{TDCOSMO. XXVII. Something}}, +} +@misc{Titled2026, + title = {{A Paper Known Only By {T}itle}}, +} +""" + +INBOX_HEAD = "# arXiv inbox\n\n---\nlast digest: {d}\n" + + +def _git(root: Path, *args: str, date: str | None = None) -> None: + env = None + if date: + import os + env = {**os.environ, "GIT_AUTHOR_DATE": f"{date}T12:00:00", + "GIT_COMMITTER_DATE": f"{date}T12:00:00"} + subprocess.run(["git", "-C", str(root), "-c", "user.name=t", + "-c", "user.email=t@t", *args], + check=True, capture_output=True, env=env) + + +def _commit(root: Path, files: dict, msg: str, date: str) -> None: + for rel, text in files.items(): + p = root / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(text) + _git(root, "add", "-A") + _git(root, "commit", "-q", "-m", msg, date=date) + + +def _repo(tmp_path: Path) -> Path: + root = tmp_path / "mem" + root.mkdir() + _git(root, "init", "-q") + _commit(root, {"reading-queue.md": QUEUE, catch_up.BIB_FILE: BIB, + "wiki/lensing/sources/a.md": "a\n", + "arxiv-inbox.md": INBOX_HEAD.format(d="2026-09-01") + + "2026-09-01 — A Pre-Cutoff Inbox Paper — 2608.99999\n"}, + "seed", "2026-09-01") + # A real ingest: bib entry + sources page, together. + _commit(root, {catch_up.BIB_FILE: BIB + "@misc{New2026,\n eprint = {2609.00005},\n}\n", + "wiki/lensing/sources/a.md": "a\nb\n"}, + "queue: file paper", "2026-09-11") + # Inbox lines that were later swept — only git remembers them. + _commit(root, {"arxiv-inbox.md": INBOX_HEAD.format(d="2026-09-12") + + "2026-09-12 — A Swept Inbox Paper — 2609.11111\n" + + "2026-09-12 — An Open Queue Paper — 2609.00010\n"}, + "inbox: appended", "2026-09-12") + _commit(root, {"arxiv-inbox.md": INBOX_HEAD.format(d="2026-09-20")}, + "inbox: swept", "2026-09-20") + # A restructure: touches sources, adds no entry. Must NOT move the cutoff. + _commit(root, {"wiki/lensing/sources/a.md": "a\nb\nc\n"}, + "seed split", "2026-09-17") + # A bib tidy-up: adds an entry, touches no sources. Must NOT either. + _commit(root, {catch_up.BIB_FILE: BIB + "@misc{New2026,\n eprint = {2609.00005},\n}\n" + "@misc{Tidy2026,\n eprint = {2609.00006},\n}\n"}, + "bib tidy", "2026-09-18") + return root + + +# --- cutoff ------------------------------------------------------------------------ +def test_cutoff_is_the_bib_plus_sources_commit_not_either_half(tmp_path): + root = _repo(tmp_path) + ev = catch_up.ingest_evidence("lensing", root) + assert ev["done"] == "2026-09-05" + assert ev["bib"]["date"] == "2026-09-11" + assert ev["last_ingested"] == "2026-09-11" + + +def test_cutoff_reads_the_scope_sections_only(tmp_path): + root = _repo(tmp_path) + # The SMBHs DONE (09-20) is newer but belongs to the interests scope. + assert catch_up.last_ingested("lensing", root) == "2026-09-11" + assert catch_up.last_ingested("interests", root) == "2026-09-20" + assert catch_up.last_ingested("all", root) == "2026-09-20" + + +def test_no_git_falls_back_to_done_lines(tmp_path): + (tmp_path / "reading-queue.md").write_text(QUEUE) + assert catch_up.last_ingested("lensing", tmp_path) == "2026-09-05" + + +def test_nothing_on_record_is_none(tmp_path): + assert catch_up.last_ingested("all", tmp_path) is None + + +def test_parse_ingest_log_takes_the_max_date_not_the_first(): + log = ("\x002026-09-10 aaa\ndiff --git a/bibliography/pyautomemory.bib " + "b/bibliography/pyautomemory.bib\n+@misc{A,\ndiff --git " + "a/wiki/lensing/sources/x.md b/wiki/lensing/sources/x.md\n" + "\x002026-09-11 bbb\ndiff --git a/bibliography/pyautomemory.bib " + "b/bibliography/pyautomemory.bib\n+@misc{B,\ndiff --git " + "a/wiki/lensing/sources/y.md b/wiki/lensing/sources/y.md\n") + assert catch_up.parse_ingest_log(log, ("lensing",)) == { + "date": "2026-09-11", "commit": "bbb"} + assert catch_up.parse_ingest_log(log, ("smbh",)) is None + + +def test_topic_narrows_and_lensing_refuses_one(): + spec = catch_up.resolve_scope("interests", "SMBHs") + assert spec["sections"] == ("SMBHs",) and spec["domains"] == ("smbh",) + for bad in (("lensing", "SMBHs"), ("interests", "Cancer"), ("nope", None)): + try: + catch_up.resolve_scope(*bad) + except ValueError: + continue + raise AssertionError(bad) + + +# --- harvests --------------------------------------------------------------------------- +def test_history_lines_parse_and_respect_the_cutoff(): + log = ("+2026-09-12 — A Swept Paper — 2609.11111\n" + "-2026-09-13 — A Removed Line Is Not An Addition — 2609.22222\n" + "+2026-09-01 — Too Old — 2608.99999\n" + "+++ b/arxiv-inbox.md\n+last digest: 2026-09-12\n") + got = catch_up.parse_history(log, "arxiv-inbox.md", "2026-09-11") + assert [(h["id"], h["date"]) for h in got] == [("2609.11111", "2026-09-12")] + + +def test_interests_history_carries_and_filters_by_topic(): + log = ("+2026-09-12 — [SMBHs] A Black Hole — 2609.1\n" + "+2026-09-12 — [Stats] A Sampler — 2609.00002\n" + "+2026-09-12 — Untagged Goes To Interests — 2609.00003\n") + got = catch_up.parse_history(log, "arxiv-interests.md", "2026-09-01", "Stats") + assert [h["title"] for h in got] == ["A Sampler"] + got = catch_up.parse_history(log, "arxiv-interests.md", "2026-09-01", "Interests") + assert [h["topic"] for h in got] == [None] + + +def test_queue_harvest_skips_done_notes_and_subheads(): + got = catch_up.harvest_queue(QUEUE, ("Strong Lensing",)) + assert [h["title"] for h in got] == [ + "An Open Queue Paper", "A Queue Paper Already In The Bib", + "A Queue Paper Without A Ref"] + + +# --- dedup --------------------------------------------------------------------------------- +def test_in_memory_index_reads_bib_ids_titles_and_done_lines(): + ids, titles = catch_up.in_memory_index(BIB, QUEUE) + assert {"2608.27566", "2608.00001", "2609.00020"} <= ids + assert catch_up.norm_title("A paper known only by title") in titles + assert catch_up.norm_title("An Old Filed Paper") in titles + + +def test_merge_dedups_across_sources_and_drops_what_memory_holds(): + harvests = [ + {"id": "2609.00010", "title": "An Open Queue Paper", "source": "queue"}, + {"id": "2609.00010", "title": "An Open Queue Paper", "source": "history", + "date": "2026-09-12"}, + {"id": "2609.00010", "title": "An Open Queue Paper", "source": "arxiv", + "abstract": "We lens.", "authors": ["Ada"]}, + {"id": None, "title": "A paper known only by TITLE", "source": "history"}, + {"id": "2608.27566", "title": "Whatever", "source": "arxiv"}, + {"id": None, "title": "No Ref Here", "source": "queue"}, + {"id": "2609.33333", "title": "no ref here", "source": "arxiv"}, + ] + ids, titles = catch_up.in_memory_index(BIB, QUEUE) + got, dropped = catch_up.merge(harvests, ids, titles) + assert dropped == 2 + by_title = {c["title"]: c for c in got} + first = by_title["An Open Queue Paper"] + assert first["source"] == "queue" + assert first["also_in"] == ["history", "arxiv"] + assert first["abstract"] == "We lens." and first["first_author"] == "Ada" + assert first["date"] == "2026-09-12" + # An id-less queue line meets its arXiv twin through the title. + assert by_title["No Ref Here"]["id"] == "2609.33333" + assert len(got) == 2 + + +# --- the whole run, arXiv stubbed ------------------------------------------------------------- +class _StubFetch: + QUERY = "cat:astro-ph.CO" + + def __init__(self): + self.queries = [] + + def fetch(self, query, max_results, start=0): + self.queries.append(query) + return b"<feed><entry>stub</entry></feed>" + + def parse(self, raw, band_start, band_end): + self.band = (band_start, band_end) + return [{"title": "A Gap Paper The Digest Never Fetched", + "authors": ["Grace"], "abstract": "Missed.", + "url": "https://arxiv.org/abs/2609.20000v1", + "primary_category": "astro-ph.CO", + "published": "2026-09-19T10:00:00Z"}, + {"title": "Already filed", "authors": [], "abstract": "", + "url": "https://arxiv.org/abs/2609.00005v1", + "published": "2026-09-10T10:00:00Z"}] + + def _get(self, params): + return b"<feed></feed>" + + +def test_catch_up_end_to_end_with_arxiv_stubbed(tmp_path): + root = _repo(tmp_path) + stub = _StubFetch() + report = catch_up.catch_up("lensing", today=datetime.date(2026, 9, 30), + root=root, digests={"arxiv_fetch": stub}, + pause=0) + assert report["cutoff"] == "2026-09-11" and report["days_lost"] == 19 + assert "submittedDate:[202609080000 TO 202609302359]" in stub.queries[0] + ids = {c["id"]: c for c in report["candidates"]} + assert ids["2609.11111"]["source"] == "history" # swept, back from git + assert ids["2609.00010"]["source"] == "queue" + assert ids["2609.20000"]["source"] == "arxiv" # the gap fill + assert ids["2609.20000"]["first_author"] == "Grace" + assert "2609.00005" not in ids # already in the bib + assert "2608.27566" not in ids + assert "2608.99999" not in ids # before the cutoff + assert report["warnings"] == [] + + +def test_arxiv_failure_is_a_warning_not_a_crash(tmp_path): + root = _repo(tmp_path) + + class Boom(_StubFetch): + def fetch(self, *a, **k): + raise OSError("HTTP 429") + + report = catch_up.catch_up("lensing", today=datetime.date(2026, 9, 30), + root=root, digests={"arxiv_fetch": Boom()}, + pause=0) + assert any("429" in w for w in report["warnings"]) + assert report["counts"]["history"] and report["counts"]["arxiv"] == 0 + + +def test_since_overrides_and_offline_skips_the_network(tmp_path): + root = _repo(tmp_path) + report = catch_up.catch_up("lensing", since="2026-08-01", + today=datetime.date(2026, 9, 30), root=root, + offline=True) + assert report["cutoff_from"] == "--since" + assert report["arxiv_window"] is None + assert "2608.99999" in {c["id"] for c in report["candidates"]} + + +def test_missing_mind_is_a_warning(tmp_path, monkeypatch): + root = _repo(tmp_path) + monkeypatch.setattr(catch_up, "mind_scripts_dir", lambda root=None: None) + report = catch_up.catch_up("lensing", today=datetime.date(2026, 9, 30), + root=root) + assert any("PyAutoMind not found" in w for w in report["warnings"]) + + +def test_text_render_groups_by_source(tmp_path): + root = _repo(tmp_path) + report = catch_up.catch_up("lensing", today=datetime.date(2026, 9, 30), + root=root, offline=True) + text = catch_up.render_text(report) + assert "cutoff 2026-09-11" in text and "19 days lost" in text + assert text.index("## queue") < text.index("## history")