From 03c456a2a44779bd52f9548cab66c4b81b3b2648 Mon Sep 17 00:00:00 2001 From: Jammy2211 Date: Thu, 15 Oct 2026 19:05:26 +0100 Subject: [PATCH 1/2] docs: link each capability-matrix search to its autofit_inference catalogue card docs/_generate_searches.py now fetches autofit_inference catalogue/search_catalogue.json by raw URL at a pinned commit (CATALOGUE_COMMIT = bea1d8746cf0) and adds a Catalogue column to the capability matrix: each search links to its card (tasks..searches.) at that commit with a line anchor, or reads "no catalogue card yet". The pin is recorded in a comment and the column note of docs/searches/index.rst. Offline, the matrix is skipped with a named stderr message and the other pages are still written or checked. Closes #1694 Co-Authored-By: Claude Opus 5.5 --- docs/_generate_searches.py | 174 ++++++++++++++++-- docs/searches/index.rst | 22 ++- .../test_generate_searches_catalogue.py | 113 ++++++++++++ 3 files changed, 292 insertions(+), 17 deletions(-) create mode 100644 test_autofit/test_generate_searches_catalogue.py diff --git a/docs/_generate_searches.py b/docs/_generate_searches.py index 0747cc32b..78593934a 100644 --- a/docs/_generate_searches.py +++ b/docs/_generate_searches.py @@ -9,11 +9,19 @@ ``files/citations.bib``; they are never edited by hand: - ``docs/api/searches.rst``: the API reference, one autosummary block per family; -- ``docs/searches/index.rst``: the capability matrix ("which search"); +- ``docs/searches/index.rst``: the capability matrix ("which search"), whose + **Catalogue** column links each search to its card in the autofit_inference + benchmark catalogue at the pinned commit ``CATALOGUE_COMMIT``; - ``docs/searches/citations.rst``: the canonical citations page for the searches. The pages deliberately omit ``autofit_version``, so a version bump never makes them -stale; only a change to the registry or the bibliography does. +stale; only a change to the registry, the bibliography or the catalogue pin does. + +The catalogue (``catalogue/search_catalogue.json`` in PyAutoLabs/autofit_inference) is +fetched by raw URL at ``CATALOGUE_COMMIT`` each run and never stored in this repository +beyond the generated page. To re-pin it, change ``CATALOGUE_COMMIT`` and re-run the +generator. Offline, the fetch fails: the capability matrix is then skipped (named on +stderr) and the other two pages are still written or checked. Run from anywhere: paths resolve relative to this file. ``--check`` is run by the Docs workflow (``.github/workflows/docs.yml``) and by ``test_autofit``. @@ -24,8 +32,10 @@ import re import subprocess import sys +import urllib.error +import urllib.request from pathlib import Path -from typing import Dict, List +from typing import Dict, List, Optional DOCS = Path(__file__).resolve().parent REPO = DOCS.parent @@ -42,6 +52,98 @@ # machine. The full entries stay in files/citations.bib. _DROPPED_BIB_FIELDS = ("abstract", "file", "keywords", "mendeley-groups") +# The autofit_inference benchmark catalogue the capability matrix links to, pinned to +# one commit so the links (and this generator's output) never move under the docs. +CATALOGUE_REPO = "PyAutoLabs/autofit_inference" +CATALOGUE_FILE = "catalogue/search_catalogue.json" +CATALOGUE_COMMIT = "bea1d8746cf009ff414ea0786c92f3e3e0909f70" +CATALOGUE_BOARD = "https://pyautolabs.github.io/PyAutoInsight/" +NO_CARD = "no catalogue card yet" + + +class CatalogueUnavailable(RuntimeError): + """ + The pinned catalogue could not be fetched or read (e.g. no network). + """ + + +def catalogue_raw_url(commit: str = CATALOGUE_COMMIT) -> str: + return ( + f"https://raw.githubusercontent.com/{CATALOGUE_REPO}/{commit}/{CATALOGUE_FILE}" + ) + + +def load_catalogue_text(commit: str = CATALOGUE_COMMIT, timeout: float = 30.0) -> str: + """ + The catalogue JSON at ``commit``, fetched by raw URL (nothing is cached on disk). + """ + url = catalogue_raw_url(commit) + try: + with urllib.request.urlopen(url, timeout=timeout) as response: + return response.read().decode("utf-8") + except (urllib.error.URLError, OSError, ValueError) as error: + raise CatalogueUnavailable(f"could not fetch {url}: {error}") from error + + +def _key_line(lines: List[str], key: str, start: int) -> Optional[int]: + """ + The index of the first line at or after ``start`` that opens the object ``key``. + """ + opening = re.compile(rf"^\s*{re.escape(json.dumps(key))}\s*:\s*\{{") + for index in range(start, len(lines)): + if opening.match(lines[index]): + return index + return None + + +def catalogue_cards(text: str) -> Dict[str, dict]: + """ + The catalogue's search cards keyed by search name: the task each card sits under + (``tasks..searches.``), the card's status and its 1-based line in the + catalogue file (``None`` when the line cannot be located), for a line anchor. + """ + try: + data = json.loads(text) + tasks = data["tasks"] + except (ValueError, KeyError, TypeError) as error: + raise CatalogueUnavailable(f"unreadable search catalogue: {error}") from error + lines = text.splitlines() + tasks_line = _key_line(lines, "tasks", 0) + cards = {} + for task_key, task in tasks.items(): + task_line = ( + None if tasks_line is None else _key_line(lines, task_key, tasks_line + 1) + ) + searches_line = ( + None if task_line is None else _key_line(lines, "searches", task_line + 1) + ) + for name, card in task.get("searches", {}).items(): + if name in cards: + continue + card_line = ( + None + if searches_line is None + else _key_line(lines, name, searches_line + 1) + ) + cards[name] = { + "task": task_key, + "label": task.get("label", task_key), + "status": card.get("status") if isinstance(card, dict) else None, + "line": None if card_line is None else card_line + 1, + } + return cards + + +def _card_cell(search: dict, cards: Dict[str, dict], commit: str) -> str: + card = cards.get(search["name"]) + if card is None: + return NO_CARD + url = f"https://github.com/{CATALOGUE_REPO}/blob/{commit}/{CATALOGUE_FILE}" + if card["line"] is not None: + url += f"#L{card['line']}" + status = f" ({card['status']})" if card["status"] else "" + return f"`{card['label']} <{url}>`__{status}" + def load_manifest() -> dict: """ @@ -181,10 +283,14 @@ def _budget(budget: dict) -> str: return ", ".join(f"``{key}={value}``" for key, value in budget.items()) -def searches_index_rst(manifest: dict) -> str: +def searches_index_rst( + manifest: dict, cards: Dict[str, dict], commit: str = CATALOGUE_COMMIT +) -> str: title = "Choosing a search: the capability matrix" lines = [ HEADER, + f".. Catalogue cards pinned at {CATALOGUE_REPO}@{commit}\n" + f" ({CATALOGUE_FILE}); change CATALOGUE_COMMIT to re-pin.\n\n", ".. _search_capability_matrix:\n\n", f"{'=' * len(title)}\n", f"{title}\n", @@ -217,6 +323,7 @@ def searches_index_rst(manifest: dict) -> str: "Install", "Status", "Example", + "Catalogue", ] rows = [] for search in manifest["searches"]: @@ -237,9 +344,10 @@ def searches_index_rst(manifest: dict) -> str: else "base", capabilities["status"], _example(search), + _card_cell(search, cards, commit), ] ) - lines += _list_table(header, rows, "14 6 8 8 7 8 7 8 8 10 9 9") + lines += _list_table(header, rows, "13 5 7 7 6 7 6 7 7 9 8 8 10") lines += [ "Objectives\n", "----------\n\n", @@ -297,7 +405,16 @@ def searches_index_rst(manifest: dict) -> str: "``consumer`` (wants a warm start, e.g. from a provider's posterior) or " "``neutral``.\n", "- **Install**: ``base`` ships with ``pip install autofit``; " - "``autofit[optional]`` needs the optional extra.\n\n", + "``autofit[optional]`` needs the optional extra.\n", + "- **Catalogue**: the search's card in the autofit_inference benchmark " + f"catalogue (`{CATALOGUE_FILE} at {commit[:12]} " + f"`__, " + "pinned), under the task it answers, with the card's status; tasks are never " + "ranked against each other. The scored wave-2 legs of each card's harness " + "samplers are in the same file's ``gaussian_x3@2`` section. The catalogue is " + "aggregated on the " + f"`PyAutoInsight board <{CATALOGUE_BOARD}>`__. A search the pinned catalogue " + f"has not benchmarked reads \"{NO_CARD}\".\n\n", "See also the :doc:`search citations ` page and the " ":doc:`API reference <../api/searches>`.\n", ] @@ -381,14 +498,27 @@ def searches_citations_rst(manifest: dict, bib_text: str) -> str: return "".join(lines) -def generated_pages(manifest: dict) -> Dict[Path, str]: - return { - DOCS / "api" / "searches.rst": api_searches_rst(manifest), - DOCS / "searches" / "index.rst": searches_index_rst(manifest), - DOCS / "searches" / "citations.rst": searches_citations_rst( - manifest, BIB.read_text() - ), - } +INDEX = DOCS / "searches" / "index.rst" + + +def generated_pages( + manifest: dict, + catalogue_text: Optional[str], + commit: str = CATALOGUE_COMMIT, +) -> Dict[Path, str]: + """ + The generated pages; without the catalogue (``None``) the capability matrix is + left out, so the other pages can still be written or checked offline. + """ + pages = {DOCS / "api" / "searches.rst": api_searches_rst(manifest)} + if catalogue_text is not None: + pages[INDEX] = searches_index_rst( + manifest, catalogue_cards(catalogue_text), commit + ) + pages[DOCS / "searches" / "citations.rst"] = searches_citations_rst( + manifest, BIB.read_text() + ) + return pages def main(argv=None) -> int: @@ -400,7 +530,19 @@ def main(argv=None) -> int: ) arguments = parser.parse_args(argv) - pages = generated_pages(load_manifest()) + try: + catalogue_text = load_catalogue_text() + catalogue_cards(catalogue_text) + except CatalogueUnavailable as error: + catalogue_text = None + print( + f"skipped {INDEX.relative_to(REPO)}: the pinned search catalogue is " + f"unavailable ({error}). Re-run with network access to " + f"{'check' if arguments.check else 'regenerate'} it.", + file=sys.stderr, + ) + + pages = generated_pages(load_manifest(), catalogue_text) if arguments.check: stale = [ @@ -423,7 +565,7 @@ def main(argv=None) -> int: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(text) print(f"wrote {path.relative_to(REPO)}") - return 0 + return 0 if catalogue_text is not None else 1 if __name__ == "__main__": diff --git a/docs/searches/index.rst b/docs/searches/index.rst index 47291b383..ec23e5426 100644 --- a/docs/searches/index.rst +++ b/docs/searches/index.rst @@ -3,6 +3,9 @@ autofit/non_linear/search/registry.py (or files/citations.bib) and re-run the generator. +.. Catalogue cards pinned at PyAutoLabs/autofit_inference@bea1d8746cf009ff414ea0786c92f3e3e0909f70 + (catalogue/search_catalogue.json); change CATALOGUE_COMMIT to re-pin. + .. _search_capability_matrix: ======================================== @@ -22,7 +25,7 @@ Capabilities .. list-table:: :header-rows: 1 - :widths: 14 6 8 8 7 8 7 8 8 10 9 9 + :widths: 13 5 7 7 6 7 6 7 7 9 8 8 10 * - Search - Family @@ -36,6 +39,7 @@ Capabilities - Install - Status - Example + - Catalogue * - :class:`~autofit.Emcee` - mcmc - none @@ -48,6 +52,7 @@ Capabilities - base - stable - `mcmc.py `__ + - `posterior `__ (measured) * - :class:`~autofit.Zeus` - mcmc - none @@ -60,6 +65,7 @@ Capabilities - ``autofit[optional]`` - stable - `mcmc.py `__ + - `posterior `__ (measured) * - :class:`~autofit.BlackJAXNUTS` - mcmc - required @@ -72,6 +78,7 @@ Capabilities - ``autofit[optional]`` - stable - `mcmc.py `__ + - `posterior `__ (measured) * - :class:`~autofit.SMC` - mcmc - required @@ -84,6 +91,7 @@ Capabilities - ``autofit[optional]`` - experimental - -- + - `posterior `__ (measured) * - :class:`~autofit.DynestyStatic` - nest - optional @@ -96,6 +104,7 @@ Capabilities - base - stable - `nest.py `__ + - `evidence `__ (measured) * - :class:`~autofit.DynestyDynamic` - nest - optional @@ -108,6 +117,7 @@ Capabilities - base - stable - `nest.py `__ + - `evidence `__ (measured) * - :class:`~autofit.Nautilus` - nest - optional @@ -120,6 +130,7 @@ Capabilities - ``autofit[optional]`` - stable - `nest.py `__ + - `evidence `__ (measured) * - :class:`~autofit.NSS` - nest - required @@ -132,6 +143,7 @@ Capabilities - ``autofit[optional]`` - experimental - -- + - `evidence `__ (deferred) * - :class:`~autofit.Drawer` - mle - none @@ -144,6 +156,7 @@ Capabilities - base - stable - `mle.py `__ + - `point/MAP `__ (unsupported) * - :class:`~autofit.BFGS` - mle - optional @@ -156,6 +169,7 @@ Capabilities - base - stable - -- + - `point/MAP `__ (measured) * - :class:`~autofit.LBFGS` - mle - optional @@ -168,6 +182,7 @@ Capabilities - base - stable - `mle.py `__ + - `point/MAP `__ (measured) * - :class:`~autofit.MultiStartAdam` - mle - required @@ -180,6 +195,7 @@ Capabilities - base - stable - `mle.py `__ + - `point/MAP `__ (measured) * - :class:`~autofit.MultiStartADABelief` - mle - required @@ -192,6 +208,7 @@ Capabilities - base - experimental - -- + - `point/MAP `__ (measured) * - :class:`~autofit.MultiStartLion` - mle - required @@ -204,6 +221,7 @@ Capabilities - base - experimental - -- + - `point/MAP `__ (measured) * - :class:`~autofit.MultiStartProdigy` - mle - required @@ -216,6 +234,7 @@ Capabilities - base - stable - -- + - `point/MAP `__ (measured) Objectives ---------- @@ -337,5 +356,6 @@ Column meanings - **Resumable**: an interrupted run resumes from its own checkpoint. - **Warm start**: ``provider`` (starts cold and can seed another search), ``consumer`` (wants a warm start, e.g. from a provider's posterior) or ``neutral``. - **Install**: ``base`` ships with ``pip install autofit``; ``autofit[optional]`` needs the optional extra. +- **Catalogue**: the search's card in the autofit_inference benchmark catalogue (`catalogue/search_catalogue.json at bea1d8746cf0 `__, pinned), under the task it answers, with the card's status; tasks are never ranked against each other. The scored wave-2 legs of each card's harness samplers are in the same file's ``gaussian_x3@2`` section. The catalogue is aggregated on the `PyAutoInsight board `__. A search the pinned catalogue has not benchmarked reads "no catalogue card yet". See also the :doc:`search citations ` page and the :doc:`API reference <../api/searches>`. diff --git a/test_autofit/test_generate_searches_catalogue.py b/test_autofit/test_generate_searches_catalogue.py new file mode 100644 index 000000000..7141b9cd7 --- /dev/null +++ b/test_autofit/test_generate_searches_catalogue.py @@ -0,0 +1,113 @@ +""" +The capability matrix's catalogue column (``docs/_generate_searches.py``): a search +with a card in the autofit_inference catalogue links to it at the pinned commit, with a +line anchor; a search without one reads "no catalogue card yet". No network is used. + +Skipped when the ``docs`` directory is absent (an installed-package test run). +""" + +import importlib.util +import json +from pathlib import Path + +import pytest + +from autofit.non_linear.search import registry + +GENERATOR = Path(__file__).resolve().parents[1] / "docs" / "_generate_searches.py" + +pytestmark = pytest.mark.skipif( + not GENERATOR.exists(), reason="docs/ is not part of this checkout" +) + +COMMIT = "0123456789abcdef0123456789abcdef01234567" + + +@pytest.fixture(name="generator") +def make_generator(): + spec = importlib.util.spec_from_file_location("_generate_searches", GENERATOR) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture(name="catalogue_text") +def make_catalogue_text(): + # A wave-2 style section repeats a "searches" key first, so the anchor must come + # from the top-level ``tasks`` block, not the first match in the file. + catalogue = { + "schema": "autofit-inference-search-catalogue", + "early": {"searches": {"Nautilus": {"status": "decoy"}}}, + "tasks": { + "point_map": { + "label": "point/MAP", + "searches": {"LBFGS": {"status": "measured", "legs": []}}, + }, + "evidence": { + "label": "evidence", + "searches": {"Nautilus": {"status": "deferred"}}, + }, + }, + } + return json.dumps(catalogue, indent=2) + + +def _line_of(text, needle): + lines = text.splitlines() + return max(index for index, line in enumerate(lines) if needle in line) + 1 + + +def test_cards_keyed_by_search_name_with_line_anchor(generator, catalogue_text): + cards = generator.catalogue_cards(catalogue_text) + + assert set(cards) == {"LBFGS", "Nautilus"} + assert cards["LBFGS"]["label"] == "point/MAP" + assert cards["Nautilus"]["status"] == "deferred" + assert cards["Nautilus"]["line"] == _line_of(catalogue_text, '"Nautilus": {') + assert catalogue_text.splitlines()[cards["LBFGS"]["line"] - 1].strip() == ( + '"LBFGS": {' + ) + + +def test_present_and_absent_cells(generator, catalogue_text): + cards = generator.catalogue_cards(catalogue_text) + + present = generator._card_cell({"name": "LBFGS"}, cards, COMMIT) + absent = generator._card_cell({"name": "Emcee"}, cards, COMMIT) + + line = cards["LBFGS"]["line"] + assert present == ( + "`point/MAP `__ (measured)" + ) + assert absent == generator.NO_CARD == "no catalogue card yet" + + +def test_matrix_has_one_catalogue_cell_per_search_and_records_the_pin( + generator, catalogue_text +): + text = generator.searches_index_rst( + registry.manifest(), generator.catalogue_cards(catalogue_text), COMMIT + ) + capabilities = text.split("Capabilities\n------------")[1].split("Objectives\n")[0] + + assert f"PyAutoLabs/autofit_inference@{COMMIT}" in text + assert capabilities.count(f"/blob/{COMMIT}/catalogue/search_catalogue.json#L") == 2 + assert capabilities.count("no catalogue card yet") == len(registry.entries()) - 2 + + +def test_unreadable_catalogue_raises(generator): + with pytest.raises(generator.CatalogueUnavailable): + generator.catalogue_cards("not json") + + +def test_offline_check_skips_only_the_matrix(generator, monkeypatch, capsys): + def offline(*args, **kwargs): + raise generator.CatalogueUnavailable("no network") + + monkeypatch.setattr(generator, "load_catalogue_text", offline) + + assert generator.main(["--check"]) == 0 + captured = capsys.readouterr() + assert "skipped docs/searches/index.rst" in captured.err + assert "2 generated search pages are up to date." in captured.out From f8157460e5571622a89acb45cfde0cbeb8af3500 Mon Sep 17 00:00:00 2001 From: Jammy2211 Date: Thu, 15 Oct 2026 19:14:36 +0100 Subject: [PATCH 2/2] review: HTTP errors fail, --require-catalogue in the Docs workflow An HTTP error (bad or unpushed pin) or an unreadable catalogue is now an error (CatalogueError), never a skip; only a connection failure skips the matrix, and --require-catalogue (used by the generated-search-docs job) makes that an error too. Parse the catalogue once; reword the board note (the benchmark records feed the PyAutoInsight board). Test the 404 path. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/docs.yml | 5 +- docs/_generate_searches.py | 61 +++++++++++++------ docs/searches/index.rst | 2 +- .../test_generate_searches_catalogue.py | 17 +++++- 4 files changed, 61 insertions(+), 24 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 71e001b6c..26a5a89e6 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -32,7 +32,8 @@ jobs: # The search pages (docs/api/searches.rst, docs/searches/index.rst and # docs/searches/citations.rst) are generated from the search manifest - # (python -m autofit search-manifest --json); fail when they are stale. + # (python -m autofit search-manifest --json) and the autofit_inference catalogue + # at a pinned commit; fail when they are stale or the catalogue is unreachable. generated-search-docs: runs-on: ubuntu-latest steps: @@ -43,4 +44,4 @@ jobs: - name: Install autofit run: pip install -e . - name: Check the generated search docs - run: python docs/_generate_searches.py --check + run: python docs/_generate_searches.py --check --require-catalogue diff --git a/docs/_generate_searches.py b/docs/_generate_searches.py index 78593934a..db309c609 100644 --- a/docs/_generate_searches.py +++ b/docs/_generate_searches.py @@ -20,8 +20,10 @@ The catalogue (``catalogue/search_catalogue.json`` in PyAutoLabs/autofit_inference) is fetched by raw URL at ``CATALOGUE_COMMIT`` each run and never stored in this repository beyond the generated page. To re-pin it, change ``CATALOGUE_COMMIT`` and re-run the -generator. Offline, the fetch fails: the capability matrix is then skipped (named on -stderr) and the other two pages are still written or checked. +generator. Offline (no connection), the capability matrix is skipped, named on stderr, +and the other two pages are still written (exit 1) or checked (exit 0); with +``--require-catalogue`` (the Docs workflow) being offline is an error too. An HTTP +error (a bad or unpushed pin) or an unreadable catalogue is always an error. Run from anywhere: paths resolve relative to this file. ``--check`` is run by the Docs workflow (``.github/workflows/docs.yml``) and by ``test_autofit``. @@ -61,9 +63,16 @@ NO_CARD = "no catalogue card yet" +class CatalogueError(RuntimeError): + """ + The pinned catalogue is wrong: an HTTP error (e.g. a pin that does not exist) or + an unreadable file. Never skipped. + """ + + class CatalogueUnavailable(RuntimeError): """ - The pinned catalogue could not be fetched or read (e.g. no network). + The pinned catalogue could not be reached (no connection, a timeout). """ @@ -81,8 +90,10 @@ def load_catalogue_text(commit: str = CATALOGUE_COMMIT, timeout: float = 30.0) - try: with urllib.request.urlopen(url, timeout=timeout) as response: return response.read().decode("utf-8") - except (urllib.error.URLError, OSError, ValueError) as error: - raise CatalogueUnavailable(f"could not fetch {url}: {error}") from error + except urllib.error.HTTPError as error: + raise CatalogueError(f"could not fetch {url}: {error}") from error + except (urllib.error.URLError, OSError) as error: + raise CatalogueUnavailable(f"could not reach {url}: {error}") from error def _key_line(lines: List[str], key: str, start: int) -> Optional[int]: @@ -106,7 +117,7 @@ def catalogue_cards(text: str) -> Dict[str, dict]: data = json.loads(text) tasks = data["tasks"] except (ValueError, KeyError, TypeError) as error: - raise CatalogueUnavailable(f"unreadable search catalogue: {error}") from error + raise CatalogueError(f"unreadable search catalogue: {error}") from error lines = text.splitlines() tasks_line = _key_line(lines, "tasks", 0) cards = {} @@ -411,8 +422,8 @@ def searches_index_rst( f"`__, " "pinned), under the task it answers, with the card's status; tasks are never " "ranked against each other. The scored wave-2 legs of each card's harness " - "samplers are in the same file's ``gaussian_x3@2`` section. The catalogue is " - "aggregated on the " + "samplers are in the same file's ``gaussian_x3@2`` section; the benchmark " + "records behind the catalogue feed the " f"`PyAutoInsight board <{CATALOGUE_BOARD}>`__. A search the pinned catalogue " f"has not benchmarked reads \"{NO_CARD}\".\n\n", "See also the :doc:`search citations ` page and the " @@ -503,18 +514,16 @@ def searches_citations_rst(manifest: dict, bib_text: str) -> str: def generated_pages( manifest: dict, - catalogue_text: Optional[str], + cards: Optional[Dict[str, dict]], commit: str = CATALOGUE_COMMIT, ) -> Dict[Path, str]: """ - The generated pages; without the catalogue (``None``) the capability matrix is - left out, so the other pages can still be written or checked offline. + The generated pages; without the catalogue cards (``None``) the capability matrix + is left out, so the other pages can still be written or checked offline. """ pages = {DOCS / "api" / "searches.rst": api_searches_rst(manifest)} - if catalogue_text is not None: - pages[INDEX] = searches_index_rst( - manifest, catalogue_cards(catalogue_text), commit - ) + if cards is not None: + pages[INDEX] = searches_index_rst(manifest, cards, commit) pages[DOCS / "searches" / "citations.rst"] = searches_citations_rst( manifest, BIB.read_text() ) @@ -528,13 +537,25 @@ def main(argv=None) -> int: action="store_true", help="Do not write; exit 1 if any generated page differs from the manifest.", ) + parser.add_argument( + "--require-catalogue", + action="store_true", + help="Fail (exit 1) instead of skipping the capability matrix when the pinned " + "catalogue cannot be reached. Offline without it, --check skips the matrix " + "and exits 0; a write skips it and exits 1.", + ) arguments = parser.parse_args(argv) try: - catalogue_text = load_catalogue_text() - catalogue_cards(catalogue_text) + cards = catalogue_cards(load_catalogue_text()) + except CatalogueError as error: + print(f"error: {error}", file=sys.stderr) + return 1 except CatalogueUnavailable as error: - catalogue_text = None + if arguments.require_catalogue: + print(f"error: {error}", file=sys.stderr) + return 1 + cards = None print( f"skipped {INDEX.relative_to(REPO)}: the pinned search catalogue is " f"unavailable ({error}). Re-run with network access to " @@ -542,7 +563,7 @@ def main(argv=None) -> int: file=sys.stderr, ) - pages = generated_pages(load_manifest(), catalogue_text) + pages = generated_pages(load_manifest(), cards) if arguments.check: stale = [ @@ -565,7 +586,7 @@ def main(argv=None) -> int: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(text) print(f"wrote {path.relative_to(REPO)}") - return 0 if catalogue_text is not None else 1 + return 0 if cards is not None else 1 if __name__ == "__main__": diff --git a/docs/searches/index.rst b/docs/searches/index.rst index ec23e5426..5ea2de6ba 100644 --- a/docs/searches/index.rst +++ b/docs/searches/index.rst @@ -356,6 +356,6 @@ Column meanings - **Resumable**: an interrupted run resumes from its own checkpoint. - **Warm start**: ``provider`` (starts cold and can seed another search), ``consumer`` (wants a warm start, e.g. from a provider's posterior) or ``neutral``. - **Install**: ``base`` ships with ``pip install autofit``; ``autofit[optional]`` needs the optional extra. -- **Catalogue**: the search's card in the autofit_inference benchmark catalogue (`catalogue/search_catalogue.json at bea1d8746cf0 `__, pinned), under the task it answers, with the card's status; tasks are never ranked against each other. The scored wave-2 legs of each card's harness samplers are in the same file's ``gaussian_x3@2`` section. The catalogue is aggregated on the `PyAutoInsight board `__. A search the pinned catalogue has not benchmarked reads "no catalogue card yet". +- **Catalogue**: the search's card in the autofit_inference benchmark catalogue (`catalogue/search_catalogue.json at bea1d8746cf0 `__, pinned), under the task it answers, with the card's status; tasks are never ranked against each other. The scored wave-2 legs of each card's harness samplers are in the same file's ``gaussian_x3@2`` section; the benchmark records behind the catalogue feed the `PyAutoInsight board `__. A search the pinned catalogue has not benchmarked reads "no catalogue card yet". See also the :doc:`search citations ` page and the :doc:`API reference <../api/searches>`. diff --git a/test_autofit/test_generate_searches_catalogue.py b/test_autofit/test_generate_searches_catalogue.py index 7141b9cd7..ef8fe63e0 100644 --- a/test_autofit/test_generate_searches_catalogue.py +++ b/test_autofit/test_generate_searches_catalogue.py @@ -8,6 +8,7 @@ import importlib.util import json +import urllib.error from pathlib import Path import pytest @@ -97,10 +98,22 @@ def test_matrix_has_one_catalogue_cell_per_search_and_records_the_pin( def test_unreadable_catalogue_raises(generator): - with pytest.raises(generator.CatalogueUnavailable): + with pytest.raises(generator.CatalogueError): generator.catalogue_cards("not json") +def test_http_error_is_an_error_not_a_skip(generator, monkeypatch, capsys): + def not_found(url, timeout): + raise urllib.error.HTTPError(url, 404, "Not Found", None, None) + + monkeypatch.setattr(generator.urllib.request, "urlopen", not_found) + + with pytest.raises(generator.CatalogueError): + generator.load_catalogue_text() + assert generator.main(["--check"]) == 1 + assert "404" in capsys.readouterr().err + + def test_offline_check_skips_only_the_matrix(generator, monkeypatch, capsys): def offline(*args, **kwargs): raise generator.CatalogueUnavailable("no network") @@ -111,3 +124,5 @@ def offline(*args, **kwargs): captured = capsys.readouterr() assert "skipped docs/searches/index.rst" in captured.err assert "2 generated search pages are up to date." in captured.out + + assert generator.main(["--check", "--require-catalogue"]) == 1