index object descriptions as their own search records

docs/src/reference.md

@@ -14,14 +14,15 @@ Set these in `conf.py`.
 ## Index format
 
 The index is a JSON array of records. Each page produces one record for the
-page and one for each section that has an anchor.
-
-| Key | Description                                                                 |
-| --- | --------------------------------------------------------------------------- |
-| `u` | URL relative to the documentation root, with `#anchor` for sections.        |
-| `t` | Title of the page.                                                          |
-| `s` | Section heading. Absent on page records.                                    |
-| `x` | Text, truncated to `searchlite_max_text`. Sections exclude nested sections. |
+page, one for each section that has an anchor, and one for each object
+description that has an anchor, such as an autodoc class, method or attribute.
+
+| Key | Description                                                                                                             |
+| --- | ----------------------------------------------------------------------------------------------------------------------- |
+| `u` | URL relative to the documentation root, with `#anchor` for sections and objects.                                        |
+| `t` | Title of the page.                                                                                                      |
+| `s` | Section heading, or the object's full name. Absent on page records.                                                     |
+| `x` | Text, truncated to `searchlite_max_text`. Sections exclude nested sections and objects; objects exclude nested objects. |
 
 `engine.load()` rewrites `u` to an absolute URL.
 

sphinx_searchlite/_index.py

@@ -10,25 +10,73 @@
 from pathlib import Path
 
 from docutils import nodes
+from sphinx import addnodes
 
 __all__ = ("collect_records", "setup_index")
 
 _WHITESPACE = re.compile(r"\s+")
 _ATTRIBUTE = "_sphinx_searchlite_records"
+_OBJECT_NAMES = "_sphinx_searchlite_object_names"
 
 
 def _flatten(text: str, limit: int) -> str:
     return _WHITESPACE.sub(" ", text).strip()[:limit]
 
 
+def _anchor(desc: addnodes.desc) -> addnodes.desc_signature | None:
+    """The object's first anchored signature, or ``None`` if it gets no record (e.g. ``:no-index:``)."""
+    return next((child for child in desc.children if isinstance(child, addnodes.desc_signature) and child["ids"]), None)
+
+
+def _without_objects(node: nodes.Node) -> str:
+    """Text of ``node``, leaving out object descriptions that get their own records."""
+    if isinstance(node, addnodes.desc) and _anchor(node) is not None:
+        return ""
+    # Defer to ``astext()`` wherever possible, since nodes like parameter lists override it.
+    if not isinstance(node, nodes.Element) or not any(_anchor(desc) is not None for desc in node.findall(addnodes.desc)):
+        return node.astext()
+    # Match ``Element.astext()`` so inline markup inside a word does not split it.
+    return node.child_text_separator.join(_without_objects(child) for child in node.children)
+
+
 def _own_text(section: nodes.Element, limit: int) -> str:
-    """Text belonging to a section, excluding its title and nested sections."""
-    parts = [child.astext() for child in section.children if not isinstance(child, (nodes.section, nodes.title))]
+    """Text belonging to a section, excluding its title, nested sections and object descriptions."""
+    parts = [_without_objects(child) for child in section.children if not isinstance(child, (nodes.section, nodes.title))]
     return _flatten(" ".join(parts), limit)
 
 
+def _object_name(signature: addnodes.desc_signature) -> str:
+    """Qualified name of the object, without the types and parameters of C/C++ signatures."""
+    if signature.get("fullname"):
+        return signature["fullname"]
+    # Multi-line signatures, such as C++ templates, put the declared name on the last line.
+    lines = [child for child in signature.children if isinstance(child, addnodes.desc_signature_line)] or [signature]
+    name = "".join(child.astext() for child in lines[-1].children if isinstance(child, (addnodes.desc_addname, addnodes.desc_name)))
+    return name or signature.astext()
+
+
+def _object_records(doctree: nodes.document, uri: str, page_title: str, limit: int, names: dict[str, str]) -> list[dict[str, str]]:
+    """One record per anchored object description, such as an autodoc class, method or field.
+
+    API reference pages are mostly object descriptions with few headings, so
+    without these anything past the first ``limit`` characters of a section is
+    unsearchable.
+    """
+    records = []
+    for desc in doctree.findall(addnodes.desc):
+        signature = _anchor(desc)
+        if signature is None:
+            continue
+        # Every signature's text, so overloads stay searchable, then the body.
+        text = " ".join(_without_objects(child) for child in desc.children)
+        anchor = signature["ids"][0]
+        name = names.get(anchor) or _object_name(signature)
+        records.append({"u": f"{uri}#{anchor}", "t": page_title, "s": name, "x": _flatten(text, limit)})
+    return records
+
+
 def collect_records(app, pagename: str, doctree: nodes.document) -> list[dict[str, str]]:
-    """Return one record for the page plus one per addressable section.
+    """Return one record for the page plus one per addressable section and object description.
 
     Keys are short because the index ships to the browser: ``u`` url, ``t`` page
     title, ``s`` section heading, ``x`` text.
@@ -48,6 +96,8 @@ def collect_records(app, pagename: str, doctree: nodes.document) -> list[dict[st
         if heading == page_title and section.parent is doctree:
             continue
         records.append({"u": f"{uri}#{section['ids'][0]}", "t": page_title, "s": heading, "x": _own_text(section, limit)})
+    names = getattr(app, _OBJECT_NAMES, {}).get(pagename, {})
+    records.extend(_object_records(doctree, uri, page_title, limit, names))
     return records
 
 
@@ -58,6 +108,15 @@ def _store(app) -> dict[str, list[dict[str, str]]]:
     return getattr(app.env, _ATTRIBUTE)
 
 
+def _on_env_updated(app, env) -> None:
+    names: dict[str, dict[str, str]] = {}
+    for domain in env.domains.values():
+        for name, _, _, docname, anchor, _ in domain.get_objects():
+            # Parameters can share their enclosing object's target; keep the object's name.
+            names.setdefault(docname, {}).setdefault(anchor, name)
+    setattr(app, _OBJECT_NAMES, names)
+
+
 def _on_page_context(app, pagename, templatename, context, doctree):
     if doctree is None or app.builder.format != "html":
         return
@@ -75,5 +134,6 @@ def _on_build_finished(app, exception):
 
 
 def setup_index(app) -> None:
+    app.connect("env-updated", _on_env_updated)
     app.connect("html-page-context", _on_page_context)
     app.connect("build-finished", _on_build_finished)

sphinx_searchlite/tests/test_searchlite.py

@@ -37,12 +37,89 @@
 Other content.
 """
 
+API_RST = """
+API
+===
+
+Intro to the API.
+
+Gadgets
+-------
+
+Things that frob\\ *nicate*.
+
+.. py:function:: legacy(x)
+   :no-index:
+
+   Kept for older callers.
+
+.. py:function:: over(a)
+                 over(a, b)
+
+   Overloaded.
+
+.. cpp:class:: template<typename T> ns::Thing
+
+   A templated thing.
+
+.. py:class:: Gadget
+
+   A gadget that frobnicates.
+
+   .. py:method:: spin(speed)
+
+      Spins the gadget sprocket.
+
+.. py:data:: TIMEOUT
+
+   Seconds before the gadget gives up.
+
+.. py:module:: north
+
+.. py:class:: Client
+
+   Northern client.
+
+   .. py:method:: connect()
+
+      Opens the northern connection.
+
+.. py:module:: south
+
+.. py:class:: Client
+
+   Southern client.
+
+   .. py:method:: connect()
+
+      Opens the southern connection.
+
+.. cpp:namespace:: tools
+
+.. cpp:class:: Socket
+
+   Network socket.
+
+   .. cpp:function:: void open()
+
+      Opens the socket.
+
+.. c:struct:: Data
+
+   C structure.
+
+   .. c:member:: int count
+
+      Counts the entries.
+"""
+
 
 def _build(tmp_path_factory, conf: str, name: str) -> Path:
     src = tmp_path_factory.mktemp(name)
     (src / "conf.py").write_text(conf)
-    (src / "index.md").write_text(INDEX_MD)
+    (src / "index.md").write_text(INDEX_MD.replace("guide\n", "guide\napi\n"))
     (src / "guide.md").write_text(GUIDE_MD)
+    (src / "api.rst").write_text(API_RST)
     out = src / "_build"
     Sphinx(str(src), str(src), str(out), str(out / ".doctrees"), "html", freshenv=True, status=None).build()
     return out
@@ -161,3 +238,67 @@ def test_index_name_is_configurable(self, tmp_path_factory):
         out = _build(tmp_path_factory, CONF + '\nsearchlite_index_filename = "docs-index.json"\n', "src_named")
         assert (out / "_static" / "docs-index.json").is_file()
         assert 'data-searchlite-index="docs-index.json"' in (out / "guide.html").read_text()
+
+
+class TestObjectDescriptions:
+    def test_objects_get_their_own_anchored_record(self, records):
+        record = next(record for record in records if record.get("s") == "Gadget")
+        assert record["u"] == "api.html#Gadget"
+        assert record["t"] == "API"
+        assert "A gadget that frobnicates" in record["x"]
+
+    def test_nested_objects_are_indexed_by_full_name(self, records):
+        record = next(record for record in records if record.get("s") == "Gadget.spin")
+        assert record["u"] == "api.html#Gadget.spin"
+        assert "sprocket" in record["x"]
+
+    @pytest.mark.parametrize(
+        ("name", "anchor"),
+        [
+            ("north.Client.connect", "north.Client.connect"),
+            ("south.Client.connect", "south.Client.connect"),
+            ("tools::Socket::open", "_CPPv4N5tools6Socket4openEv"),
+            ("Data.count", "c.Data.count"),
+        ],
+    )
+    def test_object_names_include_module_and_parent_scope(self, records, name, anchor):
+        record = next(record for record in records if record["u"] == f"api.html#{anchor}")
+        assert record["s"] == name
+
+    def test_scoped_names_do_not_depend_on_object_toc_entries(self, tmp_path_factory):
+        out = _build(tmp_path_factory, CONF + "\ntoc_object_entries = False\n", "src_no_object_toc")
+        entries = json.loads((out / "_static" / "searchlite-index.json").read_text())
+        names = {record.get("s") for record in entries}
+        assert {"north.Client.connect", "south.Client.connect", "tools::Socket::open", "Data.count"} <= names
+
+    def test_object_text_excludes_nested_objects(self, records):
+        record = next(record for record in records if record.get("s") == "Gadget")
+        assert "sprocket" not in record["x"]
+
+    def test_section_text_excludes_object_descriptions(self, records):
+        section = next(record for record in records if record.get("s") == "Gadgets")
+        assert "A gadget" not in section["x"]
+        assert "Seconds before" not in section["x"]
+
+    def test_inline_markup_does_not_split_words(self, records):
+        section = next(record for record in records if record.get("s") == "Gadgets")
+        assert "Things that frobnicate" in section["x"]
+
+    def test_unindexed_objects_stay_in_section_text(self, records):
+        section = next(record for record in records if record.get("s") == "Gadgets")
+        assert "Kept for older callers" in section["x"]
+        assert not [record for record in records if record.get("s") == "legacy"]
+
+    def test_every_overload_is_searchable(self, records):
+        record = next(record for record in records if record.get("s") == "over")
+        assert "over(a)" in record["x"]
+        assert "over(a, b)" in record["x"]
+
+    def test_names_without_fullname_omit_signature_syntax(self, records):
+        record = next(record for record in records if "_CPP" in record["u"])
+        assert record["s"] == "ns::Thing"
+
+    def test_objects_past_the_text_limit_stay_searchable(self, tmp_path_factory):
+        out = _build(tmp_path_factory, CONF + "\nsearchlite_max_text = 20\n", "src_short_api")
+        entries = json.loads((out / "_static" / "searchlite-index.json").read_text())
+        assert any(record.get("s") == "TIMEOUT" for record in entries)