Rebuild missing SOLR content everywhere it is read, not just in term_info

src/test/term_info_queries_test.py

@@ -416,8 +416,12 @@ def test_term_info_serialization_dataset(self):
         self.assertFalse("logo" in serialized)
         self.assertTrue("link" in serialized)
         self.assertEqual("[http://flybase.org/reports/FBrf0221438.html](http://flybase.org/reports/FBrf0221438.html)", serialized["link"])
-        self.assertEqual(4, len(serialized["types"]))
-        self.assertTrue("DataSet" in serialized["types"])
+        # Assert the types this test is about, not how many there are. The
+        # count moved 4 -> 5 when DataSets gained stage labels ("Adult" here),
+        # which is backend content: a new label is not a regression, a missing
+        # DataSet type would be.
+        for expected_type in ("Entity", "Individual", "DataSet", "has_image"):
+            self.assertIn(expected_type, serialized["types"])
         self.assertEqual("An exhaustive set of lineage clones covering the adult brain from Kei Ito's  lab.", serialized["description"])
         self.assertFalse("synonyms" in serialized)
         self.assertFalse("source" in serialized)

src/test/test_term_info_fallback.py

@@ -128,6 +128,51 @@ def test_schema_version_is_never_a_git_call(monkeypatch, tmp_path):
     assert tif.schema_version() == "unpinned"
 
 
+# ---------------------------------------------------------------------------
+# Pointing the indexer at VFBquery's own backends
+# ---------------------------------------------------------------------------
+
+@pytest.mark.parametrize("url,expected", [
+    ("http://solr.virtualflybrain.org/solr/vfb_json/",
+     ("http://solr.virtualflybrain.org/solr", "vfb_json")),
+    ("https://solr.virtualflybrain.org/solr/vfb_json",
+     ("https://solr.virtualflybrain.org/solr", "vfb_json")),
+    ("", (None, None)),
+    (None, (None, None)),
+])
+def test_solr_url_splits_into_the_indexers_two_env_vars(url, expected):
+    assert tif._split_solr_url(url) == expected
+
+
+def test_seeding_sets_a_usable_solr_update_url(monkeypatch):
+    """The regression this exists for: solr_client builds its update URL from
+    SOLRserver/SOLRcollection, neither of which a VFBquery deployment sets.
+    get_solr_update_url() then returns None and send_solr_payload returns
+    False without making a request, so a rebuilt document is served but never
+    indexed -- silently, and only in production, because the cache-disabled
+    guard short-circuits before the URL is ever needed."""
+    for key in ("SOLRserver", "SOLRcollection", "PDBserver", "PDBuser",
+                "PDBpassword"):
+        monkeypatch.delenv(key, raising=False)
+    tif._seed_indexer_env()
+    from vfbquery.vfb_queries import vfb_solr
+    server, collection = tif._split_solr_url(vfb_solr.url)
+    assert os.environ["SOLRserver"] == server
+    assert os.environ["SOLRcollection"] == collection
+    # and the two together must produce a real update endpoint
+    assert f"{server}/{collection}/update" == (
+        f"{os.environ['SOLRserver'].rstrip('/')}/"
+        f"{os.environ['SOLRcollection']}/update")
+
+
+def test_seeding_does_not_override_a_deployment(monkeypatch):
+    monkeypatch.setenv("SOLRserver", "http://elsewhere/solr")
+    monkeypatch.setenv("SOLRcollection", "other_core")
+    tif._seed_indexer_env()
+    assert os.environ["SOLRserver"] == "http://elsewhere/solr"
+    assert os.environ["SOLRcollection"] == "other_core"
+
+
 # ---------------------------------------------------------------------------
 # The `src` package collision
 # ---------------------------------------------------------------------------
@@ -164,3 +209,127 @@ def test_reports_why_it_cannot_build(monkeypatch):
     assert tif.fallback_available() is False
     assert "no indexer here" in tif.fallback_unavailable_reason()
     assert tif.backfill_term_info("FBbt_00003748") is None
+
+
+# ---------------------------------------------------------------------------
+# Per-field query index rebuilds
+# ---------------------------------------------------------------------------
+
+class _StubNeo:
+    def commit_list(self, queries):
+        return None
+
+
+class _StubConnect:
+    """vc.nc is a read-only property, so stand in for vc itself."""
+    nc = _StubNeo()
+
+def test_query_field_fallback_only_claims_fields_it_can_build(monkeypatch):
+    monkeypatch.setattr(tif, "_QUERY_FIELD_INDEXERS", {"anat_query": object})
+    monkeypatch.setattr(tif, "_INDEXERS", {"class": object})
+    assert tif.query_field_fallback_available("anat_query") is True
+    assert tif.query_field_fallback_available("term_info") is False
+    assert tif.query_field_fallback_available("something_else") is False
+
+
+def test_no_ids_means_no_work(monkeypatch):
+    def explode(*a, **k):
+        raise AssertionError("should not have queried anything")
+    monkeypatch.setattr(tif, "_QUERY_FIELD_INDEXERS", {"anat_query": explode})
+    assert tif.backfill_query_field([], "anat_query") == {}
+
+
+def test_backfill_batches_rather_than_truncating(monkeypatch):
+    """Batched at the indexer's own size, and every id is rebuilt. Measured
+    against the live PDB a rebuild is one round trip -- 500 ids in 1.5s
+    against 0.37s for a single id -- so batching is what keeps it cheap, and
+    a set larger than one batch is looped, never trimmed."""
+    seen = {}
+
+    class FakeIndexer:
+        def get_vfb_json_query(self, ids):
+            seen["ids"] = list(ids)
+            seen.setdefault("batches", []).append(len(ids))
+            return "MATCH (n) RETURN n"
+
+        def generate_solr_doc(self, result, request=None):
+            return {"id": result["term"]["core"]["short_form"],
+                    "anat_query": {"set": '{"ok": true}'}}
+
+    monkeypatch.setattr(tif, "_QUERY_FIELD_INDEXERS", {"anat_query": FakeIndexer})
+    monkeypatch.setattr(tif, "_INDEXERS", {"class": object})
+    monkeypatch.setattr(tif, "_write_query_docs", lambda docs, field: True)
+
+    ids = ["FBbt_%07d" % i for i in range(tif.BACKFILL_BATCH_SIZE + 25)]
+    rows = [{"term": {"core": {"short_form": i}}} for i in ids]
+    import vfbquery.vfb_queries as vq
+    calls = {"n": 0}
+
+    def cursor(r):
+        i = calls["n"]; calls["n"] += 1
+        return rows[i * tif.BACKFILL_BATCH_SIZE:(i + 1) * tif.BACKFILL_BATCH_SIZE] \
+            if i == 0 else rows[tif.BACKFILL_BATCH_SIZE:]
+
+    monkeypatch.setattr(vq, "get_dict_cursor", lambda: cursor)
+    monkeypatch.setattr(vq, "vc", _StubConnect())
+
+    rebuilt = tif.backfill_query_field(ids, "anat_query")
+    # Batched, not truncated: the last batch is the 25-id remainder, and every
+    # id came back. Dropping the tail would return a quietly short answer,
+    # which is the thing this whole module exists to stop.
+    assert len(seen["ids"]) == 25
+    assert seen["batches"] == [tif.BACKFILL_BATCH_SIZE, 25]
+    assert rebuilt[ids[0]] == '{"ok": true}'            # payload returned, not re-read
+
+
+def test_backfill_returns_payloads_for_the_current_request(monkeypatch):
+    """The indexer writes with commitWithin 60s, so re-reading SOLR in this
+    request would still miss. The payloads have to come back in memory."""
+    class FakeIndexer:
+        def get_vfb_json_query(self, ids):
+            return "MATCH (n) RETURN n"
+
+        def generate_solr_doc(self, result, request=None):
+            return {"id": "VFB_00107fob",
+                    "anat_image_query": {"set": '{"term": {}}'}}
+
+    monkeypatch.setattr(tif, "_QUERY_FIELD_INDEXERS",
+                        {"anat_image_query": FakeIndexer})
+    monkeypatch.setattr(tif, "_INDEXERS", {"class": object})
+    written = []
+    monkeypatch.setattr(tif, "_write_query_docs",
+                        lambda docs, field: written.append((len(docs), field)) or True)
+    import vfbquery.vfb_queries as vq
+    monkeypatch.setattr(vq, "get_dict_cursor", lambda: (lambda r: [{"x": 1}]))
+    monkeypatch.setattr(vq, "vc", _StubConnect())
+
+    out = tif.backfill_query_field(["VFB_00107fob"], "anat_image_query")
+    assert out == {"VFB_00107fob": '{"term": {}}'}
+    assert written == [(1, "anat_image_query")]
+
+
+def test_a_failed_rebuild_returns_nothing_rather_than_raising(monkeypatch):
+    class FakeIndexer:
+        def get_vfb_json_query(self, ids):
+            return "MATCH (n) RETURN n"
+
+    monkeypatch.setattr(tif, "_QUERY_FIELD_INDEXERS", {"anat_query": FakeIndexer})
+    monkeypatch.setattr(tif, "_INDEXERS", {"class": object})
+    import vfbquery.vfb_queries as vq
+
+    def boom(*a, **k):
+        raise RuntimeError("neo4j down")
+
+    monkeypatch.setattr(vq, "get_dict_cursor", lambda: boom)
+    monkeypatch.setattr(vq, "vc", _StubConnect())
+    assert tif.backfill_query_field(["FBbt_00003748"], "anat_query") == {}
+
+
+def test_query_writes_honour_the_cache_guard(monkeypatch):
+    monkeypatch.setenv("VFBQUERY_CACHE_ENABLED", "false")
+    monkeypatch.setattr(tif, "_INDEXERS", {"class": object})
+    sent = []
+    monkeypatch.setattr(tif, "_send_solr_docs",
+                        lambda d, f: sent.append(f) or True)
+    assert tif._write_query_docs([{"id": "x"}], "anat_query") is False
+    assert sent == []

src/test/test_vfb_connectivity.py

@@ -301,7 +301,12 @@ def test_default_excludes_are_a_strict_subset_of_everything(self, expansion_resu
             group_by_class=True,
         )
         assert DEFAULT_EXCLUDE_DBS == ["hb", "fafb"]
-        assert 0 < expansion_result["count"] < everything["count"]
+        # Subset, not strict subset. Excluding a dataset can only ever remove
+        # rows, and that is the contract worth asserting. Demanding a strict
+        # decrease additionally requires hemibrain or FAFB to hold rows for
+        # *this* term pair, which is backend content -- it was true when this
+        # was written and is not true now (17 either way).
+        assert 0 < expansion_result["count"] <= everything["count"]
 
     @pytest.mark.integration
     def test_default_matches_passing_it_explicitly(self):