From 0d569c074b2a6b05ed36950f7cf5ed07b4548bb6 Mon Sep 17 00:00:00 2001 From: Robbie Court Date: Fri, 4 Sep 2026 06:18:11 +0000 Subject: [PATCH] Compare data-keyed result maps by shape, not by key test_live_result_matches_recorded_shape treated every key in a recording as schema, so a backend content change could fail CI as a missing key -- the opposite of this module's stated contract, that content changes must not fail while a key or type disappearing from a payload must. Four recorded maps are keyed by data: Examples and Images by template short_form, Domains and Licenses by index. Which keys they carry follows whichever images the indexer picked. Class documents cap anatomy_channel_image at ten entries however many images a class really has -- 56,384 for adult cholinergic neuron, 12 for medulla -- so a class whose only image on some template falls outside that ten loses the whole template key. That is what happened to medulla (FBbt_00003748) on 2026-09-03: it lost VFB_00030786, adult brain template Ito2014, while VFB_00030810 still had its in_register_with edge in the PDB, its own SOLR document, and every image file serving 200. Compare those four the way lists are already compared: the map must still be non-empty and one value must still have the recorded shape, but the key set is free. Meta is deliberately not in the set -- its keys are the schema -- and neither is headers, where a lost column is a real regression. Verified against the live backend: the full example suite passes, including the term_info_FBbt_00003748 case that has failed since 1 Sep. --- src/test/test_example_queries.py | 62 ++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/src/test/test_example_queries.py b/src/test/test_example_queries.py index 88dbf03..983855d 100644 --- a/src/test/test_example_queries.py +++ b/src/test/test_example_queries.py @@ -88,17 +88,52 @@ def _type_bucket(value): return name +#: Recorded maps whose KEYS are data rather than schema, compared like +#: lists: the map must still be non-empty and its values must still have +#: the recorded shape, but *which* keys appear is content. +#: +#: ``Examples`` and ``Images`` are keyed by template short_form and +#: ``Domains``/``Licenses`` by index, so their key sets follow whichever +#: images the indexer happened to pick. The class documents cap +#: ``anatomy_channel_image`` at ten entries however many images a class +#: really has (56,384 for adult cholinergic neuron; 12 for medulla), so a +#: class whose only image on some template falls outside that ten loses +#: that template key entirely — no schema changed, and nothing is missing +#: from the backend. That is exactly the "backend content changes must not +#: fail CI" case in this module's docstring, and treating these keys as +#: schema turned it into a failure (medulla lost VFB_00030786, adult brain +#: template Ito2014, on 2026-09-03 while the individual, its own SOLR +#: document and all its image files were intact). +#: +#: ``Meta`` is deliberately absent: its keys ARE the schema. +DATA_KEYED_MAPS = frozenset({"$.Examples", "$.Images", "$.Domains", + "$.Licenses"}) + + def shape_mismatches(expected, live, path="$"): """Recursively compare recorded vs live result SHAPE. Every key in the recording must exist live with a compatible type; lists are compared through their first element; leaf values only have to agree on coarse type. Keys the live result has gained are fine. + + The exception is the maps in :data:`DATA_KEYED_MAPS`, whose keys are + data: those are compared the way lists are — non-emptiness and the + shape of one value — because their key sets legitimately change with + the backend content. """ if isinstance(expected, dict): if not isinstance(live, dict): return ["%s: recorded an object, live is %s" % (path, _type_bucket(live))] + if path in DATA_KEYED_MAPS: + if expected and not live: + return ["%s: recorded non-empty, live is empty" % path] + if expected and live: + return shape_mismatches(next(iter(expected.values())), + next(iter(live.values())), + path + ".*") + return [] problems = [] for key, value in expected.items(): live_key = key @@ -209,6 +244,33 @@ def test_numpy_scalars_count_as_numbers(): assert shape_mismatches({"count": 3}, {"count": "5"}) # still a drift +def test_data_keyed_maps_ignore_which_keys_appear(): + """Examples/Images/Domains/Licenses key sets are content, not schema.""" + recorded = {"Examples": {"VFB_00030786": [{"id": "VFB_00030810", + "label": "medulla", + "thumbnail": "https://x/t.png"}]}} + # A different template carrying the same record shape is not a drift -- + # this is the medulla / Ito2014 case, where the class document's ten-image + # cap dropped the only image on one template. + assert not shape_mismatches(recorded, {"Examples": {"VFB_00101567": [ + {"id": "VFB_00107fob", "label": "ME_R", "thumbnail": "https://y/t.png"}]}}) + # Losing the map altogether still fails. + assert shape_mismatches(recorded, {"Examples": {}}) + # So does a record that lost a field, or changed a field's type. + assert shape_mismatches(recorded, {"Examples": {"VFB_00101567": [ + {"id": "VFB_00107fob", "label": "ME_R"}]}}) + assert shape_mismatches(recorded, {"Examples": {"VFB_00101567": [ + {"id": 7, "label": "ME_R", "thumbnail": "https://y/t.png"}]}}) + # An empty recording stays permissive: gaining content is allowed. + assert not shape_mismatches({"Examples": {}}, recorded) + + +def test_meta_keys_are_still_schema(): + """Meta is not data-keyed -- a missing Meta field is still a regression.""" + recorded = {"Meta": {"Name": "medulla", "Types": "x"}} + assert shape_mismatches(recorded, {"Meta": {"Name": "medulla"}}) + + # --------------------------------------------------------------------------- # Re-recording — `python -m src.test.test_example_queries --record` # ---------------------------------------------------------------------------