From 4f8eaec713b94280db3970466a5344240b5a797b Mon Sep 17 00:00:00 2001 From: Jammy2211 Date: Thu, 8 Oct 2026 09:09:17 +0100 Subject: [PATCH] feat: add setup and baseline experiment evidence contract Accept inference-summary v2 alongside v1. Preserve prepared-problem and baseline provenance, distinguish cold/warm/resume from compilation/cache state, and refuse incompatible cost comparisons. Agent: Codex; model: GPT-6 --- REFERENCE.md | 138 +++++- insight/catalogue.py | 310 ++++++++++++++ insight/summary.py | 12 +- tests/fixtures/lens_summary_v2.json | 639 ++++++++++++++++++++++++++++ tests/test_catalogue.py | 273 ++++++++++++ 5 files changed, 1369 insertions(+), 3 deletions(-) create mode 100644 insight/catalogue.py create mode 100644 tests/fixtures/lens_summary_v2.json create mode 100644 tests/test_catalogue.py diff --git a/REFERENCE.md b/REFERENCE.md index 1d635d2..8f25990 100644 --- a/REFERENCE.md +++ b/REFERENCE.md @@ -13,7 +13,7 @@ There is no inference conductor and no new scientific execution engine. A source is registered once. A fixture demonstrates generic reading without being registered as a real second adopter. Initial real producer: autolens_inference. -The reader supports `schema: inference-summary`, integer `version: 1`. +The reader supports `schema: inference-summary`, integer `version: 1` or `2`. Required envelope keys: project, scope, generated_at, evidence_updated_at, producer_revision, coverage, records, comparisons, comparison_policy, limitations. `project` equals the registry's Mind identity. Times are UTC; unknown evidence time @@ -50,6 +50,142 @@ Unknown controls are not proof of parity. It displays refusals without ratios or rankings. The initial producer declares target/seed/stage context but no certified comparison pairs. Detailed original evidence remains linked at the capture commit. +## Version 2: setups and baseline-to-experiment contract + +V2 retains every v1 envelope and record field, clock and provenance distinction. +It adds `setups`, `prepared_problems` and explicit conditions on each record. +The complete synthetic [v2 fixture](tests/fixtures/lens_summary_v2.json) illustrates +a SLaM prerequisite stage, a baseline `mass_total[1]` stage and cold/warm/resumed +experiments. Its measurements are test data, not scientific recommendations. + +The reader accepts both versions, but a registry row still pins **one exact +version**. Deploy this reader first, then the project producer, then change the +registry pin with its consumer. The live registry remains `inference-summary@1` +in this phase. Rendering and scientific execution are unchanged. There is no +automatic v1 conversion and no inferred baseline, start state or acceptance. + +### Identity and ownership + +| Object | Required fields | Meaning | +|---|---|---| +| Setup | `id`, `label`, `dataset_family`, `model_family`, `instrument`, `reference_record_id` | Stable navigable setup, e.g. imaging/delaunay/HST; IDs are independent of script paths. Multiple exact configurations can live under the family. | +| Prepared problem | `id`, `setup_id`, `dataset_id`, `model_id`, `priors_id`, `stage`, `baseline_record_id`, `artifacts` | Exact scientific problem shared by baseline and experiments. Model and prior IDs identify frozen definitions, not display names. | +| Run record | All v1 fields plus `setup_id`, `problem_id`, `experiment_protocol`, `initialization`, `environment`, `work` | A particular measurement with its own seed, sampler, hardware, revisions, diagnostics and clocks. | + +`instrument` and `reference_record_id` may be null with a sibling `_reason`. +Prepared problem `dataset_id`, `model_id`, `priors_id`, `stage`, and +`baseline_record_id` may likewise be null with reasons. Stage null is appropriate +for standalone toy likelihoods; SLaM uses its actual recorded stage name, not +an alias such as `mass[1]`. Run `setup_id`, `problem_id`, and +`experiment_protocol` may be null with reasons for unmapped historical or +prerequisite/parent records. Every non-null reference must resolve. All lists +have unique IDs. Records must agree with their setup's dataset family/instrument +and their problem's setup, exact dataset and stage. + +The project producer owns the assertion that the frozen model/prior definitions +and artifacts actually match the executed analysis. Include their manifests in +the artifacts; Insight validates declarations, not scientific Python objects. +A changed dataset, model, priors, adapt image or other likelihood-affecting +preparation requires a different prepared-problem ID. A sampler change alone +does not. Keep backend, solver/configuration and measured revisions explicit in +the run; a protocol must fix every scientific control it claims to compare. + +Prepared `artifacts` and initialization `sources` use the same provenance shape: + +```json +{ + "kind": "adapt_image", + "record_id": "baseline-source", + "path": "prepared/adapt.fits", + "revision": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +} +``` + +The path is safe and relative to the producing project repository/storage root; +revision is the full source commit and sha256 identifies exact artifact bytes. +Paths may refer to retained private/gitignored output: this declaration is not +a claim of publicly downloadable or currently available bytes. The project +executor must verify the artifact digest and availability before reuse; Insight +does not fetch artifacts. The source record must resolve. Prepared sources must +belong to the same setup and may come from earlier SLaM stages. An empty artifact +list needs `artifacts_reason` (e.g. unavailable historical state, or a toy model +that needs no adapt products). Unknown evidence is never replaced by current +defaults or another run's artifacts. + +`baseline_record_id` identifies the reference run for the prepared problem. It +can retain an archived or unassessed baseline candidate. This **does not select +it as trusted**. The setup's optional `reference_record_id` is an explicit +producer selection: it must resolve to a current, complete, explicitly accepted +baseline of that setup/problem with known dataset/model/prior identities and +`diagnostics.values.max_log_likelihood` plus a nonempty numeric +`max_likelihood_parameters` mapping. Existing `samples` availability and posterior +diagnostics remain separate; a maximum-likelihood match does not prove posterior +recovery. Scientific acceptance is supplied by the project/Cortex protocol, +never inferred by Insight. A missing selected reference needs a reason. + +### Sampler start, compilation and caches + +`initialization` requires `mode`, a nonempty `recipe` and a `sources` list: + +| Mode | Meaning and source requirements | +|---|---| +| `cold` | Start from the prepared priors without reusing baseline best-fit parameters, posterior samples or sampler state. `sources` must be empty. Prepared SLaM adapt products remain allowed. | +| `warm` | Reuse named baseline information, such as positions, posterior particles, covariance or a mass matrix. At least one pinned source is required. | +| `resume` | Continue a previous run from its checkpoint; at least one source must have `kind: checkpoint`. A resumed segment is not a fresh cold or warm measurement. | +| `unknown` | Legacy or incomplete evidence; requires `reason`. Never assumed cold. | + +Initialization sources must be other records of the same prepared problem. +Self references and cycles are invalid. A checkpoint snapshot is a distinct +source record; do not overwrite it with the resumed measurement. Priors narrowed +using baseline results change the scientific problem, not merely the start mode. + +Each record's `environment` requires `hardware_id`, `compilation` and `cache`. +The project defines a hardware ID that includes device/host resources and +concurrency; it is separate from incidental scheduler job IDs. Unknown hardware +is null with `hardware_id_reason`. Compilation and cache each take `cold`, +`warm`, `not_applicable` or `unknown`; unknown requires `_reason`. +Sampler initialization does not determine either state. A cold sampler can run +with warm compilation, and a warm sampler can pay compilation costs. + +### Costs and work units + +V2 adds `timings.preparation_s` and `timings.initialization_s`, each nonnegative +seconds or null. Both require entries in `timings.definitions`; null also needs +`_reason` in the timings object. Preparation describes prerequisite work +such as the SLaM stages that built adapt products. Initialization describes +sampler startup/warmup as defined by the producer. Definitions must state the +scope, inclusion in other clocks and whether a resumed clock is cumulative or +segment-only. Never add overlapping clocks or infer unmeasured cost as zero. +The original setup/compile/sampling/fit clocks retain their original definitions. + +`work` requires separate `likelihood_evaluations`, `gradient_evaluations`, +`iterations`, `retained_samples` and `effective_sample_size`. Counts are +nonnegative integers; ESS may be a nonnegative real. All may be null with a +sibling `_reason`. `work.definitions` requires a definition for each: +include whether warmup/burn-in is counted, chain/particle aggregation and the +ESS estimator where applicable. Sampler-native iterations are not a common +cross-sampler work unit; no conversion into evaluations is inferred. + +### Comparisons and migration + +V2 comparison refusals extend v1's target/dataset/model/pipeline/stage/seed and +clock checks. Prepared problem, setup, experiment protocol, backend, precision, +dependency revisions, initialization and environment must match. Problem +dataset/model/prior identities must be known; the comparison protocol must match +the record protocols. Unknown starts or hardware/compilation/cache conditions, +resumptions and archived records are refused as current like-for-like cost +comparisons. Refused groups remain visible with reasons. Differing hardware or +starts can still be browsed as separately labelled evidence; this version does +not certify a cross-condition speed ratio. No acceptance, ranking, best seed or +universal convergence threshold is calculated even when controls match. + +Keep original record IDs, parent/stage links, archives and result paths during +migration. Unmapped records retain explicit null bindings and reasons. Empty v2 +lists remain a valid "no measurements" state. The producer and executor must +validate model/prior/artifact provenance; the dashboard and assistant consume +the same project-owned catalogue rather than independently reconstructing it. + ## Ingest and output `bin/pyauto-insight fetch` resolves each project branch once and fetches the declared diff --git a/insight/catalogue.py b/insight/catalogue.py new file mode 100644 index 0000000..d98c579 --- /dev/null +++ b/insight/catalogue.py @@ -0,0 +1,310 @@ +"""Setup and prepared-problem contract for inference-summary v2. + +Validate producer declarations only: no artifact loading, scientific acceptance, +baseline selection, sampler ranking or execution. Original v1 records survive. +""" + +from __future__ import annotations + +import re + +from insight.summary import SHA, _text, safe_relative_path + +DIGEST = re.compile(r"^[0-9a-f]{64}$") +COUNTS = ("likelihood_evaluations", "gradient_evaluations", "iterations", "retained_samples") + + +def _object(value): + return value if isinstance(value, dict) else {} + + +def _index(value, name, errors): + if not isinstance(value, list): + errors.append(f"{name} must be a list") + return {} + result = {} + for row in value: + if not isinstance(row, dict) or not _text(row.get("id")): + errors.append(f"{name} entry needs id") + elif row["id"] in result: + errors.append(f"duplicate {name} id {row['id']}") + else: + result[row["id"]] = row + return result + + +def _resolve(value, index): + return index.get(value) if isinstance(value, str) else None + + +def _nullable(row, key, errors, where): + """Unknown identifiers require explicit reasons; absent is not unknown.""" + if key not in row: + errors.append(f"{where}: missing {key}") + elif row[key] is None: + if not _text(row.get(key + "_reason")): + errors.append(f"{where}: unknown {key} needs reason") + elif not _text(row[key]): + errors.append(f"{where}: {key} must be text or null") + + +def _artifact(value, records, errors, where): + if not isinstance(value, dict): + errors.append(f"{where}: artifact must be object") + return + for key, pattern in (("revision", SHA), ("sha256", DIGEST)): + if not isinstance(value.get(key), str) or not pattern.fullmatch(value[key]): + errors.append(f"{where}: artifact needs full {key}") + if not safe_relative_path(value.get("path")): + errors.append(f"{where}: unsafe artifact path") + if not _text(value.get("kind")): + errors.append(f"{where}: artifact kind required") + if _resolve(value.get("record_id"), records) is None: + errors.append(f"{where}: artifact source record unresolved") + + +def _run_conditions(row, records, errors): + rid = row["id"] + init = row.get("initialization") + if not isinstance(init, dict): + errors.append(f"{rid}: initialization must be object") + else: + mode = init.get("mode") + if mode not in ("cold", "warm", "resume", "unknown"): + errors.append(f"{rid}: invalid initialization mode") + if not _text(init.get("recipe")): + errors.append(f"{rid}: initialization recipe required") + sources = init.get("sources") + if not isinstance(sources, list): + errors.append(f"{rid}: initialization sources must be list") + else: + if mode == "cold" and sources: + errors.append(f"{rid}: cold initialization cannot reuse sampler information") + if mode in ("warm", "resume") and not sources: + errors.append(f"{rid}: {mode} initialization requires sources") + for source in sources: + _artifact(source, records, errors, rid) + if isinstance(source, dict): + origin = _resolve(source.get("record_id"), records) + if origin and ( + origin["id"] == rid or origin.get("problem_id") != row.get("problem_id") + ): + errors.append( + f"{rid}: initialization source must be another run of the same problem" + ) + if mode == "resume" and not any( + isinstance(s, dict) and s.get("kind") == "checkpoint" for s in sources + ): + errors.append(f"{rid}: resume requires checkpoint source") + if mode == "unknown" and not _text(init.get("reason")): + errors.append(f"{rid}: unknown initialization needs reason") + environment = row.get("environment") + if not isinstance(environment, dict): + errors.append(f"{rid}: environment must be object") + else: + _nullable(environment, "hardware_id", errors, rid) + for key in ("compilation", "cache"): + if environment.get(key) not in ("cold", "warm", "not_applicable", "unknown"): + errors.append(f"{rid}: invalid {key} state") + if environment.get(key) == "unknown" and not _text(environment.get(key + "_reason")): + errors.append(f"{rid}: unknown {key} needs reason") + timing = row.get("timings") + if isinstance(timing, dict): + defs = timing.get("definitions") + for key in ("preparation_s", "initialization_s"): + value = timing.get(key) + if key not in timing or ( + value is not None and (type(value) not in (int, float) or value < 0) + ): + errors.append(f"{rid}: {key} must be nonnegative seconds or null") + if not isinstance(defs, dict) or not _text(defs.get(key)): + errors.append(f"{rid}: {key} needs timing definition") + if value is None and not _text(timing.get(key + "_reason")): + errors.append(f"{rid}: unknown {key} needs reason") + work = row.get("work") + if not isinstance(work, dict): + errors.append(f"{rid}: work must be object") + else: + for key in (*COUNTS, "effective_sample_size"): + value = work.get(key) + types = (int, float) if key == "effective_sample_size" else (int,) + if key not in work or (value is not None and (type(value) not in types or value < 0)): + errors.append(f"{rid}: invalid work.{key}") + if value is None and not _text(work.get(key + "_reason")): + errors.append(f"{rid}: unknown work.{key} needs reason") + if not _text(_object(work.get("definitions")).get(key)): + errors.append(f"{rid}: work.{key} needs definition") + + +def validate(doc): + errors = [] + setups = _index(doc.get("setups"), "setups", errors) + problems = _index(doc.get("prepared_problems"), "prepared_problems", errors) + records = _index(doc.get("records"), "records", errors) + for sid, setup in setups.items(): + for key in ("dataset_family", "model_family", "label"): + if not _text(setup.get(key)): + errors.append(f"{sid}: {key} required") + _nullable(setup, "instrument", errors, sid) + _nullable(setup, "reference_record_id", errors, sid) + ref = setup.get("reference_record_id") + if ref is not None: + record = _resolve(ref, records) + if record is None or record.get("setup_id") != sid: + errors.append(f"{sid}: reference record unresolved or wrong setup") + elif ( + record.get("archived") + or _object(record.get("scientific")).get("acceptance") != "accepted" + ): + errors.append(f"{sid}: selected reference must be current and explicitly accepted") + elif _object(record.get("execution")).get("completed") is not True: + errors.append(f"{sid}: selected reference must be complete") + else: + problem = _resolve(record.get("problem_id"), problems) + if not problem or problem.get("baseline_record_id") != ref: + errors.append(f"{sid}: selected reference must be a declared problem baseline") + elif any(not problem.get(k) for k in ("dataset_id", "model_id", "priors_id")): + errors.append(f"{sid}: selected reference needs known prepared identities") + diagnostics = _object(_object(record.get("diagnostics")).get("values")) + if type(diagnostics.get("max_log_likelihood")) not in (int, float): + errors.append(f"{sid}: selected reference needs maximum likelihood") + parameters = diagnostics.get("max_likelihood_parameters") + if ( + not isinstance(parameters, dict) + or not parameters + or any( + not _text(k) or type(v) not in (int, float) for k, v in parameters.items() + ) + ): + errors.append(f"{sid}: selected reference needs maximum likelihood parameters") + for pid, problem in problems.items(): + if _resolve(problem.get("setup_id"), setups) is None: + errors.append(f"{pid}: setup unresolved") + for key in ("dataset_id", "model_id", "priors_id", "stage", "baseline_record_id"): + _nullable(problem, key, errors, pid) + artifacts = problem.get("artifacts") + if not isinstance(artifacts, list): + errors.append(f"{pid}: artifacts must be list") + else: + if not artifacts and not _text(problem.get("artifacts_reason")): + errors.append(f"{pid}: empty prepared artifacts need reason") + for artifact in artifacts: + _artifact(artifact, records, errors, pid) + if isinstance(artifact, dict): + source = _resolve(artifact.get("record_id"), records) + if source and source.get("setup_id") != problem.get("setup_id"): + errors.append(f"{pid}: prepared artifact source belongs to another setup") + baseline = problem.get("baseline_record_id") + if baseline is not None: + r = _resolve(baseline, records) + if r is None or r.get("problem_id") != pid: + errors.append(f"{pid}: baseline record unresolved or wrong problem") + for rid, row in records.items(): + for key in ("setup_id", "problem_id", "experiment_protocol"): + _nullable(row, key, errors, rid) + setup = _resolve(row.get("setup_id"), setups) + if row.get("setup_id") is not None and setup is None: + errors.append(f"{rid}: setup unresolved") + dataset = row.get("dataset") + if ( + setup + and isinstance(dataset, dict) + and dataset.get("class") != setup.get("dataset_family") + ): + errors.append(f"{rid}: dataset family differs from setup") + if ( + setup + and isinstance(dataset, dict) + and dataset.get("instrument") != setup.get("instrument") + ): + errors.append(f"{rid}: instrument differs from setup") + problem = _resolve(row.get("problem_id"), problems) + if row.get("problem_id") is not None and problem is None: + errors.append(f"{rid}: problem unresolved") + if problem: + if row.get("setup_id") != problem.get("setup_id") or row.get("stage") != problem.get( + "stage" + ): + errors.append(f"{rid}: problem setup/stage mismatch") + if isinstance(dataset, dict) and dataset.get("id") != problem.get("dataset_id"): + errors.append(f"{rid}: problem dataset mismatch") + _run_conditions(row, records, errors) + # Source records are snapshots of earlier runs, never a cycle of resumptions. + edges = {} + for rid, row in records.items(): + sources = _object(row.get("initialization")).get("sources") + edges[rid] = ( + [ + s["record_id"] + for s in sources + if isinstance(s, dict) and _resolve(s.get("record_id"), records) is not None + ] + if isinstance(sources, list) + else [] + ) + done = set() + for rid in edges: + pending = [(rid, False)] + visiting = set() + while pending: + current, leaving = pending.pop() + if leaving: + visiting.discard(current) + done.add(current) + elif current in visiting: + errors.append("initialization source cycle") + pending.clear() + elif current not in done: + visiting.add(current) + pending.append((current, True)) + pending.extend((parent, False) for parent in edges[current]) + return errors + + +def comparison_refusals(doc, rows, comparison): + """Conservative same-problem, same-start cost comparisons only. + + Cross-condition exploration remains visible as separate records, not a + certified speed comparison. This does not certify scientific equivalence. + """ + reasons = [] + problems = {p["id"]: p for p in doc.get("prepared_problems", [])} + for row in rows: + problem = _resolve(row.get("problem_id"), problems) or {} + for key in ("dataset_id", "model_id", "priors_id"): + if not problem.get(key): + reason = f"prepared {key} unknown" + if reason not in reasons: + reasons.append(reason) + if any(r.get("experiment_protocol") != comparison.get("protocol") for r in rows): + reasons.append("comparison protocol differs from experiment protocol") + for key in ( + "setup_id", + "problem_id", + "experiment_protocol", + "backend", + "precision", + "dependency_revisions", + ): + values = [r.get(key) for r in rows] + if any(not v for v in values): + reasons.append(f"{key} unknown") + if any(v != values[0] for v in values): + reasons.append(f"{key} changed") + for key in ("initialization", "environment"): + values = [r.get(key, {}) for r in rows] + if any(v != values[0] for v in values): + reasons.append(f"{key} differs") + if any(r.get("initialization", {}).get("mode") in (None, "unknown", "resume") for r in rows): + reasons.append("unknown or resumed initialization is not a fresh-run comparison") + if any( + not r.get("environment", {}).get("hardware_id") + or any( + r.get("environment", {}).get(k) in (None, "unknown") for k in ("compilation", "cache") + ) + for r in rows + ): + reasons.append("hardware or compilation/cache conditions unknown") + if any(r.get("archived") for r in rows): + reasons.append("archived evidence is not a current comparison") + return reasons diff --git a/insight/summary.py b/insight/summary.py index 9541e93..ec90976 100644 --- a/insight/summary.py +++ b/insight/summary.py @@ -1,4 +1,4 @@ -"""Validate inference-summary v1 without interpreting scientific quality.""" +"""Validate inference-summary v1/v2 without interpreting scientific quality.""" from __future__ import annotations @@ -50,7 +50,7 @@ def supported(doc, wanted=None): if not isinstance(doc, dict): return "summary is not an object" got = doc.get("schema"), doc.get("version") - if got[0] != "inference-summary" or type(got[1]) is not int or got[1] != 1: + if got[0] != "inference-summary" or type(got[1]) is not int or got[1] not in (1, 2): return f"unsupported schema {got!r}" if wanted is not None and got != tuple(wanted): return "summary schema differs from registry" @@ -302,6 +302,10 @@ def validate(doc): errors.append("comparison endpoints must be distinct") if not _text(c.get("protocol")): errors.append("comparison requires declared protocol") + if doc.get("version") == 2: + from insight import catalogue + + errors.extend(catalogue.validate(doc)) return errors @@ -344,4 +348,8 @@ def comparison_refusals(doc, c): reasons.append("timing definitions differ") if not c.get("protocol"): reasons.append("no declared scientific protocol") + if doc.get("version") == 2: + from insight import catalogue + + reasons.extend(catalogue.comparison_refusals(doc, rows, c)) return reasons diff --git a/tests/fixtures/lens_summary_v2.json b/tests/fixtures/lens_summary_v2.json new file mode 100644 index 0000000..5c4ae41 --- /dev/null +++ b/tests/fixtures/lens_summary_v2.json @@ -0,0 +1,639 @@ +{ + "comparison_policy": { + "id": "declared-target-seed-stage-v1", + "note": "Targets/seeds/stages retained; no automatic parity certification or best-seed selection", + "ranking": "none" + }, + "comparisons": [ + { + "id": "cold-pair", + "records": [ + "baseline-mass", + "experiment-cold" + ], + "protocol": "fixed-mass-protocol-v1" + } + ], + "coverage": { + "excluded": [], + "expected": { + "reason": "No complete expected-run manifest; unrecorded jobs cannot be enumerated", + "runs": null + }, + "observed": { + "records": 5 + } + }, + "evidence_updated_at": null, + "evidence_updated_at_reason": "Legacy result rows do not record measurement timestamps", + "generated_at": "2026-10-04T07:35:14Z", + "generation_basis": "Source commit timestamp; not a measurement or check-in time", + "limitations": [ + "No scientific acceptance or convergence inferred from process completion", + "Sample archives remain in project storage; current availability unknown", + "No evidence from the retired inference programme is imported", + "Archived rows belong to this project's September 2026 runs and are historical, not current baselines" + ], + "producer_revision": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "project": "autolens_inference", + "records": [ + { + "archived": false, + "backend": "jax_cpu", + "comparison": { + "declared_target": "simple/source_plane_solved/seed0", + "group": null, + "note": "Declared target is not a verified comparison group; no ranking" + }, + "configuration": { + "free_parameters": 10 + }, + "dataset": { + "id": "simulated-hst-v1", + "instrument": "hst", + "class": "imaging" + }, + "dependency_revisions": { + "PyAutoLens": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + }, + "diagnostics": { + "status": "available", + "values": { + "max_log_likelihood": 100.0, + "max_likelihood_parameters": { + "einstein_radius": 1.6 + } + } + }, + "evidence_paths": [ + "results/baseline.json" + ], + "execution": { + "completed": true, + "reason": null, + "status": "complete" + }, + "hardware": { + "device": "cpu" + }, + "id": "baseline-mass", + "library_version": "2026.8.17.1", + "measured_at": null, + "model": "mass-model-v1", + "parent_run_id": null, + "pipeline": "slam5", + "precision": "fp64", + "sampler": "nautilus", + "samples": { + "availability": "restricted", + "path": "output/baseline/samples.csv" + }, + "scientific": { + "acceptance": "not_assessed", + "convergence": "not_assessed", + "reason": "Exporter does not make scientific rulings; consult Cortex" + }, + "seed": 0, + "stage": "mass_total[1]", + "target": "simple/source_plane_solved/seed0", + "timings": { + "compile_s": 1.9011598539655097, + "definitions": { + "compile_s": "Separate likelihood warm-up probe; not added to total_s", + "sampling_s": "wall_s: sampler clock from samples_info.json; includes sampler overhead", + "setup_s": "Not measured separately; unknown, not total minus sampling", + "total_s": "total_wall_s: elapsed search.fit call; excludes prior compile probe and dataset/model setup", + "preparation_s": "Prerequisite SLaM work, excluded from this stage fit clock", + "initialization_s": "Sampler initialization inside sampling_s; never add again" + }, + "sampling_s": 56.56000781059265, + "setup_s": null, + "total_s": 65.21035292901797, + "preparation_s": null, + "preparation_s_reason": "Baseline prerequisite cost not separately measured", + "initialization_s": null, + "initialization_s_reason": "Included in sampler clock; not separately measured" + }, + "unknown_reasons": { + "measured_at": "Not recorded in source result" + }, + "setup_id": "imaging/delaunay/hst", + "problem_id": "hst-delaunay-mass-v1", + "experiment_protocol": "fixed-mass-protocol-v1", + "initialization": { + "mode": "cold", + "recipe": "Draw from the prepared priors", + "sources": [] + }, + "environment": { + "hardware_id": "cpu-host-v1-8cores", + "compilation": "cold", + "cache": "cold" + }, + "work": { + "likelihood_evaluations": 20000, + "gradient_evaluations": 0, + "iterations": null, + "iterations_reason": "Not exposed", + "retained_samples": 1000, + "effective_sample_size": 900.0, + "definitions": { + "likelihood_evaluations": "Calls to log likelihood including initialization", + "gradient_evaluations": "Calls to log likelihood gradient", + "iterations": "Sampler-native iterations; unavailable here", + "retained_samples": "Stored posterior samples after warmup", + "effective_sample_size": "Sampler-reported posterior ESS" + } + } + }, + { + "archived": false, + "backend": "jax_cpu", + "comparison": { + "declared_target": "simple/source_plane_solved/seed0", + "group": null, + "note": "Declared target is not a verified comparison group; no ranking" + }, + "configuration": { + "free_parameters": 10 + }, + "dataset": { + "id": "simulated-hst-v1", + "instrument": "hst", + "class": "imaging" + }, + "dependency_revisions": { + "PyAutoLens": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + }, + "diagnostics": { + "status": "available", + "values": { + "max_log_likelihood": 100.0, + "max_likelihood_parameters": { + "einstein_radius": 1.6 + } + } + }, + "evidence_paths": [ + "results/baseline.json" + ], + "execution": { + "completed": true, + "reason": null, + "status": "complete" + }, + "hardware": { + "device": "cpu" + }, + "id": "baseline-source", + "library_version": "2026.8.17.1", + "measured_at": null, + "model": "mass-model-v1", + "parent_run_id": null, + "pipeline": "slam5", + "precision": "fp64", + "sampler": "nautilus", + "samples": { + "availability": "restricted", + "path": "output/baseline/samples.csv" + }, + "scientific": { + "acceptance": "not_assessed", + "convergence": "not_assessed", + "reason": "Exporter does not make scientific rulings; consult Cortex" + }, + "seed": 0, + "stage": "source_pix[2]", + "target": "simple/source_plane_solved/seed0", + "timings": { + "compile_s": 1.9011598539655097, + "definitions": { + "compile_s": "Separate likelihood warm-up probe; not added to total_s", + "sampling_s": "wall_s: sampler clock from samples_info.json; includes sampler overhead", + "setup_s": "Not measured separately; unknown, not total minus sampling", + "total_s": "total_wall_s: elapsed search.fit call; excludes prior compile probe and dataset/model setup", + "preparation_s": "Prerequisite SLaM work, excluded from this stage fit clock", + "initialization_s": "Sampler initialization inside sampling_s; never add again" + }, + "sampling_s": 56.56000781059265, + "setup_s": null, + "total_s": 65.21035292901797, + "preparation_s": null, + "preparation_s_reason": "Baseline prerequisite cost not separately measured", + "initialization_s": null, + "initialization_s_reason": "Included in sampler clock; not separately measured" + }, + "unknown_reasons": { + "measured_at": "Not recorded in source result" + }, + "setup_id": "imaging/delaunay/hst", + "problem_id": null, + "experiment_protocol": "fixed-mass-protocol-v1", + "initialization": { + "mode": "cold", + "recipe": "Draw from the prepared priors", + "sources": [] + }, + "environment": { + "hardware_id": "cpu-host-v1-8cores", + "compilation": "cold", + "cache": "cold" + }, + "work": { + "likelihood_evaluations": 20000, + "gradient_evaluations": 0, + "iterations": null, + "iterations_reason": "Not exposed", + "retained_samples": 1000, + "effective_sample_size": 900.0, + "definitions": { + "likelihood_evaluations": "Calls to log likelihood including initialization", + "gradient_evaluations": "Calls to log likelihood gradient", + "iterations": "Sampler-native iterations; unavailable here", + "retained_samples": "Stored posterior samples after warmup", + "effective_sample_size": "Sampler-reported posterior ESS" + } + }, + "problem_id_reason": "Prerequisite stage; no standalone comparison problem" + }, + { + "archived": false, + "backend": "jax_cpu", + "comparison": { + "declared_target": "simple/source_plane_solved/seed0", + "group": null, + "note": "Declared target is not a verified comparison group; no ranking" + }, + "configuration": { + "free_parameters": 10 + }, + "dataset": { + "id": "simulated-hst-v1", + "instrument": "hst", + "class": "imaging" + }, + "dependency_revisions": { + "PyAutoLens": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + }, + "diagnostics": { + "status": "available", + "values": { + "max_log_likelihood": 100.0, + "max_likelihood_parameters": { + "einstein_radius": 1.6 + } + } + }, + "evidence_paths": [ + "results/baseline.json" + ], + "execution": { + "completed": true, + "reason": null, + "status": "complete" + }, + "hardware": { + "device": "cpu" + }, + "id": "experiment-cold", + "library_version": "2026.8.17.1", + "measured_at": null, + "model": "mass-model-v1", + "parent_run_id": null, + "pipeline": "slam5", + "precision": "fp64", + "sampler": "smc", + "samples": { + "availability": "restricted", + "path": "output/baseline/samples.csv" + }, + "scientific": { + "acceptance": "not_assessed", + "convergence": "not_assessed", + "reason": "Exporter does not make scientific rulings; consult Cortex" + }, + "seed": 0, + "stage": "mass_total[1]", + "target": "simple/source_plane_solved/seed0", + "timings": { + "compile_s": 1.9011598539655097, + "definitions": { + "compile_s": "Separate likelihood warm-up probe; not added to total_s", + "sampling_s": "wall_s: sampler clock from samples_info.json; includes sampler overhead", + "setup_s": "Not measured separately; unknown, not total minus sampling", + "total_s": "total_wall_s: elapsed search.fit call; excludes prior compile probe and dataset/model setup", + "preparation_s": "Prerequisite SLaM work, excluded from this stage fit clock", + "initialization_s": "Sampler initialization inside sampling_s; never add again" + }, + "sampling_s": 56.56000781059265, + "setup_s": null, + "total_s": 65.21035292901797, + "preparation_s": null, + "preparation_s_reason": "Baseline prerequisite cost not separately measured", + "initialization_s": null, + "initialization_s_reason": "Included in sampler clock; not separately measured" + }, + "unknown_reasons": { + "measured_at": "Not recorded in source result" + }, + "setup_id": "imaging/delaunay/hst", + "problem_id": "hst-delaunay-mass-v1", + "experiment_protocol": "fixed-mass-protocol-v1", + "initialization": { + "mode": "cold", + "recipe": "Draw from the prepared priors", + "sources": [] + }, + "environment": { + "hardware_id": "cpu-host-v1-8cores", + "compilation": "cold", + "cache": "cold" + }, + "work": { + "likelihood_evaluations": 20000, + "gradient_evaluations": 0, + "iterations": null, + "iterations_reason": "Not exposed", + "retained_samples": 1000, + "effective_sample_size": 900.0, + "definitions": { + "likelihood_evaluations": "Calls to log likelihood including initialization", + "gradient_evaluations": "Calls to log likelihood gradient", + "iterations": "Sampler-native iterations; unavailable here", + "retained_samples": "Stored posterior samples after warmup", + "effective_sample_size": "Sampler-reported posterior ESS" + } + } + }, + { + "archived": false, + "backend": "jax_cpu", + "comparison": { + "declared_target": "simple/source_plane_solved/seed0", + "group": null, + "note": "Declared target is not a verified comparison group; no ranking" + }, + "configuration": { + "free_parameters": 10 + }, + "dataset": { + "id": "simulated-hst-v1", + "instrument": "hst", + "class": "imaging" + }, + "dependency_revisions": { + "PyAutoLens": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + }, + "diagnostics": { + "status": "available", + "values": { + "max_log_likelihood": 100.0, + "max_likelihood_parameters": { + "einstein_radius": 1.6 + } + } + }, + "evidence_paths": [ + "results/baseline.json" + ], + "execution": { + "completed": true, + "reason": null, + "status": "complete" + }, + "hardware": { + "device": "cpu" + }, + "id": "experiment-warm", + "library_version": "2026.8.17.1", + "measured_at": null, + "model": "mass-model-v1", + "parent_run_id": null, + "pipeline": "slam5", + "precision": "fp64", + "sampler": "smc", + "samples": { + "availability": "restricted", + "path": "output/baseline/samples.csv" + }, + "scientific": { + "acceptance": "not_assessed", + "convergence": "not_assessed", + "reason": "Exporter does not make scientific rulings; consult Cortex" + }, + "seed": 0, + "stage": "mass_total[1]", + "target": "simple/source_plane_solved/seed0", + "timings": { + "compile_s": 1.9011598539655097, + "definitions": { + "compile_s": "Separate likelihood warm-up probe; not added to total_s", + "sampling_s": "wall_s: sampler clock from samples_info.json; includes sampler overhead", + "setup_s": "Not measured separately; unknown, not total minus sampling", + "total_s": "total_wall_s: elapsed search.fit call; excludes prior compile probe and dataset/model setup", + "preparation_s": "Prerequisite SLaM work, excluded from this stage fit clock", + "initialization_s": "Sampler initialization inside sampling_s; never add again" + }, + "sampling_s": 56.56000781059265, + "setup_s": null, + "total_s": 65.21035292901797, + "preparation_s": null, + "preparation_s_reason": "Baseline prerequisite cost not separately measured", + "initialization_s": null, + "initialization_s_reason": "Included in sampler clock; not separately measured" + }, + "unknown_reasons": { + "measured_at": "Not recorded in source result" + }, + "setup_id": "imaging/delaunay/hst", + "problem_id": "hst-delaunay-mass-v1", + "experiment_protocol": "fixed-mass-protocol-v1", + "initialization": { + "mode": "warm", + "recipe": "Initialize from baseline posterior particles", + "sources": [ + { + "kind": "posterior_samples", + "record_id": "baseline-mass", + "path": "output/baseline/samples.csv", + "revision": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "sha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + } + ] + }, + "environment": { + "hardware_id": "cpu-host-v1-8cores", + "compilation": "cold", + "cache": "cold" + }, + "work": { + "likelihood_evaluations": 20000, + "gradient_evaluations": 0, + "iterations": null, + "iterations_reason": "Not exposed", + "retained_samples": 1000, + "effective_sample_size": 900.0, + "definitions": { + "likelihood_evaluations": "Calls to log likelihood including initialization", + "gradient_evaluations": "Calls to log likelihood gradient", + "iterations": "Sampler-native iterations; unavailable here", + "retained_samples": "Stored posterior samples after warmup", + "effective_sample_size": "Sampler-reported posterior ESS" + } + } + }, + { + "archived": false, + "backend": "jax_cpu", + "comparison": { + "declared_target": "simple/source_plane_solved/seed0", + "group": null, + "note": "Declared target is not a verified comparison group; no ranking" + }, + "configuration": { + "free_parameters": 10 + }, + "dataset": { + "id": "simulated-hst-v1", + "instrument": "hst", + "class": "imaging" + }, + "dependency_revisions": { + "PyAutoLens": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + }, + "diagnostics": { + "status": "available", + "values": { + "max_log_likelihood": 100.0, + "max_likelihood_parameters": { + "einstein_radius": 1.6 + } + } + }, + "evidence_paths": [ + "results/baseline.json" + ], + "execution": { + "completed": true, + "reason": null, + "status": "complete" + }, + "hardware": { + "device": "cpu" + }, + "id": "experiment-resume", + "library_version": "2026.8.17.1", + "measured_at": null, + "model": "mass-model-v1", + "parent_run_id": null, + "pipeline": "slam5", + "precision": "fp64", + "sampler": "smc", + "samples": { + "availability": "restricted", + "path": "output/baseline/samples.csv" + }, + "scientific": { + "acceptance": "not_assessed", + "convergence": "not_assessed", + "reason": "Exporter does not make scientific rulings; consult Cortex" + }, + "seed": 0, + "stage": "mass_total[1]", + "target": "simple/source_plane_solved/seed0", + "timings": { + "compile_s": 1.9011598539655097, + "definitions": { + "compile_s": "Separate likelihood warm-up probe; not added to total_s", + "sampling_s": "wall_s: sampler clock from samples_info.json; includes sampler overhead", + "setup_s": "Not measured separately; unknown, not total minus sampling", + "total_s": "total_wall_s: elapsed search.fit call; excludes prior compile probe and dataset/model setup", + "preparation_s": "Prerequisite SLaM work, excluded from this stage fit clock", + "initialization_s": "Sampler initialization inside sampling_s; never add again" + }, + "sampling_s": 56.56000781059265, + "setup_s": null, + "total_s": 65.21035292901797, + "preparation_s": null, + "preparation_s_reason": "Baseline prerequisite cost not separately measured", + "initialization_s": null, + "initialization_s_reason": "Included in sampler clock; not separately measured" + }, + "unknown_reasons": { + "measured_at": "Not recorded in source result" + }, + "setup_id": "imaging/delaunay/hst", + "problem_id": "hst-delaunay-mass-v1", + "experiment_protocol": "fixed-mass-protocol-v1", + "initialization": { + "mode": "resume", + "recipe": "Continue the warm experiment checkpoint", + "sources": [ + { + "kind": "checkpoint", + "record_id": "experiment-warm", + "path": "output/warm/checkpoint.bin", + "revision": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "sha256": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" + } + ] + }, + "environment": { + "hardware_id": "cpu-host-v1-8cores", + "compilation": "cold", + "cache": "cold" + }, + "work": { + "likelihood_evaluations": 20000, + "gradient_evaluations": 0, + "iterations": null, + "iterations_reason": "Not exposed", + "retained_samples": 1000, + "effective_sample_size": 900.0, + "definitions": { + "likelihood_evaluations": "Calls to log likelihood including initialization", + "gradient_evaluations": "Calls to log likelihood gradient", + "iterations": "Sampler-native iterations; unavailable here", + "retained_samples": "Stored posterior samples after warmup", + "effective_sample_size": "Sampler-reported posterior ESS" + } + } + } + ], + "schema": "inference-summary", + "scope": "inference", + "source_commit": "e60fd387cd5d112782888671d5a9471432abcc93", + "valid_until": null, + "version": 2, + "setups": [ + { + "id": "imaging/delaunay/hst", + "dataset_family": "imaging", + "model_family": "delaunay", + "label": "HST Delaunay", + "reference_record_id": null, + "reference_record_id_reason": "Synthetic fixture; not scientifically assessed", + "instrument": "hst" + } + ], + "prepared_problems": [ + { + "id": "hst-delaunay-mass-v1", + "setup_id": "imaging/delaunay/hst", + "dataset_id": "simulated-hst-v1", + "model_id": "mass-model-v1", + "priors_id": "mass-priors-v1", + "stage": "mass_total[1]", + "baseline_record_id": "baseline-mass", + "artifacts": [ + { + "kind": "adapt_image", + "record_id": "baseline-source", + "path": "prepared/adapt.fits", + "revision": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + } + ] + } + ] +} diff --git a/tests/test_catalogue.py b/tests/test_catalogue.py new file mode 100644 index 0000000..36bd018 --- /dev/null +++ b/tests/test_catalogue.py @@ -0,0 +1,273 @@ +"""Prepared scientific problems survive migration without invented start state.""" + +from copy import deepcopy +from dataclasses import replace + +import pytest +from conftest import LENS_ROW, SHA_A, SHA_B, fixture_doc + +from insight import ingest, registry, summary + + +def document(): + return fixture_doc("lens_summary_v2.json") + + +def test_baseline_cold_warm_and_resume_are_valid_but_not_automatically_accepted(): + doc = document() + assert summary.classify(doc) == ("ok", []) + assert summary.qualified_records(doc) == 0 + assert summary.comparison_refusals(doc, doc["comparisons"][0]) == [] + assert doc["records"][0]["timings"]["initialization_s"] is None + + +@pytest.mark.parametrize("version", [True, 0, 3, "2", None]) +def test_unsupported_versions(version): + doc = document() + doc["version"] = version + assert summary.classify(doc)[0] == "unsupported" + + +def test_registry_pins_exact_version_and_old_feed_remains_valid(body_map): + row = dict(LENS_ROW, supported_schema="inference-summary@2") + assert registry.validate({"schema": 1, "instances": [row]}, body_map) == [] + assert summary.classify(document(), ("inference-summary", 1))[0] == "unsupported" + assert summary.classify(document(), ("inference-summary", 2)) == ("ok", []) + assert summary.classify(fixture_doc("lens_summary_v1.json")) == ("ok", []) + + +@pytest.mark.parametrize("key", ["setups", "prepared_problems"]) +@pytest.mark.parametrize("value", [None, {}, "wrong", [None], [{"id": []}]]) +def test_bad_catalogue_shapes_return_errors(key, value): + doc = document() + doc[key] = value + assert summary.classify(doc)[0] == "invalid" + + +@pytest.mark.parametrize("key", ["setups", "prepared_problems"]) +def test_duplicate_id(key): + doc = document() + doc[key].append(deepcopy(doc[key][0])) + assert any("duplicate" in e for e in summary.validate(doc)) + + +@pytest.mark.parametrize( + "key,value", + [ + ("setup_id", "absent"), + ("problem_id", []), + ("problem_id", "absent"), + ("stage", "source_pix[1]"), + ("dataset", {"class": "point_source", "id": "other"}), + ], +) +def test_wrong_problem_binding(key, value): + doc = document() + doc["records"][2][key] = value + assert summary.classify(doc)[0] == "invalid" + + +@pytest.mark.parametrize("key", ["model_id", "priors_id", "dataset_id"]) +def test_unknown_prepared_identity_remains_explicit(key): + doc = document() + doc["prepared_problems"][0][key] = None + assert summary.classify(doc)[0] == "invalid" + + +@pytest.mark.parametrize( + "key,value", + [ + ("path", "../escape"), + ("path", "x/%2e%2e/y"), + ("sha256", "short"), + ("revision", "main"), + ("record_id", "absent"), + ("record_id", []), + ], +) +def test_prepared_and_initialization_artifact_provenance(key, value): + for artifacts in ("prepared", "initialization"): + doc = document() + artifact = ( + doc["prepared_problems"][0]["artifacts"][0] + if artifacts == "prepared" + else doc["records"][3]["initialization"]["sources"][0] + ) + artifact[key] = value + assert summary.classify(doc)[0] == "invalid" + + +def test_cold_does_not_mean_no_adapt_images_but_cannot_reuse_posterior(): + doc = document() + assert doc["prepared_problems"][0]["artifacts"] + doc["records"][2]["initialization"]["sources"] = doc["records"][3]["initialization"]["sources"] + assert any("cold initialization" in e for e in summary.validate(doc)) + + +@pytest.mark.parametrize("index", [3, 4]) +def test_warm_and_resume_require_source(index): + doc = document() + doc["records"][index]["initialization"]["sources"] = [] + assert summary.classify(doc)[0] == "invalid" + + +def test_resume_requires_checkpoint_and_source_cannot_be_self(): + doc = document() + source = doc["records"][4]["initialization"]["sources"][0] + source["kind"] = "posterior_samples" + assert any("checkpoint" in e for e in summary.validate(doc)) + source["kind"] = "checkpoint" + source["record_id"] = "experiment-resume" + assert summary.classify(doc)[0] == "invalid" + + +@pytest.mark.parametrize("field", ["initialization", "environment", "work"]) +@pytest.mark.parametrize("value", [None, [], 7]) +def test_malformed_run_conditions(field, value): + doc = document() + doc["records"][0][field] = value + assert summary.classify(doc)[0] == "invalid" + + +def test_archive_and_unreviewed_baselines_are_retained_but_not_selected(): + doc = document() + doc["records"][0]["archived"] = True + assert summary.classify(doc) == ("ok", []) + doc["setups"][0]["reference_record_id"] = "baseline-mass" + assert summary.classify(doc)[0] == "invalid" + doc["records"][0]["archived"] = False + assert summary.classify(doc)[0] == "invalid" + doc["records"][0]["scientific"]["acceptance"] = "accepted" + assert summary.classify(doc) == ("ok", []) + + +@pytest.mark.parametrize( + "key,value", + [("initialization_s", -1), ("preparation_s", True), ("initialization_s", float("nan"))], +) +def test_new_clocks_are_nonnegative_seconds(key, value): + doc = document() + doc["records"][0]["timings"][key] = value + assert summary.classify(doc)[0] == "invalid" + + +@pytest.mark.parametrize( + "key,value", + [ + ("iterations", 1.5), + ("retained_samples", True), + ("effective_sample_size", -1), + ("likelihood_evaluations", float("inf")), + ], +) +def test_work_counts_do_not_conflate_iterations_samples_and_ess(key, value): + doc = document() + doc["records"][0]["work"][key] = value + assert summary.classify(doc)[0] == "invalid" + + +@pytest.mark.parametrize("other", ["experiment-warm", "experiment-resume"]) +def test_different_starts_refuse_cost_comparison(other): + doc = document() + pair = dict(doc["comparisons"][0], records=["experiment-cold", other]) + assert any("initialization" in e for e in summary.comparison_refusals(doc, pair)) + + +@pytest.mark.parametrize( + "key,value", [("hardware_id", "a100-host"), ("compilation", "warm"), ("cache", "warm")] +) +def test_sampler_cold_start_is_separate_from_environment(key, value): + doc = document() + doc["records"][2]["environment"][key] = value + assert summary.classify(doc) == ("ok", []) + assert "environment differs" in summary.comparison_refusals(doc, doc["comparisons"][0]) + + +def test_v2_ingest_cache_and_offline_replay(registry_file, fake_mind, web, tmp_path): + instance = replace( + registry.load(registry_file, mind=fake_mind)[0], supported_schema="inference-summary@2" + ) + doc = document() + web.publish(instance.github, SHA_A, instance.summary_path, doc) + good = ingest.ingest(instance, tmp_path, now="2026-10-08T00:00:00Z") + assert good.outcome == "ok" and good.doc == doc + doc["records"][3]["initialization"]["sources"] = [] + web.publish(instance.github, SHA_B, instance.summary_path, doc) + bad = ingest.ingest(instance, tmp_path, now="2026-10-08T01:00:00Z") + assert bad.outcome == "invalid" and bad.cached and bad.commit == SHA_A + replay = ingest.ingest(instance, tmp_path, offline=True) + assert replay.doc == good.doc + + +def test_cycle_of_initialization_sources_is_invalid(): + doc = document() + doc["records"][3]["initialization"]["sources"][0]["record_id"] = "experiment-resume" + assert "initialization source cycle" in summary.validate(doc) + + +def test_unknown_start_is_retained_and_never_compared_as_cold(): + doc = document() + for row in (doc["records"][0], doc["records"][2]): + row["initialization"].update( + mode="unknown", reason="Legacy source did not record initialization" + ) + row["environment"].update(compilation="unknown", compilation_reason="Not recorded") + assert summary.classify(doc) == ("ok", []) + reasons = summary.comparison_refusals(doc, doc["comparisons"][0]) + assert any("initialization" in r for r in reasons) + assert any("compilation" in r for r in reasons) + + +@pytest.mark.parametrize("key", ["model_id", "priors_id"]) +def test_unknown_problem_identity_is_visible_but_never_a_comparison(key): + doc = document() + doc["prepared_problems"][0].update({key: None, key + "_reason": "Legacy metadata absent"}) + assert summary.classify(doc) == ("ok", []) + assert f"prepared {key} unknown" in summary.comparison_refusals(doc, doc["comparisons"][0]) + + +@pytest.mark.parametrize("key", ["dependency_revisions", "precision", "experiment_protocol"]) +def test_changed_run_controls_refuse_comparison(key): + doc = document() + doc["records"][2][key] = ( + {"PyAutoLens": "b" * 40} if key == "dependency_revisions" else "changed" + ) + assert summary.classify(doc) == ("ok", []) + assert f"{key} changed" in summary.comparison_refusals(doc, doc["comparisons"][0]) + + +@pytest.mark.parametrize("key", ["scientific", "execution", "diagnostics"]) +@pytest.mark.parametrize("value", [None, [], 5]) +def test_malformed_selected_reference_does_not_raise(key, value): + doc = document() + doc["setups"][0]["reference_record_id"] = "baseline-mass" + doc["records"][0]["scientific"]["acceptance"] = "accepted" + doc["records"][0][key] = value + assert summary.classify(doc)[0] == "invalid" + + +def test_selected_reference_needs_likelihood_and_parameters(): + doc = document() + doc["setups"][0]["reference_record_id"] = "baseline-mass" + doc["records"][0]["scientific"]["acceptance"] = "accepted" + doc["records"][0]["diagnostics"]["values"] = {} + assert any("maximum likelihood" in e for e in summary.validate(doc)) + + +def test_v2_empty_and_unmapped_legacy_records_are_valid(): + doc = document() + doc["setups"] = [] + doc["prepared_problems"] = [] + doc["comparisons"] = [] + row = doc["records"][0] + doc["records"] = [row] + row.update( + setup_id=None, + setup_id_reason="Unmapped legacy source", + problem_id=None, + problem_id_reason="Prepared state unavailable", + archived=True, + ) + assert summary.classify(doc) == ("ok", []) + doc["records"] = [] + assert summary.classify(doc) == ("ok", [])