From cd65f783bbfd7da72e646683a037fbd35637a660 Mon Sep 17 00:00:00 2001 From: Florent Pollet Date: Mon, 16 Nov 2026 00:18:06 -0500 Subject: [PATCH] Add temporal-leakage conformance: future-blind rerun and null-world chance test The shared suite checked label leakage and transductive fitting but had no check that information from after a prediction time cannot reach the prediction (e.g. NOAH's forward time gap to the next event). - conformance/temporal.py: - perturb_future: rewrite every evaluated subject's events after its latest indexed prediction time (log-uniform offsets, plain rows only, tie order kept) - assert_future_blind_pipeline: rerun the lifecycle on the perturbed data; predictions must be bit-identical (no statistical threshold) - build_null_world_dataset: homogeneous Poisson world where the label "any event in (T, T + ln2/rate]" is independent of the history - null_auroc_test / assert_chance_level: one-sided Mann-Whitney, one key per held-out subject, exact false-positive rate alpha - assert_profile_run now runs the future-blind rerun (test_end_to_end) - new inherited test_null_world_predictions_are_at_chance (non-zero-shot) - CONTRACT/TESTING/README updated; suite test counts 38 -> 44 Validated against NOAH (rvandewater/Multi_MEDS) on its own fixture: clean passes; an injected forward-gap leak and a vitals-bin leak both fail. Co-Authored-By: Claude Opus 5.5 --- README.md | 3 +- docs/testing.md | 2 +- src/meds_algorithm/CONTRACT.md | 20 +- src/meds_algorithm/TESTING.md | 18 +- src/meds_algorithm/conformance/end_to_end.py | 31 +- src/meds_algorithm/conformance/suite.py | 20 +- src/meds_algorithm/conformance/temporal.py | 343 +++++++++++++++++++ tests/test_suite.py | 11 +- tests/test_temporal.py | 209 +++++++++++ 9 files changed, 622 insertions(+), 35 deletions(-) create mode 100644 src/meds_algorithm/conformance/temporal.py create mode 100644 tests/test_temporal.py diff --git a/README.md b/README.md index b0753cc..4b46b24 100644 --- a/README.md +++ b/README.md @@ -135,7 +135,8 @@ uv build Tests inject publication/restoration failures, launch a child from a worker thread, check explicit algorithm provenance, reject missing/mixed/overlapping label inputs, and exercise all six lifecycles with synthetic command implementations. Full-lifecycle conformance reruns algorithm adapters with forbidden -outcomes removed and fresh artifacts; algorithm-specific cutoff, frozen-weight, compatibility and primary +outcomes removed, and again with every evaluated future rewritten, with fresh artifacts; a null-world run checks +chance-level discrimination of the future; algorithm-specific cutoff, frozen-weight, compatibility and primary behavior tests remain required. Package tests do not establish scientific validity of a downstream algorithm. MEDS-DEV provisioning requires explicit `-m meds_dev` or `-m real_data` selection; setting diff --git a/docs/testing.md b/docs/testing.md index 21464b0..d110d69 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1,7 +1,7 @@ # Tests and acceptance The following catalogues separate template checks from algorithm acceptance checks. -The [shared acceptance inventory](reference/tests.md#downstream-acceptance-suite) describes the eight +The [shared acceptance inventory](reference/tests.md#downstream-acceptance-suite) describes the nine inherited checks. Generated repositories keep only fixture wiring, execution adapters and scientific tests. See also [Shared-package tests](reference/tests.md) for tests of the runtime and suite itself. diff --git a/src/meds_algorithm/CONTRACT.md b/src/meds_algorithm/CONTRACT.md index 99b11e3..e6b85d2 100644 --- a/src/meds_algorithm/CONTRACT.md +++ b/src/meds_algorithm/CONTRACT.md @@ -325,6 +325,19 @@ dry-run, and its real end-to-end chain. The rendered `tests/test_end_to_end.py` the integer `seed` override, seed every RNG/framework/subprocess and use deterministic fixture settings. `attach_ground_truth` returns predictions unchanged on an unlabelled index. This tests outcome independence on the exercised fixture, not every possible code path. +- **Temporal cutoff (future perturbation):** the complete lifecycle runs again on a copy of the MEDS + dataset in which every evaluated subject's events after its latest indexed prediction time are deleted + and replaced by plain events copied from the same shard at log-uniform offsets, many within seconds of + `T` (`conformance.temporal.perturb_future`). Histories at every indexed time, including the row order + of ties, are unchanged, so probabilities MUST be bit-identical by key. Any path from the future to a + prediction fails this check, even one the fitted model barely uses; no statistical threshold is + involved. +- **Temporal cutoff (null world):** for non-zero-shot profiles, the full lifecycle runs on + `conformance.temporal.build_null_world_dataset`, where events follow one homogeneous Poisson process and + the label ("any event in `(T, T + ln 2 / rate]`") is independent of everything recorded up to `T`. A + one-sided Mann-Whitney test on one key per held-out subject MUST NOT reject chance at `alpha = 1e-3`. + This is a smoke alarm for leaks that training learned to exploit; the perturbation check is the gate. + `run_e2e` MUST accept this dataset (`dataset.json` names it `null_world`) with its test settings. - **Prediction validity:** validate every Parquet shard in both runs, including unique/non-null keys and finite probabilities in [0, 1]. No particular shard filename is required. - **Transductive fitting:** every tuning and held-out subject carries a canary code that occurs in no @@ -566,6 +579,7 @@ payloads. Implement these algorithm-specific assertions rather than relying only The shared `assert_profile_run` conformance helper reruns the full lifecycle through `run_e2e` with forbidden label columns removed. It requires fresh artifacts below a new run directory and preserves permitted training/context labels. Include `seed`, `external_meds_dir` and all task inputs in `E2EResult.inputs` -so the check uses the same histories and task. This detects forbidden reads on the exercised path; -it does not prove all possible code paths, temporal cutoffs, or absence of labels encoded in other -features. Implement the corresponding algorithm-specific checks as well. +so the check uses the same histories and task. The same helper reruns the lifecycle with every evaluated +subject's future rewritten (`assert_future_blind_pipeline`). These detect forbidden reads and future +information on the exercised path; they do not prove all possible code paths or absence of labels encoded +in other features. Implement the corresponding algorithm-specific checks as well. diff --git a/src/meds_algorithm/TESTING.md b/src/meds_algorithm/TESTING.md index 4332cfa..1bcc492 100644 --- a/src/meds_algorithm/TESTING.md +++ b/src/meds_algorithm/TESTING.md @@ -24,7 +24,7 @@ local processes and temporary files. The default suite does not opt in to MEDS-D Passing establishes the checked software boundaries, not clinical validity, numerical equivalence to a paper, GPU correctness or absence of all leakage. Algorithm repositories must supply scientific tests, cutoff/frozen-weight checks and end-to-end adapters for their concrete implementations. The label-blind -rerun is an adversarial check, not a filesystem sandbox or proof against malicious code. +and future-blind reruns are adversarial checks, not a filesystem sandbox or proof against malicious code. ## Downstream acceptance suite @@ -49,7 +49,8 @@ in the generated `tests/e2e.py` and the shared `E2EResult`/`MEDSDevRun` types. |---|---|---| | `test_profile_implementations_match_contract` | Compares each registered profile's commands with the core lifecycle. | Detect incorrect registration or dispatch wiring. | | `test_algorithm_configuration_roundtrips` | Saves and reloads each supplied concrete configuration. | Preserve settings across independent processes, including required constructor fields. | -| `test_end_to_end` | Runs each implemented profile and checks artifacts, manifests, coverage, fresh label-blind reruns, split canaries and zero-shot task binding. | Detect broken persistence, label leakage and transductive fitting through the real lifecycle. | +| `test_end_to_end` | Runs each implemented profile and checks artifacts, manifests, coverage, fresh label-blind reruns, a fresh rerun with every evaluated subject's events after its latest prediction time rewritten (predictions must be bit-identical), split canaries and zero-shot task binding. | Detect broken persistence, label leakage, information from after the prediction time and transductive fitting through the real lifecycle. | +| `test_null_world_predictions_are_at_chance` | Runs each completed non-zero-shot profile on a null world (homogeneous Poisson events; the label, any event in `(T, T + ln 2 / rate]`, is independent of the history) and applies a one-sided Mann-Whitney test, one key per held-out subject, at `alpha = 1e-3`. | End-to-end smoke alarm for temporal leaks that training learned to exploit, such as a forward time gap to the next real event. `run_e2e` must accept the `null_world` dataset with its test settings. | | `test_completed_algorithm_has_a_completed_report` | Validates report evidence for every completed profile. | Prevent incomplete profiles from masking unsupported completion claims. | | `test_algorithm_yaml_is_a_meds_dev_descriptor` | Checks each profile descriptor's required structure. | Catch packaging errors before integration. | | `test_slurm_dry_run_is_non_mutating` | Runs every profile's launcher with example inputs, checks the file inventory is unchanged and all commands are listed. | Inspect execution plans without submitting work; this checks file creation/removal, not changes to existing file contents. | @@ -123,6 +124,19 @@ canonical and is rendered in the template documentation rather than copied into | `test_nn_config_requires_explicit_training_settings` | Rejects missing learning rate/batch size and checks classical config independence. | Do not invent universal training defaults or require neural settings for classical algorithms. | | `test_repository_records_nn_settings_from_config_file` | Dispatches with a file override and checks resolved saved config and manifest. | Ensure experiment settings actually reach the algorithm and persisted provenance. | +### test_temporal.py + +[Source](https://github.com/McDermottHealthAI/meds-algorithm/blob/main/tests/test_temporal.py) + +| Test | What it checks | Why it exists | +|---|---|---| +| `test_null_world_shape` | Builds a null world and checks one key per held-out subject, two per training subject, prevalence near 0.5 and records that run past the label horizon. | Keep held-out keys independent and record ends uninformative, so the chance-level test is exact. | +| `test_causal_scores_are_at_chance_and_leaky_scores_are_caught` | Scores keys by hours since the last event (causal) and by hours to the next event (a NOAH-style forward-gap leak). | Show that the chance-level check passes a causal score and fails a leaky one. | +| `test_perturbation_keeps_history_and_rewrites_only_indexed_futures` | Compares histories up to every indexed time, the rewritten future, the vocabulary and untouched training subjects. | Guarantee the future-blind rerun changes nothing an honest algorithm may read. | +| `test_future_blind_rerun_passes_causal_and_fails_leaky` | Runs the future-blind rerun on the causal and the leaky toy algorithm. | Show that any dependence on the future fails without a statistical threshold. | +| `test_mann_whitney_is_calibrated_and_handles_ties` | Checks the rejection rate under the null with heavy ties, constant scores, perfect separation and one-class input. | Keep the chance-level test's false-positive rate at `alpha`. | +| `test_perturbation_keeps_tie_order_and_never_copies_objects` | Rewrites a future next to ties at one time and an object-link row. | MEDS orders ties by row and objects are single acquisitions; reordering or duplicating either would fail honest algorithms. | + ### test_conformance.py [Source](https://github.com/McDermottHealthAI/meds-algorithm/blob/main/tests/test_conformance.py) diff --git a/src/meds_algorithm/conformance/end_to_end.py b/src/meds_algorithm/conformance/end_to_end.py index 5479c79..0e70665 100644 --- a/src/meds_algorithm/conformance/end_to_end.py +++ b/src/meds_algorithm/conformance/end_to_end.py @@ -14,6 +14,7 @@ ) from .assertions import assert_prediction_keys, read_predictions +from .temporal import assert_future_blind_pipeline, assert_same_predictions def _strip_labels(labels_dir: Path, destination: Path, splits) -> Path: @@ -64,6 +65,9 @@ def assert_profile_run(result, *, profile, run_e2e, fitted_codes, tmp_path): assert_prediction_keys(artifacts[prediction_step], result.labels_dir, result.splits) assert_label_blind_pipeline(result, profile=profile, run_e2e=run_e2e, tmp_path=tmp_path) + # Temporal cutoff (CONTRACT section 8): rewriting every evaluated subject's events after its latest + # prediction time must leave every prediction unchanged. + assert_future_blind_pipeline(result, profile=profile, run_e2e=run_e2e, tmp_path=tmp_path) # Absence of transductive fitting (SPEC 2.1): codes that occur only in tuning or held-out subjects must # not be part of anything the algorithm fitted. @@ -124,31 +128,14 @@ def assert_label_blind_pipeline(result, *, profile, run_e2e, tmp_path): prediction = blind.artifacts[expected_steps[-1]] assert_prediction_keys(prediction, result.labels_dir, result.splits) # Do not require the evaluation-only boolean_value on a deliberately unlabelled run. - import polars as pl - from ..data.index import read_index from ..evaluation.predictions import check_decoded blind_predictions = read_predictions(prediction, require_truth=False) check_decoded(read_index(result.labels_dir, result.splits), blind_predictions) - keys = ["subject_id", "prediction_time"] - probability = "predicted_boolean_probability" - decisions = "predicted_boolean_value" in original.columns - columns = [*keys, probability, *(["predicted_boolean_value"] if decisions else [])] - # Explicit comparison instead of polars.testing.assert_frame_equal, whose tolerance keywords were - # renamed (rtol/atol -> rel_tol/abs_tol) within the supported polars>=1 range. - try: - a = original.select(columns).sort(keys) - b = blind_predictions.select(columns).sort(keys) - except pl.exceptions.ColumnNotFoundError as error: - raise AssertionError("Label-blind predictions lack a column of the original run") from error - same = ( - a.select(keys).equals(b.select(keys)) - and ((a[probability] - b[probability]).abs() <= 1e-8 + 1e-6 * b[probability].abs()).all() - and (not decisions or a["predicted_boolean_value"].equals(b["predicted_boolean_value"])) + assert_same_predictions( + original, + blind_predictions, + message="Predictions changed when forbidden labels were removed; fix leakage or make the " + "conformance fixture reproducible with the recorded seed", ) - if not same: - raise AssertionError( - "Predictions changed when forbidden labels were removed; fix leakage or make the " - "conformance fixture reproducible with the recorded seed" - ) diff --git a/src/meds_algorithm/conformance/suite.py b/src/meds_algorithm/conformance/suite.py index 23ef5aa..3f8fcb3 100644 --- a/src/meds_algorithm/conformance/suite.py +++ b/src/meds_algorithm/conformance/suite.py @@ -19,10 +19,11 @@ from .cases import AlgorithmCase from .end_to_end import assert_profile_run from .reports import assert_implementation_report +from .temporal import assert_chance_level, build_null_world_dataset class AlgorithmConformanceTests: - """Eight shared acceptance checks, executed against the downstream algorithm.""" + """Nine shared acceptance checks, executed against the downstream algorithm.""" @pytest.fixture def completed_profile(self, algorithm_case: AlgorithmCase, profile): @@ -61,6 +62,23 @@ def test_end_to_end(self, algorithm_case: AlgorithmCase, completed_profile, tmp_ tmp_path=tmp_path, ) + def test_null_world_predictions_are_at_chance( + self, algorithm_case: AlgorithmCase, completed_profile, tmp_path + ): + """Temporal cutoff: with a future independent of the past, held-out AUROC must be at chance.""" + if completed_profile.startswith("zero_shot_"): + pytest.skip("null world: zero-shot profiles need a task; the future-perturbation rerun covers it") + world = build_null_world_dataset(tmp_path / "null-world", seed=0) + result = algorithm_case.run_e2e( + tmp_path / "null-world-run", + profile=completed_profile, + seed=0, + external_meds_dir=world.meds_dir, + external_labels_dir=world.labels_dir, + splits=["held_out"], + ) + assert_chance_level(result, alpha=1e-3) + def test_completed_algorithm_has_a_completed_report(self, algorithm_case: AlgorithmCase): report = (algorithm_case.root / "IMPLEMENTATION_REPORT.md").read_text(encoding="utf-8") assert_implementation_report(report, algorithm_case.profiles, algorithm_case.stub_profiles) diff --git a/src/meds_algorithm/conformance/temporal.py b/src/meds_algorithm/conformance/temporal.py new file mode 100644 index 0000000..be8d21d --- /dev/null +++ b/src/meds_algorithm/conformance/temporal.py @@ -0,0 +1,343 @@ +"""Temporal-leakage checks: nothing recorded after a prediction time may reach that time's prediction. + +Two complementary, algorithm-agnostic checks run through the algorithm's own ``run_e2e`` adapter: + +* **Future perturbation (deterministic).** :func:`perturb_future` copies a MEDS dataset, deletes every event + after each evaluated subject's latest prediction time and adds random events there instead. Inputs at every + indexed prediction time are unchanged, so a leak-free pipeline must reproduce its predictions exactly + (:func:`assert_future_blind_pipeline`). This catches any path from the future to a prediction, including a + path the fitted model barely uses, and needs no statistical threshold. +* **Null world (statistical).** :func:`build_null_world_dataset` writes a dataset whose future is + independent of its past: every subject's events follow one homogeneous Poisson process (the same rate for + everyone), with codes and values drawn independently, and records continue well past every prediction + time. The label, "does any event occur in (T, T + horizon]", depends only on the future, so held-out + discrimination must be at chance. :func:`assert_chance_level` applies a one-sided Mann-Whitney test with + one key per held-out subject, so its false-positive rate is exactly ``alpha``. A leak only shows up here + if training learned to use it; read this check as an end-to-end smoke alarm and the perturbation check as + the gate. +""" + +from __future__ import annotations + +import json +import math +import random +import shutil +from dataclasses import dataclass +from datetime import datetime, timedelta +from pathlib import Path + +US_PER_HOUR = 3_600_000_000 +NULL_WORLD_DATASET = "null_world" +# (code, share of events, numeric mean, numeric sd); a None mean gives an event without a value. +NULL_WORLD_CODES: tuple[tuple[str, float, float | None, float | None], ...] = ( + ("NULL//MEASUREMENT//A", 0.30, 80.0, 12.0), + ("NULL//MEASUREMENT//B", 0.20, 96.0, 2.0), + ("NULL//MEASUREMENT//C", 0.20, 1.0, 0.3), + ("NULL//EVENT//D", 0.15, None, None), + ("NULL//EVENT//E", 0.15, None, None), +) + + +@dataclass(frozen=True) +class NullWorld: + """Paths of a written null-world dataset and the label it defines.""" + + meds_dir: Path + labels_dir: Path + horizon: timedelta + rate_per_hour: float + + +def build_null_world_dataset( + root: Path, + *, + n_train: int = 400, + n_tuning: int = 100, + n_held_out: int = 1000, + rate_per_hour: float = 0.5, + keys_per_train_subject: int = 2, + history_hours: tuple[float, float] = (12.0, 36.0), + tail_hours: float = 48.0, + codes: tuple[tuple[str, float, float | None, float | None], ...] = NULL_WORLD_CODES, + dataset_name: str = NULL_WORLD_DATASET, + seed: int = 0, +) -> NullWorld: + """Write ``root/meds`` (MEDS 0.4) and ``root/labels`` for a world in which the future is unpredictable. + + Each subject has a ``MEDS_BIRTH`` event, a static sex code and Poisson events at ``rate_per_hour``. + Prediction times are drawn uniformly in ``history_hours`` after the first possible event, independently + of the events, and every record runs ``tail_hours`` past its last prediction time, so record ends carry + no information. The label is whether any event occurs in ``(T, T + horizon]`` with + ``horizon = ln 2 / rate`` (prevalence 0.5 in expectation). Training subjects contribute + ``keys_per_train_subject`` keys; tuning and held-out subjects one each, so held-out keys are independent. + Pass ``codes`` when an algorithm needs a fixed vocabulary. + """ + import polars as pl + + if not codes or abs(sum(share for _, share, *_ in codes) - 1) > 1e-9: + raise ValueError("codes must be non-empty and their shares must sum to 1") + root = Path(root) + meds_dir, labels_dir = root / "meds", root / "labels" + rng = random.Random(seed) + names = [code for code, *_ in codes] + shares = [share for _, share, *_ in codes] + horizon_us = int(math.log(2) / rate_per_hour * US_PER_HOUR) + epoch = datetime(2100, 1, 1) + event_schema = { + "subject_id": pl.Int64, + "time": pl.Datetime("us"), + "code": pl.String, + "numeric_value": pl.Float32, + } + label_schema = {"subject_id": pl.Int64, "prediction_time": pl.Datetime("us"), "boolean_value": pl.Boolean} + + def at(microseconds: int) -> datetime: + return epoch + timedelta(microseconds=microseconds) + + subject_id, subject_splits = 0, [] + labels_dir.mkdir(parents=True, exist_ok=True) + for split, size in (("train", n_train), ("tuning", n_tuning), ("held_out", n_held_out)): + events, labels = [], [] + for _ in range(size): + subject_id += 1 + start = rng.randrange(3 * 365 * 24) * US_PER_HOUR + n_keys = keys_per_train_subject if split == "train" else 1 + offsets = sorted(rng.uniform(*history_hours) for _ in range(n_keys)) + end_hours = offsets[-1] + tail_hours + times, t = [], rng.expovariate(rate_per_hour) + while t < end_hours: + times.append(start + int(t * US_PER_HOUR)) + t += rng.expovariate(rate_per_hour) + birth = start - int(rng.uniform(20, 90) * 365.25 * 24 * US_PER_HOUR) + events.append((subject_id, at(birth), "MEDS_BIRTH", None)) + events.append((subject_id, None, f"SEX//{rng.choice('FM')}", None)) + for time in times: + code, _, mean, sd = codes[rng.choices(range(len(codes)), weights=shares)[0]] + events.append((subject_id, at(time), code, None if mean is None else rng.gauss(mean, sd))) + for offset in offsets: + prediction = start + int(offset * US_PER_HOUR) + label = any(prediction < time <= prediction + horizon_us for time in times) + labels.append((subject_id, at(prediction), label)) + subject_splits.append((subject_id, split)) + (meds_dir / "data" / split).mkdir(parents=True, exist_ok=True) + pl.DataFrame(events, schema=event_schema, orient="row").sort( + "subject_id", "time", nulls_last=False + ).write_parquet(meds_dir / "data" / split / "0.parquet") + pl.DataFrame(labels, schema=label_schema, orient="row").write_parquet(labels_dir / f"{split}.parquet") + metadata = meds_dir / "metadata" + metadata.mkdir(parents=True, exist_ok=True) + pl.DataFrame({"code": [*names, "MEDS_BIRTH", "SEX//F", "SEX//M"]}).write_parquet( + metadata / "codes.parquet" + ) + pl.DataFrame( + subject_splits, schema={"subject_id": pl.Int64, "split": pl.String}, orient="row" + ).write_parquet(metadata / "subject_splits.parquet") + (metadata / "dataset.json").write_text(json.dumps({"dataset_name": dataset_name, "meds_version": "0.4"})) + return NullWorld(meds_dir, labels_dir, timedelta(microseconds=horizon_us), rate_per_hour) + + +def perturb_future( + meds_dir: Path, + index_dir: Path, + destination: Path, + *, + splits=None, + events_per_subject: int = 8, + span: timedelta = timedelta(hours=48), + seed: int = 0, +) -> Path: + """Copy ``meds_dir`` to ``destination`` and rewrite every indexed subject's future. + + For each subject in the index (restricted to ``splits``), every timed event strictly after the subject's + latest indexed prediction time is deleted and ``events_per_subject`` new events are added at + log-uniformly random offsets in ``(T, T + span]`` (from one microsecond on, so leaks that look only a + little past ``T`` are exercised). New events copy the code, numeric value and text value of random + timed rows of the same shard, so they use the split's own vocabulary. Rows carrying any other column + (object links and other additional value modalities) are never copied: each object is one acquisition, + and linking it a second time would fabricate an impossible record. Events at or before every indexed + time are untouched and keep their row order (MEDS ties at one time are ordered by row), so every + indexed prediction must stay exactly the same. Training subjects outside the index are not changed. + """ + import polars as pl + + from ..data.index import read_index + + destination = Path(destination) + shutil.copytree(meds_dir, destination) + latest = ( + read_index(index_dir, splits).group_by("subject_id").agg(pl.col("prediction_time").max().alias("_T")) + ) + rng = random.Random(seed) + for shard in sorted((destination / "data").rglob("*.parquet")): + events = pl.read_parquet(shard) + # Joins do not promise row order; the row index restores it, since tie order at one time matters. + tagged = events.with_row_index("_row").join(latest, on="subject_id", how="left").sort("_row") + future = pl.col("_T").is_not_null() & pl.col("time").is_not_null() & (pl.col("time") > pl.col("_T")) + subjects = ( + tagged.filter(pl.col("_T").is_not_null()).select("subject_id", "_T").unique().sort("subject_id") + ) + plain = [ + c + for c in events.columns + if c not in ("subject_id", "time", "code", "numeric_value", "text_value") + ] + pool = events.filter( + pl.col("time").is_not_null() + & ~pl.col("code").str.starts_with("MEDS_") + & pl.all_horizontal([pl.col(c).is_null() for c in plain] or [pl.lit(True)]) + ) + if subjects.is_empty(): + continue + kept = tagged.filter(~future).select(events.columns) + n = subjects.height * events_per_subject if pool.height else 0 + span_us = int(span.total_seconds() * 1_000_000) + # Log-uniform: many events land just after T (where binning or rounding leaks peek), some far later. + offsets = [max(1, int(math.exp(rng.uniform(0, math.log(span_us))))) for _ in range(n)] + new = ( + pool.sample(n, with_replacement=True, seed=rng.randrange(2**32)) + .with_columns( + pl.Series( + "subject_id", + subjects["subject_id"].to_list() * events_per_subject, + dtype=events.schema["subject_id"], + ), + ( + pl.Series("_T", subjects["_T"].to_list() * events_per_subject).cast(pl.Datetime("us")) + + pl.duration(microseconds=pl.Series(offsets)) + ) + .cast(events.schema["time"]) + .alias("time"), + ) + .select(events.columns) + ) + # Stable: rows at or before T keep their order; new rows sit at unique times after T. + pl.concat([kept, new]).sort( + "subject_id", "time", nulls_last=False, maintain_order=True + ).write_parquet(shard) + return destination + + +def null_auroc_test(scores, labels) -> tuple[float, float]: + """AUROC and its one-sided Mann-Whitney p-value against chance (ties averaged, continuity corrected). + + Valid when keys are independent and the scores were fixed before ``labels`` were attached. + """ + import polars as pl + + frame = pl.DataFrame({"score": list(scores), "label": [bool(v) for v in labels]}) + positives = int(frame["label"].sum()) + negatives = frame.height - positives + if not positives or not negatives: + raise ValueError("the null-world held-out split needs both positive and negative keys") + frame = frame.with_columns(pl.col("score").rank("average").alias("rank")) + u = float(frame.filter(pl.col("label"))["rank"].sum()) - positives * (positives + 1) / 2 + n = frame.height + ties = frame.group_by("score").len()["len"].cast(pl.Float64) + variance = positives * negatives / 12 * ((n + 1) - float((ties**3 - ties).sum()) / (n * (n - 1))) + auroc = u / (positives * negatives) + if variance <= 0: + return auroc, 1.0 + z = (u - positives * negatives / 2 - 0.5) / math.sqrt(variance) + return auroc, 0.5 * math.erfc(z / math.sqrt(2)) + + +def assert_chance_level(result, *, alpha: float = 1e-3) -> tuple[float, float]: + """Fail when held-out predictions on a null world discriminate the future better than chance.""" + import polars as pl + + from meds_algorithm import split_files + + from .assertions import read_predictions + + step = list(result.artifacts)[-1] + assert step.startswith("predict"), f"the last artifact must hold predictions, got {step}" + keys = ["subject_id", "prediction_time"] + predictions = read_predictions(result.artifacts[step], require_truth=False).select( + *keys, "predicted_boolean_probability" + ) + truth = pl.concat( + pl.read_parquet(p, columns=[*keys, "boolean_value"]) + for p in split_files(result.labels_dir, result.splits) + ) + joined = truth.join(predictions, on=keys, how="inner") + assert joined.height == truth.height, "every null-world key needs a prediction" + assert joined["subject_id"].n_unique() == joined.height, "the null-world test needs one key per subject" + auroc, p_value = null_auroc_test(joined["predicted_boolean_probability"], joined["boolean_value"]) + assert p_value >= alpha, ( + f"Held-out AUROC {auroc:.3f} on a null world where the future is independent of the past " + f"(one-sided p = {p_value:.2e} < {alpha}). Information from after the prediction time reaches the " + "prediction; run the future-perturbation check to locate it." + ) + return auroc, p_value + + +def assert_same_predictions(original, other, *, message: str, rtol: float = 1e-6, atol: float = 1e-8) -> None: + """Keyed equality of probabilities (and decisions, when present) within ``atol + rtol * |p|``.""" + import polars as pl + + keys = ["subject_id", "prediction_time"] + probability = "predicted_boolean_probability" + decisions = "predicted_boolean_value" in original.columns + columns = [*keys, probability, *(["predicted_boolean_value"] if decisions else [])] + # Explicit comparison instead of polars.testing.assert_frame_equal, whose tolerance keywords were + # renamed (rtol/atol -> rel_tol/abs_tol) within the supported polars>=1 range. + try: + a = original.select(columns).sort(keys) + b = other.select(columns).sort(keys) + except pl.exceptions.ColumnNotFoundError as error: + raise AssertionError(f"{message}: predictions lack a column of the original run") from error + if not a.select(keys).equals(b.select(keys)): + raise AssertionError(f"{message}: prediction keys differ") + moved = (a[probability] - b[probability]).abs() > atol + rtol * b[probability].abs() + if moved.any() or (decisions and not a["predicted_boolean_value"].equals(b["predicted_boolean_value"])): + raise AssertionError( + f"{message} ({int(moved.sum())} of {a.height} keys moved, max |change| " + f"{float((a[probability] - b[probability]).abs().max()):.3g})" + ) + + +def assert_future_blind_pipeline(result, *, profile, run_e2e, tmp_path, tolerance: float = 0.0) -> None: + """Rerun every stage with each evaluated subject's future rewritten; predictions must not move. + + Training data, seeds and the inputs at every indexed time are identical in both runs, so a deterministic + pipeline reproduces its predictions bit for bit; the default ``tolerance`` is therefore zero. A small + leak through a briefly trained conformance model can move a probability by only ~1e-7, which a float + tolerance would hide. Pass a relative ``tolerance`` only for a pipeline that is not bitwise reproducible. + """ + from meds_algorithm import ALGORITHM_PROFILES + + from .assertions import assert_prediction_keys, read_predictions + + assert result.inputs.get("external_meds_dir"), "E2EResult.inputs must record external_meds_dir" + assert type(result.inputs.get("seed")) is int, "E2EResult.inputs must record the run_e2e seed" + steps = [step.value for step in ALGORITHM_PROFILES[profile]] + index = result.inputs.get("external_index_dir") or result.labels_dir + perturbed = perturb_future( + result.inputs["external_meds_dir"], + index, + Path(tmp_path) / "future-perturbed-meds", + splits=result.splits, + seed=result.inputs["seed"] + 1, + ) + fresh_root = Path(tmp_path) / "future-blind-run" + rerun = run_e2e( + fresh_root, + profile=profile, + **{**result.inputs, "external_meds_dir": perturbed, "splits": result.splits}, + ) + assert list(rerun.artifacts) == steps, "future-blind check must execute the full lifecycle" + for path in rerun.artifacts.values(): + assert Path(path).resolve().is_relative_to(fresh_root.resolve()), ( + "future-blind check must publish fresh outputs inside its run directory" + ) + prediction = rerun.artifacts[steps[-1]] + assert_prediction_keys(prediction, result.labels_dir, result.splits) + assert_same_predictions( + read_predictions(result.artifacts[steps[-1]], require_truth=False), + read_predictions(prediction, require_truth=False), + rtol=tolerance, + atol=0.0, + message="Predictions changed when only events after every indexed prediction time were rewritten; " + "information from after the prediction time reaches the prediction", + ) diff --git a/tests/test_suite.py b/tests/test_suite.py index fd42050..9ad0de7 100644 --- a/tests/test_suite.py +++ b/tests/test_suite.py @@ -77,10 +77,11 @@ def test_inherited_suite_collects_every_profile_without_duplicate_base(downstrea result = downstream.runpytest_subprocess("--collect-only", "-q", "--strict-markers") result.assert_outcomes() # collection succeeds without executing adapters output = result.stdout.str() - assert "38 tests collected" in output + assert "44 tests collected" in output for profile in ALGORITHM_PROFILES: for test in ( "test_end_to_end", + "test_null_world_predictions_are_at_chance", "test_meds_dev_runs_algorithm_yaml", "test_full_dag_on_mimic_iv_demo", ): @@ -92,7 +93,7 @@ def test_stubs_skip_execution_but_contract_and_configs_still_run(downstream): result = downstream.runpytest_subprocess( "-q", "-k", "end_to_end or profile_implementations or configuration_roundtrips" ) - result.assert_outcomes(passed=7, skipped=6, deselected=25) + result.assert_outcomes(passed=7, skipped=6, deselected=31) @pytest.mark.parametrize("selection", [[], ["-m", "not slow"], ["-m", "slow"]]) @@ -101,9 +102,9 @@ def test_completed_integrations_need_explicit_opt_in(downstream, selection): conftest.write_text(conftest.read_text().replace("stubs=ALGORITHM_PROFILES", "stubs=()")) result = downstream.runpytest_subprocess("-q", "-k", "meds_dev_runs or full_dag", *selection) if selection == ["-m", "not slow"]: - result.assert_outcomes(deselected=38) + result.assert_outcomes(deselected=44) else: - result.assert_outcomes(skipped=12, deselected=26) + result.assert_outcomes(skipped=12, deselected=32) def test_integration_markers_select_only_their_suite(downstream): @@ -114,7 +115,7 @@ def test_integration_markers_select_only_their_suite(downstream): result = downstream.runpytest_subprocess("--collect-only", "-q", "-m", marker) assert expected in result.stdout.str() assert excluded not in result.stdout.str() - assert "6/38 tests collected" in result.stdout.str() + assert "6/44 tests collected" in result.stdout.str() def test_mixed_completion_does_not_hide_report_requirements(tmp_path): diff --git a/tests/test_temporal.py b/tests/test_temporal.py new file mode 100644 index 0000000..5d742c9 --- /dev/null +++ b/tests/test_temporal.py @@ -0,0 +1,209 @@ +"""Temporal-leakage conformance: null world, future perturbation and the chance-level test.""" + +import math +import random +from pathlib import Path + +import polars as pl +import pytest + +from meds_algorithm import ( + ALGORITHM_PROFILES, + STEP_SPECS, + attach_ground_truth, + digest_config, + measured_artifact, + read_index, +) +from meds_algorithm.conformance.temporal import ( + NULL_WORLD_DATASET, + assert_chance_level, + assert_future_blind_pipeline, + build_null_world_dataset, + null_auroc_test, + perturb_future, +) +from meds_algorithm.conformance.types import E2EResult + + +@pytest.fixture(scope="module") +def world(tmp_path_factory): + return build_null_world_dataset( + tmp_path_factory.mktemp("null") / "world", n_train=60, n_tuning=20, n_held_out=400, seed=3 + ) + + +def _events(meds_dir, split="held_out"): + return pl.concat(pl.read_parquet(p) for p in sorted((Path(meds_dir) / "data" / split).glob("*.parquet"))) + + +def _keyed_gap(meds_dir, index, *, future: bool): + """Hours since the last event at or before T (causal) or until the first event after T (leaky).""" + events = _events(meds_dir).filter(pl.col("time").is_not_null() & (pl.col("code") != "MEDS_BIRTH")) + joined = index.join(events.select("subject_id", "time"), on="subject_id", how="left") + if future: + picked = ( + joined.filter(pl.col("time") > pl.col("prediction_time")) + .group_by("subject_id", "prediction_time") + .agg(pl.col("time").min()) + ) + gap = pl.col("time") - pl.col("prediction_time") + else: + picked = ( + joined.filter(pl.col("time") <= pl.col("prediction_time")) + .group_by("subject_id", "prediction_time") + .agg(pl.col("time").max()) + ) + gap = pl.col("prediction_time") - pl.col("time") + hours = picked.with_columns((gap.dt.total_microseconds() / 3.6e9).alias("hours")) + return index.join( + hours.select("subject_id", "prediction_time", "hours"), + on=["subject_id", "prediction_time"], + how="left", + ).with_columns(pl.col("hours").fill_null(1e3)) + + +def toy_pipeline(*, leaky: bool): + """A supervised 'algorithm' whose score is a time gap: causal, or the NOAH-style gap to the next event.""" + + def run(path, *, profile, **inputs): + inputs = {"seed": 0, **inputs} + index_dir = inputs.get("external_index_dir") or inputs["external_labels_dir"] + splits = list(inputs.get("splits") or ["held_out"]) + artifacts = {} + for step in ALGORITHM_PROFILES[profile]: + dest = Path(path) / step.value + with measured_artifact( + dest, + artifact_type=STEP_SPECS[step].artifact_type.value, + command=step.value, + producer="toy", + payload_format="test/v1", + config_digest=digest_config({}), + ) as out: + if step == ALGORITHM_PROFILES[profile][-1]: + index = read_index(index_dir, splits) + gap = _keyed_gap(inputs["external_meds_dir"], index, future=leaky) + # Short gap to the next event -> high score (a leaky model would learn exactly this). + scored = gap.with_columns( + (1 / (1 + pl.col("hours"))).alias("predicted_boolean_probability") + ) + attach_ground_truth( + scored.drop("hours"), inputs.get("external_evaluation_labels_dir", index_dir), splits + ).write_parquet(out / "predictions.parquet") + artifacts[step.value] = dest + return E2EResult(artifacts, Path(index_dir), splits, inputs, canaries={}) + + return run + + +def _run(world, tmp_path, *, leaky): + return toy_pipeline(leaky=leaky)( + tmp_path / "run", + profile="supervised", + external_meds_dir=world.meds_dir, + external_labels_dir=world.labels_dir, + splits=["held_out"], + ) + + +def test_null_world_shape(world): + held_out = pl.read_parquet(world.labels_dir / "held_out.parquet") + train = pl.read_parquet(world.labels_dir / "train.parquet") + assert held_out["subject_id"].n_unique() == held_out.height == 400 + assert train.height == 2 * 60 + assert 0.4 < held_out["boolean_value"].mean() < 0.6 + assert '"dataset_name": "null_world"' in (world.meds_dir / "metadata" / "dataset.json").read_text() + assert NULL_WORLD_DATASET == "null_world" + events = _events(world.meds_dir) + last_t = held_out.group_by("subject_id").agg(pl.col("prediction_time").max()) + # Every record continues well past its prediction time, so the end of a record carries no information. + end = events.group_by("subject_id").agg(pl.col("time").max()).join(last_t, on="subject_id") + assert (end["time"] > end["prediction_time"] + world.horizon).mean() > 0.95 + + +def test_causal_scores_are_at_chance_and_leaky_scores_are_caught(world, tmp_path): + auroc, p = assert_chance_level(_run(world, tmp_path / "causal", leaky=False)) + assert abs(auroc - 0.5) < 0.1 and p >= 1e-3 + with pytest.raises(AssertionError, match="null world"): + assert_chance_level(_run(world, tmp_path / "leaky", leaky=True)) + + +def test_perturbation_keeps_history_and_rewrites_only_indexed_futures(world, tmp_path): + index = read_index(world.labels_dir, ["held_out"]) + perturbed = perturb_future(world.meds_dir, world.labels_dir, tmp_path / "perturbed", splits=["held_out"]) + before, after = _events(world.meds_dir), _events(perturbed) + cut = pl.col("time").is_null() | (pl.col("time") <= pl.col("prediction_time")) + past = [ + frame.join(index, on="subject_id") + .filter(cut) + .drop("prediction_time") + .sort("subject_id", "time", "code", nulls_last=False) + for frame in (before, after) + ] + assert past[0].equals(past[1]) + assert not before.equals(after) + assert set(after["code"].unique()) <= set(before["code"].unique()) + # Subjects outside the index (training) are untouched. + assert _events(world.meds_dir, "train").equals(_events(perturbed, "train")) + + +def test_future_blind_rerun_passes_causal_and_fails_leaky(world, tmp_path): + causal = toy_pipeline(leaky=False) + result = _run(world, tmp_path / "causal", leaky=False) + assert_future_blind_pipeline(result, profile="supervised", run_e2e=causal, tmp_path=tmp_path / "c") + leaky = toy_pipeline(leaky=True) + result = _run(world, tmp_path / "leaky", leaky=True) + with pytest.raises(AssertionError, match="after every indexed prediction time"): + assert_future_blind_pipeline(result, profile="supervised", run_e2e=leaky, tmp_path=tmp_path / "l") + + +def test_mann_whitney_is_calibrated_and_handles_ties(): + rng = random.Random(0) + rejections = 0 + for _ in range(400): + labels = [rng.random() < 0.5 for _ in range(200)] + scores = [round(rng.random(), 1) for _ in labels] # heavy ties + rejections += null_auroc_test(scores, labels)[1] < 0.05 + assert 0.02 < rejections / 400 < 0.08 + assert null_auroc_test([0.5] * 10, [True, False] * 5) == (0.5, 1.0) + auroc, p = null_auroc_test(list(range(100)), [i >= 50 for i in range(100)]) + assert auroc == 1.0 and p < 1e-12 + with pytest.raises(ValueError, match="both positive and negative"): + null_auroc_test([0.1, 0.2], [True, True]) + assert math.isclose(null_auroc_test([0.2, 0.8, 0.4, 0.6], [False, True, True, False])[0], 0.75) + + +def test_perturbation_keeps_tie_order_and_never_copies_objects(tmp_path): + from datetime import datetime, timedelta + + t0 = datetime(2020, 1, 1) + # Ties at one time, in a deliberately non-alphabetical row order, plus an object-link row. + rows = [(1, t0, c, None, None) for c in ("Z//LAST", "A//FIRST", "M//MIDDLE")] + rows += [ + (1, t0 + timedelta(hours=1), "ECG//WAVEFORM", None, "ecg/1"), + (1, t0 + timedelta(hours=9), "X", 1.0, None), + ] + schema = { + "subject_id": pl.Int64, + "time": pl.Datetime("us"), + "code": pl.String, + "numeric_value": pl.Float32, + "ecg_link": pl.String, + } + meds, labels = tmp_path / "meds", tmp_path / "labels" + (meds / "data" / "held_out").mkdir(parents=True) + pl.DataFrame(rows, schema=schema, orient="row").write_parquet(meds / "data" / "held_out" / "0.parquet") + labels.mkdir() + pl.DataFrame( + {"subject_id": [1], "prediction_time": [t0 + timedelta(hours=2)], "boolean_value": [True]}, + schema={"subject_id": pl.Int64, "prediction_time": pl.Datetime("us"), "boolean_value": pl.Boolean}, + ).write_parquet(labels / "held_out.parquet") + for seed in range(5): + out = perturb_future(meds, labels, tmp_path / f"p{seed}", splits=["held_out"], seed=seed) + after = pl.read_parquet(out / "data" / "held_out" / "0.parquet") + assert after["code"].head(4).to_list() == ["Z//LAST", "A//FIRST", "M//MIDDLE", "ECG//WAVEFORM"] + assert after.filter(pl.col("ecg_link").is_not_null()).height == 1 + future = after.slice(4) # the old future (X) is gone; 8 new plain rows after T replace it + assert future.height == 8 and (future["time"] > t0 + timedelta(hours=2)).all() + assert set(future["code"]) <= {"Z//LAST", "A//FIRST", "M//MIDDLE", "X"}