From 42548b35bec0808c96c8901b823cb60badd58d23 Mon Sep 17 00:00:00 2001 From: Jammy2211 Date: Fri, 23 Oct 2026 16:11:44 +0100 Subject: [PATCH] Report a non-finite EP evidence term by name instead of warn-and-sum (#1713) On analytic_gaussian seeds 25 and 172 a HierarchicalFactor member sends the scatter sigma an accepted TruncatedNormalMessage whose support lies ~19 sigma into its tail; its log_partition underflows to -inf, the variable's evidence goes +inf, every adjacent factor term -inf, and log_evidence summed them to NaN with a numpy RuntimeWarning on every EP update. The posterior is unaffected (bit-identical sidecars); only the evidence readout was. log_evidence now detects a non-finite term before summing, names it once per fit (weakly keyed on the factor graph) in a logger warning and in ep_diagnostics.results, and returns an explicit NaN; evidence_divergence returns inf on a NaN so no evidence stopping rule can claim convergence. No message arithmetic, projection or optimiser changes; the tail-stable log_partition cure is filed separately. Co-Authored-By: Claude Fable 5.1 --- .../expectation_propagation/diagnostics.py | 48 +++++ .../expectation_propagation/ep_mean_field.py | 131 +++++++++++++- .../expectation_propagation/history.py | 10 +- .../expectation_propagation/optimiser.py | 14 +- .../functionality/test_evidence_nonfinite.py | 166 ++++++++++++++++++ 5 files changed, 356 insertions(+), 13 deletions(-) create mode 100644 test_autofit/graphical/functionality/test_evidence_nonfinite.py diff --git a/autofit/graphical/expectation_propagation/diagnostics.py b/autofit/graphical/expectation_propagation/diagnostics.py index 838d4db52..9909a68da 100644 --- a/autofit/graphical/expectation_propagation/diagnostics.py +++ b/autofit/graphical/expectation_propagation/diagnostics.py @@ -84,6 +84,9 @@ def __init__(self): self.scale_variables: Set[str] = set() self._previous_mean_field = None self._step = 0 + # {term label: (first step, kind)} for every non-finite term of the + # log evidence seen during the run (PyAutoFit#1713). + self.evidence_nonfinite: Dict[str, Tuple[int, str]] = {} def register_hierarchical_scales(self, factor_graph) -> None: """ @@ -133,6 +136,18 @@ def snapshot(self, factor, model_approx, status) -> None: logger.warning(f"EPDiagnostics: log_evidence failed: {e}") log_evidence = float("nan") + # A non-finite evidence is returned as NaN by `EPMeanField.log_evidence`; + # record which terms caused it, and from which step, for + # `check_evidence_nonfinite` (PyAutoFit#1713). + if not np.isfinite(log_evidence): + try: + terms = model_approx.nonfinite_evidence_terms + except Exception as e: + logger.warning(f"EPDiagnostics: evidence term scan failed: {e}") + terms = {} + for label, kind in terms.items(): + self.evidence_nonfinite.setdefault(label, (self._step, kind)) + if self._previous_mean_field is not None: try: kl = float(np.sum(mean_field.kl(self._previous_mean_field))) @@ -246,6 +261,39 @@ def plot(self, output_path: Path) -> None: plt.close(fig) +def check_evidence_nonfinite(diagnostics: EPDiagnostics) -> List[str]: + """ + Report the non-finite terms of the EP log evidence seen during a run. + + ``EPMeanField.log_evidence`` returns NaN, and logs once, when a variable's + log evidence or a factor's ``log_norm`` is NaN or ±inf (PyAutoFit#1713). + This turns the terms ``EPDiagnostics.snapshot`` recorded into a single + ``evidence_nonfinite: `` line for ``ep_diagnostics.results``, so a + batch run keeps a named trace of it. The posteriors are unaffected; the + line says that the ``log_evidence`` column of ``ep_history.csv`` is NaN + from the step named. + + Returns + ------- + A list holding that one line, or an empty list when every evidence was + finite. + """ + terms = getattr(diagnostics, "evidence_nonfinite", {}) + if not terms: + return [] + names = ", ".join( + f"{label} ({kind}, first at step {step})" + for label, (step, kind) in sorted(terms.items()) + ) + return [ + f"evidence_nonfinite: {names} — the EP log evidence is NaN from that " + f"step on (ep_history.csv log_evidence column). A message's " + f"log-partition is non-finite in floating point; the mean field and " + f"its posteriors are unaffected, only the evidence readout is " + f"(PyAutoFit#1713)." + ] + + def mean_field_summary(mean_field) -> str: """ A human-readable summary table of a mean field. diff --git a/autofit/graphical/expectation_propagation/ep_mean_field.py b/autofit/graphical/expectation_propagation/ep_mean_field.py index 3aecf7bb2..06682d87c 100644 --- a/autofit/graphical/expectation_propagation/ep_mean_field.py +++ b/autofit/graphical/expectation_propagation/ep_mean_field.py @@ -1,4 +1,5 @@ import logging +import weakref from typing import Dict, Tuple, Optional, List import numpy as np @@ -13,6 +14,14 @@ logger = logging.getLogger(__name__) +#: Non-finite evidence terms already reported by ``EPMeanField.log_evidence``, +#: keyed weakly on the factor graph (one fit = one graph, shared by every mean +#: field the EP updates build over it) with the set of reported +#: ``(label, kind)`` sets as the value — so each is logged once per fit rather +#: than on every EP update, and nothing is kept alive once the fit's graph is +#: gone (PyAutoFit#1713). +_reported_nonfinite = weakref.WeakKeyDictionary() + class EPMeanField(FactorGraph): """ @@ -306,6 +315,61 @@ def factor_evidence(self) -> Dict[Factor, np.ndarray]: for factor, meanfield in self.factor_mean_field.items() } + def _evidence_terms(self) -> Tuple[Dict[Variable, float], Dict[Factor, float]]: + """ + The per-variable log evidences ``log Zᵢ`` and per-factor ``log_norm`` + sums that ``log_evidence`` combines. + + Evaluated under ``np.errstate`` so a message whose log-partition is + non-finite does not emit numpy ``RuntimeWarning``s here; such a term + is reported by name by ``log_evidence`` instead. + """ + with np.errstate(invalid="ignore", over="ignore", divide="ignore"): + variable_evidence = { + v: np.sum(logz) for v, logz in self.variable_evidence.items() + } + factor_log_norm = { + factor: np.sum(meanfield.log_norm) + for factor, meanfield in self.factor_mean_field.items() + } + return variable_evidence, factor_log_norm + + @staticmethod + def _nonfinite_terms(variable_evidence, factor_log_norm) -> Dict: + """ + ``{term: kind}`` for every non-finite evidence term, keyed by the + ``Variable`` or ``Factor`` itself; ``kind`` is ``"nan"``, ``"+inf"`` + or ``"-inf"``. + """ + nonfinite = {} + for term, value in list(variable_evidence.items()) + list( + factor_log_norm.items() + ): + if not np.isfinite(value): + nonfinite[term] = ( + "nan" if np.isnan(value) else ("+inf" if value > 0 else "-inf") + ) + return nonfinite + + @staticmethod + def _term_label(term) -> str: + if isinstance(term, Factor): + return f"factor '{term.name}' log_norm" + return f"variable '{term.name}'" + + @property + def nonfinite_evidence_terms(self) -> Dict[str, str]: + """ + The non-finite terms of ``log_evidence`` as ``{label: kind}``, e.g. + ``{"variable 'sigma'": "+inf"}``; empty when the evidence is finite. + + A variable's term is its log evidence ``log Zᵢ`` (the log + normalisation of the product of its messages); a factor's is its mean + field's ``log_norm``. + """ + nonfinite = self._nonfinite_terms(*self._evidence_terms()) + return {self._term_label(term): kind for term, kind in nonfinite.items()} + @property def log_evidence(self): """ @@ -324,16 +388,67 @@ def log_evidence(self): Evidence for model Z = ∏ᵢ Zᵢ ∏ₐ Zₐ + + Non-finite terms + ---------------- + If any variable's ``log Zᵢ`` or any factor's ``log_norm`` is NaN or + ±inf the sum is undefined (a ``+inf`` variable term and the ``-inf`` + it induces in every adjacent factor term add to NaN). The evidence is + then returned as an explicit ``float("nan")`` without numpy + ``RuntimeWarning``s, and the offending terms are named in one + ``logger.warning`` per distinct set of terms (so once per fit, not + once per EP update). The usual cause is a message whose + log-partition is non-finite in floating point — e.g. a truncated + message whose mass inside its support underflows to zero + (PyAutoFit#1713). Only the evidence readout is affected: the mean + field and its posteriors are computed without it. """ - variable_evidence = { - v: np.sum(logz) for v, logz in self.variable_evidence.items() - } - factor_evidence = sum( - np.sum(meanfield.log_norm) - - sum(variable_evidence[v] for v in factor.all_variables) - for factor, meanfield in self.factor_mean_field.items() + variable_evidence, factor_log_norm = self._evidence_terms() + + nonfinite = self._nonfinite_terms(variable_evidence, factor_log_norm) + if nonfinite: + self._warn_nonfinite(nonfinite) + return float("nan") + + with np.errstate(invalid="ignore", over="ignore"): + factor_evidence = sum( + factor_log_norm[factor] + - sum(variable_evidence[v] for v in factor.all_variables) + for factor in self.factor_mean_field + ) + return factor_evidence + sum(variable_evidence.values()) + + def _warn_nonfinite(self, nonfinite: Dict) -> None: + """ + Log the non-finite evidence terms once per distinct set of terms. + + Every EP update builds a new ``EPMeanField`` over the same graph, and + the history re-evaluates past evidences, so the key is the set of + offending (term, kind) pairs rather than the instance. + """ + key = frozenset( + (self._term_label(term), kind) for term, kind in nonfinite.items() + ) + try: + reported = _reported_nonfinite.setdefault(self.factor_graph, set()) + except TypeError: # a graph that cannot be weakly referenced + reported = set() + if key in reported: + return + reported.add(key) + terms = ", ".join( + sorted( + f"{self._term_label(term)} ({kind})" + for term, kind in nonfinite.items() + ) + ) + logger.warning( + f"EP log evidence is non-finite and is reported as NaN: {terms}. " + f"A message's log-partition is non-finite in floating point (e.g. a " + f"truncated message whose mass inside its support underflows). The " + f"mean field and its posteriors are unaffected; only the evidence " + f"readout is. Reported once per fit (PyAutoFit#1713)." ) - return factor_evidence + sum(variable_evidence.values()) def __repr__(self) -> str: clsname = type(self).__name__ diff --git a/autofit/graphical/expectation_propagation/history.py b/autofit/graphical/expectation_propagation/history.py index a4af5ad31..ba7528c7d 100644 --- a/autofit/graphical/expectation_propagation/history.py +++ b/autofit/graphical/expectation_propagation/history.py @@ -146,11 +146,17 @@ def evidence_divergence(self) -> Union[float, np.ndarray]: successful optimisations. If there are less than two successful optimisations then this is - infinite. + infinite. It is also infinite when either evidence is NaN (a + non-finite evidence term, see ``EPMeanField.log_evidence``), so an + evidence-based stopping rule can never claim convergence on a NaN + (PyAutoFit#1713). """ - return ( + divergence = ( self.latest_successful.log_evidence - self.previous_successful.log_evidence ) + if np.any(np.isnan(divergence)): + return float("inf") + return divergence @property def evidences(self) -> List[float]: diff --git a/autofit/graphical/expectation_propagation/optimiser.py b/autofit/graphical/expectation_propagation/optimiser.py index 2294e7d31..2c81dc99e 100644 --- a/autofit/graphical/expectation_propagation/optimiser.py +++ b/autofit/graphical/expectation_propagation/optimiser.py @@ -15,7 +15,12 @@ from autofit.non_linear.paths import DirectoryPaths from autofit.non_linear.paths.abstract import AbstractPaths from autofit.tools.util import IntervalCounter -from .diagnostics import EPDiagnostics, check_sigma_collapse, mean_field_summary +from .diagnostics import ( + EPDiagnostics, + check_evidence_nonfinite, + check_sigma_collapse, + mean_field_summary, +) from .factor_optimiser import AbstractFactorOptimiser, ExactFactorFit from .visualise import Visualise @@ -653,7 +658,8 @@ def _output_diagnostics( Write the diagnostics CSVs (`ep_history.csv`, `mean_field_history.csv`) and the mean-field evolution plot to the output folder. At the end of a run (``final=True``) also write ``ep_diagnostics.results``: the - mean-field summary table plus any sigma-collapse warnings. + mean-field summary table plus any stale-factor, sigma-collapse and + non-finite-evidence warnings. """ if not self.output_path: return @@ -662,7 +668,9 @@ def _output_diagnostics( if final and model_approx is not None: warnings_list = ( - self._stale_factor_warnings() + check_sigma_collapse(self.diagnostics) + self._stale_factor_warnings() + + check_sigma_collapse(self.diagnostics) + + check_evidence_nonfinite(self.diagnostics) ) with open(self.output_path / "ep_diagnostics.results", "w+") as f: f.write(mean_field_summary(model_approx.mean_field)) diff --git a/test_autofit/graphical/functionality/test_evidence_nonfinite.py b/test_autofit/graphical/functionality/test_evidence_nonfinite.py new file mode 100644 index 000000000..3ddda2f08 --- /dev/null +++ b/test_autofit/graphical/functionality/test_evidence_nonfinite.py @@ -0,0 +1,166 @@ +""" +A non-finite term in the EP log evidence is reported by name, not summed (PyAutoFit#1713). + +On the analytic_gaussian ensemble (seeds 25 and 172, default ``projection="mode"``) +one member of a ``HierarchicalFactor`` sends the parent scatter a truncated-normal +message centred thousands of sigma below its support (mean -7374, sigma 384 on +[0, 100]). Its truncation mass ``Phi(b) - Phi(a)`` evaluates to ``1 - 1 = 0``, so its +log-partition is ``-inf``: the scatter's variable evidence is ``+inf``, every adjacent +factor term is ``-inf``, and ``EPMeanField.log_evidence`` added them, emitting numpy's +``invalid value encountered in add`` once per evaluation and returning NaN. The +posteriors are unaffected; only the evidence readout is. + +The fixture below is that state in miniature: two factors share ``x``, one message is +the production message. +""" + +import logging +import warnings + +import numpy as np +import pytest + +from autofit import graphical as g +from autofit.graphical.expectation_propagation import ep_mean_field +from autofit.graphical.expectation_propagation.diagnostics import ( + EPDiagnostics, + check_evidence_nonfinite, +) +from autofit.graphical.expectation_propagation.history import FactorHistory +from autofit.mapper.variable import Variable +from autofit.messages.truncated_normal import TruncatedNormalMessage + +LOGGER = "autofit.graphical.expectation_propagation.ep_mean_field" + + +def _ok(x): + return x + + +def _degenerate(x): + return -x + + +def make_approx(x, bad_mean=-7374.0, bad_sigma=384.0): + ok_factor = g.Factor(_ok, x, name="f_ok") + degenerate_factor = g.Factor(_degenerate, x, name="f_degenerate") + ok = TruncatedNormalMessage(mean=10.0, sigma=5.0, lower_limit=0.0, upper_limit=100.0) + bad = TruncatedNormalMessage( + mean=bad_mean, sigma=bad_sigma, lower_limit=0.0, upper_limit=100.0 + ) + return g.EPMeanField( + factor_graph=g.FactorGraph([ok_factor, degenerate_factor]), + factor_mean_field={ + ok_factor: g.MeanField({x: ok}), + degenerate_factor: g.MeanField({x: bad}), + }, + ) + + +@pytest.fixture(name="x") +def make_x(): + return Variable("x") + + +@pytest.fixture(name="degenerate") +def make_degenerate(x): + return make_approx(x) + + +@pytest.fixture(name="healthy") +def make_healthy(x): + return make_approx(x, bad_mean=20.0, bad_sigma=8.0) + + +def test_log_evidence_nan_without_runtime_warning(degenerate): + with warnings.catch_warnings(): + warnings.simplefilter("error", RuntimeWarning) + log_evidence = degenerate.log_evidence + + assert isinstance(log_evidence, float) + assert np.isnan(log_evidence) + + +def test_one_named_warning_per_run(degenerate, caplog): + with caplog.at_level(logging.WARNING, logger=LOGGER): + for _ in range(3): + degenerate.log_evidence + # A fresh EPMeanField over the same graph (what each EP update makes) + # does not repeat the warning. + g.EPMeanField( + factor_graph=degenerate.factor_graph, + factor_mean_field=degenerate.factor_mean_field, + ).log_evidence + + records = [r for r in caplog.records if r.name == LOGGER] + assert len(records) == 1 + assert "'x'" in records[0].getMessage() + assert "+inf" in records[0].getMessage() + + +def test_nonfinite_evidence_terms_names_the_variable(degenerate): + assert degenerate.nonfinite_evidence_terms == {"variable 'x'": "+inf"} + + +def test_healthy_evidence_unchanged(healthy, caplog): + with caplog.at_level(logging.WARNING, logger=LOGGER): + with warnings.catch_warnings(): + warnings.simplefilter("error", RuntimeWarning) + log_evidence = healthy.log_evidence + + variable_evidence = { + v: np.sum(logz) for v, logz in healthy.variable_evidence.items() + } + expected = sum( + np.sum(meanfield.log_norm) + - sum(variable_evidence[v] for v in factor.all_variables) + for factor, meanfield in healthy.factor_mean_field.items() + ) + sum(variable_evidence.values()) + + assert np.isfinite(log_evidence) + assert log_evidence == expected + assert healthy.nonfinite_evidence_terms == {} + assert not [r for r in caplog.records if r.name == LOGGER] + + +def test_evidence_divergence_inf_on_nan(degenerate, healthy): + factor = next(iter(healthy.factor_mean_field)) + history = FactorHistory(factor) + history(healthy, g.Status(success=True)) + history(degenerate, g.Status(success=True)) + + assert history.evidence_divergence() == float("inf") + + +def test_evidence_divergence_finite_when_healthy(healthy): + factor = next(iter(healthy.factor_mean_field)) + history = FactorHistory(factor) + history(healthy, g.Status(success=True)) + history(healthy, g.Status(success=True)) + + assert history.evidence_divergence() == 0.0 + + +def test_diagnostics_record_evidence_nonfinite(degenerate, healthy): + diagnostics = EPDiagnostics() + factor = next(iter(healthy.factor_mean_field)) + diagnostics.snapshot(factor, healthy, g.Status(success=True)) + assert check_evidence_nonfinite(diagnostics) == [] + + with warnings.catch_warnings(): + warnings.simplefilter("error", RuntimeWarning) + diagnostics.snapshot(factor, degenerate, g.Status(success=True)) + diagnostics.snapshot(factor, degenerate, g.Status(success=True)) + + assert np.isnan(diagnostics.factor_rows[-1]["log_evidence"]) + lines = check_evidence_nonfinite(diagnostics) + assert len(lines) == 1 + assert lines[0].startswith("evidence_nonfinite: ") + assert "variable 'x' (+inf, first at step 1)" in lines[0] + + +@pytest.fixture(autouse=True) +def reset_reported(): + ep_mean_field._reported_nonfinite.clear() + yield + ep_mean_field._reported_nonfinite.clear()