From 44c6081d528c5c2398240e62b772870e8693114e Mon Sep 17 00:00:00 2001 From: Jammy2211 Date: Fri, 28 Aug 2026 15:03:14 -0400 Subject: [PATCH 1/2] fix: give nested samplers a real test-mode posterior; ESS-based corner guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nautilus.apply_test_mode set n_like_max = 1, so under PYAUTO_TEST_MODE=1 the search stopped after its initial batch with exactly one non-zero weight (ESS = 1). Since 8cdcff3a0 correctly forwards weights= and caller kwargs to corner, a caller range=0.999 became a weighted quantile over that single point and collapsed to a sliver excluding every row, so corner raised "'range' is not valid or the sample is empty" (autofit_workspace scripts/plot/nautilus_plotter.py under the release profile). The data was fine (100 unique rows); the weights were degenerate, and the corner guard counted rows, not effective samples. - Nautilus.apply_test_mode: n_live=25, n_batch=25, n_networks=0, f_live=0.5, n_eff=25, n_like_max=1000 — a coarse but real posterior (measured 825-1000 calls, ESS 29-104, 11 s on nautilus_plotter.py; autolens imaging/modeling.py runtime unchanged, 60.5 s vs 60.5 s interleaved). _fit clamps n_live >= prior_count + 5 in test mode, since nautilus rejects n_live <= n_dim and apply_test_mode runs before the model is known. - Dynesty.apply_test_mode: maxcall=150, nlive/nlive_init=25 instead of maxcall=1. Dynesty's ESS only lifts at ~4000 calls, so this is a real reduced run but not a real posterior; documented, the corner guard covers it. - corner_cornerpy: skip via the existing logged path when the Kish ESS (sum(w)^2 / sum(w^2)) <= parameter count, alongside the row-count check. No try/except, kwargs forwarding unchanged. Closes #1541 Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01KeLmyZD5aG6BrTTdJzSmDv --- autofit/non_linear/plot/samples_plotters.py | 46 +++++++++++++-- .../search/nest/dynesty/search/abstract.py | 34 ++++++++++- .../non_linear/search/nest/nautilus/search.py | 46 ++++++++++++++- .../non_linear/plot/test_samples_plotters.py | 59 ++++++++++++++++++- .../non_linear/search/nest/test_dynesty.py | 18 ++++++ .../non_linear/search/nest/test_nautilus.py | 8 ++- 6 files changed, 198 insertions(+), 13 deletions(-) diff --git a/autofit/non_linear/plot/samples_plotters.py b/autofit/non_linear/plot/samples_plotters.py index 2a2c1500c..98de9a919 100644 --- a/autofit/non_linear/plot/samples_plotters.py +++ b/autofit/non_linear/plot/samples_plotters.py @@ -35,15 +35,53 @@ def _corner_range_from(data): return plot_range +def _effective_sample_size(weight_list, sample_count): + """Kish effective sample size ``sum(w)**2 / sum(w**2)`` of a weight list. + + Row count alone does not say whether a sample can be plotted: a nested + sampler stopped early returns many rows of which one carries essentially all + the weight, so every weighted statistic ``corner`` computes (quantile + ``range`` most visibly) is a statistic of that single point. ``None`` or an + empty weight list means unweighted, for which every row counts. + """ + if weight_list is None: + return float(sample_count) + + weights = np.asarray(weight_list, dtype=float) + + if weights.size == 0: + return float(sample_count) + + sum_of_squares = np.sum(weights**2) + + if sum_of_squares == 0.0: + return 0.0 + + return float(np.sum(weights) ** 2 / sum_of_squares) + + @skip_in_test_mode def corner_cornerpy(samples, path=None, filename="corner", format="show", **kwargs): data = np.asarray(samples.parameter_lists) - if data.ndim < 2 or data.shape[0] <= data.shape[1]: + + sample_count = data.shape[0] if data.ndim >= 1 else 0 + parameter_count = data.shape[1] if data.ndim >= 2 else 0 + effective_sample_size = _effective_sample_size( + getattr(samples, "weight_list", None), sample_count + ) + + if ( + data.ndim < 2 + or sample_count <= parameter_count + or effective_sample_size <= parameter_count + ): logger.info( - "corner_cornerpy: skipping corner plot, only %s sample(s) for %s parameter(s) " + "corner_cornerpy: skipping corner plot, only %s sample(s) " + "(effective sample size %.2f) for %s parameter(s) " "(e.g. PYAUTO_TEST_MODE bypass or an early-iteration update).", - data.shape[0] if data.ndim >= 1 else 0, - data.shape[1] if data.ndim >= 2 else 0, + sample_count, + effective_sample_size, + parameter_count, ) return diff --git a/autofit/non_linear/search/nest/dynesty/search/abstract.py b/autofit/non_linear/search/nest/dynesty/search/abstract.py index 0263aa6e7..89f69bd34 100644 --- a/autofit/non_linear/search/nest/dynesty/search/abstract.py +++ b/autofit/non_linear/search/nest/dynesty/search/abstract.py @@ -491,11 +491,39 @@ def checkpoint_file(self) -> str: pass def apply_test_mode(self): + """ + Reduce the sampler to a small but *real* run for test mode (level 1). + + A previous implementation set ``maxcall = 1``, which stopped dynesty + before it had drawn a single nested-sampling replacement: every sample + came from the initial live set at the same prior weight bar one, so the + returned posterior had an effective sample size of ~1 and weighted plots + (e.g. ``corner``'s quantile ``range``) collapsed onto a single point. + + The settings below instead run a genuine, if short, nested sampling + chain: a small live set with a few hundred likelihood evaluations, + capped so an expensive likelihood cannot make test mode slow. + + Note that a few hundred evaluations buys real nested-sampling + iterations, not yet a well-spread weight distribution — on the reference + ``autofit_workspace`` ``Gaussian`` fit the effective sample size only + lifts off after a few thousand evaluations, which test mode cannot + afford. Plotting code must therefore still guard against a low + effective sample size (see ``plot.samples_plotters.corner_cornerpy``) + rather than assume this run is plottable. + """ logger.warning( - "TEST MODE 1 (reduced iterations): Sampler will run with " - "minimal iterations for faster completion." + "TEST MODE 1 (reduced iterations): Sampler will run with reduced " + "live points and a capped likelihood budget, producing a coarse " + "but real posterior." ) - self.maxcall = 1 + # `nlive` (static) and `nlive_init` (dynamic) are the subclass names for + # the live set; only the one this subclass defines is reduced. + for live_point_attribute in ("nlive", "nlive_init"): + if getattr(self, live_point_attribute, None) is not None: + setattr(self, live_point_attribute, 25) + + self.maxcall = 150 def live_points_init_from(self, model, fitness): """ diff --git a/autofit/non_linear/search/nest/nautilus/search.py b/autofit/non_linear/search/nest/nautilus/search.py index 3ae499e8f..7734e46a2 100644 --- a/autofit/non_linear/search/nest/nautilus/search.py +++ b/autofit/non_linear/search/nest/nautilus/search.py @@ -150,11 +150,42 @@ def __init__( self.logger.debug("Creating Nautilus Search") def apply_test_mode(self): + """ + Reduce the sampler to a small but *real* run for test mode (level 1). + + A previous implementation set ``n_like_max = 1``, which stopped nautilus + after its very first batch of prior draws. Nautilus weights samples by + shell volume times likelihood, so before any bound has been built one + draw carries essentially all the weight: the returned posterior had an + effective sample size of 1, which is not a posterior at all and made + weighted plots (e.g. ``corner``'s quantile ``range``) collapse. + + The settings below instead let nautilus complete a coarse exploration — + fewer live points, no neural-network bounds, and an early exploration + cut-off — while capping the total likelihood budget so an expensive + likelihood cannot make test mode slow. On the reference + ``autofit_workspace`` ``Gaussian`` fit this yields 800-1000 likelihood + evaluations and an effective sample size of a few tens in a few seconds + — coarse, but a posterior weighted statistics can be computed from. + + `n_live` is a floor, not a final value: nautilus cannot build a bound + from fewer live points than the model has dimensions, so `_fit` raises + it above the model's prior count once the model is known. + """ logger.warning( - "TEST MODE 1 (reduced iterations): Sampler will run with " - "minimal iterations for faster completion." + "TEST MODE 1 (reduced iterations): Sampler will run with reduced " + "live points and a capped likelihood budget, producing a coarse " + "but real posterior." ) - self.n_like_max = 1 + self.n_live = 25 + self.n_batch = 25 + # 0 disables nautilus' neural-network bounds (multi-ellipsoid only), + # which dominate the wall time of a short run. + self.n_networks = 0 + # Terminate exploration once <= 50% of the evidence is in the live set. + self.f_live = 0.5 + self.n_eff = 25 + self.n_like_max = 1000 def _fit(self, model: AbstractPriorModel, analysis): """ @@ -175,6 +206,15 @@ def _fit(self, model: AbstractPriorModel, analysis): set of accepted ssamples of the fit. """ + if is_test_mode(): + # `apply_test_mode` runs in `__init__`, before the model is known, + # so its reduced `n_live` is only a floor: nautilus needs strictly + # more live points than dimensions to fit a bound at all, and would + # otherwise raise "Number of points must be larger than number of + # dimensions" on a model with more parameters than that floor. + self.n_live = max(self.n_live, model.prior_count + 5) + self.n_batch = self.n_live + if not isinstance(self.paths, NullPaths): checkpoint_exists = Path(self.checkpoint_file).exists() else: diff --git a/test_autofit/non_linear/plot/test_samples_plotters.py b/test_autofit/non_linear/plot/test_samples_plotters.py index 2beec0366..3a8068a3b 100644 --- a/test_autofit/non_linear/plot/test_samples_plotters.py +++ b/test_autofit/non_linear/plot/test_samples_plotters.py @@ -1,3 +1,5 @@ +import logging + import numpy as np import pytest @@ -53,12 +55,12 @@ def __init__(self, labels): class MockSamples: - def __init__(self, parameter_lists, weight_list=None): + def __init__(self, parameter_lists, weight_list=None, labels=None): self.parameter_lists = parameter_lists self.weight_list = ( weight_list if weight_list is not None else [1.0] * len(parameter_lists) ) - self.model = MockModel(["x", "y"]) + self.model = MockModel(labels if labels is not None else ["x", "y"]) @pytest.fixture @@ -177,3 +179,56 @@ def test__corner_cornerpy__degenerate_columns_still_render(monkeypatch): degenerate = MockSamples(parameter_lists=[[1.0, 5.0]] * 20) samples_plotters.corner_cornerpy(samples=degenerate, bins=5) + + +def test__effective_sample_size__uniform_and_degenerate_weights(): + assert samples_plotters._effective_sample_size([1.0] * 10, 10) == pytest.approx(10.0) + + # One point carrying all the weight is worth a single sample, however many + # rows sit beside it. + degenerate = [0.0] * 99 + [1.0] + assert samples_plotters._effective_sample_size(degenerate, 100) == pytest.approx(1.0) + + # No weights at all means unweighted, so every row counts. + assert samples_plotters._effective_sample_size(None, 7) == pytest.approx(7.0) + assert samples_plotters._effective_sample_size([], 7) == pytest.approx(7.0) + + +def test__corner_cornerpy__weight_degenerate_sample_is_skipped_not_plotted( + corner_call, caplog +): + # Regression (#1541): a nested sampler stopped after its first batch returns + # plenty of rows but one non-zero weight. `range=0.999` is then a weighted + # quantile over that single point, collapsing to a sliver that excludes every + # row, and `corner` raised "the provided 'range' is not valid or the sample + # is empty". The row-count guard never fired because the rows are all there. + rng = np.random.default_rng(0) + degenerate = MockSamples( + parameter_lists=rng.normal(size=(100, 3)).tolist(), + weight_list=[0.0] * 99 + [1.0], + labels=["x", "y", "z"], + ) + + with caplog.at_level(logging.INFO, logger=samples_plotters.__name__): + samples_plotters.corner_cornerpy( + samples=degenerate, + range=np.ones(3) * 0.999, + ) + + assert corner_call == {} + assert "skipping corner plot" in caplog.text + assert "effective sample size" in caplog.text + + +def test__corner_cornerpy__healthy_weights_still_plot(corner_call): + rng = np.random.default_rng(0) + healthy = MockSamples( + parameter_lists=rng.normal(size=(100, 3)).tolist(), + weight_list=[1.0] * 100, + labels=["x", "y", "z"], + ) + + samples_plotters.corner_cornerpy(samples=healthy, range=np.ones(3) * 0.999) + + assert corner_call != {} + assert corner_call["weights"] == healthy.weight_list diff --git a/test_autofit/non_linear/search/nest/test_dynesty.py b/test_autofit/non_linear/search/nest/test_dynesty.py index 4b62c8d25..0bd1252f4 100644 --- a/test_autofit/non_linear/search/nest/test_dynesty.py +++ b/test_autofit/non_linear/search/nest/test_dynesty.py @@ -59,3 +59,21 @@ def test__explicit_params(): assert search.facc == 0.2 assert search.dlogz_init == 0.01 assert search.number_of_cores == 1 + + +def test__test_mode(): + # #1541 (the Dynesty sibling of the Nautilus defect): `maxcall = 1` stopped + # dynesty before it drew a single replacement point, so the posterior had an + # effective sample size of ~1. Test mode must run a short but real chain. + search = af.DynestyStatic() + search.apply_test_mode() + + assert search.maxcall > 1 + assert search.maxcall <= 500 + assert search.nlive < af.DynestyStatic().nlive + + search = af.DynestyDynamic() + search.apply_test_mode() + + assert search.maxcall > 1 + assert search.nlive_init < af.DynestyDynamic().nlive_init diff --git a/test_autofit/non_linear/search/nest/test_nautilus.py b/test_autofit/non_linear/search/nest/test_nautilus.py index 87e621157..e68e2350e 100644 --- a/test_autofit/non_linear/search/nest/test_nautilus.py +++ b/test_autofit/non_linear/search/nest/test_nautilus.py @@ -59,10 +59,16 @@ def test__identifier_fields(): def test__test_mode(): + # #1541: `n_like_max = 1` stopped nautilus after its first batch of prior + # draws, returning a posterior whose effective sample size was 1. Test mode + # must instead deliver a small but real posterior. search = af.Nautilus() search.apply_test_mode() - assert search.n_like_max == 1 + assert search.n_like_max > 1 + assert search.n_like_max <= 2000 + assert search.n_live < af.Nautilus().n_live + assert search.n_networks == 0 @requires_nautilus From 8ebcbcbc65e5b33b49b5f1436347d7040810ce79 Mon Sep 17 00:00:00 2001 From: Jammy2211 Date: Fri, 28 Aug 2026 15:22:52 -0400 Subject: [PATCH 2/2] revert: keep nested-sampler test mode as-is; ESS corner guard only The human decided against raising the global PYAUTO_TEST_MODE=1 sampler budget: apply_test_mode runs for every Nautilus/Dynesty search under the release profile, so ~900 extra likelihood evaluations per search would slow every release-wave script, not just the plotter. Nautilus n_like_max=1 and Dynesty maxcall=1 are restored. The workspace script instead opts out of test mode (ENV: real_search) and caps the search with an explicit n_like_max, like other workspace examples. The corner_cornerpy effective-sample-size guard stays: a weight-degenerate sample now takes the logged skip path instead of crashing. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01KeLmyZD5aG6BrTTdJzSmDv --- .../search/nest/dynesty/search/abstract.py | 34 ++------------ .../non_linear/search/nest/nautilus/search.py | 46 ++----------------- .../non_linear/search/nest/test_dynesty.py | 18 -------- .../non_linear/search/nest/test_nautilus.py | 8 +--- 4 files changed, 7 insertions(+), 99 deletions(-) diff --git a/autofit/non_linear/search/nest/dynesty/search/abstract.py b/autofit/non_linear/search/nest/dynesty/search/abstract.py index 89f69bd34..0263aa6e7 100644 --- a/autofit/non_linear/search/nest/dynesty/search/abstract.py +++ b/autofit/non_linear/search/nest/dynesty/search/abstract.py @@ -491,39 +491,11 @@ def checkpoint_file(self) -> str: pass def apply_test_mode(self): - """ - Reduce the sampler to a small but *real* run for test mode (level 1). - - A previous implementation set ``maxcall = 1``, which stopped dynesty - before it had drawn a single nested-sampling replacement: every sample - came from the initial live set at the same prior weight bar one, so the - returned posterior had an effective sample size of ~1 and weighted plots - (e.g. ``corner``'s quantile ``range``) collapsed onto a single point. - - The settings below instead run a genuine, if short, nested sampling - chain: a small live set with a few hundred likelihood evaluations, - capped so an expensive likelihood cannot make test mode slow. - - Note that a few hundred evaluations buys real nested-sampling - iterations, not yet a well-spread weight distribution — on the reference - ``autofit_workspace`` ``Gaussian`` fit the effective sample size only - lifts off after a few thousand evaluations, which test mode cannot - afford. Plotting code must therefore still guard against a low - effective sample size (see ``plot.samples_plotters.corner_cornerpy``) - rather than assume this run is plottable. - """ logger.warning( - "TEST MODE 1 (reduced iterations): Sampler will run with reduced " - "live points and a capped likelihood budget, producing a coarse " - "but real posterior." + "TEST MODE 1 (reduced iterations): Sampler will run with " + "minimal iterations for faster completion." ) - # `nlive` (static) and `nlive_init` (dynamic) are the subclass names for - # the live set; only the one this subclass defines is reduced. - for live_point_attribute in ("nlive", "nlive_init"): - if getattr(self, live_point_attribute, None) is not None: - setattr(self, live_point_attribute, 25) - - self.maxcall = 150 + self.maxcall = 1 def live_points_init_from(self, model, fitness): """ diff --git a/autofit/non_linear/search/nest/nautilus/search.py b/autofit/non_linear/search/nest/nautilus/search.py index 7734e46a2..3ae499e8f 100644 --- a/autofit/non_linear/search/nest/nautilus/search.py +++ b/autofit/non_linear/search/nest/nautilus/search.py @@ -150,42 +150,11 @@ def __init__( self.logger.debug("Creating Nautilus Search") def apply_test_mode(self): - """ - Reduce the sampler to a small but *real* run for test mode (level 1). - - A previous implementation set ``n_like_max = 1``, which stopped nautilus - after its very first batch of prior draws. Nautilus weights samples by - shell volume times likelihood, so before any bound has been built one - draw carries essentially all the weight: the returned posterior had an - effective sample size of 1, which is not a posterior at all and made - weighted plots (e.g. ``corner``'s quantile ``range``) collapse. - - The settings below instead let nautilus complete a coarse exploration — - fewer live points, no neural-network bounds, and an early exploration - cut-off — while capping the total likelihood budget so an expensive - likelihood cannot make test mode slow. On the reference - ``autofit_workspace`` ``Gaussian`` fit this yields 800-1000 likelihood - evaluations and an effective sample size of a few tens in a few seconds - — coarse, but a posterior weighted statistics can be computed from. - - `n_live` is a floor, not a final value: nautilus cannot build a bound - from fewer live points than the model has dimensions, so `_fit` raises - it above the model's prior count once the model is known. - """ logger.warning( - "TEST MODE 1 (reduced iterations): Sampler will run with reduced " - "live points and a capped likelihood budget, producing a coarse " - "but real posterior." + "TEST MODE 1 (reduced iterations): Sampler will run with " + "minimal iterations for faster completion." ) - self.n_live = 25 - self.n_batch = 25 - # 0 disables nautilus' neural-network bounds (multi-ellipsoid only), - # which dominate the wall time of a short run. - self.n_networks = 0 - # Terminate exploration once <= 50% of the evidence is in the live set. - self.f_live = 0.5 - self.n_eff = 25 - self.n_like_max = 1000 + self.n_like_max = 1 def _fit(self, model: AbstractPriorModel, analysis): """ @@ -206,15 +175,6 @@ def _fit(self, model: AbstractPriorModel, analysis): set of accepted ssamples of the fit. """ - if is_test_mode(): - # `apply_test_mode` runs in `__init__`, before the model is known, - # so its reduced `n_live` is only a floor: nautilus needs strictly - # more live points than dimensions to fit a bound at all, and would - # otherwise raise "Number of points must be larger than number of - # dimensions" on a model with more parameters than that floor. - self.n_live = max(self.n_live, model.prior_count + 5) - self.n_batch = self.n_live - if not isinstance(self.paths, NullPaths): checkpoint_exists = Path(self.checkpoint_file).exists() else: diff --git a/test_autofit/non_linear/search/nest/test_dynesty.py b/test_autofit/non_linear/search/nest/test_dynesty.py index 0bd1252f4..4b62c8d25 100644 --- a/test_autofit/non_linear/search/nest/test_dynesty.py +++ b/test_autofit/non_linear/search/nest/test_dynesty.py @@ -59,21 +59,3 @@ def test__explicit_params(): assert search.facc == 0.2 assert search.dlogz_init == 0.01 assert search.number_of_cores == 1 - - -def test__test_mode(): - # #1541 (the Dynesty sibling of the Nautilus defect): `maxcall = 1` stopped - # dynesty before it drew a single replacement point, so the posterior had an - # effective sample size of ~1. Test mode must run a short but real chain. - search = af.DynestyStatic() - search.apply_test_mode() - - assert search.maxcall > 1 - assert search.maxcall <= 500 - assert search.nlive < af.DynestyStatic().nlive - - search = af.DynestyDynamic() - search.apply_test_mode() - - assert search.maxcall > 1 - assert search.nlive_init < af.DynestyDynamic().nlive_init diff --git a/test_autofit/non_linear/search/nest/test_nautilus.py b/test_autofit/non_linear/search/nest/test_nautilus.py index e68e2350e..87e621157 100644 --- a/test_autofit/non_linear/search/nest/test_nautilus.py +++ b/test_autofit/non_linear/search/nest/test_nautilus.py @@ -59,16 +59,10 @@ def test__identifier_fields(): def test__test_mode(): - # #1541: `n_like_max = 1` stopped nautilus after its first batch of prior - # draws, returning a posterior whose effective sample size was 1. Test mode - # must instead deliver a small but real posterior. search = af.Nautilus() search.apply_test_mode() - assert search.n_like_max > 1 - assert search.n_like_max <= 2000 - assert search.n_live < af.Nautilus().n_live - assert search.n_networks == 0 + assert search.n_like_max == 1 @requires_nautilus