From 0ab2519d71535eb8db8e8357bbfb6a59a5f365c1 Mon Sep 17 00:00:00 2001 From: Elmehdi Aitbrahim Date: Mon, 7 Sep 2026 05:38:19 -0400 Subject: [PATCH] feat(research): persist what the gauntlet computed instead of printing it (#726) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit keel computed evidence and kept the prose. `trials pbo` printed ten figures and wrote none. `trials deflate` printed a DSR whose inputs the ledger could not supply, so the number could never be recomputed or checked. `trials monte-carlo` stored a distribution's ends and not its shape. All three surfaced as "the UI cannot show this" and none of them was a UI problem. THE CONSTRAINT THAT SHAPED THE DESIGN, FOUND BEFORE WRITING ANYTHING `_decode_summary` maps a summary value to `None`, an `int`, or `Decimal(value)`. A list raises `ValueError`; a dict raises `TypeError`. And it raises ON READ, inside `read_trials` -- which every later `verify_chain`, `trials list`, `trials pbo` and web page goes through. ONE nested value would make an APPEND-ONLY, git-tracked file unreadable forever, with no way to take it back. That is not hypothetical: `_decode_summary`'s own comment records a null value doing exactly this once already. So the guard moved to the WRITE, where it is a refusal rather than a catastrophe -- and every artifact below is stored as FLAT scalar keys. `final_p05`, not a nested quantile ladder. The shape a reader can survive is the shape a writer may use. WHAT IS NOW RECORDED CSCV: every field of `PBOResult` -- pbo, both degradation coefficients, prob_loss, both dominance flags, the column/block/combination counts, rows used and dropped, and how many columns were refused. Ten figures were computed and one reached the ledger. DSR: the INPUTS as well as the outputs, and the inputs are the point. `--sharpe` is a required operator input because the ledger stores no per-trial Sharpe, so DSR was not merely expensive to recompute -- it was impossible without synthesising a number the operator had supplied, which is the one thing this codebase refuses. Recorded at the moment they were stated, the figure can be checked rather than trusted. A run with no `--trial-sharpe-variance` still refuses to compute a DSR and now also records nothing: a stored figure nobody ran is worse than an honest gap. MONTE CARLO: a seven-point quantile ladder for finals and drawdowns. `distribution_min/median/max` say how far the resampling reached; the ladder says what its shape was, which is what a histogram needs. Quantiles rather than the raw array -- thousands of Decimals per row in a git-tracked file -- and NEAREST-RANK rather than interpolated, because every value in this distribution is an equity the model actually produced and an interpolated quantile is a number no path reached. TWO NEW DIAGNOSTIC KINDS, AND WHY NEITHER FEEDS BACK `cscv` and `deflated_sharpe` are measurements ABOUT a set of trials rather than trials themselves, so both are always `series_missing` -- and `matrix.build_matrix` refuses `series_missing` rows, so a recorded PBO can never become a column in the next PBO over the same file. A diagnostic that changed the thing it measured would be worse than one nobody kept. The test for that passed for the wrong reason first: it built the matrix over session `s1` while the recorded row landed in session `all`, so the SESSION FILTER excluded it and `series_missing` did no work at all. It now records into the same session and fails against a mutant that gives the row a real series. NO BACKFILL. The six existing gauntlet rows keep exactly the fields they have and the UI keeps reading "not recorded" for the rest -- the posture #721 settled for the audit chain. The 93 tracked rows still verify. ⛔ THE STRATHERN RAIL. Every figure here is a diagnostic. Storing them makes them easier to rank by, which is why `PBOResult` carries no configuration field and why nothing written here names a winning parameter set. Nine mutants killed: the summary guard removed, the guard admitting a list, `trials pbo` back to printing and discarding, the cscv row dropping the fields beyond pbo, the cscv row carrying a series and becoming a column, deflate recording a row with no variance, the dsr row dropping its inputs, the monte carlo ladder dropped, and the quantile interpolating instead of nearest-rank. Unblocks #708's Evidence Matrix. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01KZZxmspQXe5qJ9FAsG13s6 --- keel/commands/trials.py | 83 ++++++++++++++ keel/research/ledger.py | 34 ++++++ keel/research/montecarlo.py | 34 ++++++ tests/research/test_ledger.py | 62 +++++++++++ tests/research/test_montecarlo.py | 47 ++++++++ tests/research/test_trials_cli.py | 173 ++++++++++++++++++++++++++++++ 6 files changed, 433 insertions(+) diff --git a/keel/commands/trials.py b/keel/commands/trials.py index b5863391..37daeaea 100644 --- a/keel/commands/trials.py +++ b/keel/commands/trials.py @@ -215,6 +215,41 @@ def trials_deflate( click.echo(f"\nSR_0 (rejection bar) : {sr0:.4f}") click.echo(f"DSR : {dsr:.4f}") + # #726. THE INPUTS AS WELL AS THE OUTPUTS, and the inputs are the point. + # + # `--sharpe` is a REQUIRED operator input: the ledger stores no observed annualised Sharpe per + # trial, so DSR is not merely expensive to recompute later -- it is impossible without + # synthesising a number the operator supplied, which is the one thing this codebase refuses. + # Recording them at the moment they were stated turns DSR from a re-run into a read, and lets + # the figure be CHECKED rather than trusted. + recorded = trials_ledger.append_trial( + _ledger_path(ledger), + trial_id=f"dsr-{n_decisions}n-sr{sharpe:g}", + session="deflate", + rule="(deflated sharpe over the recorded decisions)", + params={"rho": rho, "n_decisions": n_decisions}, + provenance="a_priori", + kind="deflated_sharpe", + decision="diagnostic_only", + series_missing=True, + summary={ + "observed_annual_sharpe": Decimal(str(sharpe)), + "trades_per_year": Decimal(str(trades_per_year)), + "skewness": Decimal(str(skew)), + "kurtosis": Decimal(str(kurtosis)), + "trial_sharpe_variance": Decimal(str(trial_sharpe_variance)), + "m_total": m_total, + "n_decisions": n_decisions, + "n_hat": Decimal(str(n_hat)), + "expected_max_sharpe": Decimal(str(deflate_mod.expected_max_sharpe(effective))), + "sharpe_rejection_threshold": Decimal(str(sr0)), + "min_trades": Decimal(str(deflate_mod.min_trades(effective, sharpe, trades_per_year))), + "observations": observations, + "dsr": Decimal(str(dsr)), + }, + ) + click.echo(f"\nrecorded {recorded.trial_id} hash={recorded.row_hash[:12]}") + @trials_group.command("pbo") @_LEDGER_OPTION @@ -258,6 +293,44 @@ def trials_pbo(ledger: Path | None, session: str | None, blocks: int) -> None: "near-identical configurations produces high PBO by construction." ) + # #726. Ten figures were computed and one reached the ledger, in prose. Recording the rest is + # what lets a reader ask "how overfit was this?" without a 12-second re-run -- measured on the + # real ledger, three sessions cost ~39 s of CPU, on a page that polls every 15 s. + # + # `series_missing=True` and always: this row is a measurement ABOUT a set of columns, not a + # trial with a series of its own, and `matrix.build_matrix` refuses `series_missing` rows -- + # so a recorded PBO can never become a column in the next PBO run over the same file. + # + # ⛔ THE STRATHERN RAIL. Every figure here is a diagnostic. Storing them makes them easier to + # rank by, which is exactly why `PBOResult` carries no configuration field and why nothing + # written here names a winning parameter set. + recorded = trials_ledger.append_trial( + _ledger_path(ledger), + trial_id=f"cscv-{session or 'all'}-s{blocks}", + session=session or "all", + rule="(cscv over the recorded columns)", + params={"session": session, "blocks": blocks}, + provenance="a_priori", + kind="cscv", + decision="diagnostic_only", + series_missing=True, + summary={ + "pbo": result.pbo, + "degradation_slope": result.degradation_slope, + "degradation_intercept": result.degradation_intercept, + "prob_loss": result.prob_loss, + "dominance_1st": result.dominance_1st, + "dominance_2nd": result.dominance_2nd, + "n_columns": result.n_columns, + "n_blocks": result.n_blocks, + "n_combinations": result.n_combinations, + "rows_used": result.rows_used, + "rows_dropped": result.rows_dropped, + "columns_refused": len(build.refused), + }, + ) + click.echo(f"\nrecorded {recorded.trial_id} hash={recorded.row_hash[:12]}") + # -- monte-carlo resampling (#441) ---------------------------------------------------------------- # @@ -482,6 +555,16 @@ def trials_monte_carlo( "drawdown_percentile": report.drawdown_percentile, "n_trades": len(pnls), "n_paths": paths, + # #726: the DISTRIBUTION, not just its ends. `distribution_min/median/max` say + # how far the resampling reached; the ladder says what its shape was, which is + # what a histogram needs and what nothing recorded until now -- so #708's Monte + # Carlo panel had no stored figures to draw and would have had to re-run a + # backtest inside a web request. + # + # FLAT keys, never a nested ladder: `ledger._validate_summary` refuses anything + # else, because one nested value makes this append-only file unreadable forever. + **mc_mod.quantile_ladder(finals, "final"), + **mc_mod.quantile_ladder(drawdowns, "drawdown"), }, ) except ValueError as exc: diff --git a/keel/research/ledger.py b/keel/research/ledger.py index 7c0c8964..05f79cc9 100644 --- a/keel/research/ledger.py +++ b/keel/research/ledger.py @@ -48,6 +48,14 @@ "threshold_nudge", "monte_carlo", "walk_forward", + # #726. Two more DIAGNOSTIC kinds, for the two gauntlet components that computed a full + # result and printed it. `cscv` is a PBO run over a session's columns; `deflated_sharpe` + # is one E[max SR]/MinBTL/DSR evaluation under the operator's stated inputs. Both are + # measurements ABOUT a set of trials rather than trials themselves, which is why both are + # always `series_missing` -- and `matrix.build_matrix` refuses `series_missing` rows, so + # neither can ever become a column in a later PBO run over the same ledger. + "cscv", + "deflated_sharpe", } ) DECISIONS = frozenset({"selected", "rejected", "diagnostic_only"}) @@ -128,6 +136,31 @@ def compute_row_hash(record: TrialRecord) -> str: return chain_hash(_row_payload(record)) +#: What a `summary` value may be. Everything else is refused at append time. +#: +#: `_decode_summary` maps a value to `None`, an `int` or `Decimal(value)`. A list raises +#: `ValueError` there and a dict raises `TypeError` -- and it raises on READ, inside +#: `read_trials`, which every later `verify_chain`, `trials list`, `trials pbo` and web page goes +#: through. ONE such row would make an APPEND-ONLY file unreadable forever, with no way to take it +#: back. That is not hypothetical: this module's own `_decode_summary` records a null value doing +#: exactly this once already. +#: +#: So the check is here, at the write, where it is a refusal rather than a catastrophe. It is also +#: why #726's gauntlet artifacts are FLAT keys -- `final_p05`, `final_p50` -- rather than a nested +#: quantile ladder: the shape a reader can survive is the shape a writer may use. +_SUMMARY_SCALARS = (Decimal, int, float, str, bool) + + +def _validate_summary(summary: Mapping[str, Any]) -> None: + for key, value in summary.items(): + if value is not None and not isinstance(value, _SUMMARY_SCALARS): + raise ValueError( + f"summary[{key!r}] is a {type(value).__name__}; a summary value must be a scalar " + "or None. A nested value would make this append-only ledger unreadable on the " + "next read_trials, permanently -- store a flat key per figure instead" + ) + + def _validate(record: TrialRecord) -> None: if record.provenance not in PROVENANCE: raise ValueError(f"provenance: {record.provenance!r} not in {sorted(PROVENANCE)}") @@ -135,6 +168,7 @@ def _validate(record: TrialRecord) -> None: raise ValueError(f"kind: {record.kind!r} not in {sorted(KINDS)}") if record.decision not in DECISIONS: raise ValueError(f"decision: {record.decision!r} not in {sorted(DECISIONS)}") + _validate_summary(record.summary) if not record.series_missing and not (record.per_trade_pnl or record.per_bar_pnl): raise ValueError( "series_missing is False but no P&L series was supplied; a trial with no series " diff --git a/keel/research/montecarlo.py b/keel/research/montecarlo.py index 033b9273..9cab53fb 100644 --- a/keel/research/montecarlo.py +++ b/keel/research/montecarlo.py @@ -34,6 +34,7 @@ from __future__ import annotations +import math import random from collections.abc import Sequence from dataclasses import dataclass @@ -115,6 +116,39 @@ def max_drawdown(curve: Sequence[Decimal]) -> Decimal: return deepest +#: The quantile ladder every resampled distribution is stored at (#726). +#: +#: A FIXED ladder, and a small one. The raw array of `n_paths` finals is what a chart would love +#: and what an append-only text file must not carry: at the default path count it is thousands of +#: Decimals per row, and the ledger is git-tracked. Seven quantiles are what a histogram or a +#: polyline actually needs, are bounded, and are the same seven whatever `--paths` was. +#: +#: Stored FLAT (`final_p05`, `final_p50`, ...), never as a nested list: `ledger._validate_summary` +#: refuses anything else, because a nested value makes the whole file unreadable on the next read. +QUANTILE_LADDER: tuple[int, ...] = (1, 5, 25, 50, 75, 95, 99) + + +def quantile(values: Sequence[Decimal], percent: int) -> Decimal: + """The `percent`-th quantile by NEAREST-RANK, on the sorted values. + + Nearest-rank rather than interpolating: every value in this distribution is a resampled + equity that the model actually produced, and an interpolated quantile is a number no path + reached. For a distribution being read as "what could have happened", that distinction is the + whole point -- the same reason `median` below averages the two middle values only for an even + count, where no single observation is the middle. + """ + if not values: + raise ValueError("no values to take a quantile of") + ordered = sorted(values) + rank = max(1, math.ceil(Decimal(percent) / Decimal(100) * Decimal(len(ordered)))) + return ordered[int(rank) - 1] + + +def quantile_ladder(values: Sequence[Decimal], prefix: str) -> dict[str, Decimal]: + """`{f"{prefix}_p05": ..., ...}` over `QUANTILE_LADDER` -- the flat form the ledger stores.""" + return {f"{prefix}_p{percent:02d}": quantile(values, percent) for percent in QUANTILE_LADDER} + + def median(values: Sequence[Decimal]) -> Decimal: """Exact-Decimal median: the middle of the sorted values for odd length, the mean of the two middles for even length. Raises on empty input rather than inventing a zero.""" diff --git a/tests/research/test_ledger.py b/tests/research/test_ledger.py index 9e1cf54e..bd8c802a 100644 --- a/tests/research/test_ledger.py +++ b/tests/research/test_ledger.py @@ -229,3 +229,65 @@ def test_the_tracked_ledger_still_verifies_after_the_canonicaliser_moved() -> No # errors for zero rows, which is not the same as "verified" (see `verify_records`). assert len(records) > 1, "the tracked ledger must hold a chain, not a single row" assert ledger.verify_chain(tracked) == [] + + +# -- the summary is FLAT, and the guard is at write time (#726) ----------------------------------- + + +def test_a_nested_summary_value_is_refused_at_append_time(tmp_path) -> None: + """The reason #726's quantile ladder is flat keys rather than a list. + + `_decode_summary` maps every summary value to `None`, an `int`, or `Decimal(value)`. A list + raises `ValueError` and a dict raises `TypeError` -- and it raises on READ, in `read_trials`, + which every later `verify_chain`, `trials list`, `trials pbo` and web page goes through. One + such row would make an APPEND-ONLY file unreadable forever, with no way to take it back. + + That is not hypothetical: the module's own docstring records a null summary value doing + exactly this once already ("one such row made every later read_trials/verify_chain of the + append-only chain raise forever"). + + So the check moved to the write, where it is still a refusal rather than a catastrophe. + """ + path = tmp_path / "ledger.jsonl" + for value in ([Decimal("1")], {"p50": Decimal("1")}, (1, 2)): + with pytest.raises(ValueError, match="summary"): + ledger.append_trial( + path, + trial_id="t1", + session="s", + rule="r", + provenance="a_priori", + kind="sweep_node", + decision="diagnostic_only", + series_missing=True, + summary={"quantiles": value}, + ) + assert not path.exists(), "a refused append must not have written a row" + + +def test_the_scalar_summary_values_the_gauntlet_writes_all_round_trip(tmp_path) -> None: + """Everything #726 stores: Decimals, ints, bools and explicit nulls.""" + path = tmp_path / "ledger.jsonl" + ledger.append_trial( + path, + trial_id="t1", + session="s", + rule="r", + provenance="a_priori", + kind="sweep_node", + decision="diagnostic_only", + series_missing=True, + summary={ + "pbo": Decimal("0.8810"), + "n_columns": 12, + "dominance_1st": True, + "trial_sharpe_variance": None, + }, + ) + (stored,) = ledger.read_trials(path) + + assert stored.summary["pbo"] == Decimal("0.8810") + assert stored.summary["n_columns"] == 12 + assert stored.summary["dominance_1st"] is True + assert stored.summary["trial_sharpe_variance"] is None + assert ledger.verify_chain(path) == [] diff --git a/tests/research/test_montecarlo.py b/tests/research/test_montecarlo.py index 32e33882..57210f19 100644 --- a/tests/research/test_montecarlo.py +++ b/tests/research/test_montecarlo.py @@ -20,6 +20,7 @@ import pytest +from keel.research import montecarlo from keel.research.montecarlo import ( MonteCarloReport, equity_curve, @@ -299,3 +300,49 @@ def test_a_reshuffled_sample_reports_the_median_by_construction() -> None: finals = final_equities(reshuffle(pnls, 30, seed=8), Decimal(0)) assert min(finals) == observed == max(finals) assert percentile_of(observed, finals) == Decimal("0.5") + + +# -- the quantile ladder (#726) -------------------------------------------------------------------- + + +def test_a_quantile_is_a_value_the_distribution_actually_produced() -> None: + """NEAREST-RANK, not interpolation. + + Every value here is a resampled equity the model actually reached, and an interpolated + quantile is a number no path produced. For a distribution being read as "what could have + happened", that distinction is the whole point. + """ + from decimal import Decimal as D + + values = [D("1"), D("2"), D("3"), D("4")] + for percent in (1, 5, 25, 50, 75, 95, 99): + assert montecarlo.quantile(values, percent) in values + + +def test_the_ladder_is_ordered_and_spans_the_distribution() -> None: + from decimal import Decimal as D + + values = [D(str(n)) for n in range(1, 101)] + ladder = montecarlo.quantile_ladder(values, "final") + + ordered = [ladder[f"final_p{percent:02d}"] for percent in montecarlo.QUANTILE_LADDER] + assert ordered == sorted(ordered) + assert ordered[0] == D("1") + assert ordered[-1] == D("99") + + +def test_the_ladder_keys_are_flat_and_zero_padded() -> None: + """`final_p05`, not `final_p5`: the keys sort lexically in the order they read, and the ledger + stores them side by side with every other scalar.""" + from decimal import Decimal as D + + ladder = montecarlo.quantile_ladder([D("1"), D("2")], "final") + assert sorted(ladder) == [ + "final_p01", "final_p05", "final_p25", "final_p50", "final_p75", "final_p95", "final_p99" + ] + assert all(isinstance(value, D) for value in ladder.values()) + + +def test_a_quantile_of_nothing_is_refused_rather_than_invented() -> None: + with pytest.raises(ValueError, match="quantile"): + montecarlo.quantile([], 50) diff --git a/tests/research/test_trials_cli.py b/tests/research/test_trials_cli.py index 2c880013..ff97fe37 100644 --- a/tests/research/test_trials_cli.py +++ b/tests/research/test_trials_cli.py @@ -11,6 +11,7 @@ from keel.cli import cli from keel.data.db import connect, migrate from keel.data.repository import Repository +from keel.research import ledger from keel.research import ledger as trials_ledger from keel.research.montecarlo import equity_curve, max_drawdown from keel.strategy.backtest import backtest @@ -514,3 +515,175 @@ def test_monte_carlo_caps_paths_at_2000(tmp_path): db = _mc_db(tmp_path) result = _invoke_mc(CliRunner(), db, tmp_path / "t.jsonl", "--paths", "2001", "--seed", "1") assert result.exit_code != 0 + + +# -- the gauntlet records what it computed (#726) -------------------------------------------------- +# +# keel computed evidence and kept only the prose. `trials pbo` printed ten figures and wrote none; +# `trials deflate` printed a DSR whose inputs the ledger could not supply, so the number could +# never be recomputed or checked; `trials monte-carlo` stored a distribution's ends and not its +# shape. All three surfaced as "the UI cannot show this" and none was a UI problem. + + +def _seeded_columns(path, sessions: int = 4): + """Enough synchronous per-bar series for `build_matrix` to make a CSCV matrix from.""" + from decimal import Decimal as D + + for index in range(sessions): + ledger.append_trial( + path, + trial_id=f"col-{index}", + session="s1", + rule="turtle_breakout", + params={"n": 20 + index}, + provenance="fitted", + kind="sweep_node", + decision="selected" if index == 0 else "rejected", + per_bar_pnl=[D(str((row * (index + 1)) % 7 - 3)) for row in range(64)], + ) + + +def test_trials_pbo_records_every_figure_it_computed(tmp_path) -> None: + path = tmp_path / "ledger.jsonl" + _seeded_columns(path) + + result = CliRunner().invoke(cli, ["trials", "pbo", "--ledger", str(path), "--blocks", "4"]) + assert result.exit_code == 0, result.output + + recorded = [row for row in ledger.read_trials(path) if row.kind == "cscv"] + assert len(recorded) == 1 + summary = recorded[0].summary + for field in ( + "pbo", + "degradation_slope", + "degradation_intercept", + "prob_loss", + "dominance_1st", + "dominance_2nd", + "n_columns", + "n_blocks", + "n_combinations", + "rows_used", + "rows_dropped", + ): + assert field in summary, f"{field} was computed and not recorded" + + +def test_a_recorded_cscv_row_can_never_become_a_column_in_the_next_run(tmp_path) -> None: + """The row is a measurement ABOUT a set of columns, not a trial with a series of its own. + `matrix.build_matrix` refuses `series_missing` rows, so recording one cannot feed it back into + the next PBO over the same file -- a diagnostic that changed the thing it measured.""" + from keel.research import matrix as matrix_mod + + path = tmp_path / "ledger.jsonl" + _seeded_columns(path) + + # `--session s1`, so the recorded row lands in the SAME session the matrix is built from. + # Without it the row is written under session "all" and the session filter excludes it -- the + # test would then pass whether or not `series_missing` did any work, which is how the first + # version of this passed against a mutant that gave the row a real series. + for _ in range(2): + outcome = CliRunner().invoke( + cli, + ["trials", "pbo", "--ledger", str(path), "--session", "s1", "--blocks", "4"], + ) + assert outcome.exit_code == 0, outcome.output + + recorded = [row for row in ledger.read_trials(path) if row.kind == "cscv"] + assert len(recorded) == 2, "the premise: both runs recorded into session s1" + assert all(row.session == "s1" for row in recorded) + assert all(row.series_missing for row in recorded) + + assert len(matrix_mod.build_matrix(ledger.read_trials(path), session="s1").columns) == 4 + + +def test_trials_deflate_records_the_inputs_the_operator_supplied(tmp_path) -> None: + """`--sharpe` is a REQUIRED operator input and the ledger stores no per-trial Sharpe, so DSR + is impossible to recompute later without synthesising it. Recording the inputs at the moment + they were stated turns the figure into one that can be CHECKED.""" + path = tmp_path / "ledger.jsonl" + _seeded_columns(path) + + result = CliRunner().invoke( + cli, + [ + "trials", "deflate", "--ledger", str(path), + "--sharpe", "1.8", "--trial-sharpe-variance", "0.25", "--rho", "0.5", + ], + ) + assert result.exit_code == 0, result.output + + (recorded,) = [row for row in ledger.read_trials(path) if row.kind == "deflated_sharpe"] + for field in ( + "observed_annual_sharpe", + "trades_per_year", + "skewness", + "kurtosis", + "trial_sharpe_variance", + "dsr", + "expected_max_sharpe", + "min_trades", + ): + assert field in recorded.summary, f"{field} is needed to recompute DSR and is not stored" + assert recorded.summary["observed_annual_sharpe"] == Decimal("1.8") + + +def test_deflate_without_the_variance_records_nothing_rather_than_a_guess(tmp_path) -> None: + """The command already refuses to COMPUTE a DSR it has no variance for. It must not record a + row implying it did -- a stored figure nobody ran is worse than an honest gap.""" + path = tmp_path / "ledger.jsonl" + _seeded_columns(path) + + result = CliRunner().invoke( + cli, ["trials", "deflate", "--ledger", str(path), "--sharpe", "1.8"] + ) + assert result.exit_code == 0, result.output + assert "NOT COMPUTED" in result.output + assert [row for row in ledger.read_trials(path) if row.kind == "deflated_sharpe"] == [] + + +def test_every_recorded_gauntlet_row_keeps_the_chain_intact(tmp_path) -> None: + """All three writers append to a hash-chained, append-only file. A row that broke the chain + would take the whole record with it.""" + path = tmp_path / "ledger.jsonl" + _seeded_columns(path) + CliRunner().invoke(cli, ["trials", "pbo", "--ledger", str(path), "--blocks", "4"]) + CliRunner().invoke( + cli, + ["trials", "deflate", "--ledger", str(path), "--sharpe", "1.8", + "--trial-sharpe-variance", "0.25"], + ) + + assert ledger.verify_chain(path) == [] + + +def test_monte_carlo_records_the_distributions_SHAPE_not_only_its_ends(tmp_path): + """#726. `distribution_min/median/max` say how far the resampling reached; the ladder says + what its shape WAS -- which is what a histogram needs and what nothing recorded until now, so + #708's Monte Carlo panel had no stored figures and would have had to re-run a backtest inside + a web request. + + Flat keys, because `ledger._validate_summary` refuses a nested value: one would make this + append-only file unreadable on the next read, permanently. + """ + from keel.research.montecarlo import QUANTILE_LADDER + + db = _mc_db(tmp_path) + ledger_path = tmp_path / "trials.jsonl" + result = _invoke_mc( + CliRunner(), db, ledger_path, "--mode", "trades", "--paths", "40", "--seed", "7" + ) + assert result.exit_code == 0, result.output + + (row,) = trials_ledger.read_trials(ledger_path) + for percent in QUANTILE_LADDER: + for prefix in ("final", "drawdown"): + key = f"{prefix}_p{percent:02d}" + assert key in row.summary, f"{key} was computed and not recorded" + assert isinstance(row.summary[key], Decimal) + + # A LADDER, not seven copies of one number: a distribution whose quantiles were all equal + # would satisfy a presence check and describe nothing. + ladder = [row.summary[f"drawdown_p{percent:02d}"] for percent in QUANTILE_LADDER] + assert ladder == sorted(ladder) + assert len(set(ladder)) > 1, "every drawdown quantile is identical -- the ladder says nothing"