feat(research): persist what the gauntlet computed instead of printing it (#726)

keel/commands/trials.py

@@ -215,6 +215,41 @@ def trials_deflate(
     click.echo(f"\nSR_0 (rejection bar)      : {sr0:.4f}")
     click.echo(f"DSR                       : {dsr:.4f}")
 
+    # #726. THE INPUTS AS WELL AS THE OUTPUTS, and the inputs are the point.
+    #
+    # `--sharpe` is a REQUIRED operator input: the ledger stores no observed annualised Sharpe per
+    # trial, so DSR is not merely expensive to recompute later -- it is impossible without
+    # synthesising a number the operator supplied, which is the one thing this codebase refuses.
+    # Recording them at the moment they were stated turns DSR from a re-run into a read, and lets
+    # the figure be CHECKED rather than trusted.
+    recorded = trials_ledger.append_trial(
+        _ledger_path(ledger),
+        trial_id=f"dsr-{n_decisions}n-sr{sharpe:g}",
+        session="deflate",
+        rule="(deflated sharpe over the recorded decisions)",
+        params={"rho": rho, "n_decisions": n_decisions},
+        provenance="a_priori",
+        kind="deflated_sharpe",
+        decision="diagnostic_only",
+        series_missing=True,
+        summary={
+            "observed_annual_sharpe": Decimal(str(sharpe)),
+            "trades_per_year": Decimal(str(trades_per_year)),
+            "skewness": Decimal(str(skew)),
+            "kurtosis": Decimal(str(kurtosis)),
+            "trial_sharpe_variance": Decimal(str(trial_sharpe_variance)),
+            "m_total": m_total,
+            "n_decisions": n_decisions,
+            "n_hat": Decimal(str(n_hat)),
+            "expected_max_sharpe": Decimal(str(deflate_mod.expected_max_sharpe(effective))),
+            "sharpe_rejection_threshold": Decimal(str(sr0)),
+            "min_trades": Decimal(str(deflate_mod.min_trades(effective, sharpe, trades_per_year))),
+            "observations": observations,
+            "dsr": Decimal(str(dsr)),
+        },
+    )
+    click.echo(f"\nrecorded {recorded.trial_id} hash={recorded.row_hash[:12]}")
+
 
 @trials_group.command("pbo")
 @_LEDGER_OPTION
@@ -258,6 +293,44 @@ def trials_pbo(ledger: Path | None, session: str | None, blocks: int) -> None:
         "near-identical configurations produces high PBO by construction."
     )
 
+    # #726. Ten figures were computed and one reached the ledger, in prose. Recording the rest is
+    # what lets a reader ask "how overfit was this?" without a 12-second re-run -- measured on the
+    # real ledger, three sessions cost ~39 s of CPU, on a page that polls every 15 s.
+    #
+    # `series_missing=True` and always: this row is a measurement ABOUT a set of columns, not a
+    # trial with a series of its own, and `matrix.build_matrix` refuses `series_missing` rows --
+    # so a recorded PBO can never become a column in the next PBO run over the same file.
+    #
+    # ⛔ THE STRATHERN RAIL. Every figure here is a diagnostic. Storing them makes them easier to
+    # rank by, which is exactly why `PBOResult` carries no configuration field and why nothing
+    # written here names a winning parameter set.
+    recorded = trials_ledger.append_trial(
+        _ledger_path(ledger),
+        trial_id=f"cscv-{session or 'all'}-s{blocks}",
+        session=session or "all",
+        rule="(cscv over the recorded columns)",
+        params={"session": session, "blocks": blocks},
+        provenance="a_priori",
+        kind="cscv",
+        decision="diagnostic_only",
+        series_missing=True,
+        summary={
+            "pbo": result.pbo,
+            "degradation_slope": result.degradation_slope,
+            "degradation_intercept": result.degradation_intercept,
+            "prob_loss": result.prob_loss,
+            "dominance_1st": result.dominance_1st,
+            "dominance_2nd": result.dominance_2nd,
+            "n_columns": result.n_columns,
+            "n_blocks": result.n_blocks,
+            "n_combinations": result.n_combinations,
+            "rows_used": result.rows_used,
+            "rows_dropped": result.rows_dropped,
+            "columns_refused": len(build.refused),
+        },
+    )
+    click.echo(f"\nrecorded {recorded.trial_id} hash={recorded.row_hash[:12]}")
+
 
 # -- monte-carlo resampling (#441) ----------------------------------------------------------------
 #
@@ -482,6 +555,16 @@ def trials_monte_carlo(
                 "drawdown_percentile": report.drawdown_percentile,
                 "n_trades": len(pnls),
                 "n_paths": paths,
+                # #726: the DISTRIBUTION, not just its ends. `distribution_min/median/max` say
+                # how far the resampling reached; the ladder says what its shape was, which is
+                # what a histogram needs and what nothing recorded until now -- so #708's Monte
+                # Carlo panel had no stored figures to draw and would have had to re-run a
+                # backtest inside a web request.
+                #
+                # FLAT keys, never a nested ladder: `ledger._validate_summary` refuses anything
+                # else, because one nested value makes this append-only file unreadable forever.
+                **mc_mod.quantile_ladder(finals, "final"),
+                **mc_mod.quantile_ladder(drawdowns, "drawdown"),
             },
         )
     except ValueError as exc:

keel/research/ledger.py

@@ -48,6 +48,14 @@
         "threshold_nudge",
         "monte_carlo",
         "walk_forward",
+        # #726. Two more DIAGNOSTIC kinds, for the two gauntlet components that computed a full
+        # result and printed it. `cscv` is a PBO run over a session's columns; `deflated_sharpe`
+        # is one E[max SR]/MinBTL/DSR evaluation under the operator's stated inputs. Both are
+        # measurements ABOUT a set of trials rather than trials themselves, which is why both are
+        # always `series_missing` -- and `matrix.build_matrix` refuses `series_missing` rows, so
+        # neither can ever become a column in a later PBO run over the same ledger.
+        "cscv",
+        "deflated_sharpe",
     }
 )
 DECISIONS = frozenset({"selected", "rejected", "diagnostic_only"})
@@ -128,13 +136,39 @@ def compute_row_hash(record: TrialRecord) -> str:
     return chain_hash(_row_payload(record))
 
 
+#: What a `summary` value may be. Everything else is refused at append time.
+#:
+#: `_decode_summary` maps a value to `None`, an `int` or `Decimal(value)`. A list raises
+#: `ValueError` there and a dict raises `TypeError` -- and it raises on READ, inside
+#: `read_trials`, which every later `verify_chain`, `trials list`, `trials pbo` and web page goes
+#: through. ONE such row would make an APPEND-ONLY file unreadable forever, with no way to take it
+#: back. That is not hypothetical: this module's own `_decode_summary` records a null value doing
+#: exactly this once already.
+#:
+#: So the check is here, at the write, where it is a refusal rather than a catastrophe. It is also
+#: why #726's gauntlet artifacts are FLAT keys -- `final_p05`, `final_p50` -- rather than a nested
+#: quantile ladder: the shape a reader can survive is the shape a writer may use.
+_SUMMARY_SCALARS = (Decimal, int, float, str, bool)
+
+
+def _validate_summary(summary: Mapping[str, Any]) -> None:
+    for key, value in summary.items():
+        if value is not None and not isinstance(value, _SUMMARY_SCALARS):
+            raise ValueError(
+                f"summary[{key!r}] is a {type(value).__name__}; a summary value must be a scalar "
+                "or None. A nested value would make this append-only ledger unreadable on the "
+                "next read_trials, permanently -- store a flat key per figure instead"
+            )
+
+
 def _validate(record: TrialRecord) -> None:
     if record.provenance not in PROVENANCE:
         raise ValueError(f"provenance: {record.provenance!r} not in {sorted(PROVENANCE)}")
     if record.kind not in KINDS:
         raise ValueError(f"kind: {record.kind!r} not in {sorted(KINDS)}")
     if record.decision not in DECISIONS:
         raise ValueError(f"decision: {record.decision!r} not in {sorted(DECISIONS)}")
+    _validate_summary(record.summary)
     if not record.series_missing and not (record.per_trade_pnl or record.per_bar_pnl):
         raise ValueError(
             "series_missing is False but no P&L series was supplied; a trial with no series "

keel/research/montecarlo.py

@@ -34,6 +34,7 @@
 
 from __future__ import annotations
 
+import math
 import random
 from collections.abc import Sequence
 from dataclasses import dataclass
@@ -115,6 +116,39 @@ def max_drawdown(curve: Sequence[Decimal]) -> Decimal:
     return deepest
 
 
+#: The quantile ladder every resampled distribution is stored at (#726).
+#:
+#: A FIXED ladder, and a small one. The raw array of `n_paths` finals is what a chart would love
+#: and what an append-only text file must not carry: at the default path count it is thousands of
+#: Decimals per row, and the ledger is git-tracked. Seven quantiles are what a histogram or a
+#: polyline actually needs, are bounded, and are the same seven whatever `--paths` was.
+#:
+#: Stored FLAT (`final_p05`, `final_p50`, ...), never as a nested list: `ledger._validate_summary`
+#: refuses anything else, because a nested value makes the whole file unreadable on the next read.
+QUANTILE_LADDER: tuple[int, ...] = (1, 5, 25, 50, 75, 95, 99)
+
+
+def quantile(values: Sequence[Decimal], percent: int) -> Decimal:
+    """The `percent`-th quantile by NEAREST-RANK, on the sorted values.
+
+    Nearest-rank rather than interpolating: every value in this distribution is a resampled
+    equity that the model actually produced, and an interpolated quantile is a number no path
+    reached. For a distribution being read as "what could have happened", that distinction is the
+    whole point -- the same reason `median` below averages the two middle values only for an even
+    count, where no single observation is the middle.
+    """
+    if not values:
+        raise ValueError("no values to take a quantile of")
+    ordered = sorted(values)
+    rank = max(1, math.ceil(Decimal(percent) / Decimal(100) * Decimal(len(ordered))))
+    return ordered[int(rank) - 1]
+
+
+def quantile_ladder(values: Sequence[Decimal], prefix: str) -> dict[str, Decimal]:
+    """`{f"{prefix}_p05": ..., ...}` over `QUANTILE_LADDER` -- the flat form the ledger stores."""
+    return {f"{prefix}_p{percent:02d}": quantile(values, percent) for percent in QUANTILE_LADDER}
+
+
 def median(values: Sequence[Decimal]) -> Decimal:
     """Exact-Decimal median: the middle of the sorted values for odd length, the mean of the
     two middles for even length. Raises on empty input rather than inventing a zero."""

tests/research/test_ledger.py

@@ -229,3 +229,65 @@ def test_the_tracked_ledger_still_verifies_after_the_canonicaliser_moved() -> No
     # errors for zero rows, which is not the same as "verified" (see `verify_records`).
     assert len(records) > 1, "the tracked ledger must hold a chain, not a single row"
     assert ledger.verify_chain(tracked) == []
+
+
+# -- the summary is FLAT, and the guard is at write time (#726) -----------------------------------
+
+
+def test_a_nested_summary_value_is_refused_at_append_time(tmp_path) -> None:
+    """The reason #726's quantile ladder is flat keys rather than a list.
+
+    `_decode_summary` maps every summary value to `None`, an `int`, or `Decimal(value)`. A list
+    raises `ValueError` and a dict raises `TypeError` -- and it raises on READ, in `read_trials`,
+    which every later `verify_chain`, `trials list`, `trials pbo` and web page goes through. One
+    such row would make an APPEND-ONLY file unreadable forever, with no way to take it back.
+
+    That is not hypothetical: the module's own docstring records a null summary value doing
+    exactly this once already ("one such row made every later read_trials/verify_chain of the
+    append-only chain raise forever").
+
+    So the check moved to the write, where it is still a refusal rather than a catastrophe.
+    """
+    path = tmp_path / "ledger.jsonl"
+    for value in ([Decimal("1")], {"p50": Decimal("1")}, (1, 2)):
+        with pytest.raises(ValueError, match="summary"):
+            ledger.append_trial(
+                path,
+                trial_id="t1",
+                session="s",
+                rule="r",
+                provenance="a_priori",
+                kind="sweep_node",
+                decision="diagnostic_only",
+                series_missing=True,
+                summary={"quantiles": value},
+            )
+    assert not path.exists(), "a refused append must not have written a row"
+
+
+def test_the_scalar_summary_values_the_gauntlet_writes_all_round_trip(tmp_path) -> None:
+    """Everything #726 stores: Decimals, ints, bools and explicit nulls."""
+    path = tmp_path / "ledger.jsonl"
+    ledger.append_trial(
+        path,
+        trial_id="t1",
+        session="s",
+        rule="r",
+        provenance="a_priori",
+        kind="sweep_node",
+        decision="diagnostic_only",
+        series_missing=True,
+        summary={
+            "pbo": Decimal("0.8810"),
+            "n_columns": 12,
+            "dominance_1st": True,
+            "trial_sharpe_variance": None,
+        },
+    )
+    (stored,) = ledger.read_trials(path)
+
+    assert stored.summary["pbo"] == Decimal("0.8810")
+    assert stored.summary["n_columns"] == 12
+    assert stored.summary["dominance_1st"] is True
+    assert stored.summary["trial_sharpe_variance"] is None
+    assert ledger.verify_chain(path) == []

tests/research/test_montecarlo.py

@@ -20,6 +20,7 @@
 
 import pytest
 
+from keel.research import montecarlo
 from keel.research.montecarlo import (
     MonteCarloReport,
     equity_curve,
@@ -299,3 +300,49 @@ def test_a_reshuffled_sample_reports_the_median_by_construction() -> None:
     finals = final_equities(reshuffle(pnls, 30, seed=8), Decimal(0))
     assert min(finals) == observed == max(finals)
     assert percentile_of(observed, finals) == Decimal("0.5")
+
+
+# -- the quantile ladder (#726) --------------------------------------------------------------------
+
+
+def test_a_quantile_is_a_value_the_distribution_actually_produced() -> None:
+    """NEAREST-RANK, not interpolation.
+
+    Every value here is a resampled equity the model actually reached, and an interpolated
+    quantile is a number no path produced. For a distribution being read as "what could have
+    happened", that distinction is the whole point.
+    """
+    from decimal import Decimal as D
+
+    values = [D("1"), D("2"), D("3"), D("4")]
+    for percent in (1, 5, 25, 50, 75, 95, 99):
+        assert montecarlo.quantile(values, percent) in values
+
+
+def test_the_ladder_is_ordered_and_spans_the_distribution() -> None:
+    from decimal import Decimal as D
+
+    values = [D(str(n)) for n in range(1, 101)]
+    ladder = montecarlo.quantile_ladder(values, "final")
+
+    ordered = [ladder[f"final_p{percent:02d}"] for percent in montecarlo.QUANTILE_LADDER]
+    assert ordered == sorted(ordered)
+    assert ordered[0] == D("1")
+    assert ordered[-1] == D("99")
+
+
+def test_the_ladder_keys_are_flat_and_zero_padded() -> None:
+    """`final_p05`, not `final_p5`: the keys sort lexically in the order they read, and the ledger
+    stores them side by side with every other scalar."""
+    from decimal import Decimal as D
+
+    ladder = montecarlo.quantile_ladder([D("1"), D("2")], "final")
+    assert sorted(ladder) == [
+        "final_p01", "final_p05", "final_p25", "final_p50", "final_p75", "final_p95", "final_p99"
+    ]
+    assert all(isinstance(value, D) for value in ladder.values())
+
+
+def test_a_quantile_of_nothing_is_refused_rather_than_invented() -> None:
+    with pytest.raises(ValueError, match="quantile"):
+        montecarlo.quantile([], 50)