From d31a6f628c341cc98fa180522abd817980627a18 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 01:49:49 +0200 Subject: [PATCH 01/47] deps: py3.12 floor + SACC/blinding stack (sacc, firecrown, smokescreen) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PRD #241 row 1. sacc>=2.4 joins core dependencies (sp_validation.sacc_io will be core library code). firecrown v1.15.1 + smokescreen 1.5.6 + an exact pyccl pin form the new [blinding] extra: the blind must be exactly recomputable from the seed at unblinding time, so the theory stack is pinned as a set. firecrown is not on PyPI and hard-depends on numcosmo-py (conda-forge only) plus the cosmosis/cobaya sampler connectors we never import — the new uv-overrides.txt drops those three from resolution; the Dockerfile and README carry the --overrides invocation. Plain installs without the blinding extra are unaffected. requires-python moves 3.11 → 3.12: Smokescreen and firecrown both set a 3.12 floor, and the container base (shapepipe:develop) already runs python:3.12-slim-bookworm — this aligns pyproject with the actual runtime. ruff target-version follows. Validated: uv resolution of '.[test,glass,blinding]' with overrides (235 packages, py3.12) and the full firecrown-core + smokescreen import chain in a fresh venv on candide. The CI image build on this branch is the container-side proof. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_019bGhVMAuhTy1gF6DdUc1dQ --- Dockerfile | 13 ++++++++----- README.md | 19 +++++++++++++++++++ pyproject.toml | 27 +++++++++++++++++++++++++-- uv-overrides.txt | 21 +++++++++++++++++++++ 4 files changed, 73 insertions(+), 7 deletions(-) create mode 100644 uv-overrides.txt diff --git a/Dockerfile b/Dockerfile index c791877c..0ebb57a8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,8 +28,11 @@ RUN uv pip install --no-cache-dir --upgrade 'cs_util>=0.2.1' WORKDIR /sp_validation COPY . /sp_validation -# Install with the test + glass extras so the image can run the unit suite in CI -# *and* the GLASS map-level mock test. `glass` (Generator for Large Scale -# Structure) ships `glass.ext.camb`; `cosmology` provides the `Cosmology` wrapper -# (`Cosmology.from_camb`) GLASS consumes. Both come in via the `[glass]` extra. -RUN uv pip install --no-cache-dir -e '.[test,glass]' +# Install with the test + glass + blinding extras so the image can run the unit +# suite in CI, the GLASS map-level mock test, *and* the SACC/Smokescreen blinding +# stack. `glass` (Generator for Large Scale Structure) ships `glass.ext.camb`; +# `cosmology` provides the `Cosmology` wrapper (`Cosmology.from_camb`) GLASS +# consumes. The `[blinding]` extra (firecrown + smokescreen) needs the override +# file: firecrown declares conda-forge-only / unused sampler connectors as hard +# deps — see uv-overrides.txt for the full story. +RUN uv pip install --no-cache-dir --overrides uv-overrides.txt -e '.[test,glass,blinding]' diff --git a/README.md b/README.md index e14a2071..640c6b20 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,25 @@ docker run --rm -it ghcr.io/cosmostat/sp_validation:develop python -c "import sp We do not currently build images for Apple Silicon/arm64; however the amd64 images should work on these systems, albeit with reduced performance. +## Local Installation + +Requires Python ≥ 3.12 (the floor is set by the blinding stack; the container +already runs 3.12). With [uv](https://docs.astral.sh/uv/): + +```bash +uv venv --python 3.12 +uv pip install -e '.[test]' +``` + +To also install the data-vector blinding stack (Smokescreen + firecrown, PRD +[#241](https://github.com/CosmoStat/sp_validation/issues/241)), pass the +dependency-override file — firecrown is not pip-resolvable without it (see +`uv-overrides.txt` for why): + +```bash +uv pip install --overrides uv-overrides.txt -e '.[test,blinding]' +``` + ## Flow chart diff --git a/pyproject.toml b/pyproject.toml index bb1fe413..b282d9ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,10 @@ authors = [ ] license = {text = "MIT"} readme = "README.md" -requires-python = ">=3.11" +# 3.12 floor set by Smokescreen 1.5.6 (and firecrown v1.15); the container base +# (shapepipe:develop) is already python:3.12-slim-bookworm, so this aligns +# pyproject with the actual runtime. +requires-python = ">=3.12" classifiers = [ "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3", @@ -52,6 +55,10 @@ dependencies = [ "pymaster", "regions", "reproject", + # SACC (LSST DESC's data-vector container) is the standard format for all + # data products from the tomographic round on (PRD #241); sp_validation.sacc_io + # is core library code, so sacc is a core dependency. + "sacc>=2.4,<3", # scipy 1.18 ported FITPACK from Fortran to C, changing the return shape of # RectBivariateSpline(scalar, scalar, grid=False) from 0-d `array(x)` to # shape-(1,) `array([x])`. camb's BBN Y_He predictor (bbn.py) wraps the @@ -106,6 +113,22 @@ glass = [ "glass.ext.camb==2023.6", "cosmology==2022.10.9", ] +# Data-vector blinding (PRD #241 §3-§5): Smokescreen applies the Muir et al. +# shift d → d + t(hidden) − t(fid), with firecrown + CCL as the theory engine +# (only compute_theory_vector is used; sampling stays with CosmoSIS). The blind +# must be exactly recomputable from the seed at unblinding time, so the whole +# theory stack is pinned exactly, as a set. Smokescreen 1.5.6 + firecrown v1.15 +# both set the python floor (>=3.12). +# +# firecrown is not on PyPI and declares conda-forge-only / unused sampler +# connectors as hard deps, so installing this extra requires the dependency +# override file: `uv pip install --overrides uv-overrides.txt -e '.[blinding]'` +# (see uv-overrides.txt; the Dockerfile does this for the container). +blinding = [ + "firecrown @ git+https://github.com/LSSTDESC/firecrown.git@v1.15.1", + "smokescreen==1.5.6", + "pyccl==3.3.4", +] develop = ["sp_validation[test,docs]"] [tool.pytest.ini_options] @@ -124,7 +147,7 @@ markers = [ [tool.ruff] line-length = 88 -target-version = "py311" +target-version = "py312" # Snakemake injects a `snakemake` object into rule scripts at runtime, so ruff # can't see where it's defined. Declaring it a builtin silences the false diff --git a/uv-overrides.txt b/uv-overrides.txt new file mode 100644 index 00000000..813ad055 --- /dev/null +++ b/uv-overrides.txt @@ -0,0 +1,21 @@ +# uv dependency overrides — pass via `--overrides uv-overrides.txt` (or +# UV_OVERRIDE=uv-overrides.txt) to every `uv pip install` against this project. +# +# Why this file exists: firecrown declares its sampler *connectors* as hard +# dependencies, but we use firecrown only as the theory engine for Smokescreen +# blinding (`compute_theory_vector`); sampling stays with CosmoSIS in +# cosmo_inference. Of the three connector deps: +# +# - numcosmo-py exists only on conda-forge, so pip/uv resolution of firecrown +# is *impossible* without an override; +# - cosmosis ships sdist-only (full Fortran/C build with gsl/cfitsio) — a +# heavy, fragile compile in every CI image build, for a connector we never +# import; +# - cobaya is wheel-clean but equally unused. +# +# Each line below replaces the package's requirement (wherever it appears in +# the graph) with one gated on an always-false marker, dropping it from +# resolution. firecrown's likelihood/CCL core imports none of them. +numcosmo-py; python_version < "3" +cosmosis; python_version < "3" +cobaya; python_version < "3" From 1d4826c8abb162a473d5761822361b4023f9f14b Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 02:01:48 +0200 Subject: [PATCH 02/47] deps: firecrown pip-import fixes (numpy<2.5 cap, NumCosmo patch script) Follow-up hardening after empirical validation of the blinding stack on candide. Two facts surfaced that the resolve-only check couldn't see: - firecrown 1.15.1 subclasses npt.NDArray (DataVector); numpy 2.5 made that a non-subclassable typing alias, so firecrown breaks at import on the numpy>=2.0 resolution (2.5.1). The [blinding] extra now carries numpy>=2.2,<2.5 (2.4.3 verified against the full compiled stack + the fast suite), and the Dockerfile requests the bound explicitly to dodge uv #8410 non-movement of an already-installed numpy. - pip-installed firecrown hits NumCosmo (conda-forge-only) at import time through two paths unrelated to cosmic shear: eager re-export of LSST predefined n(z) bins in generators/__init__ (defeating upstream's own lazy __getattr__), and the cluster likelihoods -> lsstdesc-crow -> Ncm.IntegralND C-subclass at module load. scripts/patch_firecrown.py makes the bin re-export lazy, the cluster imports optional, and ships a loud numcosmo_py shim (raises on any real use). Exact-string surgery against pinned v1.15.1, idempotent, fails loudly on a version bump, ends with an import check. Verified: fresh application on a pristine install, idempotent re-run, and the full toy Smokescreen blind end-to-end after patching. CI now also smoke-tests the blinding imports in the built image - the fast suite never imports firecrown, so a broken blinding stack would otherwise ship green. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_019bGhVMAuhTy1gF6DdUc1dQ --- .github/workflows/deploy-image.yml | 6 + Dockerfile | 14 ++ README.md | 1 + pyproject.toml | 9 ++ scripts/patch_firecrown.py | 225 +++++++++++++++++++++++++++++ 5 files changed, 255 insertions(+) create mode 100644 scripts/patch_firecrown.py diff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.yml index 192e87a6..13f0fafe 100644 --- a/.github/workflows/deploy-image.yml +++ b/.github/workflows/deploy-image.yml @@ -44,6 +44,12 @@ jobs: - name: Import smoke test run: docker run --rm ${{ steps.meta.outputs.tags }} python -c "import sp_validation" + # The fast suite doesn't import the blinding stack, so a broken + # firecrown/smokescreen install would otherwise ship green. Prove the + # image can actually load it (sacc + patched firecrown + smokescreen). + - name: Blinding-stack import smoke test + run: docker run --rm ${{ steps.meta.outputs.tags }} python -c "import sacc; import firecrown.likelihood; import smokescreen" + # Run the fast test suite against the freshly-built image *before* # pushing, so a failing suite blocks publication. The image carries the # full stack and the test files (COPY . + editable install), so this diff --git a/Dockerfile b/Dockerfile index 0ebb57a8..818db67d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,3 +36,17 @@ COPY . /sp_validation # file: firecrown declares conda-forge-only / unused sampler connectors as hard # deps — see uv-overrides.txt for the full story. RUN uv pip install --no-cache-dir --overrides uv-overrides.txt -e '.[test,glass,blinding]' + +# Same uv gotcha as the cs_util upgrade above (astral-sh/uv #8410): if the base +# image already carries a numpy that violates the [blinding] extra's new +# `numpy<2.5` cap (firecrown 1.15.1 breaks on numpy 2.5 at import), the +# editable install won't move it. Request the bound explicitly so the image is +# deterministic either way; numpy 2.4.x is ABI-compatible with the compiled +# stack (verified: pyccl/camb/treecorr/healpy/pymaster + fast suite). +RUN uv pip install --no-cache-dir 'numpy>=2.2,<2.5' + +# firecrown is distributed for conda-forge (where NumCosmo always exists) and +# hits NumCosmo at import time in a pip env, on paths unrelated to our use. +# This patches the installed tree (surgical, pinned-version-checked, loud on +# mismatch) and verifies `import firecrown.likelihood; import smokescreen`. +RUN python scripts/patch_firecrown.py diff --git a/README.md b/README.md index 640c6b20..ef9e601b 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,7 @@ dependency-override file — firecrown is not pip-resolvable without it (see ```bash uv pip install --overrides uv-overrides.txt -e '.[test,blinding]' +python scripts/patch_firecrown.py # make pip-installed firecrown importable without NumCosmo ``` diff --git a/pyproject.toml b/pyproject.toml index b282d9ad..ecdec6f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -124,10 +124,19 @@ glass = [ # connectors as hard deps, so installing this extra requires the dependency # override file: `uv pip install --overrides uv-overrides.txt -e '.[blinding]'` # (see uv-overrides.txt; the Dockerfile does this for the container). +# After installing this extra, run `python scripts/patch_firecrown.py` — it +# makes pip-installed firecrown importable without NumCosmo (conda-forge-only); +# see that script's docstring for the full story. blinding = [ "firecrown @ git+https://github.com/LSSTDESC/firecrown.git@v1.15.1", "smokescreen==1.5.6", "pyccl==3.3.4", + # firecrown 1.15.1 subclasses npt.NDArray (DataVector); numpy 2.5 turned + # npt.NDArray into a non-subclassable typing alias, breaking firecrown at + # import. firecrown's own env caps numpy<2.4; 2.4.3 is verified against + # the full compiled stack (pyccl/camb/treecorr/healpy/pymaster) + the + # sp_validation fast suite. + "numpy>=2.2,<2.5", ] develop = ["sp_validation[test,docs]"] diff --git a/scripts/patch_firecrown.py b/scripts/patch_firecrown.py new file mode 100644 index 00000000..7b528da1 --- /dev/null +++ b/scripts/patch_firecrown.py @@ -0,0 +1,225 @@ +"""Make pip-installed firecrown importable without NumCosmo. + +Run *inside* the target environment, after installing the ``[blinding]`` extra: + + python scripts/patch_firecrown.py + +Why this exists (PRD #241, PR 1): firecrown is the theory engine for +Smokescreen blinding — only ``compute_theory_vector`` on the SACC-read +cosmic-shear path is used. Upstream distributes firecrown via conda-forge, +where NumCosmo (a GObject-introspection C library, absent from PyPI) is always +present; in a pip/uv environment, firecrown 1.15.1 hits NumCosmo at *import +time* through two paths that have nothing to do with cosmic shear: + +1. ``firecrown/generators/__init__.py`` eagerly re-exports the LSST Y1/Y10 + predefined n(z) bin constants, defeating the lazy ``__getattr__`` that + ``_inferred_galaxy_zdist`` already provides — and computing those constants + imports NumCosmo. +2. ``firecrown/likelihood/__init__.py`` eagerly imports the cluster + likelihoods, which import ``crow`` (lsstdesc-crow), which subclasses a + NumCosmo C class at module load (``class CountsIntegralND(Ncm.IntegralND)``). + +This script (a) restores laziness in ``generators``, (b) makes the cluster +imports optional, and (c) installs a *loud* ``numcosmo_py`` shim so that any +genuine NumCosmo use raises immediately instead of being silently faked. +Everything is exact-string surgery against the pinned firecrown v1.15.1: if a +target string is missing (e.g. after a version bump), the script fails loudly +so the pin and the patch get reviewed together. Idempotent — safe to re-run. + +The right long-term fix is upstream (guarded/lazy imports in firecrown); until +then this file is the entire cost of staying pip-installable. +""" + +import importlib.metadata +import importlib.util +import subprocess +import sys +from pathlib import Path + +EXPECTED_FIRECROWN = "1.15.1" + +GENERATORS_OLD = """\ + # Lazy-loaded bins (via __getattr__) + Y1_LENS_BINS, + Y1_SOURCE_BINS, + Y10_LENS_BINS, + Y10_SOURCE_BINS, + LSST_Y1_LENS_HARMONIC_BIN_COLLECTION, + LSST_Y1_SOURCE_HARMONIC_BIN_COLLECTION, + LSST_Y10_LENS_HARMONIC_BIN_COLLECTION, + LSST_Y10_SOURCE_HARMONIC_BIN_COLLECTION, +) +""" + +GENERATORS_NEW = """\ +) + +# NOTE (sp_validation patch, scripts/patch_firecrown.py): the LSST Y1/Y10 +# predefined bin constants are computed lazily in _inferred_galaxy_zdist via a +# module-level __getattr__ that imports NumCosmo. Importing them EAGERLY here +# forced NumCosmo at `import firecrown.generators` (hence at +# `import firecrown.likelihood`), which pip cannot satisfy. Re-expose them +# lazily instead; the SACC-read cosmic-shear path never touches them. +_LAZY_BIN_NAMES = frozenset( + { + "Y1_LENS_BINS", + "Y1_SOURCE_BINS", + "Y10_LENS_BINS", + "Y10_SOURCE_BINS", + "LSST_Y1_LENS_HARMONIC_BIN_COLLECTION", + "LSST_Y1_SOURCE_HARMONIC_BIN_COLLECTION", + "LSST_Y10_LENS_HARMONIC_BIN_COLLECTION", + "LSST_Y10_SOURCE_HARMONIC_BIN_COLLECTION", + } +) + + +def __getattr__(name): + if name in _LAZY_BIN_NAMES: + from . import _inferred_galaxy_zdist as _z + + return getattr(_z, name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + +""" + +LIKELIHOOD_OLD = """\ +# Cluster statistics +from firecrown.likelihood._binned_cluster import BinnedCluster +from firecrown.likelihood._binned_cluster_number_counts import ( + BinnedClusterNumberCounts, +) +from firecrown.likelihood._binned_cluster_number_counts_shear import ( + BinnedClusterShearProfile, +) +""" + +LIKELIHOOD_NEW = """\ +# Cluster statistics. +# NOTE (sp_validation patch, scripts/patch_firecrown.py): the cluster +# likelihoods import `crow` (lsstdesc-crow), which subclasses NumCosmo C +# classes at module load. NumCosmo is conda-forge-only, so in a pip/uv env +# these imports fail. They are NOT on the cosmic-shear (TwoPoint/WeakLensing) +# path, so they become optional: without NumCosmo the cluster classes are +# unavailable but everything else loads. +try: + from firecrown.likelihood._binned_cluster import BinnedCluster + from firecrown.likelihood._binned_cluster_number_counts import ( + BinnedClusterNumberCounts, + ) + from firecrown.likelihood._binned_cluster_number_counts_shear import ( + BinnedClusterShearProfile, + ) +except (ImportError, RuntimeError, TypeError): # pragma: no cover + BinnedCluster = None # type: ignore[assignment,misc] + BinnedClusterNumberCounts = None # type: ignore[assignment,misc] + BinnedClusterShearProfile = None # type: ignore[assignment,misc] +""" + +SHIM = '''\ +"""Minimal loud shim for numcosmo_py (installed by sp_validation). + +NumCosmo is a GObject-introspection C library available only via conda-forge. +With the companion patches to firecrown (scripts/patch_firecrown.py), the +SACC-read cosmic-shear likelihood path never imports it; this shim provides +the import-time names so the patched package loads, and any genuine numerical +use of NumCosmo raises loudly rather than being silently faked. +""" + + +class _Missing: + def __init__(self, path="numcosmo_py"): + self._p = path + + def __getattr__(self, name): + return _Missing(f"{self._p}.{name}") + + def __call__(self, *a, **k): + raise RuntimeError( + f"{self._p} was called, but NumCosmo is not installed (conda-forge " + "only, not on PyPI). It is not needed for the SACC-read " + "cosmic-shear likelihood path." + ) + + def __getitem__(self, item): + return _Missing(f"{self._p}[...]") + + +Ncm = _Missing("numcosmo_py.Ncm") +Nc = _Missing("numcosmo_py.Nc") +GObject = _Missing("numcosmo_py.GObject") + + +def dict_to_var_dict(*a, **k): + raise RuntimeError("numcosmo_py.dict_to_var_dict unavailable (no NumCosmo)") + + +def var_dict_to_dict(*a, **k): + raise RuntimeError("numcosmo_py.var_dict_to_dict unavailable (no NumCosmo)") +''' + + +def patch_file(path: Path, old: str, new: str) -> str: + text = path.read_text() + if new in text: + return "already patched" + if old not in text: + sys.exit( + f"FATAL: expected text not found in {path}.\n" + "firecrown has probably been bumped past the pinned version this " + "patch targets — review scripts/patch_firecrown.py together with " + "the [blinding] pin in pyproject.toml." + ) + path.write_text(text.replace(old, new, 1)) + return "patched" + + +def main() -> None: + spec = importlib.util.find_spec("firecrown") + if spec is None or spec.origin is None: + sys.exit("FATAL: firecrown is not installed in this environment.") + pkg = Path(spec.origin).parent + + # Metadata, not `import firecrown` — pre-patch, importing is what's broken. + version = importlib.metadata.version("firecrown") + if version != EXPECTED_FIRECROWN: + sys.exit( + f"FATAL: firecrown {version} != expected {EXPECTED_FIRECROWN}; " + "review this patch against the new version before bumping " + "EXPECTED_FIRECROWN." + ) + + print( + "generators/__init__.py:", + patch_file(pkg / "generators" / "__init__.py", GENERATORS_OLD, GENERATORS_NEW), + ) + print( + "likelihood/__init__.py:", + patch_file(pkg / "likelihood" / "__init__.py", LIKELIHOOD_OLD, LIKELIHOOD_NEW), + ) + + # Loud numcosmo_py shim — only when no real NumCosmo is present. + if importlib.util.find_spec("numcosmo_py") is None: + shim_dir = pkg.parent / "numcosmo_py" + shim_dir.mkdir(exist_ok=True) + (shim_dir / "__init__.py").write_text(SHIM) + print("numcosmo_py shim: installed") + else: + print("numcosmo_py shim: skipped (numcosmo_py importable)") + + check = subprocess.run( + [ + sys.executable, + "-c", + "import firecrown.likelihood; import smokescreen", + ], + capture_output=True, + text=True, + ) + if check.returncode != 0: + sys.exit(f"FATAL: post-patch import check failed:\n{check.stderr}") + print("post-patch import check: firecrown.likelihood + smokescreen OK") + + +if __name__ == "__main__": + main() From 6050be086a9f44194ddb1a4cc7a142f7be82ccf9 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 02:36:19 +0200 Subject: [PATCH 03/47] feat(sacc_io): SACC read/write for the standard data-product layout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add sp_validation.sacc_io: the writer/reader layer for the two-file SACC layout that becomes the package's standard data-product format. {version}.sacc analysis vector — NZ tracers, coarse xi+/-, pseudo-Cl (EE/BB/EB) with a shared BandpowerWindow, COSEBIs, pure E/B, rho/tau PSF diagnostics; one FullCovariance assembled block-diagonally (zero cross-blocks). {version}_xi_fine COSEBIs/pure-EB integration input — same NZ tracers, fine-grid xi+/-, DiagonalCovariance from TreeCorr varxip/varxim. Covariance order is point-insertion order (SACC preserves it bitwise through FITS). Writers insert in the canonical order — xi+ then xi-, Cl (ee, bb, eb), COSEBIs (all En then all Bn), pure E/B in _EB_KEYS order (xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb, matching b_modes.calculate_eb_statistics), rho, then tau — but readers never assume global order: every getter resolves indices through s.indices(dtype, tracers, **tags). assemble_covariance validates that blocks are contiguous, ascending and tile the data vector exactly, failing loud otherwise. Custom data types (pure E/B, rho, tau) all parse under sacc.parse_data_type_name. Tag filters are plain kwargs; the tags={...} form silently selects nothing and is never used. Test suite (test_sacc_io.py, all synthetic and fast): per-writer round-trips (arrays/tags/windows/NZ bitwise), covariance block alignment and zero cross-blocks, assemble_covariance failure modes, DiagonalCovariance round-trip, extract() sub-covariance alignment, a tomographic multi-pair case, reader/writer mirroring on a mixed file, and the end-to-end two-file layout. 20 passed. Co-Authored-By: Claude Opus --- src/sp_validation/sacc_io.py | 537 ++++++++++++++++++++++++ src/sp_validation/tests/test_sacc_io.py | 491 ++++++++++++++++++++++ 2 files changed, 1028 insertions(+) create mode 100644 src/sp_validation/sacc_io.py create mode 100644 src/sp_validation/tests/test_sacc_io.py diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py new file mode 100644 index 00000000..cf03f12c --- /dev/null +++ b/src/sp_validation/sacc_io.py @@ -0,0 +1,537 @@ +"""SACC_IO. + +:Name: sacc_io.py + +:Description: Read/write the standard SACC data-product layout for the + weak-lensing validation package. Two files describe each + catalogue version: + + - ``{version}.sacc`` — the analysis vector: NZ tracers, coarse + ξ±, pseudo-Cℓ (EE/BB/EB) with bandpower windows, COSEBIs, + pure E/B, and ρ/τ PSF diagnostics, all sharing a single + ``FullCovariance`` assembled block-diagonally from the + per-statistic covariances (zero cross-blocks). + - ``{version}_xi_fine.sacc`` — the COSEBIs / pure-EB integration + input: the same NZ tracers, a fine-grid ξ±, and a + ``DiagonalCovariance`` from TreeCorr ``varxip``/``varxim``. + + The covariance order is the point-insertion order (SACC preserves + it bitwise through FITS save/load). Writers below insert in the + canonical order — ξ+ then ξ−, Cℓ (ee, bb, eb), COSEBIs (all Eₙ + then all Bₙ), pure E/B (xip_E, xim_E, xip_B, xim_B, xip_amb, + xim_amb — matching ``b_modes._EB_KEYS``), ρ, then τ — but readers + never assume global order: they resolve indices through + ``Sacc.indices(dtype, tracers, **tags)``. + + Tag filters are plain keyword arguments to ``indices`` / + ``get_data_points`` / ``get_tag``; the ``tags={...}`` form + silently selects nothing and must never be used. +""" + +import numpy as np +import sacc + +PSF_TRACER = "psf_stars" + +# Standard SACC data-type strings. +XI_PLUS = "galaxy_shear_xi_plus" +XI_MINUS = "galaxy_shear_xi_minus" +CL_EE = "galaxy_shear_cl_ee" +CL_BB = "galaxy_shear_cl_bb" +CL_EB = "galaxy_shear_cl_eb" +COSEBI_EE = "galaxy_shear_cosebi_ee" +COSEBI_BB = "galaxy_shear_cosebi_bb" + +# Custom data-type strings (all parse under sacc.parse_data_type_name). +PURE_TYPES = { + "xip_E": "galaxy_shear_xiPureE_plus", + "xim_E": "galaxy_shear_xiPureE_minus", + "xip_B": "galaxy_shear_xiPureB_plus", + "xim_B": "galaxy_shear_xiPureB_minus", + "xip_amb": "galaxy_shear_xiPureAmb_plus", + "xim_amb": "galaxy_shear_xiPureAmb_minus", +} +# Insertion order of the six pure-EB blocks — matches b_modes._EB_KEYS, whose +# order is the [xip_E; xim_E; xip_B; xim_B; xip_amb; xim_amb] layout of the +# treecorr/MC pure-EB covariance (b_modes.calculate_eb_statistics, ~L392). +PURE_KEYS = ("xip_E", "xim_E", "xip_B", "xim_B", "xip_amb", "xim_amb") + +RHO_PLUS = "psf_rho{k}_xi_plus" +RHO_MINUS = "psf_rho{k}_xi_minus" +TAU_PLUS = "galaxyPsf_tau{k}_xi_plus" +TAU_MINUS = "galaxyPsf_tau{k}_xi_minus" + + +def source_name(i): + """SACC tracer name for source redshift bin ``i`` (0-based).""" + return f"source_{i}" + + +def new_sacc(nz, metadata=None): + """Create a Sacc with the survey's NZ (and PSF) tracers. + + Parameters + ---------- + nz : dict or sequence + Redshift distributions, one per source bin. Either a mapping + ``{i: (z, nz)}`` keyed by 0-based bin index, or a sequence of + ``(z, nz)`` array pairs (bin index = position). Tracers are named + ``source_{i}``. + metadata : dict, optional + Key/value pairs stored on ``s.metadata``. + + Returns + ------- + sacc.Sacc + Sacc holding the ``source_{i}`` NZ tracers and the ``psf_stars`` + Misc tracer (needed by ρ/τ diagnostics). + """ + items = nz.items() if isinstance(nz, dict) else enumerate(nz) + s = sacc.Sacc() + for i, (z, nz_i) in items: + s.add_tracer("NZ", source_name(i), np.asarray(z), np.asarray(nz_i)) + s.add_tracer("Misc", PSF_TRACER) + for key, value in (metadata or {}).items(): + s.metadata[key] = value + return s + + +def _pair(bins): + """Resolve a ``(i, j)`` bin pair to the ``(source_i, source_j)`` names.""" + i, j = bins + return (source_name(i), source_name(j)) + + +def add_xi( + s, + bins, + theta, + xip, + xim, + *, + grid, + theta_nom=None, + npairs=None, + weight=None, +): + """Add a real-space shear 2PCF (ξ+ then ξ−) for one tracer pair. + + Parameters + ---------- + s : sacc.Sacc + Target, mutated in place. + bins : tuple of int + Source bin pair ``(i, j)``. + theta : array_like + Angular separations (arcmin) — TreeCorr ``meanr``. + xip, xim : array_like + ξ+ and ξ− at ``theta``. + grid : {'coarse', 'fine'} + Distinguishes the analysis grid from the fine integration grid; + stored as the ``grid`` tag on every point. + theta_nom : array_like, optional + Nominal bin centres — TreeCorr ``rnom`` — stored as ``theta_nom``. + npairs, weight : array_like, optional + TreeCorr pair counts and weights, stored per point. + """ + tracers = _pair(bins) + for dtype, xi in ((XI_PLUS, xip), (XI_MINUS, xim)): + for n, th in enumerate(theta): + tags = {"theta": float(th), "grid": grid} + if theta_nom is not None: + tags["theta_nom"] = float(theta_nom[n]) + if npairs is not None: + tags["npairs"] = float(npairs[n]) + if weight is not None: + tags["weight"] = float(weight[n]) + s.add_data_point(dtype, tracers, float(xi[n]), **tags) + + +def add_pseudo_cl( + s, + bins, + ell_eff, + cl_ee, + cl_bb, + cl_eb, + *, + window_ells, + window_weights, +): + """Add pseudo-Cℓ (EE, BB, EB) with a shared bandpower window. + + Parameters + ---------- + s : sacc.Sacc + Target, mutated in place. + bins : tuple of int + Source bin pair ``(i, j)``. + ell_eff : array_like + Effective multipole of each bandpower. + cl_ee, cl_bb, cl_eb : array_like + EE, BB and EB bandpowers at ``ell_eff``. + window_ells : array_like + Multipoles spanned by the bandpower window matrix (shape ``(nell,)``). + window_weights : array_like + Window matrix ``W`` of shape ``(nell, nbp)`` — one column per + bandpower — from NaMaster ``get_bandpower_windows``. One + ``sacc.BandpowerWindow`` is built and shared across EE/BB/EB. + """ + tracers = _pair(bins) + window = sacc.BandpowerWindow(np.asarray(window_ells), np.asarray(window_weights)) + for dtype, cl in ((CL_EE, cl_ee), (CL_BB, cl_bb), (CL_EB, cl_eb)): + s.add_ell_cl( + dtype, *tracers, np.asarray(ell_eff), np.asarray(cl), window=window + ) + + +def add_cosebis(s, bins, En, Bn, scale_cut): + """Add COSEBIs (all Eₙ then all Bₙ) for one scale cut. + + Parameters + ---------- + s : sacc.Sacc + Target, mutated in place. + bins : tuple of int + Source bin pair ``(i, j)``. + En, Bn : array_like + E- and B-mode COSEBI amplitudes, one per logarithmic mode ``n`` + (1-based). The ``[En; Bn]`` layout matches the COSEBI covariance. + scale_cut : tuple of float + ``(theta_min, theta_max)`` in arcmin, stored on every point as the + ``theta_min``/``theta_max`` tags; multiple cuts coexist in one file, + told apart by these tags. + """ + tracers = _pair(bins) + theta_min, theta_max = scale_cut + for dtype, modes in ((COSEBI_EE, En), (COSEBI_BB, Bn)): + for n, value in enumerate(modes, start=1): + s.add_data_point( + dtype, + tracers, + float(value), + n=n, + theta_min=float(theta_min), + theta_max=float(theta_max), + ) + + +def add_pure_eb(s, bins, theta, xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb): + """Add pure E/B-mode correlation functions for one tracer pair. + + Six blocks are inserted in ``PURE_KEYS`` order (xip_E, xim_E, xip_B, + xim_B, xip_amb, xim_amb), matching ``b_modes._EB_KEYS`` and the pure-EB + covariance layout. + + Parameters + ---------- + s : sacc.Sacc + Target, mutated in place. + bins : tuple of int + Source bin pair ``(i, j)``. + theta : array_like + Angular separations (arcmin), shared by all six blocks. + xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb : array_like + The six pure E/B / ambiguous mode arrays at ``theta``. + """ + tracers = _pair(bins) + values = { + "xip_E": xip_E, + "xim_E": xim_E, + "xip_B": xip_B, + "xim_B": xim_B, + "xip_amb": xip_amb, + "xim_amb": xim_amb, + } + for key in PURE_KEYS: + dtype, arr = PURE_TYPES[key], values[key] + for n, th in enumerate(theta): + s.add_data_point(dtype, tracers, float(arr[n]), theta=float(th)) + + +def add_rho(s, k, theta, rho_p, rho_m): + """Add a ρ_k PSF statistic (ρ+ then ρ−) on the ``psf_stars`` tracer. + + Parameters + ---------- + s : sacc.Sacc + Target, mutated in place. + k : int + ρ index (0…5). + theta : array_like + Angular separations (arcmin). + rho_p, rho_m : array_like + ρ_k+ and ρ_k− at ``theta``. + """ + tracers = (PSF_TRACER, PSF_TRACER) + for dtype, arr in ((RHO_PLUS.format(k=k), rho_p), (RHO_MINUS.format(k=k), rho_m)): + for n, th in enumerate(theta): + s.add_data_point(dtype, tracers, float(arr[n]), theta=float(th)) + + +def add_tau(s, bins, k, theta, tau_p, tau_m): + """Add a τ_k PSF-leakage statistic (τ+ then τ−). + + Parameters + ---------- + s : sacc.Sacc + Target, mutated in place. + bins : tuple of int + Source bin ``i`` and PSF; the τ tracers are ``(source_i, psf_stars)``. + Only ``bins[0]`` is used. + k : int + τ index (0, 2 or 5). + theta : array_like + Angular separations (arcmin). + tau_p, tau_m : array_like + τ_k+ and τ_k− at ``theta``. + """ + tracers = (source_name(bins[0]), PSF_TRACER) + for dtype, arr in ((TAU_PLUS.format(k=k), tau_p), (TAU_MINUS.format(k=k), tau_m)): + for n, th in enumerate(theta): + s.add_data_point(dtype, tracers, float(arr[n]), theta=float(th)) + + +def assemble_covariance(s, blocks): + """Assemble a block-diagonal ``FullCovariance`` from per-statistic blocks. + + Each block is validated against the current insertion order: its indices + must be contiguous and ascending, the blocks must tile ``0…len(s.mean)`` + exactly (no gap, no overlap), and each block must be square with a size + matching its index span. Any violation raises ``ValueError`` naming the + mismatch. Cross-blocks are left zero. + + Parameters + ---------- + s : sacc.Sacc + Target, mutated in place via ``add_covariance``. + blocks : sequence + Ordered ``(selector, cov)`` pairs (or a mapping of the same). Each + ``selector`` is either an index array, or a ``(data_type, tracers)`` + / ``(data_type, tracers, tags)`` tuple resolved through + ``s.indices``; ``cov`` is the block's dense covariance. + + Returns + ------- + sacc.Sacc + ``s``, with the assembled ``FullCovariance`` attached. + """ + items = blocks.items() if isinstance(blocks, dict) else blocks + ntot = len(s.mean) + full = np.zeros((ntot, ntot)) + cursor = 0 + for selector, cov in items: + idx = _resolve_indices(s, selector) + cov = np.asarray(cov) + if not np.array_equal(idx, np.arange(idx[0], idx[0] + len(idx))): + raise ValueError( + f"covariance block {selector!r} resolves to non-contiguous " + f"or non-ascending indices {idx.tolist()}" + ) + if idx[0] != cursor: + raise ValueError( + f"covariance block {selector!r} starts at index {idx[0]} but " + f"the previous blocks cover through {cursor} — blocks must tile " + "the data vector with no gap or overlap" + ) + if cov.ndim != 2 or cov.shape[0] != cov.shape[1]: + raise ValueError( + f"covariance block {selector!r} must be square; got shape {cov.shape}" + ) + if cov.shape[0] != len(idx): + raise ValueError( + f"covariance block {selector!r} has size {cov.shape[0]} but " + f"spans {len(idx)} data points" + ) + full[np.ix_(idx, idx)] = cov + cursor = idx[-1] + 1 + if cursor != ntot: + raise ValueError( + f"covariance blocks cover {cursor} of {ntot} data points — the " + "blocks must tile the whole data vector" + ) + s.add_covariance(full) + return s + + +def _resolve_indices(s, selector): + """Resolve a covariance-block selector to a sorted index array.""" + if isinstance(selector, (np.ndarray, list, tuple, range)) and not ( + len(selector) in (2, 3) and isinstance(selector[0], str) + ): + return np.asarray(selector, dtype=int) + data_type, tracers = selector[0], selector[1] + tags = selector[2] if len(selector) == 3 else {} + return np.asarray(s.indices(data_type, tuple(tracers), **tags), dtype=int) + + +def add_diagonal_covariance(s, variances): + """Attach a ``DiagonalCovariance`` from a 1-D variance array. + + The 1-D array is passed straight to ``add_covariance`` (never + ``np.diag``), which is what makes SACC store a ``DiagonalCovariance``. + + Parameters + ---------- + s : sacc.Sacc + Target, mutated in place. + variances : array_like + Per-point variances, ``len == len(s.mean)``. + + Returns + ------- + sacc.Sacc + ``s``, with the ``DiagonalCovariance`` attached. + """ + s.add_covariance(np.asarray(variances)) + return s + + +def get_nz(s, i): + """Return ``(z, nz)`` for source bin ``i``.""" + tracer = s.tracers[source_name(i)] + return tracer.z, tracer.nz + + +def get_xi(s, bins, *, grid): + """Return ``(theta, xip, xim)`` for one tracer pair and grid.""" + tracers = _pair(bins) + return ( + _sorted_tag(s, XI_PLUS, tracers, "theta", grid=grid), + _sorted_mean(s, XI_PLUS, tracers, grid=grid), + _sorted_mean(s, XI_MINUS, tracers, grid=grid), + ) + + +def get_pseudo_cl(s, bins): + """Return ``(ell_eff, cl_ee, cl_bb, cl_eb, window)`` for one tracer pair. + + ``window`` is the shared ``sacc.BandpowerWindow`` recovered via + ``get_bandpower_windows``. + """ + tracers = _pair(bins) + ell = _sorted_tag(s, CL_EE, tracers, "ell") + window = s.get_bandpower_windows(s.indices(CL_EE, tracers)) + return ( + ell, + _sorted_mean(s, CL_EE, tracers, _sort_tag="ell"), + _sorted_mean(s, CL_BB, tracers, _sort_tag="ell"), + _sorted_mean(s, CL_EB, tracers, _sort_tag="ell"), + window, + ) + + +def get_cosebis(s, bins, scale_cut=None): + """Return ``(n, En, Bn)`` for one tracer pair. + + Parameters + ---------- + scale_cut : tuple of float, optional + ``(theta_min, theta_max)`` to select when several cuts share the file. + """ + tracers = _pair(bins) + tags = ( + {"theta_min": float(scale_cut[0]), "theta_max": float(scale_cut[1])} + if scale_cut is not None + else {} + ) + modes = _sorted_tag(s, COSEBI_EE, tracers, "n", **tags) + return ( + modes.astype(int), + _sorted_mean(s, COSEBI_EE, tracers, **tags, _sort_tag="n"), + _sorted_mean(s, COSEBI_BB, tracers, **tags, _sort_tag="n"), + ) + + +def get_pure_eb(s, bins): + """Return ``(theta, {key: array})`` for the six pure-EB blocks. + + The dict is keyed by ``PURE_KEYS`` (xip_E, xim_E, …). + """ + tracers = _pair(bins) + theta = _sorted_tag(s, PURE_TYPES["xip_E"], tracers, "theta") + arrays = {key: _sorted_mean(s, PURE_TYPES[key], tracers) for key in PURE_KEYS} + return theta, arrays + + +def get_rho(s, k): + """Return ``(theta, rho_p, rho_m)`` for ρ index ``k``.""" + tracers = (PSF_TRACER, PSF_TRACER) + dt_p, dt_m = RHO_PLUS.format(k=k), RHO_MINUS.format(k=k) + return ( + _sorted_tag(s, dt_p, tracers, "theta"), + _sorted_mean(s, dt_p, tracers), + _sorted_mean(s, dt_m, tracers), + ) + + +def get_tau(s, bins, k): + """Return ``(theta, tau_p, tau_m)`` for τ index ``k`` and source bin.""" + tracers = (source_name(bins[0]), PSF_TRACER) + dt_p, dt_m = TAU_PLUS.format(k=k), TAU_MINUS.format(k=k) + return ( + _sorted_tag(s, dt_p, tracers, "theta"), + _sorted_mean(s, dt_p, tracers), + _sorted_mean(s, dt_m, tracers), + ) + + +def _order(s, data_type, tracers, sort_tag, **tag_filters): + """Indices for a selection, ordered by ascending ``sort_tag``.""" + idx = np.asarray(s.indices(data_type, tracers, **tag_filters), dtype=int) + key = np.array([s.data[i].tags[sort_tag] for i in idx]) + return idx[np.argsort(key)] + + +def _sorted_mean(s, data_type, tracers, _sort_tag="theta", **tag_filters): + """Mean values for a selection, ordered by ``_sort_tag`` ascending.""" + idx = _order(s, data_type, tracers, _sort_tag, **tag_filters) + return s.mean[idx] + + +def _sorted_tag(s, data_type, tracers, tag, **tag_filters): + """Values of ``tag`` for a selection, ordered by that tag ascending.""" + idx = _order(s, data_type, tracers, tag, **tag_filters) + return np.array([s.data[i].tags[tag] for i in idx]) + + +def extract(s, data_type=None, tracers=None, **tag_filters): + """Extract a sub-Sacc (points + aligned covariance sub-block). + + A copy is made and everything *not* matching the selection is removed, so + the covariance sub-block comes out correctly aligned and the original is + untouched. + + Parameters + ---------- + s : sacc.Sacc + Source, left unmodified. + data_type : str, optional + Data type to keep. + tracers : tuple, optional + Tracer pair to keep. + **tag_filters + Tag filters (plain kwargs, e.g. ``grid='fine'``). + + Returns + ------- + sacc.Sacc + New Sacc holding only the selected points. + """ + sub = s.copy() + selection = {} + if tracers is not None: + selection["tracers"] = tuple(tracers) + selection.update(tag_filters) + sub.keep_selection(data_type, **selection) + return sub + + +def save(s, path): + """Write ``s`` to ``path`` (FITS), overwriting any existing file.""" + s.save_fits(path, overwrite=True) + + +def load(path): + """Load a Sacc from ``path`` (FITS).""" + return sacc.Sacc.load_fits(path) diff --git a/src/sp_validation/tests/test_sacc_io.py b/src/sp_validation/tests/test_sacc_io.py new file mode 100644 index 00000000..883a2af0 --- /dev/null +++ b/src/sp_validation/tests/test_sacc_io.py @@ -0,0 +1,491 @@ +"""Tests for :mod:`sp_validation.sacc_io`. + +All synthetic, all fast: build in memory, round-trip through ``tmp_path``, +and assert arrays/tags/windows/covariances come back bitwise-identical and +correctly aligned. No cluster paths. +""" + +import numpy as np +import pytest + +from sp_validation import sacc_io as sio + + +# --------------------------------------------------------------------------- # +# Synthetic builders +# --------------------------------------------------------------------------- # +def _nz(seed, n=50): + rng = np.random.default_rng(seed) + z = np.linspace(0.01, 2.0, n) + return z, rng.uniform(0.1, 1.0, n) + + +def _theta(nbins=6): + return np.geomspace(1.0, 100.0, nbins) + + +def _spd(n, seed): + """Symmetric positive-definite matrix of size ``n``.""" + a = np.random.default_rng(seed).normal(size=(n, n)) + return a @ a.T + n * np.eye(n) + + +def _base_sacc(nbins=1): + """A Sacc with ``nbins`` NZ source tracers plus the PSF tracer.""" + return sio.new_sacc({i: _nz(i) for i in range(nbins)}) + + +# --------------------------------------------------------------------------- # +# 1. Per-writer round-trip (arrays / tags / windows / NZ bitwise) +# --------------------------------------------------------------------------- # +def _roundtrip(s, tmp_path, name="rt"): + path = tmp_path / f"{name}.sacc" + sio.save(s, str(path)) + return sio.load(str(path)) + + +def test_nz_roundtrip(tmp_path): + z, nz = _nz(3) + s = sio.new_sacc({0: (z, nz)}, metadata={"version": "v1.4.6.3"}) + s2 = _roundtrip(s, tmp_path, "nz") + z2, nz2 = sio.get_nz(s2, 0) + assert np.array_equal(z2, z) + assert np.array_equal(nz2, nz) + assert s2.metadata["version"] == "v1.4.6.3" + assert sio.PSF_TRACER in s2.tracers + + +def test_xi_roundtrip(tmp_path): + theta = _theta() + xip, xim = np.arange(6) * 1e-5, np.arange(6) * 2e-5 + npairs, weight = np.arange(6) * 1e3, np.arange(6) * 1.5 + s = _base_sacc() + sio.add_xi( + s, + (0, 0), + theta, + xip, + xim, + grid="coarse", + theta_nom=theta * 1.01, + npairs=npairs, + weight=weight, + ) + s2 = _roundtrip(s, tmp_path, "xi") + th, p, m = sio.get_xi(s2, (0, 0), grid="coarse") + assert np.array_equal(th, theta) + assert np.array_equal(p, xip) + assert np.array_equal(m, xim) + # extra tags survive + idx = s2.indices(sio.XI_PLUS, ("source_0", "source_0"), grid="coarse") + tags = s2.data[idx[0]].tags + assert tags["grid"] == "coarse" + assert set(tags) >= {"theta", "theta_nom", "npairs", "weight", "grid"} + + +def test_pseudo_cl_roundtrip(tmp_path): + ell_eff = np.array([30.0, 120.0, 210.0, 300.0]) + nell, nbp = 50, len(ell_eff) + window_ells = np.arange(2, 2 + nell).astype(float) + W = np.random.default_rng(5).uniform(size=(nell, nbp)) + ee, bb, eb = np.arange(nbp) * 1e-9, np.arange(nbp) * 2e-9, np.arange(nbp) * 3e-9 + s = _base_sacc() + sio.add_pseudo_cl( + s, + (0, 0), + ell_eff, + ee, + bb, + eb, + window_ells=window_ells, + window_weights=W, + ) + s2 = _roundtrip(s, tmp_path, "cl") + ell, cl_ee, cl_bb, cl_eb, window = sio.get_pseudo_cl(s2, (0, 0)) + assert np.array_equal(ell, ell_eff) + assert np.array_equal(cl_ee, ee) + assert np.array_equal(cl_bb, bb) + assert np.array_equal(cl_eb, eb) + assert np.array_equal(window.weight, W) + assert np.array_equal(window.values, window_ells) + + +def test_cosebis_roundtrip(tmp_path): + En, Bn = np.arange(1, 11) * 1e-6, np.arange(1, 11) * 1e-7 + s = _base_sacc() + sio.add_cosebis(s, (0, 0), En, Bn, (1.0, 100.0)) + s2 = _roundtrip(s, tmp_path, "cosebi") + n, E, B = sio.get_cosebis(s2, (0, 0)) + assert np.array_equal(n, np.arange(1, 11)) + assert np.array_equal(E, En) + assert np.array_equal(B, Bn) + idx = s2.indices(sio.COSEBI_EE, ("source_0", "source_0")) + assert s2.data[idx[0]].tags["theta_min"] == 1.0 + assert s2.data[idx[0]].tags["theta_max"] == 100.0 + + +def test_pure_eb_roundtrip(tmp_path): + theta = _theta() + arrays = {key: np.arange(6) * (i + 1) * 1e-6 for i, key in enumerate(sio.PURE_KEYS)} + s = _base_sacc() + sio.add_pure_eb(s, (0, 0), theta, **arrays) + s2 = _roundtrip(s, tmp_path, "pureeb") + th, back = sio.get_pure_eb(s2, (0, 0)) + assert np.array_equal(th, theta) + for key in sio.PURE_KEYS: + assert np.array_equal(back[key], arrays[key]) + + +def test_rho_roundtrip(tmp_path): + theta = _theta() + s = _base_sacc() + for k in range(6): + sio.add_rho( + s, k, theta, np.arange(6) * (k + 1) * 1e-6, np.arange(6) * (k + 1) * 2e-6 + ) + s2 = _roundtrip(s, tmp_path, "rho") + for k in range(6): + th, p, m = sio.get_rho(s2, k) + assert np.array_equal(th, theta) + assert np.array_equal(p, np.arange(6) * (k + 1) * 1e-6) + assert np.array_equal(m, np.arange(6) * (k + 1) * 2e-6) + + +def test_tau_roundtrip(tmp_path): + theta = _theta() + s = _base_sacc() + for k in (0, 2, 5): + sio.add_tau( + s, + (0, 0), + k, + theta, + np.arange(6) * (k + 1) * 1e-6, + np.arange(6) * (k + 1) * 2e-6, + ) + s2 = _roundtrip(s, tmp_path, "tau") + for k in (0, 2, 5): + th, p, m = sio.get_tau(s2, (0, 0), k) + assert np.array_equal(th, theta) + assert np.array_equal(p, np.arange(6) * (k + 1) * 1e-6) + assert np.array_equal(m, np.arange(6) * (k + 1) * 2e-6) + assert s2.data[s2.indices(sio.TAU_PLUS.format(k=k))[0]].tracers == ( + "source_0", + sio.PSF_TRACER, + ) + + +# --------------------------------------------------------------------------- # +# 2. Covariance alignment +# --------------------------------------------------------------------------- # +def _multi_statistic_sacc(): + """Sacc with ξ+/ξ−, Cℓ (ee/bb/eb) and COSEBIs, ready for a covariance.""" + theta = _theta() + s = _base_sacc() + sio.add_xi( + s, (0, 0), theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + ell = np.array([30.0, 120.0, 210.0]) + W = np.random.default_rng(0).uniform(size=(20, 3)) + sio.add_pseudo_cl( + s, + (0, 0), + ell, + np.arange(3) * 1e-9, + np.arange(3) * 2e-9, + np.arange(3) * 3e-9, + window_ells=np.arange(2, 22).astype(float), + window_weights=W, + ) + sio.add_cosebis( + s, (0, 0), np.arange(1, 6) * 1e-6, np.arange(1, 6) * 1e-7, (1.0, 100.0) + ) + return s + + +def test_assemble_covariance_alignment(tmp_path): + s = _multi_statistic_sacc() + tr = ("source_0", "source_0") + # Block selectors in canonical (insertion) order. + xi = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + cl = np.concatenate( + [s.indices(sio.CL_EE, tr), s.indices(sio.CL_BB, tr), s.indices(sio.CL_EB, tr)] + ) + co = np.concatenate([s.indices(sio.COSEBI_EE, tr), s.indices(sio.COSEBI_BB, tr)]) + cov_xi, cov_cl, cov_co = _spd(len(xi), 1), _spd(len(cl), 2), _spd(len(co), 3) + sio.assemble_covariance(s, [(xi, cov_xi), (cl, cov_cl), (co, cov_co)]) + s2 = _roundtrip(s, tmp_path, "cov") + assert type(s2.covariance).__name__ == "FullCovariance" + dense = s2.covariance.dense + # each block's sub-covariance is exactly what went in + assert np.array_equal(dense[np.ix_(xi, xi)], cov_xi) + assert np.array_equal(dense[np.ix_(cl, cl)], cov_cl) + assert np.array_equal(dense[np.ix_(co, co)], cov_co) + # zero cross-blocks + assert np.array_equal(dense[np.ix_(xi, cl)], np.zeros((len(xi), len(cl)))) + assert np.array_equal(dense[np.ix_(xi, co)], np.zeros((len(xi), len(co)))) + assert np.array_equal(dense[np.ix_(cl, co)], np.zeros((len(cl), len(co)))) + + +def test_assemble_covariance_selector_tuples(): + """Blocks addressed by (data_type, tracers) tuples, not raw indices.""" + s = _multi_statistic_sacc() + tr = ("source_0", "source_0") + xi = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + cl = np.concatenate( + [s.indices(sio.CL_EE, tr), s.indices(sio.CL_BB, tr), s.indices(sio.CL_EB, tr)] + ) + sio.assemble_covariance( + s, + [ + (xi, _spd(len(xi), 1)), + (cl, _spd(len(cl), 2)), + ((sio.COSEBI_EE, tr), _spd(len(s.indices(sio.COSEBI_EE, tr)), 3)), + ((sio.COSEBI_BB, tr), _spd(len(s.indices(sio.COSEBI_BB, tr)), 4)), + ], + ) + assert type(s.covariance).__name__ == "FullCovariance" + assert s.covariance.dense.shape == (len(s.mean), len(s.mean)) + + +# --------------------------------------------------------------------------- # +# 3. assemble_covariance failure modes +# --------------------------------------------------------------------------- # +def test_assemble_covariance_wrong_dimension(): + s = _base_sacc() + sio.add_xi( + s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + idx = np.arange(len(s.mean)) + with pytest.raises(ValueError, match="span"): + sio.assemble_covariance(s, [(idx, _spd(len(idx) - 1, 1))]) + + +def test_assemble_covariance_non_contiguous(): + s = _base_sacc() + sio.add_xi( + s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + idx = np.array([0, 2, 4, 6, 8, 10, 1, 3]) # not contiguous/ascending + with pytest.raises(ValueError, match="non-contiguous"): + sio.assemble_covariance(s, [(idx, _spd(len(idx), 1))]) + + +def test_assemble_covariance_missing_coverage(): + s = _multi_statistic_sacc() + tr = ("source_0", "source_0") + xi = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + with pytest.raises(ValueError, match="tile"): + sio.assemble_covariance( + s, [(xi, _spd(len(xi), 1))] + ) # leaves cl+cosebi uncovered + + +def test_assemble_covariance_non_square(): + s = _base_sacc() + sio.add_xi( + s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + idx = np.arange(len(s.mean)) + with pytest.raises(ValueError, match="square"): + sio.assemble_covariance(s, [(idx, np.ones((len(idx), len(idx) - 1)))]) + + +def test_assemble_covariance_overlap(): + s = _multi_statistic_sacc() + tr = ("source_0", "source_0") + xi = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + # second block starts before the first ended -> gap/overlap error + with pytest.raises(ValueError, match="tile|gap|overlap"): + sio.assemble_covariance(s, [(xi, _spd(len(xi), 1)), (xi, _spd(len(xi), 2))]) + + +# --------------------------------------------------------------------------- # +# 4. Fine file: DiagonalCovariance from 1-D variances +# --------------------------------------------------------------------------- # +def test_diagonal_covariance_roundtrip(tmp_path): + theta = np.geomspace(0.1, 250.0, 40) + n = len(theta) + xip, xim = np.arange(n) * 1e-5, np.arange(n) * 2e-5 + varxip, varxim = np.arange(1, n + 1) * 1e-12, np.arange(1, n + 1) * 2e-12 + s = _base_sacc() + sio.add_xi(s, (0, 0), theta, xip, xim, grid="fine") + variances = np.concatenate([varxip, varxim]) # [xip; xim] order + sio.add_diagonal_covariance(s, variances) + assert type(s.covariance).__name__ == "DiagonalCovariance" + s2 = _roundtrip(s, tmp_path, "fine") + assert type(s2.covariance).__name__ == "DiagonalCovariance" + assert np.array_equal(np.diag(s2.covariance.dense), variances) + + +# --------------------------------------------------------------------------- # +# 5. extract(): sub-covariance alignment; original untouched; tag filter +# --------------------------------------------------------------------------- # +def test_extract_subblock_and_original_untouched(): + s = _multi_statistic_sacc() + tr = ("source_0", "source_0") + xi = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + cl = np.concatenate( + [s.indices(sio.CL_EE, tr), s.indices(sio.CL_BB, tr), s.indices(sio.CL_EB, tr)] + ) + co = np.concatenate([s.indices(sio.COSEBI_EE, tr), s.indices(sio.COSEBI_BB, tr)]) + cov_co = _spd(len(co), 3) + sio.assemble_covariance( + s, [(xi, _spd(len(xi), 1)), (cl, _spd(len(cl), 2)), (co, cov_co)] + ) + n_before = len(s.mean) + sub = sio.extract(s, data_type=sio.COSEBI_EE, tracers=tr) + # original untouched + assert len(s.mean) == n_before + # subset covariance equals the COSEBI-EE diagonal sub-block + ee_local = np.arange(len(s.indices(sio.COSEBI_EE, tr))) + assert np.array_equal(sub.covariance.dense, cov_co[np.ix_(ee_local, ee_local)]) + + +def test_extract_tag_filter(): + theta = _theta() + s = _base_sacc() + sio.add_xi( + s, (0, 0), theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + sio.add_xi(s, (0, 0), theta, np.arange(6) * 3e-5, np.arange(6) * 4e-5, grid="fine") + sub = sio.extract( + s, data_type=sio.XI_PLUS, tracers=("source_0", "source_0"), grid="fine" + ) + assert len(sub.mean) == len(theta) + assert set(sub.get_tag("grid", sio.XI_PLUS)) == {"fine"} + + +# --------------------------------------------------------------------------- # +# 6. Tomographic case: >=2 bins, >=3 pairs, per-pair selection +# --------------------------------------------------------------------------- # +def test_tomographic_per_pair_selection(tmp_path): + theta = _theta() + s = _base_sacc(nbins=2) + pairs = [(0, 0), (0, 1), (1, 1)] + for k, (i, j) in enumerate(pairs): + sio.add_xi( + s, + (i, j), + theta, + np.arange(6) * (k + 1) * 1e-5, + np.arange(6) * (k + 1) * 2e-5, + grid="coarse", + ) + s2 = _roundtrip(s, tmp_path, "tomo") + for k, (i, j) in enumerate(pairs): + th, p, m = sio.get_xi(s2, (i, j), grid="coarse") + assert np.array_equal(th, theta) + assert np.array_equal(p, np.arange(6) * (k + 1) * 1e-5) + assert np.array_equal(m, np.arange(6) * (k + 1) * 2e-5) + # selecting one pair does not bleed into another + assert len(s2.indices(sio.XI_PLUS, ("source_0", "source_1"))) == len(theta) + assert len(s2.indices(sio.XI_PLUS, ("source_1", "source_1"))) == len(theta) + + +# --------------------------------------------------------------------------- # +# 7. Readers mirror writers on a mixed file +# --------------------------------------------------------------------------- # +def test_readers_on_mixed_file(tmp_path): + theta = _theta() + s = _base_sacc() + sio.add_xi( + s, (0, 0), theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + ell = np.array([30.0, 120.0, 210.0]) + W = np.random.default_rng(0).uniform(size=(20, 3)) + sio.add_pseudo_cl( + s, + (0, 0), + ell, + np.arange(3) * 1e-9, + np.arange(3) * 2e-9, + np.arange(3) * 3e-9, + window_ells=np.arange(2, 22).astype(float), + window_weights=W, + ) + sio.add_cosebis( + s, (0, 0), np.arange(1, 6) * 1e-6, np.arange(1, 6) * 1e-7, (1.0, 100.0) + ) + pure = {key: np.arange(6) * (i + 1) * 1e-6 for i, key in enumerate(sio.PURE_KEYS)} + sio.add_pure_eb(s, (0, 0), theta, **pure) + for k in range(6): + sio.add_rho( + s, k, theta, np.arange(6) * (k + 1) * 1e-7, np.arange(6) * (k + 1) * 2e-7 + ) + for k in (0, 2, 5): + sio.add_tau( + s, + (0, 0), + k, + theta, + np.arange(6) * (k + 1) * 1e-8, + np.arange(6) * (k + 1) * 2e-8, + ) + s2 = _roundtrip(s, tmp_path, "mixed") + + _, p, m = sio.get_xi(s2, (0, 0), grid="coarse") + assert np.array_equal(p, np.arange(6) * 1e-5) and np.array_equal( + m, np.arange(6) * 2e-5 + ) + ell_r, ee, bb, eb, win = sio.get_pseudo_cl(s2, (0, 0)) + assert np.array_equal(ell_r, ell) and np.array_equal(win.weight, W) + n, E, B = sio.get_cosebis(s2, (0, 0)) + assert np.array_equal(E, np.arange(1, 6) * 1e-6) + _, back = sio.get_pure_eb(s2, (0, 0)) + for key in sio.PURE_KEYS: + assert np.array_equal(back[key], pure[key]) + for k in range(6): + _, rp, rm = sio.get_rho(s2, k) + assert np.array_equal(rp, np.arange(6) * (k + 1) * 1e-7) + for k in (0, 2, 5): + _, tp, tm = sio.get_tau(s2, (0, 0), k) + assert np.array_equal(tp, np.arange(6) * (k + 1) * 1e-8) + + +# --------------------------------------------------------------------------- # +# 8. End-to-end two-file layout for a synthetic catalogue version +# --------------------------------------------------------------------------- # +def test_end_to_end_two_file_layout(tmp_path): + version = "vSYNTH" + theta_c = _theta(20) + theta_f = np.geomspace(0.1, 250.0, 200) + + # analysis file + s = _base_sacc() + sio.add_xi( + s, (0, 0), theta_c, np.arange(20) * 1e-5, np.arange(20) * 2e-5, grid="coarse" + ) + sio.add_cosebis( + s, (0, 0), np.arange(1, 11) * 1e-6, np.arange(1, 11) * 1e-7, (1.0, 100.0) + ) + tr = ("source_0", "source_0") + xi = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + co = np.concatenate([s.indices(sio.COSEBI_EE, tr), s.indices(sio.COSEBI_BB, tr)]) + sio.assemble_covariance(s, [(xi, _spd(len(xi), 1)), (co, _spd(len(co), 2))]) + sio.save(s, str(tmp_path / f"{version}.sacc")) + + # fine file + sf = _base_sacc() + sio.add_xi( + sf, (0, 0), theta_f, np.arange(200) * 1e-5, np.arange(200) * 2e-5, grid="fine" + ) + variances = np.concatenate([np.arange(1, 201) * 1e-12, np.arange(1, 201) * 2e-12]) + sio.add_diagonal_covariance(sf, variances) + sio.save(sf, str(tmp_path / f"{version}_xi_fine.sacc")) + + # reload both, verify everything + a = sio.load(str(tmp_path / f"{version}.sacc")) + f = sio.load(str(tmp_path / f"{version}_xi_fine.sacc")) + + th_c, p_c, _ = sio.get_xi(a, (0, 0), grid="coarse") + assert np.array_equal(th_c, theta_c) and np.array_equal(p_c, np.arange(20) * 1e-5) + n, E, B = sio.get_cosebis(a, (0, 0)) + assert np.array_equal(n, np.arange(1, 11)) + assert type(a.covariance).__name__ == "FullCovariance" + assert a.covariance.dense.shape == (len(a.mean), len(a.mean)) + + th_f, p_f, _ = sio.get_xi(f, (0, 0), grid="fine") + assert np.array_equal(th_f, theta_f) and np.array_equal(p_f, np.arange(200) * 1e-5) + assert type(f.covariance).__name__ == "DiagonalCovariance" + assert np.array_equal(np.diag(f.covariance.dense), variances) From 66bbbc0e6ad27b901fbd3009132d011dd78d9fa2 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 02:50:52 +0200 Subject: [PATCH 04/47] fix(sacc_io): enforce ascending grids; readers in insertion order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fresh-eyes review caught a correctness bug: readers re-sorted selections by theta/ell/n, but covariance blocks and bandpower windows stay in insertion order. On a non-ascending grid the reader output silently desynchronised from its covariance, and get_pseudo_cl returned sorted cl arrays against unsorted window columns — internally inconsistent within one return tuple. Fix by construction, not by sort: - Writers validate their grids. add_xi/add_pure_eb/add_rho/add_tau require strictly ascending theta; add_pseudo_cl requires strictly ascending ell_eff (add_cosebis is inherently safe — it enumerates the mode index). Out-of-order grids raise a loud ValueError naming the argument. - Readers drop the sort entirely and return in s.indices (insertion) order, so every getter is covariance- and window-aligned for ANY file, and ascending for canonical files. The _sorted_* helpers are replaced by plain insertion-order accessors (_mean/_tag). Also: - _pair normalises (i, j) -> sorted, so get_xi(s, (1, 0)) addresses the same symmetric shear-shear pair as (0, 1) instead of a silent empty read. - Module docstring documents the tomographic ξ covariance ordering: insertion is pair-major ([pair0 xip; pair0 xim; pair1 xip; …]), supplied to assemble_covariance as one contiguous block matching add_xi call order; type-major converters (DES 2pt-FITS) permute explicitly via s.indices. - extract() docstring states tracers takes SACC names, not integer bins. New tests (26 total, was 20): writers reject non-ascending theta/ell; (1, 0) == (0, 1) round-trip; a 3-pair tomographic ξ covariance assembled as one contiguous pair-major block with per-pair sub-blocks recovered via extract(); get_pseudo_cl window column j <-> ell_eff[j] via window_ind tags. Co-Authored-By: Claude Opus --- src/sp_validation/sacc_io.py | 114 +++++++++++++------ src/sp_validation/tests/test_sacc_io.py | 140 ++++++++++++++++++++++++ 2 files changed, 218 insertions(+), 36 deletions(-) diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index cf03f12c..4ed85d0f 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -26,6 +26,18 @@ Tag filters are plain keyword arguments to ``indices`` / ``get_data_points`` / ``get_tag``; the ``tags={...}`` form silently selects nothing and must never be used. + + Tomographic ξ ordering: each ``add_xi`` call inserts one tracer + pair as ``[xip; xim]``, so a multi-pair vector is *pair-major* + — ``[pair_0 xip; pair_0 xim; pair_1 xip; …]`` — not type-major + (``[all xip; all xim]``). A tomographic ξ covariance with + cross-pair correlations is therefore supplied to + ``assemble_covariance`` as ONE contiguous block spanning the + consecutive ``add_xi`` calls, ordered pair-by-pair to match + insertion. Writers must call ``add_xi`` in the same pair order + the covariance was built in. Converters that need a type-major + layout (e.g. the DES 2pt-FITS convention) permute explicitly via + ``s.indices`` rather than assuming global order. """ import numpy as np @@ -97,11 +109,32 @@ def new_sacc(nz, metadata=None): def _pair(bins): - """Resolve a ``(i, j)`` bin pair to the ``(source_i, source_j)`` names.""" - i, j = bins + """Resolve a ``(i, j)`` bin pair to the ``(source_i, source_j)`` names. + + The pair is normalised to ``i <= j``: shear-shear statistics are symmetric + in the tracer pair, and SACC stores each pair under one ordering, so + ``(1, 0)`` must address the same points as ``(0, 1)``. + """ + i, j = sorted(bins) return (source_name(i), source_name(j)) +def _check_ascending(name, values): + """Require ``values`` to be strictly ascending; else raise ValueError. + + Insertion order is the covariance (and bandpower-window) order, and readers + return points in insertion order, so an out-of-order grid would silently + desynchronise a data vector from its covariance. Enforce monotonicity at + write time instead. + """ + values = np.asarray(values) + if not np.all(np.diff(values) > 0): + raise ValueError( + f"{name} must be strictly ascending (insertion order is the " + f"covariance order); got {values.tolist()}" + ) + + def add_xi( s, bins, @@ -134,6 +167,7 @@ def add_xi( npairs, weight : array_like, optional TreeCorr pair counts and weights, stored per point. """ + _check_ascending("theta", theta) tracers = _pair(bins) for dtype, xi in ((XI_PLUS, xip), (XI_MINUS, xim)): for n, th in enumerate(theta): @@ -177,6 +211,7 @@ def add_pseudo_cl( bandpower — from NaMaster ``get_bandpower_windows``. One ``sacc.BandpowerWindow`` is built and shared across EE/BB/EB. """ + _check_ascending("ell_eff", ell_eff) tracers = _pair(bins) window = sacc.BandpowerWindow(np.asarray(window_ells), np.asarray(window_weights)) for dtype, cl in ((CL_EE, cl_ee), (CL_BB, cl_bb), (CL_EB, cl_eb)): @@ -234,6 +269,7 @@ def add_pure_eb(s, bins, theta, xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb): xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb : array_like The six pure E/B / ambiguous mode arrays at ``theta``. """ + _check_ascending("theta", theta) tracers = _pair(bins) values = { "xip_E": xip_E, @@ -263,6 +299,7 @@ def add_rho(s, k, theta, rho_p, rho_m): rho_p, rho_m : array_like ρ_k+ and ρ_k− at ``theta``. """ + _check_ascending("theta", theta) tracers = (PSF_TRACER, PSF_TRACER) for dtype, arr in ((RHO_PLUS.format(k=k), rho_p), (RHO_MINUS.format(k=k), rho_m)): for n, th in enumerate(theta): @@ -286,6 +323,7 @@ def add_tau(s, bins, k, theta, tau_p, tau_m): tau_p, tau_m : array_like τ_k+ and τ_k− at ``theta``. """ + _check_ascending("theta", theta) tracers = (source_name(bins[0]), PSF_TRACER) for dtype, arr in ((TAU_PLUS.format(k=k), tau_p), (TAU_MINUS.format(k=k), tau_m)): for n, th in enumerate(theta): @@ -397,9 +435,9 @@ def get_xi(s, bins, *, grid): """Return ``(theta, xip, xim)`` for one tracer pair and grid.""" tracers = _pair(bins) return ( - _sorted_tag(s, XI_PLUS, tracers, "theta", grid=grid), - _sorted_mean(s, XI_PLUS, tracers, grid=grid), - _sorted_mean(s, XI_MINUS, tracers, grid=grid), + _tag(s, XI_PLUS, tracers, "theta", grid=grid), + _mean(s, XI_PLUS, tracers, grid=grid), + _mean(s, XI_MINUS, tracers, grid=grid), ) @@ -407,16 +445,17 @@ def get_pseudo_cl(s, bins): """Return ``(ell_eff, cl_ee, cl_bb, cl_eb, window)`` for one tracer pair. ``window`` is the shared ``sacc.BandpowerWindow`` recovered via - ``get_bandpower_windows``. + ``get_bandpower_windows``; its columns are in the same insertion order as + the returned ``ell_eff``/``cl`` arrays, so window column ``j`` corresponds + to ``ell_eff[j]``. """ tracers = _pair(bins) - ell = _sorted_tag(s, CL_EE, tracers, "ell") window = s.get_bandpower_windows(s.indices(CL_EE, tracers)) return ( - ell, - _sorted_mean(s, CL_EE, tracers, _sort_tag="ell"), - _sorted_mean(s, CL_BB, tracers, _sort_tag="ell"), - _sorted_mean(s, CL_EB, tracers, _sort_tag="ell"), + _tag(s, CL_EE, tracers, "ell"), + _mean(s, CL_EE, tracers), + _mean(s, CL_BB, tracers), + _mean(s, CL_EB, tracers), window, ) @@ -435,11 +474,11 @@ def get_cosebis(s, bins, scale_cut=None): if scale_cut is not None else {} ) - modes = _sorted_tag(s, COSEBI_EE, tracers, "n", **tags) + modes = _tag(s, COSEBI_EE, tracers, "n", **tags) return ( modes.astype(int), - _sorted_mean(s, COSEBI_EE, tracers, **tags, _sort_tag="n"), - _sorted_mean(s, COSEBI_BB, tracers, **tags, _sort_tag="n"), + _mean(s, COSEBI_EE, tracers, **tags), + _mean(s, COSEBI_BB, tracers, **tags), ) @@ -449,8 +488,8 @@ def get_pure_eb(s, bins): The dict is keyed by ``PURE_KEYS`` (xip_E, xim_E, …). """ tracers = _pair(bins) - theta = _sorted_tag(s, PURE_TYPES["xip_E"], tracers, "theta") - arrays = {key: _sorted_mean(s, PURE_TYPES[key], tracers) for key in PURE_KEYS} + theta = _tag(s, PURE_TYPES["xip_E"], tracers, "theta") + arrays = {key: _mean(s, PURE_TYPES[key], tracers) for key in PURE_KEYS} return theta, arrays @@ -459,9 +498,9 @@ def get_rho(s, k): tracers = (PSF_TRACER, PSF_TRACER) dt_p, dt_m = RHO_PLUS.format(k=k), RHO_MINUS.format(k=k) return ( - _sorted_tag(s, dt_p, tracers, "theta"), - _sorted_mean(s, dt_p, tracers), - _sorted_mean(s, dt_m, tracers), + _tag(s, dt_p, tracers, "theta"), + _mean(s, dt_p, tracers), + _mean(s, dt_m, tracers), ) @@ -470,28 +509,26 @@ def get_tau(s, bins, k): tracers = (source_name(bins[0]), PSF_TRACER) dt_p, dt_m = TAU_PLUS.format(k=k), TAU_MINUS.format(k=k) return ( - _sorted_tag(s, dt_p, tracers, "theta"), - _sorted_mean(s, dt_p, tracers), - _sorted_mean(s, dt_m, tracers), + _tag(s, dt_p, tracers, "theta"), + _mean(s, dt_p, tracers), + _mean(s, dt_m, tracers), ) -def _order(s, data_type, tracers, sort_tag, **tag_filters): - """Indices for a selection, ordered by ascending ``sort_tag``.""" - idx = np.asarray(s.indices(data_type, tracers, **tag_filters), dtype=int) - key = np.array([s.data[i].tags[sort_tag] for i in idx]) - return idx[np.argsort(key)] +def _mean(s, data_type, tracers, **tag_filters): + """Mean values for a selection, in ``s.indices`` (insertion) order. - -def _sorted_mean(s, data_type, tracers, _sort_tag="theta", **tag_filters): - """Mean values for a selection, ordered by ``_sort_tag`` ascending.""" - idx = _order(s, data_type, tracers, _sort_tag, **tag_filters) - return s.mean[idx] + Never re-sort: insertion order is the covariance and bandpower-window + order, so returning in ``s.indices`` order keeps every reader aligned with + the covariance for any file (and ascending for canonically-written files, + which the writers enforce). + """ + return s.mean[s.indices(data_type, tracers, **tag_filters)] -def _sorted_tag(s, data_type, tracers, tag, **tag_filters): - """Values of ``tag`` for a selection, ordered by that tag ascending.""" - idx = _order(s, data_type, tracers, tag, **tag_filters) +def _tag(s, data_type, tracers, tag, **tag_filters): + """Values of ``tag`` for a selection, in insertion order.""" + idx = s.indices(data_type, tracers, **tag_filters) return np.array([s.data[i].tags[tag] for i in idx]) @@ -509,7 +546,12 @@ def extract(s, data_type=None, tracers=None, **tag_filters): data_type : str, optional Data type to keep. tracers : tuple, optional - Tracer pair to keep. + Tracer pair to keep, as SACC tracer **names** (e.g. + ``("source_0", "source_0")`` or ``("source_0", "psf_stars")``) — *not* + integer bin indices. This differs deliberately from the ``add_*`` / + ``get_*`` interface, whose ``bins`` argument takes integer pairs: + ``extract`` is the generic selection escape hatch, mirroring + ``Sacc.keep_selection`` and addressing non-source tracers uniformly. **tag_filters Tag filters (plain kwargs, e.g. ``grid='fine'``). diff --git a/src/sp_validation/tests/test_sacc_io.py b/src/sp_validation/tests/test_sacc_io.py index 883a2af0..28bd6627 100644 --- a/src/sp_validation/tests/test_sacc_io.py +++ b/src/sp_validation/tests/test_sacc_io.py @@ -489,3 +489,143 @@ def test_end_to_end_two_file_layout(tmp_path): assert np.array_equal(th_f, theta_f) and np.array_equal(p_f, np.arange(200) * 1e-5) assert type(f.covariance).__name__ == "DiagonalCovariance" assert np.array_equal(np.diag(f.covariance.dense), variances) + + +# --------------------------------------------------------------------------- # +# 9. Ascending-grid enforcement (writers reject out-of-order grids) +# --------------------------------------------------------------------------- # +def test_add_xi_rejects_non_ascending_theta(): + s = _base_sacc() + theta = _theta()[::-1] # descending + with pytest.raises(ValueError, match="theta must be strictly ascending"): + sio.add_xi( + s, (0, 0), theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + + +def test_add_pseudo_cl_rejects_non_ascending_ell(): + s = _base_sacc() + ell = np.array([210.0, 30.0, 120.0]) # not ascending + W = np.random.default_rng(0).uniform(size=(20, 3)) + with pytest.raises(ValueError, match="ell_eff must be strictly ascending"): + sio.add_pseudo_cl( + s, + (0, 0), + ell, + np.arange(3) * 1e-9, + np.arange(3) * 2e-9, + np.arange(3) * 3e-9, + window_ells=np.arange(2, 22).astype(float), + window_weights=W, + ) + + +def test_add_pure_eb_rho_tau_reject_non_ascending_theta(): + s = _base_sacc() + theta = _theta()[::-1] + pure = {key: np.arange(6) * 1e-6 for key in sio.PURE_KEYS} + with pytest.raises(ValueError, match="theta must be strictly ascending"): + sio.add_pure_eb(s, (0, 0), theta, **pure) + with pytest.raises(ValueError, match="theta must be strictly ascending"): + sio.add_rho(s, 0, theta, np.arange(6) * 1e-6, np.arange(6) * 2e-6) + with pytest.raises(ValueError, match="theta must be strictly ascending"): + sio.add_tau(s, (0, 0), 0, theta, np.arange(6) * 1e-6, np.arange(6) * 2e-6) + + +# --------------------------------------------------------------------------- # +# 10. Bin-pair normalisation: (1, 0) addresses the same points as (0, 1) +# --------------------------------------------------------------------------- # +def test_bin_pair_normalisation(): + theta = _theta() + xip, xim = np.arange(6) * 1e-5, np.arange(6) * 2e-5 + s = _base_sacc(nbins=2) + sio.add_xi(s, (0, 1), theta, xip, xim, grid="coarse") + th01, p01, m01 = sio.get_xi(s, (0, 1), grid="coarse") + th10, p10, m10 = sio.get_xi(s, (1, 0), grid="coarse") # reversed order + assert np.array_equal(th01, th10) + assert np.array_equal(p01, p10) and np.array_equal(p10, xip) + assert np.array_equal(m01, m10) and np.array_equal(m10, xim) + # writing under (1, 0) lands in the same tracer pair, not a new one + s2 = _base_sacc(nbins=2) + sio.add_xi(s2, (1, 0), theta, xip, xim, grid="coarse") + assert len(s2.indices(sio.XI_PLUS, ("source_0", "source_1"))) == len(theta) + + +# --------------------------------------------------------------------------- # +# 11. Reader/covariance alignment holds for ANY insertion order (the core +# regression the review caught): a tomographic multi-pair ξ covariance +# assembled as ONE contiguous pair-major block, per-pair sub-blocks +# recovered via extract(). +# --------------------------------------------------------------------------- # +def test_tomographic_xi_covariance_one_contiguous_block(): + theta = _theta() + nth = len(theta) + pairs = [(0, 0), (0, 1), (1, 1)] + s = _base_sacc(nbins=2) + for k, (i, j) in enumerate(pairs): + sio.add_xi( + s, + (i, j), + theta, + np.arange(nth) * (k + 1) * 1e-5, + np.arange(nth) * (k + 1) * 2e-5, + grid="coarse", + ) + # All ξ points as one contiguous block in insertion (pair-major) order. + xi_idx = np.arange(len(s.mean)) + assert np.array_equal(xi_idx, np.arange(3 * 2 * nth)) # 3 pairs x [xip; xim] + cov = _spd(len(xi_idx), 11) # dense, cross-pair correlations + sio.assemble_covariance(s, [(xi_idx, cov)]) + + # Per-pair xip sub-block: resolve indices, extract, compare to input. + for i, j in pairs: + idx_p = s.indices( + sio.XI_PLUS, ("source_" + str(min(i, j)), "source_" + str(max(i, j))) + ) + sub = sio.extract( + s, + data_type=sio.XI_PLUS, + tracers=("source_" + str(min(i, j)), "source_" + str(max(i, j))), + ) + assert np.array_equal(sub.covariance.dense, cov[np.ix_(idx_p, idx_p)]) + # readers stay covariance-aligned: get_xi returns in the same order + th, xip, _ = sio.get_xi(s, (i, j), grid="coarse") + assert np.array_equal(th, theta) + assert np.array_equal(s.mean[idx_p], xip) + + +# --------------------------------------------------------------------------- # +# 12. get_pseudo_cl window/cl column correspondence: window column j maps to +# the returned ell_eff[j] (verified through window_ind tags). +# --------------------------------------------------------------------------- # +def test_pseudo_cl_window_column_correspondence(tmp_path): + ell_eff = np.array([30.0, 120.0, 210.0, 300.0]) + nell, nbp = 40, len(ell_eff) + window_ells = np.arange(2, 2 + nell).astype(float) + # Distinct columns so a permutation would be detectable. + W = np.zeros((nell, nbp)) + for b in range(nbp): + W[b * 5 : (b + 1) * 5, b] = 1.0 + ee, bb, eb = np.arange(nbp) * 1e-9, np.arange(nbp) * 2e-9, np.arange(nbp) * 3e-9 + s = _base_sacc() + sio.add_pseudo_cl( + s, + (0, 0), + ell_eff, + ee, + bb, + eb, + window_ells=window_ells, + window_weights=W, + ) + s2 = _roundtrip(s, tmp_path, "clwin") + ell, cl_ee, _, _, window = sio.get_pseudo_cl(s2, (0, 0)) + # returned ell array is in insertion order + assert np.array_equal(ell, ell_eff) + assert np.array_equal(cl_ee, ee) + # window_ind tag on each EE point indexes the matching window column + idx = s2.indices(sio.CL_EE, ("source_0", "source_0")) + for pos, i in enumerate(idx): + col = s2.data[i].tags["window_ind"] + assert col == pos # insertion order preserved => column j <-> ell[j] + assert np.array_equal(window.weight[:, col], W[:, pos]) From 2f4a8a883b02a89394811f2325c29fd5c6b0b68d Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 03:19:29 +0200 Subject: [PATCH 05/47] feat(cosmo_val): SACC writers for the born-as-SACC data products MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add sacc_writers.py: pure per-statistic *_to_sacc functions (xi coarse/fine, pseudo-Cl, COSEBIs, pure-E/B, rho/tau) that turn computed arrays into single-statistic SACC "parts", plus assemble_analysis_sacc, which rebuilds the single {version}.sacc analysis file with a FullCovariance assembled block- diagonally in canonical order (per the SACC layout contract — not concatenate_data_sets, whose BlockDiagonalCovariance the contract rules out). Add bandpower_window_from_workspace to pseudo_cl.py: the NEW plumbing that threads NaMaster's get_bandpower_windows() into sacc_io.add_pseudo_cl (the EE/BB/EB diagonal window is verified identical). Tests: 12 fast tests, each writer round-tripped and checked against the contract; a real small-nside NaMaster window round-trip; analysis assembly proven to cover every point with aligned blocks and zero cross-blocks, incl. the reload-from-disk DAG path. Co-Authored-By: Claude Fable 5 --- src/sp_validation/cosmo_val/sacc_writers.py | 234 ++++++++++++++ src/sp_validation/pseudo_cl.py | 29 ++ src/sp_validation/tests/test_sacc_writers.py | 310 +++++++++++++++++++ 3 files changed, 573 insertions(+) create mode 100644 src/sp_validation/cosmo_val/sacc_writers.py create mode 100644 src/sp_validation/tests/test_sacc_writers.py diff --git a/src/sp_validation/cosmo_val/sacc_writers.py b/src/sp_validation/cosmo_val/sacc_writers.py new file mode 100644 index 00000000..d227c379 --- /dev/null +++ b/src/sp_validation/cosmo_val/sacc_writers.py @@ -0,0 +1,234 @@ +"""Born-as-SACC writers for the cosmo_val data products. + +A thin, pure layer between the ``cosmo_val`` mixins (which compute statistics as +TreeCorr / NaMaster / b_modes arrays) and :mod:`sp_validation.sacc_io` (which +knows the file layout). Each ``*_to_sacc`` function turns one already-computed +statistic into a single-statistic SACC — a *part* — carrying that statistic's +own covariance as its one covariance block. The Snakemake DAG writes one part +per rule; :func:`assemble_analysis_sacc` then loads the parts and rebuilds the +single ``{version}.sacc`` analysis file with a ``FullCovariance`` assembled +block-diagonally in canonical order (per the SACC layout contract — *not* +``sacc.concatenate_data_sets``, whose ``BlockDiagonalCovariance`` output the +contract rules out). + +The fine-grid ``{version}_xi_fine.sacc`` is a terminal product in its own right +(:func:`xi_to_sacc` with ``grid="fine"`` and a ``DiagonalCovariance`` from +TreeCorr ``varxip``/``varxim``); COSEBIs and pure-E/B consume it. + +Everything here is single-bin today (``bins=(0, 0)``); the interface is +tomography-native so a future round supplies real bin pairs unchanged. +""" + +import numpy as np + +from .. import sacc_io as sio +from ..pseudo_cl import bandpower_window_from_workspace + +# Statistics carried in the analysis file, and their custom-type k indices. +RHO_K = range(6) # ρ_0 … ρ_5 +TAU_K = (0, 2, 5) # τ_0, τ_2, τ_5 + +# NaMaster spin-2 × spin-2 decoupled-spectrum row order (EE, EB, BE, BB). +_NMT_EE, _NMT_EB, _NMT_BB = 0, 1, 3 + +BIN = (0, 0) # single-bin default until the round goes tomographic + + +def xi_to_sacc( + nz, + metadata, + theta, + xip, + xim, + *, + grid, + theta_nom=None, + npairs=None, + weight=None, + variances=None, +): + """One ξ± part (``bins=(0, 0)``) on the coarse or fine grid. + + ``variances`` (the concatenated ``[varxip; varxim]``) attaches a + ``DiagonalCovariance`` — used for the terminal fine file, where npatch=1 + leaves TreeCorr shot-noise variance as the only covariance estimate. + """ + s = sio.new_sacc(nz, metadata) + sio.add_xi( + s, + BIN, + theta, + xip, + xim, + grid=grid, + theta_nom=theta_nom, + npairs=npairs, + weight=weight, + ) + if variances is not None: + sio.add_diagonal_covariance(s, np.asarray(variances)) + return s + + +def pseudo_cl_to_sacc(nz, metadata, ell_eff, cl_all, wsp, covariance=None): + """One pseudo-Cℓ part: EE/BB/EB with the shared bandpower window. + + ``cl_all`` is NaMaster's decoupled ``(4, nbp)`` array (EE, EB, BE, BB); the + window comes from :func:`bandpower_window_from_workspace`. ``covariance``, + when given, is the dense ``[EE; BB; EB]``-ordered block matching insertion. + """ + window_ells, window_weights = bandpower_window_from_workspace(wsp) + s = sio.new_sacc(nz, metadata) + sio.add_pseudo_cl( + s, + BIN, + ell_eff, + cl_all[_NMT_EE], + cl_all[_NMT_BB], + cl_all[_NMT_EB], + window_ells=window_ells, + window_weights=window_weights, + ) + if covariance is not None: + s.add_covariance(np.asarray(covariance)) + return s + + +def cosebis_to_sacc(nz, metadata, result, scale_cut): + """One COSEBIs part at the fiducial scale cut. + + ``result`` is a single scale-cut result dict from + ``b_modes.calculate_cosebis`` — ``{"En", "Bn", "cov", ...}`` — where ``cov`` + is the ``[En; Bn]``-ordered COSEBIs covariance. Non-fiducial scale cuts are + a diagnostic (the PTE scan) and stay in the sidecar ``.npz``; only the + fiducial cut is a data product, because a ``FullCovariance`` must cover + every stored point and the cuts overlap in mode space. + """ + s = sio.new_sacc(nz, metadata) + sio.add_cosebis(s, BIN, result["En"], result["Bn"], scale_cut) + s.add_covariance(np.asarray(result["cov"])) + return s + + +def pure_eb_to_sacc(nz, metadata, theta, eb, covariance=None): + """One pure-E/B part: the six ``sacc_io.PURE_KEYS`` blocks. + + ``eb`` is a mapping with the six keys (``xip_E`` … ``xim_amb``); each array + is sampled at ``theta``. ``covariance``, when given, is the dense block in + ``PURE_KEYS`` order (matching ``b_modes._EB_KEYS`` and the insertion order). + """ + s = sio.new_sacc(nz, metadata) + sio.add_pure_eb(s, BIN, theta, **{key: eb[key] for key in sio.PURE_KEYS}) + if covariance is not None: + s.add_covariance(np.asarray(covariance)) + return s + + +def rho_tau_to_sacc(nz, metadata, rho_stats, tau_stats, tau_cov=None): + """One ρ/τ part: ρ_0…ρ_5 autos and τ_0/τ_2/τ_5 leakage. + + ``rho_stats`` / ``tau_stats`` are the ``shear_psf_leakage`` handler tables + (columns ``theta``, ``rho_{k}_p``, ``varrho_{k}_p``, ``rho_{k}_m``, … and + the τ analogue). ρ carries a diagonal (varxip/varxim) covariance — a + diagnostic placeholder, not used by inference — while τ carries ``tau_cov``, + the theoretical ``CovTauTh`` block the CosmoSIS τ-likelihood consumes. The + block order matches insertion: ρ (all +then−, per k) then τ. + """ + s = sio.new_sacc(nz, metadata) + theta_rho = np.asarray(rho_stats["theta"]) + for k in RHO_K: + sio.add_rho( + s, + k, + theta_rho, + np.asarray(rho_stats[f"rho_{k}_p"]), + np.asarray(rho_stats[f"rho_{k}_m"]), + ) + theta_tau = np.asarray(tau_stats["theta"]) + for k in TAU_K: + sio.add_tau( + s, + BIN, + k, + theta_tau, + np.asarray(tau_stats[f"tau_{k}_p"]), + np.asarray(tau_stats[f"tau_{k}_m"]), + ) + rho_var = np.concatenate( + [ + np.concatenate([rho_stats[f"varrho_{k}_p"], rho_stats[f"varrho_{k}_m"]]) + for k in RHO_K + ] + ) + tau_var = np.concatenate( + [ + np.concatenate([tau_stats[f"vartau_{k}_p"], tau_stats[f"vartau_{k}_m"]]) + for k in TAU_K + ] + ) + if tau_cov is None: + # Pure diagnostic file: diagonal covariance across ρ and τ. + s.add_covariance(np.concatenate([rho_var, tau_var])) + else: + # ρ diagonal (diagnostic) + τ dense theoretical block (inference input). + n_rho, n_tau = len(rho_var), len(tau_var) + tau_cov = np.asarray(tau_cov) + if tau_cov.shape != (n_tau, n_tau): + raise ValueError( + f"tau_cov shape {tau_cov.shape} does not match the {n_tau} τ " + "data points" + ) + full = np.zeros((n_rho + n_tau, n_rho + n_tau)) + full[:n_rho, :n_rho] = np.diag(rho_var) + full[n_rho:, n_rho:] = tau_cov + s.add_covariance(full) + return s + + +# --------------------------------------------------------------------------- # +# Analysis-file assembly +# --------------------------------------------------------------------------- # +def _copy_data_points(dst, src): + """Append every data point of ``src`` into ``dst`` (tags preserved).""" + for dp in src.data: + dst.add_data_point(dp.data_type, dp.tracers, dp.value, **dp.tags) + + +def assemble_analysis_sacc(nz, metadata, parts): + """Rebuild the single ``{version}.sacc`` analysis file from parts. + + Each part is a single-statistic Sacc (from a ``*_to_sacc`` writer, loaded + from disk) carrying its own covariance = its block. This re-adds every + part's data points into one Sacc in the order the parts are given — which + must be the canonical order (ξ± coarse, pseudo-Cℓ, COSEBIs, pure-E/B, ρ, τ) + — and assembles a single ``FullCovariance`` from the per-part covariance + blocks. Point insertion order and block order therefore agree by + construction, which ``sacc_io.assemble_covariance`` validates (contiguous, + tiling, square) and raises on if they don't. + + Parameters + ---------- + nz, metadata : see :func:`sp_validation.sacc_io.new_sacc`. + parts : sequence of sacc.Sacc + Single-statistic parts, each with a covariance, in canonical order. + + Returns + ------- + sacc.Sacc + The analysis Sacc with a ``FullCovariance`` covering every point. + """ + s = sio.new_sacc(nz, metadata) + blocks = [] + cursor = 0 + for part in parts: + if part.covariance is None: + raise ValueError( + "every analysis part must carry its own covariance block; " + f"a part with data types {sorted(set(dp.data_type for dp in part.data))} " + "has none" + ) + n = len(part.mean) + _copy_data_points(s, part) + blocks.append((np.arange(cursor, cursor + n), part.covariance.dense)) + cursor += n + return sio.assemble_covariance(s, blocks) diff --git a/src/sp_validation/pseudo_cl.py b/src/sp_validation/pseudo_cl.py index c9355ec9..34cbc68d 100644 --- a/src/sp_validation/pseudo_cl.py +++ b/src/sp_validation/pseudo_cl.py @@ -280,3 +280,32 @@ def get_pseudo_cls_catalog( cl_all = wsp.decouple_cell(cl_coupled) return ell_eff, cl_all, wsp + + +# NaMaster spin-2 × spin-2 spectrum order: EE, EB, BE, BB. +_NMT_EE = 0 + + +def bandpower_window_from_workspace(wsp): + """Extract the bandpower window matrix ``W`` for a spin-2×spin-2 workspace. + + NaMaster's ``get_bandpower_windows()`` returns a four-index array + ``(n_cl_out, n_bpw, n_cl_in, n_ell)`` describing how each output bandpower + is built from the input multipoles across the EE/EB/BE/BB spectra. SACC's + ``BandpowerWindow`` model (one window per bandpower, shared across the + stored spectra) needs the per-spectrum *decoupling* window, i.e. the + diagonal EE←EE block (equal to BB←BB and EB←EB, verified identical). + + Returns + ------- + window_ells : np.ndarray + Multipoles the window spans, ``arange(n_ell)`` — the ``ell`` axis of + ``compute_coupled_cell``. + window_weights : np.ndarray + ``W`` of shape ``(n_ell, n_bpw)`` — one column per bandpower, the layout + :func:`sp_validation.sacc_io.add_pseudo_cl` expects. + """ + bpw = wsp.get_bandpower_windows() # (n_cl_out, n_bpw, n_cl_in, n_ell) + diagonal = bpw[_NMT_EE, :, _NMT_EE, :] # (n_bpw, n_ell) + window_ells = np.arange(diagonal.shape[1], dtype=float) + return window_ells, diagonal.T diff --git a/src/sp_validation/tests/test_sacc_writers.py b/src/sp_validation/tests/test_sacc_writers.py new file mode 100644 index 00000000..057697e4 --- /dev/null +++ b/src/sp_validation/tests/test_sacc_writers.py @@ -0,0 +1,310 @@ +"""Tests for :mod:`sp_validation.cosmo_val.sacc_writers`. + +Synthetic and fast: each ``*_to_sacc`` writer is exercised with in-memory +arrays, round-tripped through ``tmp_path``, and checked against the SACC layout +contract (data types, tags, ordering, covariance alignment). The analysis-file +assembler is verified to produce a single ``FullCovariance`` covering every +point with each per-statistic block correctly placed. One real small-nside +NaMaster round-trip proves the pseudo-Cℓ window survives the writer path. +""" + +import numpy as np +import pytest + +from sp_validation import sacc_io as sio +from sp_validation.cosmo_val import sacc_writers as sw + + +def _nz(seed=0, n=40): + rng = np.random.default_rng(seed) + return np.linspace(0.01, 2.0, n), rng.uniform(0.1, 1.0, n) + + +def _spd(n, seed): + a = np.random.default_rng(seed).normal(size=(n, n)) + return a @ a.T + n * np.eye(n) + + +def _theta(n=6): + return np.geomspace(1.0, 100.0, n) + + +def _roundtrip(s, tmp_path, name): + p = tmp_path / f"{name}.sacc" + sio.save(s, str(p)) + return sio.load(str(p)) + + +META = {"catalogue_version": "vSYNTH", "npatch": 1} + + +# --------------------------------------------------------------------------- # +# Per-writer parts +# --------------------------------------------------------------------------- # +def test_xi_to_sacc_coarse(tmp_path): + theta = _theta() + xip, xim = np.arange(6) * 1e-5, np.arange(6) * 2e-5 + s = sw.xi_to_sacc( + {0: _nz()}, META, theta, xip, xim, grid="coarse", theta_nom=theta * 1.01 + ) + s2 = _roundtrip(s, tmp_path, "xic") + th, p, m = sio.get_xi(s2, (0, 0), grid="coarse") + assert np.array_equal(th, theta) + assert np.array_equal(p, xip) and np.array_equal(m, xim) + assert s2.covariance is None # coarse part has no cov until assembly + + +def test_xi_to_sacc_fine_diagonal(tmp_path): + theta = np.geomspace(0.5, 300.0, 30) + xip, xim = np.arange(30) * 1e-5, np.arange(30) * 2e-5 + varxip, varxim = np.arange(1, 31) * 1e-12, np.arange(1, 31) * 2e-12 + s = sw.xi_to_sacc( + {0: _nz()}, + META, + theta, + xip, + xim, + grid="fine", + variances=np.concatenate([varxip, varxim]), + ) + assert type(s.covariance).__name__ == "DiagonalCovariance" + s2 = _roundtrip(s, tmp_path, "xif") + th, p, _ = sio.get_xi(s2, (0, 0), grid="fine") + assert np.array_equal(th, theta) and np.array_equal(p, xip) + assert np.array_equal( + np.diag(s2.covariance.dense), np.concatenate([varxip, varxim]) + ) + + +def test_pseudo_cl_to_sacc_window_and_rows(tmp_path): + ell = np.array([30.0, 60.0, 90.0, 120.0]) + nbp = len(ell) + # NaMaster (4, nbp): EE, EB, BE, BB. + cl_all = np.vstack( + [ + np.arange(nbp) * 1e-9, + np.arange(nbp) * 2e-9, + np.zeros(nbp), + np.arange(nbp) * 3e-9, + ] + ) + + class _Wsp: + """Stand-in workspace: (n_cl_out, nbp, n_cl_in, nell) window array.""" + + def __init__(self, nbp, nell): + w = np.zeros((4, nbp, 4, nell)) + col = np.zeros((nbp, nell)) + for b in range(nbp): + col[b, b * 3 : b * 3 + 3] = 1.0 + for out in range(4): + w[out, :, out, :] = col + self._w = w + + def get_bandpower_windows(self): + return self._w + + s = sw.pseudo_cl_to_sacc({0: _nz()}, META, ell, cl_all, _Wsp(nbp, 24)) + s2 = _roundtrip(s, tmp_path, "cl") + ell_r, ee, bb, eb, window = sio.get_pseudo_cl(s2, (0, 0)) + assert np.array_equal(ell_r, ell) + assert np.array_equal(ee, cl_all[0]) # EE row + assert np.array_equal(bb, cl_all[3]) # BB row (index 3, not 2=BE) + assert np.array_equal(eb, cl_all[1]) # EB row + assert window.weight.shape == (24, nbp) + + +def test_pseudo_cl_to_sacc_real_namaster(tmp_path): + """A real small-nside NaMaster workspace's window survives the writer.""" + pytest.importorskip("pymaster") + from sp_validation.pseudo_cl import get_pseudo_cls_map + + nside = 32 + mask = np.ones(12 * nside**2) + rng = np.random.default_rng(0) + shear = ( + rng.normal(size=12 * nside**2) + 1j * rng.normal(size=12 * nside**2) + ) * 1e-2 + ell_eff, cl_all, wsp = get_pseudo_cls_map(shear, mask, nside, "linear", ell_step=8) + s = sw.pseudo_cl_to_sacc({0: _nz()}, META, ell_eff, cl_all, wsp) + s2 = _roundtrip(s, tmp_path, "clreal") + ell_r, ee, bb, eb, window = sio.get_pseudo_cl(s2, (0, 0)) + assert np.array_equal(ell_r, ell_eff) + assert np.array_equal(ee, cl_all[0]) and np.array_equal(bb, cl_all[3]) + # window columns correspond to the bandpowers, one per ell_eff + assert window.weight.shape[1] == len(ell_eff) + + +def test_cosebis_to_sacc(tmp_path): + En, Bn = np.arange(1, 11) * 1e-6, np.arange(1, 11) * 1e-7 + result = {"En": En, "Bn": Bn, "cov": _spd(20, 7)} + s = sw.cosebis_to_sacc({0: _nz()}, META, result, (1.0, 100.0)) + s2 = _roundtrip(s, tmp_path, "co") + n, E, B = sio.get_cosebis(s2, (0, 0)) + assert np.array_equal(n, np.arange(1, 11)) + assert np.array_equal(E, En) and np.array_equal(B, Bn) + assert type(s2.covariance).__name__ == "FullCovariance" + assert np.array_equal(s2.covariance.dense, result["cov"]) + + +def test_pure_eb_to_sacc(tmp_path): + theta = _theta() + eb = {key: np.arange(6) * (i + 1) * 1e-6 for i, key in enumerate(sio.PURE_KEYS)} + cov = _spd(6 * len(theta), 9) + s = sw.pure_eb_to_sacc({0: _nz()}, META, theta, eb, covariance=cov) + s2 = _roundtrip(s, tmp_path, "eb") + th, back = sio.get_pure_eb(s2, (0, 0)) + assert np.array_equal(th, theta) + for key in sio.PURE_KEYS: + assert np.array_equal(back[key], eb[key]) + assert np.array_equal(s2.covariance.dense, cov) + + +def _rho_tau_tables(nth=6, seed=0): + rng = np.random.default_rng(seed) + theta = _theta(nth) + rho = {"theta": theta} + for k in sw.RHO_K: + for suffix in ("p", "m"): + rho[f"rho_{k}_{suffix}"] = rng.normal(size=nth) * 1e-6 + rho[f"varrho_{k}_{suffix}"] = rng.uniform(1e-14, 1e-13, nth) + tau = {"theta": theta} + for k in sw.TAU_K: + for suffix in ("p", "m"): + tau[f"tau_{k}_{suffix}"] = rng.normal(size=nth) * 1e-6 + tau[f"vartau_{k}_{suffix}"] = rng.uniform(1e-14, 1e-13, nth) + return rho, tau, theta + + +def test_rho_tau_to_sacc_diagonal(tmp_path): + rho, tau, theta = _rho_tau_tables() + s = sw.rho_tau_to_sacc({0: _nz()}, META, rho, tau) + s2 = _roundtrip(s, tmp_path, "rt") + for k in sw.RHO_K: + th, p, m = sio.get_rho(s2, k) + assert np.array_equal(th, theta) + assert np.array_equal(p, rho[f"rho_{k}_p"]) + assert np.array_equal(m, rho[f"rho_{k}_m"]) + for k in sw.TAU_K: + th, p, m = sio.get_tau(s2, (0, 0), k) + assert np.array_equal(p, tau[f"tau_{k}_p"]) + assert type(s2.covariance).__name__ == "DiagonalCovariance" + + +def test_rho_tau_to_sacc_tau_theory_block(tmp_path): + rho, tau, theta = _rho_tau_tables() + n_tau = 2 * len(sw.TAU_K) * len(theta) + tau_cov = _spd(n_tau, 11) + s = sw.rho_tau_to_sacc({0: _nz()}, META, rho, tau, tau_cov=tau_cov) + assert type(s.covariance).__name__ == "FullCovariance" + # τ sub-block equals the supplied theory covariance. + tr = ("source_0", sio.PSF_TRACER) + tau_idx = np.concatenate( + [ + np.concatenate( + [ + s.indices(sio.TAU_PLUS.format(k=k), tr), + s.indices(sio.TAU_MINUS.format(k=k), tr), + ] + ) + for k in sw.TAU_K + ] + ) + s2 = _roundtrip(s, tmp_path, "rttau") + assert np.allclose(s2.covariance.dense[np.ix_(tau_idx, tau_idx)], tau_cov) + + +def test_rho_tau_to_sacc_tau_cov_shape_mismatch(): + rho, tau, _ = _rho_tau_tables() + with pytest.raises(ValueError, match="tau_cov shape"): + sw.rho_tau_to_sacc({0: _nz()}, META, rho, tau, tau_cov=_spd(3, 1)) + + +# --------------------------------------------------------------------------- # +# Analysis-file assembly +# --------------------------------------------------------------------------- # +def _make_parts(nz): + theta = _theta() + ell = np.array([30.0, 60.0, 90.0]) + + class _Wsp: + def get_bandpower_windows(self): + w = np.zeros((4, 3, 4, 20)) + for out in range(4): + for b in range(3): + w[out, b, out, b * 6 : b * 6 + 6] = 1.0 + return w + + xi = sw.xi_to_sacc( + nz, META, theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + xi.add_covariance(_spd(len(xi.mean), 1)) + cl_all = np.vstack( + [np.arange(3) * 1e-9, np.arange(3) * 2e-9, np.zeros(3), np.arange(3) * 3e-9] + ) + cl = sw.pseudo_cl_to_sacc(nz, META, ell, cl_all, _Wsp(), covariance=_spd(9, 2)) + co = sw.cosebis_to_sacc( + nz, + META, + { + "En": np.arange(1, 6) * 1e-6, + "Bn": np.arange(1, 6) * 1e-7, + "cov": _spd(10, 3), + }, + (1.0, 100.0), + ) + return [xi, cl, co] + + +def test_assemble_analysis_sacc_full_covariance(tmp_path): + nz = {0: _nz()} + parts = _make_parts(nz) + s = sw.assemble_analysis_sacc(nz, META, parts) + assert type(s.covariance).__name__ == "FullCovariance" + assert s.covariance.dense.shape == (len(s.mean), len(s.mean)) + # every point covered; blocks placed and cross-blocks zero + tr = ("source_0", "source_0") + xi_idx = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + cl_idx = np.concatenate( + [s.indices(sio.CL_EE, tr), s.indices(sio.CL_BB, tr), s.indices(sio.CL_EB, tr)] + ) + co_idx = np.concatenate( + [s.indices(sio.COSEBI_EE, tr), s.indices(sio.COSEBI_BB, tr)] + ) + assert len(xi_idx) + len(cl_idx) + len(co_idx) == len(s.mean) + dense = s.covariance.dense + assert np.array_equal(dense[np.ix_(xi_idx, xi_idx)], parts[0].covariance.dense) + assert np.array_equal(dense[np.ix_(cl_idx, cl_idx)], parts[1].covariance.dense) + assert np.array_equal(dense[np.ix_(co_idx, co_idx)], parts[2].covariance.dense) + assert np.array_equal( + dense[np.ix_(xi_idx, cl_idx)], np.zeros((len(xi_idx), len(cl_idx))) + ) + # round-trips + s2 = _roundtrip(s, tmp_path, "analysis") + assert type(s2.covariance).__name__ == "FullCovariance" + assert np.allclose(s2.covariance.dense, s.covariance.dense) + + +def test_assemble_analysis_sacc_requires_covariance(): + nz = {0: _nz()} + parts = _make_parts(nz) + parts.append( + sw.xi_to_sacc( + nz, META, _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + ) # no covariance + with pytest.raises(ValueError, match="own covariance block"): + sw.assemble_analysis_sacc(nz, META, parts) + + +def test_assemble_from_reloaded_parts(tmp_path): + """Parts written to disk then reloaded assemble identically (the DAG path).""" + nz = {0: _nz()} + parts = _make_parts(nz) + reloaded = [] + for i, part in enumerate(parts): + sio.save(part, str(tmp_path / f"part{i}.sacc")) + reloaded.append(sio.load(str(tmp_path / f"part{i}.sacc"))) + s = sw.assemble_analysis_sacc(nz, META, reloaded) + assert type(s.covariance).__name__ == "FullCovariance" + assert s.covariance.dense.shape == (len(s.mean), len(s.mean)) From 18ec7ff94cfdb751301d00eb5980c3936ee2324b Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 03:25:37 +0200 Subject: [PATCH 06/47] =?UTF-8?q?feat(twopoint):=20SACC=20=E2=86=92=202pt-?= =?UTF-8?q?FITS=20converter,=20byte-compatible=20with=20cosmosis=5Ffitting?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add sp_validation.twopoint_convert.sacc_to_twopoint_fits, converting an analysis SACC into the CosmoSIS 2pt-FITS that 2pt_like (and Sacha's rho/tau 2pt_like_xi_sys fork) reads. Output reproduces today's cosmo_inference cosmosis_fitting.py assembly HDU-for-HDU and byte-for-byte: NZDATA, XI_PLUS/MINUS, CELL_EE, the blocked COVMAT (STRT_i offsets) and separate COVMAT_CELL, and the TAU_{0,2}_PLUS + verbatim RHO_STATS tables. The data vector and covariance are permuted from SACC's pair-major order to the 2pt-FITS type-major layout via s.indices. The tau covariance is laid in as one contiguous [tau_0+; tau_2+] block, preserving the tau_0<->tau_2 cross-correlation that covdat_to_fits carries. Rho/tau HDUs need the rho/tau sidecar FITS: the RHO_STATS varrho_* variance columns are not stored in the analysis SACC, so the converter copies them verbatim (never fabricates them); xi, Cl, n(z) and the covariance are fully reconstructed from SACC alone. Tests: - test_twopoint_convert.py: byte-equal vs current cosmosis_fitting.py builders on deterministic synthetic inputs for all three product shapes (plain xi; xi+Cl; xi+rho/tau), plus the tau_0<->tau_2 cross-correlation and perturbation teeth. - test_twopoint_convert_realdata.py (skipif on candide paths): build a SACC from a real product's own contents, convert, byte-compare vs the current writer. Observed byte-equal for SP_v1.4.6_leak_corr and a glass_mock sibling; the stale on-disk files differ only by an extra CELL_BB HDU (older script version) and ~1e-17 float noise on bin edges, documented and guarded. Co-Authored-By: Claude Fable 5 --- .../tests/test_twopoint_convert.py | 378 ++++++++++++++++++ .../tests/test_twopoint_convert_realdata.py | 268 +++++++++++++ src/sp_validation/twopoint_convert.py | 330 +++++++++++++++ 3 files changed, 976 insertions(+) create mode 100644 src/sp_validation/tests/test_twopoint_convert.py create mode 100644 src/sp_validation/tests/test_twopoint_convert_realdata.py create mode 100644 src/sp_validation/twopoint_convert.py diff --git a/src/sp_validation/tests/test_twopoint_convert.py b/src/sp_validation/tests/test_twopoint_convert.py new file mode 100644 index 00000000..5e599717 --- /dev/null +++ b/src/sp_validation/tests/test_twopoint_convert.py @@ -0,0 +1,378 @@ +"""Byte-compare tests for the SACC -> 2pt-FITS converter. + +The converter (:mod:`sp_validation.twopoint_convert`) must reproduce the CosmoSIS +2pt-FITS that ``cosmo_inference/scripts/cosmosis_fitting.py`` assembles today, +so the inference chain (``2pt_like`` and Sacha Guerrini's rho/tau +``2pt_like_xi_sys`` fork) runs untouched behind it. The strongest possible check +is *byte* equality, and astropy writes FITS deterministically, so that is what we +assert: build a reference with the current script's own HDU-builder functions on +deterministic synthetic inputs, build a SACC from those same inputs via +:mod:`sp_validation.sacc_io`, convert it, and compare the two files byte for byte. + +Three configurations pin the three product shapes today's ``__main__`` emits: + +1. **plain xi** -- PRIMARY, NZ_SOURCE, COVMAT, XI_PLUS, XI_MINUS. +2. **xi + pseudo-Cl** -- adds COVMAT_CELL and CELL_EE (the harmonic block + ``2pt_like`` reads; the script builds CELL_BB and discards it, so the + converter does too). +3. **xi + rho/tau** -- adds the blocked tau covariance (TAU_0_PLUS / TAU_2_PLUS, + with the tau_0<->tau_2 cross-correlation the truncated CosmoCov tau + covariance carries) and the verbatim RHO_STATS table. + +The rho/tau product needs the rho/tau *sidecar* HDUs: the RHO_STATS table carries +per-mode ``varrho_*`` variances the analysis SACC does not store, so the +converter copies them from the sidecar exactly as today's assembly does. A teeth +test pins that a perturbed input moves the output. + +The reference builder imports ``cosmosis_fitting.py`` by path (it is a script, +not a package module), skipping cleanly if a dependency is missing -- the same +loader pattern as ``test_cosmosis_fitting.py``. +""" + +import importlib.util +from pathlib import Path + +import numpy as np +import pytest +from astropy.io import fits + +from sp_validation import sacc_io, twopoint_convert + +_SCRIPT = ( + Path(__file__).resolve().parents[3] + / "cosmo_inference" + / "scripts" + / "cosmosis_fitting.py" +) + + +def _load_cf(): + """Import cosmosis_fitting.py by path; skip cleanly if a dep is missing.""" + if not _SCRIPT.exists(): + pytest.skip(f"cosmosis_fitting.py not found at {_SCRIPT}") + spec = importlib.util.spec_from_file_location("cosmosis_fitting", _SCRIPT) + module = importlib.util.module_from_spec(spec) + try: + spec.loader.exec_module(module) + except ImportError as exc: # pragma: no cover - container has numpy/astropy + pytest.importorskip(getattr(exc, "name", "") or "cosmosis_fitting_dependency") + raise + return module + + +cf = _load_cf() + + +# --- deterministic synthetic inputs ----------------------------------------- +# +# One tomographic bin (today's analysis). N_ANG angular bins on an ascending +# theta grid; N_ELL bandpowers; a 200-point n(z) on a uniform z grid (the DES +# NZDATA table needs a uniform Z_MID axis). The xi covariance is a full +# (2*N_ANG) matrix (xi+/xi- cross-block nonzero, as CosmoCov produces); the tau +# covariance is a full (3*N_ANG) matrix that the assembly truncates to its first +# 2 statistics (tau_0, tau_2). + +N_ANG = 5 +N_ELL = 8 +SOURCE = sacc_io.source_name(0) +PSF = sacc_io.PSF_TRACER + + +def _spd(n, seed): + """A symmetric positive-definite (n, n) matrix, seeded and recognizable.""" + rng = np.random.default_rng(seed) + a = rng.normal(size=(n, n)) + return a @ a.T + n * np.eye(n) + + +def _inputs(seed=0): + """Deterministic synthetic statistics + covariances for one bin pair.""" + rng = np.random.default_rng(seed) + theta = np.sort(rng.uniform(1.0, 250.0, N_ANG)) + ell = np.sort(rng.uniform(30.0, 3000.0, N_ELL)) + z = np.linspace(0.0125, 0.4875, 200) + return { + "theta": theta, + "ell": ell, + "z": z, + "nz": np.exp(-((z - 0.25) ** 2) / 0.02), + "xip": rng.uniform(1e-6, 1e-4, N_ANG), + "xim": rng.uniform(1e-6, 1e-4, N_ANG), + "cl_ee": rng.uniform(1e-10, 1e-8, N_ELL), + "cl_bb": rng.uniform(1e-11, 1e-9, N_ELL), + "cl_eb": rng.uniform(-1e-11, 1e-11, N_ELL), + "tau0p": rng.uniform(1e-6, 1e-5, N_ANG), + "tau2p": rng.uniform(1e-6, 1e-5, N_ANG), + "tau0m": rng.uniform(1e-6, 1e-5, N_ANG), + "tau2m": rng.uniform(1e-6, 1e-5, N_ANG), + "xi_cov": _spd(2 * N_ANG, seed + 1), + "cl_cov": _spd(N_ELL, seed + 2), + "tau_cov_full": _spd(3 * N_ANG, seed + 3), + } + + +# --- reference sidecar files + HDUs (the current script's own builders) ------ + + +def _rho_sidecar_hdu(theta, seed=7): + """A rho-stats BinTableHDU with the 25-column layout (values + variances).""" + rng = np.random.default_rng(seed) + columns = [fits.Column(name="theta", format="D", array=theta)] + for k in range(6): + for suffix in ("_p", "_m"): + columns.append( + fits.Column( + name=f"rho_{k}{suffix}", + format="D", + array=rng.uniform(-1e-3, 1e-3, len(theta)), + ) + ) + columns.append( + fits.Column( + name=f"varrho_{k}{suffix}", + format="D", + array=rng.uniform(1e-15, 1e-12, len(theta)), + ) + ) + return fits.BinTableHDU.from_columns(fits.ColDefs(columns)) + + +def _tau_sidecar_hdu(theta, tau0p, tau2p): + """A tau-stats BinTableHDU with theta + tau_0_p + tau_2_p columns.""" + columns = [ + fits.Column(name="theta", format="D", array=theta), + fits.Column(name="tau_0_p", format="D", array=tau0p), + fits.Column(name="tau_2_p", format="D", array=tau2p), + ] + return fits.BinTableHDU.from_columns(fits.ColDefs(columns)) + + +def _reference_fits(tmp_path, inp, *, cl=False, rho_tau=False): + """Build the reference 2pt-FITS with cosmosis_fitting.py's own functions. + + Reproduces the exact ``__main__`` HDU list for the requested configuration, + which is the assembly the converter must match byte for byte. + """ + nz_txt = tmp_path / "nz.txt" + np.savetxt(nz_txt, np.column_stack([inp["z"], inp["nz"]])) + cov_txt = tmp_path / "cov_xi.txt" + np.savetxt(cov_txt, inp["xi_cov"]) + + nz_hdu = cf.nz_to_fits(str(nz_txt)) + xip_hdu = cf._create_2pt_hdu(inp["xip"], inp["theta"], "XI_PLUS", "G+R", "G+R") + xim_hdu = cf._create_2pt_hdu(inp["xim"], inp["theta"], "XI_MINUS", "G-R", "G-R") + + hdu_list = [fits.PrimaryHDU(), nz_hdu] + + if rho_tau: + tau_cov_npy = tmp_path / "cov_tau.npy" + np.save(tau_cov_npy, inp["tau_cov_full"]) + cov_hdu = cf.covdat_to_fits(str(cov_txt), filename_cov_tau=str(tau_cov_npy)) + else: + cov_hdu = cf.covdat_to_fits(str(cov_txt), filename_cov_tau=None) + hdu_list.append(cov_hdu) + + if cl: + cl_block = np.zeros((5, N_ELL)) + cl_block[0], cl_block[1], cl_block[4] = inp["ell"], inp["cl_ee"], inp["cl_bb"] + cl_npy = tmp_path / "cl.npy" + np.save(cl_npy, cl_block) + cl_cov_npy = tmp_path / "cl_cov.npy" + np.save(cl_cov_npy, inp["cl_cov"]) + ell_r, cl_ee_r, cl_bb_r = cf.load_pseudo_cl(str(cl_npy)) + cl_ee_hdu, _cl_bb_hdu = cf.cl_to_fits(ell_r, cl_ee_r, cl_bb_r) + cov_cl_hdu = cf.cov_cl_to_fits(str(cl_cov_npy), cov_hdu="COVAR_FULL") + hdu_list.append(cov_cl_hdu) + + hdu_list.extend([xip_hdu, xim_hdu]) + if cl: + hdu_list.append(cl_ee_hdu) + + if rho_tau: + rho_path = tmp_path / "rho.fits" + fits.HDUList([fits.PrimaryHDU(), _rho_sidecar_hdu(inp["theta"])]).writeto( + rho_path, overwrite=True + ) + tau_path = tmp_path / "tau.fits" + fits.HDUList( + [ + fits.PrimaryHDU(), + _tau_sidecar_hdu(inp["theta"], inp["tau0p"], inp["tau2p"]), + ] + ).writeto(tau_path, overwrite=True) + rho_hdu = cf.rho_to_fits(str(rho_path), theta=inp["theta"]) + tau0_hdu, tau2_hdu = cf.tau_to_fits(str(tau_path), theta=inp["theta"]) + hdu_list.extend([tau0_hdu, tau2_hdu, rho_hdu]) + + out = tmp_path / "reference.fits" + fits.HDUList(hdu_list).writeto(out, overwrite=True) + return out + + +# --- the SACC each configuration is built from ------------------------------ + + +def _sacc(inp, *, cl=False, rho_tau=False): + """Build the analysis SACC the converter reads, matching ``_inputs``. + + The covariance is laid out to match the reference exactly: the xi block is + the full (2*N_ANG) matrix; the Cl block is the EE bandpower covariance; the + tau blocks carry the tau_0<->tau_2 cross-correlation from the truncated + CosmoCov tau covariance. Blocks not consumed by the 2pt-FITS (Cl BB/EB, tau + minus) get an identity block so ``add_covariance`` sees a full matrix. + """ + s = sacc_io.new_sacc({0: (inp["z"], inp["nz"])}) + sacc_io.add_xi(s, (0, 0), inp["theta"], inp["xip"], inp["xim"], grid="coarse") + if cl: + sacc_io.add_pseudo_cl( + s, + (0, 0), + inp["ell"], + inp["cl_ee"], + inp["cl_bb"], + inp["cl_eb"], + window_ells=np.arange(2, 102), + window_weights=np.random.default_rng(9).uniform(0, 1, (100, N_ELL)), + ) + if rho_tau: + sacc_io.add_tau(s, (0, 0), 0, inp["theta"], inp["tau0p"], inp["tau0m"]) + sacc_io.add_tau(s, (0, 0), 2, inp["theta"], inp["tau2p"], inp["tau2m"]) + + n = len(s.mean) + full = np.zeros((n, n)) + ip = s.indices(sacc_io.XI_PLUS, (SOURCE, SOURCE)) + im = s.indices(sacc_io.XI_MINUS, (SOURCE, SOURCE)) + xi_all = np.concatenate([ip, im]) + full[np.ix_(xi_all, xi_all)] = inp["xi_cov"] + + if cl: + iee = s.indices(sacc_io.CL_EE, (SOURCE, SOURCE)) + full[np.ix_(iee, iee)] = inp["cl_cov"] + for dtype in (sacc_io.CL_BB, sacc_io.CL_EB): + idx = s.indices(dtype, (SOURCE, SOURCE)) + full[np.ix_(idx, idx)] = np.eye(N_ELL) + if rho_tau: + t0p = s.indices(sacc_io.TAU_PLUS.format(k=0), (SOURCE, PSF)) + t2p = s.indices(sacc_io.TAU_PLUS.format(k=2), (SOURCE, PSF)) + tau_pp = np.concatenate([t0p, t2p]) + # The joint [tau_0+; tau_2+] block is the truncated CosmoCov tau + # covariance -- cross-correlation kept, matching covdat_to_fits. + full[np.ix_(tau_pp, tau_pp)] = inp["tau_cov_full"][: 2 * N_ANG, : 2 * N_ANG] + for dtype in (sacc_io.TAU_MINUS.format(k=0), sacc_io.TAU_MINUS.format(k=2)): + idx = s.indices(dtype, (SOURCE, PSF)) + full[np.ix_(idx, idx)] = np.eye(N_ANG) + s.add_covariance(full) + return s + + +def _sidecar_hdus(tmp_path, inp): + """Return the (rho_hdu, tau_hdu) sidecar input HDUs for the rho/tau product.""" + rho_path = tmp_path / "rho_in.fits" + fits.HDUList([fits.PrimaryHDU(), _rho_sidecar_hdu(inp["theta"])]).writeto( + rho_path, overwrite=True + ) + tau_path = tmp_path / "tau_in.fits" + fits.HDUList( + [fits.PrimaryHDU(), _tau_sidecar_hdu(inp["theta"], inp["tau0p"], inp["tau2p"])] + ).writeto(tau_path, overwrite=True) + with fits.open(rho_path) as r, fits.open(tau_path) as t: + return r[1].copy(), t[1].copy() + + +# ============================================================================= +# Byte-compare: the three product shapes +# ============================================================================= + + +def test_plain_xi_byte_equal(tmp_path): + """Plain-xi product: converter matches cosmosis_fitting.py byte for byte.""" + inp = _inputs(seed=0) + reference = _reference_fits(tmp_path, inp) + s = _sacc(inp) + out = tmp_path / "converted.fits" + twopoint_convert.sacc_to_twopoint_fits(s, str(out), n_bins=1) + assert out.read_bytes() == reference.read_bytes() + + +def test_xi_cl_byte_equal(tmp_path): + """xi + pseudo-Cl product: COVMAT_CELL + CELL_EE reproduced byte for byte.""" + inp = _inputs(seed=10) + reference = _reference_fits(tmp_path, inp, cl=True) + s = _sacc(inp, cl=True) + out = tmp_path / "converted.fits" + twopoint_convert.sacc_to_twopoint_fits(s, str(out), n_bins=1) + assert out.read_bytes() == reference.read_bytes() + + +def test_xi_rho_tau_byte_equal(tmp_path): + """xi + rho/tau product: blocked tau covariance + verbatim RHO_STATS match.""" + inp = _inputs(seed=20) + reference = _reference_fits(tmp_path, inp, rho_tau=True) + s = _sacc(inp, rho_tau=True) + rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) + out = tmp_path / "converted.fits" + twopoint_convert.sacc_to_twopoint_fits( + s, str(out), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 + ) + assert out.read_bytes() == reference.read_bytes() + + +# ============================================================================= +# Structural + teeth checks +# ============================================================================= + + +def test_tau_covariance_keeps_tau0_tau2_cross(tmp_path): + """The tau covariance block couples tau_0 and tau_2 (not block-diagonal). + + covdat_to_fits truncates the 3-statistic CosmoCov tau covariance to its + first 2 blocks and lays it in as ONE contiguous block, so tau_0<->tau_2 + cross-terms survive. A block-diagonal shortcut would zero them; pin that + the converter keeps them. + """ + inp = _inputs(seed=20) + s = _sacc(inp, rho_tau=True) + rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) + out = tmp_path / "converted.fits" + twopoint_convert.sacc_to_twopoint_fits( + s, str(out), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 + ) + with fits.open(out) as hdul: + cov = hdul["COVMAT"].data + # tau_0 block rows 2*N_ANG..3*N_ANG, tau_2 block 3*N_ANG..4*N_ANG. + cross = cov[2 * N_ANG : 3 * N_ANG, 3 * N_ANG : 4 * N_ANG] + expected = inp["tau_cov_full"][:N_ANG, N_ANG : 2 * N_ANG] + assert np.array_equal(cross, expected) + assert np.any(cross != 0.0) + + +def test_perturbed_xi_changes_output(tmp_path): + """Teeth: a changed xi+ input must change the converted data vector.""" + inp = _inputs(seed=0) + s = _sacc(inp) + out = tmp_path / "base.fits" + twopoint_convert.sacc_to_twopoint_fits(s, str(out), n_bins=1) + with fits.open(out) as hdul: + base_xip = hdul["XI_PLUS"].data["VALUE"].copy() + + inp2 = _inputs(seed=0) + inp2["xip"] = inp2["xip"] + 1.0 + s2 = _sacc(inp2) + out2 = tmp_path / "perturbed.fits" + twopoint_convert.sacc_to_twopoint_fits(s2, str(out2), n_bins=1) + with fits.open(out2) as hdul: + new_xip = hdul["XI_PLUS"].data["VALUE"] + + assert not np.array_equal(base_xip, new_xip) + assert np.array_equal(new_xip, inp2["xip"]) + + +def test_rho_tau_sidecars_required_together(tmp_path): + """Supplying only one of the rho/tau sidecars is a loud error.""" + inp = _inputs(seed=20) + s = _sacc(inp, rho_tau=True) + rho_hdu, _tau_hdu = _sidecar_hdus(tmp_path, inp) + with pytest.raises(ValueError, match="together"): + twopoint_convert.sacc_to_twopoint_fits( + s, str(tmp_path / "x.fits"), rho_stats_hdu=rho_hdu, n_bins=1 + ) diff --git a/src/sp_validation/tests/test_twopoint_convert_realdata.py b/src/sp_validation/tests/test_twopoint_convert_realdata.py new file mode 100644 index 00000000..9d2739a6 --- /dev/null +++ b/src/sp_validation/tests/test_twopoint_convert_realdata.py @@ -0,0 +1,268 @@ +"""Candide-local byte-compare of the converter against real 2pt-FITS products. + +Skipped unless a real product exists on disk (candide only; never committed). +It closes the loop end to end on real data: take a real CosmoSIS 2pt-FITS, build +an analysis SACC from its own contents, convert that SACC back to a 2pt-FITS, and +byte-compare. + +The reference is *not* the on-disk file directly. The committed on-disk products +were written by an older ``cosmosis_fitting.py`` (they carry a CELL_BB HDU and +order COVMAT before NZ_SOURCE); the converter reproduces the *current* script, +which the PR-4 migration will use to regenerate them. So the meaningful contract +-- "the converter equals the current writer" -- is tested by running the current +``cosmosis_fitting.py`` builders on the same real contents and byte-comparing the +converter against that. For transparency the test also records the direct diff +against the stale on-disk file, and asserts only that it differs *by whole HDUs* +(the extra CELL_BB), not in any shared block -- i.e. the drift is purely the +known HDU-set change, with no silent data corruption. + +Observed on 2026-07-10 for ``SP_v1.4.6_leak_corr`` and a ``glass_mock`` sibling: +converter == current-script byte for byte; converter vs on-disk differs only by +the CELL_BB HDU. +""" + +import importlib.util +from pathlib import Path + +import numpy as np +import pytest +from astropy.io import fits + +from sp_validation import sacc_io, twopoint_convert + +_DATA = Path("/automnt/n17data/cdaley/unions/code/sp_validation/cosmo_inference/data") +_REAL_FILES = { + "SP_v1.4.6_leak_corr": _DATA + / "SP_v1.4.6_leak_corr_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1" + / "cosmosis_SP_v1.4.6_leak_corr_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits", + "glass_mock_00001": _DATA / "glass_mock_00001" / "cosmosis_glass_mock_00001.fits", +} + +_SCRIPT = ( + Path(__file__).resolve().parents[3] + / "cosmo_inference" + / "scripts" + / "cosmosis_fitting.py" +) + + +def _load_cf(): + spec = importlib.util.spec_from_file_location("cosmosis_fitting", _SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _sacc_from_2pt_fits(hdul): + """Build an analysis SACC from a real CosmoSIS 2pt-FITS's own contents. + + Reads the NZ, ξ±, pseudo-Cℓ (EE/BB) and blocked covariance back out of the + product and lays them into the standard SACC layout — the inverse direction + the converter then undoes. τ± minus and Cℓ EB are not in the 2pt-FITS, so + they are stored as zeros with identity covariance sub-blocks (the converter + consumes only the ``+``/EE parts). + """ + z = hdul["NZ_SOURCE"].data["Z_MID"].astype(float) + nz = hdul["NZ_SOURCE"].data["BIN1"].astype(float) + theta = hdul["XI_PLUS"].data["ANG"].astype(float) + xip = hdul["XI_PLUS"].data["VALUE"].astype(float) + xim = hdul["XI_MINUS"].data["VALUE"].astype(float) + n = len(theta) + + ell = hdul["CELL_EE"].data["ANG"].astype(float) + cl_ee = hdul["CELL_EE"].data["VALUE"].astype(float) + cl_bb = hdul["CELL_BB"].data["VALUE"].astype(float) + n_ell = len(ell) + + covmat = hdul["COVMAT"].data.astype(float) + covmat_cell = hdul["COVMAT_CELL"].data.astype(float) + xi_cov = covmat[: 2 * n, : 2 * n] + tau_joint = covmat[2 * n : 4 * n, 2 * n : 4 * n] + + tau0p = hdul["TAU_0_PLUS"].data["VALUE"].astype(float) + tau2p = hdul["TAU_2_PLUS"].data["VALUE"].astype(float) + + s = sacc_io.new_sacc({0: (z, nz)}) + sacc_io.add_xi(s, (0, 0), theta, xip, xim, grid="coarse") + sacc_io.add_pseudo_cl( + s, + (0, 0), + ell, + cl_ee, + cl_bb, + np.zeros(n_ell), + window_ells=np.arange(2, 102), + window_weights=np.ones((100, n_ell)), + ) + sacc_io.add_tau(s, (0, 0), 0, theta, tau0p, np.zeros(n)) + sacc_io.add_tau(s, (0, 0), 2, theta, tau2p, np.zeros(n)) + + source, psf = sacc_io.source_name(0), sacc_io.PSF_TRACER + idx = { + "xi": np.concatenate( + [ + s.indices(sacc_io.XI_PLUS, (source, source)), + s.indices(sacc_io.XI_MINUS, (source, source)), + ] + ), + "ee": s.indices(sacc_io.CL_EE, (source, source)), + "bb": s.indices(sacc_io.CL_BB, (source, source)), + "eb": s.indices(sacc_io.CL_EB, (source, source)), + "t0p": s.indices(sacc_io.TAU_PLUS.format(k=0), (source, psf)), + "t0m": s.indices(sacc_io.TAU_MINUS.format(k=0), (source, psf)), + "t2p": s.indices(sacc_io.TAU_PLUS.format(k=2), (source, psf)), + "t2m": s.indices(sacc_io.TAU_MINUS.format(k=2), (source, psf)), + } + full = np.zeros((len(s.mean), len(s.mean))) + full[np.ix_(idx["xi"], idx["xi"])] = xi_cov + full[np.ix_(idx["ee"], idx["ee"])] = covmat_cell + tau_pp = np.concatenate([idx["t0p"], idx["t2p"]]) + full[np.ix_(tau_pp, tau_pp)] = tau_joint + for key in ("bb", "eb"): + full[np.ix_(idx[key], idx[key])] = np.eye(n_ell) + for key in ("t0m", "t2m"): + full[np.ix_(idx[key], idx[key])] = np.eye(n) + s.add_covariance(full) + + tau_sidecar = fits.BinTableHDU.from_columns( + fits.ColDefs( + [ + fits.Column(name="theta", format="D", array=theta), + fits.Column(name="tau_0_p", format="D", array=tau0p), + fits.Column(name="tau_2_p", format="D", array=tau2p), + ] + ) + ) + return s, hdul["RHO_STATS"].copy(), tau_sidecar + + +def _current_script_reference(cf, hdul, tmp_path): + """Reference: run the current cosmosis_fitting.py on the real file's contents.""" + z = hdul["NZ_SOURCE"].data["Z_MID"].astype(float) + nz = hdul["NZ_SOURCE"].data["BIN1"].astype(float) + theta = hdul["XI_PLUS"].data["ANG"].astype(float) + xip = hdul["XI_PLUS"].data["VALUE"].astype(float) + xim = hdul["XI_MINUS"].data["VALUE"].astype(float) + n = len(theta) + ell = hdul["CELL_EE"].data["ANG"].astype(float) + cl_ee = hdul["CELL_EE"].data["VALUE"].astype(float) + cl_bb = hdul["CELL_BB"].data["VALUE"].astype(float) + covmat = hdul["COVMAT"].data.astype(float) + covmat_cell = hdul["COVMAT_CELL"].data.astype(float) + tau0p = hdul["TAU_0_PLUS"].data["VALUE"].astype(float) + tau2p = hdul["TAU_2_PLUS"].data["VALUE"].astype(float) + + np.savetxt(tmp_path / "nz.txt", np.column_stack([z, nz])) + np.savetxt(tmp_path / "cov.txt", covmat[: 2 * n, : 2 * n]) + tau_cov = np.zeros((3 * n, 3 * n)) + tau_cov[: 2 * n, : 2 * n] = covmat[2 * n : 4 * n, 2 * n : 4 * n] + np.save(tmp_path / "cov_tau.npy", tau_cov) + cl_block = np.zeros((5, len(ell))) + cl_block[0], cl_block[1], cl_block[4] = ell, cl_ee, cl_bb + np.save(tmp_path / "cl.npy", cl_block) + np.save(tmp_path / "cl_cov.npy", covmat_cell) + + nz_hdu = cf.nz_to_fits(str(tmp_path / "nz.txt")) + xip_hdu = cf._create_2pt_hdu(xip, theta, "XI_PLUS", "G+R", "G+R") + xim_hdu = cf._create_2pt_hdu(xim, theta, "XI_MINUS", "G-R", "G-R") + cov_hdu = cf.covdat_to_fits( + str(tmp_path / "cov.txt"), filename_cov_tau=str(tmp_path / "cov_tau.npy") + ) + ell_r, cl_ee_r, cl_bb_r = cf.load_pseudo_cl(str(tmp_path / "cl.npy")) + cl_ee_hdu, _ = cf.cl_to_fits(ell_r, cl_ee_r, cl_bb_r) + cov_cl_hdu = cf.cov_cl_to_fits(str(tmp_path / "cl_cov.npy"), cov_hdu="COVAR_FULL") + + fits.HDUList([fits.PrimaryHDU(), hdul["RHO_STATS"].copy()]).writeto( + tmp_path / "rho.fits", overwrite=True + ) + rho_hdu = cf.rho_to_fits(str(tmp_path / "rho.fits"), theta=theta) + tau_sidecar = fits.BinTableHDU.from_columns( + fits.ColDefs( + [ + fits.Column(name="theta", format="D", array=theta), + fits.Column(name="tau_0_p", format="D", array=tau0p), + fits.Column(name="tau_2_p", format="D", array=tau2p), + ] + ) + ) + fits.HDUList([fits.PrimaryHDU(), tau_sidecar]).writeto( + tmp_path / "tau.fits", overwrite=True + ) + tau0_hdu, tau2_hdu = cf.tau_to_fits(str(tmp_path / "tau.fits"), theta=theta) + + out = tmp_path / "reference.fits" + fits.HDUList( + [ + fits.PrimaryHDU(), + nz_hdu, + cov_hdu, + cov_cl_hdu, + xip_hdu, + xim_hdu, + cl_ee_hdu, + tau0_hdu, + tau2_hdu, + rho_hdu, + ] + ).writeto(out, overwrite=True) + return out + + +@pytest.mark.parametrize("label", list(_REAL_FILES)) +def test_realdata_roundtrip_byte_equal(label, tmp_path): + """Converter reproduces the current writer byte for byte on a real product.""" + real = _REAL_FILES[label] + if not real.exists(): + pytest.skip(f"real 2pt-FITS not on disk: {real}") + cf = _load_cf() + + with fits.open(real) as hdul: + s, rho_hdu, tau_hdu = _sacc_from_2pt_fits(hdul) + reference = _current_script_reference(cf, hdul, tmp_path) + + converted = tmp_path / "converted.fits" + twopoint_convert.sacc_to_twopoint_fits( + s, str(converted), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 + ) + + # The contract: converter == current cosmosis_fitting.py, byte for byte. + assert converted.read_bytes() == reference.read_bytes() + + +@pytest.mark.parametrize("label", list(_REAL_FILES)) +def test_realdata_ondisk_drift_is_only_cell_bb(label, tmp_path): + """The stale on-disk file differs from the converter *only* by the CELL_BB HDU. + + Documents (and guards) the known script-version drift: the on-disk products + were written before CELL_BB was dropped from the assembly, so they carry one + extra HDU. Every HDU the two share carries the same data to floating-point + precision — the drift is a whole-HDU addition, never a silent change to a + shared block. (Bin-edge columns differ by ~1e-17 float noise: the stale file + stored a clean ``Z_LOW=0``, while ``z_mid - step/2`` rounds to ``-1.7e-18``; + both are the same number, so the shared-data check is ``allclose``, not + bitwise — bitwise equality is asserted against the *current* writer above.) + """ + real = _REAL_FILES[label] + if not real.exists(): + pytest.skip(f"real 2pt-FITS not on disk: {real}") + + with fits.open(real) as hdul: + s, rho_hdu, tau_hdu = _sacc_from_2pt_fits(hdul) + ondisk_names = [h.name for h in hdul] + converted = tmp_path / "converted.fits" + twopoint_convert.sacc_to_twopoint_fits( + s, str(converted), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 + ) + with fits.open(converted) as conv: + conv_names = [h.name for h in conv] + # The only HDU the on-disk file has that the converter does not. + assert set(ondisk_names) - set(conv_names) == {"CELL_BB"} + # Every shared table HDU carries the same data to float precision. + for name in conv_names: + if name == "PRIMARY": + continue + a, b = hdul[name].data, conv[name].data + if hasattr(a, "names"): + assert all(np.allclose(a[c], b[c]) for c in a.names), name + else: + assert np.allclose(a, b), name diff --git a/src/sp_validation/twopoint_convert.py b/src/sp_validation/twopoint_convert.py new file mode 100644 index 00000000..c06e0eb3 --- /dev/null +++ b/src/sp_validation/twopoint_convert.py @@ -0,0 +1,330 @@ +"""TWOPOINT_CONVERT. + +:Name: twopoint_convert.py + +:Description: Convert an analysis SACC file into the "2pt FITS" that CosmoSIS's + ``2pt_like`` (and Sacha Guerrini's ρ/τ ``2pt_like_xi_sys`` fork) + reads. The output reproduces today's hand-assembled product from + ``cosmo_inference/scripts/cosmosis_fitting.py`` HDU-for-HDU and + byte-for-byte: an NZDATA table, XI_PLUS / XI_MINUS 2pt tables, + optional CELL_EE / CELL_BB pseudo-Cℓ tables, the blocked COVMAT + (with ``STRT_i`` block-offset headers) and separate COVMAT_CELL, + and — when the SACC carries them — the TAU_{0,2}_PLUS 2pt tables + and the RHO_STATS table. + + The converter is the *inverse* of the SACC writers in + :mod:`sp_validation.sacc_io`: it reads statistics back through + those readers and lays them into the DES ``twopoint`` FITS + convention. SACC's canonical order is pair-major (per pair + ``[ξ+; ξ−]``); the 2pt-FITS layout is type-major (all ξ+, then all + ξ−), so the data-vector and its covariance are permuted here via + ``s.indices`` rather than assuming any global order. + + Scope note (single-bin today, tomography-ready): the assembly this + mirrors is single-tomographic-bin — BIN1/BIN2 are all 1, one NZ + ``BIN1`` column. The converter reads bin ``(0, 0)`` accordingly. + A tomographic 2pt-FITS layout (multiple bin pairs, per-pair + BIN1/BIN2, one NZ column per bin) is a later extension; it is not + what today's CosmoSIS pipeline consumes, so it is out of scope for + the byte-compatible converter. + + Rho/tau caveat: the SACC layout stores ρ±/τ± *values* only, while + the 2pt-FITS RHO_STATS table also carries the per-mode *variances* + (``varrho_*``) that Sacha's fork's covariance path reads. Those + variances are not recoverable from the analysis SACC. The + converter therefore writes RHO_STATS / TAU HDUs only when a + ``rho_stats``/``tau_stats`` sidecar FITS is supplied (the same + file today's assembly copies verbatim); it never fabricates + variances. ξ±, Cℓ, n(z) and the covariance — the data vector + CosmoSIS fits — are fully reconstructed from SACC alone. +""" + +import numpy as np +from astropy.io import fits + +from . import sacc_io + +# The QUANT1/QUANT2 header pair CosmoSIS stamps on each 2pt table, keyed by the +# extension name — copied from cosmosis_fitting.py so the headers match card +# for card. +_QUANT = { + "XI_PLUS": ("G+R", "G+R"), + "XI_MINUS": ("G-R", "G-R"), + "CELL_EE": ("GEF", "GEF"), + "CELL_BB": ("GBF", "GBF"), + "TAU_0_PLUS": ("G+R", "P+R"), + "TAU_2_PLUS": ("G+R", "SR+R"), +} + + +def _twopoint_hdu(name, values, ang, *, ang_unit=None): + """Build one 2pt BinTableHDU (BIN1/BIN2/ANGBIN/VALUE/ANG). + + Reproduces ``cosmosis_fitting.py._create_2pt_hdu`` /``cl_to_fits`` exactly: + same column order and formats, the ``2PTDATA`` marker, the QUANT pair for + ``name``, and NZ_SOURCE kernels. ``ang_unit`` stamps ``TUNIT`` on the ANG + column ("arcmin" for real-space ξ/τ; unset for Cℓ, whose ANG is ℓ). + """ + nbins = len(values) + angbin = np.arange(1, nbins + 1) + columns = [ + fits.Column(name="BIN1", format="K", array=np.ones(nbins)), + fits.Column(name="BIN2", format="K", array=np.ones(nbins)), + fits.Column(name="ANGBIN", format="K", array=angbin), + fits.Column(name="VALUE", format="D", array=values), + fits.Column(name="ANG", format="D", unit=ang_unit, array=ang), + ] + hdu = fits.BinTableHDU.from_columns(fits.ColDefs(columns), name=name) + quant1, quant2 = _QUANT[name] + for key, value in { + "2PTDATA": "T", + "QUANT1": quant1, + "QUANT2": quant2, + "KERNEL_1": "NZ_SOURCE", + "KERNEL_2": "NZ_SOURCE", + "WINDOWS": "SAMPLE", + }.items(): + hdu.header[key] = value + return hdu + + +def _nz_hdu(s, n_bins): + """Build the NZDATA HDU from the SACC ``source_i`` NZ tracers. + + Reproduces ``cosmosis_fitting.py.nz_to_fits``: Z_MID from the tracer ``z`` + grid (assumed uniform), Z_LOW/Z_HIGH as ± half a step, one ``BIN{i+1}`` + column per source bin, and the NZDATA/NBIN/NZ header cards. All source bins + are required to share the ``z`` grid — the single ``Z_MID`` axis of the + DES NZDATA table. + """ + z_mid, nz0 = sacc_io.get_nz(s, 0) + z_mid = np.asarray(z_mid, dtype=float) + step = z_mid[1] - z_mid[0] + z_low = z_mid - step / 2 + z_high = z_mid + step / 2 + + columns = [ + fits.Column(name="Z_LOW", format="D", array=z_low), + fits.Column(name="Z_MID", format="D", array=z_mid), + fits.Column(name="Z_HIGH", format="D", array=z_high), + ] + for i in range(n_bins): + z_i, nz_i = sacc_io.get_nz(s, i) + if not np.array_equal(np.asarray(z_i, dtype=float), z_mid): + raise ValueError( + f"source bin {i} n(z) grid differs from source bin 0; the DES " + "NZDATA table requires one shared Z_MID axis" + ) + columns.append(fits.Column(name=f"BIN{i + 1}", format="D", array=nz_i)) + + hdu = fits.BinTableHDU.from_columns(fits.ColDefs(columns), name="NZDATA") + for key, value in { + "NZDATA": "T ", + "EXTNAME": "NZ_SOURCE", + "NBIN": n_bins, + "NZ": len(z_low), + }.items(): + hdu.header[key] = value + return hdu + + +def _cov_hdu(matrix, block_names, block_starts, extname="COVMAT", name_in_ctor=False): + """Build a covariance ImageHDU with ``NAME_i``/``STRT_i`` block headers. + + Reproduces the two covariance builders in ``cosmosis_fitting.py`` card for + card. The blocked ξ/τ ``covdat_to_fits`` builds ``ImageHDU(cov)`` unnamed + and stamps ``COVDATA`` then ``EXTNAME`` from a dict; the ``cov_cl_to_fits`` + CELL covariance builds ``ImageHDU(cov, name="COVMAT_CELL")`` (so the EXTNAME + card is created early, with astropy's standard comment) before re-stamping. + ``name_in_ctor`` selects the second form so the card order matches exactly. + """ + matrix = np.asarray(matrix, dtype=np.float64) + if matrix.shape[0] != matrix.shape[1]: + raise ValueError(f"covariance must be square; got shape {matrix.shape}") + hdu = fits.ImageHDU(matrix, name=extname) if name_in_ctor else fits.ImageHDU(matrix) + hdu.header["COVDATA"] = "True" + hdu.header["EXTNAME"] = extname + for i, (name, start) in enumerate(zip(block_names, block_starts)): + hdu.header[f"NAME_{i}"] = name + hdu.header[f"STRT_{i}"] = int(start) + return hdu + + +def _type_major_xi(s, bins): + """Return ``(theta, xip, xim)`` for one bin pair from the SACC coarse grid. + + ``sacc_io.get_xi`` already returns each statistic in insertion (= ascending + θ) order; the type-major split (all ξ+, then all ξ−) is exactly the two + arrays it hands back, so no further permutation is needed for a single pair. + """ + return sacc_io.get_xi(s, bins, grid="coarse") + + +def sacc_to_twopoint_fits( + s, + path, + *, + rho_stats_hdu=None, + tau_stats_hdu=None, + n_bins=1, +): + """Convert an analysis SACC to a CosmoSIS 2pt-FITS file. + + The assembled ``HDUList`` matches today's ``cosmosis_fitting.py`` product + for the configuration the SACC describes: PRIMARY, NZ_SOURCE, COVMAT, then + (if present) COVMAT_CELL, XI_PLUS, XI_MINUS, (if present) CELL_EE / CELL_BB, + and (if the rho/tau sidecars are supplied) TAU_0_PLUS, TAU_2_PLUS, + RHO_STATS. The data vector and its covariance are laid out type-major + (all ξ+, then all ξ−, then the τ blocks), which is the DES ``twopoint`` + convention CosmoSIS reads. + + Parameters + ---------- + s : sacc.Sacc + Analysis SACC (coarse ξ±, optional pseudo-Cℓ, covariance, and — for the + ρ/τ product — the τ data points; see ``rho_stats_hdu``). + path : str + Output FITS path (overwritten). + rho_stats_hdu, tau_stats_hdu : astropy.io.fits.BinTableHDU, optional + The rho-stats / tau-stats sidecar HDUs, copied verbatim as today's + assembly does. Required together to write the ρ/τ product; the SACC + alone cannot rebuild the ``varrho_*`` columns Sacha's fork reads. When + omitted, a pure ξ (± Cℓ) product is written. + n_bins : int, optional + Number of source tomographic bins (default 1, the current single-bin + analysis). Sets the NZDATA column count. + + Returns + ------- + astropy.io.fits.HDUList + The assembled list, also written to ``path``. + """ + if (rho_stats_hdu is None) != (tau_stats_hdu is None): + raise ValueError( + "rho_stats_hdu and tau_stats_hdu must be supplied together " + "(the ρ/τ product needs both, or neither for a pure-ξ product)" + ) + use_rho_tau = rho_stats_hdu is not None + bins = (0, 0) + + nz_hdu = _nz_hdu(s, n_bins) + theta, xip, xim = _type_major_xi(s, bins) + xip_hdu = _twopoint_hdu("XI_PLUS", xip, theta, ang_unit="arcmin") + xim_hdu = _twopoint_hdu("XI_MINUS", xim, theta, ang_unit="arcmin") + + cell_hdu, cov_cell_hdu = _build_cell(s, bins) + + cov_hdu = _build_covmat(s, bins, use_rho_tau=use_rho_tau) + + tau_hdus, rho_hdu = _build_rho_tau(rho_stats_hdu, tau_stats_hdu, theta, use_rho_tau) + + # HDU order mirrors cosmosis_fitting.py's __main__: PRIMARY, NZ, COVMAT, + # COVMAT_CELL, XI±, CELL_EE, then the τ/ρ tables. + hdu_list = [fits.PrimaryHDU(), nz_hdu, cov_hdu] + if cov_cell_hdu is not None: + hdu_list.append(cov_cell_hdu) + hdu_list.extend([xip_hdu, xim_hdu]) + if cell_hdu is not None: + hdu_list.append(cell_hdu) + if use_rho_tau: + hdu_list.extend([*tau_hdus, rho_hdu]) + + hdul = fits.HDUList(hdu_list) + hdul.writeto(path, overwrite=True) + return hdul + + +def _build_cell(s, bins): + """Build the CELL_EE 2pt HDU plus the COVMAT_CELL HDU from the SACC pseudo-Cℓ. + + Returns ``(None, None)`` when the SACC has no pseudo-Cℓ. Only CELL_EE is + emitted — the harmonic ``2pt_like`` fits ``data_sets=CELL_EE``, and today's + assembly appends CELL_EE alone (it builds a CELL_BB HDU but discards it). + The SACC still carries EE/BB/EB with bandpower windows for the B-mode + null-test path; this converter surfaces only the block CosmoSIS reads. The + CELL covariance (the EE bandpower covariance) lives in its own COVMAT_CELL + ImageHDU, matching today's product. + """ + if sacc_io.CL_EE not in s.get_data_types(): + return None, None + + ell, cl_ee, _cl_bb, _cl_eb, _window = sacc_io.get_pseudo_cl(s, bins) + cell_hdu = _twopoint_hdu("CELL_EE", cl_ee, ell) + cell_idx = s.indices(sacc_io.CL_EE, sacc_io._pair(bins)) + cov_cell = s.covariance.dense[np.ix_(cell_idx, cell_idx)] + cov_cell_hdu = _cov_hdu( + cov_cell, ["CELL_EE"], [0], extname="COVMAT_CELL", name_in_ctor=True + ) + return cell_hdu, cov_cell_hdu + + +def _build_covmat(s, bins, *, use_rho_tau): + """Assemble the blocked COVMAT (ξ± type-major, then the τ blocks). + + The ξ covariance is pulled from the SACC as the contiguous ξ+/ξ− block for + the pair and permuted from pair-major (SACC) to type-major (2pt-FITS). Under + ``use_rho_tau`` the τ_0/τ_2 covariance blocks are appended block-diagonally + with zero ξ↔τ cross-blocks, exactly as ``covdat_to_fits`` builds them. + """ + pair = sacc_io._pair(bins) + idx_p = s.indices(sacc_io.XI_PLUS, pair) + idx_m = s.indices(sacc_io.XI_MINUS, pair) + n_theta = len(idx_p) + xi_idx = np.concatenate([idx_p, idx_m]) # type-major permutation + xi_cov = s.covariance.dense[np.ix_(xi_idx, xi_idx)] + + names = ["XI_PLUS", "XI_MINUS"] + starts = [0, n_theta] + matrix = xi_cov + + if use_rho_tau: + # The τ covariance couples τ_0+ and τ_2+ (today's assembly truncates the + # 3-statistic CosmoCov τ covariance to its first 2 blocks and lays it in + # as ONE contiguous [τ_0+; τ_2+] block — cross-correlation kept). In the + # SACC those two selections are not adjacent (τ_0− sits between them), so + # gather both index sets and extract the joint sub-block, ξ↔τ zero. + tau_pair = (sacc_io.source_name(0), sacc_io.PSF_TRACER) + idx_tau0 = s.indices(sacc_io.TAU_PLUS.format(k=0), tau_pair) + idx_tau2 = s.indices(sacc_io.TAU_PLUS.format(k=2), tau_pair) + tau_idx = np.concatenate([idx_tau0, idx_tau2]) + tau_cov = s.covariance.dense[np.ix_(tau_idx, tau_idx)] + matrix = _block_diag(matrix, tau_cov) + names += ["TAU_0_PLUS", "TAU_2_PLUS"] + starts += [2 * n_theta, 2 * n_theta + len(idx_tau0)] + + return _cov_hdu(matrix, names, starts) + + +def _block_diag(*blocks): + """Stack square blocks block-diagonally with zero cross-blocks.""" + sizes = [b.shape[0] for b in blocks] + n = sum(sizes) + out = np.zeros((n, n)) + start = 0 + for b in blocks: + out[start : start + b.shape[0], start : start + b.shape[0]] = b + start += b.shape[0] + return out + + +def _build_rho_tau(rho_stats_hdu, tau_stats_hdu, theta, use_rho_tau): + """Build the TAU_{0,2}_PLUS 2pt HDUs and the verbatim RHO_STATS HDU. + + Mirrors ``tau_to_fits`` / ``rho_to_fits``: τ_0/τ_2 read their ``tau_k_p`` + columns onto the shared ξ θ grid (consistency step); RHO_STATS is copied + verbatim from the sidecar with its θ column forced onto the ξ grid. The + ``varrho_*`` columns ride along in the copy — they are why the sidecar is + required (the SACC cannot supply them). + """ + if not use_rho_tau: + return (), None + + tau = tau_stats_hdu.data + tau0_hdu = _twopoint_hdu("TAU_0_PLUS", tau["tau_0_p"], theta, ang_unit="arcmin") + tau2_hdu = _twopoint_hdu("TAU_2_PLUS", tau["tau_2_p"], theta, ang_unit="arcmin") + + rho_hdu = rho_stats_hdu.copy() + rho_hdu.name = "RHO_STATS" + rho_hdu.data = rho_hdu.data.copy() + rho_hdu.data["theta"] = theta + return (tau0_hdu, tau2_hdu), rho_hdu From f9b1d12199c414246ee8b67bf9b079bec389ff00 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 03:27:10 +0200 Subject: [PATCH 07/47] feat(one_covariance): SACC <-> OneCovariance file-format glue Add sp_validation.one_covariance_io, the two small pieces of glue between the SACC layout and OneCovariance (which knows nothing of SACC and only exchanges files): - write_nz / nz_table / nz_config_stanza: the source_i NZ tracers of an analysis SACC -> OneCovariance's combined whitespace n(z) file (column 0 = shared z grid, one n(z) column per tomographic bin, no bin edges) plus the [redshift] config stanza pointing at it. Config keys verified against upstream OneCovariance config.ini (zlens_directory/zlens_file/value_loc_in_lensbin); the UNIONS template (cosmo_val/pseudo_cl.py._modify_onecov_config) uses the z_directory alias, exposed via dir_key. Fails fast on missing bins or disagreeing z grids. - covariance_blocks: OneCovariance's flat covariance_list_*.dat output -> dense square block(s) paired with SACC selectors for sacc_io.assemble_covariance, reusing statistics.cov_from_one_covariance for the per-block reshape (col 10 gaussian / col 9 gauss+non-gaussian). Single-block and multi-block (tomography-ready) forms. Tests (test_one_covariance_io.py, 9 passing): reshape to a hand-built matrix + gaussian/gauss+ng column selection + perturbation teeth; blocks feed assemble_covariance and round-trip s.covariance.dense; n(z) write/read round-trip, UNIONS dir_key, header/no-header, and fail-fast on mismatched grids / missing bin / bad value_loc. Co-Authored-By: Claude Fable 5 --- src/sp_validation/one_covariance_io.py | 256 ++++++++++++++++ .../tests/test_one_covariance_io.py | 274 ++++++++++++++++++ 2 files changed, 530 insertions(+) create mode 100644 src/sp_validation/one_covariance_io.py create mode 100644 src/sp_validation/tests/test_one_covariance_io.py diff --git a/src/sp_validation/one_covariance_io.py b/src/sp_validation/one_covariance_io.py new file mode 100644 index 00000000..4c0f2ae4 --- /dev/null +++ b/src/sp_validation/one_covariance_io.py @@ -0,0 +1,256 @@ +"""ONE_COVARIANCE_IO. + +:Name: one_covariance_io.py + +:Description: File-format glue between the SACC data-product layout + (:mod:`sp_validation.sacc_io`) and OneCovariance + (https://github.com/rreischke/OneCovariance). Two directions: + + - **n(z) SACC -> OneCovariance input** (:func:`write_nz`): the + ``source_i`` NZ tracers of an analysis SACC are written as the + combined whitespace-delimited redshift file OneCovariance reads + (column 0 = z grid, then one ``n(z)`` column per tomographic + bin, no bin edges), and a matching ``[redshift]`` config stanza + is returned via :func:`nz_config_stanza`. + + - **OneCovariance output -> SACC covariance blocks** + (:func:`covariance_blocks`): the flat ``covariance_list_*.dat`` + table OneCovariance emits (one row per element pair) is reshaped + into dense square block(s) — reusing + :func:`sp_validation.statistics.cov_from_one_covariance` for the + per-block reshape — and paired with SACC selectors so a caller + can feed them straight to + :func:`sp_validation.sacc_io.assemble_covariance`. + + OneCovariance itself is *not* a dependency: this module only + touches its file formats, verified against the upstream + ``config.ini`` (``rreischke/OneCovariance`` @ main). + + n(z) file format (upstream ``config.ini`` comment, verbatim): + + ``redshift n_1(z) ... n_{N_source}(z)`` + + i.e. a plain whitespace-delimited text file, column 0 the shared + redshift grid and one column per tomographic bin — no ``z_low``/ + ``z_high`` edges (this is the OneCovariance convention, distinct + from the CosmoSIS NZDATA table which *does* carry edges). All + source bins must therefore share one z grid. + + ``[redshift]`` config keys (upstream canonical names): a single + combined file goes in ``zlens_directory`` + ``zlens_file``; + ``value_loc_in_lensbin`` (``mid``/``left``/``right``) says where + in each histogram bin the tabulated ``n(z)`` value sits — ``mid`` + for the bin-centred grids the SACC stores. NOTE: the UNIONS + OneCovariance template driven by + ``cosmo_val/pseudo_cl.py._modify_onecov_config`` writes the older + key names ``z_directory``/``zlens_file`` instead; pass + ``dir_key="z_directory"`` to match that template. +""" + +import os + +import numpy as np + +from . import sacc_io +from .statistics import cov_from_one_covariance + + +def nz_table(s, n_bins): + """Stack the SACC ``source_i`` NZ tracers into a OneCovariance n(z) table. + + Parameters + ---------- + s : sacc.Sacc + SACC holding ``source_0 … source_{n_bins-1}`` NZ tracers. + n_bins : int + Number of tomographic source bins to write. + + Returns + ------- + numpy.ndarray + Array of shape ``(n_z, n_bins + 1)``: column 0 the shared redshift + grid, columns ``1 … n_bins`` the per-bin ``n(z)``. This is the + OneCovariance combined-file layout (``redshift n_1(z) … n_N(z)``). + + Raises + ------ + ValueError + If any source bin is missing, or if the bins do not share one z grid + (OneCovariance's combined file has a single redshift column, so the + grids must agree bin-for-bin). + """ + z0, nz0 = sacc_io.get_nz(s, 0) + z0 = np.asarray(z0, dtype=float) + columns = [z0] + for i in range(n_bins): + if sacc_io.source_name(i) not in s.tracers: + raise ValueError( + f"SACC has no NZ tracer {sacc_io.source_name(i)!r}; cannot write " + f"a {n_bins}-bin OneCovariance n(z) file" + ) + z_i, nz_i = sacc_io.get_nz(s, i) + if not np.array_equal(np.asarray(z_i, dtype=float), z0): + raise ValueError( + f"source bin {i} n(z) grid differs from source bin 0; the " + "OneCovariance combined n(z) file has one shared redshift column" + ) + columns.append(np.asarray(nz_i, dtype=float)) + return np.column_stack(columns) + + +def write_nz(s, path, n_bins, *, dir_key="zlens_directory", header=True): + """Write the OneCovariance combined n(z) input file from a SACC. + + OneCovariance reads the source redshift distribution as a plain + whitespace-delimited text file whose column 0 is the shared redshift grid + and whose remaining columns are the per-bin ``n(z)`` (``redshift n_1(z) + … n_N(z)``) — no ``z_low``/``z_high`` edges. This writes that file from the + SACC ``source_i`` NZ tracers and returns the ``[redshift]`` config stanza + that points OneCovariance at it. + + Parameters + ---------- + s : sacc.Sacc + Analysis SACC with the ``source_i`` NZ tracers. + path : str or pathlib.Path + Output text-file path (overwritten). Its directory + basename become + the ``[redshift]`` directory/file config values. + n_bins : int + Number of tomographic source bins to write. + dir_key : str, optional + Config key for the redshift directory. Default ``"zlens_directory"`` + (upstream canonical). Pass ``"z_directory"`` for the UNIONS template + driven by ``pseudo_cl.py._modify_onecov_config``. + header : bool, optional + If ``True`` (default) prepend a ``# redshift n_1(z) …`` comment header + naming the columns; OneCovariance's ``genfromtxt``-style reader ignores + it. Set ``False`` for a bare numeric file. + + Returns + ------- + dict + The ``[redshift]`` config stanza (see :func:`nz_config_stanza`), naming + the file just written. + """ + table = nz_table(s, n_bins) + head = "" + if header: + cols = " ".join(f"n_{i + 1}(z)" for i in range(n_bins)) + head = f"redshift {cols}" + np.savetxt(str(path), table, header=head) + return nz_config_stanza( + os.path.dirname(os.path.abspath(str(path))), + os.path.basename(str(path)), + dir_key=dir_key, + ) + + +def nz_config_stanza( + directory, filename, *, dir_key="zlens_directory", value_loc="mid" +): + """Build the OneCovariance ``[redshift]`` config stanza for an n(z) file. + + Parameters + ---------- + directory : str + Directory holding the n(z) file (OneCovariance ``*_directory`` value). + filename : str + n(z) file basename (OneCovariance ``zlens_file`` value). + dir_key : str, optional + Directory config key — ``"zlens_directory"`` (upstream) or + ``"z_directory"`` (UNIONS template). Default ``"zlens_directory"``. + value_loc : str, optional + ``value_loc_in_lensbin`` — where in each histogram bin the tabulated + ``n(z)`` value sits (``mid``/``left``/``right``). Default ``"mid"``, + matching the bin-centred grids the SACC stores. + + Returns + ------- + dict + The ``[redshift]`` key/value pairs: ``{dir_key: directory, "zlens_file": + filename, "value_loc_in_lensbin": value_loc}``. Assign these under + ``config["redshift"]`` of a OneCovariance ``configparser`` config. + """ + if value_loc not in ("mid", "left", "right"): + raise ValueError( + f"value_loc_in_lensbin must be 'mid', 'left' or 'right'; got {value_loc!r}" + ) + return { + dir_key: directory, + "zlens_file": filename, + "value_loc_in_lensbin": value_loc, + } + + +def read_nz(path): + """Read a OneCovariance combined n(z) file back to ``(z, nz_columns)``. + + Inverse of :func:`write_nz` (the numeric round-trip; the config stanza is + not stored in the file). Comment/header lines are skipped. + + Parameters + ---------- + path : str or pathlib.Path + n(z) text file (column 0 = z, columns 1… = per-bin n(z)). + + Returns + ------- + tuple + ``(z, nz)`` where ``z`` is the shared redshift grid (shape ``(n_z,)``) + and ``nz`` is the per-bin distributions (shape ``(n_z, n_bins)``). + """ + table = np.atleast_2d(np.genfromtxt(str(path))) + return table[:, 0], table[:, 1:] + + +def covariance_blocks(cov_list, selectors, *, gaussian=True): + """Reshape a OneCovariance ``covariance_list`` table into SACC cov blocks. + + OneCovariance emits a flat ``covariance_list_*.dat`` table with one row per + ``(i, j)`` element pair (row-major, ``k = i·n + j``); the covariance value + lives in column 10 (Gaussian) or column 9 (Gaussian+non-Gaussian). This + reshapes the flat table into dense square block(s) — reusing + :func:`sp_validation.statistics.cov_from_one_covariance` for the per-block + reshape — and pairs each with its SACC selector, ready for + :func:`sp_validation.sacc_io.assemble_covariance`. + + Single-statistic case: pass the whole table and one selector; you get one + ``(selector, dense)`` block. Multi-statistic case (tomography-ready): pass a + sequence of ``(selector, sub_table)`` pairs — each ``sub_table`` a + contiguous slice of the flat output for one statistic / bin-pair — and each + is reshaped and re-paired with its selector in order. The API is thus shaped + to extend to multi-probe blocking without over-fitting the single-bin case. + + Parameters + ---------- + cov_list : numpy.ndarray or sequence + Either the flat OneCovariance table (2-D array, one row per pair) for a + single block, or — for the multi-block form — a sequence of + ``(selector, sub_table)`` pairs. In the multi-block form ``selectors`` + must be ``None`` (the selectors travel with the sub-tables). + selectors : selector or None + For the single-block form, the SACC selector for the whole table (a + ``(data_type, tracers[, tags])`` tuple or an index array, as + :func:`sacc_io.assemble_covariance` accepts). Must be ``None`` for the + multi-block form. + gaussian : bool, optional + Select the Gaussian-only column (``True``, default) or the + Gaussian+non-Gaussian column (``False``); passed straight through to + ``cov_from_one_covariance``. + + Returns + ------- + list + Ordered ``(selector, dense_cov)`` pairs, directly consumable by + ``sacc_io.assemble_covariance(s, blocks)``. + """ + if selectors is None: + # Multi-block form: cov_list is a sequence of (selector, sub_table). + return [ + (selector, cov_from_one_covariance(np.asarray(sub), gaussian=gaussian)) + for selector, sub in cov_list + ] + # Single-block form: one flat table, one selector. + return [ + (selectors, cov_from_one_covariance(np.asarray(cov_list), gaussian=gaussian)) + ] diff --git a/src/sp_validation/tests/test_one_covariance_io.py b/src/sp_validation/tests/test_one_covariance_io.py new file mode 100644 index 00000000..a99ad552 --- /dev/null +++ b/src/sp_validation/tests/test_one_covariance_io.py @@ -0,0 +1,274 @@ +"""Tests for :mod:`sp_validation.one_covariance_io`. + +All synthetic, all fast: the OneCovariance fixtures are built in memory +shaped exactly like its real file I/O — a flat ``covariance_list`` table with +the ``(i, j)`` index rows and the Gaussian / Gauss+non-Gaussian value columns +at the indices ``cov_from_one_covariance`` expects (col 10 / col 9), and the +combined n(z) text file (column 0 = z, one column per bin, no edges). No +cluster paths; OneCovariance is not imported. + +The two pieces: + +- **Piece 1** (n(z) SACC -> OneCovariance input): write the combined n(z) + file from a SACC's ``source_i`` NZ tracers, read it back, assert the z and + n(z) columns round-trip and the config stanza names the file. +- **Piece 2** (OneCovariance output -> SACC covariance blocks): reshape the + flat table to the dense block(s) and prove they feed + ``sacc_io.assemble_covariance`` cleanly (reshaped block -> assemble -> + ``s.covariance.dense`` matches the hand-built matrix). +""" + +import numpy as np +import numpy.testing as npt +import pytest + +from sp_validation import one_covariance_io as ocio +from sp_validation import sacc_io as sio + + +# --------------------------------------------------------------------------- # +# Synthetic OneCovariance-shaped fixtures +# --------------------------------------------------------------------------- # +def _one_cov_table(cov_gauss, cov_all): + """Flatten two n x n matrices into a OneCovariance ``covariance_list`` table. + + Reproduces the real flat output: one row per ``(i, j)`` element pair in + row-major order ``k = i·n + j``, with the Gaussian value in column 10 and + the Gaussian+non-Gaussian value in column 9. Columns 0-8 and the index + columns are filled with self-documenting placeholder values (the reshape + only reads cols 9/10, but a realistic width proves it does not spill). + """ + n = cov_gauss.shape[0] + rows = [] + for i in range(n): + for j in range(n): + row = np.arange(11.0) # placeholder cols 0-8 (+ overwritten 9,10) + row[9] = cov_all[i, j] + row[10] = cov_gauss[i, j] + rows.append(row) + return np.array(rows) + + +def _spd(n, seed): + """Symmetric positive-definite matrix of size ``n`` (a valid covariance).""" + a = np.random.default_rng(seed).normal(size=(n, n)) + return a @ a.T + n * np.eye(n) + + +def _nz(seed, n=40): + rng = np.random.default_rng(seed) + z = np.linspace(0.01, 2.0, n) + return z, rng.uniform(0.1, 1.0, n) + + +# --------------------------------------------------------------------------- # +# Piece 2 — flat covariance_list -> dense block (reshape correctness + teeth) +# --------------------------------------------------------------------------- # +def test_covariance_blocks_reshapes_to_hand_built_matrix(): + """Pin the reshape and prove gaussian vs gauss+ng select different columns. + + WHAT IS PINNED: ``covariance_blocks`` flattens/reshapes the OneCovariance + ``covariance_list`` table into a dense square block matching a hand-built + covariance. It delegates the per-block reshape to + ``statistics.cov_from_one_covariance``, so column 10 (gaussian) and column 9 + (gauss+ng) must recover the two distinct hand-built matrices. + + WHY TEETH: (a) ``gaussian=True`` vs ``False`` must return the two *different* + matrices, proving the column flag is load-bearing; (b) perturbing a single + entry of the flat input must change exactly that entry of the reshaped + block, proving the reshape actually reads the table (not a constant). + """ + cov_gauss = _spd(4, seed=1) + cov_all = _spd(4, seed=2) + table = _one_cov_table(cov_gauss, cov_all) + + selector = (sio.XI_PLUS, (sio.source_name(0), sio.source_name(0))) + + [(sel_g, block_g)] = ocio.covariance_blocks(table, selector, gaussian=True) + [(sel_a, block_a)] = ocio.covariance_blocks(table, selector, gaussian=False) + + assert sel_g == selector and sel_a == selector + npt.assert_allclose(block_g, cov_gauss, rtol=1e-12) + npt.assert_allclose(block_a, cov_all, rtol=1e-12) + + # TEETH: gaussian and gauss+ng select different columns -> different blocks. + assert not np.allclose(block_g, block_a) + + # TEETH: a perturbation of one flat-table entry moves exactly that block + # entry (row k = i·n + j, col 10 for gaussian). + perturbed = table.copy() + perturbed[2 * 4 + 1, 10] += 5.0 # element (i=2, j=1) + [(_, block_p)] = ocio.covariance_blocks(perturbed, selector, gaussian=True) + npt.assert_allclose(block_p[2, 1] - block_g[2, 1], 5.0, rtol=1e-12) + block_p[2, 1] = block_g[2, 1] + npt.assert_allclose(block_p, block_g, rtol=1e-12) # nothing else moved + + +def test_covariance_blocks_multiblock_form(): + """Prove the tomography-ready multi-block form reshapes each sub-table. + + WHAT IS PINNED: passing ``selectors=None`` and a sequence of + ``(selector, sub_table)`` pairs reshapes each sub-table independently and + returns them paired with their selectors in order — the shape needed to map + a multi-statistic OneCovariance output onto several SACC selectors. + + WHY TEETH: the two sub-tables carry different matrices; if the function + reshaped only the first or mixed them, the second block would not match its + own hand-built matrix. + """ + cov_a, cov_b = _spd(3, seed=3), _spd(2, seed=4) + table_a = _one_cov_table(cov_a, _spd(3, seed=5)) + table_b = _one_cov_table(cov_b, _spd(2, seed=6)) + sel_a = (sio.XI_PLUS, (sio.source_name(0), sio.source_name(0))) + sel_b = (sio.XI_MINUS, (sio.source_name(0), sio.source_name(0))) + + blocks = ocio.covariance_blocks( + [(sel_a, table_a), (sel_b, table_b)], None, gaussian=True + ) + + assert [s for s, _ in blocks] == [sel_a, sel_b] + npt.assert_allclose(blocks[0][1], cov_a, rtol=1e-12) + npt.assert_allclose(blocks[1][1], cov_b, rtol=1e-12) + + +# --------------------------------------------------------------------------- # +# Piece 2 — the reshaped block feeds assemble_covariance cleanly +# --------------------------------------------------------------------------- # +def test_covariance_blocks_feed_assemble_covariance(): + """Round-trip: reshaped block -> assemble_covariance -> dense matches. + + WHAT IS PINNED: the ``(selector, dense)`` pair ``covariance_blocks`` + returns is directly consumable by ``sacc_io.assemble_covariance``: assembled + onto a SACC whose only statistic is one ξ+ block, ``s.covariance.dense`` + must equal the hand-built OneCovariance matrix. This is the end-to-end + contract between the two modules. + + WHY TEETH: the block must tile the data vector exactly; if the reshape + produced the wrong size or the wrong selector, ``assemble_covariance`` would + raise (its contiguity/tiling/size checks), so a clean assemble + matching + dense is a real proof. + """ + theta = np.geomspace(1.0, 100.0, 4) + xip, xim = np.arange(4) * 1e-5, np.arange(4) * 2e-5 + s = sio.new_sacc({0: _nz(0)}) + sio.add_xi(s, (0, 0), theta, xip, xim, grid="coarse") + + # The ξ block spans ξ+ then ξ− for the pair -> 8 points, one contiguous + # block (pair-major, matching sacc_io's canonical order). + cov_gauss = _spd(8, seed=7) + table = _one_cov_table(cov_gauss, _spd(8, seed=8)) + pair = (sio.source_name(0), sio.source_name(0)) + selector = np.concatenate( + [s.indices(sio.XI_PLUS, pair), s.indices(sio.XI_MINUS, pair)] + ) + + blocks = ocio.covariance_blocks(table, selector, gaussian=True) + sio.assemble_covariance(s, blocks) + + npt.assert_allclose(s.covariance.dense, cov_gauss, rtol=1e-12) + + +# --------------------------------------------------------------------------- # +# Piece 1 — n(z) SACC -> OneCovariance input (round-trip + config stanza) +# --------------------------------------------------------------------------- # +def test_write_nz_roundtrips_and_names_file(tmp_path): + """Round-trip the n(z) file and check the config stanza names it. + + WHAT IS PINNED: ``write_nz`` writes the SACC ``source_i`` NZ tracers as the + OneCovariance combined file (column 0 = z, one column per bin, no z_low/ + z_high edges). ``read_nz`` recovers the z grid and every per-bin n(z) + column, and the returned ``[redshift]`` stanza names the exact file written + (directory + basename) plus ``value_loc_in_lensbin``. + + WHY TEETH: the z grid and each n(z) column must round-trip to the values the + SACC holds (drawn from a seeded RNG); a transposed write or a dropped column + would fail the per-bin comparison. The stanza's directory/file must match + the path actually written. + """ + z, nz0 = _nz(10) + _, nz1 = _nz(11) + s = sio.new_sacc({0: (z, nz0), 1: (z, nz1)}) + + path = tmp_path / "nz_onecov.txt" + stanza = ocio.write_nz(s, path, n_bins=2) + + z_read, nz_read = ocio.read_nz(path) + npt.assert_allclose(z_read, z, rtol=1e-12) + assert nz_read.shape == (len(z), 2) + npt.assert_allclose(nz_read[:, 0], nz0, rtol=1e-12) + npt.assert_allclose(nz_read[:, 1], nz1, rtol=1e-12) + + assert stanza["zlens_directory"] == str(tmp_path) + assert stanza["zlens_file"] == "nz_onecov.txt" + assert stanza["value_loc_in_lensbin"] == "mid" + + +def test_write_nz_unions_template_dir_key(tmp_path): + """The UNIONS-template ``z_directory`` key is selectable via ``dir_key``. + + WHAT IS PINNED: the upstream OneCovariance key is ``zlens_directory``, but + the UNIONS template (pseudo_cl.py._modify_onecov_config) writes + ``z_directory``. ``dir_key="z_directory"`` produces that variant so the + stanza drops straight into the UNIONS template's ``[redshift]`` section. + """ + z, nz0 = _nz(12) + s = sio.new_sacc({0: (z, nz0)}) + stanza = ocio.write_nz(s, tmp_path / "nz.txt", n_bins=1, dir_key="z_directory") + assert "z_directory" in stanza and "zlens_directory" not in stanza + assert stanza["z_directory"] == str(tmp_path) + assert stanza["zlens_file"] == "nz.txt" + + +def test_write_nz_fails_on_mismatched_z_grids(tmp_path): + """Fail fast when source bins do not share one redshift grid. + + WHAT IS PINNED: the OneCovariance combined file has a single redshift + column, so all bins must share the z grid. A bin on a different grid must + raise ``ValueError`` at write time, not silently mis-align. + """ + z0, nz0 = _nz(20) + z1_shifted, nz1 = _nz(21) + z1_shifted = z1_shifted + 0.1 # different grid + s = sio.new_sacc({0: (z0, nz0), 1: (z1_shifted, nz1)}) + with pytest.raises(ValueError, match="differs from source bin 0"): + ocio.write_nz(s, tmp_path / "bad.txt", n_bins=2) + + +def test_write_nz_no_header_roundtrips(tmp_path): + """The bare (header=False) file still round-trips numerically. + + WHAT IS PINNED: ``header=False`` writes a purely numeric file (no ``#`` + column-name line); ``read_nz`` recovers the same z grid and n(z) column, so + the header is cosmetic and never load-bearing for the numeric round-trip. + """ + z, nz0 = _nz(40) + s = sio.new_sacc({0: (z, nz0)}) + path = tmp_path / "bare.txt" + ocio.write_nz(s, path, n_bins=1, header=False) + z_read, nz_read = ocio.read_nz(path) + npt.assert_allclose(z_read, z, rtol=1e-12) + npt.assert_allclose(nz_read[:, 0], nz0, rtol=1e-12) + + +def test_nz_config_stanza_rejects_bad_value_loc(): + """``value_loc_in_lensbin`` outside {mid,left,right} fails fast. + + WHAT IS PINNED: OneCovariance only accepts ``mid``/``left``/``right`` for + the histogram-bin value location; an invalid value is a config bug and must + raise ``ValueError`` rather than write a stanza OneCovariance will reject. + """ + with pytest.raises(ValueError, match="value_loc_in_lensbin"): + ocio.nz_config_stanza("/dir", "nz.txt", value_loc="center") + + +def test_write_nz_fails_on_missing_bin(tmp_path): + """Fail fast when a requested source bin is absent from the SACC. + + WHAT IS PINNED: requesting more bins than the SACC carries is a real config + bug; ``write_nz`` raises ``ValueError`` naming the missing tracer rather + than writing a short file. + """ + z, nz0 = _nz(30) + s = sio.new_sacc({0: (z, nz0)}) + with pytest.raises(ValueError, match="source_1"): + ocio.write_nz(s, tmp_path / "short.txt", n_bins=2) From c631418339ce603c40a9c391e730203b6ccc67b2 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 03:29:06 +0200 Subject: [PATCH 08/47] fix(cosmo_val): tau covariance block derived from CovTauTh write-side MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The theoretical tau covariance is not (6·nbin)² over [τ+; τ−]. Read from the write-side (shear_psf_leakage.rho_tau_cov.CovTauTh.build_cov): it is a (3·nbin)² k-major matrix over {τ0, τ2, τ5} with plus/minus folded into one component per index — exactly the flavor today's CosmoSIS chain consumes via covdat_to_fits. rho_tau_to_sacc now scatters that plus-only block into the τ-plus rows/columns of the 6·nbin τ block (per-k [+;−] insertion order), leaves τ-minus a vartau diagonal, and keeps plus↔minus cross zero. The kwarg is renamed tau_cov -> tau_cov_th to name the flavor. tau_cov_th=None keeps a compact DiagonalCovariance placeholder. Co-Authored-By: Claude Fable 5 --- src/sp_validation/cosmo_val/core.py | 18 +++++ src/sp_validation/cosmo_val/pseudo_cl.py | 33 +++++--- src/sp_validation/cosmo_val/real_space.py | 81 ++------------------ src/sp_validation/cosmo_val/sacc_writers.py | 60 ++++++++++----- src/sp_validation/tests/test_sacc_writers.py | 37 +++++---- 5 files changed, 108 insertions(+), 121 deletions(-) diff --git a/src/sp_validation/cosmo_val/core.py b/src/sp_validation/cosmo_val/core.py index 48251bbb..0c9273d8 100644 --- a/src/sp_validation/cosmo_val/core.py +++ b/src/sp_validation/cosmo_val/core.py @@ -15,6 +15,7 @@ find_conservative_scale_cut_key, ) from ..statistics import chi2_and_pte +from ..version import __version__ from .catalog_characterization import CatalogCharacterizationMixin from .cosebis import CosebisMixin from .pseudo_cl import PseudoClMixin @@ -380,6 +381,23 @@ def _output_path(self, *parts): """ return os.path.abspath(os.path.join(self.cc["paths"]["output"], *parts)) + def sacc_nz(self, version): + """Single-bin ``nz`` mapping ``{0: (z, nz)}`` for the SACC writers. + + The tomography-native writer interface (``sacc_writers``) takes an nz + dict keyed by 0-based source bin; the round is single-bin, so the whole + survey n(z) is bin 0. ``get_redshift`` returns ``(z, nz)``. + """ + return {0: tuple(self.get_redshift(version))} + + def sacc_metadata(self, version): + """Provenance metadata stored on every SACC part for ``version``.""" + return { + "catalogue_version": version, + "sp_validation_version": __version__, + "npatch": self.npatch, + } + def get_redshift(self, version): """Load redshift distribution for a catalog version. diff --git a/src/sp_validation/cosmo_val/pseudo_cl.py b/src/sp_validation/cosmo_val/pseudo_cl.py index 514049a6..0dee59f4 100644 --- a/src/sp_validation/cosmo_val/pseudo_cl.py +++ b/src/sp_validation/cosmo_val/pseudo_cl.py @@ -17,6 +17,7 @@ from astropy.io import fits from cs_util.cosmo import get_theo_c_ell +from .. import sacc_io from ..pseudo_cl import ( apply_random_rotation, get_n_gal_map, @@ -26,6 +27,10 @@ ) from ..rho_tau import get_params_rho_tau from ..statistics import chi2_and_pte, cov_from_one_covariance +from .sacc_writers import BIN as SACC_BIN + +# NaMaster spin-2 × spin-2 decoupled-spectrum row order (EE, EB, BE, BB). +_NMT_EE, _NMT_EB, _NMT_BB = 0, 1, 3 class PseudoClMixin: @@ -454,6 +459,12 @@ def calculate_pseudo_cl_g_ng_cov(self, gaussian_part="iNKA"): def calculate_pseudo_cl(self): """ Compute the pseudo-Cl of given catalogs. + + Each version's spectra are born as a SACC part (``pseudo_cl_{ver}.sacc``) + via :func:`sacc_writers.pseudo_cl_to_sacc` — EE/BB/EB carrying the shared + NaMaster bandpower window. The in-memory ``self._pseudo_cls[ver]`` + ``"pseudo_cl"`` entry keeps the ``ELL``/``EE``/``EB``/``BB`` arrays the + plotting and B-mode-summary consumers read by column name. """ self.print_start("Computing pseudo-Cl's") @@ -468,11 +479,10 @@ def calculate_pseudo_cl(self): self._pseudo_cls[ver] = {} - out_path = self._output_path(f"pseudo_cl_{ver}.fits") + out_path = self._output_path(f"pseudo_cl_{ver}.sacc") if os.path.exists(out_path): self.print_done(f"Skipping Pseudo-Cl's calculation, {out_path} exists") - cl_shear = fits.getdata(out_path) - self._pseudo_cls[ver]["pseudo_cl"] = cl_shear + self._pseudo_cls[ver]["pseudo_cl"] = self._load_pseudo_cl_sacc(out_path) elif self.cell_method == "map": self.calculate_pseudo_cl_map(ver, nside, out_path) elif self.cell_method == "catalog": @@ -482,6 +492,13 @@ def calculate_pseudo_cl(self): self.print_done("Done pseudo-Cl's") + @staticmethod + def _load_pseudo_cl_sacc(out_path): + """Read a pseudo-Cl SACC part into the ELL/EE/EB/BB dict consumers use.""" + s = sacc_io.load(out_path) + ell, ee, bb, eb, _window = sacc_io.get_pseudo_cl(s, SACC_BIN) + return {"ELL": ell, "EE": ee, "EB": eb, "BB": bb} + def calculate_pseudo_cl_map(self, ver, nside, out_path): params = get_params_rho_tau(self.cc[ver], survey=ver) @@ -547,10 +564,9 @@ def calculate_pseudo_cl_map(self, ver, nside, out_path): cl_shear = cl_shear - cl_noise self.print_cyan("Saving pseudo-Cl's...") - self.save_pseudo_cl(ell_eff, cl_shear, out_path) + self.pseudo_cl_to_sacc_part(ver, out_path, ell_eff, cl_shear, wsp) - cl_shear = fits.getdata(out_path) - self._pseudo_cls[ver]["pseudo_cl"] = cl_shear + self._pseudo_cls[ver]["pseudo_cl"] = self._load_pseudo_cl_sacc(out_path) def calculate_pseudo_cl_catalog(self, ver, out_path): params = get_params_rho_tau(self.cc[ver], survey=ver) @@ -563,10 +579,9 @@ def calculate_pseudo_cl_catalog(self, ver, out_path): ) self.print_cyan("Saving pseudo-Cl's...") - self.save_pseudo_cl(ell_eff, cl_shear, out_path) + self.pseudo_cl_to_sacc_part(ver, out_path, ell_eff, cl_shear, wsp) - cl_shear = fits.getdata(out_path) - self._pseudo_cls[ver]["pseudo_cl"] = cl_shear + self._pseudo_cls[ver]["pseudo_cl"] = self._load_pseudo_cl_sacc(out_path) def get_n_gal_map(self, params, nside, cat_gal): """Weighted galaxy number-density map (thin wrapper -> primitive).""" diff --git a/src/sp_validation/cosmo_val/real_space.py b/src/sp_validation/cosmo_val/real_space.py index 76d05d0a..a0852d77 100644 --- a/src/sp_validation/cosmo_val/real_space.py +++ b/src/sp_validation/cosmo_val/real_space.py @@ -12,12 +12,11 @@ import matplotlib.ticker as mticker import numpy as np import treecorr -from astropy.io import fits from cs_util import plots as cs_plots class RealSpaceMixin: - def calculate_2pcf(self, ver, npatch=None, save_fits=False, **treecorr_config): + def calculate_2pcf(self, ver, npatch=None, **treecorr_config): """ Calculate the two-point correlation function (2PCF) ξ± for a given catalog version with TreeCorr. @@ -34,9 +33,6 @@ def calculate_2pcf(self, ver, npatch=None, save_fits=False, **treecorr_config): npatch (int, optional): The number of patches to use for the calculation. Defaults to the instance's `npatch` attribute. - save_fits (bool, optional): Whether to save the ξ± results to FITS files. - Defaults to False. - **treecorr_config: Additional TreeCorr configuration parameters that will override the instance's default `treecorr_config`. For example, `min_sep=1`. @@ -49,8 +45,11 @@ def calculate_2pcf(self, ver, npatch=None, save_fits=False, **treecorr_config): calculation is skipped, and the results are loaded from the file. - If a patch file for the given configuration does not exist, it is created during the process. - - FITS files for ξ+ and ξ− are saved with additional metadata in their - headers if `save_fits` is True. + - The ``.txt`` TreeCorr dump is the only raw byproduct written here + (read back by the covariance machinery and the skip-if-exists). The + analysis ξ± data product is born as SACC in the Snakemake scripts + (``run_2pcf.py`` coarse / ``run_2pcf_highres.py`` fine), which call + ``xi_to_sacc``; there is no DES-style ξ FITS writer anymore. """ self.print_magenta(f"Computing {ver} ξ±") @@ -101,74 +100,6 @@ def calculate_2pcf(self, ver, npatch=None, save_fits=False, **treecorr_config): gg.process(cat_gal) gg.write(out_fname, write_patch_results=True, write_cov=True) - # Save xi_p and xi_m results to fits file - # (moved outside so it runs even if txt exists) - if save_fits: - lst = np.arange(1, treecorr_config["nbins"] + 1) - - col1 = fits.Column(name="BIN1", format="K", array=np.ones(len(lst))) - col2 = fits.Column(name="BIN2", format="K", array=np.ones(len(lst))) - col3 = fits.Column(name="ANGBIN", format="K", array=lst) - col4 = fits.Column(name="VALUE", format="D", array=gg.xip) - col5 = fits.Column(name="ANG", format="D", unit="arcmin", array=gg.meanr) - coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) - xiplus_hdu = fits.BinTableHDU.from_columns(coldefs, name="XI_PLUS") - - col4 = fits.Column(name="VALUE", format="D", array=gg.xim) - coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) - ximinus_hdu = fits.BinTableHDU.from_columns(coldefs, name="XI_MINUS") - - # append xi_plus header info - xiplus_dict = { - "2PTDATA": "T", - "QUANT1": "G+R", - "QUANT2": "G+R", - "KERNEL_1": "NZ_SOURCE", - "KERNEL_2": "NZ_SOURCE", - "WINDOWS": "SAMPLE", - } - for key in xiplus_dict: - xiplus_hdu.header[key] = xiplus_dict[key] - - col1 = fits.Column(name="BIN1", format="K", array=np.ones(len(lst))) - col2 = fits.Column(name="BIN2", format="K", array=np.ones(len(lst))) - col3 = fits.Column(name="ANGBIN", format="K", array=lst) - col4 = fits.Column(name="VALUE", format="D", array=gg.xip) - col5 = fits.Column(name="ANG", format="D", unit="arcmin", array=gg.rnom) - coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) - xiplus_hdu = fits.BinTableHDU.from_columns(coldefs, name="XI_PLUS") - - col4 = fits.Column(name="VALUE", format="D", array=gg.xim) - coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) - ximinus_hdu = fits.BinTableHDU.from_columns(coldefs, name="XI_MINUS") - - # append xi_plus header info - xiplus_dict = { - "2PTDATA": "T", - "QUANT1": "G+R", - "QUANT2": "G+R", - "KERNEL_1": "NZ_SOURCE", - "KERNEL_2": "NZ_SOURCE", - "WINDOWS": "SAMPLE", - } - for key in xiplus_dict: - xiplus_hdu.header[key] = xiplus_dict[key] - # Use same naming format as txt output - fits_base = out_fname.replace(".txt", "").replace("_xi_", "_") - xiplus_hdu.writeto( - f"{fits_base.replace(ver, f'xi_plus_{ver}')}.fits", - overwrite=True, - ) - - # append xi_minus header info - ximinus_dict = {**xiplus_dict, "QUANT1": "G-R", "QUANT2": "G-R"} - for key in ximinus_dict: - ximinus_hdu.header[key] = ximinus_dict[key] - ximinus_hdu.writeto( - f"{fits_base.replace(ver, f'xi_minus_{ver}')}.fits", - overwrite=True, - ) - # Add correlation object to class if not hasattr(self, "cat_ggs"): self.cat_ggs = {} diff --git a/src/sp_validation/cosmo_val/sacc_writers.py b/src/sp_validation/cosmo_val/sacc_writers.py index d227c379..76e9e6e0 100644 --- a/src/sp_validation/cosmo_val/sacc_writers.py +++ b/src/sp_validation/cosmo_val/sacc_writers.py @@ -124,15 +124,26 @@ def pure_eb_to_sacc(nz, metadata, theta, eb, covariance=None): return s -def rho_tau_to_sacc(nz, metadata, rho_stats, tau_stats, tau_cov=None): +def rho_tau_to_sacc(nz, metadata, rho_stats, tau_stats, tau_cov_th=None): """One ρ/τ part: ρ_0…ρ_5 autos and τ_0/τ_2/τ_5 leakage. ``rho_stats`` / ``tau_stats`` are the ``shear_psf_leakage`` handler tables (columns ``theta``, ``rho_{k}_p``, ``varrho_{k}_p``, ``rho_{k}_m``, … and - the τ analogue). ρ carries a diagonal (varxip/varxim) covariance — a - diagnostic placeholder, not used by inference — while τ carries ``tau_cov``, - the theoretical ``CovTauTh`` block the CosmoSIS τ-likelihood consumes. The - block order matches insertion: ρ (all +then−, per k) then τ. + the τ analogue). Both diagnostics stay out of the blind and only τ enters + inference, so the covariance is a block-diagonal placeholder except for the + τ-plus theory block: + + - ρ (all 6·nbin points): diagonal from ``varrho`` — a diagnostic placeholder, + not consumed by inference. + - τ (6·nbin points, per-k ``[τ+; τ−]``): the ``CovTauTh`` theory covariance + ``tau_cov_th`` scattered into the τ-plus rows/columns. ``CovTauTh.build_cov`` + returns a ``(3·nbin, 3·nbin)`` k-major matrix over ``{τ0, τ2, τ5}`` with the + plus/minus contributions folded into one component per k (verified against + the write-side); it therefore aligns to our τ-plus points ``{τ0+, τ2+, τ5+}`` + in k-major order, and today's CosmoSIS chain (``covdat_to_fits``) consumes + exactly this flavor for τ. The τ-minus points carry only a ``vartau`` + diagonal (no theory covariance for them exists). ``tau_cov_th=None`` falls + back to a fully diagonal τ block (a flagged placeholder, not the design). """ s = sio.new_sacc(nz, metadata) theta_rho = np.asarray(rho_stats["theta"]) @@ -154,6 +165,7 @@ def rho_tau_to_sacc(nz, metadata, rho_stats, tau_stats, tau_cov=None): np.asarray(tau_stats[f"tau_{k}_p"]), np.asarray(tau_stats[f"tau_{k}_m"]), ) + nbin = len(theta_tau) rho_var = np.concatenate( [ np.concatenate([rho_stats[f"varrho_{k}_p"], rho_stats[f"varrho_{k}_m"]]) @@ -166,22 +178,30 @@ def rho_tau_to_sacc(nz, metadata, rho_stats, tau_stats, tau_cov=None): for k in TAU_K ] ) - if tau_cov is None: - # Pure diagnostic file: diagonal covariance across ρ and τ. + if tau_cov_th is None: + # Fully diagonal placeholder — a DiagonalCovariance (compact, honest) for + # the standalone diagnostic file; assemble reads it back via .dense. s.add_covariance(np.concatenate([rho_var, tau_var])) - else: - # ρ diagonal (diagnostic) + τ dense theoretical block (inference input). - n_rho, n_tau = len(rho_var), len(tau_var) - tau_cov = np.asarray(tau_cov) - if tau_cov.shape != (n_tau, n_tau): - raise ValueError( - f"tau_cov shape {tau_cov.shape} does not match the {n_tau} τ " - "data points" - ) - full = np.zeros((n_rho + n_tau, n_rho + n_tau)) - full[:n_rho, :n_rho] = np.diag(rho_var) - full[n_rho:, n_rho:] = tau_cov - s.add_covariance(full) + return s + tau_cov_th = np.asarray(tau_cov_th) + n_plus = len(TAU_K) * nbin + if tau_cov_th.shape != (n_plus, n_plus): + raise ValueError( + f"tau_cov_th shape {tau_cov_th.shape} does not match the " + f"{n_plus} τ-plus points ({len(TAU_K)} indices × {nbin} bins) — " + "CovTauTh.build_cov returns one (plus-folded) component per τ index" + ) + n_rho, n_tau = len(rho_var), len(tau_var) + tau_block = np.diag(tau_var) + # τ-plus local positions in the τ block, k-major (per-k layout is [+; −]). + plus = np.concatenate( + [np.arange(2 * i * nbin, 2 * i * nbin + nbin) for i in range(len(TAU_K))] + ) + tau_block[np.ix_(plus, plus)] = tau_cov_th + full = np.zeros((n_rho + n_tau, n_rho + n_tau)) + full[:n_rho, :n_rho] = np.diag(rho_var) + full[n_rho:, n_rho:] = tau_block + s.add_covariance(full) return s diff --git a/src/sp_validation/tests/test_sacc_writers.py b/src/sp_validation/tests/test_sacc_writers.py index 057697e4..d886fa0a 100644 --- a/src/sp_validation/tests/test_sacc_writers.py +++ b/src/sp_validation/tests/test_sacc_writers.py @@ -192,32 +192,35 @@ def test_rho_tau_to_sacc_diagonal(tmp_path): def test_rho_tau_to_sacc_tau_theory_block(tmp_path): + """The (3·nbin) plus-only CovTauTh block scatters into the τ-plus rows/cols; + τ-minus keeps a vartau diagonal, and cross plus↔minus stays zero.""" rho, tau, theta = _rho_tau_tables() - n_tau = 2 * len(sw.TAU_K) * len(theta) - tau_cov = _spd(n_tau, 11) - s = sw.rho_tau_to_sacc({0: _nz()}, META, rho, tau, tau_cov=tau_cov) + nbin = len(theta) + n_plus = len(sw.TAU_K) * nbin # τ-plus points (k-major, one component per k) + tau_cov_th = _spd(n_plus, 11) + s = sw.rho_tau_to_sacc({0: _nz()}, META, rho, tau, tau_cov_th=tau_cov_th) assert type(s.covariance).__name__ == "FullCovariance" - # τ sub-block equals the supplied theory covariance. tr = ("source_0", sio.PSF_TRACER) - tau_idx = np.concatenate( - [ - np.concatenate( - [ - s.indices(sio.TAU_PLUS.format(k=k), tr), - s.indices(sio.TAU_MINUS.format(k=k), tr), - ] - ) - for k in sw.TAU_K - ] + tau_plus = np.concatenate( + [s.indices(sio.TAU_PLUS.format(k=k), tr) for k in sw.TAU_K] + ) + tau_minus = np.concatenate( + [s.indices(sio.TAU_MINUS.format(k=k), tr) for k in sw.TAU_K] ) s2 = _roundtrip(s, tmp_path, "rttau") - assert np.allclose(s2.covariance.dense[np.ix_(tau_idx, tau_idx)], tau_cov) + dense = s2.covariance.dense + # τ-plus sub-block equals the supplied theory covariance (scatter is correct). + assert np.allclose(dense[np.ix_(tau_plus, tau_plus)], tau_cov_th) + # τ-minus is diagonal from vartau; plus↔minus cross is zero. + tau_minus_var = np.concatenate([np.asarray(tau[f"vartau_{k}_m"]) for k in sw.TAU_K]) + assert np.allclose(np.diag(dense[np.ix_(tau_minus, tau_minus)]), tau_minus_var) + assert np.allclose(dense[np.ix_(tau_plus, tau_minus)], 0.0) def test_rho_tau_to_sacc_tau_cov_shape_mismatch(): rho, tau, _ = _rho_tau_tables() - with pytest.raises(ValueError, match="tau_cov shape"): - sw.rho_tau_to_sacc({0: _nz()}, META, rho, tau, tau_cov=_spd(3, 1)) + with pytest.raises(ValueError, match="tau_cov_th shape"): + sw.rho_tau_to_sacc({0: _nz()}, META, rho, tau, tau_cov_th=_spd(3, 1)) # --------------------------------------------------------------------------- # From 483961ffd3469b7cceecb27d48599e798cfb5afa Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 03:30:54 +0200 Subject: [PATCH 09/47] feat(cosmo_val): define pseudo_cl_to_sacc_part, complete pseudo-Cl SACC part MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit calculate_pseudo_cl_map/_catalog already call pseudo_cl_to_sacc_part (from the prior commit); this defines it — building the EE/BB/EB part via pseudo_cl_to_sacc(nz, meta, ell_eff, cl_all, wsp) and saving it as the native pseudo_cl_{ver}.sacc product. Drops the legacy save_pseudo_cl FITS writer (no external callers) and the unused _NMT_* constants. Co-Authored-By: Claude Fable 5 --- src/sp_validation/cosmo_val/pseudo_cl.py | 36 ++++++++++-------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/src/sp_validation/cosmo_val/pseudo_cl.py b/src/sp_validation/cosmo_val/pseudo_cl.py index 0dee59f4..52d82849 100644 --- a/src/sp_validation/cosmo_val/pseudo_cl.py +++ b/src/sp_validation/cosmo_val/pseudo_cl.py @@ -28,9 +28,7 @@ from ..rho_tau import get_params_rho_tau from ..statistics import chi2_and_pte, cov_from_one_covariance from .sacc_writers import BIN as SACC_BIN - -# NaMaster spin-2 × spin-2 decoupled-spectrum row order (EE, EB, BE, BB). -_NMT_EE, _NMT_EB, _NMT_BB = 0, 1, 3 +from .sacc_writers import pseudo_cl_to_sacc class PseudoClMixin: @@ -670,26 +668,22 @@ def apply_random_rotation(self, e1, e2, rng=None): """ return apply_random_rotation(e1, e2, rng) - def save_pseudo_cl(self, ell_eff, pseudo_cl, out_path): - """ - Save pseudo-Cl's to a FITS file. + def pseudo_cl_to_sacc_part(self, version, out_path, ell_eff, cl_all, wsp): + """Write the pseudo-Cl SACC part (EE/BB/EB + shared bandpower window). - Parameters - ---------- - pseudo_cl : np.array - Pseudo-Cl's to save. - out_path : str - Path to save the pseudo-Cl's to. + ``cl_all`` is NaMaster's decoupled ``(4, nbp)`` array (EE, EB, BE, BB); + the writer takes the shared bandpower window from ``wsp``. No covariance + is attached here — the analysis file's pseudo-Cl block is supplied at + assembly (``assemble_sacc``) from the NaMaster / OneCovariance product. """ - # Create columns of the fits file - col1 = fits.Column(name="ELL", format="D", array=ell_eff) - col2 = fits.Column(name="EE", format="D", array=pseudo_cl[0]) - col3 = fits.Column(name="EB", format="D", array=pseudo_cl[1]) - col4 = fits.Column(name="BB", format="D", array=pseudo_cl[3]) - coldefs = fits.ColDefs([col1, col2, col3, col4]) - cell_hdu = fits.BinTableHDU.from_columns(coldefs, name="PSEUDO_CELL") - - cell_hdu.writeto(out_path, overwrite=True) + s = pseudo_cl_to_sacc( + self.sacc_nz(version), + self.sacc_metadata(version), + ell_eff, + cl_all, + wsp, + ) + sacc_io.save(s, out_path) def plot_pseudo_cl(self): """ From 95e150fc23093c376c32b5e3c5836f275a027034 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 03:33:26 +0200 Subject: [PATCH 10/47] feat(cosmo_val): SACC part writers on cosebis / pure_eb / psf_systematics mixins MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thin *_to_sacc_part methods that turn each already-computed diagnostic into its single-statistic SACC part via the born-as-SACC writers: - cosebis.py: cosebis_to_sacc_part picks the fiducial scale cut's {En,Bn,cov} (find_conservative_scale_cut_key, else the widest cut — mirroring plot_cosebis) and writes it; the multi-cut .npz sidecar (the PTE scan) is untouched. - pure_eb.py: pure_eb_to_sacc_part writes the six PURE_KEYS blocks at gg.meanr with the results['cov'] block. - psf_systematics.py: calculate_rho_tau_stats now writes a rho_tau_{base}.sacc part per version from the handler tables, passing cov_tau_{base}_th.npy as tau_cov_th when present (the tau-plus CovTauTh inference block), else a loud diagonal-placeholder fallback. nz/metadata come from the shared core helpers sacc_nz / sacc_metadata. Co-Authored-By: Claude Fable 5 --- src/sp_validation/cosmo_val/cosebis.py | 42 +++++++++++++++++++ .../cosmo_val/psf_systematics.py | 36 ++++++++++++++++ src/sp_validation/cosmo_val/pure_eb.py | 21 ++++++++++ 3 files changed, 99 insertions(+) diff --git a/src/sp_validation/cosmo_val/cosebis.py b/src/sp_validation/cosmo_val/cosebis.py index aa4657ba..6b1d6146 100644 --- a/src/sp_validation/cosmo_val/cosebis.py +++ b/src/sp_validation/cosmo_val/cosebis.py @@ -7,6 +7,7 @@ import numpy as np +from .. import sacc_io from ..b_modes import ( calculate_cosebis, find_conservative_scale_cut_key, @@ -15,6 +16,7 @@ plot_cosebis_scale_cut_heatmap, save_cosebis_results, ) +from .sacc_writers import cosebis_to_sacc class CosebisMixin: @@ -140,6 +142,46 @@ def calculate_cosebis( return results + @staticmethod + def _fiducial_cosebis_result(results, fiducial_scale_cut): + """Select the fiducial scale cut's result dict + its ``(min, max)`` cut. + + ``calculate_cosebis`` returns either a single result dict (full range) or + a multi-cut mapping keyed by ``(theta_min, theta_max)`` tuples. Only the + fiducial cut is a SACC data product: pick it via + ``find_conservative_scale_cut_key`` when a fiducial cut is given, else the + widest cut — mirroring ``plot_cosebis``. + """ + multi_cut = isinstance(results, dict) and all( + isinstance(k, tuple) for k in results + ) + if not multi_cut: + return results, tuple(results["scale_cut"]) + key = ( + find_conservative_scale_cut_key(results, fiducial_scale_cut) + if fiducial_scale_cut is not None + else max(results, key=lambda x: x[1] - x[0]) + ) + return results[key], tuple(key) + + def cosebis_to_sacc_part(self, version, out_path, results, fiducial_scale_cut=None): + """Write the COSEBIs SACC part at the fiducial scale cut. + + ``results`` is the object ``calculate_cosebis`` returned (single dict or + multi-cut mapping). Only the fiducial cut's ``{En, Bn, cov}`` becomes the + part — a ``FullCovariance`` must cover every stored point and the cuts + overlap in mode space, so the non-fiducial cuts stay in the diagnostic + ``.npz`` sidecar. The nz/metadata are the version's. + """ + result, scale_cut = self._fiducial_cosebis_result(results, fiducial_scale_cut) + s = cosebis_to_sacc( + self.sacc_nz(version), + self.sacc_metadata(version), + result, + scale_cut, + ) + sacc_io.save(s, out_path) + def plot_cosebis( self, version=None, diff --git a/src/sp_validation/cosmo_val/psf_systematics.py b/src/sp_validation/cosmo_val/psf_systematics.py index 9f8a247e..af819977 100644 --- a/src/sp_validation/cosmo_val/psf_systematics.py +++ b/src/sp_validation/cosmo_val/psf_systematics.py @@ -16,10 +16,12 @@ from shear_psf_leakage.rho_tau_stat import PSFErrorFit from uncertainties import ufloat +from .. import sacc_io from ..rho_tau import ( get_rho_tau_w_cov, get_samples, ) +from .sacc_writers import rho_tau_to_sacc class PSFSystematicsMixin: @@ -41,11 +43,45 @@ def calculate_rho_tau_stats(self): cov_rho=self.compute_cov_rho, npatch=self.npatch, ) + self.rho_tau_to_sacc_part( + ver, out_dir, base, rho_stat_handler, tau_stat_handler + ) self.print_done("Rho stats finished") self._rho_stat_handler = rho_stat_handler self._tau_stat_handler = tau_stat_handler + def rho_tau_to_sacc_part( + self, version, out_dir, base, rho_stat_handler, tau_stat_handler + ): + """Write the ρ/τ SACC part for one version. + + ρ_0…ρ_5 autos and τ_0/τ_2/τ_5 leakage from the handler tables. The + ``CovTauTh`` theory covariance ``cov_tau_{base}_th.npy`` — a + ``(3·nbin, 3·nbin)`` plus-folded k-major block over ``{τ0, τ2, τ5}`` — is + passed as ``tau_cov_th`` when it exists (the τ-plus inference block); its + absence falls back to a diagonal placeholder for the whole part (loudly: + the τ inference block is then only a variance diagonal, not the theory + covariance). ρ always carries a diagnostic ``varrho`` diagonal. + """ + tau_cov_path = os.path.join(out_dir, f"cov_tau_{base}_th.npy") + tau_cov_th = np.load(tau_cov_path) if os.path.exists(tau_cov_path) else None + if tau_cov_th is None: + self.print_magenta( + f"No τ theory covariance at {tau_cov_path}; writing ρ/τ SACC part " + "with a diagonal placeholder covariance (τ inference block is a " + "variance diagonal, not CovTauTh)." + ) + s = rho_tau_to_sacc( + self.sacc_nz(version), + self.sacc_metadata(version), + rho_stat_handler.rho_stats, + tau_stat_handler.tau_stats, + tau_cov_th=tau_cov_th, + ) + out_path = os.path.join(out_dir, f"rho_tau_{base}.sacc") + sacc_io.save(s, out_path) + @property def rho_stat_handler(self): if not hasattr(self, "_rho_stat_handler"): diff --git a/src/sp_validation/cosmo_val/pure_eb.py b/src/sp_validation/cosmo_val/pure_eb.py index 7524074e..416ad8ef 100644 --- a/src/sp_validation/cosmo_val/pure_eb.py +++ b/src/sp_validation/cosmo_val/pure_eb.py @@ -7,6 +7,7 @@ import numpy as np +from .. import sacc_io from ..b_modes import ( calculate_eb_statistics, calculate_pure_eb_correlation, @@ -16,6 +17,7 @@ plot_pure_eb_correlations, save_pure_eb_results, ) +from .sacc_writers import pure_eb_to_sacc class PureEBMixin: @@ -132,6 +134,25 @@ def calculate_pure_eb( return results + def pure_eb_to_sacc_part(self, version, out_path, results): + """Write the pure-E/B SACC part (six ``PURE_KEYS`` blocks + covariance). + + ``results`` is the dict ``calculate_pure_eb`` returned: the six pure-mode + arrays under ``sacc_io.PURE_KEYS``, the ``"cov"`` block (in ``PURE_KEYS`` + order), and the reporting-grid TreeCorr object ``"gg"`` whose ``meanr`` + is the shared ``theta``. + """ + theta = results["gg"].meanr + eb = {key: results[key] for key in sacc_io.PURE_KEYS} + s = pure_eb_to_sacc( + self.sacc_nz(version), + self.sacc_metadata(version), + theta, + eb, + covariance=results["cov"], + ) + sacc_io.save(s, out_path) + def plot_pure_eb( self, versions=None, From ae03ea677c21d857c4666b7fd4ea0194621fd956 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 03:38:04 +0200 Subject: [PATCH 11/47] feat(workflow): born-as-SACC scripts (coarse/fine xi, pseudo-Cl, assemble) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - run_2pcf.py: after calculate_2pcf, write {ver}_xi_coarse.sacc via xi_to_sacc(grid='coarse', theta_nom=rnom, npairs, weight) — no covariance (added at assembly). Drops the save_fits threading (mixin no longer writes DES-style xi FITS). - run_2pcf_highres.py: on rank 0, write the terminal {version}_xi_fine.sacc via xi_to_sacc(grid='fine', variances=[varxip; varxim]) as a DiagonalCovariance. Deletes write_xi_fits (DES-FITS); keeps the .txt. sacc_io import works on the bare-host MPI path (no healpy), n(z) read from shear.redshift_path. - generate_pseudo_cl.py: native product is now pseudo_cl_{ver}.sacc (the mixin writes it); report reads it back via sacc_io.get_pseudo_cl. - assemble_sacc.py (new, dual-mode): loads the per-statistic parts in canonical order and calls assemble_analysis_sacc. Injects the xi-coarse block from the CosmoCov .txt (already [xi+;xi-]-ordered) and the pseudo-Cl block from the NaMaster iNKA cov FITS as block-diagonal [EE_EE;BB_BB;EB_EB] (cross-spectrum blocks dropped, TODO flagged). --allow-placeholder attaches a documented diagonal for cov-less parts so DAG dry-runs / tests produce a valid FullCovariance; production requires the real cov inputs (fail-fast otherwise). Co-Authored-By: Claude Fable 5 --- workflow/scripts/assemble_sacc.py | 228 +++++++++++++++++++++++++ workflow/scripts/generate_pseudo_cl.py | 34 ++-- workflow/scripts/run_2pcf.py | 45 ++++- workflow/scripts/run_2pcf_highres.py | 74 +++++--- 4 files changed, 327 insertions(+), 54 deletions(-) create mode 100644 workflow/scripts/assemble_sacc.py diff --git a/workflow/scripts/assemble_sacc.py b/workflow/scripts/assemble_sacc.py new file mode 100644 index 00000000..2ad6fa27 --- /dev/null +++ b/workflow/scripts/assemble_sacc.py @@ -0,0 +1,228 @@ +"""Assemble the terminal ``{version}.sacc`` analysis file from per-statistic parts. + +Dual-mode. Under Snakemake (``script:`` directive) the injected ``snakemake`` +object supplies the parts + covariance inputs; as a standalone CLI (argparse) +the same assembly runs from explicit flags (the lightcone/ASTRA path). + +Each per-statistic ``*.sacc`` *part* (written born-as-SACC by the mixins and the +run_2pcf / generate_pseudo_cl scripts) holds one statistic. The assembler loads +them in canonical order — ξ± coarse, pseudo-Cℓ, COSEBIs, pure-E/B, ρ/τ — and +calls :func:`sacc_writers.assemble_analysis_sacc`, which rebuilds one Sacc with a +single block-diagonal ``FullCovariance`` (point-insertion order = block order, +validated by ``sacc_io.assemble_covariance``). + +Covariance sourcing (the part-by-part decision) +----------------------------------------------- +``assemble_analysis_sacc`` REQUIRES every part to carry its own covariance block. +The COSEBIs, pure-E/B and ρ/τ parts already do (their writers attach it). The +ξ± coarse and pseudo-Cℓ parts are born cov-less by design; this script injects +their blocks before assembly: + +* **ξ± coarse** — the CosmoCov theory covariance ``.txt`` (``--xi-cov``). For the + single-bin round it is already ``[ξ+; ξ−]``-ordered (CosmoCov / covdat_to_fits: + ``STRT_0=0`` XI_PLUS, ``STRT_1=len/2`` XI_MINUS), which is exactly the SACC + ξ insertion order, so ``np.loadtxt`` → ``add_covariance`` needs no permutation. +* **pseudo-Cℓ** — the NaMaster iNKA / OneCovariance covariance FITS + (``--pseudo-cl-cov`` + ``--pseudo-cl-cov-hdu``). The FITS carries the 16 + EE/EB/BE/BB cross-blocks (each ``nbp × nbp``); SACC stores EE, BB, EB (in that + order), so we assemble the block-diagonal ``[EE_EE; BB_BB; EB_EB]``. The + cross-spectrum blocks (EE↔BB, …) are dropped — matching how the B-mode PTE + today reads only ``COVAR_BB_BB``. **TODO(PR-cov):** carry the full dense + EE/BB/EB cross-covariance once the analysis needs cross-spectrum correlations. + +When a cov input is absent the assembly cannot proceed on a real product; pass +``--allow-placeholder`` to attach a documented diagonal placeholder +(``placeholder_var`` on every point of the cov-less parts) so the DAG dry-run and +the fast test can still produce a structurally-valid ``FullCovariance``. The +placeholder is a flagged stand-in, never a science covariance. +""" + +import argparse + +import numpy as np + +from sp_validation import sacc_io +from sp_validation.cosmo_val.sacc_writers import assemble_analysis_sacc + +# NaMaster iNKA covariance FITS: per-spectrum HDU names. SACC insertion order is +# EE, BB, EB, so the block-diagonal is assembled in that order. +_CL_HDU = {"EE": "COVAR_EE_EE", "BB": "COVAR_BB_BB", "EB": "COVAR_EB_EB"} +_CL_ORDER = ("EE", "BB", "EB") + +# Canonical part order — the order assemble_analysis_sacc inserts points in, which +# must match the covariance block order. Missing parts are simply skipped. +CANONICAL = ("xi_coarse", "pseudo_cl", "cosebis", "pure_eb", "rho_tau") + + +def _pseudo_cl_cov_block(cov_fits, hdu): + """Block-diagonal ``[EE_EE; BB_BB; EB_EB]`` from the NaMaster iNKA cov FITS. + + ``hdu`` selects the file flavor: for the per-spectrum iNKA file we read the + three named diagonal HDUs; for a single dense HDU (OneCovariance / g+ng + ``COVAR_FULL``) that already spans EE/BB/EB we return it as-is. + """ + from astropy.io import fits + + with fits.open(cov_fits) as hdul: + names = {h.name for h in hdul} + if all(_CL_HDU[s] in names for s in _CL_ORDER): + blocks = [np.asarray(hdul[_CL_HDU[s]].data, float) for s in _CL_ORDER] + n = blocks[0].shape[0] + full = np.zeros((3 * n, 3 * n)) + for i, block in enumerate(blocks): + full[i * n : (i + 1) * n, i * n : (i + 1) * n] = block + return full + return np.asarray(hdul[hdu].data, float) + + +def _attach_cov(part, name, xi_cov, pseudo_cl_cov, pseudo_cl_cov_hdu, placeholder_var): + """Ensure ``part`` carries a covariance, injecting the xi/pseudo-Cℓ block. + + ``part`` is mutated in place. cosebis/pure_eb/rho_tau parts already carry + their covariance and pass straight through. Raises loudly if a required xi / + pseudo-Cℓ block is missing and no placeholder was requested. + """ + if part.covariance is not None: + return part + if name == "xi_coarse": + if xi_cov is not None: + part.add_covariance(np.loadtxt(xi_cov)) + return part + elif name == "pseudo_cl": + if pseudo_cl_cov is not None: + part.add_covariance(_pseudo_cl_cov_block(pseudo_cl_cov, pseudo_cl_cov_hdu)) + return part + if placeholder_var is None: + raise ValueError( + f"the {name!r} part carries no covariance and no covariance input was " + f"given (--xi-cov / --pseudo-cl-cov). Supply the block, or pass " + "--allow-placeholder to attach a documented diagonal placeholder." + ) + part.add_covariance(np.full(len(part.mean), float(placeholder_var))) + return part + + +def assemble_sacc( + version, + part_paths, + out_path, + *, + xi_cov=None, + pseudo_cl_cov=None, + pseudo_cl_cov_hdu="COVAR_FULL", + placeholder_var=None, +): + """Assemble ``{version}.sacc`` from the per-statistic ``part_paths`` mapping. + + Parameters + ---------- + version : str + Catalogue version (stored in the assembled file's metadata). + part_paths : dict + ``{statistic: path}`` with statistic in :data:`CANONICAL`. Only the + present statistics are assembled; order is forced to canonical. + out_path : str + Destination ``{version}.sacc``. + xi_cov, pseudo_cl_cov, pseudo_cl_cov_hdu, placeholder_var + Covariance sourcing — see the module docstring. + """ + parts = [] + nz = metadata = None + for name in CANONICAL: + path = part_paths.get(name) + if path is None: + continue + part = sacc_io.load(path) + if nz is None: + # The nz tracers + metadata are identical across parts (same version); + # take them from the first loaded part for the assembled file. + nz = {i: sacc_io.get_nz(part, i) for i in range(_n_source_bins(part))} + metadata = dict(part.metadata) + parts.append( + _attach_cov( + part, name, xi_cov, pseudo_cl_cov, pseudo_cl_cov_hdu, placeholder_var + ) + ) + if not parts: + raise ValueError(f"no parts found for {version}: {part_paths}") + s = assemble_analysis_sacc(nz, metadata, parts) + sacc_io.save(s, out_path) + print(f"Assembled {len(parts)} parts -> {out_path}") + return s + + +def _n_source_bins(part): + """Count the ``source_{i}`` NZ tracers on a part (single-bin round -> 1).""" + i = 0 + while sacc_io.source_name(i) in part.tracers: + i += 1 + return i + + +def _from_snakemake(smk): + p = smk.params + inp = smk.input + part_paths = { + name: getattr(inp, name) + for name in CANONICAL + if hasattr(inp, name) and getattr(inp, name) + } + assemble_sacc( + version=p["version"], + part_paths=part_paths, + out_path=str(smk.output[0]), + xi_cov=getattr(inp, "xi_cov", None), + pseudo_cl_cov=getattr(inp, "pseudo_cl_cov", None), + pseudo_cl_cov_hdu=p.get("pseudo_cl_cov_hdu", "COVAR_FULL"), + placeholder_var=p.get("placeholder_var", None), + ) + + +def _from_cli(argv=None): + ap = argparse.ArgumentParser( + description="Assemble the terminal {version}.sacc from per-statistic parts." + ) + ap.add_argument("--version", required=True, help="Catalogue version") + ap.add_argument("--out", required=True, help="Output {version}.sacc path") + for name in CANONICAL: + ap.add_argument( + f"--{name.replace('_', '-')}", default=None, help=f"{name} part" + ) + ap.add_argument("--xi-cov", default=None, help="CosmoCov ξ covariance .txt") + ap.add_argument( + "--pseudo-cl-cov", + default=None, + help="NaMaster/OneCovariance pseudo-Cℓ cov FITS", + ) + ap.add_argument( + "--pseudo-cl-cov-hdu", + default="COVAR_FULL", + help="HDU name for a single dense pseudo-Cℓ cov (EE/BB/EB-spanning)", + ) + ap.add_argument( + "--allow-placeholder", + type=float, + default=None, + metavar="VAR", + help="Attach a diagonal placeholder (variance VAR) to cov-less parts", + ) + a = ap.parse_args(argv) + part_paths = {name: getattr(a, name) for name in CANONICAL if getattr(a, name)} + assemble_sacc( + version=a.version, + part_paths=part_paths, + out_path=a.out, + xi_cov=a.xi_cov, + pseudo_cl_cov=a.pseudo_cl_cov, + pseudo_cl_cov_hdu=a.pseudo_cl_cov_hdu, + placeholder_var=a.allow_placeholder, + ) + + +if __name__ == "__main__": + try: + snakemake # noqa: F821 — injected by Snakemake's script: directive + except NameError: + _from_cli() + else: + _from_snakemake(snakemake) # noqa: F821 diff --git a/workflow/scripts/generate_pseudo_cl.py b/workflow/scripts/generate_pseudo_cl.py index a5c19a56..81f69414 100644 --- a/workflow/scripts/generate_pseudo_cl.py +++ b/workflow/scripts/generate_pseudo_cl.py @@ -4,12 +4,13 @@ object supplies the parameters and the native product is renamed to the tagged output filename the rule declares; as a standalone CLI (argparse) the same compute runs from explicit flags and the primitive's native -``pseudo_cl_{ver}.fits`` is left in place under ``--out`` (no rename — each +``pseudo_cl_{ver}.sacc`` is left in place under ``--out`` (no rename — each lc/ASTRA recipe gets its own output directory, so the untagged native name is unambiguous and the primitives' skip-if-exists never collides across nbins -runs). The CLI form is what the lightcone/ASTRA recipe calls, so the -measurement is driven directly (no nested Snakemake) with lc handling -orchestration: +runs). The C_ell data vector is born as SACC (EE/BB/EB with a shared bandpower +window) — see ``sp_validation.cosmo_val.sacc_writers.pseudo_cl_to_sacc``. The +CLI form is what the lightcone/ASTRA recipe calls, so the measurement is driven +directly (no nested Snakemake) with lc handling orchestration: python generate_pseudo_cl.py \ --ver SP_v1.4.6.3_leak_corr \ @@ -28,8 +29,7 @@ import json import os -from astropy.io import fits - +from sp_validation import sacc_io from sp_validation.cosmo_val import CosmologyValidation @@ -52,8 +52,8 @@ def generate_pseudo_cl( version : str Catalog version (e.g., "SP_v1.4.6_leak_corr") output_dir : str - Directory the pseudo-Cl FITS file is written into. The primitive writes - its native ``pseudo_cl_{version}.fits`` here; callers that need a tagged + Directory the pseudo-Cl SACC part is written into. The primitive writes + its native ``pseudo_cl_{version}.sacc`` here; callers that need a tagged filename rename it themselves (see ``_from_snakemake``). cat_config : str Path to catalog configuration YAML @@ -76,7 +76,7 @@ def generate_pseudo_cl( Returns ------- str - Path to the primitive's native ``pseudo_cl_{version}.fits`` product. + Path to the primitive's native ``pseudo_cl_{version}.sacc`` product. """ os.makedirs(output_dir, exist_ok=True) @@ -135,17 +135,17 @@ def generate_pseudo_cl( cv = CosmologyValidation(**cv_kwargs) - # Calculate pseudo-Cls only (no covariance) + # Calculate pseudo-Cls only (no covariance). The data vector is born as a + # SACC part: pseudo_cl_{version}.sacc under output_dir. cv.calculate_pseudo_cl() - # Report on the native product (renamed by the Snakemake caller, if any) - src_cl = os.path.join(output_dir, f"pseudo_cl_{version}.fits") + # Report on the native product (renamed by the Snakemake caller, if any). + src_cl = os.path.join(output_dir, f"pseudo_cl_{version}.sacc") if os.path.exists(src_cl): - with fits.open(src_cl) as hdul: - data = hdul["PSEUDO_CELL"].data - n_ell = len(data["ELL"]) - print(f"Generated pseudo-Cl with {n_ell} ell bins") - print(f"ell range: [{data['ELL'].min():.1f}, {data['ELL'].max():.1f}]") + s = sacc_io.load(src_cl) + ell = sacc_io.get_pseudo_cl(s, (0, 0))[0] + print(f"Generated pseudo-Cl with {len(ell)} ell bins") + print(f"ell range: [{ell.min():.1f}, {ell.max():.1f}]") return src_cl diff --git a/workflow/scripts/run_2pcf.py b/workflow/scripts/run_2pcf.py index 2e1ccabf..9381d788 100644 --- a/workflow/scripts/run_2pcf.py +++ b/workflow/scripts/run_2pcf.py @@ -13,14 +13,21 @@ --out The measurement itself is unchanged — ``CosmologyValidation.calculate_2pcf`` -does the TreeCorr work and writes the ``.txt`` dump plus ξ+/ξ- FITS files into -``output_dir``. ``output_dir`` is passed explicitly (rather than via the +does the TreeCorr work and writes the ``.txt`` dump (a raw byproduct the +covariance machinery reads back). The analysis ξ± data product is then born as +SACC here: ``{ver}_xi_coarse.sacc``, a *part* on the coarse grid via +``xi_to_sacc(grid="coarse", ...)`` carrying ``theta_nom``/``npairs``/``weight`` +tags but NO covariance (the ξ block is supplied at assembly from the CosmoCov +theory covariance). ``output_dir`` is passed explicitly (rather than via the ``COSMO_VAL`` env hook) so lc can point each run at its own ``{output}`` tree. """ import argparse +import os +from sp_validation import sacc_io from sp_validation.cosmo_val import CosmologyValidation +from sp_validation.cosmo_val.sacc_writers import xi_to_sacc def run_2pcf( @@ -31,30 +38,53 @@ def run_2pcf( npatch, cat_config, output_dir, - save_fits=True, ): - """Measure ξ±(θ) for ``ver`` and write it under ``output_dir``. + """Measure ξ±(θ) for ``ver`` and write its coarse SACC part under ``output_dir``. Parameters mirror the TreeCorr reporting/integration grids: ``min_sep`` / ``max_sep`` in arcmin, ``nbins`` logarithmic bins, ``npatch`` spatial patches (1 for the paper fiducial). ``cat_config`` is an absolute path to the catalog configuration; ``output_dir`` overrides ``cat_config['paths']['output']`` so products land where lc expects. + + Returns + ------- + treecorr.GGCorrelation + The measured correlation object (also the source of the SACC part). """ cv = CosmologyValidation( versions=[ver], catalog_config=cat_config, output_dir=output_dir, ) - return cv.calculate_2pcf( + gg = cv.calculate_2pcf( ver=ver, npatch=npatch, - save_fits=save_fits, min_sep=min_sep, max_sep=max_sep, nbins=nbins, ) + # Born-as-SACC coarse ξ± part: no covariance here (added at assembly from + # the CosmoCov theory covariance). theta = meanr; theta_nom = rnom. + s = xi_to_sacc( + cv.sacc_nz(ver), + cv.sacc_metadata(ver), + gg.meanr, + gg.xip, + gg.xim, + grid="coarse", + theta_nom=gg.rnom, + npairs=gg.npairs, + weight=gg.weight, + ) + out_path = os.path.join( + output_dir or cv.cc["paths"]["output"], f"{ver}_xi_coarse.sacc" + ) + sacc_io.save(s, out_path) + print(f"Wrote coarse ξ± SACC part: {out_path}") + return gg + def _from_snakemake(smk): p = smk.params @@ -70,7 +100,6 @@ def _from_snakemake(smk): # class defaults (./cat_config.yaml, COSMO_VAL env) otherwise. cat_config=p.get("cat_config", "./cat_config.yaml"), output_dir=p.get("output_dir", None), - save_fits=True, ) @@ -97,7 +126,6 @@ def _from_cli(argv=None): "--cat-config", required=True, help="Absolute path to cat_config.yaml" ) ap.add_argument("--out", required=True, help="Output directory (lc {output})") - ap.add_argument("--no-fits", action="store_true", help="Skip ξ+/ξ- FITS export") a = ap.parse_args(argv) run_2pcf( ver=a.ver, @@ -107,7 +135,6 @@ def _from_cli(argv=None): npatch=a.npatch, cat_config=a.cat_config, output_dir=a.out, - save_fits=not a.no_fits, ) diff --git a/workflow/scripts/run_2pcf_highres.py b/workflow/scripts/run_2pcf_highres.py index eb31d2c7..ed3660bf 100644 --- a/workflow/scripts/run_2pcf_highres.py +++ b/workflow/scripts/run_2pcf_highres.py @@ -27,6 +27,12 @@ import treecorr from astropy.io import fits +# sacc_io depends only on numpy + sacc (no healpy/cs_util), so the born-as-SACC +# fine ξ± write works on the bare-host MPI path too, where the full cosmo_val +# stack is unavailable. +from sp_validation import sacc_io +from sp_validation.cosmo_val.sacc_writers import xi_to_sacc + try: # In-container path: full sp_validation stack available. from sp_validation.cosmo_val import CosmologyValidation @@ -76,6 +82,7 @@ E1_COL = None E2_COL = None W_COL = None +REDSHIFT_PATH = None # n(z) file for the SACC tracer TMIN = None # arcmin TMAX = None # arcmin NBINS = None @@ -196,35 +203,46 @@ def compute_patch_centers(ra, dec): del cat_sub -def write_xi_fits(gg, prefix, xi_data): - """Write ξ+ or ξ- to FITS matching CosmologyValidation format.""" - out_path = os.path.join( - OUTPUT_DIR, - f"{prefix}_{VERSION}_minsep={TMIN}_maxsep={TMAX}_nbins={NBINS}_npatch=1.fits", +def write_xi_fine_sacc(gg): + """Write the terminal fine-grid ξ± SACC part (``{version}_xi_fine.sacc``). + + This is a terminal product in its own right — COSEBIs and pure-E/B consume + it. It carries a ``DiagonalCovariance`` from TreeCorr ``varxip``/``varxim`` + (npatch=1 leaves shot-noise variance as the only covariance estimate). + Both run paths land here: in-container this uses the full SACC stack; on the + bare-host MPI run only ``sacc_io`` + the n(z) file are needed (no healpy). + """ + z, nz = np.loadtxt(REDSHIFT_PATH, unpack=True) + metadata = { + "catalogue_version": VERSION, + "sp_validation_version": _sp_validation_version(), + "npatch": 1, + } + s = xi_to_sacc( + {0: (z, nz)}, + metadata, + gg.meanr, + gg.xip, + gg.xim, + grid="fine", + theta_nom=gg.rnom, + variances=np.concatenate([gg.varxip, gg.varxim]), ) - n = len(xi_data) - cols = [ - fits.Column(name="BIN1", format="K", array=np.ones(n, dtype=int)), - fits.Column(name="BIN2", format="K", array=np.ones(n, dtype=int)), - fits.Column(name="ANGBIN", format="K", array=np.arange(1, n + 1)), - fits.Column(name="VALUE", format="D", array=xi_data), - fits.Column(name="ANG", format="D", unit="arcmin", array=gg.meanr), - ] - ext_name = "XI_PLUS" if "plus" in prefix else "XI_MINUS" - hdu = fits.BinTableHDU.from_columns(cols, name=ext_name) - for key, val in { - "2PTDATA": "T", - "QUANT1": "G+R", - "QUANT2": "G+R", - "KERNEL_1": "NZ_SOURCE", - "KERNEL_2": "NZ_SOURCE", - "WINDOWS": "SAMPLE", - }.items(): - hdu.header[key] = val - hdu.writeto(out_path, overwrite=True) + out_path = os.path.join(OUTPUT_DIR, f"{VERSION}_xi_fine.sacc") + sacc_io.save(s, out_path) log(f" Wrote {out_path}") +def _sp_validation_version(): + """Best-effort package version for the SACC metadata (empty if unavailable).""" + try: + from sp_validation import __version__ + + return __version__ + except Exception: + return "" + + def resolve_shear_config(cat_config_path, version): """Standalone shear-config resolver (bare-host fallback for CosmologyValidation). @@ -274,7 +292,7 @@ def resolve_paths(ver): def main(): - global CAT_PATH, VERSION, E1_COL, E2_COL, W_COL + global CAT_PATH, VERSION, E1_COL, E2_COL, W_COL, REDSHIFT_PATH global TMIN, TMAX, NBINS, NPATCH, OUTPUT_DIR, PATCH_FILE args = parse_args() @@ -301,6 +319,7 @@ def main(): E1_COL = shear_cfg["e1_col"] E2_COL = shear_cfg["e2_col"] W_COL = shear_cfg["w_col"] + REDSHIFT_PATH = shear_cfg["redshift_path"] PATCH_FILE = os.path.join( OUTPUT_DIR, @@ -383,8 +402,7 @@ def main(): gg.write(out_txt, write_patch_results=False, write_cov=False) log(f" Wrote {out_txt}") - write_xi_fits(gg, "xi_plus", gg.xip) - write_xi_fits(gg, "xi_minus", gg.xim) + write_xi_fine_sacc(gg) elapsed = time.time() - t0 log(f"Done! Total time: {elapsed / 3600:.1f}h ({elapsed:.0f}s)") From 700a88579942ac5afb6574c1949d6fa2a55fb85a Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 07:34:24 +0200 Subject: [PATCH 12/47] =?UTF-8?q?fix(twopoint):=20fail=20fast=20on=20tomog?= =?UTF-8?q?raphic/=CE=BE-less=20SACC;=20pin=20covariance=20gather?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adversarial-review hardening (one HIGH, one MEDIUM, one coverage gap): - HIGH: n_bins>1 silently truncated — n_bins drove only the NZDATA column count while data/covariance were read from bin (0, 0), so a 2-bin SACC emitted a plausible-looking FITS carrying 1/3 of the data. The converter now fails fast unless n_bins == 1 and the ξ tracer pairs are exactly {(source_0, source_0)}; tomographic emission lands with the tomographic round. - MEDIUM: a ξ-less SACC wrote an empty XI_PLUS and a (0, 0) COVMAT silently; now a loud ValueError. - Coverage: every prior covariance test exercised the identity permutation (single-pair SACC order is already type-major). A (row, col)-encoded covariance test now pins the exact np.ix_ gather of COVMAT (block-diag ξ + joint non-adjacent [τ0+; τ2+]) and COVMAT_CELL — any transposition, offset, or swapped block fails. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_016Ko89uZF84ez6HEDcTWbJe --- .../tests/test_twopoint_convert.py | 94 +++++++++++++++++++ src/sp_validation/twopoint_convert.py | 42 ++++++++- 2 files changed, 134 insertions(+), 2 deletions(-) diff --git a/src/sp_validation/tests/test_twopoint_convert.py b/src/sp_validation/tests/test_twopoint_convert.py index 5e599717..3a5bc44a 100644 --- a/src/sp_validation/tests/test_twopoint_convert.py +++ b/src/sp_validation/tests/test_twopoint_convert.py @@ -376,3 +376,97 @@ def test_rho_tau_sidecars_required_together(tmp_path): twopoint_convert.sacc_to_twopoint_fits( s, str(tmp_path / "x.fits"), rho_stats_hdu=rho_hdu, n_bins=1 ) + + +# ============================================================================= +# Fail-fast guards and permutation teeth (adversarial-review hardening) +# ============================================================================= + + +def test_tomographic_sacc_raises(tmp_path): + """A multi-bin SACC fails fast instead of silently truncating to (0, 0). + + Review finding (HIGH): ``n_bins`` alone drove the NZDATA column count while + the data vector and covariance were read from bin ``(0, 0)`` only, so a + 2-bin SACC + ``n_bins=2`` emitted a plausible-looking FITS carrying 1/3 of + the data. Both the ``n_bins`` and the tracer-pair mismatch must raise. + """ + inp = _inputs(seed=30) + s = sacc_io.new_sacc({0: (inp["z"], inp["nz"]), 1: (inp["z"], inp["nz"])}) + for pair in [(0, 0), (0, 1), (1, 1)]: + sacc_io.add_xi(s, pair, inp["theta"], inp["xip"], inp["xim"], grid="coarse") + s.add_covariance(np.eye(len(s.mean))) + + with pytest.raises(ValueError, match="single-bin only"): + twopoint_convert.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits"), n_bins=2) + with pytest.raises(ValueError, match="single-bin only"): + twopoint_convert.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits"), n_bins=1) + assert not (tmp_path / "x.fits").exists() + + +def test_sacc_without_xi_raises(tmp_path): + """A SACC with no ξ± points raises instead of writing an empty data vector.""" + inp = _inputs(seed=31) + s = sacc_io.new_sacc({0: (inp["z"], inp["nz"])}) + sacc_io.add_pseudo_cl( + s, + (0, 0), + inp["ell"], + inp["cl_ee"], + inp["cl_bb"], + inp["cl_eb"], + window_ells=np.arange(2, 102), + window_weights=np.random.default_rng(9).uniform(0, 1, (100, N_ELL)), + ) + s.add_covariance(np.eye(len(s.mean))) + + with pytest.raises(ValueError, match="nothing to convert"): + twopoint_convert.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits")) + assert not (tmp_path / "x.fits").exists() + + +def test_covmat_blocks_exact_gather_encoded_cov(tmp_path): + """Every COVMAT/COVMAT_CELL entry is the exact ``np.ix_`` gather of the SACC + covariance, pinned with a (row, col)-encoded matrix. + + Review finding (MEDIUM): for a single bin pair the ξ gather happens to be + the identity permutation, so the byte-compares alone could pass with a + transposed or block-swapped gather. Encoding ``C[i, j] = i*n + j`` (asymmetric, + every entry unique) makes any transposition, offset, or wrong block produce + detectably wrong values; the τ gather is genuinely non-identity (τ_0− sits + between τ_0+ and τ_2+ in insertion order). Expected layout per + ``covdat_to_fits``: block_diag(ξ type-major gather, joint [τ_0+; τ_2+] + gather), with COVMAT_CELL the CELL_EE gather in its own HDU. + """ + inp = _inputs(seed=32) + s = _sacc(inp, cl=True, rho_tau=True) + n = len(s.mean) + encoded = np.arange(n * n, dtype=float).reshape(n, n) + s.add_covariance(encoded, overwrite=True) + rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) + + out = tmp_path / "encoded.fits" + twopoint_convert.sacc_to_twopoint_fits( + s, str(out), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 + ) + + pair = (SOURCE, SOURCE) + xi_idx = np.concatenate( + [s.indices(sacc_io.XI_PLUS, pair), s.indices(sacc_io.XI_MINUS, pair)] + ) + tau_idx = np.concatenate( + [ + s.indices(sacc_io.TAU_PLUS.format(k=0), (SOURCE, PSF)), + s.indices(sacc_io.TAU_PLUS.format(k=2), (SOURCE, PSF)), + ] + ) + expected = twopoint_convert._block_diag( + encoded[np.ix_(xi_idx, xi_idx)], encoded[np.ix_(tau_idx, tau_idx)] + ) + cell_idx = s.indices(sacc_io.CL_EE, pair) + + with fits.open(out) as hdul: + np.testing.assert_array_equal(hdul["COVMAT"].data, expected) + np.testing.assert_array_equal( + hdul["COVMAT_CELL"].data, encoded[np.ix_(cell_idx, cell_idx)] + ) diff --git a/src/sp_validation/twopoint_convert.py b/src/sp_validation/twopoint_convert.py index c06e0eb3..8f787f3b 100644 --- a/src/sp_validation/twopoint_convert.py +++ b/src/sp_validation/twopoint_convert.py @@ -160,6 +160,33 @@ def _type_major_xi(s, bins): return sacc_io.get_xi(s, bins, grid="coarse") +def _require_single_bin(s, n_bins): + """Fail fast unless the SACC is a valid single-bin ξ product. + + The converter emits the single-bin 2pt-FITS today's CosmoSIS pipeline reads + (BIN1/BIN2 all 1, one NZ column). A tomographic SACC would otherwise slip + through silently — ``n_bins`` alone drives the NZDATA column count while the + ξ/covariance are read from bin ``(0, 0)`` only, so a 2-bin file would emit a + ``NBIN=2`` n(z) beside a data vector holding just the ``(0, 0)`` pair. + Guards both the empty-ξ case and the single-bin contract; tomographic + emission lands with the tomographic round. + """ + pairs = s.get_tracer_combinations(sacc_io.XI_PLUS) + if not pairs: + raise ValueError( + f"SACC has no {sacc_io.XI_PLUS} points — nothing to convert; the " + "2pt-FITS data vector is built from the ξ± statistics" + ) + expected = (sacc_io.source_name(0), sacc_io.source_name(0)) + if n_bins != 1 or set(pairs) != {expected}: + raise ValueError( + f"converter is single-bin only (n_bins=1, ξ pairs == {{{expected}}}); " + f"got n_bins={n_bins} and ξ pairs {sorted(pairs)}. Tomographic " + "emission (multiple bin pairs, per-pair BIN1/BIN2, one NZ column per " + "bin) lands with the tomographic round." + ) + + def sacc_to_twopoint_fits( s, path, @@ -191,19 +218,30 @@ def sacc_to_twopoint_fits( alone cannot rebuild the ``varrho_*`` columns Sacha's fork reads. When omitted, a pure ξ (± Cℓ) product is written. n_bins : int, optional - Number of source tomographic bins (default 1, the current single-bin - analysis). Sets the NZDATA column count. + Number of source tomographic bins. Must be ``1``: this converter emits + the single-bin 2pt-FITS today's CosmoSIS pipeline consumes. Tomographic + emission (multiple bin pairs, per-pair BIN1/BIN2, one NZ column per bin) + lands with the tomographic round; the converter fails fast on anything + else rather than silently truncating to bin ``(0, 0)``. Returns ------- astropy.io.fits.HDUList The assembled list, also written to ``path``. + + Raises + ------ + ValueError + If the SACC has no ξ points; if ``n_bins != 1`` or the SACC's ξ tracer + pairs are anything other than exactly ``{(source_0, source_0)}`` (the + single-bin contract); or if exactly one of the ρ/τ sidecars is supplied. """ if (rho_stats_hdu is None) != (tau_stats_hdu is None): raise ValueError( "rho_stats_hdu and tau_stats_hdu must be supplied together " "(the ρ/τ product needs both, or neither for a pure-ξ product)" ) + _require_single_bin(s, n_bins) use_rho_tau = rho_stats_hdu is not None bins = (0, 0) From 91f5d43e4fa72dfb8435134eabe2c2fdb6b8932f Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 07:39:58 +0200 Subject: [PATCH 13/47] test(pseudo_cl): fixture carries shear.redshift_path like the real cat_config The migrated pseudo-Cl SACC part calls get_redshift(), which reads cc[version]['shear']['redshift_path']; the synthetic fixture wrote the dndz file but never pointed that key at it (KeyError at core.py:422). Verified: the KeyError layer is resolved; the end-to-end test now fails one layer deeper (test asserts the legacy ELL/EE/BB/EB FITS columns while the migrated writer emits the SACC part - the open wiring seam). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_016Ko89uZF84ez6HEDcTWbJe --- src/sp_validation/tests/test_pseudo_cl.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sp_validation/tests/test_pseudo_cl.py b/src/sp_validation/tests/test_pseudo_cl.py index 45a2366d..6fc43d55 100644 --- a/src/sp_validation/tests/test_pseudo_cl.py +++ b/src/sp_validation/tests/test_pseudo_cl.py @@ -112,6 +112,7 @@ def _write_synthetic_config(tmp_path): shear_cfg = { "path": "shear.fits", + "redshift_path": str(nz_dir / "dndz_SP_A.txt"), "w_col": "w", "e1_col": "e1", "e2_col": "e2", From dd474b3e17b28c11fd913824be00552eb580642d Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 07:40:49 +0200 Subject: [PATCH 14/47] WIP(workflow): born-as-SACC rule wiring - UNVERIFIED, interrupted mid-flight Working-tree state of the wiring subagent when it died on the session limit (its final turn also went unreviewed by the safety classifier). Wires cosmo_val.smk/twopoint.smk rules and cv_* scripts to the SACC parts + assemble_sacc rule. NOT dry-run-tested (no snakemake in the shared venv), NOT reviewed; test_calculate_pseudo_cl_catalog_end_to_end still fails at the writer/consumer seam (test reads legacy ELL columns). Next session: verify this diff against the DAG design in the PR-4 plan, reconcile the pseudo-Cl end-to-end test with the born-as-SACC output, add writer-path tests, run the suite, then de-WIP. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_016Ko89uZF84ez6HEDcTWbJe --- workflow/rules/cosmo_val.smk | 40 +++++++++++++++++++++---- workflow/rules/twopoint.smk | 27 ++++++++++------- workflow/scripts/cv_cosebis.py | 18 +++++++++-- workflow/scripts/cv_pure_eb.py | 12 +++++--- workflow/scripts/cv_summarize_bmodes.py | 4 +-- 5 files changed, 76 insertions(+), 25 deletions(-) diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index 704b4fb0..000b445a 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -104,8 +104,36 @@ def cv_cosebis_npz(version): ) -def cv_pseudo_cl_fits(version): - return str(COSMO_VAL / f"pseudo_cl_{version}.fits") +def cv_pseudo_cl_sacc(version): + """Pseudo-Cl SACC part calculate_pseudo_cl writes (born as SACC).""" + return str(COSMO_VAL / f"pseudo_cl_{version}.sacc") + + +def cv_cosebis_sacc(version): + """COSEBIs SACC part (fiducial scale cut) the cv_cosebis rule writes.""" + return str(COSMO_VAL / f"{version}_cosebis.sacc") + + +def cv_pure_eb_sacc(version): + """Pure-E/B SACC part the cv_pure_eb rule writes.""" + return str(COSMO_VAL / f"{version}_pure_eb.sacc") + + +def cv_rho_tau_sacc(version): + """ρ/τ SACC part calculate_rho_tau_stats writes (rho_tau_{base}.sacc).""" + return str( + COSMO_VAL / "rho_tau_stats" / f"rho_tau_{cv_basename(version, CV_FIDUCIAL)}.sacc" + ) + + +def cv_xi_coarse_sacc(version): + """Coarse ξ± SACC part the xi rule (run_2pcf.py) writes for a version.""" + return str(COSMO_VAL / f"{version}_xi_coarse.sacc") + + +def cv_analysis_sacc(version): + """Terminal assembled analysis file {version}.sacc.""" + return str(COSMO_VAL / f"{version}.sacc") # Common params block shared by every cosmo_val rule: the cv constructor kwargs @@ -275,9 +303,9 @@ rule cv_ratio_xi_sys_xi: # --------------------------------------------------------------------------- rule cv_pseudo_cl: - """Pseudo-Cl E/B spectra for all versions (NaMaster).""" + """Pseudo-Cl E/B spectra for all versions (NaMaster), born as SACC parts.""" output: - pseudo_cl=[cv_pseudo_cl_fits(v) for v in CV_VERSIONS], + pseudo_cl=[cv_pseudo_cl_sacc(v) for v in CV_VERSIONS], params: **cv_params(), threads: 12 @@ -298,6 +326,7 @@ rule cv_pure_eb: xi=lambda w: cv_xi_txt(w.version), output: npz=cv_pure_eb_npz("{version}"), + sacc=cv_pure_eb_sacc("{version}"), params: version="{version}", min_sep_int=CV["pure_eb"]["min_sep_int"], @@ -320,6 +349,7 @@ rule cv_cosebis: xi=lambda w: cv_xi_txt(w.version), output: npz=cv_cosebis_npz("{version}"), + sacc=cv_cosebis_sacc("{version}"), params: version="{version}", min_sep_int=CV["cosebis"]["min_sep_int"], @@ -345,7 +375,7 @@ rule cv_summarize_bmodes: pure_eb=[cv_pure_eb_npz(v) for v in CV_VERSIONS], cosebis=[cv_cosebis_npz(v) for v in CV_VERSIONS], pseudo_cl=( - [cv_pseudo_cl_fits(v) for v in CV_VERSIONS] + [cv_pseudo_cl_sacc(v) for v in CV_VERSIONS] if CV.get("include_pseudo_cl", False) else [] ), output: diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index 22c09db2..1e23c6ce 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -5,9 +5,11 @@ rule xi: input: catalog=get_shear_catalog, output: - str(COSMO_VAL / "{version}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.txt"), - str(COSMO_VAL / "xi_plus_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), - str(COSMO_VAL / "xi_minus_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), + # Raw TreeCorr .txt byproduct (read back by covariance + skip-if-exists) + # and the born-as-SACC coarse ξ± part (a .part — no covariance until the + # assemble_sacc rule injects the CosmoCov block). + txt=str(COSMO_VAL / "{version}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.txt"), + xi_coarse=str(COSMO_VAL / "{version}_xi_coarse.sacc"), threads: 24 params: ver="{version}", @@ -15,7 +17,6 @@ rule xi: max_sep="{max_sep}", nbins="{nbins}", npatch="{npatch}", - fits=False, resources: mem_mb=30000, disk_mb=20000, @@ -25,12 +26,16 @@ rule xi: rule xi_highres: - """High-resolution xi for COSEBIS integration.""" + """High-resolution xi for COSEBIS integration. + + Terminal born-as-SACC product: {version}_xi_fine.sacc (a DiagonalCovariance + from TreeCorr varxip/varxim). COSEBIs and pure-E/B consume it. The raw .txt + dump is kept as a convergence byproduct. + """ container: None output: txt=str(COSMO_VAL / f"{FIDUCIAL['version']}_xi_minsep={FIDUCIAL['min_sep_int']}_maxsep={FIDUCIAL['max_sep_int']}_nbins=10000_npatch=1.txt"), - xi_plus=str(COSMO_VAL / f"xi_plus_{FIDUCIAL['version']}_minsep={FIDUCIAL['min_sep_int']}_maxsep={FIDUCIAL['max_sep_int']}_nbins=10000_npatch=1.fits"), - xi_minus=str(COSMO_VAL / f"xi_minus_{FIDUCIAL['version']}_minsep={FIDUCIAL['min_sep_int']}_maxsep={FIDUCIAL['max_sep_int']}_nbins=10000_npatch=1.fits"), + xi_fine=str(COSMO_VAL / f"{FIDUCIAL['version']}_xi_fine.sacc"), resources: tasks=30, cpus_per_task=12, @@ -92,9 +97,9 @@ wildcard_constraints: rule pseudo_cl: - """Generate pseudo-Cl data vector with configurable binning.""" + """Generate pseudo-Cl data vector (born as SACC) with configurable binning.""" output: - pseudo_cl=str(COSMO_VAL / "pseudo_cl_{version}_blind={blind}_{binning}_nbins={nbins}.fits"), + pseudo_cl=str(COSMO_VAL / "pseudo_cl_{version}_blind={blind}_{binning}_nbins={nbins}.sacc"), wildcard_constraints: blind="[ABC]", params: @@ -146,7 +151,7 @@ rule pseudo_cl_all: """Generate pseudo-Cls for all versions.""" input: expand( - str(COSMO_VAL / "pseudo_cl_{version}_blind=A_powspace_nbins=32.fits"), + str(COSMO_VAL / "pseudo_cl_{version}_blind=A_powspace_nbins=32.sacc"), version=PSEUDO_CL_VERSIONS, ), @@ -164,7 +169,7 @@ rule pseudo_cl_fine_all: """Generate fine pseudo-Cls for COSEBIS.""" input: expand( - str(COSMO_VAL / "pseudo_cl_{version}_blind={blind}_linear_nbins=2040.fits"), + str(COSMO_VAL / "pseudo_cl_{version}_blind={blind}_linear_nbins=2040.sacc"), version=config["versions"], blind=BLINDS, ), diff --git a/workflow/scripts/cv_cosebis.py b/workflow/scripts/cv_cosebis.py index 182cda99..b90ebc5b 100644 --- a/workflow/scripts/cv_cosebis.py +++ b/workflow/scripts/cv_cosebis.py @@ -3,8 +3,11 @@ Compute + plot rule (per version). plot_cosebis calls calculate_cosebis over a fine integration binning (the 2000-bin TreeCorr is the dominant cost) and evaluates the configured scale cuts. Writes the {version}_eb_..._data.npz -COSEBIs data product (declared output) plus figures, and the per-version -COSEBIs PTE that cv_summarize_bmodes collects. +COSEBIs data product plus figures, and the per-version COSEBIs PTE that +cv_summarize_bmodes collects. It also writes the born-as-SACC COSEBIs part +({version}_cosebis.sacc, the fiducial scale cut's {En,Bn,cov}) that the +assemble_sacc rule consumes — the multi-cut .npz sidecar stays the diagnostic +PTE scan. """ from cv_runner import _unbuffer_streams, make_cv, verify_outputs @@ -13,8 +16,9 @@ _unbuffer_streams() cv = make_cv(snakemake) p = snakemake.params +version = p["version"] cv.plot_cosebis( - version=p["version"], + version=version, min_sep_int=p["min_sep_int"], max_sep_int=p["max_sep_int"], nbins_int=p["nbins_int"], @@ -23,4 +27,12 @@ scale_cuts=[tuple(sc) for sc in p["scale_cuts"]], fiducial_scale_cut=tuple(p["fiducial_scale_cut"]), ) +# Born-as-SACC COSEBIs part at the fiducial scale cut (plot_cosebis stored the +# multi-cut results on the instance). +cv.cosebis_to_sacc_part( + version, + snakemake.output["sacc"], + cv._cosebis_results[version], + fiducial_scale_cut=tuple(p["fiducial_scale_cut"]), +) verify_outputs(snakemake) diff --git a/workflow/scripts/cv_pure_eb.py b/workflow/scripts/cv_pure_eb.py index d15a763f..7a453b16 100644 --- a/workflow/scripts/cv_pure_eb.py +++ b/workflow/scripts/cv_pure_eb.py @@ -3,9 +3,10 @@ Compute + plot rule (per version). plot_pure_eb calls calculate_pure_eb, which runs two TreeCorr correlations (reporting + integration binning); the reporting binning reuses the cv_2pcf data vector via calculate_2pcf's skip-if-exists -path. Writes the {version}_eb_..._data.npz data product (declared output) plus -companion figures, and the per-version E/B PTEs that cv_summarize_bmodes -collects. +path. Writes the {version}_eb_..._data.npz data product plus companion figures, +and the per-version E/B PTEs that cv_summarize_bmodes collects. It also writes +the born-as-SACC pure-E/B part ({version}_pure_eb.sacc, the six PURE_KEYS blocks ++ covariance) that the assemble_sacc rule consumes. """ from cv_runner import _unbuffer_streams, make_cv, verify_outputs @@ -14,12 +15,15 @@ _unbuffer_streams() cv = make_cv(snakemake) p = snakemake.params +version = p["version"] cv.plot_pure_eb( - versions=[p["version"]], + versions=[version], min_sep_int=p["min_sep_int"], max_sep_int=p["max_sep_int"], nbins_int=p["nbins_int"], fiducial_xip_scale_cut=tuple(p["fiducial_scale_cut"]), fiducial_xim_scale_cut=tuple(p["fiducial_scale_cut"]), ) +# Born-as-SACC pure-E/B part (plot_pure_eb stored the results on the instance). +cv.pure_eb_to_sacc_part(version, snakemake.output["sacc"], cv._pure_eb_results[version]) verify_outputs(snakemake) diff --git a/workflow/scripts/cv_summarize_bmodes.py b/workflow/scripts/cv_summarize_bmodes.py index 90df5999..0fe7b95a 100644 --- a/workflow/scripts/cv_summarize_bmodes.py +++ b/workflow/scripts/cv_summarize_bmodes.py @@ -3,8 +3,8 @@ The terminal diagnostic. summarize_bmodes reads the in-memory _pure_eb_results / _cosebis_results / _pseudo_cls dicts, which are populated by plot_pure_eb / plot_cosebis / plot_pseudo_cl. The per-version E/B and COSEBIs -npz products and the pseudo-Cl FITS are declared as inputs (so the DAG forces -those rules first), but the summary still needs the live result objects (it +npz products and the pseudo-Cl SACC parts are declared as inputs (so the DAG +forces those rules first), but the summary still needs the live result objects (it reads each version's TreeCorr `gg`, which the npz cannot hold). So this rule re-runs the three B-mode methods in-process: they reload the existing 2pcf / data-vector files via their skip-if-exists paths and recompute only the cheap From 02d9cf32d24c65c9daac1426e4a567f82e385099 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 08:01:31 +0200 Subject: [PATCH 15/47] test(pseudo_cl): read the born-as-SACC part in the end-to-end catalog test calculate_pseudo_cl_catalog is born-as-SACC (pseudo_cl_to_sacc_part writes EE/BB/EB + a shared bandpower window), so the end-to-end test can no longer read legacy ELL/EE/EB/BB FITS columns. Round-trip through sacc_io.get_pseudo_cl instead and assert the bandpower window rides the part per the layout contract. The pinned golden spectra are unchanged (identical computation, SACC serialization); they held bitwise through the round-trip. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01HPfbe2XVTGzbnPn4g7BdN6 --- src/sp_validation/tests/test_pseudo_cl.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/sp_validation/tests/test_pseudo_cl.py b/src/sp_validation/tests/test_pseudo_cl.py index 6fc43d55..b7ac16b8 100644 --- a/src/sp_validation/tests/test_pseudo_cl.py +++ b/src/sp_validation/tests/test_pseudo_cl.py @@ -57,7 +57,9 @@ import pytest import yaml +from sp_validation import sacc_io from sp_validation.cosmo_val import CosmologyValidation +from sp_validation.cosmo_val.sacc_writers import BIN as SACC_BIN from sp_validation.rho_tau import get_params_rho_tau # These tests need the full harmonic-space stack (pymaster/NaMaster + healpy), @@ -509,24 +511,24 @@ def test_apply_random_rotation_reproducible_with_seed(cv, cat_and_params): # calculate_pseudo_cl_catalog -- deterministic end-to-end catalog path # =========================================================================== def test_calculate_pseudo_cl_catalog_end_to_end(cv, tmp_path): - """End-to-end catalog path: FITS round-trip of ell + EE/EB/BB. + """End-to-end catalog path: SACC round-trip of ell + EE/EB/BB. The catalog method has no random noise debiasing, so it is reproducible to - the same ~2e-12 catalog-path float noise. save_pseudo_cl stores ELL/EE/EB/BB - (it drops the BE row); we pin the round-tripped table. + the same ~2e-12 catalog-path float noise. calculate_pseudo_cl_catalog is + born-as-SACC: it writes a pseudo-Cl part (EE/BB/EB + shared bandpower + window) via pseudo_cl_to_sacc_part; we pin the round-tripped spectra read + back through sacc_io.get_pseudo_cl. """ ver = cv._test_version cv._pseudo_cls = {ver: {}} - out_path = cv._output_path(f"pseudo_cl_cat_{ver}.fits") + out_path = cv._output_path(f"pseudo_cl_{ver}.sacc") cv.calculate_pseudo_cl_catalog(ver, out_path) assert os.path.exists(out_path) - d = fits.getdata(out_path) - # FITS gives big-endian f8; normalize for value comparison. - ell = np.asarray(d["ELL"], dtype=np.float64) - ee = np.asarray(d["EE"], dtype=np.float64) - eb = np.asarray(d["EB"], dtype=np.float64) - bb = np.asarray(d["BB"], dtype=np.float64) + s = sacc_io.load(out_path) + ell, ee, bb, eb, window = sacc_io.get_pseudo_cl(s, SACC_BIN) + # A shared BandpowerWindow rides the part per the SACC layout contract. + assert window is not None npt.assert_allclose( ell, From 0f4a6f2f23d8770ed0bbeed1e9629a83e1e3f3d5 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 08:10:51 +0200 Subject: [PATCH 16/47] fix(workflow): wire the assemble_sacc rule and the per-statistic SACC parts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The born-as-SACC parts existed but nothing assembled them. Wire the DAG: per-statistic parts (xi_coarse, pseudo_cl, cosebis, pure_eb, rho_tau) → assemble_sacc rule → terminal {version}.sacc. - rule xi: give the coarse ξ± .sacc output the same reporting-binning wildcards as the .txt (Snakemake requires one wildcard set per rule; the bare {version}_xi_coarse.sacc name left the binning wildcards unbound and broke DAG resolution). run_2pcf.py writes the part to the declared output path. - rule rho_tau_stats: declare the rho_tau .sacc part (already written by calculate_rho_tau_stats) as a real output; run_rho_tau.py verifies it. - new rule assemble_sacc + assemble_sacc_all: load the five parts in canonical order via assemble_sacc.py, injecting a documented diagonal placeholder for the cov-less ξ/pseudo-Cℓ blocks (real CosmoCov/NaMaster covariance plugs into the same --xi-cov/--pseudo-cl-cov seam later — PR-3 converter territory). - cosmo_val_all: request the terminal {version}.sacc per version. - cv_pseudo_cl.py docstring: born-as-SACC, no longer FITS. Dry-runs clean for assemble_sacc_all, cosmo_val_all, bmode_summary.json, and the fine-grid xi_fine target. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01HPfbe2XVTGzbnPn4g7BdN6 --- workflow/rules/cosmo_val.smk | 75 +++++++++++++++++++++++++++++++- workflow/rules/twopoint.smk | 14 ++++-- workflow/scripts/cv_pseudo_cl.py | 8 ++-- workflow/scripts/run_2pcf.py | 13 ++++-- workflow/scripts/run_rho_tau.py | 6 ++- 5 files changed, 103 insertions(+), 13 deletions(-) diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index 000b445a..03441872 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -127,8 +127,18 @@ def cv_rho_tau_sacc(version): def cv_xi_coarse_sacc(version): - """Coarse ξ± SACC part the xi rule (run_2pcf.py) writes for a version.""" - return str(COSMO_VAL / f"{version}_xi_coarse.sacc") + """Coarse ξ± SACC part the xi rule (run_2pcf.py) writes for a version. + + Carries the reporting-binning suffix so requesting it binds the xi job's + wildcards (the rule's txt + coarse .sacc outputs share one wildcard set). + """ + return str( + COSMO_VAL + / ( + f"{version}_xi_coarse_minsep={CV['theta_min']}_maxsep={CV['theta_max']}" + f"_nbins={CV['nbins']}_npatch={CV['npatch']}.sacc" + ) + ) def cv_analysis_sacc(version): @@ -401,6 +411,65 @@ rule cv_summarize_bmodes: "../scripts/cv_summarize_bmodes.py" +# --------------------------------------------------------------------------- +# Terminal analysis file: assemble the per-statistic SACC parts into {version}.sacc +# --------------------------------------------------------------------------- +# The five born-as-SACC parts (xi_coarse, pseudo_cl, cosebis, pure_eb, rho_tau) +# are each written by their own rule carrying its own covariance block, except +# ξ± coarse and pseudo-Cℓ which are born cov-less by design. assemble_sacc.py +# loads the parts in canonical order and rebuilds one {version}.sacc with a +# single FullCovariance (point-insertion order = block order). The cov-less +# ξ/pseudo-Cℓ blocks are supplied here: the real CosmoCov / NaMaster covariances +# plug in via --xi-cov / --pseudo-cl-cov when the analysis needs them (that +# sourcing is the PR-3 converter's territory); until then a documented diagonal +# placeholder keeps the FullCovariance structurally valid. The placeholder is a +# flagged stand-in, never a science covariance. + + +def cv_assemble_inputs(version): + """The per-statistic SACC parts assemble_sacc consumes for a version. + + Each part's filename carries enough to bind its producing rule's wildcards + (the coarse ξ± part its reporting binning, the ρ/τ part likewise). pseudo_cl + is included only when the config toggles the harmonic-space BB into the + analysis. + """ + parts = dict( + xi_coarse=cv_xi_coarse_sacc(version), + cosebis=cv_cosebis_sacc(version), + pure_eb=cv_pure_eb_sacc(version), + rho_tau=cv_rho_tau_sacc(version), + ) + if CV.get("include_pseudo_cl", False): + parts["pseudo_cl"] = cv_pseudo_cl_sacc(version) + return parts + + +rule assemble_sacc: + """Assemble the terminal {version}.sacc from the per-statistic SACC parts.""" + input: + unpack(lambda w: cv_assemble_inputs(w.version)), + output: + sacc=cv_analysis_sacc("{version}"), + params: + version="{version}", + # Real ξ / pseudo-Cℓ covariance sourcing (CosmoCov / NaMaster) plugs in + # here later; for now a documented diagonal placeholder keeps the + # assembled FullCovariance structurally valid. + placeholder_var=1.0, + resources: + mem_mb=8000, + runtime=20, + script: + "../scripts/assemble_sacc.py" + + +rule assemble_sacc_all: + """Assemble the analysis SACC file for every version.""" + input: + [cv_analysis_sacc(v) for v in CV_VERSIONS], + + # --------------------------------------------------------------------------- # Aggregate target: the whole validation suite # --------------------------------------------------------------------------- @@ -422,3 +491,5 @@ rule cosmo_val_all: str(COSMO_VAL / "ratio_xi_sys_xi.png"), # B-modes str(COSMO_VAL / "bmode_summary.json"), + # Terminal analysis file: the assembled {version}.sacc per version + [cv_analysis_sacc(v) for v in CV_VERSIONS], diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index 1e23c6ce..c39553cb 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -6,10 +6,13 @@ rule xi: catalog=get_shear_catalog, output: # Raw TreeCorr .txt byproduct (read back by covariance + skip-if-exists) - # and the born-as-SACC coarse ξ± part (a .part — no covariance until the - # assemble_sacc rule injects the CosmoCov block). + # and the born-as-SACC coarse ξ± part (no covariance until the + # assemble_sacc rule injects the CosmoCov block). Both outputs carry the + # same reporting-binning wildcards — Snakemake requires every output of a + # rule to share one wildcard set, and it keeps the coarse .sacc name + # self-describing so requesting it binds the xi job unambiguously. txt=str(COSMO_VAL / "{version}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.txt"), - xi_coarse=str(COSMO_VAL / "{version}_xi_coarse.sacc"), + xi_coarse=str(COSMO_VAL / "{version}_xi_coarse_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.sacc"), threads: 24 params: ver="{version}", @@ -75,6 +78,11 @@ rule rho_tau_stats: output: rho_stats=str(COSMO_VAL / "rho_tau_stats/rho_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), tau_stats=str(COSMO_VAL / "rho_tau_stats/tau_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), + # Born-as-SACC ρ/τ part (ρ_0…ρ_5 autos + τ_0/τ_2/τ_5 leakage, carrying + # its own covariance block) that the assemble_sacc rule consumes; + # calculate_rho_tau_stats writes it alongside the FITS via + # rho_tau_to_sacc_part. + rho_tau=str(COSMO_VAL / "rho_tau_stats/rho_tau_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.sacc"), threads: 48 params: ver="{version}", diff --git a/workflow/scripts/cv_pseudo_cl.py b/workflow/scripts/cv_pseudo_cl.py index cf04e8e8..43923e4e 100644 --- a/workflow/scripts/cv_pseudo_cl.py +++ b/workflow/scripts/cv_pseudo_cl.py @@ -1,8 +1,10 @@ """Rule cv_pseudo_cl: harmonic-space pseudo-Cl B-mode spectra. -plot_pseudo_cl triggers calculate_pseudo_cl, which writes pseudo_cl_{version}.fits -for every version (the BB spectrum cv_summarize_bmodes reads) and the cell_ee.png -figure. The per-version FITS files are the declared outputs. +plot_pseudo_cl triggers calculate_pseudo_cl, which writes the born-as-SACC +pseudo_cl_{version}.sacc part for every version (EE/BB/EB with the shared +bandpower window — the BB spectrum cv_summarize_bmodes reads) and the +cell_ee.png figure. The per-version SACC parts are the declared outputs and +feed both cv_summarize_bmodes and the assemble_sacc rule. """ from cv_runner import _unbuffer_streams, make_cv, verify_outputs diff --git a/workflow/scripts/run_2pcf.py b/workflow/scripts/run_2pcf.py index 9381d788..a773d039 100644 --- a/workflow/scripts/run_2pcf.py +++ b/workflow/scripts/run_2pcf.py @@ -38,14 +38,18 @@ def run_2pcf( npatch, cat_config, output_dir, + sacc_out=None, ): - """Measure ξ±(θ) for ``ver`` and write its coarse SACC part under ``output_dir``. + """Measure ξ±(θ) for ``ver`` and write its coarse SACC part. Parameters mirror the TreeCorr reporting/integration grids: ``min_sep`` / ``max_sep`` in arcmin, ``nbins`` logarithmic bins, ``npatch`` spatial patches (1 for the paper fiducial). ``cat_config`` is an absolute path to the catalog configuration; ``output_dir`` overrides - ``cat_config['paths']['output']`` so products land where lc expects. + ``cat_config['paths']['output']`` so the ``.txt`` byproduct lands where lc + expects. ``sacc_out`` is the exact destination for the coarse ξ± SACC part + (the Snakemake-declared output); it defaults to ``{ver}_xi_coarse.sacc`` + under the resolved output directory for the CLI path. Returns ------- @@ -78,7 +82,7 @@ def run_2pcf( npairs=gg.npairs, weight=gg.weight, ) - out_path = os.path.join( + out_path = sacc_out or os.path.join( output_dir or cv.cc["paths"]["output"], f"{ver}_xi_coarse.sacc" ) sacc_io.save(s, out_path) @@ -100,6 +104,9 @@ def _from_snakemake(smk): # class defaults (./cat_config.yaml, COSMO_VAL env) otherwise. cat_config=p.get("cat_config", "./cat_config.yaml"), output_dir=p.get("output_dir", None), + # Write the SACC part exactly where the rule declares it (the .txt + # byproduct still lands under the resolved output dir via _output_path). + sacc_out=smk.output["xi_coarse"], ) diff --git a/workflow/scripts/run_rho_tau.py b/workflow/scripts/run_rho_tau.py index ea2f35bc..9df0bbfd 100644 --- a/workflow/scripts/run_rho_tau.py +++ b/workflow/scripts/run_rho_tau.py @@ -48,9 +48,11 @@ cv.calculate_rho_tau_stats() -# Confirm CosmologyValidation produced the requested outputs +# Confirm CosmologyValidation produced the requested outputs. calculate_rho_tau_stats +# writes the rho/tau FITS *and* the born-as-SACC rho_tau part (via +# rho_tau_to_sacc_part); the part feeds the assemble_sacc rule. outputs = snakemake.output # type: ignore -for label in ("rho_stats", "tau_stats"): +for label in ("rho_stats", "tau_stats", "rho_tau"): target = Path(outputs[label]) if not target.exists(): raise FileNotFoundError( From 99db96e14d8c5ce439051197bc3275d6efb64411 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 08:14:08 +0200 Subject: [PATCH 17/47] test(assemble_sacc): integration test for the assemble_sacc.py script seam MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pure assembler is covered in test_sacc_writers; this pins the DAG-facing script that loads per-statistic .sacc part *files* and rebuilds {version}.sacc: canonical block order across all five statistics, the diagonal placeholder for the cov-less ξ± part, real CosmoCov ξ covariance .txt injection, the fail-loud path when a required covariance is absent, and the pseudo_cl config toggle. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01HPfbe2XVTGzbnPn4g7BdN6 --- src/sp_validation/tests/test_assemble_sacc.py | 206 ++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100644 src/sp_validation/tests/test_assemble_sacc.py diff --git a/src/sp_validation/tests/test_assemble_sacc.py b/src/sp_validation/tests/test_assemble_sacc.py new file mode 100644 index 00000000..a14d3f06 --- /dev/null +++ b/src/sp_validation/tests/test_assemble_sacc.py @@ -0,0 +1,206 @@ +"""Integration tests for the ``assemble_sacc.py`` workflow script. + +The pure assembler (``sacc_writers.assemble_analysis_sacc``) is covered in +``test_sacc_writers.py``. This file exercises the *script seam* the DAG uses: +``assemble_sacc.assemble_sacc`` loads per-statistic ``.sacc`` part *files* in +CANONICAL order, injects the born-cov-less ξ± / pseudo-Cℓ blocks (real CosmoCov +/ NaMaster covariance, or a flagged diagonal placeholder), and writes one +``{version}.sacc`` whose points and covariance blocks land in canonical order. + +The script lives under ``workflow/scripts`` (off the package path); it is loaded +by file path exactly as the lightcone/ASTRA CLI path imports it. +""" + +import importlib.util +from pathlib import Path + +import numpy as np +import pytest + +from sp_validation import sacc_io as sio +from sp_validation.cosmo_val import sacc_writers as sw + + +def _load_assemble_module(): + """Import ``workflow/scripts/assemble_sacc.py`` by file path.""" + repo_root = next( + p for p in Path(__file__).resolve().parents if (p / "pyproject.toml").exists() + ) + path = repo_root / "workflow" / "scripts" / "assemble_sacc.py" + spec = importlib.util.spec_from_file_location("assemble_sacc", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +asm = _load_assemble_module() + + +def _nz(seed=0, n=40): + rng = np.random.default_rng(seed) + return np.linspace(0.01, 2.0, n), rng.uniform(0.1, 1.0, n) + + +def _spd(n, seed): + a = np.random.default_rng(seed).normal(size=(n, n)) + return a @ a.T + n * np.eye(n) + + +def _theta(n=6): + return np.geomspace(1.0, 100.0, n) + + +META = {"catalogue_version": "vSYNTH", "npatch": 1} + + +def _write_parts(tmp_path, *, with_pseudo_cl=True, cov_less=("xi_coarse",)): + """Write per-statistic parts to disk; return the ``{name: path}`` mapping. + + Parts named in ``cov_less`` are written without a covariance (mimicking the + born-cov-less ξ± coarse / pseudo-Cℓ parts); the rest carry their own block. + """ + nz = {0: _nz()} + theta = _theta() + ell = np.array([30.0, 60.0, 90.0]) + + class _Wsp: + def get_bandpower_windows(self): + w = np.zeros((4, 3, 4, 20)) + for out in range(4): + for b in range(3): + w[out, b, out, b * 6 : b * 6 + 6] = 1.0 + return w + + xi = sw.xi_to_sacc( + nz, META, theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + if "xi_coarse" not in cov_less: + xi.add_covariance(_spd(len(xi.mean), 1)) + + cl_all = np.vstack( + [np.arange(3) * 1e-9, np.arange(3) * 2e-9, np.zeros(3), np.arange(3) * 3e-9] + ) + cl = sw.pseudo_cl_to_sacc( + nz, + META, + ell, + cl_all, + _Wsp(), + covariance=None if "pseudo_cl" in cov_less else _spd(9, 2), + ) + + co = sw.cosebis_to_sacc( + nz, + META, + { + "En": np.arange(1, 6) * 1e-6, + "Bn": np.arange(1, 6) * 1e-7, + "cov": _spd(10, 3), + }, + (1.0, 100.0), + ) + + eb_arrays = { + key: np.arange(6) * (i + 1) * 1e-6 for i, key in enumerate(sio.PURE_KEYS) + } + eb = sw.pure_eb_to_sacc(nz, META, theta, eb_arrays, covariance=_spd(36, 4)) + + rho = {"theta": theta} + tau = {"theta": theta} + rng = np.random.default_rng(5) + for k in sw.RHO_K: + for suffix in ("p", "m"): + rho[f"rho_{k}_{suffix}"] = rng.normal(size=6) * 1e-6 + rho[f"varrho_{k}_{suffix}"] = rng.uniform(1e-14, 1e-13, 6) + for k in sw.TAU_K: + for suffix in ("p", "m"): + tau[f"tau_{k}_{suffix}"] = rng.normal(size=6) * 1e-6 + tau[f"vartau_{k}_{suffix}"] = rng.uniform(1e-14, 1e-13, 6) + rt = sw.rho_tau_to_sacc(nz, META, rho, tau) + + parts = { + "xi_coarse": xi, + "pseudo_cl": cl, + "cosebis": co, + "pure_eb": eb, + "rho_tau": rt, + } + if not with_pseudo_cl: + parts.pop("pseudo_cl") + + paths = {} + for name, part in parts.items(): + p = tmp_path / f"{name}.sacc" + sio.save(part, str(p)) + paths[name] = str(p) + return paths + + +def test_assemble_sacc_placeholder_canonical_order(tmp_path): + """The cov-less ξ± part gets a placeholder; every point is covered and the + blocks land in canonical order (ξ±, pseudo-Cℓ, COSEBIs, pure-E/B, ρ, τ).""" + paths = _write_parts(tmp_path, cov_less=("xi_coarse",)) + out = tmp_path / "vSYNTH.sacc" + s = asm.assemble_sacc("vSYNTH", paths, str(out), placeholder_var=1.0) + assert out.exists() + assert type(s.covariance).__name__ == "FullCovariance" + assert s.covariance.dense.shape == (len(s.mean), len(s.mean)) + + # Canonical insertion order: the first data types are ξ+ then ξ−. + types_in_order = [dp.data_type for dp in s.data] + assert types_in_order[0] == sio.XI_PLUS + assert sio.XI_MINUS in types_in_order + # ξ appears before pseudo-Cℓ before COSEBIs before pure-E/B before ρ/τ. + first = {t: types_in_order.index(t) for t in set(types_in_order)} + assert first[sio.XI_PLUS] < first[sio.CL_EE] < first[sio.COSEBI_EE] + assert first[sio.COSEBI_EE] < first[sio.PURE_TYPES["xip_E"]] + assert first[sio.PURE_TYPES["xip_E"]] < first[sio.RHO_PLUS.format(k=0)] + assert first[sio.RHO_PLUS.format(k=0)] < first[sio.TAU_PLUS.format(k=0)] + + # The ξ± block is the placeholder diagonal (variance 1.0 on its own points). + tr = ("source_0", "source_0") + xi_idx = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + dense = s.covariance.dense + assert np.allclose(np.diag(dense[np.ix_(xi_idx, xi_idx)]), 1.0) + # ...and it does not bleed into the neighbouring COSEBIs block (cross zero). + co_idx = np.concatenate( + [s.indices(sio.COSEBI_EE, tr), s.indices(sio.COSEBI_BB, tr)] + ) + assert np.allclose(dense[np.ix_(xi_idx, co_idx)], 0.0) + + +def test_assemble_sacc_injects_real_xi_covariance(tmp_path): + """A CosmoCov ξ covariance .txt is loaded into the cov-less ξ± block.""" + paths = _write_parts(tmp_path, cov_less=("xi_coarse",)) + # ξ± part has 12 points ([ξ+; ξ−] over 6 θ); supply a matching cov .txt. + xi_cov = _spd(12, 21) + cov_path = tmp_path / "xi_cov.txt" + np.savetxt(str(cov_path), xi_cov) + + out = tmp_path / "vSYNTH.sacc" + s = asm.assemble_sacc("vSYNTH", paths, str(out), xi_cov=str(cov_path)) + tr = ("source_0", "source_0") + xi_idx = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + assert np.allclose(s.covariance.dense[np.ix_(xi_idx, xi_idx)], xi_cov) + + +def test_assemble_sacc_missing_cov_raises(tmp_path): + """A cov-less part with no injected block and no placeholder fails loudly.""" + paths = _write_parts(tmp_path, cov_less=("xi_coarse",)) + out = tmp_path / "vSYNTH.sacc" + with pytest.raises(ValueError, match="carries no covariance"): + asm.assemble_sacc("vSYNTH", paths, str(out)) + + +def test_assemble_sacc_respects_pseudo_cl_toggle(tmp_path): + """With pseudo_cl absent, assembly still succeeds and omits the Cℓ points.""" + paths = _write_parts(tmp_path, with_pseudo_cl=False, cov_less=("xi_coarse",)) + assert "pseudo_cl" not in paths + out = tmp_path / "vSYNTH.sacc" + s = asm.assemble_sacc("vSYNTH", paths, str(out), placeholder_var=1.0) + tr = ("source_0", "source_0") + assert len(s.indices(sio.CL_EE, tr)) == 0 + # Round-trips as a valid FullCovariance over the remaining points. + s2 = sio.load(str(out)) + assert type(s2.covariance).__name__ == "FullCovariance" + assert s2.covariance.dense.shape == (len(s2.mean), len(s2.mean)) From 9afaeaa5d127d5fbb4ddc28b87a327762db5e24b Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 08:24:56 +0200 Subject: [PATCH 18/47] fix(workflow): resolve xi_highres script path from the running checkout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The MPI xi_highres rule shells out to run_2pcf_highres.py by absolute path through the deprecated pure_eb/ compat symlink (it can't use Snakemake's script: directive — mpiexec wraps the python call). Anchor the path on common.py's own location (WORKFLOW_SCRIPTS), which resolves to the generic workflow/scripts of whatever checkout parses the DAG, regardless of which paper composes it — workflow.basedir is unreliable here (under module composition it reflects the composing paper, verified: it resolved to papers/cosmo_val/scripts). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01HPfbe2XVTGzbnPn4g7BdN6 --- workflow/common.py | 10 ++++++++++ workflow/rules/twopoint.smk | 4 +++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/workflow/common.py b/workflow/common.py index 3df3413b..17e60ce7 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -5,6 +5,16 @@ import re from pathlib import Path +# Absolute path to the generic workflow's scripts, anchored on this module's own +# location (common.py lives in workflow/, is `from common import *`'d into every +# Snakefile, and so resolves to the generic workflow dir of the running checkout +# regardless of which paper composes it — unlike workflow.basedir, which under +# `module` composition reflects the composing paper). Rules that shell out to a +# script directly (the MPI xi_highres run can't go through Snakemake's `script:` +# directive) interpolate this instead of a hardcoded pure_eb/ compat-symlink +# path. /automnt/n17data is the automount of the container-bound /n17data. +WORKFLOW_SCRIPTS = os.path.join(os.path.dirname(os.path.realpath(__file__)), "scripts") + # Output roots are env-overridable so a reproduction run can write into a # fresh tree without clobbering (or silently reusing) prior products. COSMO_VAL = Path( diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index c39553cb..ebd19f91 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -1,4 +1,6 @@ # Two-point data-vector rules: xi, rho/tau, and pseudo-Cl products. +# WORKFLOW_SCRIPTS (from common.py) is the generic workflow's scripts dir, +# resolved from the running checkout — used by the raw-shell MPI xi_highres rule. rule xi: @@ -53,7 +55,7 @@ rule xi_highres: "--bind /home,/n09data,/n17data,/n23data1,/softs " "--env LD_LIBRARY_PATH=/softs/openmpi/5.0.5-slurm-CentOS8/lib " "/n17data/cdaley/containers/containers " - "python /automnt/n17data/cdaley/unions/pure_eb/code/sp_validation/workflow/scripts/run_2pcf_highres.py" + f"python {WORKFLOW_SCRIPTS}/run_2pcf_highres.py" rule run_cosmo_val: From 00460d858fbc7f2d21fc777ba4b08459d72185d1 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 08:25:14 +0200 Subject: [PATCH 19/47] fix(workflow): assemble the analysis file from the tagged pseudo-Cl + real cov MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The analysis {version}.sacc must be byte-comparable against today's cosmosis_fitting.py assembly (the SACC layout contract's stated requirement for PR-3's converter). cosmosis consumes the tagged, blinded pseudo-Cl product (blind=A, powspace, nbins=32 per config harmonic.fiducial), not the untagged cv_pseudo_cl diagnostic — so assemble_sacc now inputs the tagged part and injects its real NaMaster covariance from the matching pseudo_cl_cov FITS (COVAR_EE_EE/BB_BB/EB_EB → block-diagonal). The untagged cv_pseudo_cl part stays the cv_summarize_bmodes B-mode diagnostic, unchanged. The ξ± coarse block keeps a documented diagonal placeholder: its real CosmoCov covariance is blind/gaussian/mask-keyed in the inference tree, and wiring it couples cosmo_val to the whole inference covariance DAG — that sourcing is PR-3 converter territory, ready at the --xi-cov seam. test_assemble_sacc: add the pseudo-Cl COVAR-FITS injection path (the live default) alongside the existing ξ .txt and placeholder paths. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01HPfbe2XVTGzbnPn4g7BdN6 --- src/sp_validation/tests/test_assemble_sacc.py | 36 ++++++++++ workflow/rules/cosmo_val.smk | 68 +++++++++++++++---- 2 files changed, 89 insertions(+), 15 deletions(-) diff --git a/src/sp_validation/tests/test_assemble_sacc.py b/src/sp_validation/tests/test_assemble_sacc.py index a14d3f06..a43747dc 100644 --- a/src/sp_validation/tests/test_assemble_sacc.py +++ b/src/sp_validation/tests/test_assemble_sacc.py @@ -184,6 +184,42 @@ def test_assemble_sacc_injects_real_xi_covariance(tmp_path): assert np.allclose(s.covariance.dense[np.ix_(xi_idx, xi_idx)], xi_cov) +def test_assemble_sacc_injects_pseudo_cl_covariance(tmp_path): + """The NaMaster cov FITS (COVAR_EE_EE/BB_BB/EB_EB) → block-diagonal pseudo-Cℓ + block (the live default: ξ± placeholder + real pseudo-Cℓ cov).""" + from astropy.io import fits + + paths = _write_parts(tmp_path, cov_less=("xi_coarse", "pseudo_cl")) + # pseudo-Cℓ part is 3 ell × {EE, BB, EB} = 9 points; per-spectrum 3×3 blocks. + ee, bb, eb = _spd(3, 31), _spd(3, 32), _spd(3, 33) + cov_fits = tmp_path / "pseudo_cl_cov.fits" + fits.HDUList( + [ + fits.PrimaryHDU(), + fits.ImageHDU(ee, name="COVAR_EE_EE"), + fits.ImageHDU(bb, name="COVAR_BB_BB"), + fits.ImageHDU(eb, name="COVAR_EB_EB"), + ] + ).writeto(str(cov_fits)) + + out = tmp_path / "vSYNTH.sacc" + s = asm.assemble_sacc( + "vSYNTH", paths, str(out), pseudo_cl_cov=str(cov_fits), placeholder_var=1.0 + ) + tr = ("source_0", "source_0") + cl_idx = np.concatenate( + [s.indices(sio.CL_EE, tr), s.indices(sio.CL_BB, tr), s.indices(sio.CL_EB, tr)] + ) + dense = s.covariance.dense + expected = np.zeros((9, 9)) + expected[0:3, 0:3], expected[3:6, 3:6], expected[6:9, 6:9] = ee, bb, eb + assert np.allclose(dense[np.ix_(cl_idx, cl_idx)], expected) + # ξ± stays the placeholder; the two blocks don't bleed into each other. + xi_idx = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + assert np.allclose(np.diag(dense[np.ix_(xi_idx, xi_idx)]), 1.0) + assert np.allclose(dense[np.ix_(xi_idx, cl_idx)], 0.0) + + def test_assemble_sacc_missing_cov_raises(tmp_path): """A cov-less part with no injected block and no placeholder fails loudly.""" paths = _write_parts(tmp_path, cov_less=("xi_coarse",)) diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index 03441872..3fe0e1df 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -105,10 +105,38 @@ def cv_cosebis_npz(version): def cv_pseudo_cl_sacc(version): - """Pseudo-Cl SACC part calculate_pseudo_cl writes (born as SACC).""" + """Untagged pseudo-Cl SACC part cv_pseudo_cl writes (B-mode diagnostic). + + This is the harmonic-space BB diagnostic cv_summarize_bmodes reads. The + *analysis* file's pseudo-Cl part is the tagged, blinded inference product + instead (see cv_pseudo_cl_analysis_sacc) so {version}.sacc stays byte- + comparable against today's cosmosis_fitting.py assembly (PR-3's converter). + """ return str(COSMO_VAL / f"pseudo_cl_{version}.sacc") +# Fiducial harmonic-binning tag the pseudo-Cl producer (twopoint.smk rules +# pseudo_cl / pseudo_cl_cov) stamps into the analysis-grade filename. Mirrors +# inference.smk's PSEUDO_CL_TAG so the analysis file carries the same pseudo-Cl +# the inference pipeline consumes (canonical: blind=A, powspace, nbins=32). +_HARMONIC_FIDUCIAL = config["harmonic"]["fiducial"] +_PSEUDO_CL_TAG = ( + f"blind={_HARMONIC_FIDUCIAL['blind']}" + f"_{_HARMONIC_FIDUCIAL['binning']}" + f"_nbins={_HARMONIC_FIDUCIAL['nbins']}" +) + + +def cv_pseudo_cl_analysis_sacc(version): + """Tagged, blinded pseudo-Cl SACC part the analysis file carries.""" + return str(COSMO_VAL / f"pseudo_cl_{version}_{_PSEUDO_CL_TAG}.sacc") + + +def cv_pseudo_cl_cov(version): + """NaMaster pseudo-Cl covariance FITS (COVAR_EE_EE/BB_BB/EB_EB extensions).""" + return str(COSMO_VAL / f"pseudo_cl_cov_{version}_{_PSEUDO_CL_TAG}.fits") + + def cv_cosebis_sacc(version): """COSEBIs SACC part (fiducial scale cut) the cv_cosebis rule writes.""" return str(COSMO_VAL / f"{version}_cosebis.sacc") @@ -418,21 +446,29 @@ rule cv_summarize_bmodes: # are each written by their own rule carrying its own covariance block, except # ξ± coarse and pseudo-Cℓ which are born cov-less by design. assemble_sacc.py # loads the parts in canonical order and rebuilds one {version}.sacc with a -# single FullCovariance (point-insertion order = block order). The cov-less -# ξ/pseudo-Cℓ blocks are supplied here: the real CosmoCov / NaMaster covariances -# plug in via --xi-cov / --pseudo-cl-cov when the analysis needs them (that -# sourcing is the PR-3 converter's territory); until then a documented diagonal -# placeholder keeps the FullCovariance structurally valid. The placeholder is a -# flagged stand-in, never a science covariance. +# single FullCovariance (point-insertion order = block order). +# +# The pseudo-Cℓ part is the TAGGED, blinded inference product (blind=A, powspace, +# nbins=32) — the same pseudo-Cℓ today's cosmosis_fitting.py consumes — so the +# analysis file stays byte-comparable against it (PR-3's converter). Its real +# NaMaster covariance is injected here from the matching pseudo_cl_cov FITS +# (COVAR_EE_EE/BB_BB/EB_EB → block-diagonal, dropping cross-spectra, matching the +# B-mode PTE's use of COVAR_BB_BB). The ξ± coarse block is the one piece not yet +# sourced from its real covariance: the CosmoCov theory .txt is blind/gaussian/ +# mask-keyed and lives deep in the inference tree, so wiring it couples cosmo_val +# to the whole inference covariance DAG — that sourcing is PR-3's converter +# territory. Until then a documented diagonal placeholder keeps the ξ block (and +# so the FullCovariance) structurally valid; it is a flagged stand-in, never a +# science covariance, and plugs out via --xi-cov the moment PR 3 lands. def cv_assemble_inputs(version): - """The per-statistic SACC parts assemble_sacc consumes for a version. + """The per-statistic SACC parts + covariance inputs assemble_sacc consumes. Each part's filename carries enough to bind its producing rule's wildcards - (the coarse ξ± part its reporting binning, the ρ/τ part likewise). pseudo_cl - is included only when the config toggles the harmonic-space BB into the - analysis. + (the coarse ξ± and ρ/τ parts their reporting binning; the pseudo-Cℓ part its + fiducial harmonic tag). pseudo_cl (+ its cov) is included only when the + config toggles the harmonic-space BB into the analysis. """ parts = dict( xi_coarse=cv_xi_coarse_sacc(version), @@ -441,7 +477,8 @@ def cv_assemble_inputs(version): rho_tau=cv_rho_tau_sacc(version), ) if CV.get("include_pseudo_cl", False): - parts["pseudo_cl"] = cv_pseudo_cl_sacc(version) + parts["pseudo_cl"] = cv_pseudo_cl_analysis_sacc(version) + parts["pseudo_cl_cov"] = cv_pseudo_cl_cov(version) return parts @@ -453,9 +490,10 @@ rule assemble_sacc: sacc=cv_analysis_sacc("{version}"), params: version="{version}", - # Real ξ / pseudo-Cℓ covariance sourcing (CosmoCov / NaMaster) plugs in - # here later; for now a documented diagonal placeholder keeps the - # assembled FullCovariance structurally valid. + # ξ± coarse block: documented diagonal placeholder until PR-3's converter + # sources the real CosmoCov theory covariance via --xi-cov. The pseudo-Cℓ + # block is real (pseudo_cl_cov input); COSEBIs / pure-E/B / ρ/τ carry + # their own. assemble_sacc.py reads pseudo_cl_cov's COVAR_* extensions. placeholder_var=1.0, resources: mem_mb=8000, From 2d1747b6839690d6b80fc555a85fb47cb6c1ffba Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 08:26:11 +0200 Subject: [PATCH 20/47] test(workflow): dry-run guard uses sys.executable, drops SNAKEMAKE_PROFILE test_bmodes_workflow_dry_runs invoked a bare python3.12 (resolves off PATH to intel-python without snakemake) and inherited the login shell's slurm SNAKEMAKE_PROFILE (forces an executor plugin the test env need not have). Use sys.executable (the interpreter pytest/snakemake live in) and drop the profile for the dry-run. The guard now passes and exercises the migrated rule graph (rule xi's coarse-sacc output, the WORKFLOW_SCRIPTS path). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01HPfbe2XVTGzbnPn4g7BdN6 --- .../tests/test_bmodes_workflow_dry_run.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/sp_validation/tests/test_bmodes_workflow_dry_run.py b/src/sp_validation/tests/test_bmodes_workflow_dry_run.py index 68c09981..60675f52 100644 --- a/src/sp_validation/tests/test_bmodes_workflow_dry_run.py +++ b/src/sp_validation/tests/test_bmodes_workflow_dry_run.py @@ -6,6 +6,7 @@ import os import subprocess +import sys from pathlib import Path import pytest @@ -31,11 +32,18 @@ def test_bmodes_workflow_dry_runs(): """The paper B-mode workflow must still parse and dry-run cleanly.""" workflow_dir = _repo_root() / "papers/bmodes" # PYTHONUNBUFFERED satisfies the Snakefile's `envvars:` declaration without - # depending on the invoking shell's environment. + # depending on the invoking shell's environment. A dry run resolves the DAG + # only — it never dispatches jobs — so drop any inherited SNAKEMAKE_PROFILE + # (e.g. the login shell's "slurm" profile), which would otherwise force an + # executor plugin the test environment need not have installed. env = os.environ | {"PYTHONNOUSERSITE": "1", "PYTHONUNBUFFERED": "1"} + env.pop("SNAKEMAKE_PROFILE", None) result = subprocess.run( [ - "python3.12", + # Invoke snakemake through the interpreter running the test — a bare + # "python3.12" resolves off PATH (e.g. intel-python without snakemake); + # sys.executable is the environment that pytest, hence snakemake, lives in. + sys.executable, "-m", "snakemake", "all_tapestry", From a592bb8350d7d9761d75d9f79e1e10e00feabc9a Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 08:31:26 +0200 Subject: [PATCH 21/47] docs(inference): mark inference_prep inputs pre-SACC, migration deferred to PR 7 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SACC migration removed the xi_plus/xi_minus FITS and switched pseudo_cl to .sacc, so inference_prep's DAG no longer resolves. It is not reachable from the cosmo_val suite (cosmo_val_all never requests it), so the cosmo_val DAG stays clean. Per the PR-4 scope this subsystem is left dormant — a comment block at the rule head names the stale inputs and states PR 7 (native-SACC inference consumption) rewires it to consume the assembled {version}.sacc directly, retiring cosmosis_fitting.py's per-product FITS assembly. Comment-only; no behavior change. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01HPfbe2XVTGzbnPn4g7BdN6 --- workflow/rules/inference.smk | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/workflow/rules/inference.smk b/workflow/rules/inference.smk index a93c0e71..6bf68645 100644 --- a/workflow/rules/inference.smk +++ b/workflow/rules/inference.smk @@ -55,11 +55,26 @@ def pseudo_cl_assets(version): cov_path = PSEUDO_CL_DIR / f"pseudo_cl_cov_{version}_{PSEUDO_CL_TAG}.fits" return str(cl_path), str(cov_path) +# --------------------------------------------------------------------------- +# DORMANT — pre-SACC cosmosis assembly. Migration to native SACC deferred to +# PR 7 (native-SACC inference consumption); do NOT deep-migrate here. +# +# The SACC migration (PR 4) removed the data products several of these inputs +# name, so this rule's DAG no longer resolves and is NOT reachable from the +# cosmo_val suite (cosmo_val_all never requests it). Stale inputs: +# - xi_plus / xi_minus FITS: the `xi` rule now emits the coarse ξ± SACC part +# ({version}_xi_coarse_...sacc), not per-sign FITS. +# - pseudo_cl / pseudo_cl_cov via pseudo_cl_assets(): the `pseudo_cl` rule now +# writes .sacc (pseudo_cl_assets still requests .fits). +# PR 7 rewires this to consume the assembled {version}.sacc (built by +# cosmo_val.smk's assemble_sacc rule) directly, retiring cosmosis_fitting.py's +# per-product FITS assembly. Until then the inference target is knowingly red. +# --------------------------------------------------------------------------- rule inference_prep: input: # Processed covariance matrix - use centralized covariance_path() cov_matrix=lambda w: covariance_path(w.version, w.blind, min_sep=w.min_sep, max_sep=w.max_sep, nbins=w.nbins), - # Xi FITS files + # Xi FITS files — PRE-SACC (no longer produced; see dormant note above) xi_plus=str(COSMO_VAL / "xi_plus_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), xi_minus=str(COSMO_VAL / "xi_minus_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), # n(z) file (using new location with base version mapping) @@ -69,6 +84,7 @@ rule inference_prep: tau_stats=str(COSMO_VAL / "rho_tau_stats/tau_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), # tau covariance (tracked as dependency) tau_cov=str(COSMO_VAL / "rho_tau_stats/cov_tau_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}_th.npy"), + # pseudo_cl / pseudo_cl_cov — PRE-SACC (.fits path; producer now writes .sacc) pseudo_cl=lambda w: pseudo_cl_assets(w.version)[0], pseudo_cl_cov=lambda w: pseudo_cl_assets(w.version)[1], output: From 60b74c60f6bc08df07e3156bfa89bdaf614e87c1 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 12:19:14 +0200 Subject: [PATCH 22/47] fix(pseudo_cl): born-at-declared-name kills the cross-rule pseudo-Cl collision MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both pseudo-Cl rules call calculate_pseudo_cl, which hardcoded the untagged native pseudo_cl_{ver}.sacc; generate_pseudo_cl.py then renamed it to the tagged name. In one cosmo_val_all DAG both fire: if the untagged cv_pseudo_cl (blind= None) ran first, the tagged rule's (blind=A) skip-if-exists silently adopted the blind=None file and the rename deleted cv_pseudo_cl's declared output — rebuild loops, and wrong-blind n(z) stamped into the terminal {version}.sacc's pseudo-Cl part. Thread out_path through calculate_pseudo_cl so each part is born directly at its final declared name (tagged for the producer, untagged for the diagnostic). No shared native basename, no rename; skip-if-exists keys on the declared path, so the two rules' paths are provably disjoint and the blind that computes each file matches its name. Multi-version + out_path now raises. generate_pseudo_cl.py and its CLI updated to pass the declared/native out_path directly. Regression tests: out_path is honoured (native name untouched) and the multi-version guard fires. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01HPfbe2XVTGzbnPn4g7BdN6 --- src/sp_validation/cosmo_val/pseudo_cl.py | 45 +++++++++++++++------ src/sp_validation/tests/test_pseudo_cl.py | 29 ++++++++++++++ workflow/scripts/generate_pseudo_cl.py | 48 ++++++++++++----------- 3 files changed, 87 insertions(+), 35 deletions(-) diff --git a/src/sp_validation/cosmo_val/pseudo_cl.py b/src/sp_validation/cosmo_val/pseudo_cl.py index 52d82849..29ff7846 100644 --- a/src/sp_validation/cosmo_val/pseudo_cl.py +++ b/src/sp_validation/cosmo_val/pseudo_cl.py @@ -454,20 +454,37 @@ def calculate_pseudo_cl_g_ng_cov(self, gaussian_part="iNKA"): f"Done Gaussian and Non-Gaussian covariance of the Pseudo-Cl's using {gaussian_part} for the Gaussian part" ) - def calculate_pseudo_cl(self): + def calculate_pseudo_cl(self, out_path=None): """ Compute the pseudo-Cl of given catalogs. - Each version's spectra are born as a SACC part (``pseudo_cl_{ver}.sacc``) - via :func:`sacc_writers.pseudo_cl_to_sacc` — EE/BB/EB carrying the shared - NaMaster bandpower window. The in-memory ``self._pseudo_cls[ver]`` - ``"pseudo_cl"`` entry keeps the ``ELL``/``EE``/``EB``/``BB`` arrays the - plotting and B-mode-summary consumers read by column name. + Each version's spectra are born as a SACC part via + :func:`sacc_writers.pseudo_cl_to_sacc` — EE/BB/EB carrying the shared + NaMaster bandpower window, with this instance's (blinded) n(z) stamped + in. The in-memory ``self._pseudo_cls[ver]`` ``"pseudo_cl"`` entry keeps + the ``ELL``/``EE``/``EB``/``BB`` arrays the plotting and B-mode-summary + consumers read by column name. + + ``out_path`` is the exact destination the part is *born at* — the + Snakemake-declared output. It must resolve per version; single-version + rules (the tagged blinded producer) pass their tagged output directly. + When ``None`` (multi-version diagnostic / the ``pseudo_cls`` property) + each part defaults to the untagged native ``pseudo_cl_{ver}.sacc``. + Skip-if-exists keys on this final path, so no two rules ever share an + undeclared native basename (a tagged product born at its native name and + then renamed would let one rule's skip-if-exists silently adopt — and + the rename delete — another rule's declared, differently-blinded file). """ self.print_start("Computing pseudo-Cl's") nside = self.nside + if out_path is not None and len(self.versions) != 1: + raise ValueError( + "calculate_pseudo_cl(out_path=...) writes one part to one path, " + f"but {len(self.versions)} versions are configured; call per version" + ) + try: self._pseudo_cls except AttributeError: @@ -477,14 +494,18 @@ def calculate_pseudo_cl(self): self._pseudo_cls[ver] = {} - out_path = self._output_path(f"pseudo_cl_{ver}.sacc") - if os.path.exists(out_path): - self.print_done(f"Skipping Pseudo-Cl's calculation, {out_path} exists") - self._pseudo_cls[ver]["pseudo_cl"] = self._load_pseudo_cl_sacc(out_path) + ver_out_path = out_path or self._output_path(f"pseudo_cl_{ver}.sacc") + if os.path.exists(ver_out_path): + self.print_done( + f"Skipping Pseudo-Cl's calculation, {ver_out_path} exists" + ) + self._pseudo_cls[ver]["pseudo_cl"] = self._load_pseudo_cl_sacc( + ver_out_path + ) elif self.cell_method == "map": - self.calculate_pseudo_cl_map(ver, nside, out_path) + self.calculate_pseudo_cl_map(ver, nside, ver_out_path) elif self.cell_method == "catalog": - self.calculate_pseudo_cl_catalog(ver, out_path) + self.calculate_pseudo_cl_catalog(ver, ver_out_path) else: raise ValueError(f"Unknown cell method: {self.cell_method}") diff --git a/src/sp_validation/tests/test_pseudo_cl.py b/src/sp_validation/tests/test_pseudo_cl.py index b7ac16b8..637b10c2 100644 --- a/src/sp_validation/tests/test_pseudo_cl.py +++ b/src/sp_validation/tests/test_pseudo_cl.py @@ -593,3 +593,32 @@ def test_calculate_pseudo_cl_catalog_end_to_end(cv, tmp_path): params = get_params_rho_tau(cv.cc[ver], survey=ver) _, cl_prim, _ = cv.get_pseudo_cls_catalog(catalog=cat_gal, params=params) npt.assert_allclose(ee, cl_prim[0], rtol=RTOL_CAT, atol=ATOL_CAT) + + +def test_calculate_pseudo_cl_out_path_born_at_declared_name(cv): + """calculate_pseudo_cl(out_path=...) writes to the given path, not the + untagged native name — the anti-collision seam. + + The tagged producer (rule pseudo_cl, blind=A) and the untagged diagnostic + (rule cv_pseudo_cl, blind=None) both call calculate_pseudo_cl; if the tagged + one wrote the native pseudo_cl_{ver}.sacc and renamed, its skip-if-exists + could silently adopt — and the rename delete — the diagnostic's differently- + blinded file. Born-at-declared-name makes the two paths provably disjoint. + """ + ver = cv._test_version + cv._pseudo_cls = {} + tagged = cv._output_path(f"pseudo_cl_{ver}_blind=A_powspace_nbins=32.sacc") + native = cv._output_path(f"pseudo_cl_{ver}.sacc") + + cv.calculate_pseudo_cl(out_path=tagged) + + assert os.path.exists(tagged) + assert not os.path.exists(native) # no undeclared native basename touched + + +def test_calculate_pseudo_cl_out_path_rejects_multiversion(cv): + """out_path targets one part; a multi-version instance must fail loudly + rather than write every version to the same path.""" + cv.versions = [cv._test_version, "SecondVersion"] + with pytest.raises(ValueError, match="one part to one path"): + cv.calculate_pseudo_cl(out_path=cv._output_path("pseudo_cl_x.sacc")) diff --git a/workflow/scripts/generate_pseudo_cl.py b/workflow/scripts/generate_pseudo_cl.py index 81f69414..9bfabf4a 100644 --- a/workflow/scripts/generate_pseudo_cl.py +++ b/workflow/scripts/generate_pseudo_cl.py @@ -35,7 +35,7 @@ def generate_pseudo_cl( version: str, - output_dir: str, + out_path: str, cat_config: str, nside: int = 1024, npatch: int = 1, @@ -45,16 +45,16 @@ def generate_pseudo_cl( nbins: int = None, power: float = 0.5, ): - """Generate a pseudo-Cl data vector into ``output_dir``. + """Generate a pseudo-Cl data vector, born as a SACC part at ``out_path``. Parameters ---------- version : str Catalog version (e.g., "SP_v1.4.6_leak_corr") - output_dir : str - Directory the pseudo-Cl SACC part is written into. The primitive writes - its native ``pseudo_cl_{version}.sacc`` here; callers that need a tagged - filename rename it themselves (see ``_from_snakemake``). + out_path : str + Exact destination the SACC part is *born at* — its final (possibly + tagged) name. No native-basename + rename step, so this producer's + skip-if-exists never collides with the untagged cv_pseudo_cl diagnostic. cat_config : str Path to catalog configuration YAML nside : int @@ -76,8 +76,9 @@ def generate_pseudo_cl( Returns ------- str - Path to the primitive's native ``pseudo_cl_{version}.sacc`` product. + ``out_path`` (the SACC part written). """ + output_dir = os.path.dirname(out_path) os.makedirs(output_dir, exist_ok=True) blind_str = f" blind={blind}" if blind else "" @@ -136,25 +137,26 @@ def generate_pseudo_cl( cv = CosmologyValidation(**cv_kwargs) # Calculate pseudo-Cls only (no covariance). The data vector is born as a - # SACC part: pseudo_cl_{version}.sacc under output_dir. - cv.calculate_pseudo_cl() - - # Report on the native product (renamed by the Snakemake caller, if any). - src_cl = os.path.join(output_dir, f"pseudo_cl_{version}.sacc") - if os.path.exists(src_cl): - s = sacc_io.load(src_cl) + # SACC part directly at out_path (its final, possibly-tagged name) — no + # shared native basename, no rename, so this producer's skip-if-exists never + # collides with the untagged cv_pseudo_cl diagnostic (which would otherwise + # let one rule adopt + delete the other's differently-blinded file). + cv.calculate_pseudo_cl(out_path=out_path) + + if os.path.exists(out_path): + s = sacc_io.load(out_path) ell = sacc_io.get_pseudo_cl(s, (0, 0))[0] print(f"Generated pseudo-Cl with {len(ell)} ell bins") print(f"ell range: [{ell.min():.1f}, {ell.max():.1f}]") - return src_cl + return out_path def _from_snakemake(smk): p = smk.params - output_cl = smk.output.pseudo_cl - src_cl = generate_pseudo_cl( + # Born directly at the rule's declared (tagged) output — no rename step. + generate_pseudo_cl( version=p["version"], - output_dir=os.path.dirname(output_cl), + out_path=smk.output.pseudo_cl, cat_config=p["cat_config"], nside=int(p["nside"]), npatch=int(p["npatch"]), @@ -164,10 +166,6 @@ def _from_snakemake(smk): nbins=int(p["nbins"]), power=float(p.get("power", 0.5)), ) - # Snakemake declares a tagged output filename; rename the native product to it. - if os.path.exists(src_cl) and src_cl != output_cl: - os.rename(src_cl, output_cl) - print(f"Saved to: {output_cl}") def _from_cli(argv=None): @@ -220,9 +218,13 @@ def _from_cli(argv=None): with open(a.cosmo_json) as f: cosmo_params = json.load(f) + # lc/ASTRA path: --out is a per-recipe directory; the untagged native name + # is unambiguous there (each recipe gets its own tree, so no cross-nbins or + # cross-blind collision). + out_path = os.path.join(a.out, f"pseudo_cl_{a.ver}.sacc") generate_pseudo_cl( version=a.ver, - output_dir=a.out, + out_path=out_path, cat_config=a.cat_config, nside=a.nside, npatch=a.npatch, From e0519c85199e9cc6df41cdf59330cc8d582f23f9 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 12:20:17 +0200 Subject: [PATCH 23/47] =?UTF-8?q?fix(workflow):=20assemble=20fails=20loudl?= =?UTF-8?q?y=20by=20default=20on=20the=20placeholder=20=CE=BE=20covariance?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit {version}.sacc is the terminal science file; shipping it with a var=1.0 diagonal as the LEADING (ξ±) covariance block — ~20 orders off the real variance — is a silent catastrophic χ²/PTE for any consumer. Drop the unconditional placeholder_var=1.0. Default: no real ξ-cov wired → assemble_sacc.py's existing ValueError fires. The placeholder is now gated behind an explicit opt-in (cosmo_val.allow_placeholder_cov: true), for dry-run / test configs only. The pseudo-Cℓ block stays real (pseudo_cl_cov input); the integration tests pass placeholder_var explicitly and are unaffected. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01HPfbe2XVTGzbnPn4g7BdN6 --- workflow/rules/cosmo_val.smk | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index 3fe0e1df..d9a56403 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -490,11 +490,16 @@ rule assemble_sacc: sacc=cv_analysis_sacc("{version}"), params: version="{version}", - # ξ± coarse block: documented diagonal placeholder until PR-3's converter - # sources the real CosmoCov theory covariance via --xi-cov. The pseudo-Cℓ - # block is real (pseudo_cl_cov input); COSEBIs / pure-E/B / ρ/τ carry - # their own. assemble_sacc.py reads pseudo_cl_cov's COVAR_* extensions. - placeholder_var=1.0, + # ξ± coarse has no real covariance wired yet (its CosmoCov theory block is + # PR-3's converter territory, plugging in via --xi-cov). By DEFAULT this + # is fatal: assemble_sacc.py raises rather than ship {version}.sacc — the + # terminal science file — with a var=1.0 placeholder as its LEADING + # covariance block (~20 orders off the real ξ± variance → silent + # catastrophic χ²/PTE for any consumer). Only an explicit config opt-in + # (cosmo_val.allow_placeholder_cov: true — dry-run / test configs) attaches + # the flagged diagonal placeholder. The pseudo-Cℓ block is real (from the + # pseudo_cl_cov input); COSEBIs / pure-E/B / ρ/τ carry their own. + placeholder_var=(1.0 if CV.get("allow_placeholder_cov", False) else None), resources: mem_mb=8000, runtime=20, From 6a5df5b701675bf66b987cc041264073cbf228b9 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 12:22:23 +0200 Subject: [PATCH 24/47] test(workflow): dry-run guard covers the cosmo_val assemble DAG MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test_bmodes_workflow_dry_runs uses papers/bmodes, whose config has no cosmo_val block, so cosmo_val.smk (the born-as-SACC + assemble rules) is never included — the guard couldn't catch a break in them. Add test_cosmo_val_workflow_assemble_ dry_runs targeting assemble_sacc_all in papers/cosmo_val (the only paper that includes cosmo_val.smk): asserts the DAG resolves and each assemble_sacc job pulls the tagged, blinded pseudo-Cl part + its NaMaster cov (not the untagged diagnostic) plus all five per-statistic parts. Shared _dry_run helper factored out of the bmodes guard. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01HPfbe2XVTGzbnPn4g7BdN6 --- .../tests/test_bmodes_workflow_dry_run.py | 70 ++++++++++++++----- 1 file changed, 52 insertions(+), 18 deletions(-) diff --git a/src/sp_validation/tests/test_bmodes_workflow_dry_run.py b/src/sp_validation/tests/test_bmodes_workflow_dry_run.py index 60675f52..87afc416 100644 --- a/src/sp_validation/tests/test_bmodes_workflow_dry_run.py +++ b/src/sp_validation/tests/test_bmodes_workflow_dry_run.py @@ -1,7 +1,10 @@ -"""Back-pressure guard #2: the B-modes Snakemake workflow dry-runs. +"""Back-pressure guard #2: the paper Snakemake workflows dry-run. -The reorg is allowed to change the rule graph; this guard only asserts that -Snakemake can still parse the workflow and construct a dry run. +The reorg is allowed to change the rule graph; these guards only assert that +Snakemake can still parse each composed workflow and construct a dry run. One +guard covers papers/bmodes (config space, no cosmo_val block); a second covers +papers/cosmo_val, whose config DOES carry a cosmo_val block — so it is the only +one that includes cosmo_val.smk and hence the born-as-SACC + assemble rules. """ import os @@ -27,38 +30,69 @@ def _repo_root() -> Path: raise RuntimeError("could not locate repo root (no pyproject.toml above test)") -@requires_candide_data -def test_bmodes_workflow_dry_runs(): - """The paper B-mode workflow must still parse and dry-run cleanly.""" - workflow_dir = _repo_root() / "papers/bmodes" - # PYTHONUNBUFFERED satisfies the Snakefile's `envvars:` declaration without - # depending on the invoking shell's environment. A dry run resolves the DAG - # only — it never dispatches jobs — so drop any inherited SNAKEMAKE_PROFILE - # (e.g. the login shell's "slurm" profile), which would otherwise force an - # executor plugin the test environment need not have installed. +def _dry_run(workflow_dir, targets, *extra_snakemake_args): + """Construct a dry run of the paper workflow at ``workflow_dir``. + + Returns the CompletedProcess. PYTHONUNBUFFERED satisfies the Snakefile's + ``envvars:`` declaration without depending on the invoking shell. A dry run + resolves the DAG only — it never dispatches jobs — so drop any inherited + SNAKEMAKE_PROFILE (e.g. the login shell's "slurm" profile), which would + otherwise force an executor plugin the test environment need not have. And + invoke snakemake through sys.executable (the interpreter pytest, hence + snakemake, lives in) — a bare python3.12 resolves off PATH to e.g. an + intel-python without snakemake. + """ env = os.environ | {"PYTHONNOUSERSITE": "1", "PYTHONUNBUFFERED": "1"} env.pop("SNAKEMAKE_PROFILE", None) - result = subprocess.run( + return subprocess.run( [ - # Invoke snakemake through the interpreter running the test — a bare - # "python3.12" resolves off PATH (e.g. intel-python without snakemake); - # sys.executable is the environment that pytest, hence snakemake, lives in. sys.executable, "-m", "snakemake", - "all_tapestry", + *targets, "--dry-run", "--cores", "1", "--configfile", "config/config.yaml", + *extra_snakemake_args, ], cwd=workflow_dir, env=env, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - timeout=60, + timeout=120, check=False, ) + + +@requires_candide_data +def test_bmodes_workflow_dry_runs(): + """The paper B-mode workflow must still parse and dry-run cleanly.""" + result = _dry_run(_repo_root() / "papers/bmodes", ["all_tapestry"]) + assert result.returncode == 0, result.stdout + + +@requires_candide_data +def test_cosmo_val_workflow_assemble_dry_runs(): + """The cosmo_val workflow (the only one including cosmo_val.smk) resolves the + born-as-SACC + assemble DAG, and assemble pulls the tagged pseudo-Cl + cov. + + Targets the assemble_sacc_all rule so every version's assemble_sacc job + appears. The dry run resolves the DAG structure only — it never executes the + assemble script — so the placeholder-cov opt-in (cosmo_val.allow_placeholder_cov) + is irrelevant here; a real run would need it (or a wired --xi-cov) to proceed, + which is the fail-loud-by-default behaviour asserted in test_assemble_sacc.""" + version = "SP_v1.4.6.3_leak_corr" + result = _dry_run(_repo_root() / "papers/cosmo_val", ["assemble_sacc_all"]) assert result.returncode == 0, result.stdout + # assemble_sacc must be in the DAG and pull the tagged, blinded pseudo-Cl + # part + its NaMaster covariance (not the untagged cv_pseudo_cl diagnostic), + # plus all five per-statistic parts. + out = result.stdout + assert "rule assemble_sacc:" in out, out + assert f"pseudo_cl_{version}_blind=A_powspace_nbins=32.sacc" in out, out + assert f"pseudo_cl_cov_{version}_blind=A_powspace_nbins=32.fits" in out, out + for part in ("_xi_coarse_", "_cosebis.sacc", "_pure_eb.sacc", "rho_tau_"): + assert part in out, f"missing {part} part in assemble DAG:\n{out}" From 3561b3692fd0444e40959055d5a70fc354615c9c Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 12:24:03 +0200 Subject: [PATCH 25/47] fix(bmodes): drop removed save_fits kwarg from run_xi_sweep's run_2pcf call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit run_xi_sweep.py called run_2pcf(..., save_fits=True) — a kwarg the SACC migration removed, so every invocation TypeErrors. The sweep consumes only the .txt dump; drop the kwarg. run_2pcf is born-as-SACC, so give its coarse part a grid-qualified sacc_out (the default {ver}_xi_coarse.sacc carries no binning, so the reporting + integration grids would collide per version). Stale "+ ξ+/ξ- FITS" docstring dropped. test_cli_seams: bind the sweep's exact run_2pcf call against the live signature (and assert save_fits no longer binds) so this CLI seam can't silently rot — the compute is cluster-only and never exercised by the fast suite. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01HPfbe2XVTGzbnPn4g7BdN6 --- papers/bmodes/scripts/run_xi_sweep.py | 10 ++-- src/sp_validation/tests/test_cli_seams.py | 65 +++++++++++++++++++++++ 2 files changed, 72 insertions(+), 3 deletions(-) create mode 100644 src/sp_validation/tests/test_cli_seams.py diff --git a/papers/bmodes/scripts/run_xi_sweep.py b/papers/bmodes/scripts/run_xi_sweep.py index b02283a5..49076435 100644 --- a/papers/bmodes/scripts/run_xi_sweep.py +++ b/papers/bmodes/scripts/run_xi_sweep.py @@ -2,8 +2,8 @@ Loops the [non-fiducial version list](sweep_versions.nonfiducial_versions) and runs the same ``run_2pcf.run_2pcf`` compute the fiducial two_point recipes call, -once per version, writing every version's ξ± text dump (+ ξ+/ξ- FITS) into one -lc ``{output}`` dir under run_2pcf's native, already-canonical name +once per version, writing every version's ξ± text dump into one lc ``{output}`` +dir under run_2pcf's native, already-canonical name ``{ver}_xi_minsep={min}_maxsep={max}_nbins={nbins}_npatch={npatch}.txt`` — the exact pattern ``cosebis_version_comparison._xi_integration`` reconstructs. @@ -72,11 +72,15 @@ def _from_cli(argv=None): versions = a.versions or nonfiducial_versions(config) for ver in versions: for grid in a.grids: + # The sweep consumes only the .txt dump (cosebis_version_comparison + # reconstructs it by binning). run_2pcf is born-as-SACC, so give its + # coarse part a grid-qualified name — the default {ver}_xi_coarse.sacc + # carries no binning, so the two grids per version would collide. run_2pcf( ver=ver, cat_config=a.cat_config, output_dir=a.out, - save_fits=True, + sacc_out=os.path.join(a.out, f"{ver}_xi_coarse_{grid}.sacc"), **GRIDS[grid], ) diff --git a/src/sp_validation/tests/test_cli_seams.py b/src/sp_validation/tests/test_cli_seams.py new file mode 100644 index 00000000..697b3426 --- /dev/null +++ b/src/sp_validation/tests/test_cli_seams.py @@ -0,0 +1,65 @@ +"""Smoke tests for workflow CLI seams — cheap guards against signature rot. + +A CLI script that calls a workflow function with a removed/renamed kwarg +TypeErrors only at invocation time (the compute is cluster-only, so it is never +exercised by the fast suite). These tests bind the exact call each seam makes +against the current signature via ``inspect.signature(...).bind(...)`` — no +compute, no data — so a drifted kwarg (e.g. run_xi_sweep's dropped save_fits) +fails here instead of on the cluster. +""" + +import importlib.util +import inspect +from pathlib import Path + +import pytest + + +def _repo_root() -> Path: + for parent in Path(__file__).resolve().parents: + if (parent / "pyproject.toml").exists(): + return parent + raise RuntimeError("could not locate repo root (no pyproject.toml above test)") + + +def _load(path, name): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_run_xi_sweep_run_2pcf_call_binds(): + """The kwargs run_xi_sweep passes to run_2pcf must bind to its signature. + + Mirrors the call in papers/bmodes/scripts/run_xi_sweep.py — if run_2pcf drops + or renames a parameter (save_fits was removed by the SACC migration), the + bind raises TypeError here rather than on every cluster invocation. + """ + root = _repo_root() + run_2pcf_mod = _load(root / "workflow/scripts/run_2pcf.py", "run_2pcf_seam") + sig = inspect.signature(run_2pcf_mod.run_2pcf) + # Exactly the keyword set run_xi_sweep._from_cli passes (grid params spread + # from GRIDS: min_sep/max_sep/nbins/npatch). + sig.bind( + ver="V", + cat_config="/cfg.yaml", + output_dir="/out", + sacc_out="/out/V_xi_coarse_reporting.sacc", + min_sep=1.0, + max_sep=250.0, + nbins=20, + npatch=1, + ) + # And the removed kwarg must NOT bind (guards against a silent re-add). + with pytest.raises(TypeError): + sig.bind( + ver="V", + cat_config="/cfg.yaml", + output_dir="/out", + save_fits=True, + min_sep=1.0, + max_sep=250.0, + nbins=20, + npatch=1, + ) From 149a50c523cc1ccfeb78f99809ecf2f9f6ccc9da Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 12:26:01 +0200 Subject: [PATCH 26/47] fix(assemble_sacc): validate expected parts, no silent statistic drop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _from_snakemake built part_paths with a hasattr filter, so a typo'd input keyword (cosebi for cosebis) silently omitted that statistic from the terminal {version}.sacc — the exact silent-truncation class this series has been bitten by. assemble_sacc now takes an `expected` list (the statistics the caller wired, from its config toggles) and raises if any is missing from part_paths or names a non-CANONICAL statistic. The rule derives expected from cv_assemble_inputs so it tracks the include_pseudo_cl toggle. CLI path is already typo-safe (argparse rejects unknown flags), so it passes expected=None. Tests: typo'd input key raises; typo in the expected list itself raises. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01HPfbe2XVTGzbnPn4g7BdN6 --- src/sp_validation/tests/test_assemble_sacc.py | 27 +++++++++++++++++++ workflow/rules/cosmo_val.smk | 6 +++++ workflow/scripts/assemble_sacc.py | 26 ++++++++++++++++++ 3 files changed, 59 insertions(+) diff --git a/src/sp_validation/tests/test_assemble_sacc.py b/src/sp_validation/tests/test_assemble_sacc.py index a43747dc..ccddab97 100644 --- a/src/sp_validation/tests/test_assemble_sacc.py +++ b/src/sp_validation/tests/test_assemble_sacc.py @@ -240,3 +240,30 @@ def test_assemble_sacc_respects_pseudo_cl_toggle(tmp_path): s2 = sio.load(str(out)) assert type(s2.covariance).__name__ == "FullCovariance" assert s2.covariance.dense.shape == (len(s2.mean), len(s2.mean)) + + +def test_assemble_sacc_expected_part_missing_raises(tmp_path): + """A typo'd input keyword drops a part from part_paths; the expected list + catches it rather than silently omitting the statistic.""" + paths = _write_parts(tmp_path, cov_less=("xi_coarse",)) + # Simulate a rule-input typo: cosebis wired under the wrong key. + paths["cosebi"] = paths.pop("cosebis") + out = tmp_path / "vSYNTH.sacc" + with pytest.raises(ValueError, match="expected parts \\['cosebis'\\] missing"): + asm.assemble_sacc( + "vSYNTH", + paths, + str(out), + expected=["xi_coarse", "pseudo_cl", "cosebis", "pure_eb", "rho_tau"], + placeholder_var=1.0, + ) + + +def test_assemble_sacc_expected_rejects_unknown_name(tmp_path): + """A typo in the expected list itself is rejected (not a valid statistic).""" + paths = _write_parts(tmp_path, cov_less=("xi_coarse",)) + out = tmp_path / "vSYNTH.sacc" + with pytest.raises(ValueError, match="not assemblable statistics"): + asm.assemble_sacc( + "vSYNTH", paths, str(out), expected=["cosebi"], placeholder_var=1.0 + ) diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index d9a56403..4f2a2e46 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -490,6 +490,12 @@ rule assemble_sacc: sacc=cv_analysis_sacc("{version}"), params: version="{version}", + # Statistics this rule wired (same toggles as cv_assemble_inputs). The + # script validates part_paths against this so a typo'd input keyword + # can't silently drop a statistic from the terminal file. + expected=lambda w: [ + k for k in cv_assemble_inputs(w.version) if k != "pseudo_cl_cov" + ], # ξ± coarse has no real covariance wired yet (its CosmoCov theory block is # PR-3's converter territory, plugging in via --xi-cov). By DEFAULT this # is fatal: assemble_sacc.py raises rather than ship {version}.sacc — the diff --git a/workflow/scripts/assemble_sacc.py b/workflow/scripts/assemble_sacc.py index 2ad6fa27..513ec976 100644 --- a/workflow/scripts/assemble_sacc.py +++ b/workflow/scripts/assemble_sacc.py @@ -107,6 +107,7 @@ def assemble_sacc( part_paths, out_path, *, + expected=None, xi_cov=None, pseudo_cl_cov=None, pseudo_cl_cov_hdu="COVAR_FULL", @@ -123,9 +124,29 @@ def assemble_sacc( present statistics are assembled; order is forced to canonical. out_path : str Destination ``{version}.sacc``. + expected : sequence of str, optional + Statistics that MUST be present in ``part_paths`` (from the caller's + config toggles). Raises loudly if any is missing or has no path — so a + typo'd input keyword (``cosebi`` for ``cosebis``) can't silently drop a + statistic from the terminal file. Names not in :data:`CANONICAL` are + rejected too (catches a typo in the expected list itself). xi_cov, pseudo_cl_cov, pseudo_cl_cov_hdu, placeholder_var Covariance sourcing — see the module docstring. """ + if expected is not None: + unknown = [name for name in expected if name not in CANONICAL] + if unknown: + raise ValueError( + f"expected parts {unknown} are not assemblable statistics; " + f"valid names are {CANONICAL}" + ) + missing = [name for name in expected if not part_paths.get(name)] + if missing: + raise ValueError( + f"expected parts {missing} missing from part_paths for {version} " + f"(got {sorted(part_paths)}); a required statistic would be " + "silently dropped from the terminal analysis file" + ) parts = [] nz = metadata = None for name in CANONICAL: @@ -167,10 +188,15 @@ def _from_snakemake(smk): for name in CANONICAL if hasattr(inp, name) and getattr(inp, name) } + # The rule declares which statistics it wired (from its config toggles); a + # typo in an input keyword drops the part from part_paths above, so validate + # against this expected list rather than trusting the hasattr filter. + expected = list(p["expected"]) assemble_sacc( version=p["version"], part_paths=part_paths, out_path=str(smk.output[0]), + expected=expected, xi_cov=getattr(inp, "xi_cov", None), pseudo_cl_cov=getattr(inp, "pseudo_cl_cov", None), pseudo_cl_cov_hdu=p.get("pseudo_cl_cov_hdu", "COVAR_FULL"), From df87be6d7ff47fb4904fa6709c4cc9aa67b746bf Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 16:34:47 +0200 Subject: [PATCH 27/47] feat(sacc_like): sp_validation shim over CosmoSIS SaccClLikelihood MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adopt CosmoSIS's native SACC likelihood for the ξ± inference path through a thin sp_validation-owned subclass that fixes two upstream defects: 1. arcmin→rad theta-tag conversion — upstream sacc_like builds its theory spline on block[section,"theta"] (radians) but evaluates it at raw SACC theta tags (arcmin), silently returning theory≈0 and collapsing χ². 2pt_like converts to radians for exactly this reason; sacc_like never does. 2. an ordering guard — the data vector (get_mean, insertion order) and the theory loop (get_data_types × tracer_combinations × points) agree only for type-major files; a pair-major tomographic file would silently misalign. The guard reconstructs the theory-loop order and requires it == arange. Factory pattern (setup imports the upstream class from csl_dir and subclasses at call time) so importing the module never needs cosmosis; not imported by __init__. build_data calls super() first (scale cuts run in arcmin, matching 2pt_like's angle_range grammar) then converts + guards. Tests (cosmosis + CSL_DIR gated, skip cleanly otherwise) prove in-process equality against 2pt_like on the PR-3 converter FITS: machine-zero equality on synthetic (χ²=1.18829133107, Δχ²=0) and real data (χ²=420310.753739, Δχ²=0, N=40), the unit-gap tripwire (raw sacc_like 631× off), scale-cut equivalence, identical perturbation response, the ordering guard, and real-type-scoped theta conversion. pyproject: cosmosis>=3.25 extra (inert in CI, which installs [test,glass,blinding]). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EnV8NPWhkxS2SxGVgGSJyt --- pyproject.toml | 9 + src/sp_validation/sacc_like_unions.py | 186 ++++++++ src/sp_validation/tests/test_sacc_like.py | 506 ++++++++++++++++++++++ 3 files changed, 701 insertions(+) create mode 100644 src/sp_validation/sacc_like_unions.py create mode 100644 src/sp_validation/tests/test_sacc_like.py diff --git a/pyproject.toml b/pyproject.toml index ecdec6f1..eef183fb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -138,6 +138,15 @@ blinding = [ # sp_validation fast suite. "numpy>=2.2,<2.5", ] +# Native SACC inference likelihood (PR 7, sp_validation.sacc_like_unions). The +# shim subclasses CosmoSIS's SaccClLikelihood; this pins the engine. Both engine +# module files are pure-python (no CAMB / compiled CSL), so the equality tests +# (tests/test_sacc_like.py) run with just this extra + a CosmoSIS Standard +# Library checkout, located via the CSL_DIR env var: +# git clone --depth 1 https://github.com/joezuntz/cosmosis-standard-library CSL +# CSL_DIR=/path/to/CSL pytest ... test_sacc_like.py +# Inert in CI (its Dockerfile installs only [test,glass,blinding]). +cosmosis = ["cosmosis>=3.25"] develop = ["sp_validation[test,docs]"] [tool.pytest.ini_options] diff --git a/src/sp_validation/sacc_like_unions.py b/src/sp_validation/sacc_like_unions.py new file mode 100644 index 00000000..2b7e5cc6 --- /dev/null +++ b/src/sp_validation/sacc_like_unions.py @@ -0,0 +1,186 @@ +"""SACC likelihood shim for CosmoSIS — the sp_validation-owned native path. + +This module is a thin subclass of CosmoSIS's ``SaccClLikelihood`` (from the +CosmoSIS Standard Library, ``likelihood/sacc/sacc_like.py``) that fixes two +upstream defects so the native SACC likelihood matches the PR-3 converter → +``2pt_like`` path bit for bit on our real-space ξ± analysis file. It exists as a +CosmoSIS *module file* (``setup``/``execute``/``cleanup`` at module scope), loaded +via ``file = .../sacc_like_unions.py`` in an ini, and is NOT imported by +``sp_validation/__init__`` (cosmosis is an optional dependency). + +Why the shim exists +------------------- +Two things in upstream ``sacc_like`` break a real-space ξ likelihood; both are +documented and empirically verified (probe: Δχ²=3184 raw vs Δχ²=0 shimmed on the +single-bin analysis file), and both are fixed here by overriding ``build_data``: + +1. **No arcmin→radian conversion (the killer).** + ``sacc_likelihoods/twopoint.py`` (L74-90 at CSL commit 4fd2f1c) builds a + ``SpectrumInterp`` over ``block[section, "theta"]`` — theory θ in *radians* + (CosmoSIS convention) — and evaluates it at each data point's raw ``theta`` + tag, which our SACC files (and firecrown) store in *arcmin*. ``2pt_like.py`` + (L179-183) converts its real-space data to radians for exactly this reason; + ``sacc_like`` never does, so the spline is evaluated ~3437× outside its grid, + ``SpectrumInterp`` returns 0 there, and χ² silently collapses to dᵀC⁻¹d. The + only prior use was the ℓ-space (unit-free) Cℓ path, which is why it was never + caught. We convert the ``theta`` tags to radians here, after scale cuts. + +2. **Theory↔data ordering is assumed, never enforced.** + The data vector is ``sacc.get_mean()`` (insertion order); the theory vector is + built by looping ``get_data_types() × get_tracer_combinations() × points``. + A comment in ``twopoint.py`` (L35) claims ``to_canonical_order`` was called on + load — it is not. The two orders agree only when the file is grouped + type-major (all of one data type, then the next). Our single-pair + ``[ξ+; ξ−]`` files satisfy this; a tomographic *pair-major* file (ξ+/ξ− per + pair) would silently misalign theory against data. We reconstruct the + theory-loop index order and require it equals ``arange`` — a hard ValueError + otherwise (see ``test_ordering_guard_raises_pair_major_tomographic``). + +When upstream fixes the units, this shim dies: the tripwire test +``test_upstream_unit_gap_tripwire`` in ``tests/test_sacc_like.py`` fails the day +raw ``sacc_like`` stops producing a wildly different χ², signalling the shim can +be retired. + +Requires a CSL checkout: ``setup`` reads ``csl_dir`` from the module options and +imports the upstream ``sacc_like`` from ``/likelihood/sacc``. The tests +locate it via the ``CSL_DIR`` environment variable (see the test module docstring +for the checkout recipe). +""" + +import os +import sys + +import numpy as np +from cosmosis.datablock import SectionOptions, option_section + +# arcmin → radian: the conversion 2pt_like applies to real-space data and that +# sacc_like omits. Applied only to `theta` tags of `real`-category data types. +ARCMIN_TO_RAD = np.pi / (180.0 * 60.0) + + +def _import_upstream_sacc_like(csl_dir): + """Import the upstream ``sacc_like`` module from a CSL checkout. + + ``/likelihood/sacc`` is prepended to ``sys.path`` so both + ``sacc_like`` and its sibling ``sacc_likelihoods`` package (imported by + ``sacc_like`` for the theory-extraction functions) resolve. Idempotent: the + path is only inserted once. + """ + sacc_dir = os.path.join(csl_dir, "likelihood", "sacc") + if not os.path.isdir(sacc_dir): + raise ValueError( + f"csl_dir={csl_dir!r} has no likelihood/sacc directory; point csl_dir " + "at a CosmoSIS Standard Library checkout (see module docstring)" + ) + if sacc_dir not in sys.path: + sys.path.insert(0, sacc_dir) + import sacc_like # noqa: E402 — resolved from the sys.path insertion above + + return sacc_like + + +def _make_subclass(sacc_like): + """Build ``SaccLikeUnions`` as a subclass of the upstream ``SaccClLikelihood``. + + A factory (not an import-time ``class ... :`` statement) so importing this + module never requires the upstream class — that dependency is deferred to + ``setup``, which has ``csl_dir`` in hand. Only ``build_data`` is overridden; + scale cuts, covariance handling (Sellentin/Hartlap), theory extraction and + ``save_theory`` all ride upstream unmodified. + """ + + class SaccLikeUnions(sacc_like.SaccClLikelihood): + """CSL ``SaccClLikelihood`` with the arcmin→rad + ordering-guard fixes.""" + + def build_data(self): + # Run the upstream build FIRST: it loads the SACC, applies data_sets + # selection and the arcmin-grammar scale cuts (matching 2pt_like's + # angle_range convention and the ini ergonomics), populates + # self.sacc_data / self.sections_for_names, and returns the + # (unit-independent) data vector we pass straight through. + x, data_vector = super().build_data() + + self._convert_real_theta_tags_to_radians() + self._assert_theory_order_matches_data() + + return x, data_vector + + def _convert_real_theta_tags_to_radians(self): + """Scale the ``theta`` tag arcmin→rad for every ``real``-category point. + + The theory spline is built on ``block[section, "theta"]`` in radians, + so the ``theta`` tag each point is evaluated at must be radians too. + Scoped to data types whose category (``sections_for_names[dt][0]``) is + ``real``: cosebis ``n`` tags and spectrum ``ell`` tags are unit-free + and must not be touched, and only the ``theta`` tag is converted + (``theta_nom`` etc. are metadata the likelihood never evaluates). + """ + real_types = { + dt + for dt, (category, _section) in self.sections_for_names.items() + if category == "real" + } + for point in self.sacc_data.data: + if point.data_type in real_types and "theta" in point.tags: + point.tags["theta"] = point.tags["theta"] * ARCMIN_TO_RAD + + def _assert_theory_order_matches_data(self): + """Require the theory-loop order to equal the data-vector order. + + The data vector is ``sacc.get_mean()`` (insertion order); upstream + builds theory by looping data types, then tracer combinations, then + points, and concatenating — assuming (never enforcing) that this + reproduces insertion order. It does only for type-major files. We + reconstruct that loop's index order and require it be ``arange``; + otherwise theory and data would be silently misaligned (the same bug + class the PR-2/PR-3 reviews caught for the converter). + """ + order = [ + int(i) + for dt in self.sacc_data.get_data_types() + for tracers in self.sacc_data.get_tracer_combinations(dt) + for i in self.sacc_data.indices(dt, tracers) + ] + expected = np.arange(len(self.sacc_data.mean)) + if not np.array_equal(order, expected): + raise ValueError( + "SACC data/theory ordering mismatch: the theory loop " + "(get_data_types × get_tracer_combinations × points) does not " + "reproduce the get_mean() insertion order, so sacc_like would " + "compare theory against data point-by-point in the WRONG order " + "and return a silently wrong χ². This happens when the file is " + "grouped pair-major (ξ+/ξ− interleaved per tracer pair) rather " + "than type-major (all ξ+, then all ξ−). Upstream assumes " + "to_canonical_order() was applied on load but never calls it; " + "write the SACC type-major, or call to_canonical_order() before " + "saving." + ) + + return SaccLikeUnions + + +def setup(options): + """CosmoSIS ``setup`` — build and instantiate the shimmed likelihood. + + Mirrors ``GaussianLikelihood.build_module``'s setup: wrap the raw options in + ``SectionOptions`` and instantiate the likelihood (whose ``__init__`` calls + ``build_data``). The one addition is reading ``csl_dir`` from the module + options to locate and import the upstream class before subclassing it. + """ + csl_dir = options.get_string(option_section, "csl_dir") + sacc_like = _import_upstream_sacc_like(csl_dir) + likelihood_class = _make_subclass(sacc_like) + return likelihood_class(SectionOptions(options)) + + +def execute(block, config): + """CosmoSIS ``execute`` — run the likelihood (mirrors ``build_module``).""" + likelihood_calculator = config + likelihood_calculator.do_likelihood(block) + return 0 + + +def cleanup(config): + """CosmoSIS ``cleanup`` — mirror of ``build_module``'s cleanup.""" + likelihood_calculator = config + likelihood_calculator.cleanup() diff --git a/src/sp_validation/tests/test_sacc_like.py b/src/sp_validation/tests/test_sacc_like.py new file mode 100644 index 00000000..f8c6c3b2 --- /dev/null +++ b/src/sp_validation/tests/test_sacc_like.py @@ -0,0 +1,506 @@ +"""Equality tests: the sp_validation SACC-likelihood shim vs CosmoSIS ``2pt_like``. + +PR 7 adopts CosmoSIS's native ``SaccClLikelihood`` for the ξ± inference path, +through the shim :mod:`sp_validation.sacc_like_unions` (which fixes the upstream +arcmin→rad gap and adds an ordering guard). The contract is *in-process module +equality*: run the shimmed ``sacc_like`` on the analysis SACC and CosmoSIS's +``2pt_like`` on the PR-3 converter's 2pt-FITS against an identical synthetic +theory DataBlock, and require the same χ², log-likelihood, theory vector and +post-cut point count. The prototype (``sacc-like-probe/probe_equality.py``) +observed *exact* equality (Δχ²=0, Δtheory=0), so the equality tests assert +``array_equal`` / rtol=1e-12. + +Environment +----------- +Both engines are pure-python CosmoSIS module files (no CAMB, no compiled CSL +modules), so they run in the shared venv with ``cosmosis`` installed. They need a +checkout of the CosmoSIS Standard Library, located via the ``CSL_DIR`` env var; +the module is skipped when cosmosis is absent (CI image) or ``CSL_DIR`` is unset +or missing. Recipe:: + + git clone --depth 1 https://github.com/joezuntz/cosmosis-standard-library CSL + CSL_DIR=/path/to/CSL pytest ... test_sacc_like.py + +The pyproject ``cosmosis`` extra pins ``cosmosis>=3.25`` for the engine itself +(inert in CI, whose Dockerfile installs only ``[test,glass,blinding]``). +""" + +import importlib.util +import os +from pathlib import Path + +import numpy as np +import pytest + +cosmosis = pytest.importorskip("cosmosis") + +# The upstream CSL checkout carrying likelihood/sacc + likelihood/2pt. Skip the +# whole module (not error) when it is not configured, mirroring the cosmo_numba +# env-gated precedent. +_CSL_DIR = os.environ.get("CSL_DIR") +if not _CSL_DIR or not Path(_CSL_DIR, "likelihood", "sacc").is_dir(): + pytest.skip( + "CSL_DIR unset or has no likelihood/sacc — set CSL_DIR to a CosmoSIS " + "Standard Library checkout to run the sacc_like equality tests", + allow_module_level=True, + ) + +from cosmosis.datablock import DataBlock, option_section # noqa: E402 + +from sp_validation import sacc_io, twopoint_convert # noqa: E402 + +# Reuse the angular-bin count from the converter tests so the two suites' single- +# bin shapes stay in lockstep. (The χ²-dynamics builders here need a covariance +# commensurate with the data, which the converter's byte-compare _sacc is not.) +from sp_validation.tests.test_twopoint_convert import N_ANG # noqa: E402 + +CSL = Path(_CSL_DIR) +ARCMIN_TO_RAD = np.pi / (180.0 * 60.0) + +# The like_name both engines are configured with, so both write identical block +# keys (_CHI2, _LIKE, _theory) — the parity the design mandates. +LIKE_NAME = "2pt_like" + + +# --------------------------------------------------------------------------- +# Shared engine harness +# --------------------------------------------------------------------------- +def _load_module_file(path, name): + """Import a CosmoSIS module file (``setup``/``execute``/``cleanup``) by path. + + ``likelihood/sacc`` and ``likelihood/2pt`` are added to ``sys.path`` so the + upstream modules resolve their siblings (``sacc_likelihoods``, ``spec_tools``, + ``twopoint_cosmosis``, …). Loading ``2pt_like.py`` runs its + ``build_module()`` at import (module-level ``setup, execute, cleanup``); + ``sacc_like_unions.py`` defines those functions directly. + """ + import sys + + for sub in ("likelihood/sacc", "likelihood/2pt"): + p = str(CSL / sub) + if p not in sys.path: + sys.path.insert(0, p) + spec = importlib.util.spec_from_file_location(name, path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +def _theory_block(): + """A DataBlock carrying the synthetic ξ± theory both engines interpolate. + + Mirrors the probe: the theory θ grid is in *radians* (CosmoSIS convention), + and the ξ+/ξ− predictions are smooth power laws sampled on it. Both engines + build a spline over this grid and evaluate it at each data point's angular + tag — so as long as both see the same block, the interpolated theory (and + hence χ²) must agree. + """ + theta_arcmin = np.geomspace(0.3, 400.0, 300) + + def t_xip(th): + return 2e-4 * (th / 10.0) ** -0.8 + + def t_xim(th): + return 1e-4 * (th / 10.0) ** -0.5 + + b = DataBlock() + for section, f in (("shear_xi_plus", t_xip), ("shear_xi_minus", t_xim)): + b[section, "theta"] = theta_arcmin * ARCMIN_TO_RAD + b[section, "bin_1_1"] = f(theta_arcmin) + b[section, "is_auto"] = True + b[section, "nbin_a"] = 1 + b[section, "nbin_b"] = 1 + b[section, "sample_a"] = "nz_source" + b[section, "sample_b"] = "nz_source" + b[section, "sep_name"] = "theta" + b[section, "save_name"] = "" + return b + + +def _run(mod, options): + """Run a CosmoSIS likelihood module file standalone; return (like, chi2, theory, n). + + Builds the options DataBlock (module options live under ``option_section``), + calls ``setup`` → ``execute`` on a fresh theory block, and reads the standard + Gaussian-likelihood outputs back out under the shared ``LIKE_NAME`` keys. + """ + opt = DataBlock() + for key, value in options.items(): + opt[option_section, key] = value + config = mod.setup(opt) + block = _theory_block() + mod.execute(block, config) + like = block["likelihoods", f"{LIKE_NAME}_LIKE"] + chi2 = block["data_vector", f"{LIKE_NAME}_CHI2"] + theory = block["data_vector", f"{LIKE_NAME}_theory"] + return like, chi2, np.asarray(theory), len(theory) + + +# --------------------------------------------------------------------------- +# The two engine module files + their options +# --------------------------------------------------------------------------- +_SHIM_PATH = Path(__file__).resolve().parents[1] / "sacc_like_unions.py" + + +@pytest.fixture(scope="module") +def m_shim(): + return _load_module_file(_SHIM_PATH, "sacc_like_unions_test") + + +@pytest.fixture(scope="module") +def m_2pt(): + return _load_module_file(CSL / "likelihood/2pt/2pt_like.py", "twopt_like_test") + + +@pytest.fixture(scope="module") +def m_raw_sacc(): + return _load_module_file(CSL / "likelihood/sacc/sacc_like.py", "raw_sacc_like_test") + + +def _shim_opts(sacc_path, **extra): + return { + "csl_dir": str(CSL), + "data_file": sacc_path, + "data_sets": "galaxy_shear_xi_plus galaxy_shear_xi_minus", + "like_name": LIKE_NAME, + **extra, + } + + +def _twopt_opts(fits_path, **extra): + return { + "data_file": fits_path, + "data_sets": "XI_PLUS XI_MINUS", + "covmat_name": "COVMAT", + "like_name": LIKE_NAME, + "gaussian_covariance": False, + "cut_zeros": False, + **extra, + } + + +def _realistic_sacc(seed=0, *, xip=None): + """A single-bin ξ± SACC with data + covariance sized like the real product. + + The converter-test ``_sacc`` builder uses a covariance ~14 orders of + magnitude larger than the ξ values (fine for byte-comparing the converter, + where covariance *content* is irrelevant), which makes every χ² collapse to + numerical zero — no teeth for the unit-gap tripwire. This builder instead + lays down realistic ξ± power laws and a covariance ~ ``(0.1·|ξ|)²`` (the + probe's recipe), so χ² is O(1)-scale and the raw-vs-shim gap is visible. + + ``xip`` overrides the ξ+ values (perturbation teeth). + """ + rng = np.random.default_rng(seed) + n = N_ANG + theta = np.geomspace(1.0, 250.0, n) # arcmin + z = np.linspace(0.01, 3.0, 200) + nz = z**2 * np.exp(-((z / 0.5) ** 1.5)) + + xip_vals = 2e-4 * (theta / 10.0) ** -0.8 * (1 + 0.05 * rng.standard_normal(n)) + xim_vals = 1e-4 * (theta / 10.0) ** -0.5 * (1 + 0.05 * rng.standard_normal(n)) + if xip is not None: + xip_vals = xip + + s = sacc_io.new_sacc({0: (z, nz)}, {"catalogue_version": "test"}) + sacc_io.add_xi(s, (0, 0), theta, xip_vals, xim_vals, grid="coarse") + + sig = 0.1 * np.abs(np.concatenate([xip_vals, xim_vals])) + a = rng.standard_normal((2 * n, 3 * n)) + cov = (a @ a.T / (3 * n)) * np.outer(sig, sig) * 0.3 + np.diag(sig**2) + s.add_covariance(cov) + return s, theta + + +def _write_pair(tmp_path, seed=0, name="probe", *, xip=None): + """Write a realistic analysis SACC and its PR-3 converter 2pt-FITS. + + Returns ``(sacc_path, fits_path)``. The SACC carries the arcmin ξ± tags the + shim converts; the FITS is the byte-compatible product ``2pt_like`` reads. + """ + s, _theta = _realistic_sacc(seed, xip=xip) + sacc_path = str(tmp_path / f"{name}.sacc") + sacc_io.save(s, sacc_path) + fits_path = str(tmp_path / f"{name}_2pt.fits") + twopoint_convert.sacc_to_twopoint_fits(sacc_io.load(sacc_path), fits_path, n_bins=1) + return sacc_path, fits_path + + +# --------------------------------------------------------------------------- +# 1. Core equality: shim on SACC ≡ 2pt_like on converter FITS +# --------------------------------------------------------------------------- +def test_shimmed_equals_2pt_like_exact(tmp_path, m_shim, m_2pt): + """The shimmed sacc_like and 2pt_like agree exactly on the same data+theory. + + Same synthetic theory block, the SACC through the shim vs the converter FITS + through 2pt_like: χ², log-likelihood and the theory vector must match to + numerical precision (the probe observed exact equality). This is the PR's + central contract — the native path reproduces the validated converter path. + """ + sacc_path, fits_path = _write_pair(tmp_path, seed=0) + + like_s, chi2_s, theory_s, n_s = _run(m_shim, _shim_opts(sacc_path)) + like_t, chi2_t, theory_t, n_t = _run(m_2pt, _twopt_opts(fits_path)) + + assert n_s == n_t == 2 * N_ANG + np.testing.assert_allclose(chi2_s, chi2_t, rtol=1e-12) + np.testing.assert_allclose(like_s, like_t, rtol=1e-12) + np.testing.assert_allclose(theory_s, theory_t, rtol=1e-12) + + +# --------------------------------------------------------------------------- +# 2. Tripwire: raw upstream sacc_like is broken on arcmin tags +# --------------------------------------------------------------------------- +def test_upstream_unit_gap_tripwire(tmp_path, m_raw_sacc, m_2pt): + """RAW upstream sacc_like (no shim) gives a wildly wrong χ² on arcmin tags. + + Documents and guards the arcmin→rad gap the shim fixes: with raw ``theta`` + tags the theory spline is evaluated outside its (radian) grid and returns 0, + collapsing χ² to dᵀC⁻¹d. We assert the relative χ² difference against + 2pt_like exceeds 10 (the probe saw Δχ²≈3184). + + IF THIS TEST EVER FAILS: upstream ``sacc_like`` has fixed its units — the + shim's arcmin→rad conversion is now redundant and the shim can be retired. + """ + sacc_path, fits_path = _write_pair(tmp_path, seed=0) + + _like_r, chi2_raw, _theory_r, _n_r = _run(m_raw_sacc, _shim_opts(sacc_path)) + _like_t, chi2_t, _theory_t, _n_t = _run(m_2pt, _twopt_opts(fits_path)) + + rel = abs(chi2_raw - chi2_t) / abs(chi2_t) + assert rel > 10, ( + f"raw sacc_like χ²={chi2_raw:.6g} is within 10× of 2pt_like χ²={chi2_t:.6g} " + f"(rel diff {rel:.3g}) — the upstream unit gap appears fixed; retire the shim" + ) + + +# --------------------------------------------------------------------------- +# 3. Scale cuts equivalent through both engines +# --------------------------------------------------------------------------- +def test_scale_cuts_equivalent(tmp_path, m_shim, m_2pt): + """The same arcmin scale cuts give the same post-cut N and χ² on both engines. + + Cuts are expressed in the each engine's grammar but the SAME numeric arcmin + values (shim cuts run before the arcmin→rad conversion, so they take arcmin + just like 2pt_like's angle_range). ξ+ ∈ [10, 200], ξ− ∈ [20, 200] arcmin. + """ + sacc_path, fits_path = _write_pair(tmp_path, seed=1) + + shim_cuts = _shim_opts( + sacc_path, + **{ + "angle_range_galaxy_shear_xi_plus_source_0_source_0": np.array( + [10.0, 200.0] + ), + "angle_range_galaxy_shear_xi_minus_source_0_source_0": np.array( + [20.0, 200.0] + ), + }, + ) + twopt_cuts = _twopt_opts( + fits_path, + **{ + "angle_range_XI_PLUS_1_1": np.array([10.0, 200.0]), + "angle_range_XI_MINUS_1_1": np.array([20.0, 200.0]), + }, + ) + + _like_s, chi2_s, _theory_s, n_s = _run(m_shim, shim_cuts) + _like_t, chi2_t, _theory_t, n_t = _run(m_2pt, twopt_cuts) + + assert n_s == n_t + assert n_s < 2 * N_ANG # the cuts actually removed points + np.testing.assert_allclose(chi2_s, chi2_t, rtol=1e-12) + + +# --------------------------------------------------------------------------- +# 4. Perturbation moves both engines identically +# --------------------------------------------------------------------------- +def test_perturbation_moves_both_identically(tmp_path, m_shim, m_2pt): + """Perturbing one data value shifts both engines' χ² by the identical amount. + + Teeth: build the base pair and a pair whose first ξ+ value is bumped, and + require Δχ²(shim) == Δχ²(2pt_like). If either engine ignored the perturbed + point (e.g. a misaligned data vector), the deltas would diverge. + """ + # Base pair, then a pair whose first ξ+ value is bumped. Rebuild the base ξ+ + # from the same seed so only the one perturbed entry differs. + base_s, theta = _realistic_sacc(seed=2) + base_xip = np.array( + [p.value for p in base_s.data if p.data_type == sacc_io.XI_PLUS] + ) + sacc_b, fits_b = _write_pair(tmp_path, seed=2, name="base") + + pert_xip = base_xip.copy() + pert_xip[0] += 5e-5 + sacc_p, fits_p = _write_pair(tmp_path, seed=2, name="pert", xip=pert_xip) + + _l, chi2_shim_b, _t, _n = _run(m_shim, _shim_opts(sacc_b)) + _l, chi2_shim_p, _t, _n = _run(m_shim, _shim_opts(sacc_p)) + _l, chi2_2pt_b, _t, _n = _run(m_2pt, _twopt_opts(fits_b)) + _l, chi2_2pt_p, _t, _n = _run(m_2pt, _twopt_opts(fits_p)) + + d_shim = chi2_shim_p - chi2_shim_b + d_2pt = chi2_2pt_p - chi2_2pt_b + assert abs(d_shim) > 0 # the perturbation actually moved χ² + np.testing.assert_allclose(d_shim, d_2pt, rtol=1e-10) + + +# --------------------------------------------------------------------------- +# 5. Ordering guard raises on a pair-major tomographic file +# --------------------------------------------------------------------------- +def test_ordering_guard_raises_pair_major_tomographic(tmp_path, m_shim): + """A pair-major 2-bin SACC trips the shim's ordering guard at setup. + + Inserting ξ± per pair — (0,0), then (0,1), then (1,1) — lays the data vector + out pair-major ([ξ+;ξ−] per pair), while ``get_data_types()`` groups the + theory loop type-major (all ξ+ pairs, then all ξ− pairs). The two orders + disagree (verified empirically), so the guard must raise a ValueError + mentioning the ordering hazard rather than silently mis-comparing. + """ + theta = np.geomspace(1.0, 250.0, N_ANG) # arcmin + z = np.linspace(0.01, 3.0, 200) + nz = z**2 * np.exp(-((z / 0.5) ** 1.5)) + xip = np.ones(N_ANG) * 1e-4 + xim = np.ones(N_ANG) * 1e-4 + s = sacc_io.new_sacc({0: (z, nz), 1: (z, nz)}) + for pair in [(0, 0), (0, 1), (1, 1)]: + sacc_io.add_xi(s, pair, theta, xip, xim, grid="coarse") + s.add_covariance(np.eye(len(s.mean))) + sacc_path = str(tmp_path / "pair_major.sacc") + sacc_io.save(s, sacc_path) + + with pytest.raises(ValueError, match="ordering"): + m_shim.setup(_as_option_block(_shim_opts(sacc_path))) + + +def _as_option_block(options): + """Build a raw options DataBlock (keys under ``option_section``) for setup().""" + opt = DataBlock() + for key, value in options.items(): + opt[option_section, key] = value + return opt + + +# --------------------------------------------------------------------------- +# 6. theta conversion scoped to real-category types (COSEBIs untouched) +# --------------------------------------------------------------------------- +def test_theta_conversion_scoped_to_real_types(tmp_path, m_shim): + """The shim converts ξ ``theta`` tags but leaves cosebi ``n`` tags untouched. + + Build a SACC carrying both ξ± (real) and COSEBIs (a non-real ``cosebis`` + category with integer ``n`` tags). After the shim's ``build_data``, the ξ + ``theta`` tags must be scaled arcmin→rad (so they equal the original arcmin + values times the conversion factor), and the cosebi ``n`` tags must be + numerically unchanged. + """ + # Build ξ± + COSEBIs, then attach the covariance last (sacc forbids adding + # points after add_covariance). + theta = np.geomspace(1.0, 250.0, N_ANG) # arcmin + z = np.linspace(0.01, 3.0, 200) + nz = z**2 * np.exp(-((z / 0.5) ** 1.5)) + n_modes = 5 + En = np.arange(1.0, n_modes + 1) + Bn = np.arange(1.0, n_modes + 1) * 0.1 + + s = sacc_io.new_sacc({0: (z, nz)}) + sacc_io.add_xi( + s, (0, 0), theta, np.ones(N_ANG) * 1e-4, np.ones(N_ANG) * 1e-4, grid="coarse" + ) + sacc_io.add_cosebis(s, (0, 0), En, Bn, scale_cut=(1.0, 250.0)) + s.add_covariance(np.eye(len(s.mean))) + + sacc_path = str(tmp_path / "with_cosebis.sacc") + sacc_io.save(s, sacc_path) + + # data_sets keeps the cosebis in (so we can check its tags survive); cosebi's + # section/category resolve from sacc_like's default_sections, so build_data + # needs no extra ini config. Only setup() runs (build_data); the theory loop + # (which would want a cosebi theory block) runs at execute, not here. + config = m_shim.setup( + _as_option_block( + _shim_opts( + sacc_path, + data_sets=( + "galaxy_shear_xi_plus galaxy_shear_xi_minus " + "galaxy_shear_cosebi_ee galaxy_shear_cosebi_bb" + ), + ) + ) + ) + + xi_thetas = [ + p.tags["theta"] for p in config.sacc_data.data if p.data_type == sacc_io.XI_PLUS + ] + cosebi_ns = [ + p.tags["n"] for p in config.sacc_data.data if p.data_type == sacc_io.COSEBI_EE + ] + # ξ theta converted to radians (original arcmin × ARCMIN_TO_RAD). + np.testing.assert_allclose( + np.sort(xi_thetas), np.sort(theta * ARCMIN_TO_RAD), rtol=1e-12 + ) + # cosebi n tags untouched (still the integer modes 1..n_modes). + np.testing.assert_array_equal(np.sort(cosebi_ns), np.arange(1, n_modes + 1)) + + +# --------------------------------------------------------------------------- +# 7. Real-data equality (candide-gated) +# --------------------------------------------------------------------------- +_REALDATA = ( + Path("/automnt/n17data/cdaley/unions/code/sp_validation/cosmo_inference/data") + / "SP_v1.4.6_leak_corr_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1" + / "cosmosis_SP_v1.4.6_leak_corr_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits" +) + + +@pytest.mark.skipif( + not _REALDATA.exists(), reason=f"real 2pt-FITS not on disk: {_REALDATA}" +) +def test_realdata_shim_equals_2pt_like(tmp_path, m_shim, m_2pt): + """On a real product, the shim on its SACC equals 2pt_like on the FITS. + + Builds a ξ-only analysis SACC from the real 2pt-FITS's own ξ± values and + covariance sub-block (via the converter test's ``_sacc_from_2pt_fits``, then + strip to ξ±), writes it, converts it to a plain-ξ FITS, and runs both engines + against the synthetic theory block. Scoping to ξ± isolates the shear + likelihood equality on the true (20-point-per-sign) data-vector shape — the + IA-only inference scope this PR targets — and keeps ``2pt_like``'s + ``twopoint.from_fits`` from tripping over the real file's separate + COVMAT_CELL / τ blocks. + """ + from astropy.io import fits + + from sp_validation.tests.test_twopoint_convert_realdata import _sacc_from_2pt_fits + + with fits.open(_REALDATA) as hdul: + full_s, _rho_hdu, _tau_hdu = _sacc_from_2pt_fits(hdul) + + # Rebuild a ξ-only SACC: same n(z), ξ± values and ξ± covariance sub-block. + source = sacc_io.source_name(0) + z, nz = sacc_io.get_nz(full_s, 0) + theta, xip, xim = sacc_io.get_xi(full_s, (0, 0), grid="coarse") + xi_idx = np.concatenate( + [ + full_s.indices(sacc_io.XI_PLUS, (source, source)), + full_s.indices(sacc_io.XI_MINUS, (source, source)), + ] + ) + xi_cov = full_s.covariance.dense[np.ix_(xi_idx, xi_idx)] + + s = sacc_io.new_sacc({0: (z, nz)}) + sacc_io.add_xi(s, (0, 0), theta, xip, xim, grid="coarse") + s.add_covariance(xi_cov) + + sacc_path = str(tmp_path / "real_xi.sacc") + sacc_io.save(s, sacc_path) + fits_path = str(tmp_path / "real_xi_2pt.fits") + twopoint_convert.sacc_to_twopoint_fits(sacc_io.load(sacc_path), fits_path, n_bins=1) + + like_s, chi2_s, theory_s, n_s = _run(m_shim, _shim_opts(sacc_path)) + like_t, chi2_t, theory_t, n_t = _run(m_2pt, _twopt_opts(fits_path)) + + assert n_s == n_t + assert n_s == 2 * len(theta) + np.testing.assert_allclose(chi2_s, chi2_t, rtol=1e-10) + np.testing.assert_allclose(like_s, like_t, rtol=1e-10) + np.testing.assert_allclose(theory_s, theory_t, rtol=1e-10) From 1b05bf7016c111df253b782f0ee6dfdf7ce3f58e Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 16:35:37 +0200 Subject: [PATCH 28/47] feat(inference): sacc_like ini template + config generator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the CosmoSIS pipeline ini template for the native SACC path (cosmosis_pipeline_A_ia_sacc.ini): copies the A_ia template, swapping load_nz_fits→load_nz_sacc (native n(z) from the SACC's source_i NZ tracers) and 2pt_like→sacc_like (the shim, via SP_VALIDATION_MODULES, csl_dir=COSMOSIS_DIR, like_name=2pt_like for block-key parity). Same numeric scale cuts as A_ia, in the sacc_like angle_range grammar (full data-type names + source_0 pairs). generate_inference_config.py fills a template's [DEFAULT] section with concrete paths (SCRATCH, FITS_FILE|SACC_FILE, COSMOSIS_DIR, SP_VALIDATION_MODULES resolved from sp_validation.__file__). Dual-mode (snakemake object OR argparse CLI) like assemble_sacc.py; plain text processing (the pipeline.sh sed idiom) that REPLACES an existing DEFAULT key in place and prepends only new ones — no configparser round-trip (which would strip comments + %(...)s interpolation) and no duplicate DEFAULT key (which CosmoSIS's strict parser rejects). Tests (no cosmosis needed): both templates fill + interpolate cleanly, the referenced module file exists, no %(...)s placeholder survives, missing-DEFAULT raises. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EnV8NPWhkxS2SxGVgGSJyt --- .../cosmosis_pipeline_A_ia_sacc.ini | 110 ++++++++++++ .../tests/test_generate_inference_config.py | 138 +++++++++++++++ workflow/scripts/generate_inference_config.py | 159 ++++++++++++++++++ 3 files changed, 407 insertions(+) create mode 100644 cosmo_inference/cosmosis_config/cosmosis_pipeline_A_ia_sacc.ini create mode 100644 src/sp_validation/tests/test_generate_inference_config.py create mode 100644 workflow/scripts/generate_inference_config.py diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_A_ia_sacc.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_A_ia_sacc.ini new file mode 100644 index 00000000..5c31a476 --- /dev/null +++ b/cosmo_inference/cosmosis_config/cosmosis_pipeline_A_ia_sacc.ini @@ -0,0 +1,110 @@ +#parameters used elsewhere in this file +[DEFAULT] +COSMOSIS_DIR = /n23data1/n06data/lgoh/scratch/cosmosis-standard-library_lisa + + +[pipeline] +modules = consistency sample_S8 camb load_nz_sacc photoz_bias linear_alignment projection add_intrinsic 2pt_shear shear_m_bias sacc_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[polychord] +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[test] + +[output] +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_sacc] +file = %(COSMOSIS_DIR)s/number_density/load_nz_sacc/load_nz_sacc.py +nz_file = %(SACC_FILE)s +data_sets = source + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_xi_plus shear_xi_minus +verbose = F + +; Native SACC likelihood via the sp_validation shim (arcmin->rad + ordering +; guard over CosmoSIS's SaccClLikelihood). data_sets/angle ranges use the SACC +; grammar (full data-type names + tracer pairs). like_name=2pt_like keeps the +; block keys identical to the 2pt_like path so chain post-processing is unchanged. +[sacc_like] +file = %(SP_VALIDATION_MODULES)s/sacc_like_unions.py +csl_dir = %(COSMOSIS_DIR)s +data_file = %(SACC_FILE)s +data_sets = galaxy_shear_xi_plus galaxy_shear_xi_minus +like_name = 2pt_like + +angle_range_galaxy_shear_xi_plus_source_0_source_0 = 10.0 200.0 +angle_range_galaxy_shear_xi_minus_source_0_source_0 = 20.0 200.0 diff --git a/src/sp_validation/tests/test_generate_inference_config.py b/src/sp_validation/tests/test_generate_inference_config.py new file mode 100644 index 00000000..4fbbb98d --- /dev/null +++ b/src/sp_validation/tests/test_generate_inference_config.py @@ -0,0 +1,138 @@ +"""Tests for the CosmoSIS inference-config generator. + +The generator (:mod:`workflow.scripts.generate_inference_config`) fills a +pipeline ini template's ``[DEFAULT]`` section with concrete paths so CosmoSIS's +ConfigParser resolves the template's ``%(KEY)s`` placeholders. These tests need +no cosmosis: they check that the substituted DEFAULT keys land, that the module +file paths the templates reference exist on disk, and that no ``%(...)s`` +placeholder is left unresolved after filling. +""" + +import configparser +import importlib.util +from pathlib import Path + +import pytest + +_REPO = Path(__file__).resolve().parents[3] +_SCRIPT = _REPO / "workflow" / "scripts" / "generate_inference_config.py" +_CONFIG_DIR = _REPO / "cosmo_inference" / "cosmosis_config" +_SACC_TEMPLATE = _CONFIG_DIR / "cosmosis_pipeline_A_ia_sacc.ini" +_FITS_TEMPLATE = _CONFIG_DIR / "cosmosis_pipeline_A_ia.ini" + + +def _load_generator(): + spec = importlib.util.spec_from_file_location("gen_inference_cfg", _SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +gen = _load_generator() + + +def _read_interpolated(ini_path): + """Parse the generated ini with interpolation ON (the way CosmoSIS reads it). + + CosmoSIS uses ``%(KEY)s`` BasicInterpolation with case-*preserved* keys, so + set ``optionxform = str`` (stdlib configparser lowercases keys by default, + which would break the uppercase ``%(FITS_FILE)s`` / ``%(SACC_FILE)s`` + lookups). The default BasicInterpolation then raises if a referenced key is + missing — exactly the failure we want to catch. + """ + parser = configparser.ConfigParser() + parser.optionxform = str + parser.read(ini_path) + return parser + + +def test_sacc_template_defaults_filled(tmp_path): + """The generated sacc ini carries the substituted DEFAULT keys and resolves.""" + out = tmp_path / "gen_sacc.ini" + gen.generate_inference_config( + _SACC_TEMPLATE, + out, + gen._substitutions( + scratch="/scratch/run", + cosmosis_dir="/csl", + sacc_file="/data/v1.sacc", + ), + ) + text = out.read_text() + assert "SCRATCH = /scratch/run" in text + assert "SACC_FILE = /data/v1.sacc" in text + assert "COSMOSIS_DIR = /csl" in text + assert "SP_VALIDATION_MODULES = " in text + # FITS_FILE is None for the sacc path — must be dropped, not written as "None". + assert "FITS_FILE" not in text + + parser = _read_interpolated(out) + # The sacc_like data_file interpolates SACC_FILE; load_nz_sacc too. + assert parser["sacc_like"]["data_file"] == "/data/v1.sacc" + assert parser["load_nz_sacc"]["nz_file"] == "/data/v1.sacc" + assert parser["sacc_like"]["csl_dir"] == "/csl" + + +def test_fits_template_defaults_filled(tmp_path): + """The generated 2pt_like ini carries the substituted DEFAULT keys and resolves.""" + out = tmp_path / "gen_fits.ini" + gen.generate_inference_config( + _FITS_TEMPLATE, + out, + gen._substitutions( + scratch="/scratch/run", + cosmosis_dir="/csl", + fits_file="/data/v1.fits", + ), + ) + text = out.read_text() + assert "SCRATCH = /scratch/run" in text + assert "FITS_FILE = /data/v1.fits" in text + assert "SACC_FILE" not in text + + parser = _read_interpolated(out) + assert parser["2pt_like"]["data_file"] == "/data/v1.fits" + assert parser["load_nz_fits"]["nz_file"] == "/data/v1.fits" + + +def test_sacc_template_module_file_exists(): + """The sacc_like module file the generated ini points at exists on disk. + + ``SP_VALIDATION_MODULES`` resolves to the installed package dir; + ``sacc_like_unions.py`` must live there (it is the shim CosmoSIS loads). + """ + modules = Path(gen._sp_validation_modules()) + assert (modules / "sacc_like_unions.py").is_file() + + +def test_all_placeholders_resolve(tmp_path): + """No ``%(...)s`` placeholder survives interpolation in either template. + + A missing DEFAULT key would make ConfigParser raise on access; iterate every + option in every section to force resolution of all placeholders. + """ + for template, subs in ( + ( + _SACC_TEMPLATE, + gen._substitutions(scratch="/s", cosmosis_dir="/csl", sacc_file="/d.sacc"), + ), + ( + _FITS_TEMPLATE, + gen._substitutions(scratch="/s", cosmosis_dir="/csl", fits_file="/d.fits"), + ), + ): + out = tmp_path / (template.stem + ".gen.ini") + gen.generate_inference_config(template, out, subs) + parser = _read_interpolated(out) + for section in parser.sections(): + for key in parser[section]: + value = parser[section][key] # raises if a placeholder is unresolved + assert "%(" not in value, f"[{section}] {key} = {value}" + + +def test_no_default_section_raises(tmp_path): + """A template with no [DEFAULT] section is a loud error.""" + bad = tmp_path / "bad.ini" + bad.write_text("[pipeline]\nmodules = a b c\n") + with pytest.raises(ValueError, match="DEFAULT"): + gen.generate_inference_config(bad, tmp_path / "out.ini", {"SCRATCH": "/s"}) diff --git a/workflow/scripts/generate_inference_config.py b/workflow/scripts/generate_inference_config.py new file mode 100644 index 00000000..68a8efe7 --- /dev/null +++ b/workflow/scripts/generate_inference_config.py @@ -0,0 +1,159 @@ +"""Generate a CosmoSIS pipeline ini by filling a template's ``[DEFAULT]`` section. + +Dual-mode, like ``assemble_sacc.py``. Under Snakemake (``script:`` directive) the +injected ``snakemake`` object supplies the template, output path and DEFAULT +substitutions; as a standalone CLI (argparse) the same fill runs from explicit +flags. + +The template carries ``%(KEY)s`` interpolation placeholders (SCRATCH, FITS_FILE +or SACC_FILE, COSMOSIS_DIR, SP_VALIDATION_MODULES) in its module sections; this +script prepends the concrete ``KEY = value`` lines into ``[DEFAULT]`` so +CosmoSIS's ConfigParser resolves them at load. It is deliberately plain text +processing — appending lines after the ``[DEFAULT]`` header, the same idiom as +``pipeline.sh``'s ``sed -i "/^\\[DEFAULT\\]/a\\KEY = value"`` — rather than a +configparser round-trip, which would strip the template's comments and its +``%(...)s`` interpolation. + +``SP_VALIDATION_MODULES`` is resolved from ``sp_validation.__file__``'s parent so +the generated ini points at the installed package's module directory (where +``sacc_like_unions.py`` lives) regardless of checkout location. +""" + +import argparse +from pathlib import Path + + +def _sp_validation_modules(): + """The directory holding the sp_validation CosmoSIS module files. + + Resolved from the installed package so the generated ini finds + ``sacc_like_unions.py`` wherever sp_validation is installed. + """ + import sp_validation + + return str(Path(sp_validation.__file__).resolve().parent) + + +def generate_inference_config(template_path, out_path, substitutions): + """Write ``out_path`` from ``template_path`` with ``substitutions`` in DEFAULT. + + Parameters + ---------- + template_path : str or Path + The pipeline ini template (carries ``%(KEY)s`` placeholders). + out_path : str or Path + Destination ini. + substitutions : dict + ``{KEY: value}`` lines prepended into the template's ``[DEFAULT]`` + section. Every referenced ``%(KEY)s`` in the template must have a value + here (COSMOSIS_DIR already sits in the template's DEFAULT and may be + overridden). ``None`` values are dropped (an absent optional key). + """ + lines = Path(template_path).read_text().splitlines(keepends=True) + + header = "[DEFAULT]" + default_idx = next( + (i for i, line in enumerate(lines) if line.strip() == header), None + ) + if default_idx is None: + raise ValueError(f"template {template_path} has no [DEFAULT] section") + + # The end of the DEFAULT section: the next `[section]` header, or EOF. + section_end = next( + ( + i + for i in range(default_idx + 1, len(lines)) + if lines[i].lstrip().startswith("[") + ), + len(lines), + ) + + # A key the template already declares in DEFAULT is REPLACED in place (e.g. the + # template's placeholder COSMOSIS_DIR); a genuinely-new key is prepended just + # after the header. This avoids a duplicate DEFAULT key, which CosmoSIS's + # ConfigParser (strict) rejects. + wanted = {key: value for key, value in substitutions.items() if value is not None} + remaining = dict(wanted) + for i in range(default_idx + 1, section_end): + stripped = lines[i].lstrip() + if not stripped or stripped.startswith(("#", ";", "[")): + continue + existing_key = stripped.split("=", 1)[0].strip() + if existing_key in remaining: + lines[i] = f"{existing_key} = {remaining.pop(existing_key)}\n" + + prepended = [f"{key} = {value}\n" for key, value in remaining.items()] + out_lines = lines[: default_idx + 1] + prepended + lines[default_idx + 1 :] + + out_path = Path(out_path) + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text("".join(out_lines)) + print( + f"Wrote {out_path} from {template_path} " + f"({len(wanted)} DEFAULT keys, {len(prepended)} new)" + ) + return str(out_path) + + +def _substitutions(scratch, cosmosis_dir, *, fits_file=None, sacc_file=None): + """Assemble the DEFAULT substitution dict, resolving SP_VALIDATION_MODULES. + + ``fits_file`` (2pt_like path) and ``sacc_file`` (sacc_like path) are mutually + the data-file placeholder for their template; whichever the template + references is filled, the other left absent. + """ + return { + "SCRATCH": scratch, + "FITS_FILE": fits_file, + "SACC_FILE": sacc_file, + "COSMOSIS_DIR": cosmosis_dir, + "SP_VALIDATION_MODULES": _sp_validation_modules(), + } + + +def _from_snakemake(smk): + p = smk.params + generate_inference_config( + template_path=smk.input[0] + if not hasattr(smk.input, "template") + else smk.input.template, + out_path=str(smk.output[0]), + substitutions=_substitutions( + scratch=p["scratch"], + cosmosis_dir=p["cosmosis_dir"], + fits_file=p.get("fits_file", None), + sacc_file=p.get("sacc_file", None), + ), + ) + + +def _from_cli(argv=None): + ap = argparse.ArgumentParser( + description="Generate a CosmoSIS pipeline ini from a template + DEFAULT subs." + ) + ap.add_argument("--template", required=True, help="Pipeline ini template") + ap.add_argument("--out", required=True, help="Output ini path") + ap.add_argument("--scratch", required=True, help="SCRATCH value") + ap.add_argument("--cosmosis-dir", required=True, help="COSMOSIS_DIR value") + ap.add_argument("--fits-file", default=None, help="FITS_FILE (2pt_like path)") + ap.add_argument("--sacc-file", default=None, help="SACC_FILE (sacc_like path)") + a = ap.parse_args(argv) + generate_inference_config( + template_path=a.template, + out_path=a.out, + substitutions=_substitutions( + scratch=a.scratch, + cosmosis_dir=a.cosmosis_dir, + fits_file=a.fits_file, + sacc_file=a.sacc_file, + ), + ) + + +if __name__ == "__main__": + try: + snakemake # noqa: F821 — injected by Snakemake's script: directive + except NameError: + _from_cli() + else: + _from_snakemake(snakemake) # noqa: F821 From 5d123508d64ce892009b655896de56389c9fe9bd Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 16:35:53 +0200 Subject: [PATCH 29/47] feat(inference): rewire inference_prep to consume the assembled SACC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Revive the real-data inference_prep from its dormant, pre-SACC state. It now consumes the assembled analysis {version}.sacc (cosmo_val.smk's assemble_sacc, bound lazily through cv_analysis_sacc) and emits the A_ia (IA-only, ξ±) file-prep products: (a) the converter 2pt-FITS (sacc_to_twopoint_fits, pure ξ — no rho/tau sidecars, A_ia scope) + a generated 2pt_like ini (the validating/legacy path), and (b) a generated sacc_like ini pointing at the SACC (the native path, validated bit-for-bit against (a)). The cosmosis_fitting.py real-data assembly is retired from this rule; the glass-mock rules keep it (their SACC migration is out of scope, noted inline). inference_fiducial extends to all three prep outputs. CSL_DIR is read lazily (inference.smk is parsed by every paper workflow, but only cosmo_val carries inference.csl_dir) — a missing key still fails loudly, just at DAG time rather than at parse time for an unrelated (bmodes) workflow. config.yaml: add inference.csl_dir. The workflow dry-run guard gains a test covering the revived inference_prep DAG (SACC in, converter FITS + both engine inis out, no cosmosis_fitting.py real-data assembly). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EnV8NPWhkxS2SxGVgGSJyt --- papers/cosmo_val/config/config.yaml | 5 +- .../tests/test_bmodes_workflow_dry_run.py | 25 +++ workflow/rules/inference.smk | 166 ++++++++++-------- 3 files changed, 122 insertions(+), 74 deletions(-) diff --git a/papers/cosmo_val/config/config.yaml b/papers/cosmo_val/config/config.yaml index 945f2f8f..a7f19a8d 100644 --- a/papers/cosmo_val/config/config.yaml +++ b/papers/cosmo_val/config/config.yaml @@ -116,11 +116,14 @@ harmonic: binning: powspace nbins: 32 -# Cosmological inference data-product locations (dormant subsystem). +# Cosmological inference data-product locations + tooling. inference: chains_dir: "/n09data/guerrini/output_chains" glass_mock_data_dir: "/n09data/guerrini/glass_mock_v1.4.6/results" glass_mock_chains_dir: "/n09data/guerrini/glass_mock_chains" + # CosmoSIS Standard Library checkout — fills COSMOSIS_DIR in the generated + # pipeline inis (the module `file =` paths and the sacc_like shim's csl_dir). + csl_dir: "/n23data1/n06data/lgoh/scratch/cosmosis-standard-library_lisa" cosebis: theta_min: 1.0 diff --git a/src/sp_validation/tests/test_bmodes_workflow_dry_run.py b/src/sp_validation/tests/test_bmodes_workflow_dry_run.py index 87afc416..b9486b5d 100644 --- a/src/sp_validation/tests/test_bmodes_workflow_dry_run.py +++ b/src/sp_validation/tests/test_bmodes_workflow_dry_run.py @@ -96,3 +96,28 @@ def test_cosmo_val_workflow_assemble_dry_runs(): assert f"pseudo_cl_cov_{version}_blind=A_powspace_nbins=32.fits" in out, out for part in ("_xi_coarse_", "_cosebis.sacc", "_pure_eb.sacc", "rho_tau_"): assert part in out, f"missing {part} part in assemble DAG:\n{out}" + + +@requires_candide_data +def test_cosmo_val_inference_prep_dry_runs(): + """The revived (PR-7) inference_prep DAG resolves end to end from the SACC. + + inference_fiducial must pull inference_prep, which consumes the assembled + {version}.sacc and emits the converter 2pt-FITS plus BOTH generated pipeline + inis (2pt_like and the native sacc_like). The old cosmosis_fitting.py real- + data assembly is retired from this path; the glass-mock rules keep it. + """ + version = "SP_v1.4.6.3_leak_corr" + result = _dry_run(_repo_root() / "papers/cosmo_val", ["inference_fiducial"]) + assert result.returncode == 0, result.stdout + out = result.stdout + assert "rule inference_prep:" in out, out + assert "rule inference_fiducial:" in out, out + # inference_prep consumes the assembled analysis SACC (not per-sign xi FITS). + assert f"{version}.sacc" in out, out + # It emits the converter FITS + both engine inis. + assert f"cosmosis_{version}.fits" in out, out + assert f"cosmosis_pipeline_{version}_A_ia.ini" in out, out + assert f"cosmosis_pipeline_{version}_A_ia_sacc.ini" in out, out + # The retired real-data assembly script must not appear in this DAG's prep. + assert "cosmosis_fitting.py --cosmosis-root" not in out, out diff --git a/workflow/rules/inference.smk b/workflow/rules/inference.smk index 6bf68645..9c73c52a 100644 --- a/workflow/rules/inference.smk +++ b/workflow/rules/inference.smk @@ -1,15 +1,19 @@ -# Imports from Snakefile: FIDUCIAL, COSMO_INFERENCE, COSMO_VAL, covariance_path, build_redshift_path, fiducial_binning_suffix -# NOTE: dormant subsystem. The file-name plumbing (config-driven paths + the -# producer-tagged pseudo-Cl names) is fixed and the DAG is valid, but it has not -# been run end-to-end. Reviving it still needs the FITS-CONTENT plumbing -# reconciled: cosmosis_fitting.py reads ELL/EE/BB + COVAR_FULL, while the -# producers write PSEUDO_CELL/ELL + COVAR_BB_BB. +# Imports from common (via `from common import *`): FIDUCIAL, COSMO_INFERENCE, +# COSMO_VAL, WORKFLOW_SCRIPTS, covariance_path, build_redshift_path, +# fiducial_binning_suffix. cv_analysis_sacc arrives from cosmo_val.smk (resolved +# lazily at DAG time, since that file is included after this one). +# +# Two paths live here: +# * Real-data inference_prep — LIVE (PR 7): consumes the assembled {version}.sacc +# and emits the converter 2pt-FITS + both engine inis (2pt_like, sacc_like). +# * glass-mock rules — still cosmosis_fitting.py-based (their SACC migration is +# out of scope); the pseudo-Cl file-name plumbing they depend on stays below. # Output root for CosmoSIS data products + configs. COSMO_INFERENCE (common.py) # already resolves to THIS repo's cosmo_inference dir, so the products land # beside the code that builds them rather than in a contributor's home. COSMO_INFERENCE_PROD = COSMO_INFERENCE -# Working directory for the cosmosis_fitting.py invocation — the same repo dir. +# Working directory for the (glass-mock) cosmosis_fitting.py invocation. COSMO_INFERENCE_RUNDIR = str(COSMO_INFERENCE) # External chain/mock locations are deployment-specific, so they live in config. @@ -56,88 +60,104 @@ def pseudo_cl_assets(version): return str(cl_path), str(cov_path) # --------------------------------------------------------------------------- -# DORMANT — pre-SACC cosmosis assembly. Migration to native SACC deferred to -# PR 7 (native-SACC inference consumption); do NOT deep-migrate here. +# Real-data inference prep — LIVE (native SACC, PR 7). Consumes the assembled +# analysis {version}.sacc (cosmo_val.smk's assemble_sacc rule) and emits the two +# file-prep products the A_ia (IA-only, ξ±) fiducial pipeline needs: +# (a) the converter 2pt-FITS (sacc_to_twopoint_fits) + a generated 2pt_like ini +# — the validating/legacy path (retiring cosmosis_fitting.py's assembly), +# (b) a generated sacc_like ini pointing at the SACC directly — the native path +# validated bit-for-bit against (a) (test_sacc_like.py). +# The converter is A_ia-scoped: no rho/tau sidecars, so it emits a pure-ξ FITS +# (it ignores the SACC's extra data types). This is file-prep only — the actual +# CosmoSIS sampling still runs via pipeline.sh against these products. # -# The SACC migration (PR 4) removed the data products several of these inputs -# name, so this rule's DAG no longer resolves and is NOT reachable from the -# cosmo_val suite (cosmo_val_all never requests it). Stale inputs: -# - xi_plus / xi_minus FITS: the `xi` rule now emits the coarse ξ± SACC part -# ({version}_xi_coarse_...sacc), not per-sign FITS. -# - pseudo_cl / pseudo_cl_cov via pseudo_cl_assets(): the `pseudo_cl` rule now -# writes .sacc (pseudo_cl_assets still requests .fits). -# PR 7 rewires this to consume the assembled {version}.sacc (built by -# cosmo_val.smk's assemble_sacc rule) directly, retiring cosmosis_fitting.py's -# per-product FITS assembly. Until then the inference target is knowingly red. +# The glass-mock rules below stay cosmosis_fitting.py-based; their SACC migration +# is out of scope for PR 7. # --------------------------------------------------------------------------- +INFERENCE_TEMPLATES = COSMO_INFERENCE_PROD / "cosmosis_config" + + +def _csl_dir(): + """The CSL checkout that fills COSMOSIS_DIR / sacc_like csl_dir in the inis. + + Read lazily (at DAG time, inside inference_prep's params) rather than at + module parse time: inference.smk is included by every paper workflow, but + only papers that run inference (cosmo_val) carry inference.csl_dir. A missing + key still fails loudly — just when the real-data inference is actually built, + not when an unrelated (bmodes) workflow merely parses this file. + """ + return INFERENCE["csl_dir"] + + rule inference_prep: input: - # Processed covariance matrix - use centralized covariance_path() - cov_matrix=lambda w: covariance_path(w.version, w.blind, min_sep=w.min_sep, max_sep=w.max_sep, nbins=w.nbins), - # Xi FITS files — PRE-SACC (no longer produced; see dormant note above) - xi_plus=str(COSMO_VAL / "xi_plus_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), - xi_minus=str(COSMO_VAL / "xi_minus_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), - # n(z) file (using new location with base version mapping) - nz_file=lambda w: build_redshift_path(w.version, w.blind), - # rho/tau stats - rho_stats=str(COSMO_VAL / "rho_tau_stats/rho_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), - tau_stats=str(COSMO_VAL / "rho_tau_stats/tau_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), - # tau covariance (tracked as dependency) - tau_cov=str(COSMO_VAL / "rho_tau_stats/cov_tau_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}_th.npy"), - # pseudo_cl / pseudo_cl_cov — PRE-SACC (.fits path; producer now writes .sacc) - pseudo_cl=lambda w: pseudo_cl_assets(w.version)[0], - pseudo_cl_cov=lambda w: pseudo_cl_assets(w.version)[1], + # The terminal assembled analysis SACC (cosmo_val.smk assemble_sacc). Bound + # lazily through its helper so the filename tracks that rule, not a literal. + sacc=lambda w: cv_analysis_sacc(w.version), output: - fits_file=str( - COSMO_INFERENCE_PROD - / "data/{version}_{blind}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}/cosmosis_{version}_{blind}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits" + fits_file=str(COSMO_INFERENCE_PROD / "data/{version}/cosmosis_{version}.fits"), + config_file_2pt=str( + INFERENCE_TEMPLATES / "cosmosis_pipeline_{version}_A_ia.ini" + ), + config_file_sacc=str( + INFERENCE_TEMPLATES / "cosmosis_pipeline_{version}_A_ia_sacc.ini" ), - config_file=str( - COSMO_INFERENCE_PROD - / "cosmosis_config/cosmosis_pipeline_{version}_{blind}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.ini" - ) params: - cosmosis_root="{version}_{blind}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}", - data_dir=f"{CHAINS_DIR}/{{version}}_{{blind}}_minsep={{min_sep}}_maxsep={{max_sep}}_nbins={{nbins}}_npatch={{npatch}}", - output_root=str(COSMO_INFERENCE_PROD), + # SCRATCH = the per-version chain output root the generated inis point at. + scratch=lambda w: f"{CHAINS_DIR}/{w.version}", + cosmosis_dir=lambda w: _csl_dir(), + template_2pt=str(INFERENCE_TEMPLATES / "cosmosis_pipeline_A_ia.ini"), + template_sacc=str(INFERENCE_TEMPLATES / "cosmosis_pipeline_A_ia_sacc.ini"), threads: 1 resources: mem_mb=8000, runtime=10, - shell: - """ - cd {COSMO_INFERENCE_RUNDIR} + run: + import os + import sys - # Run inference preparation step with cosmosis_fitting.py - python scripts/cosmosis_fitting.py \ - --cosmosis-root {params.cosmosis_root} \ - --nz-file {input.nz_file} \ - --data-dir {params.data_dir} \ - --output-root {params.output_root} \ - --xi {input.xi_plus} {input.xi_minus} \ - --cov-xi {input.cov_matrix} \ - --use-rho-tau \ - --rho-stats {input.rho_stats} \ - --tau-stats {input.tau_stats} \ - --cov-tau {input.tau_cov} \ - --cl-file {input.pseudo_cl} \ - --cov-cl {input.pseudo_cl_cov} - """ + from sp_validation import sacc_io + from sp_validation.twopoint_convert import sacc_to_twopoint_fits + + os.makedirs(os.path.dirname(output.fits_file), exist_ok=True) + + # (a) converter 2pt-FITS — pure ξ (A_ia scope; no rho/tau sidecars). + sacc_to_twopoint_fits(sacc_io.load(input.sacc), output.fits_file, n_bins=1) + + # (b) + (c) the two generated pipeline inis, from the existing templates. + # WORKFLOW_SCRIPTS (common.py) is the absolute generic-workflow scripts dir. + sys.path.insert(0, WORKFLOW_SCRIPTS) + from generate_inference_config import ( + _substitutions, + generate_inference_config, + ) + + generate_inference_config( + params.template_2pt, + output.config_file_2pt, + _substitutions( + scratch=params.scratch, + cosmosis_dir=params.cosmosis_dir, + fits_file=output.fits_file, + ), + ) + generate_inference_config( + params.template_sacc, + output.config_file_sacc, + _substitutions( + scratch=params.scratch, + cosmosis_dir=params.cosmosis_dir, + sacc_file=input.sacc, + ), + ) rule inference_fiducial: input: - # Use the same output patterns as inference_prep with FIDUCIAL params - rules.inference_prep.output.fits_file.format( - version=FIDUCIAL["version"], blind=FIDUCIAL["blind"], - min_sep=FIDUCIAL["min_sep"], max_sep=FIDUCIAL["max_sep"], - nbins=FIDUCIAL["nbins"], npatch=FIDUCIAL["npatch"] - ), - rules.inference_prep.output.config_file.format( - version=FIDUCIAL["version"], blind=FIDUCIAL["blind"], - min_sep=FIDUCIAL["min_sep"], max_sep=FIDUCIAL["max_sep"], - nbins=FIDUCIAL["nbins"], npatch=FIDUCIAL["npatch"] - ) + # The fiducial version's prep products (both engine inis + the FITS). + rules.inference_prep.output.fits_file.format(version=FIDUCIAL["version"]), + rules.inference_prep.output.config_file_2pt.format(version=FIDUCIAL["version"]), + rules.inference_prep.output.config_file_sacc.format(version=FIDUCIAL["version"]), rule inference_glass_mocks: From db286022edbc5810f6f47059fa07136f5a18ca9b Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 17:07:17 +0200 Subject: [PATCH 30/47] fix(sacc_like): keep self.sacc_data in arcmin so save_theory writes arcmin tags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review finding (MEDIUM): the shim converted theta tags arcmin→rad in place on self.sacc_data. Upstream do_likelihood's save_theory / save_realization paths copy self.sacc_data and overwrite only point values, so they were writing radian theta tags into the saved SACC — any arcmin-assuming consumer (sacc_io.get_xi, or re-ingesting the file as data_file, which double-converts to ~8.5e-8) is then silently off by 3437×. Keep self.sacc_data in arcmin always. build_data now builds a separate self._sacc_data_rad copy with the real-category theta tags converted, and extract_theory_points swaps it in around super().extract_theory_points(block) in a try/finally — so the theory spline sees radians while everything after (save_theory / save_realization) sees the untouched arcmin original. The ordering guard runs on the arcmin object (ordering is unit-independent). Theory equality is unchanged (the swap produces identical results): synthetic χ²=1.18829133107, real-data χ²=420310.753739, both Δχ²=0 vs 2pt_like. New test: run with save_theory set, reload, assert the saved theta tags equal the input file's arcmin tags exactly (and are NOT the radian conversion) and the saved values equal the theory vector; assert a second execute() yields the identical χ² (guards against a double-conversion regression). The theta-scoping test now checks the rad copy carries radians AND the original stays arcmin. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EnV8NPWhkxS2SxGVgGSJyt --- src/sp_validation/sacc_like_unions.py | 50 ++++++++++++--- src/sp_validation/tests/test_sacc_like.py | 76 ++++++++++++++++++++--- 2 files changed, 108 insertions(+), 18 deletions(-) diff --git a/src/sp_validation/sacc_like_unions.py b/src/sp_validation/sacc_like_unions.py index 2b7e5cc6..5aa610ec 100644 --- a/src/sp_validation/sacc_like_unions.py +++ b/src/sp_validation/sacc_like_unions.py @@ -23,7 +23,10 @@ ``sacc_like`` never does, so the spline is evaluated ~3437× outside its grid, ``SpectrumInterp`` returns 0 there, and χ² silently collapses to dᵀC⁻¹d. The only prior use was the ℓ-space (unit-free) Cℓ path, which is why it was never - caught. We convert the ``theta`` tags to radians here, after scale cuts. + caught. We evaluate the theory against a radian-θ *copy* of the loaded SACC, + keeping ``self.sacc_data`` in arcmin so upstream's save_theory / + save_realization paths (which copy it and overwrite only values) never write + radian tags into a file downstream consumers read as arcmin. 2. **Theory↔data ordering is assumed, never enforced.** The data vector is ``sacc.get_mean()`` (insertion order); the theory vector is @@ -100,29 +103,56 @@ def build_data(self): # (unit-independent) data vector we pass straight through. x, data_vector = super().build_data() - self._convert_real_theta_tags_to_radians() + # self.sacc_data STAYS in arcmin — save_theory / save_realization copy + # it and only overwrite point values, so its θ tags must remain the + # units the file was written in (any consumer, incl. re-ingesting the + # saved SACC as a data_file, assumes arcmin). The radian conversion the + # theory spline needs lives on a separate copy, swapped in only for the + # extraction (see extract_theory_points). + self._sacc_data_rad = self._radian_theta_copy(self.sacc_data) self._assert_theory_order_matches_data() return x, data_vector - def _convert_real_theta_tags_to_radians(self): - """Scale the ``theta`` tag arcmin→rad for every ``real``-category point. + def _radian_theta_copy(self, sacc_data): + """A copy of ``sacc_data`` with ``real``-category θ tags in radians. The theory spline is built on ``block[section, "theta"]`` in radians, - so the ``theta`` tag each point is evaluated at must be radians too. - Scoped to data types whose category (``sections_for_names[dt][0]``) is - ``real``: cosebis ``n`` tags and spectrum ``ell`` tags are unit-free - and must not be touched, and only the ``theta`` tag is converted - (``theta_nom`` etc. are metadata the likelihood never evaluates). + so the ``theta`` tag each point is evaluated against must be radians + too. Scoped to data types whose category + (``sections_for_names[dt][0]``) is ``real``: cosebis ``n`` tags and + spectrum ``ell`` tags are unit-free and left untouched, and only the + ``theta`` tag is scaled (``theta_nom`` etc. are metadata the likelihood + never evaluates). Operates on a ``.copy()`` so the original stays + arcmin for the save paths. """ real_types = { dt for dt, (category, _section) in self.sections_for_names.items() if category == "real" } - for point in self.sacc_data.data: + converted = sacc_data.copy() + for point in converted.data: if point.data_type in real_types and "theta" in point.tags: point.tags["theta"] = point.tags["theta"] * ARCMIN_TO_RAD + return converted + + def extract_theory_points(self, block): + """Extract theory against the radian-θ copy, then restore the original. + + Upstream ``extract_theory_points`` reads ``self.sacc_data`` (the θ tag + per point) to evaluate the theory spline; that read needs radians. + Swap in ``self._sacc_data_rad`` for the duration of the upstream call + and restore in ``finally`` so everything else — including the + save_theory / save_realization copies that run afterward in + ``do_likelihood`` — sees the untouched arcmin ``self.sacc_data``. + """ + original = self.sacc_data + self.sacc_data = self._sacc_data_rad + try: + return super().extract_theory_points(block) + finally: + self.sacc_data = original def _assert_theory_order_matches_data(self): """Require the theory-loop order to equal the data-vector order. diff --git a/src/sp_validation/tests/test_sacc_like.py b/src/sp_validation/tests/test_sacc_like.py index f8c6c3b2..8e8c1332 100644 --- a/src/sp_validation/tests/test_sacc_like.py +++ b/src/sp_validation/tests/test_sacc_like.py @@ -429,18 +429,78 @@ def test_theta_conversion_scoped_to_real_types(tmp_path, m_shim): ) ) - xi_thetas = [ - p.tags["theta"] for p in config.sacc_data.data if p.data_type == sacc_io.XI_PLUS + # The conversion lives on the radian copy (_sacc_data_rad); the original + # self.sacc_data stays arcmin (so save_theory writes arcmin tags — Finding 1). + rad_xi_thetas = [ + p.tags["theta"] + for p in config._sacc_data_rad.data + if p.data_type == sacc_io.XI_PLUS + ] + rad_cosebi_ns = [ + p.tags["n"] + for p in config._sacc_data_rad.data + if p.data_type == sacc_io.COSEBI_EE ] - cosebi_ns = [ - p.tags["n"] for p in config.sacc_data.data if p.data_type == sacc_io.COSEBI_EE + orig_xi_thetas = [ + p.tags["theta"] for p in config.sacc_data.data if p.data_type == sacc_io.XI_PLUS ] - # ξ theta converted to radians (original arcmin × ARCMIN_TO_RAD). + # rad copy: ξ theta scaled to radians (original arcmin × ARCMIN_TO_RAD). np.testing.assert_allclose( - np.sort(xi_thetas), np.sort(theta * ARCMIN_TO_RAD), rtol=1e-12 + np.sort(rad_xi_thetas), np.sort(theta * ARCMIN_TO_RAD), rtol=1e-12 ) - # cosebi n tags untouched (still the integer modes 1..n_modes). - np.testing.assert_array_equal(np.sort(cosebi_ns), np.arange(1, n_modes + 1)) + # rad copy: cosebi n tags untouched (still integer modes 1..n_modes). + np.testing.assert_array_equal(np.sort(rad_cosebi_ns), np.arange(1, n_modes + 1)) + # original sacc_data: ξ theta still in arcmin (unmutated). + np.testing.assert_allclose(np.sort(orig_xi_thetas), np.sort(theta), rtol=1e-12) + + +# --------------------------------------------------------------------------- +# 6b. save_theory writes arcmin tags, and re-execute is stable (Finding 1) +# --------------------------------------------------------------------------- +def test_save_theory_writes_arcmin_and_reexecute_stable(tmp_path, m_shim): + """save_theory must write a SACC whose θ tags are still arcmin, not radians. + + Finding 1: because upstream save_theory copies self.sacc_data and overwrites + only point values, self.sacc_data must stay arcmin — otherwise the saved file + carries radian θ tags and any arcmin-assuming consumer (sacc_io.get_xi, or + re-ingesting it as a data_file, which would double-convert to ~8.5e-8) is + silently off by 3437×. Assert the saved θ tags match the input file's tags + exactly (arcmin) and that the saved values equal the theory vector. Also run + execute() twice and require identical χ² — a guard against any accidental + double-conversion creeping back in. + """ + s, theta = _realistic_sacc(seed=5) + sacc_path = str(tmp_path / "in.sacc") + sacc_io.save(s, sacc_path) + input_theta = np.array( + [p.tags["theta"] for p in sacc_io.load(sacc_path).data if "theta" in p.tags] + ) + + save_path = str(tmp_path / "saved_theory.sacc") + opt = _as_option_block(_shim_opts(sacc_path, save_theory=save_path)) + config = m_shim.setup(opt) + + block1 = _theory_block() + m_shim.execute(block1, config) + chi2_1 = block1["data_vector", f"{LIKE_NAME}_CHI2"] + theory = np.asarray(block1["data_vector", f"{LIKE_NAME}_theory"]) + + saved = sacc_io.load(save_path) + saved_theta = np.array([p.tags["theta"] for p in saved.data if "theta" in p.tags]) + saved_values = np.array(saved.mean) + + # θ tags in the saved file are arcmin — identical to the input file's tags. + np.testing.assert_array_equal(saved_theta, input_theta) + # and are NOT the radian conversion (guards against the leak explicitly). + assert not np.allclose(saved_theta, input_theta * ARCMIN_TO_RAD) + # saved values are the theory vector (save_theory overwrites values in order). + np.testing.assert_allclose(saved_values, theory, rtol=1e-12) + + # A second execute() yields the identical χ² — no cumulative mutation. + block2 = _theory_block() + m_shim.execute(block2, config) + chi2_2 = block2["data_vector", f"{LIKE_NAME}_CHI2"] + np.testing.assert_allclose(chi2_2, chi2_1, rtol=1e-12) # --------------------------------------------------------------------------- From becfeb2f328d15ecfb27f54b47be7ef9b0ce1ee1 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 17:07:30 +0200 Subject: [PATCH 31/47] fix(inference): bind ini templates as inference_prep inputs, not params MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review finding (MEDIUM): the two pipeline ini templates were referenced as params, so there was no DAG edge — editing a template never regenerated the per-version config files. Bind both as rule inputs (referenced via input.* in the run block). The templates are source files, so anchor them on the running checkout (INFERENCE_TEMPLATE_DIR, off the workflow dir's parent) rather than the env-overridable COSMO_INFERENCE output root; the generated per-version configs still land in COSMO_INFERENCE (INFERENCE_CONFIG_OUT). In a normal run the two roots coincide; the split is what lets a template edit in this checkout drive the DAG. The dry-run guard now asserts both templates appear as inference_prep inputs. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EnV8NPWhkxS2SxGVgGSJyt --- .../tests/test_bmodes_workflow_dry_run.py | 4 +++ workflow/rules/inference.smk | 27 +++++++++++++------ 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/src/sp_validation/tests/test_bmodes_workflow_dry_run.py b/src/sp_validation/tests/test_bmodes_workflow_dry_run.py index b9486b5d..91a1923d 100644 --- a/src/sp_validation/tests/test_bmodes_workflow_dry_run.py +++ b/src/sp_validation/tests/test_bmodes_workflow_dry_run.py @@ -115,6 +115,10 @@ def test_cosmo_val_inference_prep_dry_runs(): assert "rule inference_fiducial:" in out, out # inference_prep consumes the assembled analysis SACC (not per-sign xi FITS). assert f"{version}.sacc" in out, out + # ...and both ini TEMPLATES, bound as inputs so a template edit regenerates the + # configs (Finding 2 — a template as params gave no DAG edge). + assert "cosmosis_pipeline_A_ia.ini" in out, out + assert "cosmosis_pipeline_A_ia_sacc.ini" in out, out # It emits the converter FITS + both engine inis. assert f"cosmosis_{version}.fits" in out, out assert f"cosmosis_pipeline_{version}_A_ia.ini" in out, out diff --git a/workflow/rules/inference.smk b/workflow/rules/inference.smk index 9c73c52a..ab7b456c 100644 --- a/workflow/rules/inference.smk +++ b/workflow/rules/inference.smk @@ -74,7 +74,15 @@ def pseudo_cl_assets(version): # The glass-mock rules below stay cosmosis_fitting.py-based; their SACC migration # is out of scope for PR 7. # --------------------------------------------------------------------------- -INFERENCE_TEMPLATES = COSMO_INFERENCE_PROD / "cosmosis_config" +# Generated per-version configs land in the (env-overridable) output root. +INFERENCE_CONFIG_OUT = COSMO_INFERENCE_PROD / "cosmosis_config" +# The ini TEMPLATES are source files: anchor them on the running checkout (repo +# root = the workflow dir's parent, via WORKFLOW_SCRIPTS), NOT on the output root +# — so a template edit in this checkout drives the DAG even when COSMO_INFERENCE +# points elsewhere. In a normal (non-worktree) run the two roots coincide. +INFERENCE_TEMPLATE_DIR = ( + Path(os.path.dirname(WORKFLOW_SCRIPTS)).parent / "cosmo_inference" / "cosmosis_config" +) def _csl_dir(): @@ -94,20 +102,23 @@ rule inference_prep: # The terminal assembled analysis SACC (cosmo_val.smk assemble_sacc). Bound # lazily through its helper so the filename tracks that rule, not a literal. sacc=lambda w: cv_analysis_sacc(w.version), + # The two pipeline ini templates are static repo files, but binding them as + # inputs (not params) puts them in the DAG, so editing a template + # regenerates the configs rather than leaving stale output on disk. + template_2pt=str(INFERENCE_TEMPLATE_DIR / "cosmosis_pipeline_A_ia.ini"), + template_sacc=str(INFERENCE_TEMPLATE_DIR / "cosmosis_pipeline_A_ia_sacc.ini"), output: fits_file=str(COSMO_INFERENCE_PROD / "data/{version}/cosmosis_{version}.fits"), config_file_2pt=str( - INFERENCE_TEMPLATES / "cosmosis_pipeline_{version}_A_ia.ini" + INFERENCE_CONFIG_OUT / "cosmosis_pipeline_{version}_A_ia.ini" ), config_file_sacc=str( - INFERENCE_TEMPLATES / "cosmosis_pipeline_{version}_A_ia_sacc.ini" + INFERENCE_CONFIG_OUT / "cosmosis_pipeline_{version}_A_ia_sacc.ini" ), params: # SCRATCH = the per-version chain output root the generated inis point at. scratch=lambda w: f"{CHAINS_DIR}/{w.version}", cosmosis_dir=lambda w: _csl_dir(), - template_2pt=str(INFERENCE_TEMPLATES / "cosmosis_pipeline_A_ia.ini"), - template_sacc=str(INFERENCE_TEMPLATES / "cosmosis_pipeline_A_ia_sacc.ini"), threads: 1 resources: mem_mb=8000, @@ -124,7 +135,7 @@ rule inference_prep: # (a) converter 2pt-FITS — pure ξ (A_ia scope; no rho/tau sidecars). sacc_to_twopoint_fits(sacc_io.load(input.sacc), output.fits_file, n_bins=1) - # (b) + (c) the two generated pipeline inis, from the existing templates. + # (b) + (c) the two generated pipeline inis, from the template inputs. # WORKFLOW_SCRIPTS (common.py) is the absolute generic-workflow scripts dir. sys.path.insert(0, WORKFLOW_SCRIPTS) from generate_inference_config import ( @@ -133,7 +144,7 @@ rule inference_prep: ) generate_inference_config( - params.template_2pt, + input.template_2pt, output.config_file_2pt, _substitutions( scratch=params.scratch, @@ -142,7 +153,7 @@ rule inference_prep: ), ) generate_inference_config( - params.template_sacc, + input.template_sacc, output.config_file_sacc, _substitutions( scratch=params.scratch, From 151d997ee0ea74e8c35a1c16a2cc023492c3e20f Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 17:24:48 +0200 Subject: [PATCH 32/47] fix(sacc_like): defer cosmosis import so bare module import needs no cosmosis MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI (test_imports.py::test_package_module_imports[sp_validation.sacc_like_unions]) failed: the CI image has the science stack but no cosmosis, and the shim imported `from cosmosis.datablock import SectionOptions, option_section` at module level. cosmosis is an optional dependency, so a bare `import sp_validation.sacc_like_unions` must succeed without it (test_imports bare-imports every top-level module). Move the cosmosis import into setup() (call time). Nothing at top level touches cosmosis now — only os/sys/numpy (all in the image). The factory's subclass of the upstream SaccClLikelihood already deferred (setup imports it from csl_dir), so setup() is the single cosmosis entry point. Verified: in the container (numpy present, cosmosis genuinely absent — the CI condition), `import sp_validation.sacc_like_unions` succeeds with setup/execute/ cleanup/ARCMIN_TO_RAD all present, and sacc_like_unions is absent from the import-sweep failures. Full function intact: test_sacc_like.py 8 passed under the venv (cosmosis present). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EnV8NPWhkxS2SxGVgGSJyt --- src/sp_validation/sacc_like_unions.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/sp_validation/sacc_like_unions.py b/src/sp_validation/sacc_like_unions.py index 5aa610ec..a25837cb 100644 --- a/src/sp_validation/sacc_like_unions.py +++ b/src/sp_validation/sacc_like_unions.py @@ -54,7 +54,12 @@ import sys import numpy as np -from cosmosis.datablock import SectionOptions, option_section + +# cosmosis is an OPTIONAL dependency: this module is a CosmoSIS module file, but +# importing it must succeed without cosmosis installed (the CI image has the +# science stack but no cosmosis, and test_imports.py bare-imports every module). +# So every cosmosis touch is deferred into setup() — nothing at top level imports +# it. numpy is fine at top level (always present). # arcmin → radian: the conversion 2pt_like applies to real-space data and that # sacc_like omits. Applied only to `theta` tags of `real`-category data types. @@ -195,8 +200,12 @@ def setup(options): Mirrors ``GaussianLikelihood.build_module``'s setup: wrap the raw options in ``SectionOptions`` and instantiate the likelihood (whose ``__init__`` calls ``build_data``). The one addition is reading ``csl_dir`` from the module - options to locate and import the upstream class before subclassing it. + options to locate and import the upstream class before subclassing it. The + cosmosis import is deferred to here (call time) so importing this module never + requires cosmosis. """ + from cosmosis.datablock import SectionOptions, option_section + csl_dir = options.get_string(option_section, "csl_dir") sacc_like = _import_upstream_sacc_like(csl_dir) likelihood_class = _make_subclass(sacc_like) From 64b2b5f8247b0f976f80bbab2e23400b3d40a850 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 11:33:59 +0200 Subject: [PATCH 33/47] =?UTF-8?q?feat:=20SACC=20=E2=86=92=202pt-FITS=20con?= =?UTF-8?q?verter=20+=20OneCovariance=20file-format=20glue?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds twopoint_convert (SACC → 2pt-FITS, byte-compatible with cosmosis_fitting) and one_covariance_io (SACC ↔ OneCovariance glue), with their test suites. Rebuilt on feat/sacc-2-sacc-io so the diff is converter-only; adapted to that branch's canonical sacc_io — ξ reads use grid='reporting' (was 'coarse'). Optional-block probing (pseudo-Cℓ, τ) stays on raw s.indices behind get_data_types/use_rho_tau guards, so the new fail-loud sacc_io readers don't trip on absent blocks. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WzUt7VbtXwr2SCHUdiQTyt --- src/sp_validation/one_covariance_io.py | 256 ++++++++++ .../tests/test_one_covariance_io.py | 274 ++++++++++ .../tests/test_twopoint_convert.py | 472 ++++++++++++++++++ .../tests/test_twopoint_convert_realdata.py | 268 ++++++++++ src/sp_validation/twopoint_convert.py | 368 ++++++++++++++ 5 files changed, 1638 insertions(+) create mode 100644 src/sp_validation/one_covariance_io.py create mode 100644 src/sp_validation/tests/test_one_covariance_io.py create mode 100644 src/sp_validation/tests/test_twopoint_convert.py create mode 100644 src/sp_validation/tests/test_twopoint_convert_realdata.py create mode 100644 src/sp_validation/twopoint_convert.py diff --git a/src/sp_validation/one_covariance_io.py b/src/sp_validation/one_covariance_io.py new file mode 100644 index 00000000..4c0f2ae4 --- /dev/null +++ b/src/sp_validation/one_covariance_io.py @@ -0,0 +1,256 @@ +"""ONE_COVARIANCE_IO. + +:Name: one_covariance_io.py + +:Description: File-format glue between the SACC data-product layout + (:mod:`sp_validation.sacc_io`) and OneCovariance + (https://github.com/rreischke/OneCovariance). Two directions: + + - **n(z) SACC -> OneCovariance input** (:func:`write_nz`): the + ``source_i`` NZ tracers of an analysis SACC are written as the + combined whitespace-delimited redshift file OneCovariance reads + (column 0 = z grid, then one ``n(z)`` column per tomographic + bin, no bin edges), and a matching ``[redshift]`` config stanza + is returned via :func:`nz_config_stanza`. + + - **OneCovariance output -> SACC covariance blocks** + (:func:`covariance_blocks`): the flat ``covariance_list_*.dat`` + table OneCovariance emits (one row per element pair) is reshaped + into dense square block(s) — reusing + :func:`sp_validation.statistics.cov_from_one_covariance` for the + per-block reshape — and paired with SACC selectors so a caller + can feed them straight to + :func:`sp_validation.sacc_io.assemble_covariance`. + + OneCovariance itself is *not* a dependency: this module only + touches its file formats, verified against the upstream + ``config.ini`` (``rreischke/OneCovariance`` @ main). + + n(z) file format (upstream ``config.ini`` comment, verbatim): + + ``redshift n_1(z) ... n_{N_source}(z)`` + + i.e. a plain whitespace-delimited text file, column 0 the shared + redshift grid and one column per tomographic bin — no ``z_low``/ + ``z_high`` edges (this is the OneCovariance convention, distinct + from the CosmoSIS NZDATA table which *does* carry edges). All + source bins must therefore share one z grid. + + ``[redshift]`` config keys (upstream canonical names): a single + combined file goes in ``zlens_directory`` + ``zlens_file``; + ``value_loc_in_lensbin`` (``mid``/``left``/``right``) says where + in each histogram bin the tabulated ``n(z)`` value sits — ``mid`` + for the bin-centred grids the SACC stores. NOTE: the UNIONS + OneCovariance template driven by + ``cosmo_val/pseudo_cl.py._modify_onecov_config`` writes the older + key names ``z_directory``/``zlens_file`` instead; pass + ``dir_key="z_directory"`` to match that template. +""" + +import os + +import numpy as np + +from . import sacc_io +from .statistics import cov_from_one_covariance + + +def nz_table(s, n_bins): + """Stack the SACC ``source_i`` NZ tracers into a OneCovariance n(z) table. + + Parameters + ---------- + s : sacc.Sacc + SACC holding ``source_0 … source_{n_bins-1}`` NZ tracers. + n_bins : int + Number of tomographic source bins to write. + + Returns + ------- + numpy.ndarray + Array of shape ``(n_z, n_bins + 1)``: column 0 the shared redshift + grid, columns ``1 … n_bins`` the per-bin ``n(z)``. This is the + OneCovariance combined-file layout (``redshift n_1(z) … n_N(z)``). + + Raises + ------ + ValueError + If any source bin is missing, or if the bins do not share one z grid + (OneCovariance's combined file has a single redshift column, so the + grids must agree bin-for-bin). + """ + z0, nz0 = sacc_io.get_nz(s, 0) + z0 = np.asarray(z0, dtype=float) + columns = [z0] + for i in range(n_bins): + if sacc_io.source_name(i) not in s.tracers: + raise ValueError( + f"SACC has no NZ tracer {sacc_io.source_name(i)!r}; cannot write " + f"a {n_bins}-bin OneCovariance n(z) file" + ) + z_i, nz_i = sacc_io.get_nz(s, i) + if not np.array_equal(np.asarray(z_i, dtype=float), z0): + raise ValueError( + f"source bin {i} n(z) grid differs from source bin 0; the " + "OneCovariance combined n(z) file has one shared redshift column" + ) + columns.append(np.asarray(nz_i, dtype=float)) + return np.column_stack(columns) + + +def write_nz(s, path, n_bins, *, dir_key="zlens_directory", header=True): + """Write the OneCovariance combined n(z) input file from a SACC. + + OneCovariance reads the source redshift distribution as a plain + whitespace-delimited text file whose column 0 is the shared redshift grid + and whose remaining columns are the per-bin ``n(z)`` (``redshift n_1(z) + … n_N(z)``) — no ``z_low``/``z_high`` edges. This writes that file from the + SACC ``source_i`` NZ tracers and returns the ``[redshift]`` config stanza + that points OneCovariance at it. + + Parameters + ---------- + s : sacc.Sacc + Analysis SACC with the ``source_i`` NZ tracers. + path : str or pathlib.Path + Output text-file path (overwritten). Its directory + basename become + the ``[redshift]`` directory/file config values. + n_bins : int + Number of tomographic source bins to write. + dir_key : str, optional + Config key for the redshift directory. Default ``"zlens_directory"`` + (upstream canonical). Pass ``"z_directory"`` for the UNIONS template + driven by ``pseudo_cl.py._modify_onecov_config``. + header : bool, optional + If ``True`` (default) prepend a ``# redshift n_1(z) …`` comment header + naming the columns; OneCovariance's ``genfromtxt``-style reader ignores + it. Set ``False`` for a bare numeric file. + + Returns + ------- + dict + The ``[redshift]`` config stanza (see :func:`nz_config_stanza`), naming + the file just written. + """ + table = nz_table(s, n_bins) + head = "" + if header: + cols = " ".join(f"n_{i + 1}(z)" for i in range(n_bins)) + head = f"redshift {cols}" + np.savetxt(str(path), table, header=head) + return nz_config_stanza( + os.path.dirname(os.path.abspath(str(path))), + os.path.basename(str(path)), + dir_key=dir_key, + ) + + +def nz_config_stanza( + directory, filename, *, dir_key="zlens_directory", value_loc="mid" +): + """Build the OneCovariance ``[redshift]`` config stanza for an n(z) file. + + Parameters + ---------- + directory : str + Directory holding the n(z) file (OneCovariance ``*_directory`` value). + filename : str + n(z) file basename (OneCovariance ``zlens_file`` value). + dir_key : str, optional + Directory config key — ``"zlens_directory"`` (upstream) or + ``"z_directory"`` (UNIONS template). Default ``"zlens_directory"``. + value_loc : str, optional + ``value_loc_in_lensbin`` — where in each histogram bin the tabulated + ``n(z)`` value sits (``mid``/``left``/``right``). Default ``"mid"``, + matching the bin-centred grids the SACC stores. + + Returns + ------- + dict + The ``[redshift]`` key/value pairs: ``{dir_key: directory, "zlens_file": + filename, "value_loc_in_lensbin": value_loc}``. Assign these under + ``config["redshift"]`` of a OneCovariance ``configparser`` config. + """ + if value_loc not in ("mid", "left", "right"): + raise ValueError( + f"value_loc_in_lensbin must be 'mid', 'left' or 'right'; got {value_loc!r}" + ) + return { + dir_key: directory, + "zlens_file": filename, + "value_loc_in_lensbin": value_loc, + } + + +def read_nz(path): + """Read a OneCovariance combined n(z) file back to ``(z, nz_columns)``. + + Inverse of :func:`write_nz` (the numeric round-trip; the config stanza is + not stored in the file). Comment/header lines are skipped. + + Parameters + ---------- + path : str or pathlib.Path + n(z) text file (column 0 = z, columns 1… = per-bin n(z)). + + Returns + ------- + tuple + ``(z, nz)`` where ``z`` is the shared redshift grid (shape ``(n_z,)``) + and ``nz`` is the per-bin distributions (shape ``(n_z, n_bins)``). + """ + table = np.atleast_2d(np.genfromtxt(str(path))) + return table[:, 0], table[:, 1:] + + +def covariance_blocks(cov_list, selectors, *, gaussian=True): + """Reshape a OneCovariance ``covariance_list`` table into SACC cov blocks. + + OneCovariance emits a flat ``covariance_list_*.dat`` table with one row per + ``(i, j)`` element pair (row-major, ``k = i·n + j``); the covariance value + lives in column 10 (Gaussian) or column 9 (Gaussian+non-Gaussian). This + reshapes the flat table into dense square block(s) — reusing + :func:`sp_validation.statistics.cov_from_one_covariance` for the per-block + reshape — and pairs each with its SACC selector, ready for + :func:`sp_validation.sacc_io.assemble_covariance`. + + Single-statistic case: pass the whole table and one selector; you get one + ``(selector, dense)`` block. Multi-statistic case (tomography-ready): pass a + sequence of ``(selector, sub_table)`` pairs — each ``sub_table`` a + contiguous slice of the flat output for one statistic / bin-pair — and each + is reshaped and re-paired with its selector in order. The API is thus shaped + to extend to multi-probe blocking without over-fitting the single-bin case. + + Parameters + ---------- + cov_list : numpy.ndarray or sequence + Either the flat OneCovariance table (2-D array, one row per pair) for a + single block, or — for the multi-block form — a sequence of + ``(selector, sub_table)`` pairs. In the multi-block form ``selectors`` + must be ``None`` (the selectors travel with the sub-tables). + selectors : selector or None + For the single-block form, the SACC selector for the whole table (a + ``(data_type, tracers[, tags])`` tuple or an index array, as + :func:`sacc_io.assemble_covariance` accepts). Must be ``None`` for the + multi-block form. + gaussian : bool, optional + Select the Gaussian-only column (``True``, default) or the + Gaussian+non-Gaussian column (``False``); passed straight through to + ``cov_from_one_covariance``. + + Returns + ------- + list + Ordered ``(selector, dense_cov)`` pairs, directly consumable by + ``sacc_io.assemble_covariance(s, blocks)``. + """ + if selectors is None: + # Multi-block form: cov_list is a sequence of (selector, sub_table). + return [ + (selector, cov_from_one_covariance(np.asarray(sub), gaussian=gaussian)) + for selector, sub in cov_list + ] + # Single-block form: one flat table, one selector. + return [ + (selectors, cov_from_one_covariance(np.asarray(cov_list), gaussian=gaussian)) + ] diff --git a/src/sp_validation/tests/test_one_covariance_io.py b/src/sp_validation/tests/test_one_covariance_io.py new file mode 100644 index 00000000..9bc7da56 --- /dev/null +++ b/src/sp_validation/tests/test_one_covariance_io.py @@ -0,0 +1,274 @@ +"""Tests for :mod:`sp_validation.one_covariance_io`. + +All synthetic, all fast: the OneCovariance fixtures are built in memory +shaped exactly like its real file I/O — a flat ``covariance_list`` table with +the ``(i, j)`` index rows and the Gaussian / Gauss+non-Gaussian value columns +at the indices ``cov_from_one_covariance`` expects (col 10 / col 9), and the +combined n(z) text file (column 0 = z, one column per bin, no edges). No +cluster paths; OneCovariance is not imported. + +The two pieces: + +- **Piece 1** (n(z) SACC -> OneCovariance input): write the combined n(z) + file from a SACC's ``source_i`` NZ tracers, read it back, assert the z and + n(z) columns round-trip and the config stanza names the file. +- **Piece 2** (OneCovariance output -> SACC covariance blocks): reshape the + flat table to the dense block(s) and prove they feed + ``sacc_io.assemble_covariance`` cleanly (reshaped block -> assemble -> + ``s.covariance.dense`` matches the hand-built matrix). +""" + +import numpy as np +import numpy.testing as npt +import pytest + +from sp_validation import one_covariance_io as ocio +from sp_validation import sacc_io as sio + + +# --------------------------------------------------------------------------- # +# Synthetic OneCovariance-shaped fixtures +# --------------------------------------------------------------------------- # +def _one_cov_table(cov_gauss, cov_all): + """Flatten two n x n matrices into a OneCovariance ``covariance_list`` table. + + Reproduces the real flat output: one row per ``(i, j)`` element pair in + row-major order ``k = i·n + j``, with the Gaussian value in column 10 and + the Gaussian+non-Gaussian value in column 9. Columns 0-8 and the index + columns are filled with self-documenting placeholder values (the reshape + only reads cols 9/10, but a realistic width proves it does not spill). + """ + n = cov_gauss.shape[0] + rows = [] + for i in range(n): + for j in range(n): + row = np.arange(11.0) # placeholder cols 0-8 (+ overwritten 9,10) + row[9] = cov_all[i, j] + row[10] = cov_gauss[i, j] + rows.append(row) + return np.array(rows) + + +def _spd(n, seed): + """Symmetric positive-definite matrix of size ``n`` (a valid covariance).""" + a = np.random.default_rng(seed).normal(size=(n, n)) + return a @ a.T + n * np.eye(n) + + +def _nz(seed, n=40): + rng = np.random.default_rng(seed) + z = np.linspace(0.01, 2.0, n) + return z, rng.uniform(0.1, 1.0, n) + + +# --------------------------------------------------------------------------- # +# Piece 2 — flat covariance_list -> dense block (reshape correctness + teeth) +# --------------------------------------------------------------------------- # +def test_covariance_blocks_reshapes_to_hand_built_matrix(): + """Pin the reshape and prove gaussian vs gauss+ng select different columns. + + WHAT IS PINNED: ``covariance_blocks`` flattens/reshapes the OneCovariance + ``covariance_list`` table into a dense square block matching a hand-built + covariance. It delegates the per-block reshape to + ``statistics.cov_from_one_covariance``, so column 10 (gaussian) and column 9 + (gauss+ng) must recover the two distinct hand-built matrices. + + WHY TEETH: (a) ``gaussian=True`` vs ``False`` must return the two *different* + matrices, proving the column flag is load-bearing; (b) perturbing a single + entry of the flat input must change exactly that entry of the reshaped + block, proving the reshape actually reads the table (not a constant). + """ + cov_gauss = _spd(4, seed=1) + cov_all = _spd(4, seed=2) + table = _one_cov_table(cov_gauss, cov_all) + + selector = (sio.XI_PLUS, (sio.source_name(0), sio.source_name(0))) + + [(sel_g, block_g)] = ocio.covariance_blocks(table, selector, gaussian=True) + [(sel_a, block_a)] = ocio.covariance_blocks(table, selector, gaussian=False) + + assert sel_g == selector and sel_a == selector + npt.assert_allclose(block_g, cov_gauss, rtol=1e-12) + npt.assert_allclose(block_a, cov_all, rtol=1e-12) + + # TEETH: gaussian and gauss+ng select different columns -> different blocks. + assert not np.allclose(block_g, block_a) + + # TEETH: a perturbation of one flat-table entry moves exactly that block + # entry (row k = i·n + j, col 10 for gaussian). + perturbed = table.copy() + perturbed[2 * 4 + 1, 10] += 5.0 # element (i=2, j=1) + [(_, block_p)] = ocio.covariance_blocks(perturbed, selector, gaussian=True) + npt.assert_allclose(block_p[2, 1] - block_g[2, 1], 5.0, rtol=1e-12) + block_p[2, 1] = block_g[2, 1] + npt.assert_allclose(block_p, block_g, rtol=1e-12) # nothing else moved + + +def test_covariance_blocks_multiblock_form(): + """Prove the tomography-ready multi-block form reshapes each sub-table. + + WHAT IS PINNED: passing ``selectors=None`` and a sequence of + ``(selector, sub_table)`` pairs reshapes each sub-table independently and + returns them paired with their selectors in order — the shape needed to map + a multi-statistic OneCovariance output onto several SACC selectors. + + WHY TEETH: the two sub-tables carry different matrices; if the function + reshaped only the first or mixed them, the second block would not match its + own hand-built matrix. + """ + cov_a, cov_b = _spd(3, seed=3), _spd(2, seed=4) + table_a = _one_cov_table(cov_a, _spd(3, seed=5)) + table_b = _one_cov_table(cov_b, _spd(2, seed=6)) + sel_a = (sio.XI_PLUS, (sio.source_name(0), sio.source_name(0))) + sel_b = (sio.XI_MINUS, (sio.source_name(0), sio.source_name(0))) + + blocks = ocio.covariance_blocks( + [(sel_a, table_a), (sel_b, table_b)], None, gaussian=True + ) + + assert [s for s, _ in blocks] == [sel_a, sel_b] + npt.assert_allclose(blocks[0][1], cov_a, rtol=1e-12) + npt.assert_allclose(blocks[1][1], cov_b, rtol=1e-12) + + +# --------------------------------------------------------------------------- # +# Piece 2 — the reshaped block feeds assemble_covariance cleanly +# --------------------------------------------------------------------------- # +def test_covariance_blocks_feed_assemble_covariance(): + """Round-trip: reshaped block -> assemble_covariance -> dense matches. + + WHAT IS PINNED: the ``(selector, dense)`` pair ``covariance_blocks`` + returns is directly consumable by ``sacc_io.assemble_covariance``: assembled + onto a SACC whose only statistic is one ξ+ block, ``s.covariance.dense`` + must equal the hand-built OneCovariance matrix. This is the end-to-end + contract between the two modules. + + WHY TEETH: the block must tile the data vector exactly; if the reshape + produced the wrong size or the wrong selector, ``assemble_covariance`` would + raise (its contiguity/tiling/size checks), so a clean assemble + matching + dense is a real proof. + """ + theta = np.geomspace(1.0, 100.0, 4) + xip, xim = np.arange(4) * 1e-5, np.arange(4) * 2e-5 + s = sio.new_sacc({0: _nz(0)}) + sio.add_xi(s, (0, 0), theta, xip, xim, grid="reporting") + + # The ξ block spans ξ+ then ξ− for the pair -> 8 points, one contiguous + # block (pair-major, matching sacc_io's canonical order). + cov_gauss = _spd(8, seed=7) + table = _one_cov_table(cov_gauss, _spd(8, seed=8)) + pair = (sio.source_name(0), sio.source_name(0)) + selector = np.concatenate( + [s.indices(sio.XI_PLUS, pair), s.indices(sio.XI_MINUS, pair)] + ) + + blocks = ocio.covariance_blocks(table, selector, gaussian=True) + sio.assemble_covariance(s, blocks) + + npt.assert_allclose(s.covariance.dense, cov_gauss, rtol=1e-12) + + +# --------------------------------------------------------------------------- # +# Piece 1 — n(z) SACC -> OneCovariance input (round-trip + config stanza) +# --------------------------------------------------------------------------- # +def test_write_nz_roundtrips_and_names_file(tmp_path): + """Round-trip the n(z) file and check the config stanza names it. + + WHAT IS PINNED: ``write_nz`` writes the SACC ``source_i`` NZ tracers as the + OneCovariance combined file (column 0 = z, one column per bin, no z_low/ + z_high edges). ``read_nz`` recovers the z grid and every per-bin n(z) + column, and the returned ``[redshift]`` stanza names the exact file written + (directory + basename) plus ``value_loc_in_lensbin``. + + WHY TEETH: the z grid and each n(z) column must round-trip to the values the + SACC holds (drawn from a seeded RNG); a transposed write or a dropped column + would fail the per-bin comparison. The stanza's directory/file must match + the path actually written. + """ + z, nz0 = _nz(10) + _, nz1 = _nz(11) + s = sio.new_sacc({0: (z, nz0), 1: (z, nz1)}) + + path = tmp_path / "nz_onecov.txt" + stanza = ocio.write_nz(s, path, n_bins=2) + + z_read, nz_read = ocio.read_nz(path) + npt.assert_allclose(z_read, z, rtol=1e-12) + assert nz_read.shape == (len(z), 2) + npt.assert_allclose(nz_read[:, 0], nz0, rtol=1e-12) + npt.assert_allclose(nz_read[:, 1], nz1, rtol=1e-12) + + assert stanza["zlens_directory"] == str(tmp_path) + assert stanza["zlens_file"] == "nz_onecov.txt" + assert stanza["value_loc_in_lensbin"] == "mid" + + +def test_write_nz_unions_template_dir_key(tmp_path): + """The UNIONS-template ``z_directory`` key is selectable via ``dir_key``. + + WHAT IS PINNED: the upstream OneCovariance key is ``zlens_directory``, but + the UNIONS template (pseudo_cl.py._modify_onecov_config) writes + ``z_directory``. ``dir_key="z_directory"`` produces that variant so the + stanza drops straight into the UNIONS template's ``[redshift]`` section. + """ + z, nz0 = _nz(12) + s = sio.new_sacc({0: (z, nz0)}) + stanza = ocio.write_nz(s, tmp_path / "nz.txt", n_bins=1, dir_key="z_directory") + assert "z_directory" in stanza and "zlens_directory" not in stanza + assert stanza["z_directory"] == str(tmp_path) + assert stanza["zlens_file"] == "nz.txt" + + +def test_write_nz_fails_on_mismatched_z_grids(tmp_path): + """Fail fast when source bins do not share one redshift grid. + + WHAT IS PINNED: the OneCovariance combined file has a single redshift + column, so all bins must share the z grid. A bin on a different grid must + raise ``ValueError`` at write time, not silently mis-align. + """ + z0, nz0 = _nz(20) + z1_shifted, nz1 = _nz(21) + z1_shifted = z1_shifted + 0.1 # different grid + s = sio.new_sacc({0: (z0, nz0), 1: (z1_shifted, nz1)}) + with pytest.raises(ValueError, match="differs from source bin 0"): + ocio.write_nz(s, tmp_path / "bad.txt", n_bins=2) + + +def test_write_nz_no_header_roundtrips(tmp_path): + """The bare (header=False) file still round-trips numerically. + + WHAT IS PINNED: ``header=False`` writes a purely numeric file (no ``#`` + column-name line); ``read_nz`` recovers the same z grid and n(z) column, so + the header is cosmetic and never load-bearing for the numeric round-trip. + """ + z, nz0 = _nz(40) + s = sio.new_sacc({0: (z, nz0)}) + path = tmp_path / "bare.txt" + ocio.write_nz(s, path, n_bins=1, header=False) + z_read, nz_read = ocio.read_nz(path) + npt.assert_allclose(z_read, z, rtol=1e-12) + npt.assert_allclose(nz_read[:, 0], nz0, rtol=1e-12) + + +def test_nz_config_stanza_rejects_bad_value_loc(): + """``value_loc_in_lensbin`` outside {mid,left,right} fails fast. + + WHAT IS PINNED: OneCovariance only accepts ``mid``/``left``/``right`` for + the histogram-bin value location; an invalid value is a config bug and must + raise ``ValueError`` rather than write a stanza OneCovariance will reject. + """ + with pytest.raises(ValueError, match="value_loc_in_lensbin"): + ocio.nz_config_stanza("/dir", "nz.txt", value_loc="center") + + +def test_write_nz_fails_on_missing_bin(tmp_path): + """Fail fast when a requested source bin is absent from the SACC. + + WHAT IS PINNED: requesting more bins than the SACC carries is a real config + bug; ``write_nz`` raises ``ValueError`` naming the missing tracer rather + than writing a short file. + """ + z, nz0 = _nz(30) + s = sio.new_sacc({0: (z, nz0)}) + with pytest.raises(ValueError, match="source_1"): + ocio.write_nz(s, tmp_path / "short.txt", n_bins=2) diff --git a/src/sp_validation/tests/test_twopoint_convert.py b/src/sp_validation/tests/test_twopoint_convert.py new file mode 100644 index 00000000..d50e33b9 --- /dev/null +++ b/src/sp_validation/tests/test_twopoint_convert.py @@ -0,0 +1,472 @@ +"""Byte-compare tests for the SACC -> 2pt-FITS converter. + +The converter (:mod:`sp_validation.twopoint_convert`) must reproduce the CosmoSIS +2pt-FITS that ``cosmo_inference/scripts/cosmosis_fitting.py`` assembles today, +so the inference chain (``2pt_like`` and Sacha Guerrini's rho/tau +``2pt_like_xi_sys`` fork) runs untouched behind it. The strongest possible check +is *byte* equality, and astropy writes FITS deterministically, so that is what we +assert: build a reference with the current script's own HDU-builder functions on +deterministic synthetic inputs, build a SACC from those same inputs via +:mod:`sp_validation.sacc_io`, convert it, and compare the two files byte for byte. + +Three configurations pin the three product shapes today's ``__main__`` emits: + +1. **plain xi** -- PRIMARY, NZ_SOURCE, COVMAT, XI_PLUS, XI_MINUS. +2. **xi + pseudo-Cl** -- adds COVMAT_CELL and CELL_EE (the harmonic block + ``2pt_like`` reads; the script builds CELL_BB and discards it, so the + converter does too). +3. **xi + rho/tau** -- adds the blocked tau covariance (TAU_0_PLUS / TAU_2_PLUS, + with the tau_0<->tau_2 cross-correlation the truncated CosmoCov tau + covariance carries) and the verbatim RHO_STATS table. + +The rho/tau product needs the rho/tau *sidecar* HDUs: the RHO_STATS table carries +per-mode ``varrho_*`` variances the analysis SACC does not store, so the +converter copies them from the sidecar exactly as today's assembly does. A teeth +test pins that a perturbed input moves the output. + +The reference builder imports ``cosmosis_fitting.py`` by path (it is a script, +not a package module), skipping cleanly if a dependency is missing -- the same +loader pattern as ``test_cosmosis_fitting.py``. +""" + +import importlib.util +from pathlib import Path + +import numpy as np +import pytest +from astropy.io import fits + +from sp_validation import sacc_io, twopoint_convert + +_SCRIPT = ( + Path(__file__).resolve().parents[3] + / "cosmo_inference" + / "scripts" + / "cosmosis_fitting.py" +) + + +def _load_cf(): + """Import cosmosis_fitting.py by path; skip cleanly if a dep is missing.""" + if not _SCRIPT.exists(): + pytest.skip(f"cosmosis_fitting.py not found at {_SCRIPT}") + spec = importlib.util.spec_from_file_location("cosmosis_fitting", _SCRIPT) + module = importlib.util.module_from_spec(spec) + try: + spec.loader.exec_module(module) + except ImportError as exc: # pragma: no cover - container has numpy/astropy + pytest.importorskip(getattr(exc, "name", "") or "cosmosis_fitting_dependency") + raise + return module + + +cf = _load_cf() + + +# --- deterministic synthetic inputs ----------------------------------------- +# +# One tomographic bin (today's analysis). N_ANG angular bins on an ascending +# theta grid; N_ELL bandpowers; a 200-point n(z) on a uniform z grid (the DES +# NZDATA table needs a uniform Z_MID axis). The xi covariance is a full +# (2*N_ANG) matrix (xi+/xi- cross-block nonzero, as CosmoCov produces); the tau +# covariance is a full (3*N_ANG) matrix that the assembly truncates to its first +# 2 statistics (tau_0, tau_2). + +N_ANG = 5 +N_ELL = 8 +SOURCE = sacc_io.source_name(0) +PSF = sacc_io.PSF_TRACER + + +def _spd(n, seed): + """A symmetric positive-definite (n, n) matrix, seeded and recognizable.""" + rng = np.random.default_rng(seed) + a = rng.normal(size=(n, n)) + return a @ a.T + n * np.eye(n) + + +def _inputs(seed=0): + """Deterministic synthetic statistics + covariances for one bin pair.""" + rng = np.random.default_rng(seed) + theta = np.sort(rng.uniform(1.0, 250.0, N_ANG)) + ell = np.sort(rng.uniform(30.0, 3000.0, N_ELL)) + z = np.linspace(0.0125, 0.4875, 200) + return { + "theta": theta, + "ell": ell, + "z": z, + "nz": np.exp(-((z - 0.25) ** 2) / 0.02), + "xip": rng.uniform(1e-6, 1e-4, N_ANG), + "xim": rng.uniform(1e-6, 1e-4, N_ANG), + "cl_ee": rng.uniform(1e-10, 1e-8, N_ELL), + "cl_bb": rng.uniform(1e-11, 1e-9, N_ELL), + "cl_eb": rng.uniform(-1e-11, 1e-11, N_ELL), + "tau0p": rng.uniform(1e-6, 1e-5, N_ANG), + "tau2p": rng.uniform(1e-6, 1e-5, N_ANG), + "tau0m": rng.uniform(1e-6, 1e-5, N_ANG), + "tau2m": rng.uniform(1e-6, 1e-5, N_ANG), + "xi_cov": _spd(2 * N_ANG, seed + 1), + "cl_cov": _spd(N_ELL, seed + 2), + "tau_cov_full": _spd(3 * N_ANG, seed + 3), + } + + +# --- reference sidecar files + HDUs (the current script's own builders) ------ + + +def _rho_sidecar_hdu(theta, seed=7): + """A rho-stats BinTableHDU with the 25-column layout (values + variances).""" + rng = np.random.default_rng(seed) + columns = [fits.Column(name="theta", format="D", array=theta)] + for k in range(6): + for suffix in ("_p", "_m"): + columns.append( + fits.Column( + name=f"rho_{k}{suffix}", + format="D", + array=rng.uniform(-1e-3, 1e-3, len(theta)), + ) + ) + columns.append( + fits.Column( + name=f"varrho_{k}{suffix}", + format="D", + array=rng.uniform(1e-15, 1e-12, len(theta)), + ) + ) + return fits.BinTableHDU.from_columns(fits.ColDefs(columns)) + + +def _tau_sidecar_hdu(theta, tau0p, tau2p): + """A tau-stats BinTableHDU with theta + tau_0_p + tau_2_p columns.""" + columns = [ + fits.Column(name="theta", format="D", array=theta), + fits.Column(name="tau_0_p", format="D", array=tau0p), + fits.Column(name="tau_2_p", format="D", array=tau2p), + ] + return fits.BinTableHDU.from_columns(fits.ColDefs(columns)) + + +def _reference_fits(tmp_path, inp, *, cl=False, rho_tau=False): + """Build the reference 2pt-FITS with cosmosis_fitting.py's own functions. + + Reproduces the exact ``__main__`` HDU list for the requested configuration, + which is the assembly the converter must match byte for byte. + """ + nz_txt = tmp_path / "nz.txt" + np.savetxt(nz_txt, np.column_stack([inp["z"], inp["nz"]])) + cov_txt = tmp_path / "cov_xi.txt" + np.savetxt(cov_txt, inp["xi_cov"]) + + nz_hdu = cf.nz_to_fits(str(nz_txt)) + xip_hdu = cf._create_2pt_hdu(inp["xip"], inp["theta"], "XI_PLUS", "G+R", "G+R") + xim_hdu = cf._create_2pt_hdu(inp["xim"], inp["theta"], "XI_MINUS", "G-R", "G-R") + + hdu_list = [fits.PrimaryHDU(), nz_hdu] + + if rho_tau: + tau_cov_npy = tmp_path / "cov_tau.npy" + np.save(tau_cov_npy, inp["tau_cov_full"]) + cov_hdu = cf.covdat_to_fits(str(cov_txt), filename_cov_tau=str(tau_cov_npy)) + else: + cov_hdu = cf.covdat_to_fits(str(cov_txt), filename_cov_tau=None) + hdu_list.append(cov_hdu) + + if cl: + cl_block = np.zeros((5, N_ELL)) + cl_block[0], cl_block[1], cl_block[4] = inp["ell"], inp["cl_ee"], inp["cl_bb"] + cl_npy = tmp_path / "cl.npy" + np.save(cl_npy, cl_block) + cl_cov_npy = tmp_path / "cl_cov.npy" + np.save(cl_cov_npy, inp["cl_cov"]) + ell_r, cl_ee_r, cl_bb_r = cf.load_pseudo_cl(str(cl_npy)) + cl_ee_hdu, _cl_bb_hdu = cf.cl_to_fits(ell_r, cl_ee_r, cl_bb_r) + cov_cl_hdu = cf.cov_cl_to_fits(str(cl_cov_npy), cov_hdu="COVAR_FULL") + hdu_list.append(cov_cl_hdu) + + hdu_list.extend([xip_hdu, xim_hdu]) + if cl: + hdu_list.append(cl_ee_hdu) + + if rho_tau: + rho_path = tmp_path / "rho.fits" + fits.HDUList([fits.PrimaryHDU(), _rho_sidecar_hdu(inp["theta"])]).writeto( + rho_path, overwrite=True + ) + tau_path = tmp_path / "tau.fits" + fits.HDUList( + [ + fits.PrimaryHDU(), + _tau_sidecar_hdu(inp["theta"], inp["tau0p"], inp["tau2p"]), + ] + ).writeto(tau_path, overwrite=True) + rho_hdu = cf.rho_to_fits(str(rho_path), theta=inp["theta"]) + tau0_hdu, tau2_hdu = cf.tau_to_fits(str(tau_path), theta=inp["theta"]) + hdu_list.extend([tau0_hdu, tau2_hdu, rho_hdu]) + + out = tmp_path / "reference.fits" + fits.HDUList(hdu_list).writeto(out, overwrite=True) + return out + + +# --- the SACC each configuration is built from ------------------------------ + + +def _sacc(inp, *, cl=False, rho_tau=False): + """Build the analysis SACC the converter reads, matching ``_inputs``. + + The covariance is laid out to match the reference exactly: the xi block is + the full (2*N_ANG) matrix; the Cl block is the EE bandpower covariance; the + tau blocks carry the tau_0<->tau_2 cross-correlation from the truncated + CosmoCov tau covariance. Blocks not consumed by the 2pt-FITS (Cl BB/EB, tau + minus) get an identity block so ``add_covariance`` sees a full matrix. + """ + s = sacc_io.new_sacc({0: (inp["z"], inp["nz"])}) + sacc_io.add_xi(s, (0, 0), inp["theta"], inp["xip"], inp["xim"], grid="reporting") + if cl: + sacc_io.add_pseudo_cl( + s, + (0, 0), + inp["ell"], + inp["cl_ee"], + inp["cl_bb"], + inp["cl_eb"], + window_ells=np.arange(2, 102), + window_weights=np.random.default_rng(9).uniform(0, 1, (100, N_ELL)), + ) + if rho_tau: + sacc_io.add_tau(s, (0, 0), 0, inp["theta"], inp["tau0p"], inp["tau0m"]) + sacc_io.add_tau(s, (0, 0), 2, inp["theta"], inp["tau2p"], inp["tau2m"]) + + n = len(s.mean) + full = np.zeros((n, n)) + ip = s.indices(sacc_io.XI_PLUS, (SOURCE, SOURCE)) + im = s.indices(sacc_io.XI_MINUS, (SOURCE, SOURCE)) + xi_all = np.concatenate([ip, im]) + full[np.ix_(xi_all, xi_all)] = inp["xi_cov"] + + if cl: + iee = s.indices(sacc_io.CL_EE, (SOURCE, SOURCE)) + full[np.ix_(iee, iee)] = inp["cl_cov"] + for dtype in (sacc_io.CL_BB, sacc_io.CL_EB): + idx = s.indices(dtype, (SOURCE, SOURCE)) + full[np.ix_(idx, idx)] = np.eye(N_ELL) + if rho_tau: + t0p = s.indices(sacc_io.TAU_PLUS.format(k=0), (SOURCE, PSF)) + t2p = s.indices(sacc_io.TAU_PLUS.format(k=2), (SOURCE, PSF)) + tau_pp = np.concatenate([t0p, t2p]) + # The joint [tau_0+; tau_2+] block is the truncated CosmoCov tau + # covariance -- cross-correlation kept, matching covdat_to_fits. + full[np.ix_(tau_pp, tau_pp)] = inp["tau_cov_full"][: 2 * N_ANG, : 2 * N_ANG] + for dtype in (sacc_io.TAU_MINUS.format(k=0), sacc_io.TAU_MINUS.format(k=2)): + idx = s.indices(dtype, (SOURCE, PSF)) + full[np.ix_(idx, idx)] = np.eye(N_ANG) + s.add_covariance(full) + return s + + +def _sidecar_hdus(tmp_path, inp): + """Return the (rho_hdu, tau_hdu) sidecar input HDUs for the rho/tau product.""" + rho_path = tmp_path / "rho_in.fits" + fits.HDUList([fits.PrimaryHDU(), _rho_sidecar_hdu(inp["theta"])]).writeto( + rho_path, overwrite=True + ) + tau_path = tmp_path / "tau_in.fits" + fits.HDUList( + [fits.PrimaryHDU(), _tau_sidecar_hdu(inp["theta"], inp["tau0p"], inp["tau2p"])] + ).writeto(tau_path, overwrite=True) + with fits.open(rho_path) as r, fits.open(tau_path) as t: + return r[1].copy(), t[1].copy() + + +# ============================================================================= +# Byte-compare: the three product shapes +# ============================================================================= + + +def test_plain_xi_byte_equal(tmp_path): + """Plain-xi product: converter matches cosmosis_fitting.py byte for byte.""" + inp = _inputs(seed=0) + reference = _reference_fits(tmp_path, inp) + s = _sacc(inp) + out = tmp_path / "converted.fits" + twopoint_convert.sacc_to_twopoint_fits(s, str(out), n_bins=1) + assert out.read_bytes() == reference.read_bytes() + + +def test_xi_cl_byte_equal(tmp_path): + """xi + pseudo-Cl product: COVMAT_CELL + CELL_EE reproduced byte for byte.""" + inp = _inputs(seed=10) + reference = _reference_fits(tmp_path, inp, cl=True) + s = _sacc(inp, cl=True) + out = tmp_path / "converted.fits" + twopoint_convert.sacc_to_twopoint_fits(s, str(out), n_bins=1) + assert out.read_bytes() == reference.read_bytes() + + +def test_xi_rho_tau_byte_equal(tmp_path): + """xi + rho/tau product: blocked tau covariance + verbatim RHO_STATS match.""" + inp = _inputs(seed=20) + reference = _reference_fits(tmp_path, inp, rho_tau=True) + s = _sacc(inp, rho_tau=True) + rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) + out = tmp_path / "converted.fits" + twopoint_convert.sacc_to_twopoint_fits( + s, str(out), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 + ) + assert out.read_bytes() == reference.read_bytes() + + +# ============================================================================= +# Structural + teeth checks +# ============================================================================= + + +def test_tau_covariance_keeps_tau0_tau2_cross(tmp_path): + """The tau covariance block couples tau_0 and tau_2 (not block-diagonal). + + covdat_to_fits truncates the 3-statistic CosmoCov tau covariance to its + first 2 blocks and lays it in as ONE contiguous block, so tau_0<->tau_2 + cross-terms survive. A block-diagonal shortcut would zero them; pin that + the converter keeps them. + """ + inp = _inputs(seed=20) + s = _sacc(inp, rho_tau=True) + rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) + out = tmp_path / "converted.fits" + twopoint_convert.sacc_to_twopoint_fits( + s, str(out), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 + ) + with fits.open(out) as hdul: + cov = hdul["COVMAT"].data + # tau_0 block rows 2*N_ANG..3*N_ANG, tau_2 block 3*N_ANG..4*N_ANG. + cross = cov[2 * N_ANG : 3 * N_ANG, 3 * N_ANG : 4 * N_ANG] + expected = inp["tau_cov_full"][:N_ANG, N_ANG : 2 * N_ANG] + assert np.array_equal(cross, expected) + assert np.any(cross != 0.0) + + +def test_perturbed_xi_changes_output(tmp_path): + """Teeth: a changed xi+ input must change the converted data vector.""" + inp = _inputs(seed=0) + s = _sacc(inp) + out = tmp_path / "base.fits" + twopoint_convert.sacc_to_twopoint_fits(s, str(out), n_bins=1) + with fits.open(out) as hdul: + base_xip = hdul["XI_PLUS"].data["VALUE"].copy() + + inp2 = _inputs(seed=0) + inp2["xip"] = inp2["xip"] + 1.0 + s2 = _sacc(inp2) + out2 = tmp_path / "perturbed.fits" + twopoint_convert.sacc_to_twopoint_fits(s2, str(out2), n_bins=1) + with fits.open(out2) as hdul: + new_xip = hdul["XI_PLUS"].data["VALUE"] + + assert not np.array_equal(base_xip, new_xip) + assert np.array_equal(new_xip, inp2["xip"]) + + +def test_rho_tau_sidecars_required_together(tmp_path): + """Supplying only one of the rho/tau sidecars is a loud error.""" + inp = _inputs(seed=20) + s = _sacc(inp, rho_tau=True) + rho_hdu, _tau_hdu = _sidecar_hdus(tmp_path, inp) + with pytest.raises(ValueError, match="together"): + twopoint_convert.sacc_to_twopoint_fits( + s, str(tmp_path / "x.fits"), rho_stats_hdu=rho_hdu, n_bins=1 + ) + + +# ============================================================================= +# Fail-fast guards and permutation teeth (adversarial-review hardening) +# ============================================================================= + + +def test_tomographic_sacc_raises(tmp_path): + """A multi-bin SACC fails fast instead of silently truncating to (0, 0). + + Review finding (HIGH): ``n_bins`` alone drove the NZDATA column count while + the data vector and covariance were read from bin ``(0, 0)`` only, so a + 2-bin SACC + ``n_bins=2`` emitted a plausible-looking FITS carrying 1/3 of + the data. Both the ``n_bins`` and the tracer-pair mismatch must raise. + """ + inp = _inputs(seed=30) + s = sacc_io.new_sacc({0: (inp["z"], inp["nz"]), 1: (inp["z"], inp["nz"])}) + for pair in [(0, 0), (0, 1), (1, 1)]: + sacc_io.add_xi(s, pair, inp["theta"], inp["xip"], inp["xim"], grid="reporting") + s.add_covariance(np.eye(len(s.mean))) + + with pytest.raises(ValueError, match="single-bin only"): + twopoint_convert.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits"), n_bins=2) + with pytest.raises(ValueError, match="single-bin only"): + twopoint_convert.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits"), n_bins=1) + assert not (tmp_path / "x.fits").exists() + + +def test_sacc_without_xi_raises(tmp_path): + """A SACC with no ξ± points raises instead of writing an empty data vector.""" + inp = _inputs(seed=31) + s = sacc_io.new_sacc({0: (inp["z"], inp["nz"])}) + sacc_io.add_pseudo_cl( + s, + (0, 0), + inp["ell"], + inp["cl_ee"], + inp["cl_bb"], + inp["cl_eb"], + window_ells=np.arange(2, 102), + window_weights=np.random.default_rng(9).uniform(0, 1, (100, N_ELL)), + ) + s.add_covariance(np.eye(len(s.mean))) + + with pytest.raises(ValueError, match="nothing to convert"): + twopoint_convert.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits")) + assert not (tmp_path / "x.fits").exists() + + +def test_covmat_blocks_exact_gather_encoded_cov(tmp_path): + """Every COVMAT/COVMAT_CELL entry is the exact ``np.ix_`` gather of the SACC + covariance, pinned with a (row, col)-encoded matrix. + + Review finding (MEDIUM): for a single bin pair the ξ gather happens to be + the identity permutation, so the byte-compares alone could pass with a + transposed or block-swapped gather. Encoding ``C[i, j] = i*n + j`` (asymmetric, + every entry unique) makes any transposition, offset, or wrong block produce + detectably wrong values; the τ gather is genuinely non-identity (τ_0− sits + between τ_0+ and τ_2+ in insertion order). Expected layout per + ``covdat_to_fits``: block_diag(ξ type-major gather, joint [τ_0+; τ_2+] + gather), with COVMAT_CELL the CELL_EE gather in its own HDU. + """ + inp = _inputs(seed=32) + s = _sacc(inp, cl=True, rho_tau=True) + n = len(s.mean) + encoded = np.arange(n * n, dtype=float).reshape(n, n) + s.add_covariance(encoded, overwrite=True) + rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) + + out = tmp_path / "encoded.fits" + twopoint_convert.sacc_to_twopoint_fits( + s, str(out), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 + ) + + pair = (SOURCE, SOURCE) + xi_idx = np.concatenate( + [s.indices(sacc_io.XI_PLUS, pair), s.indices(sacc_io.XI_MINUS, pair)] + ) + tau_idx = np.concatenate( + [ + s.indices(sacc_io.TAU_PLUS.format(k=0), (SOURCE, PSF)), + s.indices(sacc_io.TAU_PLUS.format(k=2), (SOURCE, PSF)), + ] + ) + expected = twopoint_convert._block_diag( + encoded[np.ix_(xi_idx, xi_idx)], encoded[np.ix_(tau_idx, tau_idx)] + ) + cell_idx = s.indices(sacc_io.CL_EE, pair) + + with fits.open(out) as hdul: + np.testing.assert_array_equal(hdul["COVMAT"].data, expected) + np.testing.assert_array_equal( + hdul["COVMAT_CELL"].data, encoded[np.ix_(cell_idx, cell_idx)] + ) diff --git a/src/sp_validation/tests/test_twopoint_convert_realdata.py b/src/sp_validation/tests/test_twopoint_convert_realdata.py new file mode 100644 index 00000000..1dcf920f --- /dev/null +++ b/src/sp_validation/tests/test_twopoint_convert_realdata.py @@ -0,0 +1,268 @@ +"""Candide-local byte-compare of the converter against real 2pt-FITS products. + +Skipped unless a real product exists on disk (candide only; never committed). +It closes the loop end to end on real data: take a real CosmoSIS 2pt-FITS, build +an analysis SACC from its own contents, convert that SACC back to a 2pt-FITS, and +byte-compare. + +The reference is *not* the on-disk file directly. The committed on-disk products +were written by an older ``cosmosis_fitting.py`` (they carry a CELL_BB HDU and +order COVMAT before NZ_SOURCE); the converter reproduces the *current* script, +which the PR-4 migration will use to regenerate them. So the meaningful contract +-- "the converter equals the current writer" -- is tested by running the current +``cosmosis_fitting.py`` builders on the same real contents and byte-comparing the +converter against that. For transparency the test also records the direct diff +against the stale on-disk file, and asserts only that it differs *by whole HDUs* +(the extra CELL_BB), not in any shared block -- i.e. the drift is purely the +known HDU-set change, with no silent data corruption. + +Observed on 2026-07-10 for ``SP_v1.4.6_leak_corr`` and a ``glass_mock`` sibling: +converter == current-script byte for byte; converter vs on-disk differs only by +the CELL_BB HDU. +""" + +import importlib.util +from pathlib import Path + +import numpy as np +import pytest +from astropy.io import fits + +from sp_validation import sacc_io, twopoint_convert + +_DATA = Path("/automnt/n17data/cdaley/unions/code/sp_validation/cosmo_inference/data") +_REAL_FILES = { + "SP_v1.4.6_leak_corr": _DATA + / "SP_v1.4.6_leak_corr_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1" + / "cosmosis_SP_v1.4.6_leak_corr_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits", + "glass_mock_00001": _DATA / "glass_mock_00001" / "cosmosis_glass_mock_00001.fits", +} + +_SCRIPT = ( + Path(__file__).resolve().parents[3] + / "cosmo_inference" + / "scripts" + / "cosmosis_fitting.py" +) + + +def _load_cf(): + spec = importlib.util.spec_from_file_location("cosmosis_fitting", _SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _sacc_from_2pt_fits(hdul): + """Build an analysis SACC from a real CosmoSIS 2pt-FITS's own contents. + + Reads the NZ, ξ±, pseudo-Cℓ (EE/BB) and blocked covariance back out of the + product and lays them into the standard SACC layout — the inverse direction + the converter then undoes. τ± minus and Cℓ EB are not in the 2pt-FITS, so + they are stored as zeros with identity covariance sub-blocks (the converter + consumes only the ``+``/EE parts). + """ + z = hdul["NZ_SOURCE"].data["Z_MID"].astype(float) + nz = hdul["NZ_SOURCE"].data["BIN1"].astype(float) + theta = hdul["XI_PLUS"].data["ANG"].astype(float) + xip = hdul["XI_PLUS"].data["VALUE"].astype(float) + xim = hdul["XI_MINUS"].data["VALUE"].astype(float) + n = len(theta) + + ell = hdul["CELL_EE"].data["ANG"].astype(float) + cl_ee = hdul["CELL_EE"].data["VALUE"].astype(float) + cl_bb = hdul["CELL_BB"].data["VALUE"].astype(float) + n_ell = len(ell) + + covmat = hdul["COVMAT"].data.astype(float) + covmat_cell = hdul["COVMAT_CELL"].data.astype(float) + xi_cov = covmat[: 2 * n, : 2 * n] + tau_joint = covmat[2 * n : 4 * n, 2 * n : 4 * n] + + tau0p = hdul["TAU_0_PLUS"].data["VALUE"].astype(float) + tau2p = hdul["TAU_2_PLUS"].data["VALUE"].astype(float) + + s = sacc_io.new_sacc({0: (z, nz)}) + sacc_io.add_xi(s, (0, 0), theta, xip, xim, grid="reporting") + sacc_io.add_pseudo_cl( + s, + (0, 0), + ell, + cl_ee, + cl_bb, + np.zeros(n_ell), + window_ells=np.arange(2, 102), + window_weights=np.ones((100, n_ell)), + ) + sacc_io.add_tau(s, (0, 0), 0, theta, tau0p, np.zeros(n)) + sacc_io.add_tau(s, (0, 0), 2, theta, tau2p, np.zeros(n)) + + source, psf = sacc_io.source_name(0), sacc_io.PSF_TRACER + idx = { + "xi": np.concatenate( + [ + s.indices(sacc_io.XI_PLUS, (source, source)), + s.indices(sacc_io.XI_MINUS, (source, source)), + ] + ), + "ee": s.indices(sacc_io.CL_EE, (source, source)), + "bb": s.indices(sacc_io.CL_BB, (source, source)), + "eb": s.indices(sacc_io.CL_EB, (source, source)), + "t0p": s.indices(sacc_io.TAU_PLUS.format(k=0), (source, psf)), + "t0m": s.indices(sacc_io.TAU_MINUS.format(k=0), (source, psf)), + "t2p": s.indices(sacc_io.TAU_PLUS.format(k=2), (source, psf)), + "t2m": s.indices(sacc_io.TAU_MINUS.format(k=2), (source, psf)), + } + full = np.zeros((len(s.mean), len(s.mean))) + full[np.ix_(idx["xi"], idx["xi"])] = xi_cov + full[np.ix_(idx["ee"], idx["ee"])] = covmat_cell + tau_pp = np.concatenate([idx["t0p"], idx["t2p"]]) + full[np.ix_(tau_pp, tau_pp)] = tau_joint + for key in ("bb", "eb"): + full[np.ix_(idx[key], idx[key])] = np.eye(n_ell) + for key in ("t0m", "t2m"): + full[np.ix_(idx[key], idx[key])] = np.eye(n) + s.add_covariance(full) + + tau_sidecar = fits.BinTableHDU.from_columns( + fits.ColDefs( + [ + fits.Column(name="theta", format="D", array=theta), + fits.Column(name="tau_0_p", format="D", array=tau0p), + fits.Column(name="tau_2_p", format="D", array=tau2p), + ] + ) + ) + return s, hdul["RHO_STATS"].copy(), tau_sidecar + + +def _current_script_reference(cf, hdul, tmp_path): + """Reference: run the current cosmosis_fitting.py on the real file's contents.""" + z = hdul["NZ_SOURCE"].data["Z_MID"].astype(float) + nz = hdul["NZ_SOURCE"].data["BIN1"].astype(float) + theta = hdul["XI_PLUS"].data["ANG"].astype(float) + xip = hdul["XI_PLUS"].data["VALUE"].astype(float) + xim = hdul["XI_MINUS"].data["VALUE"].astype(float) + n = len(theta) + ell = hdul["CELL_EE"].data["ANG"].astype(float) + cl_ee = hdul["CELL_EE"].data["VALUE"].astype(float) + cl_bb = hdul["CELL_BB"].data["VALUE"].astype(float) + covmat = hdul["COVMAT"].data.astype(float) + covmat_cell = hdul["COVMAT_CELL"].data.astype(float) + tau0p = hdul["TAU_0_PLUS"].data["VALUE"].astype(float) + tau2p = hdul["TAU_2_PLUS"].data["VALUE"].astype(float) + + np.savetxt(tmp_path / "nz.txt", np.column_stack([z, nz])) + np.savetxt(tmp_path / "cov.txt", covmat[: 2 * n, : 2 * n]) + tau_cov = np.zeros((3 * n, 3 * n)) + tau_cov[: 2 * n, : 2 * n] = covmat[2 * n : 4 * n, 2 * n : 4 * n] + np.save(tmp_path / "cov_tau.npy", tau_cov) + cl_block = np.zeros((5, len(ell))) + cl_block[0], cl_block[1], cl_block[4] = ell, cl_ee, cl_bb + np.save(tmp_path / "cl.npy", cl_block) + np.save(tmp_path / "cl_cov.npy", covmat_cell) + + nz_hdu = cf.nz_to_fits(str(tmp_path / "nz.txt")) + xip_hdu = cf._create_2pt_hdu(xip, theta, "XI_PLUS", "G+R", "G+R") + xim_hdu = cf._create_2pt_hdu(xim, theta, "XI_MINUS", "G-R", "G-R") + cov_hdu = cf.covdat_to_fits( + str(tmp_path / "cov.txt"), filename_cov_tau=str(tmp_path / "cov_tau.npy") + ) + ell_r, cl_ee_r, cl_bb_r = cf.load_pseudo_cl(str(tmp_path / "cl.npy")) + cl_ee_hdu, _ = cf.cl_to_fits(ell_r, cl_ee_r, cl_bb_r) + cov_cl_hdu = cf.cov_cl_to_fits(str(tmp_path / "cl_cov.npy"), cov_hdu="COVAR_FULL") + + fits.HDUList([fits.PrimaryHDU(), hdul["RHO_STATS"].copy()]).writeto( + tmp_path / "rho.fits", overwrite=True + ) + rho_hdu = cf.rho_to_fits(str(tmp_path / "rho.fits"), theta=theta) + tau_sidecar = fits.BinTableHDU.from_columns( + fits.ColDefs( + [ + fits.Column(name="theta", format="D", array=theta), + fits.Column(name="tau_0_p", format="D", array=tau0p), + fits.Column(name="tau_2_p", format="D", array=tau2p), + ] + ) + ) + fits.HDUList([fits.PrimaryHDU(), tau_sidecar]).writeto( + tmp_path / "tau.fits", overwrite=True + ) + tau0_hdu, tau2_hdu = cf.tau_to_fits(str(tmp_path / "tau.fits"), theta=theta) + + out = tmp_path / "reference.fits" + fits.HDUList( + [ + fits.PrimaryHDU(), + nz_hdu, + cov_hdu, + cov_cl_hdu, + xip_hdu, + xim_hdu, + cl_ee_hdu, + tau0_hdu, + tau2_hdu, + rho_hdu, + ] + ).writeto(out, overwrite=True) + return out + + +@pytest.mark.parametrize("label", list(_REAL_FILES)) +def test_realdata_roundtrip_byte_equal(label, tmp_path): + """Converter reproduces the current writer byte for byte on a real product.""" + real = _REAL_FILES[label] + if not real.exists(): + pytest.skip(f"real 2pt-FITS not on disk: {real}") + cf = _load_cf() + + with fits.open(real) as hdul: + s, rho_hdu, tau_hdu = _sacc_from_2pt_fits(hdul) + reference = _current_script_reference(cf, hdul, tmp_path) + + converted = tmp_path / "converted.fits" + twopoint_convert.sacc_to_twopoint_fits( + s, str(converted), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 + ) + + # The contract: converter == current cosmosis_fitting.py, byte for byte. + assert converted.read_bytes() == reference.read_bytes() + + +@pytest.mark.parametrize("label", list(_REAL_FILES)) +def test_realdata_ondisk_drift_is_only_cell_bb(label, tmp_path): + """The stale on-disk file differs from the converter *only* by the CELL_BB HDU. + + Documents (and guards) the known script-version drift: the on-disk products + were written before CELL_BB was dropped from the assembly, so they carry one + extra HDU. Every HDU the two share carries the same data to floating-point + precision — the drift is a whole-HDU addition, never a silent change to a + shared block. (Bin-edge columns differ by ~1e-17 float noise: the stale file + stored a clean ``Z_LOW=0``, while ``z_mid - step/2`` rounds to ``-1.7e-18``; + both are the same number, so the shared-data check is ``allclose``, not + bitwise — bitwise equality is asserted against the *current* writer above.) + """ + real = _REAL_FILES[label] + if not real.exists(): + pytest.skip(f"real 2pt-FITS not on disk: {real}") + + with fits.open(real) as hdul: + s, rho_hdu, tau_hdu = _sacc_from_2pt_fits(hdul) + ondisk_names = [h.name for h in hdul] + converted = tmp_path / "converted.fits" + twopoint_convert.sacc_to_twopoint_fits( + s, str(converted), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 + ) + with fits.open(converted) as conv: + conv_names = [h.name for h in conv] + # The only HDU the on-disk file has that the converter does not. + assert set(ondisk_names) - set(conv_names) == {"CELL_BB"} + # Every shared table HDU carries the same data to float precision. + for name in conv_names: + if name == "PRIMARY": + continue + a, b = hdul[name].data, conv[name].data + if hasattr(a, "names"): + assert all(np.allclose(a[c], b[c]) for c in a.names), name + else: + assert np.allclose(a, b), name diff --git a/src/sp_validation/twopoint_convert.py b/src/sp_validation/twopoint_convert.py new file mode 100644 index 00000000..fc01ac9d --- /dev/null +++ b/src/sp_validation/twopoint_convert.py @@ -0,0 +1,368 @@ +"""TWOPOINT_CONVERT. + +:Name: twopoint_convert.py + +:Description: Convert an analysis SACC file into the "2pt FITS" that CosmoSIS's + ``2pt_like`` (and Sacha Guerrini's ρ/τ ``2pt_like_xi_sys`` fork) + reads. The output reproduces today's hand-assembled product from + ``cosmo_inference/scripts/cosmosis_fitting.py`` HDU-for-HDU and + byte-for-byte: an NZDATA table, XI_PLUS / XI_MINUS 2pt tables, + optional CELL_EE / CELL_BB pseudo-Cℓ tables, the blocked COVMAT + (with ``STRT_i`` block-offset headers) and separate COVMAT_CELL, + and — when the SACC carries them — the TAU_{0,2}_PLUS 2pt tables + and the RHO_STATS table. + + The converter is the *inverse* of the SACC writers in + :mod:`sp_validation.sacc_io`: it reads statistics back through + those readers and lays them into the DES ``twopoint`` FITS + convention. SACC's canonical order is pair-major (per pair + ``[ξ+; ξ−]``); the 2pt-FITS layout is type-major (all ξ+, then all + ξ−), so the data-vector and its covariance are permuted here via + ``s.indices`` rather than assuming any global order. + + Scope note (single-bin today, tomography-ready): the assembly this + mirrors is single-tomographic-bin — BIN1/BIN2 are all 1, one NZ + ``BIN1`` column. The converter reads bin ``(0, 0)`` accordingly. + A tomographic 2pt-FITS layout (multiple bin pairs, per-pair + BIN1/BIN2, one NZ column per bin) is a later extension; it is not + what today's CosmoSIS pipeline consumes, so it is out of scope for + the byte-compatible converter. + + Rho/tau caveat: the SACC layout stores ρ±/τ± *values* only, while + the 2pt-FITS RHO_STATS table also carries the per-mode *variances* + (``varrho_*``) that Sacha's fork's covariance path reads. Those + variances are not recoverable from the analysis SACC. The + converter therefore writes RHO_STATS / TAU HDUs only when a + ``rho_stats``/``tau_stats`` sidecar FITS is supplied (the same + file today's assembly copies verbatim); it never fabricates + variances. ξ±, Cℓ, n(z) and the covariance — the data vector + CosmoSIS fits — are fully reconstructed from SACC alone. +""" + +import numpy as np +from astropy.io import fits + +from . import sacc_io + +# The QUANT1/QUANT2 header pair CosmoSIS stamps on each 2pt table, keyed by the +# extension name — copied from cosmosis_fitting.py so the headers match card +# for card. +_QUANT = { + "XI_PLUS": ("G+R", "G+R"), + "XI_MINUS": ("G-R", "G-R"), + "CELL_EE": ("GEF", "GEF"), + "CELL_BB": ("GBF", "GBF"), + "TAU_0_PLUS": ("G+R", "P+R"), + "TAU_2_PLUS": ("G+R", "SR+R"), +} + + +def _twopoint_hdu(name, values, ang, *, ang_unit=None): + """Build one 2pt BinTableHDU (BIN1/BIN2/ANGBIN/VALUE/ANG). + + Reproduces ``cosmosis_fitting.py._create_2pt_hdu`` /``cl_to_fits`` exactly: + same column order and formats, the ``2PTDATA`` marker, the QUANT pair for + ``name``, and NZ_SOURCE kernels. ``ang_unit`` stamps ``TUNIT`` on the ANG + column ("arcmin" for real-space ξ/τ; unset for Cℓ, whose ANG is ℓ). + """ + nbins = len(values) + angbin = np.arange(1, nbins + 1) + columns = [ + fits.Column(name="BIN1", format="K", array=np.ones(nbins)), + fits.Column(name="BIN2", format="K", array=np.ones(nbins)), + fits.Column(name="ANGBIN", format="K", array=angbin), + fits.Column(name="VALUE", format="D", array=values), + fits.Column(name="ANG", format="D", unit=ang_unit, array=ang), + ] + hdu = fits.BinTableHDU.from_columns(fits.ColDefs(columns), name=name) + quant1, quant2 = _QUANT[name] + for key, value in { + "2PTDATA": "T", + "QUANT1": quant1, + "QUANT2": quant2, + "KERNEL_1": "NZ_SOURCE", + "KERNEL_2": "NZ_SOURCE", + "WINDOWS": "SAMPLE", + }.items(): + hdu.header[key] = value + return hdu + + +def _nz_hdu(s, n_bins): + """Build the NZDATA HDU from the SACC ``source_i`` NZ tracers. + + Reproduces ``cosmosis_fitting.py.nz_to_fits``: Z_MID from the tracer ``z`` + grid (assumed uniform), Z_LOW/Z_HIGH as ± half a step, one ``BIN{i+1}`` + column per source bin, and the NZDATA/NBIN/NZ header cards. All source bins + are required to share the ``z`` grid — the single ``Z_MID`` axis of the + DES NZDATA table. + """ + z_mid, nz0 = sacc_io.get_nz(s, 0) + z_mid = np.asarray(z_mid, dtype=float) + step = z_mid[1] - z_mid[0] + z_low = z_mid - step / 2 + z_high = z_mid + step / 2 + + columns = [ + fits.Column(name="Z_LOW", format="D", array=z_low), + fits.Column(name="Z_MID", format="D", array=z_mid), + fits.Column(name="Z_HIGH", format="D", array=z_high), + ] + for i in range(n_bins): + z_i, nz_i = sacc_io.get_nz(s, i) + if not np.array_equal(np.asarray(z_i, dtype=float), z_mid): + raise ValueError( + f"source bin {i} n(z) grid differs from source bin 0; the DES " + "NZDATA table requires one shared Z_MID axis" + ) + columns.append(fits.Column(name=f"BIN{i + 1}", format="D", array=nz_i)) + + hdu = fits.BinTableHDU.from_columns(fits.ColDefs(columns), name="NZDATA") + for key, value in { + "NZDATA": "T ", + "EXTNAME": "NZ_SOURCE", + "NBIN": n_bins, + "NZ": len(z_low), + }.items(): + hdu.header[key] = value + return hdu + + +def _cov_hdu(matrix, block_names, block_starts, extname="COVMAT", name_in_ctor=False): + """Build a covariance ImageHDU with ``NAME_i``/``STRT_i`` block headers. + + Reproduces the two covariance builders in ``cosmosis_fitting.py`` card for + card. The blocked ξ/τ ``covdat_to_fits`` builds ``ImageHDU(cov)`` unnamed + and stamps ``COVDATA`` then ``EXTNAME`` from a dict; the ``cov_cl_to_fits`` + CELL covariance builds ``ImageHDU(cov, name="COVMAT_CELL")`` (so the EXTNAME + card is created early, with astropy's standard comment) before re-stamping. + ``name_in_ctor`` selects the second form so the card order matches exactly. + """ + matrix = np.asarray(matrix, dtype=np.float64) + if matrix.shape[0] != matrix.shape[1]: + raise ValueError(f"covariance must be square; got shape {matrix.shape}") + hdu = fits.ImageHDU(matrix, name=extname) if name_in_ctor else fits.ImageHDU(matrix) + hdu.header["COVDATA"] = "True" + hdu.header["EXTNAME"] = extname + for i, (name, start) in enumerate(zip(block_names, block_starts)): + hdu.header[f"NAME_{i}"] = name + hdu.header[f"STRT_{i}"] = int(start) + return hdu + + +def _type_major_xi(s, bins): + """Return ``(theta, xip, xim)`` for one bin pair from the SACC reporting grid. + + ``sacc_io.get_xi`` already returns each statistic in insertion (= ascending + θ) order; the type-major split (all ξ+, then all ξ−) is exactly the two + arrays it hands back, so no further permutation is needed for a single pair. + """ + return sacc_io.get_xi(s, bins, grid="reporting") + + +def _require_single_bin(s, n_bins): + """Fail fast unless the SACC is a valid single-bin ξ product. + + The converter emits the single-bin 2pt-FITS today's CosmoSIS pipeline reads + (BIN1/BIN2 all 1, one NZ column). A tomographic SACC would otherwise slip + through silently — ``n_bins`` alone drives the NZDATA column count while the + ξ/covariance are read from bin ``(0, 0)`` only, so a 2-bin file would emit a + ``NBIN=2`` n(z) beside a data vector holding just the ``(0, 0)`` pair. + Guards both the empty-ξ case and the single-bin contract; tomographic + emission lands with the tomographic round. + """ + pairs = s.get_tracer_combinations(sacc_io.XI_PLUS) + if not pairs: + raise ValueError( + f"SACC has no {sacc_io.XI_PLUS} points — nothing to convert; the " + "2pt-FITS data vector is built from the ξ± statistics" + ) + expected = (sacc_io.source_name(0), sacc_io.source_name(0)) + if n_bins != 1 or set(pairs) != {expected}: + raise ValueError( + f"converter is single-bin only (n_bins=1, ξ pairs == {{{expected}}}); " + f"got n_bins={n_bins} and ξ pairs {sorted(pairs)}. Tomographic " + "emission (multiple bin pairs, per-pair BIN1/BIN2, one NZ column per " + "bin) lands with the tomographic round." + ) + + +def sacc_to_twopoint_fits( + s, + path, + *, + rho_stats_hdu=None, + tau_stats_hdu=None, + n_bins=1, +): + """Convert an analysis SACC to a CosmoSIS 2pt-FITS file. + + The assembled ``HDUList`` matches today's ``cosmosis_fitting.py`` product + for the configuration the SACC describes: PRIMARY, NZ_SOURCE, COVMAT, then + (if present) COVMAT_CELL, XI_PLUS, XI_MINUS, (if present) CELL_EE / CELL_BB, + and (if the rho/tau sidecars are supplied) TAU_0_PLUS, TAU_2_PLUS, + RHO_STATS. The data vector and its covariance are laid out type-major + (all ξ+, then all ξ−, then the τ blocks), which is the DES ``twopoint`` + convention CosmoSIS reads. + + Parameters + ---------- + s : sacc.Sacc + Analysis SACC (reporting ξ±, optional pseudo-Cℓ, covariance, and — for the + ρ/τ product — the τ data points; see ``rho_stats_hdu``). + path : str + Output FITS path (overwritten). + rho_stats_hdu, tau_stats_hdu : astropy.io.fits.BinTableHDU, optional + The rho-stats / tau-stats sidecar HDUs, copied verbatim as today's + assembly does. Required together to write the ρ/τ product; the SACC + alone cannot rebuild the ``varrho_*`` columns Sacha's fork reads. When + omitted, a pure ξ (± Cℓ) product is written. + n_bins : int, optional + Number of source tomographic bins. Must be ``1``: this converter emits + the single-bin 2pt-FITS today's CosmoSIS pipeline consumes. Tomographic + emission (multiple bin pairs, per-pair BIN1/BIN2, one NZ column per bin) + lands with the tomographic round; the converter fails fast on anything + else rather than silently truncating to bin ``(0, 0)``. + + Returns + ------- + astropy.io.fits.HDUList + The assembled list, also written to ``path``. + + Raises + ------ + ValueError + If the SACC has no ξ points; if ``n_bins != 1`` or the SACC's ξ tracer + pairs are anything other than exactly ``{(source_0, source_0)}`` (the + single-bin contract); or if exactly one of the ρ/τ sidecars is supplied. + """ + if (rho_stats_hdu is None) != (tau_stats_hdu is None): + raise ValueError( + "rho_stats_hdu and tau_stats_hdu must be supplied together " + "(the ρ/τ product needs both, or neither for a pure-ξ product)" + ) + _require_single_bin(s, n_bins) + use_rho_tau = rho_stats_hdu is not None + bins = (0, 0) + + nz_hdu = _nz_hdu(s, n_bins) + theta, xip, xim = _type_major_xi(s, bins) + xip_hdu = _twopoint_hdu("XI_PLUS", xip, theta, ang_unit="arcmin") + xim_hdu = _twopoint_hdu("XI_MINUS", xim, theta, ang_unit="arcmin") + + cell_hdu, cov_cell_hdu = _build_cell(s, bins) + + cov_hdu = _build_covmat(s, bins, use_rho_tau=use_rho_tau) + + tau_hdus, rho_hdu = _build_rho_tau(rho_stats_hdu, tau_stats_hdu, theta, use_rho_tau) + + # HDU order mirrors cosmosis_fitting.py's __main__: PRIMARY, NZ, COVMAT, + # COVMAT_CELL, XI±, CELL_EE, then the τ/ρ tables. + hdu_list = [fits.PrimaryHDU(), nz_hdu, cov_hdu] + if cov_cell_hdu is not None: + hdu_list.append(cov_cell_hdu) + hdu_list.extend([xip_hdu, xim_hdu]) + if cell_hdu is not None: + hdu_list.append(cell_hdu) + if use_rho_tau: + hdu_list.extend([*tau_hdus, rho_hdu]) + + hdul = fits.HDUList(hdu_list) + hdul.writeto(path, overwrite=True) + return hdul + + +def _build_cell(s, bins): + """Build the CELL_EE 2pt HDU plus the COVMAT_CELL HDU from the SACC pseudo-Cℓ. + + Returns ``(None, None)`` when the SACC has no pseudo-Cℓ. Only CELL_EE is + emitted — the harmonic ``2pt_like`` fits ``data_sets=CELL_EE``, and today's + assembly appends CELL_EE alone (it builds a CELL_BB HDU but discards it). + The SACC still carries EE/BB/EB with bandpower windows for the B-mode + null-test path; this converter surfaces only the block CosmoSIS reads. The + CELL covariance (the EE bandpower covariance) lives in its own COVMAT_CELL + ImageHDU, matching today's product. + """ + if sacc_io.CL_EE not in s.get_data_types(): + return None, None + + ell, cl_ee, _cl_bb, _cl_eb, _window = sacc_io.get_pseudo_cl(s, bins) + cell_hdu = _twopoint_hdu("CELL_EE", cl_ee, ell) + cell_idx = s.indices(sacc_io.CL_EE, sacc_io._pair(bins)) + cov_cell = s.covariance.dense[np.ix_(cell_idx, cell_idx)] + cov_cell_hdu = _cov_hdu( + cov_cell, ["CELL_EE"], [0], extname="COVMAT_CELL", name_in_ctor=True + ) + return cell_hdu, cov_cell_hdu + + +def _build_covmat(s, bins, *, use_rho_tau): + """Assemble the blocked COVMAT (ξ± type-major, then the τ blocks). + + The ξ covariance is pulled from the SACC as the contiguous ξ+/ξ− block for + the pair and permuted from pair-major (SACC) to type-major (2pt-FITS). Under + ``use_rho_tau`` the τ_0/τ_2 covariance blocks are appended block-diagonally + with zero ξ↔τ cross-blocks, exactly as ``covdat_to_fits`` builds them. + """ + pair = sacc_io._pair(bins) + idx_p = s.indices(sacc_io.XI_PLUS, pair) + idx_m = s.indices(sacc_io.XI_MINUS, pair) + n_theta = len(idx_p) + xi_idx = np.concatenate([idx_p, idx_m]) # type-major permutation + xi_cov = s.covariance.dense[np.ix_(xi_idx, xi_idx)] + + names = ["XI_PLUS", "XI_MINUS"] + starts = [0, n_theta] + matrix = xi_cov + + if use_rho_tau: + # The τ covariance couples τ_0+ and τ_2+ (today's assembly truncates the + # 3-statistic CosmoCov τ covariance to its first 2 blocks and lays it in + # as ONE contiguous [τ_0+; τ_2+] block — cross-correlation kept). In the + # SACC those two selections are not adjacent (τ_0− sits between them), so + # gather both index sets and extract the joint sub-block, ξ↔τ zero. + tau_pair = (sacc_io.source_name(0), sacc_io.PSF_TRACER) + idx_tau0 = s.indices(sacc_io.TAU_PLUS.format(k=0), tau_pair) + idx_tau2 = s.indices(sacc_io.TAU_PLUS.format(k=2), tau_pair) + tau_idx = np.concatenate([idx_tau0, idx_tau2]) + tau_cov = s.covariance.dense[np.ix_(tau_idx, tau_idx)] + matrix = _block_diag(matrix, tau_cov) + names += ["TAU_0_PLUS", "TAU_2_PLUS"] + starts += [2 * n_theta, 2 * n_theta + len(idx_tau0)] + + return _cov_hdu(matrix, names, starts) + + +def _block_diag(*blocks): + """Stack square blocks block-diagonally with zero cross-blocks.""" + sizes = [b.shape[0] for b in blocks] + n = sum(sizes) + out = np.zeros((n, n)) + start = 0 + for b in blocks: + out[start : start + b.shape[0], start : start + b.shape[0]] = b + start += b.shape[0] + return out + + +def _build_rho_tau(rho_stats_hdu, tau_stats_hdu, theta, use_rho_tau): + """Build the TAU_{0,2}_PLUS 2pt HDUs and the verbatim RHO_STATS HDU. + + Mirrors ``tau_to_fits`` / ``rho_to_fits``: τ_0/τ_2 read their ``tau_k_p`` + columns onto the shared ξ θ grid (consistency step); RHO_STATS is copied + verbatim from the sidecar with its θ column forced onto the ξ grid. The + ``varrho_*`` columns ride along in the copy — they are why the sidecar is + required (the SACC cannot supply them). + """ + if not use_rho_tau: + return (), None + + tau = tau_stats_hdu.data + tau0_hdu = _twopoint_hdu("TAU_0_PLUS", tau["tau_0_p"], theta, ang_unit="arcmin") + tau2_hdu = _twopoint_hdu("TAU_2_PLUS", tau["tau_2_p"], theta, ang_unit="arcmin") + + rho_hdu = rho_stats_hdu.copy() + rho_hdu.name = "RHO_STATS" + rho_hdu.data = rho_hdu.data.copy() + rho_hdu.data["theta"] = theta + return (tau0_hdu, tau2_hdu), rho_hdu From c9a4f025b4ca07b1fc2addcc68a1f11adb8b7776 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 11:40:12 +0200 Subject: [PATCH 34/47] refactor(blinding): stack on PR-2's canonical sacc_io; sweep grid vocabulary Rebuild the per-part-at-birth blinding stack on top of feat/sacc-2-sacc-io. sacc_io.py is now PR-2's canonical module verbatim + a single appended gather(): the terminal assembly delegates its tracer/point/covariance assembly to PR-2's merge() and adds only the blind-custody call (assert_consistent_blind) and shared-stamp write. The fail-closed load gate lives in sacc_io.load() per the PRD ("sacc_io fails closed on load"); the blinding tooling uses allow_unblinded=True explicitly where it must read a not-yet-blinded real vector (blind_part). save() now carries the required type= (inherited from each part's provenance). Grid vocabulary swept coarse->reporting, fine->integration across blinding.py, blinding_theory.py, b_modes.py, blind_data_vector.py, and the blinding tests. _extract_block/_set_values stay index-based (a code comment says why): Smokescreen's ConcealDataVector aligns theory_fn output to the sub-SACC mean by row position, so the block must be carved/written by contiguous integer index, not by PR-2's tag-matching extract()/update_statistic(). Escrow/custody/CAMB<->CCL machinery is preserved exactly. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WzUt7VbtXwr2SCHUdiQTyt --- scripts/blind_data_vector.py | 173 +++ src/sp_validation/b_modes.py | 73 ++ src/sp_validation/blinding.py | 819 ++++++++++++ src/sp_validation/blinding_theory.py | 424 +++++++ src/sp_validation/sacc_io.py | 47 + src/sp_validation/tests/test_blinding.py | 1110 +++++++++++++++++ .../tests/test_camb_ccl_crosscheck.py | 174 +++ 7 files changed, 2820 insertions(+) create mode 100644 scripts/blind_data_vector.py create mode 100644 src/sp_validation/blinding.py create mode 100644 src/sp_validation/blinding_theory.py create mode 100644 src/sp_validation/tests/test_blinding.py create mode 100644 src/sp_validation/tests/test_camb_ccl_crosscheck.py diff --git a/scripts/blind_data_vector.py b/scripts/blind_data_vector.py new file mode 100644 index 00000000..9b41109d --- /dev/null +++ b/scripts/blind_data_vector.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 + +"""Script blind_data_vector.py + +Per-part data-vector blinding with :mod:`sp_validation.blinding` +(Smokescreen-fork concealment, hash-commitment custody). + +``blind-init`` runs once per catalogue version: it draws an OS-entropy seed +(never printed, never written in plaintext), publishes a repo-committable +``commitment.json`` (``sha256(seed)`` + config digest), and encrypts the seed +into a Fernet bundle. ``blind-part`` blinds one intermediate part SACC +(reporting ξ±, integration ξ±, or pseudo-Cℓ) under that fixed state, escrows +the true vector into a per-part encrypted bundle beside the blinded output, +and deletes the plaintext part. ``unblind`` verifies both commitment hashes +and restores a true part (bit-for-bit when the part's escrow bundle is beside +it); it also works on the assembled ``{version}.sacc`` (integration rows +selected by the ``grid`` tag). ``verify`` is a cheap, seedless check that a +blinded file matches a commitment. + +:Authors: Cail Daley + +Examples +-------- +Once per catalogue version:: + + blind_data_vector.py blind-init blinded/ + +Per intermediate part, at birth:: + + blind_data_vector.py blind-part parts/xi_integration.fits --blind-dir blinded/ + +Unblind one part:: + + blind_data_vector.py unblind parts/xi_integration_blinded.fits \\ + --blind-dir blinded/ -o parts/xi_integration.fits + +Verify:: + + blind_data_vector.py verify parts/xi_integration_blinded.fits \\ + blinded/commitment.json +""" + +import argparse +import json +import pathlib +import sys + +from sp_validation import blinding, sacc_io + + +def _config_from_args(args): + """A :class:`blinding.BlindingConfig` from optional CLI overrides.""" + overrides = {} + if args.s8_half_width is not None: + overrides["s8_half_width"] = args.s8_half_width + if args.omega_m_half_width is not None: + overrides["omega_m_half_width"] = args.omega_m_half_width + return blinding.BlindingConfig.from_overrides(overrides) + + +def _blind_init(args): + config = _config_from_args(args) + blind_dir = pathlib.Path(args.blind_dir) + blind_dir.mkdir(parents=True, exist_ok=True) + # Refuse before drawing anything: a blind is a one-shot custody event and + # silently overwriting a previous blind's state would destroy the record + # tying that blind to its seed. + clashes = [ + p + for p in blinding.init_paths(str(blind_dir)).values() + if pathlib.Path(p).exists() + ] + if clashes: + raise SystemExit( + "refusing to overwrite existing blind state:\n " + + "\n ".join(clashes) + + "\nPick a fresh --blind-dir (never overwrite a blind)." + ) + blinding.blind_init(str(blind_dir), config=config, label=args.label) + print( + "Commit the commitment JSON to the repo; keep the bundle + key safe " + "and separated (colocation in the blind dir is not at-rest protection)." + ) + + +def _blind_part(args): + blinding.blind_part( + args.part, + args.blind_dir, + config=_config_from_args(args), + keep_input=args.keep_input, + ) + + +def _unblind(args): + blinding.unblind_part( + args.blinded, + args.blind_dir, + args.output, + config=_config_from_args(args), + ) + + +def _verify(args): + """Seedless check: blinded-file metadata ↔ commitment JSON.""" + s = sacc_io.load(args.blinded) + with open(args.commitment, encoding="utf-8") as f: + commitment = json.load(f) + problems = [] + if not s.metadata.get("concealed"): + problems.append("file is not marked concealed") + if s.metadata.get("blind_commitment") != commitment["seed_sha256"]: + problems.append("blind_commitment does not match the committed sha256(seed)") + if s.metadata.get("blind_config_digest") != commitment["config_digest"]: + problems.append("blind_config_digest does not match the committed digest") + if "seed_smokescreen" in s.metadata: + problems.append("PLAINTEXT SEED LEAKED into file metadata (seed_smokescreen)") + if problems: + raise SystemExit("verification FAILED:\n " + "\n ".join(problems)) + print( + f"OK: {args.blinded} matches {args.commitment} " + f"(blind {s.metadata.get('blind')!r})" + ) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[1]) + sub = parser.add_subparsers(dest="mode", required=True) + + for name in ("blind-init", "blind-part", "unblind"): + p = sub.add_parser(name) + p.add_argument("--s8-half-width", type=float, default=None) + p.add_argument("--omega-m-half-width", type=float, default=None) + if name == "blind-init": + p.add_argument( + "blind_dir", + help="directory for the blind's fixed state (commitment + " + "encrypted seed bundle)", + ) + p.add_argument("--label", default="A", help="blind label (default A)") + p.set_defaults(func=_blind_init) + elif name == "blind-part": + p.add_argument("part", help="intermediate part SACC file to blind") + p.add_argument( + "--blind-dir", required=True, help="blind-init state directory" + ) + p.add_argument( + "--keep-input", + action="store_true", + help="retain the plaintext input part (default: delete it " + "after blinding — the true vector is escrowed beside the " + "blinded output)", + ) + p.set_defaults(func=_blind_part) + else: + p.add_argument("blinded", help="blinded part (or assembled) SACC file") + p.add_argument( + "--blind-dir", required=True, help="blind-init state directory" + ) + p.add_argument("-o", "--output", required=True, help="output SACC path") + p.set_defaults(func=_unblind) + + p = sub.add_parser("verify") + p.add_argument("blinded", help="blinded SACC file") + p.add_argument("commitment", help="commitment JSON") + p.set_defaults(func=_verify) + + args = parser.parse_args(argv) + args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/sp_validation/b_modes.py b/src/sp_validation/b_modes.py index 13f4c2e9..97251c3f 100644 --- a/src/sp_validation/b_modes.py +++ b/src/sp_validation/b_modes.py @@ -255,6 +255,79 @@ def pure_EB(corrs): return results +def cosebis_from_xi(theta, xip, xim, nmodes, scale_cut=None): + """COSEBIs (Eₙ, Bₙ) from ξ± arrays through the pipeline kernel (values only). + + The values-only seam of :func:`calculate_cosebis`, for callers holding + ξ± arrays rather than a TreeCorr ``GGCorrelation`` — e.g. deriving + born-blinded COSEBIs from a blinded integration-ξ± SACC part. Calls the same + ``cosmo_numba`` kernel (``COSEBIS.cosebis_from_xipm``) directly on the + values; the covariance/χ² machinery stays with :func:`calculate_cosebis`. + + ``scale_cut`` follows the :func:`sacc_io.add_cosebis` writer contract: + ``(theta_min, theta_max)`` are min/max of the *retained* bin centres + after the pipeline's ``scale_cut_to_bins``. The cut is contiguous in an + ascending grid, so selecting ``theta_min ≤ θ ≤ theta_max`` inclusively + reproduces exactly the retained set, and the kernel is built on that + set's min/max support and fed only the retained ξ± — bit-matching + :func:`calculate_cosebis`'s ``theta_cut``/``xip_cut``/``xim_cut`` path. + Identical inputs ⇒ identical numbers. + """ + from cosmo_numba.B_modes.cosebis import COSEBIS + + theta, xip, xim = (np.asarray(a) for a in (theta, xip, xim)) + tmin, tmax = scale_cut if scale_cut is not None else (theta.min(), theta.max()) + cut = (theta >= tmin) & (theta <= tmax) + theta_cut, xip_cut, xim_cut = theta[cut], xip[cut], xim[cut] + cosebis = COSEBIS( + theta_min=np.min(theta_cut), + theta_max=np.max(theta_cut), + N_max=nmodes, + precision=120, + ) + En, Bn = cosebis.cosebis_from_xipm(theta_cut, xip_cut, xim_cut, parallel=True) + return np.asarray(En), np.asarray(Bn) + + +def pure_eb_from_xi( + theta_report, xip_report, xim_report, theta_int, xip_int, xim_int, tmin, tmax +): + """Pure-E/B correlation functions from ξ± arrays through the pipeline kernel. + + The values-only seam of :func:`calculate_pure_eb_correlation`, for + callers holding ξ± arrays rather than TreeCorr correlations — e.g. + deriving born-blinded pure-E/B from blinded SACC parts. Calls the same + ``cosmo_numba`` kernel (``get_pure_EB_modes``) directly on the values. + The reporting grid must be a strict sub-range of the integration grid; + ``tmin``/``tmax`` are the reporting correlation's TreeCorr *bin edges* + (``gg.left_edges[0]`` / ``gg.right_edges[-1]``) — the pipeline's + convention, carried on SACC files by ``sacc_io.add_pure_eb``. A + reporting point coinciding with the integration boundary is degenerate + (no interior support) and comes back NaN, exactly as + :func:`calculate_pure_eb_correlation` returns it — never a spurious + finite value. + + Returns + ------- + dict + Keyed by ``_EB_KEYS`` (xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb). + """ + from cosmo_numba.B_modes.schneider2022 import get_pure_EB_modes + + modes = get_pure_EB_modes( + theta=np.asarray(theta_report), + xip=np.asarray(xip_report), + xim=np.asarray(xim_report), + theta_int=np.asarray(theta_int), + xip_int=np.asarray(xip_int), + xim_int=np.asarray(xim_int), + tmin=tmin, + tmax=tmax, + parallel=True, + ) + return dict(zip(_EB_KEYS, (np.asarray(m) for m in modes))) + + def calculate_cosebis(gg, nmodes=10, scale_cuts=None, cov_path=None): """ Calculate COSEBIs modes from a correlation function for multiple scale cuts. diff --git a/src/sp_validation/blinding.py b/src/sp_validation/blinding.py new file mode 100644 index 00000000..972fb704 --- /dev/null +++ b/src/sp_validation/blinding.py @@ -0,0 +1,819 @@ +"""Blinding — conceal each intermediate data product behind a hidden cosmology. + +:Name: blinding.py + +:Description: Smokescreen blinding wiring, per part, at birth. Each blindable + intermediate SACC product — reporting ξ±, integration ξ±, pseudo-Cℓ — is shifted, + the moment the pipeline computes it, by a difference of theory vectors + between the fiducial cosmology and a *hidden* cosmology drawn inside a + fixed amplitude envelope, so no one can read S8 off the data until the + collaboration agrees to unblind (Muir et al. 2019: + ``d → d + t(hidden) − t(fiducial)``). Only blinded parts persist on disk. + + **The fork is the concealment engine.** The hidden cosmology is drawn by + the ``UNIONS-WL/Smokescreen`` fork's own CCL-native, per-key-independent + draw; each part goes through its ``ConcealDataVector(fiducial_params, + shifts_dict, sacc_data, *, seed, theory_fn)`` entry point. This module + supplies the two sp_validation-specific pieces: the three ``theory_fn`` + backends matching each part's row layout (reporting ξ±, integration ξ±, pseudo-Cℓ) + and the SACC handling around the concealed vectors. + + **Envelope calibration.** The blinding intent is an amplitude smear of a + chosen (S8, Ωm) box. The fork draws in CCL-native primitives, so + :meth:`BlindingConfig.shifts_dict` maps the intended box to + ``{sigma8, Omega_c}`` half-widths evaluated at the fiducial — + ``σ8 = S8/√(Ωm/0.3)``, ``Ω_c = Ωm − Ω_b − Ω_ν``. The same seed yields + the same hidden cosmology across all part passes (the fork's draw depends + only on ``(key, seed)``), so the blinded parts are mutually consistent by + construction. + + **Derived statistics are born blinded.** COSEBIs and pure-E/B are never + touched by this module: the pipeline's own estimators + (:mod:`sp_validation.b_modes`) run in their normal downstream place on + the already-blinded integration ξ± part, so their outputs are born blinded. + Covariance and ρ/τ PSF diagnostics are never blinded — blinding hides + the vector, not the uncertainty, and the shift is pure E-mode so the + B-mode null tests stay honest under the blind. + + **Custody: hash commitment, no keyholder.** :func:`blind_init` runs once + per catalogue version: it draws an OS-entropy seed, publishes + ``sha256(seed)`` plus a canonical config digest as a repo-committable + ``commitment.json``, and encrypts the seed into a Fernet bundle + (``smokescreen.encryption``) — the plaintext seed is never written. + Each :func:`blind_part` call reads that fixed state, conceals one part, + escrows the part's true vector into its own encrypted bundle beside the + blinded output, and deletes the plaintext part. Terminal assembly + (:func:`sp_validation.sacc_io.gather`) calls + :func:`assert_consistent_blind` to fail closed unless every blindable + part carries the identical ``blind_commitment``. :func:`unblind_part` + verifies both hashes against the commitment *before* subtracting + anything, then restores the true part. +""" + +import dataclasses +import hashlib +import json +import os +import secrets +import warnings + +import numpy as np + +from . import sacc_io +from .blinding_theory import TheoryConfig, cl_ee, xi_ccl, xi_ell_grid + + +# --------------------------------------------------------------------------- # +# Configuration surface — the blinding envelope +# --------------------------------------------------------------------------- # +@dataclasses.dataclass(frozen=True) +class BlindingConfig: + """Blinding envelope and fiducial. + + The hidden cosmology is drawn (by the fork) uniformly and independently + per key inside ``shifts_dict()``'s half-widths about the fiducial. The + half-widths are the deliberate, configurable size of the blind — config, + not code; the group may resize the envelope. ``theory`` carries the + fiducial :class:`TheoryConfig` whose defaults *are* the blinding + fiducial. + """ + + s8_half_width: float = 0.075 + omega_m_half_width: float = 0.1 + theory: TheoryConfig = dataclasses.field(default_factory=TheoryConfig) + + def shifts_dict(self): + """The (S8, Ωm) envelope as CCL-native ``{sigma8, Omega_c}`` half-widths. + + Evaluated at the fiducial: a ΔS8 half-width maps to + ``ΔS8/√(Ωm_fid/0.3)`` in σ8 (at fixed Ωm), and a ΔΩm half-width maps + one-to-one to Ω_c (Ω_b and Ω_ν are fixed). Exact enough for a + blinding smear — the target is a characteristic amplitude, not a + precise (S8, Ωm) posterior. The fork draws each key independently as + ``U(fid − h, fid + h)``. + """ + return { + "sigma8": self.s8_half_width / np.sqrt(self.theory.Omega_m / 0.3), + "Omega_c": self.omega_m_half_width, + } + + def config_digest(self): + """sha256 of a canonical serialization of the full blinding config. + + Binds the envelope half-widths, the complete fiducial + :class:`TheoryConfig` (cosmology + the two ``halofit_version`` + tokens + IA fields) into one digest: JSON with sorted keys over the + full ordered field set. Every ``float``-declared field is coerced + through :func:`float` before serialization, so the digest depends on + the numeric *value*, not on whether a config wrote ``-1`` or ``-1.0`` + — an int-vs-float literal difference (routine when configs come from + YAML/CLI/humans) can no longer split one physical cosmology into two + digests and deny a legitimate unblind. Python's ``json`` then emits + each float via its shortest round-trip ``repr``, so two runs of the + same config produce byte-identical digests. Checked (with + ``sha256(seed)``) at unblind, so a wrong envelope or a mismatched + P(k) recipe cannot silently subtract a wrong shift. + """ + payload = { + "s8_half_width": float(self.s8_half_width), + "omega_m_half_width": float(self.omega_m_half_width), + "theory": { + f.name: ( + float(getattr(self.theory, f.name)) + if f.type in (float, "float") + else getattr(self.theory, f.name) + ) + for f in dataclasses.fields(self.theory) + }, + } + return hashlib.sha256( + json.dumps(payload, sort_keys=True).encode("utf-8") + ).hexdigest() + + @classmethod + def from_overrides(cls, overrides): + """Build from a mapping of field overrides (fail loud on unknown keys). + + ``theory`` may be given as a :class:`TheoryConfig` or a mapping of + TheoryConfig overrides, mirroring :meth:`TheoryConfig.from_overrides`. + """ + by_name = {f.name: f for f in dataclasses.fields(cls)} + unknown = set(overrides) - set(by_name) + if unknown: + raise ValueError( + f"unknown BlindingConfig fields {sorted(unknown)}; " + f"valid fields are {sorted(by_name)}" + ) + overrides = { + name: (float(v) if by_name[name].type in (float, "float") else v) + for name, v in overrides.items() + } + theory = overrides.get("theory") + if theory is not None and not isinstance(theory, TheoryConfig): + overrides["theory"] = TheoryConfig.from_overrides(dict(theory)) + return cls(**overrides) + + +# --------------------------------------------------------------------------- # +# Custody primitives +# --------------------------------------------------------------------------- # +def seed_commitment(seed): + """Public commitment for a seed: its sha256 hex digest. + + Safe to publish and commit to the repo — it ties a blinded file to its + blind without revealing the seed, and lets unblind refuse a wrong seed. + """ + return hashlib.sha256(seed.encode("utf-8")).hexdigest() + + +def hidden_params(seed, config): + """The hidden CCL parameter point the fork realizes for ``(seed, config)``. + + Re-runs the fork's own draw (``smokescreen.param_shifts.draw_param_shifts`` + — per-key-independent, local RNG) on the calibrated envelope and overlays + the deltas on the fiducial, exactly as ``ConcealDataVector`` does + internally. Deterministic: same ``(seed, config)`` ⇒ same hidden point, + forever — the reproducibility contract unblinding relies on, and what + makes every part share one hidden cosmology under one seed. + """ + from smokescreen.param_shifts import draw_param_shifts + + deltas = draw_param_shifts(config.shifts_dict(), seed) + params = dict(config.theory.ccl_params()) + for key, delta in deltas.items(): + params[key] += delta + return params + + +# --------------------------------------------------------------------------- # +# Block discovery on a SACC (a standalone part, or the assembled file) +# --------------------------------------------------------------------------- # +def source_bins(s): + """Sorted source-bin indices present in ``s`` (from ``source_i`` tracers).""" + return sorted( + int(name.split("_", 1)[1]) for name in s.tracers if name.startswith("source_") + ) + + +def xi_pairs(s, grid): + """Unordered source-bin pairs ``(i ≤ j)`` carrying ξ+ on ``grid``.""" + bins = source_bins(s) + return [ + (i, j) + for a, i in enumerate(bins) + for j in bins[a:] + if len(s.indices(sacc_io.XI_PLUS, sacc_io._pair((i, j)), grid=grid)) + ] + + +def cl_pairs(s): + """Source-bin pairs ``(i ≤ j)`` carrying pseudo-Cℓ_EE.""" + bins = source_bins(s) + return [ + (i, j) + for a, i in enumerate(bins) + for j in bins[a:] + if len(s.indices(sacc_io.CL_EE, sacc_io._pair((i, j)))) + ] + + +def _xi_indices(s, grid): + """Row indices of the ξ± block on ``grid`` (ascending).""" + return np.sort( + np.concatenate( + [ + s.indices(sacc_io.XI_PLUS, grid=grid), + s.indices(sacc_io.XI_MINUS, grid=grid), + ] + ) + ).astype(int) + + +def _cl_ee_indices(s): + """Row indices of the pseudo-Cℓ_EE block (ascending). + + Only EE: a pure E-mode cosmology shift leaves BB and EB identically + zero, so those blocks are never extracted, never concealed. + """ + return np.sort(s.indices(sacc_io.CL_EE)).astype(int) + + +def _pair_nz(s, i, j): + """The two per-bin n(z) for pair ``(i, j)`` as ``((z_i, n_i), (z_j, n_j))``.""" + z_i, n_i = sacc_io.get_nz(s, i) + z_j, n_j = sacc_io.get_nz(s, j) + return (np.asarray(z_i), np.asarray(n_i)), (np.asarray(z_j), np.asarray(n_j)) + + +# --------------------------------------------------------------------------- # +# The three theory backends — callables aligned to a sub-SACC block's rows +# --------------------------------------------------------------------------- # +def xi_theory_fn(block, theory, grid): + """``theory_fn`` for a ξ± sub-SACC block (reporting or integration grid). + + Reads each bin's n(z) directly from the block's own tracers and lays the + output out to match the block's SACC rows element-for-element: for every + pair, ξ± is computed at that pair's stored θ (the ``theta`` tag, + arcmin) and scattered to the rows ``block.indices`` reports — the + block's own row order, never an assumed pairing. IA enters from + ``theory``'s NLA fields, identically at every parameter point. + """ + pairs = xi_pairs(block, grid) + layout = [] + for i, j in pairs: + tr = sacc_io._pair((i, j)) + idx_p = block.indices(sacc_io.XI_PLUS, tr, grid=grid) + idx_m = block.indices(sacc_io.XI_MINUS, tr, grid=grid) + theta = sacc_io._tag(block, sacc_io.XI_PLUS, tr, "theta", grid=grid) + layout.append(((i, j), idx_p, idx_m, np.asarray(theta, dtype=float))) + ell = xi_ell_grid() + + def theory_fn(params): + out = np.full(len(block.mean), np.nan) + for (i, j), idx_p, idx_m, theta in layout: + nz_i, nz_j = _pair_nz(block, i, j) + xip, xim = xi_ccl(params, theory, nz_i, nz_j, theta, ell) + out[idx_p] = xip + out[idx_m] = xim + return out + + return theory_fn + + +def cl_theory_fn(block, theory): + """``theory_fn`` for the pseudo-Cℓ_EE sub-SACC block. + + Per pair: theory Cℓ_EE on the stored ``BandpowerWindows`` support, + binned by the same window matrix the measurement used (``W @ Cℓ_EE``), + scattered to the block's own rows. The concealing factor the fork forms + from this is ``W @ ΔCℓ_EE`` — the shift lands in the measured + bandpowers; ΔBB = ΔEB ≡ 0 by construction (pure E-mode shift), so those + blocks are simply not part of this backend's rows. + """ + layout = [] + for i, j in cl_pairs(block): + tr = sacc_io._pair((i, j)) + idx = block.indices(sacc_io.CL_EE, tr) + window = block.get_bandpower_windows(idx) + layout.append( + ( + (i, j), + np.asarray(idx), + np.asarray(window.values, dtype=float), # (n_ell,) + np.asarray(window.weight, dtype=float), # (n_ell, n_bp) + ) + ) + + def theory_fn(params): + out = np.full(len(block.mean), np.nan) + for (i, j), idx, w_ell, w_mat in layout: + nz_i, nz_j = _pair_nz(block, i, j) + out[idx] = w_mat.T @ cl_ee(params, theory, nz_i, nz_j, w_ell) + return out + + return theory_fn + + +# --------------------------------------------------------------------------- # +# Extract → conceal → merge, per block +# --------------------------------------------------------------------------- # +def _blindable_blocks(s): + """The blindable blocks of a SACC as ``(name, indices, factory)``. + + Works identically on a standalone part (which carries exactly one block) + and on the assembled one-file product (whose integration rows are selected by + the ``grid`` tag — the layout contract's per-block tag selection). + ``indices`` are each block's recorded row indices (ascending, so the + extracted sub-SACC preserves row order); ``factory`` builds the matching + ``theory_fn`` from the extracted sub-SACC. Blocks absent from the file + are simply not listed. + """ + blocks = [] + for grid in ("reporting", "integration"): + idx = _xi_indices(s, grid) + if len(idx): + blocks.append( + ( + f"{grid} ξ±", + idx, + lambda sub, theory, grid=grid: xi_theory_fn(sub, theory, grid), + ) + ) + idx = _cl_ee_indices(s) + if len(idx): + blocks.append(("pseudo-Cℓ_EE", idx, cl_theory_fn)) + return blocks + + +def _extract_block(s, indices): + """Extract rows ``indices`` (ascending) into a sub-SACC, order preserved. + + Deliberately index-based rather than ``sacc_io.extract`` / + ``update_statistic`` (which select and merge by ``(data_type, tracers, + tags)``): Smokescreen's ``ConcealDataVector`` aligns its ``theory_fn`` + output to the sub-SACC's ``mean`` element-for-element by row position, so + the block must be carved and written back by contiguous integer index, not + by tag-matching. This is the one place the row-index path is load-bearing. + """ + sub = s.copy() + sub.keep_indices(np.asarray(indices, dtype=int)) + return sub + + +def _concealing_factor(s, indices, factory, config, seed): + """The fork-computed additive concealing factor for one block. + + Extracts the block into a sub-SACC containing exactly the rows the + ``theory_fn`` spans (the fork's length guard enforces the agreement), + drives ``ConcealDataVector`` with the fiducial point, the calibrated + envelope, and the block's ``theory_fn``, and returns the factor + ``t(hidden) − t(fiducial)`` aligned to ``indices``. + """ + from smokescreen import ConcealDataVector + + sub = _extract_block(s, indices) + smoke = ConcealDataVector( + config.theory.ccl_params(), + config.shifts_dict(), + sub, + seed=seed, + theory_fn=factory(sub, config.theory), + ) + smoke.calculate_concealing_factor(factor_type="add") + concealed = smoke.apply_concealing_to_likelihood_datavec() + return np.asarray(concealed, dtype=float) - np.asarray(sub.mean, dtype=float) + + +def _set_values(s, indices, values): + """Overwrite ``s.data[i].value`` for ``indices`` with ``values`` (aligned).""" + for i, v in zip(indices, values): + s.data[int(i)].value = float(v) + + +def _concealed(s): + """Whether ``s`` is already a blinded file (its ``concealed`` mark is set).""" + return bool(s.metadata.get("concealed")) + + +def blind_sacc(part, seed, config=None, label="A", log=print): + """Return a blinded copy of a part SACC (covariance and tags untouched). + + Per blindable block present (a standalone part carries exactly one — + reporting ξ±, integration ξ±, or pseudo-Cℓ_EE): extract the block into a sub-SACC, + conceal through the fork, write the shifted values back at their + recorded indices (row order preserved). Provenance is stamped and any + leaked seed key stripped. A file with no blindable block (e.g. a ρ/τ + diagnostic part) is refused loudly — it should never see a blind call. + """ + config = config or BlindingConfig() + if _concealed(part): + raise ValueError("already concealed — unblind first") + blocks = _blindable_blocks(part) + if not blocks: + raise ValueError( + "no blindable block (reporting/integration ξ± or pseudo-Cℓ_EE) in this SACC " + "— ρ/τ diagnostic parts are never blinded" + ) + + blinded = part.copy() + for name, indices, factory in blocks: + factor = _concealing_factor(part, indices, factory, config, seed) + _set_values(blinded, indices, np.asarray(blinded.mean)[indices] + factor) + log(f"[blind] {name}: shifted {len(indices)} points via Smokescreen fork") + + _stamp_provenance(blinded, seed_commitment(seed), label, config.config_digest()) + return blinded + + +def unblind_sacc(blinded, seed, config=None, log=print): + """Recover the true part SACC from a blinded one + the revealed ``seed``. + + Verifies ``sha256(seed)`` and the config digest against the stamped + metadata (loud failure on either mismatch — verification precedes + subtraction), recomputes each block's shift from the seed through the + same backends, and subtracts it. Works on a standalone part or on the + assembled file (integration rows selected by the ``grid`` tag). Derived + statistics, if present (assembled file), are *not* recomputed here — the + pipeline's own estimators re-derive them from the unblinded integration ξ±. + """ + config = config or BlindingConfig() + if not _concealed(blinded): + raise ValueError("file is not concealed — nothing to unblind") + if seed_commitment(seed) != blinded.metadata["blind_commitment"]: + raise ValueError( + "seed does not match blind_commitment — refusing to unblind " + "(a wrong seed would silently produce a wrong data vector)" + ) + if config.config_digest() != blinded.metadata["blind_config_digest"]: + raise ValueError( + "blinding config does not match blind_config_digest — refusing to " + "unblind (this config would subtract a different shift than was " + "added)" + ) + + hidden = hidden_params(seed, config) + fiducial = config.theory.ccl_params() + part = blinded.copy() + for name, indices, factory in _blindable_blocks(blinded): + theory = factory(_extract_block(blinded, indices), config.theory) + factor = theory(hidden) - theory(fiducial) + _set_values(part, indices, np.asarray(part.mean)[indices] - factor) + log(f"[unblind] {name}: subtracted {len(indices)} shifts") + + for key in ("concealed", "blind", "blind_commitment", "blind_config_digest"): + part.metadata.pop(key, None) + return part + + +def _stamp_provenance(s, commitment, label, config_digest): + """Stamp the blind's public provenance; strip any leaked seed. + + ``blind_commitment`` (sha256 of the seed) ties the file to its blind + without revealing it; ``blind_config_digest`` pins the envelope + + fiducial the shift was drawn against; ``concealed``/``blind`` mark the + file. ``seed_smokescreen`` — the raw seed Smokescreen's own writer would + stamp — is popped defensively: the seed must never ride a kept file. + """ + s.metadata.pop("seed_smokescreen", None) + s.metadata["concealed"] = True + s.metadata["blind"] = label + s.metadata["blind_commitment"] = commitment + s.metadata["blind_config_digest"] = config_digest + + +# --------------------------------------------------------------------------- # +# Assembly-time custody: one blind across all parts +# --------------------------------------------------------------------------- # +def assert_consistent_blind(parts): + """Assert every blindable part shares one blind; return the shared stamp. + + Custody logic for the terminal :func:`sp_validation.sacc_io.gather`: a + part is *blindable* if it carries a blindable block (ξ± or pseudo-Cℓ_EE); + ρ/τ diagnostic and covariance-only parts are exempt. Fails closed — + ``ValueError`` — if blinded and plaintext blindable parts are mixed, or + if two parts carry different ``blind_commitment``/``blind_config_digest`` + (they were blinded under different seeds or configs and must never be + combined). The consistency key is ``(commitment, digest)`` only — the + ``blind`` *label* is informational provenance, not custody state, so + parts blinded under one seed+config but tagged with different ``--label`` + values assemble cleanly (a distinct warning is logged, not a failure). + + **Unconcealed parts must be declared mocks** (PRD §4, "Mocks vs data"): + when no blindable part is concealed, every blindable part's metadata must + carry ``type == "mock"`` — the tag :mod:`sp_validation.sacc_io` stamps + when the data vector is computed. An unconcealed ``type == "data"`` part, + or one missing the tag, fails assembly closed: skipping the blind can + never silently expose real data. A concealed+plaintext mix fails closed + regardless of ``type`` (see above). A fully-mock plaintext assembly + returns ``None``. + + Returns + ------- + dict or None + The shared blind metadata (``concealed``, ``blind``, + ``blind_commitment``, ``blind_config_digest``) for the gather to + stamp on the assembled file, or ``None`` when nothing is blinded. + """ + blindable = [p for p in parts if _blindable_blocks(p)] + concealed = [p for p in blindable if _concealed(p)] + if not concealed: + # `.get` is deliberate here: a missing `type` tag must count as + # not-a-mock and fail closed, not KeyError with less context. + exposed = sorted( + {str(p.metadata.get("type", "")) for p in blindable} - {"mock"} + ) + if exposed: + raise ValueError( + f"unconcealed blindable parts with type {exposed} in assembly " + "— only parts declared `type: mock` may assemble without a " + "blind (an unconcealed data part exposes the real vector)" + ) + return None + if len(concealed) != len(blindable): + raise ValueError( + f"blinded and plaintext blindable parts mixed in one assembly " + f"({len(concealed)} of {len(blindable)} blinded) — refusing to " + "combine (a plaintext part beside blinded ones leaks the shift)" + ) + # Custody state is (commitment, digest) only — the label is provenance. + stamps = { + (p.metadata["blind_commitment"], p.metadata["blind_config_digest"]) + for p in concealed + } + if len(stamps) != 1: + raise ValueError( + "parts carry different blind commitments — they were blinded " + "under different seeds or configs and must never be combined: " + + "; ".join(f"({c[:12]}…, {d[:12]}…)" for c, d in stamps) + ) + ((commitment, digest),) = stamps + labels = sorted({p.metadata["blind"] for p in concealed}) + if len(labels) != 1: + warnings.warn( + f"blindable parts share one blind (commitment {commitment[:12]}…, " + f"config {digest[:12]}…) but carry different labels {labels} — " + "assembling anyway; the label is provenance, not custody state. " + f"Stamping the assembled file with label {labels[0]!r}." + ) + return { + "concealed": True, + "blind": labels[0], + "blind_commitment": commitment, + "blind_config_digest": digest, + } + + +# --------------------------------------------------------------------------- # +# File-level custody: blind-init / blind-part / unblind +# --------------------------------------------------------------------------- # +def init_paths(blind_dir): + """The fixed custody state written by :func:`blind_init` in ``blind_dir``.""" + return { + "commitment": os.path.join(blind_dir, "commitment.json"), + "bundle": os.path.join(blind_dir, "blind_seed.encrpt"), + "key": os.path.join(blind_dir, "blind_seed.key"), + } + + +def part_paths(part_path): + """Blinded-output and escrow-bundle paths beside a part file.""" + stem, ext = os.path.splitext(part_path) + return { + "blinded": f"{stem}_blinded{ext or '.fits'}", + "escrow": f"{stem}_escrow.encrpt", + "escrow_key": f"{stem}_escrow.key", + } + + +def blind_init(blind_dir, config=None, label="A", log=print): + """Fix the blind for one catalogue version: seed, commitment, seed bundle. + + Runs once per catalogue version: + + 1. Draw an OS-entropy seed (never written in plaintext, never returned). + 2. Write ``commitment.json`` (repo-committable): ``sha256(seed)`` + the + canonical config digest + the blind label. + 3. Encrypt the seed into a Fernet bundle (``smokescreen.encryption``); + the temporary plaintext is deleted by the encryptor. + + These outputs are the blind's fixed state; every :func:`blind_part` and + :func:`unblind_part` call reads them. + + Custody caveat: the bundle and its Fernet key land in the *same* + ``blind_dir``. Fernet is only as protective as the key's separation — + anyone with both files can decrypt the seed. Keep the key out-of-band; + colocation is convenience, not at-rest protection. + + Returns + ------- + dict + Paths written: ``commitment``, ``bundle``, ``key``. + """ + config = config or BlindingConfig() + paths = init_paths(blind_dir) + for path in paths.values(): + if os.path.exists(path): + raise FileExistsError( + f"refusing to overwrite existing blind state {path} — a blind " + "is a one-shot custody event; choose another directory" + ) + + seed = secrets.token_hex(16) + commitment = { + "label": label, + "seed_sha256": seed_commitment(seed), + "config_digest": config.config_digest(), + } + with open(paths["commitment"], "w", encoding="utf-8") as f: + json.dump(commitment, f, indent=2, sort_keys=True) + _write_encrypted_json(paths["bundle"], {"label": label, "seed": seed}) + + log(f"[blind-init] commitment (repo-committable): {paths['commitment']}") + log(f"[blind-init] encrypted seed bundle + key: {paths['bundle']}, {paths['key']}") + log( + "[blind-init] custody: keep the bundle key out-of-band from the bundle " + "(colocation in the blind dir is not at-rest protection)" + ) + return paths + + +def _read_seed(blind_dir, config): + """Decrypt the seed bundle and verify it against the commitment. + + Both ``sha256(seed)`` and the config digest are checked before the seed + is handed to any caller — a tampered bundle or a drifted config fails + loud here, whether the caller is about to blind or to unblind. + + Returns + ------- + tuple + ``(seed, commitment_dict)``. + """ + paths = init_paths(blind_dir) + bundle = _read_encrypted_json(paths["bundle"], paths["key"]) + with open(paths["commitment"], encoding="utf-8") as f: + commitment = json.load(f) + if seed_commitment(bundle["seed"]) != commitment["seed_sha256"]: + raise ValueError( + "bundle seed does not match the committed sha256(seed) — refusing " + "to proceed" + ) + if config.config_digest() != commitment["config_digest"]: + raise ValueError( + "blinding config does not match the committed config digest — " + "refusing to proceed (a wrong envelope or P(k) recipe would " + "silently produce a wrong shift)" + ) + return bundle["seed"], commitment + + +def blind_part(part_path, blind_dir, config=None, keep_input=False, log=print): + """Blind one intermediate part SACC at birth, under the fixed blind state. + + Reads the encrypted seed and ``commitment.json`` written by + :func:`blind_init` (verifying both hashes), conceals the part through its + matching backend (:func:`blind_sacc`), writes the blinded part beside the + input with the part's true vector escrowed into a per-part Fernet bundle, + and deletes the plaintext part — only the blinded part persists on disk. + Each part's escrow is self-contained: restoring it needs only that + part's bundle plus the seed bundle; corruption of one bundle loses one + part, not all. Pass ``keep_input=True`` to retain the plaintext part + (and own the custody implication). + + Returns + ------- + dict + Paths written: ``blinded``, ``escrow``, ``escrow_key``. + """ + config = config or BlindingConfig() + seed, commitment = _read_seed(blind_dir, config) + paths = part_paths(part_path) + for path in paths.values(): + if os.path.exists(path): + raise FileExistsError( + f"refusing to overwrite existing blind output {path} — a blind " + "is a one-shot custody event" + ) + + # The plaintext part is real, unblinded data — the blinding step is the one + # legitimate reader of the true vector, so it passes the load escape hatch. + part = sacc_io.load(part_path, allow_unblinded=True) + blinded = blind_sacc(part, seed, config=config, label=commitment["label"], log=log) + + _write_encrypted_json( + paths["escrow"], + { + "label": commitment["label"], + "seed_sha256": commitment["seed_sha256"], + "true_mean": np.asarray(part.mean, dtype=float).tolist(), + }, + ) + # The blinded file inherits the part's provenance (data vs mock); it also + # carries concealed=True (stamped by blind_sacc), so it loads without the + # escape hatch. + sacc_io.save(blinded, paths["blinded"], type=part.metadata["type"]) + if not keep_input: + os.remove(part_path) + log(f"[blind-part] deleted plaintext part {part_path}") + else: + log(f"[blind-part] plaintext part RETAINED at {part_path} (keep_input=True)") + log(f"[blind-part] wrote {paths['blinded']} (escrow beside it)") + return paths + + +def unblind_part(blinded_path, blind_dir, out_path, config=None, log=print): + """Unblind one blinded part (or the assembled file), verifying first. + + Decrypts the seed bundle and verifies both ``sha256(seed)`` and the + config digest against ``commitment.json`` (fail closed on either — + verification precedes subtraction), recomputes the part's shift from the + seed and subtracts it (:func:`unblind_sacc`). **The seed-subtracted + vector is the authority** — the seed plus its commitment is the custody + root of trust, and the returned vector is what the seed math produced. + + When the part's escrow bundle exists beside the blinded file it serves + two subordinate roles, and only after its stored ``seed_sha256`` is + verified to match the same commitment (so an escrow from a different + blind can never be trusted): (1) a *tighter equality check* — the seed + subtraction and the escrowed truth must agree to ``1e-6`` relative or the + unblind fails closed; (2) *ulp-residue removal* — float add-then-subtract + leaves ~ulp residue against the pre-blind truth, and once the escrow is + bound to this commitment its exact value clears that residue so the + restore is bit-for-bit. The escrow is never the source of correctness: + if it disagrees materially the guard raises, and a mismatched or + unbound escrow never determines the output. On the assembled file — no + escrow — the subtraction stands alone, with integration rows selected by the + ``grid`` tag. + """ + config = config or BlindingConfig() + seed, commitment = _read_seed(blind_dir, config) + blinded = sacc_io.load(blinded_path) + part = unblind_sacc(blinded, seed, config=config, log=log) + + stem, ext = os.path.splitext(blinded_path) + unblinded_stem = os.path.join( + os.path.dirname(stem), os.path.basename(stem).replace("_blinded", "") + ) + escrow = part_paths(unblinded_stem + ext) + if os.path.exists(escrow["escrow"]): + bundle = _read_encrypted_json(escrow["escrow"], escrow["escrow_key"]) + if bundle.get("seed_sha256") != commitment["seed_sha256"]: + raise ValueError( + "escrow bundle beside the blinded file was written under a " + "different seed than the commitment — refusing to trust it " + "(the seed subtraction is authoritative; this escrow is not " + "bound to this blind)" + ) + true_mean = np.asarray(bundle["true_mean"], dtype=float) + recovered = np.asarray(part.mean, dtype=float) + residual = np.nanmax( + np.abs(recovered - true_mean) / (np.abs(true_mean) + 1e-30) + ) + if residual > 1e-6: + raise ValueError( + f"unblinded vector disagrees with the escrowed true vector " + f"(max rel {residual:.2e}) — wrong escrow for this part?" + ) + # Seed-bound escrow: clear the add-then-subtract ulp residue so the + # restore is bit-for-bit. Correctness already came from the seed + # subtraction above; the guard proved the escrow agrees with it. + _set_values(part, np.arange(len(true_mean)), true_mean) + log(f"[unblind] escrow verified (subtraction residual {residual:.2e})") + # unblind_sacc stripped the concealed/blind stamps, so this is the true + # revealed vector; it inherits the blinded file's provenance (data vs mock). + sacc_io.save(part, out_path, type=blinded.metadata["type"]) + log(f"[unblind] wrote {out_path}") + return out_path + + +def _write_encrypted_json(encrpt_path, payload): + """Encrypt ``payload`` (JSON) to ``encrpt_path`` + sibling ``.key``; no + plaintext survives. + + We drive ``smokescreen.encryption.encrypt_file`` (Fernet) for the crypto + but write the ciphertext and key at the exact names we control. Its + ``save_file`` mode names outputs from ``basename.split('.')[0]`` — it + truncates at the first dot — so for a dotted stem (the canonical + catalogue-version case, e.g. ``v1.4.6.3_xi_integration_escrow``) it would land at + ``v1.encrpt``/``v1.key``, diverging from what :func:`part_paths` declares + and colliding across parts that share a first-dot prefix. Instead we take + the returned ``(ciphertext, key)`` and write them ourselves. + """ + from smokescreen.encryption import encrypt_file + + key_path = encrpt_path.replace(".encrpt", ".key") + plaintext = encrpt_path.replace(".encrpt", ".json") + with open(plaintext, "w", encoding="utf-8") as f: + json.dump(payload, f) + ciphertext, key = encrypt_file(plaintext, save_file=False, keep_original=False) + with open(encrpt_path, "wb") as f: + f.write(ciphertext) + with open(key_path, "wb") as f: + f.write(key) + + +def _read_encrypted_json(encrpt_path, key_path): + """Decrypt and parse a Fernet-encrypted JSON bundle.""" + from smokescreen.encryption import decrypt_file + + return json.loads(decrypt_file(encrpt_path, key_path).decode("utf-8")) diff --git a/src/sp_validation/blinding_theory.py b/src/sp_validation/blinding_theory.py new file mode 100644 index 00000000..b44f7e73 --- /dev/null +++ b/src/sp_validation/blinding_theory.py @@ -0,0 +1,424 @@ +"""Blinding theory: fiducial configuration and the two ξ± theory paths. + +:Name: blinding_theory.py + +:Description: The blinding backend's theory surface — the fiducial + configuration (:class:`TheoryConfig`) and two independent routes to the + tomographic shear two-point prediction. + + **Division of responsibility** (per the UNIONS layering: ``cs_util`` is + the cosmology *library* — generic machinery; ``sp_validation`` holds + *configuration* and *survey-specific implementations*). This module lives + in the blinding namespace because :class:`TheoryConfig` is configuration + and the master-layout theory backends in :mod:`sp_validation.blinding` + (reporting ξ±, integration ξ±, pseudo-Cℓ) are survey-specific. The **generic** + theory machinery below — the CCL-native ξ± path (:func:`xi_ccl`, + :func:`cl_ee`), the independent CAMB P(k)→``Pk2D`` path (:func:`xi_camb`), + and the σ8/A_s rescale (:func:`camb_As_for_sigma8`) — lives here **for + now** but is destined for ``cs_util.cosmo`` (tracked in cs_util#80): it is + cosmology-library code, not blinding-specific. There is deliberately **no** + ``sp_validation/cosmology.py`` — ``develop`` removed the local cosmology + module (#223) and moved cosmology to ``cs_util.cosmo``; this module does + not resurrect it. + + Two independent routes to the shear two-point prediction: + + - **CCL-native path** (:func:`xi_ccl`, :func:`cl_ee`): CCL builds the + nonlinear P(k) through its Boltzmann-CAMB HMCode2020 route + (``matter_power_spectrum='camb'`` + ``extra_parameters``) and projects + to Cℓ/ξ± via its own Limber (``angular_cl``) + FFTLog + (``correlation``). This is the recipe the blinding theory backends use. + - **Independent-CAMB path** (:func:`xi_camb`): a direct ``pycamb`` run + produces the HMCode2020 ``P(k, z)`` (σ8-matched via the closed-form + A_s rescale of :func:`camb_As_for_sigma8`), wrapped in a ``ccl.Pk2D`` + and projected through the same CCL Limber + FFTLog machinery. + + Because both paths route their nonlinear P(k) through CAMB's HMCode2020 + and both project through CCL, a common Limber+FFTLog bug cancels between + them: the CAMB↔CCL cross-check test built on these two paths validates + the **P(k) recipe** and the **σ8/A_s amplitude convention**, not the + projection machinery. + + This module imports only ``numpy`` at module level; CCL and CAMB are + imported inside the functions that need them, so importing + :class:`TheoryConfig` never drags in a theory backend. +""" + +import dataclasses + +import numpy as np + +# Fixed constants of the fiducial — load-bearing for the CAMB↔CCL amplitude +# match, so they are emitted explicitly to both stacks rather than left to +# either stack's default. Not user-facing TheoryConfig fields. +NEFF = 3.046 +T_CMB = 2.7255 + + +# --------------------------------------------------------------------------- # +# Configuration surface — the ONE place fiducial cosmology + model choices live +# --------------------------------------------------------------------------- # +@dataclasses.dataclass(frozen=True) +class TheoryConfig: + """Fiducial cosmology and model configuration for the theory paths. + + Every field is a deliberate, configurable choice. The defaults mirror the + ``cosmo_inference`` CosmoSIS fiducial (the ``SP_v1.4.6.3_A_cell`` pipeline + + ``values_ia.ini`` central values), so the CCL theory computed here and + the CAMB theory CosmoSIS computes agree to the level the CAMB↔CCL + cross-check test asserts. Adopting a different named group fiducial is a + change to these *values*, not to any code. + + Cosmology is parametrised by the blind axes ``S8`` and ``Omega_m`` and + converted to CCL's native ``sigma8``/``Omega_c`` by :meth:`sigma8` / + :meth:`omega_c`. + + **Nonlinear-model tokens (load-bearing).** The HMCode2020+feedback recipe + is named by a token in each stack's API: CCL takes + ``extra_parameters['camb']['halofit_version']``, CAMB takes + ``NonLinearModel.set_params(halofit_version=…)``. :class:`TheoryConfig` + carries **two** tokens (``ccl_halofit_version``, ``camb_halofit_version``) + denoting one recipe, and each stack is fed its own — a single shared + string invites a silent stack disagreement the moment the two APIs name + the recipe differently (``mead2020`` vs ``mead2020_feedback`` differ by + several % at k ≳ 1/Mpc). Today both stacks accept the same string for the + feedback recipe, so the two defaults coincide; the cross-check test pins + the CCL token against the inference config independently. + """ + + # Cosmological parameters (blind axes S8, Omega_m + the rest). + S8: float = 0.80 # values_ia.ini S_8_input central + Omega_m: float = 0.30 + Omega_b: float = 0.0469 # ombh2=0.023 at h=0.7 -> 0.023/0.7^2 + h: float = 0.70 + n_s: float = 0.96 + m_nu: float = 0.06 # Σm_ν in eV, distributed under `mass_split` + w0: float = -1.0 + wa: float = 0.0 + + # Neutrino mass split: normal hierarchy (CosmoSIS `neutrino_hierarchy=normal`). + mass_split: str = "normal" + + # Boltzmann/transfer-function backend for the CCL path (#280). The default + # `boltzmann_camb` makes CCL call CAMB for the linear P(k), so blinding + # theory and the CosmoSIS+CAMB inference stack share one power-spectrum + # path. The halofit route follows the choice (see `ccl_cosmology`): only + # under `boltzmann_camb` can the nonlinear P(k) run through CAMB's HMCode + # (`matter_power_spectrum="camb"` + the tokens below); any other backend + # falls back to CCL's own halofit — consistent, but a different recipe, + # so a non-default backend is a deliberate cross-check tool, not a + # production setting. + transfer_function: str = "boltzmann_camb" + + # One nonlinear recipe (CAMB HMCode2020 + baryonic feedback), two + # stack-specific tokens — see the class docstring. + ccl_halofit_version: str = "mead2020_feedback" + camb_halofit_version: str = "mead2020_feedback" + hmcode_logT_AGN: float = 7.5 # values_ia.ini logT_AGN central + + # Intrinsic alignments: NLA. The fiducial defaults IA OFF (ia_bias=0) — + # the blinding shift is a difference of two theory vectors at the same IA, + # so IA nearly cancels there, and IA-off keeps the CAMB↔CCL cross-check a + # clean test of the shear calculation. Set `ia_bias` nonzero (CosmoSIS + # central A=1.0) to include NLA. + ia_bias: float = 0.0 + ia_z_piv: float = 0.62 + ia_alphaz: float = 0.0 + + def sigma8(self): + """CCL ``sigma8`` implied by ``S8`` and ``Omega_m``. + + ``S8 ≡ σ8 √(Ωm / 0.3)`` — the standard weak-lensing definition — so + ``σ8 = S8 / √(Ωm / 0.3)``. At the fiducial (S8=0.80, Ωm=0.30), + σ8 = 0.80. + """ + return self.S8 / np.sqrt(self.Omega_m / 0.3) + + def omega_c(self): + """CCL cold-dark-matter density ``Omega_c = Omega_m − Omega_b − Ω_ν``. + + The neutrino density ``Ω_ν h² = Σm_ν / 93.14 eV`` is subtracted so + the *total* matter density is exactly ``Omega_m`` (CCL treats massive + neutrinos as a separate species, not part of ``Omega_c``). + """ + omega_nu = self.m_nu / (93.14 * self.h**2) + return self.Omega_m - self.Omega_b - omega_nu + + def ccl_params(self): + """The fiducial point as a plain CCL-native parameter mapping. + + Exactly the keys ``Omega_c, Omega_b, h, n_s, sigma8, m_nu, + mass_split, w0, wa, Neff, T_CMB`` and no others — no CCL default + rides along. ``Neff``/``T_CMB`` are the fixed module constants. This + mapping is what the Smokescreen fork receives as ``fiducial_params`` + and what every ``theory_fn`` receives back (possibly with + ``sigma8``/``Omega_c`` overlaid by the hidden draw). + """ + return { + "Omega_c": self.omega_c(), + "Omega_b": self.Omega_b, + "h": self.h, + "n_s": self.n_s, + "sigma8": self.sigma8(), + "m_nu": self.m_nu, + "mass_split": self.mass_split, + "w0": self.w0, + "wa": self.wa, + "Neff": NEFF, + "T_CMB": T_CMB, + } + + @classmethod + def from_overrides(cls, overrides): + """Build from a mapping of field overrides (fail loud on unknown keys). + + Numeric overrides are coerced to ``float`` per the field's declared + type, so a YAML/CLI ``w0: -1`` (int) yields the same value — and the + same :meth:`config_digest` — as the float default ``-1.0``. + """ + by_name = {f.name: f for f in dataclasses.fields(cls)} + unknown = set(overrides) - set(by_name) + if unknown: + raise ValueError( + f"unknown TheoryConfig fields {sorted(unknown)}; " + f"valid fields are {sorted(by_name)}" + ) + coerced = { + name: (float(v) if by_name[name].type in (float, "float") else v) + for name, v in overrides.items() + } + return cls(**coerced) + + +# --------------------------------------------------------------------------- # +# CCL-native path: cosmology construction, Cℓ_EE, ξ± +# --------------------------------------------------------------------------- # +# The two cosmologies of a blind (fiducial + hidden) are evaluated by three +# theory backends over multiple blocks; caching the ccl.Cosmology per parameter +# point avoids re-running the CAMB P(k) computation for every block. +_COSMO_CACHE = {} + + +def ccl_cosmology(params, config): + """A ``pyccl.Cosmology`` at ``params`` with ``config``'s nonlinear recipe. + + ``params`` is a plain CCL-native mapping (:meth:`TheoryConfig.ccl_params`, + possibly with keys overlaid by the hidden draw); ``config`` supplies only + the non-sampled recipe tokens (``transfer_function``, + ``ccl_halofit_version``, ``hmcode_logT_AGN``). The Boltzmann backend is + ``config.transfer_function`` (#280); the halofit route stays consistent + with that choice: under ``boltzmann_camb`` the nonlinear P(k) runs + through CAMB's HMCode2020 (``matter_power_spectrum="camb"`` + the CAMB + tokens), while any other backend has no CAMB run to hand tokens to, so it + takes CCL's own halofit (``matter_power_spectrum="halofit"``). Either + way, the same recipe sits on both sides of any theory difference. + Cosmology objects are cached per parameter point (CCL memoises its P(k) + on the object, so the cache saves repeated Boltzmann runs across blocks). + """ + import pyccl as ccl + + key = ( + tuple(sorted(params.items())), + config.transfer_function, + config.ccl_halofit_version, + config.hmcode_logT_AGN, + ) + if key not in _COSMO_CACHE: + nonlinear = ( + { + "matter_power_spectrum": "camb", + "extra_parameters": { + "camb": { + "halofit_version": config.ccl_halofit_version, + "HMCode_logT_AGN": config.hmcode_logT_AGN, + } + }, + } + if config.transfer_function == "boltzmann_camb" + else {"matter_power_spectrum": "halofit"} + ) + _COSMO_CACHE[key] = ccl.Cosmology( + **params, + transfer_function=config.transfer_function, + **nonlinear, + ) + return _COSMO_CACHE[key] + + +def xi_ell_grid(): + """The ℓ grid the ξ± Hankel projection integrates over. + + Integers 2…49, then 200 log-spaced multipoles up to 6·10⁴ — dense enough + at low ℓ (where ξ± at large θ lives) and wide enough for the small-θ + tail. ``ccl.correlation`` interpolates C(ℓ) internally, so this fixes the + resolution of every ξ± this module produces. + """ + return np.unique( + np.concatenate([np.arange(2, 50), np.geomspace(50, 6e4, 200)]).astype(float) + ) + + +def _tracer(cosmo, z, nz, config): + """A ``WeakLensingTracer`` for one bin's n(z), NLA from ``config``. + + With the fiducial ``ia_bias = 0`` the tracer is built bare — no IA term. + A nonzero ``ia_bias`` enters as the NLA amplitude + ``A(z) = ia_bias · ((1+z)/(1+z_piv))^alphaz``. + """ + import pyccl as ccl + + z = np.asarray(z) + if config.ia_bias == 0.0: + return ccl.WeakLensingTracer(cosmo, dndz=(z, np.asarray(nz))) + a_ia = config.ia_bias * ((1 + z) / (1 + config.ia_z_piv)) ** config.ia_alphaz + return ccl.WeakLensingTracer( + cosmo, dndz=(z, np.asarray(nz)), ia_bias=(z, a_ia), use_A_ia=True + ) + + +def cl_ee(params, config, nz_i, nz_j, ell): + """Cross Cℓ_EE at ``ell`` for the bin pair with n(z) ``nz_i``, ``nz_j``. + + Two-tracer: one :class:`~pyccl.WeakLensingTracer` per bin from that bin's + own ``(z, nz)``, then ``angular_cl(cosmo, tracer_i, tracer_j, ell)`` — + the cross-spectrum for i ≠ j, the auto-spectrum when the two n(z) are the + same bin. The shear ``angular_cl`` is the E-mode spectrum; B and EB are + zero in theory, which is why only Cℓ_EE ever receives a blinding shift. + """ + import pyccl as ccl + + cosmo = ccl_cosmology(params, config) + tracer_i = _tracer(cosmo, *nz_i, config) + tracer_j = _tracer(cosmo, *nz_j, config) + return ccl.angular_cl(cosmo, tracer_i, tracer_j, np.asarray(ell, dtype=float)) + + +def xi_ccl(params, config, nz_i, nz_j, theta_arcmin, ell=None): + """CCL-native ξ± at ``theta_arcmin`` for one bin pair (Path A). + + Cross Cℓ_EE on :func:`xi_ell_grid` (or ``ell``), then ``ccl.correlation`` + (FFTLog Hankel transform) at θ in degrees, ``type="GG+"`` / ``"GG-"``. + + Returns + ------- + (np.ndarray, np.ndarray) + ``(xip, xim)`` aligned to ``theta_arcmin``. + """ + import pyccl as ccl + + ell = xi_ell_grid() if ell is None else np.asarray(ell, dtype=float) + cosmo = ccl_cosmology(params, config) + cl = cl_ee(params, config, nz_i, nz_j, ell) + theta_deg = np.asarray(theta_arcmin) / 60.0 + xip = ccl.correlation(cosmo, ell=ell, C_ell=cl, theta=theta_deg, type="GG+") + xim = ccl.correlation(cosmo, ell=ell, C_ell=cl, theta=theta_deg, type="GG-") + return xip, xim + + +# --------------------------------------------------------------------------- # +# Independent-CAMB path: A_s reconciliation + P(k) → Pk2D → CCL projection +# --------------------------------------------------------------------------- # +def make_camb_params(config, As, *, nonlinear, zmax=3.0, n_z=48, kmax=20.0): + """A ``CAMBparams`` at ``config``'s background with amplitude ``As``. + + Every :class:`TheoryConfig` field CCL sees is fed to CAMB from the same + source — ``w0``/``wa`` via ``set_dark_energy``, ``Neff``/``T_CMB`` as the + module constants, ``m_nu``/``mass_split`` through ``set_cosmology`` — so + the independent path differs from the CCL path only in who computes P(k), + never in an unmatched background parameter. + """ + import camb + + p = camb.CAMBparams() + p.set_cosmology( + H0=config.h * 100, + ombh2=config.Omega_b * config.h**2, + omch2=config.omega_c() * config.h**2, + mnu=config.m_nu, + num_massive_neutrinos=1, + neutrino_hierarchy=config.mass_split, + nnu=NEFF, + TCMB=T_CMB, + ) + p.set_dark_energy(w=config.w0, wa=config.wa, dark_energy_model="ppf") + p.InitPower.set_params(As=As, ns=config.n_s) + p.set_matter_power(redshifts=list(np.linspace(0.0, zmax, n_z)), kmax=kmax) + if nonlinear: + p.NonLinear = camb.model.NonLinear_both + p.NonLinearModel.set_params( + halofit_version=config.camb_halofit_version, + HMCode_logT_AGN=config.hmcode_logT_AGN, + ) + else: + p.NonLinear = camb.model.NonLinear_none + return p + + +def camb_linear_sigma8(config, As, **kwargs): + """CAMB's linear σ8(z=0) at amplitude ``As``.""" + import camb + + results = camb.get_results(make_camb_params(config, As, nonlinear=False, **kwargs)) + return float(results.get_sigma8_0()) + + +def camb_As_for_sigma8(config, sigma8_target, As_seed=2.1e-9, **kwargs): + """The CAMB ``A_s`` whose linear σ8 equals ``sigma8_target``. + + Closed-form: linear σ8² ∝ A_s exactly, so one CAMB linear-σ8 evaluation + at ``As_seed`` and one rescale ``As_seed · (σ8_target/σ8_seed)²`` land on + the target — no iteration. This settles the convention subtlety that our + fiducial fixes σ8 for CCL but A_s for CAMB: a nominal ``A_s = 2.1e-9`` + leaves CAMB's σ8 ≈3% off target, enough to blow a ξ± comparison to + ~9–10%. + """ + sigma8_seed = camb_linear_sigma8(config, As_seed, **kwargs) + return As_seed * (sigma8_target / sigma8_seed) ** 2 + + +def xi_camb(config, nz, theta_arcmin, *, n_ell=300, ell_max=60000, kmax=20.0, n_k=400): + """Independent-CAMB ξ± for one bin (Path B): CAMB P(k) → Pk2D → CCL. + + A direct pycamb run produces the HMCode2020 nonlinear ``P(k, z)`` at a + σ8-matched ``A_s`` (:func:`camb_As_for_sigma8`), extracted through + ``get_matter_power_interpolator(hubble_units=False, k_hunit=False)`` so + it comes out in CCL's native units (k in 1/Mpc, P in Mpc³) — **no** + ``·h`` / ``/h³`` conversion is applied (applying one would double-count + an h³ amplitude error). Both ``Pk2D`` axes are arranged ascending + (log-k ascending; scale factor ascending, i.e. CAMB's z-ascending grid + reversed). Projection is CCL's own Limber + FFTLog with a bare tracer + (IA off — this path exists for the cross-check). + + Returns + ------- + (np.ndarray, np.ndarray, float) + ``(xip, xim, As)`` — the σ8-matched amplitude is returned for + assertion by the cross-check test. + """ + import camb + import pyccl as ccl + + sigma8 = config.sigma8() + As = camb_As_for_sigma8(config, sigma8, kmax=kmax) + results = camb.get_results(make_camb_params(config, As, nonlinear=True, kmax=kmax)) + interp = results.get_matter_power_interpolator( + nonlinear=True, hubble_units=False, k_hunit=False + ) + k = np.geomspace(1e-4, kmax * config.h, n_k) # 1/Mpc + z = np.linspace(0.0, 3.0, 48) + pk = interp.P(z, k) # (n_z, n_k), Mpc^3 + a = 1.0 / (1.0 + z) + order = np.argsort(a) # Pk2D wants ascending scale factor + pk2d = ccl.Pk2D( + a_arr=a[order], lk_arr=np.log(k), pk_arr=np.log(pk[order]), is_logp=True + ) + + cosmo = ccl_cosmology(config.ccl_params(), config) + z_nz, nz_vals = nz + lens = ccl.WeakLensingTracer(cosmo, dndz=(np.asarray(z_nz), np.asarray(nz_vals))) + ells = np.unique(np.geomspace(2, ell_max, n_ell).astype(int)).astype(float) + cl = ccl.angular_cl(cosmo, lens, lens, ells, p_of_k_a=pk2d) + theta_deg = np.asarray(theta_arcmin) / 60.0 + xip = ccl.correlation(cosmo, ell=ells, C_ell=cl, theta=theta_deg, type="GG+") + xim = ccl.correlation(cosmo, ell=ells, C_ell=cl, theta=theta_deg, type="GG-") + return xip, xim, As diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index 21872d0e..fd2e8df8 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -773,3 +773,50 @@ def load(path, *, allow_unblinded=False): "allow_unblinded=True." ) return s + + +# --------------------------------------------------------------------------- # +# Terminal assembly — added on top of PR-2's canonical module (PR-6 blinding). +# Everything above this banner is byte-identical to feat/sacc-2-sacc-io; only +# gather() and its blind-custody call site live here. +# --------------------------------------------------------------------------- # +def gather(parts, metadata=None): + """Assemble standalone part SACCs into the one-file ``{version}.sacc``. + + Each part is an intermediate product as it came off its producing rule + (reporting ξ±, integration ξ±, pseudo-Cℓ, ρ/τ, …). Assembly itself — the + first-wins tracer union, in-order point concatenation with all tags + (bandpower windows included), and the block-diagonal covariance — is + exactly :func:`merge`, so gather delegates to it and adds only the one + thing merge cannot know about: **blind custody.** + + **Blind custody (the module's one blind-aware call site):** + :func:`sp_validation.blinding.assert_consistent_blind` runs before the + merge — it fails closed unless every blindable part carries the identical + ``blind_commitment``/``blind_config_digest`` (or, when nothing is blinded, + every blindable part is declared ``type='mock'``). Its returned shared + stamp is written onto the assembled file so the one-file product carries + the blind it was built from; the blinded parts already carry those keys, + so merge preserves them and this stamp is a consistent (idempotent) + re-affirmation. + + Parameters + ---------- + parts : sequence of sacc.Sacc + The part SACCs, in the assembly (covariance) order. + metadata : dict, optional + Extra key/value pairs to store on the assembled file's metadata. + + Returns + ------- + sacc.Sacc + The assembled file. + """ + from . import blinding + + parts = list(parts) + stamp = blinding.assert_consistent_blind(parts) + s = merge(parts) + for key, value in {**(metadata or {}), **(stamp or {})}.items(): + s.metadata[key] = value + return s diff --git a/src/sp_validation/tests/test_blinding.py b/src/sp_validation/tests/test_blinding.py new file mode 100644 index 00000000..7013a81a --- /dev/null +++ b/src/sp_validation/tests/test_blinding.py @@ -0,0 +1,1110 @@ +"""Tests for :mod:`sp_validation.blinding` — per-part Smokescreen blinding. + +Acceptance criteria AC1–AC9 of the blinding PRD, plus fast unit coverage of +the config/custody surface. Fast tests (no CCL import — the envelope +calibration, digest, commitment, fork-draw determinism, blind-init custody, +the merge alignment against a monkeypatched concealing factor, and the +assembly hash assertion) run in the default suite; the theory tests (fork + +CCL) are marked ``slow``; the derived-statistics tests additionally +``importorskip`` ``cosmo_numba``. + +All fixtures are synthetic and deterministic. Each blindable intermediate is +its own standalone part SACC (reporting ξ±, integration ξ±, pseudo-Cℓ), as in the +per-part-at-birth architecture; derived statistics (COSEBIs, pure-E/B) are +never stored in parts — they are computed downstream from the (blinded) integration +ξ± through the pipeline seams ``b_modes.cosebis_from_xi`` / +``b_modes.pure_eb_from_xi``, exactly as the pipeline does. The fork's +``ConcealDataVector`` carries no data-vector consistency check (only the +length guard), so fixture ξ± values are smooth synthetic templates — no +theory fill is needed to blind. +""" + +import json +import pathlib + +import numpy as np +import pytest + +from sp_validation import blinding as bd +from sp_validation import sacc_io as sio +from sp_validation.blinding_theory import TheoryConfig + +_NOLOG = lambda *a, **k: None # noqa: E731 + + +# --------------------------------------------------------------------------- # +# Synthetic part fixtures +# --------------------------------------------------------------------------- # +def _gauss_nz(z0, sigma, n=200): + z = np.linspace(0.0, 3.0, n) + nz = np.exp(-0.5 * ((z - z0) / sigma) ** 2) + return z, nz / np.trapezoid(nz, z) + + +def _reporting_theta(n=8): + return np.geomspace(5.0, 250.0, n) + + +def _integration_theta(n=80): + # The integration grid is the pure-E/B INTEGRATION grid, so it spans wider than + # the reporting range on both ends (production: ~0.08–300 arcmin). + return np.geomspace(0.1, 300.0, n) + + +def _xi_template(theta, k=0): + """Smooth synthetic ξ± for pair index ``k`` (no CCL needed).""" + theta = np.asarray(theta) + xip = 1e-4 * (1 + 0.1 * k) * (theta / 10.0) ** -0.6 + xim = 0.5e-4 * (1 + 0.1 * k) * (theta / 10.0) ** -0.9 + return xip, xim + + +def _b_mode_template(theta, amplitude): + """A smooth ξ_B(θ) template. B contributes +ξ_B to ξ+, −ξ_B to ξ−.""" + return amplitude * np.exp(-((np.log(np.asarray(theta) / 30.0)) ** 2) / 2.0) + + +def _nz_dict(nbins): + return {i: _gauss_nz(0.5 + 0.3 * i, 0.15 + 0.02 * i) for i in range(nbins)} + + +def _pairs(nbins): + return [(i, j) for i in range(nbins) for j in range(i, nbins)] + + +def make_xi_part(grid, nbins=1, b_amplitude=0.0): + """A standalone ξ± part SACC (one grid), synthetic values, eye covariance. + + ``b_amplitude`` injects a pure B-mode (+ξ_B to ξ+, −ξ_B to ξ−; + b_modes.py sign convention) — used on the integration part for AC4/AC9. + """ + theta = _reporting_theta() if grid == "reporting" else _integration_theta() + s = sio.new_sacc( + _nz_dict(nbins), metadata={"catalogue_version": "vTEST", "type": "mock"} + ) + blocks = [] + for k, (i, j) in enumerate(_pairs(nbins)): + xip, xim = _xi_template(theta, k) + xi_b = _b_mode_template(theta, b_amplitude) + sio.add_xi(s, (i, j), theta, xip + xi_b, xim - xi_b, grid=grid) + tr = sio._pair((i, j)) + idx = np.concatenate( + [ + s.indices(sio.XI_PLUS, tr, grid=grid), + s.indices(sio.XI_MINUS, tr, grid=grid), + ] + ) + blocks.append((idx, np.eye(len(idx)) * 1e-12)) + sio.assemble_covariance(s, blocks) + return s + + +def make_cl_part(nbins=1): + """A standalone pseudo-Cℓ part SACC (EE/BB/EB + bandpower windows).""" + s = sio.new_sacc( + _nz_dict(nbins), metadata={"catalogue_version": "vTEST", "type": "mock"} + ) + ell_eff = np.array([30.0, 80.0, 150.0, 280.0, 450.0]) + w_ell = np.arange(2, 501).astype(float) + w_mat = np.zeros((len(w_ell), len(ell_eff))) + for b, le in enumerate(ell_eff): + w_mat[:, b] = np.exp(-0.5 * ((w_ell - le) / 40.0) ** 2) + w_mat[:, b] /= w_mat[:, b].sum() + blocks = [] + for k, (i, j) in enumerate(_pairs(nbins)): + cl_ee = 1e-8 * (1 + 0.1 * k) * (ell_eff / 100.0) ** -1.2 + sio.add_pseudo_cl( + s, + (i, j), + ell_eff, + cl_ee, + np.zeros(5), + np.zeros(5), + window_ells=w_ell, + window_weights=w_mat, + ) + tr = sio._pair((i, j)) + idx = np.concatenate( + [s.indices(dt, tr) for dt in (sio.CL_EE, sio.CL_BB, sio.CL_EB)] + ) + blocks.append((idx, np.eye(len(idx)) * 1e-16)) + sio.assemble_covariance(s, blocks) + return s + + +def make_rho_part(): + """A standalone ρ/τ PSF-diagnostics part SACC — never blindable.""" + ctheta = _reporting_theta() + s = sio.new_sacc( + _nz_dict(1), metadata={"catalogue_version": "vTEST", "type": "mock"} + ) + blocks = [] + for k in range(2): + sio.add_rho( + s, k, ctheta, np.arange(len(ctheta)) * 1e-7, np.arange(len(ctheta)) * 2e-7 + ) + idx = np.concatenate( + [s.indices(sio.RHO_PLUS.format(k=k)), s.indices(sio.RHO_MINUS.format(k=k))] + ) + blocks.append((idx, np.eye(len(idx)) * 1e-18)) + sio.add_tau( + s, (0,), 0, ctheta, np.arange(len(ctheta)) * 3e-7, np.arange(len(ctheta)) * 4e-7 + ) + idx = np.concatenate( + [s.indices(sio.TAU_PLUS.format(k=0)), s.indices(sio.TAU_MINUS.format(k=0))] + ) + blocks.append((idx, np.eye(len(idx)) * 1e-18)) + sio.assemble_covariance(s, blocks) + return s + + +def make_parts(nbins=1, b_amplitude=0.0, with_rho=True): + """All intermediate parts of one catalogue version, keyed by name.""" + parts = { + "xi_reporting": make_xi_part("reporting", nbins), + "xi_integration": make_xi_part("integration", nbins, b_amplitude=b_amplitude), + "cl": make_cl_part(nbins), + } + if with_rho: + parts["rho_tau"] = make_rho_part() + return parts + + +def _derive_downstream(reporting_part, integration_part, nmodes=6): + """COSEBIs + pure-E/B the way the pipeline derives them downstream. + + COSEBIs from the integration ξ± (full-range scale cut); pure-E/B from the + measured reporting reporting ξ± + the integration integration ξ±, with the + edge-based bounds set to the reporting grid's span — the outermost + reporting point sits at tmax with no interior support and comes back + NaN (the AC9 boundary case). + """ + from sp_validation import b_modes + + theta_f, xip_f, xim_f = sio.get_xi(integration_part, (0, 0), grid="integration") + theta_c, xip_c, xim_c = sio.get_xi(reporting_part, (0, 0), grid="reporting") + En, Bn = b_modes.cosebis_from_xi( + theta_f, xip_f, xim_f, nmodes, scale_cut=(theta_f.min(), theta_f.max()) + ) + modes = b_modes.pure_eb_from_xi( + theta_c, + xip_c, + xim_c, + theta_f, + xip_f, + xim_f, + float(theta_c[0]), + float(theta_c[-1]), + ) + return En, Bn, modes + + +# --------------------------------------------------------------------------- # +# BlindingConfig: envelope calibration + digest (fast) +# --------------------------------------------------------------------------- # +def test_blinding_config_defaults(): + c = bd.BlindingConfig() + assert c.s8_half_width == 0.075 + assert c.omega_m_half_width == 0.1 + assert c.theory.S8 == 0.80 # fiducial TheoryConfig defaults + + +def test_blinding_config_overrides_fail_loud(): + c = bd.BlindingConfig.from_overrides({"s8_half_width": 0.05}) + assert c.s8_half_width == 0.05 + with pytest.raises(ValueError, match="unknown BlindingConfig fields"): + bd.BlindingConfig.from_overrides({"s8_half_width": 0.05, "bogus": 1}) + with pytest.raises(ValueError, match="unknown TheoryConfig fields"): + bd.BlindingConfig.from_overrides({"theory": {"nope": 1}}) + + +def test_blinding_config_is_frozen(): + with pytest.raises(Exception): + bd.BlindingConfig().s8_half_width = 0.2 + + +def test_envelope_calibration_maps_s8_box_to_ccl_halfwidths(): + """(S8, Ωm) half-widths → {sigma8, Omega_c} at the fiducial (exact forms).""" + c = bd.BlindingConfig() + shifts = c.shifts_dict() + assert set(shifts) == {"sigma8", "Omega_c"} + assert shifts["sigma8"] == pytest.approx( + c.s8_half_width / np.sqrt(c.theory.Omega_m / 0.3) + ) + assert shifts["Omega_c"] == c.omega_m_half_width + # every shift key must exist in the fiducial point (fork contract) + assert set(shifts) <= set(c.theory.ccl_params()) + + +def test_config_digest_stable_and_sensitive(): + """Canonical digest: byte-stable across runs, moves with every bound field.""" + c = bd.BlindingConfig() + assert c.config_digest() == bd.BlindingConfig().config_digest() + assert len(c.config_digest()) == 64 + assert bd.BlindingConfig(s8_half_width=0.05).config_digest() != c.config_digest() + assert ( + bd.BlindingConfig.from_overrides({"theory": {"S8": 0.79}}).config_digest() + != c.config_digest() + ) + # the P(k) recipe tokens are bound: a different halofit token = new digest + assert ( + bd.BlindingConfig.from_overrides( + {"theory": {"ccl_halofit_version": "takahashi"}} + ).config_digest() + != c.config_digest() + ) + # the Boltzmann backend (#280) is bound too — a different transfer + # function is a different P(k) path, so a different blind + assert ( + bd.BlindingConfig.from_overrides( + {"theory": {"transfer_function": "eisenstein_hu"}} + ).config_digest() + != c.config_digest() + ) + + +def test_theory_config_transfer_function_default_and_override(): + """#280: the Boltzmann backend is one config knob, CAMB by default (the + inference pipeline's Boltzmann code), overridable like any other field.""" + assert TheoryConfig().transfer_function == "boltzmann_camb" + cfg = TheoryConfig.from_overrides({"transfer_function": "eisenstein_hu"}) + assert cfg.transfer_function == "eisenstein_hu" + + +def test_config_digest_int_float_canonical(): + """One physical cosmology has one digest, regardless of int-vs-float literals. + + Configs come from YAML/CLI/humans, so a field can arrive as ``-1`` (int) or + ``-1.0`` (float). The digest must depend on the numeric *value*, not the + literal's Python type — otherwise an int-vs-float mismatch between the blind + and a later unblind would raise "config digest mismatch" and deny a + legitimate unblind. Every declared-float field must be canonical this way. + """ + for field, int_val, float_val in [ + ("w0", -1, -1.0), + ("wa", 0, 0.0), + ("Omega_m", 1, 1.0), + ("m_nu", 0, 0.0), + ("S8", 1, 1.0), + ]: + assert ( + bd.BlindingConfig.from_overrides( + {"theory": {field: int_val}} + ).config_digest() + == bd.BlindingConfig.from_overrides( + {"theory": {field: float_val}} + ).config_digest() + ), f"int-vs-float digest split on theory.{field}" + # the envelope half-widths (BlindingConfig's own float fields) too + assert ( + bd.BlindingConfig.from_overrides({"s8_half_width": 1}).config_digest() + == bd.BlindingConfig.from_overrides({"s8_half_width": 1.0}).config_digest() + ) + # and a full round-trip: an all-int override matches the float default digest + assert ( + bd.BlindingConfig.from_overrides( + {"theory": {"w0": -1, "Omega_m": 0, "wa": 0}} + ).config_digest() + == bd.BlindingConfig.from_overrides( + {"theory": {"w0": -1.0, "Omega_m": 0.0, "wa": 0.0}} + ).config_digest() + ) + + +def test_theory_config_ccl_params_exact_keyset(): + """ccl_params() carries exactly the contracted keys — nothing rides along.""" + params = TheoryConfig().ccl_params() + assert set(params) == { + "Omega_c", + "Omega_b", + "h", + "n_s", + "sigma8", + "m_nu", + "mass_split", + "w0", + "wa", + "Neff", + "T_CMB", + } + assert params["sigma8"] == pytest.approx(0.80) # S8=0.80 at Ωm=0.30 + assert params["Neff"] == 3.046 and params["T_CMB"] == 2.7255 + + +# --------------------------------------------------------------------------- # +# Commitment + the fork's draw (fast; smokescreen import is light) +# --------------------------------------------------------------------------- # +def test_commitment_is_sha256_of_seed(): + import hashlib + + seed = "the-secret" + assert bd.seed_commitment(seed) == hashlib.sha256(seed.encode()).hexdigest() + assert bd.seed_commitment("right") != bd.seed_commitment("wrong") + + +def test_hidden_params_deterministic_and_in_envelope(): + """Same (seed, config) ⇒ same hidden point; draws respect the envelope.""" + import secrets + + c = bd.BlindingConfig() + fid = c.theory.ccl_params() + h1, h2 = bd.hidden_params("a-seed", c), bd.hidden_params("a-seed", c) + assert h1 == h2 + assert bd.hidden_params("другой", c) != h1 + shifts = c.shifts_dict() + for _ in range(50): + h = bd.hidden_params(secrets.token_hex(8), c) + for key, half in shifts.items(): + assert abs(h[key] - fid[key]) <= half + # only the enveloped keys move + assert all(h[k] == fid[k] for k in fid if k not in shifts) + + +def test_hidden_params_no_global_rng_state(): + """The fork's draw uses a local RNG — global numpy state is untouched.""" + np.random.seed(0) + before = np.random.get_state()[1].copy() + bd.hidden_params("whatever", bd.BlindingConfig()) + assert np.array_equal(before, np.random.get_state()[1]) + + +# --------------------------------------------------------------------------- # +# Per-part merge + provenance with a monkeypatched factor (fast — no CCL) +# --------------------------------------------------------------------------- # +def _patch_constant_factor(monkeypatch, value=1e-6): + def fake(part, indices, factory, config, seed): + return np.arange(len(indices), dtype=float) * value + value + + monkeypatch.setattr(bd, "_concealing_factor", fake) + return fake + + +def test_merge_places_shift_at_recorded_indices_only(monkeypatch): + """AC5 (merge half), per part: the shift lands exactly on the blindable + rows, in stored order; covariance, n(z), and every tag are untouched.""" + _patch_constant_factor(monkeypatch) + for name, part in make_parts(nbins=2, with_rho=False).items(): + orig = np.array(part.mean) + orig_cov = part.covariance.dense.copy() + orig_nz = part.tracers["source_0"].nz.copy() + + blinded = bd.blind_sacc(part, "seed", log=_NOLOG) + + blocks = bd._blindable_blocks(part) + assert len(blocks) == 1, f"{name}: a part carries exactly one block" + shifted = np.zeros(len(orig), dtype=bool) + for _, indices, _ in blocks: + expected = orig[indices] + (np.arange(len(indices)) * 1e-6 + 1e-6) + assert np.array_equal(np.array(blinded.mean)[indices], expected) + shifted[indices] = True + assert np.array_equal(np.array(blinded.mean)[~shifted], orig[~shifted]) + assert np.array_equal(blinded.covariance.dense, orig_cov) + assert np.array_equal(blinded.tracers["source_0"].nz, orig_nz) + # row-order preservation: type/tracers/tags sequence is bitwise unchanged + for a, b in zip(part.data, blinded.data): + assert a.data_type == b.data_type + assert a.tracers == b.tracers + assert a.tags == b.tags + + +def test_provenance_metadata_contract(monkeypatch): + """Blinded parts carry concealed/blind/commitment/digest; no seed key.""" + _patch_constant_factor(monkeypatch) + s = make_xi_part("reporting") + s.metadata["seed_smokescreen"] = "leaked!" # must be stripped + c = bd.BlindingConfig() + blinded = bd.blind_sacc(s, "seed", config=c, label="B", log=_NOLOG) + assert blinded.metadata["concealed"] is True + assert blinded.metadata["blind"] == "B" + assert blinded.metadata["blind_commitment"] == bd.seed_commitment("seed") + assert blinded.metadata["blind_config_digest"] == c.config_digest() + assert "seed_smokescreen" not in blinded.metadata + assert blinded.metadata["catalogue_version"] == "vTEST" + + +def test_blind_refuses_double_blind(monkeypatch): + _patch_constant_factor(monkeypatch) + s = make_xi_part("reporting") + blinded = bd.blind_sacc(s, "seed", log=_NOLOG) + with pytest.raises(ValueError, match="already concealed"): + bd.blind_sacc(blinded, "seed2", log=_NOLOG) + + +def test_blind_refuses_non_blindable_part(): + """A ρ/τ diagnostic part must never see a blind call — loud refusal.""" + with pytest.raises(ValueError, match="no blindable block"): + bd.blind_sacc(make_rho_part(), "seed", log=_NOLOG) + + +def test_unblind_fails_closed_on_wrong_seed_or_config(monkeypatch): + """AC6 (in-memory half): wrong seed and wrong config both refuse loudly.""" + _patch_constant_factor(monkeypatch) + s = make_xi_part("reporting") + blinded = bd.blind_sacc(s, "right-seed", log=_NOLOG) + with pytest.raises(ValueError, match="blind_commitment"): + bd.unblind_sacc(blinded, "wrong-seed", log=_NOLOG) + with pytest.raises(ValueError, match="blind_config_digest"): + bd.unblind_sacc( + blinded, + "right-seed", + config=bd.BlindingConfig(s8_half_width=0.01), + log=_NOLOG, + ) + with pytest.raises(ValueError, match="not concealed"): + bd.unblind_sacc(s, "right-seed", log=_NOLOG) + + +# --------------------------------------------------------------------------- # +# blind-init custody + assembly hash assertion (fast — encryption only) +# --------------------------------------------------------------------------- # +def test_blind_init_writes_commitment_and_encrypted_bundle_only(tmp_path): + """AC6 (init): commitment.json + encrypted bundle; never a plaintext seed.""" + paths = bd.blind_init(str(tmp_path), log=_NOLOG) + with open(paths["commitment"], encoding="utf-8") as f: + commitment = json.load(f) + assert set(commitment) == {"label", "seed_sha256", "config_digest"} + assert len(commitment["seed_sha256"]) == 64 + assert commitment["config_digest"] == bd.BlindingConfig().config_digest() + # exactly the three custody outputs, no plaintext bundle + assert {p.name for p in tmp_path.iterdir()} == { + "commitment.json", + "blind_seed.encrpt", + "blind_seed.key", + } + # the decrypted seed matches the public commitment + bundle = bd._read_encrypted_json(paths["bundle"], paths["key"]) + assert bd.seed_commitment(bundle["seed"]) == commitment["seed_sha256"] + # one-shot custody: a second init in the same dir refuses + with pytest.raises(FileExistsError, match="refusing to overwrite"): + bd.blind_init(str(tmp_path), log=_NOLOG) + + +def test_read_seed_fails_closed_on_drifted_config(tmp_path): + bd.blind_init(str(tmp_path), log=_NOLOG) + with pytest.raises(ValueError, match="config digest"): + bd._read_seed(str(tmp_path), bd.BlindingConfig(s8_half_width=0.01)) + + +def _stamp(s, seed="s", label="A", config=None): + bd._stamp_provenance( + s, + bd.seed_commitment(seed), + label, + (config or bd.BlindingConfig()).config_digest(), + ) + return s + + +def test_assert_consistent_blind_shared_stamp(): + """One commitment across all blindable parts ⇒ the shared stamp returns; + ρ/τ parts are exempt from the assertion.""" + parts = make_parts(nbins=1) + for name in ("xi_reporting", "xi_integration", "cl"): + _stamp(parts[name]) + stamp = bd.assert_consistent_blind(list(parts.values())) + assert stamp == { + "concealed": True, + "blind": "A", + "blind_commitment": bd.seed_commitment("s"), + "blind_config_digest": bd.BlindingConfig().config_digest(), + } + + +def test_assert_consistent_blind_fails_closed(): + """AC6 (assembly): mismatched commitments and mixed states both refuse.""" + parts = make_parts(nbins=1, with_rho=False) + _stamp(parts["xi_reporting"], seed="one") + _stamp(parts["xi_integration"], seed="one") + _stamp(parts["cl"], seed="two") # different seed ⇒ different commitment + with pytest.raises(ValueError, match="different blind commitments"): + bd.assert_consistent_blind(list(parts.values())) + + parts = make_parts(nbins=1, with_rho=False) + _stamp(parts["xi_reporting"]) # blinded beside plaintext blindable parts + with pytest.raises(ValueError, match="mixed"): + bd.assert_consistent_blind(list(parts.values())) + + +def test_assert_consistent_blind_all_plaintext_is_none(): + """A declared-mock plaintext assembly (nothing blinded) asserts nothing.""" + assert bd.assert_consistent_blind(list(make_parts().values())) is None + + +def test_assert_consistent_blind_unconcealed_data_fails_closed(): + """PRD §4 "Mocks vs data": an unconcealed blindable part may assemble only + if its metadata declares ``type == "mock"`` — an unconcealed ``data`` part, + or one missing the tag, fails closed (skipping the blind can never + silently expose real data). Mixed concealed/plaintext still fails on the + mixed guard regardless of type.""" + parts = make_parts(nbins=1, with_rho=False) + parts["xi_integration"].metadata["type"] = "data" + with pytest.raises(ValueError, match="type"): + bd.assert_consistent_blind(list(parts.values())) + + parts = make_parts(nbins=1, with_rho=False) + del parts["cl"].metadata["type"] # missing tag counts as not-a-mock + with pytest.raises(ValueError, match=""): + bd.assert_consistent_blind(list(parts.values())) + + # concealed data parts assemble integration (that is the whole point of the blind) + parts = make_parts(nbins=1, with_rho=False) + for p in parts.values(): + p.metadata["type"] = "data" + _stamp(p) + assert bd.assert_consistent_blind(list(parts.values()))["concealed"] is True + + # mixed concealed/plaintext fails on the mixed guard even for mocks + parts = make_parts(nbins=1, with_rho=False) + _stamp(parts["xi_reporting"]) + with pytest.raises(ValueError, match="mixed"): + bd.assert_consistent_blind(list(parts.values())) + + +def test_gather_fails_closed_on_unconcealed_data_part(): + """The type guard reaches the terminal gather surface too.""" + parts = make_parts(nbins=1, with_rho=False) + parts["xi_reporting"].metadata["type"] = "data" + with pytest.raises(ValueError, match="type"): + sio.gather(list(parts.values())) + + +def test_assert_consistent_blind_differing_labels_warn_not_fail(): + """Same seed+config, different --label ⇒ assemble cleanly with a warning. + + The label is provenance, not custody state: parts blinded under one blind + but tagged with different labels must not be misread as different blinds. + The assembly succeeds (keyed on commitment+digest); a distinct warning + surfaces the label divergence rather than a false "different commitments". + """ + parts = make_parts(nbins=1, with_rho=False) + _stamp(parts["xi_reporting"], seed="one", label="A") + _stamp(parts["xi_integration"], seed="one", label="A") + _stamp(parts["cl"], seed="one", label="B") # same blind, different label + with pytest.warns(UserWarning, match="different labels"): + stamp = bd.assert_consistent_blind(list(parts.values())) + assert stamp["blind_commitment"] == bd.seed_commitment("one") + assert stamp["blind"] == "A" # deterministic: sorted-first label + + +def test_gather_assembles_parts_and_stamps_blind(): + """The sacc_io gather combines parts (points, tags, covariance blocks), + calls the assembly assertion, and stamps the shared blind.""" + parts = make_parts(nbins=1) + for name in ("xi_reporting", "xi_integration", "cl"): + _stamp(parts[name]) + ordered = [parts[k] for k in ("xi_reporting", "cl", "rho_tau", "xi_integration")] + s = sio.gather(ordered, metadata={"catalogue_version": "vTEST", "type": "mock"}) + + assert len(s.mean) == sum(len(p.mean) for p in ordered) + assert np.array_equal(np.array(s.mean), np.concatenate([p.mean for p in ordered])) + # covariance: block-diagonal of the parts, in order + cursor = 0 + for p in ordered: + n = len(p.mean) + assert np.array_equal( + s.covariance.dense[cursor : cursor + n, cursor : cursor + n], + p.covariance.dense, + ) + cursor += n + # the integration rows are addressable by the grid tag in the assembled file + assert len(s.indices(sio.XI_PLUS, grid="integration")) == len( + parts["xi_integration"].indices(sio.XI_PLUS, grid="integration") + ) + # bandpower windows survive assembly + assert s.get_bandpower_windows(s.indices(sio.CL_EE)) is not None + # blind stamp on the assembled file + assert s.metadata["concealed"] is True + assert s.metadata["blind_commitment"] == bd.seed_commitment("s") + assert s.metadata["catalogue_version"] == "vTEST" + + +def test_gather_fails_closed_on_mismatched_blinds(): + parts = make_parts(nbins=1, with_rho=False) + _stamp(parts["xi_reporting"], seed="one") + _stamp(parts["xi_integration"], seed="two") + _stamp(parts["cl"], seed="one") + with pytest.raises(ValueError, match="different blind commitments"): + sio.gather(list(parts.values())) + + +def test_cli_blind_init_refuses_existing_state(tmp_path): + """The blind-init CLI refuses to overwrite a previous blind's state.""" + import importlib.util + + script = ( + pathlib.Path(__file__).resolve().parents[3] / "scripts" / "blind_data_vector.py" + ) + spec = importlib.util.spec_from_file_location("_blind_cli", script) + cli = importlib.util.module_from_spec(spec) + spec.loader.exec_module(cli) + + (tmp_path / "commitment.json").write_text("{}") + with pytest.raises(SystemExit, match="refusing to overwrite"): + cli.main(["blind-init", str(tmp_path)]) + + +# --------------------------------------------------------------------------- # +# AC2 + AC3 + AC7: the shift itself (slow — fork + CCL) +# --------------------------------------------------------------------------- # +@pytest.mark.slow +@pytest.mark.parametrize( + "transfer_function", + ["boltzmann_camb", "eisenstein_hu"], + ids=["default-camb", "non-default-eh"], +) +def test_ac2_on_file_shift_equals_theory_difference_per_part(transfer_function): + """AC2: per-row shift on each blinded part == theory_fn(hidden) − + theory_fn(fiducial), hidden recovered by re-running the fork's draw — + for all three parts, on a two-bin fixture; and the recovered hidden + cosmology is identical across the three parts (one seed → one hidden). + + Scope: this verifies fork-draw recovery + placement (the shift on the + file is exactly what re-running the same backend at the recovered + hidden/fiducial points predicts, at the recorded rows). It is NOT a + backend-correctness test: both sides run the identical ``theory_fn``, so + any wrong-cosmology dependence cancels and a wrong backend would still + pass here. Backend correctness is carried by AC3. + + Parametrized over the Boltzmann backend (#280): the default CAMB route + and one non-default (Eisenstein–Hu — cheap, no CAMB run) both thread the + same ``transfer_function`` knob through all three theory backends.""" + cfg = bd.BlindingConfig.from_overrides( + {"theory": {"transfer_function": transfer_function}} + ) + seed = "ac2-seed" + parts = make_parts(nbins=2, with_rho=False) + + hiddens, worst = [], 0.0 + for name, part in parts.items(): + blinded = bd.blind_sacc(part, seed, config=cfg, log=_NOLOG) + hidden = bd.hidden_params(seed, cfg) # recovered per part + hiddens.append(hidden) + fiducial = cfg.theory.ccl_params() + ((block_name, indices, factory),) = bd._blindable_blocks(part) + theory = factory(bd._extract_block(part, indices), cfg.theory) + expected = theory(hidden) - theory(fiducial) + actual = np.array(blinded.mean)[indices] - np.array(part.mean)[indices] + gap = np.max(np.abs(actual - expected)) + scale = np.max(np.abs(expected)) + worst = max(worst, gap / scale) + assert gap <= 1e-10 * max(scale, 1e-30), f"{name}/{block_name}: |Δ|={gap:.3e}" + # one seed → one hidden cosmology across all parts + assert hiddens[0] == hiddens[1] == hiddens[2] + print(f"\nAC2 max relative shift mismatch across parts: {worst:.3e}") + + +@pytest.mark.slow +def test_ac3_cross_backend_against_independent_ccl_reference(): + """AC3: the realized shift matches an independently written direct-CCL + reference (self-contained here; does not touch the blinding backends).""" + import pyccl as ccl + + cfg = bd.BlindingConfig() + seed = "ac3-seed" + reporting = make_xi_part("reporting", nbins=2) + cl_part = make_cl_part(nbins=2) + blinded_xi = bd.blind_sacc(reporting, seed, config=cfg, log=_NOLOG) + blinded_cl = bd.blind_sacc(cl_part, seed, config=cfg, log=_NOLOG) + hidden = bd.hidden_params(seed, cfg) + fiducial = cfg.theory.ccl_params() + + # ----- independent reference (from scratch; same fixture n(z), θ, ℓ) ---- + def ref_cosmo(params): + return ccl.Cosmology( + **params, + matter_power_spectrum="camb", + extra_parameters={ + "camb": {"halofit_version": "mead2020_feedback", "HMCode_logT_AGN": 7.5} + }, + ) + + ell = np.unique( + np.concatenate([np.arange(2, 50), np.geomspace(50, 6e4, 200)]).astype(float) + ) + + def ref_xi(params, nz_i, nz_j, theta): + cosmo = ref_cosmo(params) + ti = ccl.WeakLensingTracer(cosmo, dndz=nz_i) + tj = ccl.WeakLensingTracer(cosmo, dndz=nz_j) + cl = ccl.angular_cl(cosmo, ti, tj, ell) + xip = ccl.correlation(cosmo, ell=ell, C_ell=cl, theta=theta / 60.0, type="GG+") + xim = ccl.correlation(cosmo, ell=ell, C_ell=cl, theta=theta / 60.0, type="GG-") + return xip, xim + + worst = 0.0 + for i, j in bd.xi_pairs(reporting, "reporting"): + tr = sio._pair((i, j)) + theta = sio._tag(reporting, sio.XI_PLUS, tr, "theta", grid="reporting") + nz_i, nz_j = sio.get_nz(reporting, i), sio.get_nz(reporting, j) + xip_h, xim_h = ref_xi(hidden, nz_i, nz_j, theta) + xip_f, xim_f = ref_xi(fiducial, nz_i, nz_j, theta) + for dt, ref_shift in ( + (sio.XI_PLUS, xip_h - xip_f), + (sio.XI_MINUS, xim_h - xim_f), + ): + idx = reporting.indices(dt, tr, grid="reporting") + realized = np.array(blinded_xi.mean)[idx] - np.array(reporting.mean)[idx] + worst = max(worst, np.max(np.abs(realized - ref_shift))) + print(f"\nAC3 max |realized − independent reference| (reporting ξ±): {worst:.3e}") + assert worst < 1e-8 # observed ~1e-10; factor magnitudes ~1e-6 + + # pseudo-Cℓ: W @ ΔCℓ_EE against the same independent reference + for i, j in bd.cl_pairs(cl_part): + tr = sio._pair((i, j)) + idx = cl_part.indices(sio.CL_EE, tr) + window = cl_part.get_bandpower_windows(idx) + w_ell = np.asarray(window.values, dtype=float) + w_mat = np.asarray(window.weight, dtype=float) + nz_i, nz_j = sio.get_nz(cl_part, i), sio.get_nz(cl_part, j) + + def ref_cl(params): + cosmo = ref_cosmo(params) + ti = ccl.WeakLensingTracer(cosmo, dndz=nz_i) + tj = ccl.WeakLensingTracer(cosmo, dndz=nz_j) + return ccl.angular_cl(cosmo, ti, tj, w_ell) + + ref_shift = w_mat.T @ (ref_cl(hidden) - ref_cl(fiducial)) + realized = np.array(blinded_cl.mean)[idx] - np.array(cl_part.mean)[idx] + assert np.max(np.abs(realized - ref_shift)) < 1e-12 # bandpowers ~1e-9 + + +@pytest.mark.slow +def test_ac7_reproducibility_same_seed_same_shift(): + """AC7: two blind runs of the same part with the same (seed, config) + produce identical shifts; a different seed produces a different one.""" + part = make_xi_part("reporting") + b1 = bd.blind_sacc(part, "repro-seed", log=_NOLOG) + b2 = bd.blind_sacc(part, "repro-seed", log=_NOLOG) + assert np.array_equal(np.array(b1.mean), np.array(b2.mean)) + b3 = bd.blind_sacc(part, "other-seed", log=_NOLOG) + assert not np.array_equal(np.array(b1.mean), np.array(b3.mean)) + + +# --------------------------------------------------------------------------- # +# AC1, AC4, AC5, AC9: born-blinded derived statistics (slow + cosmo_numba) +# --------------------------------------------------------------------------- # +@pytest.mark.slow +def test_ac1_zero_shift_is_identity(): + """AC1: a zero envelope reproduces every part exactly, and the integration part + run downstream through the b_modes seams yields COSEBIs and pure-E/B + identical to the unblinded run — the per-part plumbing and the + born-blinded derivation path are the identity at zero shift.""" + pytest.importorskip("cosmo_numba") + zero = bd.BlindingConfig(s8_half_width=0.0, omega_m_half_width=0.0) + parts = make_parts(nbins=1, with_rho=False) + blinded = { + name: bd.blind_sacc(p, "any-seed", config=zero, log=_NOLOG) + for name, p in parts.items() + } + for name, part in parts.items(): + assert np.array_equal(np.array(part.mean), np.array(blinded[name].mean)), ( + f"zero-shift blind changed {name}" + ) + # derived statistics downstream: identical inputs ⇒ identical numbers + En_t, Bn_t, modes_t = _derive_downstream( + parts["xi_reporting"], parts["xi_integration"] + ) + En_b, Bn_b, modes_b = _derive_downstream( + blinded["xi_reporting"], blinded["xi_integration"] + ) + assert np.array_equal(En_t, En_b) and np.array_equal(Bn_t, Bn_b) + for key in modes_t: + t, b = modes_t[key], modes_b[key] + both_nan = np.isnan(t) & np.isnan(b) + assert np.array_equal(t[~both_nan], b[~both_nan]), key + assert np.array_equal(np.isnan(t), np.isnan(b)), key + + +@pytest.mark.slow +def test_ac4_b_mode_invariance_and_leakage_floor(): + """AC4: the ΔBₙ induced by deriving B-modes from the blinded integration ξ± is + independent of the injected B amplitude (a fixed absolute E→B leakage + offset, not fractional). The magnitude is measured and reported, never + asserted against a constant.""" + pytest.importorskip("cosmo_numba") + seed = "ac4-seed" + deltas, reports = [], [] + for amp in (2e-6, 2e-5): + reporting = make_xi_part("reporting") + integration = make_xi_part("integration", b_amplitude=amp) + blinded_reporting = bd.blind_sacc(reporting, seed, log=_NOLOG) + blinded_integration = bd.blind_sacc(integration, seed, log=_NOLOG) + _, Bn_t, modes_t = _derive_downstream(reporting, integration) + _, Bn_b, modes_b = _derive_downstream(blinded_reporting, blinded_integration) + d_bn = Bn_b - Bn_t + d_xib = modes_b["xip_B"] - modes_t["xip_B"] + finite = np.isfinite(d_xib) + deltas.append((d_bn, d_xib[finite])) + reports.append( + f"B={amp:.0e}: max|ΔBₙ|={np.max(np.abs(d_bn)):.3e} " + f"(ΔBₙ/Bₙ={np.max(np.abs(d_bn)) / np.max(np.abs(Bn_t)):.2e}), " + f"max|Δξ+_B|={np.max(np.abs(d_xib[finite])):.3e} " + f"({np.max(np.abs(d_xib[finite])) / amp:.2%} of injected B)" + ) + print("\nAC4 " + "\nAC4 ".join(reports)) + + (d_bn_1, d_xib_1), (d_bn_2, d_xib_2) = deltas + scale = max(np.max(np.abs(d_bn_1)), 1e-30) + gap = np.max(np.abs(d_bn_1 - d_bn_2)) + print( + f"AC4 ΔBₙ amplitude-independence: max|ΔBₙ(2e-6) − ΔBₙ(2e-5)| = " + f"{gap:.3e} ({gap / scale:.2e} of |ΔBₙ|)" + ) + assert gap <= 1e-9 * scale + 1e-24, ( + "ΔBₙ depends on the injected B amplitude — the shift is not pure E" + ) + # The pure-ξ_B leakage is also amplitude-independent, but only to the + # adaptive-quadrature floor: cosmo_numba's Schneider integrals subdivide + # adaptively, so the estimator is not bit-linear in its inputs and the + # two runs differ at a small fraction of the (tiny) leakage itself. The + # COSEBIs assertion above carries the exact-identity criterion; this one + # bounds the quadrature wobble. + scale_x = max(np.max(np.abs(d_xib_1)), 1e-30) + gap_x = np.max(np.abs(d_xib_1 - d_xib_2)) + print( + f"AC4 Δξ+_B amplitude-independence: {gap_x:.3e} " + f"({gap_x / scale_x:.2e} of the leakage)" + ) + assert gap_x <= 0.05 * scale_x + + +@pytest.mark.slow +def test_ac5_untouched_blocks_and_row_order(): + """AC5: each part's covariance byte-identical; the Cℓ BB/EB rows and the + ρ/τ part never blinded; every part's shifted rows land at their original + within-part indices (order-preservation).""" + parts = make_parts(nbins=1) + seed = "ac5-seed" + blinded = { + name: bd.blind_sacc(parts[name], seed, log=_NOLOG) + for name in ("xi_reporting", "xi_integration", "cl") + } + for name, b in blinded.items(): + part = parts[name] + assert np.array_equal(b.covariance.dense, part.covariance.dense), name + # row order: the identity of every row (type/tracers/tags) unchanged + for a, c in zip(part.data, b.data): + assert (a.data_type, a.tracers, a.tags) == (c.data_type, c.tracers, c.tags) + # BB/EB rows of the Cℓ part untouched (pure E-mode shift) + for dt in (sio.CL_BB, sio.CL_EB): + idx = parts["cl"].indices(dt) + assert np.array_equal( + np.array(blinded["cl"].mean)[idx], np.array(parts["cl"].mean)[idx] + ), f"{dt} was touched by the blind" + # the blindable rows did move (the blind actually blinded) + for name, grid in ( + ("xi_reporting", "reporting"), + ("xi_integration", "integration"), + ): + idx = bd._xi_indices(parts[name], grid) + assert not np.allclose( + np.array(blinded[name].mean)[idx], np.array(parts[name].mean)[idx], atol=0 + ) + # ρ/τ: refused by blind_sacc (test_blind_refuses_non_blindable_part) and + # exempt in assembly — pass through gather untouched + s = sio.gather( + [ + blinded["xi_reporting"], + parts["rho_tau"], + blinded["cl"], + blinded["xi_integration"], + ] + ) + idx = s.indices(sio.RHO_PLUS.format(k=0)) + assert np.array_equal( + np.array(s.mean)[idx], + np.array(parts["rho_tau"].mean)[ + parts["rho_tau"].indices(sio.RHO_PLUS.format(k=0)) + ], + ) + + +@pytest.mark.slow +def test_ac9_pure_eb_nan_parity_under_blind(): + """AC9: the pure-E/B NaN pattern born from the blinded parts is identical + to the true parts' — blinding never moves a NaN. + + The Schneider estimator returns NaN wherever a reporting point lacks + interior support against the edge-based integration bounds. Whatever + that pattern is on the true parts (the fixture puts the outermost + reporting point at the boundary, so it is non-empty here; on production + files it is empty), the blinded derivation must reproduce it bit-for-bit: + the blind is a pure shift of the estimator's inputs, not a change of + estimator support. The finite values move (the ξ± shifted); the NaN mask + does not.""" + pytest.importorskip("cosmo_numba") + seed = "ac9-seed" + reporting, integration = make_xi_part("reporting"), make_xi_part("integration") + _, _, modes_t = _derive_downstream(reporting, integration) + _, _, modes_b = _derive_downstream( + bd.blind_sacc(reporting, seed, log=_NOLOG), + bd.blind_sacc(integration, seed, log=_NOLOG), + ) + for key in modes_t: + t, b = modes_t[key], modes_b[key] + assert np.array_equal(np.isnan(t), np.isnan(b)), ( + f"blinding moved the pure-E/B NaN pattern for {key}" + ) + # the finite values did move (the blind actually shifted the ξ±) + t, b = modes_t["xip_E"], modes_b["xip_E"] + finite = np.isfinite(t) + assert finite.any() and not np.allclose(t[finite], b[finite], atol=0) + + +# --------------------------------------------------------------------------- # +# AC6 + AC8: end-to-end custody through the file surface (slow + cosmo_numba) +# --------------------------------------------------------------------------- # +@pytest.mark.slow +def test_ac6_ac8_end_to_end_init_parts_gather_unblind(tmp_path): + """AC8: blind-init → blind-part on each intermediate part → terminal + gather (hash assertion passes, stamp lands) → unblind restores each part + bit-for-bit, and the derived statistics re-derived from the unblinded + integration part reproduce the truth. AC6: no plaintext part or seed survives + on disk, no ``seed_smokescreen`` key; unblind fails closed on a tampered + commitment.""" + pytest.importorskip("cosmo_numba") + parts = make_parts(nbins=1) + En_true, Bn_true, modes_true = _derive_downstream( + parts["xi_reporting"], parts["xi_integration"] + ) + + blind_dir = tmp_path / "blind" + blind_dir.mkdir() + init = bd.blind_init(str(blind_dir), log=_NOLOG) + + part_files, out_paths = {}, {} + for name in ("xi_reporting", "xi_integration", "cl"): + path = tmp_path / f"{name}.fits" + sio.save(parts[name], str(path)) + out_paths[name] = bd.blind_part(str(path), str(blind_dir), log=_NOLOG) + part_files[name] = path + + # -- custody hygiene (AC6) --------------------------------------------- -- + for name, path in part_files.items(): + assert not path.exists(), f"plaintext part {name} was not deleted" + blinded = sio.load(out_paths[name]["blinded"]) + assert blinded.metadata["concealed"] is True + assert "seed_smokescreen" not in blinded.metadata + assert pathlib.Path(out_paths[name]["escrow"]).exists() + assert not np.array_equal(np.array(blinded.mean), np.array(parts[name].mean)) + with open(init["commitment"], encoding="utf-8") as f: + commitment = json.load(f) + assert set(commitment) == {"label", "seed_sha256", "config_digest"} + blinded_parts = {n: sio.load(p["blinded"]) for n, p in out_paths.items()} + for b in blinded_parts.values(): + assert b.metadata["blind_commitment"] == commitment["seed_sha256"] + # no plaintext json anywhere beside the blind outputs + assert not list(tmp_path.rglob("*escrow.json")) + assert not (blind_dir / "blind_seed.json").exists() + + # -- terminal assembly: hash assertion + stamp (AC8) -------------------- -- + assembled = sio.gather( + [ + blinded_parts["xi_reporting"], + blinded_parts["cl"], + parts["rho_tau"], + blinded_parts["xi_integration"], + ], + metadata={"catalogue_version": "vTEST", "type": "mock"}, + ) + assert assembled.metadata["blind_commitment"] == commitment["seed_sha256"] + # born-blinded derived statistics from the blinded parts differ from truth + En_b, Bn_b, _ = _derive_downstream( + blinded_parts["xi_reporting"], blinded_parts["xi_integration"] + ) + assert not np.allclose(En_b, En_true, atol=0) + + # -- fail-closed on tampered commitment (AC6) --------------------------- -- + tampered = dict(commitment, seed_sha256="0" * 64) + with open(init["commitment"], "w", encoding="utf-8") as f: + json.dump(tampered, f) + with pytest.raises(ValueError, match="sha256"): + bd.unblind_part( + out_paths["xi_integration"]["blinded"], + str(blind_dir), + str(tmp_path / "never.fits"), + log=_NOLOG, + ) + with open(init["commitment"], "w", encoding="utf-8") as f: + json.dump(commitment, f) + with pytest.raises(ValueError, match="config digest"): + bd.unblind_part( + out_paths["xi_integration"]["blinded"], + str(blind_dir), + str(tmp_path / "never.fits"), + config=bd.BlindingConfig(s8_half_width=0.01), + log=_NOLOG, + ) + + # -- bit-for-bit restoration per part (AC8) ----------------------------- -- + restored = {} + for name in ("xi_reporting", "xi_integration", "cl"): + out = tmp_path / f"{name}_restored.fits" + bd.unblind_part( + out_paths[name]["blinded"], str(blind_dir), str(out), log=_NOLOG + ) + restored[name] = sio.load(str(out)) + assert np.array_equal( + np.array(restored[name].mean), np.array(parts[name].mean) + ), f"{name} not restored bit-for-bit" + assert not restored[name].metadata.get("concealed", False) + assert "blind_commitment" not in restored[name].metadata + + # unblinding then re-deriving reproduces the true derived statistics + En_r, Bn_r, modes_r = _derive_downstream( + restored["xi_reporting"], restored["xi_integration"] + ) + assert np.array_equal(En_r, En_true) and np.array_equal(Bn_r, Bn_true) + for key in modes_true: + t, r = modes_true[key], modes_r[key] + both_nan = np.isnan(t) & np.isnan(r) + assert np.array_equal(t[~both_nan], r[~both_nan]), key + assert np.array_equal(np.isnan(t), np.isnan(r)), key + + +def test_ac8_dotted_versioned_part_names_escrow_and_restore(tmp_path): + """AC8 under the canonical catalogue-version naming (dotted stems). + + Production part files carry the versioned name ``v1.4.6.3_xi_reporting.fits`` + etc. ``smokescreen.encryption.encrypt_file`` names its outputs from + ``basename.split('.')[0]``, so both these parts would misfile onto + ``v1.encrpt``/``v1.key`` and the second would silently overwrite the + first's escrowed truth. Guard: the escrow lands at the exact + :func:`part_paths` name, two dot-prefix-sharing parts do not collide, and + each restores bit-for-bit.""" + parts = make_parts(nbins=1) + blind_dir = tmp_path / "blind" + blind_dir.mkdir() + bd.blind_init(str(blind_dir), log=_NOLOG) + + version = "v1.4.6.3" + out_paths, part_files = {}, {} + for name in ("xi_reporting", "xi_integration"): + path = tmp_path / f"{version}_{name}.fits" + sio.save(parts[name], str(path)) + out_paths[name] = bd.blind_part(str(path), str(blind_dir), log=_NOLOG) + part_files[name] = path + + # escrow bundles landed at the declared names (no split('.') truncation), + # and the two dot-prefix-sharing parts did not collide onto one bundle. + escrow_files = {n: p["escrow"] for n, p in out_paths.items()} + assert escrow_files["xi_reporting"] != escrow_files["xi_integration"] + for name, path in part_files.items(): + assert not path.exists(), f"plaintext part {name} was not deleted" + assert pathlib.Path(out_paths[name]["escrow"]).exists(), name + assert pathlib.Path(out_paths[name]["escrow_key"]).exists(), name + # the truncated-name collision target must not exist + assert not (tmp_path / "v1.encrpt").exists() + assert not (tmp_path / "v1.key").exists() + assert not list(tmp_path.rglob("*escrow.json")) + + # each part restores bit-for-bit via its own escrow (not subtraction-only) + for name in ("xi_reporting", "xi_integration"): + out = tmp_path / f"{version}_{name}_restored.fits" + bd.unblind_part( + out_paths[name]["blinded"], str(blind_dir), str(out), log=_NOLOG + ) + restored = sio.load(str(out)) + assert np.array_equal(np.array(restored.mean), np.array(parts[name].mean)), ( + f"{name} not restored bit-for-bit" + ) diff --git a/src/sp_validation/tests/test_camb_ccl_crosscheck.py b/src/sp_validation/tests/test_camb_ccl_crosscheck.py new file mode 100644 index 00000000..b4cb3b3f --- /dev/null +++ b/src/sp_validation/tests/test_camb_ccl_crosscheck.py @@ -0,0 +1,174 @@ +"""CAMB↔CCL theory cross-check (blinding PRD AC10–14). + +The blinding shift is a difference of CCL theory vectors; downstream +inference runs CAMB (CosmoSIS). The shift only means what it is intended to +mean if CCL and CAMB predict the same ξ± at a fixed cosmology on our θ grid. +This module asserts that agreement between the two independent ξ± paths in +:mod:`sp_validation.blinding_theory`: + +- **Path A** (:func:`~sp_validation.blinding_theory.xi_ccl`): CCL-native — CCL's + Boltzmann-CAMB HMCode2020 P(k) route, projected by CCL Limber + FFTLog. +- **Path B** (:func:`~sp_validation.blinding_theory.xi_camb`): an independent + pycamb run produces the HMCode2020 ``P(k, z)`` (σ8-matched ``A_s``), + wrapped in a ``ccl.Pk2D`` and projected through the same CCL machinery. + +Because both paths route their nonlinear P(k) through CAMB's HMCode2020 and +both project through CCL, a common Limber+FFTLog bug cancels: this test +validates the **P(k) recipe** and the **σ8/A_s amplitude convention**, not +the projection. The one convention subtlety it settles: the fiducial fixes +σ8 for CCL but A_s for CAMB; a nominal ``A_s = 2.1e-9`` leaves CAMB's σ8 +≈3% off target — enough to blow a ξ± comparison to ~9–10%. +""" + +import pathlib +import re + +import numpy as np +import pytest + +from sp_validation import blinding_theory as cm + +# Tolerances (AC11/AC12). Observed floor on this fixture: see the printed +# numbers in the slow tests — the tolerances sit above the floor with +# headroom; version bumps move the floor and that is not a regression. +XIP_RTOL = 0.005 # 0.5 % +XIM_RTOL = 0.010 # 1.0 % +# ξ− crosses zero on this grid: the relative assertion applies only where +# |ξ−| exceeds an absolute floor set from the fixture's peak |ξ−|. +XIM_FLOOR_FRAC = 0.05 + + +# --------------------------------------------------------------------------- # +# Deterministic fixture: one Gaussian source bin, 12-point θ grid +# --------------------------------------------------------------------------- # +def _gauss_nz(n=400): + z = np.linspace(0.01, 3.0, n) + nz = np.exp(-0.5 * ((z - 0.7) / 0.2) ** 2) + return z, nz / np.trapezoid(nz, z) + + +THETA_ARCMIN = np.geomspace(5.0, 250.0, 12) + + +def _both_paths(config, **camb_kwargs): + z, nz = _gauss_nz() + xip_a, xim_a = cm.xi_ccl( + config.ccl_params(), config, (z, nz), (z, nz), THETA_ARCMIN + ) + xip_b, xim_b, As = cm.xi_camb(config, (z, nz), THETA_ARCMIN, **camb_kwargs) + return (xip_a, xim_a), (xip_b, xim_b), As + + +def _assert_xi_agreement(a, b, label): + (xip_a, xim_a), (xip_b, xim_b) = a, b + assert np.all(xip_a > 0) and np.all(xip_b > 0) # sensible cosmic shear + rel_p = np.abs(xip_b - xip_a) / np.abs(xip_a) + assert rel_p.max() < XIP_RTOL, ( + f"{label}: ξ+ max rel diff {rel_p.max():.3%} ≥ {XIP_RTOL:.1%}" + ) + floor = XIM_FLOOR_FRAC * np.max(np.abs(xim_a)) + above = np.abs(xim_a) > floor + rel_m = np.abs(xim_b - xim_a)[above] / np.abs(xim_a)[above] + assert rel_m.max() < XIM_RTOL, ( + f"{label}: ξ− max rel diff {rel_m.max():.3%} ≥ {XIM_RTOL:.1%} (on |ξ−| > floor)" + ) + # near the zero crossing: absolute agreement at the floor scale + abs_m = np.abs(xim_b - xim_a)[~above] + if len(abs_m): + assert abs_m.max() < XIM_RTOL * floor, ( + f"{label}: ξ− absolute diff {abs_m.max():.3e} near zero crossing" + ) + print( + f"\n{label}: ξ+ max rel {rel_p.max():.3%}; " + f"ξ− max rel {rel_m.max():.3%} (above floor, " + f"{above.sum()}/{len(above)} points)" + ) + + +# --------------------------------------------------------------------------- # +# AC10: σ8/A_s reconciliation +# --------------------------------------------------------------------------- # +@pytest.mark.slow +def test_ac10_sigma8_As_reconciliation(): + """(a) nominal A_s leaves CAMB's σ8 >2% off target — the convention + offset is real; (b) the closed-form rescale lands on target to <1e-4.""" + cfg = cm.TheoryConfig() + target = cfg.sigma8() + + nominal = cm.camb_linear_sigma8(cfg, 2.1e-9) + offset = abs(nominal / target - 1) + print(f"\nAC10 nominal-A_s σ8 offset: {offset:.4f}") + assert offset > 0.02 + + As = cm.camb_As_for_sigma8(cfg, target) + matched = cm.camb_linear_sigma8(cfg, As) + print(f"AC10 σ8-matched residual: {abs(matched - target):.2e} (A_s={As:.4e})") + assert abs(matched - target) < 1e-4 + + +# --------------------------------------------------------------------------- # +# AC11 + AC12: ξ± agreement at and off the fiducial +# --------------------------------------------------------------------------- # +@pytest.mark.slow +def test_ac11_xi_agreement_at_fiducial(): + cfg = cm.TheoryConfig() + a, b, _ = _both_paths(cfg) + _assert_xi_agreement(a, b, "AC11 fiducial") + + +@pytest.mark.slow +def test_ac12_xi_agreement_off_fiducial(): + """A representative in-envelope offset — the *shift* (a difference of two + theory vectors) must not inherit a stack-disagreement bias.""" + cfg = cm.TheoryConfig.from_overrides({"S8": 0.80 + 0.075, "Omega_m": 0.30 - 0.05}) + a, b, _ = _both_paths(cfg) + _assert_xi_agreement(a, b, "AC12 off-fiducial") + + +# --------------------------------------------------------------------------- # +# AC13: halofit token pinned to the inference config (fast) +# --------------------------------------------------------------------------- # +def test_ac13_halofit_token_matches_inference_config(): + """The blinding fiducial's CCL halofit token equals the CosmoSIS + inference config's ``halofit_version`` — asserted against the config + file itself. All three blinding backends share one recipe by + construction and would agree with each other while jointly diverging + from the inference stack, so this cannot be caught by the cross-backend + test and is asserted independently here.""" + ini = ( + pathlib.Path(__file__).resolve().parents[3] + / "cosmo_inference" + / "cosmosis_config" + / "cosmosis_pipeline_A_ia_cell.ini" + ) + match = re.search(r"^halofit_version\s*=\s*(\S+)", ini.read_text(), re.MULTILINE) + assert match, f"no halofit_version in {ini}" + inference_token = match.group(1) + cfg = cm.TheoryConfig() + assert cfg.ccl_halofit_version == inference_token + # the two stack tokens denote ONE recipe; a divergence is a config bug + assert cfg.camb_halofit_version == cfg.ccl_halofit_version + # #280: the shipped Boltzmann backend is CAMB-through-CCL, matching the + # CosmoSIS+CAMB inference stack — one power-spectrum path. The cross-check + # tests above (AC10–12, 14) all run at this default configuration. + assert cfg.transfer_function == "boltzmann_camb" + + +# --------------------------------------------------------------------------- # +# AC14: fast smoke — broken wiring caught in the fast suite +# --------------------------------------------------------------------------- # +def test_ac14_crosscheck_smoke(): + """Both paths run at coarse resolution: finite, positive, + few-percent-agreeing ξ+, and a σ8-matched A_s in a sane range.""" + cfg = cm.TheoryConfig() + z, nz = _gauss_nz(n=150) + theta = np.geomspace(10.0, 100.0, 4) + xip_a, _ = cm.xi_ccl(cfg.ccl_params(), cfg, (z, nz), (z, nz), theta) + xip_b, _, As = cm.xi_camb( + cfg, (z, nz), theta, n_ell=120, ell_max=30000, kmax=10.0, n_k=200 + ) + assert np.all(np.isfinite(xip_a)) and np.all(np.isfinite(xip_b)) + assert np.all(xip_a > 0) and np.all(xip_b > 0) + assert 1e-9 < As < 3e-9 + rel = np.abs(xip_b - xip_a) / np.abs(xip_a) + assert rel.max() < 0.05, f"smoke ξ+ rel diff {rel.max():.3%} unexpectedly large" From c09b063f2c46d89d7ced93d438e2e6394e1a0607 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 11:50:22 +0200 Subject: [PATCH 35/47] test(blinding): adapt to canonical sacc_io save/load and relocated config Three integration-drift fixes surfaced by the reconciled base: - test_ac6_ac8 / test_ac8_dotted: the end-to-end fixtures now pass type="mock" to sio.save (PR-2 requires the provenance stamp); the parts are mocks, blind_part re-saves inheriting that type. - test_ac13: the CosmoSIS halofit config moved to cosmo_inference/cosmosis_config/templates/ on develop; point the AC13 assertion at the new path (token unchanged: mead2020_feedback). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WzUt7VbtXwr2SCHUdiQTyt --- src/sp_validation/tests/test_blinding.py | 4 ++-- src/sp_validation/tests/test_camb_ccl_crosscheck.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/sp_validation/tests/test_blinding.py b/src/sp_validation/tests/test_blinding.py index 7013a81a..5cd0f0f7 100644 --- a/src/sp_validation/tests/test_blinding.py +++ b/src/sp_validation/tests/test_blinding.py @@ -975,7 +975,7 @@ def test_ac6_ac8_end_to_end_init_parts_gather_unblind(tmp_path): part_files, out_paths = {}, {} for name in ("xi_reporting", "xi_integration", "cl"): path = tmp_path / f"{name}.fits" - sio.save(parts[name], str(path)) + sio.save(parts[name], str(path), type="mock") out_paths[name] = bd.blind_part(str(path), str(blind_dir), log=_NOLOG) part_files[name] = path @@ -1081,7 +1081,7 @@ def test_ac8_dotted_versioned_part_names_escrow_and_restore(tmp_path): out_paths, part_files = {}, {} for name in ("xi_reporting", "xi_integration"): path = tmp_path / f"{version}_{name}.fits" - sio.save(parts[name], str(path)) + sio.save(parts[name], str(path), type="mock") out_paths[name] = bd.blind_part(str(path), str(blind_dir), log=_NOLOG) part_files[name] = path diff --git a/src/sp_validation/tests/test_camb_ccl_crosscheck.py b/src/sp_validation/tests/test_camb_ccl_crosscheck.py index b4cb3b3f..3f286474 100644 --- a/src/sp_validation/tests/test_camb_ccl_crosscheck.py +++ b/src/sp_validation/tests/test_camb_ccl_crosscheck.py @@ -139,6 +139,7 @@ def test_ac13_halofit_token_matches_inference_config(): pathlib.Path(__file__).resolve().parents[3] / "cosmo_inference" / "cosmosis_config" + / "templates" / "cosmosis_pipeline_A_ia_cell.ini" ) match = re.search(r"^halofit_version\s*=\s*(\S+)", ini.read_text(), re.MULTILINE) From b007ed12995af725f6531198604c5df5fab98984 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 18 Jul 2026 17:31:27 +0200 Subject: [PATCH 36/47] twopoint_convert: filter covariance selections by grid tag; guard empty selections Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MN9VazXKHUHQg16kiG7Ufk --- .../tests/test_twopoint_convert.py | 115 +++++++++++++++++- src/sp_validation/twopoint_convert.py | 14 ++- 2 files changed, 120 insertions(+), 9 deletions(-) diff --git a/src/sp_validation/tests/test_twopoint_convert.py b/src/sp_validation/tests/test_twopoint_convert.py index d50e33b9..59e0256d 100644 --- a/src/sp_validation/tests/test_twopoint_convert.py +++ b/src/sp_validation/tests/test_twopoint_convert.py @@ -367,6 +367,98 @@ def test_perturbed_xi_changes_output(tmp_path): assert np.array_equal(new_xip, inp2["xip"]) +def test_integration_grid_points_ignored(tmp_path): + """Extra xi/tau/Cl points tagged grid='integration' must not leak into the + converted output. + + Bug report (HIGH): ``_build_covmat`` selected xi+/xi-/tau covariance + indices with raw ``s.indices(dtype, pair)`` -- no ``grid`` filter -- while + the corresponding data-vector HDUs are built via ``sacc_io.get_xi(..., + grid='reporting')``. A real SACC carrying both 'reporting' and + 'integration' grid points under the same data type + tracer pair (e.g. the + fine COSEBIs/pure-EB integration input) would inflate/desync the + covariance relative to the data vector. Pin that the converted file is + byte-identical whether or not the integration-grid points are present. + """ + inp = _inputs(seed=40) + s_plain = _sacc(inp, cl=True, rho_tau=True) + rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) + out_plain = tmp_path / "plain.fits" + twopoint_convert.sacc_to_twopoint_fits( + s_plain, str(out_plain), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 + ) + + # Build an augmented SACC: same reporting-grid points, plus a full extra + # set of integration-grid xi/tau/Cl points on a distinct angular grid (so + # they cannot coincide with the reporting points), with the covariance + # sized to cover both blocks. + s_aug = sacc_io.new_sacc({0: (inp["z"], inp["nz"])}) + sacc_io.add_xi( + s_aug, (0, 0), inp["theta"], inp["xip"], inp["xim"], grid="reporting" + ) + sacc_io.add_pseudo_cl( + s_aug, + (0, 0), + inp["ell"], + inp["cl_ee"], + inp["cl_bb"], + inp["cl_eb"], + window_ells=np.arange(2, 102), + window_weights=np.random.default_rng(9).uniform(0, 1, (100, N_ELL)), + grid="reporting", + ) + sacc_io.add_tau(s_aug, (0, 0), 0, inp["theta"], inp["tau0p"], inp["tau0m"]) + sacc_io.add_tau(s_aug, (0, 0), 2, inp["theta"], inp["tau2p"], inp["tau2m"]) + + theta_int = inp["theta"] + 1000.0 # disjoint grid, never collides + xip_int = np.random.default_rng(41).uniform(1e-6, 1e-4, N_ANG) + xim_int = np.random.default_rng(42).uniform(1e-6, 1e-4, N_ANG) + sacc_io.add_xi(s_aug, (0, 0), theta_int, xip_int, xim_int, grid="integration") + + n = len(s_aug.mean) + full = np.zeros((n, n)) + ip = sacc_io._indices(s_aug, sacc_io.XI_PLUS, (SOURCE, SOURCE), grid="reporting") + im = sacc_io._indices(s_aug, sacc_io.XI_MINUS, (SOURCE, SOURCE), grid="reporting") + xi_all = np.concatenate([ip, im]) + full[np.ix_(xi_all, xi_all)] = inp["xi_cov"] + + iee = sacc_io._indices(s_aug, sacc_io.CL_EE, (SOURCE, SOURCE), grid="reporting") + full[np.ix_(iee, iee)] = inp["cl_cov"] + for dtype in (sacc_io.CL_BB, sacc_io.CL_EB): + idx = sacc_io._indices(s_aug, dtype, (SOURCE, SOURCE), grid="reporting") + full[np.ix_(idx, idx)] = np.eye(N_ELL) + + t0p = sacc_io._indices( + s_aug, sacc_io.TAU_PLUS.format(k=0), (SOURCE, PSF), grid="reporting" + ) + t2p = sacc_io._indices( + s_aug, sacc_io.TAU_PLUS.format(k=2), (SOURCE, PSF), grid="reporting" + ) + tau_pp = np.concatenate([t0p, t2p]) + full[np.ix_(tau_pp, tau_pp)] = inp["tau_cov_full"][: 2 * N_ANG, : 2 * N_ANG] + for dtype in (sacc_io.TAU_MINUS.format(k=0), sacc_io.TAU_MINUS.format(k=2)): + idx = sacc_io._indices(s_aug, dtype, (SOURCE, PSF), grid="reporting") + full[np.ix_(idx, idx)] = np.eye(N_ANG) + + ip_int = sacc_io._indices( + s_aug, sacc_io.XI_PLUS, (SOURCE, SOURCE), grid="integration" + ) + im_int = sacc_io._indices( + s_aug, sacc_io.XI_MINUS, (SOURCE, SOURCE), grid="integration" + ) + xi_int_all = np.concatenate([ip_int, im_int]) + full[np.ix_(xi_int_all, xi_int_all)] = _spd(2 * N_ANG, 43) + + s_aug.add_covariance(full) + + out_aug = tmp_path / "aug.fits" + twopoint_convert.sacc_to_twopoint_fits( + s_aug, str(out_aug), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 + ) + + assert out_aug.read_bytes() == out_plain.read_bytes() + + def test_rho_tau_sidecars_required_together(tmp_path): """Supplying only one of the rho/tau sidecars is a loud error.""" inp = _inputs(seed=20) @@ -450,20 +542,35 @@ def test_covmat_blocks_exact_gather_encoded_cov(tmp_path): s, str(out), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) + # Expected indices are computed via a code path independent of the shared + # `sacc_io._indices` helper the converter uses: direct boolean filtering on + # each point's data_type/tracers/grid tag, so this test does not + # self-confirm against the converter's own selection logic. + def _mask_idx(dtype, tracers): + return np.array( + [ + i + for i, p in enumerate(s.data) + if p.data_type == dtype + and p.tracers == tracers + and p.tags.get("grid") == "reporting" + ] + ) + pair = (SOURCE, SOURCE) xi_idx = np.concatenate( - [s.indices(sacc_io.XI_PLUS, pair), s.indices(sacc_io.XI_MINUS, pair)] + [_mask_idx(sacc_io.XI_PLUS, pair), _mask_idx(sacc_io.XI_MINUS, pair)] ) tau_idx = np.concatenate( [ - s.indices(sacc_io.TAU_PLUS.format(k=0), (SOURCE, PSF)), - s.indices(sacc_io.TAU_PLUS.format(k=2), (SOURCE, PSF)), + _mask_idx(sacc_io.TAU_PLUS.format(k=0), (SOURCE, PSF)), + _mask_idx(sacc_io.TAU_PLUS.format(k=2), (SOURCE, PSF)), ] ) expected = twopoint_convert._block_diag( encoded[np.ix_(xi_idx, xi_idx)], encoded[np.ix_(tau_idx, tau_idx)] ) - cell_idx = s.indices(sacc_io.CL_EE, pair) + cell_idx = _mask_idx(sacc_io.CL_EE, pair) with fits.open(out) as hdul: np.testing.assert_array_equal(hdul["COVMAT"].data, expected) diff --git a/src/sp_validation/twopoint_convert.py b/src/sp_validation/twopoint_convert.py index fc01ac9d..5e3c3156 100644 --- a/src/sp_validation/twopoint_convert.py +++ b/src/sp_validation/twopoint_convert.py @@ -288,7 +288,7 @@ def _build_cell(s, bins): ell, cl_ee, _cl_bb, _cl_eb, _window = sacc_io.get_pseudo_cl(s, bins) cell_hdu = _twopoint_hdu("CELL_EE", cl_ee, ell) - cell_idx = s.indices(sacc_io.CL_EE, sacc_io._pair(bins)) + cell_idx = sacc_io._indices(s, sacc_io.CL_EE, sacc_io._pair(bins), grid="reporting") cov_cell = s.covariance.dense[np.ix_(cell_idx, cell_idx)] cov_cell_hdu = _cov_hdu( cov_cell, ["CELL_EE"], [0], extname="COVMAT_CELL", name_in_ctor=True @@ -305,8 +305,8 @@ def _build_covmat(s, bins, *, use_rho_tau): with zero ξ↔τ cross-blocks, exactly as ``covdat_to_fits`` builds them. """ pair = sacc_io._pair(bins) - idx_p = s.indices(sacc_io.XI_PLUS, pair) - idx_m = s.indices(sacc_io.XI_MINUS, pair) + idx_p = sacc_io._indices(s, sacc_io.XI_PLUS, pair, grid="reporting") + idx_m = sacc_io._indices(s, sacc_io.XI_MINUS, pair, grid="reporting") n_theta = len(idx_p) xi_idx = np.concatenate([idx_p, idx_m]) # type-major permutation xi_cov = s.covariance.dense[np.ix_(xi_idx, xi_idx)] @@ -322,8 +322,12 @@ def _build_covmat(s, bins, *, use_rho_tau): # SACC those two selections are not adjacent (τ_0− sits between them), so # gather both index sets and extract the joint sub-block, ξ↔τ zero. tau_pair = (sacc_io.source_name(0), sacc_io.PSF_TRACER) - idx_tau0 = s.indices(sacc_io.TAU_PLUS.format(k=0), tau_pair) - idx_tau2 = s.indices(sacc_io.TAU_PLUS.format(k=2), tau_pair) + idx_tau0 = sacc_io._indices( + s, sacc_io.TAU_PLUS.format(k=0), tau_pair, grid="reporting" + ) + idx_tau2 = sacc_io._indices( + s, sacc_io.TAU_PLUS.format(k=2), tau_pair, grid="reporting" + ) tau_idx = np.concatenate([idx_tau0, idx_tau2]) tau_cov = s.covariance.dense[np.ix_(tau_idx, tau_idx)] matrix = _block_diag(matrix, tau_cov) From 3e4fd7df4273c3348a0f55dcddbe211fcf69ac15 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 19 Jul 2026 14:43:56 +0200 Subject: [PATCH 37/47] refactor: merge twopoint_convert + one_covariance_io into sacc_interop One module whose job is converting between the SACC product and external analysis-tool file formats (CosmoSIS 2pt-FITS, OneCovariance). Public API unchanged; sacc_io untouched. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01766vawzi2XqrgoyHmeHEY9 --- src/sp_validation/one_covariance_io.py | 256 ----------------- .../{twopoint_convert.py => sacc_interop.py} | 268 +++++++++++++++++- ...py => test_sacc_interop_one_covariance.py} | 4 +- ...ldata.py => test_sacc_interop_realdata.py} | 6 +- ...nvert.py => test_sacc_interop_twopoint.py} | 32 +-- 5 files changed, 277 insertions(+), 289 deletions(-) delete mode 100644 src/sp_validation/one_covariance_io.py rename src/sp_validation/{twopoint_convert.py => sacc_interop.py} (58%) rename src/sp_validation/tests/{test_one_covariance_io.py => test_sacc_interop_one_covariance.py} (99%) rename src/sp_validation/tests/{test_twopoint_convert_realdata.py => test_sacc_interop_realdata.py} (98%) rename src/sp_validation/tests/{test_twopoint_convert.py => test_sacc_interop_twopoint.py} (95%) diff --git a/src/sp_validation/one_covariance_io.py b/src/sp_validation/one_covariance_io.py deleted file mode 100644 index 4c0f2ae4..00000000 --- a/src/sp_validation/one_covariance_io.py +++ /dev/null @@ -1,256 +0,0 @@ -"""ONE_COVARIANCE_IO. - -:Name: one_covariance_io.py - -:Description: File-format glue between the SACC data-product layout - (:mod:`sp_validation.sacc_io`) and OneCovariance - (https://github.com/rreischke/OneCovariance). Two directions: - - - **n(z) SACC -> OneCovariance input** (:func:`write_nz`): the - ``source_i`` NZ tracers of an analysis SACC are written as the - combined whitespace-delimited redshift file OneCovariance reads - (column 0 = z grid, then one ``n(z)`` column per tomographic - bin, no bin edges), and a matching ``[redshift]`` config stanza - is returned via :func:`nz_config_stanza`. - - - **OneCovariance output -> SACC covariance blocks** - (:func:`covariance_blocks`): the flat ``covariance_list_*.dat`` - table OneCovariance emits (one row per element pair) is reshaped - into dense square block(s) — reusing - :func:`sp_validation.statistics.cov_from_one_covariance` for the - per-block reshape — and paired with SACC selectors so a caller - can feed them straight to - :func:`sp_validation.sacc_io.assemble_covariance`. - - OneCovariance itself is *not* a dependency: this module only - touches its file formats, verified against the upstream - ``config.ini`` (``rreischke/OneCovariance`` @ main). - - n(z) file format (upstream ``config.ini`` comment, verbatim): - - ``redshift n_1(z) ... n_{N_source}(z)`` - - i.e. a plain whitespace-delimited text file, column 0 the shared - redshift grid and one column per tomographic bin — no ``z_low``/ - ``z_high`` edges (this is the OneCovariance convention, distinct - from the CosmoSIS NZDATA table which *does* carry edges). All - source bins must therefore share one z grid. - - ``[redshift]`` config keys (upstream canonical names): a single - combined file goes in ``zlens_directory`` + ``zlens_file``; - ``value_loc_in_lensbin`` (``mid``/``left``/``right``) says where - in each histogram bin the tabulated ``n(z)`` value sits — ``mid`` - for the bin-centred grids the SACC stores. NOTE: the UNIONS - OneCovariance template driven by - ``cosmo_val/pseudo_cl.py._modify_onecov_config`` writes the older - key names ``z_directory``/``zlens_file`` instead; pass - ``dir_key="z_directory"`` to match that template. -""" - -import os - -import numpy as np - -from . import sacc_io -from .statistics import cov_from_one_covariance - - -def nz_table(s, n_bins): - """Stack the SACC ``source_i`` NZ tracers into a OneCovariance n(z) table. - - Parameters - ---------- - s : sacc.Sacc - SACC holding ``source_0 … source_{n_bins-1}`` NZ tracers. - n_bins : int - Number of tomographic source bins to write. - - Returns - ------- - numpy.ndarray - Array of shape ``(n_z, n_bins + 1)``: column 0 the shared redshift - grid, columns ``1 … n_bins`` the per-bin ``n(z)``. This is the - OneCovariance combined-file layout (``redshift n_1(z) … n_N(z)``). - - Raises - ------ - ValueError - If any source bin is missing, or if the bins do not share one z grid - (OneCovariance's combined file has a single redshift column, so the - grids must agree bin-for-bin). - """ - z0, nz0 = sacc_io.get_nz(s, 0) - z0 = np.asarray(z0, dtype=float) - columns = [z0] - for i in range(n_bins): - if sacc_io.source_name(i) not in s.tracers: - raise ValueError( - f"SACC has no NZ tracer {sacc_io.source_name(i)!r}; cannot write " - f"a {n_bins}-bin OneCovariance n(z) file" - ) - z_i, nz_i = sacc_io.get_nz(s, i) - if not np.array_equal(np.asarray(z_i, dtype=float), z0): - raise ValueError( - f"source bin {i} n(z) grid differs from source bin 0; the " - "OneCovariance combined n(z) file has one shared redshift column" - ) - columns.append(np.asarray(nz_i, dtype=float)) - return np.column_stack(columns) - - -def write_nz(s, path, n_bins, *, dir_key="zlens_directory", header=True): - """Write the OneCovariance combined n(z) input file from a SACC. - - OneCovariance reads the source redshift distribution as a plain - whitespace-delimited text file whose column 0 is the shared redshift grid - and whose remaining columns are the per-bin ``n(z)`` (``redshift n_1(z) - … n_N(z)``) — no ``z_low``/``z_high`` edges. This writes that file from the - SACC ``source_i`` NZ tracers and returns the ``[redshift]`` config stanza - that points OneCovariance at it. - - Parameters - ---------- - s : sacc.Sacc - Analysis SACC with the ``source_i`` NZ tracers. - path : str or pathlib.Path - Output text-file path (overwritten). Its directory + basename become - the ``[redshift]`` directory/file config values. - n_bins : int - Number of tomographic source bins to write. - dir_key : str, optional - Config key for the redshift directory. Default ``"zlens_directory"`` - (upstream canonical). Pass ``"z_directory"`` for the UNIONS template - driven by ``pseudo_cl.py._modify_onecov_config``. - header : bool, optional - If ``True`` (default) prepend a ``# redshift n_1(z) …`` comment header - naming the columns; OneCovariance's ``genfromtxt``-style reader ignores - it. Set ``False`` for a bare numeric file. - - Returns - ------- - dict - The ``[redshift]`` config stanza (see :func:`nz_config_stanza`), naming - the file just written. - """ - table = nz_table(s, n_bins) - head = "" - if header: - cols = " ".join(f"n_{i + 1}(z)" for i in range(n_bins)) - head = f"redshift {cols}" - np.savetxt(str(path), table, header=head) - return nz_config_stanza( - os.path.dirname(os.path.abspath(str(path))), - os.path.basename(str(path)), - dir_key=dir_key, - ) - - -def nz_config_stanza( - directory, filename, *, dir_key="zlens_directory", value_loc="mid" -): - """Build the OneCovariance ``[redshift]`` config stanza for an n(z) file. - - Parameters - ---------- - directory : str - Directory holding the n(z) file (OneCovariance ``*_directory`` value). - filename : str - n(z) file basename (OneCovariance ``zlens_file`` value). - dir_key : str, optional - Directory config key — ``"zlens_directory"`` (upstream) or - ``"z_directory"`` (UNIONS template). Default ``"zlens_directory"``. - value_loc : str, optional - ``value_loc_in_lensbin`` — where in each histogram bin the tabulated - ``n(z)`` value sits (``mid``/``left``/``right``). Default ``"mid"``, - matching the bin-centred grids the SACC stores. - - Returns - ------- - dict - The ``[redshift]`` key/value pairs: ``{dir_key: directory, "zlens_file": - filename, "value_loc_in_lensbin": value_loc}``. Assign these under - ``config["redshift"]`` of a OneCovariance ``configparser`` config. - """ - if value_loc not in ("mid", "left", "right"): - raise ValueError( - f"value_loc_in_lensbin must be 'mid', 'left' or 'right'; got {value_loc!r}" - ) - return { - dir_key: directory, - "zlens_file": filename, - "value_loc_in_lensbin": value_loc, - } - - -def read_nz(path): - """Read a OneCovariance combined n(z) file back to ``(z, nz_columns)``. - - Inverse of :func:`write_nz` (the numeric round-trip; the config stanza is - not stored in the file). Comment/header lines are skipped. - - Parameters - ---------- - path : str or pathlib.Path - n(z) text file (column 0 = z, columns 1… = per-bin n(z)). - - Returns - ------- - tuple - ``(z, nz)`` where ``z`` is the shared redshift grid (shape ``(n_z,)``) - and ``nz`` is the per-bin distributions (shape ``(n_z, n_bins)``). - """ - table = np.atleast_2d(np.genfromtxt(str(path))) - return table[:, 0], table[:, 1:] - - -def covariance_blocks(cov_list, selectors, *, gaussian=True): - """Reshape a OneCovariance ``covariance_list`` table into SACC cov blocks. - - OneCovariance emits a flat ``covariance_list_*.dat`` table with one row per - ``(i, j)`` element pair (row-major, ``k = i·n + j``); the covariance value - lives in column 10 (Gaussian) or column 9 (Gaussian+non-Gaussian). This - reshapes the flat table into dense square block(s) — reusing - :func:`sp_validation.statistics.cov_from_one_covariance` for the per-block - reshape — and pairs each with its SACC selector, ready for - :func:`sp_validation.sacc_io.assemble_covariance`. - - Single-statistic case: pass the whole table and one selector; you get one - ``(selector, dense)`` block. Multi-statistic case (tomography-ready): pass a - sequence of ``(selector, sub_table)`` pairs — each ``sub_table`` a - contiguous slice of the flat output for one statistic / bin-pair — and each - is reshaped and re-paired with its selector in order. The API is thus shaped - to extend to multi-probe blocking without over-fitting the single-bin case. - - Parameters - ---------- - cov_list : numpy.ndarray or sequence - Either the flat OneCovariance table (2-D array, one row per pair) for a - single block, or — for the multi-block form — a sequence of - ``(selector, sub_table)`` pairs. In the multi-block form ``selectors`` - must be ``None`` (the selectors travel with the sub-tables). - selectors : selector or None - For the single-block form, the SACC selector for the whole table (a - ``(data_type, tracers[, tags])`` tuple or an index array, as - :func:`sacc_io.assemble_covariance` accepts). Must be ``None`` for the - multi-block form. - gaussian : bool, optional - Select the Gaussian-only column (``True``, default) or the - Gaussian+non-Gaussian column (``False``); passed straight through to - ``cov_from_one_covariance``. - - Returns - ------- - list - Ordered ``(selector, dense_cov)`` pairs, directly consumable by - ``sacc_io.assemble_covariance(s, blocks)``. - """ - if selectors is None: - # Multi-block form: cov_list is a sequence of (selector, sub_table). - return [ - (selector, cov_from_one_covariance(np.asarray(sub), gaussian=gaussian)) - for selector, sub in cov_list - ] - # Single-block form: one flat table, one selector. - return [ - (selectors, cov_from_one_covariance(np.asarray(cov_list), gaussian=gaussian)) - ] diff --git a/src/sp_validation/twopoint_convert.py b/src/sp_validation/sacc_interop.py similarity index 58% rename from src/sp_validation/twopoint_convert.py rename to src/sp_validation/sacc_interop.py index 5e3c3156..977470a7 100644 --- a/src/sp_validation/twopoint_convert.py +++ b/src/sp_validation/sacc_interop.py @@ -1,16 +1,23 @@ -"""TWOPOINT_CONVERT. +"""SACC_INTEROP. -:Name: twopoint_convert.py +:Name: sacc_interop.py -:Description: Convert an analysis SACC file into the "2pt FITS" that CosmoSIS's - ``2pt_like`` (and Sacha Guerrini's ρ/τ ``2pt_like_xi_sys`` fork) - reads. The output reproduces today's hand-assembled product from +:Description: Converters between the SACC data product + (:mod:`sp_validation.sacc_io`) and external analysis-tool file + formats — the CosmoSIS "2pt FITS" and OneCovariance's redshift / + covariance files. Two independent, self-contained sections; the + only shared surface is the SACC on one side. + + **CosmoSIS 2pt-FITS** (:func:`sacc_to_twopoint_fits`). Convert an + analysis SACC into the "2pt FITS" that CosmoSIS's ``2pt_like`` (and + Sacha Guerrini's ρ/τ ``2pt_like_xi_sys`` fork) reads. The output + reproduces today's hand-assembled product from ``cosmo_inference/scripts/cosmosis_fitting.py`` HDU-for-HDU and - byte-for-byte: an NZDATA table, XI_PLUS / XI_MINUS 2pt tables, - optional CELL_EE / CELL_BB pseudo-Cℓ tables, the blocked COVMAT - (with ``STRT_i`` block-offset headers) and separate COVMAT_CELL, - and — when the SACC carries them — the TAU_{0,2}_PLUS 2pt tables - and the RHO_STATS table. + byte-for-byte (verified against that writer): an NZDATA table, + XI_PLUS / XI_MINUS 2pt tables, optional CELL_EE / CELL_BB pseudo-Cℓ + tables, the blocked COVMAT (with ``STRT_i`` block-offset headers) + and separate COVMAT_CELL, and — when the SACC carries them — the + TAU_{0,2}_PLUS 2pt tables and the RHO_STATS table. The converter is the *inverse* of the SACC writers in :mod:`sp_validation.sacc_io`: it reads statistics back through @@ -22,8 +29,9 @@ Scope note (single-bin today, tomography-ready): the assembly this mirrors is single-tomographic-bin — BIN1/BIN2 are all 1, one NZ - ``BIN1`` column. The converter reads bin ``(0, 0)`` accordingly. - A tomographic 2pt-FITS layout (multiple bin pairs, per-pair + ``BIN1`` column. The converter reads bin ``(0, 0)`` accordingly and + fails fast on a multi-bin SACC (single-bin v1 contract). A + tomographic 2pt-FITS layout (multiple bin pairs, per-pair BIN1/BIN2, one NZ column per bin) is a later extension; it is not what today's CosmoSIS pipeline consumes, so it is out of scope for the byte-compatible converter. @@ -37,12 +45,42 @@ file today's assembly copies verbatim); it never fabricates variances. ξ±, Cℓ, n(z) and the covariance — the data vector CosmoSIS fits — are fully reconstructed from SACC alone. + + **OneCovariance** (:func:`write_nz`, :func:`nz_config_stanza`, + :func:`read_nz`, :func:`covariance_blocks`). File-format glue + between the SACC layout and OneCovariance + (https://github.com/rreischke/OneCovariance). Two directions: the + ``source_i`` NZ tracers are written as the combined + whitespace-delimited redshift file OneCovariance reads (column 0 = + z grid, then one ``n(z)`` column per bin, no bin edges) with a + matching ``[redshift]`` config stanza; and the flat + ``covariance_list_*.dat`` table OneCovariance emits is reshaped + into dense SACC covariance block(s) paired with their selectors. + OneCovariance itself is *not* a dependency — this module only + touches its file formats, verified against the upstream + ``config.ini`` (``rreischke/OneCovariance`` @ main). + + n(z) file format (upstream ``config.ini`` comment, verbatim): + + ``redshift n_1(z) ... n_{N_source}(z)`` + + i.e. a plain whitespace-delimited text file, column 0 the shared + redshift grid and one column per tomographic bin — no ``z_low``/ + ``z_high`` edges (distinct from the CosmoSIS NZDATA table above, + which *does* carry edges). All source bins must share one z grid. """ +import os + import numpy as np from astropy.io import fits from . import sacc_io +from .statistics import cov_from_one_covariance + +# ============================================================================= +# CosmoSIS 2pt-FITS +# ============================================================================= # The QUANT1/QUANT2 header pair CosmoSIS stamps on each 2pt table, keyed by the # extension name — copied from cosmosis_fitting.py so the headers match card @@ -370,3 +408,209 @@ def _build_rho_tau(rho_stats_hdu, tau_stats_hdu, theta, use_rho_tau): rho_hdu.data = rho_hdu.data.copy() rho_hdu.data["theta"] = theta return (tau0_hdu, tau2_hdu), rho_hdu + + +# ============================================================================= +# OneCovariance +# ============================================================================= + + +def nz_table(s, n_bins): + """Stack the SACC ``source_i`` NZ tracers into a OneCovariance n(z) table. + + Parameters + ---------- + s : sacc.Sacc + SACC holding ``source_0 … source_{n_bins-1}`` NZ tracers. + n_bins : int + Number of tomographic source bins to write. + + Returns + ------- + numpy.ndarray + Array of shape ``(n_z, n_bins + 1)``: column 0 the shared redshift + grid, columns ``1 … n_bins`` the per-bin ``n(z)``. This is the + OneCovariance combined-file layout (``redshift n_1(z) … n_N(z)``). + + Raises + ------ + ValueError + If any source bin is missing, or if the bins do not share one z grid + (OneCovariance's combined file has a single redshift column, so the + grids must agree bin-for-bin). + """ + z0, nz0 = sacc_io.get_nz(s, 0) + z0 = np.asarray(z0, dtype=float) + columns = [z0] + for i in range(n_bins): + if sacc_io.source_name(i) not in s.tracers: + raise ValueError( + f"SACC has no NZ tracer {sacc_io.source_name(i)!r}; cannot write " + f"a {n_bins}-bin OneCovariance n(z) file" + ) + z_i, nz_i = sacc_io.get_nz(s, i) + if not np.array_equal(np.asarray(z_i, dtype=float), z0): + raise ValueError( + f"source bin {i} n(z) grid differs from source bin 0; the " + "OneCovariance combined n(z) file has one shared redshift column" + ) + columns.append(np.asarray(nz_i, dtype=float)) + return np.column_stack(columns) + + +def write_nz(s, path, n_bins, *, dir_key="zlens_directory", header=True): + """Write the OneCovariance combined n(z) input file from a SACC. + + OneCovariance reads the source redshift distribution as a plain + whitespace-delimited text file whose column 0 is the shared redshift grid + and whose remaining columns are the per-bin ``n(z)`` (``redshift n_1(z) + … n_N(z)``) — no ``z_low``/``z_high`` edges. This writes that file from the + SACC ``source_i`` NZ tracers and returns the ``[redshift]`` config stanza + that points OneCovariance at it. + + Parameters + ---------- + s : sacc.Sacc + Analysis SACC with the ``source_i`` NZ tracers. + path : str or pathlib.Path + Output text-file path (overwritten). Its directory + basename become + the ``[redshift]`` directory/file config values. + n_bins : int + Number of tomographic source bins to write. + dir_key : str, optional + Config key for the redshift directory. Default ``"zlens_directory"`` + (upstream canonical). Pass ``"z_directory"`` for the UNIONS template + driven by ``pseudo_cl.py._modify_onecov_config``. + header : bool, optional + If ``True`` (default) prepend a ``# redshift n_1(z) …`` comment header + naming the columns; OneCovariance's ``genfromtxt``-style reader ignores + it. Set ``False`` for a bare numeric file. + + Returns + ------- + dict + The ``[redshift]`` config stanza (see :func:`nz_config_stanza`), naming + the file just written. + """ + table = nz_table(s, n_bins) + head = "" + if header: + cols = " ".join(f"n_{i + 1}(z)" for i in range(n_bins)) + head = f"redshift {cols}" + np.savetxt(str(path), table, header=head) + return nz_config_stanza( + os.path.dirname(os.path.abspath(str(path))), + os.path.basename(str(path)), + dir_key=dir_key, + ) + + +def nz_config_stanza( + directory, filename, *, dir_key="zlens_directory", value_loc="mid" +): + """Build the OneCovariance ``[redshift]`` config stanza for an n(z) file. + + Parameters + ---------- + directory : str + Directory holding the n(z) file (OneCovariance ``*_directory`` value). + filename : str + n(z) file basename (OneCovariance ``zlens_file`` value). + dir_key : str, optional + Directory config key — ``"zlens_directory"`` (upstream) or + ``"z_directory"`` (UNIONS template). Default ``"zlens_directory"``. + value_loc : str, optional + ``value_loc_in_lensbin`` — where in each histogram bin the tabulated + ``n(z)`` value sits (``mid``/``left``/``right``). Default ``"mid"``, + matching the bin-centred grids the SACC stores. + + Returns + ------- + dict + The ``[redshift]`` key/value pairs: ``{dir_key: directory, "zlens_file": + filename, "value_loc_in_lensbin": value_loc}``. Assign these under + ``config["redshift"]`` of a OneCovariance ``configparser`` config. + """ + if value_loc not in ("mid", "left", "right"): + raise ValueError( + f"value_loc_in_lensbin must be 'mid', 'left' or 'right'; got {value_loc!r}" + ) + return { + dir_key: directory, + "zlens_file": filename, + "value_loc_in_lensbin": value_loc, + } + + +def read_nz(path): + """Read a OneCovariance combined n(z) file back to ``(z, nz_columns)``. + + Inverse of :func:`write_nz` (the numeric round-trip; the config stanza is + not stored in the file). Comment/header lines are skipped. + + Parameters + ---------- + path : str or pathlib.Path + n(z) text file (column 0 = z, columns 1… = per-bin n(z)). + + Returns + ------- + tuple + ``(z, nz)`` where ``z`` is the shared redshift grid (shape ``(n_z,)``) + and ``nz`` is the per-bin distributions (shape ``(n_z, n_bins)``). + """ + table = np.atleast_2d(np.genfromtxt(str(path))) + return table[:, 0], table[:, 1:] + + +def covariance_blocks(cov_list, selectors, *, gaussian=True): + """Reshape a OneCovariance ``covariance_list`` table into SACC cov blocks. + + OneCovariance emits a flat ``covariance_list_*.dat`` table with one row per + ``(i, j)`` element pair (row-major, ``k = i·n + j``); the covariance value + lives in column 10 (Gaussian) or column 9 (Gaussian+non-Gaussian). This + reshapes the flat table into dense square block(s) — reusing + :func:`sp_validation.statistics.cov_from_one_covariance` for the per-block + reshape — and pairs each with its SACC selector, ready for + :func:`sp_validation.sacc_io.assemble_covariance`. + + Single-statistic case: pass the whole table and one selector; you get one + ``(selector, dense)`` block. Multi-statistic case (tomography-ready): pass a + sequence of ``(selector, sub_table)`` pairs — each ``sub_table`` a + contiguous slice of the flat output for one statistic / bin-pair — and each + is reshaped and re-paired with its selector in order. The API is thus shaped + to extend to multi-probe blocking without over-fitting the single-bin case. + + Parameters + ---------- + cov_list : numpy.ndarray or sequence + Either the flat OneCovariance table (2-D array, one row per pair) for a + single block, or — for the multi-block form — a sequence of + ``(selector, sub_table)`` pairs. In the multi-block form ``selectors`` + must be ``None`` (the selectors travel with the sub-tables). + selectors : selector or None + For the single-block form, the SACC selector for the whole table (a + ``(data_type, tracers[, tags])`` tuple or an index array, as + :func:`sacc_io.assemble_covariance` accepts). Must be ``None`` for the + multi-block form. + gaussian : bool, optional + Select the Gaussian-only column (``True``, default) or the + Gaussian+non-Gaussian column (``False``); passed straight through to + ``cov_from_one_covariance``. + + Returns + ------- + list + Ordered ``(selector, dense_cov)`` pairs, directly consumable by + ``sacc_io.assemble_covariance(s, blocks)``. + """ + if selectors is None: + # Multi-block form: cov_list is a sequence of (selector, sub_table). + return [ + (selector, cov_from_one_covariance(np.asarray(sub), gaussian=gaussian)) + for selector, sub in cov_list + ] + # Single-block form: one flat table, one selector. + return [ + (selectors, cov_from_one_covariance(np.asarray(cov_list), gaussian=gaussian)) + ] diff --git a/src/sp_validation/tests/test_one_covariance_io.py b/src/sp_validation/tests/test_sacc_interop_one_covariance.py similarity index 99% rename from src/sp_validation/tests/test_one_covariance_io.py rename to src/sp_validation/tests/test_sacc_interop_one_covariance.py index 9bc7da56..f0df56e9 100644 --- a/src/sp_validation/tests/test_one_covariance_io.py +++ b/src/sp_validation/tests/test_sacc_interop_one_covariance.py @@ -1,4 +1,4 @@ -"""Tests for :mod:`sp_validation.one_covariance_io`. +"""Tests for :mod:`sp_validation.sacc_interop`. All synthetic, all fast: the OneCovariance fixtures are built in memory shaped exactly like its real file I/O — a flat ``covariance_list`` table with @@ -22,7 +22,7 @@ import numpy.testing as npt import pytest -from sp_validation import one_covariance_io as ocio +from sp_validation import sacc_interop as ocio from sp_validation import sacc_io as sio diff --git a/src/sp_validation/tests/test_twopoint_convert_realdata.py b/src/sp_validation/tests/test_sacc_interop_realdata.py similarity index 98% rename from src/sp_validation/tests/test_twopoint_convert_realdata.py rename to src/sp_validation/tests/test_sacc_interop_realdata.py index 1dcf920f..df7b1366 100644 --- a/src/sp_validation/tests/test_twopoint_convert_realdata.py +++ b/src/sp_validation/tests/test_sacc_interop_realdata.py @@ -28,7 +28,7 @@ import pytest from astropy.io import fits -from sp_validation import sacc_io, twopoint_convert +from sp_validation import sacc_interop, sacc_io _DATA = Path("/automnt/n17data/cdaley/unions/code/sp_validation/cosmo_inference/data") _REAL_FILES = { @@ -221,7 +221,7 @@ def test_realdata_roundtrip_byte_equal(label, tmp_path): reference = _current_script_reference(cf, hdul, tmp_path) converted = tmp_path / "converted.fits" - twopoint_convert.sacc_to_twopoint_fits( + sacc_interop.sacc_to_twopoint_fits( s, str(converted), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) @@ -250,7 +250,7 @@ def test_realdata_ondisk_drift_is_only_cell_bb(label, tmp_path): s, rho_hdu, tau_hdu = _sacc_from_2pt_fits(hdul) ondisk_names = [h.name for h in hdul] converted = tmp_path / "converted.fits" - twopoint_convert.sacc_to_twopoint_fits( + sacc_interop.sacc_to_twopoint_fits( s, str(converted), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) with fits.open(converted) as conv: diff --git a/src/sp_validation/tests/test_twopoint_convert.py b/src/sp_validation/tests/test_sacc_interop_twopoint.py similarity index 95% rename from src/sp_validation/tests/test_twopoint_convert.py rename to src/sp_validation/tests/test_sacc_interop_twopoint.py index 59e0256d..cd1c4a31 100644 --- a/src/sp_validation/tests/test_twopoint_convert.py +++ b/src/sp_validation/tests/test_sacc_interop_twopoint.py @@ -1,6 +1,6 @@ """Byte-compare tests for the SACC -> 2pt-FITS converter. -The converter (:mod:`sp_validation.twopoint_convert`) must reproduce the CosmoSIS +The converter (:mod:`sp_validation.sacc_interop`) must reproduce the CosmoSIS 2pt-FITS that ``cosmo_inference/scripts/cosmosis_fitting.py`` assembles today, so the inference chain (``2pt_like`` and Sacha Guerrini's rho/tau ``2pt_like_xi_sys`` fork) runs untouched behind it. The strongest possible check @@ -36,7 +36,7 @@ import pytest from astropy.io import fits -from sp_validation import sacc_io, twopoint_convert +from sp_validation import sacc_interop, sacc_io _SCRIPT = ( Path(__file__).resolve().parents[3] @@ -290,7 +290,7 @@ def test_plain_xi_byte_equal(tmp_path): reference = _reference_fits(tmp_path, inp) s = _sacc(inp) out = tmp_path / "converted.fits" - twopoint_convert.sacc_to_twopoint_fits(s, str(out), n_bins=1) + sacc_interop.sacc_to_twopoint_fits(s, str(out), n_bins=1) assert out.read_bytes() == reference.read_bytes() @@ -300,7 +300,7 @@ def test_xi_cl_byte_equal(tmp_path): reference = _reference_fits(tmp_path, inp, cl=True) s = _sacc(inp, cl=True) out = tmp_path / "converted.fits" - twopoint_convert.sacc_to_twopoint_fits(s, str(out), n_bins=1) + sacc_interop.sacc_to_twopoint_fits(s, str(out), n_bins=1) assert out.read_bytes() == reference.read_bytes() @@ -311,7 +311,7 @@ def test_xi_rho_tau_byte_equal(tmp_path): s = _sacc(inp, rho_tau=True) rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) out = tmp_path / "converted.fits" - twopoint_convert.sacc_to_twopoint_fits( + sacc_interop.sacc_to_twopoint_fits( s, str(out), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) assert out.read_bytes() == reference.read_bytes() @@ -334,7 +334,7 @@ def test_tau_covariance_keeps_tau0_tau2_cross(tmp_path): s = _sacc(inp, rho_tau=True) rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) out = tmp_path / "converted.fits" - twopoint_convert.sacc_to_twopoint_fits( + sacc_interop.sacc_to_twopoint_fits( s, str(out), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) with fits.open(out) as hdul: @@ -351,7 +351,7 @@ def test_perturbed_xi_changes_output(tmp_path): inp = _inputs(seed=0) s = _sacc(inp) out = tmp_path / "base.fits" - twopoint_convert.sacc_to_twopoint_fits(s, str(out), n_bins=1) + sacc_interop.sacc_to_twopoint_fits(s, str(out), n_bins=1) with fits.open(out) as hdul: base_xip = hdul["XI_PLUS"].data["VALUE"].copy() @@ -359,7 +359,7 @@ def test_perturbed_xi_changes_output(tmp_path): inp2["xip"] = inp2["xip"] + 1.0 s2 = _sacc(inp2) out2 = tmp_path / "perturbed.fits" - twopoint_convert.sacc_to_twopoint_fits(s2, str(out2), n_bins=1) + sacc_interop.sacc_to_twopoint_fits(s2, str(out2), n_bins=1) with fits.open(out2) as hdul: new_xip = hdul["XI_PLUS"].data["VALUE"] @@ -384,7 +384,7 @@ def test_integration_grid_points_ignored(tmp_path): s_plain = _sacc(inp, cl=True, rho_tau=True) rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) out_plain = tmp_path / "plain.fits" - twopoint_convert.sacc_to_twopoint_fits( + sacc_interop.sacc_to_twopoint_fits( s_plain, str(out_plain), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) @@ -452,7 +452,7 @@ def test_integration_grid_points_ignored(tmp_path): s_aug.add_covariance(full) out_aug = tmp_path / "aug.fits" - twopoint_convert.sacc_to_twopoint_fits( + sacc_interop.sacc_to_twopoint_fits( s_aug, str(out_aug), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) @@ -465,7 +465,7 @@ def test_rho_tau_sidecars_required_together(tmp_path): s = _sacc(inp, rho_tau=True) rho_hdu, _tau_hdu = _sidecar_hdus(tmp_path, inp) with pytest.raises(ValueError, match="together"): - twopoint_convert.sacc_to_twopoint_fits( + sacc_interop.sacc_to_twopoint_fits( s, str(tmp_path / "x.fits"), rho_stats_hdu=rho_hdu, n_bins=1 ) @@ -490,9 +490,9 @@ def test_tomographic_sacc_raises(tmp_path): s.add_covariance(np.eye(len(s.mean))) with pytest.raises(ValueError, match="single-bin only"): - twopoint_convert.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits"), n_bins=2) + sacc_interop.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits"), n_bins=2) with pytest.raises(ValueError, match="single-bin only"): - twopoint_convert.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits"), n_bins=1) + sacc_interop.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits"), n_bins=1) assert not (tmp_path / "x.fits").exists() @@ -513,7 +513,7 @@ def test_sacc_without_xi_raises(tmp_path): s.add_covariance(np.eye(len(s.mean))) with pytest.raises(ValueError, match="nothing to convert"): - twopoint_convert.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits")) + sacc_interop.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits")) assert not (tmp_path / "x.fits").exists() @@ -538,7 +538,7 @@ def test_covmat_blocks_exact_gather_encoded_cov(tmp_path): rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) out = tmp_path / "encoded.fits" - twopoint_convert.sacc_to_twopoint_fits( + sacc_interop.sacc_to_twopoint_fits( s, str(out), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) @@ -567,7 +567,7 @@ def _mask_idx(dtype, tracers): _mask_idx(sacc_io.TAU_PLUS.format(k=2), (SOURCE, PSF)), ] ) - expected = twopoint_convert._block_diag( + expected = sacc_interop._block_diag( encoded[np.ix_(xi_idx, xi_idx)], encoded[np.ix_(tau_idx, tau_idx)] ) cell_idx = _mask_idx(sacc_io.CL_EE, pair) From 33f0321ea81fe33415488ec33ee6e08b2cefcacf Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 20 Jul 2026 22:07:55 +0200 Subject: [PATCH 38/47] =?UTF-8?q?refactor:=20fold=20sacc=5Finterop=20into?= =?UTF-8?q?=20sacc=5Fio=20=E2=80=94=20one=20SACC=20module?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cail's ruling: a single sacc_io.py carries the format contract and the external-tool converters (CosmoSIS 2pt-FITS, OneCovariance). Public API unchanged; the sacc_io import name every downstream branch uses is untouched. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01766vawzi2XqrgoyHmeHEY9 --- src/sp_validation/sacc_interop.py | 616 ------------------ src/sp_validation/sacc_io.py | 552 ++++++++++++++++ ...ance.py => test_sacc_io_one_covariance.py} | 29 +- ...p_realdata.py => test_sacc_io_realdata.py} | 6 +- ...p_twopoint.py => test_sacc_io_twopoint.py} | 32 +- 5 files changed, 585 insertions(+), 650 deletions(-) delete mode 100644 src/sp_validation/sacc_interop.py rename src/sp_validation/tests/{test_sacc_interop_one_covariance.py => test_sacc_io_one_covariance.py} (92%) rename src/sp_validation/tests/{test_sacc_interop_realdata.py => test_sacc_io_realdata.py} (98%) rename src/sp_validation/tests/{test_sacc_interop_twopoint.py => test_sacc_io_twopoint.py} (96%) diff --git a/src/sp_validation/sacc_interop.py b/src/sp_validation/sacc_interop.py deleted file mode 100644 index 977470a7..00000000 --- a/src/sp_validation/sacc_interop.py +++ /dev/null @@ -1,616 +0,0 @@ -"""SACC_INTEROP. - -:Name: sacc_interop.py - -:Description: Converters between the SACC data product - (:mod:`sp_validation.sacc_io`) and external analysis-tool file - formats — the CosmoSIS "2pt FITS" and OneCovariance's redshift / - covariance files. Two independent, self-contained sections; the - only shared surface is the SACC on one side. - - **CosmoSIS 2pt-FITS** (:func:`sacc_to_twopoint_fits`). Convert an - analysis SACC into the "2pt FITS" that CosmoSIS's ``2pt_like`` (and - Sacha Guerrini's ρ/τ ``2pt_like_xi_sys`` fork) reads. The output - reproduces today's hand-assembled product from - ``cosmo_inference/scripts/cosmosis_fitting.py`` HDU-for-HDU and - byte-for-byte (verified against that writer): an NZDATA table, - XI_PLUS / XI_MINUS 2pt tables, optional CELL_EE / CELL_BB pseudo-Cℓ - tables, the blocked COVMAT (with ``STRT_i`` block-offset headers) - and separate COVMAT_CELL, and — when the SACC carries them — the - TAU_{0,2}_PLUS 2pt tables and the RHO_STATS table. - - The converter is the *inverse* of the SACC writers in - :mod:`sp_validation.sacc_io`: it reads statistics back through - those readers and lays them into the DES ``twopoint`` FITS - convention. SACC's canonical order is pair-major (per pair - ``[ξ+; ξ−]``); the 2pt-FITS layout is type-major (all ξ+, then all - ξ−), so the data-vector and its covariance are permuted here via - ``s.indices`` rather than assuming any global order. - - Scope note (single-bin today, tomography-ready): the assembly this - mirrors is single-tomographic-bin — BIN1/BIN2 are all 1, one NZ - ``BIN1`` column. The converter reads bin ``(0, 0)`` accordingly and - fails fast on a multi-bin SACC (single-bin v1 contract). A - tomographic 2pt-FITS layout (multiple bin pairs, per-pair - BIN1/BIN2, one NZ column per bin) is a later extension; it is not - what today's CosmoSIS pipeline consumes, so it is out of scope for - the byte-compatible converter. - - Rho/tau caveat: the SACC layout stores ρ±/τ± *values* only, while - the 2pt-FITS RHO_STATS table also carries the per-mode *variances* - (``varrho_*``) that Sacha's fork's covariance path reads. Those - variances are not recoverable from the analysis SACC. The - converter therefore writes RHO_STATS / TAU HDUs only when a - ``rho_stats``/``tau_stats`` sidecar FITS is supplied (the same - file today's assembly copies verbatim); it never fabricates - variances. ξ±, Cℓ, n(z) and the covariance — the data vector - CosmoSIS fits — are fully reconstructed from SACC alone. - - **OneCovariance** (:func:`write_nz`, :func:`nz_config_stanza`, - :func:`read_nz`, :func:`covariance_blocks`). File-format glue - between the SACC layout and OneCovariance - (https://github.com/rreischke/OneCovariance). Two directions: the - ``source_i`` NZ tracers are written as the combined - whitespace-delimited redshift file OneCovariance reads (column 0 = - z grid, then one ``n(z)`` column per bin, no bin edges) with a - matching ``[redshift]`` config stanza; and the flat - ``covariance_list_*.dat`` table OneCovariance emits is reshaped - into dense SACC covariance block(s) paired with their selectors. - OneCovariance itself is *not* a dependency — this module only - touches its file formats, verified against the upstream - ``config.ini`` (``rreischke/OneCovariance`` @ main). - - n(z) file format (upstream ``config.ini`` comment, verbatim): - - ``redshift n_1(z) ... n_{N_source}(z)`` - - i.e. a plain whitespace-delimited text file, column 0 the shared - redshift grid and one column per tomographic bin — no ``z_low``/ - ``z_high`` edges (distinct from the CosmoSIS NZDATA table above, - which *does* carry edges). All source bins must share one z grid. -""" - -import os - -import numpy as np -from astropy.io import fits - -from . import sacc_io -from .statistics import cov_from_one_covariance - -# ============================================================================= -# CosmoSIS 2pt-FITS -# ============================================================================= - -# The QUANT1/QUANT2 header pair CosmoSIS stamps on each 2pt table, keyed by the -# extension name — copied from cosmosis_fitting.py so the headers match card -# for card. -_QUANT = { - "XI_PLUS": ("G+R", "G+R"), - "XI_MINUS": ("G-R", "G-R"), - "CELL_EE": ("GEF", "GEF"), - "CELL_BB": ("GBF", "GBF"), - "TAU_0_PLUS": ("G+R", "P+R"), - "TAU_2_PLUS": ("G+R", "SR+R"), -} - - -def _twopoint_hdu(name, values, ang, *, ang_unit=None): - """Build one 2pt BinTableHDU (BIN1/BIN2/ANGBIN/VALUE/ANG). - - Reproduces ``cosmosis_fitting.py._create_2pt_hdu`` /``cl_to_fits`` exactly: - same column order and formats, the ``2PTDATA`` marker, the QUANT pair for - ``name``, and NZ_SOURCE kernels. ``ang_unit`` stamps ``TUNIT`` on the ANG - column ("arcmin" for real-space ξ/τ; unset for Cℓ, whose ANG is ℓ). - """ - nbins = len(values) - angbin = np.arange(1, nbins + 1) - columns = [ - fits.Column(name="BIN1", format="K", array=np.ones(nbins)), - fits.Column(name="BIN2", format="K", array=np.ones(nbins)), - fits.Column(name="ANGBIN", format="K", array=angbin), - fits.Column(name="VALUE", format="D", array=values), - fits.Column(name="ANG", format="D", unit=ang_unit, array=ang), - ] - hdu = fits.BinTableHDU.from_columns(fits.ColDefs(columns), name=name) - quant1, quant2 = _QUANT[name] - for key, value in { - "2PTDATA": "T", - "QUANT1": quant1, - "QUANT2": quant2, - "KERNEL_1": "NZ_SOURCE", - "KERNEL_2": "NZ_SOURCE", - "WINDOWS": "SAMPLE", - }.items(): - hdu.header[key] = value - return hdu - - -def _nz_hdu(s, n_bins): - """Build the NZDATA HDU from the SACC ``source_i`` NZ tracers. - - Reproduces ``cosmosis_fitting.py.nz_to_fits``: Z_MID from the tracer ``z`` - grid (assumed uniform), Z_LOW/Z_HIGH as ± half a step, one ``BIN{i+1}`` - column per source bin, and the NZDATA/NBIN/NZ header cards. All source bins - are required to share the ``z`` grid — the single ``Z_MID`` axis of the - DES NZDATA table. - """ - z_mid, nz0 = sacc_io.get_nz(s, 0) - z_mid = np.asarray(z_mid, dtype=float) - step = z_mid[1] - z_mid[0] - z_low = z_mid - step / 2 - z_high = z_mid + step / 2 - - columns = [ - fits.Column(name="Z_LOW", format="D", array=z_low), - fits.Column(name="Z_MID", format="D", array=z_mid), - fits.Column(name="Z_HIGH", format="D", array=z_high), - ] - for i in range(n_bins): - z_i, nz_i = sacc_io.get_nz(s, i) - if not np.array_equal(np.asarray(z_i, dtype=float), z_mid): - raise ValueError( - f"source bin {i} n(z) grid differs from source bin 0; the DES " - "NZDATA table requires one shared Z_MID axis" - ) - columns.append(fits.Column(name=f"BIN{i + 1}", format="D", array=nz_i)) - - hdu = fits.BinTableHDU.from_columns(fits.ColDefs(columns), name="NZDATA") - for key, value in { - "NZDATA": "T ", - "EXTNAME": "NZ_SOURCE", - "NBIN": n_bins, - "NZ": len(z_low), - }.items(): - hdu.header[key] = value - return hdu - - -def _cov_hdu(matrix, block_names, block_starts, extname="COVMAT", name_in_ctor=False): - """Build a covariance ImageHDU with ``NAME_i``/``STRT_i`` block headers. - - Reproduces the two covariance builders in ``cosmosis_fitting.py`` card for - card. The blocked ξ/τ ``covdat_to_fits`` builds ``ImageHDU(cov)`` unnamed - and stamps ``COVDATA`` then ``EXTNAME`` from a dict; the ``cov_cl_to_fits`` - CELL covariance builds ``ImageHDU(cov, name="COVMAT_CELL")`` (so the EXTNAME - card is created early, with astropy's standard comment) before re-stamping. - ``name_in_ctor`` selects the second form so the card order matches exactly. - """ - matrix = np.asarray(matrix, dtype=np.float64) - if matrix.shape[0] != matrix.shape[1]: - raise ValueError(f"covariance must be square; got shape {matrix.shape}") - hdu = fits.ImageHDU(matrix, name=extname) if name_in_ctor else fits.ImageHDU(matrix) - hdu.header["COVDATA"] = "True" - hdu.header["EXTNAME"] = extname - for i, (name, start) in enumerate(zip(block_names, block_starts)): - hdu.header[f"NAME_{i}"] = name - hdu.header[f"STRT_{i}"] = int(start) - return hdu - - -def _type_major_xi(s, bins): - """Return ``(theta, xip, xim)`` for one bin pair from the SACC reporting grid. - - ``sacc_io.get_xi`` already returns each statistic in insertion (= ascending - θ) order; the type-major split (all ξ+, then all ξ−) is exactly the two - arrays it hands back, so no further permutation is needed for a single pair. - """ - return sacc_io.get_xi(s, bins, grid="reporting") - - -def _require_single_bin(s, n_bins): - """Fail fast unless the SACC is a valid single-bin ξ product. - - The converter emits the single-bin 2pt-FITS today's CosmoSIS pipeline reads - (BIN1/BIN2 all 1, one NZ column). A tomographic SACC would otherwise slip - through silently — ``n_bins`` alone drives the NZDATA column count while the - ξ/covariance are read from bin ``(0, 0)`` only, so a 2-bin file would emit a - ``NBIN=2`` n(z) beside a data vector holding just the ``(0, 0)`` pair. - Guards both the empty-ξ case and the single-bin contract; tomographic - emission lands with the tomographic round. - """ - pairs = s.get_tracer_combinations(sacc_io.XI_PLUS) - if not pairs: - raise ValueError( - f"SACC has no {sacc_io.XI_PLUS} points — nothing to convert; the " - "2pt-FITS data vector is built from the ξ± statistics" - ) - expected = (sacc_io.source_name(0), sacc_io.source_name(0)) - if n_bins != 1 or set(pairs) != {expected}: - raise ValueError( - f"converter is single-bin only (n_bins=1, ξ pairs == {{{expected}}}); " - f"got n_bins={n_bins} and ξ pairs {sorted(pairs)}. Tomographic " - "emission (multiple bin pairs, per-pair BIN1/BIN2, one NZ column per " - "bin) lands with the tomographic round." - ) - - -def sacc_to_twopoint_fits( - s, - path, - *, - rho_stats_hdu=None, - tau_stats_hdu=None, - n_bins=1, -): - """Convert an analysis SACC to a CosmoSIS 2pt-FITS file. - - The assembled ``HDUList`` matches today's ``cosmosis_fitting.py`` product - for the configuration the SACC describes: PRIMARY, NZ_SOURCE, COVMAT, then - (if present) COVMAT_CELL, XI_PLUS, XI_MINUS, (if present) CELL_EE / CELL_BB, - and (if the rho/tau sidecars are supplied) TAU_0_PLUS, TAU_2_PLUS, - RHO_STATS. The data vector and its covariance are laid out type-major - (all ξ+, then all ξ−, then the τ blocks), which is the DES ``twopoint`` - convention CosmoSIS reads. - - Parameters - ---------- - s : sacc.Sacc - Analysis SACC (reporting ξ±, optional pseudo-Cℓ, covariance, and — for the - ρ/τ product — the τ data points; see ``rho_stats_hdu``). - path : str - Output FITS path (overwritten). - rho_stats_hdu, tau_stats_hdu : astropy.io.fits.BinTableHDU, optional - The rho-stats / tau-stats sidecar HDUs, copied verbatim as today's - assembly does. Required together to write the ρ/τ product; the SACC - alone cannot rebuild the ``varrho_*`` columns Sacha's fork reads. When - omitted, a pure ξ (± Cℓ) product is written. - n_bins : int, optional - Number of source tomographic bins. Must be ``1``: this converter emits - the single-bin 2pt-FITS today's CosmoSIS pipeline consumes. Tomographic - emission (multiple bin pairs, per-pair BIN1/BIN2, one NZ column per bin) - lands with the tomographic round; the converter fails fast on anything - else rather than silently truncating to bin ``(0, 0)``. - - Returns - ------- - astropy.io.fits.HDUList - The assembled list, also written to ``path``. - - Raises - ------ - ValueError - If the SACC has no ξ points; if ``n_bins != 1`` or the SACC's ξ tracer - pairs are anything other than exactly ``{(source_0, source_0)}`` (the - single-bin contract); or if exactly one of the ρ/τ sidecars is supplied. - """ - if (rho_stats_hdu is None) != (tau_stats_hdu is None): - raise ValueError( - "rho_stats_hdu and tau_stats_hdu must be supplied together " - "(the ρ/τ product needs both, or neither for a pure-ξ product)" - ) - _require_single_bin(s, n_bins) - use_rho_tau = rho_stats_hdu is not None - bins = (0, 0) - - nz_hdu = _nz_hdu(s, n_bins) - theta, xip, xim = _type_major_xi(s, bins) - xip_hdu = _twopoint_hdu("XI_PLUS", xip, theta, ang_unit="arcmin") - xim_hdu = _twopoint_hdu("XI_MINUS", xim, theta, ang_unit="arcmin") - - cell_hdu, cov_cell_hdu = _build_cell(s, bins) - - cov_hdu = _build_covmat(s, bins, use_rho_tau=use_rho_tau) - - tau_hdus, rho_hdu = _build_rho_tau(rho_stats_hdu, tau_stats_hdu, theta, use_rho_tau) - - # HDU order mirrors cosmosis_fitting.py's __main__: PRIMARY, NZ, COVMAT, - # COVMAT_CELL, XI±, CELL_EE, then the τ/ρ tables. - hdu_list = [fits.PrimaryHDU(), nz_hdu, cov_hdu] - if cov_cell_hdu is not None: - hdu_list.append(cov_cell_hdu) - hdu_list.extend([xip_hdu, xim_hdu]) - if cell_hdu is not None: - hdu_list.append(cell_hdu) - if use_rho_tau: - hdu_list.extend([*tau_hdus, rho_hdu]) - - hdul = fits.HDUList(hdu_list) - hdul.writeto(path, overwrite=True) - return hdul - - -def _build_cell(s, bins): - """Build the CELL_EE 2pt HDU plus the COVMAT_CELL HDU from the SACC pseudo-Cℓ. - - Returns ``(None, None)`` when the SACC has no pseudo-Cℓ. Only CELL_EE is - emitted — the harmonic ``2pt_like`` fits ``data_sets=CELL_EE``, and today's - assembly appends CELL_EE alone (it builds a CELL_BB HDU but discards it). - The SACC still carries EE/BB/EB with bandpower windows for the B-mode - null-test path; this converter surfaces only the block CosmoSIS reads. The - CELL covariance (the EE bandpower covariance) lives in its own COVMAT_CELL - ImageHDU, matching today's product. - """ - if sacc_io.CL_EE not in s.get_data_types(): - return None, None - - ell, cl_ee, _cl_bb, _cl_eb, _window = sacc_io.get_pseudo_cl(s, bins) - cell_hdu = _twopoint_hdu("CELL_EE", cl_ee, ell) - cell_idx = sacc_io._indices(s, sacc_io.CL_EE, sacc_io._pair(bins), grid="reporting") - cov_cell = s.covariance.dense[np.ix_(cell_idx, cell_idx)] - cov_cell_hdu = _cov_hdu( - cov_cell, ["CELL_EE"], [0], extname="COVMAT_CELL", name_in_ctor=True - ) - return cell_hdu, cov_cell_hdu - - -def _build_covmat(s, bins, *, use_rho_tau): - """Assemble the blocked COVMAT (ξ± type-major, then the τ blocks). - - The ξ covariance is pulled from the SACC as the contiguous ξ+/ξ− block for - the pair and permuted from pair-major (SACC) to type-major (2pt-FITS). Under - ``use_rho_tau`` the τ_0/τ_2 covariance blocks are appended block-diagonally - with zero ξ↔τ cross-blocks, exactly as ``covdat_to_fits`` builds them. - """ - pair = sacc_io._pair(bins) - idx_p = sacc_io._indices(s, sacc_io.XI_PLUS, pair, grid="reporting") - idx_m = sacc_io._indices(s, sacc_io.XI_MINUS, pair, grid="reporting") - n_theta = len(idx_p) - xi_idx = np.concatenate([idx_p, idx_m]) # type-major permutation - xi_cov = s.covariance.dense[np.ix_(xi_idx, xi_idx)] - - names = ["XI_PLUS", "XI_MINUS"] - starts = [0, n_theta] - matrix = xi_cov - - if use_rho_tau: - # The τ covariance couples τ_0+ and τ_2+ (today's assembly truncates the - # 3-statistic CosmoCov τ covariance to its first 2 blocks and lays it in - # as ONE contiguous [τ_0+; τ_2+] block — cross-correlation kept). In the - # SACC those two selections are not adjacent (τ_0− sits between them), so - # gather both index sets and extract the joint sub-block, ξ↔τ zero. - tau_pair = (sacc_io.source_name(0), sacc_io.PSF_TRACER) - idx_tau0 = sacc_io._indices( - s, sacc_io.TAU_PLUS.format(k=0), tau_pair, grid="reporting" - ) - idx_tau2 = sacc_io._indices( - s, sacc_io.TAU_PLUS.format(k=2), tau_pair, grid="reporting" - ) - tau_idx = np.concatenate([idx_tau0, idx_tau2]) - tau_cov = s.covariance.dense[np.ix_(tau_idx, tau_idx)] - matrix = _block_diag(matrix, tau_cov) - names += ["TAU_0_PLUS", "TAU_2_PLUS"] - starts += [2 * n_theta, 2 * n_theta + len(idx_tau0)] - - return _cov_hdu(matrix, names, starts) - - -def _block_diag(*blocks): - """Stack square blocks block-diagonally with zero cross-blocks.""" - sizes = [b.shape[0] for b in blocks] - n = sum(sizes) - out = np.zeros((n, n)) - start = 0 - for b in blocks: - out[start : start + b.shape[0], start : start + b.shape[0]] = b - start += b.shape[0] - return out - - -def _build_rho_tau(rho_stats_hdu, tau_stats_hdu, theta, use_rho_tau): - """Build the TAU_{0,2}_PLUS 2pt HDUs and the verbatim RHO_STATS HDU. - - Mirrors ``tau_to_fits`` / ``rho_to_fits``: τ_0/τ_2 read their ``tau_k_p`` - columns onto the shared ξ θ grid (consistency step); RHO_STATS is copied - verbatim from the sidecar with its θ column forced onto the ξ grid. The - ``varrho_*`` columns ride along in the copy — they are why the sidecar is - required (the SACC cannot supply them). - """ - if not use_rho_tau: - return (), None - - tau = tau_stats_hdu.data - tau0_hdu = _twopoint_hdu("TAU_0_PLUS", tau["tau_0_p"], theta, ang_unit="arcmin") - tau2_hdu = _twopoint_hdu("TAU_2_PLUS", tau["tau_2_p"], theta, ang_unit="arcmin") - - rho_hdu = rho_stats_hdu.copy() - rho_hdu.name = "RHO_STATS" - rho_hdu.data = rho_hdu.data.copy() - rho_hdu.data["theta"] = theta - return (tau0_hdu, tau2_hdu), rho_hdu - - -# ============================================================================= -# OneCovariance -# ============================================================================= - - -def nz_table(s, n_bins): - """Stack the SACC ``source_i`` NZ tracers into a OneCovariance n(z) table. - - Parameters - ---------- - s : sacc.Sacc - SACC holding ``source_0 … source_{n_bins-1}`` NZ tracers. - n_bins : int - Number of tomographic source bins to write. - - Returns - ------- - numpy.ndarray - Array of shape ``(n_z, n_bins + 1)``: column 0 the shared redshift - grid, columns ``1 … n_bins`` the per-bin ``n(z)``. This is the - OneCovariance combined-file layout (``redshift n_1(z) … n_N(z)``). - - Raises - ------ - ValueError - If any source bin is missing, or if the bins do not share one z grid - (OneCovariance's combined file has a single redshift column, so the - grids must agree bin-for-bin). - """ - z0, nz0 = sacc_io.get_nz(s, 0) - z0 = np.asarray(z0, dtype=float) - columns = [z0] - for i in range(n_bins): - if sacc_io.source_name(i) not in s.tracers: - raise ValueError( - f"SACC has no NZ tracer {sacc_io.source_name(i)!r}; cannot write " - f"a {n_bins}-bin OneCovariance n(z) file" - ) - z_i, nz_i = sacc_io.get_nz(s, i) - if not np.array_equal(np.asarray(z_i, dtype=float), z0): - raise ValueError( - f"source bin {i} n(z) grid differs from source bin 0; the " - "OneCovariance combined n(z) file has one shared redshift column" - ) - columns.append(np.asarray(nz_i, dtype=float)) - return np.column_stack(columns) - - -def write_nz(s, path, n_bins, *, dir_key="zlens_directory", header=True): - """Write the OneCovariance combined n(z) input file from a SACC. - - OneCovariance reads the source redshift distribution as a plain - whitespace-delimited text file whose column 0 is the shared redshift grid - and whose remaining columns are the per-bin ``n(z)`` (``redshift n_1(z) - … n_N(z)``) — no ``z_low``/``z_high`` edges. This writes that file from the - SACC ``source_i`` NZ tracers and returns the ``[redshift]`` config stanza - that points OneCovariance at it. - - Parameters - ---------- - s : sacc.Sacc - Analysis SACC with the ``source_i`` NZ tracers. - path : str or pathlib.Path - Output text-file path (overwritten). Its directory + basename become - the ``[redshift]`` directory/file config values. - n_bins : int - Number of tomographic source bins to write. - dir_key : str, optional - Config key for the redshift directory. Default ``"zlens_directory"`` - (upstream canonical). Pass ``"z_directory"`` for the UNIONS template - driven by ``pseudo_cl.py._modify_onecov_config``. - header : bool, optional - If ``True`` (default) prepend a ``# redshift n_1(z) …`` comment header - naming the columns; OneCovariance's ``genfromtxt``-style reader ignores - it. Set ``False`` for a bare numeric file. - - Returns - ------- - dict - The ``[redshift]`` config stanza (see :func:`nz_config_stanza`), naming - the file just written. - """ - table = nz_table(s, n_bins) - head = "" - if header: - cols = " ".join(f"n_{i + 1}(z)" for i in range(n_bins)) - head = f"redshift {cols}" - np.savetxt(str(path), table, header=head) - return nz_config_stanza( - os.path.dirname(os.path.abspath(str(path))), - os.path.basename(str(path)), - dir_key=dir_key, - ) - - -def nz_config_stanza( - directory, filename, *, dir_key="zlens_directory", value_loc="mid" -): - """Build the OneCovariance ``[redshift]`` config stanza for an n(z) file. - - Parameters - ---------- - directory : str - Directory holding the n(z) file (OneCovariance ``*_directory`` value). - filename : str - n(z) file basename (OneCovariance ``zlens_file`` value). - dir_key : str, optional - Directory config key — ``"zlens_directory"`` (upstream) or - ``"z_directory"`` (UNIONS template). Default ``"zlens_directory"``. - value_loc : str, optional - ``value_loc_in_lensbin`` — where in each histogram bin the tabulated - ``n(z)`` value sits (``mid``/``left``/``right``). Default ``"mid"``, - matching the bin-centred grids the SACC stores. - - Returns - ------- - dict - The ``[redshift]`` key/value pairs: ``{dir_key: directory, "zlens_file": - filename, "value_loc_in_lensbin": value_loc}``. Assign these under - ``config["redshift"]`` of a OneCovariance ``configparser`` config. - """ - if value_loc not in ("mid", "left", "right"): - raise ValueError( - f"value_loc_in_lensbin must be 'mid', 'left' or 'right'; got {value_loc!r}" - ) - return { - dir_key: directory, - "zlens_file": filename, - "value_loc_in_lensbin": value_loc, - } - - -def read_nz(path): - """Read a OneCovariance combined n(z) file back to ``(z, nz_columns)``. - - Inverse of :func:`write_nz` (the numeric round-trip; the config stanza is - not stored in the file). Comment/header lines are skipped. - - Parameters - ---------- - path : str or pathlib.Path - n(z) text file (column 0 = z, columns 1… = per-bin n(z)). - - Returns - ------- - tuple - ``(z, nz)`` where ``z`` is the shared redshift grid (shape ``(n_z,)``) - and ``nz`` is the per-bin distributions (shape ``(n_z, n_bins)``). - """ - table = np.atleast_2d(np.genfromtxt(str(path))) - return table[:, 0], table[:, 1:] - - -def covariance_blocks(cov_list, selectors, *, gaussian=True): - """Reshape a OneCovariance ``covariance_list`` table into SACC cov blocks. - - OneCovariance emits a flat ``covariance_list_*.dat`` table with one row per - ``(i, j)`` element pair (row-major, ``k = i·n + j``); the covariance value - lives in column 10 (Gaussian) or column 9 (Gaussian+non-Gaussian). This - reshapes the flat table into dense square block(s) — reusing - :func:`sp_validation.statistics.cov_from_one_covariance` for the per-block - reshape — and pairs each with its SACC selector, ready for - :func:`sp_validation.sacc_io.assemble_covariance`. - - Single-statistic case: pass the whole table and one selector; you get one - ``(selector, dense)`` block. Multi-statistic case (tomography-ready): pass a - sequence of ``(selector, sub_table)`` pairs — each ``sub_table`` a - contiguous slice of the flat output for one statistic / bin-pair — and each - is reshaped and re-paired with its selector in order. The API is thus shaped - to extend to multi-probe blocking without over-fitting the single-bin case. - - Parameters - ---------- - cov_list : numpy.ndarray or sequence - Either the flat OneCovariance table (2-D array, one row per pair) for a - single block, or — for the multi-block form — a sequence of - ``(selector, sub_table)`` pairs. In the multi-block form ``selectors`` - must be ``None`` (the selectors travel with the sub-tables). - selectors : selector or None - For the single-block form, the SACC selector for the whole table (a - ``(data_type, tracers[, tags])`` tuple or an index array, as - :func:`sacc_io.assemble_covariance` accepts). Must be ``None`` for the - multi-block form. - gaussian : bool, optional - Select the Gaussian-only column (``True``, default) or the - Gaussian+non-Gaussian column (``False``); passed straight through to - ``cov_from_one_covariance``. - - Returns - ------- - list - Ordered ``(selector, dense_cov)`` pairs, directly consumable by - ``sacc_io.assemble_covariance(s, blocks)``. - """ - if selectors is None: - # Multi-block form: cov_list is a sequence of (selector, sub_table). - return [ - (selector, cov_from_one_covariance(np.asarray(sub), gaussian=gaussian)) - for selector, sub in cov_list - ] - # Single-block form: one flat table, one selector. - return [ - (selectors, cov_from_one_covariance(np.asarray(cov_list), gaussian=gaussian)) - ] diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index 3df6488a..355f86c1 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -49,10 +49,28 @@ the covariance was built in. Converters that need a type-major layout (e.g. the DES 2pt-FITS convention) permute explicitly via ``s.indices`` rather than assuming global order. + + **Converters.** The tail of this module holds converters between + the SACC layout above and external analysis-tool file formats — + the CosmoSIS "2pt FITS" (``sacc_to_twopoint_fits``) and + OneCovariance's redshift / covariance files (``write_nz``, + ``nz_config_stanza``, ``read_nz``, ``covariance_blocks``). The + 2pt-FITS converter reproduces today's hand-assembled product from + ``cosmo_inference/scripts/cosmosis_fitting.py`` HDU-for-HDU and + byte-for-byte (verified against that writer), and is single-bin + only today — it fails fast on a multi-bin SACC (tomographic + emission lands with the tomographic round). The OneCovariance + converters are coupled to SACC by file format only; OneCovariance + itself is not a dependency. """ +import os + import numpy as np import sacc +from astropy.io import fits + +from .statistics import cov_from_one_covariance PSF_TRACER = "psf_stars" @@ -887,3 +905,537 @@ def load(path, *, allow_unblinded=False): "allow_unblinded=True." ) return s + + +# ============================================================================= +# CosmoSIS 2pt-FITS +# ============================================================================= + +# The QUANT1/QUANT2 header pair CosmoSIS stamps on each 2pt table, keyed by the +# extension name — copied from cosmosis_fitting.py so the headers match card +# for card. +_QUANT = { + "XI_PLUS": ("G+R", "G+R"), + "XI_MINUS": ("G-R", "G-R"), + "CELL_EE": ("GEF", "GEF"), + "CELL_BB": ("GBF", "GBF"), + "TAU_0_PLUS": ("G+R", "P+R"), + "TAU_2_PLUS": ("G+R", "SR+R"), +} + + +def _twopoint_hdu(name, values, ang, *, ang_unit=None): + """Build one 2pt BinTableHDU (BIN1/BIN2/ANGBIN/VALUE/ANG). + + Reproduces ``cosmosis_fitting.py._create_2pt_hdu`` /``cl_to_fits`` exactly: + same column order and formats, the ``2PTDATA`` marker, the QUANT pair for + ``name``, and NZ_SOURCE kernels. ``ang_unit`` stamps ``TUNIT`` on the ANG + column ("arcmin" for real-space ξ/τ; unset for Cℓ, whose ANG is ℓ). + """ + nbins = len(values) + angbin = np.arange(1, nbins + 1) + columns = [ + fits.Column(name="BIN1", format="K", array=np.ones(nbins)), + fits.Column(name="BIN2", format="K", array=np.ones(nbins)), + fits.Column(name="ANGBIN", format="K", array=angbin), + fits.Column(name="VALUE", format="D", array=values), + fits.Column(name="ANG", format="D", unit=ang_unit, array=ang), + ] + hdu = fits.BinTableHDU.from_columns(fits.ColDefs(columns), name=name) + quant1, quant2 = _QUANT[name] + for key, value in { + "2PTDATA": "T", + "QUANT1": quant1, + "QUANT2": quant2, + "KERNEL_1": "NZ_SOURCE", + "KERNEL_2": "NZ_SOURCE", + "WINDOWS": "SAMPLE", + }.items(): + hdu.header[key] = value + return hdu + + +def _nz_hdu(s, n_bins): + """Build the NZDATA HDU from the SACC ``source_i`` NZ tracers. + + Reproduces ``cosmosis_fitting.py.nz_to_fits``: Z_MID from the tracer ``z`` + grid (assumed uniform), Z_LOW/Z_HIGH as ± half a step, one ``BIN{i+1}`` + column per source bin, and the NZDATA/NBIN/NZ header cards. All source bins + are required to share the ``z`` grid — the single ``Z_MID`` axis of the + DES NZDATA table. + """ + z_mid, nz0 = get_nz(s, 0) + z_mid = np.asarray(z_mid, dtype=float) + step = z_mid[1] - z_mid[0] + z_low = z_mid - step / 2 + z_high = z_mid + step / 2 + + columns = [ + fits.Column(name="Z_LOW", format="D", array=z_low), + fits.Column(name="Z_MID", format="D", array=z_mid), + fits.Column(name="Z_HIGH", format="D", array=z_high), + ] + for i in range(n_bins): + z_i, nz_i = get_nz(s, i) + if not np.array_equal(np.asarray(z_i, dtype=float), z_mid): + raise ValueError( + f"source bin {i} n(z) grid differs from source bin 0; the DES " + "NZDATA table requires one shared Z_MID axis" + ) + columns.append(fits.Column(name=f"BIN{i + 1}", format="D", array=nz_i)) + + hdu = fits.BinTableHDU.from_columns(fits.ColDefs(columns), name="NZDATA") + for key, value in { + "NZDATA": "T ", + "EXTNAME": "NZ_SOURCE", + "NBIN": n_bins, + "NZ": len(z_low), + }.items(): + hdu.header[key] = value + return hdu + + +def _cov_hdu(matrix, block_names, block_starts, extname="COVMAT", name_in_ctor=False): + """Build a covariance ImageHDU with ``NAME_i``/``STRT_i`` block headers. + + Reproduces the two covariance builders in ``cosmosis_fitting.py`` card for + card. The blocked ξ/τ ``covdat_to_fits`` builds ``ImageHDU(cov)`` unnamed + and stamps ``COVDATA`` then ``EXTNAME`` from a dict; the ``cov_cl_to_fits`` + CELL covariance builds ``ImageHDU(cov, name="COVMAT_CELL")`` (so the EXTNAME + card is created early, with astropy's standard comment) before re-stamping. + ``name_in_ctor`` selects the second form so the card order matches exactly. + """ + matrix = np.asarray(matrix, dtype=np.float64) + if matrix.shape[0] != matrix.shape[1]: + raise ValueError(f"covariance must be square; got shape {matrix.shape}") + hdu = fits.ImageHDU(matrix, name=extname) if name_in_ctor else fits.ImageHDU(matrix) + hdu.header["COVDATA"] = "True" + hdu.header["EXTNAME"] = extname + for i, (name, start) in enumerate(zip(block_names, block_starts)): + hdu.header[f"NAME_{i}"] = name + hdu.header[f"STRT_{i}"] = int(start) + return hdu + + +def _type_major_xi(s, bins): + """Return ``(theta, xip, xim)`` for one bin pair from the SACC reporting grid. + + ``get_xi`` already returns each statistic in insertion (= ascending + θ) order; the type-major split (all ξ+, then all ξ−) is exactly the two + arrays it hands back, so no further permutation is needed for a single pair. + """ + return get_xi(s, bins, grid="reporting") + + +def _require_single_bin(s, n_bins): + """Fail fast unless the SACC is a valid single-bin ξ product. + + The converter emits the single-bin 2pt-FITS today's CosmoSIS pipeline reads + (BIN1/BIN2 all 1, one NZ column). A tomographic SACC would otherwise slip + through silently — ``n_bins`` alone drives the NZDATA column count while the + ξ/covariance are read from bin ``(0, 0)`` only, so a 2-bin file would emit a + ``NBIN=2`` n(z) beside a data vector holding just the ``(0, 0)`` pair. + Guards both the empty-ξ case and the single-bin contract; tomographic + emission lands with the tomographic round. + """ + pairs = s.get_tracer_combinations(XI_PLUS) + if not pairs: + raise ValueError( + f"SACC has no {XI_PLUS} points — nothing to convert; the " + "2pt-FITS data vector is built from the ξ± statistics" + ) + expected = (source_name(0), source_name(0)) + if n_bins != 1 or set(pairs) != {expected}: + raise ValueError( + f"converter is single-bin only (n_bins=1, ξ pairs == {{{expected}}}); " + f"got n_bins={n_bins} and ξ pairs {sorted(pairs)}. Tomographic " + "emission (multiple bin pairs, per-pair BIN1/BIN2, one NZ column per " + "bin) lands with the tomographic round." + ) + + +def sacc_to_twopoint_fits( + s, + path, + *, + rho_stats_hdu=None, + tau_stats_hdu=None, + n_bins=1, +): + """Convert an analysis SACC to a CosmoSIS 2pt-FITS file. + + The assembled ``HDUList`` matches today's ``cosmosis_fitting.py`` product + for the configuration the SACC describes: PRIMARY, NZ_SOURCE, COVMAT, then + (if present) COVMAT_CELL, XI_PLUS, XI_MINUS, (if present) CELL_EE / CELL_BB, + and (if the rho/tau sidecars are supplied) TAU_0_PLUS, TAU_2_PLUS, + RHO_STATS. The data vector and its covariance are laid out type-major + (all ξ+, then all ξ−, then the τ blocks), which is the DES ``twopoint`` + convention CosmoSIS reads. + + Parameters + ---------- + s : sacc.Sacc + Analysis SACC (reporting ξ±, optional pseudo-Cℓ, covariance, and — for the + ρ/τ product — the τ data points; see ``rho_stats_hdu``). + path : str + Output FITS path (overwritten). + rho_stats_hdu, tau_stats_hdu : astropy.io.fits.BinTableHDU, optional + The rho-stats / tau-stats sidecar HDUs, copied verbatim as today's + assembly does. Required together to write the ρ/τ product; the SACC + alone cannot rebuild the ``varrho_*`` columns Sacha's fork reads. When + omitted, a pure ξ (± Cℓ) product is written. + n_bins : int, optional + Number of source tomographic bins. Must be ``1``: this converter emits + the single-bin 2pt-FITS today's CosmoSIS pipeline consumes. Tomographic + emission (multiple bin pairs, per-pair BIN1/BIN2, one NZ column per bin) + lands with the tomographic round; the converter fails fast on anything + else rather than silently truncating to bin ``(0, 0)``. + + Returns + ------- + astropy.io.fits.HDUList + The assembled list, also written to ``path``. + + Raises + ------ + ValueError + If the SACC has no ξ points; if ``n_bins != 1`` or the SACC's ξ tracer + pairs are anything other than exactly ``{(source_0, source_0)}`` (the + single-bin contract); or if exactly one of the ρ/τ sidecars is supplied. + """ + if (rho_stats_hdu is None) != (tau_stats_hdu is None): + raise ValueError( + "rho_stats_hdu and tau_stats_hdu must be supplied together " + "(the ρ/τ product needs both, or neither for a pure-ξ product)" + ) + _require_single_bin(s, n_bins) + use_rho_tau = rho_stats_hdu is not None + bins = (0, 0) + + nz_hdu = _nz_hdu(s, n_bins) + theta, xip, xim = _type_major_xi(s, bins) + xip_hdu = _twopoint_hdu("XI_PLUS", xip, theta, ang_unit="arcmin") + xim_hdu = _twopoint_hdu("XI_MINUS", xim, theta, ang_unit="arcmin") + + cell_hdu, cov_cell_hdu = _build_cell(s, bins) + + cov_hdu = _build_covmat(s, bins, use_rho_tau=use_rho_tau) + + tau_hdus, rho_hdu = _build_rho_tau(rho_stats_hdu, tau_stats_hdu, theta, use_rho_tau) + + # HDU order mirrors cosmosis_fitting.py's __main__: PRIMARY, NZ, COVMAT, + # COVMAT_CELL, XI±, CELL_EE, then the τ/ρ tables. + hdu_list = [fits.PrimaryHDU(), nz_hdu, cov_hdu] + if cov_cell_hdu is not None: + hdu_list.append(cov_cell_hdu) + hdu_list.extend([xip_hdu, xim_hdu]) + if cell_hdu is not None: + hdu_list.append(cell_hdu) + if use_rho_tau: + hdu_list.extend([*tau_hdus, rho_hdu]) + + hdul = fits.HDUList(hdu_list) + hdul.writeto(path, overwrite=True) + return hdul + + +def _build_cell(s, bins): + """Build the CELL_EE 2pt HDU plus the COVMAT_CELL HDU from the SACC pseudo-Cℓ. + + Returns ``(None, None)`` when the SACC has no pseudo-Cℓ. Only CELL_EE is + emitted — the harmonic ``2pt_like`` fits ``data_sets=CELL_EE``, and today's + assembly appends CELL_EE alone (it builds a CELL_BB HDU but discards it). + The SACC still carries EE/BB/EB with bandpower windows for the B-mode + null-test path; this converter surfaces only the block CosmoSIS reads. The + CELL covariance (the EE bandpower covariance) lives in its own COVMAT_CELL + ImageHDU, matching today's product. + """ + if CL_EE not in s.get_data_types(): + return None, None + + ell, cl_ee, _cl_bb, _cl_eb, _window = get_pseudo_cl(s, bins) + cell_hdu = _twopoint_hdu("CELL_EE", cl_ee, ell) + cell_idx = _indices(s, CL_EE, _pair(bins), grid="reporting") + cov_cell = s.covariance.dense[np.ix_(cell_idx, cell_idx)] + cov_cell_hdu = _cov_hdu( + cov_cell, ["CELL_EE"], [0], extname="COVMAT_CELL", name_in_ctor=True + ) + return cell_hdu, cov_cell_hdu + + +def _build_covmat(s, bins, *, use_rho_tau): + """Assemble the blocked COVMAT (ξ± type-major, then the τ blocks). + + The ξ covariance is pulled from the SACC as the contiguous ξ+/ξ− block for + the pair and permuted from pair-major (SACC) to type-major (2pt-FITS). Under + ``use_rho_tau`` the τ_0/τ_2 covariance blocks are appended block-diagonally + with zero ξ↔τ cross-blocks, exactly as ``covdat_to_fits`` builds them. + """ + pair = _pair(bins) + idx_p = _indices(s, XI_PLUS, pair, grid="reporting") + idx_m = _indices(s, XI_MINUS, pair, grid="reporting") + n_theta = len(idx_p) + xi_idx = np.concatenate([idx_p, idx_m]) # type-major permutation + xi_cov = s.covariance.dense[np.ix_(xi_idx, xi_idx)] + + names = ["XI_PLUS", "XI_MINUS"] + starts = [0, n_theta] + matrix = xi_cov + + if use_rho_tau: + # The τ covariance couples τ_0+ and τ_2+ (today's assembly truncates the + # 3-statistic CosmoCov τ covariance to its first 2 blocks and lays it in + # as ONE contiguous [τ_0+; τ_2+] block — cross-correlation kept). In the + # SACC those two selections are not adjacent (τ_0− sits between them), so + # gather both index sets and extract the joint sub-block, ξ↔τ zero. + tau_pair = (source_name(0), PSF_TRACER) + idx_tau0 = _indices(s, TAU_PLUS.format(k=0), tau_pair, grid="reporting") + idx_tau2 = _indices(s, TAU_PLUS.format(k=2), tau_pair, grid="reporting") + tau_idx = np.concatenate([idx_tau0, idx_tau2]) + tau_cov = s.covariance.dense[np.ix_(tau_idx, tau_idx)] + matrix = _block_diag(matrix, tau_cov) + names += ["TAU_0_PLUS", "TAU_2_PLUS"] + starts += [2 * n_theta, 2 * n_theta + len(idx_tau0)] + + return _cov_hdu(matrix, names, starts) + + +def _block_diag(*blocks): + """Stack square blocks block-diagonally with zero cross-blocks.""" + sizes = [b.shape[0] for b in blocks] + n = sum(sizes) + out = np.zeros((n, n)) + start = 0 + for b in blocks: + out[start : start + b.shape[0], start : start + b.shape[0]] = b + start += b.shape[0] + return out + + +def _build_rho_tau(rho_stats_hdu, tau_stats_hdu, theta, use_rho_tau): + """Build the TAU_{0,2}_PLUS 2pt HDUs and the verbatim RHO_STATS HDU. + + Mirrors ``tau_to_fits`` / ``rho_to_fits``: τ_0/τ_2 read their ``tau_k_p`` + columns onto the shared ξ θ grid (consistency step); RHO_STATS is copied + verbatim from the sidecar with its θ column forced onto the ξ grid. The + ``varrho_*`` columns ride along in the copy — they are why the sidecar is + required (the SACC cannot supply them). + """ + if not use_rho_tau: + return (), None + + tau = tau_stats_hdu.data + tau0_hdu = _twopoint_hdu("TAU_0_PLUS", tau["tau_0_p"], theta, ang_unit="arcmin") + tau2_hdu = _twopoint_hdu("TAU_2_PLUS", tau["tau_2_p"], theta, ang_unit="arcmin") + + rho_hdu = rho_stats_hdu.copy() + rho_hdu.name = "RHO_STATS" + rho_hdu.data = rho_hdu.data.copy() + rho_hdu.data["theta"] = theta + return (tau0_hdu, tau2_hdu), rho_hdu + + +# ============================================================================= +# OneCovariance +# ============================================================================= + + +def nz_table(s, n_bins): + """Stack the SACC ``source_i`` NZ tracers into a OneCovariance n(z) table. + + Parameters + ---------- + s : sacc.Sacc + SACC holding ``source_0 … source_{n_bins-1}`` NZ tracers. + n_bins : int + Number of tomographic source bins to write. + + Returns + ------- + numpy.ndarray + Array of shape ``(n_z, n_bins + 1)``: column 0 the shared redshift + grid, columns ``1 … n_bins`` the per-bin ``n(z)``. This is the + OneCovariance combined-file layout (``redshift n_1(z) … n_N(z)``). + + Raises + ------ + ValueError + If any source bin is missing, or if the bins do not share one z grid + (OneCovariance's combined file has a single redshift column, so the + grids must agree bin-for-bin). + """ + z0, nz0 = get_nz(s, 0) + z0 = np.asarray(z0, dtype=float) + columns = [z0] + for i in range(n_bins): + if source_name(i) not in s.tracers: + raise ValueError( + f"SACC has no NZ tracer {source_name(i)!r}; cannot write " + f"a {n_bins}-bin OneCovariance n(z) file" + ) + z_i, nz_i = get_nz(s, i) + if not np.array_equal(np.asarray(z_i, dtype=float), z0): + raise ValueError( + f"source bin {i} n(z) grid differs from source bin 0; the " + "OneCovariance combined n(z) file has one shared redshift column" + ) + columns.append(np.asarray(nz_i, dtype=float)) + return np.column_stack(columns) + + +def write_nz(s, path, n_bins, *, dir_key="zlens_directory", header=True): + """Write the OneCovariance combined n(z) input file from a SACC. + + OneCovariance reads the source redshift distribution as a plain + whitespace-delimited text file whose column 0 is the shared redshift grid + and whose remaining columns are the per-bin ``n(z)`` (``redshift n_1(z) + … n_N(z)``) — no ``z_low``/``z_high`` edges. This writes that file from the + SACC ``source_i`` NZ tracers and returns the ``[redshift]`` config stanza + that points OneCovariance at it. + + Parameters + ---------- + s : sacc.Sacc + Analysis SACC with the ``source_i`` NZ tracers. + path : str or pathlib.Path + Output text-file path (overwritten). Its directory + basename become + the ``[redshift]`` directory/file config values. + n_bins : int + Number of tomographic source bins to write. + dir_key : str, optional + Config key for the redshift directory. Default ``"zlens_directory"`` + (upstream canonical). Pass ``"z_directory"`` for the UNIONS template + driven by ``pseudo_cl.py._modify_onecov_config``. + header : bool, optional + If ``True`` (default) prepend a ``# redshift n_1(z) …`` comment header + naming the columns; OneCovariance's ``genfromtxt``-style reader ignores + it. Set ``False`` for a bare numeric file. + + Returns + ------- + dict + The ``[redshift]`` config stanza (see :func:`nz_config_stanza`), naming + the file just written. + """ + table = nz_table(s, n_bins) + head = "" + if header: + cols = " ".join(f"n_{i + 1}(z)" for i in range(n_bins)) + head = f"redshift {cols}" + np.savetxt(str(path), table, header=head) + return nz_config_stanza( + os.path.dirname(os.path.abspath(str(path))), + os.path.basename(str(path)), + dir_key=dir_key, + ) + + +def nz_config_stanza( + directory, filename, *, dir_key="zlens_directory", value_loc="mid" +): + """Build the OneCovariance ``[redshift]`` config stanza for an n(z) file. + + Parameters + ---------- + directory : str + Directory holding the n(z) file (OneCovariance ``*_directory`` value). + filename : str + n(z) file basename (OneCovariance ``zlens_file`` value). + dir_key : str, optional + Directory config key — ``"zlens_directory"`` (upstream) or + ``"z_directory"`` (UNIONS template). Default ``"zlens_directory"``. + value_loc : str, optional + ``value_loc_in_lensbin`` — where in each histogram bin the tabulated + ``n(z)`` value sits (``mid``/``left``/``right``). Default ``"mid"``, + matching the bin-centred grids the SACC stores. + + Returns + ------- + dict + The ``[redshift]`` key/value pairs: ``{dir_key: directory, "zlens_file": + filename, "value_loc_in_lensbin": value_loc}``. Assign these under + ``config["redshift"]`` of a OneCovariance ``configparser`` config. + """ + if value_loc not in ("mid", "left", "right"): + raise ValueError( + f"value_loc_in_lensbin must be 'mid', 'left' or 'right'; got {value_loc!r}" + ) + return { + dir_key: directory, + "zlens_file": filename, + "value_loc_in_lensbin": value_loc, + } + + +def read_nz(path): + """Read a OneCovariance combined n(z) file back to ``(z, nz_columns)``. + + Inverse of :func:`write_nz` (the numeric round-trip; the config stanza is + not stored in the file). Comment/header lines are skipped. + + Parameters + ---------- + path : str or pathlib.Path + n(z) text file (column 0 = z, columns 1… = per-bin n(z)). + + Returns + ------- + tuple + ``(z, nz)`` where ``z`` is the shared redshift grid (shape ``(n_z,)``) + and ``nz`` is the per-bin distributions (shape ``(n_z, n_bins)``). + """ + table = np.atleast_2d(np.genfromtxt(str(path))) + return table[:, 0], table[:, 1:] + + +def covariance_blocks(cov_list, selectors, *, gaussian=True): + """Reshape a OneCovariance ``covariance_list`` table into SACC cov blocks. + + OneCovariance emits a flat ``covariance_list_*.dat`` table with one row per + ``(i, j)`` element pair (row-major, ``k = i·n + j``); the covariance value + lives in column 10 (Gaussian) or column 9 (Gaussian+non-Gaussian). This + reshapes the flat table into dense square block(s) — reusing + :func:`sp_validation.statistics.cov_from_one_covariance` for the per-block + reshape — and pairs each with its SACC selector, ready for + :func:`sp_validation.assemble_covariance`. + + Single-statistic case: pass the whole table and one selector; you get one + ``(selector, dense)`` block. Multi-statistic case (tomography-ready): pass a + sequence of ``(selector, sub_table)`` pairs — each ``sub_table`` a + contiguous slice of the flat output for one statistic / bin-pair — and each + is reshaped and re-paired with its selector in order. The API is thus shaped + to extend to multi-probe blocking without over-fitting the single-bin case. + + Parameters + ---------- + cov_list : numpy.ndarray or sequence + Either the flat OneCovariance table (2-D array, one row per pair) for a + single block, or — for the multi-block form — a sequence of + ``(selector, sub_table)`` pairs. In the multi-block form ``selectors`` + must be ``None`` (the selectors travel with the sub-tables). + selectors : selector or None + For the single-block form, the SACC selector for the whole table (a + ``(data_type, tracers[, tags])`` tuple or an index array, as + :func:`assemble_covariance` accepts). Must be ``None`` for the + multi-block form. + gaussian : bool, optional + Select the Gaussian-only column (``True``, default) or the + Gaussian+non-Gaussian column (``False``); passed straight through to + ``cov_from_one_covariance``. + + Returns + ------- + list + Ordered ``(selector, dense_cov)`` pairs, directly consumable by + ``assemble_covariance(s, blocks)``. + """ + if selectors is None: + # Multi-block form: cov_list is a sequence of (selector, sub_table). + return [ + (selector, cov_from_one_covariance(np.asarray(sub), gaussian=gaussian)) + for selector, sub in cov_list + ] + # Single-block form: one flat table, one selector. + return [ + (selectors, cov_from_one_covariance(np.asarray(cov_list), gaussian=gaussian)) + ] diff --git a/src/sp_validation/tests/test_sacc_interop_one_covariance.py b/src/sp_validation/tests/test_sacc_io_one_covariance.py similarity index 92% rename from src/sp_validation/tests/test_sacc_interop_one_covariance.py rename to src/sp_validation/tests/test_sacc_io_one_covariance.py index f0df56e9..bbe90ca3 100644 --- a/src/sp_validation/tests/test_sacc_interop_one_covariance.py +++ b/src/sp_validation/tests/test_sacc_io_one_covariance.py @@ -1,4 +1,4 @@ -"""Tests for :mod:`sp_validation.sacc_interop`. +"""Tests for :mod:`sp_validation.sacc_io`. All synthetic, all fast: the OneCovariance fixtures are built in memory shaped exactly like its real file I/O — a flat ``covariance_list`` table with @@ -22,7 +22,6 @@ import numpy.testing as npt import pytest -from sp_validation import sacc_interop as ocio from sp_validation import sacc_io as sio @@ -84,8 +83,8 @@ def test_covariance_blocks_reshapes_to_hand_built_matrix(): selector = (sio.XI_PLUS, (sio.source_name(0), sio.source_name(0))) - [(sel_g, block_g)] = ocio.covariance_blocks(table, selector, gaussian=True) - [(sel_a, block_a)] = ocio.covariance_blocks(table, selector, gaussian=False) + [(sel_g, block_g)] = sio.covariance_blocks(table, selector, gaussian=True) + [(sel_a, block_a)] = sio.covariance_blocks(table, selector, gaussian=False) assert sel_g == selector and sel_a == selector npt.assert_allclose(block_g, cov_gauss, rtol=1e-12) @@ -98,7 +97,7 @@ def test_covariance_blocks_reshapes_to_hand_built_matrix(): # entry (row k = i·n + j, col 10 for gaussian). perturbed = table.copy() perturbed[2 * 4 + 1, 10] += 5.0 # element (i=2, j=1) - [(_, block_p)] = ocio.covariance_blocks(perturbed, selector, gaussian=True) + [(_, block_p)] = sio.covariance_blocks(perturbed, selector, gaussian=True) npt.assert_allclose(block_p[2, 1] - block_g[2, 1], 5.0, rtol=1e-12) block_p[2, 1] = block_g[2, 1] npt.assert_allclose(block_p, block_g, rtol=1e-12) # nothing else moved @@ -122,7 +121,7 @@ def test_covariance_blocks_multiblock_form(): sel_a = (sio.XI_PLUS, (sio.source_name(0), sio.source_name(0))) sel_b = (sio.XI_MINUS, (sio.source_name(0), sio.source_name(0))) - blocks = ocio.covariance_blocks( + blocks = sio.covariance_blocks( [(sel_a, table_a), (sel_b, table_b)], None, gaussian=True ) @@ -162,7 +161,7 @@ def test_covariance_blocks_feed_assemble_covariance(): [s.indices(sio.XI_PLUS, pair), s.indices(sio.XI_MINUS, pair)] ) - blocks = ocio.covariance_blocks(table, selector, gaussian=True) + blocks = sio.covariance_blocks(table, selector, gaussian=True) sio.assemble_covariance(s, blocks) npt.assert_allclose(s.covariance.dense, cov_gauss, rtol=1e-12) @@ -190,9 +189,9 @@ def test_write_nz_roundtrips_and_names_file(tmp_path): s = sio.new_sacc({0: (z, nz0), 1: (z, nz1)}) path = tmp_path / "nz_onecov.txt" - stanza = ocio.write_nz(s, path, n_bins=2) + stanza = sio.write_nz(s, path, n_bins=2) - z_read, nz_read = ocio.read_nz(path) + z_read, nz_read = sio.read_nz(path) npt.assert_allclose(z_read, z, rtol=1e-12) assert nz_read.shape == (len(z), 2) npt.assert_allclose(nz_read[:, 0], nz0, rtol=1e-12) @@ -213,7 +212,7 @@ def test_write_nz_unions_template_dir_key(tmp_path): """ z, nz0 = _nz(12) s = sio.new_sacc({0: (z, nz0)}) - stanza = ocio.write_nz(s, tmp_path / "nz.txt", n_bins=1, dir_key="z_directory") + stanza = sio.write_nz(s, tmp_path / "nz.txt", n_bins=1, dir_key="z_directory") assert "z_directory" in stanza and "zlens_directory" not in stanza assert stanza["z_directory"] == str(tmp_path) assert stanza["zlens_file"] == "nz.txt" @@ -231,7 +230,7 @@ def test_write_nz_fails_on_mismatched_z_grids(tmp_path): z1_shifted = z1_shifted + 0.1 # different grid s = sio.new_sacc({0: (z0, nz0), 1: (z1_shifted, nz1)}) with pytest.raises(ValueError, match="differs from source bin 0"): - ocio.write_nz(s, tmp_path / "bad.txt", n_bins=2) + sio.write_nz(s, tmp_path / "bad.txt", n_bins=2) def test_write_nz_no_header_roundtrips(tmp_path): @@ -244,8 +243,8 @@ def test_write_nz_no_header_roundtrips(tmp_path): z, nz0 = _nz(40) s = sio.new_sacc({0: (z, nz0)}) path = tmp_path / "bare.txt" - ocio.write_nz(s, path, n_bins=1, header=False) - z_read, nz_read = ocio.read_nz(path) + sio.write_nz(s, path, n_bins=1, header=False) + z_read, nz_read = sio.read_nz(path) npt.assert_allclose(z_read, z, rtol=1e-12) npt.assert_allclose(nz_read[:, 0], nz0, rtol=1e-12) @@ -258,7 +257,7 @@ def test_nz_config_stanza_rejects_bad_value_loc(): raise ``ValueError`` rather than write a stanza OneCovariance will reject. """ with pytest.raises(ValueError, match="value_loc_in_lensbin"): - ocio.nz_config_stanza("/dir", "nz.txt", value_loc="center") + sio.nz_config_stanza("/dir", "nz.txt", value_loc="center") def test_write_nz_fails_on_missing_bin(tmp_path): @@ -271,4 +270,4 @@ def test_write_nz_fails_on_missing_bin(tmp_path): z, nz0 = _nz(30) s = sio.new_sacc({0: (z, nz0)}) with pytest.raises(ValueError, match="source_1"): - ocio.write_nz(s, tmp_path / "short.txt", n_bins=2) + sio.write_nz(s, tmp_path / "short.txt", n_bins=2) diff --git a/src/sp_validation/tests/test_sacc_interop_realdata.py b/src/sp_validation/tests/test_sacc_io_realdata.py similarity index 98% rename from src/sp_validation/tests/test_sacc_interop_realdata.py rename to src/sp_validation/tests/test_sacc_io_realdata.py index df7b1366..abe1f230 100644 --- a/src/sp_validation/tests/test_sacc_interop_realdata.py +++ b/src/sp_validation/tests/test_sacc_io_realdata.py @@ -28,7 +28,7 @@ import pytest from astropy.io import fits -from sp_validation import sacc_interop, sacc_io +from sp_validation import sacc_io _DATA = Path("/automnt/n17data/cdaley/unions/code/sp_validation/cosmo_inference/data") _REAL_FILES = { @@ -221,7 +221,7 @@ def test_realdata_roundtrip_byte_equal(label, tmp_path): reference = _current_script_reference(cf, hdul, tmp_path) converted = tmp_path / "converted.fits" - sacc_interop.sacc_to_twopoint_fits( + sacc_io.sacc_to_twopoint_fits( s, str(converted), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) @@ -250,7 +250,7 @@ def test_realdata_ondisk_drift_is_only_cell_bb(label, tmp_path): s, rho_hdu, tau_hdu = _sacc_from_2pt_fits(hdul) ondisk_names = [h.name for h in hdul] converted = tmp_path / "converted.fits" - sacc_interop.sacc_to_twopoint_fits( + sacc_io.sacc_to_twopoint_fits( s, str(converted), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) with fits.open(converted) as conv: diff --git a/src/sp_validation/tests/test_sacc_interop_twopoint.py b/src/sp_validation/tests/test_sacc_io_twopoint.py similarity index 96% rename from src/sp_validation/tests/test_sacc_interop_twopoint.py rename to src/sp_validation/tests/test_sacc_io_twopoint.py index cd1c4a31..78f6809d 100644 --- a/src/sp_validation/tests/test_sacc_interop_twopoint.py +++ b/src/sp_validation/tests/test_sacc_io_twopoint.py @@ -1,6 +1,6 @@ """Byte-compare tests for the SACC -> 2pt-FITS converter. -The converter (:mod:`sp_validation.sacc_interop`) must reproduce the CosmoSIS +The converter (:mod:`sp_validation.sacc_io`) must reproduce the CosmoSIS 2pt-FITS that ``cosmo_inference/scripts/cosmosis_fitting.py`` assembles today, so the inference chain (``2pt_like`` and Sacha Guerrini's rho/tau ``2pt_like_xi_sys`` fork) runs untouched behind it. The strongest possible check @@ -36,7 +36,7 @@ import pytest from astropy.io import fits -from sp_validation import sacc_interop, sacc_io +from sp_validation import sacc_io _SCRIPT = ( Path(__file__).resolve().parents[3] @@ -290,7 +290,7 @@ def test_plain_xi_byte_equal(tmp_path): reference = _reference_fits(tmp_path, inp) s = _sacc(inp) out = tmp_path / "converted.fits" - sacc_interop.sacc_to_twopoint_fits(s, str(out), n_bins=1) + sacc_io.sacc_to_twopoint_fits(s, str(out), n_bins=1) assert out.read_bytes() == reference.read_bytes() @@ -300,7 +300,7 @@ def test_xi_cl_byte_equal(tmp_path): reference = _reference_fits(tmp_path, inp, cl=True) s = _sacc(inp, cl=True) out = tmp_path / "converted.fits" - sacc_interop.sacc_to_twopoint_fits(s, str(out), n_bins=1) + sacc_io.sacc_to_twopoint_fits(s, str(out), n_bins=1) assert out.read_bytes() == reference.read_bytes() @@ -311,7 +311,7 @@ def test_xi_rho_tau_byte_equal(tmp_path): s = _sacc(inp, rho_tau=True) rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) out = tmp_path / "converted.fits" - sacc_interop.sacc_to_twopoint_fits( + sacc_io.sacc_to_twopoint_fits( s, str(out), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) assert out.read_bytes() == reference.read_bytes() @@ -334,7 +334,7 @@ def test_tau_covariance_keeps_tau0_tau2_cross(tmp_path): s = _sacc(inp, rho_tau=True) rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) out = tmp_path / "converted.fits" - sacc_interop.sacc_to_twopoint_fits( + sacc_io.sacc_to_twopoint_fits( s, str(out), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) with fits.open(out) as hdul: @@ -351,7 +351,7 @@ def test_perturbed_xi_changes_output(tmp_path): inp = _inputs(seed=0) s = _sacc(inp) out = tmp_path / "base.fits" - sacc_interop.sacc_to_twopoint_fits(s, str(out), n_bins=1) + sacc_io.sacc_to_twopoint_fits(s, str(out), n_bins=1) with fits.open(out) as hdul: base_xip = hdul["XI_PLUS"].data["VALUE"].copy() @@ -359,7 +359,7 @@ def test_perturbed_xi_changes_output(tmp_path): inp2["xip"] = inp2["xip"] + 1.0 s2 = _sacc(inp2) out2 = tmp_path / "perturbed.fits" - sacc_interop.sacc_to_twopoint_fits(s2, str(out2), n_bins=1) + sacc_io.sacc_to_twopoint_fits(s2, str(out2), n_bins=1) with fits.open(out2) as hdul: new_xip = hdul["XI_PLUS"].data["VALUE"] @@ -384,7 +384,7 @@ def test_integration_grid_points_ignored(tmp_path): s_plain = _sacc(inp, cl=True, rho_tau=True) rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) out_plain = tmp_path / "plain.fits" - sacc_interop.sacc_to_twopoint_fits( + sacc_io.sacc_to_twopoint_fits( s_plain, str(out_plain), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) @@ -452,7 +452,7 @@ def test_integration_grid_points_ignored(tmp_path): s_aug.add_covariance(full) out_aug = tmp_path / "aug.fits" - sacc_interop.sacc_to_twopoint_fits( + sacc_io.sacc_to_twopoint_fits( s_aug, str(out_aug), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) @@ -465,7 +465,7 @@ def test_rho_tau_sidecars_required_together(tmp_path): s = _sacc(inp, rho_tau=True) rho_hdu, _tau_hdu = _sidecar_hdus(tmp_path, inp) with pytest.raises(ValueError, match="together"): - sacc_interop.sacc_to_twopoint_fits( + sacc_io.sacc_to_twopoint_fits( s, str(tmp_path / "x.fits"), rho_stats_hdu=rho_hdu, n_bins=1 ) @@ -490,9 +490,9 @@ def test_tomographic_sacc_raises(tmp_path): s.add_covariance(np.eye(len(s.mean))) with pytest.raises(ValueError, match="single-bin only"): - sacc_interop.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits"), n_bins=2) + sacc_io.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits"), n_bins=2) with pytest.raises(ValueError, match="single-bin only"): - sacc_interop.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits"), n_bins=1) + sacc_io.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits"), n_bins=1) assert not (tmp_path / "x.fits").exists() @@ -513,7 +513,7 @@ def test_sacc_without_xi_raises(tmp_path): s.add_covariance(np.eye(len(s.mean))) with pytest.raises(ValueError, match="nothing to convert"): - sacc_interop.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits")) + sacc_io.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits")) assert not (tmp_path / "x.fits").exists() @@ -538,7 +538,7 @@ def test_covmat_blocks_exact_gather_encoded_cov(tmp_path): rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) out = tmp_path / "encoded.fits" - sacc_interop.sacc_to_twopoint_fits( + sacc_io.sacc_to_twopoint_fits( s, str(out), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) @@ -567,7 +567,7 @@ def _mask_idx(dtype, tracers): _mask_idx(sacc_io.TAU_PLUS.format(k=2), (SOURCE, PSF)), ] ) - expected = sacc_interop._block_diag( + expected = sacc_io._block_diag( encoded[np.ix_(xi_idx, xi_idx)], encoded[np.ix_(tau_idx, tau_idx)] ) cell_idx = _mask_idx(sacc_io.CL_EE, pair) From 06b24c6f4324f1bfaee7c9281a2992ff55625c86 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 20 Jul 2026 22:30:15 +0200 Subject: [PATCH 39/47] fix(Dockerfile): copy source before the blinding-extra editable install The merged layer order ran the [blinding] editable install before COPY ., so uv-overrides.txt (and the source tree) did not exist in the build context. The blinding install is itself 'uv pip install -e .[blinding]', so the old final --no-deps editable layer is redundant and is dropped. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01766vawzi2XqrgoyHmeHEY9 --- Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 52038d9f..3caa129f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,6 +33,11 @@ COPY pyproject.toml uv.lock /sp_validation/ RUN uv sync --frozen --inexact --no-install-project \ --extra test --extra glass --extra workflow +# Full source in place before the blinding-extra install below: it is an +# editable install of the project itself and needs uv-overrides.txt and +# scripts/patch_firecrown.py from the tree. +COPY . /sp_validation + # The [blinding] extra (SACC/Smokescreen blinding stack: firecrown + smokescreen) # is not in uv.lock — firecrown is not on PyPI and declares conda-forge-only / # unused sampler connectors as hard deps, so it needs the override file (see @@ -53,8 +58,3 @@ RUN uv pip install --no-cache-dir 'numpy>=2.2,<2.5' # This patches the installed tree (surgical, pinned-version-checked, loud on # mismatch) and verifies `import firecrown.likelihood; import smokescreen`. RUN python scripts/patch_firecrown.py - -# Install sp_validation itself (editable) into the same venv; deps are already -# satisfied by the sync + blinding-extra install above. -COPY . /sp_validation -RUN uv pip install --no-deps -e . From 67db6764c2f6ea044cf2c693eb3734b2c19faccc Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 20 Jul 2026 22:57:44 +0200 Subject: [PATCH 40/47] fix: reconcile pr7-lineage SACC callers with guarded sacc_io API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The feat/sacc-3 merge brought the guard-era sacc_io whose save() now requires a keyword-only `type={'data','mock'}` provenance tag (PRD #241 §4) and whose load() refuses unblinded `type='data'`. Application code and tests from the pr4/pr7 lineage still called the old `save(s, path)` and referenced a template file the #236 cleanup had relocated, leaving 21 CI failures outside the merge-conflict set. This adapts the callers to the reviewed contract (sacc_io is unchanged). save() type= threading: - CosmologyValidation gains a `sacc_type` ctor kwarg (default "data"), stamped by every part-writer: cosebis/pure_eb/psf_systematics/pseudo_cl now `save(..., type=self.sacc_type)`. - assemble_sacc inherits provenance from its parts: `type=metadata["type"]`. - run_2pcf uses `cv.sacc_type`; run_2pcf_highres stamps "data" (real-catalogue fine covariance). - Synthetic-data tests adopt the reviewed idiom `type="mock"` (loads freely): test_sacc_writers _roundtrip + reload helper, test_assemble_sacc part writer, and the test_pseudo_cl fixture (`sacc_type="mock"`). Relocated FITS engine template: - The #236 folder cleanup moved cosmosis_pipeline_A_ia.ini into cosmosis_config/templates/ (still used by the legacy cosmosis_fitting.py path). The new pr7 Snakemake regime — inference.smk's INFERENCE_TEMPLATE_DIR, generate_inference_config, and its tests — resolves templates from cosmosis_config/ directly, where pr7 placed A_ia_sacc.ini but never copied its FITS sibling. Restore A_ia.ini alongside the sacc template. Fast suite: 254 passed, 1 skipped. Two remaining failures (test_calculate_pure_eb pinned-value drift; test_configured_paths_exist_on_candide) are pre-existing and environmental — they fail identically with these changes stashed, and neither is in the API-drift set. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01766vawzi2XqrgoyHmeHEY9 --- .../cosmosis_pipeline_A_ia.ini | 108 ++++++++++++++++++ src/sp_validation/cosmo_val/core.py | 5 + src/sp_validation/cosmo_val/cosebis.py | 2 +- src/sp_validation/cosmo_val/pseudo_cl.py | 2 +- .../cosmo_val/psf_systematics.py | 2 +- src/sp_validation/cosmo_val/pure_eb.py | 2 +- src/sp_validation/tests/test_assemble_sacc.py | 2 +- src/sp_validation/tests/test_pseudo_cl.py | 1 + src/sp_validation/tests/test_sacc_writers.py | 4 +- workflow/scripts/assemble_sacc.py | 3 +- workflow/scripts/run_2pcf.py | 2 +- workflow/scripts/run_2pcf_highres.py | 3 +- 12 files changed, 126 insertions(+), 10 deletions(-) create mode 100644 cosmo_inference/cosmosis_config/cosmosis_pipeline_A_ia.ini diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_A_ia.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_A_ia.ini new file mode 100644 index 00000000..eb3ab166 --- /dev/null +++ b/cosmo_inference/cosmosis_config/cosmosis_pipeline_A_ia.ini @@ -0,0 +1,108 @@ +#parameters used elsewhere in this file +[DEFAULT] +COSMOSIS_DIR = /n23data1/n06data/lgoh/scratch/cosmosis-standard-library_lisa + + +[pipeline] +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic 2pt_shear shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[polychord] +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[test] + +[output] +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_xi_plus shear_xi_minus +verbose = F + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +data_sets=XI_PLUS XI_MINUS +like_name=2pt_like + +angle_range_XI_PLUS_1_1= 10.0 200.0 +angle_range_XI_MINUS_1_1= 20.0 200.0 \ No newline at end of file diff --git a/src/sp_validation/cosmo_val/core.py b/src/sp_validation/cosmo_val/core.py index 0c9273d8..b916e3f8 100644 --- a/src/sp_validation/cosmo_val/core.py +++ b/src/sp_validation/cosmo_val/core.py @@ -224,6 +224,7 @@ def __init__( path_onecovariance=None, cosmo_params=None, blind=None, + sacc_type="data", ): self.rho_tau_method = rho_tau_method self.cov_estimate_method = cov_estimate_method @@ -253,6 +254,10 @@ def __init__( self.nside_mask = nside_mask self.path_onecovariance = path_onecovariance self.blind = blind + # SACC provenance stamped by every part-writer via sacc_io.save(type=…): + # 'data' for real catalogues (load-gated until blinded), 'mock' for + # simulations (freely inspectable). PRD #241 §4. + self.sacc_type = sacc_type assert self.cell_method in ["map", "catalog"], ( "cell_method must be 'map' or 'catalog'" diff --git a/src/sp_validation/cosmo_val/cosebis.py b/src/sp_validation/cosmo_val/cosebis.py index 6b1d6146..9e606e07 100644 --- a/src/sp_validation/cosmo_val/cosebis.py +++ b/src/sp_validation/cosmo_val/cosebis.py @@ -180,7 +180,7 @@ def cosebis_to_sacc_part(self, version, out_path, results, fiducial_scale_cut=No result, scale_cut, ) - sacc_io.save(s, out_path) + sacc_io.save(s, out_path, type=self.sacc_type) def plot_cosebis( self, diff --git a/src/sp_validation/cosmo_val/pseudo_cl.py b/src/sp_validation/cosmo_val/pseudo_cl.py index 29ff7846..acffbdfe 100644 --- a/src/sp_validation/cosmo_val/pseudo_cl.py +++ b/src/sp_validation/cosmo_val/pseudo_cl.py @@ -704,7 +704,7 @@ def pseudo_cl_to_sacc_part(self, version, out_path, ell_eff, cl_all, wsp): cl_all, wsp, ) - sacc_io.save(s, out_path) + sacc_io.save(s, out_path, type=self.sacc_type) def plot_pseudo_cl(self): """ diff --git a/src/sp_validation/cosmo_val/psf_systematics.py b/src/sp_validation/cosmo_val/psf_systematics.py index af819977..d9900de6 100644 --- a/src/sp_validation/cosmo_val/psf_systematics.py +++ b/src/sp_validation/cosmo_val/psf_systematics.py @@ -80,7 +80,7 @@ def rho_tau_to_sacc_part( tau_cov_th=tau_cov_th, ) out_path = os.path.join(out_dir, f"rho_tau_{base}.sacc") - sacc_io.save(s, out_path) + sacc_io.save(s, out_path, type=self.sacc_type) @property def rho_stat_handler(self): diff --git a/src/sp_validation/cosmo_val/pure_eb.py b/src/sp_validation/cosmo_val/pure_eb.py index 416ad8ef..37814dbf 100644 --- a/src/sp_validation/cosmo_val/pure_eb.py +++ b/src/sp_validation/cosmo_val/pure_eb.py @@ -151,7 +151,7 @@ def pure_eb_to_sacc_part(self, version, out_path, results): eb, covariance=results["cov"], ) - sacc_io.save(s, out_path) + sacc_io.save(s, out_path, type=self.sacc_type) def plot_pure_eb( self, diff --git a/src/sp_validation/tests/test_assemble_sacc.py b/src/sp_validation/tests/test_assemble_sacc.py index ccddab97..6b4a9a7f 100644 --- a/src/sp_validation/tests/test_assemble_sacc.py +++ b/src/sp_validation/tests/test_assemble_sacc.py @@ -131,7 +131,7 @@ def get_bandpower_windows(self): paths = {} for name, part in parts.items(): p = tmp_path / f"{name}.sacc" - sio.save(part, str(p)) + sio.save(part, str(p), type="mock") paths[name] = str(p) return paths diff --git a/src/sp_validation/tests/test_pseudo_cl.py b/src/sp_validation/tests/test_pseudo_cl.py index 637b10c2..e9079bfc 100644 --- a/src/sp_validation/tests/test_pseudo_cl.py +++ b/src/sp_validation/tests/test_pseudo_cl.py @@ -169,6 +169,7 @@ def cv(tmp_path): power=0.5, n_ell_bins=N_ELL_BINS, pol_factor=True, + sacc_type="mock", ) cv._test_version = version return cv diff --git a/src/sp_validation/tests/test_sacc_writers.py b/src/sp_validation/tests/test_sacc_writers.py index d886fa0a..82080869 100644 --- a/src/sp_validation/tests/test_sacc_writers.py +++ b/src/sp_validation/tests/test_sacc_writers.py @@ -31,7 +31,7 @@ def _theta(n=6): def _roundtrip(s, tmp_path, name): p = tmp_path / f"{name}.sacc" - sio.save(s, str(p)) + sio.save(s, str(p), type="mock") return sio.load(str(p)) @@ -306,7 +306,7 @@ def test_assemble_from_reloaded_parts(tmp_path): parts = _make_parts(nz) reloaded = [] for i, part in enumerate(parts): - sio.save(part, str(tmp_path / f"part{i}.sacc")) + sio.save(part, str(tmp_path / f"part{i}.sacc"), type="mock") reloaded.append(sio.load(str(tmp_path / f"part{i}.sacc"))) s = sw.assemble_analysis_sacc(nz, META, reloaded) assert type(s.covariance).__name__ == "FullCovariance" diff --git a/workflow/scripts/assemble_sacc.py b/workflow/scripts/assemble_sacc.py index 513ec976..1398d87b 100644 --- a/workflow/scripts/assemble_sacc.py +++ b/workflow/scripts/assemble_sacc.py @@ -167,7 +167,8 @@ def assemble_sacc( if not parts: raise ValueError(f"no parts found for {version}: {part_paths}") s = assemble_analysis_sacc(nz, metadata, parts) - sacc_io.save(s, out_path) + # Provenance is inherited from the parts (all same version → same type). + sacc_io.save(s, out_path, type=metadata["type"]) print(f"Assembled {len(parts)} parts -> {out_path}") return s diff --git a/workflow/scripts/run_2pcf.py b/workflow/scripts/run_2pcf.py index a773d039..5f223b4c 100644 --- a/workflow/scripts/run_2pcf.py +++ b/workflow/scripts/run_2pcf.py @@ -85,7 +85,7 @@ def run_2pcf( out_path = sacc_out or os.path.join( output_dir or cv.cc["paths"]["output"], f"{ver}_xi_coarse.sacc" ) - sacc_io.save(s, out_path) + sacc_io.save(s, out_path, type=cv.sacc_type) print(f"Wrote coarse ξ± SACC part: {out_path}") return gg diff --git a/workflow/scripts/run_2pcf_highres.py b/workflow/scripts/run_2pcf_highres.py index ed3660bf..681e12d8 100644 --- a/workflow/scripts/run_2pcf_highres.py +++ b/workflow/scripts/run_2pcf_highres.py @@ -229,7 +229,8 @@ def write_xi_fine_sacc(gg): variances=np.concatenate([gg.varxip, gg.varxim]), ) out_path = os.path.join(OUTPUT_DIR, f"{VERSION}_xi_fine.sacc") - sacc_io.save(s, out_path) + # Fine ξ± is computed from the real catalogue for the high-res covariance. + sacc_io.save(s, out_path, type="data") log(f" Wrote {out_path}") From 9ee6eb9e3bb80f3e2ea16079e1e3251a5bb8c07a Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 21 Jul 2026 15:41:11 +0200 Subject: [PATCH 41/47] feat(blinding): wire blind-at-birth custody into the SACC migration DAG MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Author the Snakemake rules that realise sp_validation.blinding's three-verb custody surface on the migrated cosmo_val workflow (issues #247/#252). - rule blind_init (once per catalogue version): draws the seed, publishes commitment.json + the encrypted seed bundle. Refuses to overwrite state. - rule blind_part (generic over the three blindable stems — reporting ξ±, integration ξ±, analysis pseudo-Cℓ): conceals the part at birth, escrows the true vector beside the blinded output. The plaintext part is a temp() output of its producer on a data run, so only its blinded sibling persists. - common.py: run-type-aware path helpers (blindable_part / maybe_temp / blind_state_paths / version_of) so a data run binds ξ-derived consumers to the blinded siblings and a mock run bypasses blinding entirely — the whole blinding subgraph appears or vanishes with RUN_TYPE. - assemble_sacc: assert_consistent_blind across parts (assembly-time commitment check of #252), stamping the shared blind on the terminal file. - Constrain the npatch wildcard to \d+ so a producer's ξ± output cannot absorb the _blinded suffix (which made rule xi ambiguous with blind_part). The cosmo_val assemble DAG dry-runs: blind_init x2, blind_part x5, assemble consuming the blinded ξ± + pseudo-Cℓ parts. Born-blinded COSEBIs/pure-E/B and the ρ/τ concealed pass-through follow in the next commit. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EaL7prmKUHwJQcDyW3LoxD --- workflow/Snakefile | 4 ++ workflow/common.py | 97 ++++++++++++++++++++++++++++++- workflow/rules/blinding.smk | 74 +++++++++++++++++++++++ workflow/rules/cosmo_val.smk | 13 ++++- workflow/rules/twopoint.smk | 14 ++++- workflow/scripts/assemble_sacc.py | 13 +++++ workflow/scripts/blind_init.py | 12 ++++ workflow/scripts/blind_part.py | 19 ++++++ 8 files changed, 240 insertions(+), 6 deletions(-) create mode 100644 workflow/rules/blinding.smk create mode 100644 workflow/scripts/blind_init.py create mode 100644 workflow/scripts/blind_part.py diff --git a/workflow/Snakefile b/workflow/Snakefile index 6f59827b..0a1f3258 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -37,6 +37,10 @@ wildcard_constraints: # Compute rules (infrastructure — raw outputs, no evidence.json) include: "rules/twopoint.smk" +# Smokescreen blind-at-birth custody (blind_init / blind_part). Generic over the +# blindable parts twopoint.smk produces; dormant unless a data run requests a +# blinded part. Included before covariance/cosmo_val so their consumers resolve. +include: "rules/blinding.smk" include: "rules/covariance.smk" include: "rules/inference.smk" include: "rules/masks.smk" diff --git a/workflow/common.py b/workflow/common.py index 1a168041..530b9212 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -5,6 +5,8 @@ import re from pathlib import Path +from snakemake.io import temp + # Absolute path to the generic workflow's scripts, anchored on this module's own # location (common.py lives in workflow/, is `from common import *`'d into every # Snakefile, and so resolves to the generic workflow dir of the running checkout @@ -51,6 +53,10 @@ # glass-mock A/B/C variant, not Smokescreen blinding — see BLINDS above. "blind": r"[ABC]", "nbins": r"\d+", + # Constrained so a producer's ξ± output pattern cannot greedily absorb the + # "_blinded" suffix into npatch (which would make it ambiguous with the + # blind_part rule's {stem}_blinded output). npatch is always an integer. + "npatch": r"\d+", "min_sep": r"[0-9.]+", "max_sep": r"[0-9.]+", "gaussian": r"(g|ng)", @@ -65,16 +71,22 @@ DEFAULT_MASK_SUFFIX = "" CATALOG_CONFIG = None PLANCK18 = None +# Run type gates Smokescreen blind-at-birth (see the blind custody section +# below): "data" blinds the three blindable parts and binds ξ-derived consumers +# to the blinded siblings; "mock" bypasses blinding entirely. Set from +# config["cosmo_val"]["type"] in configure(); "data" is the production default. +RUN_TYPE = "data" def configure(workflow_config): """Install config-derived values after Snakemake has loaded configfiles.""" - global CATALOG_CONFIG, DEFAULT_MASK_SUFFIX, FIDUCIAL, PLANCK18 + global CATALOG_CONFIG, DEFAULT_MASK_SUFFIX, FIDUCIAL, PLANCK18, RUN_TYPE CATALOG_CONFIG = workflow_config FIDUCIAL = workflow_config["fiducial"] DEFAULT_MASK_SUFFIX = ( "_masked" if workflow_config["covariance"].get("default_masked", False) else "" ) + RUN_TYPE = workflow_config.get("cosmo_val", {}).get("type", "data") with open(COSMOLOGY_PARAMS) as f: PLANCK18 = json.load(f) @@ -196,6 +208,89 @@ def get_shear_catalog(wildcards): return str(Path(subdir) / shear_path) +# --------------------------------------------------------------------------- +# Smokescreen blind-at-birth custody (issues #247/#252, PR #253) +# --------------------------------------------------------------------------- +# Distinct from the glass-mock A/B/C `blind` wildcard above: this is Smokescreen +# concealment (the concealed=True SACC stamp). Blinding is per part, at birth. +# The three blindable parts (reporting ξ±, integration ξ±, pseudo-Cℓ) are each +# concealed the moment they are computed, so only blinded parts persist on disk. +# A `data` run binds every ξ-derived consumer (the terminal assemble, and the +# born-blinded COSEBIs / pure-E/B) to the *_blinded parts, which pulls the +# blind_part → blind_init subgraph into the DAG. A `mock` run bypasses blinding +# entirely and binds to the plaintext parts, so the subgraph never appears. +# RUN_TYPE is the single switch that flips which files exist. +# +# The path helpers below MIRROR sp_validation.blinding.init_paths / part_paths +# by hand rather than importing blinding (which pulls in numpy + smokescreen) at +# DAG-build time. test_blinding_wiring asserts the two stay in lockstep. + + +def is_data_run(): + """True when blinding is active (production data runs); False for mocks.""" + return RUN_TYPE == "data" + + +def blind_state_dir(version): + """Per-version blind-init custody directory (commitment + encrypted seed).""" + return str(COSMO_VAL / "blind" / version) + + +def blind_state_paths(version): + """The fixed custody-state files blind_init writes for a version. + + Mirrors sp_validation.blinding.init_paths(blind_state_dir(version)). + """ + d = blind_state_dir(version) + return { + "commitment": os.path.join(d, "commitment.json"), + "bundle": os.path.join(d, "blind_seed.encrpt"), + "key": os.path.join(d, "blind_seed.key"), + } + + +def blinded_path(part_path): + """The *_blinded sibling blind_part writes beside a plaintext part. + + Mirrors sp_validation.blinding.part_paths(part_path)["blinded"]. + """ + stem, ext = os.path.splitext(str(part_path)) + return f"{stem}_blinded{ext or '.fits'}" + + +def version_of(stem): + """Extract the catalogue version embedded in a blindable part's stem. + + Every blindable stem carries the version (as {version}_xi_… or + pseudo_cl_{version}_…); blind_part needs it to locate the version's blind + state. Matches the shared `version` wildcard pattern. + """ + m = re.search(WILDCARD_CONSTRAINTS["version"], stem) + if m is None: + raise ValueError(f"no catalogue version found in part stem {stem!r}") + return m.group(0) + + +def blindable_part(part_path): + """On-disk path a run persists for one blindable part. + + Data run -> the blinded sibling (binding it pulls blind_part + blind_init + into the DAG); mock run -> the plaintext part (blinding bypassed). + """ + return blinded_path(part_path) if is_data_run() else str(part_path) + + +def maybe_temp(part_path): + """Wrap a producer's blindable plaintext part temp() on data runs. + + On a data run the plaintext part's only consumer is blind_part, which + escrows the true vector before Snakemake removes the temp file — so no + plaintext blindable part persists. On a mock run the part is the real + product downstream binds to, so it is left persistent. + """ + return temp(str(part_path)) if is_data_run() else str(part_path) + + # --------------------------------------------------------------------------- # CosmologyValidation diagnostic suite (cosmo_val.py) # --------------------------------------------------------------------------- diff --git a/workflow/rules/blinding.smk b/workflow/rules/blinding.smk new file mode 100644 index 00000000..bf79433f --- /dev/null +++ b/workflow/rules/blinding.smk @@ -0,0 +1,74 @@ +# Smokescreen blind-at-birth custody rules (issues #247/#252, PR #253). +# +# Two rules realise the three-verb custody surface of sp_validation.blinding on +# the DAG. They only enter the graph on a `data` run, and only when a consumer +# binds to a *_blinded part through common.blindable_part — a `mock` run never +# requests a blinded file, so blind_part and blind_init stay dormant. +# +# blind_init (once per catalogue version) draws the seed, publishes +# commitment.json + the encrypted seed bundle. +# blind_part (once per blindable part, at birth) conceals the part, escrows +# the true vector beside the blinded output, and lets Snakemake +# remove the plaintext (temp()) once it is the sole consumer. +# +# The terminal assemble_sacc rule (cosmo_val.smk) asserts the shared commitment +# across parts — the assembly-time custody check of #252. + +# The three blindable stems: reporting ξ± (rule xi), integration ξ± (xi_highres), +# and the analysis pseudo-Cℓ (pseudo_cl). None contains "_blinded", so the +# generic blind_part rule can never blind its own output twice. The version +# pattern is the shared one from common.WILDCARD_CONSTRAINTS. +_V = WILDCARD_CONSTRAINTS["version"] +BLINDABLE_STEM = ( + rf"(?:{_V}_xi_reporting_minsep=[0-9.]+_maxsep=[0-9.]+_nbins=\d+_npatch=\d+" + rf"|{_V}_xi_integration" + rf"|pseudo_cl_{_V}_blind=[ABC]_[a-z]+_nbins=\d+)" +) + + +rule blind_init: + """Fix the blind for one catalogue version (blind-init). + + Draws an OS-entropy seed, writes the repo-committable commitment.json + (sha256(seed) + config digest) and the Fernet-encrypted seed bundle. Runs + once per version and refuses to overwrite existing state — a blind is a + one-shot custody event. + """ + output: + commitment=str(COSMO_VAL / "blind" / "{version}" / "commitment.json"), + bundle=str(COSMO_VAL / "blind" / "{version}" / "blind_seed.encrpt"), + key=str(COSMO_VAL / "blind" / "{version}" / "blind_seed.key"), + params: + blind_dir=lambda w: blind_state_dir(w.version), + resources: + runtime=5, + script: + "../scripts/blind_init.py" + + +rule blind_part: + """Blind one intermediate part SACC at birth (blind-part). + + Conceals the plaintext part through its matching theory backend, escrows the + true vector into a per-part encrypted bundle beside the blinded output, and + leaves the plaintext for Snakemake to remove (it is a temp() output of the + producing rule, and this is its only consumer on a data run). Generic over + the three blindable stems. + """ + input: + part=str(COSMO_VAL / "{stem}.sacc"), + commitment=lambda w: blind_state_paths(version_of(w.stem))["commitment"], + bundle=lambda w: blind_state_paths(version_of(w.stem))["bundle"], + key=lambda w: blind_state_paths(version_of(w.stem))["key"], + output: + blinded=str(COSMO_VAL / "{stem}_blinded.sacc"), + escrow=str(COSMO_VAL / "{stem}_escrow.encrpt"), + escrow_key=str(COSMO_VAL / "{stem}_escrow.key"), + wildcard_constraints: + stem=BLINDABLE_STEM, + params: + blind_dir=lambda w: blind_state_dir(version_of(w.stem)), + resources: + runtime=10, + script: + "../scripts/blind_part.py" diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index 73ec22c7..7a7e62c6 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -482,8 +482,15 @@ def cv_assemble_inputs(version): fiducial harmonic tag). pseudo_cl (+ its cov) is included only when the config toggles the harmonic-space BB into the analysis. """ + # blindable_part binds the raw-signal parts (reporting/integration ξ±, + # analysis pseudo-Cℓ) to their blinded siblings on a data run and to the + # plaintext on a mock run. COSEBIs and pure-E/B are born blinded (their + # writers derive them from the blinded integration ξ± and stamp + # concealed=True), so they bind by their own name in both cases; ρ/τ is a + # diagnostic carrying no cosmological vector but is stamped concealed + # pass-through so the fail-closed load gate admits it on a data run. parts = dict( - xi_reporting=cv_xi_reporting_sacc(version), + xi_reporting=blindable_part(cv_xi_reporting_sacc(version)), cosebis=cv_cosebis_sacc(version), pure_eb=cv_pure_eb_sacc(version), rho_tau=cv_rho_tau_sacc(version), @@ -493,9 +500,9 @@ def cv_assemble_inputs(version): # fiducial version's terminal file alone; other versions' {version}.sacc omit # the integration rows rather than trigger a job with no output to bind. if version == config["fiducial"]["version"]: - parts["xi_integration"] = cv_xi_integration_sacc(version) + parts["xi_integration"] = blindable_part(cv_xi_integration_sacc(version)) if CV.get("include_pseudo_cl", False): - parts["pseudo_cl"] = cv_pseudo_cl_analysis_sacc(version) + parts["pseudo_cl"] = blindable_part(cv_pseudo_cl_analysis_sacc(version)) parts["pseudo_cl_cov"] = cv_pseudo_cl_cov(version) return parts diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index 9eb90c64..f4a644ee 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -14,7 +14,9 @@ rule xi: # rule to share one wildcard set, and it keeps the reporting .sacc name # self-describing so requesting it binds the xi job unambiguously. txt=str(COSMO_VAL / "{version}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.txt"), - xi_reporting=str(COSMO_VAL / "{version}_xi_reporting_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.sacc"), + # Blindable part: temp() on a data run so only its blinded sibling + # persists (blind_part escrows the true vector first). See common.maybe_temp. + xi_reporting=maybe_temp(str(COSMO_VAL / "{version}_xi_reporting_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.sacc")), threads: 24 params: ver="{version}", @@ -42,7 +44,8 @@ rule xi_highres: container: None output: txt=str(COSMO_VAL / f"{FIDUCIAL['version']}_xi_minsep={FIDUCIAL['min_sep_int']}_maxsep={FIDUCIAL['max_sep_int']}_nbins=10000_npatch=1.txt"), - xi_integration=str(COSMO_VAL / f"{FIDUCIAL['version']}_xi_integration.sacc"), + # Blindable part: temp() on a data run (see rule xi / common.maybe_temp). + xi_integration=maybe_temp(str(COSMO_VAL / f"{FIDUCIAL['version']}_xi_integration.sacc")), resources: tasks=30, cpus_per_task=12, @@ -116,6 +119,13 @@ rule pseudo_cl: concealed=True stamp is a separate axis on the SACC file. """ output: + # This generic rule produces every pseudo-Cℓ variant — the analysis part + # (blind=A, powspace, nbins=32) folded into {version}.sacc, plus the fine + # (COSEBIS) and glass-mock variants. Only the analysis part is a terminal + # blindable, and a data run blinds it via a requested _blinded sibling + # (blind_part reads this plaintext); the fine/mock variants are B-mode / + # validation intermediates left untouched here. The output is therefore + # not temp()'d — see the PR note on residual unblinded pseudo-Cℓ. pseudo_cl=str(COSMO_VAL / "pseudo_cl_{version}_blind={blind}_{binning}_nbins={nbins}.sacc"), wildcard_constraints: blind="[ABC]", # glass-mock variant, not Smokescreen blinding diff --git a/workflow/scripts/assemble_sacc.py b/workflow/scripts/assemble_sacc.py index 4eec725d..84e5c97d 100644 --- a/workflow/scripts/assemble_sacc.py +++ b/workflow/scripts/assemble_sacc.py @@ -187,7 +187,20 @@ def assemble_sacc( ) if not parts: raise ValueError(f"no parts found for {version}: {part_paths}") + # Assembly-time custody assertion (#252): every blindable part (ξ± / pseudo-Cℓ + # EE) must share one blind commitment + config digest, or assembly fails + # closed — mixed blinded/plaintext parts and divergent-seed parts both raise. + # ρ/τ and covariance-only parts are exempt. Returns the shared blind stamp to + # carry onto the assembled file (or None for a fully-mock plaintext assembly). + from sp_validation import blinding + + shared = blinding.assert_consistent_blind(parts) s = assemble_analysis_sacc(nz, metadata, parts) + if shared is not None: + # Stamp the assembled file with the shared blind so it, too, reads as + # concealed (its parts already carried the stamp into `metadata` above; + # this makes the custody state explicit and authoritative on the union). + s.metadata.update(shared) # Assembly preserves its parts' provenance: every part was written by # sacc_io.save and therefore carries the type=data|mock stamp in its # metadata (copied into the assembled file above). diff --git a/workflow/scripts/blind_init.py b/workflow/scripts/blind_init.py new file mode 100644 index 00000000..f9e4b682 --- /dev/null +++ b/workflow/scripts/blind_init.py @@ -0,0 +1,12 @@ +"""Rule blind_init: fix the blind for one catalogue version. + +Thin wrapper over :func:`sp_validation.blinding.blind_init`. Draws the seed, +writes commitment.json + the encrypted seed bundle into the version's blind +directory. The plaintext seed is never written (the encryptor deletes it). +""" + +from snakemake.script import snakemake + +from sp_validation import blinding + +blinding.blind_init(snakemake.params["blind_dir"]) diff --git a/workflow/scripts/blind_part.py b/workflow/scripts/blind_part.py new file mode 100644 index 00000000..b93313fb --- /dev/null +++ b/workflow/scripts/blind_part.py @@ -0,0 +1,19 @@ +"""Rule blind_part: blind one intermediate part SACC at birth. + +Thin wrapper over :func:`sp_validation.blinding.blind_part`. Conceals the part, +escrows the true vector beside the blinded output, and leaves the plaintext in +place: it is a temp() output of the producing rule, so Snakemake removes it once +this (its only consumer on a data run) finishes. keep_input=True hands that +lifecycle to Snakemake rather than deleting inside the blind step, which keeps +the blinded output and its temp input in one consistent DAG accounting. +""" + +from snakemake.script import snakemake + +from sp_validation import blinding + +blinding.blind_part( + snakemake.input["part"], + snakemake.params["blind_dir"], + keep_input=True, +) From be38cd5df43f8d6ec9cacb733c740600fb74e879 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 21 Jul 2026 16:03:27 +0200 Subject: [PATCH 42/47] test(blinding): wiring tests for the blind-at-birth DAG MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - common.py path helpers mirror sp_validation.blinding init_paths/part_paths (drift guard), version_of on all three blindable stems, run-type switch. - Data-run fail-closed assembly: assemble_sacc refuses an unblinded type=data part, passes when every part is concealed under one commitment, and refuses a blinded/plaintext mix and divergent commitments. - Candide-only: the blinding subgraph (blind_init/blind_part) resolves in the cosmo_val assemble dry-run and binds the blinded ξ± + pseudo-Cℓ siblings. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EaL7prmKUHwJQcDyW3LoxD --- .../tests/test_blinding_wiring.py | 263 ++++++++++++++++++ 1 file changed, 263 insertions(+) create mode 100644 src/sp_validation/tests/test_blinding_wiring.py diff --git a/src/sp_validation/tests/test_blinding_wiring.py b/src/sp_validation/tests/test_blinding_wiring.py new file mode 100644 index 00000000..77beb7ee --- /dev/null +++ b/src/sp_validation/tests/test_blinding_wiring.py @@ -0,0 +1,263 @@ +"""Tests for the Snakemake blind-at-birth wiring (issues #247/#252, PR #253). + +Two seams are covered here, both independent of a live cluster: + +1. **The path helpers in ``workflow/common.py``** must stay in lockstep with + ``sp_validation.blinding`` — common mirrors ``init_paths`` / ``part_paths`` by + hand (to keep the DAG build from importing the heavy blinding module), so a + drift between them would silently mis-wire ``blind_part``. These tests are the + guard. +2. **The data-run fail-closed assembly**: ``assemble_sacc`` must refuse an + unblinded ``type='data'`` part and succeed once every part is concealed under + one commitment — the terminal custody gate of #252. + +A candide-only test additionally asserts the blinding subgraph resolves in the +cosmo_val DAG dry-run. +""" + +import importlib.util +import os +import subprocess +import sys +from pathlib import Path + +import numpy as np +import pytest + +from sp_validation import blinding +from sp_validation import sacc_io as sio +from sp_validation.cosmo_val import sacc_writers as sw + + +def _repo_root(): + return next( + p for p in Path(__file__).resolve().parents if (p / "pyproject.toml").exists() + ) + + +def _load_module(rel_path, name): + """Import a workflow module/script by file path (off the package path).""" + path = _repo_root() / rel_path + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +common = _load_module("workflow/common.py", "wf_common") +asm = _load_module("workflow/scripts/assemble_sacc.py", "assemble_sacc") + + +# --------------------------------------------------------------------------- # +# 1. common.py path helpers mirror sp_validation.blinding (drift guard) +# --------------------------------------------------------------------------- # +_STEMS = [ + "SP_v1.4.6.3_xi_reporting_minsep=1.0_maxsep=250.0_nbins=20_npatch=100", + "SP_v1.4.6.3_leak_corr_xi_integration", + "pseudo_cl_SP_v1.4.6.3_blind=A_powspace_nbins=32", + "pseudo_cl_SP_v1.4.6.3_leak_corr_blind=A_powspace_nbins=32", +] + + +@pytest.mark.parametrize("stem", _STEMS) +def test_blinded_path_mirrors_blinding_part_paths(stem): + part = f"/out/{stem}.sacc" + assert common.blinded_path(part) == blinding.part_paths(part)["blinded"] + + +def test_blind_state_paths_mirror_blinding_init_paths(): + version = "SP_v1.4.6.3_leak_corr" + common_paths = common.blind_state_paths(version) + ref = blinding.init_paths(common.blind_state_dir(version)) + assert common_paths == ref + + +@pytest.mark.parametrize( + "stem,expected", + [ + (_STEMS[0], "SP_v1.4.6.3"), + (_STEMS[1], "SP_v1.4.6.3_leak_corr"), + (_STEMS[2], "SP_v1.4.6.3"), + (_STEMS[3], "SP_v1.4.6.3_leak_corr"), + ], +) +def test_version_of_extracts_catalogue_version(stem, expected): + assert common.version_of(stem) == expected + + +def test_version_of_raises_without_version(): + with pytest.raises(ValueError, match="no catalogue version"): + common.version_of("cosebis_no_version_here") + + +def test_blindable_part_switches_on_run_type(monkeypatch): + part = "/out/SP_v1.4.6.3_xi_integration.sacc" + monkeypatch.setattr(common, "RUN_TYPE", "data") + assert common.blindable_part(part) == common.blinded_path(part) + monkeypatch.setattr(common, "RUN_TYPE", "mock") + assert common.blindable_part(part) == part + + +# --------------------------------------------------------------------------- # +# 2. Data-run fail-closed assembly (#252 terminal custody gate) +# --------------------------------------------------------------------------- # +META = {"catalogue_version": "vSYNTH", "npatch": 1} +# Two arbitrary-but-consistent hex stamps standing in for a real blind's +# sha256(seed) / config digest; the assembly only checks they agree across parts. +_COMMIT = "a" * 64 +_DIGEST = "b" * 64 + + +def _nz(): + return np.linspace(0.01, 2.0, 40), np.random.default_rng(0).uniform(0.1, 1.0, 40) + + +def _spd(n, seed): + a = np.random.default_rng(seed).normal(size=(n, n)) + return a @ a.T + n * np.eye(n) + + +def _data_parts(tmp_path, *, conceal, one_plaintext=False): + """Write the five per-statistic parts as ``type='data'``. + + ``conceal`` stamps every part with the shared blind (concealed=True). With + ``one_plaintext`` the ξ± reporting part is left unconcealed — a blinded / + plaintext mix the assembly must refuse. + """ + nz = {0: _nz()} + theta = np.geomspace(1.0, 100.0, 6) + + xi = sw.xi_to_sacc( + nz, META, theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" + ) + xi.add_covariance(_spd(len(xi.mean), 1)) + co = sw.cosebis_to_sacc( + nz, + META, + { + "En": np.arange(1, 6) * 1e-6, + "Bn": np.arange(1, 6) * 1e-7, + "cov": _spd(10, 3), + }, + (1.0, 100.0), + ) + eb_arrays = {k: np.arange(6) * (i + 1) * 1e-6 for i, k in enumerate(sio.PURE_KEYS)} + eb = sw.pure_eb_to_sacc(nz, META, theta, eb_arrays, covariance=_spd(36, 4)) + rho = {"theta": theta} + tau = {"theta": theta} + rng = np.random.default_rng(5) + for k in sw.RHO_K: + for s in ("p", "m"): + rho[f"rho_{k}_{s}"] = rng.normal(size=6) * 1e-6 + rho[f"varrho_{k}_{s}"] = rng.uniform(1e-14, 1e-13, 6) + for k in sw.TAU_K: + for s in ("p", "m"): + tau[f"tau_{k}_{s}"] = rng.normal(size=6) * 1e-6 + tau[f"vartau_{k}_{s}"] = rng.uniform(1e-14, 1e-13, 6) + rt = sw.rho_tau_to_sacc(nz, META, rho, tau) + + parts = {"xi_reporting": xi, "cosebis": co, "pure_eb": eb, "rho_tau": rt} + paths = {} + for name, part in parts.items(): + if conceal and not (one_plaintext and name == "xi_reporting"): + blinding._stamp_provenance(part, _COMMIT, "A", _DIGEST) + p = tmp_path / f"{name}.sacc" + sio.save(part, str(p), type="data") + paths[name] = str(p) + return paths + + +def test_data_assemble_fails_closed_on_unblinded_part(tmp_path): + """A data run refuses to assemble an unconcealed real part (fail closed).""" + paths = _data_parts(tmp_path, conceal=False) + with pytest.raises(ValueError, match="refusing to load an unblinded"): + asm.assemble_sacc( + "vSYNTH", paths, str(tmp_path / "vSYNTH.sacc"), placeholder_var=1.0 + ) + + +def test_data_assemble_passes_on_blinded_parts(tmp_path): + """With every part concealed under one blind, the data-run assembly succeeds + and stamps the shared commitment on the terminal file.""" + paths = _data_parts(tmp_path, conceal=True) + out = tmp_path / "vSYNTH.sacc" + s = asm.assemble_sacc("vSYNTH", paths, str(out), placeholder_var=1.0) + assert s.metadata["concealed"] is True + assert s.metadata["blind_commitment"] == _COMMIT + assert s.metadata["blind_config_digest"] == _DIGEST + # Round-trips through the fail-closed load gate without an escape hatch. + assert sio.load(str(out)).metadata["concealed"] is True + + +def test_data_assemble_refuses_blinded_plaintext_mix(tmp_path): + """A concealed ξ± beside a plaintext one is a custody violation — refuse.""" + paths = _data_parts(tmp_path, conceal=True, one_plaintext=True) + # The plaintext ξ± reporting part fails the load gate first (data + not + # concealed), so the mix can never even reach assembly. + with pytest.raises(ValueError, match="refusing to load an unblinded"): + asm.assemble_sacc( + "vSYNTH", paths, str(tmp_path / "vSYNTH.sacc"), placeholder_var=1.0 + ) + + +def test_assert_consistent_blind_rejects_divergent_commitments(tmp_path): + """Two ξ± parts blinded under different commitments must never combine.""" + nz = {0: _nz()} + theta = np.geomspace(1.0, 100.0, 6) + a = sw.xi_to_sacc( + nz, META, theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" + ) + b = sw.xi_to_sacc( + nz, META, theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="integration" + ) + blinding._stamp_provenance(a, _COMMIT, "A", _DIGEST) + blinding._stamp_provenance(b, "c" * 64, "A", _DIGEST) + with pytest.raises(ValueError, match="different blind commitments"): + blinding.assert_consistent_blind([a, b]) + + +# --------------------------------------------------------------------------- # +# 3. The blinding subgraph resolves in the cosmo_val DAG (candide-only) +# --------------------------------------------------------------------------- # +requires_candide_data = pytest.mark.skipif( + not Path("/n17data/cdaley/unions").exists(), + reason="candide-local workflow config/data (/n17data) absent — off-cluster", +) + + +@requires_candide_data +def test_blinding_subgraph_in_cosmo_val_dry_run(): + """A data-run cosmo_val assemble pulls blind_init + blind_part, and binds the + ξ± / pseudo-Cℓ parts to their *_blinded siblings.""" + env = os.environ | {"PYTHONNOUSERSITE": "1", "PYTHONUNBUFFERED": "1"} + env.pop("SNAKEMAKE_PROFILE", None) + result = subprocess.run( + [ + sys.executable, + "-m", + "snakemake", + "assemble_sacc_all", + "--dry-run", + "--cores", + "1", + "--configfile", + "config/config.yaml", + ], + cwd=_repo_root() / "papers/cosmo_val", + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=180, + check=False, + ) + assert result.returncode == 0, result.stdout + out = result.stdout + assert "rule blind_init:" in out, out + assert "rule blind_part:" in out, out + # assemble consumes the blinded ξ± reporting + integration and pseudo-Cℓ. + assert ( + "_xi_reporting_minsep=1.0_maxsep=250.0_nbins=20_npatch=100_blinded.sacc" in out + ) + assert "_xi_integration_blinded.sacc" in out + assert "_blind=A_powspace_nbins=32_blinded.sacc" in out From 251f4f51c3f8b9edaa7b2b0a854b1b08bc894b75 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 17 Aug 2026 20:21:43 +0200 Subject: [PATCH 43/47] Blind through the fork's vector core, and bind the draw scheme into custody MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two changes to blinding.py, both about the seam with Smokescreen. **The concealing factor is a vector, so ask for a vector.** `_concealing_factor` carved a sub-SACC per block (`s.copy()` + `keep_indices`, covariance and all), handed it to `ConcealDataVector`, applied the blind, and subtracted the result back off to recover the shift it wanted in the first place. The fork now exposes `concealing_factor(fiducial_params, shifts_dict, *, seed, theory_fn)` — the pure theory difference, no SACC, no data vector — so call that. The theory factories already read their layout off whatever SACC they are handed, so they run on the part directly and return a full-length vector, NaN outside their own block; the factor is that vector at the block's rows. `unblind_sacc` differenced the two theory vectors itself. It now goes through the same `_concealing_factor` call, so the shift added and the shift subtracted cannot drift apart. `_extract_block` is gone with its last caller. A backend that leaves a row of its *own* block unfilled would have shifted that point by NaN, silently. The slice is now checked finite and refuses instead. **A blind is (seed, config, draw scheme) — not (seed, config).** The fork versions its shift-draw semantics as `smokescreen.DRAW_SCHEME`: upstream's one global RNG stream over sorted keys is scheme 1, the fork's per-key `(seed, key)` RNG is scheme 2. Blind under one and unblind under the other and the same seed draws a *different* hidden cosmology — so the unblind subtracts a shift that was never added, leaving a smooth residual in the "revealed" vector while the seed hash, the config digest and the escrow equality check all still pass. It is the one blinding failure with no symptom. The scheme is now custody state. `blind_init` writes it into commitment.json; `_stamp_provenance` stamps `blind_draw_scheme` on every blinded file; and it is re-checked against the installed fork wherever a shift is drawn or subtracted — `_read_seed` (so a scheme change between blinding part 1 and part 2 is caught), `unblind_sacc` before any subtraction, `assert_consistent_blind` at the terminal, `stamp_concealed_passthrough`, and the CLI's seedless `verify`. A missing record fails closed: a blind whose scheme is unknown cannot be shown reproducible. Docstrings that promised same-(seed, config) reproducibility "forever", or per-key draw independence, now say what guarantees it. Closes review findings 1, 2 and 4 on PR #253. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01BRYg9fjMevgcsvgjh3KN6x --- scripts/blind_data_vector.py | 24 ++ src/sp_validation/blinding.py | 336 ++++++++++++++++------- src/sp_validation/tests/test_blinding.py | 186 ++++++++++++- 3 files changed, 433 insertions(+), 113 deletions(-) diff --git a/scripts/blind_data_vector.py b/scripts/blind_data_vector.py index 9b41109d..d9aeb502 100644 --- a/scripts/blind_data_vector.py +++ b/scripts/blind_data_vector.py @@ -113,6 +113,30 @@ def _verify(args): problems.append("blind_commitment does not match the committed sha256(seed)") if s.metadata.get("blind_config_digest") != commitment["config_digest"]: problems.append("blind_config_digest does not match the committed digest") + # The draw scheme is checked three ways: file ↔ commitment, and both against + # the installed fork. A scheme mismatch is the one blinding failure with no + # numerical symptom — the unblind would subtract a different shift than was + # added and every other check here would still pass. + installed = blinding.draw_scheme() + file_scheme = s.metadata.get("blind_draw_scheme") + committed = commitment.get("draw_scheme") + if file_scheme is None or committed is None: + problems.append( + "no draw-scheme record on the file and/or in the commitment — " + "this blind predates draw-scheme binding and cannot be verified " + "reproducible" + ) + elif int(file_scheme) != int(committed): + problems.append( + f"blind_draw_scheme {int(file_scheme)} does not match the committed " + f"draw_scheme {int(committed)}" + ) + elif int(file_scheme) != installed: + problems.append( + f"blind was drawn under Smokescreen DRAW_SCHEME={int(file_scheme)} " + f"but the installed fork implements DRAW_SCHEME={installed} — this " + "install cannot reproduce the shift" + ) if "seed_smokescreen" in s.metadata: problems.append("PLAINTEXT SEED LEAKED into file metadata (seed_smokescreen)") if problems: diff --git a/src/sp_validation/blinding.py b/src/sp_validation/blinding.py index 97a220f1..c2ff148f 100644 --- a/src/sp_validation/blinding.py +++ b/src/sp_validation/blinding.py @@ -12,20 +12,25 @@ **The fork is the concealment engine.** The hidden cosmology is drawn by the ``UNIONS-WL/Smokescreen`` fork's own CCL-native, per-key-independent - draw; each part goes through its ``ConcealDataVector(fiducial_params, - shifts_dict, sacc_data, *, seed, theory_fn)`` entry point. This module - supplies the two sp_validation-specific pieces: the three ``theory_fn`` - backends matching each part's row layout (reporting ξ±, integration ξ±, pseudo-Cℓ) - and the SACC handling around the concealed vectors. + draw. Blinding is a vector operation, so this module calls the fork's + vector core directly — ``smokescreen.concealing_factor(fiducial_params, + shifts_dict, *, seed, theory_fn)``, which returns + ``t(hidden) − t(fiducial)`` and never sees a SACC. This module supplies + the two sp_validation-specific pieces: the three ``theory_fn`` backends + matching each part's row layout (reporting ξ±, integration ξ±, pseudo-Cℓ) + and the SACC handling around the returned factors. **Envelope calibration.** The blinding intent is an amplitude smear of a chosen (S8, Ωm) box. The fork draws in CCL-native primitives, so :meth:`BlindingConfig.shifts_dict` maps the intended box to ``{sigma8, Omega_c}`` half-widths evaluated at the fiducial — - ``σ8 = S8/√(Ωm/0.3)``, ``Ω_c = Ωm − Ω_b − Ω_ν``. The same seed yields - the same hidden cosmology across all part passes (the fork's draw depends - only on ``(key, seed)``), so the blinded parts are mutually consistent by - construction. + ``σ8 = S8/√(Ωm/0.3)``, ``Ω_c = Ωm − Ω_b − Ω_ν``. Under the installed draw + scheme the deltas depend only on ``(key, seed, shift_distr)``, so one seed + yields one hidden cosmology across all part passes and the blinded parts + are mutually consistent by construction. That "only" is a property of the + *draw scheme*, not of the seed: :func:`draw_scheme` records which scheme a + blind was drawn under and every custody gate refuses an install that + disagrees (see **Custody** below). **Derived statistics are born blinded.** COSEBIs and pure-E/B are never touched by this module: the pipeline's own estimators @@ -37,17 +42,23 @@ **Custody: hash commitment, no keyholder.** :func:`blind_init` runs once per catalogue version: it draws an OS-entropy seed, publishes - ``sha256(seed)`` plus a canonical config digest as a repo-committable - ``commitment.json``, and encrypts the seed into a Fernet bundle - (``smokescreen.encryption``) — the plaintext seed is never written. + ``sha256(seed)``, a canonical config digest, and the installed fork's + ``DRAW_SCHEME`` as a repo-committable ``commitment.json``, and encrypts + the seed into a Fernet bundle (``smokescreen.encryption``) — the plaintext + seed is never written. The three together, not the seed alone, are what + reproduces a blind: seed and config fix *which* shift, the draw scheme + fixes *how* the seed becomes that shift. + Each :func:`blind_part` call reads that fixed state, conceals one part, escrows the part's true vector into its own encrypted bundle beside the - blinded output, and deletes the plaintext part. Terminal assembly - (:func:`sp_validation.sacc_io.gather`) calls - :func:`assert_consistent_blind` to fail closed unless every blindable - part carries the identical ``blind_commitment``. :func:`unblind_part` - verifies both hashes against the commitment *before* subtracting - anything, then restores the true part. + blinded output, and deletes the plaintext part. Every path that assembles + parts into the one-file product goes through + :func:`sp_validation.sacc_io.gather` — the production assembler is passed + *into* it rather than wrapped around it — and gather calls + :func:`assert_consistent_blind` to fail closed unless every blindable part + carries the identical ``blind_commitment``, ``blind_config_digest`` and + ``blind_draw_scheme``. :func:`unblind_part` verifies all three against the + commitment *before* subtracting anything, then restores the true part. """ import dataclasses @@ -89,8 +100,11 @@ def shifts_dict(self): ``ΔS8/√(Ωm_fid/0.3)`` in σ8 (at fixed Ωm), and a ΔΩm half-width maps one-to-one to Ω_c (Ω_b and Ω_ν are fixed). Exact enough for a blinding smear — the target is a characteristic amplitude, not a - precise (S8, Ωm) posterior. The fork draws each key independently as - ``U(fid − h, fid + h)``. + precise (S8, Ωm) posterior. Under the installed draw scheme the fork + draws each key independently as ``U(fid − h, fid + h)``, so adding or + resizing one key never moves another; :func:`draw_scheme` is what + makes that independence a guarantee rather than a hope, by refusing an + install whose draw semantics differ from the blind's. """ return { "sigma8": self.s8_half_width / np.sqrt(self.theory.Omega_m / 0.3), @@ -166,15 +180,68 @@ def seed_commitment(seed): return hashlib.sha256(seed.encode("utf-8")).hexdigest() +def draw_scheme(): + """The installed Smokescreen fork's shift-draw semantics version. + + ``smokescreen.DRAW_SCHEME`` is the fork's own version number for *how* a + seed becomes a set of parameter deltas — scheme 1 is upstream DESC's one + global RNG stream consumed over the sorted keys, scheme 2 (this fork) a + per-key RNG derived from ``(seed, key)``. Two installs that agree on + ``(seed, config)`` but disagree on this number draw *different* hidden + cosmologies from the same inputs. + + That is the one blinding failure with no loud symptom: a blind added under + one scheme and subtracted under another leaves a residual smooth + cosmological shift in the "unblinded" vector, which every hash, digest and + escrow check would still pass. So the scheme is bound into the blind's + custody state — ``commitment.json`` and every blinded file's + ``blind_draw_scheme`` — and re-checked against this function wherever a + shift is drawn or subtracted (:func:`_assert_draw_scheme`). + """ + from smokescreen import DRAW_SCHEME + + return int(DRAW_SCHEME) + + +def _assert_draw_scheme(recorded, what): + """Fail closed unless ``recorded`` is the installed fork's draw scheme. + + ``what`` names the surface the scheme was read from, for the message. + A missing record (``None``) is a failure, not a pass: a blind whose scheme + is unknown cannot be shown to be reproducible by this install. + """ + installed = draw_scheme() + if recorded is None: + raise ValueError( + f"{what} carries no draw-scheme record — refusing to proceed. " + f"It predates draw-scheme binding, so there is no way to tell " + f"whether the installed Smokescreen (DRAW_SCHEME={installed}) " + f"reproduces the shift it was blinded with." + ) + if int(recorded) != installed: + raise ValueError( + f"{what} was drawn under Smokescreen DRAW_SCHEME={int(recorded)} " + f"but the installed fork implements DRAW_SCHEME={installed} — " + f"refusing to proceed. The same seed draws a different hidden " + f"cosmology under a different scheme, so this install would " + f"subtract the wrong shift and pass every other check. Install " + f"the Smokescreen the blind was made with." + ) + + def hidden_params(seed, config): """The hidden CCL parameter point the fork realizes for ``(seed, config)``. Re-runs the fork's own draw (``smokescreen.param_shifts.draw_param_shifts`` — per-key-independent, local RNG) on the calibrated envelope and overlays - the deltas on the fiducial, exactly as ``ConcealDataVector`` does - internally. Deterministic: same ``(seed, config)`` ⇒ same hidden point, - forever — the reproducibility contract unblinding relies on, and what - makes every part share one hidden cosmology under one seed. + the deltas on the fiducial, exactly as + ``smokescreen.concealing_factor`` does internally. Deterministic under a + fixed draw scheme: same ``(seed, config)`` ⇒ same hidden point on any + install whose :func:`draw_scheme` matches — the reproducibility contract + unblinding relies on, and what makes every part share one hidden cosmology + under one seed. This is an introspection helper (what *was* the hidden + cosmology, once revealed); the blinding path itself never calls it, and so + does not check the scheme here — every gate that acts on a shift does. """ from smokescreen.param_shifts import draw_param_shifts @@ -315,7 +382,7 @@ def theory_fn(params): # --------------------------------------------------------------------------- # -# Extract → conceal → merge, per block +# The concealing factor, per block # --------------------------------------------------------------------------- # def _blindable_blocks(s): """The blindable blocks of a SACC as ``(name, indices, factory)``. @@ -323,10 +390,9 @@ def _blindable_blocks(s): Works identically on a standalone part (which carries exactly one block) and on the assembled one-file product (whose integration rows are selected by the ``grid`` tag — the layout contract's per-block tag selection). - ``indices`` are each block's recorded row indices (ascending, so the - extracted sub-SACC preserves row order); ``factory`` builds the matching - ``theory_fn`` from the extracted sub-SACC. Blocks absent from the file - are simply not listed. + ``indices`` are each block's recorded row indices (ascending); ``factory`` + builds the matching ``theory_fn`` from the SACC those indices point into. + Blocks absent from the file are simply not listed. """ blocks = [] for grid in ("reporting", "integration"): @@ -345,43 +411,54 @@ def _blindable_blocks(s): return blocks -def _extract_block(s, indices): - """Extract rows ``indices`` (ascending) into a sub-SACC, order preserved. - - Deliberately index-based rather than ``sacc_io.extract`` / - ``update_statistic`` (which select and merge by ``(data_type, tracers, - tags)``): Smokescreen's ``ConcealDataVector`` aligns its ``theory_fn`` - output to the sub-SACC's ``mean`` element-for-element by row position, so - the block must be carved and written back by contiguous integer index, not - by tag-matching. This is the one place the row-index path is load-bearing. - """ - sub = s.copy() - sub.keep_indices(np.asarray(indices, dtype=int)) - return sub - - def _concealing_factor(s, indices, factory, config, seed): - """The fork-computed additive concealing factor for one block. + """The fork-computed additive concealing factor for one block of ``s``. + + Blinding is a vector operation, so this goes straight to the fork's vector + core: ``smokescreen.concealing_factor`` draws the hidden deltas from + ``seed``, overlays them on the fiducial, evaluates the block's + ``theory_fn`` at both points and differences them. No data vector and no + SACC reach the fork — nothing is carved out of ``s``, and ``s`` itself is + not modified. + + The factory reads its layout off ``s`` directly, so the returned vector is + full-length: a block's ``theory_fn`` fills only its own rows and leaves + every other row NaN. Slicing to ``indices`` drops those NaNs by + construction; the finite check then proves the converse — that the + factory filled *all* of this block's rows. A row the block claims but the + factory cannot cover (a pair carrying ξ− without ξ+, say) would otherwise + write NaN into the data vector, silently. + + Both :func:`blind_sacc` and :func:`unblind_sacc` reach the fork through + this one function, so the added and the subtracted shift cannot drift + apart. - Extracts the block into a sub-SACC containing exactly the rows the - ``theory_fn`` spans (the fork's length guard enforces the agreement), - drives ``ConcealDataVector`` with the fiducial point, the calibrated - envelope, and the block's ``theory_fn``, and returns the factor - ``t(hidden) − t(fiducial)`` aligned to ``indices``. + Returns + ------- + np.ndarray + ``t(hidden) − t(fiducial)``, aligned to ``indices``. """ - from smokescreen import ConcealDataVector - - sub = _extract_block(s, indices) - smoke = ConcealDataVector( - config.theory.ccl_params(), - config.shifts_dict(), - sub, - seed=seed, - theory_fn=factory(sub, config.theory), + from smokescreen import concealing_factor + + full = np.asarray( + concealing_factor( + config.theory.ccl_params(), + config.shifts_dict(), + seed=seed, + theory_fn=factory(s, config.theory), + factor_type="add", + ), + dtype=float, ) - smoke.calculate_concealing_factor(factor_type="add") - concealed = smoke.apply_concealing_to_likelihood_datavec() - return np.asarray(concealed, dtype=float) - np.asarray(sub.mean, dtype=float) + factor = full[indices] + if not np.all(np.isfinite(factor)): + raise ValueError( + f"the theory backend left {int(np.sum(~np.isfinite(factor)))} of " + f"{len(indices)} blindable rows unfilled — refusing to blind " + "(these rows would be shifted by NaN). The block's row layout is " + "not fully covered by its theory_fn." + ) + return factor def _set_values(s, indices, values): @@ -399,11 +476,13 @@ def blind_sacc(part, seed, config=None, label="A", log=print): """Return a blinded copy of a part SACC (covariance and tags untouched). Per blindable block present (a standalone part carries exactly one — - reporting ξ±, integration ξ±, or pseudo-Cℓ_EE): extract the block into a sub-SACC, - conceal through the fork, write the shifted values back at their - recorded indices (row order preserved). Provenance is stamped and any - leaked seed key stripped. A file with no blindable block (e.g. a ρ/τ - diagnostic part) is refused loudly — it should never see a blind call. + reporting ξ±, integration ξ±, or pseudo-Cℓ_EE): ask the fork for the + block's concealing factor (:func:`_concealing_factor`) and add it at the + block's recorded indices. Row order, tags, n(z) and covariance are + untouched — only ``value`` changes, and only on blindable rows. + Provenance is stamped and any leaked seed key stripped. A file with no + blindable block (e.g. a ρ/τ diagnostic part) is refused loudly — it should + never see a blind call. """ config = config or BlindingConfig() if _concealed(part): @@ -428,17 +507,20 @@ def blind_sacc(part, seed, config=None, label="A", log=print): def unblind_sacc(blinded, seed, config=None, log=print): """Recover the true part SACC from a blinded one + the revealed ``seed``. - Verifies ``sha256(seed)`` and the config digest against the stamped - metadata (loud failure on either mismatch — verification precedes - subtraction), recomputes each block's shift from the seed through the - same backends, and subtracts it. Works on a standalone part or on the - assembled file (integration rows selected by the ``grid`` tag). Derived - statistics, if present (assembled file), are *not* recomputed here — the - pipeline's own estimators re-derive them from the unblinded integration ξ±. + Verifies the stamped draw scheme against the installed fork, then + ``sha256(seed)`` and the config digest against the stamped metadata (loud + failure on any of the three — verification precedes subtraction), then + recomputes each block's shift through :func:`_concealing_factor`, the same + call :func:`blind_sacc` added it with, and subtracts it. Works on a + standalone part or on the assembled file (integration rows selected by the + ``grid`` tag). Derived statistics, if present (assembled file), are *not* + recomputed here — the pipeline's own estimators re-derive them from the + unblinded integration ξ±. """ config = config or BlindingConfig() if not _concealed(blinded): raise ValueError("file is not concealed — nothing to unblind") + _assert_draw_scheme(blinded.metadata.get("blind_draw_scheme"), "this blinded file") if seed_commitment(seed) != blinded.metadata["blind_commitment"]: raise ValueError( "seed does not match blind_commitment — refusing to unblind " @@ -451,34 +533,46 @@ def unblind_sacc(blinded, seed, config=None, log=print): "added)" ) - hidden = hidden_params(seed, config) - fiducial = config.theory.ccl_params() part = blinded.copy() for name, indices, factory in _blindable_blocks(blinded): - theory = factory(_extract_block(blinded, indices), config.theory) - factor = theory(hidden) - theory(fiducial) + factor = _concealing_factor(blinded, indices, factory, config, seed) _set_values(part, indices, np.asarray(part.mean)[indices] - factor) log(f"[unblind] {name}: subtracted {len(indices)} shifts") - for key in ("concealed", "blind", "blind_commitment", "blind_config_digest"): + for key in ( + "concealed", + "blind", + "blind_commitment", + "blind_config_digest", + "blind_draw_scheme", + ): part.metadata.pop(key, None) return part -def _stamp_provenance(s, commitment, label, config_digest): +def _stamp_provenance(s, commitment, label, config_digest, scheme=None): """Stamp the blind's public provenance; strip any leaked seed. ``blind_commitment`` (sha256 of the seed) ties the file to its blind without revealing it; ``blind_config_digest`` pins the envelope + - fiducial the shift was drawn against; ``concealed``/``blind`` mark the - file. ``seed_smokescreen`` — the raw seed Smokescreen's own writer would - stamp — is popped defensively: the seed must never ride a kept file. + fiducial the shift was drawn against; ``blind_draw_scheme`` pins the + Smokescreen draw semantics that turned the seed into that shift (see + :func:`draw_scheme`); ``concealed``/``blind`` mark the file. The three + together are what an unblind must reproduce. ``seed_smokescreen`` — the + raw seed upstream Smokescreen's writer would stamp — is popped + defensively: the seed must never ride a kept file. + + ``scheme`` defaults to the installed fork's, which is the right answer + whenever this call is stamping a blind that was just computed. Callers + propagating an existing blind's provenance pass that blind's recorded + scheme instead. """ s.metadata.pop("seed_smokescreen", None) s.metadata["concealed"] = True s.metadata["blind"] = label s.metadata["blind_commitment"] = commitment s.metadata["blind_config_digest"] = config_digest + s.metadata["blind_draw_scheme"] = draw_scheme() if scheme is None else int(scheme) def stamp_concealed_passthrough(s, commitment_path): @@ -489,12 +583,16 @@ def stamp_concealed_passthrough(s, commitment_path): values are blind by construction and only the provenance stamp is missing. ρ/τ carries no cosmological vector at all — the stamp merely clears it for assembly under the blind (values unchanged either way). Both cases need the - four custody keys the fail-closed load gate and :func:`assert_consistent_blind` + custody keys the fail-closed load gate and :func:`assert_consistent_blind` check: ``concealed``, ``blind`` (label), ``blind_commitment`` - (= ``seed_sha256``), ``blind_config_digest``. This reads those from the - version's ``commitment.json`` (written by :func:`blind_init`) and stamps them - via :func:`_stamp_provenance`, so a pass-through part shares the exact same - ``(commitment, digest)`` custody state as the blinded ξ±/pseudo-Cℓ parts. + (= ``seed_sha256``), ``blind_config_digest``, ``blind_draw_scheme``. This + reads those from the version's ``commitment.json`` (written by + :func:`blind_init`) and stamps them via :func:`_stamp_provenance`, so a + pass-through part shares the exact same custody state as the blinded + ξ±/pseudo-Cℓ parts. The committed draw scheme is checked against the + installed fork first: a pass-through part is blind because it was derived + from parts blinded under that scheme, so stamping it from an install that + draws differently would mint a custody claim this install cannot honour. Unlike :func:`blind_sacc`, this shifts nothing and does not require a blindable block — it is the seam for parts blinded (or made blind-irrelevant) @@ -502,8 +600,15 @@ def stamp_concealed_passthrough(s, commitment_path): """ with open(commitment_path, encoding="utf-8") as f: commitment = json.load(f) + _assert_draw_scheme( + commitment.get("draw_scheme"), f"the blind at {commitment_path}" + ) _stamp_provenance( - s, commitment["seed_sha256"], commitment["label"], commitment["config_digest"] + s, + commitment["seed_sha256"], + commitment["label"], + commitment["config_digest"], + scheme=commitment["draw_scheme"], ) return s @@ -518,9 +623,12 @@ def assert_consistent_blind(parts): part is *blindable* if it carries a blindable block (ξ± or pseudo-Cℓ_EE); ρ/τ diagnostic and covariance-only parts are exempt. Fails closed — ``ValueError`` — if blinded and plaintext blindable parts are mixed, or - if two parts carry different ``blind_commitment``/``blind_config_digest`` - (they were blinded under different seeds or configs and must never be - combined). The consistency key is ``(commitment, digest)`` only — the + if two parts carry different + ``blind_commitment``/``blind_config_digest``/``blind_draw_scheme`` (they + were blinded under different seeds, configs or draw semantics and must + never be combined), or if the shared draw scheme is not the installed + fork's (this install could not unblind what it is about to assemble). + The consistency key is ``(commitment, digest, scheme)`` — the ``blind`` *label* is informational provenance, not custody state, so parts blinded under one seed+config but tagged with different ``--label`` values assemble cleanly (a distinct warning is logged, not a failure). @@ -538,8 +646,9 @@ def assert_consistent_blind(parts): ------- dict or None The shared blind metadata (``concealed``, ``blind``, - ``blind_commitment``, ``blind_config_digest``) for the gather to - stamp on the assembled file, or ``None`` when nothing is blinded. + ``blind_commitment``, ``blind_config_digest``, ``blind_draw_scheme``) + for the gather to stamp on the assembled file, or ``None`` when + nothing is blinded. """ blindable = [p for p in parts if _blindable_blocks(p)] concealed = [p for p in blindable if _concealed(p)] @@ -562,18 +671,26 @@ def assert_consistent_blind(parts): f"({len(concealed)} of {len(blindable)} blinded) — refusing to " "combine (a plaintext part beside blinded ones leaks the shift)" ) - # Custody state is (commitment, digest) only — the label is provenance. + # Custody state is (commitment, digest, scheme) — the label is provenance. + # `.get` on the scheme so a part predating scheme binding reads as None and + # fails at _assert_draw_scheme with its explanation, not with a KeyError. stamps = { - (p.metadata["blind_commitment"], p.metadata["blind_config_digest"]) + ( + p.metadata["blind_commitment"], + p.metadata["blind_config_digest"], + p.metadata.get("blind_draw_scheme"), + ) for p in concealed } if len(stamps) != 1: raise ValueError( "parts carry different blind commitments — they were blinded " - "under different seeds or configs and must never be combined: " - + "; ".join(f"({c[:12]}…, {d[:12]}…)" for c, d in stamps) + "under different seeds, configs or draw schemes and must never be " + "combined: " + + "; ".join(f"({c[:12]}…, {d[:12]}…, scheme {v})" for c, d, v in stamps) ) - ((commitment, digest),) = stamps + ((commitment, digest, scheme),) = stamps + _assert_draw_scheme(scheme, "the blind these parts share") labels = sorted({p.metadata["blind"] for p in concealed}) if len(labels) != 1: warnings.warn( @@ -587,6 +704,7 @@ def assert_consistent_blind(parts): "blind": labels[0], "blind_commitment": commitment, "blind_config_digest": digest, + "blind_draw_scheme": int(scheme), } @@ -619,7 +737,9 @@ def blind_init(blind_dir, config=None, label="A", log=print): 1. Draw an OS-entropy seed (never written in plaintext, never returned). 2. Write ``commitment.json`` (repo-committable): ``sha256(seed)`` + the - canonical config digest + the blind label. + canonical config digest + the installed fork's draw scheme + the blind + label. Those first three are the full reproducibility statement — see + :func:`draw_scheme` for why the seed and config alone are not. 3. Encrypt the seed into a Fernet bundle (``smokescreen.encryption``); the temporary plaintext is deleted by the encryptor. @@ -650,6 +770,7 @@ def blind_init(blind_dir, config=None, label="A", log=print): "label": label, "seed_sha256": seed_commitment(seed), "config_digest": config.config_digest(), + "draw_scheme": draw_scheme(), } with open(paths["commitment"], "w", encoding="utf-8") as f: json.dump(commitment, f, indent=2, sort_keys=True) @@ -667,9 +788,12 @@ def blind_init(blind_dir, config=None, label="A", log=print): def _read_seed(blind_dir, config): """Decrypt the seed bundle and verify it against the commitment. - Both ``sha256(seed)`` and the config digest are checked before the seed - is handed to any caller — a tampered bundle or a drifted config fails - loud here, whether the caller is about to blind or to unblind. + ``sha256(seed)``, the config digest, and the committed draw scheme are all + checked before the seed is handed to any caller — a tampered bundle, a + drifted config or a Smokescreen that draws differently from the one that + fixed this blind all fail loud here, whether the caller is about to blind + or to unblind. The scheme check is what stops a re-blind of a later part + from landing a different hidden cosmology than the earlier parts got. Returns ------- @@ -680,6 +804,7 @@ def _read_seed(blind_dir, config): bundle = _read_encrypted_json(paths["bundle"], paths["key"]) with open(paths["commitment"], encoding="utf-8") as f: commitment = json.load(f) + _assert_draw_scheme(commitment.get("draw_scheme"), f"the blind in {blind_dir}") if seed_commitment(bundle["seed"]) != commitment["seed_sha256"]: raise ValueError( "bundle seed does not match the committed sha256(seed) — refusing " @@ -751,8 +876,9 @@ def blind_part(part_path, blind_dir, config=None, keep_input=False, log=print): def unblind_part(blinded_path, blind_dir, out_path, config=None, log=print): """Unblind one blinded part (or the assembled file), verifying first. - Decrypts the seed bundle and verifies both ``sha256(seed)`` and the - config digest against ``commitment.json`` (fail closed on either — + Decrypts the seed bundle and verifies ``sha256(seed)``, the config digest + and the draw scheme against ``commitment.json``, and the same three + against the blinded file's own stamps (fail closed on any — verification precedes subtraction), recomputes the part's shift from the seed and subtracts it (:func:`unblind_sacc`). **The seed-subtracted vector is the authority** — the seed plus its commitment is the custody diff --git a/src/sp_validation/tests/test_blinding.py b/src/sp_validation/tests/test_blinding.py index 5cd0f0f7..a3cd9880 100644 --- a/src/sp_validation/tests/test_blinding.py +++ b/src/sp_validation/tests/test_blinding.py @@ -13,10 +13,10 @@ per-part-at-birth architecture; derived statistics (COSEBIs, pure-E/B) are never stored in parts — they are computed downstream from the (blinded) integration ξ± through the pipeline seams ``b_modes.cosebis_from_xi`` / -``b_modes.pure_eb_from_xi``, exactly as the pipeline does. The fork's -``ConcealDataVector`` carries no data-vector consistency check (only the -length guard), so fixture ξ± values are smooth synthetic templates — no -theory fill is needed to blind. +``b_modes.pure_eb_from_xi``, exactly as the pipeline does. The blinding path +hands the fork no data vector at all (``smokescreen.concealing_factor`` is a +pure theory difference), so fixture ξ± values are smooth synthetic templates — +no theory fill is needed to blind. """ import json @@ -418,6 +418,7 @@ def test_provenance_metadata_contract(monkeypatch): assert blinded.metadata["blind"] == "B" assert blinded.metadata["blind_commitment"] == bd.seed_commitment("seed") assert blinded.metadata["blind_config_digest"] == c.config_digest() + assert blinded.metadata["blind_draw_scheme"] == bd.draw_scheme() assert "seed_smokescreen" not in blinded.metadata assert blinded.metadata["catalogue_version"] == "vTEST" @@ -454,6 +455,170 @@ def test_unblind_fails_closed_on_wrong_seed_or_config(monkeypatch): bd.unblind_sacc(s, "right-seed", log=_NOLOG) +# --------------------------------------------------------------------------- # +# The vector core: full-length theory in, block slice out (fast — fake backend) +# --------------------------------------------------------------------------- # +def _fake_factory(indices, hole=None): + """A backend that fills ``indices`` with ``sigma8 * arange`` and nothing else. + + Mimics the real backends' contract — a block's ``theory_fn`` reads its + layout off the SACC it is given and returns a full-length vector, NaN on + every row outside its own block — without importing CCL. ``hole`` leaves + one row of the block itself unfilled. + """ + + def factory(s, theory): + def theory_fn(params): + out = np.full(len(s.mean), np.nan) + out[indices] = params["sigma8"] * np.arange(len(indices)) + if hole is not None: + out[hole] = np.nan + return out + + return theory_fn + + return factory + + +def test_concealing_factor_slices_its_own_block_from_a_full_length_vector(): + """The factor is the theory difference at the block's rows, and the rows + the backend does not fill are never read. + + This is the shape of the whole path after the sub-SACC carving came out: + the backend is driven off the assembled SACC directly, returns a + full-length vector that is NaN everywhere but its own block, and + ``_concealing_factor`` returns exactly that block's rows. Checked against + :func:`hidden_params`, which reaches the same hidden point by an + independent route. + """ + ordered = ["xi_reporting", "cl", "rho_tau", "xi_integration"] + s = sio.gather([make_parts(nbins=1)[k] for k in ordered]) + blocks = bd._blindable_blocks(s) + assert len(blocks) == 3, "assembled file carries all three blindable blocks" + + cfg = bd.BlindingConfig() + delta = bd.hidden_params("seed", cfg)["sigma8"] - cfg.theory.ccl_params()["sigma8"] + assert delta != 0.0 + for _, indices, _ in blocks: + factor = bd._concealing_factor(s, indices, _fake_factory(indices), cfg, "seed") + assert factor.shape == (len(indices),) + assert np.allclose(factor, delta * np.arange(len(indices))) + + +def test_concealing_factor_refuses_a_row_its_backend_cannot_fill(): + """A NaN on a row the block *claims* is a layout the backend cannot cover. + + Slicing to the block drops the NaNs outside it by construction; this is + the converse guard, and it has to be explicit — without it a row the + backend silently skipped would be shifted by NaN, destroying that point + with no error anywhere. + """ + part = make_xi_part("reporting") + ((_, indices, _),) = bd._blindable_blocks(part) + with pytest.raises(ValueError, match="unfilled"): + bd._concealing_factor( + part, + indices, + _fake_factory(indices, hole=indices[0]), + bd.BlindingConfig(), + "seed", + ) + + +# --------------------------------------------------------------------------- # +# Draw-scheme binding: the blind is (seed, config, draw semantics) +# --------------------------------------------------------------------------- # +def test_draw_scheme_is_the_installed_fork_constant(): + """The recorded scheme is read from Smokescreen, not hardcoded here.""" + import smokescreen + + assert bd.draw_scheme() == int(smokescreen.DRAW_SCHEME) + assert isinstance(bd.draw_scheme(), int) + + +def test_assert_draw_scheme_message_names_both_versions(monkeypatch): + """A scheme mismatch says which scheme made the blind and which is installed.""" + monkeypatch.setattr(bd, "draw_scheme", lambda: 2) + bd._assert_draw_scheme(2, "the blind") # matching scheme is silent + with pytest.raises(ValueError, match=r"DRAW_SCHEME=1.*DRAW_SCHEME=2"): + bd._assert_draw_scheme(1, "the blind") + with pytest.raises(ValueError, match="no draw-scheme record"): + bd._assert_draw_scheme(None, "the blind") + + +def test_unblind_refuses_a_blind_drawn_under_another_scheme(monkeypatch): + """The finding this closes: a blind added under one draw scheme and + subtracted under another silently produces a wrong data vector, because + the seed hash, the config digest and the escrow check all still pass. The + scheme must be part of the custody state, checked before any subtraction. + """ + _patch_constant_factor(monkeypatch) + blinded = bd.blind_sacc(make_xi_part("reporting"), "seed", log=_NOLOG) + assert blinded.metadata["blind_draw_scheme"] == bd.draw_scheme() + + # everything else about this file is valid — only the draw semantics moved + monkeypatch.setattr( + bd, "draw_scheme", lambda: blinded.metadata["blind_draw_scheme"] + 1 + ) + with pytest.raises(ValueError, match="DRAW_SCHEME"): + bd.unblind_sacc(blinded, "seed", log=_NOLOG) + + +def test_unblind_refuses_a_file_predating_scheme_binding(monkeypatch): + """A blinded file with no scheme record fails closed, not open.""" + _patch_constant_factor(monkeypatch) + blinded = bd.blind_sacc(make_xi_part("reporting"), "seed", log=_NOLOG) + del blinded.metadata["blind_draw_scheme"] + with pytest.raises(ValueError, match="no draw-scheme record"): + bd.unblind_sacc(blinded, "seed", log=_NOLOG) + + +def test_unblind_strips_the_scheme_stamp_with_the_rest(monkeypatch): + _patch_constant_factor(monkeypatch) + blinded = bd.blind_sacc(make_xi_part("reporting"), "seed", log=_NOLOG) + part = bd.unblind_sacc(blinded, "seed", log=_NOLOG) + assert "blind_draw_scheme" not in part.metadata + + +def test_read_seed_fails_closed_on_scheme_drift(tmp_path, monkeypatch): + """blind_part reads the seed through _read_seed, so a scheme change between + blinding part 1 and part 2 is caught before the second part is shifted.""" + bd.blind_init(str(tmp_path), log=_NOLOG) + monkeypatch.setattr(bd, "draw_scheme", lambda: 99) + with pytest.raises(ValueError, match="DRAW_SCHEME"): + bd._read_seed(str(tmp_path), bd.BlindingConfig()) + + +def test_stamp_passthrough_carries_the_committed_scheme(tmp_path, monkeypatch): + """A pass-through part inherits the blind's scheme, and cannot be stamped + from an install that draws differently.""" + paths = bd.blind_init(str(tmp_path), log=_NOLOG) + s = bd.stamp_concealed_passthrough(make_rho_part(), paths["commitment"]) + assert s.metadata["blind_draw_scheme"] == bd.draw_scheme() + + monkeypatch.setattr(bd, "draw_scheme", lambda: 99) + with pytest.raises(ValueError, match="DRAW_SCHEME"): + bd.stamp_concealed_passthrough(make_rho_part(), paths["commitment"]) + + +def test_assert_consistent_blind_refuses_divergent_or_foreign_schemes(monkeypatch): + """Parts blinded under different schemes never assemble; nor does a set + that agrees with itself but not with the installed fork.""" + parts = make_parts(nbins=1, with_rho=False) + for p in parts.values(): + _stamp(p) + parts["cl"].metadata["blind_draw_scheme"] = bd.draw_scheme() + 1 + with pytest.raises(ValueError, match="different blind commitments"): + bd.assert_consistent_blind(list(parts.values())) + + parts = make_parts(nbins=1, with_rho=False) + for p in parts.values(): + _stamp(p) + p.metadata["blind_draw_scheme"] = bd.draw_scheme() + 1 + with pytest.raises(ValueError, match="DRAW_SCHEME"): + bd.assert_consistent_blind(list(parts.values())) + + # --------------------------------------------------------------------------- # # blind-init custody + assembly hash assertion (fast — encryption only) # --------------------------------------------------------------------------- # @@ -462,7 +627,7 @@ def test_blind_init_writes_commitment_and_encrypted_bundle_only(tmp_path): paths = bd.blind_init(str(tmp_path), log=_NOLOG) with open(paths["commitment"], encoding="utf-8") as f: commitment = json.load(f) - assert set(commitment) == {"label", "seed_sha256", "config_digest"} + assert set(commitment) == {"label", "seed_sha256", "config_digest", "draw_scheme"} assert len(commitment["seed_sha256"]) == 64 assert commitment["config_digest"] == bd.BlindingConfig().config_digest() # exactly the three custody outputs, no plaintext bundle @@ -507,6 +672,7 @@ def test_assert_consistent_blind_shared_stamp(): "blind": "A", "blind_commitment": bd.seed_commitment("s"), "blind_config_digest": bd.BlindingConfig().config_digest(), + "blind_draw_scheme": bd.draw_scheme(), } @@ -681,8 +847,12 @@ def test_ac2_on_file_shift_equals_theory_difference_per_part(transfer_function): hiddens.append(hidden) fiducial = cfg.theory.ccl_params() ((block_name, indices, factory),) = bd._blindable_blocks(part) - theory = factory(bd._extract_block(part, indices), cfg.theory) - expected = theory(hidden) - theory(fiducial) + # Independent of the blinding path: the factory is driven directly off + # the part, at the hidden point recovered by hidden_params, and the two + # theory vectors are differenced here rather than by the fork. The + # factory fills only its own block, so slice to it. + theory = factory(part, cfg.theory) + expected = (theory(hidden) - theory(fiducial))[indices] actual = np.array(blinded.mean)[indices] - np.array(part.mean)[indices] gap = np.max(np.abs(actual - expected)) scale = np.max(np.abs(expected)) @@ -989,7 +1159,7 @@ def test_ac6_ac8_end_to_end_init_parts_gather_unblind(tmp_path): assert not np.array_equal(np.array(blinded.mean), np.array(parts[name].mean)) with open(init["commitment"], encoding="utf-8") as f: commitment = json.load(f) - assert set(commitment) == {"label", "seed_sha256", "config_digest"} + assert set(commitment) == {"label", "seed_sha256", "config_digest", "draw_scheme"} blinded_parts = {n: sio.load(p["blinded"]) for n, p in out_paths.items()} for b in blinded_parts.values(): assert b.metadata["blind_commitment"] == commitment["seed_sha256"] From 7cbfe7300c2b709a447d66fb1e045d09623c5011 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 17 Aug 2026 20:22:12 +0200 Subject: [PATCH 44/47] One terminal seam: route the production assembly through sacc_io.gather MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `sacc_io.gather` was the documented custody terminal — it asserts one blind across all blindable parts, then assembles — but nothing in production called it. `workflow/scripts/assemble_sacc.py` inlined its own copy of the same wrapper (assert, assemble, stamp) around a *different* assembler, and blinding.py's docstrings claimed gather guarded the terminal. Two custody implementations, one of them dead, and a docstring true of neither. The difference between them is only the assembly: `merge` unions parts that already carry covariance; `assemble_analysis_sacc` rebuilds from an n(z) and requires one covariance block per part, which is what the production terminal needs (its ξ± and pseudo-Cℓ parts are born cov-less and have blocks injected first). That is a real difference and both are worth keeping — so the assembler becomes an argument, `gather(parts, metadata=None, assemble=None)`, and assemble_sacc.py passes its own in rather than wrapping its own guard around it. The custody wrapper now exists once and cannot be routed around. Tests: the production path gets the case the fail-closed load gate cannot catch — every part concealed, so every part loads, and only `assert_consistent_blind` can see that the install's draw scheme is not the blind's. Plus the terminal file's draw-scheme stamp. Closes review finding 3 on PR #253. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01BRYg9fjMevgcsvgjh3KN6x --- src/sp_validation/sacc_io.py | 43 ++++++++++++------- .../tests/test_blinding_wiring.py | 28 ++++++++++++ workflow/scripts/assemble_sacc.py | 35 +++++++-------- 3 files changed, 74 insertions(+), 32 deletions(-) diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index 0923326e..7dcb72a7 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -994,25 +994,35 @@ def load(path, *, allow_unblinded=False): # Everything above this banner is byte-identical to feat/sacc-2-sacc-io; only # gather() and its blind-custody call site live here. # --------------------------------------------------------------------------- # -def gather(parts, metadata=None): +def gather(parts, metadata=None, assemble=None): """Assemble standalone part SACCs into the one-file ``{version}.sacc``. Each part is an intermediate product as it came off its producing rule - (reporting ξ±, integration ξ±, pseudo-Cℓ, ρ/τ, …). Assembly itself — the - first-wins tracer union, in-order point concatenation with all tags - (bandpower windows included), and the block-diagonal covariance — is - exactly :func:`merge`, so gather delegates to it and adds only the one - thing merge cannot know about: **blind custody.** + (reporting ξ±, integration ξ±, pseudo-Cℓ, ρ/τ, …). Gather is **the** terminal + seam: every path that combines parts into the one-file product goes + through here, because this is where the one thing an assembler cannot know + about is enforced — **blind custody.** - **Blind custody (the module's one blind-aware call site):** + **Blind custody.** :func:`sp_validation.blinding.assert_consistent_blind` runs before the - merge — it fails closed unless every blindable part carries the identical - ``blind_commitment``/``blind_config_digest`` (or, when nothing is blinded, - every blindable part is declared ``type='mock'``). Its returned shared - stamp is written onto the assembled file so the one-file product carries - the blind it was built from; the blinded parts already carry those keys, - so merge preserves them and this stamp is a consistent (idempotent) - re-affirmation. + assembly — it fails closed unless every blindable part carries the + identical ``blind_commitment``/``blind_config_digest``/``blind_draw_scheme`` + (or, when nothing is blinded, every blindable part is declared + ``type='mock'``). Its returned shared stamp is written onto the assembled + file so the one-file product carries the blind it was built from; the + blinded parts already carry those keys, so the assembly preserves them and + this stamp is a consistent (idempotent) re-affirmation. + + **The assembly itself is the caller's.** Two exist and both are real: + :func:`merge` (the default) does the first-wins tracer union, in-order + point concatenation with all tags, and a covariance built from whatever the + parts carry — the right thing when parts are already covariance-bearing. + :func:`sp_validation.cosmo_val.sacc_writers.assemble_analysis_sacc` rebuilds + from a given n(z) + metadata and *requires* one covariance block per part — + the right thing for the production terminal, where the ξ± and pseudo-Cℓ + parts are born cov-less and have their blocks injected first. Passing the + assembler in, rather than duplicating the custody wrapper around each one, + is what keeps the guard un-bypassable. Parameters ---------- @@ -1020,6 +1030,9 @@ def gather(parts, metadata=None): The part SACCs, in the assembly (covariance) order. metadata : dict, optional Extra key/value pairs to store on the assembled file's metadata. + assemble : callable, optional + ``assemble(parts) -> sacc.Sacc``. Defaults to :func:`merge`. Bind any + further arguments (n(z), metadata) into the callable. Returns ------- @@ -1030,7 +1043,7 @@ def gather(parts, metadata=None): parts = list(parts) stamp = blinding.assert_consistent_blind(parts) - s = merge(parts) + s = (assemble or merge)(parts) for key, value in {**(metadata or {}), **(stamp or {})}.items(): s.metadata[key] = value return s diff --git a/src/sp_validation/tests/test_blinding_wiring.py b/src/sp_validation/tests/test_blinding_wiring.py index 2eac1988..706200cf 100644 --- a/src/sp_validation/tests/test_blinding_wiring.py +++ b/src/sp_validation/tests/test_blinding_wiring.py @@ -200,6 +200,34 @@ def test_data_assemble_refuses_blinded_plaintext_mix(tmp_path): ) +def test_data_assemble_runs_behind_the_custody_guard(tmp_path, monkeypatch): + """The custody guard is reached *through* the production assembly. + + Every part here is concealed, so every part clears the fail-closed load + gate — the earlier tests all stop there. Only + ``blinding.assert_consistent_blind`` can catch what is wrong with this + assembly: the install's Smokescreen draws shifts under a different scheme + than the one that made the blind, so it could never unblind what it is + about to write. That ``assemble_sacc`` raises is the check that it runs + through :func:`sacc_io.gather` like every other assembly path, rather than + reimplementing the custody wrapper around its own assembler. + """ + paths = _data_parts(tmp_path, conceal=True) + monkeypatch.setattr(blinding, "draw_scheme", lambda: 99) + with pytest.raises(ValueError, match="DRAW_SCHEME"): + asm.assemble_sacc( + "vSYNTH", paths, str(tmp_path / "vSYNTH.sacc"), placeholder_var=1.0 + ) + + +def test_data_assemble_stamps_the_draw_scheme_on_the_terminal_file(tmp_path): + """The assembled file carries the blind's draw scheme, like its parts.""" + paths = _data_parts(tmp_path, conceal=True) + out = tmp_path / "vSYNTH.sacc" + asm.assemble_sacc("vSYNTH", paths, str(out), placeholder_var=1.0) + assert sio.load(str(out)).metadata["blind_draw_scheme"] == blinding.draw_scheme() + + def test_assert_consistent_blind_rejects_divergent_commitments(tmp_path): """Two ξ± parts blinded under different commitments must never combine.""" nz = {0: _nz()} diff --git a/workflow/scripts/assemble_sacc.py b/workflow/scripts/assemble_sacc.py index 118ec5b8..4c7d41a1 100644 --- a/workflow/scripts/assemble_sacc.py +++ b/workflow/scripts/assemble_sacc.py @@ -7,9 +7,13 @@ Each per-statistic ``*.sacc`` *part* (written born-as-SACC by the mixins and the run_2pcf / generate_pseudo_cl scripts) holds one statistic. The assembler loads them in canonical order — ξ± reporting, pseudo-Cℓ, COSEBIs, pure-E/B, ρ/τ — and -calls :func:`sacc_writers.assemble_analysis_sacc`, which rebuilds one Sacc with a -single ``BlockDiagonalCovariance`` (point-insertion order = block order, -validated by ``sacc_io.assemble_covariance``). +hands them to :func:`sacc_io.gather` along with +:func:`sacc_writers.assemble_analysis_sacc` as the assembly, which rebuilds one +Sacc with a single ``BlockDiagonalCovariance`` (point-insertion order = block +order, validated by ``sacc_io.assemble_covariance``). Going through ``gather`` +rather than calling the assembler directly is what puts this path behind the +blind-custody gate (``blinding.assert_consistent_blind``) — there is one +terminal seam, not one per assembler. Covariance sourcing (the part-by-part decision) ----------------------------------------------- @@ -176,20 +180,17 @@ def assemble_sacc( ) if not parts: raise ValueError(f"no parts found for {version}: {part_paths}") - # Assembly-time custody assertion (#252): every blindable part (ξ± / pseudo-Cℓ - # EE) must share one blind commitment + config digest, or assembly fails - # closed — mixed blinded/plaintext parts and divergent-seed parts both raise. - # ρ/τ and covariance-only parts are exempt. Returns the shared blind stamp to - # carry onto the assembled file (or None for a fully-mock plaintext assembly). - from sp_validation import blinding - - shared = blinding.assert_consistent_blind(parts) - s = assemble_analysis_sacc(nz, metadata, parts) - if shared is not None: - # Stamp the assembled file with the shared blind so it, too, reads as - # concealed (its parts already carried the stamp into `metadata` above; - # this makes the custody state explicit and authoritative on the union). - s.metadata.update(shared) + # Assembly-time custody assertion (#252) runs inside sacc_io.gather, the one + # terminal seam: every blindable part (ξ± / pseudo-Cℓ EE) must share one + # blind commitment + config digest + draw scheme, or assembly fails closed — + # mixed blinded/plaintext parts and divergent-seed parts both raise. ρ/τ and + # covariance-only parts are exempt. Gather also stamps the shared blind onto + # the assembled file, so the union reads as concealed in its own right. + # The production assembly (n(z) + per-part covariance blocks) is passed in; + # gather's own default merge() is for parts that already carry covariance. + s = sacc_io.gather( + parts, assemble=lambda ordered: assemble_analysis_sacc(nz, metadata, ordered) + ) # Assembly preserves its parts' provenance: every part was written by # sacc_io.save and therefore carries the type=data|mock stamp in its # metadata (copied into the assembled file above). From 646349c97da6e1905938603a07a961fc879410e8 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 17 Aug 2026 21:42:14 +0200 Subject: [PATCH 45/47] Take the commitment from the fork, so it stops publishing the seed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The local sha256(seed) commitment leaked the blind. Smokescreen derives its per-key RNG base seed from the first 8 bytes of the *same* undomained digest, so the published commitment carried that base seed verbatim in its first 16 hex characters. With the commitment and the (public) fiducial config, anyone could redraw the hidden cosmology and subtract the shift — from the artifact introduced to protect it. seed_commitment now delegates to smokescreen.seed_commitment, which hashes COMMITMENT_DOMAIN + str(seed). One definition of the commitment across the stack, in a different hash domain from the seed normalizer. A regression test asserts the commitment never embeds _normalize_seed(seed). The commitment.json key is renamed seed_sha256 -> seed_commitment: it is no longer sha256 of the seed, and the old name claimed otherwise. No real blind exists, so nothing migrates. Also, four honesty fixes the same review found: - The NaN refusal fires on both blind and unblind (they share one call), so its message no longer says "refusing to blind". - The CLI module docstring describes the three-way commitment (seed, config, draw scheme), and _verify's docstring says that its draw-scheme check makes the result environment-dependent by design. - _verify loads with allow_unblinded=True, so its "file is not marked concealed" diagnostic is reachable instead of an uncaught load exception. - The module docstring states what a pre-draw_scheme blind costs: recoverable only by hand from escrow. None exists; the CLI offers no override. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01BRYg9fjMevgcsvgjh3KN6x --- scripts/blind_data_vector.py | 44 ++++++++++----- src/sp_validation/blinding.py | 68 +++++++++++++++--------- src/sp_validation/tests/test_blinding.py | 61 +++++++++++++++++---- workflow/rules/blinding.smk | 6 +-- 4 files changed, 129 insertions(+), 50 deletions(-) diff --git a/scripts/blind_data_vector.py b/scripts/blind_data_vector.py index d9aeb502..692ebd94 100644 --- a/scripts/blind_data_vector.py +++ b/scripts/blind_data_vector.py @@ -7,15 +7,20 @@ ``blind-init`` runs once per catalogue version: it draws an OS-entropy seed (never printed, never written in plaintext), publishes a repo-committable -``commitment.json`` (``sha256(seed)`` + config digest), and encrypts the seed -into a Fernet bundle. ``blind-part`` blinds one intermediate part SACC -(reporting ξ±, integration ξ±, or pseudo-Cℓ) under that fixed state, escrows -the true vector into a per-part encrypted bundle beside the blinded output, -and deletes the plaintext part. ``unblind`` verifies both commitment hashes -and restores a true part (bit-for-bit when the part's escrow bundle is beside -it); it also works on the assembled ``{version}.sacc`` (integration rows -selected by the ``grid`` tag). ``verify`` is a cheap, seedless check that a -blinded file matches a commitment. +``commitment.json`` (the seed commitment + the config digest + the installed +Smokescreen fork's draw scheme), and encrypts the seed into a Fernet bundle. +Those three, not the seed alone, are what reproduces a blind: seed and config +fix *which* shift, the draw scheme fixes *how* the seed becomes that shift. +``blind-part`` blinds one intermediate part SACC (reporting ξ±, integration ξ±, +or pseudo-Cℓ) under that fixed state, escrows the true vector into a per-part +encrypted bundle beside the blinded output, and deletes the plaintext part. +``unblind`` verifies all three commitments and restores a true part +(bit-for-bit when the part's escrow bundle is beside it); it also works on the +assembled ``{version}.sacc`` (integration rows selected by the ``grid`` tag). +``verify`` is a cheap, seedless check that a blinded file matches a commitment +— seedless, but not environment-independent: it also compares both recorded +draw schemes against the installed fork, so it reports a problem on a machine +whose Smokescreen draws differently even when file and commitment agree. :Authors: Cail Daley @@ -102,15 +107,28 @@ def _unblind(args): def _verify(args): - """Seedless check: blinded-file metadata ↔ commitment JSON.""" - s = sacc_io.load(args.blinded) + """Seedless check: blinded-file metadata ↔ commitment JSON ↔ this install. + + No seed is read, so this cannot confirm that the blind is *subtractable* — + only that the file's three custody stamps agree with the commitment, and + that the recorded draw scheme is the one this install implements. That last + check makes the result environment-dependent by design: a machine carrying a + different Smokescreen reports a problem even when file and commitment agree + perfectly, because that machine could not unblind the file. + + Loads with ``allow_unblinded=True``: the whole job here is to report on a + file's custody state, including the state where the file is not concealed at + all, which the fail-closed loader would otherwise raise on before any + diagnostic could be assembled. + """ + s = sacc_io.load(args.blinded, allow_unblinded=True) with open(args.commitment, encoding="utf-8") as f: commitment = json.load(f) problems = [] if not s.metadata.get("concealed"): problems.append("file is not marked concealed") - if s.metadata.get("blind_commitment") != commitment["seed_sha256"]: - problems.append("blind_commitment does not match the committed sha256(seed)") + if s.metadata.get("blind_commitment") != commitment["seed_commitment"]: + problems.append("blind_commitment does not match the committed seed commitment") if s.metadata.get("blind_config_digest") != commitment["config_digest"]: problems.append("blind_config_digest does not match the committed digest") # The draw scheme is checked three ways: file ↔ commitment, and both against diff --git a/src/sp_validation/blinding.py b/src/sp_validation/blinding.py index c2ff148f..34b648d2 100644 --- a/src/sp_validation/blinding.py +++ b/src/sp_validation/blinding.py @@ -42,12 +42,13 @@ **Custody: hash commitment, no keyholder.** :func:`blind_init` runs once per catalogue version: it draws an OS-entropy seed, publishes - ``sha256(seed)``, a canonical config digest, and the installed fork's - ``DRAW_SCHEME`` as a repo-committable ``commitment.json``, and encrypts - the seed into a Fernet bundle (``smokescreen.encryption``) — the plaintext - seed is never written. The three together, not the seed alone, are what - reproduces a blind: seed and config fix *which* shift, the draw scheme - fixes *how* the seed becomes that shift. + the seed commitment (:func:`seed_commitment`), a canonical config digest, + and the installed fork's ``DRAW_SCHEME`` as a repo-committable + ``commitment.json``, and encrypts the seed into a Fernet bundle + (``smokescreen.encryption``) — the plaintext seed is never written. The + three together, not the seed alone, are what reproduces a blind: seed and + config fix *which* shift, the draw scheme fixes *how* the seed becomes that + shift. Each :func:`blind_part` call reads that fixed state, conceals one part, escrows the part's true vector into its own encrypted bundle beside the @@ -59,6 +60,15 @@ carries the identical ``blind_commitment``, ``blind_config_digest`` and ``blind_draw_scheme``. :func:`unblind_part` verifies all three against the commitment *before* subtracting anything, then restores the true part. + + Custody has no back door, including for its own history: a blind fixed + before the draw scheme was bound into ``commitment.json`` carries no + scheme record, and :func:`_assert_draw_scheme` refuses it ahead of every + seed read, so neither :func:`blind_part` nor :func:`unblind_part` will run + on it. Such a blind is recoverable only by hand, by decrypting its per-part + escrow bundles (:func:`_read_encrypted_json`) and reading ``true_mean`` + back out. No such blind exists — the scheme has been bound since before + the first real blind — and the CLI deliberately offers no override. """ import dataclasses @@ -125,7 +135,7 @@ def config_digest(self): digests and deny a legitimate unblind. Python's ``json`` then emits each float via its shortest round-trip ``repr``, so two runs of the same config produce byte-identical digests. Checked (with - ``sha256(seed)``) at unblind, so a wrong envelope or a mismatched + the seed commitment) at unblind, so a wrong envelope or a mismatched P(k) recipe cannot silently subtract a wrong shift. """ payload = { @@ -172,12 +182,22 @@ def from_overrides(cls, overrides): # Custody primitives # --------------------------------------------------------------------------- # def seed_commitment(seed): - """Public commitment for a seed: its sha256 hex digest. + """Public commitment for a seed: the fork's domain-separated sha256 digest. Safe to publish and commit to the repo — it ties a blinded file to its blind without revealing the seed, and lets unblind refuse a wrong seed. + + This delegates to :func:`smokescreen.seed_commitment` so the commitment has + exactly one definition across the blinding stack. That function hashes + ``smokescreen.COMMITMENT_DOMAIN + str(seed)``, *not* the bare seed, and the + domain prefix is load-bearing: the fork derives the RNG base seed from the + bare sha256 of the same string, so an undomained commitment would publish + that base seed verbatim in its first 16 hex characters and the blind would + be recoverable from the artifact meant to protect it. """ - return hashlib.sha256(seed.encode("utf-8")).hexdigest() + from smokescreen import seed_commitment as _fork_seed_commitment + + return _fork_seed_commitment(seed) def draw_scheme(): @@ -454,9 +474,9 @@ def _concealing_factor(s, indices, factory, config, seed): if not np.all(np.isfinite(factor)): raise ValueError( f"the theory backend left {int(np.sum(~np.isfinite(factor)))} of " - f"{len(indices)} blindable rows unfilled — refusing to blind " - "(these rows would be shifted by NaN). The block's row layout is " - "not fully covered by its theory_fn." + f"{len(indices)} blindable rows unfilled — refusing to apply the " + "concealing factor (these rows would be shifted by NaN). The " + "block's row layout is not fully covered by its theory_fn." ) return factor @@ -508,7 +528,7 @@ def unblind_sacc(blinded, seed, config=None, log=print): """Recover the true part SACC from a blinded one + the revealed ``seed``. Verifies the stamped draw scheme against the installed fork, then - ``sha256(seed)`` and the config digest against the stamped metadata (loud + the seed commitment and the config digest against the stamped metadata (loud failure on any of the three — verification precedes subtraction), then recomputes each block's shift through :func:`_concealing_factor`, the same call :func:`blind_sacc` added it with, and subtracts it. Works on a @@ -585,7 +605,7 @@ def stamp_concealed_passthrough(s, commitment_path): assembly under the blind (values unchanged either way). Both cases need the custody keys the fail-closed load gate and :func:`assert_consistent_blind` check: ``concealed``, ``blind`` (label), ``blind_commitment`` - (= ``seed_sha256``), ``blind_config_digest``, ``blind_draw_scheme``. This + (= ``seed_commitment``), ``blind_config_digest``, ``blind_draw_scheme``. This reads those from the version's ``commitment.json`` (written by :func:`blind_init`) and stamps them via :func:`_stamp_provenance`, so a pass-through part shares the exact same custody state as the blinded @@ -605,7 +625,7 @@ def stamp_concealed_passthrough(s, commitment_path): ) _stamp_provenance( s, - commitment["seed_sha256"], + commitment["seed_commitment"], commitment["label"], commitment["config_digest"], scheme=commitment["draw_scheme"], @@ -736,7 +756,7 @@ def blind_init(blind_dir, config=None, label="A", log=print): Runs once per catalogue version: 1. Draw an OS-entropy seed (never written in plaintext, never returned). - 2. Write ``commitment.json`` (repo-committable): ``sha256(seed)`` + the + 2. Write ``commitment.json`` (repo-committable): the seed commitment + the canonical config digest + the installed fork's draw scheme + the blind label. Those first three are the full reproducibility statement — see :func:`draw_scheme` for why the seed and config alone are not. @@ -768,7 +788,7 @@ def blind_init(blind_dir, config=None, label="A", log=print): seed = secrets.token_hex(16) commitment = { "label": label, - "seed_sha256": seed_commitment(seed), + "seed_commitment": seed_commitment(seed), "config_digest": config.config_digest(), "draw_scheme": draw_scheme(), } @@ -788,7 +808,7 @@ def blind_init(blind_dir, config=None, label="A", log=print): def _read_seed(blind_dir, config): """Decrypt the seed bundle and verify it against the commitment. - ``sha256(seed)``, the config digest, and the committed draw scheme are all + The seed commitment, the config digest, and the committed draw scheme are all checked before the seed is handed to any caller — a tampered bundle, a drifted config or a Smokescreen that draws differently from the one that fixed this blind all fail loud here, whether the caller is about to blind @@ -805,9 +825,9 @@ def _read_seed(blind_dir, config): with open(paths["commitment"], encoding="utf-8") as f: commitment = json.load(f) _assert_draw_scheme(commitment.get("draw_scheme"), f"the blind in {blind_dir}") - if seed_commitment(bundle["seed"]) != commitment["seed_sha256"]: + if seed_commitment(bundle["seed"]) != commitment["seed_commitment"]: raise ValueError( - "bundle seed does not match the committed sha256(seed) — refusing " + "bundle seed does not match the committed seed commitment — refusing " "to proceed" ) if config.config_digest() != commitment["config_digest"]: @@ -856,7 +876,7 @@ def blind_part(part_path, blind_dir, config=None, keep_input=False, log=print): paths["escrow"], { "label": commitment["label"], - "seed_sha256": commitment["seed_sha256"], + "seed_commitment": commitment["seed_commitment"], "true_mean": np.asarray(part.mean, dtype=float).tolist(), }, ) @@ -876,7 +896,7 @@ def blind_part(part_path, blind_dir, config=None, keep_input=False, log=print): def unblind_part(blinded_path, blind_dir, out_path, config=None, log=print): """Unblind one blinded part (or the assembled file), verifying first. - Decrypts the seed bundle and verifies ``sha256(seed)``, the config digest + Decrypts the seed bundle and verifies the seed commitment, the config digest and the draw scheme against ``commitment.json``, and the same three against the blinded file's own stamps (fail closed on any — verification precedes subtraction), recomputes the part's shift from the @@ -885,7 +905,7 @@ def unblind_part(blinded_path, blind_dir, out_path, config=None, log=print): root of trust, and the returned vector is what the seed math produced. When the part's escrow bundle exists beside the blinded file it serves - two subordinate roles, and only after its stored ``seed_sha256`` is + two subordinate roles, and only after its stored ``seed_commitment`` is verified to match the same commitment (so an escrow from a different blind can never be trusted): (1) a *tighter equality check* — the seed subtraction and the escrowed truth must agree to ``1e-6`` relative or the @@ -910,7 +930,7 @@ def unblind_part(blinded_path, blind_dir, out_path, config=None, log=print): escrow = part_paths(unblinded_stem + ext) if os.path.exists(escrow["escrow"]): bundle = _read_encrypted_json(escrow["escrow"], escrow["escrow_key"]) - if bundle.get("seed_sha256") != commitment["seed_sha256"]: + if bundle.get("seed_commitment") != commitment["seed_commitment"]: raise ValueError( "escrow bundle beside the blinded file was written under a " "different seed than the commitment — refusing to trust it " diff --git a/src/sp_validation/tests/test_blinding.py b/src/sp_validation/tests/test_blinding.py index a3cd9880..6d808895 100644 --- a/src/sp_validation/tests/test_blinding.py +++ b/src/sp_validation/tests/test_blinding.py @@ -334,14 +334,45 @@ def test_theory_config_ccl_params_exact_keyset(): # --------------------------------------------------------------------------- # # Commitment + the fork's draw (fast; smokescreen import is light) # --------------------------------------------------------------------------- # -def test_commitment_is_sha256_of_seed(): +def test_commitment_is_the_forks_domain_separated_digest(): + """One definition of the commitment, and it is the fork's.""" import hashlib + import smokescreen + seed = "the-secret" - assert bd.seed_commitment(seed) == hashlib.sha256(seed.encode()).hexdigest() + assert bd.seed_commitment(seed) == smokescreen.seed_commitment(seed) + assert ( + bd.seed_commitment(seed) + == hashlib.sha256( + smokescreen.COMMITMENT_DOMAIN + seed.encode("utf-8") + ).hexdigest() + ) assert bd.seed_commitment("right") != bd.seed_commitment("wrong") +def test_commitment_does_not_embed_the_rng_seed(): + """The published commitment must not carry the effective RNG seed. + + The fork derives the base seed for its per-key RNG from the *undomained* + sha256 of the seed string, taking the digest's first 8 bytes. A commitment + hashed over the bare seed would therefore publish that base seed verbatim in + its first 16 hex characters, and anyone holding the (public) commitment and + the (public) fiducial config could redraw the hidden cosmology and subtract + the blind. The domain prefix is what breaks that identity — this is the + regression guard for it. + """ + import secrets + + from smokescreen.param_shifts import _normalize_seed + + # The third seed is drawn exactly as blind_init draws a production one. + for seed in ("my_secret_seed", "the-secret", secrets.token_hex(16)): + commitment = bd.seed_commitment(seed) + assert int(commitment[:16], 16) != _normalize_seed(seed) + assert str(_normalize_seed(seed)) not in commitment + + def test_hidden_params_deterministic_and_in_envelope(): """Same (seed, config) ⇒ same hidden point; draws respect the envelope.""" import secrets @@ -627,8 +658,13 @@ def test_blind_init_writes_commitment_and_encrypted_bundle_only(tmp_path): paths = bd.blind_init(str(tmp_path), log=_NOLOG) with open(paths["commitment"], encoding="utf-8") as f: commitment = json.load(f) - assert set(commitment) == {"label", "seed_sha256", "config_digest", "draw_scheme"} - assert len(commitment["seed_sha256"]) == 64 + assert set(commitment) == { + "label", + "seed_commitment", + "config_digest", + "draw_scheme", + } + assert len(commitment["seed_commitment"]) == 64 assert commitment["config_digest"] == bd.BlindingConfig().config_digest() # exactly the three custody outputs, no plaintext bundle assert {p.name for p in tmp_path.iterdir()} == { @@ -638,7 +674,7 @@ def test_blind_init_writes_commitment_and_encrypted_bundle_only(tmp_path): } # the decrypted seed matches the public commitment bundle = bd._read_encrypted_json(paths["bundle"], paths["key"]) - assert bd.seed_commitment(bundle["seed"]) == commitment["seed_sha256"] + assert bd.seed_commitment(bundle["seed"]) == commitment["seed_commitment"] # one-shot custody: a second init in the same dir refuses with pytest.raises(FileExistsError, match="refusing to overwrite"): bd.blind_init(str(tmp_path), log=_NOLOG) @@ -1159,10 +1195,15 @@ def test_ac6_ac8_end_to_end_init_parts_gather_unblind(tmp_path): assert not np.array_equal(np.array(blinded.mean), np.array(parts[name].mean)) with open(init["commitment"], encoding="utf-8") as f: commitment = json.load(f) - assert set(commitment) == {"label", "seed_sha256", "config_digest", "draw_scheme"} + assert set(commitment) == { + "label", + "seed_commitment", + "config_digest", + "draw_scheme", + } blinded_parts = {n: sio.load(p["blinded"]) for n, p in out_paths.items()} for b in blinded_parts.values(): - assert b.metadata["blind_commitment"] == commitment["seed_sha256"] + assert b.metadata["blind_commitment"] == commitment["seed_commitment"] # no plaintext json anywhere beside the blind outputs assert not list(tmp_path.rglob("*escrow.json")) assert not (blind_dir / "blind_seed.json").exists() @@ -1177,7 +1218,7 @@ def test_ac6_ac8_end_to_end_init_parts_gather_unblind(tmp_path): ], metadata={"catalogue_version": "vTEST", "type": "mock"}, ) - assert assembled.metadata["blind_commitment"] == commitment["seed_sha256"] + assert assembled.metadata["blind_commitment"] == commitment["seed_commitment"] # born-blinded derived statistics from the blinded parts differ from truth En_b, Bn_b, _ = _derive_downstream( blinded_parts["xi_reporting"], blinded_parts["xi_integration"] @@ -1185,10 +1226,10 @@ def test_ac6_ac8_end_to_end_init_parts_gather_unblind(tmp_path): assert not np.allclose(En_b, En_true, atol=0) # -- fail-closed on tampered commitment (AC6) --------------------------- -- - tampered = dict(commitment, seed_sha256="0" * 64) + tampered = dict(commitment, seed_commitment="0" * 64) with open(init["commitment"], "w", encoding="utf-8") as f: json.dump(tampered, f) - with pytest.raises(ValueError, match="sha256"): + with pytest.raises(ValueError, match="committed seed commitment"): bd.unblind_part( out_paths["xi_integration"]["blinded"], str(blind_dir), diff --git a/workflow/rules/blinding.smk b/workflow/rules/blinding.smk index bf79433f..a5f0dedb 100644 --- a/workflow/rules/blinding.smk +++ b/workflow/rules/blinding.smk @@ -30,9 +30,9 @@ rule blind_init: """Fix the blind for one catalogue version (blind-init). Draws an OS-entropy seed, writes the repo-committable commitment.json - (sha256(seed) + config digest) and the Fernet-encrypted seed bundle. Runs - once per version and refuses to overwrite existing state — a blind is a - one-shot custody event. + (seed commitment + config digest + the installed fork's draw scheme) and the + Fernet-encrypted seed bundle. Runs once per version and refuses to overwrite + existing state — a blind is a one-shot custody event. """ output: commitment=str(COSMO_VAL / "blind" / "{version}" / "commitment.json"), From 81af22795d1b98778bfdb167a691a81c77204a55 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 17 Aug 2026 21:42:54 +0200 Subject: [PATCH 46/47] Make the run type real, and stamp rho/tau for the blind MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two claims the code could not cash, both at the terminal assembly. The mock branch of assert_consistent_blind was unreachable. Every part writer hardcoded type="data", so a mock campaign's parts declared themselves data and a mock terminal assembly failed closed — only test fixtures ever built a type="mock" part. The run type is now plumbed to all six writers: CosmologyValidation carries run_type (set from config cosmo_val.type via cv_init_params) and its four writers stamp it; run_2pcf takes run_type= and a --run-type flag; run_2pcf_highres takes --run-type. papers/cosmo_val declares `type: data` explicitly, so the switch is visible where it is made. rho/tau was never stamped. The docstring said it passed through stamp_concealed_passthrough, but nothing called it for rho/tau — psf_systematics wrote the part with no commitment, so a data run's assemble_sacc died at sacc_io.load(allow_unblinded=False) on that part, before custody was ever checked. The rho_tau_stats rule now binds commitment.json on a data run and run_rho_tau passes it through calculate_rho_tau_stats, exactly as cv_cosebis and cv_pure_eb do. rho/tau carries no cosmological vector; the stamp shifts nothing and only clears the load gate. Tests: a mock campaign's parts assemble unconcealed, and the real rho/tau writer emits a part that loads without the escape hatch. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01BRYg9fjMevgcsvgjh3KN6x --- papers/cosmo_val/config/config.yaml | 9 ++ src/sp_validation/cosmo_val/core.py | 9 ++ src/sp_validation/cosmo_val/cosebis.py | 2 +- src/sp_validation/cosmo_val/pseudo_cl.py | 2 +- .../cosmo_val/psf_systematics.py | 36 ++++++- src/sp_validation/cosmo_val/pure_eb.py | 9 +- .../tests/test_blinding_wiring.py | 95 ++++++++++++++++++- workflow/common.py | 16 ++++ workflow/rules/twopoint.smk | 25 ++++- workflow/scripts/run_2pcf.py | 16 +++- workflow/scripts/run_2pcf_highres.py | 15 ++- workflow/scripts/run_rho_tau.py | 9 +- 12 files changed, 224 insertions(+), 19 deletions(-) diff --git a/papers/cosmo_val/config/config.yaml b/papers/cosmo_val/config/config.yaml index 232513c7..a2124302 100644 --- a/papers/cosmo_val/config/config.yaml +++ b/papers/cosmo_val/config/config.yaml @@ -19,6 +19,15 @@ versions: [ # CosmologyValidation suite parameters (cosmo_val.py) # --------------------------------------------------------------------------- cosmo_val: + # Campaign run type: "data" or "mock". One switch, two effects. It gates + # Smokescreen blind-at-birth (a data run blinds the three blindable parts and + # binds every ξ-derived consumer to the blinded siblings; a mock run bypasses + # blinding), and it is stamped as the SACC `type` of every part written. The + # two are the same decision: an unconcealed blindable part only enters the + # terminal assembly when it declares itself a mock, so a mock campaign must + # say so here for its parts to assemble at all. + type: data + # CosmologyValidation constructor npatch: 100 theta_min: 1.0 diff --git a/src/sp_validation/cosmo_val/core.py b/src/sp_validation/cosmo_val/core.py index 0c9273d8..401d52af 100644 --- a/src/sp_validation/cosmo_val/core.py +++ b/src/sp_validation/cosmo_val/core.py @@ -98,6 +98,13 @@ class CosmologyValidation( noise debiasing, making those realizations reproducible run-to-run. cosmo_params : dict, optional Cosmological parameters to pass to get_cosmo(). If None, uses Planck 2018. + run_type : {'data', 'mock'}, default 'data' + The campaign's run type, stamped as the SACC ``type`` of every part this + object writes. Custody state, not decoration: + ``blinding.assert_consistent_blind`` admits an unconcealed blindable part + into a terminal assembly only when it declares itself a mock, so a mock + campaign must be built with ``run_type='mock'`` for its parts to + assemble at all. Attributes ---------- @@ -224,6 +231,7 @@ def __init__( path_onecovariance=None, cosmo_params=None, blind=None, + run_type="data", ): self.rho_tau_method = rho_tau_method self.cov_estimate_method = cov_estimate_method @@ -253,6 +261,7 @@ def __init__( self.nside_mask = nside_mask self.path_onecovariance = path_onecovariance self.blind = blind + self.run_type = run_type assert self.cell_method in ["map", "catalog"], ( "cell_method must be 'map' or 'catalog'" diff --git a/src/sp_validation/cosmo_val/cosebis.py b/src/sp_validation/cosmo_val/cosebis.py index 8ff2b18d..178bd971 100644 --- a/src/sp_validation/cosmo_val/cosebis.py +++ b/src/sp_validation/cosmo_val/cosebis.py @@ -203,7 +203,7 @@ def cosebis_to_sacc_part( from ..blinding import stamp_concealed_passthrough stamp_concealed_passthrough(s, commitment_path) - sacc_io.save(s, out_path, type="data") + sacc_io.save(s, out_path, type=self.run_type) def plot_cosebis( self, diff --git a/src/sp_validation/cosmo_val/pseudo_cl.py b/src/sp_validation/cosmo_val/pseudo_cl.py index 2a55934a..cfaa349e 100644 --- a/src/sp_validation/cosmo_val/pseudo_cl.py +++ b/src/sp_validation/cosmo_val/pseudo_cl.py @@ -708,7 +708,7 @@ def pseudo_cl_to_sacc_part(self, version, out_path, ell_eff, cl_all, wsp): cl_all, wsp, ) - sacc_io.save(s, out_path, type="data") + sacc_io.save(s, out_path, type=self.run_type) def plot_pseudo_cl(self): """ diff --git a/src/sp_validation/cosmo_val/psf_systematics.py b/src/sp_validation/cosmo_val/psf_systematics.py index 6f72ac1f..b81d8b1c 100644 --- a/src/sp_validation/cosmo_val/psf_systematics.py +++ b/src/sp_validation/cosmo_val/psf_systematics.py @@ -25,7 +25,12 @@ class PSFSystematicsMixin: - def calculate_rho_tau_stats(self): + def calculate_rho_tau_stats(self, commitment_path=None): + """Measure ρ/τ statistics per version and write each version's SACC part. + + ``commitment_path`` is custody plumbing forwarded to + :meth:`rho_tau_to_sacc_part`; see it for what it does. + """ out_dir = f"{self.cc['paths']['output']}/rho_tau_stats" if not os.path.exists(out_dir): os.mkdir(out_dir) @@ -44,7 +49,12 @@ def calculate_rho_tau_stats(self): npatch=self.npatch, ) self.rho_tau_to_sacc_part( - ver, out_dir, base, rho_stat_handler, tau_stat_handler + ver, + out_dir, + base, + rho_stat_handler, + tau_stat_handler, + commitment_path=commitment_path, ) self.print_done("Rho stats finished") @@ -52,10 +62,24 @@ def calculate_rho_tau_stats(self): self._tau_stat_handler = tau_stat_handler def rho_tau_to_sacc_part( - self, version, out_dir, base, rho_stat_handler, tau_stat_handler + self, + version, + out_dir, + base, + rho_stat_handler, + tau_stat_handler, + commitment_path=None, ): """Write the ρ/τ SACC part for one version. + ρ/τ is a PSF diagnostic carrying no cosmological vector, so it is never + blinded — but on a data run it still has to pass the fail-closed load + gate that ``assemble_sacc`` opens every part through. + ``commitment_path`` is that seam: it stamps the part concealed under the + version's blind (:func:`blinding.stamp_concealed_passthrough`), values + untouched, so the assembly admits it. A mock run passes ``None`` and the + part stays unstamped. + ρ_0…ρ_5 autos and τ_0/τ_2/τ_5 leakage from the handler tables. The ``CovTauTh`` theory covariance ``cov_tau_{base}_th.npy`` — a ``(3·nbin, 3·nbin)`` plus-folded k-major block over ``{τ0, τ2, τ5}`` — is @@ -79,8 +103,12 @@ def rho_tau_to_sacc_part( tau_stat_handler.tau_stats, tau_cov_th=tau_cov_th, ) + if commitment_path is not None: + from ..blinding import stamp_concealed_passthrough + + stamp_concealed_passthrough(s, commitment_path) out_path = os.path.join(out_dir, f"rho_tau_{base}.sacc") - sacc_io.save(s, out_path, type="data") + sacc_io.save(s, out_path, type=self.run_type) @property def rho_stat_handler(self): diff --git a/src/sp_validation/cosmo_val/pure_eb.py b/src/sp_validation/cosmo_val/pure_eb.py index 5424adf8..f0b41642 100644 --- a/src/sp_validation/cosmo_val/pure_eb.py +++ b/src/sp_validation/cosmo_val/pure_eb.py @@ -135,7 +135,12 @@ def calculate_pure_eb( return results def pure_eb_to_sacc_part( - self, version, out_path, results, eb_override=None, commitment_path=None + self, + version, + out_path, + results, + eb_override=None, + commitment_path=None, ): """Write the pure-E/B SACC part (six ``PURE_KEYS`` blocks + covariance). @@ -166,7 +171,7 @@ def pure_eb_to_sacc_part( from ..blinding import stamp_concealed_passthrough stamp_concealed_passthrough(s, commitment_path) - sacc_io.save(s, out_path, type="data") + sacc_io.save(s, out_path, type=self.run_type) def plot_pure_eb( self, diff --git a/src/sp_validation/tests/test_blinding_wiring.py b/src/sp_validation/tests/test_blinding_wiring.py index 706200cf..5bf5b0d7 100644 --- a/src/sp_validation/tests/test_blinding_wiring.py +++ b/src/sp_validation/tests/test_blinding_wiring.py @@ -16,9 +16,11 @@ """ import importlib.util +import json import os import subprocess import sys +import types from pathlib import Path import numpy as np @@ -103,7 +105,7 @@ def test_blindable_part_switches_on_run_type(monkeypatch): # --------------------------------------------------------------------------- # META = {"catalogue_version": "vSYNTH", "npatch": 1} # Two arbitrary-but-consistent hex stamps standing in for a real blind's -# sha256(seed) / config digest; the assembly only checks they agree across parts. +# seed commitment / config digest; the assembly only checks they agree across parts. _COMMIT = "a" * 64 _DIGEST = "b" * 64 @@ -117,12 +119,14 @@ def _spd(n, seed): return a @ a.T + n * np.eye(n) -def _data_parts(tmp_path, *, conceal, one_plaintext=False): - """Write the five per-statistic parts as ``type='data'``. +def _data_parts(tmp_path, *, conceal, one_plaintext=False, run_type="data"): + """Write the five per-statistic parts, stamped ``type=run_type``. ``conceal`` stamps every part with the shared blind (concealed=True). With ``one_plaintext`` the ξ± reporting part is left unconcealed — a blinded / - plaintext mix the assembly must refuse. + plaintext mix the assembly must refuse. ``run_type='mock'`` writes the parts + as a mock campaign's producers do, which is the only way an unconcealed + blindable part is allowed through the assembly. """ nz = {0: _nz()} theta = np.geomspace(1.0, 100.0, 6) @@ -162,7 +166,7 @@ def _data_parts(tmp_path, *, conceal, one_plaintext=False): if conceal and not (one_plaintext and name == "xi_reporting"): blinding._stamp_provenance(part, _COMMIT, "A", _DIGEST) p = tmp_path / f"{name}.sacc" - sio.save(part, str(p), type="data") + sio.save(part, str(p), type=run_type) paths[name] = str(p) return paths @@ -228,6 +232,87 @@ def test_data_assemble_stamps_the_draw_scheme_on_the_terminal_file(tmp_path): assert sio.load(str(out)).metadata["blind_draw_scheme"] == blinding.draw_scheme() +def test_mock_assemble_succeeds_without_a_blind(tmp_path): + """A mock campaign assembles its plaintext parts — the gate's mock branch is + reachable from real producers, not only from test fixtures. + + Every part writer stamps the campaign's run type (CosmologyValidation's + ``run_type``, ``run_2pcf``'s ``run_type=``, ``run_2pcf_highres``'s + ``--run-type``), so a ``mock`` campaign's parts declare themselves mocks and + ``assert_consistent_blind`` lets them through unconcealed. The same parts + stamped ``type='data'`` fail closed — that is + ``test_data_assemble_fails_closed_on_unblinded_part``. + """ + paths = _data_parts(tmp_path, conceal=False, run_type="mock") + out = tmp_path / "vSYNTH.sacc" + s = asm.assemble_sacc("vSYNTH", paths, str(out), placeholder_var=1.0) + assert "concealed" not in s.metadata + # No escape hatch: a mock part is not gated by the fail-closed loader. + assert sio.load(str(out)).metadata["type"] == "mock" + + +def test_rho_tau_part_is_stamped_concealed_from_the_commitment(tmp_path): + """ρ/τ carries no cosmological vector, but a data run's assembly still opens + it through the fail-closed load gate — so its writer must stamp it. + + This drives the real writer (``PSFSystematicsMixin.rho_tau_to_sacc_part``) + with the commitment the ``rho_tau_stats`` rule binds on a data run, and + checks the emitted part loads without the escape hatch. Without the stamp a + data run's ``assemble_sacc`` dies on the ρ/τ part before custody is ever + checked. + """ + from sp_validation.cosmo_val.psf_systematics import PSFSystematicsMixin + + blind_dir = tmp_path / "blind" + blind_dir.mkdir() + commitment = blinding.blind_init(str(blind_dir), log=lambda *_: None)["commitment"] + theta = np.geomspace(1.0, 100.0, 6) + rng = np.random.default_rng(11) + rho = {"theta": theta} + tau = {"theta": theta} + for k in sw.RHO_K: + for sign in ("p", "m"): + rho[f"rho_{k}_{sign}"] = rng.normal(size=6) * 1e-6 + rho[f"varrho_{k}_{sign}"] = rng.uniform(1e-14, 1e-13, 6) + for k in sw.TAU_K: + for sign in ("p", "m"): + tau[f"tau_{k}_{sign}"] = rng.normal(size=6) * 1e-6 + tau[f"vartau_{k}_{sign}"] = rng.uniform(1e-14, 1e-13, 6) + + class _Writer(PSFSystematicsMixin): + """The writer's collaborators, stubbed — the method under test is real.""" + + run_type = "data" + + def sacc_nz(self, version): + return {0: _nz()} + + def sacc_metadata(self, version): + return dict(META) + + def print_magenta(self, *args, **kwargs): + pass + + out_dir = tmp_path / "rho_tau_stats" + out_dir.mkdir() + _Writer().rho_tau_to_sacc_part( + "vSYNTH", + str(out_dir), + "vSYNTH", + types.SimpleNamespace(rho_stats=rho), + types.SimpleNamespace(tau_stats=tau), + commitment_path=commitment, + ) + + written = sio.load(str(out_dir / "rho_tau_vSYNTH.sacc")) + assert written.metadata["concealed"] is True + assert written.metadata["blind_draw_scheme"] == blinding.draw_scheme() + with open(commitment, encoding="utf-8") as f: + committed = json.load(f) + assert written.metadata["blind_commitment"] == committed["seed_commitment"] + assert written.metadata["blind_config_digest"] == committed["config_digest"] + + def test_assert_consistent_blind_rejects_divergent_commitments(tmp_path): """Two ξ± parts blinded under different commitments must never combine.""" nz = {0: _nz()} diff --git a/workflow/common.py b/workflow/common.py index 530b9212..d558773e 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -226,6 +226,19 @@ def get_shear_catalog(wildcards): # DAG-build time. test_blinding_wiring asserts the two stay in lockstep. +def run_type(): + """The campaign's run type, ``"data"`` or ``"mock"``. + + Every part writer stamps this as the SACC ``type`` metadata, which is what + ``blinding.assert_consistent_blind`` reads at assembly: a plaintext + blindable part may only assemble when it declares itself a mock. A function + rather than the ``RUN_TYPE`` global because ``from common import *`` binds + names before ``configure()`` runs, so only a call reads the configured + value. + """ + return RUN_TYPE + + def is_data_run(): """True when blinding is active (production data runs); False for mocks.""" return RUN_TYPE == "data" @@ -349,6 +362,9 @@ def cv_init_params(config, version_list=None): nrandom_cell=cv["nrandom_cell"], cell_method=cv["cell_method"], nside_mask=cv["nside_mask"], + # Stamped as the SACC `type` of every part the cv writes; RUN_TYPE reads + # from this same key (see configure()). + run_type=cv.get("type", "data"), ) if cv.get("path_onecovariance"): params["path_onecovariance"] = cv["path_onecovariance"] diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index bdfa1107..fe697893 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -24,6 +24,10 @@ rule xi: max_sep="{max_sep}", nbins="{nbins}", npatch="{npatch}", + # Stamped as the part's SACC `type`. An unconcealed blindable part only + # assembles when it declares itself a mock (assert_consistent_blind), so + # this is what makes a mock campaign's terminal assembly possible at all. + type=run_type(), resources: mem_mb=30000, disk_mb=20000, @@ -82,6 +86,7 @@ rule xi_highres: nbins=_INTEGRATION["nbins"], out=str(COSMO_VAL), scripts=WORKFLOW_SCRIPTS, + run_type=run_type(), threads: 24 resources: mem_mb=40000, @@ -90,7 +95,8 @@ rule xi_highres: "python {params.scripts}/run_2pcf_highres.py " "--version {params.version} --cat-config {params.cat_config} " "--min-sep {params.min_sep} --max-sep {params.max_sep} " - "--nbins {params.nbins} --npatch 1 --out {params.out}" + "--nbins {params.nbins} --npatch 1 --out {params.out} " + "--run-type {params.run_type}" rule run_cosmo_val: @@ -111,7 +117,23 @@ rule run_cosmo_val: """ +def rho_tau_inputs(w): + """ρ/τ has no blindable input; on a data run it binds the commitment. + + ρ/τ carries no cosmological vector, so it is never shifted — but the + fail-closed load gate assemble_sacc opens every part through admits only + concealed parts on a data run. Binding commitment.json lets the writer stamp + the part concealed pass-through (values untouched). A mock run binds nothing + and the part stays plaintext. + """ + if not is_data_run(): + return {} + return {"commitment": blind_state_paths(w.version)["commitment"]} + + rule rho_tau_stats: + input: + unpack(rho_tau_inputs), output: rho_stats=str(COSMO_VAL / "rho_tau_stats/rho_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), tau_stats=str(COSMO_VAL / "rho_tau_stats/tau_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), @@ -127,6 +149,7 @@ rule rho_tau_stats: max_sep="{max_sep}", nbins="{nbins}", npatch="{npatch}", + type=run_type(), resources: mem_mb=30000, disk_mb=20000, diff --git a/workflow/scripts/run_2pcf.py b/workflow/scripts/run_2pcf.py index 3e513479..b6281b74 100644 --- a/workflow/scripts/run_2pcf.py +++ b/workflow/scripts/run_2pcf.py @@ -39,6 +39,7 @@ def run_2pcf( cat_config, output_dir, sacc_out=None, + run_type="data", ): """Measure ξ±(θ) for ``ver`` and write its reporting SACC part. @@ -49,7 +50,10 @@ def run_2pcf( ``cat_config['paths']['output']`` so the ``.txt`` byproduct lands where lc expects. ``sacc_out`` is the exact destination for the reporting ξ± SACC part (the Snakemake-declared output); it defaults to ``{ver}_xi_reporting.sacc`` - under the resolved output directory for the CLI path. + under the resolved output directory for the CLI path. ``run_type`` + (``"data"`` or ``"mock"``) is stamped as the part's SACC ``type``: a + plaintext blindable part may only enter an assembly when it declares itself + a mock, so a mock campaign must say so here. Returns ------- @@ -85,7 +89,7 @@ def run_2pcf( out_path = sacc_out or os.path.join( output_dir or cv.cc["paths"]["output"], f"{ver}_xi_reporting.sacc" ) - sacc_io.save(s, out_path, type="data") + sacc_io.save(s, out_path, type=run_type) print(f"Wrote reporting ξ± SACC part: {out_path}") return gg @@ -107,6 +111,7 @@ def _from_snakemake(smk): # Write the SACC part exactly where the rule declares it (the .txt # byproduct still lands under the resolved output dir via _output_path). sacc_out=smk.output["xi_reporting"], + run_type=p.get("type", "data"), ) @@ -133,6 +138,12 @@ def _from_cli(argv=None): "--cat-config", required=True, help="Absolute path to cat_config.yaml" ) ap.add_argument("--out", required=True, help="Output directory (lc {output})") + ap.add_argument( + "--run-type", + default="data", + choices=("data", "mock"), + help="Campaign run type stamped as the part's SACC `type`", + ) a = ap.parse_args(argv) run_2pcf( ver=a.ver, @@ -142,6 +153,7 @@ def _from_cli(argv=None): npatch=a.npatch, cat_config=a.cat_config, output_dir=a.out, + run_type=a.run_type, ) diff --git a/workflow/scripts/run_2pcf_highres.py b/workflow/scripts/run_2pcf_highres.py index 6049e105..c5c546f5 100644 --- a/workflow/scripts/run_2pcf_highres.py +++ b/workflow/scripts/run_2pcf_highres.py @@ -91,6 +91,10 @@ NPATCH = None OUTPUT_DIR = None PATCH_FILE = None +# Campaign run type, stamped as the part's SACC `type`. Custody state, not +# decoration: a plaintext blindable part may only enter an assembly when it +# declares itself a mock (blinding.assert_consistent_blind). +RUN_TYPE = "data" def parse_args(argv=None): @@ -121,6 +125,12 @@ def parse_args(argv=None): "--max-sep", type=float, default=300.0, help="Max separation [arcmin]" ) ap.add_argument("--out", required=True, help="Output directory (lc {output})") + ap.add_argument( + "--run-type", + default="data", + choices=("data", "mock"), + help="Campaign run type stamped as the part's SACC `type`", + ) return ap.parse_args(argv) @@ -233,7 +243,7 @@ def write_xi_integration_sacc(gg): variances=np.concatenate([gg.varxip, gg.varxim]), ) out_path = os.path.join(OUTPUT_DIR, f"{VERSION}_xi_integration.sacc") - sacc_io.save(s, out_path, type="data") + sacc_io.save(s, out_path, type=RUN_TYPE) log(f" Wrote {out_path}") @@ -297,10 +307,11 @@ def resolve_paths(ver): def main(): global CAT_PATH, VERSION, E1_COL, E2_COL, W_COL, REDSHIFT_PATH - global TMIN, TMAX, NBINS, NPATCH, OUTPUT_DIR, PATCH_FILE + global TMIN, TMAX, NBINS, NPATCH, OUTPUT_DIR, PATCH_FILE, RUN_TYPE args = parse_args() VERSION = args.version + RUN_TYPE = args.run_type NBINS = args.nbins NPATCH = args.npatch TMIN = args.min_sep diff --git a/workflow/scripts/run_rho_tau.py b/workflow/scripts/run_rho_tau.py index 9df0bbfd..7b64c418 100644 --- a/workflow/scripts/run_rho_tau.py +++ b/workflow/scripts/run_rho_tau.py @@ -44,9 +44,16 @@ theta_max=float(params["max_sep"]), nbins=int(params["nbins"]), npatch=int(params["npatch"]), + run_type=params.get("type", "data"), ) -cv.calculate_rho_tau_stats() +# On a data run the rule binds the version's commitment.json, which stamps the +# emitted ρ/τ part concealed pass-through — values untouched (ρ/τ carries no +# cosmological vector) but clearing the fail-closed load gate at assembly. A mock +# run binds no commitment and the part stays plaintext, stamped type='mock'. +commitment_path = snakemake.input.get("commitment") # type: ignore + +cv.calculate_rho_tau_stats(commitment_path=commitment_path) # Confirm CosmologyValidation produced the requested outputs. calculate_rho_tau_stats # writes the rho/tau FITS *and* the born-as-SACC rho_tau part (via From 334047a5fa9cbd6baa2f2abc928d021736d93fdf Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 30 Aug 2026 02:05:41 +0200 Subject: [PATCH 47/47] Simplify the blinding rework: one home per fact sacc_io.save grows a commitment= kwarg and owns the passthrough stamp (three copy-pasted writer tails collapse); commitment_input() in workflow/common.py is the one place a data run binds commitment.json into rule inputs; the CLI's verify delegates draw-scheme checks to _assert_draw_scheme instead of re-wording them; _stamp_provenance loses its derivable scheme parameter. Draw-scheme rationale stated once (draw_scheme), mock-assembly rule once (assert_consistent_blind); dead escrow-recovery prose cut; doubled test assertions deduped; is_data_run reads run_type(). Container suite: 70/70 blinding+wiring+dry-run, 282 passed overall; 3 failures pre-existing and environmental (path checks on other users' scratch, stale-container glass import, a pure-eb value pin that fails identically on the unedited base). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01G9MahwJEQ1t9EuvUXijmy3 --- papers/cosmo_val/config/config.yaml | 10 ++--- scripts/blind_data_vector.py | 37 ++++++---------- src/sp_validation/blinding.py | 43 ++++++++----------- src/sp_validation/cosmo_val/core.py | 8 ++-- src/sp_validation/cosmo_val/cosebis.py | 12 ++---- .../cosmo_val/psf_systematics.py | 21 +++------ src/sp_validation/cosmo_val/pure_eb.py | 17 ++------ src/sp_validation/sacc_io.py | 12 +++++- src/sp_validation/tests/test_blinding.py | 9 ---- workflow/common.py | 18 ++++++-- workflow/rules/cosmo_val.smk | 19 +++----- workflow/rules/twopoint.smk | 23 +++------- workflow/scripts/run_2pcf.py | 5 +-- workflow/scripts/run_2pcf_highres.py | 3 +- 14 files changed, 89 insertions(+), 148 deletions(-) diff --git a/papers/cosmo_val/config/config.yaml b/papers/cosmo_val/config/config.yaml index a2124302..f3ead5ee 100644 --- a/papers/cosmo_val/config/config.yaml +++ b/papers/cosmo_val/config/config.yaml @@ -19,13 +19,9 @@ versions: [ # CosmologyValidation suite parameters (cosmo_val.py) # --------------------------------------------------------------------------- cosmo_val: - # Campaign run type: "data" or "mock". One switch, two effects. It gates - # Smokescreen blind-at-birth (a data run blinds the three blindable parts and - # binds every ξ-derived consumer to the blinded siblings; a mock run bypasses - # blinding), and it is stamped as the SACC `type` of every part written. The - # two are the same decision: an unconcealed blindable part only enters the - # terminal assembly when it declares itself a mock, so a mock campaign must - # say so here for its parts to assemble at all. + # Campaign run type: "data" or "mock". One switch: it gates Smokescreen + # blind-at-birth and is stamped as the SACC `type` of every part written + # (custody state at assembly — see blinding.assert_consistent_blind). type: data # CosmologyValidation constructor diff --git a/scripts/blind_data_vector.py b/scripts/blind_data_vector.py index 692ebd94..1050d226 100644 --- a/scripts/blind_data_vector.py +++ b/scripts/blind_data_vector.py @@ -9,9 +9,8 @@ (never printed, never written in plaintext), publishes a repo-committable ``commitment.json`` (the seed commitment + the config digest + the installed Smokescreen fork's draw scheme), and encrypts the seed into a Fernet bundle. -Those three, not the seed alone, are what reproduces a blind: seed and config -fix *which* shift, the draw scheme fixes *how* the seed becomes that shift. -``blind-part`` blinds one intermediate part SACC (reporting ξ±, integration ξ±, +Those three, not the seed alone, are what reproduces a blind (see +``blinding.draw_scheme``). ``blind-part`` blinds one intermediate part SACC (reporting ξ±, integration ξ±, or pseudo-Cℓ) under that fixed state, escrows the true vector into a per-part encrypted bundle beside the blinded output, and deletes the plaintext part. ``unblind`` verifies all three commitments and restores a true part @@ -113,8 +112,7 @@ def _verify(args): only that the file's three custody stamps agree with the commitment, and that the recorded draw scheme is the one this install implements. That last check makes the result environment-dependent by design: a machine carrying a - different Smokescreen reports a problem even when file and commitment agree - perfectly, because that machine could not unblind the file. + different Smokescreen could not unblind the file, so it reports a problem. Loads with ``allow_unblinded=True``: the whole job here is to report on a file's custody state, including the state where the file is not concealed at @@ -131,30 +129,19 @@ def _verify(args): problems.append("blind_commitment does not match the committed seed commitment") if s.metadata.get("blind_config_digest") != commitment["config_digest"]: problems.append("blind_config_digest does not match the committed digest") - # The draw scheme is checked three ways: file ↔ commitment, and both against - # the installed fork. A scheme mismatch is the one blinding failure with no - # numerical symptom — the unblind would subtract a different shift than was - # added and every other check here would still pass. - installed = blinding.draw_scheme() + # The draw scheme is checked two ways: file ↔ commitment, and the file's + # against the installed fork (see blinding.draw_scheme). file_scheme = s.metadata.get("blind_draw_scheme") committed = commitment.get("draw_scheme") - if file_scheme is None or committed is None: + if file_scheme != committed: problems.append( - "no draw-scheme record on the file and/or in the commitment — " - "this blind predates draw-scheme binding and cannot be verified " - "reproducible" - ) - elif int(file_scheme) != int(committed): - problems.append( - f"blind_draw_scheme {int(file_scheme)} does not match the committed " - f"draw_scheme {int(committed)}" - ) - elif int(file_scheme) != installed: - problems.append( - f"blind was drawn under Smokescreen DRAW_SCHEME={int(file_scheme)} " - f"but the installed fork implements DRAW_SCHEME={installed} — this " - "install cannot reproduce the shift" + f"blind_draw_scheme {file_scheme!r} does not match the committed " + f"draw_scheme {committed!r}" ) + try: + blinding._assert_draw_scheme(file_scheme, "the blinded file") + except ValueError as exc: + problems.append(str(exc)) if "seed_smokescreen" in s.metadata: problems.append("PLAINTEXT SEED LEAKED into file metadata (seed_smokescreen)") if problems: diff --git a/src/sp_validation/blinding.py b/src/sp_validation/blinding.py index 34b648d2..247b20e9 100644 --- a/src/sp_validation/blinding.py +++ b/src/sp_validation/blinding.py @@ -46,9 +46,8 @@ and the installed fork's ``DRAW_SCHEME`` as a repo-committable ``commitment.json``, and encrypts the seed into a Fernet bundle (``smokescreen.encryption``) — the plaintext seed is never written. The - three together, not the seed alone, are what reproduces a blind: seed and - config fix *which* shift, the draw scheme fixes *how* the seed becomes that - shift. + three together, not the seed alone, are what reproduces a blind (see + :func:`draw_scheme`). Each :func:`blind_part` call reads that fixed state, conceals one part, escrows the part's true vector into its own encrypted bundle beside the @@ -61,14 +60,10 @@ ``blind_draw_scheme``. :func:`unblind_part` verifies all three against the commitment *before* subtracting anything, then restores the true part. - Custody has no back door, including for its own history: a blind fixed - before the draw scheme was bound into ``commitment.json`` carries no - scheme record, and :func:`_assert_draw_scheme` refuses it ahead of every - seed read, so neither :func:`blind_part` nor :func:`unblind_part` will run - on it. Such a blind is recoverable only by hand, by decrypting its per-part - escrow bundles (:func:`_read_encrypted_json`) and reading ``true_mean`` - back out. No such blind exists — the scheme has been bound since before - the first real blind — and the CLI deliberately offers no override. + Custody has no back door, including for its own history: a blind carrying no + scheme record is refused by :func:`_assert_draw_scheme` ahead of every seed + read, with no CLI override — none exists, the scheme having been bound + before the first real blind. """ import dataclasses @@ -228,7 +223,8 @@ def _assert_draw_scheme(recorded, what): ``what`` names the surface the scheme was read from, for the message. A missing record (``None``) is a failure, not a pass: a blind whose scheme - is unknown cannot be shown to be reproducible by this install. + is unknown cannot be shown to be reproducible by this install (see + :func:`draw_scheme`). """ installed = draw_scheme() if recorded is None: @@ -570,7 +566,7 @@ def unblind_sacc(blinded, seed, config=None, log=print): return part -def _stamp_provenance(s, commitment, label, config_digest, scheme=None): +def _stamp_provenance(s, commitment, label, config_digest): """Stamp the blind's public provenance; strip any leaked seed. ``blind_commitment`` (sha256 of the seed) ties the file to its blind @@ -582,17 +578,16 @@ def _stamp_provenance(s, commitment, label, config_digest, scheme=None): raw seed upstream Smokescreen's writer would stamp — is popped defensively: the seed must never ride a kept file. - ``scheme`` defaults to the installed fork's, which is the right answer - whenever this call is stamping a blind that was just computed. Callers - propagating an existing blind's provenance pass that blind's recorded - scheme instead. + The stamped scheme is always the installed fork's: every caller has already + checked the blind's recorded scheme against it (:func:`_assert_draw_scheme`), + so the two agree by the time this runs. """ s.metadata.pop("seed_smokescreen", None) s.metadata["concealed"] = True s.metadata["blind"] = label s.metadata["blind_commitment"] = commitment s.metadata["blind_config_digest"] = config_digest - s.metadata["blind_draw_scheme"] = draw_scheme() if scheme is None else int(scheme) + s.metadata["blind_draw_scheme"] = draw_scheme() def stamp_concealed_passthrough(s, commitment_path): @@ -628,7 +623,6 @@ def stamp_concealed_passthrough(s, commitment_path): commitment["seed_commitment"], commitment["label"], commitment["config_digest"], - scheme=commitment["draw_scheme"], ) return s @@ -808,12 +802,11 @@ def blind_init(blind_dir, config=None, label="A", log=print): def _read_seed(blind_dir, config): """Decrypt the seed bundle and verify it against the commitment. - The seed commitment, the config digest, and the committed draw scheme are all - checked before the seed is handed to any caller — a tampered bundle, a - drifted config or a Smokescreen that draws differently from the one that - fixed this blind all fail loud here, whether the caller is about to blind - or to unblind. The scheme check is what stops a re-blind of a later part - from landing a different hidden cosmology than the earlier parts got. + The seed commitment, the config digest, and the committed draw scheme (see + :func:`draw_scheme`) are all checked before the seed is handed to any caller + — a tampered bundle, a drifted config or a Smokescreen that draws + differently from the one that fixed this blind all fail loud here, whether + the caller is about to blind or to unblind. Returns ------- diff --git a/src/sp_validation/cosmo_val/core.py b/src/sp_validation/cosmo_val/core.py index 401d52af..05ded922 100644 --- a/src/sp_validation/cosmo_val/core.py +++ b/src/sp_validation/cosmo_val/core.py @@ -100,11 +100,9 @@ class CosmologyValidation( Cosmological parameters to pass to get_cosmo(). If None, uses Planck 2018. run_type : {'data', 'mock'}, default 'data' The campaign's run type, stamped as the SACC ``type`` of every part this - object writes. Custody state, not decoration: - ``blinding.assert_consistent_blind`` admits an unconcealed blindable part - into a terminal assembly only when it declares itself a mock, so a mock - campaign must be built with ``run_type='mock'`` for its parts to - assemble at all. + object writes. Custody state, not decoration: a mock campaign must be + built with ``run_type='mock'`` for its parts to assemble at all (see + ``blinding.assert_consistent_blind``). Attributes ---------- diff --git a/src/sp_validation/cosmo_val/cosebis.py b/src/sp_validation/cosmo_val/cosebis.py index 178bd971..4fbec2ba 100644 --- a/src/sp_validation/cosmo_val/cosebis.py +++ b/src/sp_validation/cosmo_val/cosebis.py @@ -185,10 +185,8 @@ def cosebis_to_sacc_part( to the part in place of ``result["En"]`` — re-derived from the integration ξ± SACC part at the fiducial scale cut (Bn and the covariance stay from the raw estimator ``result``). ``commitment_path`` stamps the part concealed - under that version's blind (via - :func:`blinding.stamp_concealed_passthrough`) before save, so a data run's - part clears the fail-closed load gate. With both ``None`` (mock runs) the - behaviour is unchanged. + under that version's blind (see :func:`sacc_io.save`). With both ``None`` + (mock runs) the behaviour is unchanged. """ result, scale_cut = self._fiducial_cosebis_result(results, fiducial_scale_cut) if en_override is not None: @@ -199,11 +197,7 @@ def cosebis_to_sacc_part( result, scale_cut, ) - if commitment_path is not None: - from ..blinding import stamp_concealed_passthrough - - stamp_concealed_passthrough(s, commitment_path) - sacc_io.save(s, out_path, type=self.run_type) + sacc_io.save(s, out_path, type=self.run_type, commitment=commitment_path) def plot_cosebis( self, diff --git a/src/sp_validation/cosmo_val/psf_systematics.py b/src/sp_validation/cosmo_val/psf_systematics.py index b81d8b1c..c7984c26 100644 --- a/src/sp_validation/cosmo_val/psf_systematics.py +++ b/src/sp_validation/cosmo_val/psf_systematics.py @@ -26,11 +26,7 @@ class PSFSystematicsMixin: def calculate_rho_tau_stats(self, commitment_path=None): - """Measure ρ/τ statistics per version and write each version's SACC part. - - ``commitment_path`` is custody plumbing forwarded to - :meth:`rho_tau_to_sacc_part`; see it for what it does. - """ + """Measure ρ/τ statistics per version and write each version's SACC part.""" out_dir = f"{self.cc['paths']['output']}/rho_tau_stats" if not os.path.exists(out_dir): os.mkdir(out_dir) @@ -73,12 +69,9 @@ def rho_tau_to_sacc_part( """Write the ρ/τ SACC part for one version. ρ/τ is a PSF diagnostic carrying no cosmological vector, so it is never - blinded — but on a data run it still has to pass the fail-closed load - gate that ``assemble_sacc`` opens every part through. - ``commitment_path`` is that seam: it stamps the part concealed under the - version's blind (:func:`blinding.stamp_concealed_passthrough`), values - untouched, so the assembly admits it. A mock run passes ``None`` and the - part stays unstamped. + blinded; ``commitment_path`` stamps the part concealed under the + version's blind, values untouched (see :func:`sacc_io.save`), so a data + run's assembly admits it. A mock run passes ``None``. ρ_0…ρ_5 autos and τ_0/τ_2/τ_5 leakage from the handler tables. The ``CovTauTh`` theory covariance ``cov_tau_{base}_th.npy`` — a @@ -103,12 +96,8 @@ def rho_tau_to_sacc_part( tau_stat_handler.tau_stats, tau_cov_th=tau_cov_th, ) - if commitment_path is not None: - from ..blinding import stamp_concealed_passthrough - - stamp_concealed_passthrough(s, commitment_path) out_path = os.path.join(out_dir, f"rho_tau_{base}.sacc") - sacc_io.save(s, out_path, type=self.run_type) + sacc_io.save(s, out_path, type=self.run_type, commitment=commitment_path) @property def rho_stat_handler(self): diff --git a/src/sp_validation/cosmo_val/pure_eb.py b/src/sp_validation/cosmo_val/pure_eb.py index f0b41642..e7220c47 100644 --- a/src/sp_validation/cosmo_val/pure_eb.py +++ b/src/sp_validation/cosmo_val/pure_eb.py @@ -135,12 +135,7 @@ def calculate_pure_eb( return results def pure_eb_to_sacc_part( - self, - version, - out_path, - results, - eb_override=None, - commitment_path=None, + self, version, out_path, results, eb_override=None, commitment_path=None ): """Write the pure-E/B SACC part (six ``PURE_KEYS`` blocks + covariance). @@ -154,8 +149,8 @@ def pure_eb_to_sacc_part( — re-derived from the reporting + integration ξ± SACC parts (the covariance stays blind-invariant from the raw estimator ``results``). ``commitment_path`` stamps the part concealed under that version's blind - (via :func:`blinding.stamp_concealed_passthrough`) before save. With both - ``None`` (mock runs) the behaviour is unchanged. + (see :func:`sacc_io.save`). With both ``None`` (mock runs) the behaviour + is unchanged. """ theta = results["gg"].meanr source = eb_override if eb_override is not None else results @@ -167,11 +162,7 @@ def pure_eb_to_sacc_part( eb, covariance=results["cov"], ) - if commitment_path is not None: - from ..blinding import stamp_concealed_passthrough - - stamp_concealed_passthrough(s, commitment_path) - sacc_io.save(s, out_path, type=self.run_type) + sacc_io.save(s, out_path, type=self.run_type, commitment=commitment_path) def plot_pure_eb( self, diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index 7dcb72a7..8d14e6ca 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -924,7 +924,7 @@ def update_statistic(s, sub): s.data[idx[0]].value = point.value -def save(s, path, *, type): +def save(s, path, *, type, commitment=None): """Write ``s`` to ``path`` (FITS), overwriting any existing file. Parameters @@ -937,6 +937,12 @@ def save(s, path, *, type): the pipeline computing the data vector — knows whether its input catalogue is a mock; there is deliberately no default. ``load`` refuses ``type='data'`` files that are not blinded. + commitment : str, optional + Path to the version's ``commitment.json``. When given, the file is + stamped concealed under that blind + (:func:`sp_validation.blinding.stamp_concealed_passthrough`, values + untouched) before writing — the seam every born-blinded or + blind-irrelevant part uses to clear the fail-closed load gate. """ if type not in ("data", "mock"): raise ValueError(f"type must be 'data' or 'mock'; got {type!r}") @@ -946,6 +952,10 @@ def save(s, path, *, type): f"refusing to re-stamp as {type!r}" ) s.metadata["type"] = type + if commitment is not None: + from . import blinding + + blinding.stamp_concealed_passthrough(s, commitment) s.save_fits(path, overwrite=True) diff --git a/src/sp_validation/tests/test_blinding.py b/src/sp_validation/tests/test_blinding.py index 6d808895..f4aa7d58 100644 --- a/src/sp_validation/tests/test_blinding.py +++ b/src/sp_validation/tests/test_blinding.py @@ -336,18 +336,10 @@ def test_theory_config_ccl_params_exact_keyset(): # --------------------------------------------------------------------------- # def test_commitment_is_the_forks_domain_separated_digest(): """One definition of the commitment, and it is the fork's.""" - import hashlib - import smokescreen seed = "the-secret" assert bd.seed_commitment(seed) == smokescreen.seed_commitment(seed) - assert ( - bd.seed_commitment(seed) - == hashlib.sha256( - smokescreen.COMMITMENT_DOMAIN + seed.encode("utf-8") - ).hexdigest() - ) assert bd.seed_commitment("right") != bd.seed_commitment("wrong") @@ -370,7 +362,6 @@ def test_commitment_does_not_embed_the_rng_seed(): for seed in ("my_secret_seed", "the-secret", secrets.token_hex(16)): commitment = bd.seed_commitment(seed) assert int(commitment[:16], 16) != _normalize_seed(seed) - assert str(_normalize_seed(seed)) not in commitment def test_hidden_params_deterministic_and_in_envelope(): diff --git a/workflow/common.py b/workflow/common.py index d558773e..78224d02 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -230,8 +230,7 @@ def run_type(): """The campaign's run type, ``"data"`` or ``"mock"``. Every part writer stamps this as the SACC ``type`` metadata, which is what - ``blinding.assert_consistent_blind`` reads at assembly: a plaintext - blindable part may only assemble when it declares itself a mock. A function + ``blinding.assert_consistent_blind`` reads at assembly. A function rather than the ``RUN_TYPE`` global because ``from common import *`` binds names before ``configure()`` runs, so only a call reads the configured value. @@ -241,7 +240,7 @@ def run_type(): def is_data_run(): """True when blinding is active (production data runs); False for mocks.""" - return RUN_TYPE == "data" + return run_type() == "data" def blind_state_dir(version): @@ -262,6 +261,19 @@ def blind_state_paths(version): } +def commitment_input(version): + """Input mapping binding a version's commitment.json, on data runs only. + + A part writer stamps its output concealed from that file (sacc_io.save's + `commitment=`), which is what lets a born-blinded or blind-irrelevant part + clear the fail-closed load gate the terminal assembly opens every part + through. A mock run binds nothing and the part stays plaintext. + """ + if not is_data_run(): + return {} + return {"commitment": blind_state_paths(version)["commitment"]} + + def blinded_path(part_path): """The *_blinded sibling blind_part writes beside a plaintext part. diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index 705ac9bc..049b30b8 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -370,29 +370,24 @@ rule cv_pseudo_cl: # On a data run the COSEBIs / pure-E/B parts re-derive their E-mode vector from # the *blinded* integration ξ± (COSEBIs) or blinded reporting + integration ξ± -# (pure-E/B), and stamp the derived part concealed from the version's -# commitment.json. blindable_part returns the plaintext part on a mock run and -# the blinded part on a data run, so the ξ± inputs bind unconditionally for every -# version; only the commitment is gated on is_data_run(). +# (pure-E/B). blindable_part returns the plaintext part on a mock run and the +# blinded part on a data run, so the ξ± inputs bind unconditionally for every +# version; see common.commitment_input for the commitment. def cv_cosebis_inputs(w): - inputs = { + return { "xi": cv_xi_txt(w.version), "xi_integration": blindable_part(cv_xi_integration_sacc(w.version)), + **commitment_input(w.version), } - if is_data_run(): - inputs["commitment"] = blind_state_paths(w.version)["commitment"] - return inputs def cv_pure_eb_inputs(w): - inputs = { + return { "xi": cv_xi_txt(w.version), "xi_reporting": blindable_part(cv_xi_reporting_sacc(w.version)), "xi_integration": blindable_part(cv_xi_integration_sacc(w.version)), + **commitment_input(w.version), } - if is_data_run(): - inputs["commitment"] = blind_state_paths(w.version)["commitment"] - return inputs rule cv_pure_eb: diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index fe697893..31b1dd5a 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -24,9 +24,8 @@ rule xi: max_sep="{max_sep}", nbins="{nbins}", npatch="{npatch}", - # Stamped as the part's SACC `type`. An unconcealed blindable part only - # assembles when it declares itself a mock (assert_consistent_blind), so - # this is what makes a mock campaign's terminal assembly possible at all. + # Stamped as the part's SACC `type` — custody state at assembly + # (see blinding.assert_consistent_blind). type=run_type(), resources: mem_mb=30000, @@ -117,23 +116,11 @@ rule run_cosmo_val: """ -def rho_tau_inputs(w): - """ρ/τ has no blindable input; on a data run it binds the commitment. - - ρ/τ carries no cosmological vector, so it is never shifted — but the - fail-closed load gate assemble_sacc opens every part through admits only - concealed parts on a data run. Binding commitment.json lets the writer stamp - the part concealed pass-through (values untouched). A mock run binds nothing - and the part stays plaintext. - """ - if not is_data_run(): - return {} - return {"commitment": blind_state_paths(w.version)["commitment"]} - - rule rho_tau_stats: + # ρ/τ has no blindable input; it binds only the commitment, to stamp its + # part concealed pass-through (see common.commitment_input). input: - unpack(rho_tau_inputs), + unpack(lambda w: commitment_input(w.version)), output: rho_stats=str(COSMO_VAL / "rho_tau_stats/rho_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), tau_stats=str(COSMO_VAL / "rho_tau_stats/tau_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), diff --git a/workflow/scripts/run_2pcf.py b/workflow/scripts/run_2pcf.py index b6281b74..7f39660c 100644 --- a/workflow/scripts/run_2pcf.py +++ b/workflow/scripts/run_2pcf.py @@ -51,9 +51,8 @@ def run_2pcf( expects. ``sacc_out`` is the exact destination for the reporting ξ± SACC part (the Snakemake-declared output); it defaults to ``{ver}_xi_reporting.sacc`` under the resolved output directory for the CLI path. ``run_type`` - (``"data"`` or ``"mock"``) is stamped as the part's SACC ``type``: a - plaintext blindable part may only enter an assembly when it declares itself - a mock, so a mock campaign must say so here. + (``"data"`` or ``"mock"``) is stamped as the part's SACC ``type`` — custody + state at assembly (see ``blinding.assert_consistent_blind``). Returns ------- diff --git a/workflow/scripts/run_2pcf_highres.py b/workflow/scripts/run_2pcf_highres.py index c5c546f5..985c0ef4 100644 --- a/workflow/scripts/run_2pcf_highres.py +++ b/workflow/scripts/run_2pcf_highres.py @@ -92,8 +92,7 @@ OUTPUT_DIR = None PATCH_FILE = None # Campaign run type, stamped as the part's SACC `type`. Custody state, not -# decoration: a plaintext blindable part may only enter an assembly when it -# declares itself a mock (blinding.assert_consistent_blind). +# decoration (see blinding.assert_consistent_blind). RUN_TYPE = "data"