diff --git a/.gitignore b/.gitignore index 82fb2697..c5fbafd5 100644 --- a/.gitignore +++ b/.gitignore @@ -194,4 +194,7 @@ papers/catalog/plots/*.pdf papers/cosmo_val/logs/ # Ignore scratch notebooks -scratch/*/*.ipynb \ No newline at end of file +scratch/*/*.ipynb + +# Snakemake run state +.snakemake/ diff --git a/CLAUDE.md b/CLAUDE.md index e985d6c2..c35c20be 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -78,10 +78,26 @@ Main configuration in `scripts/calibration/params.py` with parameters: - pyccl for cosmological calculations ## Container Usage -Recommended installation via Apptainer/Docker: +Nothing is hand-built. CI publishes `ghcr.io/cosmostat/sp_validation:` on +every push, and each person keeps their own copy at +`~/.cache/sp_validation/sp_validation.sif`, managed by the `spv-container` CLI: + ```bash -apptainer build --sandbox sp_validation docker://ghcr.io/cosmostat/sp_validation:develop +spv-container pull # fetch :develop there (do it from a compute node) +spv-container status # which layer is live, and how current it is +spv-container exec # one-off run inside it ``` +Need a package the image lacks mid-analysis? `spv-container sandbox`, then +`spv-container exec --writable pip install `; the sandbox then takes +precedence over the SIF everywhere, workflow jobs included. + +Every rule runs inside that image, wrapped by Snakemake itself (`--profile +workflow/profiles/candide` on the cluster, `workflow/profiles/default -j N` +elsewhere). The `sp_validation` a rule imports comes from the *launched +checkout*, not the image. + +`workflow/README.md` is the full story — profiles, image resolution, refresh. + ## Notebook Configuration - The CosmologyValidation class must be initialized in cosmo_val \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 00a8d907..e385a386 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,15 +13,49 @@ inside the project container, which ships the full stack pre-built. ### Container (recommended) +CI builds and pushes an image on **every** push, tagged by the sanitized branch +name (see +[`.github/workflows/deploy-image.yml`](.github/workflows/deploy-image.yml)), so +`:develop` tracks the integration branch and your branch has an image of its +own. Nothing is built by hand. + +You keep your own copy of the image. `spv-container` — stdlib-only, so it runs +straight from a checkout: symlink it onto your PATH (`ln -s +"$PWD/src/sp_validation/container.py" ~/.local/bin/spv-container`, the README's +install step) or call it as `python3 src/sp_validation/container.py` — pulls it to +`~/.cache/sp_validation/sp_validation.sif` and runs things inside it: + ```bash -# build a writeable sandbox from the published image -apptainer build --sandbox sp_validation docker://ghcr.io/cosmostat/sp_validation:develop -apptainer shell --writable sp_validation +spv-container pull # fetch :develop; ~1.5 GB, so do it from a compute node +spv-container status # which layer is live, and how current it is +spv-container exec bash # an interactive shell inside it ``` -The image is rebuilt and pushed on every push to `develop` (see -[`.github/workflows/deploy-image.yml`](.github/workflows/deploy-image.yml)), so -`:develop` always tracks the latest integration branch. +That image is read-only. When you need a package it does not carry yet, unpack a +writable sandbox once with `spv-container sandbox` and install into it with +`spv-container exec --writable pip install `; the sandbox then takes +precedence everywhere, workflow jobs included. Treat it as an exploration tool — +the real fix is adding the dependency to `pyproject.toml` — and reset it with +`spv-container pull && spv-container sandbox --force`. + +Analysis runs through Snakemake, which wraps every job in `apptainer exec` +against that same image for you — see +[`workflow/README.md`](workflow/README.md) for the profiles and the details. + +Two things worth knowing while developing: + +- **Your checkout's code is what runs.** The workflow prepends the launched + checkout's `src/` to the container's `PYTHONPATH`, so the image supplies the + dependency stack and your working tree supplies `sp_validation`. No rebuild + needed to test a change. (Caveat: `rerun-triggers: code` does not watch + `src/`, so force reruns after editing a module.) +- **To test a branch's own image** — when the *stack* changed, not just `src/` — + point the workflow at its CI tag: + + ```bash + snakemake --profile workflow/profiles/candide \ + --config container=docker://ghcr.io/cosmostat/sp_validation:my-branch + ``` ### Local install with `uv` diff --git a/Dockerfile b/Dockerfile index ccc930bf..d24e3040 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,10 @@ # Development image with more bells and whistles FROM ghcr.io/cosmostat/shapepipe:im_sims +# liblapack-dev: cosmosis's MultiNest links -llapack, and the base image ships +# only the runtime liblapack.so.3 (no dev symlink). The gsl/cfitsio/fftw3 dev +# packages are what the CosmoSIS Standard Library's C sources compile against +# (they are the headers CSL's own CI installs); git is for cloning it. RUN apt-get update -y --quiet --fix-missing && \ apt-get dist-upgrade -y --quiet --fix-missing && \ apt-get install -y --quiet \ @@ -8,14 +12,24 @@ RUN apt-get update -y --quiet --fix-missing && \ automake \ libtool \ pkg-config \ + git \ htop \ npm \ - tmux + tmux \ + liblapack-dev \ + libgsl-dev \ + libcfitsio-dev \ + libfftw3-dev # The base shapepipe image provides a uv-managed venv at /app/.venv (exported as # VIRTUAL_ENV); install sp_validation's deps into that same venv rather than # spawning a second one under /sp_validation. ENV UV_PROJECT_ENVIRONMENT=/app/.venv +# $HOME is bind-mounted under apptainer, so uv would otherwise discover the +# host's managed CPythons -- including newer ones that satisfy requires-python +# -- and build a venv against an interpreter carrying none of this stack. +ENV UV_PYTHON=/app/.venv/bin/python \ + UV_PYTHON_DOWNLOADS=never WORKDIR /sp_validation @@ -30,9 +44,59 @@ WORKDIR /sp_validation # numba-safe numpy 2.4.6 come straight from the lock, so the old ad-hoc snakemake # and cs_util `--upgrade` layers are gone. COPY pyproject.toml uv.lock /sp_validation/ + +# cosmosis builds MPI-enabled polychord/multinest only when MPIFC is set: its +# setup.py exports MPIFC for conda builds only, and the sampler Makefiles gate on +# `which $(MPIFC)`. Absolute path, not a bare name: /opt/ompi/bin is not always on +# PATH, and a miss silently omits libchord_mpi.so while the install still succeeds, +# and `cosmosis --mpi` fails at load time -- which is how the pipeline runs, since +# the --smp pool is broken upstream. Must precede the sync that builds cosmosis. +ENV MPIFC=/opt/ompi/bin/mpif90 + RUN uv sync --frozen --inexact --no-install-project \ --extra test --extra glass --extra workflow +# The CosmoSIS Standard Library: the module files (camb interface, projection, +# 2pt likelihood, ...) the cosmo_inference pipelines name. The `workflow` extra +# above installs cosmosis itself; CSL is a separate tree of modules that is not +# on PyPI and has to be built against that install, so it is cloned and compiled +# here rather than left to each user (which is what the .ini templates used to +# assume, hard-coding one person's home directory). +# +# Pinned to Sacha Guerrini's fork, which carries the four commits the UNIONS +# pipelines depend on: tau-stats, sample_S8, and two z-dependent linear-alignment +# modules. See cosmo_inference/README.md for the standing of that fork. +ARG CSL_REPO=https://github.com/sachaguer/cosmosis-standard-library.git +ARG CSL_REF=b26fa7ff666ab4d607b2e32e36f799a53bfb1d9c +ENV CSL_DIR=/opt/cosmosis-standard-library + +# `python -m cosmosis.configure` emits the exports (COSMOSIS_SRC_DIR et al.) +# every CSL Makefile includes its compiler config from. Evaluated directly +# rather than through the `cosmosis-configure` wrapper: that wrapper's +# am-I-sourced probe reads unset zsh/ksh variables, which `set -u` turns into +# an error, and its `exit` then ends the sourcing shell with status 0 — make +# never runs and the layer still "succeeds". The trailing `test -f` keeps any +# such silent no-op loud. +# +# `make -C shear` rather than a bare `make`: the top-level target also descends +# into likelihood/, which builds the Planck, WMAP and ACT likelihoods -- large, +# data-dependent, and unused by any UNIONS pipeline. Everything our .ini +# templates reference is either pure Python (consistency, sample_S8, camb, +# load_nz_fits, photoz_bias, linear_alignment, add_intrinsic, shear_m_bias, +# xi_sys, 2pt_like -- no Makefile in those trees at all) or lives under shear/: +# `limber`, which project_2d.py links, and `cl_to_xi_nicaea`, whose +# nicaea_interface.so the 2pt_shear stage loads. +RUN bash -c 'set -eo pipefail; \ + export PATH=/app/.venv/bin:$PATH; \ + git clone --filter=blob:none "$CSL_REPO" "$CSL_DIR"; \ + cd "$CSL_DIR"; \ + git checkout --detach "$CSL_REF"; \ + cmds=$(python -m cosmosis.configure); \ + eval "$cmds"; \ + export GSL_INC=/usr/include GSL_LIB=/usr/lib/x86_64-linux-gnu; \ + make -C shear; \ + test -f shear/cl_to_xi_nicaea/nicaea_interface.so' + # Install sp_validation itself (editable) into the same venv; deps are already # satisfied by the sync above. COPY . /sp_validation diff --git a/README.md b/README.md index e14a2071..455ddd91 100644 --- a/README.md +++ b/README.md @@ -62,30 +62,35 @@ directive imports the shared rules under each run's own config and an output `prefix`, so runs namespace under `results//` without clobbering one another. -## Container Installation (Recommended) +## Installation -The easiest way to install sp_validation is via a container. Docker images are automatically built and pushed to the [GitHub Container Registry (GHCR)](https://github.com/CosmoStat/sp_validation/pkgs/container/sp_validation) on every push to `develop`. This image can be installed and run on most systems (including clusters) with just a few lines of code. - -We recommend running the image with **Apptainer** (formerly Singularity) which is installed on most HPC clusters. To simply run the image, use the following command: +`sp_validation` runs from a pre-built container: CI builds an image carrying +the full scientific stack on every push and publishes it to the +[GitHub Container Registry](https://github.com/CosmoStat/sp_validation/pkgs/container/sp_validation). +The bundled `spv-container` CLI installs and manages your personal copy of it: ```bash -# build writeable "sandbox" container in the current directory -# ./sp_validation will be a directory that functions like a vm -apptainer build --sandbox sp_validation docker://ghcr.io/cosmostat/sp_validation:develop - -# open a shell in the container -apptainer shell --writable sp_validation -# and confirm that the installation was successful -python -c "import sp_validation" -``` +git clone https://github.com/CosmoStat/sp_validation.git +cd sp_validation +ln -s "$PWD/src/sp_validation/container.py" ~/.local/bin/spv-container -You can also run the image with **Docker**: - -```bash -docker run --rm -it ghcr.io/cosmostat/sp_validation:develop python -c "import sp_validation" +spv-container pull # fetch the image (~1.5 GB) +spv-container exec python -c "import sp_validation" # confirm it works ``` -We do not currently build images for Apple Silicon/arm64; however the amd64 images should work on these systems, albeit with reduced performance. +That is the whole install. `pull` puts the image at its canonical per-user +path (`~/.cache/sp_validation/`), and everything else finds it there — +`spv-container exec` for one-off commands (`spv-container exec bash` for an +interactive shell) and the Snakemake workflow for cluster jobs. +`spv-container status` says what you have and how current it is; +`spv-container sandbox` gives you a writable copy for mid-analysis +`pip install`s. On a cluster, run the pull from a compute node. + +To run the analysis workflow (`workflow/`), see +[`workflow/README.md`](workflow/README.md): Snakemake runs on the host, and +the profile puts each job in the container itself. For Docker, development +installs, and more depth, see the +[installation docs](https://cosmostat.github.io/sp_validation/installation.html). diff --git a/cosmo_inference/README.md b/cosmo_inference/README.md index 5d753010..e6f4ca83 100644 --- a/cosmo_inference/README.md +++ b/cosmo_inference/README.md @@ -4,15 +4,46 @@ by Lisa Goh and Sacha Guerrini, CEA Paris-Saclay This folder contains the files neccessary to run the cosmological inference pipeline on the UNIONS galaxy catalogues. ### Requirements -To run the pipeline, one would need to have installed [CosmoSIS](https://cosmosis.readthedocs.io/en/latest/). To sample the PSF leakage parameters, the fork of [cosmosis-standard-library](https://github.com/sachaguer/cosmosis-standard-library/) of Sacha Guerrini has to be used. +Everything the pipeline needs ships in the container: nothing to install, and no +paths to edit before a run. + +[CosmoSIS](https://cosmosis.readthedocs.io/en/latest/) comes in via the +`workflow` extra, built with MPI support. The CosmoSIS Standard Library — the +tree of modules the `.ini` pipelines name — is built into the image at +`/opt/cosmosis-standard-library`, with `CSL_DIR` pointing there. CosmoSIS reads +environment variables into an `.ini`'s `[DEFAULT]` section, so the templates' +`COSMOSIS_DIR = %(CSL_DIR)s` resolves to it. Outside the container, export +`CSL_DIR` at a build of your own and the same templates work unchanged. + +CSL is pinned to **Sacha Guerrini's fork** +([sachaguer/cosmosis-standard-library](https://github.com/sachaguer/cosmosis-standard-library/)) +at `b26fa7ff`, not to upstream: the UNIONS pipelines depend on four commits that +exist only there — tau statistics, `sample_S8`, and two z-dependent +linear-alignment modules. The fork is 4 commits ahead of +`cosmosis-developers/cosmosis-standard-library` and 373 behind it; carrying those +four forward onto current upstream is future work. + +Launch sampling under MPI (`mpiexec -n N cosmosis --mpi ...`), not `--smp`: +CosmoSIS's shared-memory pool is unmaintained and still crashes after sampling +completes (`Pool` has no attribute `data`, `runtime/process_pool.py`) as of +3.25.2. ### To Run -The inference pipeline is now orchestrated through Python. Run the main Snakemake workflow from the parent directory: +The inference pipeline is orchestrated through Snakemake. On the candide +cluster, drive it with the committed profile — see +[`workflow/README.md`](../workflow/README.md) for the one-time +`uv tool install` setup and the full explanation. From the repository root: ```bash -snakemake -j inference_fiducial +snakemake --profile workflow/profiles/candide \ + -s workflow/Snakefile \ + inference_fiducial --configfile ``` +Off-cluster, drop `--profile` and add `-j ` instead. Each job runs +inside the sp_validation container automatically — no `apptainer shell` or +`apptainer exec` needed by hand. + This will automatically execute all steps: 1. Calculate 2PCF ($\xi_{pm}$) via `cosmo_val.py` 2. Compute covariance matrices using CosmoCov diff --git a/cosmo_inference/cosmosis_config/templates/cosmosis_pipeline_A_ia.ini b/cosmo_inference/cosmosis_config/templates/cosmosis_pipeline_A_ia.ini index eb3ab166..656c6e5c 100644 --- a/cosmo_inference/cosmosis_config/templates/cosmosis_pipeline_A_ia.ini +++ b/cosmo_inference/cosmosis_config/templates/cosmosis_pipeline_A_ia.ini @@ -1,6 +1,8 @@ #parameters used elsewhere in this file [DEFAULT] -COSMOSIS_DIR = /n23data1/n06data/lgoh/scratch/cosmosis-standard-library_lisa +# The CosmoSIS Standard Library; CSL_DIR comes from the environment, set in the +# container (see cosmo_inference/README.md). +COSMOSIS_DIR = %(CSL_DIR)s [pipeline] diff --git a/cosmo_inference/cosmosis_config/templates/cosmosis_pipeline_A_ia_cell.ini b/cosmo_inference/cosmosis_config/templates/cosmosis_pipeline_A_ia_cell.ini index 87f06064..4a365195 100644 --- a/cosmo_inference/cosmosis_config/templates/cosmosis_pipeline_A_ia_cell.ini +++ b/cosmo_inference/cosmosis_config/templates/cosmosis_pipeline_A_ia_cell.ini @@ -1,6 +1,8 @@ #parameters used elsewhere in this file [DEFAULT] -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library +# The CosmoSIS Standard Library; CSL_DIR comes from the environment, set in the +# container (see cosmo_inference/README.md). +COSMOSIS_DIR = %(CSL_DIR)s [pipeline] diff --git a/cosmo_inference/cosmosis_config/templates/cosmosis_pipeline_A_psf.ini b/cosmo_inference/cosmosis_config/templates/cosmosis_pipeline_A_psf.ini index f9f4da51..341baf21 100644 --- a/cosmo_inference/cosmosis_config/templates/cosmosis_pipeline_A_psf.ini +++ b/cosmo_inference/cosmosis_config/templates/cosmosis_pipeline_A_psf.ini @@ -1,6 +1,8 @@ #parameters used elsewhere in this file [DEFAULT] -COSMOSIS_DIR = /home/guerrini/cosmosis-standard-library +# The CosmoSIS Standard Library; CSL_DIR comes from the environment, set in the +# container (see cosmo_inference/README.md). +COSMOSIS_DIR = %(CSL_DIR)s [pipeline] diff --git a/docs/source/installation.rst b/docs/source/installation.rst index f84acebc..56de599f 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -2,28 +2,59 @@ Installation ============ ``sp_validation`` is **not** distributed on PyPI. -Install it from a pre-built container, or check out the source with ``uv`` when you need to edit it. +It runs from a pre-built container, managed by the bundled ``spv-container`` CLI; check out the source with ``uv`` only when you need to edit the package itself. -Container (recommended) ------------------------ +Container via ``spv-container`` (recommended) +--------------------------------------------- -Every push to ``develop`` builds an image carrying the full scientific stack and pushes it to the `GitHub Container Registry (GHCR) -`_. +Every push builds an image carrying the full scientific stack and pushes it to the `GitHub Container Registry (GHCR) +`_, tagged by branch — ``:develop`` tracks the integration branch. The image runs on most systems, including HPC clusters, with no further setup. +``spv-container`` installs your personal copy of it and manages it from then on: -`Apptainer `_ (formerly Singularity) is installed on most clusters and is the path we recommend: +.. code-block:: bash + + git clone https://github.com/CosmoStat/sp_validation.git + cd sp_validation + ln -s "$PWD/src/sp_validation/container.py" ~/.local/bin/spv-container + + spv-container pull # fetch the image (~1.5 GB) + spv-container exec python -c "import sp_validation" # confirm it works + +The symlink works because ``container.py`` is deliberately stdlib-only: it runs on the *host*, where the science stack is not installed. +(Inside the container the same CLI is on ``PATH`` as a console script.) +``pull`` requires `Apptainer `_ (formerly Singularity), which is installed on most clusters, and writes the image to one canonical per-user path, ``~/.cache/sp_validation/sp_validation.sif``. +Each user owns their copy: you refresh it when you want to, and nobody else's refresh moves the ground under your running jobs. +On a cluster, run the pull from a compute node — it moves ~1.5 GB. + +The subcommands: .. code-block:: bash - # Build a writeable "sandbox" container in the current directory. - # ./sp_validation is a directory that behaves like a small VM. - apptainer build --sandbox sp_validation docker://ghcr.io/cosmostat/sp_validation:develop + spv-container pull # fetch the published image to the canonical path + spv-container status # what is here, which commit built it, how current + spv-container exec # run a command inside it (exec bash for a shell) + spv-container sandbox # unpack into a writable dir, for pip installs + spv-container exec --writable # ... with writes that persist + +``status`` compares the image's build commit against your checkout's ``HEAD``, so you always know whether a ``pull`` would refresh anything. +The **sandbox** is the escape hatch for exploratory work that needs a package the image does not carry yet: once built, it takes precedence over the SIF everywhere — Snakemake workflow jobs included — until you reset with ``spv-container pull`` + ``spv-container sandbox --force``. + +Everything resolves the image in one order — sandbox if it exists, else your SIF, else the registry tag — and that includes the analysis workflow. +How the workflow uses the image (Snakemake runs on the host; the profile puts each job in the container) is covered in ``workflow/README.md``. + +Other ways to run the image +--------------------------- + +The published image is a normal OCI image; ``spv-container`` is a convenience, not a gatekeeper. +Run it directly with Apptainer: + +.. code-block:: bash - # Open a shell in the container, then confirm the install works. - apptainer shell --writable sp_validation - python -c "import sp_validation" + apptainer pull sp_validation.sif docker://ghcr.io/cosmostat/sp_validation:develop + apptainer shell sp_validation.sif -The image also runs under Docker: +or with Docker: .. code-block:: bash diff --git a/papers/bmodes/Snakefile b/papers/bmodes/Snakefile index 0adf66a1..c58663d6 100644 --- a/papers/bmodes/Snakefile +++ b/papers/bmodes/Snakefile @@ -4,8 +4,6 @@ configfile: "config/config.yaml" configfile: "/n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_val/cat_config.yaml" -container: "/n17data/cdaley/containers/containers" - envvars: "PYTHONUNBUFFERED", @@ -27,6 +25,9 @@ import common common.configure(config) from common import * +# The one image every rule runs in; see common.resolve_container. +container: common.resolve_container(config.get("container")) + # Wildcard constraints — centralized in common.py, not in individual rule files wildcard_constraints: **WILDCARD_CONSTRAINTS diff --git a/papers/bmodes/rules/presentation.smk b/papers/bmodes/rules/presentation.smk index a9ef5dc1..c59f0aa3 100644 --- a/papers/bmodes/rules/presentation.smk +++ b/papers/bmodes/rules/presentation.smk @@ -59,7 +59,11 @@ rule talk_previews: rule talk_figure: - """Convert a single paper PDF to high-res PNG for the talk.""" + """Convert a single paper PDF to high-res PNG for the talk. + + `container: None` on purpose: ImageMagick's `convert` is a host tool and is + not installed in the sp_validation image. + """ input: pdf=lambda w: TALK_FIGURES[w.name], output: @@ -73,7 +77,11 @@ rule talk_figure: rule talk_figure_preview: - """Downscale a talk figure to < 1800px for safe AI reading.""" + """Downscale a talk figure to < 1800px for safe AI reading. + + `container: None` for the same reason as talk_figure: `convert` is a host + tool, absent from the image. + """ input: f"{TALK_DIR}/images/{{name}}.png", output: @@ -120,8 +128,6 @@ rule presentation_omega_m_difference: mock_summary="/n09data/guerrini/glass_mock_chains/summary_parameter_constraints_merged_v6.txt", output: f"{TALK_DIR}/images/omega_m_difference_config_harm.png", - container: - None shell: "python {TALK_DIR}/plot_omega_m_difference.py" @@ -132,8 +138,6 @@ rule presentation_s8_scatter_mocks: mock_summary="/n09data/guerrini/glass_mock_chains/summary_parameter_constraints_merged_v6.txt", output: f"{TALK_DIR}/images/s8_scatter_config_vs_harmonic.png", - container: - None shell: "python {TALK_DIR}/plot_s8_scatter_mocks.py" @@ -147,11 +151,5 @@ rule presentation_pte_cosebis: ], output: f"{TALK_DIR}/images/pte_cosebis_talk.png", - container: - None shell: - """ - apptainer exec --bind /home,/scratch,/automnt,/n17data,/n23data1,/n09data \ - /n17data/cdaley/containers/containers/ \ - python {TALK_DIR}/plot_pte_cosebis_talk.py - """ + "python {TALK_DIR}/plot_pte_cosebis_talk.py" diff --git a/papers/bmodes/rules/synthesis.smk b/papers/bmodes/rules/synthesis.smk index fb9208e1..64b60770 100644 --- a/papers/bmodes/rules/synthesis.smk +++ b/papers/bmodes/rules/synthesis.smk @@ -39,7 +39,7 @@ def _claim_outputs(): # Paper Macros # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -localrules: xi_cosmology_paper, paper_macros, bmodes_paper_spec, all_tapestry, unblinding_ceremony +localrules: xi_cosmology_paper, paper_macros, bmodes_paper_spec, all_tapestry rule xi_cosmology_paper: """Spec for B-mode reporting in configuration-space paper (Goh et al.). @@ -128,40 +128,6 @@ rule bmodes_paper_spec: # Aggregate Targets # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -_CEREMONY_BLIND = config.get("ceremony_blind", "A") -_CEREMONY_CHAIN_ROOT = "/n09data/guerrini/output_chains" -_CEREMONY_COSMOSIS_DIR = "/home/guerrini/sp_validation/cosmo_inference/data" - -rule unblinding_ceremony: - """Generate unblinding ceremony figure sequence. - - Via snakemake: - snakemake unblinding_ceremony --config ceremony_blind=B --nolock - Standalone: - app python workflow/scripts/unblinding_ceremony.py B - """ - input: - xi_data=f"{_CEREMONY_COSMOSIS_DIR}/{FIDUCIAL_VERSION}_{_CEREMONY_BLIND}/cosmosis_{FIDUCIAL_VERSION}_{_CEREMONY_BLIND}.fits", - pure_eb=f"results/paper_plots/intermediate/{FIDUCIAL_VERSION}_{_CEREMONY_BLIND}_pure_eb_semianalytic.npz", - pseudo_cl=_pseudo_cl_path(FIDUCIAL_VERSION, blind=_CEREMONY_BLIND), - pseudo_cl_cov=_pseudo_cl_cov_path(FIDUCIAL_VERSION, blind=_CEREMONY_BLIND), - cosmosis_cell_fits=f"{_CEREMONY_COSMOSIS_DIR}/{FIDUCIAL_VERSION}_{_CEREMONY_BLIND}_fid/cosmosis_{FIDUCIAL_VERSION}_{_CEREMONY_BLIND}_fid_cell.fits", - bestfit_dir=f"{_CEREMONY_CHAIN_ROOT}/best_fit/{FIDUCIAL_VERSION}_{_CEREMONY_BLIND}_10_80/shear_xi_plus/theta.txt", - output: - evidence=f"{TAPESTRY_DIR}/unblinding_ceremony/evidence.json", - params: - blind=_CEREMONY_BLIND, - chain_version=FIDUCIAL_VERSION.replace("SP_", "").replace("_leak_corr", ""), - chain_prefix=FIDUCIAL_VERSION, - chain_root_dir=_CEREMONY_CHAIN_ROOT, - results_dir="results/unblinding", - bestfit_root_fid_cell=f"{FIDUCIAL_VERSION}_{_CEREMONY_BLIND}_fid_cell", - bestfit_root_halofit_cell=f"{FIDUCIAL_VERSION}_{_CEREMONY_BLIND}_halofit_cell", - bestfit_root_config=f"{FIDUCIAL_VERSION}_{_CEREMONY_BLIND}_10_80", - shell: - "python workflow/scripts/unblinding_ceremony.py {params.blind} --chain-version {params.chain_version}" - - rule all_tapestry: """Aggregate target for all claim evidence and paper outputs.""" input: diff --git a/papers/bmodes/scripts/container_env.sh b/papers/bmodes/scripts/container_env.sh new file mode 100644 index 00000000..a63b40f5 --- /dev/null +++ b/papers/bmodes/scripts/container_env.sh @@ -0,0 +1,45 @@ +# Shared environment for the run_*.sh sweep drivers. Source, don't run: +# +# . "$(dirname "${BASH_SOURCE[0]}")/container_env.sh" +# +# Sets the checkout the drivers run out of, the container to run in, and +# `spv_python`, which is how every driver invokes python inside it. +WT=/n17data/cdaley/unions/code/sp_validation.worktrees/repro-paper-ii-astra +SRC=$WT/src +WSCRIPTS=$WT/workflow/scripts +PSCRIPTS=$WT/papers/bmodes/scripts + +# CONTAINER and BIND are resolved exactly as `sp_validation/container.py` does: +# the writable sandbox if there is one, else the SIF. +_spv_cache=${XDG_CACHE_HOME:-$HOME/.cache}/sp_validation +_spv_sandbox=${SPV_SANDBOX:-$_spv_cache/sandbox} +if [ -d "$_spv_sandbox" ]; then + CONTAINER=$_spv_sandbox +else + CONTAINER=${SPV_CONTAINER:-$_spv_cache/sp_validation.sif} +fi +BIND=${SPV_APPTAINER_BINDS:-/home,/scratch,/automnt,/n17data,/n23data1,/n09data} +unset _spv_cache _spv_sandbox + +# Every math library pinned to one thread -- pass as SPV_EXEC_EXTRA where the +# parallelism is by process, not by thread. +SINGLE_THREAD_ENV="--env OMP_NUM_THREADS=1 --env OPENBLAS_NUM_THREADS=1 + --env MKL_NUM_THREADS=1 --env NUMBA_NUM_THREADS=1 --env NUMEXPR_NUM_THREADS=1 + --env VECLIB_MAXIMUM_THREADS=1" + +# Run python inside the container against the checkout's src. Extra +# `apptainer exec` flags go in SPV_EXEC_EXTRA (word-split on purpose). +spv_python() { + apptainer exec --bind "$BIND" --env PYTHONPATH="$SRC" ${SPV_EXEC_EXTRA:-} \ + "$CONTAINER" /usr/local/bin/python "$@" +} + +# Echo the version list a sweep runs over: $VERSIONS if the caller set one, +# else whatever sweep_versions.py resolves from $1 (a config path). +sweep_versions() { + if [ -n "${VERSIONS:-}" ]; then + echo "$VERSIONS" + else + spv_python "$PSCRIPTS/sweep_versions.py" --config "$1" + fi +} diff --git a/papers/bmodes/scripts/filter_catalog_ellipticity.py b/papers/bmodes/scripts/filter_catalog_ellipticity.py index ade5dd6d..baab3ffb 100644 --- a/papers/bmodes/scripts/filter_catalog_ellipticity.py +++ b/papers/bmodes/scripts/filter_catalog_ellipticity.py @@ -18,7 +18,6 @@ sys.stderr if hasattr(sys, "ps1") else open(sys.stderr.fileno(), "w", buffering=1) ) -from snakemake.script import snakemake # noqa: E402 input_path = snakemake.input["catalog"] output_fits = snakemake.output["catalog"] diff --git a/papers/bmodes/scripts/plot_pure_eb_covariance.py b/papers/bmodes/scripts/plot_pure_eb_covariance.py index 3fdba8db..5522ef59 100644 --- a/papers/bmodes/scripts/plot_pure_eb_covariance.py +++ b/papers/bmodes/scripts/plot_pure_eb_covariance.py @@ -27,8 +27,6 @@ def _load_snakemake(): "results/paper_plots/pure_eb_covariance.png", str(Path.cwd()), ) - from snakemake.script import snakemake - return snakemake diff --git a/papers/bmodes/scripts/run_cl_sweep.py b/papers/bmodes/scripts/run_cl_sweep.py index 8cb2106e..d5fd6eb0 100644 --- a/papers/bmodes/scripts/run_cl_sweep.py +++ b/papers/bmodes/scripts/run_cl_sweep.py @@ -13,7 +13,7 @@ ``mask:`` entry), so no per-version mask wiring is needed here. Serial over versions; the estimator uses the recipe's full core allocation per call. - apptainer exec ... /usr/local/bin/python run_cl_sweep.py \ + python run_cl_sweep.py \ --config .../config.yaml --cat-config .../cat_config.yaml \ --nside 1024 --npatch 1 --binning powspace --nbins 32 --power 0.5 \ --blind A --out diff --git a/papers/bmodes/scripts/run_cosebis_ptes_sweep.py b/papers/bmodes/scripts/run_cosebis_ptes_sweep.py index 26b3d996..df3b9b79 100644 --- a/papers/bmodes/scripts/run_cosebis_ptes_sweep.py +++ b/papers/bmodes/scripts/run_cosebis_ptes_sweep.py @@ -14,7 +14,7 @@ config_space_pte_matrices.py adapts via ``_cosebis_matrix_from_npz`` — into ``--out``. Serial over versions (~25 min/version, 206 pairs). - apptainer exec ... /usr/local/bin/python run_cosebis_ptes_sweep.py \ + python run_cosebis_ptes_sweep.py \ --config .../config.yaml \ --xi-sweep-dir \ --cov-sweep-dir \ diff --git a/papers/bmodes/scripts/run_cov_sweep.sh b/papers/bmodes/scripts/run_cov_sweep.sh index 88d7f593..f86f20dc 100755 --- a/papers/bmodes/scripts/run_cov_sweep.sh +++ b/papers/bmodes/scripts/run_cov_sweep.sh @@ -26,12 +26,7 @@ # [--blind A] [--versions "v1 v2 ..."] set -euo pipefail -CONTAINER=/n17data/cdaley/containers/containers/ -WT=/n17data/cdaley/unions/code/sp_validation.worktrees/repro-paper-ii-astra -SRC=$WT/src -WSCRIPTS=$WT/workflow/scripts -PSCRIPTS=$WT/papers/bmodes/scripts -BIND=/home,/scratch,/automnt,/n17data,/n23data1,/n09data +. "$(dirname "${BASH_SOURCE[0]}")/container_env.sh" CONFIG=""; CATCONFIG=""; PLANCK18=""; MASKBASE=""; OUT=""; BLIND="A"; VERSIONS="" MINSEP=0.5; MAXSEP=300.0; NBINS=1000 @@ -50,10 +45,7 @@ done mkdir -p "$OUT" -if [ -z "$VERSIONS" ]; then - VERSIONS=$(apptainer exec --bind "$BIND" --env PYTHONPATH="$SRC" "$CONTAINER" \ - /usr/local/bin/python "$PSCRIPTS/sweep_versions.py" --config "$CONFIG") -fi +VERSIONS=$(sweep_versions "$CONFIG") for ver in $VERSIONS; do base="covariance_${ver}_${BLIND}_g_minsep=${MINSEP}_maxsep=${MAXSEP}_nbins=${NBINS}_masked" diff --git a/papers/bmodes/scripts/run_pure_eb_ptes_sweep.sh b/papers/bmodes/scripts/run_pure_eb_ptes_sweep.sh index 34ba5f7b..0ea3a862 100644 --- a/papers/bmodes/scripts/run_pure_eb_ptes_sweep.sh +++ b/papers/bmodes/scripts/run_pure_eb_ptes_sweep.sh @@ -19,12 +19,7 @@ # --out [--blind A] [--versions "v1 v2 ..."] set -euo pipefail -CONTAINER=/n17data/cdaley/containers/containers/ -WT=/n17data/cdaley/unions/code/sp_validation.worktrees/repro-paper-ii-astra -SRC=$WT/src -WSCRIPTS=$WT/workflow/scripts -PSCRIPTS=$WT/papers/bmodes/scripts -BIND=/home,/scratch,/automnt,/n17data,/n23data1,/n09data +. "$(dirname "${BASH_SOURCE[0]}")/container_env.sh" CONFIG=""; CATCONFIG=""; PUREEBSWEEP=""; COVSWEEP=""; OUT=""; BLIND="A"; VERSIONS="" while [ $# -gt 0 ]; do @@ -42,10 +37,7 @@ done mkdir -p "$OUT" -if [ -z "$VERSIONS" ]; then - VERSIONS=$(apptainer exec --bind "$BIND" --env PYTHONPATH="$SRC" "$CONTAINER" \ - /usr/local/bin/python "$PSCRIPTS/sweep_versions.py" --config "$CONFIG") -fi +VERSIONS=$(sweep_versions "$CONFIG") for ver in $VERSIONS; do pureeb="$PUREEBSWEEP/${ver}_${BLIND}_pure_eb_semianalytic.npz" @@ -55,8 +47,7 @@ for ver in $VERSIONS; do [ -f "$f" ] || { echo "MISSING upstream input for $ver: $f" >&2; exit 1; } done echo "[pure_eb_ptes_sweep] $ver" - apptainer exec --bind "$BIND" --env PYTHONPATH="$SRC" "$CONTAINER" \ - /usr/local/bin/python "$PSCRIPTS/calculate_pure_eb_ptes.py" \ + spv_python "$PSCRIPTS/calculate_pure_eb_ptes.py" \ --version "$ver" --blind "$BLIND" \ --pure-eb-data "$pureeb" --cov-integration "$covint" \ --npatch 1 --n-samples 2000 --out "$OUT" diff --git a/papers/bmodes/scripts/run_pure_eb_semianalytic.sh b/papers/bmodes/scripts/run_pure_eb_semianalytic.sh index 609f2a18..472cd81d 100644 --- a/papers/bmodes/scripts/run_pure_eb_semianalytic.sh +++ b/papers/bmodes/scripts/run_pure_eb_semianalytic.sh @@ -14,10 +14,7 @@ # --out [--n-chunks 20] [--n-samples 2000] [--nproc 16] set -euo pipefail -CONTAINER=/n17data/cdaley/containers/containers/ -SRC=/n17data/cdaley/unions/code/sp_validation.worktrees/repro-paper-ii-astra/src -SCRIPTS=/n17data/cdaley/unions/code/sp_validation.worktrees/repro-paper-ii-astra/papers/bmodes/scripts -BIND=/home,/scratch,/automnt,/n17data,/n23data1,/n09data +. "$(dirname "${BASH_SOURCE[0]}")/container_env.sh" VERSION=""; BLIND="A"; CATCONFIG=""; XIREP=""; XIINT=""; COVINT=""; OUT="" NCHUNKS=20; NSAMPLES=2000; NPROC="${SLURM_CPUS_PER_TASK:-16}" @@ -52,11 +49,8 @@ mkdir -p "$OUT/chunks" echo "[pure_eb] $NCHUNKS chunks, $NSAMPLES samples, nproc=$NPROC, version=$VERSION blind=$BLIND" for i in $(seq 0 $((NCHUNKS-1))); do ( - apptainer exec --bind "$BIND" --env PYTHONPATH="$SRC" \ - --env OMP_NUM_THREADS=1 --env OPENBLAS_NUM_THREADS=1 --env MKL_NUM_THREADS=1 \ - --env NUMBA_NUM_THREADS=1 --env NUMEXPR_NUM_THREADS=1 --env VECLIB_MAXIMUM_THREADS=1 \ - "$CONTAINER" \ - /usr/local/bin/python "$SCRIPTS/precompute_pure_eb_chunk.py" \ + SPV_EXEC_EXTRA=$SINGLE_THREAD_ENV + spv_python "$PSCRIPTS/precompute_pure_eb_chunk.py" \ --chunk-id "$i" --n-chunks "$NCHUNKS" --n-samples "$NSAMPLES" \ --version "$VERSION" --blind "$BLIND" --cat-config "$CATCONFIG" \ --xi-reporting "$XIREP" --xi-integration "$XIINT" --cov-integration "$COVINT" \ @@ -76,8 +70,7 @@ done [ "$missing" -eq 0 ] || { echo "[pure_eb] chunk failures — aborting gather" >&2; exit 1; } echo "[pure_eb] all $NCHUNKS chunks done; gathering" -apptainer exec --bind "$BIND" --env PYTHONPATH="$SRC" "$CONTAINER" \ - /usr/local/bin/python "$SCRIPTS/gather_pure_eb_chunks.py" \ +spv_python "$PSCRIPTS/gather_pure_eb_chunks.py" \ --version "$VERSION" --blind "$BLIND" \ --xi-reporting "$XIREP" --xi-integration "$XIINT" \ --chunks-dir "$OUT/chunks" \ diff --git a/papers/bmodes/scripts/run_pure_eb_sweep.sh b/papers/bmodes/scripts/run_pure_eb_sweep.sh index 74b6dab3..8195f150 100755 --- a/papers/bmodes/scripts/run_pure_eb_sweep.sh +++ b/papers/bmodes/scripts/run_pure_eb_sweep.sh @@ -19,12 +19,7 @@ # --out [--blind A] [--versions "v1 v2 ..."] set -euo pipefail -CONTAINER=/n17data/cdaley/containers/containers/ -WT=/n17data/cdaley/unions/code/sp_validation.worktrees/repro-paper-ii-astra -SRC=$WT/src -WSCRIPTS=$WT/workflow/scripts -PSCRIPTS=$WT/papers/bmodes/scripts -BIND=/home,/scratch,/automnt,/n17data,/n23data1,/n09data +. "$(dirname "${BASH_SOURCE[0]}")/container_env.sh" CONFIG=""; CATCONFIG=""; XISWEEP=""; COVSWEEP=""; OUT=""; BLIND="A"; VERSIONS="" while [ $# -gt 0 ]; do @@ -42,10 +37,7 @@ done mkdir -p "$OUT" -if [ -z "$VERSIONS" ]; then - VERSIONS=$(apptainer exec --bind "$BIND" --env PYTHONPATH="$SRC" "$CONTAINER" \ - /usr/local/bin/python "$PSCRIPTS/sweep_versions.py" --config "$CONFIG") -fi +VERSIONS=$(sweep_versions "$CONFIG") for ver in $VERSIONS; do xirep="$XISWEEP/${ver}_xi_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.txt" diff --git a/papers/bmodes/scripts/run_xi_sweep.py b/papers/bmodes/scripts/run_xi_sweep.py index b02283a5..dd7219a3 100644 --- a/papers/bmodes/scripts/run_xi_sweep.py +++ b/papers/bmodes/scripts/run_xi_sweep.py @@ -13,7 +13,7 @@ binning. Serial over versions — lc's dask handles cross-output concurrency, and TreeCorr already uses the recipe's full OpenMP allocation per call. - apptainer exec ... /usr/local/bin/python run_xi_sweep.py \ + python run_xi_sweep.py \ --config .../config.yaml --cat-config .../cat_config.yaml --out """ diff --git a/papers/bmodes/scripts/unblinding_ceremony.py b/papers/bmodes/scripts/unblinding_ceremony.py deleted file mode 100644 index 61ab3a95..00000000 --- a/papers/bmodes/scripts/unblinding_ceremony.py +++ /dev/null @@ -1,1389 +0,0 @@ -#!/usr/bin/env python3 -"""Run the UNIONS unblinding ceremony figure sequence. - -Usage (standalone) ------------------- -app python workflow/scripts/unblinding_ceremony.py A -app python workflow/scripts/unblinding_ceremony.py A --output-dir /path/to/output - -Usage (snakemake) ------------------ -snakemake unblinding_ceremony --config ceremony_blind=A -""" - -import argparse -import json -import shutil -import sys -from dataclasses import dataclass -from datetime import datetime, timezone -from pathlib import Path - -_SCRIPT_DIR = Path(__file__).resolve().parent -if str(_SCRIPT_DIR) not in sys.path: - sys.path.insert(0, str(_SCRIPT_DIR)) - -import matplotlib.pyplot as plt -import numpy as np -from astropy.io import fits -from getdist import plots -from matplotlib import scale as mscale -from matplotlib.gridspec import GridSpec -from plotting_utils import PAPER_MPLSTYLE, SquareRootScale -from scipy.interpolate import interp1d - -# ── Plotting environment ──────────────────────────────────────────────────── -mscale.register_scale(SquareRootScale) -plt.style.use(PAPER_MPLSTYLE) -plt.rc("text", usetex=True) - - -# ── Data types ────────────────────────────────────────────────────────────── - - -@dataclass(frozen=True) -class ChainSpec: - root: str - label: str - color: str - base_dir: Path - alpha: float = 1.0 - - -FULL_PARAMS = [ - "OMEGA_M", - "ombh2", - "h0", - "n_s", - "SIGMA_8", - "s_8_input", - "logt_agn", - "a", - "m1", - "bias_1", -] -COSMO_PARAMS = ["OMEGA_M", "s_8_input", "SIGMA_8", "a"] - -MUTED_ALPHA = 0.25 - - -@dataclass -class CeremonyConfig: - """All paths and parameters needed by the ceremony, sourced from snakemake or CLI.""" - - blind: str - chain_version: str - chain_prefix: str - chain_root_dir: Path - external_root_dir: Path - results_dir: Path - evidence_dir: Path - - xi_data_path: Path - pure_eb_path: Path - pseudo_cl_path: Path - pseudo_cl_cov_path: Path - cosmosis_cell_fits: Path - bestfit_dir: Path - bestfit_root_fid_cell: str - bestfit_root_halofit_cell: str - bestfit_root_config: str - - -def _save_path(cfg: CeremonyConfig, index: int, slug: str) -> Path: - return cfg.results_dir / f"{index:02d}_{slug}.pdf" - - -# ── Chain loading (extracted from Sasha's notebooks) ──────────────────────── - - -def _load_xi_table(path: Path | str, nrows: int = 20) -> np.ndarray: - rows: list[np.ndarray] = [] - with Path(path).open("r", encoding="utf-8") as handle: - for line in handle: - stripped = line.strip() - if not stripped or stripped.startswith("#"): - continue - values = np.fromstring(stripped, sep=" ") - if values.size < 9: - continue - rows.append(values) - if len(rows) >= nrows: - break - if len(rows) < nrows: - raise ValueError( - f"Expected at least {nrows} xi rows in {path}, found {len(rows)}" - ) - return np.vstack(rows) - - -def ensure_getdist_chain(base_dir: Path, root: str) -> Path: - """MAKE PARAMNAMES FILE + READ CHAIN conversion (from notebooks).""" - chain_dir = base_dir / root - samples_path = chain_dir / f"samples_{root}.txt" - gd_samples_path = chain_dir / f"getdist_{root}.txt" - paramnames_path = chain_dir / f"getdist_{root}.paramnames" - - if not samples_path.exists(): - return gd_samples_path - - with samples_path.open("r", encoding="utf-8") as file: - params = file.readline()[1:].split("\t")[:-4] - - with paramnames_path.open("w", encoding="utf-8") as file: - for param in params: - if len(param.split("--")) > 1: - file.write(param.split("--")[1] + "\n") - else: - file.write(param.split("--")[0] + "\n") - - samples = np.loadtxt(samples_path) - if "nautilus" in root: - samples = np.column_stack( - (np.exp(samples[:, -3]), samples[:, -1] - samples[:, -2], samples[:, 0:-3]) - ) - else: - samples = np.column_stack((samples[:, -1], samples[:, -3], samples[:, 0:-4])) - np.savetxt(gd_samples_path, samples) - return gd_samples_path - - -def _build_plotter( - width_inch: float, - axes_fontsize: float, - axes_labelsize: float, - legend_fontsize: float, -): - g = plots.get_subplot_plotter(width_inch=width_inch) - g.settings.axes_fontsize = axes_fontsize - g.settings.axes_labelsize = axes_labelsize - g.settings.alpha_filled_add = 0.7 - g.settings.legend_fontsize = legend_fontsize - return g - - -def _set_param_labels(chain) -> None: - name_list = [ - "OMEGA_M", - "ombh2", - "h0", - "n_s", - "SIGMA_8", - "S_8", - "s_8_input", - "logt_agn", - "a", - "m1", - "bias_1", - ] - label_list = [ - r"\Omega_{\rm m}", - r"\omega_{\rm b} h^2", - r"h_0", - r"n_{\rm s}", - r"\sigma_8", - r"S_8", - r"S_8", - r"\log T_{\rm AGN}", - r"A_{\rm IA}", - r"m_1", - r"\Delta z_1", - ] - - param_names = chain.getParamNames() - for name, label in zip(name_list, label_list): - try: - param_names.parWithName(name).label = label - except Exception: - pass - - try: - param_names.parWithName("S_8") - except Exception: - try: - s8_input = chain.getParams().s_8_input - chain.addDerived(s8_input, name="S_8", label=r"S_8") - except Exception: - pass - - -def _adjust_paramname_chain( - chain, current_name: str, target_name: str, label: str -) -> None: - try: - param_names = chain.getParamNames() - par = param_names.parWithName(current_name) - par.label = label - par.name = target_name - chain.setParamNames(param_names) - except Exception: - pass - - -def _derive_parameter_s8(chain): - if "S_8" in chain.getParamNames().list(): - return chain - omega_m = chain.getParams().OMEGA_M - sigma_8 = chain.getParams().SIGMA_8 - s_8 = sigma_8 * (omega_m / 0.3) ** 0.5 - chain.addDerived(s_8, name="S_8", label=r"S_8") - return chain - - -def _harmonize_external_chain(chain, root: str) -> None: - if root in {"Planck18", "KiDS-1000", "HSC_Y3", "HSC_Y3_cell", "DES+KiDS", "DES_Y3"}: - _adjust_paramname_chain(chain, "omega_m", "OMEGA_M", r"\Omega_{\rm m}") - if root == "DES_Y3": - _derive_parameter_s8(chain) - - -def load_getdist_chains( - chain_specs: list[ChainSpec], - width_inch: float, - axes_fontsize: float, - axes_labelsize: float, - legend_fontsize: float, -): - g = _build_plotter( - width_inch=width_inch, - axes_fontsize=axes_fontsize, - axes_labelsize=axes_labelsize, - legend_fontsize=legend_fontsize, - ) - - chains = [] - for spec in chain_specs: - ensure_getdist_chain(spec.base_dir, spec.root) - chain = g.samples_for_root( - str(spec.base_dir / spec.root / f"getdist_{spec.root}"), - cache=False, - settings={"ignore_rows": 0, "smooth_scale_2D": 0.5, "smooth_scale_1D": 0.5}, - ) - _set_param_labels(chain) - if spec.base_dir.name == "ext_data" or spec.root in { - "Planck18", - "DES_Y3", - "KiDS-1000", - "DES+KiDS", - "HSC_Y3", - "HSC_Y3_cell", - }: - _harmonize_external_chain(chain, spec.root) - chains.append(chain) - - return g, chains - - -# ── Plot functions (extracted from Sasha's notebooks) ─────────────────────── - - -def plot_triangle( - chain_specs: list[ChainSpec], - param_names: list[str], - output_path: Path, - width_inch: float = 20.0, - axes_fontsize: float = 35.0, - axes_labelsize: float = 50.0, - legend_fontsize: float = 40.0, - legend_loc: str = "upper right", -) -> None: - """Extracted triangle_plot pattern from contour notebooks.""" - g, chains = load_getdist_chains( - chain_specs, width_inch, axes_fontsize, axes_labelsize, legend_fontsize - ) - - colours = [spec.color for spec in chain_specs] - linestyle = ["solid" for _ in chain_specs] - line_args = [dict(color=col, ls=ls) for col, ls in zip(colours, linestyle)] - - g.triangle_plot( - chains, - param_names, - legend_labels=[spec.label for spec in chain_specs], - line_args=line_args, - contour_colors=colours, - legend_loc=legend_loc, - filled=True, - ) - - output_path.parent.mkdir(parents=True, exist_ok=True) - g.export(str(output_path)) - plt.close(g.fig) - - -def plot_xipm_data_vector(xipm_path: str, output_path: Path) -> None: - """Extracted from 2D_cosmic_shear_paper_plots/corr_func.ipynb (xi+ / xi- blocks).""" - xipm = _load_xi_table(xipm_path, nrows=20) - theta = xipm[:, 1] - xip = xipm[:, 3] - xim = xipm[:, 4] - varxip = xipm[:, 7] - varxim = xipm[:, 8] - - fig, (ax1, ax2) = plt.subplots(ncols=2, nrows=1, figsize=(10, 4.5)) - - ax1.tick_params( - axis="both", - which="both", - direction="in", - length=6, - width=1, - top=True, - bottom=True, - left=True, - right=True, - ) - ax1.yaxis.minorticks_on() - ax1.plot( - theta, - xip * 1e4, - marker="o", - markersize=4, - ls="solid", - lw=1.8, - color="royalblue", - ) - ax1.fill_between( - theta, (xip - varxip) * 1e4, (xip + varxip) * 1e4, color="powderblue", alpha=0.7 - ) - ax1.text( - 0.85, - 0.88, - "1-1", - transform=ax1.transAxes, - bbox=dict(facecolor="white", edgecolor="black", boxstyle="round", pad=0.5), - ) - ax1.axvspan(0, 10, color="gray", alpha=0.3) - ax1.axvspan(150, 200, color="gray", alpha=0.3) - ax1.set_xscale("log") - ax1.set_xlabel(r"$\theta$ [arcmin]") - ax1.set_ylabel(r"$\xi_+\times 10^4$") - - ax2.tick_params( - axis="both", - which="both", - direction="in", - length=6, - width=1, - top=True, - bottom=True, - left=True, - right=True, - ) - ax2.yaxis.minorticks_on() - ax2.plot( - theta, - xim * 1e4, - marker="o", - markersize=4, - ls="solid", - lw=1.8, - color="orangered", - ) - ax2.fill_between( - theta, (xim - varxim) * 1e4, (xim + varxim) * 1e4, color="pink", alpha=0.7 - ) - ax2.text( - 0.85, - 0.88, - "1-1", - transform=ax2.transAxes, - bbox=dict(facecolor="white", edgecolor="black", boxstyle="round", pad=0.5), - ) - ax2.axvspan(0, 10, color="gray", alpha=0.3) - ax2.axvspan(150, 200, color="gray", alpha=0.3) - ax2.set_xscale("log") - ax2.set_xlabel(r"$\theta$ [arcmin]") - ax2.set_ylabel(r"$\xi_-\times 10^4$") - - fig.tight_layout() - output_path.parent.mkdir(parents=True, exist_ok=True) - fig.savefig(output_path, dpi=300, bbox_inches="tight") - plt.close(fig) - - -def plot_cell_ee_data_vector( - pseudo_cl_path: str, pseudo_cl_cov_path: str, output_path: Path -) -> None: - """Extracted from 2025_10_08_plot_data_vectors.py (EE panel logic).""" - cell = fits.getdata(pseudo_cl_path) - cov_cell = fits.open(pseudo_cl_cov_path) - - ell = cell["ell"] - cl_ee = cell["EE"] - cov_cl_ee = cov_cell["COVAR_EE_EE"].data - cov_cell.close() - - fig, ax0 = plt.subplots(ncols=1, nrows=1, figsize=(7, 5)) - ax0.errorbar( - ell, - cl_ee * ell, - yerr=np.sqrt(np.diag(cov_cl_ee)) * ell, - label=r"$C_\ell^{EE}$", - color="royalblue", - fmt="o", - capsize=2, - ) - - ax0.set_xscale("squareroot") - ax0.set_xticks(np.array([100, 400, 900, 1600])) - ax0.minorticks_on() - ax0.tick_params(axis="x", which="minor", length=2, width=0.8) - minor_ticks = [i * 10 for i in range(1, 10)] + [i * 100 for i in range(1, 21)] - ax0.set_xticks(minor_ticks, minor=True) - ax0.legend() - - ax0.set_xlabel(r"$\ell$") - ax0.set_ylabel(r"$\ell \, C_\ell^{EE}$") - - plt.tight_layout() - output_path.parent.mkdir(parents=True, exist_ok=True) - plt.savefig(output_path, dpi=300, bbox_inches="tight") - plt.close(fig) - - -def plot_xipm_bestfit_with_bmodes( - xi_data_path: Path | str, - pure_eb_data_path: Path, - bestfit_dir: Path | None, - output_path: Path, - scale_cut_xip: tuple[float, float] = (12.0, 83.0), - scale_cut_xim: tuple[float, float] = (12.0, 83.0), -) -> None: - """Extracted from 2D_cosmic_shear_paper_plots/workflow/scripts/plot_xi_bestfit.py. - - If bestfit_dir is None, plots data + B-modes only (no theory curves). - - xi_data_path can be: - - A CosmoSIS FITS file with XI_PLUS/XI_MINUS HDUs and COVMAT (preferred) - - A plain-text TreeCorr output table (legacy) - """ - xi_path = Path(xi_data_path) - if xi_path.suffix == ".fits": - xip_hdu = fits.getdata(str(xi_path), "XI_PLUS") - xim_hdu = fits.getdata(str(xi_path), "XI_MINUS") - cov = fits.getdata(str(xi_path), "COVMAT") - theta_data = xip_hdu["ANG"] - xip_data = xip_hdu["VALUE"] - xim_data = xim_hdu["VALUE"] - n = len(xip_data) - sigma_xip = np.sqrt(np.diag(cov[:n, :n])) - sigma_xim = np.sqrt(np.diag(cov[n : 2 * n, n : 2 * n])) - else: - data = _load_xi_table(xi_path, nrows=20) - theta_data = data[:, 1] - xip_data = data[:, 3] - xim_data = data[:, 4] - sigma_xip = data[:, 7] - sigma_xim = data[:, 8] - - eb_data = np.load(pure_eb_data_path) - theta_eb = eb_data["theta"] - xip_B = eb_data["xip_B"] - xim_B = eb_data["xim_B"] - cov_pure_eb = eb_data["cov_pure_eb"] - - nbins = len(theta_eb) - sigma_xip_B = np.sqrt( - np.diag(cov_pure_eb[2 * nbins : 3 * nbins, 2 * nbins : 3 * nbins]) - ) - sigma_xim_B = np.sqrt( - np.diag(cov_pure_eb[3 * nbins : 4 * nbins, 3 * nbins : 4 * nbins]) - ) - - min_sep, max_sep = 1.0, 250.0 - bin_edges = np.geomspace(min_sep, max_sep, nbins + 1) - bin_centers_nominal = np.sqrt(bin_edges[:-1] * bin_edges[1:]) - - def get_bin_edge_cuts(centers, edges, scale_cut): - mask = (centers >= scale_cut[0]) & (centers <= scale_cut[1]) - idx_first = np.where(mask)[0][0] - idx_last = np.where(mask)[0][-1] - return edges[idx_first], edges[idx_last + 1] - - edge_cut_xip = get_bin_edge_cuts(bin_centers_nominal, bin_edges, scale_cut_xip) - edge_cut_xim = get_bin_edge_cuts(bin_centers_nominal, bin_edges, scale_cut_xim) - - has_theory = bestfit_dir is not None - if has_theory: - theta_theory_rad = np.loadtxt( - bestfit_dir / "shear_xi_plus" / "theta.txt", comments="#" - ) - theta_theory = np.rad2deg(theta_theory_rad) * 60 - xip_theory = np.loadtxt( - bestfit_dir / "shear_xi_plus" / "bin_1_1.txt", comments="#" - ) - xim_theory = np.loadtxt( - bestfit_dir / "shear_xi_minus" / "bin_1_1.txt", comments="#" - ) - - theta_sys_rad = np.loadtxt(bestfit_dir / "xi_sys" / "theta.txt", comments="#") - theta_sys = np.rad2deg(theta_sys_rad) * 60 - xip_sys = np.loadtxt(bestfit_dir / "xi_sys" / "shear_xi_plus.txt", comments="#") - xim_sys = np.loadtxt( - bestfit_dir / "xi_sys" / "shear_xi_minus.txt", comments="#" - ) - - theta_fine = np.geomspace(0.5, 300, 500) - if has_theory: - xip_th_interp = interp1d( - theta_theory, xip_theory, kind="cubic", fill_value="extrapolate" - )(theta_fine) - xim_th_interp = interp1d( - theta_theory, xim_theory, kind="cubic", fill_value="extrapolate" - )(theta_fine) - xip_sys_interp = interp1d( - theta_sys, xip_sys, kind="cubic", fill_value="extrapolate" - )(theta_fine) - xim_sys_interp = interp1d( - theta_sys, xim_sys, kind="cubic", fill_value="extrapolate" - )(theta_fine) - - scale_factor = 1e-4 - xlim = [1, 250] - ylim = [-0.15, 1.25] - - ms_data = 3 - ms_bmode = 3 - capsize = 1.5 - elinewidth = 0.8 - - fig, axes = plt.subplots(1, 2, figsize=(10, 4.5), sharey=True) - - plot_configs = [ - ( - axes[0], - xip_data, - sigma_xip, - xip_B, - sigma_xip_B, - xip_th_interp if has_theory else None, - xip_sys_interp if has_theory else None, - edge_cut_xip, - r"$\xi_+$", - "+", - ), - ( - axes[1], - xim_data, - sigma_xim, - xim_B, - sigma_xim_B, - xim_th_interp if has_theory else None, - xim_sys_interp if has_theory else None, - edge_cut_xim, - r"$\xi_-$", - "-", - ), - ] - - for idx, ( - ax, - xi_data_arr, - sigma_xi, - xi_B, - sigma_B, - xi_th, - xi_sys_arr, - edge_cut, - label, - _pm, - ) in enumerate(plot_configs): - show_legend = idx == 1 - - ax.axvspan(xlim[0], edge_cut[0], color="0.90", zorder=0, alpha=0.7) - ax.axvspan(edge_cut[1], xlim[1], color="0.90", zorder=0, alpha=0.7) - - if has_theory: - ax.plot( - theta_fine, - theta_fine * (xi_th + xi_sys_arr) / scale_factor, - "-", - color="k", - lw=1.5, - label=r"Best-fit $\xi^{\mathrm{th}}_\pm + \xi^{\mathrm{sys}}_\pm$" - if show_legend - else None, - zorder=2, - ) - ax.plot( - theta_fine, - theta_fine * xi_sys_arr / scale_factor, - "-", - color="C0", - lw=1.2, - label=r"Best-fit $\xi^{\mathrm{sys}}_\pm$" if show_legend else None, - zorder=2, - ) - - ax.errorbar( - theta_data, - theta_data * xi_data_arr / scale_factor, - yerr=theta_data * sigma_xi / scale_factor, - fmt="o", - color="k", - markersize=ms_data, - capsize=capsize, - elinewidth=elinewidth, - label=r"$\xi_\pm$" if show_legend else None, - zorder=3, - ) - - theta_eb_offset = theta_eb * 1.03 - ax.errorbar( - theta_eb_offset, - theta_eb_offset * xi_B / scale_factor, - yerr=theta_eb_offset * sigma_B / scale_factor, - fmt="o", - color="C3", - markersize=ms_bmode, - capsize=capsize, - elinewidth=elinewidth, - alpha=0.85, - label=r"$\xi^B_\pm$" if show_legend else None, - zorder=3, - ) - - ax.axhline(0, color="gray", linestyle="--", alpha=0.8, linewidth=0.8, zorder=1) - ax.set_xscale("log") - ax.set_xlim(xlim) - ax.set_ylim(ylim) - ax.set_xlabel(r"$\theta$ (arcmin)") - ax.set_title(label) - if show_legend: - ax.legend(loc="upper left") - - axes[0].set_ylabel(r"$\theta\xi \times 10^4$") - - fig.tight_layout() - output_path.parent.mkdir(parents=True, exist_ok=True) - fig.savefig(output_path, dpi=150, bbox_inches="tight") - plt.close(fig) - - -def _get_stats_row(chain, label: str, color: str) -> list[str | float]: - margestats = chain.getMargeStats() - try: - s8_stats = margestats.parWithName("S_8") - except Exception: - s8_stats = margestats.parWithName("s_8_input") - sigma8_stats = margestats.parWithName("SIGMA_8") - omegam_stats = margestats.parWithName("OMEGA_M") - return [ - label, - color, - s8_stats.mean, - s8_stats.mean - s8_stats.limits[0].lower, - s8_stats.limits[0].upper - s8_stats.mean, - sigma8_stats.mean, - sigma8_stats.mean - sigma8_stats.limits[0].lower, - sigma8_stats.limits[0].upper - sigma8_stats.mean, - omegam_stats.mean, - omegam_stats.mean - omegam_stats.limits[0].lower, - omegam_stats.limits[0].upper - omegam_stats.mean, - ] - - -def get_sigma_tension(mean1, low1, high1, mean2, low2, high2): - sigma1 = 0.5 * (high1 + low1) - sigma2 = 0.5 * (high2 + low2) - delta_mean = np.abs(mean1 - mean2) - sigma_tension = delta_mean / np.sqrt(sigma1**2 + sigma2**2) - sign = 1 if mean1 > mean2 else -1 - return sigma_tension * sign - - -def plot_cell_ee_with_bestfit( - cosmosis_data_path: str, - bestfit_specs: list[tuple[str, str, dict]], - output_folder: str, - output_path: Path, - ell_min: float = 10.0, - ell_max: float = 2048.0, - label_data: str = "Fiducial data", -) -> None: - """Extracted from get_chi2_cell.ipynb plot_best_fit() (cell 21-22). - - Parameters - ---------- - cosmosis_data_path : str - Path to CosmoSIS FITS file with CELL_EE and COVMAT HDUs. - bestfit_specs : list of (label, root, line_args_dict) - Each entry is (legend label, chain root name, dict of plot kwargs). - output_folder : str - Base path to best_fit directories (e.g. /n09data/guerrini/output_chains/). - output_path : Path - Where to save the figure. - """ - data = fits.getdata(cosmosis_data_path, "CELL_EE") - cov_mat = fits.getdata(cosmosis_data_path, "COVMAT") - - fig, ax = plt.subplots(1, 1, figsize=(8, 5)) - - ell = data["ANG"] - cell = data["VALUE"] - ax.errorbar( - ell, - ell * cell, - yerr=ell * np.sqrt(np.diag(cov_mat)), - fmt="o", - label=label_data, - color="black", - capsize=2, - ) - - for label, root, line_kw in bestfit_specs: - ell_th = np.loadtxt(f"{output_folder}/best_fit/{root}/shear_cl/ell.txt") - shear_cl = np.loadtxt(f"{output_folder}/best_fit/{root}/shear_cl/bin_1_1.txt") - mask = (ell_th > ell_min) & (ell_th < ell_max) - ax.plot(ell_th[mask], ell_th[mask] * shear_cl[mask], label=label, **line_kw) - - ax.axvline(x=1800, color="black", linestyle="--", alpha=0.5) - ax.axvline(x=2048, color="black", linestyle="--", alpha=1.0) - ax.axvline(x=500, color="black", linestyle="--", alpha=0.3) - - ax.text( - 1740, - 0.90, - r"$k_\mathrm{max} = 3 h$ Mpc$^{-1}$", - transform=ax.get_xaxis_transform(), - ha="center", - va="top", - fontsize=10, - rotation=90, - ) - ax.text( - 1978, - 0.90, - r"$k_\mathrm{max} = 5 h$ Mpc$^{-1}$", - transform=ax.get_xaxis_transform(), - ha="center", - va="top", - fontsize=10, - rotation=90, - ) - ax.text( - 470, - 0.90, - r"$k_\mathrm{max} = 1 h$ Mpc$^{-1}$", - transform=ax.get_xaxis_transform(), - ha="center", - va="top", - fontsize=10, - rotation=90, - ) - - ax.set_ylabel(r"$\ell C_\ell$", fontsize=16) - ax.set_xlabel(r"$\ell$", fontsize=16) - ax.set_xlim(ell.min() - 10, ell.max() + 100) - ax.set_xscale("squareroot") - ax.set_xticks(np.array([100, 400, 900, 1600])) - ax.minorticks_on() - ax.tick_params(axis="x", which="minor", length=2, width=0.8) - minor_ticks = [i * 10 for i in range(1, 10)] + [i * 100 for i in range(1, 21)] - ax.xaxis.set_ticks(minor_ticks, minor=True) - ax.tick_params(axis="both", which="major", labelsize=14) - ax.tick_params(axis="both", which="minor", labelsize=10) - ax.yaxis.get_offset_text().set_fontsize(14) - - plt.legend(loc="lower center", bbox_to_anchor=(0.685, 0.70), fontsize=12) - - output_path.parent.mkdir(parents=True, exist_ok=True) - plt.savefig(output_path, bbox_inches="tight") - plt.close(fig) - - -def plot_s8_whisker( - chain_specs: list[ChainSpec], - output_path: Path, - reference_labels: list[str] | None = None, - reference_colors: list[str] | None = None, - reference_label: str | None = None, -) -> None: - """Extracted from 2025_10_28_plot_whisker.ipynb (cells 5-11). - - Supports multiple reference bands: pass reference_labels and - reference_colors as parallel lists. Each reference gets its own - shaded band in the corresponding color. For backwards compatibility, - a single reference_label still works. - """ - if reference_labels is None and reference_label is not None: - reference_labels = [reference_label] - reference_colors = reference_colors or [None] - - g, chains = load_getdist_chains( - chain_specs, - width_inch=30, - axes_fontsize=60, - axes_labelsize=60, - legend_fontsize=60, - ) - plt.close(g.fig) - - labels = [spec.label for spec in chain_specs] - colours = [spec.color for spec in chain_specs] - alphas = [spec.alpha for spec in chain_specs] - - param_values = np.array( - [ - [ - "# Expt", - "Colour", - "S8_Mean", - "S8_low", - "S8_high", - "sigma_8_Mean", - "sigma_8_low", - "sigma_8_high", - "Omega_m_Mean", - "Omega_m_low", - "Omega_m_high", - ] - ], - dtype=object, - ) - - escaped_labels = np.char.replace(np.array(labels), "\\", "\\\\") - for i, chain in enumerate(chains): - row = _get_stats_row(chain, escaped_labels[i], colours[i]) - param_values = np.vstack((param_values, row)) - - expt = np.char.replace(param_values[1:, 0].astype(str), "\\\\", "\\") - colours_arr = param_values[1:, 1].astype(str) - s8_mean = param_values[1:, 2].astype(np.float64) - s8_low = param_values[1:, 3].astype(np.float64) - s8_high = param_values[1:, 4].astype(np.float64) - sigma8_mean = param_values[1:, 5].astype(np.float64) - sigma8_low = param_values[1:, 6].astype(np.float64) - sigma8_high = param_values[1:, 7].astype(np.float64) - omegam_mean = param_values[1:, 8].astype(np.float64) - omegam_low = param_values[1:, 9].astype(np.float64) - omegam_high = param_values[1:, 10].astype(np.float64) - - ref_indices = [] - for rl in reference_labels or []: - matches = np.where(expt == rl)[0] - if len(matches): - ref_indices.append(matches[0]) - ref_label_set = set(reference_labels or []) - - n_rows = len(expt) - fig_height = max(6, 0.5 * n_rows + 1) - fig = plt.figure(figsize=(10, fig_height)) - gs = GridSpec(1, 3, width_ratios=[1, 0.5, 0.5]) - ax1 = fig.add_subplot(gs[0]) - ax2 = fig.add_subplot(gs[1], sharey=ax1) - ax3 = fig.add_subplot(gs[2], sharey=ax1) - - axs = [ax1, ax2, ax3] - - params = [ - (s8_mean, s8_low, s8_high, r"$S_8$"), - (sigma8_mean, sigma8_low, sigma8_high, r"$\sigma_8$"), - (omegam_mean, omegam_low, omegam_high, r"$\Omega_{\rm m}$"), - ] - - row_spacing = 0.1 - y = np.arange(len(expt)) - - for ax, param in zip(axs, params): - means, lows, highs, label = param - for i, mean, low, high, color, alpha in zip( - y, means, lows, highs, colours_arr, alphas - ): - ax.errorbar( - mean, - 0.05 + i * row_spacing, - xerr=np.array([low, high])[:, None], - fmt="o", - color=color, - ecolor=color, - elinewidth=2, - capsize=3, - alpha=alpha, - ) - ax.set_xlabel(label, fontsize=14) - - for ri, ref_idx in enumerate(ref_indices): - band_color = ( - reference_colors[ri] - if reference_colors and ri < len(reference_colors) - else colours_arr[ref_idx] - ) or colours_arr[ref_idx] - ax.axvspan( - means[ref_idx] - lows[ref_idx], - means[ref_idx] + highs[ref_idx], - color=band_color, - alpha=0.15, - zorder=0, - ) - - ax.grid(False) - ax.tick_params(axis="y", left=False, labelleft=False) - if label == r"$S_8$": - ax.set_xlim(0.25, 1.05) - elif label == r"$\sigma_8$": - ax.set_xlim(0.5, 1.2) - elif label == r"$\Omega_{\rm m}$": - ax.set_xlim(0.1, 0.5) - - axs[0].set_yticks(0.05 + y * row_spacing) - axs[0].set_yticklabels([]) - for label, color, alpha in zip(expt, colours_arr, alphas): - idx = np.where(expt == label)[0][0] - yloc = 0.05 + row_spacing * idx - axs[0].text( - 0.26, - yloc, - label, - fontsize=12, - ha="left", - va="center", - color=color, - alpha=alpha, - ) - if label not in ref_label_set and ref_indices: - ri0 = ref_indices[0] - s8_tension = get_sigma_tension( - s8_mean[idx], - s8_low[idx], - s8_high[idx], - s8_mean[ri0], - s8_low[ri0], - s8_high[ri0], - ) - sign_str = "+" if s8_tension > 0 else "-" - axs[0].text( - 1.045, - yloc, - rf"${sign_str}{np.abs(s8_tension):.2f}" + r"\, \sigma$", - fontsize=10, - ha="right", - va="center", - color=color, - alpha=alpha, - ) - - plt.gca().invert_yaxis() - plt.tight_layout() - - output_path.parent.mkdir(parents=True, exist_ok=True) - plt.savefig(output_path, dpi=300, bbox_inches="tight") - plt.close(fig) - - -# ── Snakemake entry ────────────────────────────────────────────────────────── - - -def _config_from_snakemake(smk) -> CeremonyConfig: - """Build config from snakemake.input / snakemake.output / snakemake.params.""" - chain_root_dir = Path(smk.params.chain_root_dir) - - return CeremonyConfig( - blind=smk.params.blind, - chain_version=smk.params.chain_version, - chain_prefix=smk.params.chain_prefix, - chain_root_dir=chain_root_dir, - external_root_dir=chain_root_dir / "ext_data", - results_dir=Path(smk.params.results_dir), - evidence_dir=Path(smk.output.evidence).parent, - xi_data_path=Path(smk.input.xi_data), - pure_eb_path=Path(smk.input.pure_eb), - pseudo_cl_path=Path(smk.input.pseudo_cl), - pseudo_cl_cov_path=Path(smk.input.pseudo_cl_cov), - cosmosis_cell_fits=Path(smk.input.cosmosis_cell_fits), - bestfit_dir=Path(smk.input.bestfit_dir).parent.parent, - bestfit_root_fid_cell=smk.params.bestfit_root_fid_cell, - bestfit_root_halofit_cell=smk.params.bestfit_root_halofit_cell, - bestfit_root_config=smk.params.bestfit_root_config, - ) - - -# ── CLI entry ──────────────────────────────────────────────────────────────── - -_CHAIN_ROOT_DIR = Path("/n09data/guerrini/output_chains") -_COSMOSIS_DATA_DIR = Path("/home/guerrini/sp_validation/cosmo_inference/data") -_DEFAULT_CHAIN_VERSION = "v1.4.6" - - -def _require_path(path: Path, label: str) -> Path: - if path.exists(): - return path - raise FileNotFoundError(f"{label}: {path}") - - -def _require_bestfit_root(chain_root_dir: Path, root: str) -> str: - if (chain_root_dir / "best_fit" / root / "shear_cl" / "ell.txt").exists(): - return root - raise FileNotFoundError(f"No best-fit shear_cl for {root}") - - -def _config_from_cli() -> CeremonyConfig: - """Build config from command-line arguments + path resolution. - - Uses the exact data vectors from inference (Lisa's xi_pm, Sasha's pseudo-Cl - and CosmoSIS FITS) — the same files the chains were fit to. - """ - _PROJECT_ROOT = _SCRIPT_DIR.parent.parent - - parser = argparse.ArgumentParser( - description="Run the UNIONS unblinding ceremony plot sequence." - ) - parser.add_argument("blind", choices=["A", "B", "C"], help="Revealed blind letter") - parser.add_argument( - "--chain-version", - default=_DEFAULT_CHAIN_VERSION, - help="Chain version (default: %(default)s)", - ) - parser.add_argument( - "--output-dir", - type=Path, - default=None, - help="Output directory for results (default: /results/unblinding)", - ) - args = parser.parse_args() - - blind = args.blind - chain_version = args.chain_version - chain_prefix = f"SP_{chain_version}_leak_corr" - output_dir = args.output_dir or (_PROJECT_ROOT / "results" / "unblinding") - - xi_data_path = _require_path( - _COSMOSIS_DATA_DIR - / f"{chain_prefix}_{blind}" - / f"cosmosis_{chain_prefix}_{blind}.fits", - f"CosmoSIS xi FITS for blind {blind}", - ) - - pseudo_cl_path = _require_path( - Path( - f"/home/guerrini/sp_validation/cosmo_val/output/pseudo_cl_{chain_prefix}.fits" - ), - f"pseudo-Cl for {chain_prefix} (Guerrini)", - ) - pseudo_cl_cov_path = _require_path( - Path( - f"/home/guerrini/sp_validation/cosmo_val/output/pseudo_cl_cov_{chain_prefix}.fits" - ), - f"pseudo-Cl covariance for {chain_prefix} (Guerrini)", - ) - - cosmosis_cell_fits = _require_path( - _COSMOSIS_DATA_DIR - / f"{chain_prefix}_{blind}_fid" - / f"cosmosis_{chain_prefix}_{blind}_fid_cell.fits", - f"CosmoSIS C_ell FITS for blind {blind}", - ) - - bestfit_root_config = f"{chain_prefix}_{blind}_10_80" - bestfit_dir = _require_path( - _CHAIN_ROOT_DIR / "best_fit" / bestfit_root_config, - f"Best-fit directory for blind {blind}", - ) - - return CeremonyConfig( - blind=blind, - chain_version=chain_version, - chain_prefix=chain_prefix, - chain_root_dir=_CHAIN_ROOT_DIR, - external_root_dir=_CHAIN_ROOT_DIR / "ext_data", - results_dir=output_dir, - evidence_dir=output_dir / "claims" / "unblinding_ceremony", - xi_data_path=xi_data_path, - pure_eb_path=_require_path( - _PROJECT_ROOT - / "results" - / "paper_plots" - / "intermediate" - / f"{chain_prefix}_{blind}_pure_eb_semianalytic.npz", - f"Pure E/B file for blind {blind}", - ), - pseudo_cl_path=pseudo_cl_path, - pseudo_cl_cov_path=pseudo_cl_cov_path, - cosmosis_cell_fits=cosmosis_cell_fits, - bestfit_dir=bestfit_dir, - bestfit_root_fid_cell=_require_bestfit_root( - _CHAIN_ROOT_DIR, f"{chain_prefix}_{blind}_fid_cell" - ), - bestfit_root_halofit_cell=_require_bestfit_root( - _CHAIN_ROOT_DIR, f"{chain_prefix}_{blind}_halofit_cell" - ), - bestfit_root_config=bestfit_root_config, - ) - - -# ── Main ceremony ──────────────────────────────────────────────────────────── - - -def run_ceremony(cfg: CeremonyConfig) -> None: - blind = cfg.blind - cfg.results_dir.mkdir(parents=True, exist_ok=True) - - reveal_harmonic_root = f"{cfg.chain_prefix}_{blind}_lmin=300_lmax=1600_cell" - reveal_config_root = f"{cfg.chain_prefix}_{blind}_10_80" - - # ── Act 1 — The Data (naked, then with fits) ────────────────────── - - # 01: xi+/- data + B-modes, no theory - plot_xipm_bestfit_with_bmodes( - xi_data_path=cfg.xi_data_path, - pure_eb_data_path=cfg.pure_eb_path, - bestfit_dir=None, - output_path=_save_path(cfg, 1, "xi_pm_data"), - ) - - # 02: C_ell^EE data, no theory - plot_cell_ee_data_vector( - str(cfg.pseudo_cl_path), - str(cfg.pseudo_cl_cov_path), - _save_path(cfg, 2, "cell_ee_data"), - ) - - # 03: xi+/- with config-space best-fit (Paper IV Fig 1) - plot_xipm_bestfit_with_bmodes( - xi_data_path=cfg.xi_data_path, - pure_eb_data_path=cfg.pure_eb_path, - bestfit_dir=cfg.bestfit_dir, - output_path=_save_path(cfg, 3, f"xi_bestfit_blind_{blind}"), - ) - - # 04: C_ell^EE with harmonic + config best-fit (Paper V Fig 2) - plot_cell_ee_with_bestfit( - cosmosis_data_path=str(cfg.cosmosis_cell_fits), - bestfit_specs=[ - ( - rf"UNIONS $C_\ell$, Blind {blind}", - cfg.bestfit_root_fid_cell, - {"color": "royalblue", "linestyle": "-"}, - ), - ( - r"UNIONS $C_\ell$, Halofit", - cfg.bestfit_root_halofit_cell, - {"color": "royalblue", "linestyle": "--"}, - ), - ( - r"UNIONS $\xi_\pm(\vartheta)$ (Goh et al., 2026)", - cfg.bestfit_root_config, - {"color": "orange", "linestyle": "-"}, - ), - ], - output_folder=str(cfg.chain_root_dir), - output_path=_save_path(cfg, 4, f"cell_ee_bestfit_blind_{blind}"), - ) - - # ── Act 2 — The Reveal ──────────────────────────────────────────── - - # 05: Consistency — Omega_m-S8, harmonic vs config vs Planck - plot_triangle( - [ - ChainSpec( - root=reveal_harmonic_root, - label=rf"UNIONS $C_\ell$, Blind {blind}", - color="royalblue", - base_dir=cfg.chain_root_dir, - ), - ChainSpec( - root=reveal_config_root, - label=rf"UNIONS $\xi_\pm(\vartheta)$, Blind {blind}", - color="orange", - base_dir=cfg.chain_root_dir, - ), - ChainSpec( - root="Planck18", - label=r"\textit{Planck} 2018", - color="violet", - base_dir=cfg.external_root_dir, - ), - ], - ["OMEGA_M", "S_8"], - _save_path(cfg, 5, f"consistency_blind_{blind}"), - width_inch=12, - axes_fontsize=24, - axes_labelsize=28, - ) - - # 06: 4-param triangle — revealed blind, harmonic + config overlaid - plot_triangle( - [ - ChainSpec( - root=reveal_harmonic_root, - label=rf"UNIONS $C_\ell$, Blind {blind}", - color="royalblue", - base_dir=cfg.chain_root_dir, - ), - ChainSpec( - root=reveal_config_root, - label=rf"UNIONS $\xi_\pm(\vartheta)$, Blind {blind}", - color="orange", - base_dir=cfg.chain_root_dir, - ), - ], - COSMO_PARAMS, - _save_path(cfg, 6, f"triangle_cosmo_blind_{blind}"), - width_inch=20, - ) - - # 07: Full-param triangle — revealed blind, harmonic + config overlaid - plot_triangle( - [ - ChainSpec( - root=reveal_harmonic_root, - label=rf"UNIONS $C_\ell$, Blind {blind}", - color="royalblue", - base_dir=cfg.chain_root_dir, - ), - ChainSpec( - root=reveal_config_root, - label=rf"UNIONS $\xi_\pm(\vartheta)$, Blind {blind}", - color="orange", - base_dir=cfg.chain_root_dir, - ), - ], - FULL_PARAMS, - _save_path(cfg, 7, f"triangle_full_blind_{blind}"), - width_inch=30, - axes_fontsize=26, - axes_labelsize=28, - ) - - # ── Act 3 — In Context ──────────────────────────────────────────── - - # 08: S8 whisker — all 6 UNIONS blinds (3 harmonic + 3 config), - # revealed blind highlighted, rest muted. Plus external surveys. - whisker_specs = [] - - for b in ("A", "B", "C"): - is_revealed = b == blind - alpha = 1.0 if is_revealed else MUTED_ALPHA - whisker_specs.append( - ChainSpec( - root=f"{cfg.chain_prefix}_{b}_lmin=300_lmax=1600_cell", - label=rf"UNIONS $C_\ell$, Blind {b}", - color="royalblue", - base_dir=cfg.chain_root_dir, - alpha=alpha, - ) - ) - whisker_specs.append( - ChainSpec( - root=f"{cfg.chain_prefix}_{b}_10_80", - label=rf"UNIONS $\xi_\pm$, Blind {b}", - color="orange", - base_dir=cfg.chain_root_dir, - alpha=alpha, - ) - ) - - whisker_specs.extend( - [ - ChainSpec( - root="Planck18", - label=r"\textit{Planck} 2018", - color="black", - base_dir=cfg.external_root_dir, - ), - ChainSpec( - root="DES_Y3", - label=r"DES Y3 $\xi_\pm$", - color="black", - base_dir=cfg.external_root_dir, - ), - ChainSpec( - root="KiDS-1000", - label=r"KiDS-1000 $\xi_\pm$", - color="black", - base_dir=cfg.external_root_dir, - ), - ChainSpec( - root="HSC_Y3", - label=r"HSC Y3 $\xi_\pm$", - color="black", - base_dir=cfg.external_root_dir, - ), - ] - ) - - plot_s8_whisker( - whisker_specs, - reference_labels=[ - rf"UNIONS $C_\ell$, Blind {blind}", - rf"UNIONS $\xi_\pm$, Blind {blind}", - ], - reference_colors=["royalblue", "orange"], - output_path=_save_path(cfg, 8, f"s8_whisker_blind_{blind}"), - ) - - # ── Write evidence ─────────────────────────────────────────────── - - produced_figures = [ - _save_path(cfg, 1, "xi_pm_data"), - _save_path(cfg, 2, "cell_ee_data"), - _save_path(cfg, 3, f"xi_bestfit_blind_{blind}"), - _save_path(cfg, 4, f"cell_ee_bestfit_blind_{blind}"), - _save_path(cfg, 5, f"consistency_blind_{blind}"), - _save_path(cfg, 6, f"triangle_cosmo_blind_{blind}"), - _save_path(cfg, 7, f"triangle_full_blind_{blind}"), - _save_path(cfg, 8, f"s8_whisker_blind_{blind}"), - ] - output_dict = {} - for fig_path in produced_figures: - output_dict[fig_path.stem] = fig_path.name - - cfg.evidence_dir.mkdir(parents=True, exist_ok=True) - - evidence = { - "id": "unblinding_ceremony", - "spec_id": "unblinding_ceremony", - "generated": datetime.now(timezone.utc).isoformat(timespec="seconds"), - "input": { - "xi_data": str(cfg.xi_data_path), - "pure_eb_data": str(cfg.pure_eb_path), - "pseudo_cl": str(cfg.pseudo_cl_path), - "pseudo_cl_cov": str(cfg.pseudo_cl_cov_path), - "cosmosis_cell_fits": str(cfg.cosmosis_cell_fits), - "harmonic_chains": str( - cfg.chain_root_dir - / f"{cfg.chain_prefix}_{{A,B,C}}_lmin=300_lmax=1600_cell" - ), - "config_chains": str( - cfg.chain_root_dir / f"{cfg.chain_prefix}_{{A,B,C}}_10_80" - ), - "external_chains": str( - cfg.external_root_dir / "{Planck18,DES_Y3,KiDS-1000,HSC_Y3}" - ), - }, - "output": output_dict, - "params": { - "chain_version": cfg.chain_version, - "blind": blind, - "scale_cut_arcmin": "12-83", - "ell_range": "300-1600", - "n_figures": len(output_dict), - }, - "evidence": { - "ceremony_date": "2026-02-27", - "script": "workflow/scripts/unblinding_ceremony.py", - }, - } - - evidence_path = cfg.evidence_dir / "evidence.json" - evidence_path.write_text(json.dumps(evidence, indent=2) + "\n") - - for fig_path in produced_figures: - shutil.copy2(fig_path, cfg.evidence_dir / fig_path.name) - - print(f"Saved ceremony figures to {cfg.results_dir.resolve()}") - print(f"Wrote evidence to {evidence_path}") - - -# ── Entry point dispatch ───────────────────────────────────────────────────── - -try: - snakemake # injected by snakemake's script: directive -except NameError: - snakemake = None - -if snakemake is not None: - run_ceremony(_config_from_snakemake(snakemake)) -elif __name__ == "__main__": - run_ceremony(_config_from_cli()) diff --git a/papers/cosmo_val/Snakefile b/papers/cosmo_val/Snakefile index 2585bb7a..716a3ac6 100644 --- a/papers/cosmo_val/Snakefile +++ b/papers/cosmo_val/Snakefile @@ -9,8 +9,6 @@ configfile: "config/config.yaml" configfile: "/n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_val/cat_config.yaml" -container: "/n17data/cdaley/containers/containers" - envvars: "PYTHONUNBUFFERED", @@ -32,6 +30,9 @@ import common common.configure(config) from common import * +# The one image every rule runs in; see common.resolve_container. +container: common.resolve_container(config.get("container")) + # Wildcard constraints — centralized in common.py, not in individual rule files wildcard_constraints: **WILDCARD_CONSTRAINTS diff --git a/pyproject.toml b/pyproject.toml index c6a68dd0..f325121f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -112,6 +112,13 @@ dependencies = [ [project.urls] Homepage = "https://github.com/CosmoStat/sp_validation" +[project.scripts] +# Host-side management of your own container image (sp_validation.container). +# Stdlib-only by design -- it has to run on the host, outside the container -- +# so it also works straight from a checkout with nothing installed: +# `python3 src/sp_validation/container.py status`. +spv-container = "sp_validation.container:main" + [tool.uv] # The reproducibility target is the Linux container; scope the lock to Linux # (mirrors shapepipe) so `uv lock` resolves the linux-centric stack (pymaster, @@ -134,8 +141,8 @@ docs = [ "sphinxawesome-theme>=5.3,!=6.0.3" ] # GLASS mock generation (sp_validation.glass_mock). Kept optional: the core -# library and its import guard resolve without GLASS, and the production -# container does not yet ship it. Pinned set (verified 2026-06-13, fiber +# library and its import guard resolve without GLASS (the production container +# ships it via the ``glass`` extra). Pinned set (verified 2026-06-13, fiber # glass-cosmology-api-pin): glass 2025.1 is the unique version with the flat API # the map path uses AND the legacy ``cosmo.dc``/``xm``/``ef`` interface that # ``cosmology`` 2022.10.9 (its newest release) provides — newer glass calls @@ -155,8 +162,16 @@ glass = [ workflow = [ "snakemake", # run_2pcf_highres.py drives the MPI convergence run; the container ships - # OpenMPI (/opt/ompi) so mpi4py builds against it. + # OpenMPI under /opt/ompi so mpi4py builds against it. "mpi4py", + # CosmoSIS drives the cosmo_inference sampling step. Sdist-only, so this is a + # source build (~2 min) against the base image's gfortran/GSL/cfitsio/OpenMPI — + # no extra apt packages needed. Floor at 3.25 simply to stay current; there is + # no known-good older version to pin back to. Note the shared-memory pool is + # still broken upstream at 3.25.2 (bcast/gather/allreduce in + # runtime/process_pool.py return an unassigned `self.data`), so chains must run + # under MPI -- which is why the Dockerfile sets MPIFC before syncing this extra. + "cosmosis>=3.25", # NOTE: workflow/scripts/cv_*.py also import `cv_runner`, which is not # published or resolvable (no public repo found) — left undeclared pending # its source. Same for `unions_wl` (scripts/check_footprint.py). diff --git a/src/sp_validation/container.py b/src/sp_validation/container.py new file mode 100755 index 00000000..7aa5f2ed --- /dev/null +++ b/src/sp_validation/container.py @@ -0,0 +1,390 @@ +#!/usr/bin/env python3 +"""Manage this user's local copy of the sp_validation container image. + +Everyone runs their own image: the canonical paths are under your own cache, you +refresh them when you want to, and nobody else's refresh moves the ground under +a running job. + +There are two layers, and you only need the second when you want it: + +* the **SIF** (``~/.cache/sp_validation/sp_validation.sif``) -- a pristine, + read-only copy of the published image. This is the default and the normal case. +* an optional **sandbox** (``~/.cache/sp_validation/sandbox/``) -- the same image + unpacked into a writable directory, so ``pip install`` inside it sticks. This + is the escape hatch for exploratory work that needs a package the image does + not carry yet, and it is opt-in: nothing builds one for you. + +Subcommands, exposed as the ``spv-container`` console script:: + + spv-container pull # fetch the tag to the canonical path + spv-container status # what is here, and how current is it + spv-container sandbox # unpack the SIF into a writable dir + spv-container exec # run something inside it + spv-container exec --writable # ... with writes that persist + +Everything resolves the same image in the same order -- **sandbox if it exists, +else the SIF, else the registry tag** -- and that includes the Snakemake +workflow, so a package you installed into your sandbox is there for your +workflow jobs too. + +This module is deliberately **stdlib-only** (``argparse``/``subprocess``/ +``pathlib``). It runs on the *host*, outside the container, where the science +stack is not installed -- so it must import without it. That also means it works +straight from a checkout with no install at all:: + + python3 src/sp_validation/container.py pull +""" + +import argparse +import os +import shutil +import subprocess +import sys +from pathlib import Path + +# The image every entry point names, written down here once. CI pushes one tag +# per branch, sanitized, so ``:develop`` tracks the integration branch. +CONTAINER_URI = "docker://ghcr.io/cosmostat/sp_validation:develop" + +CACHE_DIR = Path(os.environ.get("XDG_CACHE_HOME", "~/.cache")) / "sp_validation" + +# Where this user's image lives. Per-user by construction: one file, one owner, +# no coordination. Override with ``SPV_CONTAINER`` (an absolute path). +DEFAULT_SIF = CACHE_DIR / "sp_validation.sif" + +# The optional writable unpacking of that image. Override with ``SPV_SANDBOX``. +DEFAULT_SANDBOX = CACHE_DIR / "sandbox" + +# Bind mounts for interactive `exec`. candide's disks; override wholesale with +# ``SPV_APPTAINER_BINDS`` or per-call with ``--bind``. +DEFAULT_BINDS = "/home,/scratch,/automnt,/n17data,/n23data1,/n09data" + + +def local_sif(): + """Return this user's canonical image path (may not exist yet).""" + override = os.environ.get("SPV_CONTAINER") + path = Path(override) if override else DEFAULT_SIF + return path.expanduser() + + +def local_sandbox(): + """Return this user's writable sandbox directory (may not exist).""" + override = os.environ.get("SPV_SANDBOX") + path = Path(override) if override else DEFAULT_SANDBOX + return path.expanduser() + + +def resolve_image(): + """Return ``(path_or_uri, kind)`` for the image everything should run. + + The one resolution order, shared by the CLI and the workflow: the writable + sandbox if it exists, else the pristine SIF if it exists, else the registry + tag for Snakemake to pull. ``kind`` is ``"sandbox"``, ``"sif"`` or ``"tag"``. + """ + sandbox = local_sandbox() + if sandbox.is_dir(): + return str(sandbox), "sandbox" + sif = local_sif() + if sif.exists(): + return str(sif), "sif" + return CONTAINER_URI, "tag" + + +def image_labels(sif): + """Return the image's OCI labels as a dict, or ``{}`` if unreadable. + + Never raises: a missing file, a missing ``apptainer``, or a corrupt image + all mean "we don't know", which every caller here treats as non-fatal. + """ + sif = Path(sif) + if not sif.exists() or shutil.which("apptainer") is None: + return {} + try: + out = subprocess.run( + ["apptainer", "inspect", "--labels", str(sif)], + capture_output=True, + text=True, + timeout=60, + ) + except (OSError, subprocess.SubprocessError): + return {} + if out.returncode != 0: + return {} + labels = {} + for line in out.stdout.splitlines(): + key, sep, value = line.partition(":") + if sep: + labels[key.strip()] = value.strip() + return labels + + +def image_revision(sif): + """Return the sp_validation commit the image was built from, or ``None``.""" + return image_labels(sif).get("org.opencontainers.image.revision") + + +def _require_apptainer(): + """Exit unless ``apptainer`` is on PATH.""" + if shutil.which("apptainer") is None: + sys.exit("apptainer is not on PATH") + + +def _git(*args, cwd=None): + """Run a git command, returning stripped stdout or ``None`` on any failure.""" + try: + out = subprocess.run( + ["git", *args], capture_output=True, text=True, cwd=cwd, timeout=30 + ) + except (OSError, subprocess.SubprocessError): + return None + return out.stdout.strip() if out.returncode == 0 else None + + +def compare_revision(revision, repo=None): + """Place an image revision relative to a checkout's HEAD. + + Returns one of ``"in-sync"``, ``"behind"`` (the image predates HEAD), + ``"ahead"`` (HEAD predates the image), ``"diverged"``, or ``"unknown"`` + (no revision label, no git, or a commit this clone has never fetched). + """ + if not revision: + return "unknown" + repo = repo or Path(__file__).resolve().parents[2] + head = _git("rev-parse", "HEAD", cwd=repo) + if head is None: + return "unknown" + if head == revision: + return "in-sync" + if _git("cat-file", "-e", f"{revision}^{{commit}}", cwd=repo) is None: + return "unknown" + if _git("merge-base", "--is-ancestor", revision, head, cwd=repo) is not None: + return "behind" + if _git("merge-base", "--is-ancestor", head, revision, cwd=repo) is not None: + return "ahead" + return "diverged" + + +def cmd_pull(args): + """Pull ``--tag`` to the canonical path, atomically.""" + _require_apptainer() + sif = local_sif() + sif.parent.mkdir(parents=True, exist_ok=True) + # Pull to a sibling temp name and rename: an atomic rename within one + # directory, so a job gets either the whole old image or the whole new one. + # Pulling in place would leave the file half-written for the ~15 minutes the + # pull takes. Jobs already running hold the old inode open and finish on it. + tmp = sif.with_name(sif.name + f".pull.{os.getpid()}") + print(f"pulling {args.tag}\n -> {sif}") + try: + subprocess.run( + ["apptainer", "pull", "--force", "--name", str(tmp), args.tag], check=True + ) + os.replace(tmp, sif) + except subprocess.CalledProcessError as exc: + tmp.unlink(missing_ok=True) + sys.exit(f"pull failed ({exc.returncode})") + except KeyboardInterrupt: + tmp.unlink(missing_ok=True) + raise + labels = image_labels(sif) + print(f"revision: {labels.get('org.opencontainers.image.revision', 'unknown')}") + print(f"version: {labels.get('org.opencontainers.image.version', 'unknown')}") + return 0 + + +def cmd_sandbox(args): + """Unpack the image into a writable directory -- the opt-in escape hatch.""" + _require_apptainer() + sandbox = local_sandbox() + if sandbox.exists() and not args.force: + sys.exit( + f"sandbox already exists at {sandbox}\n" + "pass --force to discard it and rebuild from a clean image" + ) + source = args.source or ( + str(local_sif()) if local_sif().exists() else CONTAINER_URI + ) + sandbox.parent.mkdir(parents=True, exist_ok=True) + print(f"building sandbox from {source}\n -> {sandbox}") + # Build beside the target and swap it in, as `pull` does -- and for a sharper + # reason here. A half-written .sif fails loudly, but a half-unpacked sandbox + # *directory* is still a directory, so resolve_image() would elect it as the + # live image and every job would silently run a broken tree. + # + # Building first also means a `--force` rebuild that fails (a typo in + # --source, a network blip) leaves the sandbox you already had untouched, + # rather than deleting a working environment on the way to not replacing it. + # + # `--fix-perms` so the tree can be deleted again later (apptainer warns about + # exactly this otherwise). No `--fakeroot`: an unprivileged build from an + # existing image works through user namespaces, which is what candide has. + staging = sandbox.with_name(f"{sandbox.name}.build.{os.getpid()}") + shutil.rmtree(staging, ignore_errors=True) + try: + subprocess.run( + ["apptainer", "build", "--sandbox", "--fix-perms", str(staging), source], + check=True, + ) + except subprocess.CalledProcessError as exc: + shutil.rmtree(staging, ignore_errors=True) + sys.exit(f"sandbox build failed ({exc.returncode}); {sandbox} is unchanged") + except (KeyboardInterrupt, OSError): + shutil.rmtree(staging, ignore_errors=True) + raise + + if sandbox.exists(): + print(f"replacing {sandbox}") + shutil.rmtree(sandbox, ignore_errors=True) + if sandbox.exists(): + shutil.rmtree(staging, ignore_errors=True) + sys.exit(f"could not remove {sandbox}; remove it by hand and retry") + os.replace(staging, sandbox) + print( + "\nthis sandbox now takes precedence over the SIF everywhere, including " + "workflow jobs.\ninstall into it with: spv-container exec --writable pip " + "install \nreset to a clean image with: spv-container pull && " + "spv-container sandbox --force" + ) + return 0 + + +def cmd_status(args): + """Report which image layer is live, its revision, and how current it is.""" + sif = local_sif() + sandbox = local_sandbox() + active, kind = resolve_image() + + if sif.exists(): + print(f"SIF: {sif} ({sif.stat().st_size / 1e9:.1f} GB)") + else: + print(f"SIF: absent ({sif})") + if sandbox.is_dir(): + print(f"sandbox: {sandbox} (writable; may carry local modifications)") + else: + print("sandbox: none") + + if kind == "tag": + print(f"\nactive: {active} (registry tag -- nothing pulled locally)") + print("run: spv-container pull") + return 1 + + print(f"\nactive: {active} ({kind})") + labels = image_labels(active) + revision = labels.get("org.opencontainers.image.revision") + source = "" + if revision is None and kind == "sandbox" and sif.exists(): + # Some sandbox trees do not carry the original labels through. The SIF + # beside it is the best remaining evidence of what it was built from -- + # a guess, so it is labelled as one rather than printed as fact. + revision = image_revision(sif) + if revision: + source = " (inferred from the SIF beside it, not read from the sandbox)" + print(f"revision: {revision or 'unknown'}{source}") + print(f"version: {labels.get('org.opencontainers.image.version', 'unknown')}") + if kind == "sandbox": + print( + " (the revision above is what the sandbox was built from; " + "anything\n installed into it since is not reflected in " + "any label)" + ) + verdict = compare_revision(revision) + explain = { + "in-sync": "matches this checkout's HEAD", + "behind": "older than this checkout's HEAD -- pull to refresh", + "ahead": "newer than this checkout's HEAD", + "diverged": "on a different branch from this checkout", + "unknown": "cannot compare (no label, or a commit this clone lacks)", + }[verdict] + print(f"checkout: {verdict} ({explain})") + return 0 + + +def cmd_exec(args): + """Run a command inside the image -- the one-off path for humans and agents.""" + _require_apptainer() + if not args.command: + sys.exit("nothing to run; pass a command after `exec`") + binds = args.bind or os.environ.get("SPV_APPTAINER_BINDS", DEFAULT_BINDS) + + if args.writable: + # A SIF is a read-only filesystem, so `--writable` against one fails + # obscurely; only a sandbox takes writes. + sandbox = local_sandbox() + if not sandbox.is_dir(): + sys.exit( + f"--writable needs a sandbox, and there is none at {sandbox}\n" + "build one with: spv-container sandbox" + ) + image, extra = str(sandbox), ["--writable"] + else: + image, kind = resolve_image() + if kind == "tag": + sys.exit(f"no local image; run: spv-container pull ({image})") + extra = [] + + cmd = [ + "apptainer", + "exec", + *extra, + "--cleanenv", + "--bind", + binds, + image, + *args.command, + ] + return subprocess.run(cmd).returncode + + +def build_parser(): + parser = argparse.ArgumentParser( + prog="spv-container", description=__doc__.splitlines()[0] + ) + sub = parser.add_subparsers(dest="subcommand", required=True) + + p_pull = sub.add_parser("pull", help="fetch the image to the canonical path") + p_pull.add_argument( + "--tag", + default=CONTAINER_URI, + help=f"image to pull (default: {CONTAINER_URI})", + ) + p_pull.set_defaults(func=cmd_pull) + + p_status = sub.add_parser( + "status", help="report which image layer is live and how current it is" + ) + p_status.set_defaults(func=cmd_status) + + p_sandbox = sub.add_parser( + "sandbox", help="unpack the image into a writable directory (opt-in)" + ) + p_sandbox.add_argument( + "--source", + help="image to unpack (default: the local SIF, or the registry tag)", + ) + p_sandbox.add_argument( + "--force", + action="store_true", + help="discard an existing sandbox and rebuild from a clean image", + ) + p_sandbox.set_defaults(func=cmd_sandbox) + + p_exec = sub.add_parser("exec", help="run a command inside the local image") + p_exec.add_argument("--bind", help=f"bind mounts (default: {DEFAULT_BINDS})") + p_exec.add_argument( + "--writable", + action="store_true", + help="run against the sandbox so writes (e.g. pip install) persist", + ) + p_exec.add_argument("command", nargs=argparse.REMAINDER) + p_exec.set_defaults(func=cmd_exec) + + return parser + + +def main(argv=None): + args = build_parser().parse_args(argv) + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/sp_validation/tests/data/container_smoke/Snakefile b/src/sp_validation/tests/data/container_smoke/Snakefile new file mode 100644 index 00000000..75527690 --- /dev/null +++ b/src/sp_validation/tests/data/container_smoke/Snakefile @@ -0,0 +1,21 @@ +# Standalone workflow exercised by src/sp_validation/tests/test_container_smoke.py. +# +# The module-level `container:` below mirrors what every real workflow does +# (workflow/Snakefile) -- Snakemake has no way to take a default image from a +# profile. Everything else under test arrives from the driving profile +# (workflow/profiles/candide). See container_smoke.py for what the job checks. + + +# Literal rather than the package's CONTAINER_URI: this Snakefile is test data, +# deliberately standalone. test_container_smoke.py asserts the two agree, so +# drift fails the test rather than the run. +container: config.get("container", "docker://ghcr.io/cosmostat/sp_validation:develop") + + +rule container_smoke: + output: + "results/container_smoke.yaml", + resources: + runtime=5, + script: + "container_smoke.py" diff --git a/src/sp_validation/tests/data/container_smoke/container_smoke.py b/src/sp_validation/tests/data/container_smoke/container_smoke.py new file mode 100644 index 00000000..b36eba11 --- /dev/null +++ b/src/sp_validation/tests/data/container_smoke/container_smoke.py @@ -0,0 +1,88 @@ +"""Rule container_smoke: exercise the containerized-SLURM path end to end. + +Cheap sanity check of the profile-driven container path -- same executor +(slurm), same software-deployment-method (apptainer), same apptainer-args +binds, same container image every real rule uses. Four things it proves, each +written to the output YAML: + + * the job really ran inside the image (``APPTAINER_CONTAINER``, set by + apptainer itself -- without it the rest could all pass on the bare host); + * the editable ``sp_validation`` install resolves on the container's + PYTHONPATH (import provenance: file + version, not just import success); + * the numeric stack works (numpy eigh on a small fixed matrix). + ``OMP_NUM_THREADS`` is recorded but not asserted -- see the assertions; + * which commit of this checkout is running (git rev-parse from inside the + container -- proves /home is bound and usable, not just readable). + +Driven by the co-located Snakefile; the assertions on the output YAML live in +src/sp_validation/tests/test_container_smoke.py (marked ``slow``, cluster only). +""" + +import os +import platform +import subprocess + +import numpy as np +import yaml + + +# --- the job is actually inside the image --------------------------------- +container_info = { + "apptainer_container": os.environ.get("APPTAINER_CONTAINER", "unset"), +} + +# --- editable install resolves inside the container ------------------------ +import sp_validation + +sp_validation_info = { + "version": getattr(sp_validation, "__version__", "unknown"), + "file": sp_validation.__file__, +} + +# --- numeric stack + threading ----------------------------------------- +rng = np.random.default_rng(seed=42) +a = rng.standard_normal((8, 8)) +symmetric = a + a.T +eigenvalues = np.linalg.eigh(symmetric)[0] + +numeric_info = { + "numpy_version": np.__version__, + "eigenvalues": [float(v) for v in eigenvalues], + "omp_num_threads": os.environ.get("OMP_NUM_THREADS", "unset"), +} + +# --- provenance: what commit is actually running in the container --------- +# src/sp_validation/tests/data/container_smoke/ -> repo root, five levels up. +# (This is the checkout the Snakefile came from, which is what we want to +# report; the editable install may well resolve to a *different* checkout.) +repo_dir = os.path.abspath( + os.path.join(os.path.dirname(os.path.abspath(__file__)), *([os.pardir] * 5)) +) +try: + commit = subprocess.run( + ["git", "-C", repo_dir, "rev-parse", "HEAD"], + capture_output=True, + text=True, + check=True, + ).stdout.strip() +except (subprocess.CalledProcessError, FileNotFoundError) as exc: + commit = f"unavailable ({exc})" + +provenance = { + "repo_dir": repo_dir, + "commit": commit, + "hostname": platform.node(), + "python": platform.python_version(), +} + +with open(snakemake.output[0], "w") as f: + yaml.safe_dump( + { + "container": container_info, + "sp_validation": sp_validation_info, + "numeric": numeric_info, + "provenance": provenance, + }, + f, + sort_keys=False, + ) diff --git a/src/sp_validation/tests/test_container_smoke.py b/src/sp_validation/tests/test_container_smoke.py new file mode 100644 index 00000000..e10d9ab0 --- /dev/null +++ b/src/sp_validation/tests/test_container_smoke.py @@ -0,0 +1,119 @@ +"""Smoke test of the profile-driven containerized-SLURM path. + +Submits one real (tiny, 5-minute) SLURM job through the committed candide +profile. The executor, the apptainer deployment method and the bind mounts come +from that profile; the image is the module-level ``container:`` in the test +Snakefile, exactly as real workflows declare it. That contract is what's under +test, so this can only run on candide -- marked ``slow``, skipped elsewhere. + +The job writes a YAML report (see data/container_smoke/container_smoke.py); the +assertions below check what it reports. +""" + +import os +import re +import shutil +import subprocess +import tempfile +from pathlib import Path + +import numpy as np +import pytest +import yaml + +requires_cluster = pytest.mark.skipif( + not Path("/n17data/cdaley/unions").exists() or shutil.which("sbatch") is None, + reason="needs candide: /n17data and a SLURM submit host", +) + + +def _repo_root() -> Path: + for parent in Path(__file__).resolve().parents: + if (parent / "pyproject.toml").exists(): + return parent + raise RuntimeError("could not locate repo root (no pyproject.toml above test)") + + +def _reference_eigenvalues() -> np.ndarray: + """The same deterministic computation the job runs inside the container.""" + rng = np.random.default_rng(seed=42) + a = rng.standard_normal((8, 8)) + return np.linalg.eigh(a + a.T)[0] + + +def test_smoke_snakefile_names_the_workflow_image(): + """The test Snakefile's literal image must track the package's CONTAINER_URI.""" + repo_root = _repo_root() + uri = re.search( + r'^CONTAINER_URI = "(.+)"$', + (repo_root / "src/sp_validation/container.py").read_text(), + re.MULTILINE, + ).group(1) + snakefile = ( + repo_root / "src/sp_validation/tests/data/container_smoke/Snakefile" + ).read_text() + assert f'"{uri}"' in snakefile, uri + + +@pytest.mark.slow +@requires_cluster +def test_container_smoke(): + repo_root = _repo_root() + workflow_dir = repo_root / "src/sp_validation/tests/data/container_smoke" + + # Not pytest's tmp_path: that lives in the login node's /tmp, which the + # compute node cannot see, so the job's output would "go missing". The + # workdir must be on a shared filesystem. + tmp_path = Path(tempfile.mkdtemp(prefix="container_smoke_", dir=Path.home())) + + env = os.environ | {"PYTHONNOUSERSITE": "1", "PYTHONUNBUFFERED": "1"} + result = subprocess.run( + [ + "snakemake", + "--profile", + str(repo_root / "workflow/profiles/candide"), + "-s", + str(workflow_dir / "Snakefile"), + "--directory", + str(tmp_path), + "--jobs", + "1", + "container_smoke", + ], + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=600, + check=False, + ) + assert result.returncode == 0, result.stdout + + report = yaml.safe_load((tmp_path / "results/container_smoke.yaml").read_text()) + + # The job ran inside the image, not on the bare host. Everything below would + # pass on the host too, so this is the assertion that makes them mean + # something: apptainer sets APPTAINER_CONTAINER in every process it starts. + assert report["container"]["apptainer_container"] != "unset", report["container"] + + # The install must resolve to an editable src/ checkout, not a site-packages + # copy. Note it need not be *this* checkout: the container's editable install + # points at the shared /n17data working tree, while the Snakefile under test + # is read from wherever the test runs. + module_file = Path(report["sp_validation"]["file"]) + assert module_file.parts[-3:] == ("src", "sp_validation", "__init__.py"), module_file + assert "site-packages" not in module_file.parts, module_file + + # The numeric stack agrees with the same computation run here. + np.testing.assert_allclose( + report["numeric"]["eigenvalues"], _reference_eigenvalues(), rtol=1e-10, atol=1e-12 + ) + + # numeric.omp_num_threads is recorded but deliberately NOT asserted: the + # profile leaves OMP_NUM_THREADS unset by design, and rules that need it + # pinned set it themselves, so "unset" here is correct rather than a gap. + + # git worked inside the container, so /home is bound and usable. + assert re.fullmatch(r"[0-9a-f]{40}", report["provenance"]["commit"]), report["provenance"] + + shutil.rmtree(tmp_path) # keep only on failure, for post-mortem diff --git a/uv.lock b/uv.lock index bef8072e..40669e2a 100644 --- a/uv.lock +++ b/uv.lock @@ -561,6 +561,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2e/2c/c80db37593ad593e3f6c60705fbdf690b33f22c89edd5507d2731346cd87/cosmology-2022.10.9-py3-none-any.whl", hash = "sha256:3903658c2474177a1a1c75771ae14458d93200c516f8fc6c3f4d776f3287b288", size = 9341, upload-time = "2022-10-10T10:18:14.928Z" }, ] +[[package]] +name = "cosmosis" +version = "3.25.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "dulwich", marker = "sys_platform == 'linux'" }, + { name = "dynesty", marker = "sys_platform == 'linux'" }, + { name = "emcee", marker = "sys_platform == 'linux'" }, + { name = "h5py", marker = "sys_platform == 'linux'" }, + { name = "matplotlib", marker = "sys_platform == 'linux'" }, + { name = "nautilus-sampler", marker = "sys_platform == 'linux'" }, + { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "py-bobyqa", marker = "sys_platform == 'linux'" }, + { name = "pybind11", marker = "sys_platform == 'linux'" }, + { name = "pyyaml", marker = "sys_platform == 'linux'" }, + { name = "scikit-learn", marker = "sys_platform == 'linux'" }, + { name = "scipy", marker = "sys_platform == 'linux'" }, + { name = "threadpoolctl", marker = "sys_platform == 'linux'" }, + { name = "zeus-mcmc", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/80/63/83898440e5486294cb17a7ddf01896af66dd153d7f3e77897ed120abd055/cosmosis-3.25.2.tar.gz", hash = "sha256:4a8333395b600a5e8339c637b947a982bac4c88a7b242e4b6de8274be03004a8", size = 416233, upload-time = "2026-03-09T12:01:08.269Z" } + [[package]] name = "coverage" version = "7.15.0" @@ -852,6 +874,44 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9a/91/428eca0cc8a1142b9952bef02c06bfe54526d730dd070de873424ab8c0bf/ducc0-0.41.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a3a71fd23661ddd6a7f6b44433e778d35a25b3f1492c9239217cdfedbe8178b1", size = 5613706, upload-time = "2026-03-26T18:55:24.971Z" }, ] +[[package]] +name = "dulwich" +version = "1.2.13" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "urllib3", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ab/7d/022515919b5ee3175a70a7e52049c4a2637008a4a49024ad15db1531bda9/dulwich-1.2.13.tar.gz", hash = "sha256:77f0d7012710da1ba0742a197fce215b0e1a05aeb9bc8a03ce94ccd2644317d8", size = 1361288, upload-time = "2026-08-24T12:38:30.179Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/42/2383b8eb8e910d18d4e37156d2e191bb5b3b202648636db266065611835b/dulwich-1.2.13-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:8816b7c1e5e527b9c8d8aa322b9b80cb1878c0b4990707ceffa12c2529eb6dc5", size = 1459316, upload-time = "2026-08-24T12:37:18.387Z" }, + { url = "https://files.pythonhosted.org/packages/47/ee/d75a0249a21f22c03db2ab1a8ee9f727ca1fafc6bd7da86ddd0564a05b7b/dulwich-1.2.13-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:1c1ff65f48a81fce6fa78edffc289655556082543f34a4024a0eca5161f2fb85", size = 1490824, upload-time = "2026-08-24T12:37:20.045Z" }, + { url = "https://files.pythonhosted.org/packages/65/7e/52324b023c728e7e0a3397d8388ef85b2980d9f436d34a4b0530f076f5ae/dulwich-1.2.13-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:7805fbcd07a542502e50f7bd63cb6a2ab1b04fa5803bf2ee163866e8c97c6f94", size = 1459564, upload-time = "2026-08-24T12:37:32.602Z" }, + { url = "https://files.pythonhosted.org/packages/5c/2e/f321b0ffd3168d795aad5043bf17190ba8481e293cd09154f0cd3e155134/dulwich-1.2.13-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:d258d59a7af83147d5c888e5ab3a53f2d37ad4ae41728d70969e0bce1ac7ba83", size = 1490259, upload-time = "2026-08-24T12:37:34.311Z" }, + { url = "https://files.pythonhosted.org/packages/47/0d/ab0e5dd1558b15a78fa2d75c03b5af44823b8e1297ffcfef4bb66f24f9d6/dulwich-1.2.13-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:b484a49c166c2e5b2deecdb964d4725a454cfcc87970746f2b2faac252d0bb72", size = 1461868, upload-time = "2026-08-24T12:37:46.208Z" }, + { url = "https://files.pythonhosted.org/packages/45/75/d94f4317f09df37638c81ce90d0988154cb8d537504a9d4c0f5837cf320a/dulwich-1.2.13-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:c11ba8725d8808dee69e3aec1aceed3b716b16da3136c31499b271e0a79913c4", size = 1491986, upload-time = "2026-08-24T12:37:47.977Z" }, + { url = "https://files.pythonhosted.org/packages/46/0f/10fadbbc7efbb103f747af24e690393e6f6c29d85acac5febb34e35cb6e7/dulwich-1.2.13-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:63c26d34f53fe972e64fa6a9f6d7b013fa616a8357893e648710799e2e238db7", size = 1458718, upload-time = "2026-08-24T12:37:56.561Z" }, + { url = "https://files.pythonhosted.org/packages/5d/0e/8b9157e4536e78ab80b2d10759408e026d5f2ea62fc1fe5a386b89f7b4c3/dulwich-1.2.13-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:f0a79903a43491804852333f4e85fd484bc5e35ecf4fc836409f8194e3f1fc0b", size = 1539867, upload-time = "2026-08-24T12:37:58.202Z" }, + { url = "https://files.pythonhosted.org/packages/cc/95/4a840bf858b3d83087de36f5f768ff20cc464f0495a37d7bd8fd97cf1ac1/dulwich-1.2.13-cp315-cp315-manylinux_2_28_aarch64.whl", hash = "sha256:c243c278cb3281c8f6876b3621f2396548041aa316d91f3c78aa2d7a22043ef9", size = 1462669, upload-time = "2026-08-24T12:38:09.971Z" }, + { url = "https://files.pythonhosted.org/packages/e4/de/e3d9e52ce2bf06c63eb360bd467008cdb030655ccf98100bfc72577dd926/dulwich-1.2.13-cp315-cp315-manylinux_2_28_x86_64.whl", hash = "sha256:33ac85b270d2410fa1fc962caeef14a63e04de46dd76c5080f558c5d9752fa04", size = 1540300, upload-time = "2026-08-24T12:38:11.568Z" }, + { url = "https://files.pythonhosted.org/packages/82/52/9c7a9ff65cdd99237220f88018f3b4b2f4cb910d4341b9625c903796ca64/dulwich-1.2.13-cp315-cp315t-manylinux_2_28_aarch64.whl", hash = "sha256:c226635fe7dffdc8718b177a11b904a4bd5ff83dc9b5a443109458603f944489", size = 1460164, upload-time = "2026-08-24T12:38:20.434Z" }, + { url = "https://files.pythonhosted.org/packages/ee/6b/dbeece5fa1dd39416ba7b525e6ae8f9f96f34c1bc9de58f6a398d889cba4/dulwich-1.2.13-cp315-cp315t-manylinux_2_28_x86_64.whl", hash = "sha256:b832922190662896e057b8d84fd28d07d7c8eb9cbe2519348142bb909cd693b4", size = 1493098, upload-time = "2026-08-24T12:38:22.242Z" }, + { url = "https://files.pythonhosted.org/packages/5e/50/838d9c3da15f94b8f36c69e7abe3fea5294569a1dbda3812a07fba429f54/dulwich-1.2.13-py3-none-any.whl", hash = "sha256:edadd2f1019d879f80ab107a8587cb7f3b72d077b23facb8564700e7e3ce1758", size = 730094, upload-time = "2026-08-24T12:38:28.562Z" }, +] + +[[package]] +name = "dynesty" +version = "3.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "matplotlib", marker = "sys_platform == 'linux'" }, + { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "scipy", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c4/75/db494edc8abfe2273b9cb08c87a2a1165c6ae90d76f746980ef07ec5303a/dynesty-3.1.0.tar.gz", hash = "sha256:851717431f04f749bca45e86704c28fecf9f3b10f85a2a69820b26e42cefab68", size = 35564761, upload-time = "2026-07-17T14:38:31.465Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/f9/41c72d0b5fb511fdf23cf62ed50da6c73d4497f03ed1cad04f214dc477c8/dynesty-3.1.0-py3-none-any.whl", hash = "sha256:4e5b77bb261abdcb98e3a538d7498a94d5882a848f0ea46924b404396ef036d5", size = 106027, upload-time = "2026-07-17T14:38:28.523Z" }, +] + [[package]] name = "emcee" version = "3.1.6" @@ -2267,6 +2327,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/09/dc/f3dfb7488b770f3f67e6545085bf2abea5172e88f57b8ad25ef860ca704c/myst_parser-5.1.0-py3-none-any.whl", hash = "sha256:9c91c52b3cdb4d94a6506e4fab4e2f296c7623a0da0dcbe6de1565c3dad67a8a", size = 85817, upload-time = "2026-05-13T09:38:17.904Z" }, ] +[[package]] +name = "narwhals" +version = "2.25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6f/7b/6248dada39781db1ab3ebf08943080df0796098515a87f6f8696d14ec744/narwhals-2.25.0.tar.gz", hash = "sha256:62c036c810662bf7820b7737077176313bc59350eeeefb808510f388c743e4b2", size = 677076, upload-time = "2026-08-20T18:10:15.454Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/dc/55481808fd70ef1567cf13540ffd4702af3f74b112e35427564b03f79c2d/narwhals-2.25.0-py3-none-any.whl", hash = "sha256:1f0f403e8c7e4463cde9bfe78b12fdd809e3ae3dda6d9b2f802934fb9c7a6a8f", size = 467373, upload-time = "2026-08-20T18:10:13.834Z" }, +] + +[[package]] +name = "nautilus-sampler" +version = "1.0.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "scikit-learn", marker = "sys_platform == 'linux'" }, + { name = "scipy", marker = "sys_platform == 'linux'" }, + { name = "threadpoolctl", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/2c/a50c1a550ba43d86a0b41cf6410986d478d8640139c8e963f4dde8a8bfc2/nautilus_sampler-1.0.6.tar.gz", hash = "sha256:4e90b6d97be742be2e255c35a79e5a388cee043b4d2ce970eec327ed34139b78", size = 43118, upload-time = "2025-12-29T14:08:25.629Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/54/372c42bcfe324b650946f6cfee0e5adbd55aafc25cab555460bf5d9e8d36/nautilus_sampler-1.0.6-py3-none-any.whl", hash = "sha256:de6f3b9d249d87f05a673e8597e56621037e84dc712a6c2d3c18ff3672735b9b", size = 35772, upload-time = "2025-12-29T14:08:24.603Z" }, +] + [[package]] name = "nbclient" version = "0.11.0" @@ -2758,6 +2842,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842, upload-time = "2024-07-21T12:58:20.04Z" }, ] +[[package]] +name = "py-bobyqa" +version = "1.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "pandas", marker = "sys_platform == 'linux'" }, + { name = "scipy", marker = "sys_platform == 'linux'" }, + { name = "setuptools", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6b/41/c4c74daf208ed27e14071e92efb7ea238ffdf77ee93a3a7777ff02d2b0e4/py_bobyqa-1.5.0.tar.gz", hash = "sha256:3c7719b68b28834ea6d538f54603f6a891263f7c21f1a673de79e3a5e0e7e413", size = 51486, upload-time = "2024-09-16T03:52:32.303Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/1e/0d44a4e3a291c009a357fbd1d61511d9306c2c4db9a7ceb6e8104d8d385f/Py_BOBYQA-1.5.0-py3-none-any.whl", hash = "sha256:457afc04d6f2c9f1814934854dc4e542c5e5982a0f80add4b211fcdb0b5811e3", size = 57978, upload-time = "2024-09-16T03:52:30.667Z" }, +] + [[package]] name = "py-cpuinfo" version = "9.0.0" @@ -3354,6 +3453,29 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a1/9f/56288a8aabafa24679d32d01e7db732e8e89e4074cc91aedc0bb7b7c8e46/sacc-2.4-py3-none-any.whl", hash = "sha256:da6b648998738c7c307cbdff644429633726844f8c048642725be5efc1859fa3", size = 50931, upload-time = "2026-07-02T11:29:26.286Z" }, ] +[[package]] +name = "scikit-learn" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "joblib", marker = "sys_platform == 'linux'" }, + { name = "narwhals", marker = "sys_platform == 'linux'" }, + { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "scipy", marker = "sys_platform == 'linux'" }, + { name = "threadpoolctl", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fa/6f/37092bdb25f712817231799fc5674d8e704066a8a70c1d2d40517e18b4ab/scikit_learn-1.9.0.tar.gz", hash = "sha256:8833266989d3a5110178a9fae30783675460724d0e1efb13b14901d2c660c557", size = 7750767, upload-time = "2026-06-02T11:54:32.706Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/ee/5adbc77656b71f9456a2f5a7a9fdb4bcf9207a6b962889f1c2f9323afa4e/scikit_learn-1.9.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5e50ed4da51974e86e940690e9a3d82e729b62b5a49f7c9bac534d515d39d86f", size = 8837603, upload-time = "2026-06-02T11:53:30.328Z" }, + { url = "https://files.pythonhosted.org/packages/6c/c2/63fdda36c56437eeb44aaf9493c8bcd62ce230ab1598924fc626ffbfa943/scikit_learn-1.9.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:056c92bb67ad4c28463c2f2653d9701449201e7e7a9e94e321be0f71c4fef2b8", size = 9132097, upload-time = "2026-06-02T11:53:33.456Z" }, + { url = "https://files.pythonhosted.org/packages/0c/39/ffe829a5b8ecb40a518724a997794657fdc354ada5e8fe8e64d998c0bac9/scikit_learn-1.9.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:38c3dcb9a1ffb85505ec53d54c7b4aea0cff70050425a7760c2af661ac85df05", size = 8789690, upload-time = "2026-06-02T11:53:47.461Z" }, + { url = "https://files.pythonhosted.org/packages/1f/88/8dab5de10c638c083772a6be83a3d8106ced492f74a928c8693638e5bb50/scikit_learn-1.9.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:da76d09304a4706db7cc1e3ebaa3b6b98a67365cc11d2996c4f1e58ba47df714", size = 9087723, upload-time = "2026-06-02T11:53:50.702Z" }, + { url = "https://files.pythonhosted.org/packages/7d/79/f4a0c4fe9711154cddabf913471153af79056382ddc612cfe5ee0ff4b72e/scikit_learn-1.9.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5162ad10a418c8a282dde04c9aa06965de3e9a65f33c1440c0ae69bb1a09d913", size = 8847671, upload-time = "2026-06-02T11:54:04.448Z" }, + { url = "https://files.pythonhosted.org/packages/f0/af/4d72d9e475ac83719160c662619e4bf7b95c19507cd582e7d0167a3c3dae/scikit_learn-1.9.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1fea2cc5677ab49d6f5bade978c866da44957b712d92e9635e8b4f723013c3cb", size = 9118104, upload-time = "2026-06-02T11:54:07.205Z" }, + { url = "https://files.pythonhosted.org/packages/18/3c/e440e039bb82cd19004edaaad00acbde0fb9b461083c3ecf37941c557312/scikit_learn-1.9.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:147e9329ef0e39f75d4cffa02b2aa48d827832684926cd5210d9a2cb5c57246b", size = 8855050, upload-time = "2026-06-02T11:54:21.699Z" }, + { url = "https://files.pythonhosted.org/packages/43/26/b341b8dab5998da6270a3a42c2152c578501354d36f944b5856757035ef8/scikit_learn-1.9.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5bad8f8b9950321b54c965fdcbac6c6c55e79e16646b49977bcf3668d3870a1a", size = 9097190, upload-time = "2026-06-02T11:54:24.454Z" }, +] + [[package]] name = "scipy" version = "1.17.1" @@ -3748,6 +3870,7 @@ test = [ { name = "ruff", marker = "sys_platform == 'linux'" }, ] workflow = [ + { name = "cosmosis", marker = "sys_platform == 'linux'" }, { name = "mpi4py", marker = "sys_platform == 'linux'" }, { name = "snakemake", marker = "sys_platform == 'linux'" }, ] @@ -3761,6 +3884,7 @@ requires-dist = [ { name = "colorama" }, { name = "cosmo-numba", git = "https://github.com/aguinot/cosmo-numba.git?rev=main" }, { name = "cosmology", marker = "extra == 'glass'", specifier = "==2022.10.9" }, + { name = "cosmosis", marker = "extra == 'workflow'", specifier = ">=3.25" }, { name = "cryptography" }, { name = "cs-util", git = "https://github.com/CosmoStat/cs_util.git?rev=develop" }, { name = "emcee" }, @@ -4421,6 +4545,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/88/0a/469e2bd01be1490336e6c8707386845655d59261543315778a3ccc7e8019/zarr-3.2.1-py3-none-any.whl", hash = "sha256:f78cdd3d9687ad0e9f9cba2c5683b64f0c52589c19f685eeabe872e93cc0d2c7", size = 319617, upload-time = "2026-05-05T12:37:20.66Z" }, ] +[[package]] +name = "zeus-mcmc" +version = "2.5.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "matplotlib", marker = "sys_platform == 'linux'" }, + { name = "numpy", marker = "sys_platform == 'linux'" }, + { name = "pytest", marker = "sys_platform == 'linux'" }, + { name = "scikit-learn", marker = "sys_platform == 'linux'" }, + { name = "scipy", marker = "sys_platform == 'linux'" }, + { name = "seaborn", marker = "sys_platform == 'linux'" }, + { name = "setuptools", marker = "sys_platform == 'linux'" }, + { name = "tqdm", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/15/c0/248a600ae0f3d1437612821f7c528c895b6a8408a052c352d4cac46dcf94/zeus-mcmc-2.5.4.tar.gz", hash = "sha256:594baa90de4ad4488c4db5ed6a0446f7103bc4b3de787f4d7d23c91c9aa88769", size = 35051, upload-time = "2023-01-12T04:14:20.688Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/cb/99ee8021fa305d65b1fb369fabf96ccc65c60282407390e5355bb9e62f9f/zeus_mcmc-2.5.4-py3-none-any.whl", hash = "sha256:a64a7dae15f413200c6d590a3edfc3b8bc63c6bea3acdf123c0397c8089e123f", size = 24122, upload-time = "2023-01-12T04:14:19.24Z" }, +] + [[package]] name = "zipp" version = "4.1.0" diff --git a/workflow/README.md b/workflow/README.md index f20ffe8f..4171074e 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -51,9 +51,231 @@ Give the target *before* `--configfile`: `--configfile` takes one-or-more paths, so a target after it is read as a config file ("No such file: im_mbias"). Always dry-run first with `-n`. -The profile carries only cluster policy — no container settings (the image-sims -rules own their `apptainer exec` call) and no `OMP_NUM_THREADS` (pinned to 1 at -that same `apptainer exec` line, since the slurm executor's `--export=ALL` -propagates the driver's env, not a profile flag). Per-rule `mem_mb` / `runtime` -stay on the rules. Off-cluster, drop `--profile` and add `-j N`. See the -profile's own comments for the full rationale. +Every rule runs inside the sp_validation container: the profile sets +`software-deployment-method: apptainer` and `apptainer-args` (the bind mounts), +and Snakemake wraps each job's `shell:`/`script:` command in `apptainer exec` +itself — no rule writes its own `apptainer exec` call. The image name comes +from the `container:` directive in `workflow/Snakefile` (or a rule's own +override, e.g. the image-sims `SIF`). + +A few rules shell out to a host toolchain (CosmoCov, ImageMagick) and keep +`container: None`; each says why in its own docstring. + +`OMP_NUM_THREADS` is not set by the profile either: the slurm executor's +`--export=ALL` propagates the driver's env, not a profile flag, so a rule that +needs it pinned sets it itself. Per-rule `mem_mb` / `runtime` stay on the rules. + +### Off candide — the default profile + +Candide is where the analysis runs, so the candide profile is the one to reach +for. `profiles/default/config.yaml` exists for anywhere else — a laptop, another +cluster — and carries the machine-independent half (container wrapping, rerun +triggers, latency wait) with no SLURM layer: + +```bash +snakemake --profile workflow/profiles/default -s workflow/Snakefile \ + --configfile -j 4 +``` + +Snakemake cannot compose profiles, so both files carry that block (marked +`GENERIC` in each) — change one, change the other. `apptainer-args` is not part +of it: expect to edit the default profile's `--bind` list for your machine. + +### Which `sp_validation` a rule imports: the launched checkout + +The image is the frozen *dependency stack*; the `sp_validation` that runs is +the one in the checkout you launched from — `common.configure()` puts that +checkout's `src/` on each job's `PYTHONPATH` +(`common.inject_checkout_pythonpath` has the mechanics). + +This is the default because the alternative is incoherent: Snakemake's +`script:` directive already runs the checkout's *script files*, so without it a +rule executes new script code against an old `import sp_validation` — the two +halves of one commit, split. + +**Caveat:** `rerun-triggers: code` watches rule bodies and `script:` files, not +`src/`. Editing a module under `src/` does not by itself mark outputs stale — +force with `-F` or `--forcerun `. + +To reproduce a run from the image alone, opt out: + +```bash +snakemake --profile workflow/profiles/candide --config checkout_pythonpath=false +``` + +Either way the checkout has to sit under one of the profile's bind mounts to be +visible inside the job. + +Most of the time this default is all you need. Reach for a different *image* +only when the dependency stack changed — a new package, a lockfile bump — not +when only `src/` did; "Running an image other than your own" below has the +override. + +### Never write `/automnt/nXXdataN` in a path + +Use the plain form `/nXXdataN/...` in every rule, config, and invocation +directory. `/automnt/nXXdataN` works only from a node that does *not* own that +disk. On the owning node the disk is mounted directly at `/nXXdataN` and there +is no `/automnt/nXXdataN` entry at all, so a job that lands there dies about one +second after the allocation starts, before any log file is written. This is why +`n17` is in the profile's exclude list. Every canonical path in `common.py` +already uses the plain form; keep new paths the same. + +### Run Snakemake from the host, never from inside the container + +`snakemake` is a thin host-side tool, pinned once per machine: + +```bash +uv tool install snakemake==9.23.1 --with snakemake-executor-plugin-slurm +``` + +(match the version to `snakemake` in this repo's `uv.lock`). Run every +`snakemake` command directly on the host — do not `apptainer shell` first. +Snakemake itself never touches the science stack; it only reads rule +definitions and submits jobs. Each job carries its own `apptainer exec` +wrapping from the profile (see above), so the container is where the science +code runs, not where the orchestrator runs — one container per job, never a +nested one. + +Check for a stray `~/.local/bin/snakemake` (any host-side `pip install --user +snakemake` leaves one): Apptainer passes your `PATH` and mounts your `$HOME` by +default, so it can silently shadow the one `uv tool install` set up. `which +snakemake` should resolve under `uv tool dir`, not `~/.local/bin`. + +### The container image — one per person + +Everything runs one image, published by CI as a registry tag: + +``` +docker://ghcr.io/cosmostat/sp_validation:develop +``` + +**Each person keeps their own copy of it.** There is no shared image directory: +you pull your own file, you refresh it when you want to, and nobody else's +refresh moves the ground under your running jobs. The canonical path is + +``` +~/.cache/sp_validation/sp_validation.sif +``` + +and a small CLI, `spv-container`, is what puts it there and tells you about it: + +```bash +spv-container pull # fetch :develop to the canonical path (~1.5 GB / ~15 min) +spv-container status # what is here, which commit it was built from, how current +spv-container exec # run something inside it, candide binds already applied +``` + +It ships as a console script with the package, and — being stdlib-only, because +it has to run on the *host* — also works straight from a checkout with nothing +installed: run `python3 src/sp_validation/container.py`, or put it on your PATH +once (the README's install step): + +```bash +ln -s "$PWD/src/sp_validation/container.py" ~/.local/bin/spv-container +``` + +**Do the pull from a compute node**, not the login node: it moves ~1.5 GB and +takes about fifteen minutes. `pull` writes to a temporary name and renames, so a +job either gets the whole old image or the whole new one; jobs already running +hold the old file open and finish against it unharmed. + +```bash +salloc -p comp -c 4 --time=01:00:00 --exclude=n17,n09,n36 --no-shell # note the job id +srun --jobid= spv-container pull +scancel +``` + +**How the workflow finds it.** One resolution order, shared by the CLI and the +workflow: your **sandbox** if you have built one (below), else your **`.sif`** if +you have pulled one, else the **registry tag** — which Snakemake autopulls into +`.snakemake/singularity` under the working directory. That works, but re-pulls +once per run directory, so `spv-container pull` is the path to prefer. Snakemake +accepts all three forms, a sandbox directory included. The tag itself is written +down once, as +`CONTAINER_URI` in `sp_validation/container.py`, which `workflow/common.py` +re-exports; the image-sims `sif:` config key defaults to `null` and resolves the +same way. Override any of it with `--config container=...` (below), or point +somewhere else entirely with `SPV_CONTAINER`. + +At launch the workflow prints one advisory line if your image was built from a +commit behind your checkout. It never fails the run — an older image is normally +fine, since the checkout's `src/` is what rules import (see above). It matters +when the *dependency stack* moved: a new package, a lockfile bump. + +#### When you need to install something: the sandbox + +The pristine SIF is read-only, which is what you want almost always — it is +exactly the published image, and two people running it run the same thing. But +mid-analysis you sometimes need a package the image does not carry yet, and +rebuilding through CI to find out whether it helps is too slow a loop. + +For that, unpack the image into a writable directory once: + +```bash +spv-container sandbox # ~/.cache/sp_validation/sandbox/ +spv-container exec --writable pip install +``` + +Writes into a sandbox persist. It is opt-in — nothing builds one for you — and +once it exists **it takes precedence over the SIF everywhere, workflow jobs +included**, so a package you install this way is available to your Snakemake runs +without any further wiring. Jobs exec it read-only; only `--writable` writes. + +The cost is that what you are running is no longer fully described by a revision +label. `spv-container status` says which layer is live and flags that, and the +workflow prints one line at launch when a sandbox is in play — the divergence is +visible, never silent. When you are done exploring, either fold the dependency +into `pyproject.toml` (the real fix) or reset to a clean image: + +```bash +spv-container pull # refresh the pristine SIF +spv-container sandbox --force # discard the sandbox, rebuild from it +``` + +**Where the image comes from.** CI (`.github/workflows/deploy-image.yml`) builds +it on every push, `FROM ghcr.io/cosmostat/shapepipe:im_sims` with `uv sync +--frozen` against `uv.lock`, and publishes to `ghcr.io/cosmostat/sp_validation` +tagged by branch — so `:develop` tracks the tip of `develop`. The package is +public; no credentials are needed. Your pulled file is a *snapshot*: CI +publishing a new image changes nothing until you pull again. + +`spv-container status` reads `org.opencontainers.image.revision` — the +sp_validation commit the image was built from — and places it against your +checkout's `HEAD`, naming which layer (sandbox or SIF) it read. The image-sims workflow records the same label in +`m_bias_config.yaml` as `ghcr_revision`, so a result file says which image +produced the number. + +#### Running an image other than your own + +`container` is a config key read by every entry Snakefile +(`common.resolve_container`), so one flag overrides the default everywhere at +once — a local file, or any CI tag: + +```bash +snakemake --profile workflow/profiles/candide --config container=/path/to/my.sif +snakemake --profile workflow/profiles/candide \ + --config container=docker://ghcr.io/cosmostat/sp_validation:my-branch +``` + +CI tags an image for **every** branch, by sanitized branch name (`/` → `-`), so +the second form is how you test a branch's own stack. Snakemake autopulls a tag +per run directory (~15 minutes — from a compute node); `spv-container pull --tag +` instead puts it at your canonical path, where it becomes your default. + +For the image-sims workflow, set `image_sims: {sif: ...}` in your run config. +Either way the image has to sit under one of the profile's bind mounts to be +visible. + +One trap to know: the `script:` directive bind-mounts the host orchestrator's +`snakemake` into the job and *appends* it to `sys.path`, so a `snakemake` +importable inside the image wins the lookup. If `script:` rules start failing +with `ModuleNotFoundError: No module named 'snakemake.iocontainers'` or similar, +an in-image snakemake older than the host's is the first thing to check. + +### `snakemake` in `script:` files + +Every script run via a rule's `script:` directive uses a bare `snakemake` +name (`snakemake.input[...]`, etc.) with no import — Snakemake injects it as +a module global before the script runs. `from snakemake.script import +snakemake` is IDE-hint-only and raises `ImportError` if actually executed. diff --git a/workflow/Snakefile b/workflow/Snakefile index 7d91db81..fe05f02b 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -8,19 +8,15 @@ # or run standalone with --configfile pointing at a paper config # (e.g. papers/bmodes/config/config.yaml). -container: "/n17data/cdaley/containers/containers" - envvars: "PYTHONUNBUFFERED", import os import sys -# Host OpenMPI libs for MPI rules inside the container. -# Apptainer passes APPTAINERENV_* vars into the container as their unprefixed names. -# Only affects rules that import mpi4py; safe for all other rules. -os.environ["APPTAINERENV_LD_LIBRARY_PATH"] = "/softs/openmpi/5.0.5-slurm-CentOS8/lib" - +# Machine-specific env belongs to the machine, so it rides the profile's +# `apptainer-args --env`, not this file. See workflow/profiles/. +# # Shared helpers live in common.py next to this Snakefile. Snakemake's `module` # imports rules, not Python globals, so helpers travel by plain Python import; # when composed, the paper Snakefile has already imported common and this hits @@ -28,9 +24,14 @@ os.environ["APPTAINERENV_LD_LIBRARY_PATH"] = "/softs/openmpi/5.0.5-slurm-CentOS8 sys.path.insert(0, os.path.realpath(str(workflow.basedir))) import common +# configure() also puts this checkout's src/ on the container's PYTHONPATH -- +# see common.inject_checkout_pythonpath. common.configure(config) from common import * +# The one image every rule runs in; see common.resolve_container. +container: common.resolve_container(config.get("container")) + # Wildcard constraints — centralized in common.py, not in individual rule files wildcard_constraints: **WILDCARD_CONSTRAINTS diff --git a/workflow/common.py b/workflow/common.py index 3df3413b..506f7cb2 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -1,10 +1,36 @@ """Shared helpers for the B-modes Snakemake workflow.""" +import importlib.util import json import os import re +import sys from pathlib import Path +# This checkout's importable source tree: workflow/common.py -> /src. +REPO_SRC = Path(__file__).resolve().parent.parent / "src" + +# The container model lives in the package (``sp_validation/container.py``). +# Taken from *this checkout's* src/, so the workflow and the ``spv-container`` +# CLI can never disagree about which image to run. +# +# Loaded by file path rather than as ``sp_validation.container``: snakemake runs +# on the host, where sp_validation is usually not installed, and importing the +# package would drag in ``__init__`` -> ``version`` -> a metadata warning on +# every launch. The module itself is stdlib-only, so this costs nothing. +_container = importlib.util.module_from_spec( + importlib.util.spec_from_file_location( + "_spv_container", REPO_SRC / "sp_validation" / "container.py" + ) +) +sys.modules["_spv_container"] = _container +_container.__loader__.exec_module(_container) + +compare_revision = _container.compare_revision +image_revision = _container.image_revision +resolve_image = _container.resolve_image + + # Output roots are env-overridable so a reproduction run can write into a # fresh tree without clobbering (or silently reusing) prior products. COSMO_VAL = Path( @@ -50,9 +76,90 @@ PLANCK18 = None +def inject_checkout_pythonpath(workflow_config): + """Make the launched checkout's ``src`` win over the image's baked copy. + + Snakemake's ``script:`` directive already runs the *checkout's* script + files, so without this a rule executes new script code against an old + ``import sp_validation`` -- the two halves of one commit, split. Prepending + ``REPO_SRC`` closes that: the image stays the frozen dependency stack, the + checkout supplies sp_validation. + + Apptainer forwards ``APPTAINERENV_``-prefixed host variables into the job as + their unprefixed names, surviving the profile's ``--cleanenv``; setting it + here on the driver reaches every containerized rule. Any value the user + already exported is preserved behind ours. + + Opt out with ``--config checkout_pythonpath=false`` to reproduce a run from + the image alone. + """ + flag = workflow_config.get("checkout_pythonpath", True) + # `--config key=false` can arrive as the *string* "false" depending on how + # Snakemake parses the value, so don't lean on truthiness alone. + if isinstance(flag, str): + flag = flag.strip().lower() not in ("false", "no", "0", "off", "") + if not flag: + return + if not REPO_SRC.is_dir(): + return + existing = os.environ.get("APPTAINERENV_PYTHONPATH", "") + parts = [str(REPO_SRC)] + [p for p in existing.split(":") if p] + os.environ["APPTAINERENV_PYTHONPATH"] = ":".join(parts) + + +def resolve_container(override=None): + """Return the image every rule should run in. + + ``override`` wins if set (a ``docker://`` tag, a ``.sif`` path or a sandbox + directory -- Snakemake's ``container:`` accepts all three); otherwise + ``resolve_image()``, so jobs run what interactive ``spv-container`` work + runs. + """ + if override: + return str(override) + return resolve_image()[0] + + +def warn_if_image_stale(): + """Print one advisory line about a local image that is not pristine or current. + + Never fatal. Two things worth saying at launch: + + * a sandbox is in play, so what jobs run is not fully described by any + revision label -- deliberate, but it should not be a silent difference + from a clean run; + * the image predates the checkout. Usually fine, because the checkout's + ``src/`` is what rules import; it matters when the *dependency stack* + moved -- a new package, a lockfile bump. + + Silent when there is no local image, no apptainer, or no revision label. + """ + image, kind = resolve_image() + if kind == "tag": + return + revision = image_revision(image) + if kind == "sandbox": + built = f"built from {revision[:12]}" if revision else "revision unknown" + print( + f"[container] running the writable sandbox at {image} ({built}). " + "Anything installed into it is part of this run; " + "`spv-container status` for detail.", + file=sys.stderr, + ) + if compare_revision(revision) == "behind": + print( + f"[container] image was built from {revision[:12]}, which is behind this " + "checkout. Fine unless the dependency stack moved; refresh with " + "`spv-container pull`.", + file=sys.stderr, + ) + + def configure(workflow_config): """Install config-derived values after Snakemake has loaded configfiles.""" global CATALOG_CONFIG, DEFAULT_MASK_SUFFIX, FIDUCIAL, PLANCK18 + inject_checkout_pythonpath(workflow_config) + warn_if_image_stale() CATALOG_CONFIG = workflow_config FIDUCIAL = workflow_config["fiducial"] DEFAULT_MASK_SUFFIX = ( diff --git a/workflow/image_sims/Snakefile b/workflow/image_sims/Snakefile index d7a89065..7420ae68 100644 --- a/workflow/image_sims/Snakefile +++ b/workflow/image_sims/Snakefile @@ -4,38 +4,37 @@ Run the sp_validation-side chain (merge -> extract -> calibrate -> m-bias), optionally including the ShapePipe pipeline stage, without pulling in the cosmology-validation config the top-level ``workflow/Snakefile`` requires. -Layer a run config over the operational defaults -- the workflow config.yaml -carries operational defaults but *no* science keys, so it is incomplete on its -own (by design); the run config supplies the science knobs. The one drive -command on candide, with the committed SLURM profile owning all scheduling: +The one drive command on candide, with the committed SLURM profile owning +scheduling and container wrapping: snakemake --profile workflow/profiles/candide \\ -s workflow/image_sims/Snakefile \\ im_mbias --configfile my_run.yaml -``configfile: "workflow/image_sims/config.yaml"`` below loads the operational -defaults automatically, so only ``my_run.yaml`` (the science knobs, and any -operational override that run wants) is passed on the command line; Snakemake -deep-merges the two. The profile supplies the executor, account, partition, -node excludes and job floor -- no ``-j`` needed (the slurm executor sets the -job cap). Off-cluster, drop ``--profile`` and add ``-j N`` to run locally. - -The target (``im_mbias``) is given *before* ``--configfile``: Snakemake's -``--configfile`` takes one-or-more paths, so a target placed after it is -swallowed as a config path ("No such file: im_mbias"). Put targets ahead of -``--configfile`` (or make ``--configfile`` the last flag on the line). Always +``configfile:`` below loads the operational defaults, so ``my_run.yaml`` need +only carry the science knobs (and any operational override); Snakemake +deep-merges the two. Off-cluster, drop ``--profile`` and add ``-j N``. + +Put targets *before* ``--configfile``: it takes one-or-more paths, so a target +after it is swallowed as a config path ("No such file: im_mbias"). Always dry-run first with ``-n``. -The same rules are also available inside the main workflow: they are included -there under ``if "image_sims" in config``. +The same rules are also included in the main workflow under +``if "image_sims" in config``. """ -configfile: "workflow/image_sims/config.yaml" +import os +import sys +# Shared helpers from the generic workflow one directory up -- resolve_container +# in particular, so `sif: null` here falls back to the same image every other +# entry point runs. Snakemake's `include:` shares Python globals, so +# image_sims.smk sees this import. +sys.path.insert(0, os.path.realpath(os.path.join(str(workflow.basedir), ".."))) +import common -# The image-sims rules own their container invocation explicitly, so no -# top-level container is needed here. -container: None + +configfile: "workflow/image_sims/config.yaml" include: "../rules/image_sims.smk" diff --git a/workflow/image_sims/config.yaml b/workflow/image_sims/config.yaml index 8f719c68..f5f3e3c0 100644 --- a/workflow/image_sims/config.yaml +++ b/workflow/image_sims/config.yaml @@ -1,44 +1,31 @@ # Image-simulation m-bias workflow configuration. # -# Two kinds of keys live under `image_sims:`, and the split is the point: -# -# * OPERATIONAL keys default here (active lines below) and *nowhere else* -- -# the .smk reads them bare, so this file is their single home. Override in -# a run config only when a run genuinely differs from the shared setup. -# -# * SCIENCE keys have NO default -- not here, not in code. They fix the -# estimator's scientific behaviour and must be stated per run, so they -# appear below only as commented template lines. Supply them in a run -# config layered on top: -# -# snakemake -s workflow/image_sims/Snakefile \ -# --configfile workflow/image_sims/config.yaml \ -# --configfile my_run.yaml \ -# -j 4 im_mbias -# -# A run config that omits a science key fails at DAG parse, naming the key; an -# unknown key under `image_sims:` fails as a typo. The structural keys below -# (sif, repos, data roots, num, tile_ids) also have no default and must be set. +# OPERATIONAL keys default here and nowhere else -- the .smk reads them bare, so +# this file is their single home. SCIENCE keys have no default anywhere and +# appear below only as commented template lines: they fix the estimator's +# behaviour, so each run must state them in a run config layered on top. A run +# config that omits one fails at DAG parse, naming the key; an unknown key under +# `image_sims:` fails as a typo. image_sims: - # --- containers ------------------------------------------------------- - # Two images, one per half of the chain (the split gate766 ran). One image - # is the eventual target -- the sp_validation image is FROM the ShapePipe - # image -- but until sp_validation is uv-locked with cosmo_numba declared, - # its published image can drift NumPy past numba's window (seen 2026-07-11: - # "Numba needs NumPy 2.4 or less. Got NumPy 2.5" at ngmix). PYTHONPATH - # shadows pure-Python code only, never binary deps. - sif: /n17data/cdaley/containers/sp_validation_im_sims.sif # extract/calibrate/m-bias - sif_pipeline: /n17data/cdaley/containers/shapepipe_im_sims-runtime.sif # pipeline/merge - # Apptainer bind mounts. /automnt is required when repos/data are - # automounted (candide gotcha); harmless otherwise. [operational] - binds: /n17data,/n09data,/home,/automnt + # --- container -------------------------------------------------------- + # One image for the whole chain: the sp_validation image is built FROM the + # ShapePipe image, so it carries both stacks. CI builds it from the uv lock -- + # an unlocked build drifts NumPy past numba's ceiling and the ngmix stage dies + # ("Numba needs NumPy 2.4 or less"). + # + # `null` means the workflow's one image: your own .sif if you have pulled one + # with `spv-container pull`, else the registry tag for Snakemake to autopull + # (workflow/README.md). Neither is repeated here. + # Override with a local .sif path, or with a branch tag + # (docker://ghcr.io/cosmostat/sp_validation:) to run a + # branch's own CI image. + sif: null # --- repositories ----------------------------------------------------- # Bound into the image; both repos' src go on PYTHONPATH so this branch's - # code wins over the baked copies: ShapePipe's #766 build, and sp_validation's - # image_sims.py / catalog.match_catalogs_radec. + # code wins over the baked copies. shapepipe_repo: /n17data/cdaley/unions/code/shapepipe sp_validation_repo: /n17data/cdaley/unions/code/sp_validation diff --git a/workflow/profiles/candide/config.yaml b/workflow/profiles/candide/config.yaml index 0316b909..19afd89a 100644 --- a/workflow/profiles/candide/config.yaml +++ b/workflow/profiles/candide/config.yaml @@ -1,71 +1,60 @@ # Committed SLURM profile for the candide cluster (IAP). # -# This is the "one run command" half of the workflow: drive any target with +# Drive any target with # # snakemake --profile workflow/profiles/candide \ # -s workflow/image_sims/Snakefile \ # --configfile # -# and Snakemake owns all scheduling -- it fans out one SLURM job per branch x -# tile and drives them against the cluster, MPI-free. Everything here is -# cluster policy (executor, account, partition, node excludes, per-job -# defaults); it carries no science and no workflow logic. -# -# What is deliberately NOT here: -# -# * Container / apptainer settings. The image-sims rules set -# ``container: None`` and own their ``apptainer exec`` call through the -# shared ``EXEC`` prefix (one image for every stage, with PYTHONPATH / -# PSF_DICT / OMP_NUM_THREADS injected there). So no -# ``software-deployment-method: apptainer`` / ``apptainer-args`` -- those -# would wrap a *second*, redundant container around jobs that already run -# inside one. -# -# * OMP_NUM_THREADS. It is pinned to 1 on the ``apptainer exec`` line in -# workflow/rules/image_sims.smk, not here. The slurm executor submits with -# ``--export=ALL``, which propagates the *driver's* ambient environment; a -# profile only sets CLI flags, never the driver's own env, so an -# ``OMP_NUM_THREADS`` set here would silently depend on the operator having -# exported it by hand. Injecting it at the container boundary puts it where -# the compute runs, committed and independent of the launching shell. -# -# * Per-rule resources (mem_mb, runtime). Those live on each rule in the -# .smk; the ``default-resources`` below are only the floor for rules that -# set none. +# Snakemake owns scheduling and the container wrapping; run it host-side, never +# inside an ``apptainer shell``. workflow/README.md is the full story. executor: slurm -# Cluster policy applied to every job unless a rule overrides it. The excludes -# are the flaky/no-internet candide nodes (n17 mount issues, n09 no internet, -# n36); ``slurm_extra`` is passed verbatim onto the sbatch line by the executor -# plugin, so the quoting is what sbatch must see. +# --- GENERIC: mirrored in workflow/profiles/default/config.yaml ------------- +software-deployment-method: apptainer + +# Rerun a job when its code / params / inputs change, not only on mtime. +# Caveat: "code" watches rule bodies and ``script:`` files, not ``src/`` -- +# editing a module under src/ does not mark outputs stale on its own. +rerun-triggers: ["mtime", "params", "input", "code"] + +# Give an appearing output file a moment on networked filesystems before +# Snakemake calls a job failed for a missing output. +latency-wait: 5 +# --- end GENERIC ------------------------------------------------------------ + +# No ``apptainer-prefix``: the entry Snakefiles resolve ``container:`` to this +# user's own image path, so there is nothing for Snakemake to cache. # -# Three candide-specific SLURM lessons are baked into the values below (learned -# the hard way on the earlier hand-driven im-sims runs; see shapepipe's retired -# image_sims_pipeline/Snakefile docstring): +# candide's disks, plus the one machine-specific env var: the host OpenMPI libs +# MPI rules need to find libmpi inside the container (only rules importing +# mpi4py care; harmless for the rest). The bind list matches ``spv-container +# exec``'s default; keep the two in step. +apptainer-args: >- + --cleanenv + --bind /home,/scratch,/automnt,/n17data,/n23data1,/n09data + --env LD_LIBRARY_PATH=/softs/openmpi/5.0.5-slurm-CentOS8/lib + +# Cluster policy applied to every job unless a rule overrides it. Excludes are +# the flaky/no-internet candide nodes (n17 mount issues, n09 no internet, n36). # -# * ``runtime`` MUST carry a unit (``60m``, ``6h``, ``2d``). Snakemake's -# resource parser reads a *bare* number as SECONDS, so ``runtime: 60`` would -# silently give every job a 60-second wall clock and kill it on start. The -# quoted-with-unit form here is deliberate; keep it that way, and prefer the -# same in any ``--default-resources`` passed on the command line. (A bare -# integer in a *rule's* ``resources: runtime=720`` is fine -- snakemake -# reads rule-level numeric runtime as minutes -- the seconds trap is only +# * ``runtime`` MUST carry a unit (``60m``, ``6h``, ``2d``). Snakemake's +# resource parser reads a bare number as SECONDS, so ``runtime: 60`` would +# silently give every job a 60-second wall clock and kill it on start. +# (A bare integer in a rule's own ``resources: runtime=720`` is fine -- +# rule-level numeric runtime is read as minutes; the seconds trap is only # the CLI/default-resources parser.) # # * ``cpus_per_task`` is pinned to 12 to CAP JOBS PER NODE, not because a job -# needs 12 cores (the chain is MPI-free and pins ``OMP_NUM_THREADS=1`` at -# the container). candide's per-user process limit is ``ulimit -u 1200`` -# *per node*, and apptainer crashes ("can't start new thread") beyond ~4 -# concurrent jobs on a 48-core node. Requesting 12 CPUs/job holds SLURM to -# ~4 jobs per 48-core node, under the ceiling. Dropping this to 1 would let -# SLURM pack ~48 jobs onto a node and crash the compute-heavy im_pipeline -# stage (which inherits this default -- it sets mem/runtime but not cpus). +# needs 12 cores. candide's per-user process limit is ``ulimit -u 1200`` +# per node, and apptainer crashes ("can't start new thread") beyond ~4 +# concurrent jobs on a 48-core node; 12 CPUs/job holds SLURM to ~4 jobs per +# node. # -# * After launching a real fan-out, VERIFY the request actually landed: -# ``squeue -u $USER -o "%C %l"`` must show 12 (CPUs) and the wall clock you -# intended (e.g. 12:00:00 for im_pipeline). A silently-misparsed runtime or -# cpus shows up here before it wastes a queue slot. +# * After launching a real fan-out, verify the request landed: +# ``squeue -u $USER -o "%C %l"`` should show 12 (CPUs) and the intended +# wall clock. default-resources: slurm_account: "cusers" slurm_partition: "comp,pscomp" @@ -73,13 +62,7 @@ default-resources: cpus_per_task: 12 slurm_extra: "'--exclude=n17,n09,n36'" -# Give an appearing output file a moment on candide's automounted filesystems -# before Snakemake calls a job failed for a missing output, and retry a job -# once on transient node failure. -latency-wait: 5 +# Retry a job once on transient node failure, and keep the SLURM logs of +# successful jobs (candide debugging). retries: 1 - -# Keep the SLURM logs of successful jobs (candide debugging), and rerun a job -# when its code / params / inputs change, not only on mtime. slurm-keep-successful-logs: true -rerun-triggers: ["mtime", "params", "input", "code"] diff --git a/workflow/profiles/default/config.yaml b/workflow/profiles/default/config.yaml new file mode 100644 index 00000000..bf272c03 --- /dev/null +++ b/workflow/profiles/default/config.yaml @@ -0,0 +1,39 @@ +# Machine-independent profile: the container model, and nothing else. +# +# Use it anywhere that is not candide -- a laptop, a workstation, another +# cluster's interactive node: +# +# snakemake --profile workflow/profiles/default -s workflow/Snakefile \ +# --configfile -j 4 +# +# On candide -- where the analysis actually runs -- use +# `--profile workflow/profiles/candide` instead: the GENERIC block below plus +# the SLURM executor and candide's machine layer. +# +# Requirements are the same everywhere: `apptainer` on PATH, and `snakemake` +# installed host-side (`uv tool install ...`, see workflow/README.md) -- never +# run from inside an apptainer shell. + +# --- GENERIC: mirrored in workflow/profiles/candide/config.yaml ------------- +# Wrap each job's `shell:`/`script:` command in `apptainer exec`, using the +# image named by the entry Snakefile's `container:` directive. +software-deployment-method: apptainer + +# Rerun a job when its code / params / inputs change, not only on mtime. +# Caveat: "code" watches rule bodies and `script:` files, not `src/` -- editing +# a module under src/ does not mark outputs stale on its own. +rerun-triggers: ["mtime", "params", "input", "code"] + +# Give an appearing output file a moment on networked filesystems before +# Snakemake calls a job failed for a missing output. +latency-wait: 5 +# --- end GENERIC ------------------------------------------------------------ + +# Binds are the one thing you almost certainly need to edit for your machine: +# whatever paths your inputs, outputs and checkout live under. `--cleanenv` so a +# job's environment is the image's, not your shell's. If $HOME and the working +# directory cover everything (apptainer mounts both by default), drop `--bind`. +apptainer-args: "--cleanenv --bind /home" + +# No `apptainer-prefix`, here or on candide: the entry Snakefiles resolve +# `container:` to this user's own image path (workflow/README.md). diff --git a/workflow/rules/covariance.smk b/workflow/rules/covariance.smk index 49e40201..e9185428 100644 --- a/workflow/rules/covariance.smk +++ b/workflow/rules/covariance.smk @@ -11,7 +11,6 @@ def get_cat_params(version): # covariance_dir(), covariance_base(), covariance_path() defined in Snakefile -# Additional wildcard constraints defined locally for pseudo-Cl rules (line 327) # DEFAULT_MASK_SUFFIX defined in Snakefile # Footprint mask power spectra (nside=4096, from comprehensive catalog with spatial cuts only) @@ -145,6 +144,12 @@ EOF rule covariance_cosmocov: + """Run the host-compiled CosmoCov binary. + + `container: None` on purpose: CosmoCov is a host-compiled Fortran/C binary + loaded through environment-modules (`module load gcc intelpython openmpi`), + not a Python entry point the container ships. + """ input: rules.covariance_ini.output, output: @@ -203,13 +208,13 @@ rule covariance_glass_mock: seed=range(config["glass_mocks"]["seed_range"][0], config["glass_mocks"]["seed_range"][1] + 1), ), output: - xi_covariance="/automnt/n17data/cdaley/unions/pure_eb/results/covariance/glass_mock_v1.4.6/xi_covariance.npy", - cl_covariance="/automnt/n17data/cdaley/unions/pure_eb/results/covariance/glass_mock_v1.4.6/cl_covariance.npy", - combined_covariance="/automnt/n17data/cdaley/unions/pure_eb/results/covariance/glass_mock_v1.4.6/combined_covariance.npy", - correlation_plot="/automnt/n17data/cdaley/unions/pure_eb/results/covariance/glass_mock_v1.4.6/combined_correlation.png", - xi_mean="/automnt/n17data/cdaley/unions/pure_eb/results/covariance/glass_mock_v1.4.6/xi_mean.npy", - cl_mean="/automnt/n17data/cdaley/unions/pure_eb/results/covariance/glass_mock_v1.4.6/cl_mean.npy", - combined_mean="/automnt/n17data/cdaley/unions/pure_eb/results/covariance/glass_mock_v1.4.6/combined_mean.npy", + xi_covariance="results/covariance/glass_mock_v1.4.6/xi_covariance.npy", + cl_covariance="results/covariance/glass_mock_v1.4.6/cl_covariance.npy", + combined_covariance="results/covariance/glass_mock_v1.4.6/combined_covariance.npy", + correlation_plot="results/covariance/glass_mock_v1.4.6/combined_correlation.png", + xi_mean="results/covariance/glass_mock_v1.4.6/xi_mean.npy", + cl_mean="results/covariance/glass_mock_v1.4.6/cl_mean.npy", + combined_mean="results/covariance/glass_mock_v1.4.6/combined_mean.npy", script: "../scripts/compute_glass_mock_covariance.py" @@ -231,30 +236,21 @@ rule generate_glass_mock_rhotau_samples: mock_id="{mock_id}", output_dir="results/glass_mock_rhotau_samples", threads: 1 - shell: - """ - python /n17data/cdaley/unions/pure_eb/code/sp_validation/workflow/scripts/generate_glass_mock_rhotau_samples.py \ - --cov-tau {input.cov_tau} \ - --ref-tau {input.ref_tau} \ - --output-dir {params.output_dir} \ - --mock-ids {params.mock_id} - """ + script: + "../scripts/generate_glass_mock_rhotau_samples.py" rule covariance_process: + """Post-process a raw CosmoCov matrix into the analysis-ready form.""" input: str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}.txt") output: matrix=str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}_processed.txt"), gaussian=str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}_processed_g.txt"), plot=str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}_processed_plot.pdf") - params: - output_stub=str(COSMO_INFERENCE / "data/covariance/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}/covariance_{version}_{blind}_{gaussian}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}{mask_suffix}_processed") threads: 1 - shell: - """ - python /n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_inference/scripts/cosmocov_process.py {input} {params.output_stub} - """ + script: + "../scripts/cosmocov_process.py" def fiducial_covariance_outputs(mask_suffix=""): diff --git a/workflow/rules/glass_mock.smk b/workflow/rules/glass_mock.smk index da8cb491..243076d6 100644 --- a/workflow/rules/glass_mock.smk +++ b/workflow/rules/glass_mock.smk @@ -119,7 +119,7 @@ rule mock_cosebis_bias_test: ), xi_ref=f"{MOCK_RESULTS}/gg_glass_mock_00001_nbins=1000.fits", cov=str( - Path("/n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_inference/data/covariance") + COSMO_INFERENCE / "data/covariance" / "covariance_SP_v1.4.6_leak_corr_A_g_minsep=0.5_maxsep=500.0_nbins=1000_masked" / "covariance_SP_v1.4.6_leak_corr_A_g_minsep=0.5_maxsep=500.0_nbins=1000_masked_processed.txt" ), diff --git a/workflow/rules/image_sims.smk b/workflow/rules/image_sims.smk index a4eb92d0..cc992391 100644 --- a/workflow/rules/image_sims.smk +++ b/workflow/rules/image_sims.smk @@ -1,43 +1,18 @@ """Image-simulation orchestration: raw SKiLLS sim images -> shear m/c bias. -This rule set drives the image-simulation validation chain end to end and is -the sp_validation-side half of the split described in +The sp_validation-side half of the split described in ``UNIONS-WL/MultiBand_ImSim#1``: ShapePipe turns the simulated tiles into -per-tile shape catalogues, then sp_validation merges, extracts, calibrates and -finally measures the multiplicative/additive shear bias. - -Two images, one prefix shape. Architecturally one image could run every -stage -- the sp_validation image is built ``FROM`` the ShapePipe image, so it -carries both stacks -- but the *published* sp_validation image's environment -is not yet trustworthy for the ShapePipe half: sp_validation has no lockfile -and does not declare its numba-bearing dependency (``cosmo_numba``), so -unpinned install layers can drift NumPy past numba's window (a 2026-07-11 -gate run hit exactly this: ``Numba needs NumPy 2.4 or less. Got NumPy 2.5`` -at the ngmix stage). PYTHONPATH shadowing covers pure-Python *code*, never -binary deps, so until sp_validation is uv-locked with its deps declared -(spun off as its own task), each half runs in its own repo's image -- the -same split the gate766 baseline ran: - -* ShapePipe stages -> ``pipeline`` (raw images -> per-tile cats) and ``merge`` - (``create_final_cat`` -> ``final_cat_{sim}.hdf5``) run in ``sif_pipeline`` - (the ShapePipe image). -* sp_validation stages -> ``manifest``, ``extract`` (-> comprehensive cat), - ``calibrate`` (-> cut cat) and ``m_bias`` (-> ``m_bias_results.yaml``) run - in ``sif`` (the sp_validation image). - -Every rule sets ``container: None`` and calls ``apptainer exec`` explicitly -through a shared prefix template (``EXEC_PIPELINE`` / ``EXEC`` -- identical -env injections, different image), because the images are not the workflow's -top-level container. Everything is parameterised under -``config["image_sims"]`` -- the two ``sif`` keys, repository roots, data roots, -the PSF dictionary, the explicit ``tile_ids`` list and the sim/calibration -knobs -- so a fresh user drives it from config alone, with no hard-coded clone -layout. Configuration is fail-fast: a schema check at load rejects an unknown -key (typo) and a missing science key (see ``workflow/image_sims/config.yaml`` -for the operational/science split). The ``PYTHONPATH`` override injects both -repos' ``src`` so the *branch* source (ShapePipe's ``#766`` build; -sp_validation's ``image_sims.py``, ``catalog.match_catalogs_radec``) wins over -whatever is baked into the image. +per-tile shape catalogues (``pipeline``, ``merge``), then sp_validation +extracts, calibrates and measures the multiplicative/additive shear bias +(``manifest``, ``extract``, ``calibrate``, ``m_bias``). + +One image runs the whole chain: the sp_validation image is built ``FROM`` the +ShapePipe image, so it carries both stacks. Everything else is parameterised +under ``config["image_sims"]`` -- repository roots, data roots, the PSF +dictionary, the explicit ``tile_ids`` list and the sim/calibration knobs -- so +a fresh user drives it from config alone. Configuration is fail-fast: a schema +check at load rejects an unknown key (typo) and a missing science key (see +``workflow/image_sims/config.yaml`` for the operational/science split). The five simulations per grid are the reference ``1z2z`` (no input shear) plus the ``+/-`` shear pairs ``1p2z``/``1m2z`` (g1) and ``1z2p``/``1z2m`` (g2); the @@ -45,24 +20,15 @@ m-bias estimator matches each to the reference by RA/Dec. """ import os -from pathlib import Path IMSIM = config["image_sims"] # --- fail-fast schema check ---------------------------------------------- -# One home for every fact: the run config carries the science knobs, the -# workflow config.yaml carries the operational defaults, and *this* block is -# where a typo or a missing knob dies -- at DAG parse, before any compute. +# An unknown key under ``image_sims:`` is a hard error (typo protection); a +# missing key is a hard error naming it. Both fire at DAG parse, before compute. # -# Every key must be declared below. An unknown key under ``image_sims:`` is a -# hard error (typo protection); a missing *science* key is a hard error naming -# the key (no silent code default anywhere). Operational keys default in the -# workflow config.yaml and nowhere else: the .smk reads them as bare -# ``IMSIM[key]`` (never ``.get`` with a second literal), so their value comes -# from config.yaml alone -- the single home for an operational default. -# -# Science keys: required from the *run* config; no default in config.yaml (only -# a commented template line) and no default in code. These fix the estimator's +# Science keys: required from the *run* config; no default here or in +# config.yaml (only a commented template line). These fix the estimator's # scientific behaviour, so they must be stated per run, never inherited. _SCIENCE_KEYS = { "w_cols", @@ -72,16 +38,14 @@ _SCIENCE_KEYS = { "bootstrap_seed", "mask_config", } -# Deprecated science keys: accepted (so a pre-``w_cols`` run config still parses -# and the estimator's back-compat path runs) but not *required* -- our configs -# state ``w_cols``. Listed here only to keep them out of the unknown-key error. +# Deprecated but still accepted, so a pre-``w_cols`` run config keeps parsing +# into the estimator's back-compat path. _DEPRECATED_KEYS = { "w_col", } -# Operational keys: default (visibly) in the workflow config.yaml; the .smk -# reads them bare, so config.yaml is their one home. +# Operational keys: default in the workflow config.yaml; the .smk reads them +# bare (never ``.get`` with a literal), so config.yaml is their one home. _OPERATIONAL_KEYS = { - "binds", "sims_type", "branches", "shape", @@ -94,7 +58,6 @@ _OPERATIONAL_KEYS = { # Structural keys: paths/identifiers the run must supply (no sensible default). _STRUCTURAL_KEYS = { "sif", - "sif_pipeline", "shapepipe_repo", "sp_validation_repo", "grids_base", @@ -128,13 +91,13 @@ if _missing_structural: f"{_missing_structural} -- set them in the run config" ) -# --- containers ----------------------------------------------------------- -# Two images (see module docstring): the ShapePipe image for the pipeline and -# merge stages, the sp_validation image for everything downstream. Collapse -# back to one image once sp_validation's env is lock-managed. -SIF = IMSIM["sif"] # sp_validation stages -SIF_PIPELINE = IMSIM["sif_pipeline"] # ShapePipe stages -BINDS = IMSIM["binds"] +# --- container ------------------------------------------------------------ +# Every compute rule carries ``container: SIF`` rather than inheriting a +# module-level default: these rules are also included from the top-level +# workflow/Snakefile, whose module default is the cosmology image (no ShapePipe +# stack). Binds come from the driving profile's ``apptainer-args``. A null +# ``sif`` resolves to the workflow's one image (see workflow/image_sims/config.yaml). +SIF = common.resolve_container(IMSIM["sif"]) # --- repositories (bound into the image; branch code overrides) ----------- SHAPEPIPE_REPO = IMSIM["shapepipe_repo"] @@ -181,48 +144,27 @@ CALIBRATE = IMSIM["calibrate_script"] # m-bias is *this branch's* extracted core, injected on PYTHONPATH. COMPUTE_M_BIAS = f"{SPV_REPO}/scripts/compute_m_bias_image_sims.py" -# --- container exec prefixes ---------------------------------------------- -# One prefix *shape* for every stage -- two instances, one per image. Three -# env injections make the on-disk branch -# code and the sim PSF win over the image's baked copies: +# --- in-command env prefix ------------------------------------------------- +# Snakemake wraps each rule's whole ``shell:`` string inside the container, so +# these ``VAR=value`` tokens land inside it. Three settings: # -# * PYTHONPATH prepends BOTH repos' ``src`` (ShapePipe first, then -# sp_validation), so Python resolves the worktree build before -# ``/app``/``/sp_validation`` -- the local-testing counterpart of the -# git-ref deps, letting the branch code run without an image rebuild. This -# covers the Python *packages* only: the bash entry points (run_job) and -# the ShapePipe/sp_validation *scripts* are still invoked at the repo paths -# resolved from config (RUN_JOB, CREATE_FINAL_CAT, EXTRACT_INFO, ...), not -# shadowed by PYTHONPATH. +# * PYTHONPATH prepends both repos' ``src`` so Python resolves the worktree +# build ahead of the copies baked into the image -- the branch's code runs +# without an image rebuild. Packages only: the bash and python entry points +# are invoked at the repo paths from config (RUN_JOB, CREATE_FINAL_CAT, +# EXTRACT_INFO, ...), not shadowed by PYTHONPATH. # * PSF_DICT points the fake_psf module (PSF_DICT_PATH = $PSF_DICT, expanded # via getexpanded) at this run's PSF dictionary. -# -# The SLURM env vars are stripped (``env -u ...``) so that when the ShapePipe -# pipeline stage's OpenMPI initialises inside the image it does not try to -# attach to the host SLURM launcher (cf. apptainer_noslurm.sh). The strip is -# harmless for the pure-Python sp_validation stages, so one prefix serves all. -# -# ``OMP_NUM_THREADS=1`` is injected here, at the ``apptainer exec`` call, and -# not left to the SLURM profile. The chain is MPI-free: Snakemake fans out one -# job per branch x tile and each job's parallelism is ShapePipe's own internal -# multiprocessing (``-N n_smp``), so the OpenMP/BLAS thread pool inside the -# container must be pinned to 1 to avoid oversubscription. The SLURM profile -# cannot pin it reliably: the slurm executor submits with ``--export=ALL``, -# which propagates the *driver's* ambient environment -- but a Snakemake -# profile only sets CLI flags, never the driver's own env, so an -# ``OMP_NUM_THREADS`` there would depend on the operator having exported it by -# hand (the implicit, uncommitted state the "one run command" is meant to -# retire). Injecting it on the ``apptainer exec`` line puts it where the -# compute actually runs -- inside the container, independent of the driver's -# env -- the same lever this prefix already uses for PYTHONPATH/PSF_DICT. -_EXEC_PREFIX = ( - "env -u SLURM_JOBID -u SLURM_JOB_ID -u SLURM_PROCID " - f"apptainer exec --bind {BINDS} " - f"--env PYTHONPATH={SHAPEPIPE_REPO}/src:{SPV_REPO}/src " - f"--env PSF_DICT={PSF_DICT} --env OMP_NUM_THREADS=1 " +# * OMP_NUM_THREADS=1 rides here rather than in the SLURM profile: the chain +# is MPI-free (Snakemake fans out one job per branch x tile; in-job +# parallelism is ShapePipe's own ``-N n_smp``), so the OpenMP/BLAS pool must +# be pinned to 1 to avoid oversubscription, and a profile can only set CLI +# flags, never the driver env the slurm executor's ``--export=ALL`` +# propagates. +_ENV_PREFIX = ( + f"PYTHONPATH={SHAPEPIPE_REPO}/src:{SPV_REPO}/src " + f"PSF_DICT={PSF_DICT} OMP_NUM_THREADS=1 " ) -EXEC = _EXEC_PREFIX + SIF # sp_validation stages -EXEC_PIPELINE = _EXEC_PREFIX + SIF_PIPELINE # ShapePipe stages JOB_MASK = sum([1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048]) @@ -300,8 +242,10 @@ rule im_manifest: input_sims_base=INPUT_SIMS_BASE, sims_type=SIMS_TYPE, num=NUM, + container: + SIF shell: - "{EXEC} python {BUILD_MANIFEST} " + "{_ENV_PREFIX} python {BUILD_MANIFEST} " "--input-sims-base {params.input_sims_base} " "--sims-type {params.sims_type} --num {params.num} " "{params.branch_args} -o {output.manifest}" @@ -349,22 +293,17 @@ rule im_init: rule im_pipeline: - """Run ShapePipe on one simulated tile (ShapePipe stage). + """Run ShapePipe on one simulated tile. Delegates the module DAG to ShapePipe's own job runner; the sentinel log - marks tile completion for the merge step. This is the compute-heavy, - MPI-bearing stage. + marks tile completion for the merge step. The compute-heavy stage. """ input: - # ``params.py`` is a *tracked* output of ``im_init``, so this one input - # supplies the im_init -> im_pipeline edge. The ``cfis`` symlink the - # shell reads (via {RUN_JOB}) is created by that same im_init shell block - # as an *untracked* side effect -- no rule declares it as an output - # (snakemake will not track a symlink/directory output). Declaring it an - # input here therefore asked the DAG for a file no rule produces: on a - # fresh grids_base it aborted the build with MissingInputException before - # any job ran. It is safe to drop -- cfis exists whenever params does, - # since im_init stages both together. + # params.py alone supplies the im_init -> im_pipeline edge. The `cfis` + # symlink {RUN_JOB} also reads is an untracked side effect of the same + # im_init shell (snakemake will not track a symlink output), so it must + # not be declared here -- doing so asks the DAG for a file no rule + # produces and aborts on a fresh grids_base. params=f"{GRIDS_BASE}/{{sim}}/params.py", output: done=touch(f"{GRIDS_BASE}/{{sim}}/logs/pipeline_{{tile}}.done"), @@ -375,9 +314,11 @@ rule im_pipeline: resources: mem_mb=16000, runtime=720, + container: + SIF shell: "cd {params.run_dir} && " - "{EXEC_PIPELINE} bash {RUN_JOB} " + "{_ENV_PREFIX} bash {RUN_JOB} " "-e {wildcards.tile} -t image_sims -j {JOB_MASK} " "-p {params.psf} -N {params.n_smp}" @@ -397,9 +338,11 @@ rule im_merge: cat=f"{GRIDS_BASE}/{{sim}}/final_cat_{{sim}}.hdf5", params: run_dir=lambda wc: f"{GRIDS_BASE}/{wc.sim}", + container: + SIF shell: "cd {params.run_dir} && " - "{EXEC_PIPELINE} python {CREATE_FINAL_CAT} " + "{_ENV_PREFIX} python {CREATE_FINAL_CAT} " "-I -m final_cat_{wildcards.sim}.hdf5 -i .. " "-p cfis/final_cat.param -P {wildcards.sim} " "-o n_tiles_final.txt -v" @@ -418,8 +361,10 @@ rule im_extract: cat=f"{GRIDS_BASE}/{{sim}}/shape_catalog_comprehensive_{SHAPE}.fits", params: run_dir=lambda wc: f"{GRIDS_BASE}/{wc.sim}", + container: + SIF shell: - "cd {params.run_dir} && {EXEC} python {EXTRACT_INFO}" + "cd {params.run_dir} && {_ENV_PREFIX} python {EXTRACT_INFO}" rule im_calibrate: @@ -436,20 +381,16 @@ rule im_calibrate: cat=f"{GRIDS_BASE}/{{sim}}/shape_catalog_cut_{SHAPE}.fits", params: run_dir=lambda wc: f"{GRIDS_BASE}/{wc.sim}", + container: + SIF shell: "cd {params.run_dir} && " - "{EXEC} python {CALIBRATE} -s calibrate" + "{_ENV_PREFIX} python {CALIBRATE} -s calibrate" -rule im_mbias: - """Multiplicative/additive shear bias from the calibrated grids. - - Produces the workflow's headline artifact, ``m_bias_results.yaml``. The - injected shear (``shear_amplitude`` and the branch map) comes from - ``manifest.yaml`` alone -- no literal amplitude here or in config.yaml. The - generated ``m_bias_config.yaml`` carries the manifest's ``branches`` and - ``pairs``, so the estimator's sim list and pairing are the campaign's, not a - hard-coded default. +rule im_mbias_config: + """Assemble ``m_bias_config.yaml`` for the m-bias step: the manifest's + shear/branch facts, this run's science knobs, and git/container provenance. """ input: manifest=MANIFEST, @@ -457,112 +398,39 @@ rule im_mbias: f"{GRIDS_BASE}/{{sim}}/shape_catalog_cut_{SHAPE}.fits", sim=SIMS ), output: - results=f"{GRIDS_BASE}/results/m_bias_results.yaml", - params: cfg=f"{GRIDS_BASE}/results/m_bias_config.yaml", + params: grids_base=GRIDS_BASE, num=NUM, cat_name=f"shape_catalog_cut_{SHAPE}.fits", sif=SIF, - sif_pipeline=SIF_PIPELINE, shapepipe_repo=SHAPEPIPE_REPO, sp_validation_repo=SPV_REPO, + results_dir=f"{GRIDS_BASE}/results", + results=f"{GRIDS_BASE}/results/m_bias_results.yaml", # Science knobs, read bare from the run config (no default here). match_radius_deg=IMSIM["match_radius_deg"], w_cols=IMSIM["w_cols"], n_bootstrap=IMSIM["n_bootstrap"], pair_match=IMSIM["pair_match"], bootstrap_seed=IMSIM["bootstrap_seed"], - run: - import hashlib - import re - import subprocess - - import yaml - - with open(input.manifest) as fh: - manifest = yaml.safe_load(fh) - - def _git(repo, *args): - """Read a git fact from ``repo``; ``None`` if it is not a checkout.""" - try: - return subprocess.run( - ["git", "-C", repo, *args], - capture_output=True, - text=True, - check=True, - ).stdout.strip() - except (subprocess.CalledProcessError, FileNotFoundError): - return None - - def _sif_revision(sif_path): - """GHCR revision baked into the SIF's OCI labels. - - A plain-text scan of the image file (login-safe: no exec, no - container start), reading org.opencontainers.image.revision -- the - source commit GHCR built the image from. ``None`` if absent. - """ - try: - with open(sif_path, "rb") as fh: - blob = fh.read() - except OSError: - return None - m = re.search( - rb'org\.opencontainers\.image\.revision"?[:=]"?([0-9a-f]{7,40})', - blob, - ) - return m.group(1).decode() if m else None - - # Manifest hash: sha256 of the exact bytes im_manifest wrote, so the - # result records which injected-shear facts it was computed against. - with open(input.manifest, "rb") as fh: - manifest_sha256 = hashlib.sha256(fh.read()).hexdigest() - - provenance = { - "manifest_sha256": manifest_sha256, - "sp_validation": { - "branch": _git(params.sp_validation_repo, "rev-parse", "--abbrev-ref", "HEAD"), - "commit": _git(params.sp_validation_repo, "rev-parse", "HEAD"), - }, - "shapepipe": { - "branch": _git(params.shapepipe_repo, "rev-parse", "--abbrev-ref", "HEAD"), - "commit": _git(params.shapepipe_repo, "rev-parse", "HEAD"), - }, - "containers": { - "sif": params.sif, - "ghcr_revision": _sif_revision(params.sif), - "sif_pipeline": params.sif_pipeline, - "ghcr_revision_pipeline": _sif_revision(params.sif_pipeline), - }, - } - - os.makedirs(os.path.dirname(output.results), exist_ok=True) - # Emit *every* key the estimator requires -- pair_match and - # bootstrap_seed included. Requiring a key without emitting it would - # be a KeyError at run time, so the generated config is the complete - # contract between rule and estimator. ``provenance`` rides along as a - # top-level block: the compute script copies it verbatim into the output - # results yaml, so a result file is self-describing (which manifest, - # which repo commits, which container built the number). - mbias_cfg = { - "grids_dir": params.grids_base, - "num": params.num, - "catalog_name": params.cat_name, - # Injected shear: from the manifest, the single source of truth. - "shear_amplitude": manifest["shear_amplitude"], - "branches": list(manifest["branches"]), - "pairs": manifest["pairs"], - "match_radius_deg": params.match_radius_deg, - "w_cols": list(params.w_cols), - "pair_match": params.pair_match, - "n_bootstrap": params.n_bootstrap, - "bootstrap_seed": params.bootstrap_seed, - "results_dir": os.path.dirname(output.results), - "output_path": output.results, - "provenance": provenance, - } - with open(params.cfg, "w") as fh: - yaml.safe_dump(mbias_cfg, fh) - shell( - "{EXEC} python {COMPUTE_M_BIAS} -c {params.cfg} -v" - ) + container: + SIF + script: + "../scripts/im_mbias_config.py" + + +rule im_mbias: + """Multiplicative/additive shear bias from the calibrated grids. + + Produces the workflow's headline artifact, ``m_bias_results.yaml``, running + the estimator against the config ``im_mbias_config`` assembled. + """ + input: + cfg=f"{GRIDS_BASE}/results/m_bias_config.yaml", + output: + results=f"{GRIDS_BASE}/results/m_bias_results.yaml", + container: + SIF + shell: + "{_ENV_PREFIX} python {COMPUTE_M_BIAS} -c {input.cfg} -v" diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index 22c09db2..861c3b05 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -24,46 +24,53 @@ rule xi: "../scripts/run_2pcf.py" -rule xi_highres: - """High-resolution xi for COSEBIS integration.""" - container: None - output: - txt=str(COSMO_VAL / f"{FIDUCIAL['version']}_xi_minsep={FIDUCIAL['min_sep_int']}_maxsep={FIDUCIAL['max_sep_int']}_nbins=10000_npatch=1.txt"), - xi_plus=str(COSMO_VAL / f"xi_plus_{FIDUCIAL['version']}_minsep={FIDUCIAL['min_sep_int']}_maxsep={FIDUCIAL['max_sep_int']}_nbins=10000_npatch=1.fits"), - xi_minus=str(COSMO_VAL / f"xi_minus_{FIDUCIAL['version']}_minsep={FIDUCIAL['min_sep_int']}_maxsep={FIDUCIAL['max_sep_int']}_nbins=10000_npatch=1.fits"), - resources: - tasks=30, - cpus_per_task=12, - nodes=6, - mem_mb_per_cpu=2000, - runtime=2880, - slurm_extra="'--exclude=n17,n09,n36 --partition=pscomp'", - mpi="/softs/openmpi/5.0.5-slurm-CentOS8/bin/mpiexec", - shell: - "{resources.mpi} -n {resources.tasks} " - "apptainer exec " - "--bind /home,/n09data,/n17data,/n23data1,/softs " - "--env LD_LIBRARY_PATH=/softs/openmpi/5.0.5-slurm-CentOS8/lib " - "/n17data/cdaley/containers/containers " - "python /automnt/n17data/cdaley/unions/pure_eb/code/sp_validation/workflow/scripts/run_2pcf_highres.py" - - -rule run_cosmo_val: - """Full CosmoVal diagnostic suite.""" - output: - sentinel=str(COSMO_VAL / "run_cosmo_val.done"), - threads: 24 - resources: - mem_mb=60000, - disk_mb=20000, - runtime=360, - shell: - """ - export PYTHONPATH="/home/cdaley/.local/lib/python3.12/site-packages:${{PYTHONPATH:-}}" - cd /n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_val \ - && python run_cosmo_val.py \ - && touch {output.sentinel} - """ +# PARKED: xi_highres (high-resolution xi for COSEBIS integration). Not runnable +# as written -- the shell invokes run_2pcf_highres.py bare, but the script has +# required --cat-config and --out arguments. Revive it with those supplied. +# +# The MPI reasoning below is hard-won and must survive the revival: +# +# Exception to the profile-driven container model: this is multi-node MPI, one +# `apptainer exec` per rank. Snakemake's own container wrapping puts the +# *whole* shell command -- `mpiexec` included -- inside a single container +# instance, so only rank 0's node would run inside it; the other ranks, +# spawned by SLURM/PMI on their own nodes, would land bare on the host. +# `container: None` plus an explicit `mpiexec -n N apptainer exec ...` +# per-rank is therefore required. +# Snakemake's slurm-jobstep plugin deliberately does NOT prepend `srun` to a +# job carrying an `mpi` resource, which is what lets the rule's own launcher +# run on the host, outside the container. +# Because this rule builds its own apptainer call, reaching the source-cache +# copy of the script relies on our `--bind /home` rather than on Snakemake's +# automatic mount -- and on a concrete image file, since `apptainer exec` +# takes no `docker://` URI. Take that path from `resolve_image()[0]` rather +# than naming a second image path that can drift. +# +# rule xi_highres: +# container: None +# params: +# image=resolve_image()[0], +# input: +# script=workflow.source_path("../scripts/run_2pcf_highres.py"), +# output: +# txt=str(COSMO_VAL / f"{FIDUCIAL['version']}_xi_minsep={FIDUCIAL['min_sep_int']}_maxsep={FIDUCIAL['max_sep_int']}_nbins=10000_npatch=1.txt"), +# xi_plus=str(COSMO_VAL / f"xi_plus_{FIDUCIAL['version']}_minsep={FIDUCIAL['min_sep_int']}_maxsep={FIDUCIAL['max_sep_int']}_nbins=10000_npatch=1.fits"), +# xi_minus=str(COSMO_VAL / f"xi_minus_{FIDUCIAL['version']}_minsep={FIDUCIAL['min_sep_int']}_maxsep={FIDUCIAL['max_sep_int']}_nbins=10000_npatch=1.fits"), +# resources: +# tasks=30, +# cpus_per_task=12, +# nodes=6, +# mem_mb_per_cpu=2000, +# runtime=2880, +# slurm_extra="'--exclude=n17,n09,n36 --partition=pscomp'", +# mpi="/softs/openmpi/5.0.5-slurm-CentOS8/bin/mpiexec", +# shell: +# "{resources.mpi} -n {resources.tasks} " +# "apptainer exec " +# "--bind /home,/n09data,/n17data,/n23data1,/softs " +# "--env LD_LIBRARY_PATH=/softs/openmpi/5.0.5-slurm-CentOS8/lib " +# "{params.image} " +# "python {input.script} --cat-config <...> --out <...>" rule rho_tau_stats: diff --git a/workflow/scripts/analyze_mask_power_spectrum.py b/workflow/scripts/analyze_mask_power_spectrum.py index 2e0e5b28..65c02132 100644 --- a/workflow/scripts/analyze_mask_power_spectrum.py +++ b/workflow/scripts/analyze_mask_power_spectrum.py @@ -72,8 +72,6 @@ def export_power_spectrum( def main(): """Process single mask power spectrum (Snakemake script entry point).""" - from snakemake.script import snakemake - mask_path = snakemake.input.mask output_path = str(snakemake.output.power_spectrum) diff --git a/workflow/scripts/cosmocov_process.py b/workflow/scripts/cosmocov_process.py new file mode 100644 index 00000000..6e6c1723 --- /dev/null +++ b/workflow/scripts/cosmocov_process.py @@ -0,0 +1,73 @@ +"""Assemble a raw CosmoCov block dump into an analysis-ready covariance matrix. + +CosmoCov writes one row per (i, j) element with the Gaussian term in column 8 +and the non-Gaussian term in column 9; this rebuilds the symmetric matrices, +checks positive-definiteness, and plots the correlation matrix. + +Run through Snakemake's ``script:`` directive, which injects ``snakemake`` as a +module global before this file executes. +""" + +import sys + +import matplotlib + +matplotlib.use("Agg") + +import matplotlib.pyplot as plt # noqa: E402 +import numpy as np # noqa: E402 + + +def get_cov(filename): + """Return (gaussian, non-gaussian, ndata) from a CosmoCov element list.""" + + data = np.loadtxt(filename) + ndata = int(np.max(data[:, 0])) + 1 + + cov_g = np.zeros((ndata, ndata)) + cov_ng = np.zeros((ndata, ndata)) + for i in range(data.shape[0]): + row, col = int(data[i, 0]), int(data[i, 1]) + cov_g[row, col] = cov_g[col, row] = data[i, 8] + cov_ng[row, col] = cov_ng[col, row] = data[i, 9] + + return cov_g, cov_ng, ndata + + +def plot_correlation(cov, ndata, plot_path): + """Save the correlation matrix with xi+/xi- block annotations.""" + + diag = np.sqrt(np.diag(cov)) + correlation = cov / np.outer(diag, diag) + + fig, ax = plt.subplots() + extent = (0, ndata, ndata, 0) + image = ax.imshow(correlation, cmap="seismic", vmin=-1, vmax=1, extent=extent) + + ax.axvline(x=ndata // 2, color="black", linewidth=1.0) + ax.axhline(y=ndata // 2, color="black", linewidth=1.0) + + fig.colorbar(image, orientation="vertical") + + ax.text(ndata // 4, ndata + 5, r"$\xi_+^{ij}(\theta)$", fontsize=12) + ax.text(3 * (ndata // 4), ndata + 5, r"$\xi_-^{ij}(\theta)$", fontsize=12) + ax.text(-9, ndata // 4, r"$\xi_+^{ij}(\theta)$", fontsize=12) + ax.text(-9, 3 * (ndata // 4), r"$\xi_-^{ij}(\theta)$", fontsize=12) + + fig.savefig(plot_path, dpi=300) + plt.close(fig) + + +cov_g, cov_ng, ndata = get_cov(snakemake.input[0]) # noqa: F821 +print(f"Dimension of cov: {ndata}x{ndata}") + +cov = cov_g + cov_ng + +eigenvalues = np.linalg.eigvalsh(cov) +print(f"min+max eigenvalues cov: {eigenvalues.min():e}, {eigenvalues.max():e}") +if eigenvalues.min() <= 0.0: + sys.exit("non-positive eigenvalue encountered! Covariance invalid!") + +np.savetxt(snakemake.output.matrix, cov) # noqa: F821 +np.savetxt(snakemake.output.gaussian, cov_g) # noqa: F821 +plot_correlation(cov, ndata, snakemake.output.plot) # noqa: F821 diff --git a/workflow/scripts/cv_additive_bias.py b/workflow/scripts/cv_additive_bias.py index 3fdec487..60df4c3d 100644 --- a/workflow/scripts/cv_additive_bias.py +++ b/workflow/scripts/cv_additive_bias.py @@ -10,7 +10,6 @@ import json from cv_runner import _unbuffer_streams, make_cv, verify_outputs -from snakemake.script import snakemake _unbuffer_streams() cv = make_cv(snakemake) diff --git a/workflow/scripts/cv_cosebis.py b/workflow/scripts/cv_cosebis.py index 182cda99..c1ca8275 100644 --- a/workflow/scripts/cv_cosebis.py +++ b/workflow/scripts/cv_cosebis.py @@ -8,7 +8,6 @@ """ from cv_runner import _unbuffer_streams, make_cv, verify_outputs -from snakemake.script import snakemake _unbuffer_streams() cv = make_cv(snakemake) diff --git a/workflow/scripts/cv_footprints.py b/workflow/scripts/cv_footprints.py index 5ed89f07..e4a1af6a 100644 --- a/workflow/scripts/cv_footprints.py +++ b/workflow/scripts/cv_footprints.py @@ -6,7 +6,6 @@ """ from cv_runner import _unbuffer_streams, make_cv, touch_sentinels -from snakemake.script import snakemake _unbuffer_streams() cv = make_cv(snakemake) diff --git a/workflow/scripts/cv_objectwise_leakage.py b/workflow/scripts/cv_objectwise_leakage.py index 8b6d5694..ae0f012a 100644 --- a/workflow/scripts/cv_objectwise_leakage.py +++ b/workflow/scripts/cv_objectwise_leakage.py @@ -8,7 +8,6 @@ """ from cv_runner import _unbuffer_streams, make_cv, touch_sentinels -from snakemake.script import snakemake _unbuffer_streams() cv = make_cv(snakemake) diff --git a/workflow/scripts/cv_plot_2pcf.py b/workflow/scripts/cv_plot_2pcf.py index 9d5c8900..5251e7a5 100644 --- a/workflow/scripts/cv_plot_2pcf.py +++ b/workflow/scripts/cv_plot_2pcf.py @@ -7,7 +7,6 @@ """ from cv_runner import _unbuffer_streams, make_cv, touch_sentinels -from snakemake.script import snakemake _unbuffer_streams() cv = make_cv(snakemake) diff --git a/workflow/scripts/cv_plot_rho_stats.py b/workflow/scripts/cv_plot_rho_stats.py index 95a37b09..c99bd646 100644 --- a/workflow/scripts/cv_plot_rho_stats.py +++ b/workflow/scripts/cv_plot_rho_stats.py @@ -6,7 +6,6 @@ """ from cv_runner import _unbuffer_streams, make_cv, touch_sentinels -from snakemake.script import snakemake _unbuffer_streams() cv = make_cv(snakemake) diff --git a/workflow/scripts/cv_plot_tau_stats.py b/workflow/scripts/cv_plot_tau_stats.py index ed90e334..2a00b3a5 100644 --- a/workflow/scripts/cv_plot_tau_stats.py +++ b/workflow/scripts/cv_plot_tau_stats.py @@ -5,7 +5,6 @@ """ from cv_runner import _unbuffer_streams, make_cv, touch_sentinels -from snakemake.script import snakemake _unbuffer_streams() cv = make_cv(snakemake) diff --git a/workflow/scripts/cv_pseudo_cl.py b/workflow/scripts/cv_pseudo_cl.py index cf04e8e8..7dfebc54 100644 --- a/workflow/scripts/cv_pseudo_cl.py +++ b/workflow/scripts/cv_pseudo_cl.py @@ -6,7 +6,6 @@ """ from cv_runner import _unbuffer_streams, make_cv, verify_outputs -from snakemake.script import snakemake _unbuffer_streams() cv = make_cv(snakemake) diff --git a/workflow/scripts/cv_pure_eb.py b/workflow/scripts/cv_pure_eb.py index d15a763f..bcc46d84 100644 --- a/workflow/scripts/cv_pure_eb.py +++ b/workflow/scripts/cv_pure_eb.py @@ -9,7 +9,6 @@ """ from cv_runner import _unbuffer_streams, make_cv, verify_outputs -from snakemake.script import snakemake _unbuffer_streams() cv = make_cv(snakemake) diff --git a/workflow/scripts/cv_ratio_xi_sys_xi.py b/workflow/scripts/cv_ratio_xi_sys_xi.py index ba737d7e..79afd89a 100644 --- a/workflow/scripts/cv_ratio_xi_sys_xi.py +++ b/workflow/scripts/cv_ratio_xi_sys_xi.py @@ -8,7 +8,6 @@ """ from cv_runner import _unbuffer_streams, make_cv, verify_outputs -from snakemake.script import snakemake _unbuffer_streams() cv = make_cv(snakemake) diff --git a/workflow/scripts/cv_rho_tau_fits.py b/workflow/scripts/cv_rho_tau_fits.py index 39d2e371..64b24b67 100644 --- a/workflow/scripts/cv_rho_tau_fits.py +++ b/workflow/scripts/cv_rho_tau_fits.py @@ -9,7 +9,6 @@ """ from cv_runner import _unbuffer_streams, make_cv, touch_sentinels -from snakemake.script import snakemake _unbuffer_streams() cv = make_cv(snakemake) diff --git a/workflow/scripts/cv_summarize_bmodes.py b/workflow/scripts/cv_summarize_bmodes.py index 90df5999..85f0fa1d 100644 --- a/workflow/scripts/cv_summarize_bmodes.py +++ b/workflow/scripts/cv_summarize_bmodes.py @@ -17,7 +17,6 @@ import json from cv_runner import _unbuffer_streams, make_cv, verify_outputs -from snakemake.script import snakemake _unbuffer_streams() cv = make_cv(snakemake) diff --git a/workflow/scripts/cv_weights.py b/workflow/scripts/cv_weights.py index 3cb3f3ca..0316fe1e 100644 --- a/workflow/scripts/cv_weights.py +++ b/workflow/scripts/cv_weights.py @@ -5,7 +5,6 @@ """ from cv_runner import _unbuffer_streams, make_cv, verify_outputs -from snakemake.script import snakemake _unbuffer_streams() cv = make_cv(snakemake) diff --git a/workflow/scripts/generate_glass_mock_rhotau_samples.py b/workflow/scripts/generate_glass_mock_rhotau_samples.py index 63d99095..87b7d06c 100644 --- a/workflow/scripts/generate_glass_mock_rhotau_samples.py +++ b/workflow/scripts/generate_glass_mock_rhotau_samples.py @@ -116,6 +116,36 @@ def generate_samples_for_mock(mock_id, cov_tau, theta, ref_tau_header, output_di return f"Generated tau samples for mock {mock_id:05d}" +def run(cov_tau_path, ref_tau_path, output_dir, mock_ids): + """Generate sampled tau statistics for every mock in ``mock_ids``.""" + print("Loading tau covariance...") + cov_tau = np.load(cov_tau_path) + print(f" cov_tau shape: {cov_tau.shape}") + + print("Loading reference FITS...") + ref_tau_data, ref_tau_header = load_reference_fits(ref_tau_path) + theta = ref_tau_data["theta"] + print( + f" theta range: {theta.min():.3f} - {theta.max():.3f} arcmin, nbins: {len(theta)}" + ) + + print(f"Generating tau samples for {len(mock_ids)} mocks...") + for mock_id in mock_ids: + msg = generate_samples_for_mock( + mock_id, cov_tau, theta, ref_tau_header, output_dir + ) + print(msg) + print("Done!") + + +def parse_mock_ids(spec): + """Parse a mock-ID spec, either a single ID or an inclusive ``lo-hi`` range.""" + if "-" in spec: + lo, hi = spec.split("-") + return list(range(int(lo), int(hi) + 1)) + return [int(spec)] + + def main(): parser = argparse.ArgumentParser( description="Generate zero-mean tau samples for GLASS mocks" @@ -146,37 +176,22 @@ def main(): ) args = parser.parse_args() - # Load tau covariance and reference - print("Loading tau covariance...") - cov_tau = np.load(args.cov_tau) - print(f" cov_tau shape: {cov_tau.shape}") - - print("Loading reference FITS...") - ref_tau_data, ref_tau_header = load_reference_fits(args.ref_tau) - theta = ref_tau_data["theta"] - print( - f" theta range: {theta.min():.3f} - {theta.max():.3f} arcmin, nbins: {len(theta)}" - ) + run(args.cov_tau, args.ref_tau, args.output_dir, parse_mock_ids(args.mock_ids)) - # Parse mock ID range - mock_ids = ( - list( - range( - int(args.mock_ids.split("-")[0]), int(args.mock_ids.split("-")[1]) + 1 - ) - ) - if "-" in args.mock_ids - else [int(args.mock_ids)] - ) - print(f"Generating tau samples for {len(mock_ids)} mocks...") - for mock_id in mock_ids: - msg = generate_samples_for_mock( - mock_id, cov_tau, theta, ref_tau_header, args.output_dir - ) - print(msg) - print("Done!") +# ── Entry point dispatch ───────────────────────────────────────────────────── +try: + snakemake # injected by snakemake's script: directive +except NameError: + snakemake = None -if __name__ == "__main__": +if snakemake is not None: + run( + snakemake.input.cov_tau, + snakemake.input.ref_tau, + snakemake.params.output_dir, + parse_mock_ids(snakemake.params.mock_id), + ) +elif __name__ == "__main__": main() diff --git a/workflow/scripts/im_mbias_config.py b/workflow/scripts/im_mbias_config.py new file mode 100644 index 00000000..39073ca1 --- /dev/null +++ b/workflow/scripts/im_mbias_config.py @@ -0,0 +1,107 @@ +"""Assemble ``m_bias_config.yaml`` for the image-sims m-bias estimator. + +Combines the manifest's injected-shear facts, this run's science knobs and +git/container provenance into the single config ``compute_m_bias_image_sims.py`` +reads. ``provenance`` rides along as a top-level block: the estimator copies it +verbatim into its results yaml, so a result file records which manifest, repo +commits and container produced the number. + +`snakemake` is injected as a module global by Snakemake's `script:` preamble +before this file runs (`from snakemake.script import snakemake` is +IDE-hint-only and raises ImportError if actually executed). +""" + +import hashlib +import os +import re +import subprocess + +import yaml + +params = snakemake.params # noqa: F821 +manifest_path = snakemake.input["manifest"] # noqa: F821 + + +def _git(repo, *args): + """Read a git fact from ``repo``; ``None`` if it is not a checkout.""" + try: + return subprocess.run( + ["git", "-C", repo, *args], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + except (subprocess.CalledProcessError, FileNotFoundError): + return None + + +def _sif_revision(): + """``org.opencontainers.image.revision`` from the running image's OCI labels. + + Read the image actually mounted, which Apptainer names in + ``APPTAINER_CONTAINER``, rather than the configured ``sif`` -- a run may + override it, and a registry tag says nothing about which build was pulled. + Chunked plain-text scan: no exec, no container start, no 1.5 GB in memory. + """ + sif_path = os.environ.get("APPTAINER_CONTAINER") + if not sif_path: + return None + pattern = re.compile( + rb'org\.opencontainers\.image\.revision"?[:=]"?([0-9a-f]{7,40})' + ) + tail = b"" + try: + with open(sif_path, "rb") as fh: + while chunk := fh.read(8 << 20): + m = pattern.search(tail + chunk) + if m: + return m.group(1).decode() + tail = chunk[-128:] + except OSError: + return None + return None + + +with open(manifest_path) as fh: + manifest = yaml.safe_load(fh) +with open(manifest_path, "rb") as fh: + manifest_sha256 = hashlib.sha256(fh.read()).hexdigest() + +mbias_cfg = { + "grids_dir": params.grids_base, + "num": params.num, + "catalog_name": params.cat_name, + # Injected shear: from the manifest, the single source of truth. + "shear_amplitude": manifest["shear_amplitude"], + "branches": list(manifest["branches"]), + "pairs": manifest["pairs"], + "match_radius_deg": params.match_radius_deg, + "w_cols": list(params.w_cols), + "pair_match": params.pair_match, + "n_bootstrap": params.n_bootstrap, + "bootstrap_seed": params.bootstrap_seed, + "results_dir": params.results_dir, + "output_path": params.results, + "provenance": { + "manifest_sha256": manifest_sha256, + "sp_validation": { + "branch": _git( + params.sp_validation_repo, "rev-parse", "--abbrev-ref", "HEAD" + ), + "commit": _git(params.sp_validation_repo, "rev-parse", "HEAD"), + }, + "shapepipe": { + "branch": _git( + params.shapepipe_repo, "rev-parse", "--abbrev-ref", "HEAD" + ), + "commit": _git(params.shapepipe_repo, "rev-parse", "HEAD"), + }, + "container": { + "sif": params.sif, + "ghcr_revision": _sif_revision(), + }, + }, +} + +with open(snakemake.output["cfg"], "w") as fh: # noqa: F821 + yaml.safe_dump(mbias_cfg, fh) diff --git a/workflow/scripts/process_mask.py b/workflow/scripts/process_mask.py index c0368147..c9590368 100644 --- a/workflow/scripts/process_mask.py +++ b/workflow/scripts/process_mask.py @@ -138,8 +138,6 @@ def save_area_summary( def main(): """Main processing function.""" - # Snakemake script execution only (no interactive mode) - from snakemake.script import snakemake # Get parameters from Snakemake source_mask_path = snakemake.input.mask diff --git a/workflow/scripts/run_rho_tau.py b/workflow/scripts/run_rho_tau.py index ea2f35bc..fbf62a3c 100644 --- a/workflow/scripts/run_rho_tau.py +++ b/workflow/scripts/run_rho_tau.py @@ -28,8 +28,6 @@ "/n17data/cdaley/unions/pure_eb/code/sp_validation/cosmo_val/output/rho_tau_stats/rho_stats_SP_v1.4.5.fits", "/home/cdaley/n17data/unions/pure_eb", ) -else: - from snakemake.script import snakemake params = snakemake.params # type: ignore