From f92cb44b6747a010c4cf369d7e515e9278824244 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 15:46:49 -0400 Subject: [PATCH 01/41] feat(orchestration): Snakemake workflow for the real-data pipeline (#848) Implements the round-2 design in #848: static two-phase execution driven by a parse-time SQLite run index (never a rule input, so appending tiles never invalidates completed work); rules wrap the existing per-unit shapepipe_run config chains at tile/exposure granularity with deterministic RUN_DATETIME=False directory() outputs; in-job completeness count-floor (the ported bash complete_check table, workflow/scripts/completeness.py); one-allocation execution profile for nibi with apptainer software-deployment (the workflow never calls apptainer itself); out-of-DAG run_report verb + onsuccess/onerror hooks; workflow/bin/sp as the committed launcher. First pass touches no module code. Validated by dry-run against a 4-tile P3 subset (219-job compute DAG); prepare phase runs green on nibi. Co-Authored-By: Claude Fable 5 --- .gitignore | 1 + profiles/nibi/config.yaml | 101 +++++++ workflow/README.md | 105 ++++++++ workflow/Snakefile | 124 +++++++++ workflow/bin/sp | 63 +++++ workflow/config.yaml | 35 +++ workflow/rules/exposure.smk | 70 +++++ workflow/rules/prepare.smk | 67 +++++ workflow/rules/tile.smk | 147 ++++++++++ workflow/scripts/build_forest.py | 42 +++ workflow/scripts/build_index.py | 139 ++++++++++ workflow/scripts/clean_exposures.py | 64 +++++ workflow/scripts/completeness.py | 86 ++++++ workflow/scripts/run_report.py | 107 ++++++++ workflow/scripts/sp_rule.py | 405 ++++++++++++++++++++++++++++ 15 files changed, 1556 insertions(+) create mode 100644 profiles/nibi/config.yaml create mode 100644 workflow/README.md create mode 100644 workflow/Snakefile create mode 100755 workflow/bin/sp create mode 100644 workflow/config.yaml create mode 100644 workflow/rules/exposure.smk create mode 100644 workflow/rules/prepare.smk create mode 100644 workflow/rules/tile.smk create mode 100644 workflow/scripts/build_forest.py create mode 100644 workflow/scripts/build_index.py create mode 100644 workflow/scripts/clean_exposures.py create mode 100644 workflow/scripts/completeness.py create mode 100644 workflow/scripts/run_report.py create mode 100644 workflow/scripts/sp_rule.py diff --git a/.gitignore b/.gitignore index db20f3086..4c4b6d0b0 100644 --- a/.gitignore +++ b/.gitignore @@ -139,3 +139,4 @@ code # .felt here is a machine-local symlink into it. Never track it in this repo. /.felt/ /.felt +.snakemake/ diff --git a/profiles/nibi/config.yaml b/profiles/nibi/config.yaml new file mode 100644 index 000000000..e977803c1 --- /dev/null +++ b/profiles/nibi/config.yaml @@ -0,0 +1,101 @@ +# Snakemake profile for the Nibi cluster (Digital Research Alliance). +# +# ONE-ALLOCATION mode: sbatch one node up front, then run snakemake with the +# LOCAL scheduler inside it (no per-job sbatch; the local scheduler bin-packs the +# DAG into the allocation against `cores` + `resources: mem_mb`). This is the +# proven nibi path (job_p3_batch1) and the single honest first-pass mode; +# multi-node scaling = multiple tile-disjoint invocations, deferred. +# +# Launch via `workflow/bin/sp` (loads apptainer/1.4.5, uses the /project venv). +# software-deployment-method wraps every job's shell in `apptainer exec` — the +# user never types apptainer; the container is set in the Snakefile (local .sif). + +# Local scheduler (no `executor:` -> built-in local). Size to the allocation. +cores: 192 # node cores available to the DAG +local-cores: 4 # cap localrule/aggregation concurrency +resources: + mem_mb: 720000 # node RAM cap the local scheduler bin-packs under + +software-deployment-method: [apptainer] +# Explicit container environment (finding 2/3): the apptainer SDM otherwise drops +# the proven-recipe env and hardcodes --home , hiding ~/.ssl/cadcproxy.pem. +# --cleanenv strict host-env isolation (APPTAINERENV_*/SINGULARITYENV_* survive) +# OMP_NUM_THREADS=1 caps OpenBLAS fork-explosion (verified: pool 32->1) +# MALLOC_ARENA_MAX=2 bounded allocator (im_sims/nibi lesson) +# --home /home/cdaley wins over the SDM's --home ; restores cadcproxy.pem for vos/vcp +# PYTHONPATH SETTLED CALL 3 — P0 pins the prod worktree src so science +# content matches the bash validation runs (which use this +# override). Production later rebuilds the sif at the exact +# validation commit and DROPS this --env line. +apptainer-args: "--cleanenv --env OMP_NUM_THREADS=1 --env MALLOC_ARENA_MAX=2 --env PYTHONPATH=/project/def-mjhudson/cdaley/shapepipe-prod/src --home /home/cdaley --bind /project --bind /scratch" + +latency-wait: 60 # NFS: wait for outputs to appear after a job +keep-going: true # a failed job poisons only its cone; siblings run on +rerun-incomplete: true # re-do jobs left incomplete by an unclean death +show-failed-logs: true +printshellcmds: true + +# rerun-triggers left at the full Snakemake v9 default +# {mtime,params,input,code,software-env}: params/code fixes SHOULD propagate. +# No override — the wrapper writes outputs only on change so mtimes move only +# when reality moves; protected() guards finished science products. + +default-resources: + mem_mb: 4000 + runtime: 120 # minutes + cpus_per_task: 1 + +# Per-stage resources — measured footprints from the nibi P3 runs (#808). +# threads == cpus_per_task == ShapePipe SMP fork width (sp_rule sets +# SMP_BATCH_SIZE={threads}); mem_mb == threads x measured per-worker footprint. +# +# set-threads is REQUIRED for {threads} to resolve above 1: the local scheduler +# derives cpus_per_task from threads, NOT the reverse (verified snakemake 9.23 — +# set-resources cpus_per_task alone leaves {threads}==1, i.e. SMP_BATCH_SIZE=1, +# serial). Keep the two blocks in lockstep. +set-threads: + exp_split: 4 + exp_mask: 4 + exp_psf: 8 + tile_mask: 16 + tile_detect: 16 + tile_merge_headers: 8 + tile_vignets: 8 + tile_ngmix: 8 + tile_merge_cats: 4 + tile_make_cat: 4 + +set-resources: + exp_split: + cpus_per_task: 4 + mem_mb: 8000 + exp_mask: + cpus_per_task: 4 + mem_mb: 8000 + exp_psf: + cpus_per_task: 8 + mem_mb: 16000 + runtime: 240 + tile_mask: + cpus_per_task: 16 + mem_mb: 16000 + tile_detect: + cpus_per_task: 16 + mem_mb: 16000 + tile_merge_headers: + cpus_per_task: 8 + mem_mb: 8000 + tile_vignets: + cpus_per_task: 8 + mem_mb: 24000 + runtime: 240 + tile_ngmix: + cpus_per_task: 8 + mem_mb: 24000 + runtime: 360 + tile_merge_cats: + cpus_per_task: 4 + mem_mb: 8000 + tile_make_cat: + cpus_per_task: 4 + mem_mb: 8000 diff --git a/workflow/README.md b/workflow/README.md new file mode 100644 index 000000000..4d8abd1f1 --- /dev/null +++ b/workflow/README.md @@ -0,0 +1,105 @@ +# ShapePipe Snakemake orchestration + +Snakemake workflow that orchestrates real-data ShapePipe runs, replacing the +`curl_canfar_local.sh → run_job_sp_canfar_v2.0.bash → job_sp_canfar_v2.0.bash` +bash layers (and per-site sbatch reimplementations). **Module code is untouched**: +rules call `shapepipe_run -c ` on the existing config chains. Design and +rationale: [CosmoStat/shapepipe#848](https://github.com/CosmoStat/shapepipe/issues/848) +(the living PRD). + +## Quick start (nibi) + +**One-allocation mode.** `sbatch` one node, then run Snakemake with the *local* +scheduler inside it: it bin-packs the DAG into the allocation (no per-job +sbatch). Each job's shell runs `shapepipe_run` **inside the container** (the +profile's apptainer software-deployment — you never type `apptainer`). + +```bash +# One-time: a snakemake env on a SHARED filesystem (/project — NOT /tmp, which is +# node-local; the executor re-invokes this python inside jobs). +uv venv /project/def-mjhudson/cdaley/snakemake-env --python 3.12 +source /project/def-mjhudson/cdaley/snakemake-env/bin/activate +uv pip install 'snakemake>=9,<10' + +# Edit workflow/config.yaml: tile_list, run_dir, config_src, container, star_cats. + +# The committed launcher loads apptainer/1.4.5 + the /project venv, so a fresh +# shell always has the right state. Inside your allocation: +workflow/bin/sp prepare # prepare_tiles -> build_index.py -> prepare_exposures +workflow/bin/sp run # the main compute DAG +``` + +`sp` subcommands: `prepare` (chains all three prepare steps), `prepare_tiles`, +`index`, `prepare_exposures`, `run`, `rerun ` (forced recompute of +protected products), `cancel ` (scancel sweep before `--unlock`), +`clean-exposures [--apply]`, `report`. + +## Execution is three static invocations + +The exposure job set is data-derived from the tiles' `find_exposures` output, so +it cannot be scheduled in the same static DAG that produces it. Hence: + +1. `snakemake prepare_tiles` — per-tile static DAG (Git_vos → Uz → Fe), + `keep-going` so tile failures are independent. +2. `build_index.py` — a **plain script**, not a DAG node. It iterates the + declared tile list, checks each tile's Fe output at its deterministic path (no + glob), writes `run_index.sqlite` + `missing.json`, and exits nonzero only if + the missing *fraction* exceeds a threshold. (There is no `--allow-missing` + Snakemake flag; that mechanism does not exist.) +3. `snakemake prepare_exposures` — per-exposure static DAG (Gie_vos + per-unit + star cats), parseable now the index exists. +4. `snakemake` (main DAG) — the tile/exposure compute chain. + +`sp prepare` chains 1–3 so the UX stays one command. + +## Layout + +``` +workflow/ + Snakefile parse-time index load; global container:; onsuccess/onerror report hooks + config.yaml the run: tile list, paths, container, chunk count + bin/sp committed launcher (module load + /project venv + subcommands) + rules/ + prepare.smk tile get_images/uncompress/find_exposures + per-unit star cats + exposure.smk per-exposure: get_images, split, mask, psf (no temp()) + tile.smk per-tile: exp forest, merge_headers, mask, detect, vignets, ngmix, merge, make_cat + scripts/ + sp_rule.py the thin per-unit wrapper (isolation furniture, config copy, log-sync, count floor) + build_index.py prepare-phase run_index.sqlite builder (plain script) + build_forest.py per-tile exposure symlink forest (group-compatible shell) + completeness.py the ported count-floor table (shared by sp_rule + run_report) + run_report.py standalone report (NOT a DAG node; run_report hooks call it) + clean_exposures.py index-driven exposure-store reclaim (replaces temp()) +profiles/nibi/config.yaml local scheduler; apptainer SDM; resources; keep-going +``` + +## How it works + +- **Completeness = the count floor.** Each stage declares its deterministic + `output/run_sp_/` directory as `directory()` output (the per-unit + config copy forces `RUN_DATETIME = False`, so the path is known at DAG time). + After the run, `sp_rule.py` counts products per mandatory runner against + `completeness.py`'s floor and exits nonzero below it; per-CCD attrition between + floor and `expect` is tolerated (the directory simply lacks that product). + This *is* the failure policy — no 3-class taxonomy, no error-signature + whitelist. `--keep-going` isolates a failure to its own cone. +- **Per-unit isolation is by work-dir content, not `-e/--exclusive`.** Every + tile/exposure runs its own `shapepipe_run` in `tiles//` or `exp//`, + with a `cfis` config symlink, `star_cat_{exp,tiles}` symlinks, a + `tile_numbers.txt` (tiles) or fabricated `exp_numbers-000-000.txt` pseudo-Fe + (exposures), and a config copy setting `RUN_DATETIME=False` and + `SMP_BATCH_SIZE={threads}`. `NUMBER_LIST=-` is injected **only** for + stages whose numbering scheme is the unit ID (tile-scheme stages + exp split), + never for get_images / exp_mask / exp_psf. +- **No `temp()` on shared exposure dirs.** Exposures overlap tiles, so temp() + would cascade destructive reruns when a tile is appended. The store is + persistent; `sp clean-exposures` reclaims space only for exposures all of + whose consuming tiles have a final_cat. +- **The index is parse-time data, never a rule input.** Appending tiles changes + which jobs exist without invalidating completed work. Star cats are re-keyed + per unit for the same reason. `final_cat` is `protected()`. +- **Failure is a report, not a gate.** `run_report.py` disk-scans the trees + against the count table and enumerates shortfalls (whole-unit absence vs + per-CCD attrition). It is a standalone script — a DAG report node would be + poisoned by the very failures it must enumerate — emitted automatically by the + `onsuccess`/`onerror` hooks and runnable any time via `sp report`. diff --git a/workflow/Snakefile b/workflow/Snakefile new file mode 100644 index 000000000..04722a73c --- /dev/null +++ b/workflow/Snakefile @@ -0,0 +1,124 @@ +"""ShapePipe real-data orchestration — Snakemake workflow. + +Design / rationale: CosmoStat/shapepipe#848 (the living PRD). + +Execution is honestly THREE static invocations, chained by ``workflow/bin/sp`` +so the UX stays one command (``sp prepare`` / ``sp run``): + + snakemake --profile profiles/nibi prepare_tiles # Git_vos + Uz + Fe (keep-going) + python workflow/scripts/build_index.py ... # PLAIN SCRIPT, not a DAG node + snakemake --profile profiles/nibi prepare_exposures # Gie_vos + star cats + snakemake --profile profiles/nibi # the main compute DAG + +The run index is loaded ONCE here at parse time into plain dicts; it is never a +rule input, so appending tiles and rebuilding it changes which jobs exist without +invalidating completed work. build_index.py is a plain script *between* +prepare_tiles and prepare_exposures (the exposure job set is data-derived from +the tiles' find_exposures output, so it cannot be scheduled in the same static +DAG that produces it). +""" + +import sqlite3 +from pathlib import Path + +configfile: "workflow/config.yaml" + +# Every job's shell runs inside this container (apptainer software-deployment in +# the profile); the user never types apptainer. +container: config["container"] + +# --- paths ----------------------------------------------------------------- +RUN_DIR = Path(config["run_dir"]) +CONFIG_SRC = Path(config["config_src"]) +STAR_CATS = Path(config["star_cats"]) +INDEX_DB = Path(config["index_db"]) +SCRIPTS = Path(workflow.basedir) / "scripts" +NGMIX_CHUNKS = config.get("ngmix_chunks", 8) + +with open(config["tile_list"]) as f: + TILES = [ln.strip() for ln in f if ln.strip()] + +# --- parse-time index load (compute phase only) ---------------------------- +# EXP: global set of exposure base-ids. TILE_EXP: tile -> [exp_ids]. +EXP, TILE_EXP = set(), {} +if INDEX_DB.exists(): + _con = sqlite3.connect(INDEX_DB) + EXP = {r[0] for r in _con.execute("SELECT exp_id FROM exposures")} + for tile, exp in _con.execute("SELECT tile_id, exp_id FROM tile_exposures"): + TILE_EXP.setdefault(tile, []).append(exp) + _con.close() + +wildcard_constraints: + tile = r"\d{3}\.\d{3}", + exp = r"\d{6,7}", + chunk = r"\d+", + +# --- shared wrapper invocation --------------------------------------------- +def sp_rule(stage, config_name, level, unit, *, isolate=True, exp_forest=None, + extra=""): + """Build the shell command that runs one unit through sp_rule.py. + + NUMBER_LIST is injected by default (tile-scheme stages + exp split); pass + ``isolate=False`` for get_images / exp_mask / exp_psf, whose per-CCD/download + numbering must not be constrained. ``{threads}`` is filled by Snakemake and + becomes SMP_BATCH_SIZE (fork width == cpus_per_task). + """ + cmd = ( + f"python {SCRIPTS}/sp_rule.py --stage {stage} --config {config_name} " + f"--unit {unit} --level {level} --run-dir {RUN_DIR} " + f"--config-src {CONFIG_SRC} --star-cats {STAR_CATS} --threads {{threads}}" + ) + if not isolate: + cmd += " --no-isolate" + if exp_forest: + cmd += f" --exp-forest {exp_forest}" + if extra: + cmd += " " + extra + return cmd + +def tile_dir(tile): + return RUN_DIR / "tiles" / tile + +def exp_dir(exp): + return RUN_DIR / "exp" / exp + +include: "rules/prepare.smk" +include: "rules/exposure.smk" +include: "rules/tile.smk" + +# --- top-level targets ------------------------------------------------------ +# Only aggregation targets are localrules (no-ops under the local executor, but +# they keep the compute chain group-compatible: no mid-chain localrules). +localrules: all, prepare_tiles, prepare_exposures + +rule all: + input: + expand(str(RUN_DIR / "tiles/{tile}/final_cat-{tile}.fits"), tile=TILES) + +# Phase A.1 — per-tile static DAG (download + find_exposures). keep-going so +# tile failures are independent; build_index.py runs next as a plain script. +rule prepare_tiles: + input: + expand(str(RUN_DIR / "tiles/{tile}/output/run_sp_tile_Fe"), tile=TILES) + +# Phase A.2 — per-exposure static DAG (download + star cats), parseable now the +# index exists. Star cats are re-keyed per unit (see prepare.smk). +rule prepare_exposures: + input: + expand(str(RUN_DIR / "exp/{exp}/output/run_sp_exp_Gie"), exp=sorted(EXP)), + expand(str(RUN_DIR / "exp/{exp}/star_cat-{exp}.fits"), exp=sorted(EXP)), + expand(str(RUN_DIR / "tiles/{tile}/star_cat-{tile}.fits"), tile=TILES) + +# --- report hooks ----------------------------------------------------------- +# run_report is NOT a DAG node (a descendant of every job would be poisoned by +# any hard failure — the exact case it exists for). It is a standalone script, +# emitted automatically at the end of every invocation, runnable any time. +def _report(status): + shell(f"python {SCRIPTS}/run_report.py --run-dir {RUN_DIR} " + f"--index {INDEX_DB} --status {status} || true") + +onsuccess: + _report("success") + +onerror: + _report("error") diff --git a/workflow/bin/sp b/workflow/bin/sp new file mode 100755 index 000000000..f226aa441 --- /dev/null +++ b/workflow/bin/sp @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# sp — the committed launcher for the ShapePipe Snakemake workflow. +# +# Removes the shell-state dependencies the review flagged (finding 20): it loads +# the apptainer module (Snakemake resolves `apptainer` via PATH at job runtime) +# and runs the Snakemake that lives on the shared /project venv (must be on a +# shared FS — the executor re-invokes it inside jobs). One entry point, so a +# fresh tmux / a restart after a crash always launches with the right state. +# +# Usage: +# sp prepare # prepare_tiles -> build_index.py -> prepare_exposures +# sp prepare_tiles # phase A.1 only +# sp index # build_index.py only (plain script, threshold-gated) +# sp prepare_exposures # phase A.2 only +# sp run [ARGS...] # the main compute DAG +# sp rerun # forced recompute of protected/finished products +# sp cancel # scancel this workflow's jobs (name sweep) before --unlock +# sp clean-exposures [--apply] # index-driven exposure-store reclaim +# sp report # emit run_report.json now (mid-run ok) +# Extra args after the subcommand pass through to snakemake. +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # workflow/ +REPO="$(dirname "$HERE")" +VENV="${SP_SNAKEMAKE_ENV:-/project/def-mjhudson/cdaley/snakemake-env}" +PROFILE="$REPO/profiles/nibi" +SCRIPTS="$HERE/scripts" +CONFIG="$HERE/config.yaml" + +module load apptainer/1.4.5 2>/dev/null || true +# shellcheck disable=SC1091 +source "$VENV/bin/activate" + +# Minimal scalar reader for workflow/config.yaml (key: value, no nesting). +cfg() { sed -n "s/^$1:[[:space:]]*//p" "$CONFIG" | head -1; } +RUN_DIR="$(cfg run_dir)"; INDEX_DB="$(cfg index_db)" +TILE_LIST="$(cfg tile_list)"; MISSING_THRESH="${SP_MISSING_THRESHOLD:-0.0}" + +sm() { snakemake --profile "$PROFILE" "$@"; } +build_index() { + python "$SCRIPTS/build_index.py" --tile-list "$TILE_LIST" \ + --run-dir "$RUN_DIR" --db "$INDEX_DB" --missing-threshold "$MISSING_THRESH" +} + +cmd="${1:-}"; shift || true +case "$cmd" in + prepare) sm prepare_tiles "$@"; build_index; sm prepare_exposures "$@" ;; + prepare_tiles) sm prepare_tiles "$@" ;; + index) build_index ;; + prepare_exposures) sm prepare_exposures "$@" ;; + run) sm "$@" ;; + rerun) sm --forcerun "$@" ;; + cancel) + run="${1:?usage: sp cancel }" + squeue --me --noheader --format='%i %j' \ + | awk -v r="$run" '$2 ~ r {print $1}' | xargs -r scancel + echo "cancelled jobs matching '$run'; safe to --unlock / rerun now" ;; + clean-exposures) python "$SCRIPTS/clean_exposures.py" \ + --run-dir "$RUN_DIR" --index "$INDEX_DB" "$@" ;; + report) python "$SCRIPTS/run_report.py" \ + --run-dir "$RUN_DIR" --index "$INDEX_DB" --status manual "$@" ;; + *) echo "sp: unknown subcommand '${cmd:-}' (see the header of $0)"; exit 2 ;; +esac diff --git a/workflow/config.yaml b/workflow/config.yaml new file mode 100644 index 000000000..53542d1c2 --- /dev/null +++ b/workflow/config.yaml @@ -0,0 +1,35 @@ +# Run configuration for the ShapePipe Snakemake workflow. +# +# A "run" is declared by a tile list plus the paths below. Everything here is +# read at parse time; none of it is a rule input, so editing it (e.g. appending +# tiles) never invalidates completed work — it only changes which jobs exist. + +# The tile list that scopes this run (one "IDra.IDdec" per line). +tile_list: /scratch/cdaley/shapepipe-output/snakemake-p0/tiles4.txt + +# The container every job runs inside (apptainer software-deployment in the profile). +container: /project/def-mjhudson/cdaley/containers/shapepipe-develop-runtime.sif + +# Where products land ($SP_RUN root). Per-unit work-dir forest is built under here: +# /tiles/// and /exp/// +run_dir: /scratch/cdaley/shapepipe-output/snakemake-p0 + +# The canonical config chain (example/cfis in the repo, or a run-specific copy). +config_src: /home/cdaley/projects/def-mjhudson/cdaley/runs/p3-batch1/cfis + +# Pre-staged inputs (P3 data already on /project; get_images RETRIEVE=symlink). +# Star catalogues for masking, pre-generated (network step done in prepare): +star_cats: /home/cdaley/projects/def-mjhudson/cdaley/runs/p3-batch1/star_cats + +# The run index. Durable, low-volume products (index, missing.json, run_report, +# final_cat) should land on /project, not /scratch (60-day purge, finding 15). +index_db: /project/def-mjhudson/cdaley/shapepipe-output/snakemake-p0/index/run_index.sqlite + +# ngmix within-tile chunking: static N chunks (closed ID ranges computed +# per-tile, in-job, from the tile's own sexcat). +ngmix_chunks: 8 + +# The container's installed shapepipe is overridden by the prod worktree via +# --env PYTHONPATH in profiles/nibi (settled call 3); no config knob here. +# build_index.py's missing-tile fraction floor is passed by workflow/bin/sp +# (SP_MISSING_THRESHOLD, default 0.0 = any missing tile is fatal). diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk new file mode 100644 index 000000000..50403d386 --- /dev/null +++ b/workflow/rules/exposure.smk @@ -0,0 +1,70 @@ +"""Exposure chain — per exposure, keyed by exp_id (dedup is structural). + +Gie -> Sp -> Ma -> psf, each in the exposure's own work dir, chained by +ShapePipe's last:/run_sp_exp_* INPUT_DIR resolver. Completeness is the count +floor (completeness.py). + +NO temp() on these shared exposure directories: exposures overlap tiles by +construction (~7-10 tiles per exposure), so temp() would delete an exposure the +moment its forests are built and cascade destructive reruns across spatial +neighbours whenever a tile is appended (findings 4/5/6/19). The store is +persistent; space is reclaimed by the explicit index-driven `sp clean-exposures` +verb (scripts/clean_exposures.py), which deletes an exposure's intermediates +only once every consuming tile in the index has its final_cat present. + +NUMBER_LIST is injected only for exp_split (numbering scheme == exposure id); +NEVER for get_images (download) / exp_mask / exp_psf (per-CCD schemes where the +#746 startup validation would turn tolerated per-CCD attrition into a +whole-exposure hard failure). +""" + +EXP_OUT = str(RUN_DIR / "exp/{exp}/output") + +# Symlink/download exposure image/weight/flag; reads the fabricated pseudo-Fe +# exp_numbers-000-000.txt the wrapper drops (via last:find_exposures_runner). +rule exp_get_images: + output: + directory(f"{EXP_OUT}/run_sp_exp_Gie") + params: + cmd=lambda wc: sp_rule("exp_get_images", "config_exp_Gie_vos.ini", + "exp", wc.exp, isolate=False) + shell: + "{params.cmd}" + +# Split multi-HDU exposure into single-CCD files (+ headers-*.npy). Bulk output; +# persistent (see module docstring on temp()). +rule exp_split: + input: + rules.exp_get_images.output + output: + directory(f"{EXP_OUT}/run_sp_exp_Sp") + params: + cmd=lambda wc: sp_rule("exp_split", "config_exp_Sp.ini", "exp", wc.exp) + shell: + "{params.cmd}" + +# Mask per-CCD; this exposure's star cat is a declared input (per-unit re-key). +rule exp_mask: + input: + rules.exp_split.output, + star=str(RUN_DIR / "exp/{exp}/star_cat-{exp}.fits"), + output: + directory(f"{EXP_OUT}/run_sp_exp_Ma") + params: + cmd=lambda wc: sp_rule("exp_mask", "config_exp_Ma_onthefly.ini", + "exp", wc.exp, isolate=False) + shell: + "{params.cmd}" + +# SExtractor -> setools star selection -> PSFEx model -> psfex_interp, per CCD. +# setools may reject a sparse CCD (~0.2% attrition) — tolerated by the floor. +rule exp_psf: + input: + rules.exp_mask.output + output: + directory(f"{EXP_OUT}/run_sp_exp_SxSePsfPi") + params: + cmd=lambda wc: sp_rule("exp_psf", "config_exp_psfex.ini", "exp", wc.exp, + isolate=False) + shell: + "{params.cmd}" diff --git a/workflow/rules/prepare.smk b/workflow/rules/prepare.smk new file mode 100644 index 000000000..2911c3b19 --- /dev/null +++ b/workflow/rules/prepare.smk @@ -0,0 +1,67 @@ +"""Phase A — prepare: tile staging, find_exposures, per-unit star cats. + +Split into two static invocations with build_index.py (a plain script) between: + * prepare_tiles — tile_get_images -> tile_uncompress -> tile_find_exposures + * [build_index.py] — plain script, threshold-gated (not a DAG node) + * prepare_exposures — exp_get_images (exposure.smk) + per-unit star cats + +Nibi compute nodes have internet, so downloads run in-DAG (no login-node tier). +Rules stay group-compatible: no mid-chain localrules, no run:/pipe outputs. +""" + +# get_images reads a one-line tile_numbers.txt the wrapper drops in $SP_RUN. +# NUMBER_LIST is NEVER injected here (download stage; nothing on disk to +# validate against — #746 would hard-fail). +rule tile_get_images: + output: + directory(str(RUN_DIR / "tiles/{tile}/output/run_sp_tile_Git")) + params: + cmd=lambda wc: sp_rule("tile_get_images", "config_tile_Git_vos.ini", + "tile", wc.tile, isolate=False) + shell: + "{params.cmd}" + +rule tile_uncompress: + input: + rules.tile_get_images.output + output: + directory(str(RUN_DIR / "tiles/{tile}/output/run_sp_tile_Uz")) + params: + cmd=lambda wc: sp_rule("tile_uncompress", "config_tile_Uz.ini", + "tile", wc.tile, isolate=True) + shell: + "{params.cmd}" + +# find_exposures reads the tile FITS HISTORY header -> exp_numbers-.txt, the +# data-derived tile->exposure edge build_index.py aggregates (as a plain script). +rule tile_find_exposures: + input: + rules.tile_uncompress.output + output: + directory(str(RUN_DIR / "tiles/{tile}/output/run_sp_tile_Fe")) + params: + cmd=lambda wc: sp_rule("tile_find_exposures", "config_tile_Fe.ini", + "tile", wc.tile, isolate=True) + shell: + "{params.cmd}" + +# Star catalogues for masking are pre-generated offline (create_star_cat.py on a +# networked login node — the one genuinely networked prepare step). They are +# re-keyed PER UNIT here: each tile/exposure's cat is its own DAG node, so +# appending tile 51 adds nodes rather than mutating a run-level output that would +# dirty every finished mask (the growth-invariant fix, finding 11). +rule exp_star_cat: + output: + str(RUN_DIR / "exp/{exp}/star_cat-{exp}.fits") + params: + src=lambda wc: str(STAR_CATS / "exp" / f"star_cat-{wc.exp}.fits") + shell: + "ln -sf {params.src} {output}" + +rule tile_star_cat: + output: + str(RUN_DIR / "tiles/{tile}/star_cat-{tile}.fits") + params: + src=lambda wc: str(STAR_CATS / "tiles" / f"star_cat-{wc.tile}.fits") + shell: + "ln -sf {params.src} {output}" diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk new file mode 100644 index 000000000..845167710 --- /dev/null +++ b/workflow/rules/tile.smk @@ -0,0 +1,147 @@ +"""Tile post-chain — per tile: gather exposures, then detect/PSF/shape/catalogue. + +A per-tile exposure "forest" (symlink view of exactly this tile's exposures' +products) gives the tile modules a deterministic $SP_EXP to glob. But the forest +is only a runtime convenience, NEVER the DAG edge: every tile rule that reads +through $SP_EXP declares the underlying exposure directory() outputs as input via +an index dict lookup, so the last real reader is the last DAG consumer (findings +5/19). All rules are group-compatible (shell only, no mid-chain localrules). +""" + +TILE_OUT = str(RUN_DIR / "tiles/{tile}/output") +FOREST = str(RUN_DIR / "tiles/{tile}/exp_forest") + + +def _exp_out(wc, prefix): + return [str(exp_dir(e) / "output" / prefix) for e in TILE_EXP.get(wc.tile, [])] + +def tile_exp_split(wc): return _exp_out(wc, "run_sp_exp_Sp") +def tile_exp_mask(wc): return _exp_out(wc, "run_sp_exp_Ma") +def tile_exp_psf(wc): return _exp_out(wc, "run_sp_exp_SxSePsfPi") +def tile_exp_products(wc): + """All of this tile's exposures' product dirs — the forest's DAG inputs.""" + return tile_exp_split(wc) + tile_exp_mask(wc) + tile_exp_psf(wc) + + +# Build the per-tile symlink forest (exp_forest//output/run_sp_exp_* -> the +# real exposure products). Declaring the exposure dirs as input makes this the +# tile's wait-on-its-exposures edge; the forest itself is just the SP_EXP view. +rule tile_exp_forest: + input: + tile_exp_products + output: + directory(FOREST) + params: + cmd=lambda wc: (f"python {SCRIPTS}/build_forest.py --tile {wc.tile} " + f"--run-dir {RUN_DIR} --index {INDEX_DB} --forest {{output}}") + shell: + "{params.cmd}" + +# Merge single-exposure WCS headers into the tile-level sqlite log +# (log_exp_headers--.sqlite — the tile suffix downstream Sx/PiViVi/Ng +# consume via NUMBERING_SCHEME -000-000). Reads headers-*.npy through the forest, +# so the exposure split dirs are explicit inputs. +rule tile_merge_headers: + input: + forest=rules.tile_exp_forest.output, + split=tile_exp_split, + output: + directory(f"{TILE_OUT}/run_sp_tile_Mh_exp") + params: + cmd=lambda wc: sp_rule("tile_merge_headers", "config_tile_Mh_exp.ini", + "tile", wc.tile, isolate=True, + exp_forest=FOREST.format(tile=wc.tile)) + shell: + "{params.cmd}" + +# Mask tiles; this tile's star cat is a declared input (per-unit re-key). +rule tile_mask: + input: + git=rules.tile_get_images.output, + star=str(RUN_DIR / "tiles/{tile}/star_cat-{tile}.fits"), + output: + directory(f"{TILE_OUT}/run_sp_tile_Ma") + params: + cmd=lambda wc: sp_rule("tile_mask", "config_tile_Ma_onthefly.ini", + "tile", wc.tile, isolate=True) + shell: + "{params.cmd}" + +# SExtractor object detection on the tile. +rule tile_detect: + input: + mask=rules.tile_mask.output, + mh=rules.tile_merge_headers.output, + output: + directory(f"{TILE_OUT}/run_sp_tile_Sx") + params: + cmd=lambda wc: sp_rule("tile_detect", "config_tile_Sx.ini", + "tile", wc.tile, isolate=True) + shell: + "{params.cmd}" + +# PSFEx interpolation to galaxies + vignet postage stamps (reads exposure PSF +# models + split through the forest — those dirs are explicit inputs). +rule tile_vignets: + input: + sx=rules.tile_detect.output, + forest=rules.tile_exp_forest.output, + split=tile_exp_split, + psf=tile_exp_psf, + output: + directory(f"{TILE_OUT}/run_sp_tile_PiViVi") + params: + cmd=lambda wc: sp_rule("tile_vignets", "config_tile_PiViVi_canfar_sx.ini", + "tile", wc.tile, isolate=True, + exp_forest=FOREST.format(tile=wc.tile)) + shell: + "{params.cmd}" + +# ngmix shape measurement — static N chunks. Each chunk computes its own closed +# object-ID range in-job from the tile's own sexcat (a declared input); chunks +# never rewrite the shared log (--no-log-sync) and read literal INPUT_DIRs. +rule tile_ngmix: + input: + vignets=rules.tile_vignets.output, + sx=rules.tile_detect.output, + output: + directory(f"{TILE_OUT}/run_sp_tile_ngmix_Ng{{chunk}}u") + params: + cmd=lambda wc: sp_rule( + "tile_ngmix", "config_tile_Ng_template_batch.ini", "tile", wc.tile, + extra=f"--ngmix-chunk {wc.chunk} --ngmix-nchunks {NGMIX_CHUNKS}"), + shell: + "{params.cmd}" + +def ngmix_chunks(wc): + return [f"{RUN_DIR}/tiles/{wc.tile}/output/run_sp_tile_ngmix_Ng{k}u" + for k in range(1, NGMIX_CHUNKS + 1)] + +# Merge the N chunk catalogues (DAG-serialized: the single per-tile log-sync +# point before make_cat). N_SPLIT_MAX == chunk count. +rule tile_merge_cats: + input: + ngmix_chunks + output: + directory(f"{TILE_OUT}/run_sp_Ms") + params: + cmd=lambda wc: sp_rule("tile_merge_cats", "config_merge_sep_cats_template.ini", + "tile", wc.tile, isolate=True, + extra=f"--n-split-max {NGMIX_CHUNKS}"), + shell: + "{params.cmd}" + +# Build the final catalogue: the run's science product. protected() so a +# code/params-drift rerun cannot silently discard a finished catalogue — the +# recompute must go through `sp rerun` (--forcerun). +rule tile_make_cat: + input: + rules.tile_merge_cats.output + output: + protected(str(RUN_DIR / "tiles/{tile}/final_cat-{tile}.fits")) + params: + cmd=lambda wc: sp_rule("tile_make_cat", "config_make_cat_psfex_nosm.ini", + "tile", wc.tile, isolate=True, + extra=f"--final-cat {RUN_DIR}/tiles/{wc.tile}/final_cat-{wc.tile}.fits"), + shell: + "{params.cmd}" diff --git a/workflow/scripts/build_forest.py b/workflow/scripts/build_forest.py new file mode 100644 index 000000000..82df8b0c6 --- /dev/null +++ b/workflow/scripts/build_forest.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +"""Build a tile's exposure symlink forest (the SP_EXP view). + +A plain script (not a run: block) so the tile chain stays group-compatible. +Reads the tile's exposures from run_index.sqlite and symlinks each exposure's +``exp//output`` into ``//output`` by exact name (no glob), so +the tile modules' $SP_EXP globs resolve to exactly this tile's exposures' +products. The forest is a convenience view; the DAG edge to the exposures is +declared in the rule's input (tile.smk), not here. +""" + +import argparse +import sqlite3 +from pathlib import Path + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--tile", required=True) + p.add_argument("--run-dir", required=True, type=Path) + p.add_argument("--index", required=True, type=Path) + p.add_argument("--forest", required=True, type=Path) + args = p.parse_args() + + con = sqlite3.connect(args.index) + exps = [r[0] for r in con.execute( + "SELECT exp_id FROM tile_exposures WHERE tile_id=?", (args.tile,))] + con.close() + + args.forest.mkdir(parents=True, exist_ok=True) + for e in exps: + src = args.run_dir / "exp" / e / "output" + dst = args.forest / e / "output" + dst.parent.mkdir(parents=True, exist_ok=True) + if dst.is_symlink() or dst.exists(): + dst.unlink() + dst.symlink_to(src) + print(f"[build_forest] {args.tile}: {len(exps)} exposures -> {args.forest}") + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/build_index.py b/workflow/scripts/build_index.py new file mode 100644 index 000000000..13783314a --- /dev/null +++ b/workflow/scripts/build_index.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +"""Build the run index (``run_index.sqlite``) that drives the compute DAG. + +The index is *parse-time data*, never a rule input: the Snakefile loads it once +at parse time into plain dicts, so extending a run's tile list changes which +jobs exist without touching the mtime chain of completed work. + +It records, per run: + + tiles(tile_id, ra_dir, n_exp, status) + exposures(exp_id) -- deduplicated union over tiles + tile_exposures(tile_id, exp_id) -- the tile->exposure edges + +The tile->exposure edges are *data-derived*: they are read from each tile's +``find_exposures`` output (``exp_numbers--.txt``), which +``find_exposures_runner`` produces by parsing the tile FITS ``HISTORY`` header. +So this is a **plain script**, not a DAG node: it runs between the two static +invocations — after ``snakemake prepare_tiles`` (Git+Uz+Fe, keep-going) and +before ``snakemake prepare_exposures`` parses the now-existing index. It +iterates the *declared* tile list and checks each tile's Fe output at its +deterministic path (no globbing — O(tile-list) existence checks, respecting the +no-``ls``-at-scale ban). A bad tile costs only that tile: it is recorded in +``missing.json`` and the index is built over the rest. The script exits nonzero +only if the missing *fraction* exceeds ``--missing-threshold`` (default 0.0), +so a keep-going download storm that lost a few tiles does not cost the run. +There is NO ``--allow-missing`` flag (that Snakemake flag does not exist). + +Exposure IDs are stored with their trailing single-char suffix stripped +(``2243881p`` -> ``2243881``); that ``exp_base`` is the dedup key and the +exposure-rule wildcard, matching the ``exp///`` forest layout. +""" + +import argparse +import json +import sqlite3 +import sys +from pathlib import Path + + +def read_exposure_list(exp_numbers_file: Path) -> list[str]: + """Return the exp_base IDs listed in one tile's find_exposures output. + + Each line is an exposure name like ``2243881p``; we strip a trailing + alphabetic suffix so the base ID is the dedup key. + """ + ids = [] + for line in exp_numbers_file.read_text().splitlines(): + name = line.strip() + if not name: + continue + ids.append(name[:-1] if name[-1].isalpha() else name) + return ids + + +def build(tile_ids: list[str], run_dir: Path, db_path: Path, + missing_threshold: float = 0.0) -> dict: + """Build the index over ``tile_ids``; return a summary dict. + + For each tile, check its ``exp_numbers--.txt`` at the tile's + deterministic ``find_exposures`` run dir (RUN_DATETIME=False, no glob). A + tile whose exposure list is missing is recorded in ``missing.json`` and the + index is built over the rest (a bad tile costs that tile, not the run). The + build is fatal only if the missing fraction exceeds ``missing_threshold``. + """ + db_path.parent.mkdir(parents=True, exist_ok=True) + con = sqlite3.connect(db_path) + con.executescript( + """ + DROP TABLE IF EXISTS tiles; + DROP TABLE IF EXISTS exposures; + DROP TABLE IF EXISTS tile_exposures; + CREATE TABLE tiles(tile_id TEXT PRIMARY KEY, ra_dir TEXT, n_exp INTEGER); + CREATE TABLE exposures(exp_id TEXT PRIMARY KEY); + CREATE TABLE tile_exposures( + tile_id TEXT, exp_id TEXT, + PRIMARY KEY (tile_id, exp_id)); + """ + ) + + missing = [] + all_exposures: set[str] = set() + for tile_id in tile_ids: + ra_dir = tile_id.split(".")[0] + idra, iddec = tile_id.split(".") + # Flat forest, deterministic run dir (RUN_DATETIME=False). + exp_file = (run_dir / "tiles" / tile_id / "output" / "run_sp_tile_Fe" / + "find_exposures_runner" / "output" / + f"exp_numbers-{idra}-{iddec}.txt") + if not exp_file.exists(): + missing.append(tile_id) + continue + exp_ids = read_exposure_list(exp_file) + con.execute("INSERT INTO tiles VALUES (?,?,?)", + (tile_id, ra_dir, len(exp_ids))) + for exp_id in exp_ids: + all_exposures.add(exp_id) + con.execute("INSERT OR IGNORE INTO tile_exposures VALUES (?,?)", + (tile_id, exp_id)) + + con.executemany("INSERT OR IGNORE INTO exposures VALUES (?)", + [(e,) for e in sorted(all_exposures)]) + con.commit() + con.close() + + (db_path.parent / "missing.json").write_text(json.dumps(missing, indent=2)) + frac = len(missing) / len(tile_ids) if tile_ids else 0.0 + if frac > missing_threshold: + raise SystemExit( + f"Missing exposure lists for {len(missing)}/{len(tile_ids)} tile(s) " + f"(fraction {frac:.3f} > threshold {missing_threshold}): {missing}. " + f"Re-run prepare_tiles for them, or raise --missing-threshold.") + return {"n_tiles": len(tile_ids) - len(missing), + "n_exposures": len(all_exposures), + "n_missing": len(missing)} + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--tile-list", required=True, type=Path, + help="file of tile IDs, one per line") + p.add_argument("--run-dir", required=True, type=Path, + help="$SP_RUN: root of the tiles/ work-dir forest") + p.add_argument("--db", required=True, type=Path, + help="output run_index.sqlite path") + p.add_argument("--missing-threshold", type=float, default=0.0, + help="fatal if the missing-tile fraction exceeds this " + "(default 0.0: any missing tile is fatal)") + args = p.parse_args() + + tile_ids = [ln.strip() for ln in args.tile_list.read_text().splitlines() + if ln.strip()] + summary = build(tile_ids, args.run_dir, args.db, args.missing_threshold) + print(f"run_index: {summary['n_tiles']} tiles, " + f"{summary['n_exposures']} exposures, " + f"{summary['n_missing']} missing -> {args.db}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/clean_exposures.py b/workflow/scripts/clean_exposures.py new file mode 100644 index 000000000..4f46bf19b --- /dev/null +++ b/workflow/scripts/clean_exposures.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""Index-driven exposure-store cleanup (`sp clean-exposures`). + +temp() is deliberately NOT used on the shared exposure directories (findings +4/5/6/19): they overlap tiles by construction, so temp() deletion would cascade +destructive reruns across spatial neighbours whenever a tile is appended. Instead +the store is persistent and space is reclaimed explicitly here. + +For each exposure in run_index.sqlite, delete its bulky intermediates +(run_sp_exp_Sp / run_sp_exp_Ma / run_sp_exp_SxSePsfPi) ONLY when every tile that +consumes it (tile_exposures) has its final_cat-.fits present across the whole +output root. An exposure still feeding an unfinished tile is left untouched. + +Caveat: if a later batch appends a tile that shares a deleted exposure, +Snakemake will regenerate that exposure — cleanup trades peak storage for +recompute. Run it between campaign phases, not mid-DAG. --dry-run by default. +""" + +import argparse +import shutil +import sqlite3 +import sys +from pathlib import Path + +PREFIXES = ("run_sp_exp_Sp", "run_sp_exp_Ma", "run_sp_exp_SxSePsfPi") + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--run-dir", required=True, type=Path) + p.add_argument("--index", required=True, type=Path) + p.add_argument("--apply", action="store_true", + help="actually delete (default: dry-run, list only)") + args = p.parse_args() + + con = sqlite3.connect(args.index) + edges = con.execute("SELECT exp_id, tile_id FROM tile_exposures").fetchall() + con.close() + + consumers: dict[str, list[str]] = {} + for exp_id, tile_id in edges: + consumers.setdefault(exp_id, []).append(tile_id) + + def tile_done(tile): + return (args.run_dir / "tiles" / tile / f"final_cat-{tile}.fits").exists() + + freed, held = 0, 0 + for exp_id, tiles in consumers.items(): + if not all(tile_done(t) for t in tiles): + held += 1 + continue + for prefix in PREFIXES: + d = args.run_dir / "exp" / exp_id / "output" / prefix + if d.is_dir(): + freed += 1 + print(f"{'DELETE' if args.apply else 'would delete'} {d}") + if args.apply: + shutil.rmtree(d) + print(f"[clean_exposures] {freed} dirs {'freed' if args.apply else 'reclaimable'}, " + f"{held} exposures held (consuming tile unfinished)", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/completeness.py b/workflow/scripts/completeness.py new file mode 100644 index 000000000..f35e81aee --- /dev/null +++ b/workflow/scripts/completeness.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""The count-floor completeness table — the single failure policy. + +This is the ported ``complete_check`` count table from the v2.0 bash layer +(``run_job_sp_canfar_v2.0.bash`` job dispatch, survey §4). It is the *only* +failure policy in the design: there is no 3-class taxonomy and no error-signature +whitelist. A stage is a real failure iff a mandatory runner produced fewer than +its ``floor`` files; per-CCD attrition (a sparse CCD setools rejects, ~0.2%) +sits between ``floor`` and ``expect`` and is tolerated. + +Two consumers share this table: + * ``sp_rule.py`` — after ``shapepipe_run``, counts products per runner and + exits nonzero if any mandatory runner is below its floor (``keep-going`` + then isolates that unit's cone; ``retries`` handle transient failures). + * ``run_report.py`` — disk-scans finished trees against ``expect`` to + enumerate shortfalls for the human, mid-run or after. + +Per-runner fields: + expect nominal file count for a fully complete unit (report yardstick) + floor the fail-loud minimum (below this the job exits nonzero) + warn if True the runner never fails the unit at all (bash ``:warn`` — + e.g. psfex_interp on tiles missing some epochs) + subpath count files in ``/output//`` instead of + ``/output/`` (bash ``:rand_split`` — setools split cats) + +Counts are file counts in the runner's output dir, matching the bash +``ls / | wc -l`` semantics (broken symlinks excluded by the caller). +""" + +# stage -> {runner_subdir: {expect, floor, [warn], [subpath]}} +COMPLETENESS = { + # --- tile prepare (phase A) --- + "tile_get_images": {"get_images_runner": dict(expect=4, floor=4)}, + "tile_uncompress": {"uncompress_fits_runner": dict(expect=1, floor=1)}, + "tile_find_exposures": {"find_exposures_runner": dict(expect=1, floor=1)}, + + # --- exposure chain --- + "exp_get_images": {"get_images_runner": dict(expect=6, floor=6)}, + "exp_split": {"split_exp_runner": dict(expect=121, floor=41)}, + "exp_mask": {"mask_runner": dict(expect=40, floor=1)}, + "exp_psf": { + "sextractor_runner": dict(expect=80, floor=2), + "setools_runner": dict(expect=80, floor=2, subpath="rand_split"), + "psfex_runner": dict(expect=80, floor=2), + "psfex_interp_runner": dict(expect=40, floor=0, warn=True), + }, + + # --- tile post --- + "tile_merge_headers": {"merge_headers_runner": dict(expect=1, floor=1)}, + "tile_mask": {"mask_runner": dict(expect=1, floor=1)}, + "tile_detect": {"sextractor_runner": dict(expect=2, floor=2)}, + "tile_vignets": { + "psfex_interp_runner": dict(expect=1, floor=1), + "vignetmaker_runner_run_1": dict(expect=1, floor=1), + "vignetmaker_runner_run_2": dict(expect=4, floor=4), + }, + "tile_ngmix": {"ngmix_runner": dict(expect=1, floor=1)}, + "tile_merge_cats": {"merge_sep_cats_runner": dict(expect=1, floor=1)}, + "tile_make_cat": {"make_cat_runner": dict(expect=1, floor=1)}, +} + + +def count_products(run_dir, runner, spec): + """Count files in ``run_dir//output[/]/`` (live links only).""" + out = run_dir / runner / "output" + if "subpath" in spec: + out = out / spec["subpath"] + if not out.is_dir(): + return 0 + return sum(1 for p in out.iterdir() if p.exists()) # p.exists() drops dead links + + +def check_floor(stage, run_dir): + """Return (ok, details). ok is False iff a mandatory runner is below floor. + + ``details`` is a list of (runner, n_found, floor, expect, warn) tuples. + """ + table = COMPLETENESS[stage] + details, ok = [], True + for runner, spec in table.items(): + n = count_products(run_dir, runner, spec) + details.append((runner, n, spec["floor"], spec["expect"], + spec.get("warn", False))) + if not spec.get("warn", False) and n < spec["floor"]: + ok = False + return ok, details diff --git a/workflow/scripts/run_report.py b/workflow/scripts/run_report.py new file mode 100644 index 000000000..bb3300405 --- /dev/null +++ b/workflow/scripts/run_report.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Standalone run report — NOT a DAG node. + +A report rule that declares all tiles' outputs as inputs is a descendant of every +job, so any single hard failure (under --keep-going) poisons its cone and the +report never runs — the exact scenario it exists for (finding 8). So this is a +plain script: it disk-scans the tiles/ and exp/ trees against the count-floor +table (completeness.py) and the index, and emits run_report.json. It runs +automatically via the Snakefile's onsuccess/onerror hooks, and by hand any time +(``sp report``), mid-run included. + +It distinguishes whole-exposure absence (all runners missing — a real gap or a +deletion/forest bug) from tolerated per-CCD attrition (counts between floor and +expect), so a future deletion bug cannot be silently absorbed (finding 5). +""" + +import argparse +import json +import sqlite3 +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from completeness import COMPLETENESS, check_floor # noqa: E402 + +# stage -> (level, run_sp_ dir name) +STAGE_DIR = { + "tile_get_images": ("tile", "run_sp_tile_Git"), + "tile_uncompress": ("tile", "run_sp_tile_Uz"), + "tile_find_exposures": ("tile", "run_sp_tile_Fe"), + "exp_get_images": ("exp", "run_sp_exp_Gie"), + "exp_split": ("exp", "run_sp_exp_Sp"), + "exp_mask": ("exp", "run_sp_exp_Ma"), + "exp_psf": ("exp", "run_sp_exp_SxSePsfPi"), + "tile_merge_headers": ("tile", "run_sp_tile_Mh_exp"), + "tile_mask": ("tile", "run_sp_tile_Ma"), + "tile_detect": ("tile", "run_sp_tile_Sx"), + "tile_vignets": ("tile", "run_sp_tile_PiViVi"), + "tile_merge_cats": ("tile", "run_sp_Ms"), + "tile_make_cat": ("tile", "run_sp_Mc"), +} + + +def classify(stage, run_dir): + """'absent' | 'under' | 'attrition' | 'complete' for one unit's stage.""" + if not run_dir.is_dir(): + return "absent", [] + ok, details = check_floor(stage, run_dir) + if not ok: + # all mandatory runners empty -> whole-unit absence; else below floor + mand = [d for d in details if not d[4]] + if mand and all(n == 0 for _, n, *_ in mand): + return "absent", details + return "under", details + if any(n < expect for _, n, floor, expect, warn in details if not warn): + return "attrition", details + return "complete", details + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--run-dir", required=True, type=Path) + p.add_argument("--index", required=True, type=Path) + p.add_argument("--status", default="manual") + p.add_argument("--out", type=Path, default=None) + args = p.parse_args() + + tiles, exps = [], [] + if args.index.exists(): + con = sqlite3.connect(args.index) + tiles = [r[0] for r in con.execute("SELECT tile_id FROM tiles")] + exps = [r[0] for r in con.execute("SELECT exp_id FROM exposures")] + con.close() + + missing_json = args.index.parent / "missing.json" + missing = json.loads(missing_json.read_text()) if missing_json.exists() else [] + + report = {"status": args.status, "n_tiles": len(tiles), "n_exposures": len(exps), + "missing_tiles": missing, "stages": {}} + for stage, (level, prefix) in STAGE_DIR.items(): + if stage not in COMPLETENESS: + continue + units = tiles if level == "tile" else exps + sub = "tiles" if level == "tile" else "exp" + tally = {"complete": 0, "attrition": 0, "under": [], "absent": []} + for u in units: + run_dir = args.run_dir / sub / u / "output" / prefix + verdict, _ = classify(stage, run_dir) + if verdict in ("complete", "attrition"): + tally[verdict] += 1 + else: + tally[verdict].append(u) + report["stages"][stage] = tally + + finals = [t for t in tiles + if (args.run_dir / "tiles" / t / f"final_cat-{t}.fits").exists()] + report["final_cats"] = {"present": len(finals), "of": len(tiles), + "missing": [t for t in tiles if t not in finals]} + + out = args.out or (args.index.parent / "run_report.json") + out.write_text(json.dumps(report, indent=2)) + print(f"[run_report] {report['final_cats']['present']}/{len(tiles)} final " + f"cats -> {out}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/sp_rule.py b/workflow/scripts/sp_rule.py new file mode 100644 index 000000000..cce92e2c8 --- /dev/null +++ b/workflow/scripts/sp_rule.py @@ -0,0 +1,405 @@ +#!/usr/bin/env python3 +"""Thin per-unit ShapePipe rule wrapper — the single entrypoint every rule calls. + +Under the one-allocation local executor + apptainer software-deployment, +Snakemake runs each rule's shell *inside* the container on the allocation's node. +So this script already runs in-container: it calls ``shapepipe_run`` directly +(never apptainer itself), which is why the workflow never hand-invokes apptainer. + +What it owns — only what is intrinsically ShapePipe's, nothing Snakemake already +does: + +1. **Per-unit work dir + v2.0 isolation furniture.** Each tile/exposure gets its + own ``$SP_RUN`` (``tiles//`` or ``exp//``). Isolation is by work-dir + *content*, the proven v2.0 mechanism — NOT ``-e/--exclusive``: + - a ``cfis`` symlink to the canonical config dir (so ``$SP_CONFIG`` / ``.mask`` + refs resolve); + - ``star_cat_exp`` / ``star_cat_tiles`` symlinks to the shared cat pool; + - for a tile: ``tile_numbers.txt`` = this tile ID (dot format, real data — + what get_images reads); + - for an exposure: a fabricated + ``output/run_sp_tile_Fe/find_exposures_runner/output/exp_numbers-000-000.txt`` + = this exposure ID (v2.0 ``init_exp_work_dir`` pseudo-Fe), so the Gie + config's ``last:find_exposures_runner`` resolver finds exactly one unit. + +2. **Per-unit config copy.** Forces ``RUN_DATETIME = False`` (deterministic + ``output/run_sp_/`` the rule declares as its ``directory()`` output), + sets ``SMP_BATCH_SIZE = {threads}`` (fork width == cpus_per_task, so SLURM + packing density and true per-job process count are one number), and — only + when ``isolate`` (tile-scheme stages + exp split; NEVER get_images / exp_mask / + exp_psf, whose per-CCD/download numbering would make #746 turn tolerated + per-CCD attrition into a whole-exposure hard failure) — ``NUMBER_LIST = -``. + +3. **Log sync at START and end.** ``update_runs_log_file.py`` regenerates + ``log_run_sp.txt`` before ``shapepipe_run`` (after the pre-delete, so + resolution never sees a just-deleted dir) and again after. Skipped for ngmix + chunks: the log is already complete from the prior tile stages, and the N + concurrent chunk jobs share the tile work dir — a full log *rewrite* would + race, whereas ShapePipe's own single-line ``O_APPEND`` of each chunk's run is + atomic. ngmix chunk configs additionally carry *literal* INPUT_DIRs + (deterministic under RUN_DATETIME=False), so they need no log read at all. + +4. **Count-floor check** (``completeness.py`` — the ported bash ``complete_check`` + table, the single failure policy: no 3-class taxonomy, no whitelist). After + the run, count products per mandatory runner and exit nonzero below the floor. + ``keep-going`` isolates that unit's cone; per-CCD attrition between floor and + ``expect`` is tolerated. Stages absent from the table fall back to a + zero-output floor. + +Stage-specific verbs (grounded in ``runs/p3-batch1/job_p3_batch1.sh``): + +- ``--ngmix-chunk K --ngmix-nchunks N`` — read this tile's SExtractor object + count from its ``run_sp_tile_Sx`` output, split ``[1, N_obj]`` into N disjoint + closed ID ranges, and generate chunk K's config from + ``config_tile_Ng_template_batch.ini``. +- ``--n-split-max N`` — generate the merge config from + ``config_merge_sep_cats_template.ini`` with ``N_SPLIT_MAX = N``. +- ``--final-cat PATH`` — after ``make_cat``, copy the produced ``final_cat`` FITS + to PATH (the rule's declared, protected science-product output). + +Run by hand to reproduce exactly what a rule does (the escape hatch — it +pre-deletes the run dir, just like Snakemake's delete-before-rerun):: + + python sp_rule.py --stage exp_split --config config_exp_Sp.ini \\ + --unit 2243881 --level exp --run-dir $SP_RUN --config-src $SP_CONFIG \\ + --star-cats $STAR_CATS --threads 8 +""" + +import argparse +import os +import shutil +import subprocess +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from completeness import COMPLETENESS, check_floor # noqa: E402 + + +def unit_work_dir(run_dir: Path, level: str, unit: str) -> Path: + """Per-unit work dir in the forest: tiles/ or exp/. + + Flat (no IDra/prefix sharding): the shard existed only to keep the bash + layer's ``ls exp/`` bearable, but Snakemake never globs the forest — the + run index drives every path — so flattening keeps single-wildcard rules. + ``run`` level (aggregation stages) work directly in ``run_dir``. + """ + if level == "tile": + return run_dir / "tiles" / unit + if level == "exp": + base = unit[:-1] if unit[-1].isalpha() else unit + return run_dir / "exp" / base + return run_dir + + +def materialise_unit(work: Path, level: str, unit: str, config_src: Path, + star_cats: Path) -> None: + """Create the v2.0 unit-isolation furniture in the work dir.""" + (work / "output").mkdir(parents=True, exist_ok=True) + + cfis = work / "cfis" # $SP_CONFIG -> canonical config dir (.mask refs resolve) + if not cfis.exists(): + cfis.symlink_to(config_src) + + for name, sub in (("star_cat_exp", "exp"), ("star_cat_tiles", "tiles")): + link = work / name + if not link.exists() and (star_cats / sub).exists(): + link.symlink_to(star_cats / sub) + + if level == "tile": + (work / "tile_numbers.txt").write_text(unit + "\n") # dot format, real data + elif level == "exp": + fe = work / "output" / "run_sp_tile_Fe" / "find_exposures_runner" / "output" + fe.mkdir(parents=True, exist_ok=True) + exp_numbers = fe / "exp_numbers-000-000.txt" + if not exp_numbers.exists(): + exp_numbers.write_text(unit + "\n") + + +def normalize_config_text(text: str, unit: str, isolate: bool, threads: int) -> str: + """Force RUN_DATETIME=False, SMP_BATCH_SIZE=threads, and (if isolating) + NUMBER_LIST=-. + + ``NUMBER_LIST`` uses ShapePipe's image-number convention (dots -> dashes), + the ``set_config_number_list`` mechanism that replaced the retired + ``-e/--exclusive`` flag (#746). + """ + lines = text.splitlines() + out, in_file, saw_datetime, saw_numberlist, saw_smp = [], False, False, False, False + number = "-" + unit.replace(".", "-") + for line in lines: + stripped = line.strip() + if stripped.startswith("[") and stripped.endswith("]"): + in_file = stripped.upper() == "[FILE]" + if stripped.upper().startswith("RUN_DATETIME"): + out.append("RUN_DATETIME = False") + saw_datetime = True + continue + if stripped.upper().startswith("SMP_BATCH_SIZE"): + out.append(f"SMP_BATCH_SIZE = {threads}") + saw_smp = True + continue + if isolate and stripped.upper().startswith("NUMBER_LIST"): + out.append(f"NUMBER_LIST = {number}") + saw_numberlist = True + continue + out.append(line) + if in_file and stripped.upper() == "[FILE]" and isolate and not saw_numberlist: + out.append(f"NUMBER_LIST = {number}") + saw_numberlist = True + if not saw_datetime: + # ShapePipe reads RUN_DATETIME from [DEFAULT]; inject after its header. + patched, done = [], False + for line in out: + patched.append(line) + if not done and line.strip().upper() == "[DEFAULT]": + patched.append("RUN_DATETIME = False") + done = True + out = patched if done else ["[DEFAULT]", "RUN_DATETIME = False"] + out + if not saw_smp: + # SMP_BATCH_SIZE lives in [JOB]; inject after its header. + patched, done = [], False + for line in out: + patched.append(line) + if not done and line.strip().upper() == "[JOB]": + patched.append(f"SMP_BATCH_SIZE = {threads}") + done = True + out = patched if done else out + ["[JOB]", f"SMP_BATCH_SIZE = {threads}"] + return "\n".join(out) + "\n" + + +def run_prefix_from_text(text: str, config_label: str) -> str: + """The RUN_NAME (== run_sp_) the module will write under.""" + for line in text.splitlines(): + s = line.strip() + if s.upper().startswith("RUN_NAME"): + return s.split("=", 1)[1].strip() + raise SystemExit(f"no RUN_NAME in {config_label}") + + +# --- ngmix chunking -------------------------------------------------------- + +# Literal, deterministic INPUT_DIRs for the ngmix chunk (RUN_DATETIME=False), so +# concurrent chunks need no log read at all (finding 18). +NGMIX_LITERAL_INPUT = ( + "./output/run_sp_tile_Sx/sextractor_runner/output, " + "./output/run_sp_tile_PiViVi/psfex_interp_runner/output, " + "./output/run_sp_tile_PiViVi/vignetmaker_runner_run_2/output, " + "./output/run_sp_tile_Mh_exp/merge_headers_runner/output") + + +def ngmix_id_ranges(n_obj: int, n_chunks: int) -> list[tuple[int, int]]: + """Split ``[1, n_obj]`` into ``n_chunks`` disjoint *closed* ID ranges. + + SExtractor's ``NUMBER`` column (what ngmix reads as ``obj_id``) runs 1..N + contiguous, so covering ``[1, N]`` processes every object exactly once. The + first n_chunks-1 chunks get ``n_obj // n_chunks`` each; the last gets the + remainder — a *closed* upper bound at ``n_obj`` (never ``ID_OBJ_MAX = -1``: + ngmix treats ``id_obj_max <= 0`` as unbounded, which would double-count). + + Deviation from the nibi monolith (deliberate): the monolith sizes every + tile's chunks from the *average* object count across all tiles and leaves the + last chunk open (``-1``); here each tile is an independent job, so we size + from *this* tile's own count and close the last chunk. The merged catalogue + is identical either way (merge_sep_cats recombines all chunks, ngmix fits + each object independently) — this just removes the open-ended overflow. + """ + base, rem = divmod(n_obj, n_chunks) + ranges, lo = [], 1 + for k in range(1, n_chunks + 1): + size = base + (rem if k == n_chunks else 0) + hi = lo + size - 1 + ranges.append((lo, hi)) + lo = hi + 1 + return ranges + + +def tile_object_count(work: Path) -> int: + """Object count for this tile from its run_sp_tile_Sx SExtractor catalogue. + + Matches ``get_number_objects.py``: the ``NAXIS2`` of the catalogue's last + HDU. Per-unit isolation means exactly one ``sexcat*.fits`` lives here. + """ + from astropy.io import fits + + sexcats = sorted((work / "output" / "run_sp_tile_Sx").glob( + "sextractor_runner/output/sexcat*.fits")) + if not sexcats: + raise SystemExit( + f"[sp_rule] FATAL ngmix: no sexcat under {work}/output/run_sp_tile_Sx") + with fits.open(sexcats[0]) as hdul: + return int(hdul[-1].header["NAXIS2"]) + + +def apply_ngmix_subs(text: str, chunk: int, lo: int, hi: int) -> str: + """Fill the ngmix batch template for one chunk (the monolith's sed recipe) + plus literal INPUT_DIRs so the chunk needs no shared-log read.""" + text = text.replace("NgXu", f"Ng{chunk}u").replace("X_interp", "psfex_interp") + out = [] + for line in text.splitlines(): + s = line.strip() + if s.startswith("ID_OBJ_MIN"): + out.append(f"ID_OBJ_MIN = {lo}") + elif s.startswith("ID_OBJ_MAX"): + out.append(f"ID_OBJ_MAX = {hi}") + elif s.startswith("INPUT_DIR") and "sextractor_runner" in s: + out.append(f"INPUT_DIR = {NGMIX_LITERAL_INPUT}") + else: + out.append(line) + return "\n".join(out) + + +def apply_merge_subs(text: str, n_split_max: int) -> str: + """Fill N_SPLIT_MAX in the merge template (the monolith's sed recipe).""" + out = [] + for line in text.splitlines(): + if line.strip().startswith("N_SPLIT_MAX"): + out.append(f"N_SPLIT_MAX = {n_split_max}") + else: + out.append(line) + return "\n".join(out) + + +# --- in-job count floor ---------------------------------------------------- + +def count_floor(run_out: Path, stage: str) -> None: + """Fail loudly if any mandatory runner is below its ``floor`` (completeness.py). + + Stages absent from the table skip this floor (only the zero-output floor + applies to them). + """ + if stage not in COMPLETENESS: + return + ok, details = check_floor(stage, run_out) + for runner, n, floor, expect, warn in details: + tag = "warn" if warn else ("OK" if n >= floor else " <-- BELOW floor") + print(f"[sp_rule] {runner}: {n}/{expect} (floor {floor}) {tag}", + file=sys.stderr) + if not ok: + print(f"[sp_rule] FATAL {stage}: count floor breached in {run_out}", + file=sys.stderr) + sys.exit(1) + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--stage", required=True) + p.add_argument("--config", required=True, help="config filename in --config-src") + p.add_argument("--unit", required=True) + p.add_argument("--level", required=True, choices=["tile", "exp", "run"]) + p.add_argument("--run-dir", required=True, type=Path, help="$SP_RUN root") + p.add_argument("--config-src", required=True, type=Path, + help="dir holding the canonical configs") + p.add_argument("--star-cats", type=Path, default=None, + help="shared star-cat pool (tiles/ + exp/ subdirs)") + p.add_argument("--threads", type=int, default=1, + help="SMP_BATCH_SIZE == cpus_per_task (fork width)") + p.add_argument("--exp-forest", type=Path, default=None, + help="$SP_EXP: per-tile exposure forest (tile post-stages)") + p.add_argument("--no-isolate", action="store_true", + help="skip NUMBER_LIST (get_images / exp_mask / exp_psf)") + p.add_argument("--ngmix-chunk", type=int, default=None, + help="1-indexed ngmix chunk K (with --ngmix-nchunks)") + p.add_argument("--ngmix-nchunks", type=int, default=None, + help="total ngmix chunks N (with --ngmix-chunk)") + p.add_argument("--n-split-max", type=int, default=None, + help="N_SPLIT_MAX for the merge_sep_cats template") + p.add_argument("--final-cat", type=Path, default=None, + help="copy the produced final_cat FITS here (make_cat)") + args = p.parse_args() + + ngmix_mode = args.ngmix_chunk is not None + if ngmix_mode and args.ngmix_nchunks is None: + raise SystemExit("--ngmix-chunk requires --ngmix-nchunks") + + work = unit_work_dir(args.run_dir, args.level, args.unit) + materialise_unit(work, args.level, args.unit, args.config_src, + args.star_cats or args.run_dir) + + # Build the config copy. ngmix chunks and merge fill template placeholders + # first; every copy is then normalized (RUN_DATETIME=False, SMP_BATCH_SIZE, + # [+ NUMBER_LIST]). + src_text = (args.config_src / args.config).read_text() + isolate = not args.no_isolate + if ngmix_mode: + n_obj = tile_object_count(work) + lo, hi = ngmix_id_ranges(n_obj, args.ngmix_nchunks)[args.ngmix_chunk - 1] + print(f"[sp_rule] ngmix chunk {args.ngmix_chunk}/{args.ngmix_nchunks}: " + f"n_obj={n_obj} -> ID_OBJ [{lo}, {hi}]", file=sys.stderr) + src_text = apply_ngmix_subs(src_text, args.ngmix_chunk, lo, hi) + cfg_name = f"config_tile_Ng{args.ngmix_chunk}u.ini" # distinct per chunk + elif args.n_split_max is not None: + src_text = apply_merge_subs(src_text, args.n_split_max) + cfg_name = args.config + else: + cfg_name = args.config + + cfg_mod_dir = work / "cfis_mod" + cfg_mod = cfg_mod_dir / cfg_name + cfg_text = normalize_config_text(src_text, args.unit, isolate, args.threads) + cfg_mod_dir.mkdir(parents=True, exist_ok=True) + cfg_mod.write_text(cfg_text) + + prefix = run_prefix_from_text(cfg_text, str(cfg_mod)) + run_out = work / "output" / prefix + + # Pre-delete the deterministic run dir (delete-before-rerun for the hand-run + # escape hatch; ShapePipe's FileHandler.mkdir raises on an existing run dir). + if run_out.exists(): + shutil.rmtree(run_out) + + env = dict(os.environ) + env["SP_RUN"] = str(work) + env["SP_CONFIG"] = str(work / "cfis") # canonical dir (.mask refs), not cfis_mod + if args.exp_forest: + env["SP_EXP"] = str(args.exp_forest) + # Thread caps also come via apptainer-args; keep here for the hand-run path. + env.setdefault("OMP_NUM_THREADS", "1") + env.setdefault("OPENBLAS_NUM_THREADS", "1") + env.setdefault("MKL_NUM_THREADS", "1") + env.setdefault("NUMEXPR_NUM_THREADS", "1") + env.setdefault("MALLOC_ARENA_MAX", "2") + env.setdefault("MALLOC_TRIM_THRESHOLD_", "0") + + # Log sync before the run (after pre-delete). Skipped for ngmix chunks: N + # concurrent chunks would race a full rewrite (literal INPUT_DIRs make it moot). + if not ngmix_mode: + subprocess.call(["update_runs_log_file.py"], cwd=str(work), env=env) + + print(f"[sp_rule] stage={args.stage} unit={args.unit} work={work}", + file=sys.stderr) + rc = subprocess.call(["shapepipe_run", "-c", str(cfg_mod)], + cwd=str(work), env=env) + if rc != 0: + print(f"[sp_rule] WARN shapepipe_run exit {rc} — checking count floor", + file=sys.stderr) + + if not ngmix_mode: + subprocess.call(["update_runs_log_file.py"], cwd=str(work), env=env) + + # Zero-output floor for stages without a count table; count floor otherwise. + if args.stage in COMPLETENESS: + count_floor(run_out, args.stage) + else: + produced = list(run_out.glob("**/output/*")) if run_out.exists() else [] + if not produced: + print(f"[sp_rule] FATAL {args.stage} {args.unit}: zero output in {run_out}", + file=sys.stderr) + sys.exit(1) + + # make_cat: publish the science product to its declared, protected path. + if args.final_cat is not None: + finals = sorted(run_out.glob("**/output/final_cat*.fits")) + if not finals: + print(f"[sp_rule] FATAL {args.stage} {args.unit}: no final_cat under {run_out}", + file=sys.stderr) + sys.exit(1) + args.final_cat.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(finals[0], args.final_cat) + print(f"[sp_rule] final_cat -> {args.final_cat}", file=sys.stderr) + + print(f"[sp_rule] OK {args.stage} {args.unit}", file=sys.stderr) + + +if __name__ == "__main__": + main() From 187950c4ec95035b0e1b40ffaf69841122c31f89 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 15:57:40 -0400 Subject: [PATCH 02/41] fix(orchestration): resolve {threads} at rule level, not inside params.cmd MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Snakemake formats params once, so a {threads} placeholder inside params.cmd survives literally and SMP_BATCH_SIZE never resolves (fork width silently 1, serial modules — the p3-first10 failure mode). Move '--threads {threads}' into each rule's shell: string; tile_exp_forest takes none (build_forest.py is single-threaded symlinking and has no such flag). Completeness table doc touch-ups ride along. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NA8M1oLZLbWJxwoyTNrTAi --- workflow/Snakefile | 6 ++++-- workflow/rules/exposure.smk | 8 ++++---- workflow/rules/prepare.smk | 6 +++--- workflow/rules/tile.smk | 15 ++++++++------- workflow/scripts/completeness.py | 8 ++++++-- 5 files changed, 25 insertions(+), 18 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 04722a73c..09d218318 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -60,13 +60,15 @@ def sp_rule(stage, config_name, level, unit, *, isolate=True, exp_forest=None, NUMBER_LIST is injected by default (tile-scheme stages + exp split); pass ``isolate=False`` for get_images / exp_mask / exp_psf, whose per-CCD/download - numbering must not be constrained. ``{threads}`` is filled by Snakemake and + numbering must not be constrained. ``--threads {threads}`` is appended in + each rule's ``shell:`` string (NOT here: Snakemake formats the shell string + once, so a ``{threads}`` inside params.cmd would survive literally) and becomes SMP_BATCH_SIZE (fork width == cpus_per_task). """ cmd = ( f"python {SCRIPTS}/sp_rule.py --stage {stage} --config {config_name} " f"--unit {unit} --level {level} --run-dir {RUN_DIR} " - f"--config-src {CONFIG_SRC} --star-cats {STAR_CATS} --threads {{threads}}" + f"--config-src {CONFIG_SRC} --star-cats {STAR_CATS}" ) if not isolate: cmd += " --no-isolate" diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index 50403d386..3e934b185 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -29,7 +29,7 @@ rule exp_get_images: cmd=lambda wc: sp_rule("exp_get_images", "config_exp_Gie_vos.ini", "exp", wc.exp, isolate=False) shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" # Split multi-HDU exposure into single-CCD files (+ headers-*.npy). Bulk output; # persistent (see module docstring on temp()). @@ -41,7 +41,7 @@ rule exp_split: params: cmd=lambda wc: sp_rule("exp_split", "config_exp_Sp.ini", "exp", wc.exp) shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" # Mask per-CCD; this exposure's star cat is a declared input (per-unit re-key). rule exp_mask: @@ -54,7 +54,7 @@ rule exp_mask: cmd=lambda wc: sp_rule("exp_mask", "config_exp_Ma_onthefly.ini", "exp", wc.exp, isolate=False) shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" # SExtractor -> setools star selection -> PSFEx model -> psfex_interp, per CCD. # setools may reject a sparse CCD (~0.2% attrition) — tolerated by the floor. @@ -67,4 +67,4 @@ rule exp_psf: cmd=lambda wc: sp_rule("exp_psf", "config_exp_psfex.ini", "exp", wc.exp, isolate=False) shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" diff --git a/workflow/rules/prepare.smk b/workflow/rules/prepare.smk index 2911c3b19..4ca369ec9 100644 --- a/workflow/rules/prepare.smk +++ b/workflow/rules/prepare.smk @@ -19,7 +19,7 @@ rule tile_get_images: cmd=lambda wc: sp_rule("tile_get_images", "config_tile_Git_vos.ini", "tile", wc.tile, isolate=False) shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" rule tile_uncompress: input: @@ -30,7 +30,7 @@ rule tile_uncompress: cmd=lambda wc: sp_rule("tile_uncompress", "config_tile_Uz.ini", "tile", wc.tile, isolate=True) shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" # find_exposures reads the tile FITS HISTORY header -> exp_numbers-.txt, the # data-derived tile->exposure edge build_index.py aggregates (as a plain script). @@ -43,7 +43,7 @@ rule tile_find_exposures: cmd=lambda wc: sp_rule("tile_find_exposures", "config_tile_Fe.ini", "tile", wc.tile, isolate=True) shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" # Star catalogues for masking are pre-generated offline (create_star_cat.py on a # networked login node — the one genuinely networked prepare step). They are diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 845167710..8abfdd18b 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -35,6 +35,7 @@ rule tile_exp_forest: cmd=lambda wc: (f"python {SCRIPTS}/build_forest.py --tile {wc.tile} " f"--run-dir {RUN_DIR} --index {INDEX_DB} --forest {{output}}") shell: + # no --threads: build_forest.py is single-threaded symlinking, takes no such flag "{params.cmd}" # Merge single-exposure WCS headers into the tile-level sqlite log @@ -52,7 +53,7 @@ rule tile_merge_headers: "tile", wc.tile, isolate=True, exp_forest=FOREST.format(tile=wc.tile)) shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" # Mask tiles; this tile's star cat is a declared input (per-unit re-key). rule tile_mask: @@ -65,7 +66,7 @@ rule tile_mask: cmd=lambda wc: sp_rule("tile_mask", "config_tile_Ma_onthefly.ini", "tile", wc.tile, isolate=True) shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" # SExtractor object detection on the tile. rule tile_detect: @@ -78,7 +79,7 @@ rule tile_detect: cmd=lambda wc: sp_rule("tile_detect", "config_tile_Sx.ini", "tile", wc.tile, isolate=True) shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" # PSFEx interpolation to galaxies + vignet postage stamps (reads exposure PSF # models + split through the forest — those dirs are explicit inputs). @@ -95,7 +96,7 @@ rule tile_vignets: "tile", wc.tile, isolate=True, exp_forest=FOREST.format(tile=wc.tile)) shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" # ngmix shape measurement — static N chunks. Each chunk computes its own closed # object-ID range in-job from the tile's own sexcat (a declared input); chunks @@ -111,7 +112,7 @@ rule tile_ngmix: "tile_ngmix", "config_tile_Ng_template_batch.ini", "tile", wc.tile, extra=f"--ngmix-chunk {wc.chunk} --ngmix-nchunks {NGMIX_CHUNKS}"), shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" def ngmix_chunks(wc): return [f"{RUN_DIR}/tiles/{wc.tile}/output/run_sp_tile_ngmix_Ng{k}u" @@ -129,7 +130,7 @@ rule tile_merge_cats: "tile", wc.tile, isolate=True, extra=f"--n-split-max {NGMIX_CHUNKS}"), shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" # Build the final catalogue: the run's science product. protected() so a # code/params-drift rerun cannot silently discard a finished catalogue — the @@ -144,4 +145,4 @@ rule tile_make_cat: "tile", wc.tile, isolate=True, extra=f"--final-cat {RUN_DIR}/tiles/{wc.tile}/final_cat-{wc.tile}.fits"), shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" diff --git a/workflow/scripts/completeness.py b/workflow/scripts/completeness.py index f35e81aee..d52ad8d0a 100644 --- a/workflow/scripts/completeness.py +++ b/workflow/scripts/completeness.py @@ -30,12 +30,16 @@ # stage -> {runner_subdir: {expect, floor, [warn], [subpath]}} COMPLETENESS = { # --- tile prepare (phase A) --- - "tile_get_images": {"get_images_runner": dict(expect=4, floor=4)}, + # get_images counts are CONFIG-FLAVOR-DEPENDENT: the v2.0 bash table said 4/6 + # for the canfar vos flavor; the nibi symlink configs produce one file per + # INPUT_FILE_PATTERN entry (tile: image+weight=2; exp: image+weight+flag=3), + # verified against the p3-batch1 baseline tree (100 files / 50 tiles). + "tile_get_images": {"get_images_runner": dict(expect=2, floor=2)}, "tile_uncompress": {"uncompress_fits_runner": dict(expect=1, floor=1)}, "tile_find_exposures": {"find_exposures_runner": dict(expect=1, floor=1)}, # --- exposure chain --- - "exp_get_images": {"get_images_runner": dict(expect=6, floor=6)}, + "exp_get_images": {"get_images_runner": dict(expect=3, floor=3)}, "exp_split": {"split_exp_runner": dict(expect=121, floor=41)}, "exp_mask": {"mask_runner": dict(expect=40, floor=1)}, "exp_psf": { From ac6bddd7f5eddf1c9e514a4e759a1c4db7660dff Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 16:07:59 -0400 Subject: [PATCH 03/41] fix(orchestration): carry original exposure names through the index; star-cat store naming MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three P0-blocking fixes found by running against real data: - exposures(name): the fabricated per-unit exp_numbers list must carry the original exposure name verbatim (2605805p — get_images matches .fits.fz; the bare dedup id matches nothing). The index now stores both; exp_get_images passes --exp-name from the parse-time dict. - tile_star_cat: the pre-generated store names tiles in ShapePipe's image-number convention (dots->dashes); translate when linking. - profile: PYTHONPATH pinned to this branch's src/ (identical to develop@97e16d50 — orchestration commits never touch src/); NOT shapepipe-prod (drifted to a PR branch mid-run, and the live p3-batch1 job reads it) and not the sif default (frozen pre-#843). Run scoped to the 210/211 overlap quad (19 unique exposures, 15 shared — exercises structural dedup; the zero-overlap 196 quad is kept for the append-invariant test). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NA8M1oLZLbWJxwoyTNrTAi --- profiles/nibi/config.yaml | 12 ++++++----- workflow/Snakefile | 8 +++++--- workflow/config.yaml | 9 ++++++--- workflow/rules/exposure.smk | 3 ++- workflow/rules/prepare.smk | 4 +++- workflow/scripts/build_index.py | 35 ++++++++++++++++++--------------- workflow/scripts/sp_rule.py | 17 ++++++++++++---- 7 files changed, 55 insertions(+), 33 deletions(-) diff --git a/profiles/nibi/config.yaml b/profiles/nibi/config.yaml index e977803c1..07f68833a 100644 --- a/profiles/nibi/config.yaml +++ b/profiles/nibi/config.yaml @@ -23,11 +23,13 @@ software-deployment-method: [apptainer] # OMP_NUM_THREADS=1 caps OpenBLAS fork-explosion (verified: pool 32->1) # MALLOC_ARENA_MAX=2 bounded allocator (im_sims/nibi lesson) # --home /home/cdaley wins over the SDM's --home ; restores cadcproxy.pem for vos/vcp -# PYTHONPATH SETTLED CALL 3 — P0 pins the prod worktree src so science -# content matches the bash validation runs (which use this -# override). Production later rebuilds the sif at the exact -# validation commit and DROPS this --env line. -apptainer-args: "--cleanenv --env OMP_NUM_THREADS=1 --env MALLOC_ARENA_MAX=2 --env PYTHONPATH=/project/def-mjhudson/cdaley/shapepipe-prod/src --home /home/cdaley --bind /project --bind /scratch" +# PYTHONPATH SETTLED CALL 3 — P0 pins THIS branch's src/ (identical to +# develop@97e16d50: orchestration commits never touch src/). +# NOT shapepipe-prod (drifted to a PR branch mid-run, and the +# live p3-batch1 job reads it) and not the sif default +# (frozen pre-#843). Production later rebuilds the sif at the +# validated commit and DROPS this --env line. +apptainer-args: "--cleanenv --env OMP_NUM_THREADS=1 --env MALLOC_ARENA_MAX=2 --env PYTHONPATH=/project/def-mjhudson/cdaley/shapepipe-snakemake/src --home /home/cdaley --bind /project --bind /scratch" latency-wait: 60 # NFS: wait for outputs to appear after a job keep-going: true # a failed job poisons only its cone; siblings run on diff --git a/workflow/Snakefile b/workflow/Snakefile index 09d218318..6e391bda2 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -39,11 +39,13 @@ with open(config["tile_list"]) as f: TILES = [ln.strip() for ln in f if ln.strip()] # --- parse-time index load (compute phase only) ---------------------------- -# EXP: global set of exposure base-ids. TILE_EXP: tile -> [exp_ids]. -EXP, TILE_EXP = set(), {} +# EXP: exposure base-id -> original name (2605805 -> 2605805p; the name goes +# verbatim into the fabricated per-unit exp_numbers list so get_images matches +# .fits.fz in the store). TILE_EXP: tile -> [exp_ids]. +EXP, TILE_EXP = {}, {} if INDEX_DB.exists(): _con = sqlite3.connect(INDEX_DB) - EXP = {r[0] for r in _con.execute("SELECT exp_id FROM exposures")} + EXP = dict(_con.execute("SELECT exp_id, name FROM exposures")) for tile, exp in _con.execute("SELECT tile_id, exp_id FROM tile_exposures"): TILE_EXP.setdefault(tile, []).append(exp) _con.close() diff --git a/workflow/config.yaml b/workflow/config.yaml index 53542d1c2..ec529c4dc 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -5,14 +5,17 @@ # tiles) never invalidates completed work — it only changes which jobs exist. # The tile list that scopes this run (one "IDra.IDdec" per line). -tile_list: /scratch/cdaley/shapepipe-output/snakemake-p0/tiles4.txt +# 210/211 quad: 19 unique exposures, 15 reused across tiles — exercises the +# structural exposure dedup (the 196 quad had zero overlap; kept for the +# append-invariant test). +tile_list: /scratch/cdaley/shapepipe-output/smk-p0/tiles4.txt # The container every job runs inside (apptainer software-deployment in the profile). container: /project/def-mjhudson/cdaley/containers/shapepipe-develop-runtime.sif # Where products land ($SP_RUN root). Per-unit work-dir forest is built under here: # /tiles/// and /exp/// -run_dir: /scratch/cdaley/shapepipe-output/snakemake-p0 +run_dir: /scratch/cdaley/shapepipe-output/smk-p0 # The canonical config chain (example/cfis in the repo, or a run-specific copy). config_src: /home/cdaley/projects/def-mjhudson/cdaley/runs/p3-batch1/cfis @@ -23,7 +26,7 @@ star_cats: /home/cdaley/projects/def-mjhudson/cdaley/runs/p3-batch1/star_cats # The run index. Durable, low-volume products (index, missing.json, run_report, # final_cat) should land on /project, not /scratch (60-day purge, finding 15). -index_db: /project/def-mjhudson/cdaley/shapepipe-output/snakemake-p0/index/run_index.sqlite +index_db: /project/def-mjhudson/cdaley/shapepipe-output/smk-p0/index/run_index.sqlite # ngmix within-tile chunking: static N chunks (closed ID ranges computed # per-tile, in-job, from the tile's own sexcat). diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index 3e934b185..f39f6be23 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -27,7 +27,8 @@ rule exp_get_images: directory(f"{EXP_OUT}/run_sp_exp_Gie") params: cmd=lambda wc: sp_rule("exp_get_images", "config_exp_Gie_vos.ini", - "exp", wc.exp, isolate=False) + "exp", wc.exp, isolate=False, + extra=f"--exp-name {EXP[wc.exp]}") shell: "{params.cmd} --threads {threads}" diff --git a/workflow/rules/prepare.smk b/workflow/rules/prepare.smk index 4ca369ec9..530ac9b31 100644 --- a/workflow/rules/prepare.smk +++ b/workflow/rules/prepare.smk @@ -62,6 +62,8 @@ rule tile_star_cat: output: str(RUN_DIR / "tiles/{tile}/star_cat-{tile}.fits") params: - src=lambda wc: str(STAR_CATS / "tiles" / f"star_cat-{wc.tile}.fits") + # the store names tiles in ShapePipe's image-number convention (dots->dashes) + src=lambda wc: str(STAR_CATS / "tiles" / + f"star_cat-{wc.tile.replace('.', '-')}.fits") shell: "ln -sf {params.src} {output}" diff --git a/workflow/scripts/build_index.py b/workflow/scripts/build_index.py index 13783314a..7ba865d94 100644 --- a/workflow/scripts/build_index.py +++ b/workflow/scripts/build_index.py @@ -37,19 +37,22 @@ from pathlib import Path -def read_exposure_list(exp_numbers_file: Path) -> list[str]: - """Return the exp_base IDs listed in one tile's find_exposures output. - - Each line is an exposure name like ``2243881p``; we strip a trailing - alphabetic suffix so the base ID is the dedup key. +def read_exposure_list(exp_numbers_file: Path) -> list[tuple[str, str]]: + """Return ``(exp_id, name)`` pairs from one tile's find_exposures output. + + Each line is an exposure *name* like ``2243881p``; the bare base ID (suffix + stripped) is the dedup key everywhere in the DAG, but the original name is + kept in the index — the fabricated per-unit ``exp_numbers`` list must carry + it verbatim (``get_images`` matches ``.fits.fz`` in the store; the + bare ID matches nothing). """ - ids = [] + pairs = [] for line in exp_numbers_file.read_text().splitlines(): name = line.strip() if not name: continue - ids.append(name[:-1] if name[-1].isalpha() else name) - return ids + pairs.append((name[:-1] if name[-1].isalpha() else name, name)) + return pairs def build(tile_ids: list[str], run_dir: Path, db_path: Path, @@ -70,7 +73,7 @@ def build(tile_ids: list[str], run_dir: Path, db_path: Path, DROP TABLE IF EXISTS exposures; DROP TABLE IF EXISTS tile_exposures; CREATE TABLE tiles(tile_id TEXT PRIMARY KEY, ra_dir TEXT, n_exp INTEGER); - CREATE TABLE exposures(exp_id TEXT PRIMARY KEY); + CREATE TABLE exposures(exp_id TEXT PRIMARY KEY, name TEXT NOT NULL); CREATE TABLE tile_exposures( tile_id TEXT, exp_id TEXT, PRIMARY KEY (tile_id, exp_id)); @@ -78,7 +81,7 @@ def build(tile_ids: list[str], run_dir: Path, db_path: Path, ) missing = [] - all_exposures: set[str] = set() + all_exposures: set[tuple[str, str]] = set() for tile_id in tile_ids: ra_dir = tile_id.split(".")[0] idra, iddec = tile_id.split(".") @@ -89,16 +92,16 @@ def build(tile_ids: list[str], run_dir: Path, db_path: Path, if not exp_file.exists(): missing.append(tile_id) continue - exp_ids = read_exposure_list(exp_file) + exp_pairs = read_exposure_list(exp_file) con.execute("INSERT INTO tiles VALUES (?,?,?)", - (tile_id, ra_dir, len(exp_ids))) - for exp_id in exp_ids: - all_exposures.add(exp_id) + (tile_id, ra_dir, len(exp_pairs))) + for exp_id, _name in exp_pairs: con.execute("INSERT OR IGNORE INTO tile_exposures VALUES (?,?)", (tile_id, exp_id)) + all_exposures.update(exp_pairs) - con.executemany("INSERT OR IGNORE INTO exposures VALUES (?)", - [(e,) for e in sorted(all_exposures)]) + con.executemany("INSERT OR IGNORE INTO exposures VALUES (?,?)", + sorted(all_exposures)) con.commit() con.close() diff --git a/workflow/scripts/sp_rule.py b/workflow/scripts/sp_rule.py index cce92e2c8..b114fad08 100644 --- a/workflow/scripts/sp_rule.py +++ b/workflow/scripts/sp_rule.py @@ -93,7 +93,7 @@ def unit_work_dir(run_dir: Path, level: str, unit: str) -> Path: def materialise_unit(work: Path, level: str, unit: str, config_src: Path, - star_cats: Path) -> None: + star_cats: Path, exp_name: str | None = None) -> None: """Create the v2.0 unit-isolation furniture in the work dir.""" (work / "output").mkdir(parents=True, exist_ok=True) @@ -112,8 +112,13 @@ def materialise_unit(work: Path, level: str, unit: str, config_src: Path, fe = work / "output" / "run_sp_tile_Fe" / "find_exposures_runner" / "output" fe.mkdir(parents=True, exist_ok=True) exp_numbers = fe / "exp_numbers-000-000.txt" - if not exp_numbers.exists(): - exp_numbers.write_text(unit + "\n") + # Written UNCONDITIONALLY (cheap, deterministic): an exists-guard once + # pinned a stale pre-fix file with the bare base id. Content is the + # ORIGINAL exposure name (with its 'p'-style suffix, from the index) — + # the on-disk store is .fits.fz; the bare base id matches nothing. + # v2.0 copies the name verbatim from the tile's Fe output; the index + # carries it for us. + exp_numbers.write_text((exp_name or unit) + "\n") def normalize_config_text(text: str, unit: str, isolate: bool, threads: int) -> str: @@ -296,6 +301,10 @@ def main() -> None: help="SMP_BATCH_SIZE == cpus_per_task (fork width)") p.add_argument("--exp-forest", type=Path, default=None, help="$SP_EXP: per-tile exposure forest (tile post-stages)") + p.add_argument("--exp-name", default=None, + help="original exposure name incl. suffix (e.g. 2605805p); " + "written into the fabricated exp_numbers list so " + "get_images finds .fits.fz in the store") p.add_argument("--no-isolate", action="store_true", help="skip NUMBER_LIST (get_images / exp_mask / exp_psf)") p.add_argument("--ngmix-chunk", type=int, default=None, @@ -314,7 +323,7 @@ def main() -> None: work = unit_work_dir(args.run_dir, args.level, args.unit) materialise_unit(work, args.level, args.unit, args.config_src, - args.star_cats or args.run_dir) + args.star_cats or args.run_dir, exp_name=args.exp_name) # Build the config copy. ngmix chunks and merge fill template placeholders # first; every copy is then normalized (RUN_DATETIME=False, SMP_BATCH_SIZE, From 9b0456e72438ab7f6f10e304d231100c0e3423ae Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 16:09:45 -0400 Subject: [PATCH 04/41] fix(orchestration): exposure-name pass-through, star-cat dirs, P0 run config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - get_images looks up .fits.fz: the store keeps the CFHT 'p' suffix the index's base-id key strips. exposures gains a name column; rules pass --exp-name; the wrapper writes the ORIGINAL name into the fabricated exp_numbers list — and writes it unconditionally (an exists-guard pinned a stale pre-fix file). - Star cats are consumed as DIRECTORIES via the wrapper's $SP_RUN symlinks (the v2.0 mechanism; exp cats are per-CCD, 40/exposure) — dropped the per-unit star-cat file rules that linked nonexistent names nothing read. - get_images completeness counts follow the nibi symlink configs (tile 2, exp 3; the v2.0 4/6 were the canfar vos flavor), verified against p3-batch1. - P0 run config: 210/211 quad (19 unique exps, 13 shared across tiles — exercises structural dedup; the 196 quad had zero overlap), PYTHONPATH pinned to this branch's src (= develop@97e16d50; shapepipe-prod drifted mid-run). Co-Authored-By: Claude Fable 5 --- workflow/Snakefile | 4 +--- workflow/rules/prepare.smk | 31 ++++++++++--------------------- workflow/rules/tile.smk | 3 +-- 3 files changed, 12 insertions(+), 26 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 6e391bda2..774ae3ad2 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -109,9 +109,7 @@ rule prepare_tiles: # index exists. Star cats are re-keyed per unit (see prepare.smk). rule prepare_exposures: input: - expand(str(RUN_DIR / "exp/{exp}/output/run_sp_exp_Gie"), exp=sorted(EXP)), - expand(str(RUN_DIR / "exp/{exp}/star_cat-{exp}.fits"), exp=sorted(EXP)), - expand(str(RUN_DIR / "tiles/{tile}/star_cat-{tile}.fits"), tile=TILES) + expand(str(RUN_DIR / "exp/{exp}/output/run_sp_exp_Gie"), exp=sorted(EXP)) # --- report hooks ----------------------------------------------------------- # run_report is NOT a DAG node (a descendant of every job would be poisoned by diff --git a/workflow/rules/prepare.smk b/workflow/rules/prepare.smk index 530ac9b31..d8698c826 100644 --- a/workflow/rules/prepare.smk +++ b/workflow/rules/prepare.smk @@ -46,24 +46,13 @@ rule tile_find_exposures: "{params.cmd} --threads {threads}" # Star catalogues for masking are pre-generated offline (create_star_cat.py on a -# networked login node — the one genuinely networked prepare step). They are -# re-keyed PER UNIT here: each tile/exposure's cat is its own DAG node, so -# appending tile 51 adds nodes rather than mutating a run-level output that would -# dirty every finished mask (the growth-invariant fix, finding 11). -rule exp_star_cat: - output: - str(RUN_DIR / "exp/{exp}/star_cat-{exp}.fits") - params: - src=lambda wc: str(STAR_CATS / "exp" / f"star_cat-{wc.exp}.fits") - shell: - "ln -sf {params.src} {output}" - -rule tile_star_cat: - output: - str(RUN_DIR / "tiles/{tile}/star_cat-{tile}.fits") - params: - # the store names tiles in ShapePipe's image-number convention (dots->dashes) - src=lambda wc: str(STAR_CATS / "tiles" / - f"star_cat-{wc.tile.replace('.', '-')}.fits") - shell: - "ln -sf {params.src} {output}" +# networked login node) and consumed as DIRECTORIES: the mask configs read +# INPUT_DIR $SP_RUN/star_cat_{exp,tiles} with per-CCD numbering (exp cats are +# star_cat--.fits, 40/exposure; tile cats star_cat--.fits) +# — the v2.0 mechanism. The wrapper materialises the two dir symlinks in every +# unit work dir (sp_rule.materialise_unit); there is NO per-unit star-cat DAG +# node: the store is pre-run input like the image store, and a missing cat fails +# the mask stage's count-floor loudly. (Earlier per-unit file rules linked names +# that don't exist — one file per exposure vs the store's 40 — and nothing read +# them: the configs only see the dir symlinks.) When star-cat GENERATION moves +# in-workflow (P2), it becomes real per-unit rules producing into the store. diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 8abfdd18b..1bf7d4244 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -55,11 +55,10 @@ rule tile_merge_headers: shell: "{params.cmd} --threads {threads}" -# Mask tiles; this tile's star cat is a declared input (per-unit re-key). +# Mask tiles (star cats: pre-run store, read via the wrapper's dir symlink). rule tile_mask: input: git=rules.tile_get_images.output, - star=str(RUN_DIR / "tiles/{tile}/star_cat-{tile}.fits"), output: directory(f"{TILE_OUT}/run_sp_tile_Ma") params: From 6fe141fc01221eb06b3016a883999195369282fe Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 16:35:07 -0400 Subject: [PATCH 05/41] fix(orchestration): {output} placeholder inside params.cmd survives literally MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same single-format trap as {threads} (187950c4): Snakemake formats the shell string once, so the escaped {{output}} in tile_exp_forest's params reached build_forest as a literal './{output}' — all four forest jobs racing one garbage dir (FileExistsError at 50% of the P0 run). Move --forest {output} into the shell string. Swept the rules for further escaped placeholders: none. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ac/code/session_01NA8M1oLZLbWJxwoyTNrTAi --- workflow/rules/tile.smk | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 1bf7d4244..d36e91d95 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -32,11 +32,14 @@ rule tile_exp_forest: output: directory(FOREST) params: + # --forest {output} lives in the shell string: Snakemake formats shell + # ONCE — an {output} placeholder inside params.cmd survives literally + # (same trap as {threads}; all four forest jobs then race one './{output}'). cmd=lambda wc: (f"python {SCRIPTS}/build_forest.py --tile {wc.tile} " - f"--run-dir {RUN_DIR} --index {INDEX_DB} --forest {{output}}") + f"--run-dir {RUN_DIR} --index {INDEX_DB}") shell: - # no --threads: build_forest.py is single-threaded symlinking, takes no such flag - "{params.cmd}" + # no --threads: build_forest.py is single-threaded symlinking + "{params.cmd} --forest {output}" # Merge single-exposure WCS headers into the tile-level sqlite log # (log_exp_headers--.sqlite — the tile suffix downstream Sx/PiViVi/Ng From a110d89faeca7f0ac6bb2bf6a0ac79862ea044fe Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 16:42:12 -0400 Subject: [PATCH 06/41] =?UTF-8?q?fix(orchestration):=20shard=20the=20expos?= =?UTF-8?q?ure=20forest=20=E2=80=94=20the=20module=20glob's=20shape=20is?= =?UTF-8?q?=20not=20cosmetic?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit exp_utils.get_exp_output_files hardwires the v2.0 sharded layout into its $SP_EXP glob (///output/run_sp_*/...); a flat forest fails every tile gather stage with 'No split_exp_runner output found'. The exposure STORE stays flat (nothing globs it — the index drives every path); only the module-facing forest view carries the 2-digit shard. Verified by hand-running tile_merge_headers on 210.296 (1/1 OK). Co-Authored-By: Claude Fable 5 --- workflow/scripts/build_forest.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/workflow/scripts/build_forest.py b/workflow/scripts/build_forest.py index 82df8b0c6..86dc4354e 100644 --- a/workflow/scripts/build_forest.py +++ b/workflow/scripts/build_forest.py @@ -3,10 +3,14 @@ A plain script (not a run: block) so the tile chain stays group-compatible. Reads the tile's exposures from run_index.sqlite and symlinks each exposure's -``exp//output`` into ``//output`` by exact name (no glob), so -the tile modules' $SP_EXP globs resolve to exactly this tile's exposures' -products. The forest is a convenience view; the DAG edge to the exposures is -declared in the rule's input (tile.smk), not here. +``exp//output`` into ``///output`` by exact name (no +glob). The 2-digit ```` shard level is NOT cosmetic: ShapePipe's +``exp_utils.get_exp_output_files`` hardwires the sharded v2.0 layout into its +$SP_EXP glob (``///output/run_sp_*/...``), so a flat +forest makes every tile gather stage fail "No split_exp_runner output found". +(The exposure STORE stays flat — only the module-facing view needs the shard.) +The forest is a convenience view; the DAG edge to the exposures is declared in +the rule's input (tile.smk), not here. """ import argparse @@ -30,7 +34,7 @@ def main() -> None: args.forest.mkdir(parents=True, exist_ok=True) for e in exps: src = args.run_dir / "exp" / e / "output" - dst = args.forest / e / "output" + dst = args.forest / e[:2] / e / "output" # sharded: the module glob's shape dst.parent.mkdir(parents=True, exist_ok=True) if dst.is_symlink() or dst.exists(): dst.unlink() From 28c10d8cdca5947184fb5a2d2f831e84468b2c95 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 16:47:15 -0400 Subject: [PATCH 07/41] fix(orchestration): carry a wrapper-scripts content hash into every rule's params MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The default rerun-triggers' code trigger hashes only the rule's own shell string — NOT external scripts it calls. Bitten live: the forest-shard fix changed build_forest.py but no rule text, so stale flat forests survived a rerun and tile_merge_headers kept failing. SCRIPT_HASH (md5 of sp_rule.py + completeness.py + build_forest.py, computed at parse time) rides into params via an accept-and-ignore --script-hash flag, so script edits propagate exactly like code edits under the full default triggers. One-time cost: every rule's params change once, re-running completed work — the same tradeoff as any code change under full triggers, and the P0-scale rerun doubles as a live test of rerun semantics. Co-Authored-By: Claude Fable 5 --- workflow/Snakefile | 14 +++++++++++++- workflow/rules/tile.smk | 3 ++- workflow/scripts/build_forest.py | 5 +++++ workflow/scripts/sp_rule.py | 5 +++++ 4 files changed, 25 insertions(+), 2 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 774ae3ad2..3c869482c 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -70,7 +70,8 @@ def sp_rule(stage, config_name, level, unit, *, isolate=True, exp_forest=None, cmd = ( f"python {SCRIPTS}/sp_rule.py --stage {stage} --config {config_name} " f"--unit {unit} --level {level} --run-dir {RUN_DIR} " - f"--config-src {CONFIG_SRC} --star-cats {STAR_CATS}" + f"--config-src {CONFIG_SRC} --star-cats {STAR_CATS} " + f"--script-hash {SCRIPT_HASH}" ) if not isolate: cmd += " --no-isolate" @@ -86,6 +87,17 @@ def tile_dir(tile): def exp_dir(exp): return RUN_DIR / "exp" / exp +# Content hash of the wrapper scripts, computed once at parse time. Exposed as a +# param on every rule (via sp_rule() and tile_exp_forest) so the default +# rerun-triggers' *params* trigger covers script edits: the `code` trigger only +# hashes the rule's own shell string, NOT external scripts it calls — without +# this, a wrapper/forest-script fix silently leaves stale outputs in place +# (bitten live: the forest-shard fix rebuilt nothing on rerun). +import hashlib as _hashlib +SCRIPT_HASH = _hashlib.md5(b"".join( + (SCRIPTS / n).read_bytes() + for n in ("sp_rule.py", "completeness.py", "build_forest.py"))).hexdigest()[:12] + include: "rules/prepare.smk" include: "rules/exposure.smk" include: "rules/tile.smk" diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index d36e91d95..04cc4c6c9 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -36,7 +36,8 @@ rule tile_exp_forest: # ONCE — an {output} placeholder inside params.cmd survives literally # (same trap as {threads}; all four forest jobs then race one './{output}'). cmd=lambda wc: (f"python {SCRIPTS}/build_forest.py --tile {wc.tile} " - f"--run-dir {RUN_DIR} --index {INDEX_DB}") + f"--run-dir {RUN_DIR} --index {INDEX_DB} " + f"--script-hash {SCRIPT_HASH}") shell: # no --threads: build_forest.py is single-threaded symlinking "{params.cmd} --forest {output}" diff --git a/workflow/scripts/build_forest.py b/workflow/scripts/build_forest.py index 86dc4354e..e945820e0 100644 --- a/workflow/scripts/build_forest.py +++ b/workflow/scripts/build_forest.py @@ -24,6 +24,11 @@ def main() -> None: p.add_argument("--run-dir", required=True, type=Path) p.add_argument("--index", required=True, type=Path) p.add_argument("--forest", required=True, type=Path) + p.add_argument("--script-hash", default=None, + help="ignored at runtime: carries the wrapper-scripts content " + "hash into the rule's params so the params rerun-trigger " + "covers script edits (the code trigger only hashes the " + "rule's own shell string)") args = p.parse_args() con = sqlite3.connect(args.index) diff --git a/workflow/scripts/sp_rule.py b/workflow/scripts/sp_rule.py index b114fad08..8ee71c693 100644 --- a/workflow/scripts/sp_rule.py +++ b/workflow/scripts/sp_rule.py @@ -315,6 +315,11 @@ def main() -> None: help="N_SPLIT_MAX for the merge_sep_cats template") p.add_argument("--final-cat", type=Path, default=None, help="copy the produced final_cat FITS here (make_cat)") + p.add_argument("--script-hash", default=None, + help="ignored at runtime: carries the wrapper-scripts content " + "hash into the rule's params so the params rerun-trigger " + "covers script edits (the code trigger only hashes the " + "rule's own shell string)") args = p.parse_args() ngmix_mode = args.ngmix_chunk is not None From a273e76369119113d05251c3c8b33beb3e1ec654 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 19:59:57 -0400 Subject: [PATCH 08/41] fix(orchestration): keep snakemake state and index off /project (hard-full quota) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit def-mjhudson /project sits at its 27/27 TiB ceiling: a .snakemake metadata write died mid-run (killing an otherwise-green invocation after all 8 ngmix chunks of the first tile passed), and by end of night even git and file edits on /project failed. bin/sp now runs snakemake with --directory -state on /scratch (state only — data paths are absolute), the Snakefile resolves its configfile relative to itself so --directory can't break it, and the run index lives with the run on /scratch until project space is reclaimed (finding 15's /project placement stays the design intent; noted in config). Co-Authored-By: Claude Fable 5 --- workflow/Snakefile | 5 ++++- workflow/bin/sp | 7 ++++++- workflow/config.yaml | 8 +++++--- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 3c869482c..5d93a1810 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -21,7 +21,10 @@ DAG that produces it). import sqlite3 from pathlib import Path -configfile: "workflow/config.yaml" +# Resolved relative to THIS file, not the working directory: snakemake runs with +# --directory on /scratch (bin/sp) so .snakemake/ state never lands on /project +# (group quota is a hard 27/27 TiB — a metadata write mid-run died on it live). +configfile: str(Path(workflow.snakefile).parent / "config.yaml") # Every job's shell runs inside this container (apptainer software-deployment in # the profile); the user never types apptainer. diff --git a/workflow/bin/sp b/workflow/bin/sp index f226aa441..d1f105548 100755 --- a/workflow/bin/sp +++ b/workflow/bin/sp @@ -34,9 +34,14 @@ source "$VENV/bin/activate" # Minimal scalar reader for workflow/config.yaml (key: value, no nesting). cfg() { sed -n "s/^$1:[[:space:]]*//p" "$CONFIG" | head -1; } RUN_DIR="$(cfg run_dir)"; INDEX_DB="$(cfg index_db)" +# Snakemake state (.snakemake: metadata, locks, incomplete markers) lives NEXT TO +# THE RUN on /scratch, never on /project (hard 27/27 TiB group quota killed a +# metadata write mid-run live). --directory only moves state: all data paths are +# absolute, and the Snakefile resolves its own configfile. +STATE_DIR="${SP_STATE_DIR:-${RUN_DIR}-state}"; mkdir -p "$STATE_DIR" TILE_LIST="$(cfg tile_list)"; MISSING_THRESH="${SP_MISSING_THRESHOLD:-0.0}" -sm() { snakemake --profile "$PROFILE" "$@"; } +sm() { snakemake --profile "$PROFILE" --directory "$STATE_DIR" "$@"; } build_index() { python "$SCRIPTS/build_index.py" --tile-list "$TILE_LIST" \ --run-dir "$RUN_DIR" --db "$INDEX_DB" --missing-threshold "$MISSING_THRESH" diff --git a/workflow/config.yaml b/workflow/config.yaml index ec529c4dc..f92cbb69f 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -24,9 +24,11 @@ config_src: /home/cdaley/projects/def-mjhudson/cdaley/runs/p3-batch1/cfis # Star catalogues for masking, pre-generated (network step done in prepare): star_cats: /home/cdaley/projects/def-mjhudson/cdaley/runs/p3-batch1/star_cats -# The run index. Durable, low-volume products (index, missing.json, run_report, -# final_cat) should land on /project, not /scratch (60-day purge, finding 15). -index_db: /project/def-mjhudson/cdaley/shapepipe-output/smk-p0/index/run_index.sqlite +# The run index. Design intent (finding 15) is durable products on /project — +# BLOCKED for now: def-mjhudson /project is hard-full (27/27 TiB), so index and +# report live with the run on /scratch until space is reclaimed. Mind the +# 60-day purge for anything that must survive. +index_db: /scratch/cdaley/shapepipe-output/smk-p0/index/run_index.sqlite # ngmix within-tile chunking: static N chunks (closed ID ranges computed # per-tile, in-job, from the tile's own sexcat). From 0010def6c3928185b3c84334983d6625538fd749 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 17 Jul 2026 01:45:14 -0400 Subject: [PATCH 09/41] fix(orchestration): drop exp_mask's dead per-unit star-cat input MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-unit star_cat file rules were removed (the store is per-CCD and the mask config reads it as a dir symlink), but exp_mask kept declaring exp/{exp}/star_cat-{exp}.fits as input. No producer exists, so the main DAG became unbuildable the moment the tile list grew past the prepared set — MissingInputException on every new exposure. Dormant for P0 (all 19 exposures pre-staged); caught live by the append-invariant test (196 quad appended: DAG build died before scheduling anything). With the fix the append dry-run schedules exactly the appended cone (185 jobs: 30 new exposures x4 stages once each, 4 tile chains) and nothing for the finished tiles; restored-state dry-run returns to 'Nothing to be done'. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0188niYp9MmhLmZou1KXWbye --- workflow/rules/exposure.smk | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index f39f6be23..82cdb4cd4 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -44,11 +44,14 @@ rule exp_split: shell: "{params.cmd} --threads {threads}" -# Mask per-CCD; this exposure's star cat is a declared input (per-unit re-key). +# Mask per-CCD. Star cats are NOT a per-unit input: the store is per-CCD +# (star_cat--.fits) and the mask config reads it as a DIR symlink +# (see prepare.smk) — a per-unit file input here has no producer and makes the +# DAG unbuildable the moment a new exposure is appended (caught live by the +# append-invariant test). A missing cat fails the mask count-floor loudly. rule exp_mask: input: rules.exp_split.output, - star=str(RUN_DIR / "exp/{exp}/star_cat-{exp}.fits"), output: directory(f"{EXP_OUT}/run_sp_exp_Ma") params: From 833aa9369232d37f0ccfeb35fecad1ef223fc00c Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 30 Jul 2026 21:50:51 -0400 Subject: [PATCH 10/41] fix(orchestration): nibi-flavor completeness counts; file-level attrition reporting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sextractor writes 3 files/CCD on nibi (120/exp, not v2.0's 80); vignetmaker run_2 writes 5 sqlites/tile (not canfar's 4) — both verified against the P0 tree and the bash baseline. run_report now aggregates found-vs-expected at file granularity per runner, so warn-runner attrition (psfex_interp) is reported instead of hidden behind unit-level completeness. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NRBKnHV6oUXEqRpsjqJpex --- workflow/scripts/completeness.py | 10 +++++++-- workflow/scripts/run_report.py | 36 ++++++++++++++++++++++++++++++-- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/workflow/scripts/completeness.py b/workflow/scripts/completeness.py index d52ad8d0a..b2246abef 100644 --- a/workflow/scripts/completeness.py +++ b/workflow/scripts/completeness.py @@ -42,8 +42,11 @@ "exp_get_images": {"get_images_runner": dict(expect=3, floor=3)}, "exp_split": {"split_exp_runner": dict(expect=121, floor=41)}, "exp_mask": {"mask_runner": dict(expect=40, floor=1)}, + # sextractor expect is nibi-flavor: 3 files/CCD (sexcat + background + + # background_rms; v2.0's 80 assumed 2/CCD), verified against the P0 tree + # AND the bash baseline (both 120/exposure). "exp_psf": { - "sextractor_runner": dict(expect=80, floor=2), + "sextractor_runner": dict(expect=120, floor=2), "setools_runner": dict(expect=80, floor=2, subpath="rand_split"), "psfex_runner": dict(expect=80, floor=2), "psfex_interp_runner": dict(expect=40, floor=0, warn=True), @@ -56,7 +59,10 @@ "tile_vignets": { "psfex_interp_runner": dict(expect=1, floor=1), "vignetmaker_runner_run_1": dict(expect=1, floor=1), - "vignetmaker_runner_run_2": dict(expect=4, floor=4), + # 5 sqlites/tile on nibi (image/weight/flag/background/background_rms); + # v2.0's 4 was the canfar flavor. floor follows the tile-post pattern + # (expect=floor: all-or-nothing, every vignette feeds ngmix). + "vignetmaker_runner_run_2": dict(expect=5, floor=5), }, "tile_ngmix": {"ngmix_runner": dict(expect=1, floor=1)}, "tile_merge_cats": {"merge_sep_cats_runner": dict(expect=1, floor=1)}, diff --git a/workflow/scripts/run_report.py b/workflow/scripts/run_report.py index bb3300405..683433ecb 100644 --- a/workflow/scripts/run_report.py +++ b/workflow/scripts/run_report.py @@ -12,6 +12,14 @@ It distinguishes whole-exposure absence (all runners missing — a real gap or a deletion/forest bug) from tolerated per-CCD attrition (counts between floor and expect), so a future deletion bug cannot be silently absorbed (finding 5). + +Attrition is counted at file granularity, not unit granularity: each stage's +``products`` block aggregates found-vs-expected file counts per runner across +all passing units, with a per-unit shortfall map where files are missing. +``warn`` runners (psfex_interp) are exempt from *failing* a unit, not from +being *reported* — the P0 validation found 46/760 psfex_interp CCDs missing +(6%) behind a unit-level "zero attrition" claim, which is the failure mode +this granularity exists to prevent. """ import argparse @@ -52,7 +60,9 @@ def classify(stage, run_dir): if mand and all(n == 0 for _, n, *_ in mand): return "absent", details return "under", details - if any(n < expect for _, n, floor, expect, warn in details if not warn): + # warn runners are exempt from FAILING a unit (check_floor), not from + # attrition reporting — psfex_interp is precisely the runner that attrits. + if any(n < expect for _, n, _, expect, _ in details): return "attrition", details return "complete", details @@ -83,13 +93,28 @@ def main() -> None: units = tiles if level == "tile" else exps sub = "tiles" if level == "tile" else "exp" tally = {"complete": 0, "attrition": 0, "under": [], "absent": []} + # File-level aggregate per runner over PASSING units (complete/attrition); + # under/absent units are enumerated by name above, not folded in here — + # mixing them would double-count whole-unit absence as attrition. + products = {r: {"found": 0, "expect": 0, "by_unit": {}} + for r in COMPLETENESS[stage]} for u in units: run_dir = args.run_dir / sub / u / "output" / prefix - verdict, _ = classify(stage, run_dir) + verdict, details = classify(stage, run_dir) if verdict in ("complete", "attrition"): tally[verdict] += 1 + for runner, n, _, expect, _ in details: + agg = products[runner] + agg["found"] += n + agg["expect"] += expect + if n < expect: + agg["by_unit"][u] = expect - n else: tally[verdict].append(u) + for agg in products.values(): + if not agg["by_unit"]: + del agg["by_unit"] + tally["products"] = products report["stages"][stage] = tally finals = [t for t in tiles @@ -101,6 +126,13 @@ def main() -> None: out.write_text(json.dumps(report, indent=2)) print(f"[run_report] {report['final_cats']['present']}/{len(tiles)} final " f"cats -> {out}", file=sys.stderr) + for stage, tally in report["stages"].items(): + for runner, agg in tally["products"].items(): + if agg["found"] < agg["expect"]: + pct = 100 * (agg["expect"] - agg["found"]) / agg["expect"] + print(f"[run_report] attrition {stage}/{runner}: " + f"{agg['found']}/{agg['expect']} files (-{pct:.1f}%) " + f"across {len(agg['by_unit'])} units", file=sys.stderr) if __name__ == "__main__": From 20cc50630b0a9c3cfca922a1e136b19ffd89fb34 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 30 Jul 2026 22:02:25 -0400 Subject: [PATCH 11/41] feat(ngmix): env-expanded chunk fields; position-seeded RNG becomes the only mode The three module touches named in #848: - ID_OBJ_MIN/MAX read via getexpanded, so the committed ngmix template can interpolate $NGMIX_ID_MIN/$NGMIX_ID_MAX per chunk. Unset vars fail loudly. - SEED_FROM_POSITION is retired as an option: each object's RNG derives from its own (ra, dec, ccd), so results are bit-identical under any chunking. Tile-seed path, Ngmix.get_prior, and self._rng removed; a config that still sets SEED_FROM_POSITION=False raises with a clear message. - RUN_NAME read via getexpanded, so per-chunk run dirs can come from $SP_NGMIX_CHUNK in the committed template. Tests: tests/module/test_ngmix.py 24 passed / 1 pre-existing failure (azgauss needs ngmix>=2.4.1, absent from the runtime image; fails on clean tree too); test_ngmix_weight_validation.py 14 passed. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NRBKnHV6oUXEqRpsjqJpex --- .../modules/ngmix_package/__init__.py | 16 +++- src/shapepipe/modules/ngmix_package/ngmix.py | 73 ++++++------------- src/shapepipe/modules/ngmix_runner.py | 32 ++++---- src/shapepipe/run.py | 2 +- 4 files changed, 58 insertions(+), 65 deletions(-) diff --git a/src/shapepipe/modules/ngmix_package/__init__.py b/src/shapepipe/modules/ngmix_package/__init__.py index 230606bba..fca6ee0bd 100644 --- a/src/shapepipe/modules/ngmix_package/__init__.py +++ b/src/shapepipe/modules/ngmix_package/__init__.py @@ -40,15 +40,27 @@ (no batch saving) ID_OBJ_MIN : int ID of first galaxy object to be processed; not used if set to ``-1`` - (default) + (default). Environment variables are expanded, so an orchestrator can + set the object range per chunk, for example + ``ID_OBJ_MIN = $SP_NGMIX_ID_OBJ_MIN``. ID_OBJ_MAX : int ID of last galaxy object to be processed; not used if set to ``-1`` - (default) + (default). Environment variables are expanded, as for ``ID_OBJ_MIN``. BKG_RMS_VIGNET_PATH : str, optional Path to a ``background_rms_vignet*.sqlite`` file produced by ``vignetmaker_runner``. The string may contain ``{file_number_string}``, which is replaced by the current tile ID. +Random number generation +======================== + +Each object gets its own random number stream, seeded from its sky position +and CCD (``position_seed``, ngmix#796). The output is therefore identical +whether a tile is processed in one go or split into object chunks with +``ID_OBJ_MIN``/``ID_OBJ_MAX``. The older tile-seeded mode is retired; the +config option ``SEED_FROM_POSITION`` is obsolete. Setting it to ``False`` +raises an error, so a stale config cannot silently change the RNG. + """ __all__ = ["ngmix"] diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index d4f64a74e..fc9779c6a 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -335,13 +335,12 @@ class Ngmix(object): (robust for galaxies); ``"wcs"`` uses the catalog sky position projected through the WCS (better for stars, whose HSM moments are noisy). See :func:`make_ngmix_observation`. - seed_from_position : bool, optional - If ``True``, replace the tile-level RNG with a per-object RNG seeded - from the object's sky position (:func:`position_seed`) inside the - object loop, so metacal's ``fixnoise`` counter-noise and the fit - guesses cancel across Pujol image-simulation branches (ngmix#796). The - default ``False`` leaves the production path byte-identical. See - :func:`position_seed` for the physics and the seed construction. + Notes + ----- + The RNG is always per object and seeded from that object's sky position + (:func:`position_seed`). Results therefore do not depend on how the tile is + split into object chunks, and metacal's ``fixnoise`` counter-noise and the + fit guesses cancel across Pujol image-simulation branches (ngmix#796). Raises ------ @@ -364,7 +363,6 @@ def __init__( id_obj_max=-1, bkg_sub=True, centroid_source="hsm", - seed_from_position=False, metacal_psf="fitgauss", ): @@ -418,20 +416,14 @@ def __init__( self._id_obj_max = id_obj_max self._bkg_sub = bkg_sub self._centroid_source = centroid_source - self._seed_from_position = seed_from_position self._metacal_psf = metacal_psf self._w_log = w_log - # Initiatlise random generator - seed = int(''.join(re.findall(r'\d+', self._file_number_string))) - self._rng = np.random.RandomState(seed) - self._w_log.info(f'Random generator initialisation seed = {seed}') - if self._seed_from_position: - self._w_log.info( - 'SEED_FROM_POSITION on: per-object RNG seeded from sky position' - ' for Pujol noise cancellation (image sims, ngmix#796)' - ) + self._w_log.info( + 'Per-object RNG seeded from sky position (ngmix#796): results are' + ' invariant to how the tile is split into object chunks' + ) @classmethod def MegaCamFlip(self, vign, ccd_nb): @@ -461,18 +453,6 @@ def MegaCamFlip(self, vign, ccd_nb): # swap y axis so origin is on bottom-left return vign - def get_prior(self, T_range=None, F_range=None): - """Get Prior. - - Returns - ------- - ngmix.joint_prior.PriorSimpleSep - """ - return get_prior( - self._pixel_scale, self._rng, - T_range=T_range, F_range=F_range, - ) - def compile_results(self, results): """Compile Results. @@ -809,7 +789,6 @@ def process(self): vignet_cat = self._vignet_cat final_res = [] - prior = self.get_prior() count = 0 n_empty_cat = 0 @@ -843,24 +822,20 @@ def process(self): n_no_epoch += 1 continue - # Position-seeded per-object RNG for Pujol noise cancellation in - # image sims (ngmix#796): the same object gets the same fixnoise - # counter-noise and fit guesses in every shear branch, so both - # cancel in the branch difference. The prior is rebuilt from the - # same per-object RNG because the guesser draws its initial guess - # via prior.sample() (ngmix guessers.py), which consumes the RNG the - # prior was CONSTRUCTED with — so a per-object rng alone would leave - # the guess drawing from the shared tile stream and break - # cancellation. Off in production, where the single tile-level - # self._rng and the tile-level prior carry the whole loop. - if self._seed_from_position: - obj_rng = np.random.RandomState( - position_seed(stamp.ra[0], stamp.dec[0], stamp.ccd) - ) - obj_prior = get_prior(self._pixel_scale, obj_rng) - else: - obj_rng = self._rng - obj_prior = prior + # Position-seeded per-object RNG (ngmix#796). Each object draws from + # a stream fixed by its own (ra, dec, ccd), so the result is + # independent of which chunk the object lands in and of detection + # order, and the same object gets the same fixnoise counter-noise + # and fit guesses in every Pujol shear branch, so both cancel in the + # branch difference. The prior is rebuilt from the same per-object + # RNG because the guesser draws its initial guess via prior.sample() + # (ngmix guessers.py), which consumes the RNG the prior was + # CONSTRUCTED with — a per-object rng alone would leave the guess + # drawing from a shared stream and break both properties. + obj_rng = np.random.RandomState( + position_seed(stamp.ra[0], stamp.dec[0], stamp.ccd) + ) + obj_prior = get_prior(self._pixel_scale, obj_rng) try: flux_guess = ( diff --git a/src/shapepipe/modules/ngmix_runner.py b/src/shapepipe/modules/ngmix_runner.py index 3d98356c4..fcb50be9f 100644 --- a/src/shapepipe/modules/ngmix_runner.py +++ b/src/shapepipe/modules/ngmix_runner.py @@ -86,9 +86,12 @@ def ngmix_runner( # No batch saving save_batch = -1 - # First and last galaxy ID to process - id_obj_min = config.getint(module_config_sec, "ID_OBJ_MIN") - id_obj_max = config.getint(module_config_sec, "ID_OBJ_MAX") + # First and last galaxy ID to process. Read via ``getexpanded`` so an + # orchestrator can drive the chunk bounds from environment variables + # (``$SP_NGMIX_ID_OBJ_MIN`` and friends); ``getexpanded`` is the only + # accessor in ShapePipe's config that expands ``$VAR``. + id_obj_min = int(config.getexpanded(module_config_sec, "ID_OBJ_MIN")) + id_obj_max = int(config.getexpanded(module_config_sec, "ID_OBJ_MAX")) # Centroid source for the galaxy Jacobian origin: "wcs" (default -- the # catalog sky position projected through the WCS, trusting the astrometry) @@ -99,16 +102,20 @@ def ngmix_runner( else: centroid_source = "wcs" - # Seed the per-object RNG from sky position instead of per tile, so - # metacal's fixnoise counter-noise (and the fit guesses) cancel across - # Pujol image-simulation shear branches (ngmix#796). Default False leaves - # the production path byte-identical. + # Position-seeded RNG is the only mode: every object's RNG comes from its + # own (ra, dec, ccd), so results do not depend on how the tile is split + # into chunks, and metacal's fixnoise counter-noise cancels across Pujol + # image-simulation shear branches (ngmix#796). The retired tile-seed mode + # had neither property. Old configs that disable it must fail loudly. if config.has_option(module_config_sec, "SEED_FROM_POSITION"): - seed_from_position = config.getboolean( - module_config_sec, "SEED_FROM_POSITION" - ) - else: - seed_from_position = False + if not config.getboolean(module_config_sec, "SEED_FROM_POSITION"): + raise ValueError( + "SEED_FROM_POSITION = False is no longer supported: the" + " tile-seeded RNG mode has been retired because it makes" + " results depend on the object chunking. Remove the" + " SEED_FROM_POSITION entry from the ngmix config section" + " (position-seeded RNG is now the only mode)." + ) # Check PSF vignets first: if all are empty dicts {}, the exposures for this # tile are absent from the PSF dictionary and no shape measurement is possible. @@ -161,7 +168,6 @@ def ngmix_runner( id_obj_max=id_obj_max, bkg_sub=bkg_sub, centroid_source=centroid_source, - seed_from_position=seed_from_position, metacal_psf=metacal_psf, ) diff --git a/src/shapepipe/run.py b/src/shapepipe/run.py index 962953ede..fee0dad3e 100644 --- a/src/shapepipe/run.py +++ b/src/shapepipe/run.py @@ -84,7 +84,7 @@ def _set_run_name(self): Set the name of the current pipeline run. """ - self._run_name = self.config.get("DEFAULT", "RUN_NAME") + self._run_name = self.config.getexpanded("DEFAULT", "RUN_NAME") if self.config.getboolean("DEFAULT", "RUN_DATETIME"): self._run_name += datetime.now().strftime("_%Y-%m-%d_%H-%M-%S") From d1c0afb01edbb3d4eee59bb76c38c73e8c39315a Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 30 Jul 2026 22:06:58 -0400 Subject: [PATCH 12/41] =?UTF-8?q?feat(orchestration):=20committed=20workfl?= =?UTF-8?q?ow=20configs=20=E2=80=94=20no=20wrapper,=20no=20runtime=20rewri?= =?UTF-8?q?ting=20(#848=20D2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New workflow/config/cfis/ (example/cfis + p3-batch1 deltas, rewritten; example/cfis untouched for the bash flow): - RUN_DATETIME=False committed; every 'last:' INPUT_DIR replaced by the fixed $SP_RUN/output/run_sp_//output path of its producer. - NUMBER_LIST=$SP_UNIT_NUM on unit-ID-numbered stages (getlist env-expands; SP_UNIT_NUM carries the dashed form, e.g. -210-282 — rules do the transform). - ngmix template: $NGMIX_ID_MIN/$NGMIX_ID_MAX, SEED_FROM_POSITION=True, per-chunk RUN_NAME=run_sp_tile_ngmix_Ng${SP_NGMIX_CHUNK}u, SAVE_BATCH=250. - merge_sep_cats: N_SPLIT_MAX=$NGMIX_N_CHUNKS; module reads it env-expanded (same getexpanded treatment as the ngmix chunk fields). - make_cat from the nosm variant (what the P3 runs used) — no spread_model dependency. - p3 deltas folded: real INPUT_PATH/RETRIEVE=symlink, exp_Ma external star cat ($SP_RUN/star_cat_exp), PiViVi RUN_DATETIME fix. Rejected: SMP_BATCH_SIZE deltas (superseded by -b {threads}). - tile_Sx from the nomask variant (plain Sx needs an unlisted tile_Ma stage); tile_Uc kept but its run_sp_tile_Gic input path is unverified (Uc is not the production detect variant). All 14 configs pass a strict $VAR-expansion round-trip with the workflow's env stubbed. workflow/config.yaml config_src now points in-repo. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NRBKnHV6oUXEqRpsjqJpex --- .../modules/merge_sep_cats_runner.py | 2 +- workflow/config.yaml | 8 +- workflow/config/cfis/config_exp_Gie.ini | 99 ++++++++++ workflow/config/cfis/config_exp_Ma.ini | 86 +++++++++ workflow/config/cfis/config_exp_Sp.ini | 78 ++++++++ workflow/config/cfis/config_exp_psfex.ini | 181 ++++++++++++++++++ .../config/cfis/config_merge_sep_cats.ini | 87 +++++++++ workflow/config/cfis/config_onthefly.mask | 86 +++++++++ workflow/config/cfis/config_tile_Fe.ini | 76 ++++++++ workflow/config/cfis/config_tile_Git.ini | 93 +++++++++ workflow/config/cfis/config_tile_Mc.ini | 80 ++++++++ workflow/config/cfis/config_tile_Mh_exp.ini | 76 ++++++++ .../config/cfis/config_tile_Ng_template.ini | 104 ++++++++++ workflow/config/cfis/config_tile_PiViVi.ini | 172 +++++++++++++++++ workflow/config/cfis/config_tile_Sx.ini | 118 ++++++++++++ workflow/config/cfis/config_tile_Uc.ini | 92 +++++++++ workflow/config/cfis/config_tile_Uz.ini | 73 +++++++ .../config/cfis/config_tile_onthefly.mask | 90 +++++++++ workflow/config/cfis/default.conv | 5 + workflow/config/cfis/default.param | 68 +++++++ workflow/config/cfis/default.psfex | 85 ++++++++ workflow/config/cfis/default_exp.sex | 133 +++++++++++++ workflow/config/cfis/default_noimaflags.param | 65 +++++++ workflow/config/cfis/default_tile.sex | 133 +++++++++++++ workflow/config/cfis/final_cat.param | 114 +++++++++++ .../mask_default/MEGAPRIME_star_i_13.8.reg | 24 +++ .../cfis/mask_default/Messier_catalog.npy | Bin 0 -> 4209 bytes .../mask_default/Messier_catalog_updated.fits | Bin 0 -> 8640 bytes workflow/config/cfis/mask_default/default.ww | 40 ++++ .../config/cfis/mask_default/halo_mask.reg | 50 +++++ .../config/cfis/mask_default/ngc_cat.fits | Bin 0 -> 201600 bytes workflow/config/cfis/star_selection.setools | 103 ++++++++++ 32 files changed, 2418 insertions(+), 3 deletions(-) create mode 100644 workflow/config/cfis/config_exp_Gie.ini create mode 100644 workflow/config/cfis/config_exp_Ma.ini create mode 100644 workflow/config/cfis/config_exp_Sp.ini create mode 100644 workflow/config/cfis/config_exp_psfex.ini create mode 100644 workflow/config/cfis/config_merge_sep_cats.ini create mode 100644 workflow/config/cfis/config_onthefly.mask create mode 100644 workflow/config/cfis/config_tile_Fe.ini create mode 100644 workflow/config/cfis/config_tile_Git.ini create mode 100644 workflow/config/cfis/config_tile_Mc.ini create mode 100644 workflow/config/cfis/config_tile_Mh_exp.ini create mode 100644 workflow/config/cfis/config_tile_Ng_template.ini create mode 100644 workflow/config/cfis/config_tile_PiViVi.ini create mode 100644 workflow/config/cfis/config_tile_Sx.ini create mode 100644 workflow/config/cfis/config_tile_Uc.ini create mode 100644 workflow/config/cfis/config_tile_Uz.ini create mode 100644 workflow/config/cfis/config_tile_onthefly.mask create mode 100644 workflow/config/cfis/default.conv create mode 100644 workflow/config/cfis/default.param create mode 100644 workflow/config/cfis/default.psfex create mode 100644 workflow/config/cfis/default_exp.sex create mode 100644 workflow/config/cfis/default_noimaflags.param create mode 100644 workflow/config/cfis/default_tile.sex create mode 100644 workflow/config/cfis/final_cat.param create mode 100644 workflow/config/cfis/mask_default/MEGAPRIME_star_i_13.8.reg create mode 100644 workflow/config/cfis/mask_default/Messier_catalog.npy create mode 100644 workflow/config/cfis/mask_default/Messier_catalog_updated.fits create mode 100644 workflow/config/cfis/mask_default/default.ww create mode 100644 workflow/config/cfis/mask_default/halo_mask.reg create mode 100644 workflow/config/cfis/mask_default/ngc_cat.fits create mode 100644 workflow/config/cfis/star_selection.setools diff --git a/src/shapepipe/modules/merge_sep_cats_runner.py b/src/shapepipe/modules/merge_sep_cats_runner.py index 927c79b76..6e99cf3ec 100644 --- a/src/shapepipe/modules/merge_sep_cats_runner.py +++ b/src/shapepipe/modules/merge_sep_cats_runner.py @@ -28,7 +28,7 @@ def merge_sep_cats_runner( ): """Define The Merge SEP Catalogues Runner.""" # Get config entries - n_split_max = config.getint(module_config_sec, "N_SPLIT_MAX") + n_split_max = int(config.getexpanded(module_config_sec, "N_SPLIT_MAX")) file_pattern = config.getlist(module_config_sec, "FILE_PATTERN") file_ext = config.getlist(module_config_sec, "FILE_EXT") diff --git a/workflow/config.yaml b/workflow/config.yaml index f92cbb69f..592ecf989 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -17,8 +17,12 @@ container: /project/def-mjhudson/cdaley/containers/shapepipe-develop-runtime.sif # /tiles/// and /exp/// run_dir: /scratch/cdaley/shapepipe-output/smk-p0 -# The canonical config chain (example/cfis in the repo, or a run-specific copy). -config_src: /home/cdaley/projects/def-mjhudson/cdaley/runs/p3-batch1/cfis +# The canonical config chain: the committed, wrapper-free configs (#848 D2 config +# sweep) -- RUN_DATETIME=False, fixed INPUT_DIRs, NUMBER_LIST=$SP_UNIT_NUM baked +# in. example/cfis stays untouched for the bash flow; runs/p3-batch1/cfis's real +# divergences (real data paths, external star cats, SAVE_BATCH memory tuning) +# were folded in -- see the config-sweep report for the full list. +config_src: /home/cdaley/shapepipe-smk/workflow/config/cfis # Pre-staged inputs (P3 data already on /project; get_images RETRIEVE=symlink). # Star catalogues for masking, pre-generated (network step done in prepare): diff --git a/workflow/config/cfis/config_exp_Gie.ini b/workflow/config/cfis/config_exp_Gie.ini new file mode 100644 index 000000000..7c6297cf0 --- /dev/null +++ b/workflow/config/cfis/config_exp_Gie.ini @@ -0,0 +1,99 @@ +# ShapePipe configuration file for: get images + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = False + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_exp_Gie + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = get_images_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = $SP_RUN + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Get exposures +[GET_IMAGES_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output + +FILE_PATTERN = exp_numbers + +FILE_EXT = .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + + +# Paths + +# Output path (optional, default is [FILE]:OUTPUT_DIR +# OUTPUT_PATH = input_images + +# Input path where original images are stored. Can be local path or vos url. +# Single string or list of strings +INPUT_PATH = /project/def-mjhudson/unions-wl/exposures, /project/def-mjhudson/unions-wl/exposures, /project/def-mjhudson/unions-wl/exposures + +# Input file pattern including tile number as dummy template +INPUT_FILE_PATTERN = 000000, 000000.weight, 000000.flag + +# Input file extensions +INPUT_FILE_EXT = .fits.fz, .fits.fz, .fits.fz + +# Input numbering scheme, python regexp +INPUT_NUMBERING = \d{6} + +# Output file pattern without number +OUTPUT_FILE_PATTERN = image-, weight-, flag- + +# Method to retrieve images, one in 'vos', 'symlink' +RETRIEVE = symlink + +# If RETRIEVE=vos, number of attempts to download +# Optional, default=3 +N_TRY = 3 + +# Retrieve command options, optional +RETRIEVE_OPTIONS = --certfile=$HOME/.ssl/cadcproxy.pem + +#CHECK_EXISTING_DIR = $SP_RUN/output/run_sp_Gie_prev diff --git a/workflow/config/cfis/config_exp_Ma.ini b/workflow/config/cfis/config_exp_Ma.ini new file mode 100644 index 000000000..0b85fe86c --- /dev/null +++ b/workflow/config/cfis/config_exp_Ma.ini @@ -0,0 +1,86 @@ +# ShapePipe configuration file for masking of exposures + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_exp_Ma + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = mask_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 4 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +### Mask exposures +[MASK_RUNNER] + +# Parent module +INPUT_DIR = $SP_RUN/output/run_sp_exp_Sp/split_exp_runner/output, $SP_RUN/star_cat_exp + +# Update numbering convention, accounting for HDU number of +# single-exposure single-HDU files +NUMBERING_SCHEME = -0000000-0 + +# Input file patterns: image, weight, external flag, external star catalogue +# (folded from runs/p3-batch1/cfis: offline cluster, no online star-cat fetch) +FILE_PATTERN = image, weight, flag, star_cat + +FILE_EXT = .fits, .fits, .fits, .fits + +# Path of mask config file +MASK_CONFIG_PATH = $SP_CONFIG/config_onthefly.mask + +# External mask file flag, use if True, otherwise ignore +USE_EXT_FLAG = True + +# External star catalogue flag, use external cat if True, +# obtain from online catalogue if False +# (folded from p3-batch1: True, using the pre-staged star_cat_exp pool) +USE_EXT_STAR = True + +# File name suffix for the output flag files (optional) +PREFIX = pipeline + +# Path to check for existing output mask files +CHECK_EXISTING_DIR = $SP_RUN/output/run_sp_exp_Ma/mask_runner/output diff --git a/workflow/config/cfis/config_exp_Sp.ini b/workflow/config/cfis/config_exp_Sp.ini new file mode 100644 index 000000000..dd27d6ccd --- /dev/null +++ b/workflow/config/cfis/config_exp_Sp.ini @@ -0,0 +1,78 @@ +# ShapePipe configuration file for single-exposures, +# split images + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_exp_Sp + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = split_exp_runner + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed exposure ID (exp split is a "tile-scheme" stage per sp_rule.py). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 8 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[SPLIT_EXP_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_exp_Gie/get_images_runner/output + +FILE_PATTERN = image, weight, flag + +# Matches compressed single-exposure files +FILE_EXT = .fitsfz, .fitsfz, .fitsfz + +NUMBERING_SCHEME = -0000000 + +# OUTPUT_SUFFIX, actually file name prefixes. +# Expected keyword "flag" will lead to a behavior where the data are saved as int. +# The code also expects the image data to use the "image" suffix +# (default value in the pipeline). +OUTPUT_SUFFIX = image, weight, flag + +# Number of HDUs/CCDs of mosaic +N_HDU = 40 diff --git a/workflow/config/cfis/config_exp_psfex.ini b/workflow/config/cfis/config_exp_psfex.ini new file mode 100644 index 000000000..0af871d8e --- /dev/null +++ b/workflow/config/cfis/config_exp_psfex.ini @@ -0,0 +1,181 @@ +# ShapePipe configuration file for single-exposures. PSFex PSF model. +# Process exposures after masking, from star detection to PSF model. + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_exp_SxSePsfPi +#RUN_NAME = run_sp_exp_SxSePsf + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = sextractor_runner, setools_runner, psfex_runner, psfex_interp_runner + + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = $SP_RUN/output + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 8 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[SEXTRACTOR_RUNNER] + +# Input from two modules +INPUT_DIR = $SP_RUN/output/run_sp_exp_Sp/split_exp_runner/output, $SP_RUN/output/run_sp_exp_Ma/mask_runner/output + +# Read pipeline flag files created by mask module +FILE_PATTERN = image, weight, pipeline_flag + +# Explicit extensions: a 3-entry FILE_PATTERN override must not fall back on +# the decorator's 4-entry FILE_EXT default (length check fails at startup) +FILE_EXT = .fits, .fits, .fits + +NUMBERING_SCHEME = -0000000-0 + +# SExtractor executable path +EXEC_PATH = source-extractor + +# SExtractor configuration files +DOT_SEX_FILE = $SP_CONFIG/default_exp.sex +DOT_PARAM_FILE = $SP_CONFIG//default.param +DOT_CONV_FILE = $SP_CONFIG/default.conv + +# Use input weight image if True +WEIGHT_IMAGE = True + +# Use input flag image if True +FLAG_IMAGE = True + +# Use input PSF file if True +PSF_FILE = False + +# Use distinct image for detection (SExtractor in +# dual-image mode) if True. +DETECTION_IMAGE = False + +# Distinct weight image for detection (SExtractor +# in dual-image mode) +DETECTION_WEIGHT = False + +# True if photometry zero-point is to be read from exposure image header +ZP_FROM_HEADER = True + +# If ZP_FROM_HEADER is True, zero-point key name +ZP_KEY = PHOTZP + +# Background information from image header. +# If BKG_FROM_HEADER is True, background value will be read from header. +# In that case, the value of BACK_TYPE will be set atomatically to MANUAL. +# This is used e.g. for the LSB images. +BKG_FROM_HEADER = False +# LSB images: +# BKG_FROM_HEADER = True + +# If BKG_FROM_HEADER is True, background value key name +# LSB images: +#BKG_KEY = IMMODE + +# Type of image check (optional), default not used, can be a list of +# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, MINIBACK_RMS, -BACKGROUND, +# FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, APERTURES +CHECKIMAGE = BACKGROUND, BACKGROUND_RMS + +# File name suffix for the output sextractor files (optional) SUFFIX = tile +SUFFIX = sexcat + +## Post-processing + +# Not required for single exposures +MAKE_POST_PROCESS = FALSE + + +[SETOOLS_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_exp_SxSePsfPi/sextractor_runner/output + +# Note: Make sure this doe not match the SExtractor background images +# (sexcat_background*) +FILE_PATTERN = sexcat + +NUMBERING_SCHEME = -0000000-0 + +# SETools config file +SETOOLS_CONFIG_PATH = $SP_CONFIG/star_selection.setools + + +[PSFEX_RUNNER] + +# Use 80% sample for PSF model +FILE_PATTERN = star_split_ratio_80 + +NUMBERING_SCHEME = -0000000-0 + +# Path to executable for the PSF model (optional) +EXEC_PATH = psfex + +# Default psfex configuration file +DOT_PSFEX_FILE = $SP_CONFIG/default.psfex + +[PSFEX_INTERP_RUNNER] + +# Use 20% sample for PSF validation +FILE_PATTERN = star_split_ratio_80, star_split_ratio_20, psfex_cat + +FILE_EXT = .psf, .fits, .cat + +NUMBERING_SCHEME = -0000000-0 + +# Run mode for psfex interpolation: +# CLASSIC: 'classical' run, interpolate to object positions +# MULTI-EPOCH: interpolate for multi-epoch images +# VALIDATION: validation for single-epoch images +MODE = VALIDATION + +# Column names of position parameters +POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE + +# If True, measure and store ellipticity of the PSF (using moments) +GET_SHAPES = True + +# Minimum number of stars per CCD for PSF model to be computed +STAR_THRESH = 22 + +# Maximum chi^2 for PSF model to be computed on CCD +CHI2_THRESH = 2 diff --git a/workflow/config/cfis/config_merge_sep_cats.ini b/workflow/config/cfis/config_merge_sep_cats.ini new file mode 100644 index 000000000..a033f28d5 --- /dev/null +++ b/workflow/config/cfis/config_merge_sep_cats.ini @@ -0,0 +1,87 @@ +# ShapePipe post-run configuration file: merge separated catalogues + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_Ms + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = merge_sep_cats_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +# NOTE: only chunk 1's ngmix output is listed here; merge_sep_cats_runner +# derives the other chunks' paths itself from N_SPLIT_MAX and this pattern +# (chunk dirs are run_sp_tile_ngmix_Ngu, k=1..N_SPLIT_MAX), all under the +# same fixed $SP_RUN/output. +INPUT_DIR = $SP_RUN/output/run_sp_tile_ngmix_Ng1u/ngmix_runner/output + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 8 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[MERGE_SEP_CATS_RUNNER] + +# Input file pattern(s), list of strings with length matching number of expected input file types +# Cannot contain wild cards +FILE_PATTERN = ngmix + +# FILE_EXT (optional) list of string extensions to identify input files +FILE_EXT = .fits + +# Numbering convention, string that exemplifies a numbering pattern. +NUMBERING_SCHEME = -000-000 + +# WARNING (optional, default is 'error'). Use 'always'/'ignore' to +# display/ignore warnings, and not raise error +WARNING = always + +# Maximum number of separated catalogues per input. +# BLOCKER: merge_sep_cats_runner.py reads this with plain config.getint (not +# getexpanded) -- $NGMIX_N_CHUNKS would NOT expand here, unlike ID_OBJ_MIN/MAX +# in the ngmix module. Committed as a constant matching workflow/config.yaml's +# ngmix_chunks: 8; if that value changes, this line must be edited by hand +# (or the module gets the same getexpanded treatment ngmix_runner got). +# Chunk count comes from the workflow (single source of truth: the +# scattergather count); env-expanded by the module. +N_SPLIT_MAX = $NGMIX_N_CHUNKS diff --git a/workflow/config/cfis/config_onthefly.mask b/workflow/config/cfis/config_onthefly.mask new file mode 100644 index 000000000..7c185c602 --- /dev/null +++ b/workflow/config/cfis/config_onthefly.mask @@ -0,0 +1,86 @@ +# Mask module configuration file for single-exposure images + +## Paths to executables +[PROGRAM_PATH] + +WW_PATH = weightwatcher +WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww + +# Indicate cds client executable if no external star catalogue is available +# (e.g. no internet access on run nodes) +CDSCLIENT_PATH = findgsc2.2 + + +## Border mask +[BORDER_PARAMETERS] + +BORDER_MAKE = True + +BORDER_WIDTH = 50 +BORDER_FLAG_VALUE = 4 + + +## Halo mask +[HALO_PARAMETERS] + +HALO_MAKE = True + +HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg +HALO_MAG_LIM = 13. +HALO_SCALE_FACTOR = 0.05 +HALO_MAG_PIVOT = 13.8 +HALO_FLAG_VALUE = 2 +HALO_REG_FILE = halo.reg + + +## Diffraction spike mask +[SPIKE_PARAMETERS] + +SPIKE_MAKE = True + +SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg +SPIKE_MAG_LIM = 18. +SPIKE_SCALE_FACTOR = 0.3 +SPIKE_MAG_PIVOT = 13.8 +SPIKE_FLAG_VALUE = 128 +SPIKE_REG_FILE = spike.reg + + +## Messier mask +[MESSIER_PARAMETERS] + +MESSIER_MAKE = True + +MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits +MESSIER_SIZE_PLUS = 0. +MESSIER_FLAG_VALUE = 16 + + +## NGC mask +[NGC_PARAMETERS] + +NGC_MAKE = True + +NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits +NGC_SIZE_PLUS = 0. +NGC_FLAG_VALUE = 32 + + + +## Missing data parameters +[MD_PARAMETERS] + +MD_MAKE = False + +MD_THRESH_FLAG = 0.3 +MD_THRESH_REMOVE = 0.75 +MD_REMOVE = False + + +## Other parameters +[OTHER] + +TEMP_DIRECTORY = .temp + +KEEP_REG_FILE = False +KEEP_INDIVIDUAL_MASK = False diff --git a/workflow/config/cfis/config_tile_Fe.ini b/workflow/config/cfis/config_tile_Fe.ini new file mode 100644 index 000000000..9546f062e --- /dev/null +++ b/workflow/config/cfis/config_tile_Fe.ini @@ -0,0 +1,76 @@ +# ShapePipe configuration file for: find exposures + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = False + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Fe + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = find_exposures_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names +INPUT_DIR = $SP_RUN + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Get tiles +[FIND_EXPOSURES_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Git/get_images_runner/output + +FILE_PATTERN = CFIS_image + +FILE_EXT = .fits + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# Column number of exposure name in FITS header +COLNUM = 3 + +# Prefix to remove from exposure name +EXP_PREFIX = p + diff --git a/workflow/config/cfis/config_tile_Git.ini b/workflow/config/cfis/config_tile_Git.ini new file mode 100644 index 000000000..72a0f0be7 --- /dev/null +++ b/workflow/config/cfis/config_tile_Git.ini @@ -0,0 +1,93 @@ +# ShapePipe configuration file for: get tile images + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = False + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Git + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = get_images_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = $SP_RUN + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Get tiles +[GET_IMAGES_RUNNER] + +FILE_PATTERN = tile_numbers + +FILE_EXT = .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = + +# Paths + +# Input path where original images are stored. Can be local path or vos url. +# Single string or list of strings +INPUT_PATH = /project/def-mjhudson/unions-wl/tiles, /project/def-mjhudson/unions-wl/tiles + +# Input file pattern including tile number as dummy template +INPUT_FILE_PATTERN = CFIS.000.000.r, CFIS.000.000.r.weight + +# Input file extensions +INPUT_FILE_EXT = .fits, .fits.fz + +# Input numbering scheme, python regexp +INPUT_NUMBERING = \d{3}\.\d{3} + +# Output file pattern without number +OUTPUT_FILE_PATTERN = CFIS_image-, CFIS_weight- + +# Copy/download method, one in 'vos', 'symlink' +RETRIEVE = symlink + +# If RETRIEVE=vos, number of attempts to download +# Optional, default=3 +N_TRY = 3 + +# Copy command options, optional +RETRIEVE_OPTIONS = --certfile=$HOME/.ssl/cadcproxy.pem + +#CHECK_EXISTING_DIR = $SP_RUN/data_tiles diff --git a/workflow/config/cfis/config_tile_Mc.ini b/workflow/config/cfis/config_tile_Mc.ini new file mode 100644 index 000000000..502731431 --- /dev/null +++ b/workflow/config/cfis/config_tile_Mc.ini @@ -0,0 +1,80 @@ +# ShapePipe post-run configuration file: create final catalogs, with +# no spread model on input + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_Mc + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = make_cat_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = ./output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 8 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[MAKE_CAT_RUNNER] + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_PiViVi/psfex_interp_runner/output, $SP_RUN/output/run_sp_Ms/merge_sep_cats_runner/output + +# Input file pattern(s), list of strings with length matching number of expected input file types +# Cannot contain wild cards +FILE_PATTERN = sexcat, galaxy_psf, ngmix + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282) +NUMBER_LIST = $SP_UNIT_NUM + +# FILE_EXT (optional) list of string extensions to identify input files +FILE_EXT = .fits, .sqlite, .fits + +# Numbering convention, string that exemplifies a numbering pattern. +# Matches input single exposures (with 'p' removed) +# Needs to be given in this section, will be updated in module +# sections below +NUMBERING_SCHEME = -000-000 + +SM_DO_CLASSIFICATION = False + +SHAPE_MEASUREMENT_TYPE = ngmix diff --git a/workflow/config/cfis/config_tile_Mh_exp.ini b/workflow/config/cfis/config_tile_Mh_exp.ini new file mode 100644 index 000000000..96512df1a --- /dev/null +++ b/workflow/config/cfis/config_tile_Mh_exp.ini @@ -0,0 +1,76 @@ +# ShapePipe configuration file for merging per-exposure WCS headers +# at the tile level. Input is the exp_numbers file produced by +# find_exposures_runner; EXP_BASE_DIR tells the runner where to find +# the per-exposure split_exp_runner header .npy files. + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Mh_exp + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = merge_headers_runner + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 16 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[MERGE_HEADERS_RUNNER] + +# Input: exp_numbers txt file from find_exposures_runner +INPUT_DIR = $SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output + +FILE_PATTERN = exp_numbers + +FILE_EXT = .txt + +# Tile numbering scheme (RA-Dec, e.g. -301-279) +NUMBERING_SCHEME = -000-000 + +# Root directory containing all per-exposure work directories. +# The runner will walk this tree to collect headers-.npy files. +EXP_BASE_DIR = $SP_EXP diff --git a/workflow/config/cfis/config_tile_Ng_template.ini b/workflow/config/cfis/config_tile_Ng_template.ini new file mode 100644 index 000000000..6e3e33292 --- /dev/null +++ b/workflow/config/cfis/config_tile_Ng_template.ini @@ -0,0 +1,104 @@ +# ShapePipe configuration file for tiles: ngmix + KSB + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +# Per-chunk run dir: the workflow exports SP_NGMIX_CHUNK= for each chunk. +# RUN_NAME is env-expanded (run.py getexpanded); braces keep the trailing "u" +# out of the variable name. +RUN_NAME = run_sp_tile_ngmix_Ng${SP_NGMIX_CHUNK}u + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = ngmix_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Model-fitting shapes with ngmix +[NGMIX_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_PiViVi/psfex_interp_runner/output, $SP_RUN/output/run_sp_tile_PiViVi/vignetmaker_runner_run_2/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output + +FILE_PATTERN = sexcat, image_vignet, background_vignet, galaxy_psf, weight_vignet, flag_vignet, log_exp_headers + +FILE_EXT = .fits, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# BKG_RMS_VIGNET_PATH (optional): per-pixel BACKGROUND_RMS vignets, used as +# 1/RMS^2 inverse-variance ngmix weights. When set, the file must exist for +# every tile (missing file -> error, no per-tile fallback); omit the option +# entirely to fall back to the scalar sigma_mad noise estimate. +BKG_RMS_VIGNET_PATH = $SP_RUN/output/run_sp_tile_PiViVi/vignetmaker_runner_run_2/output/background_rms_vignet{file_number_string}.sqlite + +# Number of objects to batch save during processing, optional. Omit or set +# to -1 for no batch saving. +# 250 (folded from runs/p3-batch1/cfis): worker RSS grows ~4.8 MB/object +# until the flush recycles it -> peak ~1.1 GB + 250 x 4.8 MB ~= 2.3 GB/worker +# (A/B test, job 17607877). Not superseded by -b {threads} (that sets fork +# width, this bounds per-worker memory). +SAVE_BATCH = 250 + +# Magnitude zero-point +MAG_ZP = 30.0 + +# Pixel scale in arcsec +PIXEL_SCALE = 0.186 + +# SEED_FROM_POSITION: per-object RNG seeded from sky position (ra, dec, ccd) +# instead of one ordered per-tile stream, so results are bit-identical under +# any chunking (D4/#796). Required for the chunked ngmix scatter/gather. +SEED_FROM_POSITION = True + +# ID_OBJ_MIN/MAX: this chunk's closed SExtractor NUMBER-column range, +# computed at execution time from the tile's own object count and expanded +# via ShapePipe's getexpanded (ngmix_runner.py verified: env-expanded, not +# plain getint). +ID_OBJ_MIN = $NGMIX_ID_MIN +ID_OBJ_MAX = $NGMIX_ID_MAX diff --git a/workflow/config/cfis/config_tile_PiViVi.ini b/workflow/config/cfis/config_tile_PiViVi.ini new file mode 100644 index 000000000..246ee18f9 --- /dev/null +++ b/workflow/config/cfis/config_tile_PiViVi.ini @@ -0,0 +1,172 @@ +# ShapePipe configuration file for tile, from detection up to shape measurement. +# PSFEx PSF model. + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_PiViVi + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +#MODULE = psfex_interp_runner, + +MODULE = psfex_interp_runner, vignetmaker_runner, vignetmaker_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 16 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[PSFEX_INTERP_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output, $SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output + +FILE_PATTERN = sexcat, log_exp_headers, exp_numbers + +FILE_EXT = .fits, .sqlite, .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# Run mode for psfex interpolation: +# CLASSIC: 'classical' run, interpolate to object positions +# MULTI-EPOCH: interpolate for multi-epoch images +# VALIDATION: validation for single-epoch images +MODE = MULTI-EPOCH + +# Column names of position parameters +POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD + +# If True, measure and store ellipticity of the PSF +GET_SHAPES = True + +# Number of stars threshold +STAR_THRESH = 20 + +# chi^2 threshold +CHI2_THRESH = 2 + +# Multi-epoch mode parameters + +# Root directory of per-exposure work directories; replaces ME_DOT_PSF_DIR +# for v2.0 per-exposure pipeline. psfex_runner/output/ dirs are discovered +# by scanning $SP_EXP for the exposures listed in the exp_numbers input file. +ME_DOT_PSF_EXP_DIR = $SP_EXP + +# Input psf file pattern +ME_DOT_PSF_PATTERN = star_split_ratio_80 + + +# Create vignets for tiles weights +[VIGNETMAKER_RUNNER_RUN_1] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_Uz/uncompress_fits_runner/output + +FILE_PATTERN = sexcat, CFIS_weight + +FILE_EXT = .fits, .fits + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +MASKING = False +MASK_VALUE = 0 + +# Run mode for psfex interpolation: +# CLASSIC: 'classical' run, interpolate to object positions +# MULTI-EPOCH: interpolate for multi-epoch images +# VALIDATION: validation for single-epoch images +MODE = CLASSIC + +# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) +COORD = PIX +POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE + +# Vignet size in pixels +STAMP_SIZE = 51 + +# Output file name prefix, file name is _vignet.fits +PREFIX = weight + + +[VIGNETMAKER_RUNNER_RUN_2] + +# Create multi-epoch vignets for tiles corresponding to +# positions on single-exposures + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output, $SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output + +FILE_PATTERN = sexcat, log_exp_headers, exp_numbers + +FILE_EXT = .fits, .sqlite, .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +MASKING = False +MASK_VALUE = 0 + +# Run mode for psfex interpolation: +# CLASSIC: 'classical' run, interpolate to object positions +# MULTI-EPOCH: interpolate for multi-epoch images +# VALIDATION: validation for single-epoch images +MODE = MULTI-EPOCH + +# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) +COORD = SPHE +POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD + +# Vignet size in pixels +STAMP_SIZE = 51 + +# Output file name prefix, file name is vignet.fits +PREFIX = + +# Additional parameters for path and file pattern corresponding to single-exposure +# run outputs. ME_IMAGE_EXP_DIR/ME_IMAGE_EXP_RUNNERS replace ME_IMAGE_DIR for +# the v2.0 per-exposure pipeline; output dirs are discovered by scanning $SP_EXP. +ME_IMAGE_EXP_DIR = $SP_EXP +ME_IMAGE_EXP_RUNNERS = split_exp_runner, split_exp_runner, split_exp_runner, sextractor_runner, sextractor_runner +ME_IMAGE_PATTERN = flag, image, weight, background, background_rms diff --git a/workflow/config/cfis/config_tile_Sx.ini b/workflow/config/cfis/config_tile_Sx.ini new file mode 100644 index 000000000..0ce9ea226 --- /dev/null +++ b/workflow/config/cfis/config_tile_Sx.ini @@ -0,0 +1,118 @@ +# ShapePipe configuration file for tile detection + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Sx + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = sextractor_runner + + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = $SP_RUN/output + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 16 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[SEXTRACTOR_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Git/get_images_runner/output, $SP_RUN/output/run_sp_tile_Uz/uncompress_fits_runner/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output + +FILE_PATTERN = CFIS_image, CFIS_weight, log_exp_headers + +FILE_EXT = .fits, .fits, .sqlite + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# SExtractor executable path +EXEC_PATH = source-extractor + +# SExtractor configuration files +DOT_SEX_FILE = $SP_CONFIG/default_tile.sex +DOT_PARAM_FILE = $SP_CONFIG/default_noimaflags.param +DOT_CONV_FILE = $SP_CONFIG/default.conv + +# Use input weight image if True +WEIGHT_IMAGE = True + +# Use input flag image if True +FLAG_IMAGE = False + +# Use input PSF file if True +PSF_FILE = False + +# Use distinct image for detection (SExtractor in +# dual-image mode) if True +DETECTION_IMAGE = False + +# Distinct weight image for detection (SExtractor +# in dual-image mode) +DETECTION_WEIGHT = False + +ZP_FROM_HEADER = False + +BKG_FROM_HEADER = False + +# Type of image check (optional), default not used, can be a list of +# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, +# MINIBACK_RMS, -BACKGROUND, #FILTERED, +# OBJECTS, -OBJECTS, SEGMENTATION, APERTURES +CHECKIMAGE = BACKGROUND + +# File name suffix for the output sextractor files (optional) +SUFFIX = sexcat + +## Post-processing + +# Necessary for tiles, to enable multi-exposure processing +MAKE_POST_PROCESS = True + +# World coordinate keywords, SExtractor output. Format: KEY_X,KEY_Y +WORLD_POSITION = XWIN_WORLD,YWIN_WORLD + +# Number of pixels in x,y of a CCD. Format: Nx,Ny +CCD_SIZE = 33,2080,1,4612 diff --git a/workflow/config/cfis/config_tile_Uc.ini b/workflow/config/cfis/config_tile_Uc.ini new file mode 100644 index 000000000..f2ff51e4a --- /dev/null +++ b/workflow/config/cfis/config_tile_Uc.ini @@ -0,0 +1,92 @@ +# ShapePipe configuration file for tile object selection using +# an (external) catalogue + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Uc + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = read_ext_sexcat_runner + + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = $SP_RUN/output + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 16 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[READ_EXT_SEXCAT_RUNNER] + +# NOTE(blocker): run_sp_tile_Gic (external-catalog get_images) is not one of +# the 13 configs in this sweep -- no committed config produces it, so this +# path is written by naming-convention analogy, unverified. Uc is otherwise +# fully de-wrapper-ified; do not rely on this input until Gic exists. +INPUT_DIR = $SP_RUN/output/run_sp_tile_Gic/get_images_runner/output, $SP_RUN/output/run_sp_tile_Git/get_images_runner/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output + +FILE_PATTERN = CFIS_cat, CFIS_image, log_exp_headers + +FILE_EXT = .cat, .fits, .sqlite + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# File name suffix for the output sextractor files (optional) +SUFFIX = sexcat + +# Side length of the square postage stamp (vignet) extracted from the tile +# image, in pixels (must be odd). Default: 51 +VIGNET_SIZE = 51 + +## Post-processing + +# Necessary for tiles, to enable multi-exposure processing +MAKE_POST_PROCESS = True + +# World coordinate keywords, SExtractor output. Format: KEY_X,KEY_Y +WORLD_POSITION = ALPHA_J2000,DELTA_J2000 + +# Number of pixels in x,y of a CCD. Format: Nx,Ny +CCD_SIZE = 33,2080,1,4612 diff --git a/workflow/config/cfis/config_tile_Uz.ini b/workflow/config/cfis/config_tile_Uz.ini new file mode 100644 index 000000000..fc8550af4 --- /dev/null +++ b/workflow/config/cfis/config_tile_Uz.ini @@ -0,0 +1,73 @@ +# ShapePipe configuration file for: uncompress FITS image + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Uz + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = uncompress_fits_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 16 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options +[UNCOMPRESS_FITS_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Git/get_images_runner/output + +FILE_PATTERN = CFIS_weight + +FILE_EXT = .fitsfz + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# Input HDU of image data, optional, default=0 +HDU_DATA = 1 + +# Output file pattern +OUTPUT_PATTERN = CFIS_weight diff --git a/workflow/config/cfis/config_tile_onthefly.mask b/workflow/config/cfis/config_tile_onthefly.mask new file mode 100644 index 000000000..18cf5db2d --- /dev/null +++ b/workflow/config/cfis/config_tile_onthefly.mask @@ -0,0 +1,90 @@ +# Mask module config file for tiles + +## Paths to executables +[PROGRAM_PATH] + +WW_PATH = weightwatcher +WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww + +# Indicate cds client executable if no external star catalogue is available +# (e.g. no internet access on run nodes) +CDSCLIENT_PATH = findgsc2.2 + +## Border parameters +[BORDER_PARAMETERS] + +BORDER_MAKE = False + +BORDER_WIDTH = 0 +BORDER_FLAG_VALUE = 4 + + +## Halo parameters +[HALO_PARAMETERS] + +HALO_MAKE = True + +HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg +HALO_MAG_LIM = 13. +HALO_SCALE_FACTOR = 0.05 +HALO_MAG_PIVOT = 13.8 +HALO_FLAG_VALUE = 2 +HALO_REG_FILE = halo.reg + + +## Diffraction pike parameters +[SPIKE_PARAMETERS] + +SPIKE_MAKE = True + +SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg +SPIKE_MAG_LIM = 18. +SPIKE_SCALE_FACTOR = 0.3 +SPIKE_MAG_PIVOT = 13.8 +SPIKE_FLAG_VALUE = 128 +SPIKE_REG_FILE = spike.reg + + +## Messier parameters +[MESSIER_PARAMETERS] + +MESSIER_MAKE = True + +MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits +MESSIER_PIXEL_SCALE = 0.187 +MESSIER_SIZE_PLUS = 0. +MESSIER_FLAG_VALUE = 16 + +## NGC mask +[NGC_PARAMETERS] + +NGC_MAKE = True + +NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits +NGC_SIZE_PLUS = 0. +NGC_FLAG_VALUE = 32 + + +## External flag +[EXTERNAL_FLAG] + +EF_MAKE = False + + +## Missing data parameters +[MD_PARAMETERS] + +MD_MAKE = False + +MD_THRESH_FLAG = 0.3 +MD_THRESH_REMOVE = 0.75 +MD_REMOVE = False + + +## Other parameters +[OTHER] + +KEEP_REG_FILE = False +KEEP_INDIVIDUAL_MASK = False + +TEMP_DIRECTORY = .temp_tiles diff --git a/workflow/config/cfis/default.conv b/workflow/config/cfis/default.conv new file mode 100644 index 000000000..2590b9cba --- /dev/null +++ b/workflow/config/cfis/default.conv @@ -0,0 +1,5 @@ +CONV NORM +# 3x3 ``all-ground'' convolution mask with FWHM = 2 pixels. +1 2 1 +2 4 2 +1 2 1 diff --git a/workflow/config/cfis/default.param b/workflow/config/cfis/default.param new file mode 100644 index 000000000..09ad8405e --- /dev/null +++ b/workflow/config/cfis/default.param @@ -0,0 +1,68 @@ +NUMBER #Running object number +EXT_NUMBER #FITS extension number + +FLUX_AUTO #Flux within a Kron-like elliptical aperture [count] +FLUXERR_AUTO #RMS error for AUTO flux [count] +MAG_AUTO #Kron-like elliptical aperture magnitude [mag] +MAGERR_AUTO #RMS error for AUTO magnitude [mag] +FLUX_WIN #Gaussian-weighted flux [count] +FLUXERR_WIN #RMS error for WIN flux [count] +MAG_WIN #Gaussian-weighted magnitude [mag] +MAGERR_WIN #RMS error for MAG_WIN [mag] +FLUX_APER(1) +FLUXERR_APER(1) + +FLUX_RADIUS #Fraction-of-light radii [pixel] + +SNR_WIN #Gaussian-weighted SNR + +BACKGROUND #Background at centroid position [count] +THRESHOLD #Detection threshold above background [count] + +X_IMAGE #Object position along x [pixel] +Y_IMAGE #Object position along y [pixel] + +X_WORLD #Barycenter position along world x axis [deg] +Y_WORLD #Barycenter position along world y axis [deg] + +X2_IMAGE #Variance along x [pixel**2] +Y2_IMAGE #Variance along y [pixel**2] +XY_IMAGE #Covariance between x and y [pixel**2] +ERRX2_IMAGE #Variance of position along x [pixel**2] +ERRY2_IMAGE #Variance of position along y [pixel**2] +ERRXY_IMAGE #Covariance of position between x and y [pixel**2] + +XWIN_IMAGE #Windowed position estimate along x [pixel] +YWIN_IMAGE #Windowed position estimate along y [pixel] + +XWIN_WORLD #Windowed position along world x axis [deg] +YWIN_WORLD #Windowed position along world y axis [deg] + +X2WIN_IMAGE #Windowed variance along x [pixel**2] +Y2WIN_IMAGE #Windowed variance along y [pixel**2] +XYWIN_IMAGE #Windowed covariance between x and y [pixel**2] +ERRX2WIN_IMAGE #Variance of windowed pos along x [pixel**2] +ERRY2WIN_IMAGE #Variance of windowed pos along y [pixel**2] +ERRXYWIN_IMAGE #Covariance of windowed pos between x and y [pixel**2] + +MU_THRESHOLD #Analysis threshold above background [mag * arcsec**(-2)] +MU_MAX #Peak surface brightness above background [mag * arcsec**(-2)] + +FLAGS #Extraction flags +FLAGS_WIN #Flags for WINdowed parameters + +# The following flag requires a flag image +IMAFLAGS_ISO #FLAG-image flags OR'ed over the iso. profile !!! REQUIRE FLAG_IMAGE !!! + +FWHM_IMAGE #FWHM assuming a gaussian core [pixel] +FWHM_WORLD #FWHM assuming a gaussian core [deg] +ELONGATION #A_IMAGE/B_IMAGE +ELLIPTICITY #1 - B_IMAGE/A_IMAGE + +VIGNET(51,51) #Pixel data around detection [count] + +# For GaaP photometry +A_WORLD +B_WORLD +THETA_J2000 + diff --git a/workflow/config/cfis/default.psfex b/workflow/config/cfis/default.psfex new file mode 100644 index 000000000..a9d1a906c --- /dev/null +++ b/workflow/config/cfis/default.psfex @@ -0,0 +1,85 @@ +# Default configuration file for PSFEx 3.17.1 +# EB 2017-11-30 +# + +#-------------------------------- PSF model ---------------------------------- + +BASIS_TYPE PIXEL # NONE, PIXEL, GAUSS-LAGUERRE or FILE +BASIS_NUMBER 20 # Basis number or parameter +BASIS_NAME basis.fits # Basis filename (FITS data-cube) +BASIS_SCALE 1.0 # Gauss-Laguerre beta parameter +NEWBASIS_TYPE NONE # Create new basis: NONE, PCA_INDEPENDENT + # or PCA_COMMON +NEWBASIS_NUMBER 8 # Number of new basis vectors +PSF_SAMPLING 1. # Sampling step in pixel units (0.0 = auto) +PSF_PIXELSIZE 1.0 # Effective pixel size in pixel step units +PSF_ACCURACY 0.01 # Accuracy to expect from PSF "pixel" values +PSF_SIZE 51,51 # Image size of the PSF model +PSF_RECENTER N # Allow recentering of PSF-candidates Y/N ? +MEF_TYPE INDEPENDENT # INDEPENDENT or COMMON + +#------------------------- Point source measurements ------------------------- + +CENTER_KEYS XWIN_IMAGE,YWIN_IMAGE # Catalogue parameters for source pre-centering +PHOTFLUX_KEY FLUX_AUTO # Catalogue parameter for photometric norm. +PHOTFLUXERR_KEY FLUXERR_AUTO # Catalogue parameter for photometric error + +#----------------------------- PSF variability ------------------------------- + +PSFVAR_KEYS XWIN_IMAGE,YWIN_IMAGE # Catalogue or FITS (preceded by :) params +PSFVAR_GROUPS 1,1 # Group tag for each context key +PSFVAR_DEGREES 2 # Polynom degree for each group +PSFVAR_NSNAP 9 # Number of PSF snapshots per axis +HIDDENMEF_TYPE COMMON # INDEPENDENT or COMMON +STABILITY_TYPE EXPOSURE # EXPOSURE or SEQUENCE + +#----------------------------- Sample selection ------------------------------ + +SAMPLE_AUTOSELECT N # Automatically select the FWHM (Y/N) ? + +BADPIXEL_FILTER N # Filter bad-pixels in samples (Y/N) ? +BADPIXEL_NMAX 0 # Maximum number of bad pixels allowed + +#----------------------- PSF homogeneisation kernel -------------------------- + +HOMOBASIS_TYPE NONE # NONE or GAUSS-LAGUERRE +HOMOBASIS_NUMBER 10 # Kernel basis number or parameter +HOMOBASIS_SCALE 1.0 # GAUSS-LAGUERRE beta parameter +HOMOPSF_PARAMS 2.0, 3.0 # Moffat parameters of the idealised PSF +HOMOKERNEL_DIR # Where to write kernels (empty=same as input) +HOMOKERNEL_SUFFIX .homo.fits # Filename extension for homogenisation kernels + +#----------------------------- Output catalogs ------------------------------- + +OUTCAT_TYPE FITS_LDAC # NONE, ASCII_HEAD, ASCII, FITS_LDAC + +#------------------------------- Check-plots ---------------------------------- + +CHECKPLOT_DEV NULL # NULL, XWIN, TK, PS, PSC, XFIG, PNG, + # JPEG, AQT, PDF or SVG +CHECKPLOT_RES 0 # Check-plot resolution (0 = default) +CHECKPLOT_ANTIALIAS Y # Anti-aliasing using convert (Y/N) ? +CHECKPLOT_TYPE NONE # FWHM,ELLIPTICITY,COUNTS, COUNT_FRACTION, CHI2, RESIDUALS +CHECKPLOT_TYPE FWHM,ELLIPTICITY,COUNTS, COUNT_FRACTION, CHI2, RESIDUALS + # or NONE +CHECKPLOT_NAME fwhm, ellipticity, counts, countfrac, chi2, resi + +#------------------------------ Check-Images --------------------------------- + +# Note: Check-image types can be set the ShapePipe config file, psfex_runner section +####### +CHECKIMAGE_TYPE NONE # CHI,PROTOTYPES,SAMPLES,RESIDUALS,SNAPSHOTS + # or MOFFAT,-MOFFAT,-SYMMETRICAL +#CHECKIMAGE_NAME chi.fits,proto.fits,samp.fits,resi.fits,snap.fits + # Check-image filenames +#CHECKIMAGE_CUBE N # Save check-images as datacubes (Y/N) ? + +#----------------------------- Miscellaneous --------------------------------- + +PSF_SUFFIX .psf # Filename extension for output PSF filename +VERBOSE_TYPE NORMAL # can be QUIET,NORMAL,LOG or FULL +WRITE_XML N # Write XML file (Y/N)? + +NTHREADS 1 # Number of simultaneous threads for + # the SMP version of PSFEx + # 0 = automatic diff --git a/workflow/config/cfis/default_exp.sex b/workflow/config/cfis/default_exp.sex new file mode 100644 index 000000000..b87275ecb --- /dev/null +++ b/workflow/config/cfis/default_exp.sex @@ -0,0 +1,133 @@ +# Default configuration file for SExtractor 2.19.5 +# EB 2017-11-30 +# + +#-------------------------------- Catalog ------------------------------------ + +CATALOG_TYPE FITS_LDAC + +PARAMETERS_NAME default.param + +#------------------------------- Extraction ---------------------------------- + +DETECT_TYPE CCD # CCD (linear) or PHOTO (with gamma correction) +DETECT_MINAREA 5 # min. # of pixels above threshold +DETECT_MAXAREA 0 # max. # of pixels above threshold (0=unlimited) +THRESH_TYPE RELATIVE # threshold type: RELATIVE (in sigmas) + # or ABSOLUTE (in ADUs) +DETECT_THRESH 1.5 # or , in mag.arcsec-2 +ANALYSIS_THRESH 1.5 # or , in mag.arcsec-2 + +FILTER Y # apply filter for detection (Y or N)? +FILTER_NAME default.conv +FILTER_THRESH # Threshold[s] for retina filtering + +DEBLEND_NTHRESH 32 # Number of deblending sub-thresholds +DEBLEND_MINCONT 0.001 # Minimum contrast parameter for deblending + +CLEAN Y # Clean spurious detections? (Y or N)? +CLEAN_PARAM 1.0 # Cleaning efficiency + +MASK_TYPE CORRECT # type of detection MASKing: can be one of + # NONE, BLANK or CORRECT + +#-------------------------------- WEIGHTing ---------------------------------- + +WEIGHT_TYPE MAP_WEIGHT # type of WEIGHTing: NONE, BACKGROUND, + # MAP_RMS, MAP_VAR or MAP_WEIGHT +RESCALE_WEIGHTS Y # Rescale input weights/variances (Y/N)? +WEIGHT_IMAGE weight.fits # weight-map filename +WEIGHT_GAIN Y # modulate gain (E/ADU) with weights? (Y/N) +WEIGHT_THRESH # weight threshold[s] for bad pixels + +#-------------------------------- FLAGging ----------------------------------- + +FLAG_IMAGE flag.fits # filename for an input FLAG-image +FLAG_TYPE OR # flag pixel combination: OR, AND, MIN, MAX + # or MOST + +#------------------------------ Photometry ----------------------------------- + +PHOT_APERTURES 5 # MAG_APER aperture diameter(s) in pixels +PHOT_AUTOPARAMS 2.5, 3.5 # MAG_AUTO parameters: , +PHOT_PETROPARAMS 2.0, 3.5 # MAG_PETRO parameters: , + # +PHOT_AUTOAPERS 0.0,0.0 # , minimum apertures + # for MAG_AUTO and MAG_PETRO +PHOT_FLUXFRAC 0.5 # flux fraction[s] used for FLUX_RADIUS + +SATUR_KEY SATURATE # keyword for saturation level (in ADUs) + +MAG_ZEROPOINT 30.0 # magnitude zero-point +MAG_GAMMA 4.0 # gamma of emulsion (for photographic scans) + +GAIN_KEY GAIN # keyword for detector gain in e-/ADU +PIXEL_SCALE 0. # size of pixel in arcsec (0=use FITS WCS info) + +#------------------------- Star/Galaxy Separation ---------------------------- + +SEEING_FWHM 0.6 # stellar FWHM in arcsec +STARNNW_NAME default.nnw + +#------------------------------ Background ----------------------------------- + +BACK_TYPE AUTO # AUTO or MANUAL +BACK_VALUE 0.0 # Default background value in MANUAL mode +BACK_SIZE 64 # Background mesh: or , +BACK_FILTERSIZE 3 # Background filter: or , + +BACKPHOTO_TYPE GLOBAL # can be GLOBAL or LOCAL +BACKPHOTO_THICK 24 # thickness of the background LOCAL annulus +BACK_FILTTHRESH 0.0 # Threshold above which the background- + # map filter operates + +#------------------------------ Check Image ---------------------------------- + +####### +## AG : This parameter is set in pipeline config file. +####### +# CHECKIMAGE_TYPE NONE #BACKGROUND_RMS,BACKGROUND +# can be NONE, BACKGROUND, BACKGROUND_RMS, + # MINIBACKGROUND, MINIBACK_RMS, -BACKGROUND, + # FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, + # or APERTURES +# CHECKIMAGE_NAME check.fits,back.fits +# Filename for the check-image + +#--------------------- Memory (change with caution!) ------------------------- + +MEMORY_OBJSTACK 3000 # number of objects in stack +MEMORY_PIXSTACK 300000 # number of pixels in stack +MEMORY_BUFSIZE 1024 # number of lines in buffer + +#------------------------------- ASSOCiation --------------------------------- + +ASSOC_NAME sky.list # name of the ASCII file to ASSOCiate +ASSOC_DATA 2,3,4 # columns of the data to replicate (0=all) +ASSOC_PARAMS 2,3,4 # columns of xpos,ypos[,mag] +ASSOCCOORD_TYPE PIXEL # ASSOC coordinates: PIXEL or WORLD +ASSOC_RADIUS 2.0 # cross-matching radius (pixels) +ASSOC_TYPE NEAREST # ASSOCiation method: FIRST, NEAREST, MEAN, + # MAG_MEAN, SUM, MAG_SUM, MIN or MAX +ASSOCSELEC_TYPE MATCHED # ASSOC selection type: ALL, MATCHED or -MATCHED + +#----------------------------- Miscellaneous --------------------------------- + +VERBOSE_TYPE NORMAL # can be QUIET, NORMAL or FULL +HEADER_SUFFIX .head # Filename extension for additional headers +WRITE_XML N # Write XML file (Y/N)? + +NTHREADS 1 # 1 single thread + +FITS_UNSIGNED N # Treat FITS integer values as unsigned (Y/N)? +INTERP_MAXXLAG 16 # Max. lag along X for 0-weight interpolation +INTERP_MAXYLAG 16 # Max. lag along Y for 0-weight interpolation +INTERP_TYPE ALL # Interpolation type: NONE, VAR_ONLY or ALL + +#--------------------------- Experimental Stuff ----------------------------- + +#PSF_NAME default.psf # File containing the PSF model +#PSF_NMAX 1 # Max.number of PSFs fitted simultaneously +#PATTERN_TYPE RINGS-HARMONIC # can RINGS-QUADPOLE, RINGS-OCTOPOLE, + # RINGS-HARMONICS or GAUSS-LAGUERRE +#SOM_NAME default.som # File containing Self-Organizing Map weights diff --git a/workflow/config/cfis/default_noimaflags.param b/workflow/config/cfis/default_noimaflags.param new file mode 100644 index 000000000..b251f5b74 --- /dev/null +++ b/workflow/config/cfis/default_noimaflags.param @@ -0,0 +1,65 @@ +NUMBER #Running object number +EXT_NUMBER #FITS extension number + +FLUX_AUTO #Flux within a Kron-like elliptical aperture [count] +FLUXERR_AUTO #RMS error for AUTO flux [count] +MAG_AUTO #Kron-like elliptical aperture magnitude [mag] +MAGERR_AUTO #RMS error for AUTO magnitude [mag] +FLUX_WIN #Gaussian-weighted flux [count] +FLUXERR_WIN #RMS error for WIN flux [count] +MAG_WIN #Gaussian-weighted magnitude [mag] +MAGERR_WIN #RMS error for MAG_WIN [mag] +FLUX_APER(1) +FLUXERR_APER(1) + +FLUX_RADIUS #Fraction-of-light radii [pixel] + +SNR_WIN #Gaussian-weighted SNR + +BACKGROUND #Background at centroid position [count] +THRESHOLD #Detection threshold above background [count] + +X_IMAGE #Object position along x [pixel] +Y_IMAGE #Object position along y [pixel] + +X_WORLD #Barycenter position along world x axis [deg] +Y_WORLD #Barycenter position along world y axis [deg] + +X2_IMAGE #Variance along x [pixel**2] +Y2_IMAGE #Variance along y [pixel**2] +XY_IMAGE #Covariance between x and y [pixel**2] +ERRX2_IMAGE #Variance of position along x [pixel**2] +ERRY2_IMAGE #Variance of position along y [pixel**2] +ERRXY_IMAGE #Covariance of position between x and y [pixel**2] + +XWIN_IMAGE #Windowed position estimate along x [pixel] +YWIN_IMAGE #Windowed position estimate along y [pixel] + +XWIN_WORLD #Windowed position along world x axis [deg] +YWIN_WORLD #Windowed position along world y axis [deg] + +X2WIN_IMAGE #Windowed variance along x [pixel**2] +Y2WIN_IMAGE #Windowed variance along y [pixel**2] +XYWIN_IMAGE #Windowed covariance between x and y [pixel**2] +ERRX2WIN_IMAGE #Variance of windowed pos along x [pixel**2] +ERRY2WIN_IMAGE #Variance of windowed pos along y [pixel**2] +ERRXYWIN_IMAGE #Covariance of windowed pos between x and y [pixel**2] + +MU_THRESHOLD #Analysis threshold above background [mag * arcsec**(-2)] +MU_MAX #Peak surface brightness above background [mag * arcsec**(-2)] + +FLAGS #Extraction flags +FLAGS_WIN #Flags for WINdowed parameters + +FWHM_IMAGE #FWHM assuming a gaussian core [pixel] +FWHM_WORLD #FWHM assuming a gaussian core [deg] +ELONGATION #A_IMAGE/B_IMAGE +ELLIPTICITY #1 - B_IMAGE/A_IMAGE + +VIGNET(51,51) #Pixel data around detection [count] + +# For GaaP photometry +A_WORLD +B_WORLD +THETA_J2000 + diff --git a/workflow/config/cfis/default_tile.sex b/workflow/config/cfis/default_tile.sex new file mode 100644 index 000000000..ff3b25213 --- /dev/null +++ b/workflow/config/cfis/default_tile.sex @@ -0,0 +1,133 @@ +# Default configuration file for SExtractor 2.19.5 +# EB 2017-11-30 +# + +#-------------------------------- Catalog ------------------------------------ + +CATALOG_TYPE FITS_LDAC + +PARAMETERS_NAME default.param + +#------------------------------- Extraction ---------------------------------- + +DETECT_TYPE CCD # CCD (linear) or PHOTO (with gamma correction) +DETECT_MINAREA 5 # min. # of pixels above threshold +DETECT_MAXAREA 0 # max. # of pixels above threshold (0=unlimited) +THRESH_TYPE RELATIVE # threshold type: RELATIVE (in sigmas) + # or ABSOLUTE (in ADUs) +DETECT_THRESH 1.5 # or , in mag.arcsec-2 +ANALYSIS_THRESH 1.5 # or , in mag.arcsec-2 + +FILTER Y # apply filter for detection (Y or N)? +FILTER_NAME default.conv +FILTER_THRESH # Threshold[s] for retina filtering + +DEBLEND_NTHRESH 32 # Number of deblending sub-thresholds +DEBLEND_MINCONT 0.0005 # Minimum contrast parameter for deblending + +CLEAN Y # Clean spurious detections? (Y or N)? +CLEAN_PARAM 1.0 # Cleaning efficiency + +MASK_TYPE CORRECT # type of detection MASKing: can be one of + # NONE, BLANK or CORRECT + +#-------------------------------- WEIGHTing ---------------------------------- + +WEIGHT_TYPE MAP_WEIGHT # type of WEIGHTing: NONE, BACKGROUND, + # MAP_RMS, MAP_VAR or MAP_WEIGHT +RESCALE_WEIGHTS Y # Rescale input weights/variances (Y/N)? +WEIGHT_IMAGE weight.fits # weight-map filename +WEIGHT_GAIN Y # modulate gain (E/ADU) with weights? (Y/N) +WEIGHT_THRESH # weight threshold[s] for bad pixels + +#-------------------------------- FLAGging ----------------------------------- + +FLAG_IMAGE flag.fits # filename for an input FLAG-image +FLAG_TYPE OR # flag pixel combination: OR, AND, MIN, MAX + # or MOST + +#------------------------------ Photometry ----------------------------------- + +PHOT_APERTURES 5 # MAG_APER aperture diameter(s) in pixels +PHOT_AUTOPARAMS 2.5, 3.5 # MAG_AUTO parameters: , +PHOT_PETROPARAMS 2.0, 3.5 # MAG_PETRO parameters: , + # +PHOT_AUTOAPERS 0.0,0.0 # , minimum apertures + # for MAG_AUTO and MAG_PETRO +PHOT_FLUXFRAC 0.5 # flux fraction[s] used for FLUX_RADIUS + +SATUR_KEY SATURATE # keyword for saturation level (in ADUs) + +MAG_ZEROPOINT 30.0 # magnitude zero-point +MAG_GAMMA 4.0 # gamma of emulsion (for photographic scans) + +GAIN_KEY GAIN # keyword for detector gain in e-/ADU +PIXEL_SCALE 0. # size of pixel in arcsec (0=use FITS WCS info) + +#------------------------- Star/Galaxy Separation ---------------------------- + +SEEING_FWHM 0.6 # stellar FWHM in arcsec +STARNNW_NAME default.nnw + +#------------------------------ Background ----------------------------------- + +BACK_TYPE MANUAL # AUTO or MANUAL +BACK_VALUE 0.0 # Default background value in MANUAL mode +BACK_SIZE 64 # Background mesh: or , +BACK_FILTERSIZE 3 # Background filter: or , + +BACKPHOTO_TYPE GLOBAL # can be GLOBAL or LOCAL +BACKPHOTO_THICK 24 # thickness of the background LOCAL annulus +BACK_FILTTHRESH 0.0 # Threshold above which the background- + # map filter operates + +#------------------------------ Check Image ---------------------------------- + +####### +## AG : This parameter is set in pipeline config file. +####### +# CHECKIMAGE_TYPE NONE #BACKGROUND_RMS,BACKGROUND +# can be NONE, BACKGROUND, BACKGROUND_RMS, + # MINIBACKGROUND, MINIBACK_RMS, -BACKGROUND, + # FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, + # or APERTURES +# CHECKIMAGE_NAME check.fits,back.fits +# Filename for the check-image + +#--------------------- Memory (change with caution!) ------------------------- + +MEMORY_OBJSTACK 3000 # number of objects in stack +MEMORY_PIXSTACK 300000 # number of pixels in stack +MEMORY_BUFSIZE 1024 # number of lines in buffer + +#------------------------------- ASSOCiation --------------------------------- + +ASSOC_NAME sky.list # name of the ASCII file to ASSOCiate +ASSOC_DATA 2,3,4 # columns of the data to replicate (0=all) +ASSOC_PARAMS 2,3,4 # columns of xpos,ypos[,mag] +ASSOCCOORD_TYPE PIXEL # ASSOC coordinates: PIXEL or WORLD +ASSOC_RADIUS 2.0 # cross-matching radius (pixels) +ASSOC_TYPE NEAREST # ASSOCiation method: FIRST, NEAREST, MEAN, + # MAG_MEAN, SUM, MAG_SUM, MIN or MAX +ASSOCSELEC_TYPE MATCHED # ASSOC selection type: ALL, MATCHED or -MATCHED + +#----------------------------- Miscellaneous --------------------------------- + +VERBOSE_TYPE NORMAL # can be QUIET, NORMAL or FULL +HEADER_SUFFIX .head # Filename extension for additional headers +WRITE_XML N # Write XML file (Y/N)? + +NTHREADS 1 # 1 single thread + +FITS_UNSIGNED N # Treat FITS integer values as unsigned (Y/N)? +INTERP_MAXXLAG 16 # Max. lag along X for 0-weight interpolation +INTERP_MAXYLAG 16 # Max. lag along Y for 0-weight interpolation +INTERP_TYPE ALL # Interpolation type: NONE, VAR_ONLY or ALL + +#--------------------------- Experimental Stuff ----------------------------- + +#PSF_NAME default.psf # File containing the PSF model +#PSF_NMAX 1 # Max.number of PSFs fitted simultaneously +#PATTERN_TYPE RINGS-HARMONIC # can RINGS-QUADPOLE, RINGS-OCTOPOLE, + # RINGS-HARMONICS or GAUSS-LAGUERRE +#SOM_NAME default.som # File containing Self-Organizing Map weights diff --git a/workflow/config/cfis/final_cat.param b/workflow/config/cfis/final_cat.param new file mode 100644 index 000000000..8ddfebd37 --- /dev/null +++ b/workflow/config/cfis/final_cat.param @@ -0,0 +1,114 @@ +# coordinates +XWIN_WORLD +YWIN_WORLD + +# tile ID, for plot of tile-dependent additive bias. +# Can maybe be removed. +TILE_ID + +# flags +FLAGS +IMAFLAGS_ISO +NGMIX_MCAL_FLAGS + +# PSF ellipticity (original image PSF) +NGMIX_G1_PSF_ORIG_NOSHEAR +NGMIX_G2_PSF_ORIG_NOSHEAR + +# spread class +#SPREAD_CLASS + +# spread model flag and error +#SPREAD_MODEL +#SPREADERR_MODEL + +# Number of epochs (exposures) +N_EPOCH +NGMIX_N_EPOCH + +## Shape measurement outputs +## Ngmix: model fitting + +# galaxy ellipticity +NGMIX_G1_1M +NGMIX_G2_1M +NGMIX_G1_1P +NGMIX_G2_1P +NGMIX_G1_2M +NGMIX_G2_2M +NGMIX_G1_2P +NGMIX_G2_2P +NGMIX_G1_NOSHEAR +NGMIX_G2_NOSHEAR +#NGMIX_G1_ERR_1M +#NGMIX_G2_ERR_1M +#NGMIX_G1_ERR_1P +#NGMIX_G2_ERR_1P +#NGMIX_G1_ERR_2M +#NGMIX_G2_ERR_2M +#NGMIX_G1_ERR_2P +#NGMIX_G2_ERR_2P +NGMIX_G1_ERR_NOSHEAR +NGMIX_G2_ERR_NOSHEAR + +# flags +NGMIX_FLAGS_1M +NGMIX_FLAGS_1P +NGMIX_FLAGS_2M +NGMIX_FLAGS_2P +NGMIX_FLAGS_NOSHEAR + +# size and error +NGMIX_T_1M +NGMIX_T_1P +NGMIX_T_2M +NGMIX_T_2P +NGMIX_T_NOSHEAR +NGMIX_T_ERR_1M +NGMIX_T_ERR_1P +NGMIX_T_ERR_2M +NGMIX_T_ERR_2P +NGMIX_T_ERR_NOSHEAR +NGMIX_T_PSF_RECONV_1M +NGMIX_T_PSF_RECONV_1P +NGMIX_T_PSF_RECONV_2M +NGMIX_T_PSF_RECONV_2P +NGMIX_T_PSF_RECONV_NOSHEAR + +# flux and error +NGMIX_FLUX_1M +NGMIX_FLUX_1P +NGMIX_FLUX_2M +NGMIX_FLUX_2P +NGMIX_FLUX_NOSHEAR +NGMIX_FLUX_ERR_1M +NGMIX_FLUX_ERR_1P +NGMIX_FLUX_ERR_2M +NGMIX_FLUX_ERR_2P +NGMIX_FLUX_ERR_NOSHEAR + +# magnitudes +MAG_AUTO +MAGERR_AUTO +MAG_WIN +MAGERR_WIN +FLUX_AUTO +FLUXERR_AUTO +FLUX_APER +FLUXERR_APER +FLUX_RADIUS + +# SNR from SExtractor +SNR_WIN + +FWHM_IMAGE +FWHM_WORLD + +# PSF size measured on original image +NGMIX_T_PSF_ORIG_NOSHEAR + +# PSF size measured on reconvolved image +# NGMIX_T_PSF_RECONV_NOSHEAR + +# ngmix moment failure flag +NGMIX_MOM_FAIL diff --git a/workflow/config/cfis/mask_default/MEGAPRIME_star_i_13.8.reg b/workflow/config/cfis/mask_default/MEGAPRIME_star_i_13.8.reg new file mode 100644 index 000000000..4e4164aaf --- /dev/null +++ b/workflow/config/cfis/mask_default/MEGAPRIME_star_i_13.8.reg @@ -0,0 +1,24 @@ +-11.5 68 +-6 186.5 +7 188 +10 64.5 +31 55 +50 38.5 +56.5 11.5 +188 8 +192 -4 +59.5 -11.5 +45 -33 +13.5 -64 +5 -154 +-6 -155 +-11 -64.5 +-40 -44.5 +-51.5 -30.5 +-62.5 -22.5 +-68 -9.5 +-177 -2 +-176 3 +-78 12.5 +-67.5 14.5 +-38.5 50 diff --git a/workflow/config/cfis/mask_default/Messier_catalog.npy b/workflow/config/cfis/mask_default/Messier_catalog.npy new file mode 100644 index 0000000000000000000000000000000000000000..ef07eb032b08de4fcf508418524692e27f92ad75 GIT binary patch literal 4209 zcmb7{dvKIj6~H$S0z_V+Av_zF=Wf!F0D%A@VJ{fCEf+Q}Ge}K(fi%99nrA zDunWq77DdgDz+6YRTR<6s3U8&j-3Igj&%@bimd}oA2U?zlbKdM_uP9w(m(ve`$y*c zzCFM5y62pGb0i#Dx@=XB?^WN{Ks?dak_puMR|f+dThsACg+H)u`P4vJg+CbRYTc6H z4~^zMo@mjpb;;T0Jrn)RYq*17Tj|%eB$n*RU?!Sg*O7@Qw1q{{wyp$y?@C2G6Z9UO zJ*|SwZ1?}atFSw_CB3P=b8}@&M!PQkRQogQ9WJxR1Qh>397tE&bDwclCSdu1(AH6 z4(J=nqclLjNNy$y{Uf>I4k$$U+;$icOckc`Be|`!t%iZDp)MQ_Fa8fL{-b;tQzLp( z1zK|XR4X~i+6ssD2bS^mr21)@q|-L|e`)L2lj@%utcLWY`f8xj0Yf~cz08Nmysu}I z@{^g-7~8uRih{w2Z!HWBN-@I6ONt@pAt6OgeuG|8G~c*H>$1v0xQBsaDf~%ZQ5pK+ zj9#xu(RJZAt&dbL!Y~#qk)qmYi%L-!8?ToXsYiGI;Jwj}Di2{ei;a+?!V|0AWsRl> zym+Os9#lTUNERCeCHYt zJUw6aL%5fLQZM+Z%ud_-6Iwr&qT_?AH)5Y0Q~eRfvY5YD%-O+3yrf8%k}`2kQ5lS5 zF$!?}vw9tm%8M`a(@7E=S+VZ$>vz-ugb6G*QHqIP%nRl)Kw>5JofNeih%kx8CQH%J ziDe7S6Z!TxZ4=cXgaC_C$oH}-GAe=*UV1+mP=gW5Sgf4nY({85j5bP5FwOf$skxcg z`ew}uH3Z>41}X@U&gz10ej=di1-q4`b^hq(-Jdoa+e?F#{-Xl{Y#R$_Fm@dV%C0vB0w`Q0Pu=9N?r10aa z1fiP6W{_UL)7z_%e`NK-A8(x7p+->k!%PNhq!_BzUb;+rXK_@tft|MwtqK197itW` zEEbzBMTsX?^IN<3T|d4!Y5&KoR4Kw77ON%ms|DMaqS5$4ebivn6}t8M9V&xgsj&!i z8K@&*oT+IkM%@MMhIoJxJ1e4yYTsCB3-YHfhMU}t3w%~Us)i{I@1A>5M z-PR@B7L@6soDj-%^Uc>N)_z{GGL!si)iAfUj; z5chgC? z3g;oLX0bJ-x6I~7j4@k7dKc!~37<-tWQGvdGVllib}C9S%nNnqSzCL_=S?yN!a4@l z6EN0g|6g7*%bn5qnc^)}C2BrGlz|ul+q+-jSP-A>&68b=j-t$X$xCiGfE6@H@}#1%&`}s!}C4|3xRHS%ncEW8jNY^fzTg z8JXpd^5&~XtPx=|i*1pj&>|L+d5Nq!9g?sS0L0-4w@4|Wsp3qZx=0U6~aLVzCyqRm{F~3&hHc>Jfya3>+iC-sHUVhk$^KK6XW4(DKwegl8D|DgmX=&0@1}8gR!+><@H9 z+WAAZ9^qLAjuT*e#Csq>?XndVKW$FmD8h3Le2oCVEeJOk2$!g6{6`KDwc_*QyqTuQPCx0DJ5Z>D8!r%S;}ikXzqWafEL$@B#sK&c5wX z9YclwMt!znzPT4B5ME^9B?1bpOTA8Mbw^hpRnUe7=8}*^_$C9V2=Fdu-YK1(=oR%* zt|gU1IL*MfJ`;oX=kB5NT zfba?f-uzXxoJl@$H4aq@J{Kw+tpw#gl=-AJ5(Csd;(*Y{{>v*ks<&9 literal 0 HcmV?d00001 diff --git a/workflow/config/cfis/mask_default/Messier_catalog_updated.fits b/workflow/config/cfis/mask_default/Messier_catalog_updated.fits new file mode 100644 index 0000000000000000000000000000000000000000..6a9f00096170565aeb7fab3429c2256ce56f3533 GIT binary patch literal 8640 zcmeHL4NO&K82bxk?Xcg61^5lKgn(d%puWZoviN(qOUV;yoAdgbQ9S23R5C zGJc%mUqCidTB!L;nwUE8!IV-Emzk_l)0xbfKNvc7R`!19<}PlRM&quncFy*U!+YNM z`JV52-?M$^Ch6l6Vxt8?D|on!!gOJNj?I!|&(0GZIf71aOcL@OCY#x0H+#s(5YrnI z^eKJtg(9EHZZ{PQW|PAtI0|zm&n)h!0aEly_^(HOFCm{TKRZLR3ptjD%+_qlmS@ee z<+(|E-9@JuqYY7g@ELn0_VZRD>u0W-raGWHpgN#BpgN#BpgN#B@ZWcUA19Oa@dm9B z$n*vyu-NFpUb~tuWLRw`d!gVkWn@W$RN!zOPJ5+1di$@QpJ)8@^Rsdu5B7Y14xWX4 zS&}W&v4Ee)%@0rF0_OL1DSR_L^MwZY!H4;MT?t}*vH{QW{rFzUwCCjK3b`h`DO++# zcF*4nulyY!*XREJQ20EvxYZhU`smoGB+vRn9^rG1jKwNtnfqonipd7O(vLv1l&P%K zd;Z)D_w!ESQ~KdWJ`L|jpvgWz+iDYpf$N*;MLuI{LbS_2yUFFz!1U?j6XRTb_*Jc3 z1LNcR9`<9t7x=h77oYd^x%!)DEt2Npr~knA%k{bVyr<7KFN(fYZ|HOJc~4)6GS3UG zHswc^f&1s;18>Zq`uoOTE~2Q~q|Mla|%g za-h=JE%N?j_uw*?bnuLd%4718+1H`mV&Nd-x*iGk9UB7p&}C4$PlbH^_CmVeV&Nd- z=89JNK<*p#3k)C#Ou1FXUhvhhA8^?Mtb-}voMsd9Z@}Xk4L!{R2OH5KblZ)dP$=zM!+^*|xtz<#td_j+GQ1-&6ZGat%$AEx3wsX?S2kT*O z(P(-PbvTh(obuFT5Z{>yK3MWT-;4Ig4*6`)85)KG1QE-xCN>W;=W>HMh*)Vk)ja6M z@syJyu$7@FVSU62dK5DdMBK3RxExb`isC~K0*QVOCn9^$ zbp(OdhJ(xyr{#o-Fcz?5&!N=z7J3m$omj$lnD%Km2LvbsqXagOsg}PA-bQ25LIe>V zY_+_lgL<|@I z07LL-iY?$qV<2@4R4q?dbnI3#Dr+G=U=KWr-9NxdBlb3dj*Wz&SaOD!L_8P^mIHNs zdT16GHv7tAi0Nvf^(agkZEeVSNc<=n#-hoUN=9c0WOjG+Cb?dqlKPu$YDEYqwBwk(z^jfXBY2yG-{#>NlWyeBp( z@5@r2p?zSM!|o1OUcz)9@V|8#^oBHrL)mTU;l)flYcOm;Vdr^9f&XuELZ^<7!rdZ> z7d$Tc4;qc|FXe?ZLlMI-okBI`F@A;7=F| BE!zM9 literal 0 HcmV?d00001 diff --git a/workflow/config/cfis/mask_default/default.ww b/workflow/config/cfis/mask_default/default.ww new file mode 100644 index 000000000..c2797f904 --- /dev/null +++ b/workflow/config/cfis/mask_default/default.ww @@ -0,0 +1,40 @@ +#--------------------------------- Weights ------------------------------------ + +WEIGHT_NAMES weightin.fits # Filename(s) of the input WEIGHT map(s) + +WEIGHT_MIN 0. # Pixel below those thresholds will be flagged +WEIGHT_MAX 1000. # Pixels above those thresholds will be flagged +WEIGHT_OUTFLAGS 1 # FLAG values for thresholded pixels + +#---------------------------------- Flags ------------------------------------- + +FLAG_NAMES flagin.fits # Filename(s) of the input FLAG map(s) + +FLAG_WMASKS 0xff # Bits which will nullify the WEIGHT-map pixels +FLAG_MASKS 0x01 # Bits which will be converted as output FLAGs +FLAG_OUTFLAGS 2 # Translation of the FLAG_MASKS bits + +#---------------------------------- Polygons ---------------------------------- + +POLY_NAMES "" # Filename(s) of input DS9 regions +POLY_OUTFLAGS # FLAG values for polygon masks +POLY_OUTWEIGHTS 0.0 # Weight values for polygon masks +POLY_INTERSECT Y # Use inclusive OR for polygon intersects (Y/N)? + +#---------------------------------- Output ------------------------------------ + +OUTWEIGHT_NAME "w.fits" # Output WEIGHT-map filename +OUTFLAG_NAME flag.fits # Output FLAG-map filename + +#----------------------------- Miscellaneous --------------------------------- + +GETAREA N # Compute area for flags and weights (Y/N)? +GETAREA_WEIGHT 0.0 # Weight threshold for area computation +GETAREA_FLAGS 1 # Bit mask for flag pixels not counted in area +MEMORY_BUFSIZE 256 # Buffer size in lines +VERBOSE_TYPE NORMAL # can be QUIET, NORMAL or FULL +WRITE_XML N # Write XML file (Y/N)? +XML_NAME ww.xml # Filename for XML output +XSL_URL file:///usr/local/share/weightwatcher/ww.xsl + # Filename for XSL style-sheet +NTHREADS 1 # 1 single thread \ No newline at end of file diff --git a/workflow/config/cfis/mask_default/halo_mask.reg b/workflow/config/cfis/mask_default/halo_mask.reg new file mode 100644 index 000000000..c44f25167 --- /dev/null +++ b/workflow/config/cfis/mask_default/halo_mask.reg @@ -0,0 +1,50 @@ + 274.66813 -1.25966 + 272.54579 32.47406 + 266.21222 65.67579 + 255.76731 97.82190 + 241.37579 128.40544 + 223.26462 156.94408 + 201.71942 182.98775 + 177.07997 206.12573 + 149.73486 225.99312 + 120.11532 242.27660 + 88.68848 254.71937 + 55.94996 263.12519 + 22.41606 267.36151 + -11.38436 267.36151 + -44.91826 263.12519 + -77.65678 254.71937 +-109.08362 242.27660 +-138.70315 225.99312 +-166.04827 206.12573 +-190.68772 182.98775 +-212.23292 156.94408 +-230.34409 128.40544 +-244.73561 97.82190 +-255.18052 65.67579 +-261.51409 32.47406 +-263.63643 -1.25966 +-261.51409 -34.99339 +-255.18052 -68.19511 +-244.73561 -100.34123 +-230.34409 -130.92476 +-212.23292 -159.46341 +-190.68772 -185.50708 +-166.04827 -208.64506 +-138.70315 -228.51245 +-109.08362 -244.79593 + -77.65678 -257.23870 + -44.91826 -265.64452 + -11.38436 -269.88084 + 22.41606 -269.88084 + 55.94996 -265.64452 + 88.68848 -257.23870 + 120.11532 -244.79593 + 149.73486 -228.51245 + 177.07997 -208.64506 + 201.71942 -185.50708 + 223.26462 -159.46341 + 241.37579 -130.92476 + 255.76731 -100.34123 + 266.21222 -68.19511 + 272.54579 -34.99339 diff --git a/workflow/config/cfis/mask_default/ngc_cat.fits b/workflow/config/cfis/mask_default/ngc_cat.fits new file mode 100644 index 0000000000000000000000000000000000000000..f51546da7fc88ae1ca919a3c13882fe1a7df6768 GIT binary patch literal 201600 zcmeGF2~?NW_CJoJC^(`d4mctzj+u&vn#6MsJc>gOJW`TaIFXu~17=1VYN#lVC{9`A zfF^3U*7~iL>ptto=A6A> zdpOfR16KEE`t~vd0zw1+=3oDSCjy4PGB*B|31cP(B)k&PwR?XnU}D1HvBL&W81^^i zL6Y73_w7F5Kk(}aeuF1W7(69l*x-c00SQya4gY&CZfb8apu2@1{}x{x@Ebd6%#h&| z0$z!CaUMBl_}GafUl}{m&EzltWe(_X_Kx@u{QCc;;Q#PHnc4ra>_5f+vA{nT_{ReO zSl}NE{9}QCEbxy7{=aVlnoe5X`}7VCXezt+?k{97v*};7dLm%R$gzVbObJLBJY@9n zfZ?wt=*j6{iu|Ad`Ty$tx!wQ&^JkuqxBdJ1bK4N`8$EpNO9>-rK0eGf6AwW7|MZXf zwfh@>ZQHi*(EdO0L;3&ouWxvt=-!ye{}0v+FHLx5(zt+egC`6gGdyAVguh=ebo>9s zukHU~Kj!!MT-<2;ckOQWim?73Ux)vLpAMP$k;6w1`_Ekahe!AB&i-gRZ1_to=dbbS z7W_Z_FXqSo_%HGcCVw;?JYndVkz)e_{`dIW{}=i7kLhb_`%M_EZS=px*R@a7GnyY( z)%@>&$B*#2{4w;uz>n~0e*Zmu+P@P=zCQd#EdBp?`Vv0P@4tsn#|z_&`ETIU{Qi6R zI&eI{JaQ~2RsQ$()BMDLBmVx$ZXxzixrB3$GW6Q(0ez6|7tpwsPev zE9_$Zz2elE+^{4Mk1&5CT)25il=*6ESZeMuO}OIgE6V;pk$!lZtK9R-la(jFLG{Ce zgM&?@R`xVmw#zfjG&lE{X_QTFpo;r{VpIYT4-XH^@R^e$4$l1-Ges54DBmR2@g6(K z08_?cRP44*R{!E(VYY_{OED^5k1^7gTU-U;EThuiC|UO7iAJ^HUxETctHr1ZQx=%Kcc z>e6aXiYz-~7y%<~Eu(H>f;j#D?66GD)H3QnxkmhJSO7Bx5rmtE)MtC1F!n(?CNwx6 z;tccqBy1<3L#$~8U8p0z+%$}G(2~t@Ljv7JmJ`7|UZzeZ5xO&1b%ycChkTEkj*#08u z1Gi0Q8in>3eKv%}$H!}6b9P-&&wXxi37FF|7NCiRz)1zJ){t5u)GmYsGGQu}SpIc} z+EXEvLmO30W5q_H_DtR9YJ`@t_>XjPZTfC%JWq3p#cMppwecNDq$VsG(^*_wn#fha z)HIeL=&r$0umx2Efq1;Ph{dwS(gFj)v=LVUeA7rfGL);JKIvl_OP3rL*Sg+(pRn$*MJnDbjw~neYf%DumDycd@N&)SDq-I(u$P>({Ll>$u(-1_>-aM z64G_!)`wXOI=OASZgW;xbaXUV2`1ledNEDxO4&kfMYost6MV#u?>}Z;n8*9hLb21= zmuwLVf@NeOaCbtijrhQ48d>r2VwZn8cYAr1P!uV4Rj$FVgtp*uJVfk5UkD6Zx%baz zJ96glRDy`zjm=k!9l5ZYyS-$Ej(6nyVQf>NwTx|Gy0iVWVK;Q|U>RGUIO51R_b`Et zydApiG)Hi&NH!l@A( zO!(x~Y4u{Sr`&$|!YCBI8<}p;efo8Vy&gSGyf%(1pUvGVWI1sCEHIs*7+BJ^!xPi^yWs?kwXT2H@Q-yKogI z{OOlyufM#Bbs^}Y%xBxA^d2rOj170uDT(r4dFs#ZHw}VLRfIR9@aB=544n)l%2&Lu zPLEq}OhMa+2`o_#eNdfl+l+~@M=n))giyX`E!AUMPeuywo1((zEY(W<$=y~a!a-E5 zc}{sfJH?2@phe1w3bD$hiy(&F$z`8CnqVixkC|^XXUAn9*lk1qQqlM z4y#=`dq{mEb3}!vUvc2F)&1vh6qk++A;*}glV3rKn0^Af7(b%z3;%6?Mi1~H=*+af=O{HQ zX$d`9*~>RgXeRVxl=9&FRPrFN^Tq2o!cwnyFb z@nM5Hdoxnjw%$|wjcCl2NGz(DqF%=+L+Xxzh^DANI8s%s(}yEjS9#({*hsV}*g_+% z7ZwyY3=N2+;Il-%S$Re<)CWWQEK$Gx1tapCpP9fV>h}#XI--LuK#3$$uD+EK7K`5U zJeb-{fVM-r&U43DeN1^`(a{5g_I`a#zAX)#nusMDPH82B4(to`9x^iE7cd8~e-|5o!87jd<{ax_ zY>LL4Zi$)d&xc)(XmaYbIQr))HZtqdNG^2ZzU$*|$W6YSp*F>SY{bVu}_=hRQ~-uHgymaniOq8qk1M2iPwFW{DQR z`N$^K284(Duad8l`Yo5Alcl5Da|HRXT-8^!7`Hy>5Y{h1Nxm^f%e8@GzgHeR3ihx> ztG*#(f3*SJIYO)@n&%!<2fS~SE;^E1qGQ{SUJ4?0IJ6#HEPmTjm3c7J)=h;Nj`%2% z0vt0(sr{JqXQB#+L(B9wYJcp<-1b?@<7iQ#eLrU&NfZu`f8YF|N8}0agIKx8cU^EI zKW<@o06JG`m*ioOTbIv%0nrle1I8GKR{!Y2VTq0vuZ!ZXGu`13;=f8w!Hf+kbrg0S zbUNpI48Q~o3R+fY(`cr6!Ek7sTK0J@-{5L3IjC!8p z`#;jTMKScJ+h0&$^g1RkAfyVYO6WJy>i4+D+!rz-tf^2T(|Q_RF+*uZVvRa|M46l* zcb8*eJ$EX+q09Ry@#Gr2Efx};blbOtv@*o0l-lG1?S}A1`QpK*Ta1)-3GeK!`eEFM z#dQM`!?K-P*B)b6t|VaHo;V^))opITsKnS3kr`v`Ow9Un!IXLQXm(3g7}NRT#98z> zS}W(b`8BvdWnFr9_O`#rkmLmu`V)^{_4nFE$3xsbOs`(Y>@zi#I~@9=qu)KggFD8a zX_BhSVS4 z*Qj|!VNG}wa|if@BES^=e@IhNYdo0;J1QEJyeLfa0GkYLis(PmRn($4*2IGJmEHMIesT(7)XvvquC6tF#(z=(k#z zSF8;C`kQ0PFFYF;uZ9KL*U=4sjY!TQH%vgke)IVR9->*f3H>hH-=RZ+Z*|a3#MJt+ zdwp3fBw6B7LaR9S*q^6dNW;ZM@c8z_imnD^ijm9DseNc-2sn*&WcD<*FYgm}WiMhn zY4T+Id(2=-IVOybj&>G3e?Y_`)-iE4#jDR%u=A%JWaV-hM}m*j|6UOrnIn1f;7Gdw z10P&Td`vO<)4O&7G9>t|Zu^r96LM~Ux11ftQeIo#$DUo#k2`uMOj(zob8l)h4hNPp zwMHAULoYS7J>Hmf$o^uW7rC;80O&uw)09mJv6h&c;A4MLn853(0wN?W3dsrIv6jWA zQUgjHTkOmXv8glB?e?wYyXq}0iu`piWr^vr&2yfa@hwLoGfjGDuTu%YAXa;S=6jJ! zR^G^s(1To=^ih%Xcc~>v>M{ZFO)$G=g*$9{%70VuAF6igpKhW#TH{6a}MW8N&tYNZiFeF zi)8(@0X*RBB&2gr@0Dc`QYB0+F()#`SoB9ai$yg{q?AjMWyTG3$5;xcaHTP8gC5$q zDR-h|={YA{Vl6QjQ(DhbyWMSW?zvRy^Ftc1_~J<0SxDQ?vzKrW1NAL2KV-5jlhmE6 z=%AZdr;k&m<5@OxDp_p)%|o)x+s!$WVfCS6am89T^02HNw6(+n_@zw3vxHRhNNrv0 zRK8Ww-5M6OE>?@%40J^ym|{8m>nA4-5A7&REb4bzdQ24SyHottWzz+9fgb6oIbb|u?_g)9qafvV zZ)e2)W<1`q=N6aR?TpC#gbbvarv2H|C^NqXG1LZHda$Q)VEsMj0W6kCYn<)WTn zz`!?_SiVt+I}m%r<)~#?Cfr0K(!Qd<3Sdg`&9 z<48l{x5U^KD116Kf~pqeF? zVryW~s+Qb5Y_iAFuQQxM>B_Ck@=M9a24tQH80^|tq+w#TWLdNrhi)97Dc(lzA~lH< zOu8RiwY9V3J*XF3j6L`EAWviV(@VLD0lq2LTw5<@Kx~pr{Z$i#otM`waN~IA**1pN z{U#F%D1zQy<0(CT*STZ;UH?^L%nh9`1yhv3)cpA;j__PtS3wGT^b*}$vKumPt&yd_ ziFU;ps^le#w_=;~1Rly+BBN6)lA!5vtp>F6ISVktb@OOJ|?bgn+ z;cw4!n`3P=dR|vMv73M&wD;F0-!&$O*xdDTiT4_3tNPX7VIJ&;wU|3pi~Ep)p@byFkE)v%m}`E+|(U-O)ig%lEsYu~Qj z66cgdNI@id4;E{Y<#@xDH$Y%r)=vyJwm$c^yUP0BGr}fxooB?LPfjIAZTL1vmf3#A zfMUAJhEu0y*)xG0Fw7%!$bIQmS0~)rV(X7y&w24mt~>pc`C^(Z3tfl@K1*Z{36y0| zDh`-TGU0|kvBnlmN5PPwn<6tONS57S)>Q>1uv)8%IUPW0T{5r7$g)`QLL|XQmEt3^ z%<<3N_So>V5H*qQp%P$fiH*_GvaDAgwJ7>bv2kD@S++qbW(vmOacrZgakZG+t*%n` zu4unexJzuru;!idfJ$hqZ^G!|-L9%z%BF^qqW6|xjnQzFLE66GBUG0CX@R>B+JyOf zEp&b47q-XylMczUCFx@(Y+7DHRL6V_QZ)2Uzg@`r3K4|g>f!bMW_64Sm`akP3P`#4 zv4TGJPE%U!>b&Lqv$u%`=n!BWEoJle^|GA*8SYR(L=~*FYoQ8$pbrFWmDItK*oLE+ zu<>89P9+dlLJI5wi=9h#Ds{~xN0IsU$68_y7+}Y{k|OMKw zu=L6V(o|x)JvusPBla3W&~DgKD@T?>XF$O40eZCz)fh(;;06u-0|c(m>Al=Zqnasp zbV(QOkb|WXFrg)~Zv^K|C|R1Lge7)j_F3Aaki{|~`^hyad9=cCXKtO}vM* zwk&o>?F7dM6L_H14QThg>mn<8JNI;4BKzWTr}watEVdUR-Tiz*&I6;64aD}?b!&~< z9UX0~MDs8R*q#M5q}Qaj959UIkt%zxt(Rrb-(lt0Q9ECYQcHARN~{0jwqpA`yE|=C zgC_!J&xkrXW51Z?9<=sQk_kgnfu4<9Vqe|nvNDFHO3;Oo?(3hZ3?xKSL5L>abRm_ku{(-^8h$Y9K`71mC@MkhL11DIlgz!ven~B?0%%O zF2(etoF#39S|aDbc~U*-WU;L&Ei{mV}aIOD9VRF775(bS=sjW_5UyvHmK2UH0c>b*$mQ|TD@mR!O{1U5hOGPf5d91NKZ9)}DMFty2*IXG4jQ0Xm3 zs!QOIO%vOL&!M2*0od?Vr|zGA+Pesrun*@s)gx2fCj8``y|VJ8*WGE#PgdlpJzz>m@mb<< z$YlE(PRsnTwN69MLN{Ec*R4s#g%R$J^u>tb2s zh|X(6A4~je_BLa7Nd}&!oWOQV5LTW5rJ3dh2g~v}ngCt2qfVT;ul68wMJ1R=o?R$? zOU_)eawlT;y5)=3zYAE--@S8hPBC1p))`fZGBcJ@;tNjfdp5rRtVz;{E zb8mB%0y5BPEM|6m-<;N_TK&Sv6ww!3&mfSIEm3fxjOB7C?Ec#-T+e3$g zDGI;N5Fgw4>h zLEw^KQmMpu*uZ=Wn{5zDpC!J3ElQkf*@ax7Ln?nvXUD4a6lI;rCP0ca69v9S$dt3F& zG%uuy9;A$@Am z;yZg8DT{r?4c{RfSkW55lL?I@!ABXG80G4zf+c>$QLm~9+=on02aHj5j6!Z=tnKM_ z`S$eB>bi9)Lb{{s_JPb4I80F_cFU^M|8yfQiinU^-yG{|gyEtHi;=1r#a8OsmL-1J zzTP;3yyXozicNSLBTzZaq%+}LxWW{tvB6!g?gZ)?=mU>=*Jah!KT?a*%Y>p2j>xKK z^_cBJHZ96MCaXTVhIyb0l)D@!Qjz?Fw%TH6a7wLOx8d%VeSUA0^3?~IbRTpEJ%80R zySS;NnkCMRJg%O>{tFDG-B9%Ve$f+E;58kmKOs+7^->oStGlf;7@lW|KWPR`?zF^D zJ!^?~KJs>jC75Ezd*F%ZjdTQ^Kh00i%^W4`%`w=ONu)e4I<*h0tpPwTUtDx>2lzWxK@+$m30wRI+og=kY; zxEv>|b$FejGq5wiT~M=eP6?jZJ9-qlbZ5jzLCO42}N;0f4 zqa`k$xGc*e5dlYQ92a52GF7gU6zzr!J7SDo^_H=-J28%98;xCFd9H9U#aZk$4>_z4 z95U0hgCm_G2lf##Duohaw==}%0gL<+E-qoaB^dJ{C`J*`U0i}2YT_6Ue$cvH_{iH{ zaN`Zm)4@nM+BC>h6=SM!1KI+dB`(!Kz7T8VT$m6JH^rse39`(;V%f+n_7XBiRo*|& zJwK{h;%xRbXDH4p_-j|frgow7MHE}QmqLl#lT|V{Fo9Czc*G6=e#A;#LdwMV#UAVz z00WMVLYAFAnh72hK^L$3*;_F1;kN|NdQ|L{RSDv=My7kAc?J7oWfycZP@g_iT)~cn z|L|}M7#;jqkm>OTflLUrmMCsJP25IK`XThI6NANaWF=9Bh^Dxj8822;i8at(j1~+iaEnlm{5G;vN+IGxWdvB*RW6c`?$xbBhW~% zZAcaOa5R#{0y==boNOP&egme~TK!*JCW|8Mo5E`pc;JCn*#nygwrh;K^lTgFrRK+} zMbgk_iR&Gs!~>sB>};R~0Sn&qyPl<;A|hJiMs&3J6Ip!lAkn6{4y3PReH)At;A4qj z-`T6G;-qWPF%M>X3jyQNM+Yt} z2BstG95{w!!1#d-4W`4g50*vOmjpsQ;kf-~kn}IVh35`nYKg1ueVo|`mbn}9ZTKZA zY!3%NOS$vdVQ~wg4NY~+zJu=bJkmcvNx;CgLa3^bBWFOWW9+tHqN@KxDIPs~k#ctn zR*9d_-{BqxOigk7OrZMc&9SVUKOwz?ZJ&zHLMAZMdpMTVA3MCzgKT1odoU0P)DeW{ zl7a4ecqrfOX|7gjiTnFn$#y+Lc>rdyH=x{{AN4|$n99-_W7HcsQv+Stx3cu6I%;aE zMlPh`(xXYL_@#0UCO}HKcz`4CDQlLwIwebbV0BWb&X1g7g~lMH1&fP6en<>ied#eO zR19D7>ci$CJz`dgCop|N3DUbBeK;_oce`GZNgI6 z@DIcg{jfx-Jrz_S5{TqEmVzqA@4mhqQb}YFPhI^|B}T?x(va&hZhLrcD^`9bOHn9@Dl-;3Rj_P25BUDX)T_!7 z@dH-#K_GySDa&F*Hv}sjvMHLxQ%4_oQgU zu5y(pMsZZ7oH0h6MXwED+Q^SoDGzJCi(~UJpGN9^cU8`ludQb2Nu+K0pgb{RP%nZ` zU$`v)cxN#J-35tdVBXj~SRh8bk(Nt2t~y}Cnu`)_k8gU|l3(&(zpVVWr`OpR8Dd1opm9R?MdtobK z(ArkKea>m}wmz_&N>VDX-fwS5!W|4L@GR;3>Kgf4wTmJE*$sTAtc>kiuQP$XjASEM zdA5yg3TK~pHRQ@?7OG4%4-m)=-DQ_DJAuWrwpF}^+OYkK zyG2z&BJ+Km$UcNAek;<}H?B}N!(ugFwW5^rrR_K@Q(1}=C<;oBHnIA?*ljAntt0Tz0rOZu zp?J{iDfS#B0T|Borrn;*+vrdbOuOcZ0_cJW(pCHpjglegKOjN7^05WQYHCS04l{y> zPq8|TbF8SZy=NY>P8HcdW zL)#?A@otciZ0E;E7ZSxZapG3IF%4*MaRsype6xXp{?bpj1&X7Eg*%sWWEmpl*ih3MVrxF5~V)8u#DQJMwe(}Rp zvKewpFfy+zWsPp5*8oQ(IE>Es;6KWxCT=`xR2MoYjvgY;><%fL_m{5mh!}3GzQ#iAE1JJod zt<9n0$Hl&^3tPR$TiN!kbKg+F(AQnotQF-{MPkMaz&t3%YQ0=bEo-1JqA;+UpWjmN zB1uik0hlRk;cin3Qa7-L_FV1S-m?7Vn(Qc>Tv;3ElRe?v0_|H%)^2-Qdj9+s$0;+d z-7!b}k~opqbKrv#(T8MdjJKn~lyTG^vr&4UxXe5N-ID%sg~A`_9a2$(dH6p_6B|!F z&p2RHl<=9MS_OSzpgs}7^z#(yIXIH7p2WZ&Asp*0Jzv<%eIYZgxuJp6t`=uD39TjT z^e&c-cDH35?3Y>(g47cj=fF@mu{v12HhR4;hm=1dt$$^stnkJU9NJu^_MXY&<380G z80%8|-85AR-39SLi74s6c$!nUbuph*VH~w~20Qgy58}4Zj;i~0uq@lB8f%+M%GF6o zRMl}WfL3d)t^X$x>aChO13WpHkfosHYr()K?-mzO;l}=yC57awjQ?8$|L0pJ!qBFSJuNWb6MQ= zq7opQvVPz7Mm6MucLIm5@(7O&sj3a;Qz)*(p~2I8#jj6Ru^q8LSHcw9 zww;!Y+g06nTg|OdB zvb1m(m>%#neDqOKjkJDloKYFGVB!JBC^0w7{ssX9hPp2dh|iOCk+-~z{CWVzN5Go# zvTo=;_ARP)mkrv*h{qSb$%BPWJnHtp;0(msFNzOl+MwTM;{ zErUF|Gj#B=WFss_f~v&2JA(iuM7F`V1zI^6X!u>_g_|V^I+~J$^X0hf#IXX;sfK| z&Tgn8pFV@>`3=_;F7V44;_Ql7dA`n$YE%(NG;dFKx9mo5rpfAqmQj4s2P5PBSg9Sq z@F)hmnz9j&oRvB~j-{~rfh|Jq9XKbAfYC7)ICPY%j&TkWwD$vRC#dmQi@|GT6Y$8p zAWI?E!H^6MLOw3D;ADln$~M3K{bJ7W5i6T;1U^VpWpLq=q_8U+4LWVSgYA}TAl#s^ zY}7DPRT^@ib@5>6$*E7T!Cf!Z*T5QOT@Tydc%<<*8W82_WFu^1%)sRh6xXex5$1){ zam|E~vT}_O4x@2{l1c!aC4)*YlmSQ)5>k``p;DX-K*|Iqw6hxr2CAB9>Oe^)pxlAe z>H@|}+Q02eQP`wqEjt|@>_Z++YF1EFP(0Tqwh6YcOBY2l4_FLUc1NqyS8^$6=?j-l zwr*6Du+fexI_R1_z!7gu6j7Xv1Yj>G8}XPsoCo7f65A9z&5d!H#fC0PWFV9{_+ZIl z>MqEPC{cC2n66LUY4w9zWU28;bU)N3XtD3Xsd&!-RrDmU$%{cMH6nsbu*I58J*`ro ztKhEEEFxQ$-{!}jW3f$NU2|{=QsDrlaWu^+G?rqKNCYSczkC(ryn*t)qubn_I!66gpVwL6)~b_+}IHfbEq>z!7!0gR-8F>s8I6a6uP2Pj>o#flse zjlkUr0^nP+B~rgp7;gzW+q7k8Z()^c6bexEh0B&96@-QUl*FQnDVyI*P!iMgO0WlO z+bl9gE&Q2p`Rd?rj{CPS&3d?JVagVnc{!&b1=i9fT4ATT8BXjH(o8bpBZQV{mB+Bo zPC}frE-kaJtK|0jun+j4N}nS+8!&OkZ?$r*PI-#{xDQmKizVA;`x-Wm69c+dt}X7G z+Iyqigw_d(GN|7=GBVAkP1$-xiVVUD_Y!TxW$VShGN_9_^9_5L5>j&31w5dRk)VTs zL%x+ThXE)ZFpqx_EBa#ho=PxMT7dlkOvlQMiNSbFd4J zXc92!1H%8cMC%1J)Uq0Fc-t6NV8X{@?3*w$A!V^`ho>HV2k%Vy>##Is+p?)@9QI8~ z3W%tJL2KNTYsk?$VA?K>weRScD0GQ-Et8Gskm`i$I)d6^Gqwo^VCbQxw8IgaCb;?d zkPy5xLmWBx4X5@BXmZuAZgVvr#~LA0M{?UX>s7qIO{%4||MaeGh1(0zLkC@lj!`lg zcZqFC;lS1d5Y+QxdZBJq#;WaWv6+Yc_9%* z-Vgb`k9-{a`!`^9%>%7r%#i!;0l3r5eAyD~>|WpjGg`7^%5m8oxg4qzOnp#CoQG*X z@eOuEE+Kt#eS})xI+NG?6RAXpf>ml9kivvyBvW?4?b>lTg9fI$MTJahrD}h)f_$q@ z7@;ENd+8Nf>;n<5^Bnw z^4ureOc-0NUdPq|iDl(NaJ}?(lz50VBrQX|j%PNYsgALb6Vr0$;~MmNS8E8Fm#mD8 zF@LFo8>s@j<~v~^e3tCoF-JB<+Jy)JX37wx)2HBwW;GBBsHY3LkfOrA|CoCR5KY;! z;cj&tcOwBzcNd*1gxdeY1)&mPmh6P<$MNn%i>B<Bq#ZYhDiiad5(3o! zcO!RhK2%lK52R?nbiu&`|JlPC7`w7_+smpJ&eYrhQec5DmEzQt&+2mjkxIIBR*~vh z$x#hm;)k@Hp1o&q&H&Xc8H%uMv}CH=4WZqS%i1e)*n~UTKv#=-{gUZ&!&A5`>W_dq z4Mx{b5B1gZg8tsJK=4x_`-IzGC*5^K!F4NUx2hjy3A z=L|3G&?LFqeiyuFFbCZQ8Hly*a{aI!iYXivBW+FDrT+!{Ev!9If~pObVVzdVCU{7r zga=9-Id05hj35VfFQE19@?FoFe&Y?`>*5mer7;Tct6-GnUqQY#;^}ni+4q zc4!#KBPGHnR}ii6oCERTDo_77MzzF&c0#(6;`Hfr4eUl3)$xNa5%zRX<%1^>Vn9GC zs2>%hd@xu%EYbO`2q$@dXNn+gJMn-`ENLQeo8WRW>T4sL z3CK*ms}I&Q^l&^0F%5GV6xSufH>IfYxbHz#{3$-n`=@0PF0w%}id9Q?ZImyYVrR94 zV9JQn_w72pXRy^lLH$KKIn=u#t~V& zNCv&79|XyzDv>yX*JQHI-e*Te%0SsiaVGllR$C-8W~+CfFDPLlO<;aLx5a9QL6SU`%3tR9~D# zpb{X$>gd;3z$euZFbpgbhe^E=$!;FaZrPs?E^s%oZdjbH?lH?ajKKo=K1)U}ub`HF zvS0|Bm)1AIK z`gj8~?TQzd-nzd`P?hs!kI0%8oaC2=leRj>dVCOTC&Pp`8W7N0vS*$BYBOGt$ptV? zcyG5;?XPD9dTvT_s;zyQ+^Bi|S%C1No)%*1II0>;|*u8^ftKh{Sy zA=;9?angP>W^BX*%9*k^)?%B{<1W%}Fl9fyQ}H{_^TRLHBBErJw~$TyC$gj19#K(; z(OuNyNQM;fz_mRe94R7QP|cKm=~1F5*BBpaf+hQ2sG}r?lbz5dlft2Io&8R!#v_?& zDz&J-SeoCNH<+&xbfwO*&z&5{Z)G`d9w=eSn7l-}xbb|R(IW!D@0Yk!^kGE?=0Tl-l^Um3VRPxB*#4C}t4vG-W8i(5z>@vnNLLf* zq;T)R@ew^N!hRR2PC(ZkY}D3`PWLY9JjabEi$zD?QgnR~#dRb{zdzJo`)zBsIx~$v zdR?u)(wSEl;A6>WdnVhTqanjuWJXg)<9rZ3J%Asyfu8+3Sk=REQ%KMTdiMAIsvdH% z&_l<1^xNr9Rh+q`7Im0$AZ2oQ-gOqsz@Ed47@tj^YyblVFT~_W%4 z>~{7GBYiG(yjttdDVg#S9s~Z|tEOS5N%VqpH^}M(GUL_8CoA&Uz$FG$4RI@%df9JGzqW5rYH%IA6$Zj&*syXt!EY@&pt7Ysu%aeo4clss%171M8-% zB{-x9kva;W$J-ryLu`8efqDptMZ2x+bR)5PG#&W)EqfV`6%Y@I1=CZf)mNAXo`){r z143dSr*iIF#`kd4apEy>#(lMHdLxQc@(l<$ny?#FIH1%4GjRHDwKU`jZiFCOav-MG zJ4-AE(O~+0y5sX=DPAKR#55MS!Cphc%ytFtz#)O^UfM+pN$t7Vb}?${`jJK&;uLvY zQx0ksmy?ML4@s!Po*UHKFDDc2mTU^K&^9N{X%nvx*zKhFh{Xi{lnI z(!PWTO56xmTd{`&9J)`A#kTRTnWMPZE+Ab7RlKft+lAZ+S2B*@eC+ha>$o!j!ICeS z4eaH30F;;lhb3dNPgnydW#bVr_)Pi2)bZ-MW7jx7H1y)oT!{*XT(9A54?sd_@d;C@AP!4jLM?!*WeN_mb>Nn*3<{NY^pN& z0tl~kWaupRkt&1HeeT6>6Y7&qEE$)Vs6N00T(AYTIa3bCl6Xl&ebFG7#+Si+CfhS` zIIh4IL64M(D?Xy?Vb==`sUd?0w$ZC%XiS0>5FjNv)oF*TNH<_H@PTrbj@fVNJgzP= zc=di&tz=3IEa0&!K~>u~iH0Nk!sU?qJ?$BAC4MN8g6#RSs&8^q#b%4)c+|Ag-M9fk z5_tT3iAQieqRWum6;v)>)gWzCN&O+6y`9JK#@$Xphd!1ZHZeoZ#72nWGGYACeB%df zq?V|UDl*V{rDSVyC29N8h&t+3%uonAl(XbWY%G_$vzn!_fnNHtP*uiuEkx>OI-*0g zeH4p#)Te5u9EamkDNVQWJctP+njcqHfg#g1+j>;Wk>jd7a^u0eAzO0jD@U9;$X3lakkSAi7HZ`p^rklEsy0ae$Z|h)gVZf!*&Y@ zx~Y#!?4#C|v|ETa<){sJ)q2n0*)QN@$q`8YgtV`>nn)NVyxsIhdE=aeUV zp9vH2%Go{~6}17KtBh}vMOUgM+N>PMQgF++U*8ZmGT@tX^a4NUEi9Tz7p^h^<4biU zBJhK@>4a59syed84@->0bCywPQ7d6}=8=F{uk=`Z?oX2_EGOZp*HM!nMAasoc&V0Z zhhyRoc}$#}(9?{3VFxJPHd^zb^)3U+3%=|%mz9{9#xNO5L&>c<3# zw&cX~@y_GJ!-Ws}pq=EG1oS}z@WU2FP$n^*_)WAkVB)9luJ5$~4^htVJWZG->XvQ{)@wbefwIUBWe?7pRFJ^ zmwfwbz!)bAPuh`zdQrRmzl)vJKGB(py)_iq7JDr|&k0L!!(%1fY01er0g$LC-#WBs z?i(u8)0L5mK|Fv;Bw{}1`I2z^<&CN#j_0(}q7(Yl7_!)uQ{$aD#Az_b+iGO7*FJ6F zBw~xAz=br?qtF?A)sK~9u~V@EzTd*1*FZrOFw>Axw6N*tuiKm@r@!SRPU32p2Rw%_ zTux6uBu=i~#S{3IbcXfKR>#y&$n*lKHu9S|Q1+Ai^0_6GlE#EZ<2?g-jd+0Yj-Rm- z^V5ffH};%!;;;Y010Ud9a``iz<(3J5Ng++cv^XwwABWTpDv`7*R4w>kzaRs&mV6V< z^abpR)3z~I^QN4BBGRbc{#jnlqsre9X5b~?ws@!JJb*zLFs-r1d9i*|ntG8LLHHro ziQH7j-F?nzIL&F_;Ct>rl3157dpe1MyWGbb=M0a=ji7F#F6?1biJ3D3)l_V@k%15a zrkrsuRlR{HL&(SseddJwDiLWHh$Oq3a_02QDiIqYAZUxt#5yVwIoNn01rJNkDCO%6 z%-JJ6;q2oiWyX?)jwo}IZ3?@Z@~xKFbGjfa35)9{_Ev|xIbpa?S%Slovq$8}={wr< z=`fg}yPWyheW!IvK1kzugfOs{6Zq0x?h8@Ek|}d;$tANs;K72$F6s!vwSnB@f&gIm zJ)NQ07=pE^4>4tO%l)e1vdUx}-H?-;p0i)VqM3j}-UtE z>yHx#I#?(eC1)X#xdA*NK^y3;7mqjtOYUeg(^;*Wt5gukruZy5tNJ08S~9-C4G?=c z-f4y_B2Y}1nB8yM!MF7jD!N0NjkEmC@!JA?I|=FRL7l@|lniUkV|H9&*dWhQ9H-D2 zNS7Tle#5y70;a{PY3dt1?MqVFm9tLfsqezw53|ZSl|93JaqB1-_5f4x7<1a*H+wk8 zDcf{bYdOvFxcyWj)|9il2?r^4?i^X{?5_^nv^t>@ zn($2ndo8Z~2f!~-5Xyaf%YF~XaDYK~O37Fh#g;r}n@Xel>|YY>bqFccr#M5EKaSa1 z2wbSHwVmT#!F~@-9lzC)JjZK|y#RwJO6ahBMB6!W9`-yOJAs4}9;tD5O41U8c(6U@ ztiEn1m%P&pK9-!b`>s6)&z<52t;-yoLn)8*4w+Dn?J>8Ek9c>reo_}`O*toKsM8w# z$9Y$Y*?AGp^CO1wDI_m4;and-yZel9Io44Hgg&u&C}0c^7A$ryrcUo7Z;4PKv8J5c z+t2P*c_X`lot^SYwi*Ck@LR3^-1*~eYr-$w1w$V&eT2vHLi=q@SaVUu<`Ty|?eq(E zsEKKd&CMuO%duL6gc7Fj^i<2k_R#D#6xA#_FL;#N5d0}OF;;)xAIDV9l6f+lFtyq! z;}U|OEXE~L{Vo{4eA}8N=sqa5YGF<#%;`wEOtMF6^dVIqz0^tsuCB7bx&0(swuL^Z z6^TtfwlU{t?3|s4O;O5{^XnI@b+7m3oIy2eb18_^nJGg>RdfbWu8*8QsXMxA7D{Ta3jnXcwr;Asi~fKmcaS1?>*2atmYKL(+nfbE@2; zH(8fnC@TJY;` zdp%ylq7q5umxWc*>@xbkSV@V}F;0`h16;y`OBf#ZN?b&tDq5GC+ngqNPa5?#j)jZH z*q-Qd$;hnSqQ(*SRvd026LOVB=3V=%zaE@k)VtXJ0tsfSqH$D8ciLUmZ!>_0Dd#77 zI5RT}36u_)MTJFn8*~P+mTpmtKCkoibS#=tT(_tN-}P}u;{lA5uG!YYe}}&KXtLcD zlRSXZ+Ag{kuO96Gfql!`F6ws92}7GB&oN=~FGcn~Jgi7!*|$qtc-vjk_DL*0Q!Z%| zZHsx=Ne_*5iHxzkVqdHhq!dsev?TJD-393%;(;obT#}HjCSflLztx@6;;o*Jx6-fj zXdLaZC_+g31IG;tx<@RQ*1T2Ni4r$Jv?**wxp{L>U z`BH2x;sTPtQvCRRLSAu_Xza_bOB0j zk7e5nRXgN^NEa=(V5ofuyIut#KoX>U+()&=-;p7v!9+-J7->I?GXm}shceVuyvbC} zB_;i*V6|8-WLyMk*hA}*j=MQ%a|9&QHkX@g2e$mu|v16z50 zp_6c^A&JyRUU|EMlYlGk{;s6`${&s>4@``5(M*}g%G1U6gOa%=N&(oDZPbIcy9_WS zKHcSty(vytQ|r=;EVi;R%c+5}60=vG#$H~rk25duK2M#v#H#1B)GtUM!^paKST%0F z`UOvfu5`7i)n@{YOn4vKY8)V_30Tfj3GnDHSK^QTj75SOKWN{sE4PUF+NT@JA3jmTJUeS-JOm_E8}?3h1wuGvzx=Bb@1%HuLO)OROE1 zsNTdfAs#qjMoX?Q<72;%#9c1LLP1Ne#g|BPS7$%$F*tI3&7fAwHPtyGO4(t0%p~JXmbzq*OJe1KqZXI55SB63$_J)52Jh0m3*Y;6TVNbZmqpm5^=> z&sVow=@+Qr7fWvZ5Pw*rWI=;crp$bAnj=5E?Y8MAY{s6%on{XJ!v`KyH#$9WM*^PL zZGYpOI!@11y;uq}eXmE6^X{&8B$7&)a`TC4cGdJYA^^bffl#n2=L(()y#@k4F#R;z zncebB!+;cgrrcCD#F>p%<3mDLMu@uh?)Uh)WOlJ>`8H=>YMfXE9$Y006V`H`zteJu zz7}$8Q}I*|j+-XwDp7K4M8wlxc)SL`CB`7cd4#URUjQP4O@#NK&vNDlXE3k~!m$<8 zrt~$Q^!G5xQBiUWvgaTCsnhywv8}@!s9E6+U9&AqzMp%{nb-Hthi`_;&6m@iq*Z;n z5pq~=35;=Gz>DOF9&N&{SH`PUtO{=EIEC1mqnz>{g}hdW2`#x5f7+;TEWgP_e7ei6 zb8b1kpX_gVK)HAdx(|jNR$EePa~=l3EV&)edSAr}L~<0wn(~9qp-zvI85=w5gRSG8 z9>~B$f^I-N_6-$bcvJ+%$!ezDQJkjmJSQ_{DIZ3r$oLrxT{AXQ?i`k_zUefHoeeDA z<*voP>cHRv_v~WxYjvFUU34}OAz{e_{dUW@KWxJWCte5{ zJ$*DYg^?_|8*9Qk7;jMm|AQN{&)j$BH3(v)3~U$jg*|SqaaY-Mag|tx=dm7E*@Nbh zi}@Jp(-$T8Cao76kgAB&1XJ$)Dn_is)6D^{u-rX7+IbVtNS;1@3O5k}-~SS7)Y6nK{W5Ucqm7{Gx+cjF)7 zMp^63I}9EEwd6+*au>MSM$l2plKafN&g)He?*OKz{P;nvI@zEHdoGhgYJY1S3BV5C z6ulz>N8y}OAIy_;!GnPvFvh5*80QdhdQx+sS97%#(;nM};~+e@9dleN!Br08l2HL( zi%v%e%eNN}Vm~{+79u!D{Aop`)5veGq!wR^>>4|l)Lt_&hz_V3ARJD#`jf`L%V zk_X?r>?B>PX_RI%h|OCv1ii6-$0Twx?4H<6Hs4A(QM;&?H)J!h;X3;1}j`WL%;8_)C3O5|%gRk>}2- z@3zil-{z9G$9yl#i9-S@<5a@br&x}|GjQOcL+WUFupA#dftj+Ak8Wrny)Z+;59CTf z&uk;ThVwI$IxK&{yTw^E7P2lks1N#LM3g#JcLF^xO!`>zC?+GDaCr+q&=)1YxE7)o zZ_8vH5N*k0WB1Ck1w5hAemOQFP?kk%4Tksh-lypSvs_{6}&4 z=^HF0k{~6H#eUaEokYM8p%+V8QAc{!`H_lKd_Wi6;r#}04|*Y_{3)bzahPLv2OV@E z0BQROXBtk+K{0LQGcEhbte0zXBlI9w9&ge>cEBY88=k{w%G^F_PSU*1%v2LbA9Iqj zlG$?*YsnLRJ!E?ve>v}p*aqJSi0Pbo4}i1^dnqrC3{ojQj{8uVAR`5%rhEOy2#CvL`NEBUp4Z{l;Ux z>`{UL@~Cb=Un9G5ZdqjxOV;INO(A>WXf@db@GW_=dn?&vRvveKY{I-Mv8n@@Cb=T{ z>t{TjW~R<-GxU>-3Z;BiKLiGSOnGvDoQ!;1|J5DD83>=(k&$>L5+aEYq&(hPR>EEZ zF%6~|%f}x+i^NH{c$g4`aQVaRZ8Bq^7hk%XLaw|T5F+39`hcxoK!p5yp0e%N7uhe+ zxVt>rvX4l|g@;zU3q^_aJQ=w|-%ZdsR<083acU2+wO>wVt`QsXzya#(M)+;0r|g28 zI)oHeELm7CMNS|66#EvwvE;WakBbPr2253e6eaS7_^99v1;6f-zcV~kf1K6_5AvQV ze+)Y!=MLHBT1a%0r=Ca?+c#%X&=DU?{)or-=8nx2rF0YPE(`mPai-ulSR8P`rk4CJ z_L!51ds=Xk2KIeetn(W3V}z7>oSqph4n~}0S53A5o|YOMRp_&Rjtc z?S>zErHj2dVMZnJS@QI2w^VIp1PVZ4ZGYHvPI`xpWxv2`mOS-ag!;bo3bMMvOuxIA z;JgtrpY6do9x3r7?l*Wp&u^z|i~Wesz#I3Yk^r5>{x~;OdXJw*47Dyr6%r(#qvKwR z#eV<(Drc_8heiNOtR$r0FDi8AKIO#?D4zWC{jEGFCHvuyW_MW>xKJGWc`gOMZc#

4h9YH;BjCK1aktyi9r? zzQt$C^H#pBd$rgdmOtYKs(xsJeBM`jmHUSF9~8T`SZDG*>9{Lg_Q0t9wl6z zA^Qy*Nt%Y{5|0aS$I1RBmwVU^KaDx<%*C-2s7@mgO4Lmk_e(aYSjr<+F8vlEmw10g z7zjFaX?NI}SLFydpbU1yzY{Kw_my>#=LZHdk|i%rik5Ye%qY--8zq0SGUW4kK?cUr zv3@ZtS>)p~Z6*u^aH#T0w)hrPIMk;x%959QB#0Tm`A{fm-(IPaCkOrBf-MH3DX$Ek zCSSl_BarGUSMbc&ko<=)E}8P`xb<>;$vqtQ%hd=UIpi;YFi2iqNcQ2=1j@o@p~ zurBy2hIQFVt}HH-EeF4H#ErDrD^Cu=+0i5zD40rI!Of)wd!r2C(7F^4xFv_+nky0T zp~M$ia>(_n?l>*RP=J)YJJzq>A1^H&(|U;GYTOvv0PjoygKiC1)7OZDnDdjxSnSm` zodxD~T!N((KXqN!#NrGlq$=p7Upi+LW3Mq6eqmj%-boQhO8zPWN~y68J$I`P#3(vInkocS5YUBK3>W^AE=#=OrEh1~*(e?ey#1m&d3~ z*0%QyQSw_|qPRHL*a;l4Cu!SPUiaxKpKbS-I}+Fa6(kZ%GHBrlwinEpV7zKtzf5*`fw+PX=I&f@ts<0}Jp-Br^LKt+FJEwcfC2POsVGV{hcA28zYX4)Au~#C{)4CwvgLTaF@eUZsl*2A(%P( z>y~ZFo4EKn0+)L6TW!-@7;wJAB@!|+9unAg@E*jW*k^#NV)h_-}eLE%2N2CsS z5439&Uf|KPEqOYMP3L&wDIvCUVi!=Zl&8Z)FQlrjm*bAM*UiWfNI?T3*;i!L<6?QWccHeFt* zIxKcrBek=&)h|p6cFI_$Oahp$^5?cjqJ$j@B&dd%F8|z-D7LTyUnlmponk$UD!|rl zUwll?zgM@0X{HK?y*=Q7_tn$_-PGZeWpA4m?0uCHiU(jY9V!3J-Qcw3>Tih8QGez5 z2`_mjz+mk0*MiaFAhRS0M*Y7gtQD`(Nh^zi1z@|1oN=U&4#FPVgnx@_v8MFVy7>5; z@hNYfXsfSR_W0|Gqn;x7))DHb%fFrr_H1<9i$W^@dgG*L1D&*zg2l4r-w&tlYoFCh zOVP?T7@r`YY0*wQN>`~}H9~w!_9&1lXQj(phE6No0X*7f$=c!t;?sbsb-iV6`DC$` z^wCH#5mdWppIE^#P#H)w{kO?Dv6W376w?ITkB2p1yML`VGLo`vKavn<9@FDO6WV`s z2z(ivk7d{+ALzEq)OVb7tZ|)f`+XZIJ~YRG zU0`!(IhhA0>meLc(gBv~yuky8hp-1W&9WQ5S}Irlx21(Wm4Ya@d9L@Rf9MXy_WNO>XVIr27sJ8!yAYf@DE?<(N1tUkxTRE_Wcf!qTk~jeQ@r?~;07%P zNtC#C0^3Hbl^)v2c7sQY#0RlD|Cmcyzgv95OdQy{$F&=*ND&(ls|be2x?-_`c_ju? zM3k!x_C~Y#SHuU0U%K5eEJExZd!t6GaWo=V)+!EhdeD{#NVgj|J}uV_D80C-VmFRR zkOgOkXwQ|=8tleBi)F#_)k+rwtTE}N3a&TfFNAM}Av&W6u-U$oW=-|iD>2{;qWV!028XcAzX~TXKJd0V}@dGK<;BiZ|wUK+# z{%!-Z8$TuF>gwNgl@eWITZPy_2AK>TMK#-Q_|hCPW#t6DWuuukYFaDPNZ*^SN@%j+NNdht7FHyU$PbYML>53#ulX`=_{iIMa{U{LTOaO8a!ckjWVoP|9` z9Q6kD4YhJ`C0HPky|Qp-vgHR4kBXqi%(T|^oT4L$YX6JCv}s-TSY1LRZG3~DY;)wK zR$t?2e7z9s$PWWo)kIJ(d$(BUUfqEyglj^bDXdC@!8F1qO~%RV?ro-dXrzr#UXa7N zm_u>(1II&?#X9Z*U#!x2QKc9(y+%YKh2Lnq@pob3+)qoi)wTLfZrLiYwpQy;W0q=_ zP2yV0CM3@bbw{vv6HYOv6-O!anby$cWU`$8>v(-90H#!VcaKb4Rjs_HklJ<=&vFQx z)gP)efbjk-nYQtm`Xi-Gd#*HI6WkK;_`ge(#mck?X8D*>=&S zlg78Q{GyRxKIkc8PKf!`j}rTYCzZ=^)HhvDld8j>QFM+T0>1Zs9c*iAZ_B zLS9b>{~!q39{vL=Wv55B`vN!1_HPmC>Bc}nF%4I~^=~@Ub3Iv#;;2IKsuEi~quCeB zfM8106mcD))Iy}eI9~`s{j%(U*2S_V zXV74%oHY`{IoYzZTy(n&%nrX#v|c(9Sr|O!qxLM8mX|utsri zJOiZsAc$(&Do7_pG*tpf(nMAaUnf1rZaQqOyp!{Vh&6txXVZY&i$s6=Acey)y=jqk z2Yb={VX6c?aa1hj)J-|S431^{L}R`U7fS@*xK%#O+7_ytF2}zfbf-`(W#L~b=cg5_5s8V-}@`TD|u&kDtA9;7S74M@qdn|?Q4&ib)NkApl) zO0x%M$$S6)R$C0Tl;9TDhfSB)8K`OOZc*_123-R9l!zT63Q9UFhAz|CYC#U7OQbwF zPqrXC5MoUfHqBZh3RZktS0Zp`g_&Cqc_Z6eW zEi1)pmO2q@da0%}S*UtLw@SsB5+CjpAMwprB`_g84g`yjJV$l^0dxqX&N}Nj74DXx z%0KZk=F4YwX8;jkOJsigVVH9l6HWVE zn9ty7c3*%T`EpKOx7GBuG-ux5Yt=x&0vJ--tQ1=xouvDZJVj_)b;io(Xp;)5e2rtk zO7TmzdHKs!$zRJ2?7HLocC%9_72kUs@@igg{uD z;BWTiEUS_M8hZumR8^K8*lM0k_-wQ4gVZO9Dlf0Ks_3;Ln7|F}5G{jP9o+>XPeBjt z94~|1vyy;=M?WE(bNp6Gp&C5C8RyKTb3{`UQh~&J6k8NC+|jrl$O6Eb-#*X*1ALg) zBTNosPkpuWtqI(~(4CGmX@pKD7No4s%W|^o%k6#YkF*1)R){Ov^HexAu;yPB$raNq z9k^AhmuhA&@jmj6PE>2G9>9az%jUFHlQWlal4AwG=%+XgA#mSz@$->qw3HIXBhXtZ zHmuQk6x|Zu0Fh5Vlnm2-X4%dA@0QI7?e%yLi$Q2`NaVlKPZJLSI zEB+vl5uP_DY~DUbJVtgP^ibX(V>iEHrtE1(C&rb{Ck4wMydhKLi}mKu&yty~LbRhW zLAu?%UyRtu`47d^;E0PAPqFBx2r#BZ-#y}~O|NN!sWK>8JjWWna<)bq)ONqj8@N%g z85ro+K2gNZF4RvEuuq-@^y~QZ(kxd^7s)_+!A-q6dPH_#wQAXmK~Il#zLodQ~^WmiU9G@lh11PVGPoGRR+{LJhP;$ zP>pby8|i&?Qa-1-ogky=Ps;(8q2puA1aDZg-!HCi*g=7FoQGNXk3w6IVj47T zr{`JL=oJp#DFsDkI*ZSlCk;`b5}(eKQ7=r<-v%^>fajtPn)EoD(4tY8Oyhl117r76u+5om~%NZu;GN_bx@XP8!oEL_w(1e)*=66T_V%BpBZY)#Gttgq)gpR9DeBTJX&tT>XOn&S@SkX73e= ztf3&mL}81&YsFx)N1?3|+v1u%p0C{drq~4Ncc)k_-OmZo$3R$VH6cHLhA-AzPB|^- zpR2O?uj+Er?H2cCJE4 zZHtWubUHdD7!$VGo8y$SJFBYTsC2s(+14HzW?stEgqEur2WLK@b1beKf#+r z#)Av^7UCGY7HWec6JlWkOC30f!Si~KcmD*1_Hp>Vx1Zy2p1@( z=;FUY1h`A`2FIyDXGw*5LxxgWb}*YM8*er515pL0vkFBtN1MQ)zPS*X@U~}-9t=h~ zMd-q?7=|X+TRgtVyPmU7${t!ut9QnUZJctALb-A!r4=*2C^kV$e7$9>U(Sfy;@fm5 zjU2_2260n@8W#Ut){sc;3o z_08MGn}ihgO$XbW<*+x4{;hLm>uC|sRs^NG=`LE|zSr47|50|ainCiRGdELQZT;h&E8MIayHsmv&5BcnJ0Q{V(VBIs zrSJf@Yt!0fv2@f+x?|J21as)<_FF&KQ}Zeng{^;_^X@E`!_Y%j0qIXm-o12~ngg|gKnf9R%etp0g`vyO=-|1~(;9LkidF2wn? zO}E>85hho$uTF6zHaIRv3} zl87x-wf|KjsnB|!9CtD95ed7^?~!s9-KTB`eoT69OZf9@e=)s&Q{8`X?@OlDQT|WO~BCjc1EoA>?Kbd*==5}6+1{th51aZ zx4E&>S?g!z(oo z!(&R0jK3|_*Ti7Ti;n$y7J%ZeO^Xl8{-Y+37&zNu#PFOC`OVFMwKw(z_ae72s%KyJeHmjSCfdtLQ0vkN60%I zCrq5D-C)GtcF6NQd*b}pEpcyzx|FA@7$eW$o!aTwHma?pP|CJLHbjaH`qMh3H=S{| zae~N$6iOlGtvP}>F|{kfGSUwHu~t4YZMnYl?lL{(tao!EQGzN-lN~>%k^EOF2;tEL z$0pkve<=0S?a=S`$`#SKiw`(-G(?pMZCWgT;O0^}>X)dH3cXM$AD#H7z!v)WtsU~` zO3zUytEz+%TYkv-nvW;NO(cgL%Jep3V>t>bxe`xwR@$@B!pX^Y( zNIbp#OS_)RD%Eo#7ioZOi2{6VUd(*^XGU~uk`GykjREWha z5K-K;hS1jzicmK z+Q`pta1IvztFP-DT|Te1mT_jHuF9rL=LGkos(eOnKRsF&t!}M*ZNoHok#i->4E*7$ zKXsOK<9tt`!MwYXJBOblI009$*E)YvgUeP?X*tS7Cx}66 zuWOI$SkK4w{E8{REW6!(rSb{#o|oenU8UWMt?~&LL-Ps#JY}G^YtP6AT-||(X>)B~ zDix;~OsEpMDhlfpUMLn-=bLHhNU`m<_b(7JW~8KC0nx0^6FLVmOny-f}hz4omc6k$1Q5Y)GRAJKGf;{jcgtotOL)*PAMJ`h>RU$)6nZ{vb zP9UR)f!^-REpqw2nHEM?aYhxU8b_FW#1W_PNw?cQkS}j~^IbK+HO_APSCJUwUJW%& zzn$m2&U3dxs9&Xd?3wADb#G25XkGq$+B1Y9e!j2EXM%fQ>jRc3pCu@P81lS}Ar7-k?mFQl`~s%zuK` zpv2}WSGJ2v^rUdlvmE8L)!VfSbT;smAR?5ZkuqSV$8xWihHI-Q1$%~bof5^AJ1H@~ zNZe1>Y9uJMF!c|0{$@)r*Efb~*LAeBfrrl!Y246m)gkddsRotB4AYV`vf<$Y>JNn$ zrbXvu!~Z;07clL}nNiEbRvA<&*HWfdcv5*|8IdM%+jlu6@1mnetck+KI3kr%A`3%*|dGb z0MDn~6Ibd34#FcRWl9T^vm64t{By=B5&tOlO_$T2yprEN{;|TLLV_xz13l+>lM4*0 zRi)eQcle3(2PW%Y3dRsPYnkk(2Zj#>HtZVnI%SmaOW2+v%M`MSV?On>?e;fhdYiIa zt(3z^NO?nusSJ!TpK*45=)6w3{{0JjFacw#Y@Y6%Azd;kl}*y^u(lV(Kc~0pO)ij9 zqA=Teh_M3_T;(2G=;R><*rwYGYY{0TNn@P{FkK}qW`kJ8DJ1?dCE6d%cTzcCsX~fs zS$242ptvI75B-VGP=v6z0%YPrV^bq`_H^%3XM~!Us-}*Szf17)!jy&TDjY`IVPy7M z+Hj@rP~ZXdD^EJ7Nt;+ol_1S@>~1G@%!eAf78|x?mNN_-9*~x(a0r{Y#aY6ciCj!* zaEyy|mL$#9eldilhn)a!qbuboMU{tRosn*)c=SP+6De}dT|X-+u02+)bgtm#VxXF~ zANJ84r-YvFA`f1vlB>&p)7NftbTj>Bl(&ctJ3p$_Xr|-O$_~$&3uH#hO=o0>1--Pz zv@YQc;UaE6rU*uC_@D}L73aqNP4fVlh!e}0dI8&5y~Ffux#rc@8V8an(ff!sg3VY& zy2|~W?9`HPD+(N@#M}xQ!6T6Tq2esv4(Gx34$tn<+QJyZeU;#7pzLXyY50SIKz~95 z!bTc26hMxaPDW_ zau-#!lnz{G={EO1-EOs!JA9WQr@5z0JjyQ-vyaF%H@>5nvZ$72cUVm#V@`)v(7+X&LD{;lMD!^N<9J(S-)c5O0y7~ zZr2OCuL0%BMHiy?0`F_;aNk%z(EkpT;b0 zkL*J4t*;zV8@l=;tlgpHu(R?%PwB%g+Ak4ZcFVZ7r*xcZrX9XJ;(Ym3R$a#y!TGTY zUPGd|3I2#d<798Sc-2Zd zjVxy{bgLYL2d95vsECL?K2to)6Ml;!%vUiM!D`@0-YE((M$WQ3^03u$4wGUF*B%dy zb6(}Rhq7s|YGNHzg*Z;y00j&JV(Srh70J0kCnQzyQsRzLvh!L;Yg?juY}+rVoHAD- z4I#ZkEFWR%E8SEZX?Kh|=WM_BJ^dxjQ|fo*&_Dz3Nbq0NUOKMbL@0w6gOYwj%Vh1yS&@5ZnV|$=SXi9pT=^e zPz9#7TfC8EOM>9Wn03?}b?t(>{-e{_LUD-HM-(?7i_T9SkOhx))XoNNmffjSq!`8O z`FfaYq@Bhki=Rlv@C4pBx^T;;-^K=dQsl4ganB-e-`ZK)4Tgt^^Y&*tTZ0KTuufOR zd;8^@i(4>83aLw4^5M&aX|8g1ry=LOEtyv)vHlUILaI|}j<>~Uo3*11kHS)KOQDag zxyqH3B9sjXP@0hH^cVSHIfPXP0j61Y=MHnc&HrhrqS#dF-0h%T#M!TMl+z_TU9rg9 z@)mtX)x{wedIvl=LLY2PP@YpIR7x2o#SpHE_r^c)rv5f)dvtD_EOW-4(*1`~?gJsJ zIMa&+RVCf-OkV4U9z3Q)%9NN*vfHbcY8;pVDS4UR7FXY-1{1FO_aF2IowRiX88;mF z6Q%#%s2g&Iil9!2~X{t2~_JZNq|lp3gv?4(#?ueqfGRg)2vO9vUXT4{ur*{GCo#cq2bA)y~!??7VH3 zNZ?bDWLhOv`tEVAA`uuEl=o3Tez_*_hjIgi{*#@ny4|L~K%m9he{A^{hh4bq4TP_K0 zGC(Bxt=3XJ%IcBr_Q>gswv?k_3b5RG=Ra(7#9=~RqI1Go=NO3=fNf%|^OhLr8~QQ+ ztBC)L$E$P1Hqx;8nb4*LM^>kEvtCItetF~o-v6|&A$R_KtFyxWSWpcUe!tW6K8F|b zFGlk5bDsBE{-M5!0ou{`-$NK-j)jP*$I-^+bP&Va<`WXwHt4Ldvfmn?r z^6Ex%#H9v`3|}ZljrmYRFW2hxCA@_y{Gk*?+o4OuD{Q1@!8A|nLXwpRcbw5-X)Km{ zQmo<>lB!aot4y0I+kSC_CWLFIK?@x9!nRVMU%DO19?n%fK)=g`6jIt`JI97<&$;$! zf57<65RoF@~6B2CB$8&atIZLtn3P zj8AYr;^R4=P6?YJ_Ue4+8!YC*x5hv`FaM2PsUw*x@VFM}Y=SE2s1M=R?andJ(Ym%D zIc>RftZb-)ZTj`dF@D|{26^BxA=SBZgg54#e(l#pa^&PC-tJ`PP-0;W4)p@+zBlw^ z*x4Fs)ft(x*4zm|EG2eVILVXB^#=z@|BLX$Zt)Q(W+-l2ROGd5z0*lq20`R#=wI%C zH-ljh^<6hqR$H5ynukN+2h-Xu!oy)frL8tlm)JtN{@FF!>KbWOvqI-x9@$YmbbLf{ zccl%7Yn3bcVW8g|d0Th;Q}r-L+o(x#vaNN=z$o(j9A_W>sgh!N1Xnv1BoFrkdYQ6E z)ZJUHjm3W|P$nFrmPLs1T+4bLXJ;#I zT#YnCBh5?nhP?ZY?t=^=X|gw%YXr(3`Km-0uI;VjhCj-wA7X#MV0}%VJpM327j-_- z`kLT}ptK!hcL|#*erH-8?(<6&ei_OPA`yOiva_kX`r zuYVX_f=4(Be4;6CtllLu&`G#p&iwRuriNb`~1YkRkiWHuAy?|d}4B?gS z&H?UuDyzdI-R|;UnpnXRycl?Rls%$@4~eTk4@gybE#Cn&XBgM_E)Zirb?FYw^ z?P8L7h}J|43>s%_ExIagBJENLd~AQSS0CCz=Bg6X|k5X*z@GiL=B*o{`GxCO)EfEEnHF zfT8&!M?ZbUxq3;YlA?a;cGvmUvVdca7V4L1d#tW?1lb$-uW5wQcf<>om;}XK9(M}y z5Gxa|6qNgAgUsNnTP{_Mwr2`tMhAUb&a~UEZEEG*ht0it_#ieYP2T31G^561Pv$$z z#y_QdDHx;vg*chv|EQLNUm#o#J$lbkab4j%dWR-RSNSSbHfsG~UGs=$zw%pph8%(} zMl1Jovg}Lx0;ODuCe$pE{W!D;Z1atZb`FUE9{5xQZT0B#h2l7`x#A?#ZAE`xD{f}X zm;Wlu1MJI4k;4myWxn}K^okT`IXCQ(;8uBujSwDRtb*XtRX%4)jqeL3Tq&PUu->X1 zp+CS`D)^&+8zCF5KcNR`#vb*;vU-6iC?BhR>y|hx8oz%*6O6^aohL%wkGL`N8vr~AwOps*X1q(>)It)wweE~I)Z1uKwTTBiD+)T! zVCEc@Q^4U*6~m)Tuyx9Wr0HS0lISUb`Aax{F(s@Bk?*dM?9#weCy1_OkIM6HFLWgx z%w+dfH&Y_JP%Mr6P0#qWJ-X&(ilv+l^w2y&I@0d?;$)dUQFm1~`2K^0A1SAG)jd=`0?wnu_i?xi`-3y=JvhhLOJ zY$a*V-HE2QN4M59MMpx4Ka_7M(JxSRWI+U57+~EdjgVarbkOZZGvzp@_FVR-7 z$KzP6=QoC5$}dK2@d9u1b@6pQe>YEr^BwCjiic*}z2QnRb8dh2MK=ro-8l|)inmJC zcuY&_-ha2q-=!D1jib6}$H`pI($?_^DH1m^K+7cxgS2uNszqC#MZ~x2mu`3OzF$a= zHc>+TAf!Z!o*c7K7Srl?AGTKXy!4z>_laA?H#Ad97<+UdbJUtfLNc;y7!$^PG1__h z{b|}jI4jHU{&A7aVV6fq(du`4@d|$imcDYTvXYwlbdcc~b#a><| zrbB>LNNOw?x>bJJAWl(bv9AGjFCXFTVHXg^RH-aGreUQxo}rIen0~$cuScCXxx|VD zGal>y?@4Fv@HKV7VtO19m8^|}(%2*BRYt*Gr`Lr;%&imTE#W70d&y9qi@CKx-a;Z{ zKU$?G)Y}G&IWWR`g7*na4D@t6W=_j@xYI1KxEj~NV{4cwVn<>2_IAm4ZTON zS~=yXPZS0dLG_t_=;saHIjC-3yT{N3ak#R)?vvDG*gF^%Tdlq z8N5M!$KkkqbTNdEX-+rytdrKZN5_LsH~Jue85})EP8KI8=`}WEakaHb5L(9+wjVZC9Nc48E06+j(J*0cr%5jg~q*u^TkPHLWR0@L~+;I^#}(m#7XuE6;o}op2J7URbF#18pcRDS}Bfjur2>0k9vgi z5#k8Si%m;5c-ygxuYyi9jX7KE6!lr`8-Kv`i?h~SoDflB0Y{ZDFIaEU<0>BE z+NQgTy!6TX<7t4AfUCtPAJ=tTJsT|%$2jDR3DplC^+M{4(cX|eeP!2_=*cYU8^6`s zFJ($>z5U7_Cn~*j$;D{-{L=Hz15&Wxpkf^pAm!fz@0|1Biz45CRL`-&)^sLCpft|z z`RHCL()DX!97;nm(|^R~L|Cid`^r)!v>RgiWG1d#HBz=6ZJ*=gQ& z5&At00~r}qVZss(Y|~2dHWSTU2%x3PS7+p{O_%$CL1;40yN>t8Kx$$vmZWBlhOZH6 zKrdIG>(!!>T)N=6Rt~FCsQ97N=mq{(q)9amGOr%vLQ4& zB36_5TS-AVn6{1(qe++>Os9nRAh6t#eOIv<3*ic-ZLcne#F1U^>s84z1-)0B04wFq zxtgigrI)Of*KRA*YZo(giF#5Z&RSDB_?VCP4G(VnwC1d}G9lp7kuuq7%^`3!b&+=1 z>U3kx0P`6W_Ue`5bYp?23Ie6T5xCN6Ko*G*pBs8{!LWejYG`VLuGf_@)@aV+=MmZ< zMasy@RtCo!m4Wy{SiM!w$T+BPE$ZW9y>2AIEJ-xh!-FbFnM^*&|2(Cm5VZeG{X0U% zb2rY>bJHMQ<(9415RO>6IFRz(WEuI;KUx>Vbmal@97uNool-D;e!qBrbGm8?rvK>m zOiSVA@O8ck2Y5VvR(Nl^OLtp_$BTuseX9w2C~1HdkwJzOT`*Lo((T^OxnIe0y3)l+ zc`I8y%q?FJ;ebsL`$L?(-hHu8SLyXvn9OodK^vz3P%!U2ZpZL-xltB4DVCiwJD>%G;R*~fn%4d(z`>cbqyB{@~NVM z^=7bnEf<7Ti5gvkbJ7tlZmPQ(+dJZ{;2A)DR>4$h6W~mASFYd#(|UwND3GVLb%sY&SKS?|0*Nfc5n6yI=<0Q z#h5Y@CDOaU(;4%Zq025+B`&&DP8a`J ze!g&s3k(pCkc*N3nsA5Iav{-^{$s8@cvv2a;RRkM_lF{*^;R_ z7+h`p9(D#kqPIAW3F8Ls7bRriR_Yrb^$5drMG2tzujx?YQV)pd$%g_`We=d&lX6?I zY{`?T3Ws5O(;ng3_l7o*vB%@1oEA(pOD+N%S?C0q`M%5a;nU6#H}k?S6%KJ`(tI&F zqaC+#o@mEDAq-Otjyo2KkdeQu`sVW*cV~<@_=yj-#i~?K7nh&r454MCt#M`C2NT5W zF*-AtiJ&+(x%yvvek^YMUZ;QG64fE8pKZroh!ZC&?y76Zan(CTCDTa$>++~qg5F^= z0|E4K!{Mj1*3BeoQZ@w(CgcHwvpnuwiaoGemffd&BYADwSNZ~~A+*d*9ZSnDM+#P> zMEk(ht1+Q6a)xq4pJka&dsgxk!B6v8x6kRoWCQ~lV11&NJCUq>s}c~??LM67?!YVV zKsQ$JbNh5Tk0p7f3ngHBUA!E|sc;Wp5B+KEb9<3<6In(SQw_b(n0;dZngtdjO*8E? zXMw!-p&5Ejx< zv@?lK(2|P=;@InB+?-ZCjQZ6Jw2j^RU71Z48(s;6!u{$e7t#Y%{5TXba`Xg zJ$h&j(vfzb(>&a``##0Pe1ZA~otAgPNaC-f6=P0|L9X2 zDYkK7NZErQMrQkI57%z6mI=$gcg_<#dJpr(X}aB)z}*q@gN9BO)Mv=D#`#CVc5%$f z5jzGq*Mu@{kMqmD?Q3W0vtQa`edh&=SIUMektVeJ&R-~AS@?}6fD|0jbL5kvTt^U$ zv+Ta}CyQ6gPiTS`+n0-dubilR_}%W?b-D~Lj?z8ME?s3nt<3I|tj`zvtN7?^hsoPs z3fIuF8U#+czFhjKPkjhUljZoUbjoE_3PMVSoKVtP$CxJ6+a&MH;@=x=9$z)3FCqD-e1**p~wqsuc3oL-FjR24%g2=iP=J`}3B0TUpc zk*rLp62^0D4|}iTqO@WPIK=YeMJ$WMHNKBU-y;*`B+><0v|EGYy;$*il{TNN%a;{0 zxrKSm1;%LW?Dn=}7f|siSBToU#{EwV;}>4d7%^T?jsuluNFT>=SY1QSapummNcWge7f}Gvz9~J z_aM~wD8KXzh>_E0>+2A%#lG4jZwt|l(0up%E{~J9H?29E?3+EWtjGB887BCxbnkM+JyartI2m^F6EUei}edRD;{T|`y%vy zZRUu_Z#$&br$oA4A3FCdH}NG@%2ACH{SS%9nKcHXT$Kja_l($eSDU)9?ALF**kux= z7z6bib57pLIt*?wU0=W9i$o$L1C=oaxBwelD2J0dUoq9L>^CA)lyba*KNMrcX6==2 z`W?{9!G|gn^F=A=r7(^w<+f^3${MVyVk~xdttjF;Lydt>m3uQqH@+_lhbi$uq~|?O z{QE&zuhuU5EsYo5IVGQA0vE!PND;$jCyZ<&sNeEJv59+p${r0@aZ6 zM1(VuaPYvC67X0P>rCVt0TN8}=*Pl;R_8t1w=ky4`pKdjpI3@2a1gi$`~};Spj4va zQL;g--QPtMjFfld#bT~8Dx|=t$~TcR^Xq%Ga^WhH>+|^Hh;vQQ!+IKNN>nZsyIHEJ z0eYE+UU67N^W>${#Q-}^k~h9UilBkTlc4Rc1-;@DcphW$Z)sMp@^utm?(F!f`PMabY%n#@n6j-NYbJ<*DtmF=Gd14}O#=-YFZ;T{`!@)7`c1XuCk$$x zsQpr-d5p=FIoFvNrDHUXhYyM`++3o zzm(y*63v4gRqcY!abzQw%#%MR_EW}D3W78}*@@sH8yJj{`%kZwf~y%-SWHXlpA)g~ zQLZ5=BO`?>^$6L)`&O8<3MPX3-y0$1U$cEJiYgBZ@e&u66fpQeSUO64FyZaG!0lg8 z+ZWEsiPLXUD@N*vkvElEpPOrprg`+wKIGX<=jic`a{9l%K~5-WbqRsPlcPCqT7)a} zRXFtjWRx7!z`X2~s{*F~%W0yB8{GV1BB=lM1>ytJ(dAKvA5}gOa>CdfbrhCp9Pk*y zw-V!+HrL6 zR1Tvjp-Z`HQ3J>lkh&s8TMVngbOujoxL-m)&2)OH=Rx-Of$dssc99&*qAWL`)c(9; zxS9-H^d+xRr1 zi-Hf~g$U1icg@jAd1stl@xo`i1f?j!*^Qgm-J{z}khTY@TW{mV&p6-G-he&l#0icB zP~VlZuGs0rM;r;J-6r&lk$!DY)va6&_^8(NCy6FOsj6U)iVY(5!8>$EU!##8*zG*U zF^gU3Vt{>f!Lhi3iTPaUReQvlKjvLx^dvxf@agOGuOesw>7?%Ne3!ytkPfJAWF6t8>};Pa6B?FUU-B-G(j`MxNeDQ=dAdX$L4bqM ze6+lqEv#~wdX!BQ8pL_}c6w5Wl!jivO29m4W1q$P&|;LX(upZ?o^DiByp1IF6tRM;kr@c0Pi&h!J|w=&psr+os~~k6CeI3MWJq4lEB3a&~>xYpV`^Ls(EGCm#Ax$5@Ciaeu7%@X@o{ zl>np4{RhPXCK7oU{qjhROu0VG=gKK|0^f!=nP-F2w5WuIXT+QAi+Ox)KY@fo-v*fX z)iJ>syFTHbJ)+`lwl9*g*ov*ZKcF4_*QI@I&ho<=D}Ta%b}ZLtFMv`Z02(z6tGw63$y*+B_1rjT0Y|2n>t6 zQAmF656O6dwcJMd>=~gh2`KfA33u!h|Jit~ZVZ#Kb+q`1%Yo&-a2Pl$K}_OyEf`!L zd)JB@j?mQjdYA++_SkBcvakutIljEmpg?~GgVJ#RyXR|V<4lghAdJm z90`i)z_4@s@_9XMDc!hMe_)r%;$McN{MQ8Tz^)vBA)Pj$RJD=z!0wHl$JwDlq#~r- z18**nE4bHK?o)1HWQ_A5mwWPkUCzJ>2j#pcpH(w)Rf;Ovlf`vpUVw*LnHYF)rnC0P zTlA6|B>*;mvh!5iXLYY_2ut>h$Dhj5210#!W*d|W96Yu)O09J#t47MU{L~94Zr5$UOtq-`C3atslA}EQ z*NplG?w%!{U{F&h<*jJ3mL)aUFYtJCp|j|pQF^FABtqb7#(=2TE%Z=db_i^vZ{T~o z{vy(Yk@8mDKyede13%dyZ++rJ6{E^Gwmqmpp?Hp7N>xD;y0D7lNwiuzH)e~);M zORQ+Bng=ER8*5GGNPwymuEY)sYAH5zM3euT5`!89i%qXxS0}cfltE-$`#1KHx;?Ey z{-aW>r)^VG4AMc(_lg4MrUgWDxt4M{jzR3IQPJQUCTs^nFL_^0&@aT9t^(;?-@`oCrAtiFV*tEQZ);3%N z>zQda<~p{*VF=Ns;yE+DH=V(tp376ecei{EumP1Kma97oDMm)hpkzn+X>)_dWQrGgGlk;nhbyl? zDMxZ75zo88ZWiJ>cc@^1P2J)g=HuxPuySqFDW~OCIBFN20YrdJ^Ap8f-av}_VULLw zVw0Ow#UPztsY}!W|={Dsam&RwtxVmjDC^ACw)_quJK z3c5jg=VZz$pSbdOFKWk>;oYq51lW#*BSdbEXskn-XPQOVKB7%Wz%J;!FsFWg54l(HDmcPw{) z;n^zwYnu9?dftCU$fa3DN~Gze2hHUm#zSUZ3iwDV%aKd(eOd=hzLxU(BH5dy0Lp~n z8pr-3vBph@YwW=dh&b*@6i@>A@HlYD8$}8xg#+Ix0tpQb4>w>AHgawupEL$`wLLY@ ziX$P}Y~S>L5INoA9-5&`7#`K-l=G&&N$6#4OL$76M43Ir3%sii^q;)v5vD}%3??%9A4WgJAHqfR^Q-wZ>Jpj z>v-!c{9*_^IrSVXSLmUf1ygicXT7egK!osJn9QxwI|;5E&ie@u`7RXFT?$8HvqIVW zx#gDKQk%B9*Tag?B|!u0xKsSV(MUuZBPVtV7C&&R z6CS1!CiW0c8}`;HZd{qz`>d10QQ~rdX#*wpA0hgDQKXqdL`q^oY6+VuOMRi8XeU^e zoDB8&qA+n_tjuC59I>iX0{Yl$+4rMQ^&+GJcKvqoJv(PH3|TM|OqU)Ox4Y@+s0QI? zo`_}US%Q>2<;wbmoA$_20UdooXWNMrcFL=cWNE9HX^TB_(izE}0Hqu_V0w49xXwN1 zX>_@Bf^4(QydpvsJU4fXRYjVw^{8)fupx8Dlqh}l%|IW2R?Hwph60wak=BEL?>Uj_ zZVZ_cE3)N{5&nAOZKM>8mUnPEj6d86+E^gIoBX&YAR44E##wKIG#eb@n!wvWInB@N zrfvwkj>tQUx+!g4&%F^N@7xsavj-M?MF`0aZVx3gG}4!MI!W9+$pyM53|St$Zy6t8 zM7ja<{y5qAqZGZCLkW2B4B+!bAyw202f*H{l@D_1s{A6b&kl((goDy0UrRY2A;R6G zURp}x4~N7}cTChnC*$lxN9E%4-&+bg)nHl`FDl8|r*N1m^$4dptxqZ!io1bZk159r z_L1H~0pkbsA12FSwx5xrE=}QEwp+@0hqns#jVmt{ip+@_IzbymaQuEj4&$Is2_-x_ zEPo4@7Ms%wn65IUQ?j?uRVjM?BVR!u@?e3tFG<~1m2hRCAwg?pDrcSMQ$_P2nWB97 zp80kAghP0P>RJv^T|pJVQDyuR>npyMQP^XbYPUl;NB;{qmlPpSm*{!aNoQUO2DeJb zmU0;DLHySxbS3ROpH~lrT&>IH>JNz>A^qJO5ZcH?Xr?J&w$REMT?VGf6vw2}H`vO+VTVGRoPY*$D@`)&|-)5jh8EazDc~oP@zhf=$k87xaZg~ z<4Ai51J=1Lh5J**XghSTe4yCmW%1M6jy@?KCKojSRT`(;L$a%#NS0hnr~(fN51zCR zyXnOZj~qWa_`U#(5^6F5uzFQ)J}KJ|?Ij*2PHVK0d2p=ySgZyGWWb{ zKyEk`owIgg50IKZdB`)d;umIGs)X^&YimXB_E5cYRiKf+eZ*SNtwpN1&fc~~F8=sz z-F@UCrDv=+*f><)Hzs^%i%dx#Bi8WEuF`HeQeeHzrCuCu8qkm<+pYcPz_RPguZyj> zCY7sCyh4kSj~9zOnorX0rA&1OLw;`L*la$cx~uKU3)b70u1eM~QGMFl$+09bxME35 zlfm_4i&mMSeS0>=deJ@djv2A~e-~ID-aAp+8hiYAr?s6g5Q@1mcA}DJX3d5Sr2uxe zRGcMSV>!SK;jg{cOaFYMUFp%`a4ud{lYNd7Dw>gUK0+L0XBB&xCYIEcgzQYb(bR-x zQllf*OC(lQB{YttfEe-P@P>L6JX=9e>afyDX2z#Vpi7pW)IQL9nZ6LHrejMAKV-ei zBGY=G#gfQ^_ak`&0i}NMXt703ySbole45m2j+1N-#~D{9#hn&Ea(h8lF*p*6tk+09 z0aG)3P3pN^CNWzJ0!NwZl#+N}KB>txUvG&Nxzf6WgOBrlgYTpPTjjKy_FPh>%^bOw zv85kU%2g9f8oSUcV0sTj(;AY-9M}V@-1gdO&7(%!^mg*mvV{dG zSGgH2t(VD@=K-lnG}4>H6i71ELkX=1P5jl(>Jb z`1L8B#oP2_No0;YnbpeoiMH*e$0DuoNoiMwfecbQgCAZB2Zr18Cq1`N+{^r>fKJJX z5uw47R-tjiQ zRMID!3vK-4xdQalRh5qF@@c>N>C#jI2#8;ut2=|2ttPbJxXfcS4evJ1`Q%@=hmFup% zR=Gh%^GG{s=OMXxN230Ec(ii#V-C3(GXREIke2VZ&d{+@#TDxfkhihdSOX5KAcZqm zPcsMdyC|ie)lVi{Ue>i$m2wU2WWHGPp{3)4YDhU>Y<4+D%5Nr zIwUCHQsQ!^NfnFalc%QY2r|kAwUoEbTB!VLJ}1eYL#-csp3^&cMas9y1EQ^;cxnMq zhH28}VVN9rm^c0YP!VJy8tBPetzVe(pqOz~G8w3bvSgA+c;h`vWLA1313%TH;20IS z$iezu*=yP`9OU4025E5X`8?~uf2El#^(^mZ}d zy)r`;c(5lP<)+A}(aoctE3+4hRZ9`SPh&A-c;k1U6DL&Q)n$@->dT1H@H?l+xCtnNA%i2fs5*o6uC5 zTOoR|4F+t}p(I}$D<7&bDX9#Ptg|w4@)6}E70K9xt)pY4u+LzWV`-)fMvGOiZ&1@F zMFiNKmeRl90g*+L$9P@h!5DEL>$nhHU6y5vv7d(M@j6nd^4J#7?>viG1`a=6WmTN# zckZjN$1kXlO&=)`53#zTN|-A7X<~G@+jO%1H+8IYhj%1Xz!0SD(h}0>k92kkCw7~ zzX;Dhp-pI%tJi<*jTLvHe%3{?^|WPAYaYX%IK+Y$=w$~zZ>=)d2MGt1JFwf@%Z^ou zZ$&Y=ELOB*gpz}D<+=fV5-&zMGt@BBbtT)^4e4_DuiKE1M>q{RDVsrOzz-?MT8jRA zGWB{P)gb)mto0#VM@VsTe3LBFIQ*n|*eD6^H1yMVVKed z$CIa~lDShL`FqXpOkP@yuH?)7Xb%;A> zU@(3eI_Sc_^Ne7X8~ja`Gvf7Cn>A82#vXrccYbwuc?^%?hpj1`yNCe?)nJ;EFS3qz z)2$NG>Gn{|Pu$FMSPtqVniBELWdi9OmZDtA|F+n5xz3v`{9i!{l%vXZ8$?5{GUXb3 zz?8>Z{n!*%(AR5Sa*B0k4vPB-nbyDWeWe<)X{JMS&WRibSC~&Rg~!ATB8X&BvmqE9 z(`H(KPR!69ERqmA>ySW#=t{TMSXY^SR$O8xXNgY*hSIeMm|Y1h@dQOvDI@#y7)(z@Mxw5OGGwzy@2ilduD<4$Tt;r zW2vF!gUus{NP_P$$IunEB97*v#O7%pY;d=Bd1Po44oP&9SyDrSiS?muw8xO^MnMN0 z5%f;JNaJ}|iYvw--4rdZH0MuMz(9Cyr??{Z4P^_pmIL9vwT>TYc@e4FKNfp^k7zULTANZ041K6lvPj0P9;lz*G{E+r7K!fdCqvk`T{Hl?(uFGU`0!|IC)OR1 zqQJr9dRNahBZm{M*b4&^DRh^%#*hsi`Aaf zC5HZUM&xo^5M7kRslsOc5|$bGuW5wC*aRKGhY#4sV#C_3^qloStb3Rst^Vb}hBZ$R z*KxGE3?A@->8U_(vgchI$aq40Vd2NU}AKMj0M(T>C=>N&G&X#J;JC2>rYNo!qBwrVL6E+kZehSGF|Yn z8?!}o9va1KrXL$NH`((O$#0Z$5Ye`ta?`U#8}2p2)O=rt%}x+^)6pwuS7~7P1?U4* zC6qAg-?K=Z9g?a0LWn50sMb2iGQ)9n!DxfdgIO@#9J{g86L@p5vEpr!-|E;armI zX4p@A#bJ(kD{V~`Mg~XHFVu}bhp}>XgtgBM6JtnWPWbJ%#iBPx&R6XvWe9h8t8{!B zBc~)y7T?tN)VknXs$3pHDLnR5$k-BFn4W*w_Xou%Y(OYiCMYAPw3sE1a8|*!2Lv{k z5=n}P9;zC;w5<@IR-G63(>(mO`j_(8n^f?=4SStlH!qhKO-e7Xf zn1-AZT`2~VSl|k*9;8cjTObk$%L24DghAs(=Ekc;7&gs!)wfRZ-pLFT$uJ#8RzTV< zQrsx~b-7%#@RBUeDR*q}_T|`X6uMw}s@#$5ZP~`8f-+3WI2|3M+lm=Ur%c{!J<0() z>}i^5%E4sI!=R=Nu^AdiPO%8%YPo_QVwi3aDdfz+TIz#=l-WzHtsit$)-s`;vSe23 zJA81!p!yJO`tVNKgu{aT;o4($tr*K>WEUm!RN$sO9x6_;8kpc4C#O6cZ><>6T8}P_ zl;^ity|^2xc=+imJ7!v6bCRjV*RoS?JiMGH;><`tR zt&Cyj^LuA{@D!)a z(xYHXQ6+h+x9h1DTDd3<{gm)V|9+n~a=0!LS>Ww<<#r|3bo9gfRCv3#Oj1luoDN?$ z(@R=A-8>AiekVOIGS$d4y~9X5Sr zdpcvFX@24m*P+UAwvn=`&>OufSeM9E6RY7@gvyZxMH+`;IypghV`hQZl#u|&u4s@o zQV&Oo5eOXf9L=Dn$`_}@7iEj&@&0 zzu=ALB3u4NV2_TrelPx0k1aLQ)GHQQHu)WLOy>v>QXq5>>#KFyr7iY!n5U9#jr`%N z|6qY^IO-)WHc$0I!wYLY-!eW`IE-?eD)+t2nJXnWLj!v*&Uzr*T*zIozH`HuMaYJ| zwu;%fLE{*{Y0f*vT$!;jG9|#Vr9ez2vmgvjBOJbUiI~D7LK0H2DJ3}lvytwz24H}L zu)C4fn-8BlGf|*g)bKYCiA3|-jSGxyYl%#Xf&fMpO1yDET-)hM#Z*ax2aB?4WGUjm zF7$c?r%x@4@uI*33p(d-eLIXrTe;bZLj8{#lD=CTFWHh3Y;CKrEHre zC#*5YfsELCggu#Z0wIN_<~x@fxY8T*uX*v)K!5*&xBFdf^i13sxiZe%eYQ^2;6~va z1#-Ome3mJ3Vyj5XYNfRe(H0}<)_mPp#~ARj$B%Qw5FYyhx*FJ|+o_>Dz0p4}*2Awj zrETg!6Jw}u43s+hq_-;<*r{*6*~7VKu{fr)9`)%e!@q1KZ$EaoZnp-V7OMSwKeeYH?WKqlMI0{`Z54zMz)%%*UDa?Oe6ZR)50Mi78)F_u3Aj%u;)+N?! zZSUUdIl*2udMGSFf1=nM^LeR`WK(5oBk?smc*5#mU9Xv%a zRaTUW-gl^w>R8X$eqmke={GzUB2^Viyu@)2PE;vExVBjNWZ7cc6x9+GffBzJS}Qp8 zoIwO>T~dGDCnoUC=CqvZ{y_Ry!oDxun;4Wzx2=|=Jma`{5l59MUE)vX`|N5cVEkZu zpin-3V{biTLo^0zbyDZlQl;SW{|GzxxT>o3 z|5KJ0sYzmLmM9vAk=a;z$;)1Aa})(bIijMdc%!`F6-6@+;-=Ag@7Jw_A(b`8O~%S0xOOm!}37@}x2tR28H=6D5vu79W&m*kk=O+aboT z@!Ww9j-QpQYO!y{Ib&I*Ry;5=JT}COG8U@T+~cBl_O1wLOoD!8*L*CjcWXIgIg?j* z)j%7Mi_hmw*M8}sL#hU0+iCGRx8iY<8Gc#Y(#7Ybg8{aO^rr;3H7Rx!Q+_^MgcZ82 zSI^c=sRl47B3Q;2#m4|!q>WsDIrt&V`dfjHKD}*gagOx!2Nw5qc|S(9W5iY$@AHEP zi&X1LCj~>#m2MsJ&BX1hxQAo!7TNmAz5;s)p}oI1OWs5-dCx$2?oh6%;#ua}SKG@s z#ZGOqYAC-LXV)-LX}awG${}3>1Ht2B2d9n6Ijp)av?XEP-3;Ir4{YMMMP#|#y8HD? zib6}3&nv{|q{2?ZK>Yp}p(00ozHFP;#gw2$2`^DT3R2~KmiRkE1wT@@Diex&Q=Ng_ zCsa%|Fj4P-Gmv2q7(7Rn*OEoPDD_G647BK+Txs$rs~!R#4_$WKK3}1mX+RtKM7T&V zh*Atq716l36vOCQl96)TLGciMGR4jJR&?Cy%*1O*QNXarLtUKK+?~ZZ28U>ORLm{@ zR!f0LZ(GEdI}N?t*Hi;X&q${shZYqN7(?i_(`obFC<|Rw1Au9tG^ZhVR?$`ggYd{^ z(c#sh;uh>-c%+|kmT)GI{~9BUUNKH&^S!FL3Vy027hB6&yFh|hBB9iY;oN6lD3Zmj zEVnrenK333sUk+ZQG?9W-|zw0E2rJ4apr_srgp=qG^Yv2PJ*GZ6ay1`JV&fg2bd`_ z{G1ie`LQ%(BzR;dI^pymTTOf*g=c5NIf+naX5tWfh0BI)#x^xWrxaUExfJ~PhqCIv zu%-!fGo8BhK^GB=w*SkfGb2T_!KFGZt2Fe2fO9iz7yO~%Q)NN26VB_DILY&VNt|;J zo>S0w>wwwXUgu>|RVuX`c#(}m1G+y&AE0l`abihQ4Gz!y>1SNY_%sXy5~e_{9^uGC>@pWuu2jST#p{_S8SLat3L^aZ$-xNreL*gw&hqPAj%J z`LAd7oks<0(%Q(GTH6B^PA`_^tCia^UVJmZmG1cs)5A@i-lZ`*3Mqv>+$-Wb?g2N% zruYWJduPOTw>_vGRigGh#oJfJ=WGk2m|~3jH3&zttj?p%t8Tax9@|r7mlkGc1Rjde zOb$qYNB2@j?9XQ$i|qx9dt$%3AZk^Fhk9Jwmi4+gXT6)`h=#`rA--TZs&HU6aGX3Q zzF=Y%Mra!X7laB~f<}VZ#K;;M%aElKDG)K?8yl@yF7;whRjs!zJ05b|wBJ%~LfNUA z+eWlfP0aXK#$*PM;Rde?I4Xo3M`8g#9X;^vys9LqAG$mk z>)ga(MBPmYDqSQGOH`b;s~a+DUYwyOiI@3`$tWQzxE)7*6(2CQE1Wnw6#g)7kVBK5 zp)6IPm{+B5EoaD|S6Eb0>QklNUh(qGD$#;r&r8a;vfV~enN?Q(DUkPV6DPT&sVv5i z5<{XL(rbmh507C7oQ=EyQkM>8xAMMxEZjMk`i-_JK?h-4ksDR_5gn(1gU1~QW&d{P zl(S7}%OxG%&g65<0~ka~^gZr&<~bmMc}5;H$4R{QsRc<&DsjJgvp!5cttcOgk^+?Q z@T$zIaQe|2$_y|Cy^i?&_fuNRMU8aS74ZcN|DJLn9TiP-=ofSq_&^vFFTSDy@rNn} zfzRsaKfSNjhp{3Y6`!AMt@JQ4Ci5-@A0xSIZ4*I=&AZg&5T~-#cxGuf&=dae{WM zbTL)d2yu!GcJT1LzlH*Pev0tyz#myEcxfl!v&kc-FJ|#7l($+A2FJ7GVMtu-q>YSCg zOnXjmf|hBqCyzO!NOu|ln5ptZs*}Nfb`=3*O8h-ioWN0@w(#H}*U6VMbpL^92w#+o zQA{-`?zN~Rh2jK@e;zPOFgs~P&d?wx+^s{qGD4gnTO%Spdwg_7Msisp^J<)Oko?3} zoyW*CGN-~%us){z!VjijH4!IBTcmgx-+sMXobU!D#$x2kI#K;v7A8DLXTTBx#g%eC z+rBf_$zCLNi^@~pw;N3j9^lZTGGRa&$Y$587c+~VbQvvld|WvoPVvG6#nlfUH=GeC zzHXsI3dR8Y*9CEsMf)(*UBKg}W6Fej{sNIIt?LIr2Zt&F4oduyhaG#?8BdN~Fc{_R zn2kY~eGf!>RcbJe%?%cN=Phv-Olwt$3fe8Ycy0f`r)6a6o2tGSr~fUMkq4J*0u}>U z1Q*!JmjkeF23W%m;uKdiszMFOCW+T6?jvb%oxavfNK`}H!5y7nY3eXk8bj!wx$g^}(1C{uOFLtWbBU&|bO~syx6KwO%t-{L zF~AIF4LU^Qu?D|!OZXQPL%*f!18Q$42ue9cNE6?2?*!whYKR?q zRvaghASgZMmK|^=^Fqu;ni!?1viMN&60hxfQXr7>@C%+bQ#cHdC0oRo#26TqJ1LQO zNqpI5Z74n<%$+JudfTLwKy0wkxsE$DibtlF@=A_4$$%s&gd_gOsqWn`*m}%wO7P(6 zN&G^6#TcgB!(?w~kG7aW`tLbT$6pRuVb}x4l&C@YJ=*C+?sU%{K0E%p1gGOM^E?ua zsq*1yagzLWC}x_6{pm)L3G@<72+?oH-xlrs{P4z5Xrs%!U1Z;$M>TZqm-t(Sb2o`e zI7??1cjB#%Zr2OXX%iYA^^OO3vF11%N5v_z@saVtpXn`?az>Z)VdA8>`G{zM{gNt9 zF=-B>T$si&w8HgO{!?tCQ!-4Szu@+KFJARfiVsXj#JKH-9oJ0Zfwrjuw_ch#GNPHr z-yZ4KC2c?@${Fa76a@dnf*BG#LeKWj^(?6cA!uOn4U@&0NA^|wHa_#H+b+!H8#g=> z!h-)`ac4RaWN?t%_RR06LTwHlPhD|uXU_r*#{2P0wz#*C_GzZNioI&JT>Xzu+NMpF z`tb!(&W|_W5eiFqhzMENJF*HmO7u+?Uofp!O!HJ|$7jbnVH};E4Pm!xFYy!h$s5Q? zon&GR0$01fJW;F-6wnr{L8t*eCtb$;^Yv?Z49S)wy~LXa>7gUeDK0DUhZUxQameP3 zmk-tuGLOoEj6e9p^o8*UGo5$6S3`BI$Gb&hh<9EaKIn3wKoRs{qD0$+H>ZPM+V?yy zh4b29bAgOMOq|v<;iztMAV&dllKR1v{2v3o1tP=rmu!3 z2-6Na?_hOWf}DkIPK7|QBKRp_(k%L#WDlFo8B>VpFw;a6lI zX7l`EOz0auzQ?C*3^lxJplLhAH(ZB;p^0Q48+NI`*HIWukL%lBDsL+stxKQ}V(&?j z?O3+0R*s#bAG~8pRHMXer^F8o;gax+F%Sp*&vN8l0fR`1cPeCL*BkX&$nUo%Nk3*)>-tG=z@f zZXDa3E$A(4D|_?|Pjl~kWO{YL^h`SA#xa5gl+m_l(rWiU(yx_Z4}(rA-j#D=6Svy~L zA`iR*X4KCpbvAICvPEc2*Rw7OxY)6JE2LacPA~t*oZL~N36JH8?!9EN^M{tgdk##r z9B;1xZ7DJ3nA6Y8*p{h*wK*-jdJ#kkpwHgpEa2jWQm#ZZWgRAuP8xz|sRO?MV@K$kx}rf#$|&t%jwBiNpA<^&5^L8FRC7r_L1@+t8& ziS8|vv^}2NA%5Y4q0%;}OEBdr_43rz)g>x(odpdCtMx+l1IPI=C!awt^_3(D7dJZ# zz3XyXO0S!aIbU(n0EVg*B{%@Fz*`_{s%?5U!|Fu(#y}`=do@aR{z(<3zA14!Rs8UH zGu;|6qu=h;y}eVwOIOchLd2rW&++n}mX)G6F;=BZa6tE0PVDlBX=1(Z3W(nr7;S}6 zqQv&tCeRM1mRA^y)v_a@cPGn1W6fmgPQu&7^SC>h~&%bL)Nb zjqZaqj$W(gxb@!uRXNExs@MAQZoP4fbqND380PkPe!2ccBMGJ)fBB7(qXK56Y)W)m zd0!x7!ndQH?S0Q%-=bWZ#!;h0`BW!?6e@Y9z4+{2+ls9#q^3ZEX+XVdl=!s0lf|{0 zG}JfH*B6L?d2@Ab^wJ1}5)b2?!0F$Qwwg3`+HbFreJ&$CS95z(bO^($NE3@j;g! zIpS7QVnXo5PK|IrpO&UisuZghmB2Iiw{d0vW`ZtGIVzz=n2h7}@os>j93=*4b$^8s z8=yQM*}MTk5Bc}E+ejckB@e8IhLcpJ?W#z^VZOlC5)u0%OgVqdJ2 zoYh53fe5e~1Tx0nasCGth3Z@PKX~-MZjXCiM(a@T0FPjn7{UEpr5tvp$~)slMdd%W z6hq*y#y_uJ|48_MS2`Y>A9(B-dw7_hp zglWR9r=7MeA<;}tC!KJZEHsSyRTB%S!0r7|z_~E9N;eOr`0Uc8BJb3FkYCf72Q$uBuUG_&GPO!8}o3vQBK)V#9Zc5ng&vs!`&uT23pNg2OZN zzstq1Rr-{y0oJ?uQ8AJ=0|ksyN^DjzpK(wg&NjgO`<%bfC-a8_M~O-2oF8{|)k_ss zy2`^x#ZZ#Hs1i+;=X!Sv6C?R(@rNg6O^jP-_6+?sg$E@bJ}M)4=v;|K5==#cjG$F| zVikcAN>F3)qVTNRt~z7Y#+8atrmOsAv=zyaRzc?nkHS6jR(fsz>)E5~sB<Ye6Ja3b7xam0?lE z*SO(A9kJXyWn!vC#97@hK3N?OeIDKD-oQr-re=|;&(O`HJxfS&K!*s^xAsW?4f+GD zfT4cbE;-DbiDxR$^@-gh9tIS8nE2?^AWVjFQ@CL$=xP#LCCf18`@lB7?bG&@NE}(I zW6UrO-yyraI#k<38>mlqeWz&e*P<4Z@|3oH28D}97>))IO9`LdXF$AIVqS4Kp6heR zDcOZ{^b}W&De*_;`V1)&UAdV84)vo1nJ2rN@0|I}_ZcDdH3H0M{L&}mn0SO6?byPE zRG;)+V$u3!?H2=V;w~|u`ZW`fzLepdBCqvDpl{U{Ta;g_G8`&`Q7#Z$den2aaTI3* z2Qs0l3WY--5^%}#^Gc*=vFYQ*`=Ush zM>@-hv!O$J5RRTH{X3Vx5NQIY&#@h)pLy$+M!7(&?6UKmg~e58fYo0P$XmQ6S0urs ze}Wv~CGpa_B!(51Rx%ac8p4rSKhden>5eLNp#(fK^W`{hP;EA&Pk1aymQU0z*T-Kd z<+l?j?RBm7YxDsbzY6Wd`bWiZZ~Hn=g;e5Q2dwY7bOk7769`O3zGEg1Y-L8Do%rA` z>-3Z@)r~OGx2<#`hiqztqfFT&@sSGa+m69%MRIHb_Aauf+S|?u}>75edWeZ)xt0JSe4bU^RHW6fbjr`&toRvkw#u^&g4hr{o;(H1<|ykEEAYyB#NVt&X6?f6sAm)~5qzer~ zR;R>}Z1FirJ6yHcic8i?HZhb5jgkLV;9QB0R*lfOA?Zw^)A39*-3WmX4}TLmX39{F z)Bt}O9)@t~UD?i&$dJ;$Ow42=mBzm1TqS;nS{(gz;@N)})JN$1BkCHk@{d@k`t zJwgF`Z@VuU${J47>G@3W(APQZS}%9eiwF%BkG^ekTrqq~^(I%}wmW1qrn6u$uI#&X zv$X`z2SUpA&FUh4BIzCGGf~)=w2AdMKBEU{EtJLju01Ax^pftP4^m3v#EGei_tv7bQOC3bI(QVR!J3jj4zVI$;7!8DS5iYq>XYo7p73%)ApSX;z!=p!9ePV zlw;ZAytj~P2poA|Jb1l^u6aCqFw>oQ=8`zi(Y4KCVafKR&Bj}G_~2aS@hC&MAI(<0 z3Cl-&C&-vuyLGHLRn6nk0U0vp*uDCiiD6oK*16c-WPs3OAN_2LbJ5&uG)N!)Zin+L zqrNJ1DO1Wly0@9g=Rm3=1eJ1+`p3&*u|1Uf%If&`FK6YZ?@jVws`%_ji!0>)tQjb) z8yssd$@@7{s2qin{C0B8UgzSZbJ}ydL~?^m)|1TUr}N=6RZ5P?PLm?lL_%pym4~97 zpPS6oUk{_p`HoUOCx#RcdQC!>YaVRHJ{Ewl-7Cp;c0kWm*JsK7O2x@2C5s?GNWlmg)K~*OMWIG?`AT> ze-&d&(Dl_`V(D-&z>-hPDeh;XC)!XhGgFpq*Dc!s>sPyd(BU9gLyTNoOL=RZY})kd z3(7F9LD);u|4X-M9L9uuj?3nK&6J^7)jTnc6sqKf1+RF=f$$9m>TyV}B=a@JO_$Rzx|?hqzf3C^rb~Rj zSx)8~1$0lBz?|R(a=KC9*dsYmE{o}LtFshX2D_sna7q-c!Wsh8AQ;w|OEm~cS zQM7+9gH13fSHP4JY}B_$EQ+gNirw$oaG92JPzRk>uKzt9gRhunN`)5Xn2Jt&qe{nV zK&$`V4%w0$A&ZE@G{v;vtC!^X7cW*jyMM3K)^i`+b_``U=|0A2l zx9s%z(bgDQ0mHmqjSs$>+mr3(5JbYbbn^6;Vz{DwC`Y+pN9pQFf0Nwbuf9<1k(yuxCx?JFY+N1xmqF`TUjU}OgNr{ejzCnV*p|$P*`~}(Q z$UNP$jkepfWFubQ0F(*-8U{*9?BLd6XJof&**-gEP>g$n^>p=uNPpgZyoJpr#2Tb2 zZFUA9K9!)ALp1jIF;h0UsGmpjbXmDhPDVK}c!W!dvI#NPGlVpyPqcN(40}xQr!<-> zoBgGSh7K0Lo$^qsypz=)Y@v({uun7HyBC;GG-m8=r|@1<<4UvCsV$b$C0e#JI6S8F z!o{uEwbIHZsi5o6OC-fzZ>(;&{bxtW`5)=0^OPBpvS6QR?!CxHm40^ruB+v;{387Y z(#}o^#EQnuZuO!vZ;@FnhBibw!{hy( zq7kcK3WrgSL<`r4FVcZqrmHZqcYQF$D-qlxZy}#ST}tF>MT9|GNDW^uux_s4x=wggq1IL~NGXB*qy0_F! z`<0xP8>akV@rSDBxBcx)MJByAe<*z*bl)qwaZp&{$kRy6yU4V$pNJ}EYZc03eyf>C z;T|9MFeUs;lSQMg=7CQbQ=$f8S*mDM{e+*-_P>|moF^#&UQ-sM$_uH%os5*QhZ$|9 zMx7SnESJL2n9%=R9V__bIiht=bGBO&&blnd3 zzNB(}49B=(cd|^6x=x1_jA2?9E7Px=sACaT;km@P}CLI;m;Y$861{zSS ziF+eQz0yLiObsh_JMmRlRq~Xh2F#5T0kSuML74zxH3$pqiva5ml<=4?P8R_dRh0L& z`UBqWVjbqioZX=wX281v>+rWdlnd05*0u+{5D*(U(p(m5Km$&s2meg0S0WZHemk{k zg7eEgW2y(NsZArC&&ZswkirL~XHrU6kvj~pdD{MYoorTYo_xSSlz4BKC}$}P#Y`g{ z@Y*SP>$ayC30BxyoqH_*cLkA&0)rDCpk zY%QQl)FhCcU@pD|l(AT9gG(aUEFgM>mV3ki`cs8eLr+aU6zto5m2T=_foW<~u(x+f zK>Iee`8n~|M&|ClDbcn_JnuabYY2}OJ4KuzQf>f#Z#&g@K!o4*j1p;P7O4rx#LWhn z5^MjD68Bb!OWb=6qlv)-1N9y)o@YhzB1{7s*!ViuFLf7crZ9%bxCHALIs+xv;J9at zyo+ONTd_J)(B=1Tt~KL{>S;pi!+T|Yq)#fs;_8Rk)rr=5uDT&Xm7+wG zj%7# zj>*SvcT~`s?jkk&jC?HePdXe7^o_M-#I=`<1~!TkH*hR^w?T^7uj6HdGh@_T%xF7s z!x{I!z$5xdGg$ogz;=G ztp_9*l}!hZzY_d`b*W0!->RjIj1#r`pVe&+&)N3CrU|mScgH(V8ED{RIo7FQZ*6sL zv4QOGog+^#5=?vye7vK${_{23x9~u@w~ojuw=@Yg51&1-4_A-!3QS;$# zs$*>6tWxVILK_LjrUMtoTR$`7Qv{=nERwfzDK8N6?Lc3Yye(;=0_IgI++(S2V-!=R z=#-{3lN%qbuS6h*Qvr^$178tdkt^CKODJ4cwjVl-yNIBh@$twT^eb{c%`m zq=eY@$ui>djOzZhrj+tjagam4M3B*C&<(q+d1M->j*r19assCqv6vA%h-YV7b74y1 z;ODmo)sM7JG2Gw}&mMn7I;c^qHHYgiA2Wn&r~~GsgR(9+>;f1FNg)07sI2QHN5ZZ! zjZU+2=?g2dm{xA!>20zB51%PuMr>TH^(E&|m4UDtRUX-FC3~5kVeGR9_25N&@5G&! zGRVGQ^=Bn-IymBViAN)?pkqU~<2Ewk6UJk?SK4t|L3pCI1h zK9J%8FvPZtl+6z=4NU?dOz0r|q_b2djB+KT_aA(vlRh2=4%ELu+GcLpy~S|DAYED@ zhVP8j(@3fzWk`{%*QlPhSm1vXY5;q_Kn!J!yA`BH>A_I_ zkWz!NFJ88M@uG!d>gTry-?uH;m-ib0WqR$w31LnO-DePOwJrm{m?MksyIqGAqG3u{ zwz=b8JwPi_AFV+b3*;>$t_w9&m~ub+J?|=H2c_-c8>8e692mN4MOt_RH~1*=bF<4oevO?X;Hb#OJ=tdcbN3dShSH>uDz0h6{hQ*`H4IGUOkmBvB840f9!81L641TR^oT*nThd1!m2*R8*Q6}h20 z(Oyu8=Ehyrv<%;#hVt35$@1dPrNh>!bHbPwSu6KlKdz{=Ndwrs>s(8BO?6h|B-AAOc>5#T*GUA~F z`ul<;aFEWy_wl!Sb)!nzW5}|p;xOkO6m%_i2#-NMw(B9aeqww(2d{oGacpgd59BEN;N+KO9$iDX+otYBMQ! z4$~cqanzfqrQE7}InOV@q{`dkwpVx3LmQ-sa~u`LTI%PshhD!!-o|bd^*sZ?SB*S0xmY%(8&^DF z43C$)h>LI6(Izy?i9J%b8>T=Rn+`o(VeKG^1{ZSGxQ;4s?+{z)J{41(1%au?#izPe zH;~hJAS=OuUuiE|0Oo#n0%D{A1#0md`gUE$2)bCcD4alL&}E< zVsl!5?H90M_0Ph@v&_VSZv2uqF+*;A`&|`ZreCkop$rY2kzJMzS54h$o9;XAEajbM zP@2whDA%#OFlnZQ0hWH>Hfxyog0dD{qq`XT{CJrtKhrIHx7K!Vq3rV7bD_XRIUeci z@?z6#l;cT|E*xkFscQQ^JH30Pbqn(bR5wOWf2Y7HX27}?&!HeCnpMa;WGZP0QcBTn zm8^4~x?!2_wtx>|ak|rh^QSRXfiYr#EtYNB2FLqK+unBi&1dDkcj^6XPyN_jdB?Kh zy4%uK(qBEce>dG0B29^5z4tg|xy=jcrN=D?_PlsyL9vZF}rfL z8-m(EKWE5P_SN~rtHRZcJHFeYP%23<9Z+93>%3ogD2B&?Gx84RQT(BtMTsHNvKgyi zO4~fG?N=9^EVAL?BoEk6qn)*+>8VW#6r_sF(G@P$U@g_uvDiOOi$O_`Xl)VO+aC5q zEwPFl{=txehLZr}d2hiTNq&1cPmuQ_rHm?J2;DcTkQy9uB=Je zbKH8K&HxOi#E3r8&JNt5fKd%8V^+&61oJwSQGWz4fIP#=t`cxi3Si-NWR|BVCy*+p_O* zkrIaR#}XO)V%>YZ)D&O=`jITzmS+H!E{KM}GjILwD^uq4aKuE4pBNeNUxOnfGf~{Y zbfua=x}bH@uj@W}t5PmwZL`v)%R8&xdlO?B$I={LywGldF;Xta$)4W`Ef&#!J0mh) z`qt_+fyVn8mFt{^SMJeFt2EQ5xw0o)ScH@b?Tp8>#JT+@?HO2l+oQ%FcUOG*j0T30 z{PxIZo1Nh^9?%vGQ&L7oop#i*69wH6TJLg(lZXRQiU4#{sPuekyLOcJZAQ`@*>k~e zWeWw2+qPjHdw3a^6&7^4xLMBQr9A4ZpKXuGKP8{Gmuo3VLM*v%MtOrg1MKBgXSugm zkfeBwWOd{p_%;t7rogxq%W&xo~F@ zO$>&{b0cqza`H;wRG>VYeqZ3sX9TO%*HSX}x0k)xenK%tM3+B8&v>y&4qDnodoG}m zW|SV1gD~OlP*{#)H*y%g)KwlMXGe&CykYWO8RgnvlD)|6;~5#G?Nem0A5DMyf0THs zT)fRCfl7b{wQ@5vWskgL)dm_>NA%c0-s0Nma#7hMV@jzkNeQ2iqq0sv6QA77xwaKTn2p-_@*`tPa z>G2_lIkrMTV@mJ@4_8zbQ)Bf}9gD^Gq91j@lqnpezTD1|X^(ddcU;B2V`e==MmPrH;!G#&mL11IKj91*L;z_9?cF$#J92-NK087$RJ_zf< zlj7=dof>Z=cJDFK_wBVxITH?}_SF}Cy(3me+gYWud1Jrs#^FPi3kR*%oG*+EeZ@wR z4rbnsuZEPfQ7%esAaJpTvDoMbGn@|IZBozz{d53sgAd z98vGQe!Hco)}?=(9Fd%^7#gdOsk2j#Xl<0csJe?$?C}l%p-c5wy#~|K*}J;)fPo%M zm1!gYG5ANnlk9Y)5eAWBO!#ZM9D4m{)m27w4khKzcQnE1(rTaV&)s_u7|M;zl|yPT z)gvWcV$76MXAF5ZGhm7dC^7Y{9254m4$G}dxiL8$*CnlBn|D9H%9(cfeH5KOnTvlC3%9O9|y)pFL*h(H_MtRe-@XpfSBp%e%R@ zhe#7+WBQfKyLrY-RnbVt&dhfL9BNf1@|2XZ^&_2*?42kaM(p%tCxb@}nGKVzk@PT0RN<%W(E%R6hYUoCM>+)WW;|9F^6l828)ciB zx2WQt8)^_-#c^^P+~J5PjOOs#7!Z^Sr8uG==A5x9se~_ zGDqzQp5>s9!l6ryJ$6ubEO=axKfpqj$AtB3onN(BL<2oXSeLxPW>e+;%qQzx&oQ6l zzsjx5i+Cjh zDS$F2d~8XfjCtM6_cbu}THggubt2OE<+0u4rQ}e&V#?2FKjy5Hl2d8~i?P__^`^>u z_ui~STSI?5VvoG{{joZNfbX{-&pRfPN!+^|i)~fVAK#S_e8^k43n*aYNvm1#=4oY9 zHLw9``(v_j#wLvvNmOCEl<$5B2ZZlPAN^T79Un%@H^Tb%%3IX|^Z2B5OwP>1BStK} z_P3mpM-LDBF`ao-879;?#<7Y&y<@OC_{WuZv7W1K)-9VV zi;df#DY`O|P*wOr;82OhqeIF-G0KGFUe6GxIgbk{)1izXbjVpqvZyc!l`5phH{5LP z;Y^K+Q^S<`{w0oA0^0=4c(#of^2iQ#l!7oFwoQ&Ay|_OlWqjsFtCTMg>lTVgiTr4( zav`CIVhmx{B{}NAZXL-mhVbi78AV3nDoO-Yz+^>4xN8Ql*WXHj`R$4O_quDI{!Swe zDw~e~ce(TP#Oao?oNkt~;%g<^gDTYJP3 zdWSsJ*T5z;*y7wwtIP~_!Q)9M^`C~{DqDE!UpX#zaF~=T9)Uz)S!+yoLKp*#FX1Fs zt`t*#etSZz&7zQ50iZm>hBJSfnGX9%lndSdhXI-(3#dgWZ`#)d(otxOYzG%%p zay76~MRM|udfH_ApouGUtvk3|tt^I3a6|ti;;-z!DY1rd`#ITqka6YmiRde^eB4<*pLwrplL7#f?l>!^}sEaL5RA zzUIpTL5Xf_XJzklzUO$hvKZh1cK(u3SvB~t`XOavx%k@bR~i#$aXWvlEYe~Pj2JoqDydAayuuxc85BpNi_%wrDFT~WTlwW z7^Z1^gmvxK^Q2X=J#Ha+XBbHN0sXlmxnkf(P0%Ksv^-yoWAU#x1`4VYlR9O|Xim#1 zu^RfME2-AIESM=^T1wXEP2_Ao^DmjtD^-3k5*3{6$_q)E+%L;2<;o28jnyZQI3(sm zP)v>3oA=21JIzx*F!kA!6Sj!mEQYEQ@PUv~D&{f4^Kd|z-bCcHrbKbEDe^}Lm}90d#_gu@raRu-zNc^uv>wlW|AgYn$tven`R(m8~M zz8;gykBhC1>Z$(N^vN}tP8ks{*0GYe)pRI+drIzJ(Yrc3p>0o@xmOfzzog?7K9nGz z(zX`@I@UF?sdpU+*5N{30E>Z^690@Ag{vJ6ohm+i3U3{4BWH$+WF5FuTa;QkT)Ux) z@&44PLsm9ouc|*~M-$nID@2MZKfgV-ZM0a%LKO^EDW5&{&N*T^#!=7>jyvPUGKP|> z1V5_8v=__BF6JpmmAg~LvVA8)7K2cOsaPkLKX^tl^nh8k%IX93$?Q=yekBHDxG>#fZp4xkzSjM<8V34YS=_r?xin1;QVijO%5n?%m zUaaLw8FWZ2zxj?3bd>9wAxc?hD5DB|sM2?;*g`%>RmIqYS0c7}mq^Rxlf}!WAm2u)} z{!l!UG_bWP4Av*-NHFoRRyGedwSQ^;w}~t(bSl5I#wUW1CC%& zk|?oqj}C36984kHJym?iCPokfd=%lb_@49Hl`w?>(wDo)v0o->9=gP|SHk631`~>@ zE-}6B6?sGJ7S*9WJuXAu&_<~Ls-JC7`+mIafAl>)71c9vjbrMT`f|mU zsTw*8V$;L>8Fp2 z^*pJpJjc&xXLmf*V-V-*lz{@;9^a&kQ|xJh(ge)(b2;L_<_#)ipzIb~WPLXLi7&;} zZ)f*DB5zr^OgmfKH2bw^>jy@CU{F#hk;*+!-bPn8MG9hvc93;h{o)VfsO$+4xQ(um z@}mmR|JF^@{g1L3ZeZ_q?kz7WktX=F^9p6%sYW@l!1Qpk7-d#!j6JeT$b&|&t#knk zOy4Fo7I*OKA~sV4`*^&J9JxZ5&{DE%5Gul@dWV<#2FHx)7lOC(uBbZ3cTuZ71(X@n9o0pLK^r%wCH%^cp2T&6JI9DzpIQ@yLpj>4Rr#<+N{Sa;~ox z>7X#A?PXI7pS8+el|K)}EU z9y7Ygmespn@TfuHsg{;3%mB;`NM>fRou58U4dE18cyJR`%u3NXG}2jno4M1TsH4Ag zc%Kr}3*<1?lfj|*AcduyyLexW;))1?6@j*U*H?FaGrzBpt@a$y4LPW)Fem#f_{i`~dgo`C`? zK5{so%~F=urA!f?n7G<~YX9EqFHjEmqoNrn1Ih&MtduLVo_8TV7 zg6TKm96b!iF8@jz_6Gx*2|>VL50H; zx2B8zQGG&4A$(RUbI$2xm5PWx?uruo$(iA~0Rm56jwf|1^-VAJ#DNR4A(K_)8ujPC z8*o>~WoR6R$FQmLvFO)zd(j3`Qlh-1s+6$pCmuT_A7g?CQxj6NPo0$uBL{}G_1jOr zpW{yW?Q8WZP>lWd+zuDq35QTANR*aW#@>}9=X&_b1%!%kuYRs^mho%8p|{Q=xq zNl0m{&wU`yoyZ}(43-&ml_%btBWL!TrwK9Yd;4U=eb%8XZ)qHP%5zUP4!7R-wimF; z|FGDT*IyAIGCoz3C;{|cXRTs}l1UJZ)j#PXm)RzxUr-@^a>W+0kDPA1u_;m@T&ZOR z5}Sk~*=Nsb&9y9EMd%FlGUb;!M=r@(YxEAUanzFq0onGKC-g@g(HN-vUaKjO9@Y+Z zpHD8`Cm&)p5DBJ<&Aqd}c$@rE5Ii0?#L8L37!r&do}5)Ehu>>f@r^Fa&&gqhzgS_R zg_$QKcgZM*$^(I*Yk5|_;4KgZl?i7t+IpeQQ`OF%%P9D^ z>-Oj|E+v2-wNp;>rW)E}PyVt^=6-%sn-DYl?c5ew;sp7VkziV7?#l<{n6Uw6aidG_ zeV4`YRdaOP$JstRw||^C&N&nqdL@RR6UUwZRJZKh!UXZA_g-V3!jU`oxHw)BZiQhn zYyyv0!o_i}%T@DugKXk&1~m^vQ)NGdX=(a%qSep+LjQ7xA*!tJh3tb4zmNgpUqtn^Fp0{=7#{2%71v z5u0{aF8QvT(pG`P9xSL9vH=M}>EpBKb&Zu1{u!--X%o(SWFI_wS6gh}fqgPN?kC+D zPz~ruQsh*o3Fu+`lDFoNHJgtG1XYa^^!&5#FvpTm@P82Uq7&upPf`^IgJb@)74Do~ zzv@F+nn&KssWNtGa`nJDuVt<*J`(Ha$Xak(|yY_=}+!%d6-hW7fT^?GeySE4Rtm7=5`%DMx+wa+GJWHw)yPugB>U z8v1;O3N0g^*RF&BkECPbImV}8Fi7Xe2IT!M{;LQckNa}v>LVYh|0)wwMa0Q=YiFwe zs-JD=&p6`d-|)NcVK5RU66(u#@0ux3m1<)1nRtKy$UfaCYaa8yUM)Au35ucOL5Y!5 zWxFjcs=c4zbe-E^+!0xqPvd6QJIr5FF7M}c%FK}W=TEFJ$G%`|rrPTBIZE7$G^3qy zcBMj^S5_)lY|tBJYQY>}+j+_8;afshx9#~8OXb*1z58a`-291`+?ASpHrPGj1r$5k<%t^)l(k$`0f1E1lgMB8Um)N`|SMX zMdEur7es7{F2S+JS${cF?c4l89po&gkyJ5A^S6}B##~|rwoxv>DpGVJt5-(I+4)(c z<>)GXdPp?@q{CMU#yTHPt-lPzgNpG69TVY+~9uZP(8P%R3F`|SlOJ48NN6OdqRn*VJrIi-H* z>Nb~uWvU!)X3>g|&t4EXW1YH?rW5F}Zvl_6a+!JiY+VJ$5SkUs6s`|~(lA|6w^U}f zGxL`u71|4)i4zMtn%7+}Y995k$SJH$poeK53&_$u;V}^kOP{^)e2Tl2%z_HKkrH1h zC$)G(Dds7cm@X%AtN?o&9{EQKqPYIo+_{geq!uU*jms5_0%d~L)6juL5I#2WUJ_(Ms8aNALt*x&>0Y|UfAPow2@ zuDB~6AjM)GcFHx6>!$=27R36C<*MK@Z6Hd)orE@#hSXp~#%t0l*OF)h?yC~^H2 zxg`0J`mfP;;Z0lQk~>c6;4e|1f`zx9mPyU)iy2saGm?&KYrAETJT7* z3x=kPt?V~c88`Usf{4q~V(qiUDCf5e;=^S9Uiyxe(gi~AC|RF`xk|ZA6_yKsJ0?eW z%+XfYVhiSV5L6ryQL@U!LE?;wBslJ{c_noXLv;N2ON(=owMSEnnQ|DOaaLF78%K zM^KUmR(mhX&DZS(eL#A1u3X$Mzq&;gT(?dxmal7qsWR%E9LxL`)jg)S!n9MVhHiMQ z%92UsRZt~N1T8I=gG)?qN{C3gd|B*$rk-w5Rmy~m&d0mC9DPH5&!)d!5(ij7R2Jh$ zm4C;If4ycB=I&O6MZD_vV_>0fa{zSg?1OT*1}Qtr&i>{y z9pYLRBF$%Q5sPlq`Fc>?#8^$jkc09GE^H&g#Ob07iQ@gHdfC?m|Kfp}&YOR|tS8=^ zmDoiu=gK~3E@>$m$0F|FJ@N7a%_B)``>POD=Wo+~0gG)fnteb%!H8Y~Gf2sl_C!ul zna_M)7thRe4up4;5AvBO2^UhnIW0RbGn;K%%HjnboC8Y=b;c!2l~~+qi`dA3Rk?u@ z7?}<3j33O4Y?&%7Yck!wOV0OBxCb;+I+Uj?@Ol<&4Sm6-ou)Z!cMV!JT(U zwvG>&BKCz;afriDS4%7|Ef%Y|xzw43JM=;9+i7Bz*(WqSY7%PglLe2|)?!VS-=js~ z)7iQKRjPJd_;Q?E(sq=#m`1v!`(7t_@(lq>m68q|+tTGD%z=nv73+xIoFzxrJFlAu zOnIs0WQOcdmer<|;ODcKj6N+!vORebiy5Xv62)J>y?U925_6KpWZn}G5-_I3@Ezhe z^3zp?Vru=$>C*pCh)vU8lO+TJla4v6`)C zB?ag{yRccYv#IlpP!`P;dx>*dY|rndy1s5@+P7V<$Qdl#VpGKwV9r_b8Xp|QdIWhz zW-s}$+EGg$i;>xH&A6t-U3=tHtxS?g#G>t`Q{sK@1SlLx`d@^cVmad9X>yoWWU1>U%u?=*q$y z8Cv&`2$ityr&^tMpJXd-w+RPuaJ*pvvql9yQ#GK%hmMO~EbOWh_y(~~K+eEGXlkSs zmP9!7$??CLb~{CdW#O=O;@$2h=^dr8>Benx$y?^_YwQY-XD`Slds3AIyBnVgeY9-5@Mx)+!>u&01i%g#i+8w>d=X#}{r1w^GTpgtFY1871U~zz zmfOS^y+`XMHQ@X0r>5@{Sxs-!Mn*I|SO6&K8mrozk_5rsC=0rq*E%)fsf8)>=7$bw zZ8h|#HWi6C+4rEnr~a%I`AqZ`ov^M-F@5UwEi#G?h%(dDL1=eME~#~OpAf5u=gXo8 z2kH_a_1l3}tKFv;y`pCYL1n_HZm2Jx9`=m-?kjL8_d~8MjvcLsP8!G37UlA$?q(BI z`(pHK_lPyRFd!ED*z-JTx{J2)k*yH(hcXs>R^oK7`*<YRv`BI(S@$B(3z{okQblIv6ba#|2FzFmT z_1kxpi+VO!@udh@{f|hOb_kb?I_ce66NO8E%@MCH{#7>*t^Ts+8{PcDGu38{>f8PH zvcPJ0{$F0wA%*DP_R{~7d$2k&tIrP9+9_M`(aHoE%GvhP9sA^@r{`7&ZlDoqz3(v- zJOh2mM!BqRhJ~h{E;R^MnQ}Pq;h>8$5(auWTHcb873wY!JMgG%dF@kR0d5Os(OrPS zd|m_FE{HOYiDL^@&1Wy0U&|evU}>uxrf#CVu6dj4uW2B2ubj5QWPi~}1IPEuhgmPY zNSmVyV!w-&4?kF6g{2x#`RwJ*I=D~1I7%}mw2|`tPIpkxqx!RkSWI}~iaU@YoWg;T zAf!Z#PZ{J@8ekAw%#q#yoS*~7lo(Vj>s0tduJqf>Um5Q%>@-s&ML`H(opuXZ4pTfd zVcBbY|UofKICE%JO$}-G)cs(+d%sm9|Aw&$@-!qY7sSwfc3B$^PX1DglR9e|f#j z5PEe(toqq@k#CPXixcju3Z?vZ(PKN^{O>R5_z0-pvS{*VH~;hLA@w25zUo4N8v7D@3)zNnRu!#OI73se;P?G;_a+>u>R=y@fj&~|txIc9c+LYoq! z^4&h0^&*02F%nGA+qYU-+<30227$fC#CN7^SC%NGD^^Z*TX2|zKa^d4_KKnkx5d{j zR3Bz|tSF0hr-dEURbWh&kN3G#|9zt-pgx38YPr+ypRbXIsS+z+Z12t^p&|b@A+_@P zCT{LKZPg#94_Z02i93Rk<3+U9JXYqOawoH5gjh@SSW%KHJ8`@Lg8JF^$|GCc1suJ{ ze9H1Zd*#RB?t-qd)r6uicFCtdN(;pYzzWvMsXGd^a(Noqv3z%E`Eb3or`@n(bvHTh zE0gAn620w}&5LB5_h#%?6);ahUN!`QA>JTMSd0_EZuw_7Y)B|n&cvd+1iH^~*DgMz+_C;YzR zs28AAXF!R#LOHwe4&ARC9`_14dr6_{7}d|Vi~oL3?A?8J&A?}`s(U7QjO{0kqeR2B zd3$GK)o)@Z>DNZ97FGPqI+?*VLHSl&toYpsndhC^M6_+M8pC&v?6+k|;Rlc0d^wp< z0mV&~XZq(ly?NIQPzs#iekL{7N#rbkC4``I1cPPU^Bp6q&>00iJf+&46MN3(Idy!n2{hp3> z&0~!(;0oT4ia|L|Slc|)T~>I9&SDkRNVgWbop|yR3C8&?XHMkxr{yWFnLPU^2mK=nauv$JFY?Y0uJ5GXP9khAK%d3(VSI7l8#))mWi zY$SP$Sa)wh@Oo0oCBamO)Vip2ahLa~9egP9aHR7PbJM)gKwy3RYPXodL|v+& zF(v-C!|g|&O~tfIL(i?_{Ef{e{!rBr`&pLTuXwN;y{b}F2^0pore=kOps$bY=%zh% z&0fL!m6zSZdDp4&x{^edvR&2*a0ICWbeR72vh@XrCo_>^pr4EnKFfkTpp-t87PvDH>KmJ`FQmI+2CR5&Robq9a*Igdc=kmI{2-J}weD_XUbP3U-`;Th zUbpd;xAbx;C4Bb!uPVBK!P*v@n$B@Oxo&=WXP*pU4@>}$(}C`9bB~V_%HdS`F0%U; zvdwp9*i%<)d-hECb(E;21UOWQbKQSsR-l+_vFk5HcmMd>bJ80k5}X^IiQ32kWx@^h z_gcSld4oR;jtvF1T>H*o^g}8B2op@5=B`5=XK5lTu{b*suYCHrOpG~t_7u0ZexQKXVqeJvmPEkdm~>D z$qoPACLFNG?~&pjE`U%){g9HI=??EPNlEYsH?I>{emo~xrBREBK;KvH9A@uTmB`ao zI7ff^%j4>JmHNTs^=8gVhNFm7Kg5aA%#CLYD+n+o!DDriSngc`(AsXgF2aq^KBJ9{6u-Tx-BCB?p$CBARoT?2 zmg6ImFTylWh190O@oopw@+x3Df;RqiR-B9bNzbAY4Uakd+^*ybucQP7s&qIl9wwQG zvU-^Uw(0RL&S7q}gF*c$G5(x$xJvgw#$ubY0?y${=QJ==qVECcz_r1@>AQ_?UfrMc z9}PNK zSF!7ho}e3{(dC;a&hxaFt8RF9Vn=sLv9`Ez<)+=qVj7#m)K|J7<@FRXjdXOFPjyNV zK8ttW;?w965L?;Bd5hiwo2o|Wv!5Lk=5{@`LZ3p?HhuQ-FtAE2jT7bP5R``}1AFDQFlfH&Ej3Y*%oE22dunpY4<((pX`EU^?k%B{@WH zd{tW=a8wy~+@sLKnvl+b>idV zZ}q1Dg3sREn)5-tA_7At$+kDghlyu6{SyQS`uOe5&vtNQEBv|yIH>Y{u{-6~hU!wE z3M)$ND|BPG>_Vb9cG&!5H#eU4^8Xln*SM;R?|&Rc#Vd+p;uS+fK~#)P#qjKz%~2G* zu|*?MQHs=}BrP>k5k*Chn%9IvQMsrnN@Q7H&fdq;C@eGzv9$1$W~HH+rk3`9ubF)~ zyuSS&{2%NGXPud~-s`ew&t=Vw3CybQeEhIO!v$<}Q6+@b&-pG*{}Bh10D~#zQiV?E zTS9_KIE{>M%qfvgC62S9jAKC;^`*sjy8GDAVQ&*)%fQ_^I@u9~TXWzO*^KX8=;i2x zVG~fwSl_7~a|FF7hmiuu&d~*qAnfFda=LP7-%G|IER&L05H)otZio67NKLV-x-;44 z=(Ipw(?q49oaLC(Y4z8F2TFmkvCt80#SsQhd+cQH>{}#XUj|sZ0$+H5fWV)il5!k!4C;_Hcb=T%W zj)334a=i_^n`St+4z>%X{sev3Yl+MgiA$s{9#!2nA9;=#0+u;_yDKHmD8Xz`3A`_v zE~=UXZ;GKk=jR^b?>K2BqOldvb(@mf`u z=r|(+P-5>%$7^%iiW2$4Q9sV+%ZFSRpXi_HMc%zS($V7UJmDx|)7^7JjA--N)l@R_ z?wNMh18aPQ1Cl^ElHq8vw1;rRRkHf-6*a6F3n67F0gX{2Cxb;`m_XXP5s%!ao{p&_ zUJ(0$mH#&Zoh#PpJ6b%T7SpQc2G%$%E9K$x?Z_{???)&!6a^^L9{>6%;pSzOr~)ZX zf)x!rmZ7N(huj%{j!`?S#PpA_SgvKh@f(f|Az-jPszg^9*Kn%nZD)g$`x0}sU@wc# zCX_>o^iap}#}?A80+5p~<1Z=C;KNr19zxsP?O_Hbm5_mCiM0hRw&4hpU=sc;cJv*j z3pW71rslQ7#oimd#HvP1a%Jvs6~@0vumcY1%3NE9r3l075D+BO8`q8HP#>DgMk&uL z+0hp%yd(ugtD4s)$@mPr^avPC04eGf#~7RsLrj4bC1US8hJ6v{e8oWM=b;}!!va?N;j=cKcD0j8|L_!qAjRgq#=%@i~!F*?D~IzVQa3FY##{2Zx1a-5A)C^7Pqk&7ejfk6gc-XMSb z0}LcmL69Yert2-Sm;wy)CCrN)YrKQQJmxx0m=_aaLNH8#Sm~F%@5efphR+hg-%+T4 z0sWwzRl@35gyf4HQ`k=}W(Z)R}=8|$T#<-VMCYTE4^1Ti_Uii#cwEU9CEicCl-;Z^M19%)R zXDhInl3M4vBhhRH&i+AtIpEE26=YnSWf3M6ObV6hs(>MVbl{gSbBqS~G`e_H z2eBR)YGfklB#z#gr$bvf9Lr$BFZn$^jZ2sgYYANt>zbP1Hqqi2e6H;jY0#%0l8OYfb!Ed7|%+IwumSO{06~v+x2y;@5k1#0^f8OQoF>Bop2C5>a1)gDS z*=P5uz64hA^bunrj`Rm88Tc)WAP_zVZ=z#dczgjbLlTYo97V2&I6T~3C|cfa79 z5aYm4AG#b>@N>Aa04b~aPMiL>#5eQxWf(=1SipyrOAn39a>|vA1j6Nq#xJCC+_Ib4sB!M$eQ6%9%mP3sq&4&h$71)!y~g(>)|khiJS zoc&~2{ona|8n#lye3I#&7V{lV;0Dx}NcRjZFz#dgVgo@S-Sc#saX+h%lVDZ%-~zU2 z%xxjQOknq8cmu4P5>u(%qe9*Qd*4Ew-`hQHF0kcT^Q00{V&sm;?d2U%H=AU(Gy z@ra`(a%nT2;jm}S5w;XBVH4nh6o`EWCz7Lc)g8K|3c`kDwgwCSBo;P7iA_GnDMMa> zA_xJ&H*456(}d6%RkFN|;_bbtkvE~QtknnO)W~)S2U%s`5l1W60u`#P z!$||U><1D|9P5kN3god}1=1=Cm_2I`v)X6-3Fs&_Qr+7q!1%KNV8sp4AqgdL9Bzj( z^|spYx$w|Pz;Vg>;9(QG+{$uv2(c^e;*D1Vv z$9oxLZnhPEfuT`l)I;M5B$Pp<)ON3yW0cgNQQWI;Qr^czCRCRZv^P3i`2+(A9aW-8 zxxL8|Mme@35DxGGVL^&fo_J4WGf2Xm3%mu^=ZFU$lvs41?ZORYHV}XVgp?xgar!Uk z8wFsC!?*{MIALJ&3kXYmK#&v6sD>&_FL4i?xKskMlCU9&d-NYHctnX3MGv_L66;|c z84i0_S8xwpLQRCSIv!v8h|ikjv96l!zb{u4B;n|6hxhec;sruAh?8Q!uS|R$$ z0@z+`L}`YNY;b`YL1-KgPpG$Ve(&k#ywxL_rj;*%RRr*+BNOT|z0LG=+*G62g+pa6 z_Y7D`z)Y*xRVYWPBwL; zklNQTR$qs%w;NO+IeB1GErhSPVFT|S`rwJ3TuWu#}a3J>?5R8fgXj047W5TUlRxUh%JNKlc5Hfc(?G|63=JMhR`tda*tG<0}W9)(fGj&@1Q zL`cCVDDh+s8!S(omq-i4eAy68W0Nj60#?Yp*ifXdTyFn6OSH=|K{h?g+7LL?fLF9AFH>u2{=Pe1KghEWPgNii|rQCBPKA zobcAqt>D7yvI=gdbz3e^D3kVh6y{ibTHZ4PEMWR^7TbyGV-z=W{1olzhHF*xVdOHQ z+`08^r#V{;9*}Zxt0NG{Yt2Q&CTzOD$1z7AOsT~KhlF08#k7q5POGEJ^}9^_d#*Fa zAolxBY!_yWVLsWQ6#lT4?ZS~%s4v@(!n?uxK`iM(G3mMep}~$c9IytY64?HB{toZ1 zTSzQ*(KU5{`}>Y|9W;UCs__0TFZKr-CE_9MvA?UYqdoSmK?=1HsM2J;l7ea5RB1vG z0?Hjd5k)?jUvEGL+TT0c=r6;`3OGurq2K>RC~FE`U<(szV4{6Rq>n&~ z65!FllC}I$-cLY+4gsE zC?fPA1EEBiJ3EW9DSnt@!|t=QPwpb+=%Et-AzY^tf``OmO<>QX?*MDj17Ow&_AAE7 zsjvqmfiOOXw|-jQDr*zaAM@6)18F|0NFJ{Qvjym};a~YS>|dU1L}4-ms>^WLzr5HO zjm6k1AeBsCs^wi?$tT#P9FV>`n|F~3hcc4)FTKu|m{<8gLExC@$-8WcafQ_W%w%H> zCJj|Sex;85aNFf>~v1wZw4s&7>fbje`p=IpsF@#7}899P5= z4oCshYvHUF=08ZeR5Ia#Ub7t$*myw%lmg+Yt%{06L`{pK%ALgNIMgV}Q`0Up57B$(ykBaRX>k`MSNI)a|BxA*=)`@_nJji=~e8T<#j`6)f` zoOJ2}9Nog073bvur93hAK(7#`2e$0yqdi46g8qLA-LENr81EBwS>pfda$v||Wd^c; z;5F%}12eK6UP$-23IkPH=h+~2qsY4P7dBlMqMtw&=pw`6zz%=If#cSrfCKRX9^12-0;>}afP-=$yF2kO(P2`cpi64jmHwFg%RweUl?P#LEEembzVu5`i#Rq5Ggl~1k5w)5^k9>bEJy^1 zCguFueJoGvz=3Kg(QB-6`C^o7y0oZO7H@$sA=v{8qDsf>Y!tTcKy_)tqRu&NCytdM zrr?1Rz2bDz6rLj-VCr9?e1UEt@sK5OtQ8i@oE;KW=@M-mMFvqmsznLqo~*E6#N0S| zm`I;YW7DvPgZk1QMT5_X5-4tJi-QH?%`KUz2C)N^S^SKLF3%M`9mY<>gd_#Jpv0&` z)&qBQlt~^SumaW^%HdxV$M7(H0FoDo;3kxdEqC;LL8T>5G6A4RR_gsQS|jD)3L+fS z8zKV$KTKd_{PZAsRwofhszt*qSgd)of-aa2f5;{xZ|im)kLY>I`ny*}z(}!Ah4VLB z)QT)Xk^q6R1CDZZ%w#6)gQV^R}!8Tz=0I*G_>pYBN?A6EzQ=0(xA1Q_Y%3-wpZguU) z{?7V4*yT%Aq?EnB%I_asbD9uR@?+VS_v+n%b49V=Eajw$7$ucP;$*IcvK zMW5#?=g^+s1`b)}%VYXlbb6_ZG|-7yHW0fwQ{kwNLdxmctS8?6k~;5~cV^wN#0ix2 zAlCB}^^4{ViG=)nqi~;?JV!am2aoxc*RWPmcW$mz_ z>jG8@zo6R#P$sZT-oxHBx7wo$cpNQKZaCIErGW4st6yGk;J$g+;jTs30$5`l2%^`; z`b{_HgU~i(3`p+=Gr!w!31F}qnBJbx8kl#bg$Tl5>8u&TvJARJ2{$}+XH9VQBJ63t zK>wG=!A5r0u>MHA=zb{ggGC*pSYx~% zz#zSU@Tm;P0!$85H~@=PJ=nUIHNoeEbP>Ki*wT|-z)^#+mW+>sZ920I3`J2~y5Zns zN0|G9cPU^@vE!21FpNE_oL{knU1QkS!6I9Zm_kY*w*ARti=_Godl$2(;{ph!RR3VV zFy@ZlIPnk`JE){kMuZp?0WEZyUf@_cCf^nO2l1WrM8M%+6S|KZpLgC)8v~ge#mB+% z1;%d{fBIL3!@;k89J|jg5+5AlsDtCP*g%}c*bzMNKne~J9*EZi^-Tk*;^-3A8}_(L zAB%(k(Kar{as>TT;(=-?@mw{tAi1T)`6B<9urR{-9sB2WFa>;6Syp6(;RSLbU7XEU z;3G~vQiU#WU1Oavkx6a<0aaEXVY6`vPd=2BB?__Y2n`!mfn8BzZ8@8U#HA1r+yoD- znonCOPXvt;B|h+Uyl4|cNYmgbvUBa?Fvo zM&3>-4fJgg`#g7+Xr&~uU#=LBVbY$I!vk$?(Z?@gj|4%Nz^t|_u@CDlP9J;}##$nA8ud*h|2vkoT)|z<#UXbe))EW4q+A)jK!>_tbG%~C zNyB>}xHEQ48R>G=A_a|hlz6xjKV;w@YI}+|!0T}vy5IqiHfMN) zLpR8L^r-3~T%5QXdE6b%(9?A^NZNWHFTM*bF-yd8b;6vlR!t(Lmhqa_%4voj0 zB}T6#MVRo=6n7<|e(KvH9Nb$NxsQrVM;+Q@cRU-PB>FI@2IxB@9Ahw9-UK)R41~45 zY>{~s1FC_r?V4VKTe{t(*pcd?Jexfatpg#IRrYMv`{GUv6esmjqM%mqhs>RhuqjG` z$0w(hckva=cfPlWzQ5u~`{HY=VwTX)u+zxvONC7()2|ZsA&}tWQQlb(!|oR1fl^>v zenx-#NVMPqJ|NV@IpSLm6}m{V=kK!AcgKoe4&bB2PbtcKuI~%{f~&JKv1S^h<5#IfanD@GW4KrXKj@tS!6RLvd)~>nlF(w4(i~I2 zL9iIK9jO+#DIT_M{B_p^K=B}EB%=$5;)De}cs4s@JaV-`#Vo=22y?KBkP;%Kuo&Z` zbBib>!@v|(#^$i|xCakE$k0|0jhAp@rPpS!F)s}F?(tp;f7`%U9n5WZq$ zQ#>HFV65A%fnv}sw0+-qlcNvrp-1`G! z*af_V042q~KP1MW{%IL_$PyC*l)2`*X_Tlk&0qQ6>=&kzw(n01VZYnZTrW6@}Vn5Ja=k)RW+rpLfw;GF{ zfz&oT!8kNvvCu_I$@VptYdKCgK+5(=5Lyc8O&xBaR+842VjA={U6{2-9+*dbum|SZ=|Z`Z#szE;raK@-27gH_cXqX(x3Gu6 zQ4(>+h(&f7j3fP0;$O%vBgV);vP58_@;73gc;KS7oon?w&WSR@-Oc6Ybo0gl&~fY`!rnc2LgN$h9?d)FkeU(0}n_U5jPB5 zBAhJ@2-bpOrR9SNrHS-K#)>efgyQt5YRPPG)wjUW0sUQe9lwf~^ zC-x2J<>N;Qq@WrIw~w)x5mN0yC_^JXY^j%*avB3(8|*>q58Jn| zkd4R9fy6@)j@&uP{>I*WXe*cw)1S^}e`CKXRUxEdm;c#nY&|yGX)(=$>tc*P*xE%% zNvx%LlM3Y-EQ7h3pJ6sl$|k_A(xg+=texYC51W`;M+jz3m&_1 zg4IQI;Q(Fo*o}L@^UWnaQKbkQrV0=rb=rP4&(oL)SE4>KjkbK9?rTgmzxyE}A<>rK zB7JznS4#<))b{HqYL$*Fya{{I zT}gU?JWpDxzYD7~cd;X+#Ppo<ZBDMliol*XEVn60DHmHw|QEeQgg> zq>s_?UNe_2{jrndy15JnJ{lW5J;LyTa;8nKY)H1dF$v=ah!hPQW1CZyG|a*Qy3}^m zhuJIuX%>(|kClx%dtLbpqXO`dV#m0-8KV(n#1!B(HWt2(vd$FRW)fj?s0RzfLaHfN zV{yIj8qZ-Hl4(;bdjTh=%-htOb%$?qNJ>1c9s|EXq{Q(&+OeCM{sBX(rn1?8)Uaoe zssW2ju}Rn}H#hfz;GqzYq?{yn9wR;*VjZHbZ0^BQKI{_@N}`aCdad_ecDLR@g1x@+ z6uXBrnTRQ*Xlx!|9~Ql+H)$%G3$_$_#oJ^?H-XdGvWxCY+O^J3ZLMs@3_sQvy{gS``gbW0RkeWa;pa{Hd1kLFZ7U1Z4Z+z=aKybh7!lR zsw~Snxo#?T*;pHEEJgAYl@Lrfuc)wun{R{Ac1sH%%P7~ht;V+bZc;K{?x5gbdJ#32 z*>Ju7uX!SdfS!%J=6k^VC?QRVtR3!-WaBy<+uNQpRxRpX_l;uPk>xtF?=oRiVW6E| z0@(FGrZ@veV>xXGv7}OY9*QjSO^z`VTQb3v9xKbecGnn-WDWv`M`L+k*6N#)%waQI zZMFwV2i^X+P6U5PiqkzmC9!TYZCx6>p`HyfaohYQK-24Z?k|_*e4SUzNo& zGdj7{->>RZ{tiWI#e z)UV@lC@Iu_2MNn{zz%&Jq;xlTlnIL+dbQL@LW_$Z5_<86*-97l)*TTa@4pvdOu+V1 zQ;Nn)-f3=sh_5=~zyreO6#d88%c7Bnqcry6m$k-7tO8OA7)WCuH4igJ;go<9curLL z5L=lRqW@9H^b;>HdoWI&H9e=Xk3SAFlF(1i2aiO2#b}`#YH(uqH_gFBU#ync}5!k#Sa+{XMRstR@BR< zW97}m*!bWZ!ckEezp$!K^VrC9i^Mwz_!>L68Y^dL40NXn&)th=z0vZ6AUapk!^GHm%(#n2V5X~<3c}8Ua2zy+@>v1qjjF7~X3H#x< zLZc~;2S$BqkBejDj8SklAr;~9$+pexvdgnq8D zn5gp0FU*pQgduVRJg2d1841c9=#dWscz}R|O!uvi6IIHH>93z>=nV$GNj*ntvELf| z9{(`g-Np1qMG9|QB0gy{q;Age<38}JFD?3nol5;$(Q%VEc@_nlT(Sq>;4v{Ib@53d^;q zFRR?FykbAK;9X%LA@+7-oTX--ydc!SjWhngLAE}ANSofq0Xh}Pw(J@IylX>#eQqC9J%IAeH-TPiOWL=GaJV2|4OsK)LcHN~AhV{Da{4Hv9ZQtFeD@ z`$s!$5ziBU>rM zFo6t&N8@e}JoIXev&kNU&|rbx=!9)I?EpOl+H!Y4Kjn=TToA5OmBzasvgKIgO9T&5 zqKVgawhS%IZB!xUcoUy!B@gQ3U!fdt(yK_XLZbwq2v>5C-)nU@j0!T%_k}m@8)^*0 z@+6Q7tMg`PpmXJfyRZjup69FofS@C*!zNbV;>lA+6nX~0A(?s>d9%x&L!7a$anG`N z7K)o#qnu1V8-*Krf6W(tDS#Wv+vpdJk(kDIb6TwJf@sTT*TCD#+d2GLHHJK7AfcSM zXBw+Uj~><{##Y|8c(E}P-+fZ94DT)XE=@4{j!B~*1kB3Y-yCGwTCabK>!!RcyU_b2 z9>8Jc9S#&IA+7`}jd#eoaDrwPps7^f2a8jA=7q^%I*$AN;eDbpR-32<2paE*F3Sdl z19YLs$~(v7=5uUw#lO-GUFJ9Eo#ER}m=0S(uIzfPv*L|qbU>G`?6&Hz{sS6msuD%I z_|8gD`pHC4sa!V<-k+E6z7*SiVV2Sj{SOqEp6mXEuQ3#zJmQf`;qW-d9_x@u4o68G zk2g!uFJcUI8}U&>G1mPTX2f94lS;rIs1jOjyo&B(-C7fL?Lj1a4f(`K=-Q(Hq|LU`aeuI6wIO2`6akhwK2oI{zfnz z(6>MtfZ#`Q>D#An6z~Cw2gzE}Vo&{%#s|!Ji6qdY@~0knpu{~F;mM*JBSpEOz{rwE zNacKG06IowAc)rZz@>+I2OMD!r1V(%z^6Rf(*5yHduV*{_9HxC?puOTM!JL!w)0-* zggeAq`Jl@AC&q(moq<9F!+5}3Tg84;paqXk^9%*uVn~qo7<@O|U}$AfU)pr&npz%! z5l*TM2aSh)n9X}!mHi-;()iG1oP&lrr4o2w7--m_=DbH6nQJWxHF3Nrc)$h3)XFWb zaMKlVG=a7-qxS!YFvi=m2YH_mNm8P@MGLX)!6tb8kkBp73ytx1f2TdHoU5~Wz@EEK zzi6C)iJRbmcHI%el?&C%Rt&4JI&mmP-o{XK&Hx6oa<%b&-eINdz+A4jF6AAP2MQd} z$I6-81!E}o6T+1wN#jhvaN-R#235}HB3#A$^6?4r?u!!Po#yc#gPVyM6V49*Xg+@; zYPr+d8V^5>gGd{Rja#yH7~Z3j_d;U;H;^PNkNB%O@3kpM@PJLMe0W8Q{T~>abg_|N zM)WFDmZAwJ0|~JsahiCq53f@FFt|bEg7C>Wm*<9Gk2fMQTmdNQ%7{;@jae9sfFOB{ z{%0%ifrA#nP)dnfbjf%X*~cB7jnbHge#%RjAR-DOlWKywjTKoC@xD(c>M6N z4=^bUU=rzs@m|A_Brn9Gl*T8Z?I^~S34X|TZhFLBo;f^Gq}7Z?Hnl8x|0c zxNecgCisPTM3J^r7G2V}V4|uH`V{{PVgqy013gDFvVcA#CWhIOLjupy z6V2l>(s~B7fmA{;P1v!?XyMAFw(?mGBG^|r7{z%YJD>HLjqUi-$H_zE^FOokH9J0{ zz2>y_M&omjjumGj5L2P;T+H_DKmr7QNF%?sFo<2mUgCV15Ra8FL}o}iS~y4%=LhlB z=jItI2F+FAVWY5Iij6(I^&hULZ|OR3CCc{)@tOiUN^Fl*CgQUODbj>X&u1tdJijD2 zQZ*}2n>62e8U51|*}_=)vdKQiNHb1_`peLaKZ7nDj6;u=FL$pr9>aVgRRIo-uWWi( z&p|`qQ8EQ#vXA~QGVQ5^F!CzveU^?R?Z8lKyK+T^F%~JB_*Vj3nH_2j$5@PblnJK0 zLyQq}6N%)p@@Tpdj&xJAgq5$nzE%GPtANl&I%?(l&c+x(NAyS@>z>)l-x;x5OhUq@ z8ee@k*zm#XN;?1pIE}AKImWt;C=wQfSSw$XU!gz18&xM|jpL#IKXV2i>Z1y-m+yf2 z(>j~lE=B5>kq?9Vghk`Fno|AOqsN4iK~TBvcBTHSYYqj3fBp4e%~f-tweq!1JL^~R zd9^tWwE3Vve{XsZB1j{zAJkll$3TZ|G7a-V&_x2v zc^IlpHK&lkN8>rz!We_WI~dAlE9bx^V}VS5lIrig7j3M@K~E(RE2`wS+{&B1--eEE zWZg+@ep^5NA@X4~^k9dOP4j^x!(4qs6_hBdv3HFcsz3@oR=$V(o!Dmgcb=&d&G-6O z=+#j}SumL5iRSydUa}v>(oYVmpbiMH$Jq4?^C%IFvi?Efu|_DeTp@uTE8myurgwFp zElw!wPEra%@NTr0ic1~`zushwlufWy?og_mu^!jnp*~q&0@Gm_o zubk@5Bff4&|H_c6M4U!w^}eQ+ORs~+I8h~vtX`Rs#UpT9G=7*&(f^2m)nQ=rt;)ZP zeyFd;t|M5B9xJbY;y&MT;t1`2kPXU>M`?WD_<`aDLJ5uka;AU}-`|1Mr^m{F`Q3*P zM^6dBh=<01ZM}(AVB!uxB+_3u<*?i47DP8u0>kP`9Gi$Avcz@IBkbsi^2e|8?*87M3vR^vB{BiOenQ6&we@tXAe%#g=@$o8Y==WFZ>%sZe8sju;y z4gJ`+NGm{n$@FGm82c9M9+`E7zV567-3`+~V7fle-UA}*2)FiCv#;>cBIttYtvfNS z0!wK`fT1;BiyOz!PHHZ^pGw-^{-m0fVk#VBWk}r)skAId@U;9xMN6dAPj;PQz*Gd>bD1-^4yg6gpk0y15@? zI@&_wLC{sV7CCxza{&`bHMP+@jBmUy@1lS{n(FaqI4^X&AiBkZ;AV|6thj^;^@%Zf zRF>-Qt|=rCsw3>Tk(uEP2i5Ivu)U_ZFPsA}Vv zuknq4%VTJTSoeeeiXVa>stez$jq`l@^Oz!l=Y@gP#*wwGE#{M{gaFoLZGn-w@-I;W z27)fny4h(G5>4F+SD77QiRwhaqe zf6T*>a%JS(wn{Yn9lLBuEFP=cw%-xf5%W?MFhXqG1$R$m%y?MWT&QgqS12RRo}i?r^l+cuW8Q4V&b24K?xA<*mUPMHGtF9$3C;^ z9g*lqB_v_xd_~23Tlac&=$N1%M#q`r17_6J4)d;A{4kY8CWL6K+Trz+mdDK@AQVK2 zi^(h;;XoxsoOXDWt5N}NbLjt|)O{z0r%zsu)?STOPnp$9!ywaZ6LpMrNEKPVic)h?BBYyehQY)*@HdvZNngzcrN z5Q|4syFD4p7GWNjs^C%8ZvBoJqp;{_df%#c3ywA-T}c!mydJ~GU~QgwpbB^-iqLP510}n~?#ks|eaWZL;GaS0_Niyun zy=dq3t#6pS{u9QPU`W-hs&9<1J`p1vh^0qUeG{{l5~QucYclA3`8?x?6MKgS}iGT+sjwc%vKlIUgi=^gxcH?dM{GO;4cObD@|19)b=SB=nveBA7c~4-=_X?YZ1j z{~Yr$C6EFmf$(0ur5H#dlB$64VTI+J`Wte4ev1=|u$z=hfR9yuqE{*NM`HkyG8~== zPGY^$fzL+?ltKyLE5;P$(h{&T()Nk5rK}fv(&Pq!v#S32H7q$P)M+tQ_21{g=3sjm zB*-fMXS0-2EIyJhAZn`rrEKLpgf?IkWALaxY>B|7b|e<1G_{xeY-0q{tVjwr@5rw@PsldI!d`Hpne*0v_O_slDHdXW{r*=#bKpnD$r2_r!e@ll1=GOkO#3bZRc^z^L5kdni`0dfG*}oE0drH zUb|pSkuRY{+Nb>`qcO5O2$(3*r>KU%jNSz>kfd>HpM_q`SH?MMtEqiTD)k&7ty3#LurXAgG9s-W6u6#wxz;N+HK=E?y?n_lLKgu&NUdaEkdo?r_k)Jl zu-33TKuKUhxQaB_++inh1nIeKH+FLXsRS1ENtm83_nONpeV_5<5onYkfh4JF-_glN zj2u`=V0~ZR#0qhvG#F9|P3^m+IS)sCl*o6^s`gz!pN9v@;|m}Oglu2-b;A)Z=zWW` zxcW_N;z18ed=SsWv4UrlR-&kn2Vh9PV0Y!hWIij3$_r1qOny zs{Kx;pFm=ptF`R+VXzT^<^0?7H3g4*gIFspgP}Ny9jW$jcZ{_`8-$>hy7Ygil(obT z2oekEn%aN+5u-8uLM2cQgraQL8oNoU1h9Z`JjQ4&7ayq#m|`T}+B-#jUl1cG@nf9* zTVzt#Y1{v125-FaDPkyX8r&_L1>q$mU7|?a;DMLyWf<^6ti%zlrSry78oc<3(Qt93u&ETg+|y`; zBUDnQZ>?(Z@&Ka|Hqw{LaM08NtqPcihK-ypstjl!!y++!t5a?O_IXAkM*y}UqE#Ib z8mn)S8AQZqq&lEFft{$A7Of80bC*w>X@{vj=%T41SmX-)xw}|h5mG`vy21l7dyXGcY)H8q z4-8-H;|0pQc*U;5`y$2$ zHk`+!<;L`7tT>>2)>6jVr1Gl;mGceHuw=qu^Uap;^N5kTF;cOr>w1t}SF7 z-9QwVNC)}FGKSk2yW`CU7EK+*=jj^`O?F0*RULHZK8wa^jkFaw29;l6cC1AwLR(?v z!MI`fQ#7zq&R1;khuQjhnFKDy4#t6XW6ZBQs%h%rlRm5!E`TMAp&AHPLF_W1lP*9D z9GFyzZ&52+ONd5^Un5uz#vUYA@^}=j6EaE<s=)dIgU>-|0TPXqknI7rvjRNL!_A{@C>c5)@q@apbeFkIG*Wql!0i>Gx%&Vn* ze53D#2_aTf!+vSbmcx}%vMn5^4jXqt&#AW#qz;?x!zN>S0re%YVXN*MtFb2$K9R}| zTN`Su3Ed_LAX?R784>ImbL*c_ZdgVF8;LfVa6my#wLIx(^gvRp4Fre=;ps?yEm|r3 zpb}Qq619~b!3l@Z)GYCQywO08U*zjy+18m2L(V3OOE*|BZQE5IZy}@5ax8^)#mRB7 zg~{|_9J_%@CNPz{SXQ56cd(1I#HovAO)a}~x=73@N?_ktvOlnfLrC!e>CFswbH^yB z)iqT~^3^wD{sWpy<&>9>=^JIxnS{;pdLe)ThlH+t8_Mos2R8x310D|wS(;iRv=s(o zZE}r<7y$!=tirnb8Vm65*I8XTyO>Y>JVTTyBVepw1*4V@C8Ryr;u`i1zUp8o)#ttK zdL_nlnbMV3mE((i4t?!96Y@b<*f&?|u4M+(`-k~r94Cc1l@{aE7qbd;eIMX7m9K+X zbXiO(AiV9ysvC+EbtH7Y{|pO2nkR}&ZFy}hD?$rHZU7cdRWVt9&A*!nOJNT+p_2XD zV!E)pBz!a5_%pDOB1pDl;bAe1u}4A|!6STP8asj}*aj)7wC5~*Mi`%n0UlWlk5vsf zvREnxymdIje~qvw<8uP0q@bpT-?7=JA}y&dKCC@)=d^r9O|ifD^Eu|hMkr-ftwHfz z!$egb59{lp?9=TcA4Y0xEr`?C<0>Bl23=5Pe>}^L+(#&-O^3gp%vNE|0t^YMrVjro zj%8pbuLRIh4TR(FECZ>pR6>Xye)1Um6icm9U<%8l#OH3j8PcQb2qW4gv7G4<&N#KI zBSyO$tDEf?Mg|W}9T9Pt<>4H7)R#C$U^3$qd~k>$Y&rthL+4>W85m4Lj62KgC_Y#O zA3UZevRurtqKd>ZBEB=r{Y%7HSQ$Yd5pQR?vbm5Z9I-e}v0=nlh2RHTl-N|Md?8cL zOeq^8ly%j0&{cJWo?%?Xs}4}6*a$bCh&eVgkmQkjO<9jv#}8@35x+e&zJH}gl#sgo z4kxJmPsu2%EbsP4J zfgwFsb>!0(Y%&f)M{()fk$c=&*zf7i5~@0?vyYLAy@0T|RBqIg(qZpms9{>ns*bvO zLGKI2beK@sbX4tny_b2YyMP|q*qyCHnwHH8eKamS^~41z5KG^FO^w9N_q#|&M{$WX z5|h;HaDhQbXR95#+K)}Z=^jKVBdbT+*iVkQg^!Lx~`k9#tLPypVk& zZ*iAF7j=FRk9ul{tA!aoAzWXB-WJ6=s(=SJ_MQCvEQyq2M@K}nPjJ2iOlT6GgOq<7 z*DEo#hJAc@g76DUfycBW_HoQFE?q{yG?smg3xiD0p~OXBV>L$eC{B-69la-)oy6CJ zO5g$XW8r-2ta7Imt2*YnSYssS*j!aU&*f9I`jBGufXCNysA6%30|;jm_*9&aiz-t6 zF*64lBVc4`D&u4Hx4wMp^WEv?p$Aod^WanE)Si5!#>_7@M%3HLrjEWjh)-R0ww{!m z7x>gI+w0-D=f+yJeF`v5hk+2x5Y3i68mz3wQQbfL$Odp<9;?UB;z*7COPuyoZK72%g z`l#}IC7t=O`LYTKUghiq**}#ysKkr-ie)-OdtB$S?8S;t zUG^AT8_OfH7ELAaK_rCMfdd&(73<6m3PZ1TG=Z90BkjdB3P zCgLH1ZA)S=VxA5Jscp1>jUI{&04gDvM)ywAhhP~DKcusxlY{j1`lp;l=f<;{NRESI zW{G=)48Lbai{U-YsHqyd*00NjAu3^2C-e)pZ^n0D$9pSqObEKBD{`GswjUG1F6qPS zB`m8GX3S$rm|%t!v&uWA`fu{=Faigr*_ZTmbkD(1;+TNG)LQiUsR|xVov?f>TZ<&{ zg*ax^pMXtR)tPPlwOq7?-~qz_Li8~h^5BQbS#2 z5H?k^kT)g}1{n?$x4E$pykf8?NkWOoBlwG0N&`dbxrvh)Pn5T*nN=>F($`}Nyu=wF z6Orn*4jZ?KAgn&g%}rm2jgBrJSWH2pjw>7{{#MP_!M8vu#ZC$gQx0Gi5Ckc9QeR)? zAchHbR-ZJjRM~~S)pbZG4RmK~k!L`<;89gUK+u^MQ`Lz-r1159#f>>6T2m*Ts9`52 z%R>@`k)Q2_(+RKMb0P(g8_Ddlx%vjC<3Mm{mp_|Hs?(#Y&u%PaRxB?NFes?0&whK- zvJ4S)8|q&rtH(AfwxpUzatmP3e&NUd!e<^oB#visy4Y7Z@t{t**mjjHZc%SqDp7P@LVm${fL z{`W+DbR=z`+k2Q>ew2BifUm00?cd5RIJ^@`Nn@)zc}yCc4ZjdURGGZGnzc7i;skt6 zoxE)jd;90^0(z7nWIbdXWCDaV;pB^4rETH`W2G#9(L= zdM7E*)}LUG3y4sr;#Ic+2OjXiK^&9npPn5zwSql{>>nxt=$blpp9c@Tn@t4Cbn*f_ zU+wWiJ+WJSX%{CIm&(N*KdHQj8x2}Ic})Ia2w&j-xUiUDIt5+Yx3R>DACl>mmf>uh znPmbKTGc77{aNmBw}n^`G0QJez281h8puWGmB<9R^#-64Ta(D%;G&dRb!a zeI*N0Odddr6B09VJv1ax73lKNosY(be4EqR(>_RHZ=aW!luKPOh4gk*z2$}YNzIK+ z9DYLvLN!f|?^eUwV{n%0Z0O_t(^z|SH^>brVO8Uo6zP>1H&B&C8k@#{QfOr2lSBl7 z1CNicC_iEG0RNiy_}?+U>g+%fFhHuQ)5n(THRkdp;9J${MZs)!6Yi=q{m15P6-GGt zm;9osGm4HV$>wUc;4$O-9OZxbs#6I;Nbrm_o`hI}F0`F-HIcPP71LtiaW$CD#Bwc_ zkWA48&o&Qc1rJr7fsfz9cc8K1=R8BOKAZ@>azTki9e>m8hd3#Laam5;FM1yB?c zRyEjSKh+CWkkI2+Irha$WG)NoV^v@DiDvV$gawh(l`mT2*d#Nf3d(8fi?0^xv(bekivb4_n(MP%NByYk zi)(V&*8BAiG6L}vcFr{C0qM=h4Rh{{4hH@K8tt1vDtm@3vyX-icEr60z zW)1hYm!pec1tUWrO`V1Lk7JE}MXZCUsk0YE+b@|DM6fGLxLx3@u)1P{8w6qYD>W6{&zmdp|B4csB=%KxELZmL5d?wsm2QdLhHo|)%AlL$@6Fz< zp9?n!YxCL2J$IEDTFVx|`v65&)6_Z7r&unSr_NRh0?T4=Ak~XXl#xB=@KbC)j-zu` z!OYd0^{*$GGd_W>#i?&lo&16-@wse)dHNiPR&@@vO)(d^QR4qa$OvL7?YEOiauiBr zEN15<2I;vu8CmQcPR<7odLZRq5_=WNf+Q9X2!D@duV8t>B!I_%3AOAMbKfhZXlk-& zB%6%B0~ts#O@90^TZ{f_9gbwnCbkaK9d%-pN88w%>GDQ;p>1+pF`Hwi&q=1!gV=h6 zHljz`BYDCZwlVz>{g5_IP6%N(WHOOh5o5`*UTht{gj7Z1m{iKvHR~jBWRkYY&j&D@ z{Ug_locy9cTZhj)@sK!>9kvcbb6_x~V1awBJRZj^@v1*tjTuFhkc3yiIK|g?O?GMv zDJx6anxg!Ag!P-)2IR8ArY6(*SJ(!O3G^nV)NIa{|FyRBlv~)NIRxGpJYMrZ z!mOCIrz*lgujLlA?&z-rm?@=OHeVAXQ{Pf4=#pQ?TNx~po@JG}&unE~u^;(roz=^u z`I?LJa96=}?%Nlbj}$4H&OLL9J%tI?I$(2aAM&}+APUr3OV+%0gr4mrzs?N=E z<8StS;F?33Hz2r1$6kWap`KMMsgOsOsyhtL+AMl%c-lF|TC+UsfXz1A@ky zI`>KpU-hM&#TUUp_rXKH>V1(HOiVR(UawNV>cx5!%x~b7)pwDtPrx8WQ|HgLGY>e5 zaNx133-*WeAsf7e2~kQ@7nEl4!55CZ1|th@gz|86L%$IF=FwV~WlmK9n5r%uRLWO! zKiAjv&EYkCu>0R4q=Zf1Y?{PpFKFg`O*Qq+6%~9Ly0m2grW3G~VcC52(7CQEDQqzx z4JlAeDwkq!&SxFEA^PipuBnUM$MUz{xatb+#XE{vj+P^ogBdk-ahtn5uJb!$Tq&w7 z3C6AElbg7tESYkjFGgqsB~{bZ)UDOZQlv68fWiL`kzq{pf*9Xp@DQ230%Axfm4s^)K> z6}N|(Oy4ZzFU;5Jhm5hM4}BRf8Wl)|8&Yqb=m?T;`%yUpjW*E9`tpR;~B=On0V+Dq7n zgAKWWP3}aMq)U7qcKMUV1Yy}uFW$e|9f4G6yKMI{-XDvPghPm3hMb9h*xHo|dkA35 zz3*}ykfH!K@zK=f&!+ICMajbI&_`3#E3UB&Oqrm*RBna4Kiee%d*L!rHD%zAduT;>o=SIr#6+jta-K{J4%O2$cc2J;^PMvtnlnitJmW62jdWOI?x z`JE*K`#y~G4JH97dg?&lbn*@SK5ub{Uuq94lt8ZK~f^dH=FtF~}%A~H| z@5Z}*-$cBGLjBc75cmE5C(=bqS@Ttmq2ts8HyKia>YAao+=|;wuEM5Ztf^};zHOxb zEqFwcD>qC`WO+CM$E-3|-LSNPPh8?Jj4ZU>aMqniq{KNz7P@!)}{^aGP*!yB2N$~s%(u?awlLbECre~1xyI(EavccpH8peeVI>qv@u^T>_$4#_ zlCcrX3pNCd0G9b_ih;w#2$+PP)g;YmgVY2aRfK`EIvru%a9b836+E(h<5*Wr+)<1P zDOqoL9Nmvbnv@ev-yN1{WVcyDKP1z4PhTVcm%y^ScpA%5f(U|XcDGZ;GVIiW zktL4ouqNDJ2_;_4j5xbZ(c^)a+FkjA_^(1qm{wve#xAFQN-a%0VBL$cd*D z%3Q5O_MS7wMBH(hiYjH~{p`Y0VNWlkUkB1r){l)1GVUouQ4`V`H0M$23{D+6# z$B4$A2PjUDRn7h(+<41O)c{kgx_wxh(FePI^5H0$QB${%tu&VA{41)U1PEJe*a%m` zG6)wclwC;GAgc?DZLdjJcA~Fc=j`qOcq;ADVKsqS)g72&9e^W;8WN>A+N zvB{PWCBg?;{%0<739&n@zKSLHfs==(?ihYVX(7);kp|lFVvf=hDH3HeJ_f2eO{4Yq zuuG3{0Gw6LY38e!#9kJ@MJY|qX;*4l^K0F@7zmFiSiZ;YlyxS|@vT&rU=f3mf`_W+ z6h$hl&DDEhpd5XxGJNpQq6(@(%9V1Z@1HxI1XQ_o-M-D7b%OUGCESf|!5(iNOaTsr zJ6`s+t{y!I|AgBQnrGUh%1Cu*XJ%h@eUH;~nz}Qj*4T@DZQ=o|S=F6S#@hG)*g%vJ zICc&#wLgxvHE1fsVdwI23(N$Ur35DVKX@u}-(i69GsyHTSDXeK;~zfYKzCW{Eiz)R(TzYq80Q zMrh|d4U}i8Rcu&os#7j+TAb0%+}k9W=8ZeUwxd3@C41mD`5E4ZFJ3X|K_g?_Q5bO{EbV=_3fJ zf)aPUjAyah1Psy*`P^H1xwkz21Vr4go}Ohyp>d=*g~sF5`~%mGL!ZrbwjU^QG|uRO znQt?aaTWSswMru}wSg(n;->G`^NqsSVrkAmt^^dcPc{nCJ%`1}NGRcT*!cfAd;2h( zsy6T+0ANJ%<_NtvQX16h$hfQXVShnUqRRDay-~CrMOF2{o0> z>@7-0ROBtg5UG?JPl{5%&suArIp_QMUf1`!et+yg&N}z|z1G@muXV5cE@=0(d5anK zm;aEenqASzSe*tSu0P?f;5^EglsevJs{_ky-axGnWtmeG-Px5kY7fFRES*xaAQGfU zdFYa(r4$~Cy!g(q!c5lthHO)!sU}LT){T{Jd#-lTV4eN?d4*RGkpGnXmA?8^$t<4T z|1DmPC}v0Y%J0{?Irh8K_MkV;tH#*A(2Lz$6hLe}h$zsPoE&&A`UU((Zid`tGp1O< z-Z)zBUg)2i8%f_dHo?{9#I)$fVsHF+Tj}Mz#j139v*nc1q3p6?LJRTcMMq1AFn<)< zS)1x|!@9d8qsPme?F-eN94!{0J`Z)@7FKfykyk;}Fl^s^d6qoh^n25j@h>zu^!w@J zvu05P3T1c_M+e>~y+sw5RavX$zY}jT7Q<>lEdSrKMCVm9+Oyj2z#2x=FmkN=DKD^| zE1Afl=}=arWq$v1w$zZpc8Bwe%H(B9Pr5c@TQ{6vW#<1~-DT};`4<8H_|~WKMUtE4 zw2Kg3b97zl!;FT#jBF#=+g)PanZ$lo3;X5mQ1?Dwg*{fzrAZ* zX*Q`#P~Qr+x_+@*kTXl$)IzK#bHk$O1?%e9rvwU+LvukmN-eBeUpk)sG1S*INXMVB zF|dG`D}LDS@XqB$CPA$3$p+<}jBye;J3PP-zv1S6?i50J5!)*F&KG%sn(RqK!;Y7C zxIzt%u&9@?Vo{Uz6M3hLBq01GBEN+rs1PlcPw&L2}W}H(r<}_MHh_*M8%H&-;lVbAne;U zN2Ue#`|A%&^nbg0vJ`7&r89&Sv>cJ`=PFVIAtf^s6PM zM?Ps~wp-kT7kUSDo#2t-j_GXp=G7D`4eV<`4D2=tOpD zpSNe?=o1Qz3G7n=4*pFkK!)`K580<=zmDPxS^|=>5TUS zOENR%ZPwZe{wg@|%B&<=3tOK?D6ry|9kPn~R5$R10E4Q0hwCROXgRJIdwuX#RUr3kg{=%21b_1+K(kLgsv=^&S?R_n& z@LlSD_V)pOCsF$*BuzP8De;3AH5X|~l4_4ba(-wu8c3Dy@v_h+EY%IIs;_e!`>$6+ zqZKzNOX*_;{&yQ8z3 zCl6c|*poP{YTAlMdK zAL5W42rF3EjLD@B`L9CJI8RGz`iGpCUi|XnT=!F3)6ZJHXZu2TC%Y_I z+$xvW;;6fw3%T0+unkG?HpQ;j*OZ&HD1Av*Sxf?*YW^UUj6Un7oQ$uCZThM7CD^!_ zKv3{Clq3B=&Xjllnjh|CjwrGNt;fh~!jk6fDRI}^%|FX!`0Xd%b(|ETnAIik^O=ES zMgt9|C(G3_6TN=y!qH(Y#MQ~k-W3}TXq#HmN9`$*H!ic)kB#R>T(sM}c~Y64U@ezX z#GM!On%6|D>HaaQYn=NbM_7gaOG`$R)qFiu4!T$_*9}i|UEavrKxiD2F%(ir;$`9Y zkD!b@TjBLs9x47T6K3Ri4ps+TOB<3~)tvs#nfjoipO8#i9OKoy^5p)RPbj_GHV<^a@qe+hdGO9|=l9dL3^tp~SGX zzJ0K~m;l{tBaSL~YNj{1U6vtg3`AyhRUOg6xAu6Xq=Tx-Z6B?iG0-Diw;e+OH!@f^ zJY2fZOFw3`wf1Ki>8qKt}Y( zMtAahjlI~+9|mIfb9wH!go+jd!4s6ni>3rFj_lGp^4OfwUZ0wo#{1A9A^QW9z1|3> z*%}&8-o<$XvG{|nZ-es0-XbsU{SA7rPfsX8c`Bhuo*Zl(t#zTHO|d-5p)dsX6O=jq za#XEbZLl$PLh=-;uupN@crO;i&>?yH{S;Npo{U%n%_-bg(uppm4%vpQ9q#dye zz8sD`ef+fC?_VD@Y@g1WZLU%Sp-~|EzYv_g$aFq=bcZp1nLB;GN__kg&C+)5c{eAj zng{Kr2}BRcoYgbDG4_PlLOe}bvus{Th;1c(?r^gF!*a9`0~wV00g`8N!%oWq;gHN9 zzgm8sW7EGHhy|o?WY1A=xa%h*&(i5?P^KvXd+>y0!3vQ-?2TpHKNenbTq;?gg0F27 zl!dqKP&MTu&B5w&9*q~tbX1){Pz1A`fAP(U?qd`FFuf2OAz8fXjH<~kEQngsm*~+z z5IFtRP6mrfzV`@Y|5SL&SxgCxM{Y9}*sWma<+Oi^TE0Ko8OE}G>4|t%v+MFOm!K@= z2)r&402s)!eTnPomE@uWdMaf7ma*=kjbgAgq`W^`ovFUwTTt3Ec8(h!sti;7guUb&_x z8(mf`pP*_kvG?YIARRyRN3A83!YCnmEqAl3`O8n?hM>IKbV-T6%%Zs(r7jNGowHY{FGn#>#CFEesFBH)rIOj%AUj zYulFLtObyrlw(L%3Ld9gWb#R5o=-#a>AQ>F@0c!1p@BXj`D{m;)TJ;Dwy;D$Z&BZ?TRPW37%rcaT)t7S zZ}nL@UER;L$dq6 z3a|Ffwkne$x_fthcX#qyBPn(b%Ko3@yfL?gvhj*Ka zzDx^t;PyEBX}7Q|gF@hfSA%#5{not-XK;^Q|ZDj*v z?M(O0**acbwxFxzICx~Y`^aE3AKN~5_`W)_j&H&$h>AbgOAgZ>XJxqUiXrg@k{SMO$5zdhxI0FCH&nI1`2hjKUq=5XET67B$ zS|T?y2$91Y=!cZC8%{Dw>joe~^1}_u8!EYcuKO4b!{vMOT0e}oenUup)K*K=%_mQ2FL_BUa8hAqT+obfs@3?_~_2bANO*YHvSIu=OYq8&dpCqmcsCG0h2(EOthRE4YA-VkoPw%G0)FuZ2DC1NflzgOE><-b zmY7ZlYZ~&4RE^h9nJxv5psL$qT4325zZ#+z<$sgh*ZDftJ@fn_Gi?=WqV@JwsOpqN zZRn3ERqv&BlEpQo0)Nb@dT*!6Kz|>G#*k`IbG52RL?-y!1R-_7>~X3_mvrMwq@|%? zlH6~ryI6?Wsf*P;O~046Oe{P>)o9BOul@A(hQoZJ>2`3CYN;Sh#@3);~XbfgVsbu+1fi_OT9Pg~l{KAa%R|C`=AZ}tThGRXiHG(DTtw$z>xNd`ZXxu(>NT%gO`aj6 zDa)+=f*C`q&Emd2R$r} zwq)Tg;(A9Zg3&l!wdH*~-k+r5;;8dzw4=7*LaUq>jtu^!`l%pwnxqRxq#9nU#|9# zNkMZ|$2}z_ncQjCk^%(;l}K*kaJowUPHg|29sa5DKCFRgoaUY(;!d-*a?IJ{o_Y95 zv)bCLCGFC>T(%NQTm^9{pXRkjchc`lYoM-sa;0IUQjMd!yqf6MrRcIo!LHOA;Qev| zM--*F0pUWbYsEVEK3>3DQVUUDEOC4UXb8|kDmA`Dnm4&3V#02hp9p;5pC%YdFaI{P zv^jAo+H+-^Xd0Vt3GC(gL}_4sny_(=Hn4sim45AH)tG?x0=R$%#IG~F)~x8)*l`i$tpv&~pLU0pQyfr#hMt45H% zveJv0a@>e+Q2Ji5q7>>8<^F$^C&)OHB8yhquoGjTe(dzVuy0CuREAVvc4~_8+$k&u zw~*>LV1m4G@Mww~2ZLGY&YaXGj`Wb9hd`5gwY1sRHD%N=;3Dkju?LqxE?sij% zECHNzH^yoX;GXXXQ{aV+-bNNwwd);s<^v1MI4oR#c>t) z?p4Mw20|{*^al06T@MW{7!?hAZ1e^V``Mu2EQ%n+>~??SrG-!I0Lo|-`N_XP}6XlZS<}guru;HaX`5y`wb%)XdxBM%kye`d(3EnH8l^4OT0l`wMRK? zpn>zcd({aC!lqWafw#tc1IXQz&j7M)55#zbK7KMX_OpUXUS?;TWW^92bj5V^Gz1)2t83<59YVdp0yg{cML{J7FPWA>x z#gM2WZL4`h37*uxH7G-1TLyY=mn#+~JGhXh4CW^p1`^sy}yB3LpNRLedB13j(LmXVoyWcY3?QVmqUSPR`u}Ltju<&YP4d0$4?F8#=+pRRnRnHL z7a1C$g;XB3NduB47tZ~r`7g&tn*4O?8Ur*Sc#VmQ{)xw^R9aPC!?H^Bd_aB?*>0I)Mz`R zQJgmskH|Tk5WJ6M5CNfv~sh%_MwSp`rNB^x#sIsukTQM=4ptIo}cRVxNU6YWik5lSZ{RS z`Nqg)x))Nc)EnKaURZQTjlQChH@f$~q8hq)_eQgK0YUvZ>e}OpUJk3RLfdhJYV4*n z-q84|T1t%;Qsd&Ld$mc$(3bD)hPt0#|6{y2lz0aX0gaHFF!ZF9y;A7ZVI9SjtFCXi z&btv;@~?#$dtkD6TcPR^n=6Jz3(~Vyew4{?>ovoU7Fhew>%%I#;;$2vNr+M+w5LCC0 zo#a-KJ)0h6iJsSR%e*C?bNZNZ6zXTIoBqi4?x550!wPoukCom;76P!eqv6)k^Sv90 z5zvAeXWyQ>%xm|Q{Rl8}O#Ww{yyQoA4c3C)_DYgBiIt;9G2LMjZU`mW{KR;5%;7?b^(=42YrmOS0s4g0jI&d`{E<^5Hk~&4g!jUOdn0lbTyfmH^Q)2Jz8+FDf1eh3 zkU%cFgSBbFe>1%?l#1*f?zl74j=JaA{aPM2P(fv~cLTQuO6{tTS~htb-TUaLh;D8A zQ1NN^zSGCF>iVJl&4-oV7+!0O!bBhXc(pf*oIB`Y+0Gg|)9X0q8{;_|L+WAr$4#8g zYpWaeXSJE>UG;ozLlin8HEW=FSNRD+j0qo|R9^Z%!7az|9MXo=V;98qk0llk87!Aa zWQI55)TW3WkDlyMTI{cbfC{Nc9t2{_h_D<%^~lr9ysMc|AgHCK9?o@f-qk0@M!f$h z?}q%>6Ld-l>+DBAp6-oUP(Px6AsaH=ercjZ8ugQ_9=o;DTM%asQmt}Nb9}v$gF`r2 z6FzI(XKpVpec^GlFtTzyv$v>J z7gE?8w)!)_Z7ltd{eH1*pE$bFTf{9rEM{pu+cDR@f8!c+v{aUVz-Yf@wY5mF29?p}JF5l#HMuB?)) z7Tq@AUHWmRF(Dlv3y9atedMIclQRK#!&;bw##4WUa=e!r9&ZE?ulp%hOz3pmYwr{PN<|Sl(VFt31<86wzKQ-7yWmreO^XoeI zeLl``rREe;YwKrtb>6gb(x41iYoCdAKe~ReIa)lWZTjx=RJVM8ZG(cdLTdF#>E5E< zmxW^{Lh6IUZ8C?5e+1DFb!hsmcURWfV^)ra%X@ch@#cSEZ)(y7f?(A{Ky+r<&C5#)M zpxU(~P1U%-UNbQU+WYZw_29lsO%E#6qjL9aTy*^1Y<5{U?0N2tEa2c!bE&3#;ocV0 zWWk)#`iE^ev@S_KEqgV%mE*wDd{zCd$%kU)IB-p+ypd>vjqUt#VDwm7>E~uB(`?U! zICN^aYWBHJjAqzYv`-R`_idH7hhNW-zoODMtHbZ)%b%RzmLeEXA$6D{kY|ayK`}cj zkHoE(KiOqrGSH1db);jioPB4hmeNXkq*IZcU3**)K>axCNQ+fc;^*xF3Y4jf?PMoyD%UjxArT>YlJ!G>tCbPsx<8iC_lB()i)TSXGDU(yE3NTtD7VTuwol z=+A9()MVbbh4{tL__-oW)#zJGJIZQ%^4=Y)#@!~tfpz7{C(C6&Cq+P5N1d$PtZH@Y zAMx!k6EozSYIF6!mc}oXuGu){MbiKp6>~nHDH|X9Q=2cNOgH>G1fm>sXnPoo{XB{F zX+gHW8)fcCsmf{X&R3s`A51b>S9!QQDaB z6pqUBHxO;ZoDQlnuSS08=l@n9=bqWJZTj8Xp4QpF%{&)ait5^gJVAAO`Y;t6v&Kxp zKv5v)T)FD`+kLu3thLzbpX1a8XHpCeAcE?5F7VHoA4PIz3|D`U7;R-%!01v<3--tLv9h_J-OR8wrZf_l zb7VBN{o_nbciD8rY{~l#7nIB9>+Xzn?KA9-7V}zzpmo%l&UNJDUb#jNh&t-bOIfmc zq^Y~2dq7B?d2610L{<&l=o5Wuh7`@TIjN2MXFi@TE7^jk!)aMU>g+YMo7seS)OaQ6ZAZGUc+C_^4$4X95?T| zV+P{p*|L=EFMwN=zb_jtr#QqwC@bl?PCMMI$cuj}Y@ol>R>>*8J4W4aWqDQwmDw9K{J9hOE6d08s1EjNzBNI@s&lMYhBjnqtLm-Og%mUwGa8(lE%RmWvRkSq6sKnE z7+KhHe3W2q(_~@(m8Jp0(D+xL%pciR%b=g2Q*+-+ne)`vs_3P40&l|J0pkTQoCD)YW=7U(ksv?wUAi*=%Kq64eMg$OWxUlYlxszcib>{ z5rKXT9?dD})a!U!)!;Y|JuK0B51x>BiT?4~0x|QPyi3>#_|~TNW=)oNxc~x$et;ld z;jzWfnVyVIsi)O5Reohc@r_Kw5TXc(+CvVFLc{grI`vO+%e^RggI(5ZhePd@xKU<9)eXOm?G$6#=;#ua0T3f|ZO`4g8wZ&T^r?Up4A>~ch{M5Sg(u`(&?}LKH;p|o5Jk(}8D90WhXeC*c7e9~ z1)LIpz}SzV42dZvoX2!JOXKDBveFN*Gm>7g=d_$6$`yio4uQtjN+}MWiP+ zE_5p8Br%GmbSYzx3%7Sr^|@SJ2wQ`4;jk@U3)mv1CEB>jO4W?mK!oxe&MThw=utW3 zr$8_!Z2Z^fM-mdB-L=)EfL?wGSnM>H8PAsX|#c(Wgk~EQB0eF1~R7XBg2rkhY_p^C%k`i zR}_0_ZL^&wTpX`mH*&h-H2Hag7sGbP@-Rx%{2gA*HN(uNJ<^7p^BW45c`>tFnlC;J z@xKjm+?#Jk^s&~(XdAa?mTF#VTwtvp*E7zm&G7>G*86dPPf&3rcP+E(J5Jo+#j0+5 zN83WvpsZQ56|(n>6Q(DFHr#18yhPPGK3?PdG`7s}Y8Lh}AKs_*@QC}Yj#rafdRjRn zN3(J9s`m8G#tm?yidoSd6?b!0v2?gN@m}>ht;`3nF=6xCCsq90KN&fY1(eM*-Mt)O z9)qZ*ad4XZE|EJ8=~6UO*6jLN701mstsD<1e@=8y{MsY%4G^ZGd996J-S2HuGt(`b zw<~hbung2hkv8ZwAG5`)d$);9wEr`!IVo z+vb0b^%|rvjLfGkVq@L2$4pWIYq1uqcgUYCC3O#iFxAxV?(RO}FY>T!$Z7FgxvHo3 zM21*=Lab_Wt-bROO2~<)GHr{oleCkpqvDh1saiMO8tJ$#HsyJ>8k$XeD|-CH+fOtUj6TPL?Bw$ zPF3-@pA7fHkdyG%Xs=#=-N?cy;q6?nE+3kRZO2^8rn|k`B+wb)N8LCrx5VsVbDno#mm7vmzf4*k3_Ch9*o>22IAS( zs)?U0%5X`1V54m6S0geRB$6Jd$%Lo`^@-HP_;|nF_+O-7Yl-9Md$qYl0teea5^tn5 zeC#L_I-8;;Z8dO}9Af`;0{zr-Ih5vIz~*o%v!&6cb&I3wv0Zr*BHH#1gOXq5)nIwf zXtF4+X6*E07Mox}Yq8eVXQ&viGbDs(?N)ozygCm}G(-_B^EkKE9xd9gofKOnyIBVnVnQCu zymd-Z4Y^AUxYf4Jr^~#qo0S>rgAGLE({4^t;2zCi2bUxaVrcty$ zKpf8%VHzK7C+VQm@siOZgz;eutsrI12INRO$|0qGfEct{di&v~7UH^8xrERrjBLkC z$MKnx=I6_?5L1pz0@H=&!UK(IBAsF;8cB`SJ5KK|Eu*i=cbru&mqlOe?D*mqN%=pB zF4-sjxFnDeUZHe+HBqwub$Pfuz~${_auNO17ad(z&ya3ZVTPqI={s|z8wIv}2*>I8 z-ch;MPYO`Rjx!Cb#M$*-m?$*f+ad$m5bqFw=-+V@MxO)F#o zr(O}8Zpo5sT32}=8s#}cx!W)e8p!vV&+MQ(E_5LJ<1Fc83tL%NcKo?mMssbCe{GlQ z`145-{}!ip)Zq;G252m&(^(q-Oq0Hk-WcW*ayr!PvsCz7?z!k#2!V`K+MRDY3h>B5_`oww-#P5SgE9jBMoSB(ZX(y*{X+ zGM!EoxERI<9zU#dok)u}jCf1mw~o_k+%QR@vbA;;G-$XbRz}o(GD@(Cb!0@$ME$Q1 zarbVy#!nz@?9plJOzFo_i59F(`?k})>m+dK;xJJ}zkjt{e)!gK0|w%Hdh_QGg^2>e zcjc4xa|F{5qCeb822v3Pf*J~xC8wnY+WHNkEZZhIbbi?Chlb)bX-NVTc`@=M_$I&w)1f=?d5TZZtlrdcI(Arv} zr{X1m_aj_REs;wJzm3RoE?q`o4=p+m#RfT=&?u1xl6Q`HqB0^_=L^zhfS=~vaOs>d zP6pawQY&fa%topntDJ)HgwlEBY*mjh94#pi)dz==3pz4hU`w*>hP8JZNf8c++?CRV zi{t$8iH3H|B^TckCC8-6s$SG)xkKSZ89^|I9zX~NjR`xY&z5mvqEx@RqrNnZBoBw} zooQ0f`qoF88Ye9un;pgF(W9!~G?M{S+ZCcSljZtzX2Z-nsxt-Y+C(SJ?L2di+(2#@ z?FOsMgA`7Tc4g<88B*Jau#yt`a+b6Cgz&`H`KdVZX1@{Th9?VU5H+X(_leF;mK&I* zd;_7&ym-msqd8)+c}wI=F!FHI3WLUeJtWDR>Is)3}4S=|~(H>{LAVw`#D zX^k;5d%*$Xf;9~q@EirD@~mGp8W3xSNitbiv^{uWyJn&!-|?Wa7!bpq&g;%dka8Xo zi+va`S2HIUGSk5c8muiF5iS;SR9U86$*loc`doJ8$_NfYH3W^Ie7RCakZVu(F}TpM zbF7SFbwPvQwZEAzIb7c0U*BTiFd1ChH%g9gGbR527<0#~$xxe<$wxVnDLwsqz?Mtp z6lsxGY6L?L>`}Q{TBMAPNP20_Oi>Xol-{hnPnuydz%^^Cb=EyC&B&I-56k7!<`blk z|9Z6$T!bA!miH>rFQpX6wF^QK(JyVaO&mhat1NbDo0F1ELNeU|4T^-e&D9WEef=Ou z`xVj}Cn1N=mP|(>@;Yf}1A$v_Q?|)O|D4d$xMP(x zVUHegU)$+Rq}Tnmq9o;V&lL%EjV?%vU?iDs8F@$1@Jx{m-oGHqrt@Y=XL3|Uh`umi zhL&`Qvd7}daxHJ6|ZU2dK0~>eBUu{{7 zomegHqplx2m;RC_V=vnhg+c+gE_TV{`-QTIBfc1Al*66lolDvKY{>^6GWZbZsDk)w z4`U!2Go4FmNDJ%>_Ydm5R+}VQ_uC3WMs(fKjX)i(oUum=^$D+IQG=A0XiB4*GTe4t zO?bSM!pT`Iu}<1zMwgTer%BJ~Oui|NPD(3ZQXgWAG$ELyAiNArX*ov{t6vn>cDR!g zJI)=-Sv26*4JqwnC6+8S$HM(Ig>#@$tjCZ+qd86r8B@km9$KUD1f7(AJ5}P%<|IH1 zM?=?CN#@<)qo98Oc@p&BK%meNm?W|OM>9~MkkUVcjc(of3-xwbO5Pc%$DI+tt@l$1 zTbs_xkC2l9y$FVM9nl;LeqarM~;}Wdsu)VpO40(0D5e$e- z8cHPTzKVWps%%Pn;Kn3Vr#opVQ0`;aw%4GwUts%miFCyt2ir zuEe*#EB~|8y^@VgtsGDgWf~u}yM9;>8rEh?CtKxC)1jfbL`HCk;S<#jr)5OOMIrjwfFWPo)5Y#7UKR*twe7;v>1yI}6T-7p7XlzI zOHS@bgGK2=ZvAl)&xM>W?KjG}rX$1ZQ@Qwx>2fjgxK)1X(mP9X`8e|pMAhM;ajIVQ zw?~&TtYMrr{R_h{zAn@jRFQgG*-jT0M(zBSqcLHZCks`*K(=<2HUV;Qvn`L!j0oHH zt7Z5_!7y6{J60^CDzdb4wx4$SPGr>9*CQtEa&)7Niqs^hwlOuz+H$r=|F8z?+Hjj> z`067^rqi`cjJoEAWJANsQ8~rCKdzh38)`ymoUWXV4D~NefkKVtG4oV)igi>;+La94 zXW_!|u!41EukQh_%Iml-=mee1Zq8LLh?Uf$gGO!ReRX6phr)~ojYb*bCK;;MO^+Kn z7U|KDnn>kh!jbu*#it>dCokBPcDliFQgb)U*JQ!d5C{guP1EEoSGxIOZJIhMN!46j zZX`9zrLv<~ll>sg*2t0iT#RZ+dW$MuQvbV6&Ted{t!_Ka<@q~GU*=9?X?TUw?ZG1V zg(sU=1++VtKe^F+aKj}gP6;|xF8-iGPP3t^t&Va*r(4_E@=A0Cp>FN6N~ayN^^%O} z-I!s1dv?3#WvbdGrow|YQ1{+DRJEnIM*2thvDLhJ*A*L^7HD0%{gkJw z`D5SsrTbKl^QqheOD!$ck3U(ZMUPC5-M66&C-2}}BX`H9r#*A>Zgsh0UxrGIerff% zIJ5sT0zs=#dQej*)sAY-CFJzrB(bxkGr^mG$ zC8z&;5u5h7E7?6rk>CJ=SsHh*cTaA9LL121qsIeza+dqqSWEW+*gmsE)o%Hb=~7mX zkEg3zoB^Z0Rj$WjHq6ikaI0JozC1&0|7cxI$L+CQ++TX!7daN}ai|0?zl8-0Iz4_D z$>-GwxG%@KB)5_U$tn8jG2Qul)=!oW7ycZWO&T5v(%@@|Y^NvT zgh87b-KyV{ucV78Orza^ZxHO2Gt$>zC|X_Km@j#J;30<(K^V?e9O;2#$8FC~b7j0= zC&04(Y^CJ;S0RCdD2uCcgzzGR6+NxtdsBgmi|sRrGwT(}(@F;E&c zF?!e)z3mRVA=7%_cHFd!x5?#x-VLK|T9-l@9BtFI%d?~l*G!hsVfGp>-SVZat^4K+ z)_sYLq2v_K_5~w6YAB}(n9%1kY@Q7BPeqM^(#B1coBZ{yeY9w0llVE+M=P(Y8=qJ?-vUk`sAXQiFTiG#TfA0b8PTrpZV@l^hK~P?1gi4=Zai zb_1^`n!$JFa3}49&2lySQh;mmvz;`LWeA;Yg79MD7yjp)ol6L2Lze2aaNC0E-S#<%H?|0y4ywx>jTu?q)5 z-=@2gB$D_$iH|X!KbkqiPI{k~ICC zlyIS66SXL5TjXZ$crU6Fj7`YqtdlhajX|f^p!~qG#9@I)P#+V(CHsU7WVSrTxCMfS z;0nnk|5&ND2PkY)_9tDkrhz^*Tr(^1HB(<(AlB${+lw>47pJBMZo{{G_59I`#rFU_ z2T_ei17&Br@YM*o^?t8OlVt=)Hr>P1MX%dWN-lZfB5t6bc(X-^`!@`Cx?XHl)n-ux zgfA(hVMqTCS`Xh151a`cB{;t+T&yntJ{s6b0475Dl0Fb4pOSM}(?AzQe`2@9CWuR> zk_Zj=gp47=%r_z3fyrPDH>b567KJFreK*`1?l7?ZEMGp0)QhHJTe@s}uWECLhA%TD zV0WE-1ASE~0Y2U%f)U;RXCyHRD0_EEChM`mbUIkmP(C4W=IMVM^=YKx@HuHt8Ww)| zf*s3ozvyUVt{{x)-$~$&k`7Uh`nHpl5mta6+8#jsk`^c>|0&>FABdjpAWhim<%cgR zJI(|AOaq_+@kfcIlMYuK2(*yXn;Odpk@6I-EYuLa;~GgkuYxr+7}0xk0CDgddz;ks zkKWCTrHl^=@U0xZ*_Uem*&{|U!=*PnS7k&pA*H4fbb7Z;mhvZU8cEP-XqzZKqW3L( zCry)!cq4;i?a}+9S<;&CeJv@C2-dcmlyNI9($FzYzQ}#R$YF`59G4$>HsstG_3kmGP;m8KM=m79$F$5_nXV3nk~vbmL?lcMO85DO@+%H z1aKb>pL1<0*)CBn|Vh?CMc}Bi!X$~KK-#&XhaDeO{S`HqRd%BuzXU0Vr zA7ViV=|s3`L6!Q?O_tAWHKZ_&ZE}nlG^QNi4a+Bpifrr-cY4286xc;x7A>iD6wA@l zx3)x?@bx<4v4s>dVR3F?7oR+^^$o=3jRStzJVrv7Rf}aaGcLx_bZGcsj%3?&Q(v(6 z=E)(-yJ8Fdpxj3rWh4VxqtE~f;e_pY%|~c_R4zv_GE(|PH}902*e&)A1Paw$`;gZO z88i)i`w3f}k@*8I1cC4F-yy6V4O`>o3vMiH8lcfo)=0{uD_i&8mLwC>O*(Dg*M7ipLl+R@*93DmkPqw@vV>Y`)DbfQdONH=%iOm3!JfKsx?Hm zlU}PlP(eV*BDnB`obiqS@-fB1A?kDKbV5E^a5T&oh>Uc1N52|T+GdaQl3Gp)K+rBo(!0-+Kx7LV6dIn1 z9zf~6W911ZM^x8@heNbbHgb!?R}K_nHy-1KQ)nTO)WO--TE1YxB-zgQTtSsz*kroF zF98Z+Xbd_h+w0sI)`f=QX_7-i5Ky$mXc#4HWY&f^jqCcB6mUlX2_)AI`D2inAhz}#4HG8F7sG8b4=d>{ngN<_ct_%E1rp2a63Q+``R2Vo$;A4K3`crpGiJz&&LfpfoS2MA4JWwueiEzha`auqkpT6PJl}6zi=%nA@NKO;VQ;1-e=&TizflnalTb($A z&xqXdxj^IbWI2M70oTuXC;iz>`J!Ic$*SX|KNllk`>Rjrpv(MX`7HXimcC$~Y;T^b zO{fV2v80+@P?;alL{FLVvUIaNQo7GL+v>8gL=OB>FYE?TC`OqdxkH0sR3q@BBN5Wy z;{1gdFkG~xK+w=N`g|_^)!9<^ca9+n7a+L3Fwn6jl@XnWx2ws)oJiUtC;iqIY zXf4LH^=_*4;`^l((HZW#VW+gRXD9ZRlupoMI&Tj&)pmvGyT$V1xsF;WU;Ph~r8DnJ z{`Dn&|Ag$a*BNvJ+HUMFSzM`FZf9I*Y>JVx2iHaQAWn&A2u*0>B|=|KeW|B;W#dz2pDzd}`NpJ_H+8Te*{{MThlxrw{A(5U*&!eL46DIQV-cOS995X?#dzy92ffDN@c( zFSgJR4Q*JVunhEx!j_VT^KxxAEg)#<8Yg%BHQx;N0`1#ADV!DH7mTCjpyATp^7Z(( znudO8;7IS&fvY0U?vt7*pWe7DiY-h0ZTlkGJe@xMisdW+EXi7|zsRv;W=Fwx%RaJB zHu)iERYN^zCI%pC5gmF8U>=+j&A75u2TWpVtGt>-Oj)s)`g{ZM8WpPa!KJ!T>I7< z=*A>T@mH``(n)#JlV!0+feu9Pm?w7<>jXjHmCVUsu-5jwB}DJbkXxC?t63w%_Reau zku@#^eFNRkp&ux))M%hk_dbPl!o74lxPU^<*!6y)9xK?~4AChN_}7<|i+#OW)A6rw z!o@4(2Ck}T!7R#usIW~DYTtwqT^u9x2|Um!gH3mMeU9A4Jf{1HZFuphEaqJqfq470 z^dPLjHxM*RR!MdC#QTz=Es|rz2jM8)2+HBjl8)C-F>QfH1F@r%IDFJrrvbL#XUTqU zqiePn;^aK}JSz~z<@8P&#AX)|zH%I0SJ+55Uzb0Zh+<2=5bs-}Bxl@D?FDUQ&_Yh% z8t0^6-yx<;CFqXZmz2v>Y>0%V(f5KR2}G`A15xXYWN>2$aH~sSb_Dt&16-`6eJ`vp z7n1=(3q~U-4R*`tx3$p5(U0Twjo&9%aC{97{Qwc4CI#&awUju%sMZ+Z$=uGHsL+d#jE*&O=$-A)^2)IBrh5clbO=5Mue9CtEUe-az`2~Y8v_@Gk zTRADxY`PY|}t4~3P3%b5DI>-d|72CN~B}s&lQ`0tf80X!(GHTs>lGTBC~O~IEuV2e3KZYS zkB*jIq;S)tk_VK+ObPHQsvAH9<>_j2Io-i;0OA>}PM#|*m=&Gf(fuS9L3F=?P|+rQ zHWh|vy1p-slLR{MBIH1Hy#MNHxtBRv(js2>L`noF(@*+8S0JwvY#0XI~GI!cDK8*s612Hm~nIEyWJ~aGTDe14TG$=;9pBO7&aOr!n6&)H}KkiDNZwP8SIQ#DjvYUMexLCFs zb(hI3PToP$PteI={>UJX4RGs*jF@=ISh`mK(2wI}5OlQrn9ZC|L&(Xf*+}-V?7@US zQT8o&Bb4T1AsS4SHf;WB8t9VkWH1MP-SA=kLqD*MPn3Lr3I+|L2v)?Ajds*>-K8>` zrOS>;nR{roIxc-Lv7al@0HSrF42WdVg=q2|xrP}gA>2PQI_3txI((~M<=6oKjLyk& zIh~Ha;fv1c%G~HROh(6LvXP9S8luc-%N2kx4!s*?kFLc!+iUo+4Z%83k*`@e7h+S; z5G;L;WUGfws~2L?vLPobs3e1R&;~xuag^;r8KW{}!|^wbv#oLz z#I0aqvbuGbTcqpV=Ju9v;U%awyj0Ku+F=!l5lgs{d zNtg=|73*XhRa7+uXh0mUBm4c$9oz8h2^mT9EcEcT{qJ-cNyJQ~;irv~zh+aIC^SyR z%P=Z{fbW~|pQEyiYbI4(`ZcJOHX}=nO=%1|{Tg5PE7-8n@(Rga z0oRY?^cz|%Ygrip?h8hoziE=)&+>! zS<;t`u?1FrAnvKA+Q)Y_7F%O{%hA9J_Je%ofOvAUtf;>|+yD)-HZ4CB$$Refdm&T$ zu;wqc;{^z|^*eE{4j12ZPji8cTsLiDTdKdb`f-qY==A4qQ>=~4v2yh1GeNA8wWQfj ze>SPRFY2}PA{hJrd5U6~urz=}H6*Q* zYZ2ehns5t^PaG0@kDt~od_N`0@6N*)(9>Z2eucAXK$0hLxC8Z;bOp^}|Hb%Srh-_FSM zO}{rN2!?XMuaY7Qt{Z9iPE@PQs^Ts}PN2>fng8o2nyr3x!!&uxKbyBGhm%!4A7_Q( z2_-YSpl^xfM}hhnhZAG$%-CGEm0R-^|2Xq zgAI*7@p8K#bz|k|SFDn6NH*G98kwtA>~B?BHG)py-f^X0^L?uYvnVsCxZAnOwVbYP z`e}d!OG|7&x9SIOiIpNYb^+I}M6l~}R2*qFY0!`31a8lfcOy5shC2b0XRV5fxAf7v;T9oyPLOHzBEhlbVqhJCczFP0zk}}pTHm&+=GgVSl z!YL|6-?fM~_t*^_G?y~sPNFp#N zaGsaG88^s~$?BpT zok(|(Vbjz_biy7r)`fkb-)DBAu%5&*%ETf?yJoC%c zRF%-qZvKG+#62adG3!&^PZNwT_s&oW^g@2vZYfVD$$}T6l2^!_4l8c)xb zcl?4<)*ftkymQFB=6xE^Zj%M|AhAo_lhEVNrlnlMD)r$Vk3 z$`}3f32t>8JgJ3)#z%#+*pE@90UB?TU5w8aUl$-sb7f(RP*_qLR*zF{vQ3bUZlqz| zWEJ<336SwS-G)@vxQC6Yu@IXls8-+J66M=ZSt$Pdc4Ht5vE_tnwd3DrWSa)nWrdeL z?kTgBWMm@Pp?zK!iCLR6vT1&2wY|7yY3v=RnnyZ6C_fI9SK_BeabYiKUyVIc zCj4c#ie+|SGSOWk*>P!vpVL}Ugu^aDhjV%Zn6Yq|v8KU930ik)mW|9pscS*qofs#-Qc za1>8vS~D6TN<;Tb)pC@H+|ekGqk812q_6F{DGfmMovvE?aj>=_BTps#kPw-J6v628 z1N*dM`a!TK;@$U%P%UJFrPI+6nx)!(oEfDY`wlO6jWad{1uj<~Rf!iijZ7Dc5X~2U zX%dApccsk7o`CBI6fOpTMI8w}Nw3kut0~^emi)|nQXwrrnQ4nj#N;wfCINO(l!bRmaeWQ=^5m87F zydNb8JBsB5=;~?965TvOo?{)PxmY)Rwpr%4xg<*c_vWd@`xYAYt)yGlNihe@;OhsQ zZYz``--JHVnR3uSzP1hPJE>+1tCH`b{-@QXoJ&__(BOeaStlvwL;`T1=%@RnJo9TU zhkk(gbhea#eQy{A1AR6{%84#TD2+mcdqOq1YpDq(hA0iBT`gxLQya)CN9x7-8=lqR ze*f5ZMqN13Rz$JmW!nnX_~EY&7fW>iFgf^1)M=zTTp^38Xu!YvL6?dh@-Fv*B61um zk%Q5h(lYZ3y$kZ|n*7ODjvdQX;$wj*&mF{bc7>yfqV0D_<%KgFqI&XIyHzVctA-Wq zr!(>bp&CA0hZpWXulmM6O)o4pV}IXD)zS~YEi**FKOx`QC5u)N1N|^XzO#wqG#%I; znxv9R6sL*WhSPDf?t_J4Z9(~NnrebQe72xe){%o$wMQuJC?F2)b8ld8ypRSOHA;Qe zCjW4l1{OP>tGiUx*#B{&YU*!=g960&CsZQK9z3TXX#BK7B@#%kxfr4jE7%wPh+V&- zgS^N~iyzu*pq!bZTHM(r;!3AZeK}Oyq)oJ4%3&|~5Jgj~ba5K(lr@A%Ktn$$_s={P zx7l7nwD$PtsER9z8kG(yhSyI1B#Q01bk#08g^CliM76y08ZEXJ%(;-faG?vsJvrz& ztukey|B5zTOv5LwB5rV6Y>`8p6KY9$LL|y7R&h_K=^sAP*di6nl{;;*1Ostl2bEaQ zTp{uyT9?bgNM;UbG@h+maigqCTPJar5Qbv}vs~IRkfZnMoMuTXu4~ozKBC9&Q0={m*-#VB0=j#bwQod}%p+?;j+KrY{E&S2U7C#Z`4h5j|tF zEG{rRJ(?2`9ZOUavBf@H8u+}ALiR}&zVXdBhLY^ls zC&G64Bq_SAmua98l(eHNnWTu?QD(fbjj!79n)gw#N5&Z%s&-ju=(kuU-O`L-Mmb-iImJmj!EUJ2|LcdZy{PRNnRuqCo=fD+&)2$ z6DD3-)e9foAuq6CMRi~OCs)Xe(FszV852~CuGfa;(4wDIEjU~P#W(VED`cU~DeKdi zTSrPG>r-v@v8u(7v%+o|?l?q;mG(UtW(&luB-M^PnrrA%g~ko@x~s;|F9>r1;@Pn( zzVW9~o|`vOwTxWH))1Xk0{f@hggi977AuF?5YhI)zli?Eba{(oQr&TR#yieSMJ&H= zi0We-Qe_d@#e91}gKd{X(KjHRjZ5TZ?lt-l`1Xx$s_m_PHA~+fe9p%C3z9OU%X{@z zLdvVU!)Ykw*pVoUzUdTJjs|WDzchDql<1rJ79)>yWV$$-F7r8T(=;H8U|+6O@t5_A zBFakoC41Y)=fdUlmsltwsTK@L$?UVhKa8aj8o#cSqD%YgY0Dq``}y)@10Vqtf`X&C zuYT~D*Gob5Q@ZBcFiel}v{%(%?b|8k-UzW%yExqmvyI{$kP8jh1$ zBUxVYZ$KbN$jM~OwTQSJ{W~f-gw6y1}V5z5Nb!5kR zxRXh-!S^mc8y+4A)}D#&_-Tz|mCJOx%QAc4!{?GUNtTnS1+jGlGzh<4HrAx1@=^L8 zS4kxQ_4_(RLzVL9?3Tt+5T;>liM$nAYomS+n=0LY2zwuhA$ck>zN;2nKWKYnp1j7! zT~tH5yw*Hbv(yg|S2a?NZ$2Hx<>iU;ZY1ak&z((nd)b&8tsrdgnj=g7s7~V-=dbB* z+HV^J{M*0KxbiX#Y z~TBsu1B(XlsXQaP&6SVo=mab8x}Y(xZgWtqH&8gYQ(xPj8HkOu++coGU%Z z?b-nxgQ0QA8PKqiYV_8FW{becF>q3$SM8=RP53?vhMa+%S8nZl(403IIR?rJS0CA& zf{Se!8t3MsEgjc-ZlJTIv@sKk7Twr%;EP+_b7bQyM0BJ6!29aB`COL+Ut1n7OD4Iu z5Mc(2PxOa#?*7|P8eJ@n4=Ms@NO=gjPjpjEX*;4!CV*mSWRckUA1cJa!9rxU%~jR> z%yEWBRM?WVi%o(~R%tca^zZ-Ls39y$c4EBzZhdRz$Znk{zq9RvP(H-vMJA#O zmUi4`6ZZNCcf4t^2Fjkg*gX&l)CoG-TrvB~KbwbB$jP2@(%p4yq%g0OU6AJPvIjCg z+qXu$l}8=pIip~->$pQgw7CHt`w`O0RV%s_nD z?p^2p$hkWB*0)Tmgw&D!z*b- zxsAi*_wBDl8R(CF@-5}nAZQIV=(;%fSMIG9gh%C|TjSlsWbxKL+2}IpuFdWtE<9)m z&_d3jh0EN-9Jio{wdtUXmZ(1KF4x1uYCCA|O7|dH@b-o~%%Fvn+|TKBS~+8}K?CAc zb56W94a;Tu1ounEi`Ldc42en#Oo1hnTA)eBL92Za~|^byS}&Wl{EE6S7aXxg%la$e9r5 z9%$6tIJ-=DsT?YiY~}C~#Vn1S&Ka_mlK`~UkN$kgb@!359}ZUioOjN-dl?=D;mIIp z=kfoqy|0a{s@VRgNKGx1La&O2Qdtz4g@tw_&z$Wk3>0OHg5o2FiK3z?8lrFz6*+)B z$djU^C@P8)S>D8CFS`~gX@;qkrIrtsY3aR|b+zAj*6hRKbN#>lzq@|iFPJm4)^|P3 z?AbGG));5}esvAO!M`3gPOI@p87k!NNaF}j#n+<=gtKWOz5R>}(3s@HI6@w)F)rc0 z0uVI|e50w|X?Ho|%EzV!HzkQLFv6g{N%V1dqX}c7B#Iik7V>-*-K)j&5Uf<#jQfFh zwCDq}U}vO_8Tbg+5dT83wat&$rAsLVh3K)yd7Q+M!nw&oDfrd_Ynwgo*^?P*Fz)wg zW8PU0tw9#gOV&2;s3a>)Gz3v5r+$5bJCFjlJKXqrD94_g(CZqF&&=!%++^sz85Dw9 zpJdBThW2WvkRfW8qi7&z;>#}gJ1wVaA*EH~ICc_&K|VHZsC%e!2tEpw%9KMNsxvP7 zyz3-7L>oH0S)4p|PGV3>9(ppsxQL4s9B2mCx;FG{U*ig{@~U@=Kg|7_aSAc4QtPbK zVX)kV!Bj=5C=us_>8#*RZG~kv@5R`1uB=NjxKnes&1COrhBO7T;=4Y%z)X=ri zxv}DB6)0@t*nt&+FMe=w8Sa@RrnF9HIs7G`{n;yza7>LwN;`ZCW^7&6G6M@YJQ}x$ zWnOdI<^NTqC{Z*F2$P_cDTnWk6`%Z}4j2NyO&gAt-cPX5Eo(5+;h)8bPjD2$G!$z5 z)D1Mg=K}bLu8o+vhrCs)bl`Fe(}w@zCK}5Bl7h(xO@0j$4FQ~5PZ1q4tjTEjEs=XR zTYtpDXyeCaR!bc;rV2_AQU}anFTg)oxDn;e#t?M_yMjJqS&aBj#lx5dyWFf@aBn2+ zA|Fi)>su!NdO}@!$w0lP#r(+ikuJDhCwu6v9Z5q*s6il~0Co6*xu zMX8#M&Zj3{ZQ{ss2BzQdZS+9ThNMb3y^FWujp+t{Rh}G%ypr936oQ#GLOM~qj@?`X zn;z3dJ&#;rX<6&~bJ2#kO6)9~*fbsK=i3bFA^$Leb;v<(eRYH=gng`IxwXyT=eq2o z$9ouk-)Qy4r^nqC4cOw8G!#+1flfa52jhTnHf^MPH)>;jjV~Bakv=x+6+0EZex0R- zPIN82gAcWTfb*X!#|`grklJ@y$Kx2|7-PA}9#A53&4C5XeY z3ka%?9DaYiNJe-RG$w)R+DIH$IB5n@G0|{Jpl46c=d4Zj5?HuVO3&Wt$d=<8qulF^ z+em=nfGHL9=Yq)R#gVKJHq@v=g8XsSf^;dCWmL~b;}&{0?4o>ZR6>%0%ulX(hf$;Y zi7&tQ%9tAajy=uL4IF%ytfX>4?Kx%dwj z0H6!yAEUFcicAEhfu&i1lvKxhuVSJs{^)&}<@DSuf0d7>js7`aoWq**VMSv&3f#rH z*2@eLJ(EQGBYU_8IO$r%gE`{ot_ByNnWjYy^b;$F*(DCi+NMQ}!@)lEeXx|ZP=JF< z>8_~)3Z`8ahj2s$)s?i7y@RN0_&TS?P-964aUH$VMKtfA_#8*N9pJ*UM4U^ol%ktL zK}y<)okE;ink9cVYaH_t%P{>i?E<1WLi8n8zd+P%f_9AEV$Spmcx@JFqS29 zWFfWr{4#IoZkFy4frIqFnZZw>qic~bxYHvqJSTyv8CwJJ$?{18pgph-QrFw)3=0pjBO^-PpC78#Z!Qs*vmxzwFaZeFGVkM(lRP$I=*YnSGI ztYn2l?b8+TOLT997aE$SL+^K^yzYT*4d4(RoVbaSl7mk*e$Y<_{v3eW|=XqwJFEolP7hdAJ4OIiXv)9a*xI$VVCz`|z{wiQYjiGj;3F{yD)s*F5gGGvY zjNq_@J~qLZ9zwhW{!mUg=7wD?RLf9-DAcvF{@p~LIsb@}9yAvHCC}VQRW#NX8t)*X z(0!@{p+@*kk&9H=_*Ef|>=tG%`zPOMpbEzN_*nbw`v><1MjF-8pPtitGYtp`8VgHB zGG-a4Pg(0e-t@d zXbje(5Ye5ELAu^Ko{qbEp7P3`lvgb%Geq@XEY6Q@>tqWroE|F*%p3rUXkxV}z#V1y z!}QZVUZTL9DxlD&MSboe3SLrM;HsdaleG)JUL`O*nl`ShCqnM#xM+;SA(BLRp((Ad zje8(b)GcW5(nr)EKGrsv2c-ZyYTG>cmPJ#dagIXl}U0eGXVabX$V8 z&6-fBaG-JdlE}v`BB-wXV;nYxZzHznu&V&K{$UMP*1%)a#v@7fZAAXTKa}OhFT5;H zmp<)mf*Odo{2Jm*4`S%!Kki`d@@GpbeO;SSd_&lA1G`iZYS^?1o8rYj3`4uiKo5A z;nrF3qQ9-CqW9}9U@KYc+T>t=>f8R`Tu>_q=}F7V#9{6}Mt7HLsqOSAaE zJp9GLCIhLjy7^p*KPkG-I(V*nohUAAV#&{3sDc-P#xHT=C?rjBia)uhuh@&@KhU^j zt7(%528j=_Ac6vAsL99#{taT#&_r>Wj6IxvIF?n90%%^>qOp+K^|jyFE@-A}Q^FFA z-0r2U(+jf38_+P<@I;kR@dH4ilwZluJfCa zp~s|uW4))iuM?@R-C3|K#(GcTLdiuLYU-dUF&e8q5<1hEKHg3nE$_L$v!{L-EH2=V zL->Z$`qZOt;sO>!WdWMlw5g{Siwg+O2T}Rhv^PVmt?olBWp(92u`y+q{YZF;HEnsn zkFHJI=Pv57LITm{v=#+OJ+K#HCz1v$DDHsC99$*=1EOthosa0=vRe;pKeV^RH zucQ=d%GhE3tj{1~25v1iuWQrq^%7^z2^z~X9Y@U0jqbqxw!0K=dW4%edpw*Al!D~- zS-3$+(qJyr7akR7v9t$*$_uAwxr+~Des>8seF<(J#TF+xDCk8pooAX7Ts9pL z`^@CwitVlp@nM8=DX`VFnIRKtHIg)ehRGJGDLzC3E=fb-I3mP3EJUEaGUbfB7K?K# zH@4K8uFZ&TC(i7COKMDFK|xWJ_z(#W09_T#zCWCn9yr1NQ7;uV`@uk3y2)Gi21R4$ zt#Au2`{9qG!;x3*U0uX{=_O=J%nSV32iEk_q1BJ3m41-j`B(&vpkDllI^@7a1IAk zS~O8QogZ0h9N1fDy$>}|D@>ao?QiVOye3gvaX(L(mY5Z8eDdRUT;b^^9~7)=79Sz- z034*mHZ2k7_&&nke28;TJn0|1sq_!E{DVSWTOdlsd**Q$6xg%{(~epnZddIx<$@WP ztUVDN=ca&Z+JbrR)`zjx+TA(DUpOby+IikvJO-gq*A^~}wswB~h($JG91E8;SReSa z&edPuT2Wzru&z-4suZ+%UO1Kf$CqsjAJMfWoLA4r_LIq0*B1416Q%JSX|Eh+(R+4r zAC_HAL!kze5x$F67PzJI!bLc?mua4ig-!-*$*=6ObjQX}Ti7;54k*bPe)a#tFb>fr z(c#)+iyQU+qTKqfatGAFr^a#f!wOh*Ep?zbt$8rgg7y{*7tC`=oytE{CtNbjZf)C1 zE$g$AmwXT@-bW-8Fetm&aRFO-uQ*rFwx{o=^599VAQnF*H{R+yHq<#i*EM^7H_AVg zCkIERvD8slt$wlUv@r{3Uz9|%-G=a<)+uQQI|WjC^a&n=7)R<`Rn|8W`C~*WD0N|w z^$mPBb$9k#`)58h6W{8nuDIC0tE0I~2lE^kwRCOiV*ylfpf?kBNDo>XkWPz#dfD|# zvJY^x7QK_~>gwrdgQx%@5pu50O2GAK1@hjXoOh?agmol_K%0rh;%1Bs59dMYjI6P8Q3d4$oaY*Vj)HUtne^1&~KAN_&LnKXFsO}_ZX;&81Q2K8=AGU_&bZzB#`IHlFV`ym#SzsJ|m$I zrR2;$fi(HbIM+Kn^SOAMwr)1Rl31siL;YylttTzANxf81=CDYL^_lJ*wrpDF@EV%x zH__$%naMXP8HZRQn91e%Vsf$a1angKa8Qbyz?z? z&LJoBb~{?u{4{d`7T{=ZqU4yJ44pNW`Q<@M!j~sds!=Ml(LrzA7|GJIp)&udqNHO7 z<*!a)FVWPzWNCS&)6Bp6&|A^j?1fC!(J6-#v7HYSs3s1td`c|c$4!{1W8z?1VZKcv zkFGfa1TFbAkb$X!zn`EbNYZo)G-~C?h~wFEO2qm)+N+N22#lsh$Oa6JNka|S9C`;z zJ$KU8Zyo!>DHqX$QXhCw(On&bOUl#K6v{Hk2r@k1pg>kr8Orgm2x*KScVLab2Tl=en2C9l7W z{R1il^xqO`5k4cYLpYWttIc9cUZF1h05Dz4y03;7VTA-5Qv+M(ZBJ`2FBOZ=II^ZE zQl{yaZ0xKy4w{O|glq!6f<~uyG^MVMltm%UYIB)J7V^#3O6ysc0GfX$j(tqg7#={2 z?jP^!9$ByWP@H=$D@XxL@S-Gqnxnb`mNm|u#$nyE);TlCivNbDJ6Y-ZF+n7NVf&JWV!Jb%e+o*>`(V#<+MkwrXVp&v;_D3lLgF(I1 zgR(vRDD^AVU;LS9FXYT{N301LrAnKzp9{pq5R`#&`AtOui*Bg>-x*EkOvcEITW!qgZ;OvArN{yV?s$h+v1+TP9K%QOp5(qiMg1z+PY*NOT@xv$caUo`k&PRrqcMJgx7v4Z;DPs>Z8(Xd+g7@Q31(BODZ!IZ5hN=_6-~ z8^wJyTVBRs7F7CEG7>9phL5pp=ODX!6lO{nP{97NDk6+l;VfW_PFG=LdC5h!w+7a_ zwyIqsEx|?SZcb@eJ&{fe#_Z$qG(}EGR&AL`x#kHfxD}dojG`&<(^~ktqVcq#xw9YS znIq_+#tW6S0Fx_+QrZO|mCoF*P`+2+u-+^$Wu zYVHs0K>&vuKYP%e{BmdauxYD)N~gG&3)xSV#?HpmM6V)NVT#mPZpQ|SM<|a(stRUb zqBIBxT_{;{$9YgR@}HXGqe*QyNDS-7uYNfank{&$F4PKbXuu#!c=lh`2;c%kvl* zSV1g)-jmG~g8+yY9C`Diu`;1aeW*T{pXN_%9uASvRYBgd97=g4*gsW%YS#a8$2B6PDFy-3$ z9b+l5b3iLD{VQnr(5~F04$BbD{1>Z8pYS>>2*%R2{J;Yg`rEUzK#iyQV{+)tOFLY$ z(pf~CCE?U%_!2e)8*253J`|)f zwVKdBcc-x2%T5IW*r$Os#0)!OqN`88L?duM8`@Itu=>;i8sXKJai9=fE_R?1-T5plu=o-GXX0#k(*0rL`jg<8ue~BxXDw-8aBUAF&>DZLTT@xr9 zFD+>P^53rE(#0~Ovkn6i3tVrPj1z_1{a zTFfhXCa|*0G`f1TG?aYMfSe|QQSV9!eD)AD1d|bMd_4<%z_Y1DX6%QCq+G0W5KVo7HT{dOd}8q2rjC>6wj!j z*#4K;P>i&AhlA2TZk5`m_(}}r^qwK->&leHAGV|P-{-p&RGe5&;SJkag~|*|5-AKi zmB3&*u>EHxb$^*Z0j1_qaJ!vx6gV!R27Ckshte&L2%E2UnxW+NgESV01n{d8t|UC1 z#y%e@+big6rY0I!j()-uMdniSuSVlWg;ufd$!_I<3W?5br;gS zGV1(vI_m>8Y+C7?VU%}~uNF|Ul+Ha*xyN{eTPdg%ADS_yZJF&I8Y^i^qBh9#K2Qyx zx>ovpFpcSQ(Piv4k491P1*=QzrMMu8Fd?>IDpW#!$(GnXOd#-~O@Y_d>Eib)=Qk!3@#(7K4!#+qI)S zDF-=aWdVe?K;Ql<5^sU9@twVvE7 zp?TCO2%{-iJXsbt!`cOL#uc24b^r%P2wf}dT}gSts=p{h{ZVCo4tL>wq88kt)8TGb z-ye=zd#P_ETIY$YIq^{hq^5#?#UX)$B+=Qqs<3WtBjvxwvG}S< z+0hDPS>`qVSXFF4FBK)2p_obrurn2+Bu$+q<{Im|xEnTP1HKMF1rF5sGf}*ZFkm!M z#$H!hP06iyn%85$vY`J*lBJ0>*TcAi0}mLG(2RPZZx7(ekv<36o*BR;C zSRczBu~_Ot5wf=USqn1ao6%W`-L2rNLMGuy(W*n#F z5d8M|1o3h}y(Fk~y20ll6r^*%1t(qG@Nq0N9TnBJeC9-T!VQtTDWx{T z0yz|y4cq$Bk_|rmzGdhe&NGWWIxD8QaCm?`ceMvJ!}TB*0qgsi>V`0 zFr|P79-Fq|+gR(;f#>9+oqTNC#+AX;@yI3?zh2s<;^{=nxOj|}>?R8~HdY$H;m~GV zST032sR)XrWm}h9@T*c`1@?jB5dU!s&JX*7M(iF+YojsI5ZN0l(qE!iv9|RPX5(mv z0-L*-jdNN+sf4Qt2@rEI$AvB=Yt-;-v<7;3h+fW7ssb}dJ6^yg&R$vZZLIMXVrpc8 zBKozjWwz_Qn5JzS6-h&%9l*13;25H9%4wjbFSXpxfP$%Y)DG!S?gLv!x@l$%bws8T z$e}!WQ)3VMt*rBvf0#9j>Zse!23N1#T(X!l{Be z!Qy3PCj=Lzpv}3vjrWn1S=L~-mG?B#Llghv8Jm(NCdTSzrWi&Io3^dVm7>e0e4k@LXt-mZPhu)-_Iak<} zK&xw&6O(8bf=WgREWrIf0+4po{6&E?rc<3nj~M{#H-(;xiOLmK#3SE-$uZ zCDydcW$nm*WPs$NI&$S(ku*DV8!L!aSoz`}W1%_QQW}eQP*5F$sZ>(&<}O2>^CsIItd z8{#m2nRh@si$c2X*=sbno%+y(_-OL?2J-h9!&?-9kAnUgmM3v2PXbeHpX?;|B5oXi zm<7*xh`mTs4W!D0wk`Lkl&&4+a)-$lOM7a_Ds4`GOuezs(GCmu~UZ{}b{;2Bt z7o{!Mj4Z4uv@xGqE##Q)3PtiSnG$SlB;Gniap4_59LTYHm$1R zGCgd4jk^c9prA3{*nq{*ddRX_YOLzVtJE%QJo_o*sQT$VwS(h=gJQe=H9;$T@DWu7 zYt=gE66@srQ=s@X9iN&WBs|tDQT$fNZMUA zCc))F+ryFE1ZnYFe0}?V*!GP#rN-og94jl5jJt7k7-le4cs80=PFHyq6c$vz; z_5B_DzM&NeDuFK4D79l-(;#7fN2N#)+VNXCt>_oPE~RL|aqog#I#hoN)2e%h8GEtH zBDKyns$cLH51sx-x`KQ(t$Luh7=sOepj6$X8aL86^!iJpRBWr$E9t3GhWxNnYxLE# z(#0oOM1U+xW7UbF;*vWk0d`WUCvttMgER z&;+PY`DyjHQIy$>_Xy1P`s9HRVd_7M@*z4v1a|6n9aM%W^xr#~U>W@&z#Y*tDJA9Ym=^ zT^ztTc3P6i_ovTX(~X^vWr$Lk(rlt>JAILfqH8~=_@MD}qu9FsYo~%xFe_ASMMz?b zvp4Kat2P2L=Wo&J&Qeb?!OWG#(06q#r{y>q+uixn+WBdfk%RY3i?Mfgjzf(%orco2 zo!7Ar^Y#q(G5CnC?P?!Q%PXd_!+^C-+tu|JJs!SJj>&Rzgc{EuG~{ir_*FUXuE=m> zG=NF1qYyQ=)KGuiCoXZ+GmT0?{kyA!O6+vI_D4~FEHIyPBHgvgPi#G}$pYnLyLJSL zt!6Iy5^39ApJ7RD>M_v?+fVM&F1x-uPxeob@Svn>d=Xopi^Gx^oKCmvrXMZ)^tx++ zt!aN#Y;he_(zTinXT_G*$z5yij}}{TDh((D;S#9PIY4anZ8=e9(`u|uv;vz;kW~3t z&9Fdm;@otX2L0&v-gTkg!`9cTtF zwdvwxWU-PofW@Z0*WP*4%j(`c zh{Z=O6wKIP-xZsF@DJq<@5~rn{}A5$5(ij==z>nh+1{&o2zIb6iQSB|eN-fq5-!2F z-cz0PG-bIOWsJhoFHowX@15)A#<|P;n2QDh?XLU!+1b$UD zGXHko>h8x0mQ=wX$kOcIi?4TOSzgIF8emz4>S)iy)+;L$I$2~K8@Q+%+Y&liJE)sj zrK0sgZBBn9mg0j(;((He40>|JWs#1x3#gB6JLpM#ai+gm#{ISyzy>{??k$#nX6JDX zX!Sw;4>!_MTnd3dSo}f3I4!sYq18~I;u3X#CE2Icv 0 +MAG_AUTO < 21 +FWHM_IMAGE > 0.3 / 0.187 +FWHM_IMAGE < 1.5 / 0.187 +FLAGS == 0 +IMAFLAGS_ISO == 0 +NO_SAVE + +[MASK:flag] +FLAGS == 0 +IMAFLAGS_ISO == 0 +NO_SAVE + + +[MASK:star_selection] +# Star selection using the FWHM mode +MAG_AUTO > 18. +MAG_AUTO < 22. +FWHM_IMAGE <= mode(FWHM_IMAGE{preselect}) + 0.2 +FWHM_IMAGE >= mode(FWHM_IMAGE{preselect}) - 0.2 +FLAGS == 0 +IMAFLAGS_ISO == 0 + +[MASK:fwhm_mag_cut] +FWHM_IMAGE > 0 +FWHM_IMAGE < 40 +MAG_AUTO < 35 +FLAGS == 0 +IMAFLAGS_ISO == 0 +NO_SAVE + +# Split the 'star_selection' sample into +# two random sub-samples with ratio 80/20 +[RAND_SPLIT:star_split] +RATIO = 20 +MASK = star_selection + +# The following selection is only used for plotting + +[PLOT:size_mag] +TYPE = plot +FORMAT = png +X_1 = FWHM_IMAGE{fwhm_mag_cut} +Y_1 = MAG_AUTO{fwhm_mag_cut} +X_2 = FWHM_IMAGE{star_selection} +Y_2 = MAG_AUTO{star_selection} +MARKER_1 = + +MARKER_2 = . +MARKERSIZE_1 = 3 +MARKERSIZE_2 = 3 +LABEL_1 = All +LABEL_2 = "Stars, mean FWHM: @mean(FWHM_IMAGE{star_selection})*0.187@ arcsec" +TITLE = "Stellar locus" +XLABEL = "FWHM (pix)" +YLABEL = Mag + +[PLOT:hist_mag_stars] +TYPE = hist +FORMAT = png +Y = MAG_AUTO{star_selection} +BIN = 20 +LABEL = "stars" +XLABEL = "Magnitude" +YLABEL = "Number" +TITLE = "Magnitude of stars" + +[PLOT:fwhm_field] +TYPE = scatter +FORMAT = png +X = X_IMAGE{star_selection} +Y = Y_IMAGE{star_selection} +SCATTER = FWHM_IMAGE{star_selection}*0.186 +MARKER = . +LABEL = "FWHM (arcsec)" +TITLE = "FWHM of stars" +XLABEL = "X (pix)" +YLABEL = "Y (pix)" + +[PLOT:mag_star_field] +TYPE = scatter +FORMAT = png +X = X_IMAGE{star_selection} +Y = Y_IMAGE{star_selection} +SCATTER = MAG_AUTO{star_selection} +MARKER = . +LABEL = "Magnitude" +TITLE = "Magnitude of stars" +XLABEL = "X (pix)" +YLABEL = "Y (pix)" + +[STAT:star_stat] +"Nb objects full cat" = len(FWHM_IMAGE) +"Nb objects not masked" = len(FWHM_IMAGE{flag}) +"Nb stars" = len(FWHM_IMAGE{star_selection}) +"stars/deg^2" = len(FWHM_IMAGE{star_selection})/4612./0.187*3600.*1./2048./0.187*3600. +"Mean star fwhm selected (arcsec)" = mean(FWHM_IMAGE{star_selection})*0.187 +"Standard deviation fwhm star selected (arcsec)" = std(FWHM_IMAGE{star_selection})*0.187 +"Mode fwhm used (arcsec)" = mode(FWHM_IMAGE{preselect})*0.187 +"Min fwhm cut (arcesec)" = mode(FWHM_IMAGE{preselect})*0.187-0.1*0.187 +"Max fwhm cut (arcsec)" = mode(FWHM_IMAGE{preselect})*0.187+0.1*0.187 From 1b22b9884d658affc2816e3b9ec38b169826a31f Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 30 Jul 2026 22:22:51 -0400 Subject: [PATCH 13/41] =?UTF-8?q?feat(orchestration):=20manifests=20become?= =?UTF-8?q?=20the=20DAG's=20currency=20=E2=80=94=20completeness=20check=20?= =?UTF-8?q?CLI=20+=20manifest-driven=20report=20(#848=20D2/D3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit completeness.py gains the 'check ' CLI every rule's shell line calls after shapepipe_run: counts products under $SP_RUN against the floor table, writes a byte-stable JSON manifest (per-runner found/expect/ floor/warn, log-scraped failure reasons, no wall-clock), exits nonzero iff a mandatory runner is below floor — manifest always written first. STAGE_DIR maps stages to the committed configs' fixed RUN_NAMEs; the ngmix entry env-expands ${SP_NGMIX_CHUNK} so chunk K checks chunk K's dir. run_report.py rewritten to read manifests instead of disk-scanning: per-tile / per-exposure tables with failure reasons, attrition aggregate (failed units excluded), tile-blocked-by-exposure join, 'not run' for missing manifests. CLI interface unchanged (--run-dir --index --status; --out/--limit added). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01B5KZRTAZKSBAnpdDhE7yzK --- workflow/scripts/completeness.py | 209 ++++++++++++++++++++- workflow/scripts/run_report.py | 313 ++++++++++++++++++++----------- 2 files changed, 409 insertions(+), 113 deletions(-) diff --git a/workflow/scripts/completeness.py b/workflow/scripts/completeness.py index b2246abef..dd0836c66 100644 --- a/workflow/scripts/completeness.py +++ b/workflow/scripts/completeness.py @@ -8,12 +8,18 @@ its ``floor`` files; per-CCD attrition (a sparse CCD setools rejects, ~0.2%) sits between ``floor`` and ``expect`` and is tolerated. -Two consumers share this table: - * ``sp_rule.py`` — after ``shapepipe_run``, counts products per runner and - exits nonzero if any mandatory runner is below its floor (``keep-going`` - then isolates that unit's cone; ``retries`` handle transient failures). - * ``run_report.py`` — disk-scans finished trees against ``expect`` to - enumerate shortfalls for the human, mid-run or after. +This file is also the ``check`` CLI — the second half of every rule's shell line +(PRD D2/D3):: + + shapepipe_run -c $SP_CONFIG/config_exp_Ma.ini -b {threads} \\ + && completeness.py check exp_mask {output} + +It counts the unit's products under ``$SP_RUN``, writes the manifest at +``{output}``, and exits nonzero iff a mandatory runner is below its floor. The +manifest is ALWAYS written first, failure included: it is the DAG's currency and +the only thing ``run_report.py`` reads, so a failed unit must still leave a +record of *why*. Manifests carry no wall-clock — identical on-disk state must +produce a byte-identical manifest, or the mtime rerun-trigger churns the cone. Per-runner fields: expect nominal file count for a fully complete unit (report yardstick) @@ -27,6 +33,13 @@ ``ls / | wc -l`` semantics (broken symlinks excluded by the caller). """ +import argparse +import json +import os +import re +import sys +from pathlib import Path + # stage -> {runner_subdir: {expect, floor, [warn], [subpath]}} COMPLETENESS = { # --- tile prepare (phase A) --- @@ -94,3 +107,187 @@ def check_floor(stage, run_dir): if not spec.get("warn", False) and n < spec["floor"]: ok = False return ok, details + + +# --- where a stage writes ------------------------------------------------- +# +# stage -> (level, run_sp_ dir under $SP_RUN/output/). These are the +# committed configs' RUN_NAMEs (RUN_DATETIME=False makes them fixed, PRD D2), so +# the check never resolves a run-log. The ngmix entry interpolates the same env +# var its config does, so chunk K's check looks at chunk K's dir. +STAGE_DIR = { + "tile_get_images": ("tile", "run_sp_tile_Git"), + "tile_uncompress": ("tile", "run_sp_tile_Uz"), + "tile_find_exposures": ("tile", "run_sp_tile_Fe"), + "exp_get_images": ("exp", "run_sp_exp_Gie"), + "exp_split": ("exp", "run_sp_exp_Sp"), + "exp_mask": ("exp", "run_sp_exp_Ma"), + "exp_psf": ("exp", "run_sp_exp_SxSePsfPi"), + "tile_merge_headers": ("tile", "run_sp_tile_Mh_exp"), + "tile_mask": ("tile", "run_sp_tile_Ma"), + "tile_detect": ("tile", "run_sp_tile_Sx"), + "tile_detect_uc": ("tile", "run_sp_tile_Uc"), + "tile_vignets": ("tile", "run_sp_tile_PiViVi"), + "tile_ngmix": ("tile", "run_sp_tile_ngmix_Ng${SP_NGMIX_CHUNK}u"), + "tile_merge_cats": ("tile", "run_sp_Ms"), + "tile_make_cat": ("tile", "run_sp_Mc"), +} + + +# --- failure reasons ------------------------------------------------------ + +# Lines worth showing a human who asks "why is this runner short?". Deliberately +# crude: the point is a pointer into the logs, not a taxonomy (there is no error +# whitelist in this design — the count floor is the policy). +_ERROR_RE = re.compile( + r"traceback|exception|\berror\b|\bfailed\b|no such file|not found|" + r"killed|out of memory|oom|segmentation fault|bad chi2", + re.IGNORECASE) +_TS_RE = re.compile(r"^\d{2}/\d{2}/\d{4} \d{2}:\d{2}:\d{2}\s*") +_NOISE_RE = re.compile(r"A total of 0 errors were recorded") + +MAX_LOG_FILES = 40 # logs are per-CCD; a handful is enough to characterise +MAX_TAIL_LINES = 120 # per file +MAX_REASONS = 3 # per runner + + +def _normalise(line: str) -> str: + """Collapse a log line to its shape, so 40 per-CCD copies dedupe to one.""" + line = _TS_RE.sub("", line.strip()) + line = re.sub(r"/\S+", "", line) # paths differ per CCD + line = re.sub(r"\d+", "N", line) + return line[:200] + + +def scrape_reasons(stage_dir, runner): + """Best-effort, bounded: distinct error-looking lines from a runner's logs. + + Two sources, in order of usefulness: the runner's per-process worker logs + (``/logs/process-*.log`` — where the module's own exception lands), + and the stage's ``logs/log_sp.log`` (where ShapePipe records its error + tally). Sorted, truncated, deduped by shape — a manifest must stay + byte-stable for a given tree. + """ + seen, reasons = {}, [] + candidates = [] + for d in (stage_dir / runner / "logs", stage_dir / "logs"): + if d.is_dir(): + candidates += sorted(p for p in d.iterdir() if p.is_file()) + for path in candidates[:MAX_LOG_FILES]: + try: + lines = path.read_text(errors="replace").splitlines()[-MAX_TAIL_LINES:] + except OSError: + continue + for raw in lines: + if not _ERROR_RE.search(raw) or _NOISE_RE.search(raw): + continue + shape = _normalise(raw) + if shape in seen: + seen[shape] += 1 + continue + seen[shape] = 1 + reasons.append([path.name, _TS_RE.sub("", raw.strip())[:300], shape]) + out = [] + for name, text, shape in reasons[:MAX_REASONS]: + n = seen[shape] + out.append(f"{name}: {text}" + (f" [x{n}]" if n > 1 else "")) + return out + + +# --- manifest ------------------------------------------------------------- + +def build_manifest(stage, run_dir, unit, stage_subdir=None): + """Count, classify and (on shortfall) scrape. Returns (manifest, ok). + + Stages absent from the table fall back to the zero-output floor: any product + anywhere under the stage dir passes, nothing at all fails. + """ + level, subdir = STAGE_DIR.get(stage, (None, None)) + subdir = stage_subdir or (os.path.expandvars(subdir) if subdir else None) + stage_dir = run_dir / "output" / subdir if subdir else run_dir + manifest = { + "stage": stage, + "level": level, + "unit": unit, + "run_dir": str(run_dir), + "stage_dir": str(stage_dir), + "runners": {}, + "failures": [], + } + + if stage not in COMPLETENESS: + produced = list(stage_dir.glob("**/output/*")) if stage_dir.is_dir() else [] + ok = bool(produced) + manifest["status"] = "complete" if ok else "failed" + manifest["n_products"] = len(produced) + if not ok: + manifest["failures"].append( + {"runner": None, "found": 0, "floor": 1, + "reasons": [f"zero output under {stage_dir}"]}) + return manifest, ok + + ok, details = check_floor(stage, stage_dir) + short = False + for runner, n, floor, expect, warn in details: + below = n < floor + if n < expect: + short = True + manifest["runners"][runner] = { + "found": n, "expect": expect, "floor": floor, "warn": warn, + "status": ("complete" if n >= expect else + "warn" if (warn or not below) else "below_floor"), + } + if below: + manifest["failures"].append({ + "runner": runner, "found": n, "floor": floor, "expect": expect, + "warn": warn, "reasons": scrape_reasons(stage_dir, runner), + }) + manifest["status"] = "failed" if not ok else ("warn" if short else "complete") + return manifest, ok + + +def _unit_from_env(): + """``SP_UNIT_NUM`` carries the rules' dashed, dash-prefixed form + (``-210-282``, ``-2605805``); the manifest records the human ID.""" + raw = os.environ.get("SP_UNIT_NUM", "") + return raw.lstrip("-") or "unknown" + + +def main(argv=None) -> int: + p = argparse.ArgumentParser(description="ShapePipe per-unit completeness check") + sub = p.add_subparsers(dest="cmd", required=True) + c = sub.add_parser("check", help="count products, write the manifest") + c.add_argument("stage") + c.add_argument("manifest", type=Path) + c.add_argument("--run-dir", type=Path, default=None, + help="the unit's $SP_RUN (default: the env var)") + c.add_argument("--unit", default=None, help="override $SP_UNIT_NUM") + c.add_argument("--stage-dir", default=None, + help="override the run_sp_* subdir (default: the stage table)") + args = p.parse_args(argv) + + run_dir = args.run_dir or Path(os.environ.get("SP_RUN", "")) + if not str(run_dir): + print("[completeness] FATAL: $SP_RUN unset and --run-dir not given", + file=sys.stderr) + return 2 + unit = args.unit or _unit_from_env() + + manifest, ok = build_manifest(args.stage, Path(run_dir), unit, args.stage_dir) + args.manifest.parent.mkdir(parents=True, exist_ok=True) + args.manifest.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") + + for runner, r in manifest["runners"].items(): + tag = {"complete": "OK", "warn": "warn", "below_floor": "<-- BELOW floor"} + print(f"[completeness] {runner}: {r['found']}/{r['expect']} " + f"(floor {r['floor']}) {tag[r['status']]}", file=sys.stderr) + print(f"[completeness] {args.stage} {unit}: {manifest['status']} " + f"-> {args.manifest}", file=sys.stderr) + for f in manifest["failures"]: + for reason in f["reasons"]: + print(f"[completeness] {f['runner']}: {reason}", file=sys.stderr) + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/workflow/scripts/run_report.py b/workflow/scripts/run_report.py index 683433ecb..05ca92478 100644 --- a/workflow/scripts/run_report.py +++ b/workflow/scripts/run_report.py @@ -1,70 +1,169 @@ #!/usr/bin/env python3 -"""Standalone run report — NOT a DAG node. - -A report rule that declares all tiles' outputs as inputs is a descendant of every -job, so any single hard failure (under --keep-going) poisons its cone and the -report never runs — the exact scenario it exists for (finding 8). So this is a -plain script: it disk-scans the tiles/ and exp/ trees against the count-floor -table (completeness.py) and the index, and emits run_report.json. It runs -automatically via the Snakefile's onsuccess/onerror hooks, and by hand any time -(``sp report``), mid-run included. - -It distinguishes whole-exposure absence (all runners missing — a real gap or a -deletion/forest bug) from tolerated per-CCD attrition (counts between floor and -expect), so a future deletion bug cannot be silently absorbed (finding 5). - -Attrition is counted at file granularity, not unit granularity: each stage's -``products`` block aggregates found-vs-expected file counts per runner across -all passing units, with a per-unit shortfall map where files are missing. -``warn`` runners (psfex_interp) are exempt from *failing* a unit, not from -being *reported* — the P0 validation found 46/760 psfex_interp CCDs missing -(6%) behind a unit-level "zero attrition" claim, which is the failure mode -this granularity exists to prevent. +"""``sp report`` — the run's success/failure tables, read from the manifests. + +NOT a DAG node. A report rule that declared all tiles' outputs as inputs would be +a descendant of every job, so one hard failure under --keep-going would poison +its cone and the report would never run — the exact scenario it exists for. So it +is a plain script, runnable at any time including mid-run; the Snakefile's +onsuccess/onerror hooks call it so every invocation ends with one. + +It reads two things and nothing else (PRD D3): + + * the **index** (``run_index.sqlite``) — the units the run declared, and the + tile->exposure edges that let an exposure failure be blamed on the tiles it + blocks; + * the **manifests** — one per unit per stage, written by + ``completeness.py check``, carrying per-runner found/expect/floor and + log-scraped failure reasons. + +No disk scanning: counting products is the *check's* job, done once at the moment +the products were fresh. A unit with no manifest for a stage is "not run" — which +is a real and distinct answer from "ran and produced nothing". + +Manifests are discovered by glob (``tiles/**/manifests/*.json``), not by +constructed path: the unit stores are sharded (``tiles///``) and the +sharding depth is not this script's business. """ import argparse import json import sqlite3 import sys +from collections import defaultdict from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parent)) -from completeness import COMPLETENESS, check_floor # noqa: E402 - -# stage -> (level, run_sp_ dir name) -STAGE_DIR = { - "tile_get_images": ("tile", "run_sp_tile_Git"), - "tile_uncompress": ("tile", "run_sp_tile_Uz"), - "tile_find_exposures": ("tile", "run_sp_tile_Fe"), - "exp_get_images": ("exp", "run_sp_exp_Gie"), - "exp_split": ("exp", "run_sp_exp_Sp"), - "exp_mask": ("exp", "run_sp_exp_Ma"), - "exp_psf": ("exp", "run_sp_exp_SxSePsfPi"), - "tile_merge_headers": ("tile", "run_sp_tile_Mh_exp"), - "tile_mask": ("tile", "run_sp_tile_Ma"), - "tile_detect": ("tile", "run_sp_tile_Sx"), - "tile_vignets": ("tile", "run_sp_tile_PiViVi"), - "tile_merge_cats": ("tile", "run_sp_Ms"), - "tile_make_cat": ("tile", "run_sp_Mc"), -} - - -def classify(stage, run_dir): - """'absent' | 'under' | 'attrition' | 'complete' for one unit's stage.""" - if not run_dir.is_dir(): - return "absent", [] - ok, details = check_floor(stage, run_dir) - if not ok: - # all mandatory runners empty -> whole-unit absence; else below floor - mand = [d for d in details if not d[4]] - if mand and all(n == 0 for _, n, *_ in mand): - return "absent", details - return "under", details - # warn runners are exempt from FAILING a unit (check_floor), not from - # attrition reporting — psfex_interp is precisely the runner that attrits. - if any(n < expect for _, n, _, expect, _ in details): - return "attrition", details - return "complete", details +# Stage order per level — the report's column order, and the definition of +# "expected" (a declared unit with no manifest for one of these is not run). +TILE_STAGES = ["tile_get_images", "tile_uncompress", "tile_find_exposures", + "tile_merge_headers", "tile_detect", "tile_vignets", + "tile_ngmix", "tile_merge_cats", "tile_make_cat"] +EXP_STAGES = ["exp_get_images", "exp_split", "exp_mask", "exp_psf"] + +STATUSES = ("complete", "warn", "failed", "not_run") + + +def load_manifests(run_dir: Path, sub: str) -> dict: + """``{unit: {stage: manifest}}`` for one store (``tiles`` or ``exp``). + + The unit key is the manifest dir's *parent directory name* — shard-depth + agnostic, and the only form that joins to the index (the manifest's own + ``unit`` field carries ``SP_UNIT_NUM``'s dashed form, ``210-282``, which is + not the index's ``210.282``). The stage comes from the manifest body, never + the filename: ngmix chunks share a stage under per-chunk filenames, and they + collapse to one worst-case entry. + """ + out: dict = defaultdict(dict) + for path in sorted((run_dir / sub).glob("**/manifests/*.json")): + try: + m = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError) as exc: + print(f"[run_report] unreadable manifest {path}: {exc}", file=sys.stderr) + continue + unit = path.parent.parent.name + stage = m.get("stage", path.stem) + prev = out[unit].get(stage) + # Worst status wins when several manifests share a stage (ngmix chunks). + rank = lambda d: STATUSES.index(d.get("status")) if d.get("status") in STATUSES else len(STATUSES) # noqa: E731 + if prev is None or rank(m) > rank(prev): + out[unit][stage] = m + return out + + +def shortfalls(m: dict) -> dict: + """``{runner: (found, expect, floor)}`` for every runner under expect.""" + return {r: (d["found"], d["expect"], d["floor"]) + for r, d in m.get("runners", {}).items() if d["found"] < d["expect"]} + + +def reasons(m: dict) -> list: + """Flattened failure reasons, runner-tagged, for the report's why column.""" + out = [] + for f in m.get("failures", []): + head = f"{f['runner']} {f['found']}/{f.get('expect', '?')} (floor {f['floor']})" + out += [f"{head}: {r}" for r in f["reasons"]] or [head] + return out + + +def tally_level(units, stages, manifests) -> dict: + """Per-stage counts + named unit lists, for one level.""" + per_stage = {} + for stage in stages: + t = {"complete": 0, "warn": [], "failed": [], "not_run": []} + agg = defaultdict(lambda: {"found": 0, "expect": 0, "by_unit": {}}) + for u in units: + m = manifests.get(u, {}).get(stage) + if m is None: + t["not_run"].append(u) + continue + status = m.get("status", "failed") + status = status if status in ("complete", "warn") else "failed" + if status == "complete": + t["complete"] += 1 + else: + t[status].append(u) + if status == "failed": + # Failed units are named above, never folded into the attrition + # aggregate: a whole-unit failure is not per-CCD attrition, and + # mixing them hides real deletion bugs behind a big denominator. + continue + for runner, d in m.get("runners", {}).items(): + a = agg[runner] + a["found"] += d["found"] + a["expect"] += d["expect"] + if d["found"] < d["expect"]: + a["by_unit"][u] = d["expect"] - d["found"] + for a in agg.values(): + if not a["by_unit"]: + del a["by_unit"] + t["products"] = dict(agg) + per_stage[stage] = t + return per_stage + + +def unit_rows(units, stages, manifests) -> list: + """One row per non-clean unit: its first bad stage, shortfalls, why.""" + rows = [] + for u in units: + got = manifests.get(u, {}) + bad = [s for s in stages + if got.get(s) is None or got[s].get("status") != "complete"] + if not bad: + continue + stage = bad[0] + m = got.get(stage) + rows.append({ + "unit": u, + "stage": stage, + "status": "not_run" if m is None else m.get("status", "failed"), + "shortfalls": shortfalls(m) if m else {}, + "reasons": reasons(m) if m else [], + "n_bad_stages": len(bad), + }) + return rows + + +def print_table(title, rows, limit=25): + print(f"\n{title} ({len(rows)} affected)") + if not rows: + print(" — none") + return + print(f" {'unit':<14} {'stage':<20} {'status':<8} why") + for r in rows[:limit]: + short = ", ".join(f"{k} {v[0]}/{v[1]}" for k, v in r["shortfalls"].items()) + why = (r["reasons"][0] if r["reasons"] else short) or "-" + print(f" {r['unit']:<14} {r['stage']:<20} {r['status']:<8} {why[:90]}") + if len(rows) > limit: + print(f" … and {len(rows) - limit} more (see the JSON report)") + + +def print_stage_table(title, per_stage, n_units): + print(f"\n{title} ({n_units} units declared)") + print(f" {'stage':<20} {'ok':>6} {'warn':>6} {'fail':>6} {'not run':>8} attrition") + for stage, t in per_stage.items(): + att = [f"{r} {a['found']}/{a['expect']}" + for r, a in t["products"].items() if a["found"] < a["expect"]] + print(f" {stage:<20} {t['complete']:>6} {len(t['warn']):>6} " + f"{len(t['failed']):>6} {len(t['not_run']):>8} {', '.join(att)[:60]}") def main() -> None: @@ -73,66 +172,66 @@ def main() -> None: p.add_argument("--index", required=True, type=Path) p.add_argument("--status", default="manual") p.add_argument("--out", type=Path, default=None) + p.add_argument("--limit", type=int, default=25, + help="rows per stdout table (the JSON report is complete)") args = p.parse_args() - tiles, exps = [], [] + tiles, exps, tile_exp = [], [], defaultdict(list) if args.index.exists(): con = sqlite3.connect(args.index) - tiles = [r[0] for r in con.execute("SELECT tile_id FROM tiles")] - exps = [r[0] for r in con.execute("SELECT exp_id FROM exposures")] + tiles = [r[0] for r in con.execute("SELECT tile_id FROM tiles ORDER BY 1")] + exps = [r[0] for r in con.execute("SELECT exp_id FROM exposures ORDER BY 1")] + for tile_id, exp_id in con.execute("SELECT tile_id, exp_id FROM tile_exposures"): + tile_exp[tile_id].append(exp_id) con.close() + else: + print(f"[run_report] no index at {args.index} — reporting manifests only", + file=sys.stderr) + + tile_m = load_manifests(args.run_dir, "tiles") + exp_m = load_manifests(args.run_dir, "exp") + tiles = tiles or sorted(tile_m) + exps = exps or sorted(exp_m) missing_json = args.index.parent / "missing.json" missing = json.loads(missing_json.read_text()) if missing_json.exists() else [] - report = {"status": args.status, "n_tiles": len(tiles), "n_exposures": len(exps), - "missing_tiles": missing, "stages": {}} - for stage, (level, prefix) in STAGE_DIR.items(): - if stage not in COMPLETENESS: - continue - units = tiles if level == "tile" else exps - sub = "tiles" if level == "tile" else "exp" - tally = {"complete": 0, "attrition": 0, "under": [], "absent": []} - # File-level aggregate per runner over PASSING units (complete/attrition); - # under/absent units are enumerated by name above, not folded in here — - # mixing them would double-count whole-unit absence as attrition. - products = {r: {"found": 0, "expect": 0, "by_unit": {}} - for r in COMPLETENESS[stage]} - for u in units: - run_dir = args.run_dir / sub / u / "output" / prefix - verdict, details = classify(stage, run_dir) - if verdict in ("complete", "attrition"): - tally[verdict] += 1 - for runner, n, _, expect, _ in details: - agg = products[runner] - agg["found"] += n - agg["expect"] += expect - if n < expect: - agg["by_unit"][u] = expect - n - else: - tally[verdict].append(u) - for agg in products.values(): - if not agg["by_unit"]: - del agg["by_unit"] - tally["products"] = products - report["stages"][stage] = tally - - finals = [t for t in tiles - if (args.run_dir / "tiles" / t / f"final_cat-{t}.fits").exists()] - report["final_cats"] = {"present": len(finals), "of": len(tiles), - "missing": [t for t in tiles if t not in finals]} + report = { + "status": args.status, + "n_tiles": len(tiles), "n_exposures": len(exps), + "missing_tiles": missing, + "tile_stages": tally_level(tiles, TILE_STAGES, tile_m), + "exp_stages": tally_level(exps, EXP_STAGES, exp_m), + "tiles": unit_rows(tiles, TILE_STAGES, tile_m), + "exposures": unit_rows(exps, EXP_STAGES, exp_m), + } + + # Blame propagation: an incomplete exposure blocks every tile that reads it. + # Without this, a tile stalled at tile_vignets looks like its own failure. + bad_exp = {r["unit"] for r in report["exposures"]} + blocked = {t: sorted(set(tile_exp.get(t, [])) & bad_exp) for t in tiles} + report["tiles_blocked_by_exposures"] = {t: e for t, e in blocked.items() if e} + + done = report["tile_stages"]["tile_make_cat"]["complete"] + report["final_cats"] = {"present": done, "of": len(tiles)} out = args.out or (args.index.parent / "run_report.json") - out.write_text(json.dumps(report, indent=2)) - print(f"[run_report] {report['final_cats']['present']}/{len(tiles)} final " - f"cats -> {out}", file=sys.stderr) - for stage, tally in report["stages"].items(): - for runner, agg in tally["products"].items(): - if agg["found"] < agg["expect"]: - pct = 100 * (agg["expect"] - agg["found"]) / agg["expect"] - print(f"[run_report] attrition {stage}/{runner}: " - f"{agg['found']}/{agg['expect']} files (-{pct:.1f}%) " - f"across {len(agg['by_unit'])} units", file=sys.stderr) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n") + + print(f"[run_report] status={args.status} {done}/{len(tiles)} final cats" + + (f" ({len(missing)} tiles missing exposure lists)" if missing else "")) + print_stage_table("EXPOSURES", report["exp_stages"], len(exps)) + print_stage_table("TILES", report["tile_stages"], len(tiles)) + print_table("exposures not complete", report["exposures"], args.limit) + print_table("tiles not complete", report["tiles"], args.limit) + nb = report["tiles_blocked_by_exposures"] + if nb: + print(f"\ntiles waiting on incomplete exposures ({len(nb)})") + for t, e in list(nb.items())[:args.limit]: + print(f" {t:<14} {', '.join(e[:6])}" + + (f" (+{len(e) - 6})" if len(e) > 6 else "")) + print(f"\n[run_report] -> {out}") if __name__ == "__main__": From 83d26f2088447c5989060b6e5c4bac3b23c39c6d Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 30 Jul 2026 22:36:11 -0400 Subject: [PATCH 14/41] =?UTF-8?q?feat(orchestration):=20the=20rules=20rewr?= =?UTF-8?q?ite=20=E2=80=94=20manifests=20as=20DAG=20currency,=20sharded=20?= =?UTF-8?q?stores,=20parse-time=20index,=20native=20scattergather=20(#848?= =?UTF-8?q?=20D1=E2=80=93D5)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sp_rule.py is deleted; its responsibilities dissolve into the design: config normalization -> the committed S2 configs; log sync -> dead (zero 'last:'); unit furniture -> inline shell (tile_numbers.txt, pseudo-Fe exp_numbers with the ORIGINAL exposure name, star-cat symlinks); count floor -> completeness.py check writing the manifest each rule declares as its only output. SP_UNIT_NUM carries the leading-dash dashed form; the rules do the transform. Stores shard to tiles/<2ch>// and exp/<2ch>//; build_index accumulates (no DROP TABLE) and is built at parse time of the compute invocation; bin/sp reduces to run (two invocations) + report + cancel. ngmix scatters via the scattergather knob with in-job closed ID ranges (scripts/ngmix_range.py); retries with attempt-scaled mem on transient- exposed rules; protected() dropped; temp(directory()) rides as a scoped secondary output on the vignette store and chunk dirs — the one exception to no-directory-outputs, so native temp() reclaims intra-tile bulk. Deviations proven by implementation: merge_sep_cats INPUT_DIR must stay relative (MergeSep re.sub's the first '1' in the path — absolute sharded paths break chunk discovery); failed jobs need keep-incomplete in the profile or Snakemake deletes the failure manifests sp report exists to read (-> S6). Dry-runs: prepare 13 jobs, compute 141 on the 210/211 quad (19 exposures, 8 ngmix chunks); --set-scatter ngmix=1 -> 113. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01B5KZRTAZKSBAnpdDhE7yzK --- workflow/Snakefile | 293 ++++++++---- workflow/bin/sp | 81 ++-- workflow/config.yaml | 14 +- .../config/cfis/config_merge_sep_cats.ini | 11 +- workflow/rules/exposure.smk | 108 +++-- workflow/rules/prepare.smk | 85 ++-- workflow/rules/tile.smk | 269 ++++++----- workflow/scripts/build_forest.py | 13 +- workflow/scripts/build_index.py | 61 ++- workflow/scripts/ngmix_range.py | 55 +++ workflow/scripts/sp_rule.py | 419 ------------------ 11 files changed, 644 insertions(+), 765 deletions(-) create mode 100644 workflow/scripts/ngmix_range.py delete mode 100644 workflow/scripts/sp_rule.py diff --git a/workflow/Snakefile b/workflow/Snakefile index 5d93a1810..b707fdc17 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -1,24 +1,31 @@ """ShapePipe real-data orchestration — Snakemake workflow. -Design / rationale: CosmoStat/shapepipe#848 (the living PRD). - -Execution is honestly THREE static invocations, chained by ``workflow/bin/sp`` -so the UX stays one command (``sp prepare`` / ``sp run``): - - snakemake --profile profiles/nibi prepare_tiles # Git_vos + Uz + Fe (keep-going) - python workflow/scripts/build_index.py ... # PLAIN SCRIPT, not a DAG node - snakemake --profile profiles/nibi prepare_exposures # Gie_vos + star cats - snakemake --profile profiles/nibi # the main compute DAG - -The run index is loaded ONCE here at parse time into plain dicts; it is never a -rule input, so appending tiles and rebuilding it changes which jobs exist without -invalidating completed work. build_index.py is a plain script *between* -prepare_tiles and prepare_exposures (the exposure job set is data-derived from -the tiles' find_exposures output, so it cannot be scheduled in the same static -DAG that produces it). +Design / rationale: CosmoStat/shapepipe#848 (the living PRD), D1-D5. + +`sp run` is TWO snakemake invocations over this one Snakefile: + + SP_PHASE=prepare snakemake prepare_all_tiles # Git -> Uz -> Fe, per tile + SP_PHASE=compute snakemake all # everything else + +They are two because the exposure job set is *data-derived*: it comes from the +tiles' find_exposures output, and a Snakemake DAG is fixed at parse time. The +join between them is the tile<->exposure index, built HERE at parse time of +invocation 2 (build_index.build(), imported — there is no `sp index` verb) and +loaded into plain dicts. The index is never a rule input, so appending tiles and +rebuilding it changes which jobs exist without invalidating completed work; it +ACCUMULATES across invocations, so a later clean_exposure (S5) sees every +consuming tile of the whole campaign, not just this tile list. + +The atom (D2): one rule == one `shapepipe_run` on one unit; its single declared +output is its MANIFEST (`/manifests/.json`), written by +`completeness.py check`. Product files are not declared — a missing CCD is often +legitimate, and at DR6 scale per-CCD declaration means millions of paths. """ +import hashlib +import os import sqlite3 +import sys from pathlib import Path # Resolved relative to THIS file, not the working directory: snakemake runs with @@ -32,16 +39,49 @@ container: config["container"] # --- paths ----------------------------------------------------------------- RUN_DIR = Path(config["run_dir"]) -CONFIG_SRC = Path(config["config_src"]) STAR_CATS = Path(config["star_cats"]) INDEX_DB = Path(config["index_db"]) SCRIPTS = Path(workflow.basedir) / "scripts" -NGMIX_CHUNKS = config.get("ngmix_chunks", 8) +# The config chain is the repo's own committed dir BY CONSTRUCTION (D2): the +# configs and the rules that set the env vars they interpolate are one artefact +# and must version together. Hence no `config_src` knob. +CONFIG_DIR = Path(workflow.basedir) / "config" / "cfis" + +sys.path.insert(0, str(SCRIPTS)) +import build_index # noqa: E402 +from completeness import STAGE_DIR # noqa: E402 + +# `prepare` suppresses the missing-tile threshold (tiles are still being +# prepared) and the report hooks. Unset == compute (a bare hand-run of `all`). +PHASE = os.environ.get("SP_PHASE", "compute") with open(config["tile_list"]) as f: TILES = [ln.strip() for ln in f if ln.strip()] -# --- parse-time index load (compute phase only) ---------------------------- +# --- ngmix scatter (D4) ---------------------------------------------------- +# Native directive: `--set-scatter ngmix=N` overrides it, N=1 degenerates to one +# ngmix job per tile. We take the count and drive our own integer `chunk` +# wildcard rather than snakemake's `{scatteritem}` ("3-of-8") token, because the +# chunk number is not ours alone: it names the run dir +# (`run_sp_tile_ngmix_Ngu`, from the template's RUN_NAME), and +# merge_sep_cats derives chunks 2..N from chunk 1's path by substituting the +# first "1" — which only works for bare integers. +scattergather: + ngmix=int(config.get("ngmix_chunks", 8)) + +NGMIX_CHUNKS = workflow._scatter["ngmix"] + +# --- parse-time index build + load (D1) ------------------------------------ +# Invocation 2's parse IS the index build. Tiles whose find_exposures output is +# absent land in index/missing.json and are dropped from the DAG; the build is +# fatal only above SP_MISSING_THRESHOLD (and never in the prepare phase, where +# absent Fe output is the normal state). +if any(build_index.exp_list_path(RUN_DIR, t).exists() for t in TILES): + build_index.build( + TILES, RUN_DIR, INDEX_DB, + missing_threshold=(None if PHASE == "prepare" + else float(os.environ.get("SP_MISSING_THRESHOLD", "0.0")))) + # EXP: exposure base-id -> original name (2605805 -> 2605805p; the name goes # verbatim into the fabricated per-unit exp_numbers list so get_images matches # .fits.fz in the store). TILE_EXP: tile -> [exp_ids]. @@ -49,93 +89,186 @@ EXP, TILE_EXP = {}, {} if INDEX_DB.exists(): _con = sqlite3.connect(INDEX_DB) EXP = dict(_con.execute("SELECT exp_id, name FROM exposures")) - for tile, exp in _con.execute("SELECT tile_id, exp_id FROM tile_exposures"): - TILE_EXP.setdefault(tile, []).append(exp) + for _tile, _exp in _con.execute("SELECT tile_id, exp_id FROM tile_exposures"): + TILE_EXP.setdefault(_tile, []).append(_exp) _con.close() +# Tiles this run can actually compute: declared AND indexed. The index spans the +# campaign, so it is intersected with the declared list, not used as it. +TILES_READY = [t for t in TILES if t in TILE_EXP] + wildcard_constraints: - tile = r"\d{3}\.\d{3}", - exp = r"\d{6,7}", + tile = r"\d{3}\.\d{3}", + exp = r"\d{6,7}", + shard = r"\d{2}", chunk = r"\d+", -# --- shared wrapper invocation --------------------------------------------- -def sp_rule(stage, config_name, level, unit, *, isolate=True, exp_forest=None, - extra=""): - """Build the shell command that runs one unit through sp_rule.py. - - NUMBER_LIST is injected by default (tile-scheme stages + exp split); pass - ``isolate=False`` for get_images / exp_mask / exp_psf, whose per-CCD/download - numbering must not be constrained. ``--threads {threads}`` is appended in - each rule's ``shell:`` string (NOT here: Snakemake formats the shell string - once, so a ``{threads}`` inside params.cmd would survive literally) and - becomes SMP_BATCH_SIZE (fork width == cpus_per_task). - """ - cmd = ( - f"python {SCRIPTS}/sp_rule.py --stage {stage} --config {config_name} " - f"--unit {unit} --level {level} --run-dir {RUN_DIR} " - f"--config-src {CONFIG_SRC} --star-cats {STAR_CATS} " - f"--script-hash {SCRIPT_HASH}" - ) - if not isolate: - cmd += " --no-isolate" - if exp_forest: - cmd += f" --exp-forest {exp_forest}" - if extra: - cmd += " " + extra - return cmd +# --- the sharded stores (D2) ---------------------------------------------- +# tiles/<2-char prefix>// and exp/<2-char prefix>// — no directory +# exceeds ~1k entries at full-UNIONS scale. Rules carry the shard as its own +# wildcard because an output pattern cannot compute it; every path the DAG uses +# is built by these helpers, so a mismatched (shard, id) pair is never requested. +TILE_DIR = str(RUN_DIR / "tiles" / "{shard}" / "{tile}") +EXP_DIR = str(RUN_DIR / "exp" / "{shard}" / "{exp}") def tile_dir(tile): - return RUN_DIR / "tiles" / tile + return f"{RUN_DIR}/tiles/{tile[:2]}/{tile}" def exp_dir(exp): - return RUN_DIR / "exp" / exp - -# Content hash of the wrapper scripts, computed once at parse time. Exposed as a -# param on every rule (via sp_rule() and tile_exp_forest) so the default -# rerun-triggers' *params* trigger covers script edits: the `code` trigger only -# hashes the rule's own shell string, NOT external scripts it calls — without -# this, a wrapper/forest-script fix silently leaves stale outputs in place -# (bitten live: the forest-shard fix rebuilt nothing on rerun). -import hashlib as _hashlib -SCRIPT_HASH = _hashlib.md5(b"".join( - (SCRIPTS / n).read_bytes() - for n in ("sp_rule.py", "completeness.py", "build_forest.py"))).hexdigest()[:12] + return f"{RUN_DIR}/exp/{exp[:2]}/{exp}" + +def tile_manifest(tile, stage): + return f"{tile_dir(tile)}/manifests/{stage}.json" + +def exp_manifest(exp, stage): + return f"{exp_dir(exp)}/manifests/{stage}.json" + +def forest_dir(tile): + return f"{tile_dir(tile)}/exp_forest" + +def final_cat(tile): + return f"{tile_dir(tile)}/final_cat-{tile}.fits" + +def unit_num(unit): + """$SP_UNIT_NUM: ShapePipe's image-number convention, dot -> dash, leading + dash (tile ``210.282`` -> ``-210-282``; exposure ``2605805`` -> ``-2605805``). + The RULES do this transform; the configs just interpolate $SP_UNIT_NUM into + NUMBER_LIST (the `set_config_number_list` mechanism that replaced the retired + -e/--exclusive flag, #746).""" + return "-" + unit.replace(".", "-") + +# Content hash of completeness.py, computed once at parse time and carried as a +# param on every rule: the default rerun-triggers' `code` trigger hashes only the +# rule's own shell string, NOT external scripts it calls — without this, a fix to +# the count table silently leaves stale manifests in place (bitten live). Scoped +# to completeness.py alone, the one script every shell line runs; build_forest.py +# gets its own hash, on the forest rule only. +SCRIPT_HASH = hashlib.md5((SCRIPTS / "completeness.py").read_bytes()).hexdigest()[:12] +FOREST_HASH = hashlib.md5((SCRIPTS / "build_forest.py").read_bytes()).hexdigest()[:12] + +# --- the shell every rule runs (D2) ---------------------------------------- + +_THREAD_CAPS = " ".join( + f"{k}={v}" for k, v in ( + ("OMP_NUM_THREADS", 1), ("OPENBLAS_NUM_THREADS", 1), + ("MKL_NUM_THREADS", 1), ("NUMEXPR_NUM_THREADS", 1), + ("MALLOC_ARENA_MAX", 2), ("MALLOC_TRIM_THRESHOLD_", 0))) + + +def unit_pre(stage, level, unit, *, exp_name=None, forest=None, env=None, + pre_run=()): + """The unit-furniture + environment prologue, as bash. + + Returned as a rule ``params`` value, NEVER inlined into the ``shell:`` + string: snakemake formats a shell string ONCE, so a ``{output}``/``{threads}`` + placeholder inside a params value would survive literally — and, conversely, + the literal ``${SP_NGMIX_CHUNK}`` braces this prologue needs would blow up + that formatting if they lived in the shell string. Params values are + substituted after formatting, so both hazards go away together. + + What it materialises (the proven v2.0 isolation-by-work-dir-content, NOT + -e/--exclusive): + * ``output/`` and ``manifests/``; + * tile: ``tile_numbers.txt`` (dot format — what get_images reads); + * exposure: a fabricated pseudo-Fe ``exp_numbers-000-000.txt`` holding the + ORIGINAL exposure name from the index (``2605805p``), so get_images + matches ``.fits.fz`` in the store — the bare base id matches + nothing. Written UNCONDITIONALLY: an exists-guard once pinned a stale + pre-fix file with the bare id; + * ``star_cat_exp`` / ``star_cat_tiles`` dir symlinks into the shared + pre-generated pool (the mask configs read them as INPUT_DIRs). + There is no per-unit ``cfis`` symlink any more: $SP_CONFIG points straight at + the committed config dir. + + Finally it ``rm -rf``s this stage's own fixed run dir — ShapePipe's + FileHandler raises on an existing run dir, and it is how a rerun never sees + stale products (D2: the job clears its run dir at start). + """ + work = tile_dir(unit) if level == "tile" else exp_dir(unit) + _, subdir = STAGE_DIR[stage] + lines = [ + "set -euo pipefail", + f"export SP_RUN='{work}'", + f"export SP_UNIT_NUM='{unit_num(unit)}'", + f"export SP_CONFIG='{CONFIG_DIR}'", + # Also set via apptainer-args in the profile; kept here so a hand-run of + # this same line outside snakemake behaves identically. + f"export {_THREAD_CAPS}", + 'mkdir -p "$SP_RUN/output" "$SP_RUN/manifests"', + ] + if forest: + lines.append(f"export SP_EXP='{forest}'") + for k, v in (env or {}).items(): + lines.append(f"export {k}='{v}'") + + if level == "tile": + lines.append(f"printf '%s\\n' '{unit}' > \"$SP_RUN/tile_numbers.txt\"") + else: + fe = "$SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output" + lines += [f'mkdir -p "{fe}"', + f"printf '%s\\n' '{exp_name or unit}' > \"{fe}/exp_numbers-000-000.txt\""] + + for name, sub in (("star_cat_exp", "exp"), ("star_cat_tiles", "tiles")): + src = STAR_CATS / sub + lines.append(f"[ -d '{src}' ] && ln -sfn '{src}' \"$SP_RUN/{name}\" || true") + + lines += list(pre_run) + lines += [f'rm -rf "$SP_RUN/output/{subdir}"', 'cd "$SP_RUN"'] + return "\n".join(lines) + + +def sp_shell(stage, config_name): + """The rule's shell string: prologue, one shapepipe_run, one completeness check. + + ``{threads}`` and ``{output}`` are placeholders HERE and nowhere else (see + unit_pre). ``-b {threads}`` makes SMP fork width and cpus_per_task one number + by construction (D4). + + The check runs even when shapepipe_run failed — the manifest is the only + thing `sp report` reads, so a failed unit must still leave a record of why. + """ + return ( + "{params.pre}\n" + "rc=0\n" + f"shapepipe_run -c \"$SP_CONFIG/{config_name}\" -b {{threads}} || rc=$?\n" + f"python {SCRIPTS}/completeness.py check {stage} {{output.manifest}} || rc=1\n" + "exit $rc\n" + ) + include: "rules/prepare.smk" include: "rules/exposure.smk" include: "rules/tile.smk" # --- top-level targets ------------------------------------------------------ -# Only aggregation targets are localrules (no-ops under the local executor, but -# they keep the compute chain group-compatible: no mid-chain localrules). -localrules: all, prepare_tiles, prepare_exposures +# Only the aggregation targets are localrules: a mid-chain localrule would break +# `group:` fusion of the compute chains (D4). +localrules: all, prepare_all_tiles rule all: input: - expand(str(RUN_DIR / "tiles/{tile}/final_cat-{tile}.fits"), tile=TILES) - -# Phase A.1 — per-tile static DAG (download + find_exposures). keep-going so -# tile failures are independent; build_index.py runs next as a plain script. -rule prepare_tiles: - input: - expand(str(RUN_DIR / "tiles/{tile}/output/run_sp_tile_Fe"), tile=TILES) + [final_cat(t) for t in TILES_READY] -# Phase A.2 — per-exposure static DAG (download + star cats), parseable now the -# index exists. Star cats are re-keyed per unit (see prepare.smk). -rule prepare_exposures: +# Invocation 1 — the static per-tile DAG, known from the tile list alone. +# keep-going makes tile failures independent; the ones that lose their exposure +# list are dropped by the index build at invocation 2's parse. +rule prepare_all_tiles: input: - expand(str(RUN_DIR / "exp/{exp}/output/run_sp_exp_Gie"), exp=sorted(EXP)) + [tile_manifest(t, "tile_find_exposures") for t in TILES] # --- report hooks ----------------------------------------------------------- # run_report is NOT a DAG node (a descendant of every job would be poisoned by # any hard failure — the exact case it exists for). It is a standalone script, -# emitted automatically at the end of every invocation, runnable any time. +# emitted automatically at the end of the COMPUTE invocation, runnable any time +# via `sp report`. def _report(status): shell(f"python {SCRIPTS}/run_report.py --run-dir {RUN_DIR} " f"--index {INDEX_DB} --status {status} || true") -onsuccess: - _report("success") +if PHASE == "compute": + + onsuccess: + _report("success") -onerror: - _report("error") + onerror: + _report("error") diff --git a/workflow/bin/sp b/workflow/bin/sp index d1f105548..1ead411a3 100755 --- a/workflow/bin/sp +++ b/workflow/bin/sp @@ -1,23 +1,23 @@ #!/usr/bin/env bash -# sp — the committed launcher for the ShapePipe Snakemake workflow. +# sp — the committed launcher for the ShapePipe Snakemake workflow (PRD #848 D1). # -# Removes the shell-state dependencies the review flagged (finding 20): it loads -# the apptainer module (Snakemake resolves `apptainer` via PATH at job runtime) -# and runs the Snakemake that lives on the shared /project venv (must be on a -# shared FS — the executor re-invokes it inside jobs). One entry point, so a -# fresh tmux / a restart after a crash always launches with the right state. +# Two verbs, nothing else: # -# Usage: -# sp prepare # prepare_tiles -> build_index.py -> prepare_exposures -# sp prepare_tiles # phase A.1 only -# sp index # build_index.py only (plain script, threshold-gated) -# sp prepare_exposures # phase A.2 only -# sp run [ARGS...] # the main compute DAG -# sp rerun # forced recompute of protected/finished products -# sp cancel # scancel this workflow's jobs (name sweep) before --unlock -# sp clean-exposures [--apply] # index-driven exposure-store reclaim -# sp report # emit run_report.json now (mid-run ok) -# Extra args after the subcommand pass through to snakemake. +# sp run [ARGS...] bring the products on disk up to date with the tile list. +# Two snakemake invocations over one Snakefile: +# 1. PREPARE snakemake prepare_all_tiles +# 2. COMPUTE snakemake all <- its PARSE builds the index +# ARGS (--jobs, -n, --forcerun, ...) pass through to BOTH. +# sp report [ARGS...] emit run_report.json now (mid-run is fine). +# +# Anything else is passed straight through to snakemake with the same profile and +# state dir (the escape hatch: `sp --unlock`, `sp exp_psf ...`, `sp --dag`). +# +# It also loads the apptainer module (snakemake resolves `apptainer` via PATH at +# job runtime) and activates the snakemake venv on the shared /project FS (the +# executor re-invokes it inside jobs, so it cannot live on a node-local path). +# One entry point, so a fresh tmux or a restart after a crash always launches +# with the right state. set -euo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # workflow/ @@ -34,35 +34,40 @@ source "$VENV/bin/activate" # Minimal scalar reader for workflow/config.yaml (key: value, no nesting). cfg() { sed -n "s/^$1:[[:space:]]*//p" "$CONFIG" | head -1; } RUN_DIR="$(cfg run_dir)"; INDEX_DB="$(cfg index_db)" + # Snakemake state (.snakemake: metadata, locks, incomplete markers) lives NEXT TO -# THE RUN on /scratch, never on /project (hard 27/27 TiB group quota killed a -# metadata write mid-run live). --directory only moves state: all data paths are +# THE RUN on /scratch, never on /project (a hard 27/27 TiB group quota killed a +# metadata write mid-run, live). --directory only moves state: all data paths are # absolute, and the Snakefile resolves its own configfile. STATE_DIR="${SP_STATE_DIR:-${RUN_DIR}-state}"; mkdir -p "$STATE_DIR" -TILE_LIST="$(cfg tile_list)"; MISSING_THRESH="${SP_MISSING_THRESHOLD:-0.0}" + +# SP_MISSING_THRESHOLD gates the compute parse's index build: the fraction of +# declared tiles allowed to be missing their exposure list (default 0.0). +export SP_MISSING_THRESHOLD="${SP_MISSING_THRESHOLD:-0.0}" sm() { snakemake --profile "$PROFILE" --directory "$STATE_DIR" "$@"; } -build_index() { - python "$SCRIPTS/build_index.py" --tile-list "$TILE_LIST" \ - --run-dir "$RUN_DIR" --db "$INDEX_DB" --missing-threshold "$MISSING_THRESH" -} -cmd="${1:-}"; shift || true +cmd="${1:-}" case "$cmd" in - prepare) sm prepare_tiles "$@"; build_index; sm prepare_exposures "$@" ;; - prepare_tiles) sm prepare_tiles "$@" ;; - index) build_index ;; - prepare_exposures) sm prepare_exposures "$@" ;; - run) sm "$@" ;; - rerun) sm --forcerun "$@" ;; + run) + shift + SP_PHASE=prepare sm prepare_all_tiles "$@" + SP_PHASE=compute sm all "$@" + ;; + report) + shift + python "$SCRIPTS/run_report.py" --run-dir "$RUN_DIR" --index "$INDEX_DB" \ + --status manual "$@" + ;; cancel) - run="${1:?usage: sp cancel }" + # Kept only because it is two lines: scancel this workflow's jobs by name + # before an --unlock. Not part of the design surface. + run="${2:?usage: sp cancel }" squeue --me --noheader --format='%i %j' \ | awk -v r="$run" '$2 ~ r {print $1}' | xargs -r scancel - echo "cancelled jobs matching '$run'; safe to --unlock / rerun now" ;; - clean-exposures) python "$SCRIPTS/clean_exposures.py" \ - --run-dir "$RUN_DIR" --index "$INDEX_DB" "$@" ;; - report) python "$SCRIPTS/run_report.py" \ - --run-dir "$RUN_DIR" --index "$INDEX_DB" --status manual "$@" ;; - *) echo "sp: unknown subcommand '${cmd:-}' (see the header of $0)"; exit 2 ;; + echo "cancelled jobs matching '$run'; safe to --unlock / rerun now" + ;; + *) + sm "$@" + ;; esac diff --git a/workflow/config.yaml b/workflow/config.yaml index 592ecf989..942199b14 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -13,16 +13,14 @@ tile_list: /scratch/cdaley/shapepipe-output/smk-p0/tiles4.txt # The container every job runs inside (apptainer software-deployment in the profile). container: /project/def-mjhudson/cdaley/containers/shapepipe-develop-runtime.sif -# Where products land ($SP_RUN root). Per-unit work-dir forest is built under here: -# /tiles/// and /exp/// +# Where products land ($SP_RUN root). The sharded per-unit stores live under here: +# /tiles/<2-char prefix>// and /exp/// run_dir: /scratch/cdaley/shapepipe-output/smk-p0 -# The canonical config chain: the committed, wrapper-free configs (#848 D2 config -# sweep) -- RUN_DATETIME=False, fixed INPUT_DIRs, NUMBER_LIST=$SP_UNIT_NUM baked -# in. example/cfis stays untouched for the bash flow; runs/p3-batch1/cfis's real -# divergences (real data paths, external star cats, SAVE_BATCH memory tuning) -# were folded in -- see the config-sweep report for the full list. -config_src: /home/cdaley/shapepipe-smk/workflow/config/cfis +# There is no config_src knob: the config chain is workflow/config/cfis, resolved +# relative to the Snakefile. The configs interpolate $SP_RUN / $SP_UNIT_NUM / +# $SP_CONFIG / $SP_EXP / $NGMIX_* and the rules export them -- configs and rules +# are one artefact and must version together, so the dir is fixed by construction. # Pre-staged inputs (P3 data already on /project; get_images RETRIEVE=symlink). # Star catalogues for masking, pre-generated (network step done in prepare): diff --git a/workflow/config/cfis/config_merge_sep_cats.ini b/workflow/config/cfis/config_merge_sep_cats.ini index a033f28d5..dbaf12334 100644 --- a/workflow/config/cfis/config_merge_sep_cats.ini +++ b/workflow/config/cfis/config_merge_sep_cats.ini @@ -41,8 +41,15 @@ NUMBER_LIST = $SP_UNIT_NUM # NOTE: only chunk 1's ngmix output is listed here; merge_sep_cats_runner # derives the other chunks' paths itself from N_SPLIT_MAX and this pattern # (chunk dirs are run_sp_tile_ngmix_Ngu, k=1..N_SPLIT_MAX), all under the -# same fixed $SP_RUN/output. -INPUT_DIR = $SP_RUN/output/run_sp_tile_ngmix_Ng1u/ngmix_runner/output +# same fixed output dir. +# +# RELATIVE ON PURPOSE (do not "fix" this to $SP_RUN/...): merge_sep_cats derives +# chunk k's path with re.sub("1", str(k), input_file, 1) -- it replaces the FIRST +# "1" in the path. An absolute path under the sharded store +# (.../tiles/21/210.282/...) has digits before "Ng1u", so chunk 2 would be looked +# for in a nonexistent directory. Every rule runs shapepipe_run with cwd=$SP_RUN, +# so "./output/..." resolves and "Ng1u" carries the first "1". +INPUT_DIR = ./output/run_sp_tile_ngmix_Ng1u/ngmix_runner/output # Output directory OUTPUT_DIR = $SP_RUN/output diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index 82cdb4cd4..667906223 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -1,74 +1,90 @@ -"""Exposure chain — per exposure, keyed by exp_id (dedup is structural). +"""Exposure chain — per exposure, keyed by exp base id (dedup is structural). -Gie -> Sp -> Ma -> psf, each in the exposure's own work dir, chained by -ShapePipe's last:/run_sp_exp_* INPUT_DIR resolver. Completeness is the count -floor (completeness.py). + exp_get_images -> exp_split -> exp_mask -> exp_psf -NO temp() on these shared exposure directories: exposures overlap tiles by -construction (~7-10 tiles per exposure), so temp() would delete an exposure the -moment its forests are built and cascade destructive reruns across spatial -neighbours whenever a tile is appended (findings 4/5/6/19). The store is -persistent; space is reclaimed by the explicit index-driven `sp clean-exposures` -verb (scripts/clean_exposures.py), which deletes an exposure's intermediates -only once every consuming tile in the index has its final_cat present. +Each in the exposure's own sharded work dir, chained by manifests; every config +reads fixed ``$SP_RUN/output/run_sp_exp_*`` INPUT_DIRs, so nothing resolves a +run log. There is no `prepare_exposures` aggregation target: these chains hang +off the compute DAG (`all` <- final_cat <- tile chain <- exposure manifests). -NUMBER_LIST is injected only for exp_split (numbering scheme == exposure id); -NEVER for get_images (download) / exp_mask / exp_psf (per-CCD schemes where the -#746 startup validation would turn tolerated per-CCD attrition into a -whole-exposure hard failure). -""" +NO temp() anywhere in this file, ever (D5). Exposures overlap tiles by +construction (~7-10 tiles each), so their consumer set closes over the CAMPAIGN, +not over one invocation — reclamation here is clean_exposure's job (S5), driven +by the accumulating index. A temp() here would delete an exposure the moment +this invocation's readers finished and cascade destructive reruns across spatial +neighbours the next time a tile is appended. -EXP_OUT = str(RUN_DIR / "exp/{exp}/output") +NUMBER_LIST is set only for exp_split (its numbering scheme IS the exposure id); +never for get_images / exp_mask / exp_psf, whose per-CCD or download numbering +would make the #746 startup validation turn tolerated per-CCD attrition into a +whole-exposure hard failure. That is now a property of the committed configs +(config_exp_Sp.ini has NUMBER_LIST = $SP_UNIT_NUM; Gie/Ma/psfex have none). +""" -# Symlink/download exposure image/weight/flag; reads the fabricated pseudo-Fe -# exp_numbers-000-000.txt the wrapper drops (via last:find_exposures_runner). rule exp_get_images: output: - directory(f"{EXP_OUT}/run_sp_exp_Gie") + manifest = f"{EXP_DIR}/manifests/exp_get_images.json" params: - cmd=lambda wc: sp_rule("exp_get_images", "config_exp_Gie_vos.ini", - "exp", wc.exp, isolate=False, - extra=f"--exp-name {EXP[wc.exp]}") + pre = lambda wc: unit_pre("exp_get_images", "exp", wc.exp, + exp_name=EXP[wc.exp]), + script_hash = SCRIPT_HASH + threads: 1 + retries: 2 + resources: + mem_mb = lambda wc, attempt: 4000 * attempt, + runtime = 60 shell: - "{params.cmd} --threads {threads}" + sp_shell("exp_get_images", "config_exp_Gie.ini") -# Split multi-HDU exposure into single-CCD files (+ headers-*.npy). Bulk output; -# persistent (see module docstring on temp()). +# Split the multi-HDU exposure into single-CCD files (+ headers-*.npy, which the +# tiles' merge_headers reads). rule exp_split: input: - rules.exp_get_images.output + rules.exp_get_images.output.manifest output: - directory(f"{EXP_OUT}/run_sp_exp_Sp") + manifest = f"{EXP_DIR}/manifests/exp_split.json" params: - cmd=lambda wc: sp_rule("exp_split", "config_exp_Sp.ini", "exp", wc.exp) + pre = lambda wc: unit_pre("exp_split", "exp", wc.exp), + script_hash = SCRIPT_HASH + threads: 8 + resources: + mem_mb = lambda wc, attempt: 8000 * attempt, + runtime = 120 shell: - "{params.cmd} --threads {threads}" + sp_shell("exp_split", "config_exp_Sp.ini") -# Mask per-CCD. Star cats are NOT a per-unit input: the store is per-CCD -# (star_cat--.fits) and the mask config reads it as a DIR symlink -# (see prepare.smk) — a per-unit file input here has no producer and makes the -# DAG unbuildable the moment a new exposure is appended (caught live by the -# append-invariant test). A missing cat fails the mask count-floor loudly. rule exp_mask: input: - rules.exp_split.output, + rules.exp_split.output.manifest output: - directory(f"{EXP_OUT}/run_sp_exp_Ma") + manifest = f"{EXP_DIR}/manifests/exp_mask.json" params: - cmd=lambda wc: sp_rule("exp_mask", "config_exp_Ma_onthefly.ini", - "exp", wc.exp, isolate=False) + pre = lambda wc: unit_pre("exp_mask", "exp", wc.exp), + script_hash = SCRIPT_HASH + threads: 4 + resources: + mem_mb = lambda wc, attempt: 8000 * attempt, + runtime = 120 shell: - "{params.cmd} --threads {threads}" + sp_shell("exp_mask", "config_exp_Ma.ini") # SExtractor -> setools star selection -> PSFEx model -> psfex_interp, per CCD. -# setools may reject a sparse CCD (~0.2% attrition) — tolerated by the floor. +# setools may reject a sparse CCD (~0.2% attrition) — tolerated by the floor's +# :warn on psfex_interp_runner. rule exp_psf: input: - rules.exp_mask.output + rules.exp_mask.output.manifest output: - directory(f"{EXP_OUT}/run_sp_exp_SxSePsfPi") + manifest = f"{EXP_DIR}/manifests/exp_psf.json" params: - cmd=lambda wc: sp_rule("exp_psf", "config_exp_psfex.ini", "exp", wc.exp, - isolate=False) + pre = lambda wc: unit_pre("exp_psf", "exp", wc.exp), + script_hash = SCRIPT_HASH + threads: 8 + retries: 2 + benchmark: + f"{EXP_DIR}/manifests/exp_psf.benchmark.tsv" + resources: + mem_mb = lambda wc, attempt: 16000 * attempt, + runtime = 240 shell: - "{params.cmd} --threads {threads}" + sp_shell("exp_psf", "config_exp_psfex.ini") diff --git a/workflow/rules/prepare.smk b/workflow/rules/prepare.smk index d8698c826..92998e184 100644 --- a/workflow/rules/prepare.smk +++ b/workflow/rules/prepare.smk @@ -1,58 +1,67 @@ -"""Phase A — prepare: tile staging, find_exposures, per-unit star cats. +"""Invocation 1 — PREPARE: one static chain per tile. -Split into two static invocations with build_index.py (a plain script) between: - * prepare_tiles — tile_get_images -> tile_uncompress -> tile_find_exposures - * [build_index.py] — plain script, threshold-gated (not a DAG node) - * prepare_exposures — exp_get_images (exposure.smk) + per-unit star cats + tile_get_images -> tile_uncompress -> tile_find_exposures -Nibi compute nodes have internet, so downloads run in-DAG (no login-node tier). -Rules stay group-compatible: no mid-chain localrules, no run:/pipe outputs. +Known from the tile list alone, cheap, wide, idempotent. find_exposures parses +the tile FITS HISTORY header into ``exp_numbers--.txt`` — the +data-derived tile->exposure edge that invocation 2's parse aggregates into the +index. Nibi compute nodes have internet, so downloads run in-DAG (no login-node +tier). + +Rules stay group-compatible: shell only, no mid-chain localrules, no pipe outputs. + +Star catalogues for masking are pre-generated offline (create_star_cat.py on a +networked login node) and consumed as DIRECTORIES: the mask configs read +``$SP_RUN/star_cat_{exp,tiles}`` with per-CCD numbering. Every rule's prologue +materialises the two dir symlinks; there is NO per-unit star-cat DAG node — the +store is pre-run input like the image store, and a missing cat fails the mask +stage's count floor loudly. """ -# get_images reads a one-line tile_numbers.txt the wrapper drops in $SP_RUN. -# NUMBER_LIST is NEVER injected here (download stage; nothing on disk to -# validate against — #746 would hard-fail). +# NUMBER_LIST is never set for get_images (download stage; nothing on disk to +# validate against — #746 would hard-fail the unit). The committed +# config_tile_Git.ini simply has no NUMBER_LIST, so this is now a property of the +# config, not of an injection step. rule tile_get_images: output: - directory(str(RUN_DIR / "tiles/{tile}/output/run_sp_tile_Git")) + manifest = f"{TILE_DIR}/manifests/tile_get_images.json" params: - cmd=lambda wc: sp_rule("tile_get_images", "config_tile_Git_vos.ini", - "tile", wc.tile, isolate=False) + pre = lambda wc: unit_pre("tile_get_images", "tile", wc.tile), + script_hash = SCRIPT_HASH + threads: 1 + retries: 2 + resources: + mem_mb = lambda wc, attempt: 4000 * attempt, + runtime = 60 shell: - "{params.cmd} --threads {threads}" + sp_shell("tile_get_images", "config_tile_Git.ini") rule tile_uncompress: input: - rules.tile_get_images.output + rules.tile_get_images.output.manifest output: - directory(str(RUN_DIR / "tiles/{tile}/output/run_sp_tile_Uz")) + manifest = f"{TILE_DIR}/manifests/tile_uncompress.json" params: - cmd=lambda wc: sp_rule("tile_uncompress", "config_tile_Uz.ini", - "tile", wc.tile, isolate=True) + pre = lambda wc: unit_pre("tile_uncompress", "tile", wc.tile), + script_hash = SCRIPT_HASH + threads: 4 + resources: + mem_mb = lambda wc, attempt: 8000 * attempt, + runtime = 60 shell: - "{params.cmd} --threads {threads}" + sp_shell("tile_uncompress", "config_tile_Uz.ini") -# find_exposures reads the tile FITS HISTORY header -> exp_numbers-.txt, the -# data-derived tile->exposure edge build_index.py aggregates (as a plain script). rule tile_find_exposures: input: - rules.tile_uncompress.output + rules.tile_uncompress.output.manifest output: - directory(str(RUN_DIR / "tiles/{tile}/output/run_sp_tile_Fe")) + manifest = f"{TILE_DIR}/manifests/tile_find_exposures.json" params: - cmd=lambda wc: sp_rule("tile_find_exposures", "config_tile_Fe.ini", - "tile", wc.tile, isolate=True) + pre = lambda wc: unit_pre("tile_find_exposures", "tile", wc.tile), + script_hash = SCRIPT_HASH + threads: 1 + resources: + mem_mb = lambda wc, attempt: 2000 * attempt, + runtime = 30 shell: - "{params.cmd} --threads {threads}" - -# Star catalogues for masking are pre-generated offline (create_star_cat.py on a -# networked login node) and consumed as DIRECTORIES: the mask configs read -# INPUT_DIR $SP_RUN/star_cat_{exp,tiles} with per-CCD numbering (exp cats are -# star_cat--.fits, 40/exposure; tile cats star_cat--.fits) -# — the v2.0 mechanism. The wrapper materialises the two dir symlinks in every -# unit work dir (sp_rule.materialise_unit); there is NO per-unit star-cat DAG -# node: the store is pre-run input like the image store, and a missing cat fails -# the mask stage's count-floor loudly. (Earlier per-unit file rules linked names -# that don't exist — one file per exposure vs the store's 40 — and nothing read -# them: the configs only see the dir symlinks.) When star-cat GENERATION moves -# in-workflow (P2), it becomes real per-unit rules producing into the store. + sp_shell("tile_find_exposures", "config_tile_Fe.ini") diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 04cc4c6c9..a0f048a59 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -1,151 +1,218 @@ -"""Tile post-chain — per tile: gather exposures, then detect/PSF/shape/catalogue. - -A per-tile exposure "forest" (symlink view of exactly this tile's exposures' -products) gives the tile modules a deterministic $SP_EXP to glob. But the forest -is only a runtime convenience, NEVER the DAG edge: every tile rule that reads -through $SP_EXP declares the underlying exposure directory() outputs as input via -an index dict lookup, so the last real reader is the last DAG consumer (findings -5/19). All rules are group-compatible (shell only, no mid-chain localrules). -""" +"""Tile post-chain — per tile: gather exposures, then detect / PSF / shape / catalogue. + + tile_exp_forest + tile_merge_headers -> tile_detect -> tile_vignets -> tile_ngmix x N + -> tile_merge_cats -> tile_make_cat -TILE_OUT = str(RUN_DIR / "tiles/{tile}/output") -FOREST = str(RUN_DIR / "tiles/{tile}/exp_forest") +The DAG edge to the exposures is always the exposures' MANIFESTS, looked up +through the index (TILE_EXP). The per-tile exposure "forest" — a symlink view of +exactly this tile's exposures' products — exists only so the ShapePipe configs +have one deterministic ``$SP_EXP`` to glob; it is NEVER the edge. Its 2-char +shard level is not cosmetic: ``exp_utils.get_exp_output_files`` hardwires +``///output/run_sp_*`` into its glob, so a flat forest +makes every tile gather stage fail "No split_exp_runner output found". +All rules are group-compatible (shell only, no mid-chain localrules). + +Note there is no `tile_mask` rule: the committed config chain is the +"sx_nomask" tile_detect variant (config_tile_Sx.ini reads Git + Uz + Mh, no mask +run), and no tile-mask config was committed in the S2 sweep. Adding the masked +variant is a config + one rule, at the config selector the PRD describes. +""" -def _exp_out(wc, prefix): - return [str(exp_dir(e) / "output" / prefix) for e in TILE_EXP.get(wc.tile, [])] +def tile_exp(wc): + return TILE_EXP.get(wc.tile, []) -def tile_exp_split(wc): return _exp_out(wc, "run_sp_exp_Sp") -def tile_exp_mask(wc): return _exp_out(wc, "run_sp_exp_Ma") -def tile_exp_psf(wc): return _exp_out(wc, "run_sp_exp_SxSePsfPi") -def tile_exp_products(wc): - """All of this tile's exposures' product dirs — the forest's DAG inputs.""" - return tile_exp_split(wc) + tile_exp_mask(wc) + tile_exp_psf(wc) +def exp_manifests(wc, stage): + return [exp_manifest(e, stage) for e in tile_exp(wc)] +def tile_exp_split(wc): return exp_manifests(wc, "exp_split") +def tile_exp_mask(wc): return exp_manifests(wc, "exp_mask") +def tile_exp_psf(wc): return exp_manifests(wc, "exp_psf") +def tile_exp_all(wc): return tile_exp_split(wc) + tile_exp_mask(wc) + tile_exp_psf(wc) -# Build the per-tile symlink forest (exp_forest//output/run_sp_exp_* -> the -# real exposure products). Declaring the exposure dirs as input makes this the -# tile's wait-on-its-exposures edge; the forest itself is just the SP_EXP view. + +# Build the per-tile symlink forest. Declaring the exposure manifests as input +# makes this wait on its exposures; the forest itself is only the $SP_EXP view. +# Its output stays a directory() (it has no ShapePipe run dir and no manifest — +# it is not a shapepipe_run at all). rule tile_exp_forest: input: - tile_exp_products + tile_exp_all output: - directory(FOREST) + forest = directory(f"{TILE_DIR}/exp_forest") params: - # --forest {output} lives in the shell string: Snakemake formats shell - # ONCE — an {output} placeholder inside params.cmd survives literally - # (same trap as {threads}; all four forest jobs then race one './{output}'). - cmd=lambda wc: (f"python {SCRIPTS}/build_forest.py --tile {wc.tile} " - f"--run-dir {RUN_DIR} --index {INDEX_DB} " - f"--script-hash {SCRIPT_HASH}") + cmd = lambda wc: (f"python {SCRIPTS}/build_forest.py --tile {wc.tile} " + f"--run-dir {RUN_DIR} --index {INDEX_DB}"), + # build_forest.py's own content hash rides here and nowhere else. + script_hash = FOREST_HASH + threads: 1 + resources: + mem_mb = 2000, + runtime = 20 shell: - # no --threads: build_forest.py is single-threaded symlinking - "{params.cmd} --forest {output}" + # --forest {output} lives in the shell string: snakemake formats shell + # ONCE, so an {output} placeholder inside params.cmd would survive + # literally and every forest job would race one './{output}'. + "{params.cmd} --forest {output.forest}" -# Merge single-exposure WCS headers into the tile-level sqlite log -# (log_exp_headers--.sqlite — the tile suffix downstream Sx/PiViVi/Ng -# consume via NUMBERING_SCHEME -000-000). Reads headers-*.npy through the forest, -# so the exposure split dirs are explicit inputs. +# Merge single-exposure WCS headers into the tile-level sqlite +# (log_exp_headers--.sqlite, which Sx / PiViVi / ngmix consume). +# Reads headers-*.npy through the forest -> the split manifests are the edge. rule tile_merge_headers: input: - forest=rules.tile_exp_forest.output, - split=tile_exp_split, + forest = rules.tile_exp_forest.output.forest, + split = tile_exp_split, output: - directory(f"{TILE_OUT}/run_sp_tile_Mh_exp") + manifest = f"{TILE_DIR}/manifests/tile_merge_headers.json" params: - cmd=lambda wc: sp_rule("tile_merge_headers", "config_tile_Mh_exp.ini", - "tile", wc.tile, isolate=True, - exp_forest=FOREST.format(tile=wc.tile)) + pre = lambda wc: unit_pre("tile_merge_headers", "tile", wc.tile, + forest=forest_dir(wc.tile)), + script_hash = SCRIPT_HASH + threads: 4 + resources: + mem_mb = lambda wc, attempt: 8000 * attempt, + runtime = 120 shell: - "{params.cmd} --threads {threads}" - -# Mask tiles (star cats: pre-run store, read via the wrapper's dir symlink). -rule tile_mask: - input: - git=rules.tile_get_images.output, - output: - directory(f"{TILE_OUT}/run_sp_tile_Ma") - params: - cmd=lambda wc: sp_rule("tile_mask", "config_tile_Ma_onthefly.ini", - "tile", wc.tile, isolate=True) - shell: - "{params.cmd} --threads {threads}" + sp_shell("tile_merge_headers", "config_tile_Mh_exp.ini") # SExtractor object detection on the tile. rule tile_detect: input: - mask=rules.tile_mask.output, - mh=rules.tile_merge_headers.output, + uz = f"{TILE_DIR}/manifests/tile_uncompress.json", + mh = rules.tile_merge_headers.output.manifest, output: - directory(f"{TILE_OUT}/run_sp_tile_Sx") + manifest = f"{TILE_DIR}/manifests/tile_detect.json" params: - cmd=lambda wc: sp_rule("tile_detect", "config_tile_Sx.ini", - "tile", wc.tile, isolate=True) + pre = lambda wc: unit_pre("tile_detect", "tile", wc.tile), + script_hash = SCRIPT_HASH + threads: 8 + resources: + mem_mb = lambda wc, attempt: 16000 * attempt, + runtime = 180 shell: - "{params.cmd} --threads {threads}" + sp_shell("tile_detect", "config_tile_Sx.ini") -# PSFEx interpolation to galaxies + vignet postage stamps (reads exposure PSF -# models + split through the forest — those dirs are explicit inputs). +# PSFEx interpolation to galaxies + vignet postage stamps: the last stage that +# reads exposure products, and the bulk intra-tile intermediate. +# +# The vignette store is declared as a SECOND, temp(directory()) output alongside +# the manifest. This is the ONE scoped exception to "no directory() outputs" +# (D5): the store is ~tens of GB per tile and must be reclaimed when its last +# intra-tile reader finishes, but it must not become DAG currency — so the +# manifest stays the edge, and the directory rides along purely so native temp() +# fires at the right moment. Its readers (tile_ngmix, tile_make_cat) declare +# BOTH. `--notemp` keeps it for debugging. rule tile_vignets: input: - sx=rules.tile_detect.output, - forest=rules.tile_exp_forest.output, - split=tile_exp_split, - psf=tile_exp_psf, + sx = rules.tile_detect.output.manifest, + forest = rules.tile_exp_forest.output.forest, + split = tile_exp_split, + psf = tile_exp_psf, output: - directory(f"{TILE_OUT}/run_sp_tile_PiViVi") + manifest = f"{TILE_DIR}/manifests/tile_vignets.json", + store = temp(directory(f"{TILE_DIR}/output/run_sp_tile_PiViVi")), params: - cmd=lambda wc: sp_rule("tile_vignets", "config_tile_PiViVi_canfar_sx.ini", - "tile", wc.tile, isolate=True, - exp_forest=FOREST.format(tile=wc.tile)) + pre = lambda wc: unit_pre("tile_vignets", "tile", wc.tile, + forest=forest_dir(wc.tile)), + script_hash = SCRIPT_HASH + threads: 16 + resources: + mem_mb = lambda wc, attempt: 32000 * attempt, + runtime = 240 shell: - "{params.cmd} --threads {threads}" + sp_shell("tile_vignets", "config_tile_PiViVi.ini") -# ngmix shape measurement — static N chunks. Each chunk computes its own closed -# object-ID range in-job from the tile's own sexcat (a declared input); chunks -# never rewrite the shared log (--no-log-sync) and read literal INPUT_DIRs. +# ngmix shape measurement — N chunks per tile (D4). Each chunk computes its own +# CLOSED object-ID range at EXECUTION time from this tile's own sexcat: a params +# function cannot, because params evaluate before the sexcat exists. Closed, not +# open-ended: `ID_OBJ_MAX = -1` on the last chunk was the 13-hour straggler's +# root cause (ngmix treats id_obj_max <= 0 as unbounded). +# +# Chunks write nothing shared: each has its own run_sp_tile_ngmix_Ngu, and +# merge_sep_cats — DAG-serialised after all chunks — is the gather. rule tile_ngmix: input: - vignets=rules.tile_vignets.output, - sx=rules.tile_detect.output, + vignets = rules.tile_vignets.output.manifest, + store = rules.tile_vignets.output.store, + sx = rules.tile_detect.output.manifest, output: - directory(f"{TILE_OUT}/run_sp_tile_ngmix_Ng{{chunk}}u") + manifest = f"{TILE_DIR}/manifests/tile_ngmix_{{chunk}}.json", + # temp(directory()) for the same reason as the vignette store above. + chunkdir = temp(directory(f"{TILE_DIR}/output/run_sp_tile_ngmix_Ng{{chunk}}u")), params: - cmd=lambda wc: sp_rule( - "tile_ngmix", "config_tile_Ng_template_batch.ini", "tile", wc.tile, - extra=f"--ngmix-chunk {wc.chunk} --ngmix-nchunks {NGMIX_CHUNKS}"), + pre = lambda wc: unit_pre( + "tile_ngmix", "tile", wc.tile, + env={"SP_NGMIX_CHUNK": wc.chunk, "NGMIX_N_CHUNKS": NGMIX_CHUNKS}, + pre_run=[f'eval "$(python {SCRIPTS}/ngmix_range.py --run-dir ' + f'"$SP_RUN" --chunk {wc.chunk} --n-chunks {NGMIX_CHUNKS})"']), + script_hash = SCRIPT_HASH + threads: 4 + retries: 2 + benchmark: + f"{TILE_DIR}/manifests/tile_ngmix_{{chunk}}.benchmark.tsv" + resources: + mem_mb = lambda wc, attempt: 14000 * attempt, + runtime = 720 shell: - "{params.cmd} --threads {threads}" + sp_shell("tile_ngmix", "config_tile_Ng_template.ini") + + +def ngmix_manifests(wc): + return [f"{tile_dir(wc.tile)}/manifests/tile_ngmix_{k}.json" + for k in range(1, NGMIX_CHUNKS + 1)] -def ngmix_chunks(wc): - return [f"{RUN_DIR}/tiles/{wc.tile}/output/run_sp_tile_ngmix_Ng{k}u" +def ngmix_chunkdirs(wc): + return [f"{tile_dir(wc.tile)}/output/run_sp_tile_ngmix_Ng{k}u" for k in range(1, NGMIX_CHUNKS + 1)] -# Merge the N chunk catalogues (DAG-serialized: the single per-tile log-sync -# point before make_cat). N_SPLIT_MAX == chunk count. +# The gather: merge the N chunk catalogues. N_SPLIT_MAX comes from the workflow's +# own chunk count via $NGMIX_N_CHUNKS (env-expanded by the module). rule tile_merge_cats: input: - ngmix_chunks + manifests = ngmix_manifests, + chunkdirs = ngmix_chunkdirs, output: - directory(f"{TILE_OUT}/run_sp_Ms") + manifest = f"{TILE_DIR}/manifests/tile_merge_cats.json" params: - cmd=lambda wc: sp_rule("tile_merge_cats", "config_merge_sep_cats_template.ini", - "tile", wc.tile, isolate=True, - extra=f"--n-split-max {NGMIX_CHUNKS}"), + pre = lambda wc: unit_pre("tile_merge_cats", "tile", wc.tile, + env={"NGMIX_N_CHUNKS": NGMIX_CHUNKS}), + script_hash = SCRIPT_HASH + threads: 8 + resources: + mem_mb = lambda wc, attempt: 16000 * attempt, + runtime = 120 shell: - "{params.cmd} --threads {threads}" + sp_shell("tile_merge_cats", "config_merge_sep_cats.ini") -# Build the final catalogue: the run's science product. protected() so a -# code/params-drift rerun cannot silently discard a finished catalogue — the -# recompute must go through `sp rerun` (--forcerun). +# The run's science product. make_cat also reads the vignette store's +# psfex_interp output, so it — not ngmix — is the store's last reader. +# +# No protected(): the full default rerun-triggers govern, and protected() only +# ever forced people through a `--forcerun` detour. rule tile_make_cat: input: - rules.tile_merge_cats.output + ms = rules.tile_merge_cats.output.manifest, + store = rules.tile_vignets.output.store, output: - protected(str(RUN_DIR / "tiles/{tile}/final_cat-{tile}.fits")) + manifest = f"{TILE_DIR}/manifests/tile_make_cat.json", + final_cat = f"{TILE_DIR}/final_cat-{{tile}}.fits", params: - cmd=lambda wc: sp_rule("tile_make_cat", "config_make_cat_psfex_nosm.ini", - "tile", wc.tile, isolate=True, - extra=f"--final-cat {RUN_DIR}/tiles/{wc.tile}/final_cat-{wc.tile}.fits"), + pre = lambda wc: unit_pre("tile_make_cat", "tile", wc.tile), + script_hash = SCRIPT_HASH + threads: 8 + resources: + mem_mb = lambda wc, attempt: 16000 * attempt, + runtime = 120 shell: - "{params.cmd} --threads {threads}" + # Publish the catalogue next to the manifest: a real file, so it is a + # real declared output (and it persists — never temp()). + "{params.pre}\n" + "rc=0\n" + 'shapepipe_run -c "$SP_CONFIG/config_tile_Mc.ini" -b {threads} || rc=$?\n' + f"python {SCRIPTS}/completeness.py check tile_make_cat {{output.manifest}} || rc=1\n" + "if [ $rc -eq 0 ]; then\n" + ' cp -f "$(ls -1 "$SP_RUN"/output/run_sp_Mc/make_cat_runner/output/final_cat*.fits' + ' | head -1)" {output.final_cat}\n' + "fi\n" + "exit $rc\n" diff --git a/workflow/scripts/build_forest.py b/workflow/scripts/build_forest.py index e945820e0..7ccaa308e 100644 --- a/workflow/scripts/build_forest.py +++ b/workflow/scripts/build_forest.py @@ -3,12 +3,12 @@ A plain script (not a run: block) so the tile chain stays group-compatible. Reads the tile's exposures from run_index.sqlite and symlinks each exposure's -``exp//output`` into ``///output`` by exact name (no -glob). The 2-digit ```` shard level is NOT cosmetic: ShapePipe's +``exp///output`` into ``///output`` by exact +name (no glob). The 2-char ```` shard level is NOT cosmetic: ShapePipe's ``exp_utils.get_exp_output_files`` hardwires the sharded v2.0 layout into its $SP_EXP glob (``///output/run_sp_*/...``), so a flat forest makes every tile gather stage fail "No split_exp_runner output found". -(The exposure STORE stays flat — only the module-facing view needs the shard.) +(The exposure STORE is sharded the same way, for the filesystem's sake.) The forest is a convenience view; the DAG edge to the exposures is declared in the rule's input (tile.smk), not here. """ @@ -24,11 +24,6 @@ def main() -> None: p.add_argument("--run-dir", required=True, type=Path) p.add_argument("--index", required=True, type=Path) p.add_argument("--forest", required=True, type=Path) - p.add_argument("--script-hash", default=None, - help="ignored at runtime: carries the wrapper-scripts content " - "hash into the rule's params so the params rerun-trigger " - "covers script edits (the code trigger only hashes the " - "rule's own shell string)") args = p.parse_args() con = sqlite3.connect(args.index) @@ -38,7 +33,7 @@ def main() -> None: args.forest.mkdir(parents=True, exist_ok=True) for e in exps: - src = args.run_dir / "exp" / e / "output" + src = args.run_dir / "exp" / e[:2] / e / "output" dst = args.forest / e[:2] / e / "output" # sharded: the module glob's shape dst.parent.mkdir(parents=True, exist_ok=True) if dst.is_symlink() or dst.exists(): diff --git a/workflow/scripts/build_index.py b/workflow/scripts/build_index.py index 7ba865d94..31fc6178b 100644 --- a/workflow/scripts/build_index.py +++ b/workflow/scripts/build_index.py @@ -5,7 +5,11 @@ at parse time into plain dicts, so extending a run's tile list changes which jobs exist without touching the mtime chain of completed work. -It records, per run: +It ACCUMULATES: tables are created if absent and rows upserted, never dropped +(D1). Successive invocations widen it, so a later ``clean_exposure`` sees every +consuming tile of the whole campaign rather than of the current tile list. + +It records: tiles(tile_id, ra_dir, n_exp, status) exposures(exp_id) -- deduplicated union over tiles @@ -14,20 +18,21 @@ The tile->exposure edges are *data-derived*: they are read from each tile's ``find_exposures`` output (``exp_numbers--.txt``), which ``find_exposures_runner`` produces by parsing the tile FITS ``HISTORY`` header. -So this is a **plain script**, not a DAG node: it runs between the two static -invocations — after ``snakemake prepare_tiles`` (Git+Uz+Fe, keep-going) and -before ``snakemake prepare_exposures`` parses the now-existing index. It +So the build is not a DAG node: ``build()`` is imported and called at PARSE TIME +by the Snakefile of the COMPUTE invocation, after the PREPARE invocation has +produced the tiles' find_exposures output. (There is no ``sp index`` verb; the +CLI below stays for hand-inspection.) It iterates the *declared* tile list and checks each tile's Fe output at its deterministic path (no globbing — O(tile-list) existence checks, respecting the no-``ls``-at-scale ban). A bad tile costs only that tile: it is recorded in -``missing.json`` and the index is built over the rest. The script exits nonzero -only if the missing *fraction* exceeds ``--missing-threshold`` (default 0.0), -so a keep-going download storm that lost a few tiles does not cost the run. -There is NO ``--allow-missing`` flag (that Snakemake flag does not exist). +``missing.json`` and the index is built over the rest. The build fails only if +the missing *fraction* exceeds ``missing_threshold`` (``None`` disables the +check — what the PREPARE phase passes, where absent Fe output is normal), so a +keep-going download storm that lost a few tiles does not cost the run. Exposure IDs are stored with their trailing single-char suffix stripped (``2243881p`` -> ``2243881``); that ``exp_base`` is the dedup key and the -exposure-rule wildcard, matching the ``exp///`` forest layout. +exposure-rule wildcard, matching the sharded ``exp///`` store. """ import argparse @@ -55,8 +60,20 @@ def read_exposure_list(exp_numbers_file: Path) -> list[tuple[str, str]]: return pairs +def exp_list_path(run_dir: Path, tile_id: str) -> Path: + """This tile's find_exposures output, at its deterministic path. + + Sharded store (D2), fixed run dir (RUN_DATETIME=False) — an existence check, + never a glob (no ``ls`` at scale). + """ + idra, iddec = tile_id.split(".") + return (run_dir / "tiles" / tile_id[:2] / tile_id / "output" / + "run_sp_tile_Fe" / "find_exposures_runner" / "output" / + f"exp_numbers-{idra}-{iddec}.txt") + + def build(tile_ids: list[str], run_dir: Path, db_path: Path, - missing_threshold: float = 0.0) -> dict: + missing_threshold: float | None = 0.0) -> dict: """Build the index over ``tile_ids``; return a summary dict. For each tile, check its ``exp_numbers--.txt`` at the tile's @@ -67,14 +84,14 @@ def build(tile_ids: list[str], run_dir: Path, db_path: Path, """ db_path.parent.mkdir(parents=True, exist_ok=True) con = sqlite3.connect(db_path) + # No DROP: the index accumulates across invocations (D1). con.executescript( """ - DROP TABLE IF EXISTS tiles; - DROP TABLE IF EXISTS exposures; - DROP TABLE IF EXISTS tile_exposures; - CREATE TABLE tiles(tile_id TEXT PRIMARY KEY, ra_dir TEXT, n_exp INTEGER); - CREATE TABLE exposures(exp_id TEXT PRIMARY KEY, name TEXT NOT NULL); - CREATE TABLE tile_exposures( + CREATE TABLE IF NOT EXISTS tiles( + tile_id TEXT PRIMARY KEY, ra_dir TEXT, n_exp INTEGER); + CREATE TABLE IF NOT EXISTS exposures( + exp_id TEXT PRIMARY KEY, name TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS tile_exposures( tile_id TEXT, exp_id TEXT, PRIMARY KEY (tile_id, exp_id)); """ @@ -84,30 +101,26 @@ def build(tile_ids: list[str], run_dir: Path, db_path: Path, all_exposures: set[tuple[str, str]] = set() for tile_id in tile_ids: ra_dir = tile_id.split(".")[0] - idra, iddec = tile_id.split(".") - # Flat forest, deterministic run dir (RUN_DATETIME=False). - exp_file = (run_dir / "tiles" / tile_id / "output" / "run_sp_tile_Fe" / - "find_exposures_runner" / "output" / - f"exp_numbers-{idra}-{iddec}.txt") + exp_file = exp_list_path(run_dir, tile_id) if not exp_file.exists(): missing.append(tile_id) continue exp_pairs = read_exposure_list(exp_file) - con.execute("INSERT INTO tiles VALUES (?,?,?)", + con.execute("INSERT OR REPLACE INTO tiles VALUES (?,?,?)", (tile_id, ra_dir, len(exp_pairs))) for exp_id, _name in exp_pairs: con.execute("INSERT OR IGNORE INTO tile_exposures VALUES (?,?)", (tile_id, exp_id)) all_exposures.update(exp_pairs) - con.executemany("INSERT OR IGNORE INTO exposures VALUES (?,?)", + con.executemany("INSERT OR REPLACE INTO exposures VALUES (?,?)", sorted(all_exposures)) con.commit() con.close() (db_path.parent / "missing.json").write_text(json.dumps(missing, indent=2)) frac = len(missing) / len(tile_ids) if tile_ids else 0.0 - if frac > missing_threshold: + if missing_threshold is not None and frac > missing_threshold: raise SystemExit( f"Missing exposure lists for {len(missing)}/{len(tile_ids)} tile(s) " f"(fraction {frac:.3f} > threshold {missing_threshold}): {missing}. " diff --git a/workflow/scripts/ngmix_range.py b/workflow/scripts/ngmix_range.py new file mode 100644 index 000000000..9e3058052 --- /dev/null +++ b/workflow/scripts/ngmix_range.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Print one ngmix chunk's closed object-ID range as bash exports. + +Run inside the chunk's shell, from the tile's own sexcat, because the range is +only knowable at execution time (PRD D4):: + + eval "$(ngmix_range.py --run-dir $SP_RUN --chunk 3 --n-chunks 8)" + # -> export NGMIX_ID_MIN=751; export NGMIX_ID_MAX=1125 + +SExtractor's NUMBER column (ngmix's obj_id) runs 1..N contiguous, so covering +[1, N] processes every object exactly once. The first N-1 chunks get equal +shares; the last takes the remainder, with a CLOSED upper bound at n_obj — +never ID_OBJ_MAX = -1, which ngmix reads as unbounded and would double-count. +Each tile is sized from its OWN count (the bash monolith used the campaign +average and left the last chunk open). +""" + +import argparse +from pathlib import Path + + +def id_ranges(n_obj: int, n_chunks: int) -> list[tuple[int, int]]: + base, rem = divmod(n_obj, n_chunks) + ranges, lo = [], 1 + for k in range(1, n_chunks + 1): + hi = lo + base + (rem if k == n_chunks else 0) - 1 + ranges.append((lo, hi)) + lo = hi + 1 + return ranges + + +def object_count(run_dir: Path) -> int: + """NAXIS2 of the last HDU of this tile's sexcat (get_number_objects.py).""" + from astropy.io import fits + + cats = sorted((run_dir / "output" / "run_sp_tile_Sx").glob( + "sextractor_runner/output/sexcat*.fits")) + if not cats: + raise SystemExit(f"[ngmix_range] FATAL: no sexcat under {run_dir}") + with fits.open(cats[0]) as hdul: + return int(hdul[-1].header["NAXIS2"]) + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--run-dir", required=True, type=Path) + p.add_argument("--chunk", required=True, type=int) + p.add_argument("--n-chunks", required=True, type=int) + a = p.parse_args() + lo, hi = id_ranges(object_count(a.run_dir), a.n_chunks)[a.chunk - 1] + print(f"export NGMIX_ID_MIN={lo}; export NGMIX_ID_MAX={hi}") + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/sp_rule.py b/workflow/scripts/sp_rule.py deleted file mode 100644 index 8ee71c693..000000000 --- a/workflow/scripts/sp_rule.py +++ /dev/null @@ -1,419 +0,0 @@ -#!/usr/bin/env python3 -"""Thin per-unit ShapePipe rule wrapper — the single entrypoint every rule calls. - -Under the one-allocation local executor + apptainer software-deployment, -Snakemake runs each rule's shell *inside* the container on the allocation's node. -So this script already runs in-container: it calls ``shapepipe_run`` directly -(never apptainer itself), which is why the workflow never hand-invokes apptainer. - -What it owns — only what is intrinsically ShapePipe's, nothing Snakemake already -does: - -1. **Per-unit work dir + v2.0 isolation furniture.** Each tile/exposure gets its - own ``$SP_RUN`` (``tiles//`` or ``exp//``). Isolation is by work-dir - *content*, the proven v2.0 mechanism — NOT ``-e/--exclusive``: - - a ``cfis`` symlink to the canonical config dir (so ``$SP_CONFIG`` / ``.mask`` - refs resolve); - - ``star_cat_exp`` / ``star_cat_tiles`` symlinks to the shared cat pool; - - for a tile: ``tile_numbers.txt`` = this tile ID (dot format, real data — - what get_images reads); - - for an exposure: a fabricated - ``output/run_sp_tile_Fe/find_exposures_runner/output/exp_numbers-000-000.txt`` - = this exposure ID (v2.0 ``init_exp_work_dir`` pseudo-Fe), so the Gie - config's ``last:find_exposures_runner`` resolver finds exactly one unit. - -2. **Per-unit config copy.** Forces ``RUN_DATETIME = False`` (deterministic - ``output/run_sp_/`` the rule declares as its ``directory()`` output), - sets ``SMP_BATCH_SIZE = {threads}`` (fork width == cpus_per_task, so SLURM - packing density and true per-job process count are one number), and — only - when ``isolate`` (tile-scheme stages + exp split; NEVER get_images / exp_mask / - exp_psf, whose per-CCD/download numbering would make #746 turn tolerated - per-CCD attrition into a whole-exposure hard failure) — ``NUMBER_LIST = -``. - -3. **Log sync at START and end.** ``update_runs_log_file.py`` regenerates - ``log_run_sp.txt`` before ``shapepipe_run`` (after the pre-delete, so - resolution never sees a just-deleted dir) and again after. Skipped for ngmix - chunks: the log is already complete from the prior tile stages, and the N - concurrent chunk jobs share the tile work dir — a full log *rewrite* would - race, whereas ShapePipe's own single-line ``O_APPEND`` of each chunk's run is - atomic. ngmix chunk configs additionally carry *literal* INPUT_DIRs - (deterministic under RUN_DATETIME=False), so they need no log read at all. - -4. **Count-floor check** (``completeness.py`` — the ported bash ``complete_check`` - table, the single failure policy: no 3-class taxonomy, no whitelist). After - the run, count products per mandatory runner and exit nonzero below the floor. - ``keep-going`` isolates that unit's cone; per-CCD attrition between floor and - ``expect`` is tolerated. Stages absent from the table fall back to a - zero-output floor. - -Stage-specific verbs (grounded in ``runs/p3-batch1/job_p3_batch1.sh``): - -- ``--ngmix-chunk K --ngmix-nchunks N`` — read this tile's SExtractor object - count from its ``run_sp_tile_Sx`` output, split ``[1, N_obj]`` into N disjoint - closed ID ranges, and generate chunk K's config from - ``config_tile_Ng_template_batch.ini``. -- ``--n-split-max N`` — generate the merge config from - ``config_merge_sep_cats_template.ini`` with ``N_SPLIT_MAX = N``. -- ``--final-cat PATH`` — after ``make_cat``, copy the produced ``final_cat`` FITS - to PATH (the rule's declared, protected science-product output). - -Run by hand to reproduce exactly what a rule does (the escape hatch — it -pre-deletes the run dir, just like Snakemake's delete-before-rerun):: - - python sp_rule.py --stage exp_split --config config_exp_Sp.ini \\ - --unit 2243881 --level exp --run-dir $SP_RUN --config-src $SP_CONFIG \\ - --star-cats $STAR_CATS --threads 8 -""" - -import argparse -import os -import shutil -import subprocess -import sys -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parent)) -from completeness import COMPLETENESS, check_floor # noqa: E402 - - -def unit_work_dir(run_dir: Path, level: str, unit: str) -> Path: - """Per-unit work dir in the forest: tiles/ or exp/. - - Flat (no IDra/prefix sharding): the shard existed only to keep the bash - layer's ``ls exp/`` bearable, but Snakemake never globs the forest — the - run index drives every path — so flattening keeps single-wildcard rules. - ``run`` level (aggregation stages) work directly in ``run_dir``. - """ - if level == "tile": - return run_dir / "tiles" / unit - if level == "exp": - base = unit[:-1] if unit[-1].isalpha() else unit - return run_dir / "exp" / base - return run_dir - - -def materialise_unit(work: Path, level: str, unit: str, config_src: Path, - star_cats: Path, exp_name: str | None = None) -> None: - """Create the v2.0 unit-isolation furniture in the work dir.""" - (work / "output").mkdir(parents=True, exist_ok=True) - - cfis = work / "cfis" # $SP_CONFIG -> canonical config dir (.mask refs resolve) - if not cfis.exists(): - cfis.symlink_to(config_src) - - for name, sub in (("star_cat_exp", "exp"), ("star_cat_tiles", "tiles")): - link = work / name - if not link.exists() and (star_cats / sub).exists(): - link.symlink_to(star_cats / sub) - - if level == "tile": - (work / "tile_numbers.txt").write_text(unit + "\n") # dot format, real data - elif level == "exp": - fe = work / "output" / "run_sp_tile_Fe" / "find_exposures_runner" / "output" - fe.mkdir(parents=True, exist_ok=True) - exp_numbers = fe / "exp_numbers-000-000.txt" - # Written UNCONDITIONALLY (cheap, deterministic): an exists-guard once - # pinned a stale pre-fix file with the bare base id. Content is the - # ORIGINAL exposure name (with its 'p'-style suffix, from the index) — - # the on-disk store is .fits.fz; the bare base id matches nothing. - # v2.0 copies the name verbatim from the tile's Fe output; the index - # carries it for us. - exp_numbers.write_text((exp_name or unit) + "\n") - - -def normalize_config_text(text: str, unit: str, isolate: bool, threads: int) -> str: - """Force RUN_DATETIME=False, SMP_BATCH_SIZE=threads, and (if isolating) - NUMBER_LIST=-. - - ``NUMBER_LIST`` uses ShapePipe's image-number convention (dots -> dashes), - the ``set_config_number_list`` mechanism that replaced the retired - ``-e/--exclusive`` flag (#746). - """ - lines = text.splitlines() - out, in_file, saw_datetime, saw_numberlist, saw_smp = [], False, False, False, False - number = "-" + unit.replace(".", "-") - for line in lines: - stripped = line.strip() - if stripped.startswith("[") and stripped.endswith("]"): - in_file = stripped.upper() == "[FILE]" - if stripped.upper().startswith("RUN_DATETIME"): - out.append("RUN_DATETIME = False") - saw_datetime = True - continue - if stripped.upper().startswith("SMP_BATCH_SIZE"): - out.append(f"SMP_BATCH_SIZE = {threads}") - saw_smp = True - continue - if isolate and stripped.upper().startswith("NUMBER_LIST"): - out.append(f"NUMBER_LIST = {number}") - saw_numberlist = True - continue - out.append(line) - if in_file and stripped.upper() == "[FILE]" and isolate and not saw_numberlist: - out.append(f"NUMBER_LIST = {number}") - saw_numberlist = True - if not saw_datetime: - # ShapePipe reads RUN_DATETIME from [DEFAULT]; inject after its header. - patched, done = [], False - for line in out: - patched.append(line) - if not done and line.strip().upper() == "[DEFAULT]": - patched.append("RUN_DATETIME = False") - done = True - out = patched if done else ["[DEFAULT]", "RUN_DATETIME = False"] + out - if not saw_smp: - # SMP_BATCH_SIZE lives in [JOB]; inject after its header. - patched, done = [], False - for line in out: - patched.append(line) - if not done and line.strip().upper() == "[JOB]": - patched.append(f"SMP_BATCH_SIZE = {threads}") - done = True - out = patched if done else out + ["[JOB]", f"SMP_BATCH_SIZE = {threads}"] - return "\n".join(out) + "\n" - - -def run_prefix_from_text(text: str, config_label: str) -> str: - """The RUN_NAME (== run_sp_) the module will write under.""" - for line in text.splitlines(): - s = line.strip() - if s.upper().startswith("RUN_NAME"): - return s.split("=", 1)[1].strip() - raise SystemExit(f"no RUN_NAME in {config_label}") - - -# --- ngmix chunking -------------------------------------------------------- - -# Literal, deterministic INPUT_DIRs for the ngmix chunk (RUN_DATETIME=False), so -# concurrent chunks need no log read at all (finding 18). -NGMIX_LITERAL_INPUT = ( - "./output/run_sp_tile_Sx/sextractor_runner/output, " - "./output/run_sp_tile_PiViVi/psfex_interp_runner/output, " - "./output/run_sp_tile_PiViVi/vignetmaker_runner_run_2/output, " - "./output/run_sp_tile_Mh_exp/merge_headers_runner/output") - - -def ngmix_id_ranges(n_obj: int, n_chunks: int) -> list[tuple[int, int]]: - """Split ``[1, n_obj]`` into ``n_chunks`` disjoint *closed* ID ranges. - - SExtractor's ``NUMBER`` column (what ngmix reads as ``obj_id``) runs 1..N - contiguous, so covering ``[1, N]`` processes every object exactly once. The - first n_chunks-1 chunks get ``n_obj // n_chunks`` each; the last gets the - remainder — a *closed* upper bound at ``n_obj`` (never ``ID_OBJ_MAX = -1``: - ngmix treats ``id_obj_max <= 0`` as unbounded, which would double-count). - - Deviation from the nibi monolith (deliberate): the monolith sizes every - tile's chunks from the *average* object count across all tiles and leaves the - last chunk open (``-1``); here each tile is an independent job, so we size - from *this* tile's own count and close the last chunk. The merged catalogue - is identical either way (merge_sep_cats recombines all chunks, ngmix fits - each object independently) — this just removes the open-ended overflow. - """ - base, rem = divmod(n_obj, n_chunks) - ranges, lo = [], 1 - for k in range(1, n_chunks + 1): - size = base + (rem if k == n_chunks else 0) - hi = lo + size - 1 - ranges.append((lo, hi)) - lo = hi + 1 - return ranges - - -def tile_object_count(work: Path) -> int: - """Object count for this tile from its run_sp_tile_Sx SExtractor catalogue. - - Matches ``get_number_objects.py``: the ``NAXIS2`` of the catalogue's last - HDU. Per-unit isolation means exactly one ``sexcat*.fits`` lives here. - """ - from astropy.io import fits - - sexcats = sorted((work / "output" / "run_sp_tile_Sx").glob( - "sextractor_runner/output/sexcat*.fits")) - if not sexcats: - raise SystemExit( - f"[sp_rule] FATAL ngmix: no sexcat under {work}/output/run_sp_tile_Sx") - with fits.open(sexcats[0]) as hdul: - return int(hdul[-1].header["NAXIS2"]) - - -def apply_ngmix_subs(text: str, chunk: int, lo: int, hi: int) -> str: - """Fill the ngmix batch template for one chunk (the monolith's sed recipe) - plus literal INPUT_DIRs so the chunk needs no shared-log read.""" - text = text.replace("NgXu", f"Ng{chunk}u").replace("X_interp", "psfex_interp") - out = [] - for line in text.splitlines(): - s = line.strip() - if s.startswith("ID_OBJ_MIN"): - out.append(f"ID_OBJ_MIN = {lo}") - elif s.startswith("ID_OBJ_MAX"): - out.append(f"ID_OBJ_MAX = {hi}") - elif s.startswith("INPUT_DIR") and "sextractor_runner" in s: - out.append(f"INPUT_DIR = {NGMIX_LITERAL_INPUT}") - else: - out.append(line) - return "\n".join(out) - - -def apply_merge_subs(text: str, n_split_max: int) -> str: - """Fill N_SPLIT_MAX in the merge template (the monolith's sed recipe).""" - out = [] - for line in text.splitlines(): - if line.strip().startswith("N_SPLIT_MAX"): - out.append(f"N_SPLIT_MAX = {n_split_max}") - else: - out.append(line) - return "\n".join(out) - - -# --- in-job count floor ---------------------------------------------------- - -def count_floor(run_out: Path, stage: str) -> None: - """Fail loudly if any mandatory runner is below its ``floor`` (completeness.py). - - Stages absent from the table skip this floor (only the zero-output floor - applies to them). - """ - if stage not in COMPLETENESS: - return - ok, details = check_floor(stage, run_out) - for runner, n, floor, expect, warn in details: - tag = "warn" if warn else ("OK" if n >= floor else " <-- BELOW floor") - print(f"[sp_rule] {runner}: {n}/{expect} (floor {floor}) {tag}", - file=sys.stderr) - if not ok: - print(f"[sp_rule] FATAL {stage}: count floor breached in {run_out}", - file=sys.stderr) - sys.exit(1) - - -def main() -> None: - p = argparse.ArgumentParser(description=__doc__) - p.add_argument("--stage", required=True) - p.add_argument("--config", required=True, help="config filename in --config-src") - p.add_argument("--unit", required=True) - p.add_argument("--level", required=True, choices=["tile", "exp", "run"]) - p.add_argument("--run-dir", required=True, type=Path, help="$SP_RUN root") - p.add_argument("--config-src", required=True, type=Path, - help="dir holding the canonical configs") - p.add_argument("--star-cats", type=Path, default=None, - help="shared star-cat pool (tiles/ + exp/ subdirs)") - p.add_argument("--threads", type=int, default=1, - help="SMP_BATCH_SIZE == cpus_per_task (fork width)") - p.add_argument("--exp-forest", type=Path, default=None, - help="$SP_EXP: per-tile exposure forest (tile post-stages)") - p.add_argument("--exp-name", default=None, - help="original exposure name incl. suffix (e.g. 2605805p); " - "written into the fabricated exp_numbers list so " - "get_images finds .fits.fz in the store") - p.add_argument("--no-isolate", action="store_true", - help="skip NUMBER_LIST (get_images / exp_mask / exp_psf)") - p.add_argument("--ngmix-chunk", type=int, default=None, - help="1-indexed ngmix chunk K (with --ngmix-nchunks)") - p.add_argument("--ngmix-nchunks", type=int, default=None, - help="total ngmix chunks N (with --ngmix-chunk)") - p.add_argument("--n-split-max", type=int, default=None, - help="N_SPLIT_MAX for the merge_sep_cats template") - p.add_argument("--final-cat", type=Path, default=None, - help="copy the produced final_cat FITS here (make_cat)") - p.add_argument("--script-hash", default=None, - help="ignored at runtime: carries the wrapper-scripts content " - "hash into the rule's params so the params rerun-trigger " - "covers script edits (the code trigger only hashes the " - "rule's own shell string)") - args = p.parse_args() - - ngmix_mode = args.ngmix_chunk is not None - if ngmix_mode and args.ngmix_nchunks is None: - raise SystemExit("--ngmix-chunk requires --ngmix-nchunks") - - work = unit_work_dir(args.run_dir, args.level, args.unit) - materialise_unit(work, args.level, args.unit, args.config_src, - args.star_cats or args.run_dir, exp_name=args.exp_name) - - # Build the config copy. ngmix chunks and merge fill template placeholders - # first; every copy is then normalized (RUN_DATETIME=False, SMP_BATCH_SIZE, - # [+ NUMBER_LIST]). - src_text = (args.config_src / args.config).read_text() - isolate = not args.no_isolate - if ngmix_mode: - n_obj = tile_object_count(work) - lo, hi = ngmix_id_ranges(n_obj, args.ngmix_nchunks)[args.ngmix_chunk - 1] - print(f"[sp_rule] ngmix chunk {args.ngmix_chunk}/{args.ngmix_nchunks}: " - f"n_obj={n_obj} -> ID_OBJ [{lo}, {hi}]", file=sys.stderr) - src_text = apply_ngmix_subs(src_text, args.ngmix_chunk, lo, hi) - cfg_name = f"config_tile_Ng{args.ngmix_chunk}u.ini" # distinct per chunk - elif args.n_split_max is not None: - src_text = apply_merge_subs(src_text, args.n_split_max) - cfg_name = args.config - else: - cfg_name = args.config - - cfg_mod_dir = work / "cfis_mod" - cfg_mod = cfg_mod_dir / cfg_name - cfg_text = normalize_config_text(src_text, args.unit, isolate, args.threads) - cfg_mod_dir.mkdir(parents=True, exist_ok=True) - cfg_mod.write_text(cfg_text) - - prefix = run_prefix_from_text(cfg_text, str(cfg_mod)) - run_out = work / "output" / prefix - - # Pre-delete the deterministic run dir (delete-before-rerun for the hand-run - # escape hatch; ShapePipe's FileHandler.mkdir raises on an existing run dir). - if run_out.exists(): - shutil.rmtree(run_out) - - env = dict(os.environ) - env["SP_RUN"] = str(work) - env["SP_CONFIG"] = str(work / "cfis") # canonical dir (.mask refs), not cfis_mod - if args.exp_forest: - env["SP_EXP"] = str(args.exp_forest) - # Thread caps also come via apptainer-args; keep here for the hand-run path. - env.setdefault("OMP_NUM_THREADS", "1") - env.setdefault("OPENBLAS_NUM_THREADS", "1") - env.setdefault("MKL_NUM_THREADS", "1") - env.setdefault("NUMEXPR_NUM_THREADS", "1") - env.setdefault("MALLOC_ARENA_MAX", "2") - env.setdefault("MALLOC_TRIM_THRESHOLD_", "0") - - # Log sync before the run (after pre-delete). Skipped for ngmix chunks: N - # concurrent chunks would race a full rewrite (literal INPUT_DIRs make it moot). - if not ngmix_mode: - subprocess.call(["update_runs_log_file.py"], cwd=str(work), env=env) - - print(f"[sp_rule] stage={args.stage} unit={args.unit} work={work}", - file=sys.stderr) - rc = subprocess.call(["shapepipe_run", "-c", str(cfg_mod)], - cwd=str(work), env=env) - if rc != 0: - print(f"[sp_rule] WARN shapepipe_run exit {rc} — checking count floor", - file=sys.stderr) - - if not ngmix_mode: - subprocess.call(["update_runs_log_file.py"], cwd=str(work), env=env) - - # Zero-output floor for stages without a count table; count floor otherwise. - if args.stage in COMPLETENESS: - count_floor(run_out, args.stage) - else: - produced = list(run_out.glob("**/output/*")) if run_out.exists() else [] - if not produced: - print(f"[sp_rule] FATAL {args.stage} {args.unit}: zero output in {run_out}", - file=sys.stderr) - sys.exit(1) - - # make_cat: publish the science product to its declared, protected path. - if args.final_cat is not None: - finals = sorted(run_out.glob("**/output/final_cat*.fits")) - if not finals: - print(f"[sp_rule] FATAL {args.stage} {args.unit}: no final_cat under {run_out}", - file=sys.stderr) - sys.exit(1) - args.final_cat.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(finals[0], args.final_cat) - print(f"[sp_rule] final_cat -> {args.final_cat}", file=sys.stderr) - - print(f"[sp_rule] OK {args.stage} {args.unit}", file=sys.stderr) - - -if __name__ == "__main__": - main() From b149e187618b4c086fc6637b4768a8ec4a2346a8 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 30 Jul 2026 22:55:48 -0400 Subject: [PATCH 15/41] fix(orchestration): harden the rewrite against its adversarial review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All eleven findings from the fresh-eyes review, repro-verified: keep-incomplete:true (failure manifests are the post-mortem record — Snakemake otherwise deletes exactly what sp report reads); sp run survives a partial prepare (the missing-threshold is the gate, not set -e); the parse-time index build runs only under SP_PHASE=compute, always evaluates the threshold there, and refuses a silent empty run (WorkflowError on zero ready tiles) while passthrough invocations (--unlock, --dag, targets) never build; profile set-threads/set-resources blocks deleted — they silently replaced the rules' attempt-scaled lambdas and fork widths; manifests write only on content change (a byte-identical rewrite churned the whole cone via mtime); per-tile edge refresh in build_index (stale tile→exposure edges are unrepresentable-shrink no more); blame join blocks on failed/missing only, never warn (else all-tiles-blocked-by-all- exposures at production attrition); ngmix range eval no longer swallows script failure; compute DAG gets a regeneration edge to tile_find_exposures; threshold check precedes any durable index write. Manifest 'unit' is the human ID (basename of SP_RUN); build_forest handles a real-dir dst; stale comments (N_SPLIT_MAX expansion, && check form, sp_rule/protected refs) corrected. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01B5KZRTAZKSBAnpdDhE7yzK --- profiles/nibi/config.yaml | 71 ++++------------- workflow/Snakefile | 37 ++++++--- workflow/bin/sp | 26 +++++- .../config/cfis/config_merge_sep_cats.ini | 12 ++- workflow/rules/tile.smk | 19 ++++- workflow/scripts/build_forest.py | 9 ++- workflow/scripts/build_index.py | 79 +++++++++++++------ workflow/scripts/completeness.py | 48 +++++++---- workflow/scripts/run_report.py | 19 ++++- 9 files changed, 200 insertions(+), 120 deletions(-) diff --git a/profiles/nibi/config.yaml b/profiles/nibi/config.yaml index 07f68833a..029329aa0 100644 --- a/profiles/nibi/config.yaml +++ b/profiles/nibi/config.yaml @@ -37,67 +37,26 @@ rerun-incomplete: true # re-do jobs left incomplete by an unclean deat show-failed-logs: true printshellcmds: true +# D3 DEPENDS ON THIS. Snakemake's default is to DELETE the declared outputs of a +# failed job — and this workflow's declared output IS the manifest, the only +# record of *why* a unit failed and the only thing `sp report` reads. Without +# keep-incomplete, every failed unit reads back as "not_run" and the post-mortem +# debris is gone. Safe here because each rule `rm -rf`s its own run dir at start, +# so a rerun never sees stale products. +keep-incomplete: true + # rerun-triggers left at the full Snakemake v9 default # {mtime,params,input,code,software-env}: params/code fixes SHOULD propagate. -# No override — the wrapper writes outputs only on change so mtimes move only -# when reality moves; protected() guards finished science products. +# No override — completeness.py writes the manifest only on change, so mtimes +# move only when reality moves. default-resources: mem_mb: 4000 runtime: 120 # minutes cpus_per_task: 1 -# Per-stage resources — measured footprints from the nibi P3 runs (#808). -# threads == cpus_per_task == ShapePipe SMP fork width (sp_rule sets -# SMP_BATCH_SIZE={threads}); mem_mb == threads x measured per-worker footprint. -# -# set-threads is REQUIRED for {threads} to resolve above 1: the local scheduler -# derives cpus_per_task from threads, NOT the reverse (verified snakemake 9.23 — -# set-resources cpus_per_task alone leaves {threads}==1, i.e. SMP_BATCH_SIZE=1, -# serial). Keep the two blocks in lockstep. -set-threads: - exp_split: 4 - exp_mask: 4 - exp_psf: 8 - tile_mask: 16 - tile_detect: 16 - tile_merge_headers: 8 - tile_vignets: 8 - tile_ngmix: 8 - tile_merge_cats: 4 - tile_make_cat: 4 - -set-resources: - exp_split: - cpus_per_task: 4 - mem_mb: 8000 - exp_mask: - cpus_per_task: 4 - mem_mb: 8000 - exp_psf: - cpus_per_task: 8 - mem_mb: 16000 - runtime: 240 - tile_mask: - cpus_per_task: 16 - mem_mb: 16000 - tile_detect: - cpus_per_task: 16 - mem_mb: 16000 - tile_merge_headers: - cpus_per_task: 8 - mem_mb: 8000 - tile_vignets: - cpus_per_task: 8 - mem_mb: 24000 - runtime: 240 - tile_ngmix: - cpus_per_task: 8 - mem_mb: 24000 - runtime: 360 - tile_merge_cats: - cpus_per_task: 4 - mem_mb: 8000 - tile_make_cat: - cpus_per_task: 4 - mem_mb: 8000 +# NO set-threads / set-resources here, deliberately. Profile overrides REPLACE a +# rule's own values (verified snakemake 9.23), which would kill the +# attempt-scaled `mem_mb = lambda wc, attempt: ...` OOM retries and the tuned +# ngmix thread count. The RULES own threads and resources; this profile only +# sizes the allocation and sets the defaults for rules that state nothing. diff --git a/workflow/Snakefile b/workflow/Snakefile index b707fdc17..7e9300c5f 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -28,6 +28,8 @@ import sqlite3 import sys from pathlib import Path +from snakemake.exceptions import WorkflowError + # Resolved relative to THIS file, not the working directory: snakemake runs with # --directory on /scratch (bin/sp) so .snakemake/ state never lands on /project # (group quota is a hard 27/27 TiB — a metadata write mid-run died on it live). @@ -51,9 +53,12 @@ sys.path.insert(0, str(SCRIPTS)) import build_index # noqa: E402 from completeness import STAGE_DIR # noqa: E402 -# `prepare` suppresses the missing-tile threshold (tiles are still being -# prepared) and the report hooks. Unset == compute (a bare hand-run of `all`). -PHASE = os.environ.get("SP_PHASE", "compute") +# SP_PHASE is set by bin/sp on the two invocations of `sp run` and NOWHERE else. +# It gates the two parse-time side effects — the index build and the report +# hooks — so that a passthrough invocation (`sp --unlock`, `sp --dag`, `sp +# exp_psf ...`) never mutates durable state or dies on a threshold it was not +# asked about. Unset means "read the index, build nothing". +PHASE = os.environ.get("SP_PHASE", "") with open(config["tile_list"]) as f: TILES = [ln.strip() for ln in f if ln.strip()] @@ -72,15 +77,18 @@ scattergather: NGMIX_CHUNKS = workflow._scatter["ngmix"] # --- parse-time index build + load (D1) ------------------------------------ -# Invocation 2's parse IS the index build. Tiles whose find_exposures output is -# absent land in index/missing.json and are dropped from the DAG; the build is -# fatal only above SP_MISSING_THRESHOLD (and never in the prepare phase, where -# absent Fe output is the normal state). -if any(build_index.exp_list_path(RUN_DIR, t).exists() for t in TILES): +# The COMPUTE invocation's parse IS the index build, and it runs UNCONDITIONALLY +# there — no "some Fe output exists" guard. That guard used to make a +# totally-failed prepare produce an empty index, an empty DAG and a green exit 0; +# with it gone, zero Fe outputs means a missing fraction of 1.0, which trips the +# SP_MISSING_THRESHOLD gate and fails loudly, as it should. +# +# In every other phase (prepare, or unset for a passthrough invocation) the parse +# builds NOTHING and only loads whatever index is already on disk. +if PHASE == "compute": build_index.build( TILES, RUN_DIR, INDEX_DB, - missing_threshold=(None if PHASE == "prepare" - else float(os.environ.get("SP_MISSING_THRESHOLD", "0.0")))) + missing_threshold=float(os.environ.get("SP_MISSING_THRESHOLD", "0.0"))) # EXP: exposure base-id -> original name (2605805 -> 2605805p; the name goes # verbatim into the fabricated per-unit exp_numbers list so get_images matches @@ -97,6 +105,15 @@ if INDEX_DB.exists(): # campaign, so it is intersected with the declared list, not used as it. TILES_READY = [t for t in TILES if t in TILE_EXP] +# A compute invocation with nothing to compute is never a success. Without this, +# an empty intersection yields `rule all` with no inputs, an empty DAG and exit +# 0 — the silent green run the threshold gate exists to prevent. +if PHASE == "compute" and not TILES_READY: + raise WorkflowError( + f"No declared tile has an indexed exposure list: 0 of {len(TILES)} tiles " + f"are ready to compute (index {INDEX_DB}). Run the prepare phase first " + f"(`sp run`), or check {INDEX_DB.parent / 'missing.json'}.") + wildcard_constraints: tile = r"\d{3}\.\d{3}", exp = r"\d{6,7}", diff --git a/workflow/bin/sp b/workflow/bin/sp index 1ead411a3..3424584d1 100755 --- a/workflow/bin/sp +++ b/workflow/bin/sp @@ -51,8 +51,30 @@ cmd="${1:-}" case "$cmd" in run) shift - SP_PHASE=prepare sm prepare_all_tiles "$@" - SP_PHASE=compute sm all "$@" + # PREPARE failing is NOT fatal to the run: keep-going means a failed tile + # poisons only its own cone, and the tiles that lost their exposure list are + # dropped at the compute parse. The real gate is SP_MISSING_THRESHOLD, which + # the compute parse's index build enforces over the WHOLE tile list — so we + # record the failure and go on rather than letting `set -e` abort here. + prep_rc=0 + SP_PHASE=prepare sm prepare_all_tiles "$@" || prep_rc=$? + if [ "$prep_rc" -ne 0 ]; then + echo "" >&2 + echo "############################################################" >&2 + echo "## WARNING: the PREPARE phase exited $prep_rc." >&2 + echo "## Some tiles may be missing their exposure list and will be" >&2 + echo "## dropped from the compute DAG. Continuing to COMPUTE; the" >&2 + echo "## SP_MISSING_THRESHOLD gate (now $SP_MISSING_THRESHOLD) decides" >&2 + echo "## whether that is tolerable." >&2 + echo "############################################################" >&2 + echo "" >&2 + fi + + comp_rc=0 + SP_PHASE=compute sm all "$@" || comp_rc=$? + if [ "$prep_rc" -ne 0 ] || [ "$comp_rc" -ne 0 ]; then + exit "$([ "$comp_rc" -ne 0 ] && echo "$comp_rc" || echo "$prep_rc")" + fi ;; report) shift diff --git a/workflow/config/cfis/config_merge_sep_cats.ini b/workflow/config/cfis/config_merge_sep_cats.ini index dbaf12334..750fe1124 100644 --- a/workflow/config/cfis/config_merge_sep_cats.ini +++ b/workflow/config/cfis/config_merge_sep_cats.ini @@ -84,11 +84,9 @@ NUMBERING_SCHEME = -000-000 WARNING = always # Maximum number of separated catalogues per input. -# BLOCKER: merge_sep_cats_runner.py reads this with plain config.getint (not -# getexpanded) -- $NGMIX_N_CHUNKS would NOT expand here, unlike ID_OBJ_MIN/MAX -# in the ngmix module. Committed as a constant matching workflow/config.yaml's -# ngmix_chunks: 8; if that value changes, this line must be edited by hand -# (or the module gets the same getexpanded treatment ngmix_runner got). -# Chunk count comes from the workflow (single source of truth: the -# scattergather count); env-expanded by the module. +# merge_sep_cats_runner.py reads this with getexpanded (runner .py line ~31), so +# $NGMIX_N_CHUNKS DOES expand here, exactly like ID_OBJ_MIN/MAX in the ngmix +# module. The tile_merge_cats rule exports it from the workflow's scattergather +# chunk count, which is the single source of truth; this value must equal that +# count, so do not replace it with a literal. N_SPLIT_MAX = $NGMIX_N_CHUNKS diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index a0f048a59..e894309b9 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -63,6 +63,12 @@ rule tile_merge_headers: input: forest = rules.tile_exp_forest.output.forest, split = tile_exp_split, + # config_tile_Mh_exp.ini reads run_sp_tile_Fe output, which the PREPARE + # phase produced. Declaring the Fe manifest gives the COMPUTE DAG a + # regeneration path for it instead of a silent dependency on a + # previous invocation (prepare.smk is included in every parse, so the + # rule exists here too). Normally a satisfied no-op. + fe = f"{TILE_DIR}/manifests/tile_find_exposures.json", output: manifest = f"{TILE_DIR}/manifests/tile_merge_headers.json" params: @@ -109,6 +115,9 @@ rule tile_vignets: forest = rules.tile_exp_forest.output.forest, split = tile_exp_split, psf = tile_exp_psf, + # config_tile_PiViVi.ini reads run_sp_tile_Fe output — same reason as + # tile_merge_headers above. + fe = f"{TILE_DIR}/manifests/tile_find_exposures.json", output: manifest = f"{TILE_DIR}/manifests/tile_vignets.json", store = temp(directory(f"{TILE_DIR}/output/run_sp_tile_PiViVi")), @@ -144,8 +153,14 @@ rule tile_ngmix: pre = lambda wc: unit_pre( "tile_ngmix", "tile", wc.tile, env={"SP_NGMIX_CHUNK": wc.chunk, "NGMIX_N_CHUNKS": NGMIX_CHUNKS}, - pre_run=[f'eval "$(python {SCRIPTS}/ngmix_range.py --run-dir ' - f'"$SP_RUN" --chunk {wc.chunk} --n-chunks {NGMIX_CHUNKS})"']), + # Two steps, not `eval "$(...)"`: a command substitution inside eval + # discards the script's exit status, so a missing sexcat would fall + # through to shapepipe_run with an unset range and fail as something + # else. Capture, check, then eval — the range script fails as itself. + pre_run=[f'ngmix_range_out=$(python {SCRIPTS}/ngmix_range.py --run-dir ' + f'"$SP_RUN" --chunk {wc.chunk} --n-chunks {NGMIX_CHUNKS}) ' + f'|| exit 1', + 'eval "$ngmix_range_out"']), script_hash = SCRIPT_HASH threads: 4 retries: 2 diff --git a/workflow/scripts/build_forest.py b/workflow/scripts/build_forest.py index 7ccaa308e..427e9c647 100644 --- a/workflow/scripts/build_forest.py +++ b/workflow/scripts/build_forest.py @@ -14,6 +14,7 @@ """ import argparse +import shutil import sqlite3 from pathlib import Path @@ -36,8 +37,14 @@ def main() -> None: src = args.run_dir / "exp" / e[:2] / e / "output" dst = args.forest / e[:2] / e / "output" # sharded: the module glob's shape dst.parent.mkdir(parents=True, exist_ok=True) + # A symlink (the normal case) is unlinked; a REAL directory left behind + # by a hand-run or an older layout must be removed as a tree — unlink() + # raises IsADirectoryError on it and would kill the job. if dst.is_symlink() or dst.exists(): - dst.unlink() + if dst.is_dir() and not dst.is_symlink(): + shutil.rmtree(dst) + else: + dst.unlink() dst.symlink_to(src) print(f"[build_forest] {args.tile}: {len(exps)} exposures -> {args.forest}") diff --git a/workflow/scripts/build_index.py b/workflow/scripts/build_index.py index 31fc6178b..73d08609a 100644 --- a/workflow/scripts/build_index.py +++ b/workflow/scripts/build_index.py @@ -5,9 +5,19 @@ at parse time into plain dicts, so extending a run's tile list changes which jobs exist without touching the mtime chain of completed work. -It ACCUMULATES: tables are created if absent and rows upserted, never dropped -(D1). Successive invocations widen it, so a later ``clean_exposure`` sees every -consuming tile of the whole campaign rather than of the current tile list. +It ACCUMULATES ACROSS INVOCATIONS, but is AUTHORITATIVE FOR THE CURRENT TILE +LIST (D1). Precisely: + + * a tile in the current list that has find_exposures output has its ``tiles`` + row replaced and its ``tile_exposures`` edges DELETED AND REBUILT — the Fe + output is the truth, so a tile whose exposure list shrank or changed must + not keep stale edges to exposures it no longer reads; + * a tile NOT in the current list is left completely untouched, which is what + makes the index span the campaign and lets a later ``clean_exposure`` see + every consuming tile; + * ``exposures`` rows only ever accumulate (``INSERT OR IGNORE``). An exposure + no tile references any more is a harmless orphan: nothing reads that table + except by joining through ``tile_exposures``. It records: @@ -19,16 +29,20 @@ ``find_exposures`` output (``exp_numbers--.txt``), which ``find_exposures_runner`` produces by parsing the tile FITS ``HISTORY`` header. So the build is not a DAG node: ``build()`` is imported and called at PARSE TIME -by the Snakefile of the COMPUTE invocation, after the PREPARE invocation has -produced the tiles' find_exposures output. (There is no ``sp index`` verb; the -CLI below stays for hand-inspection.) It +by the Snakefile of the COMPUTE invocation ONLY — ``SP_PHASE == "compute"``, +which ``bin/sp`` sets — after the PREPARE invocation has produced the tiles' +find_exposures output. No other parse builds anything: a prepare parse or a +passthrough invocation (``sp --unlock``, ``sp --dag``) just loads whatever is +already on disk. (There is no ``sp index`` verb; the CLI below stays for +hand-inspection.) It iterates the *declared* tile list and checks each tile's Fe output at its deterministic path (no globbing — O(tile-list) existence checks, respecting the no-``ls``-at-scale ban). A bad tile costs only that tile: it is recorded in ``missing.json`` and the index is built over the rest. The build fails only if -the missing *fraction* exceeds ``missing_threshold`` (``None`` disables the -check — what the PREPARE phase passes, where absent Fe output is normal), so a -keep-going download storm that lost a few tiles does not cost the run. +the missing *fraction* exceeds ``missing_threshold`` (``None`` disables the check +entirely), so a keep-going download storm that lost a few tiles does not cost the +run. The threshold is checked BEFORE anything is written, so a failed build +leaves the previous index and ``missing.json`` intact. Exposure IDs are stored with their trailing single-char suffix stripped (``2243881p`` -> ``2243881``); that ``exp_base`` is the dedup key and the @@ -81,7 +95,24 @@ def build(tile_ids: list[str], run_dir: Path, db_path: Path, tile whose exposure list is missing is recorded in ``missing.json`` and the index is built over the rest (a bad tile costs that tile, not the run). The build is fatal only if the missing fraction exceeds ``missing_threshold``. + + ORDER MATTERS: the threshold is evaluated FIRST, from the missing set, and + the database + ``missing.json`` are written only if it passes. A build that + aborts must leave no trace — an aborted parse that had already mutated + durable state was the bug this ordering fixes. + + The write is idempotent, which is what makes it acceptable that the compute + parse runs it even under ``-n``: re-running over an unchanged tree produces + an identical database. """ + missing = [t for t in tile_ids if not exp_list_path(run_dir, t).exists()] + frac = len(missing) / len(tile_ids) if tile_ids else 0.0 + if missing_threshold is not None and frac > missing_threshold: + raise SystemExit( + f"Missing exposure lists for {len(missing)}/{len(tile_ids)} tile(s) " + f"(fraction {frac:.3f} > threshold {missing_threshold}): {missing}. " + f"Re-run prepare_tiles for them, or raise --missing-threshold.") + db_path.parent.mkdir(parents=True, exist_ok=True) con = sqlite3.connect(db_path) # No DROP: the index accumulates across invocations (D1). @@ -97,34 +128,32 @@ def build(tile_ids: list[str], run_dir: Path, db_path: Path, """ ) - missing = [] + missing_set = set(missing) all_exposures: set[tuple[str, str]] = set() for tile_id in tile_ids: - ra_dir = tile_id.split(".")[0] - exp_file = exp_list_path(run_dir, tile_id) - if not exp_file.exists(): - missing.append(tile_id) + if tile_id in missing_set: continue - exp_pairs = read_exposure_list(exp_file) + ra_dir = tile_id.split(".")[0] + exp_pairs = read_exposure_list(exp_list_path(run_dir, tile_id)) con.execute("INSERT OR REPLACE INTO tiles VALUES (?,?,?)", (tile_id, ra_dir, len(exp_pairs))) - for exp_id, _name in exp_pairs: - con.execute("INSERT OR IGNORE INTO tile_exposures VALUES (?,?)", - (tile_id, exp_id)) + # Replace this tile's edge set wholesale. INSERT OR IGNORE alone only + # ever added, so a tile whose exposure list shrank kept edges to + # exposures it no longer reads — and those stale edges would block it in + # the report and pin those exposures against cleanup. + con.execute("DELETE FROM tile_exposures WHERE tile_id = ?", (tile_id,)) + con.executemany("INSERT INTO tile_exposures VALUES (?,?)", + [(tile_id, exp_id) for exp_id, _ in exp_pairs]) all_exposures.update(exp_pairs) - con.executemany("INSERT OR REPLACE INTO exposures VALUES (?,?)", + # Exposures accumulate: OR IGNORE, never REPLACE (the name never changes, + # and orphans left by a shrunken tile are harmless). + con.executemany("INSERT OR IGNORE INTO exposures VALUES (?,?)", sorted(all_exposures)) con.commit() con.close() (db_path.parent / "missing.json").write_text(json.dumps(missing, indent=2)) - frac = len(missing) / len(tile_ids) if tile_ids else 0.0 - if missing_threshold is not None and frac > missing_threshold: - raise SystemExit( - f"Missing exposure lists for {len(missing)}/{len(tile_ids)} tile(s) " - f"(fraction {frac:.3f} > threshold {missing_threshold}): {missing}. " - f"Re-run prepare_tiles for them, or raise --missing-threshold.") return {"n_tiles": len(tile_ids) - len(missing), "n_exposures": len(all_exposures), "n_missing": len(missing)} diff --git a/workflow/scripts/completeness.py b/workflow/scripts/completeness.py index dd0836c66..1a8820c10 100644 --- a/workflow/scripts/completeness.py +++ b/workflow/scripts/completeness.py @@ -9,17 +9,24 @@ sits between ``floor`` and ``expect`` and is tolerated. This file is also the ``check`` CLI — the second half of every rule's shell line -(PRD D2/D3):: +(PRD D2/D3). The rules capture ShapePipe's return code rather than ``&&``-ing +onto it, so the check runs — and the manifest is written — even when +``shapepipe_run`` failed:: - shapepipe_run -c $SP_CONFIG/config_exp_Ma.ini -b {threads} \\ - && completeness.py check exp_mask {output} + rc=0 + shapepipe_run -c $SP_CONFIG/config_exp_Ma.ini -b {threads} || rc=$? + completeness.py check exp_mask {output} || rc=1 + exit $rc It counts the unit's products under ``$SP_RUN``, writes the manifest at ``{output}``, and exits nonzero iff a mandatory runner is below its floor. The -manifest is ALWAYS written first, failure included: it is the DAG's currency and -the only thing ``run_report.py`` reads, so a failed unit must still leave a -record of *why*. Manifests carry no wall-clock — identical on-disk state must -produce a byte-identical manifest, or the mtime rerun-trigger churns the cone. +manifest is ALWAYS written, failure included: it is the DAG's currency and the +only thing ``run_report.py`` reads, so a failed unit must still leave a record of +*why*. (The profile sets ``keep-incomplete`` so snakemake does not delete that +record on the way out.) Manifests carry no wall-clock, and are rewritten ONLY +when their content changes — identical on-disk state must leave a byte-identical +manifest with an UNMOVED mtime, or the mtime rerun-trigger churns the cone on +every unrelated ``--forcerun``. Per-runner fields: expect nominal file count for a fully complete unit (report yardstick) @@ -246,11 +253,15 @@ def build_manifest(stage, run_dir, unit, stage_subdir=None): return manifest, ok -def _unit_from_env(): - """``SP_UNIT_NUM`` carries the rules' dashed, dash-prefixed form - (``-210-282``, ``-2605805``); the manifest records the human ID.""" - raw = os.environ.get("SP_UNIT_NUM", "") - return raw.lstrip("-") or "unknown" +def _unit_from_run_dir(run_dir): + """The human unit ID: the basename of ``$SP_RUN`` (``210.282``, ``2605805``). + + NOT ``SP_UNIT_NUM``, which carries ShapePipe's dashed numbering form + (``-210-282``) and would put ``210-282`` in the manifest — a key that joins + to nothing. ``run_report`` keys units on the store directory name, which is + exactly this basename, so the two now agree. + """ + return Path(str(run_dir)).name or "unknown" def main(argv=None) -> int: @@ -261,7 +272,8 @@ def main(argv=None) -> int: c.add_argument("manifest", type=Path) c.add_argument("--run-dir", type=Path, default=None, help="the unit's $SP_RUN (default: the env var)") - c.add_argument("--unit", default=None, help="override $SP_UNIT_NUM") + c.add_argument("--unit", default=None, + help="override the unit ID (default: basename of $SP_RUN)") c.add_argument("--stage-dir", default=None, help="override the run_sp_* subdir (default: the stage table)") args = p.parse_args(argv) @@ -271,11 +283,17 @@ def main(argv=None) -> int: print("[completeness] FATAL: $SP_RUN unset and --run-dir not given", file=sys.stderr) return 2 - unit = args.unit or _unit_from_env() + unit = args.unit or _unit_from_run_dir(run_dir) manifest, ok = build_manifest(args.stage, Path(run_dir), unit, args.stage_dir) args.manifest.parent.mkdir(parents=True, exist_ok=True) - args.manifest.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") + # Write ONLY on change. An unconditional write moves the mtime on every run, + # and mtime is a rerun-trigger: a `--forcerun` of one upstream stage would + # then rewrite this manifest byte-identically and drag the whole downstream + # cone along with it. + text = json.dumps(manifest, indent=2, sort_keys=True) + "\n" + if not args.manifest.exists() or args.manifest.read_text() != text: + args.manifest.write_text(text) for runner, r in manifest["runners"].items(): tag = {"complete": "OK", "warn": "warn", "below_floor": "<-- BELOW floor"} diff --git a/workflow/scripts/run_report.py b/workflow/scripts/run_report.py index 05ca92478..1a8d128fd 100644 --- a/workflow/scripts/run_report.py +++ b/workflow/scripts/run_report.py @@ -206,9 +206,24 @@ def main() -> None: "exposures": unit_rows(exps, EXP_STAGES, exp_m), } - # Blame propagation: an incomplete exposure blocks every tile that reads it. + # Blame propagation: a BLOCKING exposure blocks every tile that reads it. # Without this, a tile stalled at tile_vignets looks like its own failure. - bad_exp = {r["unit"] for r in report["exposures"]} + # + # Blocking means "failed" or "never ran" — NOT "warn". Warn is the expected + # per-CCD attrition (setools rejecting a sparse CCD, psfex_interp short an + # epoch); it is present in essentially every exposure at production scale, so + # counting it here made every exposure block every tile and the table said + # nothing. + # Judged over ALL the exposure's stages, not just the first bad one, so an + # exposure that warns early and fails late still blocks. + def _blocks(unit): + for stage in EXP_STAGES: + m = exp_m.get(unit, {}).get(stage) + if m is None or m.get("status", "failed") == "failed": + return True + return False + + bad_exp = {e for e in exps if _blocks(e)} blocked = {t: sorted(set(tile_exp.get(t, [])) & bad_exp) for t in tiles} report["tiles_blocked_by_exposures"] = {t: e for t, e in blocked.items() if e} From 12ecfc2b1c1cc4b9c88e60c38798420d392e5a89 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 30 Jul 2026 23:13:06 -0400 Subject: [PATCH 16/41] feat(orchestration): clean_exposure becomes an in-DAG rule with tombstones (#848 D5, S5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exposure-store reclamation moves from a standalone script you remember to run into the DAG itself. One `clean_exposure` job per exposure takes every consuming tile's `tile_vignets` manifest as `input:` — vignets is the last stage that reads exposure products — deletes the store, and leaves a `cleaned.json` tombstone as its output. Writer, then readers, then cleaner: DAG-ordered, race-free, rolling mid-run while ngmix still grinds. The consumer set comes from the accumulating index (new EXP_TILES inverse edge), so it spans the campaign, not one invocation. The tombstone / late-append interaction, which is the whole design problem here: * The job deletes the exposure's `manifests/` along with its `output/`. That is load-bearing. The manifests are the exposure rules' DECLARED outputs; had they survived, a tile appended after the clean would find the exposure chain "up to date" and run its vignets against a store that is no longer on disk. With them gone the DAG sees an unbuilt chain and regenerates it — the accepted cost of a late append, expressed as ordinary Snakemake bookkeeping rather than a special case. The tombstone deliberately does NOT stand in for those manifests: it is input to nothing, so it can never mask their absence. * Tiles already finished are not disturbed: Snakemake demands a missing intermediate only when something downstream of it must run. * `params.consumers` carries the consumer set, so growing it makes the tombstone stale under the default `params` rerun-trigger; the clean job then reruns after the new tile's vignets, against the enlarged set. Cleaned once per consumer set, not once per campaign. * Nothing is lost to the report: each manifest's content is copied verbatim into the tombstone before deletion. Scope guard: an exposure is only eligible when every consuming tile is either in this run's scope or already has its vignets on disk. Without that parse-time test, requesting a tombstone for an exposure shared with a later batch would drag that batch's whole tile chain into this DAG through the clean rule's input — scope expansion by cleanup. Ineligible exposures are deferred, never lost: the invocation that finishes their last consumer picks them up. Gated by `clean:` in workflow/config.yaml (default false for the 4-tile P0 run, where the exposure store is exactly what you want to inspect) and by SP_PHASE=compute, so prepare and passthrough parses schedule no deletions. Flipping it on later reclaims retroactively — the missing tombstones schedule exactly the outstanding clean jobs. Invariants held: no temp() on exposure-level outputs; the loud WorkflowError on zero ready tiles; keep-incomplete untouched. Dry-runs on the 210/211 quad (19 exposures): prepare 13; compute 145 with clean off (identical to b149e187 under the same disk state); compute 164 with clean on = 145 + 19 clean_exposure. Prepare and passthrough parses show no clean rules and leave the index untouched. Deviation from the gap-map wording: one wildcarded rule with an index-driven input function, not literally one generated rule object per exposure. The DAG is identical, and at DR6 scale ~20k rule objects is a parse cost with nothing bought. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UxXRKzfu6RUGNLv7ytNsHw --- workflow/README.md | 173 ++++++++++++++++++---------- workflow/Snakefile | 54 ++++++++- workflow/config.yaml | 10 ++ workflow/rules/exposure.smk | 40 +++++++ workflow/scripts/clean_exposure.py | 82 +++++++++++++ workflow/scripts/clean_exposures.py | 64 ---------- 6 files changed, 293 insertions(+), 130 deletions(-) create mode 100644 workflow/scripts/clean_exposure.py delete mode 100644 workflow/scripts/clean_exposures.py diff --git a/workflow/README.md b/workflow/README.md index 4d8abd1f1..f42a76300 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -1,56 +1,88 @@ # ShapePipe Snakemake orchestration -Snakemake workflow that orchestrates real-data ShapePipe runs, replacing the +Snakemake workflow that orchestrates real-data ShapePipe runs. It replaces the `curl_canfar_local.sh → run_job_sp_canfar_v2.0.bash → job_sp_canfar_v2.0.bash` -bash layers (and per-site sbatch reimplementations). **Module code is untouched**: -rules call `shapepipe_run -c ` on the existing config chains. Design and -rationale: [CosmoStat/shapepipe#848](https://github.com/CosmoStat/shapepipe/issues/848) +bash layers and the per-site sbatch reimplementations. **Module code is +untouched**: rules call `shapepipe_run -c ` on the existing config +chains. Design and rationale: +[CosmoStat/shapepipe#848](https://github.com/CosmoStat/shapepipe/issues/848) (the living PRD). -## Quick start (nibi) +Use `workflow/bin/sp` for everything. Bare `snakemake all` outside `sp` is +unsupported: `sp` sets the state directory, the SLURM profile, and +`SP_PHASE`, which the Snakefile needs to build the tile/exposure index at +parse time. Running snakemake directly skips all of that. -**One-allocation mode.** `sbatch` one node, then run Snakemake with the *local* -scheduler inside it: it bin-packs the DAG into the allocation (no per-job -sbatch). Each job's shell runs `shapepipe_run` **inside the container** (the -profile's apptainer software-deployment — you never type `apptainer`). +## Quick start (nibi) ```bash -# One-time: a snakemake env on a SHARED filesystem (/project — NOT /tmp, which is -# node-local; the executor re-invokes this python inside jobs). +# One-time: a snakemake env on a SHARED filesystem (/project — NOT /tmp, which +# is node-local; the SLURM executor re-invokes this python inside every job). uv venv /project/def-mjhudson/cdaley/snakemake-env --python 3.12 source /project/def-mjhudson/cdaley/snakemake-env/bin/activate -uv pip install 'snakemake>=9,<10' +uv pip install 'snakemake>=9,<10' 'snakemake-executor-plugin-slurm>=2.7,<3' -# Edit workflow/config.yaml: tile_list, run_dir, config_src, container, star_cats. +# Edit workflow/config.yaml: tile_list, run_dir, container, star_cats. -# The committed launcher loads apptainer/1.4.5 + the /project venv, so a fresh -# shell always has the right state. Inside your allocation: -workflow/bin/sp prepare # prepare_tiles -> build_index.py -> prepare_exposures -workflow/bin/sp run # the main compute DAG +# The committed launcher loads apptainer/1.4.5 + the /project venv, so a +# fresh shell always has the right state. +workflow/bin/sp run # bring products on disk up to date with the tile list +workflow/bin/sp report # emit run_report.json now (mid-run is fine) +workflow/bin/sp cancel # scancel this workflow's jobs ``` -`sp` subcommands: `prepare` (chains all three prepare steps), `prepare_tiles`, -`index`, `prepare_exposures`, `run`, `rerun ` (forced recompute of -protected products), `cancel ` (scancel sweep before `--unlock`), -`clean-exposures [--apply]`, `report`. +Installed and pinned versions on nibi (`/project/def-mjhudson/cdaley/snakemake-env`, +queried 2026-07-30): `snakemake==9.23.1`, `snakemake-executor-plugin-slurm==2.7.1`. +Pin range: `snakemake>=9,<10`, `snakemake-executor-plugin-slurm>=2.7,<3`. The +v8→v9 breaks matter here: `--use-singularity` became `--sdm`, executors became +plugins, and full `rerun-triggers` became the default. + +Anything other than `run`, `report`, `cancel` passes straight through to +snakemake with the workflow's profile and state dir — the escape hatch for +`sp --unlock`, `sp --dag`, `sp exp_psf ...`. + +## Execution: two static invocations + +The exposure job set is data-derived from the tiles' `find_exposures` output, +so it cannot live in the same static DAG that produces it. `sp run` is +therefore two snakemake invocations over one Snakefile: + +1. **PREPARE** — `snakemake prepare_all_tiles`: per-tile static DAG + (`Git_vos → Uz → Fe`), `keep-going` so tile failures are independent. A + nonzero exit here is not fatal to the run — tiles that lost their + exposure list are dropped at the compute parse — but it is a warning: + `SP_MISSING_THRESHOLD` (default 0.0) is the real gate. +2. **COMPUTE** — `snakemake all`: this invocation's *parse* builds the + tile↔exposure index (`build_index.py`, imported at parse time, not a DAG + node) and runs the full tile/exposure compute chain. The index + accumulates across invocations, so appending tiles later changes which + jobs exist without invalidating completed work. + +`sp run` chains both so the UX is one command; both exit codes are checked +and the run fails if either phase failed. -## Execution is three static invocations +## Execution mode: one SLURM job per rule -The exposure job set is data-derived from the tiles' `find_exposures` output, so -it cannot be scheduled in the same static DAG that produces it. Hence: +The profile (`profiles/nibi/config.yaml`) sets `executor: slurm`. Every rule +instance becomes its own SLURM job carrying that rule's own attempt-scaled +resources (`cpus_per_task = threads`, `mem_mb`, `runtime`) and runs inside +`apptainer exec` via the profile's software-deployment method — the workflow +never calls apptainer directly. Snakemake feeds the queue as jobs finish, so +the full campaign's job count never needs to queue at once, and multi-node +scaling is inherent. `jobs:` in the profile caps concurrent submissions at a +fraction of the cluster's per-user submit limit (queried, not invented — see +the comment in the profile file for the query and date). -1. `snakemake prepare_tiles` — per-tile static DAG (Git_vos → Uz → Fe), - `keep-going` so tile failures are independent. -2. `build_index.py` — a **plain script**, not a DAG node. It iterates the - declared tile list, checks each tile's Fe output at its deterministic path (no - glob), writes `run_index.sqlite` + `missing.json`, and exits nonzero only if - the missing *fraction* exceeds a threshold. (There is no `--allow-missing` - Snakemake flag; that mechanism does not exist.) -3. `snakemake prepare_exposures` — per-exposure static DAG (Gie_vos + per-unit - star cats), parseable now the index exists. -4. `snakemake` (main DAG) — the tile/exposure compute chain. +The profile intentionally sets no `set-resources` / `set-threads` overrides: +those replace a rule's own values wholesale, which would kill the +attempt-scaled `mem_mb = lambda wc, attempt: ...` OOM retries and the tuned +ngmix thread count. Rules own their own resources; the profile only supplies +defaults for rules that state nothing. -`sp prepare` chains 1–3 so the UX stays one command. +`group:` labels that fuse short rules (uncompress, merges) into their chunky +neighbours (queue-latency amortization, per the PRD) are **not yet wired**: +they require labels in `workflow/rules/*.smk`, out of scope for this +profile-only pass. ## Layout @@ -58,7 +90,7 @@ it cannot be scheduled in the same static DAG that produces it. Hence: workflow/ Snakefile parse-time index load; global container:; onsuccess/onerror report hooks config.yaml the run: tile list, paths, container, chunk count - bin/sp committed launcher (module load + /project venv + subcommands) + bin/sp committed launcher (module load + /project venv + run/report/cancel) rules/ prepare.smk tile get_images/uncompress/find_exposures + per-unit star cats exposure.smk per-exposure: get_images, split, mask, psf (no temp()) @@ -69,37 +101,50 @@ workflow/ build_forest.py per-tile exposure symlink forest (group-compatible shell) completeness.py the ported count-floor table (shared by sp_rule + run_report) run_report.py standalone report (NOT a DAG node; run_report hooks call it) - clean_exposures.py index-driven exposure-store reclaim (replaces temp()) -profiles/nibi/config.yaml local scheduler; apptainer SDM; resources; keep-going + clean_exposure.py ONE exposure's store + manifests -> tombstone (the clean_exposure rule) +profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; keep-going ``` ## How it works -- **Completeness = the count floor.** Each stage declares its deterministic - `output/run_sp_/` directory as `directory()` output (the per-unit - config copy forces `RUN_DATETIME = False`, so the path is known at DAG time). - After the run, `sp_rule.py` counts products per mandatory runner against - `completeness.py`'s floor and exits nonzero below it; per-CCD attrition between - floor and `expect` is tolerated (the directory simply lacks that product). - This *is* the failure policy — no 3-class taxonomy, no error-signature - whitelist. `--keep-going` isolates a failure to its own cone. -- **Per-unit isolation is by work-dir content, not `-e/--exclusive`.** Every - tile/exposure runs its own `shapepipe_run` in `tiles//` or `exp//`, - with a `cfis` config symlink, `star_cat_{exp,tiles}` symlinks, a - `tile_numbers.txt` (tiles) or fabricated `exp_numbers-000-000.txt` pseudo-Fe - (exposures), and a config copy setting `RUN_DATETIME=False` and - `SMP_BATCH_SIZE={threads}`. `NUMBER_LIST=-` is injected **only** for - stages whose numbering scheme is the unit ID (tile-scheme stages + exp split), - never for get_images / exp_mask / exp_psf. -- **No `temp()` on shared exposure dirs.** Exposures overlap tiles, so temp() - would cascade destructive reruns when a tile is appended. The store is - persistent; `sp clean-exposures` reclaims space only for exposures all of - whose consuming tiles have a final_cat. -- **The index is parse-time data, never a rule input.** Appending tiles changes - which jobs exist without invalidating completed work. Star cats are re-keyed - per unit for the same reason. `final_cat` is `protected()`. +- **The atom is one rule == one `shapepipe_run` on one unit.** Its single + declared output is that unit's manifest + (`/manifests/.json`), not its product files — a missing + CCD is often legitimate, and at DR6 scale per-CCD declaration means + millions of paths. +- **Manifests are the DAG's currency.** `completeness.py check` writes the + manifest, and it is the only record of *why* a unit failed. That is why + the profile sets `keep-incomplete: true` — Snakemake's default deletes a + failed job's declared outputs, which would erase the manifest the report + needs. +- **Completeness is a count floor, not a taxonomy.** After a run, + `sp_rule.py` counts products per mandatory runner against + `completeness.py`'s floor and exits nonzero below it. Per-CCD attrition + between floor and `expect` is tolerated. No 3-class taxonomy, no + error-signature whitelist. `--keep-going` isolates a failure to its own + DAG cone. +- **Stores are sharded.** Every tile/exposure runs its own `shapepipe_run` + in `tiles/<2-char prefix>//` or `exp///`, with a `cfis` + config symlink, `star_cat_{exp,tiles}` symlinks, and a config copy setting + `RUN_DATETIME=False`. Configs are committed under `workflow/config/cfis/` + and version with the rules that set the env vars they interpolate — there + is no `config_src` knob, by construction. +- **The index is parse-time data, never a rule input.** Appending tiles + changes which jobs exist without invalidating completed work. Star cats + are re-keyed per unit for the same reason. `final_cat` is `protected()`. +- **Exposure products are not `temp()`.** Exposures overlap tiles, so + `temp()` would cascade destructive reruns when a tile is appended later. + Reclamation is the in-DAG `clean_exposure` rule instead: one job per + exposure, taking every consuming tile's `tile_vignets` manifest as input + (the campaign-wide consumer set comes from the accumulating index), which + deletes the store *and* the exposure's manifests and leaves a `cleaned.json` + tombstone. Deleting the manifests is what makes a late append correct: the + appended tile finds an unbuilt chain and regenerates it. The `clean:` flag + in `config.yaml` gates it; flipping it on later reclaims retroactively, + since the missing tombstones schedule exactly the outstanding clean jobs. - **Failure is a report, not a gate.** `run_report.py` disk-scans the trees against the count table and enumerates shortfalls (whole-unit absence vs - per-CCD attrition). It is a standalone script — a DAG report node would be - poisoned by the very failures it must enumerate — emitted automatically by the - `onsuccess`/`onerror` hooks and runnable any time via `sp report`. + per-CCD attrition). It runs standalone — a DAG report node would itself be + poisoned by the failures it must enumerate — and fires automatically from + the COMPUTE invocation's `onsuccess`/`onerror` hooks, or on demand via + `sp report`. diff --git a/workflow/Snakefile b/workflow/Snakefile index 7e9300c5f..50dc75779 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -93,12 +93,15 @@ if PHASE == "compute": # EXP: exposure base-id -> original name (2605805 -> 2605805p; the name goes # verbatim into the fabricated per-unit exp_numbers list so get_images matches # .fits.fz in the store). TILE_EXP: tile -> [exp_ids]. -EXP, TILE_EXP = {}, {} +# EXP_TILES is the inverse edge — the CAMPAIGN-WIDE consumer set clean_exposure +# is keyed on (D5). +EXP, TILE_EXP, EXP_TILES = {}, {}, {} if INDEX_DB.exists(): _con = sqlite3.connect(INDEX_DB) EXP = dict(_con.execute("SELECT exp_id, name FROM exposures")) for _tile, _exp in _con.execute("SELECT tile_id, exp_id FROM tile_exposures"): TILE_EXP.setdefault(_tile, []).append(_exp) + EXP_TILES.setdefault(_exp, []).append(_tile) _con.close() # Tiles this run can actually compute: declared AND indexed. The index spans the @@ -162,6 +165,52 @@ def unit_num(unit): # gets its own hash, on the forest rule only. SCRIPT_HASH = hashlib.md5((SCRIPTS / "completeness.py").read_bytes()).hexdigest()[:12] FOREST_HASH = hashlib.md5((SCRIPTS / "build_forest.py").read_bytes()).hexdigest()[:12] +CLEAN_HASH = hashlib.md5((SCRIPTS / "clean_exposure.py").read_bytes()).hexdigest()[:12] + +# --- exposure reclamation (D5, S5) ----------------------------------------- +# `clean:` in config.yaml gates the whole mechanism. Off => the rule generates no +# jobs at all (nothing requests a tombstone); flipping it on later reclaims +# RETROACTIVELY, because the exposures already cleaned are exactly the ones with +# a tombstone, so the missing tombstones schedule exactly the clean jobs. +# Only ever active under SP_PHASE=compute: the prepare and passthrough parses +# read no index of their own and must schedule no deletions. +CLEAN = bool(config.get("clean", False)) and PHASE == "compute" + + +def tombstone(exp): + """The clean_exposure output. Lives BESIDE manifests/, not inside it: the + clean job deletes manifests/ wholesale, and `sp report` scans it.""" + return f"{exp_dir(exp)}/cleaned.json" + + +def clean_consumers(exp): + """Every tile in the campaign that reads this exposure — the set whose + vignets must all exist before the store may go.""" + return sorted(EXP_TILES.get(exp, [])) + + +def clean_targets(): + """Which exposures this invocation may clean. + + An exposure is eligible only when every consuming tile is either in this + run's scope or has already produced its vignets on disk. Without that test, + requesting a tombstone for an exposure shared with a LATER batch would drag + that batch's whole tile chain into this DAG through the clean rule's input — + scope expansion by cleanup, which is not a trade anyone asked for. Ineligible + exposures are simply skipped; the invocation that finishes their last + consumer picks them up. Deferral, never loss. + """ + if not CLEAN: + return [] + ready = set(TILES_READY) + out = [] + for exp, tiles in EXP_TILES.items(): + if not tiles: + continue + if all(t in ready or Path(tile_manifest(t, "tile_vignets")).exists() + for t in tiles): + out.append(tombstone(exp)) + return sorted(out) # --- the shell every rule runs (D2) ---------------------------------------- @@ -264,7 +313,8 @@ localrules: all, prepare_all_tiles rule all: input: - [final_cat(t) for t in TILES_READY] + [final_cat(t) for t in TILES_READY], + clean_targets(), # Invocation 1 — the static per-tile DAG, known from the tile list alone. # keep-going makes tile failures independent; the ones that lose their exposure diff --git a/workflow/config.yaml b/workflow/config.yaml index 942199b14..ab06a459e 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -32,6 +32,16 @@ star_cats: /home/cdaley/projects/def-mjhudson/cdaley/runs/p3-batch1/star_cats # 60-day purge for anything that must survive. index_db: /scratch/cdaley/shapepipe-output/smk-p0/index/run_index.sqlite +# Rolling exposure-store reclamation (D5). When true, the COMPUTE DAG grows one +# `clean_exposure` job per exposure. It fires once every campaign tile that reads +# that exposure has its vignets, deletes the exposure's store AND its manifests, +# and leaves `cleaned.json` (which absorbs the manifests, so the report keeps +# them). Off here because P0 is a 4-tile debugging run where the exposure store +# is exactly what you want to inspect; on for any run big enough to care about +# disk. Flipping it on later reclaims retroactively — the missing tombstones +# schedule exactly the outstanding clean jobs. +clean: false + # ngmix within-tile chunking: static N chunks (closed ID ranges computed # per-tile, in-job, from the tile's own sexcat). ngmix_chunks: 8 diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index 667906223..b752bbee7 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -88,3 +88,43 @@ rule exp_psf: runtime = 240 shell: sp_shell("exp_psf", "config_exp_psfex.ini") + + +# --- reclamation (D5) ------------------------------------------------------- +# The one exception to "no reclamation in this file": clean_exposure OWNS +# exposure-level deletion, and it is a real job, not temp() bookkeeping, because +# an exposure's consumer set closes over the CAMPAIGN. The index supplies that +# set (EXP_TILES, accumulated across invocations); the input is every consuming +# tile's tile_vignets manifest — vignets is the last stage that reads exposure +# products, everything after it reads tile-level files. +# +# Three properties make the late append behave (see clean_exposure.py): +# * the job deletes the exposure's manifests too, so a tile appended after the +# clean sees an unbuilt chain and regenerates it instead of running against +# an empty store. The tombstone deliberately does NOT stand in for those +# manifests — it is not an input to anything but itself. +# * a finished tile is not disturbed: Snakemake demands a missing intermediate +# only when something downstream of it must run. +# * params.consumers carries the consumer set, so growing it makes the +# tombstone stale under the default `params` rerun-trigger; the clean job +# reruns after the new tile's vignets, against the enlarged set. +# +# Not a localrule: at DR6 scale these are ~20k rmtrees and the local-cores cap +# would serialise them behind the aggregation targets. +rule clean_exposure: + input: + lambda wc: [tile_manifest(t, "tile_vignets") + for t in clean_consumers(wc.exp)] + output: + tombstone = f"{EXP_DIR}/cleaned.json" + params: + consumers = lambda wc: ",".join(clean_consumers(wc.exp)), + script_hash = CLEAN_HASH + threads: 1 + resources: + mem_mb = 2000, + runtime = 30 + shell: + f"python {SCRIPTS}/clean_exposure.py" + " --exp-dir $(dirname {output.tombstone}) --exp {wildcards.exp}" + " --tombstone {output.tombstone} --consumers '{params.consumers}'" diff --git a/workflow/scripts/clean_exposure.py b/workflow/scripts/clean_exposure.py new file mode 100644 index 000000000..2d3d40cc5 --- /dev/null +++ b/workflow/scripts/clean_exposure.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Reclaim ONE exposure's store and leave a tombstone (PRD #848 D5, S5). + +Run as the shell of the in-DAG ``clean_exposure`` rule, never by hand: the rule's +``input:`` is every consuming tile's ``tile_vignets`` manifest, so by the time +this executes, every campaign tile that reads this exposure has already extracted +its postage stamps. Writer, then readers, then cleaner — DAG-ordered, race-free. + +What it deletes: the exposure's whole ``output/`` tree (the bulk store — +run_sp_exp_Gie/Sp/Ma/SxSePsfPi) AND its ``manifests/``. Deleting the manifests is +deliberate and load-bearing, not tidiness: + + * the manifests are the exposure rules' DECLARED outputs. If they survived, a + tile appended later would find the exposure chain "up to date" and run + tile_vignets against products that are no longer on disk. With them gone the + DAG sees the chain as unbuilt and regenerates it — the accepted cost of a + late append (D5), expressed as ordinary Snakemake bookkeeping rather than as + a special case. + * Snakemake only demands a missing intermediate when something downstream of it + needs to run, so tiles already finished are NOT rerun by their exposures' + manifests vanishing. + +Nothing is lost to the report: each manifest's content is copied verbatim into +the tombstone under ``manifests`` before deletion, so `sp report` can still say +what this exposure produced and why any runner was short. + +The tombstone records the consumer set it was cleaned against. The rule carries +that same set as a ``params`` value, so when the index grows a new consumer the +tombstone goes stale under the default ``params`` rerun-trigger and the clean job +is rescheduled after the new tile's vignets — the exposure is cleaned once per +consumer set, not once per campaign. +""" + +import argparse +import json +import shutil +import time +from pathlib import Path + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--exp-dir", required=True, type=Path) + p.add_argument("--exp", required=True) + p.add_argument("--tombstone", required=True, type=Path) + p.add_argument("--consumers", default="", + help="comma-separated tile ids this exposure was cleaned against") + args = p.parse_args() + + consumers = [t for t in args.consumers.split(",") if t] + + # Absorb the manifests before they go: the tombstone becomes the exposure's + # surviving record. + manifests = {} + mdir = args.exp_dir / "manifests" + if mdir.is_dir(): + for f in sorted(mdir.glob("*.json")): + try: + manifests[f.stem] = json.loads(f.read_text()) + except (OSError, json.JSONDecodeError) as exc: + manifests[f.stem] = {"unreadable": str(exc)} + + removed = [] + for target in (args.exp_dir / "output", mdir): + if target.exists(): + shutil.rmtree(target) + removed.append(str(target)) + + args.tombstone.parent.mkdir(parents=True, exist_ok=True) + args.tombstone.write_text(json.dumps({ + "exp": args.exp, + "cleaned_at": time.strftime("%Y-%m-%dT%H:%M:%S"), + "consumers": consumers, + "removed": removed, + "manifests": manifests, + }, indent=2) + "\n") + print(f"[clean_exposure] {args.exp}: removed {len(removed)} tree(s) after " + f"{len(consumers)} consuming tile(s)") + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/clean_exposures.py b/workflow/scripts/clean_exposures.py deleted file mode 100644 index 4f46bf19b..000000000 --- a/workflow/scripts/clean_exposures.py +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env python3 -"""Index-driven exposure-store cleanup (`sp clean-exposures`). - -temp() is deliberately NOT used on the shared exposure directories (findings -4/5/6/19): they overlap tiles by construction, so temp() deletion would cascade -destructive reruns across spatial neighbours whenever a tile is appended. Instead -the store is persistent and space is reclaimed explicitly here. - -For each exposure in run_index.sqlite, delete its bulky intermediates -(run_sp_exp_Sp / run_sp_exp_Ma / run_sp_exp_SxSePsfPi) ONLY when every tile that -consumes it (tile_exposures) has its final_cat-.fits present across the whole -output root. An exposure still feeding an unfinished tile is left untouched. - -Caveat: if a later batch appends a tile that shares a deleted exposure, -Snakemake will regenerate that exposure — cleanup trades peak storage for -recompute. Run it between campaign phases, not mid-DAG. --dry-run by default. -""" - -import argparse -import shutil -import sqlite3 -import sys -from pathlib import Path - -PREFIXES = ("run_sp_exp_Sp", "run_sp_exp_Ma", "run_sp_exp_SxSePsfPi") - - -def main() -> None: - p = argparse.ArgumentParser(description=__doc__) - p.add_argument("--run-dir", required=True, type=Path) - p.add_argument("--index", required=True, type=Path) - p.add_argument("--apply", action="store_true", - help="actually delete (default: dry-run, list only)") - args = p.parse_args() - - con = sqlite3.connect(args.index) - edges = con.execute("SELECT exp_id, tile_id FROM tile_exposures").fetchall() - con.close() - - consumers: dict[str, list[str]] = {} - for exp_id, tile_id in edges: - consumers.setdefault(exp_id, []).append(tile_id) - - def tile_done(tile): - return (args.run_dir / "tiles" / tile / f"final_cat-{tile}.fits").exists() - - freed, held = 0, 0 - for exp_id, tiles in consumers.items(): - if not all(tile_done(t) for t in tiles): - held += 1 - continue - for prefix in PREFIXES: - d = args.run_dir / "exp" / exp_id / "output" / prefix - if d.is_dir(): - freed += 1 - print(f"{'DELETE' if args.apply else 'would delete'} {d}") - if args.apply: - shutil.rmtree(d) - print(f"[clean_exposures] {freed} dirs {'freed' if args.apply else 'reclaimable'}, " - f"{held} exposures held (consuming tile unfinished)", file=sys.stderr) - - -if __name__ == "__main__": - main() From e677f679f41b398d7630ce1aeee9494abbef9726 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 30 Jul 2026 23:13:49 -0400 Subject: [PATCH 17/41] feat(orchestration): nibi profile goes slurm-executor (#848 D4, S6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The profile flips from the P0 one-allocation/local-scheduler mode to the PRD's executor: slurm. jobs: 800 = 80% of the queried MaxSubmitPU=1000 on def-mjhudson (sacctmgr, 2026-07-30). default-resources carry mem_mb, runtime, slurm_account: def-mjhudson; keep-incomplete stays (a failed job's manifest is its only post-mortem record). No set-resources/set-threads — rules own their attempt-scaled lambdas. Env pins recorded in the README: snakemake>=9,<10, snakemake-executor-plugin-slurm>=2.7,<3 (installed: 9.23.1 / 2.7.1). group: fusion deferred — it needs rule edits, tracked in the README. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UxXRKzfu6RUGNLv7ytNsHw --- profiles/nibi/config.yaml | 41 ++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/profiles/nibi/config.yaml b/profiles/nibi/config.yaml index 029329aa0..7819a0fdc 100644 --- a/profiles/nibi/config.yaml +++ b/profiles/nibi/config.yaml @@ -1,20 +1,29 @@ # Snakemake profile for the Nibi cluster (Digital Research Alliance). # -# ONE-ALLOCATION mode: sbatch one node up front, then run snakemake with the -# LOCAL scheduler inside it (no per-job sbatch; the local scheduler bin-packs the -# DAG into the allocation against `cores` + `resources: mem_mb`). This is the -# proven nibi path (job_p3_batch1) and the single honest first-pass mode; -# multi-node scaling = multiple tile-disjoint invocations, deferred. +# SLURM-EXECUTOR mode (PRD #848 D-profile): one SLURM job per rule instance, +# each carrying its rule's own attempt-scaled resources (cpus_per_task, +# mem_mb, runtime). Snakemake feeds the queue as jobs finish, so DR6's ~170k +# total jobs never queue at once, and multi-node scaling is inherent — this +# supersedes the earlier one-allocation/local-scheduler mode. # # Launch via `workflow/bin/sp` (loads apptainer/1.4.5, uses the /project venv). # software-deployment-method wraps every job's shell in `apptainer exec` — the # user never types apptainer; the container is set in the Snakefile (local .sif). -# Local scheduler (no `executor:` -> built-in local). Size to the allocation. -cores: 192 # node cores available to the DAG -local-cores: 4 # cap localrule/aggregation concurrency -resources: - mem_mb: 720000 # node RAM cap the local scheduler bin-packs under +executor: slurm + +# Per-user submit cap, queried 2026-07-30 on nibi: +# sacctmgr show assoc user=cdaley format=Account,MaxSubmitPU -P +# -> def-mjhudson_cpu 1000, def-mjhudson_gpu 1000 (MaxSubmitPU; no site-wide +# MaxSubmitJobs in `scontrol show config`, so the association limit governs) +# Set to ~80% of that (1000) so this workflow never starves other submissions +# under the same account. Re-query if the association limits change. +jobs: 800 + +default-resources: + mem_mb: 2000 + runtime: 120 # minutes + slurm_account: def-mjhudson software-deployment-method: [apptainer] # Explicit container environment (finding 2/3): the apptainer SDM otherwise drops @@ -50,13 +59,13 @@ keep-incomplete: true # No override — completeness.py writes the manifest only on change, so mtimes # move only when reality moves. -default-resources: - mem_mb: 4000 - runtime: 120 # minutes - cpus_per_task: 1 - # NO set-threads / set-resources here, deliberately. Profile overrides REPLACE a # rule's own values (verified snakemake 9.23), which would kill the # attempt-scaled `mem_mb = lambda wc, attempt: ...` OOM retries and the tuned # ngmix thread count. The RULES own threads and resources; this profile only -# sizes the allocation and sets the defaults for rules that state nothing. +# sets defaults for rules that state nothing (default-resources above). +# +# `group:` fusion of the short rules (uncompress, merges) into their chunky +# neighbours (PRD D-profile) is NOT set here — it requires labels in the rule +# files themselves, which is out of scope for this profile-only pass. Deferred +# to whichever slice next touches workflow/rules/*.smk. From 96c78cdb1fbb2f48c46b96f1f9c351738718ec60 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 31 Jul 2026 00:08:07 -0400 Subject: [PATCH 18/41] fix(orchestration): stop the clean_exposure cascade; report reads tombstones (#848 D5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An adversarial review of the in-DAG clean_exposure design found eight problems. This fixes all eight. The central one is a rerun cascade. THE CASCADE, AND WHAT ACTUALLY CUT IT clean_exposure deletes an exposure's manifests on purpose: that is what makes a tile appended later rebuild the chain instead of running against an empty store. But those manifests are the tile side's inputs, and an exposure is read by ~7-10 tiles. So one rebuilt exposure made every other tile that reads it rerun, and that rerun rebuilt ITS exposures, spreading across the whole exposure-overlap connected component. On fixture t4 (four tiles chained by shared exposures, all complete, all exposures cleaned, one tile's catalogue deleted), asking for the one damaged tile scheduled all four tiles' chains. ancient() alone does NOT fix it, and the measurement says why. Snakemake suppresses the pending-producer propagation only when every shared file is ancient AND EXISTS (dag.py); reclamation has deleted these. Job counts with ancient() alone were identical to before. Three changes cut it, and t4 proves each: * a FINISHED tile (final_cat on disk) drops the exposure manifests that are gone from its input list, and holds the rest through ancient(). It has already extracted everything it will ever read from them, so the propagation has nowhere to go. Only missing manifests are dropped, so a campaign that has cleaned nothing declares exactly the edges it always did. * clean_exposure declares only IN-SCOPE consumers' vignets manifests. An out-of-scope consumer is checked for existence at parse time instead; declaring it pulled that finished tile's whole chain into the DAG. * the profile drops the `input` rerun-trigger. With it on, the conditional edge reads as "set of input files has changed" and reruns the finished tiles anyway — against a store that is gone. Nothing is lost: a changed exposure list still arrives through the find_exposures manifest's mtime, and the ngmix chunk count rides in params. t4, one damaged tile of four: 82 jobs before, 70 with the edge cut but the trigger on, 28 with all three — exactly the damaged tile's chain, its own two exposures, and the clean jobs. Same 28 whether the tile list is the one tile or all four. Appending a tile that shares a cleaned exposure still rebuilds that exposure and reschedules its clean, and touches no other tile (25 jobs). A complete campaign schedules nothing, clean on or off. The cost to know: --forcerun on a tile whose final_cat exists will not rebuild its reclaimed exposures. Delete the final_cat first. THE OTHER SEVEN * sp report read no tombstones, so every cleaned exposure came back "not_run" and blocked the complete tiles whose completion had authorised the deletion. It now reads the absorbed manifests out of cleaned.json, reports the unit as `cleaned` with its warn counts intact, and never blames it for a tile. Fixture t3: 3 exposures "not run" + 2 tiles blocked, before; 3 cleaned + 0 blocked, after. * clean_ignore_tiles: in config.yaml. A permanently-failed tile pinned its ~80 exposures forever; listed there, it leaves the consumer sets. Retrying it later rebuilds those chains from scratch, and the config says so. * the exp_psf benchmark tsv moved out of manifests/, which reclamation deletes. It is the measured-memory feed for mem_mb sizing (D4). * clean_exposure is a localrule. At DR6 scale it was ~20k sbatch submissions to run rmtree. * `--config clean=false` was truthy, so switching reclamation off switched it on. Config booleans now parse strings, and reject what they cannot read. * the tombstone is written, complete and atomically, BEFORE anything is deleted. The crash window now costs disk, never the record. * the false comments in clean_exposure.py and config.yaml are true again. Dry-runs only; no jobs were run. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UxXRKzfu6RUGNLv7ytNsHw --- profiles/nibi/config.yaml | 22 +++++++-- workflow/README.md | 22 +++++++++ workflow/Snakefile | 71 +++++++++++++++++++++++++----- workflow/config.yaml | 25 ++++++++--- workflow/rules/exposure.smk | 26 +++++++++-- workflow/rules/tile.smk | 63 +++++++++++++++++++++++++- workflow/scripts/clean_exposure.py | 36 +++++++++++---- workflow/scripts/run_report.py | 69 ++++++++++++++++++++++++++--- 8 files changed, 294 insertions(+), 40 deletions(-) diff --git a/profiles/nibi/config.yaml b/profiles/nibi/config.yaml index 7819a0fdc..b79985e62 100644 --- a/profiles/nibi/config.yaml +++ b/profiles/nibi/config.yaml @@ -54,10 +54,24 @@ printshellcmds: true # so a rerun never sees stale products. keep-incomplete: true -# rerun-triggers left at the full Snakemake v9 default -# {mtime,params,input,code,software-env}: params/code fixes SHOULD propagate. -# No override — completeness.py writes the manifest only on change, so mtimes -# move only when reality moves. +# rerun-triggers: the v9 default MINUS `input`. params/code/mtime fixes still +# propagate — completeness.py writes the manifest only on change, so mtimes move +# only when reality moves. +# +# `input` is dropped because reclamation needs the tile->exposure edge to be +# CONDITIONAL: a tile whose final_cat is on disk declares no exposure inputs, so +# a neighbour rebuilding a shared exposure cannot drag it along (see tile.smk). +# With the `input` trigger on, that same conditional reads as "set of input files +# has changed" and reruns every finished tile — against an exposure store that +# reclamation has already deleted. Measured on fixture t4: 70 jobs with the +# trigger, 28 without, for one damaged tile in a four-tile chain. +# +# Nothing this workflow relied on is lost. The two genuinely data-derived input +# sets are covered another way: a changed exposure list arrives through the +# (non-ancient) find_exposures manifest's mtime, and the ngmix chunk count rides +# in params. And the Snakefile refuses to schedule reclamation while the trigger +# is enabled, so a bare snakemake invocation cannot quietly undo this. +rerun-triggers: [mtime, params, code, software-env] # NO set-threads / set-resources here, deliberately. Profile overrides REPLACE a # rule's own values (verified snakemake 9.23), which would kill the diff --git a/workflow/README.md b/workflow/README.md index f42a76300..c64047e7f 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -142,6 +142,28 @@ profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; kee appended tile finds an unbuilt chain and regenerates it. The `clean:` flag in `config.yaml` gates it; flipping it on later reclaims retroactively, since the missing tombstones schedule exactly the outstanding clean jobs. + The tombstone is written *before* anything is deleted, so a crash can cost + disk but never the record. +- **A finished tile declares no reclaimed exposures.** Deleting an exposure's + manifests would otherwise rerun every other tile that reads it, and those + reruns spread across the exposure-overlap component. So a tile whose + `final_cat` exists drops the exposure manifests that are gone from its input + list, and holds the rest through `ancient()`. This is why the profile runs + with `rerun-triggers: [mtime, params, code, software-env]`: the `input` + trigger reads that cut as a reason to rerun the very tiles it protects. + Know the consequence — `--forcerun` on a tile whose `final_cat` exists will + not rebuild its reclaimed exposures. Delete the `final_cat` first. +- **A dead tile can be told to stop pinning exposures.** An exposure is + cleanable only once every consuming tile has its vignets, so one + permanently-failed tile holds its ~80 exposures for the life of the + campaign. List it under `clean_ignore_tiles:` in `config.yaml` and it leaves + the consumer sets. Retrying an ignored tile later is legal and expensive: + its exposure chains are gone and rebuild from scratch. +- **A reclaimed exposure reports as `cleaned`.** `run_report.py` reads the + absorbed manifests out of `cleaned.json`, so a reclaimed exposure keeps its + per-runner counts and blocks no tile. The `exp_psf` benchmark tsv lives + beside `manifests/`, not inside it, so reclamation does not eat the + memory-sizing data. - **Failure is a report, not a gate.** `run_report.py` disk-scans the trees against the count table and enumerates shortfalls (whole-unit absence vs per-CCD attrition). It runs standalone — a DAG report node would itself be diff --git a/workflow/Snakefile b/workflow/Snakefile index 50dc75779..419e56cb7 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -107,6 +107,7 @@ if INDEX_DB.exists(): # Tiles this run can actually compute: declared AND indexed. The index spans the # campaign, so it is intersected with the declared list, not used as it. TILES_READY = [t for t in TILES if t in TILE_EXP] +READY_SET = set(TILES_READY) # A compute invocation with nothing to compute is never a success. Without this, # an empty intersection yields `rule all` with no inputs, an empty DAG and exit @@ -168,13 +169,50 @@ FOREST_HASH = hashlib.md5((SCRIPTS / "build_forest.py").read_bytes()).hexdigest( CLEAN_HASH = hashlib.md5((SCRIPTS / "clean_exposure.py").read_bytes()).hexdigest()[:12] # --- exposure reclamation (D5, S5) ----------------------------------------- + + +def flag(value, default=False): + """Truthiness for a config value that may arrive as a STRING. + + `--config clean=false` delivers the string "false", and every non-empty + string is truthy in Python — a plain bool() read that as ON and scheduled + the deletions the user had just switched off. YAML booleans pass through + unchanged; only strings are parsed, and an unparseable one is an error, not + a guess. + """ + if value is None: + return default + if isinstance(value, str): + v = value.strip().lower() + if v in ("1", "true", "yes", "on"): + return True + if v in ("0", "false", "no", "off", ""): + return False + raise WorkflowError(f"Cannot read {value!r} as a boolean (use true/false).") + return bool(value) + + # `clean:` in config.yaml gates the whole mechanism. Off => the rule generates no # jobs at all (nothing requests a tombstone); flipping it on later reclaims # RETROACTIVELY, because the exposures already cleaned are exactly the ones with # a tombstone, so the missing tombstones schedule exactly the clean jobs. # Only ever active under SP_PHASE=compute: the prepare and passthrough parses # read no index of their own and must schedule no deletions. -CLEAN = bool(config.get("clean", False)) and PHASE == "compute" +CLEAN = flag(config.get("clean", False)) and PHASE == "compute" + +# Reclamation REQUIRES the `input` rerun-trigger to be off (profiles/nibi sets +# the list; tile.smk explains why). The two are already tied together: CLEAN is +# gated on SP_PHASE=compute, which only workflow/bin/sp sets, and bin/sp always +# launches with that profile. A bare snakemake invocation without SP_PHASE +# schedules no clean job at all, so it cannot meet the incompatible combination. +# There is no runtime assertion because the trigger set is not readable at parse +# time (workflow.dag_settings is None until the DAG is built). + +# Tiles that must not pin an exposure's store. See config.yaml: a permanently +# failed tile otherwise holds every exposure it touches (~80) forever, because +# its vignets manifest will never exist and its exposures are therefore never +# eligible. Listing it here drops it from the consumer sets. +CLEAN_IGNORE_TILES = set(config.get("clean_ignore_tiles") or []) def tombstone(exp): @@ -185,8 +223,9 @@ def tombstone(exp): def clean_consumers(exp): """Every tile in the campaign that reads this exposure — the set whose - vignets must all exist before the store may go.""" - return sorted(EXP_TILES.get(exp, [])) + vignets must all exist before the store may go — minus the ignored tiles.""" + return sorted(t for t in EXP_TILES.get(exp, []) + if t not in CLEAN_IGNORE_TILES) def clean_targets(): @@ -199,15 +238,23 @@ def clean_targets(): scope expansion by cleanup, which is not a trade anyone asked for. Ineligible exposures are simply skipped; the invocation that finishes their last consumer picks them up. Deferral, never loss. + + Consumer sets are the IGNORE-FILTERED ones (clean_consumers), so a tile in + `clean_ignore_tiles` neither gates eligibility nor appears in the job's + input — which is the whole point of that list. """ if not CLEAN: return [] - ready = set(TILES_READY) out = [] - for exp, tiles in EXP_TILES.items(): - if not tiles: + for exp, raw in EXP_TILES.items(): + if not raw: continue - if all(t in ready or Path(tile_manifest(t, "tile_vignets")).exists() + # An empty set AFTER filtering means every consumer is ignored: nothing + # is left that could ever read this exposure, so it is eligible now. + # That is the whole point of clean_ignore_tiles — all() of an empty set + # is True, and it is true here in the intended sense. + tiles = clean_consumers(exp) + if all(t in READY_SET or Path(tile_manifest(t, "tile_vignets")).exists() for t in tiles): out.append(tombstone(exp)) return sorted(out) @@ -307,9 +354,13 @@ include: "rules/exposure.smk" include: "rules/tile.smk" # --- top-level targets ------------------------------------------------------ -# Only the aggregation targets are localrules: a mid-chain localrule would break -# `group:` fusion of the compute chains (D4). -localrules: all, prepare_all_tiles +# Only the aggregation targets and clean_exposure are localrules: a mid-chain +# localrule would break `group:` fusion of the compute chains (D4). +# clean_exposure is safe there because it is not in any chain — it hangs off +# `all` — and it is seconds of rmtree. As an sbatch job it would be ~20k +# scheduler submissions at DR6 scale to delete directories; the local-cores cap +# serialises them instead, which costs nothing at rmtree speed. +localrules: all, prepare_all_tiles, clean_exposure rule all: input: diff --git a/workflow/config.yaml b/workflow/config.yaml index ab06a459e..e52aa4457 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -35,13 +35,28 @@ index_db: /scratch/cdaley/shapepipe-output/smk-p0/index/run_index.sqlite # Rolling exposure-store reclamation (D5). When true, the COMPUTE DAG grows one # `clean_exposure` job per exposure. It fires once every campaign tile that reads # that exposure has its vignets, deletes the exposure's store AND its manifests, -# and leaves `cleaned.json` (which absorbs the manifests, so the report keeps -# them). Off here because P0 is a 4-tile debugging run where the exposure store -# is exactly what you want to inspect; on for any run big enough to care about -# disk. Flipping it on later reclaims retroactively — the missing tombstones -# schedule exactly the outstanding clean jobs. +# and leaves `cleaned.json`, which absorbs the manifests — `sp report` reads them +# back out of the tombstone and reports the exposure as `cleaned`. Off here +# because P0 is a 4-tile debugging run where the exposure store is exactly what +# you want to inspect; on for any run big enough to care about disk. Flipping it +# on later reclaims retroactively — the missing tombstones schedule exactly the +# outstanding clean jobs. clean: false +# Tiles that may NOT pin an exposure store (default: empty). +# +# An exposure is eligible for cleaning only once EVERY consuming tile has its +# vignets. One permanently-failed tile therefore holds all ~80 exposures it +# touches for the life of the campaign. A tile listed here is dropped from the +# consumer sets, and its exposures become eligible. +# +# READ THIS BEFORE ADDING A TILE. Ignoring a tile is a decision to give up its +# exposures' stores. If you later retry that tile, those exposure chains are +# gone and will be REBUILT from scratch — get_images, split, mask, psf, per +# exposure. That is correct, and expensive. Ignore a tile when you have decided +# it is dead, not while you are still debugging it. +clean_ignore_tiles: [] + # ngmix within-tile chunking: static N chunks (closed ID ranges computed # per-tile, in-job, from the tile's own sexcat). ngmix_chunks: 8 diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index b752bbee7..e04dd76fc 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -82,7 +82,11 @@ rule exp_psf: threads: 8 retries: 2 benchmark: - f"{EXP_DIR}/manifests/exp_psf.benchmark.tsv" + # BESIDE manifests/, not inside it: clean_exposure deletes manifests/ + # wholesale, and this tsv is the measured-memory feed for mem_mb sizing + # (D4). Inside manifests/ it died with the first reclamation and took + # the campaign's only record of exp_psf's real footprint with it. + f"{EXP_DIR}/exp_psf.benchmark.tsv" resources: mem_mb = lambda wc, attempt: 16000 * attempt, runtime = 240 @@ -109,12 +113,26 @@ rule exp_psf: # tombstone stale under the default `params` rerun-trigger; the clean job # reruns after the new tile's vignets, against the enlarged set. # -# Not a localrule: at DR6 scale these are ~20k rmtrees and the local-cores cap -# would serialise them behind the aggregation targets. +# The tile side reads the exposure manifests through ancient() (see tile.smk), +# which is what keeps this deletion from rebuilding every neighbouring tile. +# This rule's OWN inputs are deliberately not ancient: a tile that really did +# rebuild its vignets must reschedule the cleans of the exposures it read. +# +# A localrule (declared in the Snakefile): it is an rmtree, not science, and one +# sbatch per exposure would be ~20k scheduler submissions at DR6 scale. Local +# execution serialises them under local-cores, which costs nothing at rmtree +# speed and never blocks the compute chains (this rule is in none of them). rule clean_exposure: input: + # ONLY the consumers this invocation may actually build. A consumer that + # is out of scope had its vignets manifest checked for existence at parse + # time (clean_targets' eligibility test) — declaring it here as well would + # pull that finished tile's whole chain into the DAG, where a rebuilt + # shared exposure then reruns it. That is how one damaged tile reached its + # spatial neighbours. In-scope consumers keep their edge: they may run in + # this DAG, so the clean must be ordered after them. lambda wc: [tile_manifest(t, "tile_vignets") - for t in clean_consumers(wc.exp)] + for t in clean_consumers(wc.exp) if t in READY_SET] output: tombstone = f"{EXP_DIR}/cleaned.json" params: diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index e894309b9..8b665d1cf 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -23,8 +23,69 @@ variant is a config + one rule, at the config selector the PRD describes. def tile_exp(wc): return TILE_EXP.get(wc.tile, []) +# --- the tile->exposure edge, and why it is cut for finished tiles ---------- +# +# THE cascade fix. clean_exposure deletes the exposure's manifests on purpose: +# that is what makes a tile appended later rebuild the chain instead of running +# against an empty store. But those manifests are the tile side's inputs, and an +# exposure is read by ~7-10 tiles. So the moment ONE tile's chain rebuilt an +# exposure, every other tile reading it saw "input files updated by another job" +# and reran — and that rerun rebuilt ITS exposures, which reran THEIR other +# consumers, propagating across the whole exposure-overlap connected component. +# On fixture t4, asking for one damaged tile scheduled all four tiles' chains. +# +# Two mechanisms, and only the second one actually cuts it: +# +# 1. ancient() on every exposure-manifest edge. Correct on its own terms — a +# tile has no business rerunning because an exposure manifest is NEWER — and +# it is what keeps a pure-mtime disturbance (a re-touched manifest, a +# restored backup) from waking finished tiles. But ancient() governs +# TIMESTAMPS only. Snakemake propagates "my input is produced by a job that +# will run in this DAG" separately, and ancient does not suppress it +# (measured: t4 counts were identical with ancient alone). +# +# 2. Cutting the RECLAIMED edges of a FINISHED tile — the mechanism that works. +# A tile whose final_cat is on disk needs nothing further from its +# exposures: it has already extracted everything it will ever read. So for +# such a tile the input list drops the manifests that are GONE, and the +# propagation has nowhere to go. An UNfinished tile keeps its full edge set +# and therefore still drags in — and rebuilds — every exposure it needs, +# which is the accepted price of a late append, unchanged. +# +# Only the missing ones are dropped, never a manifest that still exists: a +# campaign that has cleaned nothing then declares exactly the edges it always +# did, and the cut cannot perturb it. +# +# The marker is final_cat, not the tile's own vignets manifest: on a tile whose +# catalogue was lost, the vignets manifest still exists while the vignette store +# (temp()) does not, so keying on vignets would cut the edge on exactly the tile +# that has to rerun, and run it against a deleted exposure store. +# +# THE CUT REQUIRES THE `input` RERUN-TRIGGER TO BE OFF (profiles/nibi sets the +# trigger list). Dropping an input is itself a change in the set of input files, +# which that trigger reads as a reason to rerun — reinstating the very cascade, +# now as "Set of input files has changed", and running finished tiles against a +# store that is gone. Measured on fixture t4, one damaged tile of four: 82 jobs +# with neither fix, 70 with the cut but the trigger on, 28 with both (= exactly +# the damaged tile's own chain, its two exposures, and the clean jobs). +# +# The cost, stated plainly: `--forcerun` on a tile whose final_cat exists will +# NOT rebuild its reclaimed exposures, because those edges are not in the DAG. +# Delete that tile's final_cat first and the whole chain comes back. +# +# What none of this weakens: clean_exposure's own inputs are neither ancient nor +# cut, so a tile that really did rebuild its vignets still reschedules the cleans +# of the exposures it read, and a grown consumer set still travels through +# params.consumers. +def tile_finished(tile): + return Path(final_cat(tile)).exists() + + def exp_manifests(wc, stage): - return [exp_manifest(e, stage) for e in tile_exp(wc)] + paths = [exp_manifest(e, stage) for e in tile_exp(wc)] + if tile_finished(wc.tile): + paths = [p for p in paths if Path(p).exists()] + return [ancient(p) for p in paths] def tile_exp_split(wc): return exp_manifests(wc, "exp_split") def tile_exp_mask(wc): return exp_manifests(wc, "exp_mask") diff --git a/workflow/scripts/clean_exposure.py b/workflow/scripts/clean_exposure.py index 2d3d40cc5..e441fc3b5 100644 --- a/workflow/scripts/clean_exposure.py +++ b/workflow/scripts/clean_exposure.py @@ -21,8 +21,20 @@ manifests vanishing. Nothing is lost to the report: each manifest's content is copied verbatim into -the tombstone under ``manifests`` before deletion, so `sp report` can still say -what this exposure produced and why any runner was short. +the tombstone under ``manifests``, and ``run_report.py`` reads a cleaned +exposure's record out of the tombstone — it reports the unit as ``cleaned``, +warn counts and shortfalls intact, instead of "not run". + +Order matters, and it is the reverse of the obvious one: the tombstone is +written FIRST, complete, and only then is anything deleted. A crash between the +two leaves a tombstone beside a store that is still there — the next invocation +treats the exposure as cleaned and only the disk is lost. Deleting first would +put the crash window where the manifests are already gone and the record that +replaces them was never written, and the report would be blind to that exposure +forever. + +The exp_psf benchmark tsv lives beside ``manifests/``, not inside it, so it +survives this job — it is the measured-memory feed for resource sizing (D4). The tombstone records the consumer set it was cleaned against. The rule carries that same set as a ``params`` value, so when the index grows a new consumer the @@ -60,20 +72,26 @@ def main() -> None: except (OSError, json.JSONDecodeError) as exc: manifests[f.stem] = {"unreadable": str(exc)} - removed = [] - for target in (args.exp_dir / "output", mdir): - if target.exists(): - shutil.rmtree(target) - removed.append(str(target)) + targets = [t for t in (args.exp_dir / "output", mdir) if t.exists()] + # Tombstone first, complete, fsync'd — then delete. See the module docstring: + # the crash window has to sit where the data still exists, not where the + # record does not. args.tombstone.parent.mkdir(parents=True, exist_ok=True) - args.tombstone.write_text(json.dumps({ + tmp = args.tombstone.with_suffix(".json.tmp") + tmp.write_text(json.dumps({ "exp": args.exp, "cleaned_at": time.strftime("%Y-%m-%dT%H:%M:%S"), "consumers": consumers, - "removed": removed, + "removed": [str(t) for t in targets], "manifests": manifests, }, indent=2) + "\n") + tmp.replace(args.tombstone) # atomic: no half-written tombstone, ever + + removed = [] + for target in targets: + shutil.rmtree(target) + removed.append(str(target)) print(f"[clean_exposure] {args.exp}: removed {len(removed)} tree(s) after " f"{len(consumers)} consuming tile(s)") diff --git a/workflow/scripts/run_report.py b/workflow/scripts/run_report.py index 1a8d128fd..43ce8dc62 100644 --- a/workflow/scripts/run_report.py +++ b/workflow/scripts/run_report.py @@ -16,6 +16,11 @@ ``completeness.py check``, carrying per-runner found/expect/floor and log-scraped failure reasons. +A reclaimed exposure has no manifests: ``clean_exposure`` deleted them after +copying them into ``/cleaned.json``. That tombstone is read as the +unit's record and the unit is reported as **cleaned** — not "not run", and it +blocks no tile. + No disk scanning: counting products is the *check's* job, done once at the moment the products were fresh. A unit with no manifest for a stage is "not run" — which is a real and distinct answer from "ran and produced nothing". @@ -69,6 +74,36 @@ def load_manifests(run_dir: Path, sub: str) -> dict: return out +def absorb_tombstones(run_dir: Path, sub: str, manifests: dict) -> set: + """Fill in reclaimed units from their ``cleaned.json``; return their ids. + + A cleaned exposure has NO ``manifests/`` — ``clean_exposure`` deleted it, + after copying every manifest verbatim into the tombstone. Read them back, or + the report inverts the truth exactly when reclamation works: the exposure + shows as "not run" and blocks the very tiles whose completion authorised the + deletion. + + Manifests on disk win if both exist — that is a re-built chain, and the + tombstone is then a stale record of the previous generation. + """ + cleaned = set() + for path in sorted((run_dir / sub).glob("**/cleaned.json")): + unit = path.parent.name + try: + tomb = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError) as exc: + print(f"[run_report] unreadable tombstone {path}: {exc}", file=sys.stderr) + continue + if manifests.get(unit): + continue + for key, m in (tomb.get("manifests") or {}).items(): + if not isinstance(m, dict): + continue + manifests[unit][m.get("stage", key)] = m + cleaned.add(unit) + return cleaned + + def shortfalls(m: dict) -> dict: """``{runner: (found, expect, floor)}`` for every runner under expect.""" return {r: (d["found"], d["expect"], d["floor"]) @@ -84,17 +119,25 @@ def reasons(m: dict) -> list: return out -def tally_level(units, stages, manifests) -> dict: - """Per-stage counts + named unit lists, for one level.""" +def tally_level(units, stages, manifests, cleaned=frozenset()) -> dict: + """Per-stage counts + named unit lists, for one level. + + ``cleaned`` units are counted by the status their absorbed manifests carry + (complete or warn — attrition is preserved) and additionally listed under + ``cleaned``, so a reclaimed campaign reads as reclaimed rather than as a + campaign that never ran. + """ per_stage = {} for stage in stages: - t = {"complete": 0, "warn": [], "failed": [], "not_run": []} + t = {"complete": 0, "warn": [], "failed": [], "not_run": [], "cleaned": []} agg = defaultdict(lambda: {"found": 0, "expect": 0, "by_unit": {}}) for u in units: m = manifests.get(u, {}).get(stage) if m is None: t["not_run"].append(u) continue + if u in cleaned: + t["cleaned"].append(u) status = m.get("status", "failed") status = status if status in ("complete", "warn") else "failed" if status == "complete": @@ -158,12 +201,14 @@ def print_table(title, rows, limit=25): def print_stage_table(title, per_stage, n_units): print(f"\n{title} ({n_units} units declared)") - print(f" {'stage':<20} {'ok':>6} {'warn':>6} {'fail':>6} {'not run':>8} attrition") + print(f" {'stage':<20} {'ok':>6} {'warn':>6} {'fail':>6} {'not run':>8} " + f"{'cleaned':>8} attrition") for stage, t in per_stage.items(): att = [f"{r} {a['found']}/{a['expect']}" for r, a in t["products"].items() if a["found"] < a["expect"]] print(f" {stage:<20} {t['complete']:>6} {len(t['warn']):>6} " - f"{len(t['failed']):>6} {len(t['not_run']):>8} {', '.join(att)[:60]}") + f"{len(t['failed']):>6} {len(t['not_run']):>8} " + f"{len(t.get('cleaned', [])):>8} {', '.join(att)[:60]}") def main() -> None: @@ -190,6 +235,8 @@ def main() -> None: tile_m = load_manifests(args.run_dir, "tiles") exp_m = load_manifests(args.run_dir, "exp") + # Reclaimed exposures speak through their tombstones (D5, S5). + cleaned_exp = absorb_tombstones(args.run_dir, "exp", exp_m) tiles = tiles or sorted(tile_m) exps = exps or sorted(exp_m) @@ -201,7 +248,8 @@ def main() -> None: "n_tiles": len(tiles), "n_exposures": len(exps), "missing_tiles": missing, "tile_stages": tally_level(tiles, TILE_STAGES, tile_m), - "exp_stages": tally_level(exps, EXP_STAGES, exp_m), + "exp_stages": tally_level(exps, EXP_STAGES, exp_m, cleaned_exp), + "cleaned_exposures": sorted(cleaned_exp), "tiles": unit_rows(tiles, TILE_STAGES, tile_m), "exposures": unit_rows(exps, EXP_STAGES, exp_m), } @@ -216,7 +264,13 @@ def main() -> None: # nothing. # Judged over ALL the exposure's stages, not just the first bad one, so an # exposure that warns early and fails late still blocks. + # A CLEANED exposure never blocks: its store is gone precisely because every + # consuming tile already had its vignets. Its absorbed manifests are read + # above, so a cleaned exposure that genuinely failed still shows in the + # tables — it just does not get to hold complete tiles hostage. def _blocks(unit): + if unit in cleaned_exp: + return False for stage in EXP_STAGES: m = exp_m.get(unit, {}).get(stage) if m is None or m.get("status", "failed") == "failed": @@ -235,7 +289,8 @@ def _blocks(unit): out.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n") print(f"[run_report] status={args.status} {done}/{len(tiles)} final cats" - + (f" ({len(missing)} tiles missing exposure lists)" if missing else "")) + + (f" ({len(missing)} tiles missing exposure lists)" if missing else "") + + (f" ({len(cleaned_exp)} exposures reclaimed)" if cleaned_exp else "")) print_stage_table("EXPOSURES", report["exp_stages"], len(exps)) print_stage_table("TILES", report["tile_stages"], len(tiles)) print_table("exposures not complete", report["exposures"], args.limit) From 4d28897550cc566b80ce5ea3b171ca5f75fb294a Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 31 Jul 2026 00:09:29 -0400 Subject: [PATCH 19/41] docs(orchestration): profile comment tells the truth about the clean/trigger gate Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UxXRKzfu6RUGNLv7ytNsHw --- profiles/nibi/config.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/profiles/nibi/config.yaml b/profiles/nibi/config.yaml index b79985e62..63997cb05 100644 --- a/profiles/nibi/config.yaml +++ b/profiles/nibi/config.yaml @@ -69,8 +69,10 @@ keep-incomplete: true # Nothing this workflow relied on is lost. The two genuinely data-derived input # sets are covered another way: a changed exposure list arrives through the # (non-ancient) find_exposures manifest's mtime, and the ngmix chunk count rides -# in params. And the Snakefile refuses to schedule reclamation while the trigger -# is enabled, so a bare snakemake invocation cannot quietly undo this. +# in params. And clean scheduling is structurally gated to bin/sp (SP_PHASE=compute +# + this profile), so a bare snakemake invocation cannot quietly recombine +# reclamation with the `input` trigger; a runtime assertion is impossible +# (the trigger set is unreadable at parse time) — the gate is the launcher. rerun-triggers: [mtime, params, code, software-env] # NO set-threads / set-resources here, deliberately. Profile overrides REPLACE a From 831a1273b369762c859622625bb2bbb557e5cbd9 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 31 Jul 2026 00:10:40 -0400 Subject: [PATCH 20/41] run: point the quad at a fresh smk-g3 run dir (old smk-p0 tree is unsharded, pre-manifest) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UxXRKzfu6RUGNLv7ytNsHw --- workflow/config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/workflow/config.yaml b/workflow/config.yaml index e52aa4457..e06235f1a 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -8,14 +8,14 @@ # 210/211 quad: 19 unique exposures, 15 reused across tiles — exercises the # structural exposure dedup (the 196 quad had zero overlap; kept for the # append-invariant test). -tile_list: /scratch/cdaley/shapepipe-output/smk-p0/tiles4.txt +tile_list: /scratch/cdaley/shapepipe-output/smk-g3/tiles4.txt # The container every job runs inside (apptainer software-deployment in the profile). container: /project/def-mjhudson/cdaley/containers/shapepipe-develop-runtime.sif # Where products land ($SP_RUN root). The sharded per-unit stores live under here: # /tiles/<2-char prefix>// and /exp/// -run_dir: /scratch/cdaley/shapepipe-output/smk-p0 +run_dir: /scratch/cdaley/shapepipe-output/smk-g3 # There is no config_src knob: the config chain is workflow/config/cfis, resolved # relative to the Snakefile. The configs interpolate $SP_RUN / $SP_UNIT_NUM / @@ -30,7 +30,7 @@ star_cats: /home/cdaley/projects/def-mjhudson/cdaley/runs/p3-batch1/star_cats # BLOCKED for now: def-mjhudson /project is hard-full (27/27 TiB), so index and # report live with the run on /scratch until space is reclaimed. Mind the # 60-day purge for anything that must survive. -index_db: /scratch/cdaley/shapepipe-output/smk-p0/index/run_index.sqlite +index_db: /scratch/cdaley/shapepipe-output/smk-g3/index/run_index.sqlite # Rolling exposure-store reclamation (D5). When true, the COMPUTE DAG grows one # `clean_exposure` job per exposure. It fires once every campaign tile that reads From f82e7f46e93f95fb348bab05162fedfc509bb7ea Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 31 Jul 2026 00:11:55 -0400 Subject: [PATCH 21/41] fix(orchestration): slurm_account is the association name (def-mjhudson_cpu) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The executor plugin validates against sacctmgr associations, which on nibi carry the _cpu/_gpu suffix — the bare account that sbatch scripts accept is rejected at submission ('appears to be invalid'). First-contact finding from the first real slurm-executor run. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UxXRKzfu6RUGNLv7ytNsHw --- profiles/nibi/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/profiles/nibi/config.yaml b/profiles/nibi/config.yaml index 63997cb05..e4ad8e2c0 100644 --- a/profiles/nibi/config.yaml +++ b/profiles/nibi/config.yaml @@ -23,7 +23,7 @@ jobs: 800 default-resources: mem_mb: 2000 runtime: 120 # minutes - slurm_account: def-mjhudson + slurm_account: def-mjhudson_cpu software-deployment-method: [apptainer] # Explicit container environment (finding 2/3): the apptainer SDM otherwise drops From 5cfdc040cdc181ab6d10aa17458ccfd3dcc70eb5 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 15 Aug 2026 18:47:55 -0400 Subject: [PATCH 22/41] fix(orchestration): index build is the main compute parse's alone MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SLURM executor re-invokes snakemake inside every job, and that re-invocation inherits SP_PHASE=compute — so every job's parse re-ran build_index.build. Hundreds of concurrent sqlite writers on Lustre produced the 'database is locked' storm that killed the first real run (smk-g3, 2026-07-31): the top-level compute parse died first, and the compute jobs a later invocation submitted kept dying in their own parses. - gate the build on workflow.is_main_process as well as SP_PHASE; job parses now only LOAD the index - timeout=60 on every sqlite3.connect (Snakefile loader, build_index, run_report, build_forest): Lustre lock handoffs outlive the 5 s default even under pure reader traffic - config.yaml: the quad is 186/187 (real files); 210/211's tiles are broken symlinks into anaennis' moved processed_tiles tree, as are ~8.5k of the 10.3k staged tiles Co-Authored-By: Claude Fable 5 --- workflow/Snakefile | 13 +++++++++++-- workflow/config.yaml | 7 ++++--- workflow/scripts/build_forest.py | 2 +- workflow/scripts/build_index.py | 2 +- workflow/scripts/run_report.py | 2 +- 5 files changed, 18 insertions(+), 8 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 419e56cb7..54d5a95a0 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -85,7 +85,14 @@ NGMIX_CHUNKS = workflow._scatter["ngmix"] # # In every other phase (prepare, or unset for a passthrough invocation) the parse # builds NOTHING and only loads whatever index is already on disk. -if PHASE == "compute": +# +# is_main_process matters as much as PHASE: the SLURM executor re-invokes +# snakemake inside every job, that re-invocation parses this file again, and it +# inherits SP_PHASE=compute from the submitting environment. Without the guard, +# every one of the run's jobs re-runs the build — hundreds of concurrent sqlite +# writers on Lustre, which is exactly the "database is locked" storm that killed +# the first real run (2026-07-31). Job parses only LOAD the index below. +if PHASE == "compute" and workflow.is_main_process: build_index.build( TILES, RUN_DIR, INDEX_DB, missing_threshold=float(os.environ.get("SP_MISSING_THRESHOLD", "0.0"))) @@ -97,7 +104,9 @@ if PHASE == "compute": # is keyed on (D5). EXP, TILE_EXP, EXP_TILES = {}, {}, {} if INDEX_DB.exists(): - _con = sqlite3.connect(INDEX_DB) + # timeout=60: Lustre lock handoffs are slow; the default 5 s trips on + # nothing more sinister than a reader in another job's parse. + _con = sqlite3.connect(INDEX_DB, timeout=60) EXP = dict(_con.execute("SELECT exp_id, name FROM exposures")) for _tile, _exp in _con.execute("SELECT tile_id, exp_id FROM tile_exposures"): TILE_EXP.setdefault(_tile, []).append(_exp) diff --git a/workflow/config.yaml b/workflow/config.yaml index e06235f1a..d3b417149 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -5,9 +5,10 @@ # tiles) never invalidates completed work — it only changes which jobs exist. # The tile list that scopes this run (one "IDra.IDdec" per line). -# 210/211 quad: 19 unique exposures, 15 reused across tiles — exercises the -# structural exposure dedup (the 196 quad had zero overlap; kept for the -# append-invariant test). +# 186/187 quad: 18 unique exposures with cross-tile reuse — exercises the +# structural exposure dedup. (The 210/211 quad used earlier is unusable: its +# tile images are symlinks into anaennis' moved processed_tiles tree. ~8.5k of +# the 10.3k staged tiles are such broken links; ~600 tiles are real files.) tile_list: /scratch/cdaley/shapepipe-output/smk-g3/tiles4.txt # The container every job runs inside (apptainer software-deployment in the profile). diff --git a/workflow/scripts/build_forest.py b/workflow/scripts/build_forest.py index 427e9c647..f780531e1 100644 --- a/workflow/scripts/build_forest.py +++ b/workflow/scripts/build_forest.py @@ -27,7 +27,7 @@ def main() -> None: p.add_argument("--forest", required=True, type=Path) args = p.parse_args() - con = sqlite3.connect(args.index) + con = sqlite3.connect(args.index, timeout=60) exps = [r[0] for r in con.execute( "SELECT exp_id FROM tile_exposures WHERE tile_id=?", (args.tile,))] con.close() diff --git a/workflow/scripts/build_index.py b/workflow/scripts/build_index.py index 73d08609a..8dc9cb13a 100644 --- a/workflow/scripts/build_index.py +++ b/workflow/scripts/build_index.py @@ -114,7 +114,7 @@ def build(tile_ids: list[str], run_dir: Path, db_path: Path, f"Re-run prepare_tiles for them, or raise --missing-threshold.") db_path.parent.mkdir(parents=True, exist_ok=True) - con = sqlite3.connect(db_path) + con = sqlite3.connect(db_path, timeout=60) # No DROP: the index accumulates across invocations (D1). con.executescript( """ diff --git a/workflow/scripts/run_report.py b/workflow/scripts/run_report.py index 43ce8dc62..567a0c96b 100644 --- a/workflow/scripts/run_report.py +++ b/workflow/scripts/run_report.py @@ -223,7 +223,7 @@ def main() -> None: tiles, exps, tile_exp = [], [], defaultdict(list) if args.index.exists(): - con = sqlite3.connect(args.index) + con = sqlite3.connect(args.index, timeout=60) tiles = [r[0] for r in con.execute("SELECT tile_id FROM tiles ORDER BY 1")] exps = [r[0] for r in con.execute("SELECT exp_id FROM exposures ORDER BY 1")] for tile_id, exp_id in con.execute("SELECT tile_id, exp_id FROM tile_exposures"): From acfb91a0783799d76f0177b3b5986ba2a1ba70f8 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 15 Aug 2026 18:50:59 -0400 Subject: [PATCH 23/41] fix(orchestration): sp pins --snakefile; works from any cwd MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An sbatch head job starts in the submission directory; snakemake's Snakefile discovery is cwd-relative and found nothing. The launcher already promised cwd-independence — now it delivers it. Co-Authored-By: Claude Fable 5 --- workflow/bin/sp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/workflow/bin/sp b/workflow/bin/sp index 3424584d1..ec563f032 100755 --- a/workflow/bin/sp +++ b/workflow/bin/sp @@ -45,7 +45,9 @@ STATE_DIR="${SP_STATE_DIR:-${RUN_DIR}-state}"; mkdir -p "$STATE_DIR" # declared tiles allowed to be missing their exposure list (default 0.0). export SP_MISSING_THRESHOLD="${SP_MISSING_THRESHOLD:-0.0}" -sm() { snakemake --profile "$PROFILE" --directory "$STATE_DIR" "$@"; } +# --snakefile pins the workflow to this checkout: sp must work from any cwd +# (an sbatch head job starts in the submission directory, not the repo). +sm() { snakemake --snakefile "$HERE/Snakefile" --profile "$PROFILE" --directory "$STATE_DIR" "$@"; } cmd="${1:-}" case "$cmd" in From dedc1f0064c349b8b784a1982c8b26e70f3c8d40 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 15 Aug 2026 18:55:03 -0400 Subject: [PATCH 24/41] docs(orchestration): a failed tile pins ~8 exposures, not ~80 P3 fixture measures 7.9 exposures/tile (50 tiles / 181 exposures, sp_p3_batch1-17716364.log). Co-Authored-By: Claude Fable 5 --- workflow/config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow/config.yaml b/workflow/config.yaml index d3b417149..dfb3bb3a0 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -47,8 +47,8 @@ clean: false # Tiles that may NOT pin an exposure store (default: empty). # # An exposure is eligible for cleaning only once EVERY consuming tile has its -# vignets. One permanently-failed tile therefore holds all ~80 exposures it -# touches for the life of the campaign. A tile listed here is dropped from the +# vignets. One permanently-failed tile therefore holds all its exposures +# (~8 measured on P3: 7.9 exposures/tile) for the life of the campaign. A tile listed here is dropped from the # consumer sets, and its exposures become eligible. # # READ THIS BEFORE ADDING A TILE. Ignoring a tile is a decision to give up its From 2a24d3409f3652c152846d98f70e946a6685b5c0 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 15 Aug 2026 19:07:35 -0400 Subject: [PATCH 25/41] fix(orchestration): manifests are success-only DAG currency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A manifest was written on failure too, and `keep-incomplete: true` kept it. A head process that died without recording job failures therefore left failed stages looking complete, and a resume scheduled downstream work on top of them (exp_psf on an empty exp_mask output dir, live). completeness.py now forks on the result: success writes `.json` and removes a stale `.failed.json`; failure writes `.failed.json` and removes a stale `.json`. Exactly one exists, so `.json` means the stage succeeded. Both keep the write-only-on-change behaviour. The failed manifest is never a declared output, so snakemake neither tracks nor deletes it — which lets the profile drop `keep-incomplete` and get the wanted default back: a failed job's declared output is removed. run_report.py reads either name and takes the status from the manifest body, so manifests written under the old convention still report as failed. Co-Authored-By: Claude Fable 5 --- profiles/nibi/config.yaml | 14 +++---- workflow/README.md | 15 +++++--- workflow/Snakefile | 7 +++- workflow/scripts/clean_exposure.py | 8 +++- workflow/scripts/completeness.py | 59 +++++++++++++++++++++--------- workflow/scripts/run_report.py | 18 +++++++-- 6 files changed, 86 insertions(+), 35 deletions(-) diff --git a/profiles/nibi/config.yaml b/profiles/nibi/config.yaml index e4ad8e2c0..f78bea7f7 100644 --- a/profiles/nibi/config.yaml +++ b/profiles/nibi/config.yaml @@ -46,13 +46,13 @@ rerun-incomplete: true # re-do jobs left incomplete by an unclean deat show-failed-logs: true printshellcmds: true -# D3 DEPENDS ON THIS. Snakemake's default is to DELETE the declared outputs of a -# failed job — and this workflow's declared output IS the manifest, the only -# record of *why* a unit failed and the only thing `sp report` reads. Without -# keep-incomplete, every failed unit reads back as "not_run" and the post-mortem -# debris is gone. Safe here because each rule `rm -rf`s its own run dir at start, -# so a rerun never sees stale products. -keep-incomplete: true +# No `keep-incomplete` here (it was the pre-S7a way to preserve failed manifests +# as report evidence): the declared output IS the manifest, so letting snakemake +# delete a failed job's output is exactly the semantics this workflow wants — a +# `.json` on disk then means the stage succeeded, and a resume cannot +# schedule downstream work on top of a failure. The post-mortem evidence lives in +# `.failed.json`, which completeness.py writes on failure and which is +# never a declared output, so snakemake never touches it. # rerun-triggers: the v9 default MINUS `input`. params/code/mtime fixes still # propagate — completeness.py writes the manifest only on change, so mtimes move diff --git a/workflow/README.md b/workflow/README.md index c64047e7f..2183f03e8 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -112,11 +112,16 @@ profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; kee (`/manifests/.json`), not its product files — a missing CCD is often legitimate, and at DR6 scale per-CCD declaration means millions of paths. -- **Manifests are the DAG's currency.** `completeness.py check` writes the - manifest, and it is the only record of *why* a unit failed. That is why - the profile sets `keep-incomplete: true` — Snakemake's default deletes a - failed job's declared outputs, which would erase the manifest the report - needs. +- **Manifests are the DAG's currency, and they are success-only.** + `completeness.py check` writes `.json` when the stage passed its count + floor, and `.failed.json` when it did not — removing the other file + either way. So `.json` on disk means "this stage succeeded", and a + resume after an unclean death cannot schedule downstream work on top of a + failure. The failed manifest carries the same content plus the scraped + reasons; it is never a declared output, so Snakemake never tracks or deletes + it, and `sp report` reads it when the success manifest is absent. The profile + runs *without* `keep-incomplete` for the same reason: deleting a failed job's + declared output is the wanted semantics. - **Completeness is a count floor, not a taxonomy.** After a run, `sp_rule.py` counts products per mandatory runner against `completeness.py`'s floor and exits nonzero below it. Per-CCD attrition diff --git a/workflow/Snakefile b/workflow/Snakefile index 54d5a95a0..55aae67bf 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -346,8 +346,11 @@ def sp_shell(stage, config_name): unit_pre). ``-b {threads}`` makes SMP fork width and cpus_per_task one number by construction (D4). - The check runs even when shapepipe_run failed — the manifest is the only - thing `sp report` reads, so a failed unit must still leave a record of why. + The check runs even when shapepipe_run failed, so a failed unit still leaves + a record of why — but that record goes to `.failed.json`, and the + declared `.json` is removed if it was there. Success manifests are + success-only currency (completeness.py); the failed one is evidence for + `sp report` and is never a declared output. """ return ( "{params.pre}\n" diff --git a/workflow/scripts/clean_exposure.py b/workflow/scripts/clean_exposure.py index e441fc3b5..ce6fef80a 100644 --- a/workflow/scripts/clean_exposure.py +++ b/workflow/scripts/clean_exposure.py @@ -20,11 +20,17 @@ needs to run, so tiles already finished are NOT rerun by their exposures' manifests vanishing. -Nothing is lost to the report: each manifest's content is copied verbatim into +Nothing is lost to the report: every ``manifests/*.json`` is copied verbatim into the tombstone under ``manifests``, and ``run_report.py`` reads a cleaned exposure's record out of the tombstone — it reports the unit as ``cleaned``, warn counts and shortfalls intact, instead of "not run". +The absorption is by GLOB, so it takes whatever is in ``manifests/``, keyed by +file stem; the report re-keys on each manifest's own ``stage`` field. A +``.failed.json`` is therefore carried through unremarkably — it should +never be there (an exposure with a failed stage has no complete vignets consumer +and so is not eligible for cleaning), but it costs nothing to be right about. + Order matters, and it is the reverse of the obvious one: the tombstone is written FIRST, complete, and only then is anything deleted. A crash between the two leaves a tombstone beside a store that is still there — the next invocation diff --git a/workflow/scripts/completeness.py b/workflow/scripts/completeness.py index 1a8820c10..51c2521d8 100644 --- a/workflow/scripts/completeness.py +++ b/workflow/scripts/completeness.py @@ -18,15 +18,25 @@ completeness.py check exp_mask {output} || rc=1 exit $rc -It counts the unit's products under ``$SP_RUN``, writes the manifest at -``{output}``, and exits nonzero iff a mandatory runner is below its floor. The -manifest is ALWAYS written, failure included: it is the DAG's currency and the -only thing ``run_report.py`` reads, so a failed unit must still leave a record of -*why*. (The profile sets ``keep-incomplete`` so snakemake does not delete that -record on the way out.) Manifests carry no wall-clock, and are rewritten ONLY -when their content changes — identical on-disk state must leave a byte-identical -manifest with an UNMOVED mtime, or the mtime rerun-trigger churns the cone on -every unrelated ``--forcerun``. +It counts the unit's products under ``$SP_RUN`` and exits nonzero iff a mandatory +runner is below its floor. WHERE it writes the record is the whole point: + + * success -> ``.json`` (the rule's declared output, the DAG's currency) + and any stale ``.failed.json`` is removed; + * failure -> ``.failed.json`` (same content, ``status: failed``) and any + stale ``.json`` is REMOVED. + +``.json`` therefore means "this stage succeeded", full stop — a resume can +never schedule a downstream stage on top of a failed one, not even after a head +process died without recording the failure. The failed manifest is never a +declared output of any rule: it is post-mortem evidence for ``run_report.py``, +which reads it when the success manifest is absent, and snakemake neither tracks +nor deletes it. + +Manifests carry no wall-clock, and either file is rewritten ONLY when its content +changes — identical on-disk state must leave a byte-identical manifest with an +UNMOVED mtime, or the mtime rerun-trigger churns the cone on every unrelated +``--forcerun``. Per-runner fields: expect nominal file count for a fully complete unit (report yardstick) @@ -253,6 +263,18 @@ def build_manifest(stage, run_dir, unit, stage_subdir=None): return manifest, ok +def failed_path(manifest: Path) -> Path: + """``.json`` -> ``.failed.json``, in the same manifests/ dir.""" + return manifest.with_name(manifest.name[:-len(".json")] + ".failed.json") + + +def write_if_changed(path: Path, text: str) -> None: + """Write only when the bytes differ — see the module docstring on mtime.""" + path.parent.mkdir(parents=True, exist_ok=True) + if not path.exists() or path.read_text() != text: + path.write_text(text) + + def _unit_from_run_dir(run_dir): """The human unit ID: the basename of ``$SP_RUN`` (``210.282``, ``2605805``). @@ -286,21 +308,24 @@ def main(argv=None) -> int: unit = args.unit or _unit_from_run_dir(run_dir) manifest, ok = build_manifest(args.stage, Path(run_dir), unit, args.stage_dir) - args.manifest.parent.mkdir(parents=True, exist_ok=True) - # Write ONLY on change. An unconditional write moves the mtime on every run, - # and mtime is a rerun-trigger: a `--forcerun` of one upstream stage would - # then rewrite this manifest byte-identically and drag the whole downstream - # cone along with it. text = json.dumps(manifest, indent=2, sort_keys=True) + "\n" - if not args.manifest.exists() or args.manifest.read_text() != text: - args.manifest.write_text(text) + + # The success/failure fork. Exactly one of the two files exists afterwards, + # so the presence of .json IS the statement "this stage succeeded" and + # the DAG can never build on top of a failure. The removal of the stale + # counterpart is what makes a success->failure or failure->success transition + # complete rather than additive. + written = args.manifest if ok else failed_path(args.manifest) + stale = failed_path(args.manifest) if ok else args.manifest + write_if_changed(written, text) + stale.unlink(missing_ok=True) for runner, r in manifest["runners"].items(): tag = {"complete": "OK", "warn": "warn", "below_floor": "<-- BELOW floor"} print(f"[completeness] {runner}: {r['found']}/{r['expect']} " f"(floor {r['floor']}) {tag[r['status']]}", file=sys.stderr) print(f"[completeness] {args.stage} {unit}: {manifest['status']} " - f"-> {args.manifest}", file=sys.stderr) + f"-> {written}", file=sys.stderr) for f in manifest["failures"]: for reason in f["reasons"]: print(f"[completeness] {f['runner']}: {reason}", file=sys.stderr) diff --git a/workflow/scripts/run_report.py b/workflow/scripts/run_report.py index 567a0c96b..69c2e4b5f 100644 --- a/workflow/scripts/run_report.py +++ b/workflow/scripts/run_report.py @@ -16,6 +16,13 @@ ``completeness.py check``, carrying per-runner found/expect/floor and log-scraped failure reasons. +A stage that failed leaves ``.failed.json`` instead of ``.json`` +(``completeness.py``): the success name is reserved for success, so the DAG +cannot build on a failure. Both names are read here, and the status comes from +the manifest BODY, so a stage is reported as failed either way — including +manifests written before that split, where the failure content sat under the +success name. + A reclaimed exposure has no manifests: ``clean_exposure`` deleted them after copying them into ``/cleaned.json``. That tombstone is read as the unit's record and the unit is reported as **cleaned** — not "not run", and it @@ -42,7 +49,7 @@ TILE_STAGES = ["tile_get_images", "tile_uncompress", "tile_find_exposures", "tile_merge_headers", "tile_detect", "tile_vignets", "tile_ngmix", "tile_merge_cats", "tile_make_cat"] -EXP_STAGES = ["exp_get_images", "exp_split", "exp_mask", "exp_psf"] +EXP_STAGES = ["exp_get_images", "exp_star_cat", "exp_split", "exp_mask", "exp_psf"] STATUSES = ("complete", "warn", "failed", "not_run") @@ -54,8 +61,13 @@ def load_manifests(run_dir: Path, sub: str) -> dict: agnostic, and the only form that joins to the index (the manifest's own ``unit`` field carries ``SP_UNIT_NUM``'s dashed form, ``210-282``, which is not the index's ``210.282``). The stage comes from the manifest body, never - the filename: ngmix chunks share a stage under per-chunk filenames, and they - collapse to one worst-case entry. + the filename: ngmix chunks share a stage under per-chunk filenames, and + ``.failed.json`` names the same stage as ``.json``. + + Several files can therefore map to one (unit, stage), and the WORST status + wins. That is what collapses the ngmix chunks to one entry, and what makes a + failed manifest speak even in the transient window where its success-named + counterpart has not yet been removed. """ out: dict = defaultdict(dict) for path in sorted((run_dir / sub).glob("**/manifests/*.json")): From a1d4dfac3501e6dd531ebf385e29632665d28e7e Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 15 Aug 2026 19:11:56 -0400 Subject: [PATCH 26/41] feat(orchestration): star-cat creation moves into the DAG MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The PRD assumed mask star catalogues were pre-generated offline. The pre-staged store covers 1 of the current quad's 18 exposures, and symlinking the whole store into a unit breaks the mask module outright: the file handler intersects the image numbers found across a config's INPUT_DIRs, and the store contributes 7240 foreign numbers, so the intersection is empty ("numbers ... do not intersect"). New rule `exp_star_cat`, one per exposure: it runs create_star_cat.py in the container against the exp_get_images symlink farm (one Vizier cone query over the focal plane, ~6 s), then builds a REAL per-unit `star_cat_exp/` of 40 per-CCD symlinks — exactly this exposure's numbers, so the intersection is right by construction and config_exp_Ma.ini needs no change. exp_mask takes its manifest as an input. It is a localrule, so the queries run serially in the head process (an sbatch job on a compute node, which has internet): CDS is protected at any campaign scale and the scheduler never sees a six-second job. The catalogue itself is cached run-independently under `star_cats:`, now the cache root on /scratch, and create_star_cat.py skips an existing cat, so reruns and later campaigns are free. unit_pre no longer materialises the star_cat_exp/star_cat_tiles store symlinks. tile.smk records what a future tile_mask needs on the tile side. Co-Authored-By: Claude Fable 5 --- workflow/README.md | 29 ++++--- workflow/Snakefile | 34 ++++---- workflow/config.yaml | 13 +++- workflow/config/cfis/config_exp_Ma.ini | 4 +- workflow/rules/exposure.smk | 104 ++++++++++++++++++++++++- workflow/rules/prepare.smk | 10 +-- workflow/rules/tile.smk | 8 ++ 7 files changed, 165 insertions(+), 37 deletions(-) diff --git a/workflow/README.md b/workflow/README.md index 2183f03e8..c286ebdcb 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -22,7 +22,8 @@ uv venv /project/def-mjhudson/cdaley/snakemake-env --python 3.12 source /project/def-mjhudson/cdaley/snakemake-env/bin/activate uv pip install 'snakemake>=9,<10' 'snakemake-executor-plugin-slurm>=2.7,<3' -# Edit workflow/config.yaml: tile_list, run_dir, container, star_cats. +# Edit workflow/config.yaml: tile_list, run_dir, container, star_cats (the +# star-catalogue cache root). # The committed launcher loads apptainer/1.4.5 + the /project venv, so a # fresh shell always has the right state. @@ -92,8 +93,8 @@ workflow/ config.yaml the run: tile list, paths, container, chunk count bin/sp committed launcher (module load + /project venv + run/report/cancel) rules/ - prepare.smk tile get_images/uncompress/find_exposures + per-unit star cats - exposure.smk per-exposure: get_images, split, mask, psf (no temp()) + prepare.smk tile get_images/uncompress/find_exposures + exposure.smk per-exposure: get_images, star_cat, split, mask, psf (no temp()) tile.smk per-tile: exp forest, merge_headers, mask, detect, vignets, ngmix, merge, make_cat scripts/ sp_rule.py the thin per-unit wrapper (isolation furniture, config copy, log-sync, count floor) @@ -129,14 +130,22 @@ profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; kee error-signature whitelist. `--keep-going` isolates a failure to its own DAG cone. - **Stores are sharded.** Every tile/exposure runs its own `shapepipe_run` - in `tiles/<2-char prefix>//` or `exp///`, with a `cfis` - config symlink, `star_cat_{exp,tiles}` symlinks, and a config copy setting - `RUN_DATETIME=False`. Configs are committed under `workflow/config/cfis/` - and version with the rules that set the env vars they interpolate — there - is no `config_src` knob, by construction. + in `tiles/<2-char prefix>//` or `exp///`. Configs are + committed under `workflow/config/cfis/` and version with the rules that set + the env vars they interpolate — there is no `config_src` knob, and no per-unit + config symlink; `$SP_CONFIG` points straight at the committed directory. +- **Mask star catalogues are built in the DAG.** `exp_star_cat` runs one Vizier + cone query per exposure into the run-independent cache at `star_cats:`, then + fans it out into a real per-unit `star_cat_exp/` directory of 40 per-CCD + symlinks, which `exp_mask` consumes. The directory must be per-unit and real: + the file handler intersects the image numbers it finds across a config's + `INPUT_DIR`s, so a symlink to the whole cache contributes every other + exposure's numbers and the intersection comes out empty. It is a `localrule`, + so the queries run serially in the head process — CDS is never hammered, and + the scheduler never sees a six-second job. The cache makes reruns and later + campaigns free. - **The index is parse-time data, never a rule input.** Appending tiles - changes which jobs exist without invalidating completed work. Star cats - are re-keyed per unit for the same reason. `final_cat` is `protected()`. + changes which jobs exist without invalidating completed work. - **Exposure products are not `temp()`.** Exposures overlap tiles, so `temp()` would cascade destructive reruns when a tile is appended later. Reclamation is the in-DAG `clean_exposure` rule instead: one job per diff --git a/workflow/Snakefile b/workflow/Snakefile index 55aae67bf..2b2ec322e 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -23,6 +23,7 @@ legitimate, and at DR6 scale per-CCD declaration means millions of paths. """ import hashlib +import json import os import sqlite3 import sys @@ -41,7 +42,13 @@ container: config["container"] # --- paths ----------------------------------------------------------------- RUN_DIR = Path(config["run_dir"]) +# Run-independent cache root for the mask star catalogues, keyed by exposure and +# filled by the exp_star_cat rule (config.yaml explains the placement). STAR_CATS = Path(config["star_cats"]) +# This checkout's own tree: exp_star_cat runs its scripts/python/create_star_cat.py +# against its src/, for the same reason CONFIG_DIR is the committed config dir — +# script, library and rule are one artefact and version together. +REPO_DIR = Path(workflow.basedir).parent INDEX_DB = Path(config["index_db"]) SCRIPTS = Path(workflow.basedir) / "scripts" # The config chain is the repo's own committed dir BY CONSTRUCTION (D2): the @@ -296,11 +303,10 @@ def unit_pre(stage, level, unit, *, exp_name=None, forest=None, env=None, ORIGINAL exposure name from the index (``2605805p``), so get_images matches ``.fits.fz`` in the store — the bare base id matches nothing. Written UNCONDITIONALLY: an exists-guard once pinned a stale - pre-fix file with the bare id; - * ``star_cat_exp`` / ``star_cat_tiles`` dir symlinks into the shared - pre-generated pool (the mask configs read them as INPUT_DIRs). + pre-fix file with the bare id. There is no per-unit ``cfis`` symlink any more: $SP_CONFIG points straight at - the committed config dir. + the committed config dir, and ``star_cat_exp`` is a real per-unit directory + built by the ``exp_star_cat`` rule, not a symlink into a shared pool. Finally it ``rm -rf``s this stage's own fixed run dir — ShapePipe's FileHandler raises on an existing run dir, and it is how a rerun never sees @@ -330,10 +336,6 @@ def unit_pre(stage, level, unit, *, exp_name=None, forest=None, env=None, lines += [f'mkdir -p "{fe}"', f"printf '%s\\n' '{exp_name or unit}' > \"{fe}/exp_numbers-000-000.txt\""] - for name, sub in (("star_cat_exp", "exp"), ("star_cat_tiles", "tiles")): - src = STAR_CATS / sub - lines.append(f"[ -d '{src}' ] && ln -sfn '{src}' \"$SP_RUN/{name}\" || true") - lines += list(pre_run) lines += [f'rm -rf "$SP_RUN/output/{subdir}"', 'cd "$SP_RUN"'] return "\n".join(lines) @@ -366,13 +368,15 @@ include: "rules/exposure.smk" include: "rules/tile.smk" # --- top-level targets ------------------------------------------------------ -# Only the aggregation targets and clean_exposure are localrules: a mid-chain -# localrule would break `group:` fusion of the compute chains (D4). -# clean_exposure is safe there because it is not in any chain — it hangs off -# `all` — and it is seconds of rmtree. As an sbatch job it would be ~20k -# scheduler submissions at DR6 scale to delete directories; the local-cores cap -# serialises them instead, which costs nothing at rmtree speed. -localrules: all, prepare_all_tiles, clean_exposure +# The aggregation targets, clean_exposure and exp_star_cat run in the head +# process. clean_exposure is seconds of rmtree and hangs off `all`; exp_star_cat +# is ~6 s dominated by one CDS query, and running it locally serialises those +# queries (see exposure.smk). Both would otherwise be ~20k sbatch submissions at +# DR6 scale for work shorter than the scheduling latency. +# +# exp_star_cat is the one MID-CHAIN localrule, so it must stay out of any future +# `group:` label: a local job cannot be fused into a submitted group. +localrules: all, prepare_all_tiles, clean_exposure, exp_star_cat rule all: input: diff --git a/workflow/config.yaml b/workflow/config.yaml index dfb3bb3a0..2c5b3008f 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -24,8 +24,17 @@ run_dir: /scratch/cdaley/shapepipe-output/smk-g3 # are one artefact and must version together, so the dir is fixed by construction. # Pre-staged inputs (P3 data already on /project; get_images RETRIEVE=symlink). -# Star catalogues for masking, pre-generated (network step done in prepare): -star_cats: /home/cdaley/projects/def-mjhudson/cdaley/runs/p3-batch1/star_cats + +# The mask star-catalogue CACHE root — run-independent, shared by every campaign. +# `exp_star_cat` writes /exp/star_cat-.fits, one GSC 2.3 cone query per +# exposure (~787 KiB, ~6 s), and skips a cat that is already there. Exposures +# recur across tiles and batches, so the cache is mostly hits. +# +# On /scratch because def-mjhudson /project is hard-full (27/27 TiB). The 60-day +# purge therefore applies, and that is acceptable: a purged cache regenerates one +# query per exposure, and exp_star_cat is a localrule, so the head process runs +# those queries serially and never hammers CDS. +star_cats: /scratch/cdaley/shapepipe-output/star-cat-cache # The run index. Design intent (finding 15) is durable products on /project — # BLOCKED for now: def-mjhudson /project is hard-full (27/27 TiB), so index and diff --git a/workflow/config/cfis/config_exp_Ma.ini b/workflow/config/cfis/config_exp_Ma.ini index 0b85fe86c..d5b521080 100644 --- a/workflow/config/cfis/config_exp_Ma.ini +++ b/workflow/config/cfis/config_exp_Ma.ini @@ -63,7 +63,6 @@ INPUT_DIR = $SP_RUN/output/run_sp_exp_Sp/split_exp_runner/output, $SP_RUN/star_c NUMBERING_SCHEME = -0000000-0 # Input file patterns: image, weight, external flag, external star catalogue -# (folded from runs/p3-batch1/cfis: offline cluster, no online star-cat fetch) FILE_PATTERN = image, weight, flag, star_cat FILE_EXT = .fits, .fits, .fits, .fits @@ -76,7 +75,8 @@ USE_EXT_FLAG = True # External star catalogue flag, use external cat if True, # obtain from online catalogue if False -# (folded from p3-batch1: True, using the pre-staged star_cat_exp pool) +# True: the cat comes from $SP_RUN/star_cat_exp, the per-unit farm the +# exp_star_cat rule builds (40 per-CCD links to this exposure's one cat). USE_EXT_STAR = True # File name suffix for the output flag files (optional) diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index e04dd76fc..da3e49e4f 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -1,6 +1,7 @@ """Exposure chain — per exposure, keyed by exp base id (dedup is structural). - exp_get_images -> exp_split -> exp_mask -> exp_psf + exp_get_images -> exp_split ---> exp_mask -> exp_psf + -> exp_star_cat -/ Each in the exposure's own sharded work dir, chained by manifests; every config reads fixed ``$SP_RUN/output/run_sp_exp_*`` INPUT_DIRs, so nothing resolves a @@ -36,6 +37,102 @@ rule exp_get_images: shell: sp_shell("exp_get_images", "config_exp_Gie.ini") +# --- mask star catalogues --------------------------------------------------- +# One GSC 2.3 (Vizier) cone query per exposure, covering the whole MegaCam focal +# plane, plus the 40 per-CCD symlinks the mask module's numbering scheme needs. +# +# It reads the exp_get_images output dir directly: that is a symlink farm of +# image-.fitsfz, exactly what create_star_cat.py's `-k exp` mode consumes — +# one multi-extension header read, one query, ~6 s measured. +# +# A LOCALRULE (declared in the Snakefile), so it runs in the head process — an +# sbatch job on a compute node, and nibi compute nodes have internet (verified by +# curl to vizier.cds.unistra.fr from a compute job). Local execution serialises +# the queries, which throttles CDS at any campaign scale for free, and it spares +# the scheduler one submission per exposure for six seconds of work. +# +# The CATALOGUE is cached run-independently under config `star_cats`, keyed by +# exposure. create_star_cat.py skips a cat that exists, so retries, reruns and +# later campaigns re-link an existing file and issue no query. +# +# The per-unit farm is a REAL directory holding exactly this exposure's 40 +# numbers, and that is load-bearing: config_exp_Ma.ini reads it as an INPUT_DIR +# and the file handler INTERSECTS the numbers found across INPUT_DIRs, so a +# symlink to a shared whole-store pool contributes every other exposure's numbers +# and the intersection is empty ("numbers ... do not intersect", live). +# +# Declared output is a manifest rather than a symlink or a directory(): it is +# written last, it is the one per-exposure file that is unique to this rule, it +# records what the farm points at, and it keeps the "one rule, one manifest" +# currency of every other rule — including being deleted by clean_exposure, so a +# reclaimed exposure rebuilds its farm from the cache at no query cost. + +# The container's certifi bundle. The host leaks SSL_CERT_FILE / CURL_CA_BUNDLE +# pointing at a path that does not exist inside the image, so requests is pointed +# at the bundle explicitly (proven in the p3-batch1 bash precedent). +STAR_CAT_CA = "/app/.venv/lib/python3.12/site-packages/certifi/cacert.pem" + + +def star_cat_cmd(exp): + """The whole rule body, as bash — carried as a params value, never inlined + in ``shell:``: it contains literal ``{}`` (the manifest JSON) and snakemake + formats a shell string once, which would consume those braces.""" + cache = f"{STAR_CATS}/exp" + cat = f"{cache}/star_cat-{exp}.fits" + work = exp_dir(exp) + farm = f"{work}/star_cat_exp" + images = f"{work}/output/run_sp_exp_Gie/get_images_runner/output" + manifest = exp_manifest(exp, "exp_star_cat") + body = json.dumps({ + "stage": "exp_star_cat", "level": "exp", "unit": exp, + "status": "complete", "cache": cat, "link_dir": farm, "n_links": 40, + }, indent=2, sort_keys=True) + return "\n".join([ + "set -euo pipefail", + f"mkdir -p '{cache}' '{farm}' '{work}/manifests'", + # Same binds and isolation as the profile's apptainer-args; the SDM does + # not wrap this rule (container: None below), because the farm loop and + # the manifest write belong to the host side of the job. + f"apptainer exec --cleanenv --home {Path.home()} --bind /project --bind /scratch" + f" --env PYTHONPATH={REPO_DIR}/src,REQUESTS_CA_BUNDLE={STAR_CAT_CA}" + f",SSL_CERT_FILE={STAR_CAT_CA},CURL_CA_BUNDLE={STAR_CAT_CA}" + f" '{config['container']}'" + f" python {REPO_DIR}/scripts/python/create_star_cat.py" + f" -i '{images}' -o '{cache}' -k exp", + f"test -s '{cat}'", + # The fan-out the file handler's NUMBERING_SCHEME wants: 40 links to the + # one focal-plane catalogue (pattern from the p3-batch1 precedent). + f"for ccd in $(seq 0 39); do ln -sfn '{cat}' " + f"'{farm}/star_cat-{exp}-'\"$ccd\"'.fits'; done", + # Byte-stable, and written only after the links exist: an unconditional + # write would move the mtime, which is a rerun-trigger. + f"tmp='{manifest}.tmp'", + "cat > \"$tmp\" <<'SP_STAR_CAT_JSON'", + body, + "SP_STAR_CAT_JSON", + f"cmp -s \"$tmp\" '{manifest}' && rm -f \"$tmp\" || mv -f \"$tmp\" '{manifest}'", + ]) + + +rule exp_star_cat: + input: + rules.exp_get_images.output.manifest + output: + manifest = f"{EXP_DIR}/manifests/exp_star_cat.json" + params: + cmd = lambda wc: star_cat_cmd(wc.exp) + # No SDM wrapping: the rule calls apptainer itself, because only the query + # runs in the container (bin/sp has loaded the apptainer module). + container: + None + threads: 1 + retries: 2 + resources: + mem_mb = 2000, + runtime = 10 + shell: + "{params.cmd}" + # Split the multi-HDU exposure into single-CCD files (+ headers-*.npy, which the # tiles' merge_headers reads). rule exp_split: @@ -55,7 +152,10 @@ rule exp_split: rule exp_mask: input: - rules.exp_split.output.manifest + # Both inputs are real INPUT_DIRs of config_exp_Ma.ini: the split CCDs + # and this exposure's own star_cat_exp farm. + rules.exp_split.output.manifest, + rules.exp_star_cat.output.manifest output: manifest = f"{EXP_DIR}/manifests/exp_mask.json" params: diff --git a/workflow/rules/prepare.smk b/workflow/rules/prepare.smk index 92998e184..4b1e401a6 100644 --- a/workflow/rules/prepare.smk +++ b/workflow/rules/prepare.smk @@ -10,12 +10,10 @@ tier). Rules stay group-compatible: shell only, no mid-chain localrules, no pipe outputs. -Star catalogues for masking are pre-generated offline (create_star_cat.py on a -networked login node) and consumed as DIRECTORIES: the mask configs read -``$SP_RUN/star_cat_{exp,tiles}`` with per-CCD numbering. Every rule's prologue -materialises the two dir symlinks; there is NO per-unit star-cat DAG node — the -store is pre-run input like the image store, and a missing cat fails the mask -stage's count floor loudly. +Star catalogues for masking are NOT a prepare-phase concern and not pre-run +input: they are built in the compute DAG, one node per exposure +(``exp_star_cat`` in exposure.smk), into a run-independent cache. The tile side +has no star-cat node because it has no mask rule yet — see tile.smk. """ # NUMBER_LIST is never set for get_images (download stage; nothing on disk to diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 8b665d1cf..d52b76e0a 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -18,6 +18,14 @@ Note there is no `tile_mask` rule: the committed config chain is the "sx_nomask" tile_detect variant (config_tile_Sx.ini reads Git + Uz + Mh, no mask run), and no tile-mask config was committed in the S2 sweep. Adding the masked variant is a config + one rule, at the config selector the PRD describes. + +That rule also needs a tile-side analogue of ``exp_star_cat``: tile star cats key +on TILE id, so they are a separate cache namespace and a separate node, and the +earliest point it can run is after ``tile_uncompress`` (create_star_cat.py's +``-k tile`` mode reads the uncompressed tile image's primary header). The mask +config would then read a real per-unit ``$SP_RUN/star_cat_tiles`` directory, +built the same way and for the same reason (the file handler intersects numbers +across INPUT_DIRs, so a shared pool cannot be symlinked in wholesale). """ def tile_exp(wc): From 6a8e71a09d6a0e7d3f1f8cb729a140a5b347ac19 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 15 Aug 2026 20:08:28 -0400 Subject: [PATCH 27/41] fix(orchestration): the verdict composes counts with shapepipe_run's rc completeness.py forked on its count verdict alone. A job whose floors were met but whose shapepipe_run exited nonzero therefore wrote the SUCCESS manifest and still exited nonzero. Snakemake then deleted that manifest as a failed job's output, the check had already unlinked any prior failed.json, and the failure left no trace on disk at all. The two statements are now composed. sp_shell passes `--job-rc "$rc"`; a nonzero rc forces the failure branch, records job_rc and a shapepipe_run failure entry that run_report surfaces, and the fork writes failed.json and exits 1 as usual. The rc is recorded only when nonzero, so every existing success manifest stays byte-identical and the mtime rerun-trigger does not churn. Co-Authored-By: Claude Fable 5 --- workflow/Snakefile | 14 +++++++++++++- workflow/scripts/completeness.py | 32 ++++++++++++++++++++++++++++++-- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 2b2ec322e..eb60f7c39 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -353,12 +353,24 @@ def sp_shell(stage, config_name): declared `.json` is removed if it was there. Success manifests are success-only currency (completeness.py); the failed one is evidence for `sp report` and is never a declared output. + + The verdict is COMPOSED, which is why `--job-rc "$rc"` is passed: the count + floors and shapepipe_run's exit status are two independent statements about + the same job, and the manifest must reflect both. Without it, a job whose + counts cleared their floors but whose shapepipe_run died (a late runner + raising after the counted ones wrote their files) wrote the SUCCESS manifest + and still exited nonzero — snakemake then deleted that manifest as a failed + job's output, no failed.json was written (the check thought it had won), and + any prior failed.json had already been unlinked. The failure left no trace at + all. With the rc composed in, the fork lands on the failure side and the + evidence survives. """ return ( "{params.pre}\n" "rc=0\n" f"shapepipe_run -c \"$SP_CONFIG/{config_name}\" -b {{threads}} || rc=$?\n" - f"python {SCRIPTS}/completeness.py check {stage} {{output.manifest}} || rc=1\n" + f"python {SCRIPTS}/completeness.py check {stage} {{output.manifest}}" + " --job-rc \"$rc\" || rc=1\n" "exit $rc\n" ) diff --git a/workflow/scripts/completeness.py b/workflow/scripts/completeness.py index 51c2521d8..138399af5 100644 --- a/workflow/scripts/completeness.py +++ b/workflow/scripts/completeness.py @@ -15,11 +15,14 @@ rc=0 shapepipe_run -c $SP_CONFIG/config_exp_Ma.ini -b {threads} || rc=$? - completeness.py check exp_mask {output} || rc=1 + completeness.py check exp_mask {output} --job-rc "$rc" || rc=1 exit $rc It counts the unit's products under ``$SP_RUN`` and exits nonzero iff a mandatory -runner is below its floor. WHERE it writes the record is the whole point: +runner is below its floor OR ``--job-rc`` is nonzero — the verdict is COMPOSED of +the counts and shapepipe_run's own exit status, because a runner can raise after +the counted ones have written their files. WHERE it writes the record is the +whole point: * success -> ``.json`` (the rule's declared output, the DAG's currency) and any stale ``.failed.json`` is removed; @@ -298,6 +301,8 @@ def main(argv=None) -> int: help="override the unit ID (default: basename of $SP_RUN)") c.add_argument("--stage-dir", default=None, help="override the run_sp_* subdir (default: the stage table)") + c.add_argument("--job-rc", type=int, default=0, + help="shapepipe_run's exit status, composed into the verdict") args = p.parse_args(argv) run_dir = args.run_dir or Path(os.environ.get("SP_RUN", "")) @@ -308,6 +313,29 @@ def main(argv=None) -> int: unit = args.unit or _unit_from_run_dir(run_dir) manifest, ok = build_manifest(args.stage, Path(run_dir), unit, args.stage_dir) + + # The verdict is COMPOSED of two independent statements: the count floors + # (above) and shapepipe_run's own exit status (here). Counts alone are not + # enough — a runner can raise AFTER the counted runners have written their + # files, so the floors are met while the job died. That combination used to + # write the SUCCESS manifest, which snakemake then deleted as a failed job's + # output, leaving no manifest at all: failure invisible, and any earlier + # failed.json already unlinked by the success branch. An rc-driven override + # puts the fork on the failure side, so failed.json is written and kept. + # + # Recorded only when nonzero, which keeps every existing success manifest + # byte-identical (the mtime rerun-trigger reads those bytes). + if args.job_rc != 0: + ok = False + manifest["status"] = "failed" + manifest["job_rc"] = args.job_rc + manifest["failures"].append({ + "runner": "shapepipe_run", "found": 0, "floor": 1, "expect": 1, + "warn": False, + "reasons": [f"shapepipe_run exited {args.job_rc} " + f"(counts above floor)"], + }) + text = json.dumps(manifest, indent=2, sort_keys=True) + "\n" # The success/failure fork. Exactly one of the two files exists afterwards, From 734101201c2ff6ae9be94fe77bc8e708b8ac627b Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 15 Aug 2026 20:08:44 -0400 Subject: [PATCH 28/41] fix(orchestration): exp_star_cat stops trusting what it cannot see MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four ways the star-catalogue stage lied about its own state, plus one comment that lied about the scheduler. The farm loop wrote into the shared pool. Legacy unit dirs carry star_cat_exp as a symlink into the old shared star-cat pool, `mkdir -p` no-ops on it, and the 40-link loop then followed the link and wrote this exposure's links into the pool (520 stray links found live). The rule now unlinks a symlinked farm before the mkdir. It never rm -rf's a real farm: that farm is its own output, and ln -sfn refreshes it in place. A killed cache write stayed poisonous forever. create_star_cat.py wrote the cache FITS in place, so an interrupted write left a truncated file that the `isfile -> continue` skip trusted for the rest of the campaign — and `test -s` passes on partial bytes. Both branches now write a hidden, PID-tagged temp with a .fits suffix (astropy picks its writer from the extension) and os.replace it into position; a leftover temp is removed in a finally. The dot-prefix keeps the temp out of the `star_cat*` globs. The manifest attested forever. It was the rule's only declared output, so a purged cache (/scratch, 60 days) or a deleted farm left the stage looking complete while exp_mask had nothing to read. The ccd-0 farm link is now declared alongside it. Snakemake's existence test follows symlinks, so a purge leaves that link dangling, the rule reruns, and it re-queries or re-links — the recovery config.yaml already promised. The script was outside the fingerprint. exp_star_cat's params held only paths, so an edit to create_star_cat.py never reran it. STAR_CAT_HASH follows the existing completeness/forest/clean pattern. The hash deliberately does NOT key the cache path: a rerun still takes the skip-if-exists branch, so a semantic change to the query needs the cache cleared by hand. config.yaml now says so. And the localrule comment claimed local execution serialises the queries. It does not: local jobs run in a pool sized by --local-cores (unset here, so the head process's CPU count, 5 as measured), which is the real concurrency cap. The localrule stays — 5-way is ordinary CDS politeness, and the alternative is ~20k sbatch submissions for six seconds of work each. Co-Authored-By: Claude Fable 5 --- scripts/python/create_star_cat.py | 27 +++++++++++++-- workflow/Snakefile | 8 +++++ workflow/config.yaml | 11 +++++-- workflow/rules/exposure.smk | 55 +++++++++++++++++++++++++------ 4 files changed, 87 insertions(+), 14 deletions(-) diff --git a/scripts/python/create_star_cat.py b/scripts/python/create_star_cat.py index 0e3586f88..404cee4ac 100755 --- a/scripts/python/create_star_cat.py +++ b/scripts/python/create_star_cat.py @@ -118,6 +118,29 @@ def query_vizier(ra, dec, radius_arcmin): return _query_vizier(ra, dec, radius_arcmin, CDS_CAT_ID) +def _write_atomic(table, output_dir, img_number, output_name): + """Write ``table`` to ``output_name`` atomically. + + The catalogue is a run-independent CACHE, and the caller's only test for a + cache hit is ``os.path.isfile``. An in-place ``table.write`` that is killed + part-way (job timeout, OOM, node failure) therefore leaves a truncated FITS + that every later run trusts forever — and ``test -s`` passes on partial + bytes. Writing to a temp and renaming makes the visible file all-or-nothing: + ``os.replace`` is atomic within a directory. + + The temp keeps a ``.fits`` suffix, because astropy picks the writer from the + extension. It is dot-prefixed and PID-tagged so it stays out of the + ``star_cat*`` globs the rules use, and two concurrent writers cannot collide. + """ + tmp = f"{output_dir}/.tmp-{os.getpid()}-star_cat{img_number}.fits" + try: + table.write(tmp, overwrite=True) + os.replace(tmp, output_name) + finally: + if os.path.exists(tmp): + os.remove(tmp) + + def main(input_dir, output_dir, kind): file_list = os.listdir(input_dir) @@ -140,7 +163,7 @@ def main(input_dir, output_dir, kind): f"Focal plane center: ra={ra:.4f}, dec={dec:.4f}, radius={radius:.2f} arcmin" ) table = query_vizier(ra, dec, radius) - table.write(output_name, overwrite=True) + _write_atomic(table, output_dir, img_number, output_name) else: h = fits.getheader(fpath, 0) @@ -156,7 +179,7 @@ def main(input_dir, output_dir, kind): continue table = query_vizier(ra, dec, radius) - table.write(output_name, overwrite=True) + _write_atomic(table, output_dir, img_number, output_name) return 0 diff --git a/workflow/Snakefile b/workflow/Snakefile index eb60f7c39..1c8222271 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -183,6 +183,14 @@ def unit_num(unit): SCRIPT_HASH = hashlib.md5((SCRIPTS / "completeness.py").read_bytes()).hexdigest()[:12] FOREST_HASH = hashlib.md5((SCRIPTS / "build_forest.py").read_bytes()).hexdigest()[:12] CLEAN_HASH = hashlib.md5((SCRIPTS / "clean_exposure.py").read_bytes()).hexdigest()[:12] +# Same argument for the star-cat query script, which lives in the repo's own +# scripts/python/ rather than workflow/scripts/: exp_star_cat's params otherwise +# fingerprint nothing but paths, so an edit to create_star_cat.py would never +# rerun the rule. The hash does NOT key the cache path — see config.yaml's +# star_cats block on clearing the cache after a semantic change. +STAR_CAT_HASH = hashlib.md5( + (REPO_DIR / "scripts" / "python" / "create_star_cat.py").read_bytes() +).hexdigest()[:12] # --- exposure reclamation (D5, S5) ----------------------------------------- diff --git a/workflow/config.yaml b/workflow/config.yaml index 2c5b3008f..55962297e 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -30,10 +30,17 @@ run_dir: /scratch/cdaley/shapepipe-output/smk-g3 # exposure (~787 KiB, ~6 s), and skips a cat that is already there. Exposures # recur across tiles and batches, so the cache is mostly hits. # +# THE CACHE IS NOT KEYED BY SCRIPT VERSION. A semantic change to +# scripts/python/create_star_cat.py (query radius, catalogue ID, column set) +# does rerun exp_star_cat — the script's hash is a rule param — but the rerun +# takes the skip-if-exists branch and only re-links the OLD catalogues. Clear +# this cache by hand when the change must reach the data. +# # On /scratch because def-mjhudson /project is hard-full (27/27 TiB). The 60-day # purge therefore applies, and that is acceptable: a purged cache regenerates one -# query per exposure, and exp_star_cat is a localrule, so the head process runs -# those queries serially and never hammers CDS. +# query per exposure. exp_star_cat declares its ccd-0 farm link as an output, so +# a purge leaves that link dangling and the rule reruns, instead of trusting a +# manifest that outlived its data. star_cats: /scratch/cdaley/shapepipe-output/star-cat-cache # The run index. Design intent (finding 15) is durable products on /project — diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index da3e49e4f..ee34f639a 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -47,9 +47,19 @@ rule exp_get_images: # # A LOCALRULE (declared in the Snakefile), so it runs in the head process — an # sbatch job on a compute node, and nibi compute nodes have internet (verified by -# curl to vizier.cds.unistra.fr from a compute job). Local execution serialises -# the queries, which throttles CDS at any campaign scale for free, and it spares -# the scheduler one submission per exposure for six seconds of work. +# curl to vizier.cds.unistra.fr from a compute job). Local execution does NOT +# serialise the queries — an earlier version of this comment claimed it did, and +# that was false. In cluster mode snakemake runs local jobs in a worker pool +# sized by `--local-cores`, and this rule takes one thread each, so as many CDS +# queries (and apptainer execs) run at once as that pool is wide. The nibi +# profile leaves --local-cores unset, so it is the head process's CPU count — +# 5 in the measured allocation. +# +# The localrule stays anyway. 5-way concurrency against Vizier is ordinary +# politeness, not abuse, and the alternative is one sbatch per exposure: ~20k +# submissions for six seconds of work each, which is exactly the sub-15-minute +# scheduler churn cluster policy asks us to bundle away. If CDS ever objects, +# --local-cores is the knob — it caps these queries directly. # # The CATALOGUE is cached run-independently under config `star_cats`, keyed by # exposure. create_star_cat.py skips a cat that exists, so retries, reruns and @@ -61,11 +71,23 @@ rule exp_get_images: # symlink to a shared whole-store pool contributes every other exposure's numbers # and the intersection is empty ("numbers ... do not intersect", live). # -# Declared output is a manifest rather than a symlink or a directory(): it is -# written last, it is the one per-exposure file that is unique to this rule, it -# records what the farm points at, and it keeps the "one rule, one manifest" -# currency of every other rule — including being deleted by clean_exposure, so a -# reclaimed exposure rebuilds its farm from the cache at no query cost. +# TWO declared outputs, and the second one is the point. +# +# The manifest keeps the "one rule, one manifest" currency of every other rule: +# written last, unique to this rule, a record of what the farm points at, and +# deleted by clean_exposure so a reclaimed exposure rebuilds its farm from the +# cache at no query cost. +# +# But a manifest attests FOREVER, and the two things it attests to both live +# outside the unit's manifests/ dir: the cache FITS on /scratch (60-day purge) +# and the farm itself. Either can vanish under a manifest that still says +# "complete", and then exp_mask runs against nothing. So the ccd-0 farm link is +# declared too — one link stands for all 40, they are created by the same loop +# in the same instant, and declaring 40 buys nothing. Snakemake's existence test +# is os.path.exists, which FOLLOWS symlinks and is therefore False for a link +# whose target the purge removed. A purged cache or a deleted farm makes the +# rule out of date, it reruns, and it re-queries or re-links as needed — the +# recovery config.yaml's star_cats comment promises, now actually wired up. # The container's certifi bundle. The host leaks SSL_CERT_FILE / CURL_CA_BUNDLE # pointing at a path that does not exist inside the image, so requests is pointed @@ -89,6 +111,14 @@ def star_cat_cmd(exp): }, indent=2, sort_keys=True) return "\n".join([ "set -euo pipefail", + # LEGACY-SYMLINK HAZARD. Unit dirs built before this rule existed carry + # star_cat_exp as a SYMLINK into the old shared star-cat pool. `mkdir -p` + # is a no-op on an existing symlink-to-directory, so the 40-link loop + # below followed it and wrote this exposure's links INTO THE SHARED POOL + # (520 stray links found live). Replace the link — never `rm -rf` it, + # which would recurse into the pool, and never touch a real directory: + # a real farm is this rule's own output and `ln -sfn` refreshes it. + f"[ -L '{farm}' ] && rm -f '{farm}' || true", f"mkdir -p '{cache}' '{farm}' '{work}/manifests'", # Same binds and isolation as the profile's apptainer-args; the SDM does # not wrap this rule (container: None below), because the farm loop and @@ -118,9 +148,14 @@ rule exp_star_cat: input: rules.exp_get_images.output.manifest output: - manifest = f"{EXP_DIR}/manifests/exp_star_cat.json" + manifest = f"{EXP_DIR}/manifests/exp_star_cat.json", + # The sentinel: ccd-0 of the 40-link farm (see above). + link = f"{EXP_DIR}/star_cat_exp/star_cat-{{exp}}-0.fits" params: - cmd = lambda wc: star_cat_cmd(wc.exp) + cmd = lambda wc: star_cat_cmd(wc.exp), + # create_star_cat.py is external to the shell string, so the `code` + # rerun-trigger does not see it — same reason SCRIPT_HASH exists. + script_hash = STAR_CAT_HASH # No SDM wrapping: the rule calls apptainer itself, because only the query # runs in the container (bin/sp has loaded the apptainer module). container: From cba14de8b9bd828c6f3d1ff56a6f16c9ed0e7c31 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 15 Aug 2026 20:08:56 -0400 Subject: [PATCH 29/41] fix(orchestration): reclaim the star-cat farm, stop the home-quota leak MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit clean_exposure targeted only output/ and manifests/, so the star_cat_exp link farm outlived the exp_star_cat manifest that owned it — 40 symlinks belonging to no rule. The farm and the legacy star_cat_tiles link are now targets too. This is consistency, not bytes: the catalogue stays in the run-independent cache, so rebuilding the farm is a relink and no query. Deletion is symlink-safe. Legacy unit dirs carry star_cat_exp as a link into the shared pool, and rmtree would recurse through it and empty that pool for every other exposure in the campaign. A symlink target is unlinked; only a real directory is rmtree'd. Existence is tested as `is_symlink() or exists()`, because exists() follows the link and would skip a dangling legacy one. Separately, astroquery pickled every Vizier response into $HOME/.astropy/cache/astroquery/Vizier at ~2 MB a query (10 queries, 10 pickles, observed live). The workflow already caches the result as a FITS on scratch and skips the query on a hit, so the pickle is a duplicate — and at campaign scale (~25k exposures) it is ~50 GB against a 50 GB home quota. query_vizier now passes cache=False. Co-Authored-By: Claude Fable 5 --- src/shapepipe/utilities/vizier.py | 10 +++++++++- workflow/scripts/clean_exposure.py | 28 ++++++++++++++++++++++++---- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/src/shapepipe/utilities/vizier.py b/src/shapepipe/utilities/vizier.py index 1f4fd3472..0bf077a78 100644 --- a/src/shapepipe/utilities/vizier.py +++ b/src/shapepipe/utilities/vizier.py @@ -64,8 +64,16 @@ def query_vizier(ra, dec, radius_arcmin, cat_id): v = Vizier( row_limit=-1, timeout=timeout, vizier_server=server ) + # cache=False: astroquery otherwise pickles every HTTP response into + # $HOME/.astropy/cache/astroquery/Vizier, ~2 MB per query. The + # workflow already caches the RESULT as a FITS catalogue on scratch + # and skips the query when it hits, so the pickle is pure duplicate — + # and at campaign scale (~25k exposures) it is ~50 GB against a + # 50 GB home quota. Home is for source and config, not for a second + # copy of the survey. result = v.query_region( - coord, radius=radius_arcmin * u.arcmin, catalog=cat_id + coord, radius=radius_arcmin * u.arcmin, catalog=cat_id, + cache=False, ) if len(result) > 0: print( diff --git a/workflow/scripts/clean_exposure.py b/workflow/scripts/clean_exposure.py index ce6fef80a..e8b575469 100644 --- a/workflow/scripts/clean_exposure.py +++ b/workflow/scripts/clean_exposure.py @@ -7,8 +7,19 @@ its postage stamps. Writer, then readers, then cleaner — DAG-ordered, race-free. What it deletes: the exposure's whole ``output/`` tree (the bulk store — -run_sp_exp_Gie/Sp/Ma/SxSePsfPi) AND its ``manifests/``. Deleting the manifests is -deliberate and load-bearing, not tidiness: +run_sp_exp_Gie/Sp/Ma/SxSePsfPi), its ``manifests/``, and its star-catalogue link +farms (``star_cat_exp``, plus the legacy ``star_cat_tiles``). The farms are +reclaimed for consistency, not for bytes: ``exp_star_cat``'s manifest is deleted +here like every other, so the exposure's chain must read as unbuilt, and 40 +symlinks left behind are a farm no rule now owns. The catalogue itself lives in +the run-independent cache, so rebuilding the farm costs a relink and no query. + +Deletion is SYMLINK-SAFE: a target that is itself a symlink is ``unlink``ed, not +``rmtree``d. Legacy unit dirs carry ``star_cat_exp`` as a link into the old +shared pool, and an rmtree would recurse through it and delete the shared cache +for every other exposure in the campaign. + +Deleting the manifests is deliberate and load-bearing, not tidiness: * the manifests are the exposure rules' DECLARED outputs. If they survived, a tile appended later would find the exposure chain "up to date" and run @@ -78,7 +89,11 @@ def main() -> None: except (OSError, json.JSONDecodeError) as exc: manifests[f.stem] = {"unreadable": str(exc)} - targets = [t for t in (args.exp_dir / "output", mdir) if t.exists()] + # is_symlink() first, and OR'd with exists(): exists() follows the link, so a + # dangling legacy star_cat_exp would otherwise be skipped and survive. + candidates = (args.exp_dir / "output", mdir, + args.exp_dir / "star_cat_exp", args.exp_dir / "star_cat_tiles") + targets = [t for t in candidates if t.is_symlink() or t.exists()] # Tombstone first, complete, fsync'd — then delete. See the module docstring: # the crash window has to sit where the data still exists, not where the @@ -96,7 +111,12 @@ def main() -> None: removed = [] for target in targets: - shutil.rmtree(target) + # NEVER rmtree a symlink: star_cat_exp is a link into the shared pool in + # legacy unit dirs, and rmtree would follow it and empty that pool. + if target.is_symlink(): + target.unlink() + else: + shutil.rmtree(target) removed.append(str(target)) print(f"[clean_exposure] {args.exp}: removed {len(removed)} tree(s) after " f"{len(consumers)} consuming tile(s)") From 30567ac17ef7db2875b19484105d7d30418c0230 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 24 Aug 2026 16:55:22 -0400 Subject: [PATCH 30/41] =?UTF-8?q?run(smk-g3):=20flip=20clean=20on=20?= =?UTF-8?q?=E2=80=94=20quad=20validated,=20batch1=20append=20needs=20the?= =?UTF-8?q?=20room?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The quad is proven equivalent to the bash baseline (validation verdict p0night1), so its exposure stores' inspection value is spent. The campaign now scales into the p3-batch1 tiles: 181 exposures un-cleaned would be ~1.2 TB against the 1 TiB scratch quota, so rolling reclamation (D5) is load-bearing from here on. Dry-run: exactly 18 clean_exposure jobs, no recompute. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014jAmAvmxJ3UCwYQrpHghkb --- workflow/config.yaml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/workflow/config.yaml b/workflow/config.yaml index 55962297e..3f7deab87 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -53,12 +53,14 @@ index_db: /scratch/cdaley/shapepipe-output/smk-g3/index/run_index.sqlite # `clean_exposure` job per exposure. It fires once every campaign tile that reads # that exposure has its vignets, deletes the exposure's store AND its manifests, # and leaves `cleaned.json`, which absorbs the manifests — `sp report` reads them -# back out of the tombstone and reports the exposure as `cleaned`. Off here -# because P0 is a 4-tile debugging run where the exposure store is exactly what -# you want to inspect; on for any run big enough to care about disk. Flipping it -# on later reclaims retroactively — the missing tombstones schedule exactly the -# outstanding clean jobs. -clean: false +# back out of the tombstone and reports the exposure as `cleaned`. On here: +# the quad is validated against the bash baseline (validation verdict +# p0night1), so its exposure stores' inspection value is spent, and the +# campaign is scaling into the p3-batch1 tiles — 181 exposures un-cleaned +# would be ~1.2 TB against a 1 TiB scratch quota, so rolling reclamation is +# load-bearing from here on. Flipping it on reclaims retroactively — the +# missing tombstones schedule exactly the outstanding clean jobs. +clean: true # Tiles that may NOT pin an exposure store (default: empty). # From fadc27739af81e6249ce61452987268d19b67e8e Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 24 Aug 2026 16:59:03 -0400 Subject: [PATCH 31/41] run(smk-g3): the campaign appends the first 30 batch1 tiles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tiles4.txt grows from 4 to 34 (parse-time config; nothing completed is invalidated). 30 of p3-batch1's 50 tiles now, not all: 30 tiles dedup to 100 unique exposures — worst-case un-cleaned high-water ~660G against ~840G free scratch — while 40 would hit ~1.1T. The remaining 20 are the next append, exercising append-at-scale a second time with the first wave's exposures already reclaimed. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014jAmAvmxJ3UCwYQrpHghkb --- workflow/config.yaml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/workflow/config.yaml b/workflow/config.yaml index 3f7deab87..94edd70c9 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -5,10 +5,14 @@ # tiles) never invalidates completed work — it only changes which jobs exist. # The tile list that scopes this run (one "IDra.IDdec" per line). -# 186/187 quad: 18 unique exposures with cross-tile reuse — exercises the -# structural exposure dedup. (The 210/211 quad used earlier is unusable: its -# tile images are symlinks into anaennis' moved processed_tiles tree. ~8.5k of -# the 10.3k staged tiles are such broken links; ~600 tiles are real files.) +# The campaign grows by appending to this file — parse-time config, so +# completed work is never invalidated. Current contents: the 186/187 quad +# (18 exposures, complete and validated against the bash baseline), plus the +# first 30 tiles of the p3-batch1 list (100 unique exposures — chosen so the +# worst-case un-cleaned exposure high-water ~660G fits the scratch quota with +# margin; the remaining 20 batch1 tiles are the next append). (The 210/211 +# quad used earlier is unusable: its tile images are symlinks into anaennis' +# moved processed_tiles tree — ~8.5k of the 10.3k staged tiles are broken.) tile_list: /scratch/cdaley/shapepipe-output/smk-g3/tiles4.txt # The container every job runs inside (apptainer software-deployment in the profile). From 93c7b4e059b5fd2221ceca3b68b893d22ae86346 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 28 Aug 2026 09:54:37 -0400 Subject: [PATCH 32/41] Move failure evidence onto snakemake's native log: directive MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each shapepipe_run rule now declares `log: /logs/.json`. `completeness.py check` writes its full verdict there on EVERY run — counts against floors, per-runner detail, scraped reasons, job_rc when nonzero — and writes the declared `manifests/.json` only when that verdict is a success. Snakemake deletes a failed job's declared output natively and never touches its log, so the two directives say exactly the two things the workflow needs, and the success/failure fork with its unlink of the stale counterpart is gone (`failed_path` deleted). Layout: `logs/` is a sibling of `manifests/`, and clean_exposure reclaims both. A log attesting "complete" for a reclaimed store would contradict the unbuilt chain the DAG must then see, and for a successful stage its content is byte-identical to the manifest, so the tombstone keeps absorbing manifests only. The benchmark tsv stays outside both dirs — it measures the job, it makes no claim about the store. run_report reads both dirs, worst status wins per (unit, stage): the manifest for success, the log for failure, and legacy `.failed.json` still parses as failed. Records without a `stage` key are skipped, so a stray JSON deeper in a unit tree is inert. tile_make_cat gains `--job-rc`, which it was missing: without it a job whose counts clear their floors but whose shapepipe_run died writes a "complete" log for a manifest snakemake is about to delete. exp_star_cat gets no log — it computes no verdict. The one guarantee this trades away is stated at the definition site: removing a previous success's manifest is now snakemake's job, so a head process SIGKILLed between a job's failure and that deletion leaves a stale success-named manifest. rerun-incomplete covers the DAG; the report's worst-status-wins covers the tables. Verified: `--lint` parses (the only remaining log-directive lints are the three rules deliberately without one); a `--dry-run` against the live state dir schedules 579 jobs, byte-identical in count and reasons to the same dry run with these changes stashed, so nothing here enlarges the pending rerun set; unit tests of the check (byte-stable manifest across a no-op rerun, log-only on count shortfall, log-only on nonzero job-rc), of run_report against a fixture mixing all three record conventions, and of clean_exposure reclaiming logs/ and still reporting the exposure as cleaned. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011Vg5ZoFShkKRdkk9bpbpXy --- profiles/nibi/config.yaml | 14 +++--- workflow/README.md | 32 ++++++------ workflow/Snakefile | 49 +++++++++++------- workflow/rules/exposure.smk | 12 +++++ workflow/rules/prepare.smk | 6 +++ workflow/rules/tile.smk | 23 +++++++-- workflow/scripts/clean_exposure.py | 27 ++++++---- workflow/scripts/completeness.py | 81 ++++++++++++++++-------------- workflow/scripts/run_report.py | 79 +++++++++++++++++------------ 9 files changed, 200 insertions(+), 123 deletions(-) diff --git a/profiles/nibi/config.yaml b/profiles/nibi/config.yaml index f78bea7f7..8bc71a9d1 100644 --- a/profiles/nibi/config.yaml +++ b/profiles/nibi/config.yaml @@ -46,13 +46,13 @@ rerun-incomplete: true # re-do jobs left incomplete by an unclean deat show-failed-logs: true printshellcmds: true -# No `keep-incomplete` here (it was the pre-S7a way to preserve failed manifests -# as report evidence): the declared output IS the manifest, so letting snakemake -# delete a failed job's output is exactly the semantics this workflow wants — a -# `.json` on disk then means the stage succeeded, and a resume cannot -# schedule downstream work on top of a failure. The post-mortem evidence lives in -# `.failed.json`, which completeness.py writes on failure and which is -# never a declared output, so snakemake never touches it. +# No `keep-incomplete` here. The declared output IS the manifest, so letting +# snakemake delete a failed job's output is exactly the semantics this workflow +# wants: a `.json` on disk means the stage succeeded, and a resume cannot +# schedule downstream work on top of a failure. The post-mortem evidence is the +# rule's `log:` (`/logs/.json`), which completeness.py writes on +# every run and which snakemake preserves through the failure that produced it — +# which is also what `show-failed-logs` above surfaces on the console. # rerun-triggers: the v9 default MINUS `input`. params/code/mtime fixes still # propagate — completeness.py writes the manifest only on change, so mtimes move diff --git a/workflow/README.md b/workflow/README.md index c286ebdcb..1d219617e 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -102,7 +102,7 @@ workflow/ build_forest.py per-tile exposure symlink forest (group-compatible shell) completeness.py the ported count-floor table (shared by sp_rule + run_report) run_report.py standalone report (NOT a DAG node; run_report hooks call it) - clean_exposure.py ONE exposure's store + manifests -> tombstone (the clean_exposure rule) + clean_exposure.py ONE exposure's store + manifests + logs -> tombstone (the clean_exposure rule) profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; keep-going ``` @@ -114,15 +114,16 @@ profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; kee CCD is often legitimate, and at DR6 scale per-CCD declaration means millions of paths. - **Manifests are the DAG's currency, and they are success-only.** - `completeness.py check` writes `.json` when the stage passed its count - floor, and `.failed.json` when it did not — removing the other file - either way. So `.json` on disk means "this stage succeeded", and a + `completeness.py check` writes its full verdict — per-runner counts against + floors, scraped failure reasons, the `shapepipe_run` exit status when nonzero + — to the rule's `log:` (`/logs/.json`) on *every* run, and + additionally to the declared `.json` manifest only when that verdict is + a success. So `.json` on disk means "this stage succeeded", and a resume after an unclean death cannot schedule downstream work on top of a - failure. The failed manifest carries the same content plus the scraped - reasons; it is never a declared output, so Snakemake never tracks or deletes - it, and `sp report` reads it when the success manifest is absent. The profile - runs *without* `keep-incomplete` for the same reason: deleting a failed job's - declared output is the wanted semantics. + failure. Nothing unlinks anything: Snakemake deletes a failed job's declared + output natively and never touches its log, which is why the profile runs + *without* `keep-incomplete`. `sp report` reads both dirs — the manifest for + success, the log for failure — and a unit with neither ran nothing. - **Completeness is a count floor, not a taxonomy.** After a run, `sp_rule.py` counts products per mandatory runner against `completeness.py`'s floor and exits nonzero below it. Per-CCD attrition @@ -151,8 +152,9 @@ profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; kee Reclamation is the in-DAG `clean_exposure` rule instead: one job per exposure, taking every consuming tile's `tile_vignets` manifest as input (the campaign-wide consumer set comes from the accumulating index), which - deletes the store *and* the exposure's manifests and leaves a `cleaned.json` - tombstone. Deleting the manifests is what makes a late append correct: the + deletes the store *and* the exposure's `manifests/` and `logs/`, and leaves a + `cleaned.json` tombstone. Deleting the manifests is what makes a late append + correct: the appended tile finds an unbuilt chain and regenerates it. The `clean:` flag in `config.yaml` gates it; flipping it on later reclaims retroactively, since the missing tombstones schedule exactly the outstanding clean jobs. @@ -175,9 +177,11 @@ profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; kee its exposure chains are gone and rebuild from scratch. - **A reclaimed exposure reports as `cleaned`.** `run_report.py` reads the absorbed manifests out of `cleaned.json`, so a reclaimed exposure keeps its - per-runner counts and blocks no tile. The `exp_psf` benchmark tsv lives - beside `manifests/`, not inside it, so reclamation does not eat the - memory-sizing data. + per-runner counts and blocks no tile. The logs go with the manifests — a log + claiming `complete` for a store that is gone would contradict the unbuilt + chain the DAG must now see, and its content duplicates the manifest anyway. + The `exp_psf` benchmark tsv lives beside both dirs, not inside either, so + reclamation does not eat the memory-sizing data. - **Failure is a report, not a gate.** `run_report.py` disk-scans the trees against the count table and enumerates shortfalls (whole-unit absence vs per-CCD attrition). It runs standalone — a DAG report node would itself be diff --git a/workflow/Snakefile b/workflow/Snakefile index 1c8222271..3b18fd1b8 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -20,6 +20,13 @@ The atom (D2): one rule == one `shapepipe_run` on one unit; its single declared output is its MANIFEST (`/manifests/.json`), written by `completeness.py check`. Product files are not declared — a missing CCD is often legitimate, and at DR6 scale per-CCD declaration means millions of paths. + +Failure evidence rides on the rule's `log:` (`/logs/.json`, +`unit_log` below): the check writes its full verdict there on every run, and the +manifest only when that verdict is a success. Snakemake deletes a failed job's +declared output and never touches its log, so the two directives say exactly the +two things this workflow needs — "this stage succeeded" and "here is what +happened". """ import hashlib @@ -240,8 +247,8 @@ CLEAN_IGNORE_TILES = set(config.get("clean_ignore_tiles") or []) def tombstone(exp): - """The clean_exposure output. Lives BESIDE manifests/, not inside it: the - clean job deletes manifests/ wholesale, and `sp report` scans it.""" + """The clean_exposure output. Lives BESIDE manifests/ and logs/, not inside + either: the clean job deletes both wholesale, and `sp report` scans it.""" return f"{exp_dir(exp)}/cleaned.json" @@ -305,7 +312,12 @@ def unit_pre(stage, level, unit, *, exp_name=None, forest=None, env=None, What it materialises (the proven v2.0 isolation-by-work-dir-content, NOT -e/--exclusive): - * ``output/`` and ``manifests/``; + * ``output/``, ``manifests/`` and ``logs/`` — the last two are a SIBLING + pair, not one dir with two naming conventions: clean_exposure deletes + BOTH wholesale, because a log attesting "complete" for a store that has + been reclaimed contradicts the unbuilt chain the DAG must now see. (The + benchmark tsv is deliberately outside both: it measures the job, it does + not claim anything about the store, so reclamation must not eat it.) * tile: ``tile_numbers.txt`` (dot format — what get_images reads); * exposure: a fabricated pseudo-Fe ``exp_numbers-000-000.txt`` holding the ORIGINAL exposure name from the index (``2605805p``), so get_images @@ -330,7 +342,7 @@ def unit_pre(stage, level, unit, *, exp_name=None, forest=None, env=None, # Also set via apptainer-args in the profile; kept here so a hand-run of # this same line outside snakemake behaves identically. f"export {_THREAD_CAPS}", - 'mkdir -p "$SP_RUN/output" "$SP_RUN/manifests"', + 'mkdir -p "$SP_RUN/output" "$SP_RUN/manifests" "$SP_RUN/logs"', ] if forest: lines.append(f"export SP_EXP='{forest}'") @@ -352,33 +364,32 @@ def unit_pre(stage, level, unit, *, exp_name=None, forest=None, env=None, def sp_shell(stage, config_name): """The rule's shell string: prologue, one shapepipe_run, one completeness check. - ``{threads}`` and ``{output}`` are placeholders HERE and nowhere else (see - unit_pre). ``-b {threads}`` makes SMP fork width and cpus_per_task one number - by construction (D4). + ``{threads}``, ``{output}`` and ``{log}`` are placeholders HERE and nowhere + else (see unit_pre). ``-b {threads}`` makes SMP fork width and cpus_per_task + one number by construction (D4). The check runs even when shapepipe_run failed, so a failed unit still leaves - a record of why — but that record goes to `.failed.json`, and the - declared `.json` is removed if it was there. Success manifests are - success-only currency (completeness.py); the failed one is evidence for - `sp report` and is never a declared output. + a record of why: the verdict goes to the rule's `log:` on EVERY run, and to + the declared `.json` manifest only when it is a success. Snakemake + deletes a failed job's output and never touches its log, so the manifest + stays success-only currency and the log is the evidence `sp report` reads. The verdict is COMPOSED, which is why `--job-rc "$rc"` is passed: the count floors and shapepipe_run's exit status are two independent statements about - the same job, and the manifest must reflect both. Without it, a job whose + the same job, and the record must reflect both. Without it, a job whose counts cleared their floors but whose shapepipe_run died (a late runner - raising after the counted ones wrote their files) wrote the SUCCESS manifest - and still exited nonzero — snakemake then deleted that manifest as a failed - job's output, no failed.json was written (the check thought it had won), and - any prior failed.json had already been unlinked. The failure left no trace at - all. With the rc composed in, the fork lands on the failure side and the - evidence survives. + raising after the counted ones wrote their files) publishes a SUCCESS + manifest and still exits nonzero — snakemake deletes that manifest as a + failed job's output, and the log is left claiming "complete" for a stage + with nothing to show for it. With the rc composed in, both files agree that + the job failed. """ return ( "{params.pre}\n" "rc=0\n" f"shapepipe_run -c \"$SP_CONFIG/{config_name}\" -b {{threads}} || rc=$?\n" f"python {SCRIPTS}/completeness.py check {stage} {{output.manifest}}" - " --job-rc \"$rc\" || rc=1\n" + " --log {log} --job-rc \"$rc\" || rc=1\n" "exit $rc\n" ) diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index ee34f639a..f56653e6c 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -25,6 +25,8 @@ whole-exposure hard failure. That is now a property of the committed configs rule exp_get_images: output: manifest = f"{EXP_DIR}/manifests/exp_get_images.json" + log: + f"{EXP_DIR}/logs/exp_get_images.json" params: pre = lambda wc: unit_pre("exp_get_images", "exp", wc.exp, exp_name=EXP[wc.exp]), @@ -151,6 +153,10 @@ rule exp_star_cat: manifest = f"{EXP_DIR}/manifests/exp_star_cat.json", # The sentinel: ccd-0 of the 40-link farm (see above). link = f"{EXP_DIR}/star_cat_exp/star_cat-{{exp}}-0.fits" + # No `log:`. Every other rule's log carries a completeness VERDICT, and this + # rule computes none: it is not a shapepipe_run, it has no count floor, and + # under `set -euo pipefail` it either completes or aborts at the failing + # step. Snakemake's own captured job stderr is the evidence for that. params: cmd = lambda wc: star_cat_cmd(wc.exp), # create_star_cat.py is external to the shell string, so the `code` @@ -175,6 +181,8 @@ rule exp_split: rules.exp_get_images.output.manifest output: manifest = f"{EXP_DIR}/manifests/exp_split.json" + log: + f"{EXP_DIR}/logs/exp_split.json" params: pre = lambda wc: unit_pre("exp_split", "exp", wc.exp), script_hash = SCRIPT_HASH @@ -193,6 +201,8 @@ rule exp_mask: rules.exp_star_cat.output.manifest output: manifest = f"{EXP_DIR}/manifests/exp_mask.json" + log: + f"{EXP_DIR}/logs/exp_mask.json" params: pre = lambda wc: unit_pre("exp_mask", "exp", wc.exp), script_hash = SCRIPT_HASH @@ -211,6 +221,8 @@ rule exp_psf: rules.exp_mask.output.manifest output: manifest = f"{EXP_DIR}/manifests/exp_psf.json" + log: + f"{EXP_DIR}/logs/exp_psf.json" params: pre = lambda wc: unit_pre("exp_psf", "exp", wc.exp), script_hash = SCRIPT_HASH diff --git a/workflow/rules/prepare.smk b/workflow/rules/prepare.smk index 4b1e401a6..d4a8d1cdd 100644 --- a/workflow/rules/prepare.smk +++ b/workflow/rules/prepare.smk @@ -23,6 +23,8 @@ has no star-cat node because it has no mask rule yet — see tile.smk. rule tile_get_images: output: manifest = f"{TILE_DIR}/manifests/tile_get_images.json" + log: + f"{TILE_DIR}/logs/tile_get_images.json" params: pre = lambda wc: unit_pre("tile_get_images", "tile", wc.tile), script_hash = SCRIPT_HASH @@ -39,6 +41,8 @@ rule tile_uncompress: rules.tile_get_images.output.manifest output: manifest = f"{TILE_DIR}/manifests/tile_uncompress.json" + log: + f"{TILE_DIR}/logs/tile_uncompress.json" params: pre = lambda wc: unit_pre("tile_uncompress", "tile", wc.tile), script_hash = SCRIPT_HASH @@ -54,6 +58,8 @@ rule tile_find_exposures: rules.tile_uncompress.output.manifest output: manifest = f"{TILE_DIR}/manifests/tile_find_exposures.json" + log: + f"{TILE_DIR}/logs/tile_find_exposures.json" params: pre = lambda wc: unit_pre("tile_find_exposures", "tile", wc.tile), script_hash = SCRIPT_HASH diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index d52b76e0a..5f8dfacd0 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -140,6 +140,8 @@ rule tile_merge_headers: fe = f"{TILE_DIR}/manifests/tile_find_exposures.json", output: manifest = f"{TILE_DIR}/manifests/tile_merge_headers.json" + log: + f"{TILE_DIR}/logs/tile_merge_headers.json" params: pre = lambda wc: unit_pre("tile_merge_headers", "tile", wc.tile, forest=forest_dir(wc.tile)), @@ -158,6 +160,8 @@ rule tile_detect: mh = rules.tile_merge_headers.output.manifest, output: manifest = f"{TILE_DIR}/manifests/tile_detect.json" + log: + f"{TILE_DIR}/logs/tile_detect.json" params: pre = lambda wc: unit_pre("tile_detect", "tile", wc.tile), script_hash = SCRIPT_HASH @@ -190,6 +194,8 @@ rule tile_vignets: output: manifest = f"{TILE_DIR}/manifests/tile_vignets.json", store = temp(directory(f"{TILE_DIR}/output/run_sp_tile_PiViVi")), + log: + f"{TILE_DIR}/logs/tile_vignets.json" params: pre = lambda wc: unit_pre("tile_vignets", "tile", wc.tile, forest=forest_dir(wc.tile)), @@ -218,6 +224,8 @@ rule tile_ngmix: manifest = f"{TILE_DIR}/manifests/tile_ngmix_{{chunk}}.json", # temp(directory()) for the same reason as the vignette store above. chunkdir = temp(directory(f"{TILE_DIR}/output/run_sp_tile_ngmix_Ng{{chunk}}u")), + log: + f"{TILE_DIR}/logs/tile_ngmix_{{chunk}}.json" params: pre = lambda wc: unit_pre( "tile_ngmix", "tile", wc.tile, @@ -258,6 +266,8 @@ rule tile_merge_cats: chunkdirs = ngmix_chunkdirs, output: manifest = f"{TILE_DIR}/manifests/tile_merge_cats.json" + log: + f"{TILE_DIR}/logs/tile_merge_cats.json" params: pre = lambda wc: unit_pre("tile_merge_cats", "tile", wc.tile, env={"NGMIX_N_CHUNKS": NGMIX_CHUNKS}), @@ -281,6 +291,8 @@ rule tile_make_cat: output: manifest = f"{TILE_DIR}/manifests/tile_make_cat.json", final_cat = f"{TILE_DIR}/final_cat-{{tile}}.fits", + log: + f"{TILE_DIR}/logs/tile_make_cat.json" params: pre = lambda wc: unit_pre("tile_make_cat", "tile", wc.tile), script_hash = SCRIPT_HASH @@ -289,12 +301,17 @@ rule tile_make_cat: mem_mb = lambda wc, attempt: 16000 * attempt, runtime = 120 shell: - # Publish the catalogue next to the manifest: a real file, so it is a - # real declared output (and it persists — never temp()). + # sp_shell's body, plus the catalogue publish: a real file, so it is a + # real declared output (and it persists — never temp()). `--job-rc` is + # composed in for the same reason it is everywhere else (see sp_shell) — + # without it a job whose counts cleared their floors but whose + # shapepipe_run died writes a "complete" log for a manifest snakemake is + # about to delete. "{params.pre}\n" "rc=0\n" 'shapepipe_run -c "$SP_CONFIG/config_tile_Mc.ini" -b {threads} || rc=$?\n' - f"python {SCRIPTS}/completeness.py check tile_make_cat {{output.manifest}} || rc=1\n" + f"python {SCRIPTS}/completeness.py check tile_make_cat {{output.manifest}}" + ' --log {log} --job-rc "$rc" || rc=1\n' "if [ $rc -eq 0 ]; then\n" ' cp -f "$(ls -1 "$SP_RUN"/output/run_sp_Mc/make_cat_runner/output/final_cat*.fits' ' | head -1)" {output.final_cat}\n' diff --git a/workflow/scripts/clean_exposure.py b/workflow/scripts/clean_exposure.py index e8b575469..e65c0dbcf 100644 --- a/workflow/scripts/clean_exposure.py +++ b/workflow/scripts/clean_exposure.py @@ -7,8 +7,8 @@ its postage stamps. Writer, then readers, then cleaner — DAG-ordered, race-free. What it deletes: the exposure's whole ``output/`` tree (the bulk store — -run_sp_exp_Gie/Sp/Ma/SxSePsfPi), its ``manifests/``, and its star-catalogue link -farms (``star_cat_exp``, plus the legacy ``star_cat_tiles``). The farms are +run_sp_exp_Gie/Sp/Ma/SxSePsfPi), its ``manifests/`` and its ``logs/``, and its +star-catalogue link farms (``star_cat_exp``, plus the legacy ``star_cat_tiles``). The farms are reclaimed for consistency, not for bytes: ``exp_star_cat``'s manifest is deleted here like every other, so the exposure's chain must read as unbuilt, and 40 symlinks left behind are a farm no rule now owns. The catalogue itself lives in @@ -31,16 +31,25 @@ needs to run, so tiles already finished are NOT rerun by their exposures' manifests vanishing. +``logs/`` goes with them, and for the same reason rather than for bytes. Each +log holds the completeness verdict of one stage, written on every run and kept by +snakemake through failures; a log left behind would attest "complete" for a store +that is no longer there, contradicting the unbuilt chain the DAG must now see. +Its content for a successful stage is byte-identical to the manifest beside it, +so absorbing the logs into the tombstone would duplicate what the manifests +already carry — they are deleted, not copied. + Nothing is lost to the report: every ``manifests/*.json`` is copied verbatim into the tombstone under ``manifests``, and ``run_report.py`` reads a cleaned exposure's record out of the tombstone — it reports the unit as ``cleaned``, warn counts and shortfalls intact, instead of "not run". The absorption is by GLOB, so it takes whatever is in ``manifests/``, keyed by -file stem; the report re-keys on each manifest's own ``stage`` field. A -``.failed.json`` is therefore carried through unremarkably — it should -never be there (an exposure with a failed stage has no complete vignets consumer -and so is not eligible for cleaning), but it costs nothing to be right about. +file stem; the report re-keys on each manifest's own ``stage`` field. A legacy +``.failed.json`` from the pre-``log:`` convention is therefore carried +through unremarkably — it should never be there (an exposure with a failed stage +has no complete vignets consumer and so is not eligible for cleaning), but it +costs nothing to be right about. Order matters, and it is the reverse of the obvious one: the tombstone is written FIRST, complete, and only then is anything deleted. A crash between the @@ -50,8 +59,8 @@ replaces them was never written, and the report would be blind to that exposure forever. -The exp_psf benchmark tsv lives beside ``manifests/``, not inside it, so it -survives this job — it is the measured-memory feed for resource sizing (D4). +The exp_psf benchmark tsv lives beside ``manifests/`` and ``logs/``, not inside +either, so it survives this job — it is the measured-memory feed for resource sizing (D4). The tombstone records the consumer set it was cleaned against. The rule carries that same set as a ``params`` value, so when the index grows a new consumer the @@ -91,7 +100,7 @@ def main() -> None: # is_symlink() first, and OR'd with exists(): exists() follows the link, so a # dangling legacy star_cat_exp would otherwise be skipped and survive. - candidates = (args.exp_dir / "output", mdir, + candidates = (args.exp_dir / "output", mdir, args.exp_dir / "logs", args.exp_dir / "star_cat_exp", args.exp_dir / "star_cat_tiles") targets = [t for t in candidates if t.is_symlink() or t.exists()] diff --git a/workflow/scripts/completeness.py b/workflow/scripts/completeness.py index 138399af5..58c1ee9eb 100644 --- a/workflow/scripts/completeness.py +++ b/workflow/scripts/completeness.py @@ -10,33 +10,41 @@ This file is also the ``check`` CLI — the second half of every rule's shell line (PRD D2/D3). The rules capture ShapePipe's return code rather than ``&&``-ing -onto it, so the check runs — and the manifest is written — even when +onto it, so the check runs — and the verdict is recorded — even when ``shapepipe_run`` failed:: rc=0 shapepipe_run -c $SP_CONFIG/config_exp_Ma.ini -b {threads} || rc=$? - completeness.py check exp_mask {output} --job-rc "$rc" || rc=1 + completeness.py check exp_mask {output} --log {log} --job-rc "$rc" || rc=1 exit $rc It counts the unit's products under ``$SP_RUN`` and exits nonzero iff a mandatory runner is below its floor OR ``--job-rc`` is nonzero — the verdict is COMPOSED of the counts and shapepipe_run's own exit status, because a runner can raise after -the counted ones have written their files. WHERE it writes the record is the -whole point: - - * success -> ``.json`` (the rule's declared output, the DAG's currency) - and any stale ``.failed.json`` is removed; - * failure -> ``.failed.json`` (same content, ``status: failed``) and any - stale ``.json`` is REMOVED. - -``.json`` therefore means "this stage succeeded", full stop — a resume can -never schedule a downstream stage on top of a failed one, not even after a head -process died without recording the failure. The failed manifest is never a -declared output of any rule: it is post-mortem evidence for ``run_report.py``, -which reads it when the success manifest is absent, and snakemake neither tracks -nor deletes it. - -Manifests carry no wall-clock, and either file is rewritten ONLY when its content +the counted ones have written their files. + +WHERE it writes the verdict is the whole point, and it is two files with two +different jobs: + + * the LOG (``--log``, the rule's snakemake ``log:``) gets the full verdict on + EVERY run, success or failure — counts against floors, per-runner detail, + scraped failure reasons, ``job_rc`` when nonzero. Snakemake never deletes a + log file, so it survives the failed job that wrote it and is the post-mortem + evidence ``run_report.py`` reads. + * the MANIFEST (the rule's declared ``output:``) gets that same verdict ONLY + when it is a success. Snakemake deletes a failed job's declared output + natively, so nothing here has to unlink anything. + +``.json`` therefore means "this stage succeeded": a resume cannot schedule +a downstream stage on top of a failed one. The removal of a PREVIOUS success's +manifest is snakemake's to do, not this script's, and the one gap that leaves is +a head process SIGKILLed between the job's failure and that deletion — a +success-named manifest then outlives the failure it no longer describes. The +profile's ``rerun-incomplete`` covers the DAG side, and ``run_report.py`` takes +the WORST status across a stage's log and manifest, so the report is right even +in that window. + +Neither file carries wall-clock, and each is rewritten ONLY when its content changes — identical on-disk state must leave a byte-identical manifest with an UNMOVED mtime, or the mtime rerun-trigger churns the cone on every unrelated ``--forcerun``. @@ -266,11 +274,6 @@ def build_manifest(stage, run_dir, unit, stage_subdir=None): return manifest, ok -def failed_path(manifest: Path) -> Path: - """``.json`` -> ``.failed.json``, in the same manifests/ dir.""" - return manifest.with_name(manifest.name[:-len(".json")] + ".failed.json") - - def write_if_changed(path: Path, text: str) -> None: """Write only when the bytes differ — see the module docstring on mtime.""" path.parent.mkdir(parents=True, exist_ok=True) @@ -295,6 +298,9 @@ def main(argv=None) -> int: c = sub.add_parser("check", help="count products, write the manifest") c.add_argument("stage") c.add_argument("manifest", type=Path) + c.add_argument("--log", type=Path, required=True, + help="the rule's log: path — the verdict is written here every " + "run, success or failure") c.add_argument("--run-dir", type=Path, default=None, help="the unit's $SP_RUN (default: the env var)") c.add_argument("--unit", default=None, @@ -317,11 +323,10 @@ def main(argv=None) -> int: # The verdict is COMPOSED of two independent statements: the count floors # (above) and shapepipe_run's own exit status (here). Counts alone are not # enough — a runner can raise AFTER the counted runners have written their - # files, so the floors are met while the job died. That combination used to - # write the SUCCESS manifest, which snakemake then deleted as a failed job's - # output, leaving no manifest at all: failure invisible, and any earlier - # failed.json already unlinked by the success branch. An rc-driven override - # puts the fork on the failure side, so failed.json is written and kept. + # files, so the floors are met while the job died. Without this, such a job + # publishes a SUCCESS manifest that snakemake then deletes as a failed job's + # output — and the log would claim "complete" for a stage with no manifest, + # which reads as a bookkeeping bug rather than as the failure it is. # # Recorded only when nonzero, which keeps every existing success manifest # byte-identical (the mtime rerun-trigger reads those bytes). @@ -338,22 +343,22 @@ def main(argv=None) -> int: text = json.dumps(manifest, indent=2, sort_keys=True) + "\n" - # The success/failure fork. Exactly one of the two files exists afterwards, - # so the presence of .json IS the statement "this stage succeeded" and - # the DAG can never build on top of a failure. The removal of the stale - # counterpart is what makes a success->failure or failure->success transition - # complete rather than additive. - written = args.manifest if ok else failed_path(args.manifest) - stale = failed_path(args.manifest) if ok else args.manifest - write_if_changed(written, text) - stale.unlink(missing_ok=True) + # The log ALWAYS gets the verdict; the manifest gets it only on success. No + # unlink of anything: snakemake deletes a failed job's declared output, so + # the presence of .json IS the statement "this stage succeeded" and + # the DAG can never build on top of a failure. A failure->success transition + # publishes the manifest; a success->failure has the manifest removed for us, + # and the log is overwritten with the new verdict either way. + write_if_changed(args.log, text) + if ok: + write_if_changed(args.manifest, text) for runner, r in manifest["runners"].items(): tag = {"complete": "OK", "warn": "warn", "below_floor": "<-- BELOW floor"} print(f"[completeness] {runner}: {r['found']}/{r['expect']} " f"(floor {r['floor']}) {tag[r['status']]}", file=sys.stderr) print(f"[completeness] {args.stage} {unit}: {manifest['status']} " - f"-> {written}", file=sys.stderr) + f"-> {args.log}" + (f" + {args.manifest}" if ok else ""), file=sys.stderr) for f in manifest["failures"]: for reason in f["reasons"]: print(f"[completeness] {f['runner']}: {reason}", file=sys.stderr) diff --git a/workflow/scripts/run_report.py b/workflow/scripts/run_report.py index 69c2e4b5f..22201f7e7 100644 --- a/workflow/scripts/run_report.py +++ b/workflow/scripts/run_report.py @@ -12,19 +12,21 @@ * the **index** (``run_index.sqlite``) — the units the run declared, and the tile->exposure edges that let an exposure failure be blamed on the tiles it blocks; - * the **manifests** — one per unit per stage, written by - ``completeness.py check``, carrying per-runner found/expect/floor and - log-scraped failure reasons. - -A stage that failed leaves ``.failed.json`` instead of ``.json`` -(``completeness.py``): the success name is reserved for success, so the DAG -cannot build on a failure. Both names are read here, and the status comes from -the manifest BODY, so a stage is reported as failed either way — including -manifests written before that split, where the failure content sat under the -success name. - -A reclaimed exposure has no manifests: ``clean_exposure`` deleted them after -copying them into ``/cleaned.json``. That tombstone is read as the + * the **verdicts** written by ``completeness.py check`` — per-runner + found/expect/floor and log-scraped failure reasons — which land in two places + with two different lifetimes. Every run writes the unit's ``logs/.json`` + (the rule's snakemake ``log:``, which snakemake never deletes); a run whose + verdict is a success ADDITIONALLY writes ``manifests/.json``, the + rule's declared output, which snakemake deletes when the job fails. + +Both dirs are read here and the status comes from the file BODY, so a failed +stage speaks through its log while a successful one is corroborated by two +identical files. A unit with neither ran nothing. Also read, for continuity with +stores written before this convention: ``manifests/.failed.json``, which +is where failure evidence used to go. + +A reclaimed exposure has neither dir: ``clean_exposure`` deleted both, after +copying the manifests into ``/cleaned.json``. That tombstone is read as the unit's record and the unit is reported as **cleaned** — not "not run", and it blocks no tile. @@ -32,9 +34,9 @@ the products were fresh. A unit with no manifest for a stage is "not run" — which is a real and distinct answer from "ran and produced nothing". -Manifests are discovered by glob (``tiles/**/manifests/*.json``), not by -constructed path: the unit stores are sharded (``tiles///``) and the -sharding depth is not this script's business. +Records are discovered by glob (``tiles/**/manifests/*.json`` and +``tiles/**/logs/*.json``), not by constructed path: the unit stores are sharded +(``tiles///``) and the sharding depth is not this script's business. """ import argparse @@ -55,31 +57,41 @@ def load_manifests(run_dir: Path, sub: str) -> dict: - """``{unit: {stage: manifest}}`` for one store (``tiles`` or ``exp``). + """``{unit: {stage: verdict}}`` for one store (``tiles`` or ``exp``). - The unit key is the manifest dir's *parent directory name* — shard-depth - agnostic, and the only form that joins to the index (the manifest's own + Reads BOTH of a unit's record dirs: ``manifests/`` (the rules' declared + outputs, success-only) and ``logs/`` (the rules' ``log:``, written every run + and never deleted by snakemake, so this is where a failure survives). + + The unit key is the record dir's *parent directory name* — shard-depth + agnostic, and the only form that joins to the index (the record's own ``unit`` field carries ``SP_UNIT_NUM``'s dashed form, ``210-282``, which is - not the index's ``210.282``). The stage comes from the manifest body, never - the filename: ngmix chunks share a stage under per-chunk filenames, and - ``.failed.json`` names the same stage as ``.json``. - - Several files can therefore map to one (unit, stage), and the WORST status - wins. That is what collapses the ngmix chunks to one entry, and what makes a - failed manifest speak even in the transient window where its success-named - counterpart has not yet been removed. + not the index's ``210.282``). The stage comes from the body, never the + filename: ngmix chunks share a stage under per-chunk filenames, and a log + names the same stage as the manifest beside it. + + Several files therefore map to one (unit, stage), and the WORST status wins. + That is what collapses the ngmix chunks to one entry, and it is why a + successful stage's two byte-identical records cost nothing while a failure + always speaks. A body with no ``stage`` field is skipped: it is not one of + ours, which is what keeps a stray JSON deeper in the tree inert. """ out: dict = defaultdict(dict) - for path in sorted((run_dir / sub).glob("**/manifests/*.json")): + paths = sorted((run_dir / sub).glob("**/manifests/*.json")) \ + + sorted((run_dir / sub).glob("**/logs/*.json")) + for path in paths: try: m = json.loads(path.read_text()) except (OSError, json.JSONDecodeError) as exc: - print(f"[run_report] unreadable manifest {path}: {exc}", file=sys.stderr) + print(f"[run_report] unreadable record {path}: {exc}", file=sys.stderr) + continue + if not isinstance(m, dict) or "stage" not in m: continue unit = path.parent.parent.name - stage = m.get("stage", path.stem) + stage = m["stage"] prev = out[unit].get(stage) - # Worst status wins when several manifests share a stage (ngmix chunks). + # Worst status wins when several records share a stage (ngmix chunks; + # a stage's manifest and its identical log). rank = lambda d: STATUSES.index(d.get("status")) if d.get("status") in STATUSES else len(STATUSES) # noqa: E731 if prev is None or rank(m) > rank(prev): out[unit][stage] = m @@ -89,8 +101,9 @@ def load_manifests(run_dir: Path, sub: str) -> dict: def absorb_tombstones(run_dir: Path, sub: str, manifests: dict) -> set: """Fill in reclaimed units from their ``cleaned.json``; return their ids. - A cleaned exposure has NO ``manifests/`` — ``clean_exposure`` deleted it, - after copying every manifest verbatim into the tombstone. Read them back, or + A cleaned exposure has neither ``manifests/`` nor ``logs/`` — + ``clean_exposure`` deleted both, after copying every manifest verbatim into + the tombstone (the logs duplicate them). Read them back, or the report inverts the truth exactly when reclamation works: the exposure shows as "not run" and blocks the very tiles whose completion authorised the deletion. From b2cfa3bd62121128cf783389ce8a9a4dfbade54d Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 29 Aug 2026 10:02:06 -0400 Subject: [PATCH 33/41] Fetch star catalogues by sky chunk, not by exposure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mask star catalogue's network cost was a function of the exposure count: one GSC 2.3 cone per exposure, ~25k queries at DR6 scale, re-fetching the same sky ~8 times over because exposures overlap that deep. Make it a function of sky area instead. `star_catalogue` fetches the tile list's footprint once, one query per NSIDE=32 HEALPix chunk, into a run-independent chunk store under the `star_cats` root; `exp_star_cat` keeps its DAG position and both its outputs but is now a pure local cut of those chunks to the exposure's focal-plane disc. ~16x fewer queries and ~4x less queried area, and the campaign's star catalogue becomes a durable product in its own right. Chunk-need comes from the tile list, the one thing known before any download, padded 2 deg for the exposure offsets the fixed tile grid admits. Chunks already on disk are skipped, so a campaign growing past its footprint fetches only what it adds and one growing within it fetches nothing; the tile list's hash rides in params so that growth actually reaches the DAG. A chunk the pad failed to cover is a loud failure in the cut, never a quietly under-masked exposure. fetch and cut live in one module because they must agree on which pixel holds which star, and one shared NSIDE cannot drift. Verified on exposure 2086324: the cut reproduces the catalogue the old per-exposure query produced exactly — 10936 stars, identical GSC2.3 id sets, identical columns. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011Vg5ZoFShkKRdkk9bpbpXy --- workflow/Snakefile | 43 +++-- workflow/config.yaml | 35 ++-- workflow/rules/exposure.smk | 170 ++++++++++------ workflow/rules/prepare.smk | 7 +- workflow/scripts/star_cats.py | 354 ++++++++++++++++++++++++++++++++++ 5 files changed, 515 insertions(+), 94 deletions(-) create mode 100644 workflow/scripts/star_cats.py diff --git a/workflow/Snakefile b/workflow/Snakefile index 3b18fd1b8..352761501 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -49,12 +49,14 @@ container: config["container"] # --- paths ----------------------------------------------------------------- RUN_DIR = Path(config["run_dir"]) -# Run-independent cache root for the mask star catalogues, keyed by exposure and -# filled by the exp_star_cat rule (config.yaml explains the placement). +# Run-independent root for the mask star catalogues: the HEALPix chunk store the +# star_catalogue rule fills, and the per-exposure cuts exp_star_cat makes from it +# (config.yaml explains the placement). STAR_CATS = Path(config["star_cats"]) -# This checkout's own tree: exp_star_cat runs its scripts/python/create_star_cat.py -# against its src/, for the same reason CONFIG_DIR is the committed config dir — -# script, library and rule are one artefact and version together. +# This checkout's own tree: the star-cat rules run in the container against its +# src/ (shapepipe.utilities.vizier and .cfis), for the same reason CONFIG_DIR is +# the committed config dir — script, library and rule are one artefact and +# version together. REPO_DIR = Path(workflow.basedir).parent INDEX_DB = Path(config["index_db"]) SCRIPTS = Path(workflow.basedir) / "scripts" @@ -190,14 +192,14 @@ def unit_num(unit): SCRIPT_HASH = hashlib.md5((SCRIPTS / "completeness.py").read_bytes()).hexdigest()[:12] FOREST_HASH = hashlib.md5((SCRIPTS / "build_forest.py").read_bytes()).hexdigest()[:12] CLEAN_HASH = hashlib.md5((SCRIPTS / "clean_exposure.py").read_bytes()).hexdigest()[:12] -# Same argument for the star-cat query script, which lives in the repo's own -# scripts/python/ rather than workflow/scripts/: exp_star_cat's params otherwise -# fingerprint nothing but paths, so an edit to create_star_cat.py would never -# rerun the rule. The hash does NOT key the cache path — see config.yaml's -# star_cats block on clearing the cache after a semantic change. +# Same argument for star_cats.py, which both star-cat rules call: their params +# otherwise fingerprint nothing but paths, so an edit to the chunking or the cut +# would never rerun them. ONE hash for both rules because it is one script — and +# that is also why fetch and cut live in one module (they must agree on which +# pixel holds which star). The hash does NOT key the store path — see +# config.yaml's star_cats block on clearing the store after a semantic change. STAR_CAT_HASH = hashlib.md5( - (REPO_DIR / "scripts" / "python" / "create_star_cat.py").read_bytes() -).hexdigest()[:12] + (SCRIPTS / "star_cats.py").read_bytes()).hexdigest()[:12] # --- exposure reclamation (D5, S5) ----------------------------------------- @@ -399,15 +401,16 @@ include: "rules/exposure.smk" include: "rules/tile.smk" # --- top-level targets ------------------------------------------------------ -# The aggregation targets, clean_exposure and exp_star_cat run in the head -# process. clean_exposure is seconds of rmtree and hangs off `all`; exp_star_cat -# is ~6 s dominated by one CDS query, and running it locally serialises those -# queries (see exposure.smk). Both would otherwise be ~20k sbatch submissions at -# DR6 scale for work shorter than the scheduling latency. +# The aggregation targets, clean_exposure, star_catalogue and exp_star_cat run in +# the head process. clean_exposure is seconds of rmtree and hangs off `all`; +# exp_star_cat is seconds of local FITS work; both would otherwise be ~20k sbatch +# submissions at DR6 scale for work shorter than the scheduling latency. +# star_catalogue is one job either way, and local keeps its CDS concurrency the +# explicit number its thread pool sets (see exposure.smk). # -# exp_star_cat is the one MID-CHAIN localrule, so it must stay out of any future -# `group:` label: a local job cannot be fused into a submitted group. -localrules: all, prepare_all_tiles, clean_exposure, exp_star_cat +# star_catalogue and exp_star_cat are MID-CHAIN localrules, so they must stay out +# of any future `group:` label: a local job cannot be fused into a submitted group. +localrules: all, prepare_all_tiles, clean_exposure, star_catalogue, exp_star_cat rule all: input: diff --git a/workflow/config.yaml b/workflow/config.yaml index 94edd70c9..c81d4e45d 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -29,22 +29,31 @@ run_dir: /scratch/cdaley/shapepipe-output/smk-g3 # Pre-staged inputs (P3 data already on /project; get_images RETRIEVE=symlink). -# The mask star-catalogue CACHE root — run-independent, shared by every campaign. -# `exp_star_cat` writes /exp/star_cat-.fits, one GSC 2.3 cone query per -# exposure (~787 KiB, ~6 s), and skips a cat that is already there. Exposures -# recur across tiles and batches, so the cache is mostly hits. +# The mask star-catalogue root — run-independent, shared by every campaign, and +# holding two things: +# /I_305_out/nside32/star_chunk-.fits the SKY store, one GSC 2.3 +# query per HEALPix chunk (~3.4 deg^2, ~25k rows), written by +# `star_catalogue` over the tile list's footprint and never fetched twice; +# /exp/star_cat-.fits the per-exposure cuts +# `exp_star_cat` makes from those chunks, with no network at all. +# Network therefore scales with SKY AREA, not exposure count: exposures overlap +# ~7-10 deep, so a full-UNIONS footprint is ~1.5k queries against ~25k exposures. +# Design intent is /project — this is a durable science product, not scratch +# work — blocked only by the quota below. # -# THE CACHE IS NOT KEYED BY SCRIPT VERSION. A semantic change to -# scripts/python/create_star_cat.py (query radius, catalogue ID, column set) -# does rerun exp_star_cat — the script's hash is a rule param — but the rerun -# takes the skip-if-exists branch and only re-links the OLD catalogues. Clear -# this cache by hand when the change must reach the data. +# THE STORE IS NOT KEYED BY SCRIPT VERSION. A semantic change to +# workflow/scripts/star_cats.py (padding, catalogue ID, column set) does rerun +# both rules — the script's hash is a param on each — but a chunk already on disk +# is skipped and only re-cut. Clear the store by hand when the change must reach +# the data. Changing NSIDE or the catalogue ID is the exception: those name the +# directory, so a change there fetches into a new one beside the old. # # On /scratch because def-mjhudson /project is hard-full (27/27 TiB). The 60-day -# purge therefore applies, and that is acceptable: a purged cache regenerates one -# query per exposure. exp_star_cat declares its ccd-0 farm link as an output, so -# a purge leaves that link dangling and the rule reruns, instead of trusting a -# manifest that outlived its data. +# purge therefore applies. A purged per-exposure cut is free to remake, and +# exp_star_cat declares its ccd-0 farm link as an output so a purge leaves that +# link dangling and the rule reruns rather than trusting a manifest that outlived +# its data. A purged CHUNK costs its query again: exp_star_cat fails loudly on a +# missing chunk, and re-running `star_catalogue` refetches it. star_cats: /scratch/cdaley/shapepipe-output/star-cat-cache # The run index. Design intent (finding 15) is durable products on /project — diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index f56653e6c..c6fea1cae 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -1,7 +1,11 @@ """Exposure chain — per exposure, keyed by exp base id (dedup is structural). - exp_get_images -> exp_split ---> exp_mask -> exp_psf - -> exp_star_cat -/ + exp_get_images -> exp_split -----> exp_mask -> exp_psf + -> exp_star_cat --/ + star_catalogue ---------------/ + +``star_catalogue`` is campaign-level, not per-exposure: one fetch of the whole +footprint's stars, which every exposure's ``exp_star_cat`` then cuts locally. Each in the exposure's own sharded work dir, chained by manifests; every config reads fixed ``$SP_RUN/output/run_sp_exp_*`` INPUT_DIRs, so nothing resolves a @@ -40,32 +44,98 @@ rule exp_get_images: sp_shell("exp_get_images", "config_exp_Gie.ini") # --- mask star catalogues --------------------------------------------------- -# One GSC 2.3 (Vizier) cone query per exposure, covering the whole MegaCam focal -# plane, plus the 40 per-CCD symlinks the mask module's numbering scheme needs. +# Two rules, and the split between them is the design: the NETWORK is a function +# of the campaign's sky area, the per-exposure catalogue is a local cut. +# +# `star_catalogue` fetches the footprint's GSC 2.3 stars once, one Vizier query +# per HEALPix chunk, into a run-independent chunk store under config `star_cats`. +# `exp_star_cat` then reads the chunks covering an exposure's focal plane and +# cuts them to it — no network at all. workflow/scripts/star_cats.py holds both +# halves and the geometry they must agree on; its docstring is the reference for +# the chunking and the padding. # -# It reads the exp_get_images output dir directly: that is a symlink farm of -# image-.fitsfz, exactly what create_star_cat.py's `-k exp` mode consumes — -# one multi-extension header read, one query, ~6 s measured. +# The arithmetic: exposures overlap ~7-10 deep and a tile's exposures all look at +# the same square degree, so the old one-cone-per-exposure design re-fetched the +# same sky ~8 times over. Chunked by sky, a full-UNIONS footprint is ~1.5k +# queries where the exposure count would have been ~25k, and a campaign that +# grows within the fetched footprint issues none. + +# The container's certifi bundle. The host leaks SSL_CERT_FILE / CURL_CA_BUNDLE +# pointing at a path that does not exist inside the image, so requests is pointed +# at the bundle explicitly (proven in the p3-batch1 bash precedent). +STAR_CAT_CA = "/app/.venv/lib/python3.12/site-packages/certifi/cacert.pem" + +# The rules run star_cats.py inside the container (healpy, astroquery, astropy) +# but call apptainer THEMSELVES rather than letting the SDM wrap them +# (`container: None` on both): the CA bundle above and the exposure rule's +# host-side farm loop both need the explicit exec. bin/sp has loaded the +# apptainer module. PYTHONPATH pins this checkout's src/ for +# shapepipe.utilities.{vizier,cfis} — the mirror-retry query and the tile-ID +# grid convention are library code, not copies. +def in_container(cmd, *, network=False): + ca = ("," + ",".join(f"{k}={STAR_CAT_CA}" for k in + ("REQUESTS_CA_BUNDLE", "SSL_CERT_FILE", "CURL_CA_BUNDLE")) + if network else "") + return (f"apptainer exec --cleanenv --home {Path.home()}" + f" --bind /project --bind /scratch" + f" --env PYTHONPATH={REPO_DIR}/src{ca}" + f" '{config['container']}' {cmd}") + + +# The campaign's star catalogue: a first-class durable science product, keyed by +# sky rather than by run. Chunk-need is recomputed from the tile list on every +# run and only the missing chunks are fetched, so appending tiles costs exactly +# the chunks they add. # -# A LOCALRULE (declared in the Snakefile), so it runs in the head process — an -# sbatch job on a compute node, and nibi compute nodes have internet (verified by -# curl to vizier.cds.unistra.fr from a compute job). Local execution does NOT -# serialise the queries — an earlier version of this comment claimed it did, and -# that was false. In cluster mode snakemake runs local jobs in a worker pool -# sized by `--local-cores`, and this rule takes one thread each, so as many CDS -# queries (and apptainer execs) run at once as that pool is wide. The nibi -# profile leaves --local-cores unset, so it is the head process's CPU count — -# 5 in the measured allocation. +# A LOCALRULE (declared in the Snakefile): it is one job of network I/O, and the +# fetch loop is a 4-wide thread pool inside it — the same modest concurrency the +# per-exposure rule reached by accident through --local-cores, now an explicit +# number that does not scale with the head node's CPU count. # -# The localrule stays anyway. 5-way concurrency against Vizier is ordinary -# politeness, not abuse, and the alternative is one sbatch per exposure: ~20k -# submissions for six seconds of work each, which is exactly the sub-15-minute -# scheduler churn cluster policy asks us to bundle away. If CDS ever objects, -# --local-cores is the knob — it caps these queries directly. +# `tile_list_hash` is what makes the incremental behaviour visible to the DAG. +# The tile list is parse-time config, not a rule input (and the profile drops the +# `input` rerun-trigger anyway), so appending tiles would otherwise leave this +# rule up to date against a footprint that has grown. Hashing the list into a +# param reruns it, and the rerun fetches only what is new. +STAR_CAT_MANIFEST = f"{RUN_DIR}/manifests/star_catalogue.json" + + +rule star_catalogue: + output: + manifest = STAR_CAT_MANIFEST + # No `log:` — see write_manifest() in star_cats.py: this rule runs no + # shapepipe_run and computes no completeness verdict, so there is nothing to + # split between a manifest and a log. Under `set -euo pipefail` it either + # completes or aborts, and snakemake's captured stderr is the evidence. + # `cmd` is a params value, so it is substituted AFTER the shell string is + # formatted: a `{output.manifest}` placeholder in here would survive + # literally (see unit_pre in the Snakefile). Hence the explicit path. + params: + cmd = in_container( + f"python {SCRIPTS}/star_cats.py fetch" + f" --tile-list '{config['tile_list']}' --store '{STAR_CATS}'" + f" --manifest '{STAR_CAT_MANIFEST}'", network=True), + tile_list_hash = hashlib.md5( + Path(config["tile_list"]).read_bytes()).hexdigest()[:12], + script_hash = STAR_CAT_HASH + container: + None + threads: 4 + retries: 2 + resources: + mem_mb = 4000, + runtime = 720 + shell: + "set -euo pipefail\n{params.cmd}" + + +# The per-exposure catalogue and the 40 per-CCD symlinks the mask module's +# numbering scheme needs. Local: one header read for the focal-plane footprint, +# a load of the chunks covering it, a radial cut. # -# The CATALOGUE is cached run-independently under config `star_cats`, keyed by -# exposure. create_star_cat.py skips a cat that exists, so retries, reruns and -# later campaigns re-link an existing file and issue no query. +# A LOCALRULE for the same reason as clean_exposure: seconds of work, and one +# sbatch per exposure would be ~20k submissions well under the 15-minute floor +# cluster policy asks us to bundle away. # # The per-unit farm is a REAL directory holding exactly this exposure's 40 # numbers, and that is load-bearing: config_exp_Ma.ini reads it as an INPUT_DIR @@ -78,38 +148,30 @@ rule exp_get_images: # The manifest keeps the "one rule, one manifest" currency of every other rule: # written last, unique to this rule, a record of what the farm points at, and # deleted by clean_exposure so a reclaimed exposure rebuilds its farm from the -# cache at no query cost. +# chunk store at no network cost. # # But a manifest attests FOREVER, and the two things it attests to both live -# outside the unit's manifests/ dir: the cache FITS on /scratch (60-day purge) +# outside the unit's manifests/ dir: the cut catalogue on /scratch (60-day purge) # and the farm itself. Either can vanish under a manifest that still says # "complete", and then exp_mask runs against nothing. So the ccd-0 farm link is # declared too — one link stands for all 40, they are created by the same loop # in the same instant, and declaring 40 buys nothing. Snakemake's existence test # is os.path.exists, which FOLLOWS symlinks and is therefore False for a link -# whose target the purge removed. A purged cache or a deleted farm makes the -# rule out of date, it reruns, and it re-queries or re-links as needed — the -# recovery config.yaml's star_cats comment promises, now actually wired up. - -# The container's certifi bundle. The host leaks SSL_CERT_FILE / CURL_CA_BUNDLE -# pointing at a path that does not exist inside the image, so requests is pointed -# at the bundle explicitly (proven in the p3-batch1 bash precedent). -STAR_CAT_CA = "/app/.venv/lib/python3.12/site-packages/certifi/cacert.pem" - - +# whose target the purge removed. A purged cut or a deleted farm makes the rule +# out of date, it reruns, and it re-cuts or re-links as needed. def star_cat_cmd(exp): """The whole rule body, as bash — carried as a params value, never inlined in ``shell:``: it contains literal ``{}`` (the manifest JSON) and snakemake formats a shell string once, which would consume those braces.""" - cache = f"{STAR_CATS}/exp" - cat = f"{cache}/star_cat-{exp}.fits" + cut_dir = f"{STAR_CATS}/exp" + cat = f"{cut_dir}/star_cat-{exp}.fits" work = exp_dir(exp) farm = f"{work}/star_cat_exp" images = f"{work}/output/run_sp_exp_Gie/get_images_runner/output" manifest = exp_manifest(exp, "exp_star_cat") body = json.dumps({ "stage": "exp_star_cat", "level": "exp", "unit": exp, - "status": "complete", "cache": cat, "link_dir": farm, "n_links": 40, + "status": "complete", "cat": cat, "link_dir": farm, "n_links": 40, }, indent=2, sort_keys=True) return "\n".join([ "set -euo pipefail", @@ -121,16 +183,10 @@ def star_cat_cmd(exp): # which would recurse into the pool, and never touch a real directory: # a real farm is this rule's own output and `ln -sfn` refreshes it. f"[ -L '{farm}' ] && rm -f '{farm}' || true", - f"mkdir -p '{cache}' '{farm}' '{work}/manifests'", - # Same binds and isolation as the profile's apptainer-args; the SDM does - # not wrap this rule (container: None below), because the farm loop and - # the manifest write belong to the host side of the job. - f"apptainer exec --cleanenv --home {Path.home()} --bind /project --bind /scratch" - f" --env PYTHONPATH={REPO_DIR}/src,REQUESTS_CA_BUNDLE={STAR_CAT_CA}" - f",SSL_CERT_FILE={STAR_CAT_CA},CURL_CA_BUNDLE={STAR_CAT_CA}" - f" '{config['container']}'" - f" python {REPO_DIR}/scripts/python/create_star_cat.py" - f" -i '{images}' -o '{cache}' -k exp", + f"mkdir -p '{cut_dir}' '{farm}' '{work}/manifests'", + in_container(f"python {SCRIPTS}/star_cats.py cut" + f" --images '{images}' --store '{STAR_CATS}'" + f" --out '{cat}'"), f"test -s '{cat}'", # The fan-out the file handler's NUMBERING_SCHEME wants: 40 links to the # one focal-plane catalogue (pattern from the p3-batch1 precedent). @@ -148,28 +204,26 @@ def star_cat_cmd(exp): rule exp_star_cat: input: - rules.exp_get_images.output.manifest + rules.exp_get_images.output.manifest, + # The chunks this cut reads. star_cats.py fails loudly on a chunk that is + # missing anyway, but the edge is what makes the fetch happen first. + rules.star_catalogue.output.manifest output: manifest = f"{EXP_DIR}/manifests/exp_star_cat.json", # The sentinel: ccd-0 of the 40-link farm (see above). link = f"{EXP_DIR}/star_cat_exp/star_cat-{{exp}}-0.fits" - # No `log:`. Every other rule's log carries a completeness VERDICT, and this - # rule computes none: it is not a shapepipe_run, it has no count floor, and - # under `set -euo pipefail` it either completes or aborts at the failing - # step. Snakemake's own captured job stderr is the evidence for that. + # No `log:`, for the same reason as star_catalogue above. params: cmd = lambda wc: star_cat_cmd(wc.exp), - # create_star_cat.py is external to the shell string, so the `code` + # star_cats.py is external to the shell string, so the `code` # rerun-trigger does not see it — same reason SCRIPT_HASH exists. script_hash = STAR_CAT_HASH - # No SDM wrapping: the rule calls apptainer itself, because only the query - # runs in the container (bin/sp has loaded the apptainer module). container: None threads: 1 retries: 2 resources: - mem_mb = 2000, + mem_mb = 4000, runtime = 10 shell: "{params.cmd}" diff --git a/workflow/rules/prepare.smk b/workflow/rules/prepare.smk index d4a8d1cdd..08ac0d66f 100644 --- a/workflow/rules/prepare.smk +++ b/workflow/rules/prepare.smk @@ -11,9 +11,10 @@ tier). Rules stay group-compatible: shell only, no mid-chain localrules, no pipe outputs. Star catalogues for masking are NOT a prepare-phase concern and not pre-run -input: they are built in the compute DAG, one node per exposure -(``exp_star_cat`` in exposure.smk), into a run-independent cache. The tile side -has no star-cat node because it has no mask rule yet — see tile.smk. +input: the compute DAG fetches the campaign footprint's stars once +(``star_catalogue``) and cuts them per exposure (``exp_star_cat``), both in +exposure.smk, into a run-independent store. The tile side has no star-cat node +because it has no mask rule yet — see tile.smk. """ # NUMBER_LIST is never set for get_images (download stage; nothing on disk to diff --git a/workflow/scripts/star_cats.py b/workflow/scripts/star_cats.py new file mode 100644 index 000000000..56886cfa0 --- /dev/null +++ b/workflow/scripts/star_cats.py @@ -0,0 +1,354 @@ +#!/usr/bin/env python3 +"""The campaign's GSC 2.3 star catalogue, as a HEALPix-chunked sky store. + +Masking needs, for every exposure, the bright stars over its focal plane. The +sky does not change between exposures, so the network cost of that is a property +of the campaign's SKY AREA, not of its exposure count: exposures overlap each +other ~7-10 deep, and a tile's exposures all look at the same square degree. + +So the store is chunked by sky, not by exposure. One GSC 2.3 cone query per +HEALPix pixel of NSIDE=32, written run-independently under the ``star_cats`` +config root and never fetched twice. A campaign that grows past the fetched +footprint queries only the chunks its new tiles add; one that grows within it +queries nothing. + +Two numbers set the scale. A full-UNIONS footprint is ~1.5k chunks against ~25k +exposures, so the QUERY COUNT drops ~16x. The queried AREA drops ~4x: the old +design covered the footprint ~8 times over (that is just the exposure overlap +depth), the new one ~2 times, the 2x being the price of bounding a HEALPix +quadrilateral by the cone Vizier speaks (see ``pixel_cone``) — 5.6-9 deg^2 for a +3.36 deg^2 pixel, ~40-60k rows and ~3-4 MB per chunk. + +Two subcommands, one module, deliberately: ``fetch`` and ``cut`` must agree +EXACTLY on which pixel holds which star, and a shared NSIDE constant in one file +is the only version of that agreement which cannot drift. + + fetch --tile-list ... --store ... --manifest ... + The campaign side. Turns the tile list into the set of pixels its + exposures can possibly need, fetches the missing ones, writes a manifest. + + cut --images ... --store ... --out ... + The per-exposure side, purely local: read the focal-plane footprint from + the exposure's image headers, load the chunks covering it, deduplicate, + and cut to the focal-plane disc. Reproduces byte-for-byte the same sky + selection the old one-query-per-exposure cone did. + +Geometry, and why the fetch pad is what it is. Chunk-need is computed from the +TILE list rather than from exposure pointings, because tile IDs are the one thing +known before any download: a pointing center means reading a FITS header of an +image get_images has not fetched yet, and the DAG needs the chunk set at parse +time. Tiles sit on a fixed 0.5 deg grid (``cfis.get_tile_coord_from_nixy``), so +each tile is a disc of half-diagonal 0.354 deg; find_exposures gives a tile every +exposure whose footprint covers it, and the MegaCam focal plane is a disc of +radius 0.73 deg (measured on the cached catalogues). An exposure center is +therefore at most 0.354 + 0.73 deg from the tile center, and its stars 0.73 deg +beyond that: 1.81 deg, padded to ``PAD_DEG`` = 2.0. The pad is a perimeter cost +— negligible for a contiguous campaign, and paid once. + +The pad is a bound, not a promise: ``cut`` verifies that every chunk covering the +exposure it was handed is on disk, and fails loudly if one is not. A missing +chunk means the geometry above is wrong, and that must not degrade quietly into +an under-masked exposure. +""" + +import argparse +import json +import os +import sys +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +import numpy as np +import healpy as hp +from astropy import units as u +from astropy.coordinates import SkyCoord +from astropy.io import fits +from astropy.table import Table, vstack +from astropy.wcs import WCS + +# GSC 2.3. The same catalogue the mask module's own CDS path uses +# (mask.py: _CDS_cat_ID), so the store is a drop-in for it. +CAT_ID = "I/305/out" + +# NSIDE=32 -> 3.36 deg^2 per pixel, 12288 pixels over the sky. Chosen so one +# chunk is a couple of MegaCam focal planes: small enough that a Vizier query +# stays within a small multiple of the per-exposure queries this replaces, large +# enough that a full-UNIONS footprint is ~1.5k chunks rather than ~25k. The +# cone-vs-quadrilateral overhead is scale-free, so NSIDE trades query count +# against query size and nothing else. NESTED, so a chunk id is a hierarchical +# sky address and a future NSIDE change is a subdivision. +NSIDE = 32 +NEST = True + +# Angular padding on the disc used to select chunks (see the module docstring). +PAD_DEG = 2.0 + +# The MegaCam focal-plane disc, and the margin added to a pixel's own bounding +# cone. Both in degrees. +MARGIN_DEG = 0.02 + +# GSC 2.3's object id: the deduplication key where chunk cones overlap. +ID_COL = "GSC2.3" + + +# --- the chunk store -------------------------------------------------------- + + +def store_dir(store: Path) -> Path: + """Chunks live under the catalogue and resolution that produced them, so a + later NSIDE or catalogue change is a new directory beside the old one rather + than a silent reinterpretation of files already on disk.""" + return Path(store) / CAT_ID.replace("/", "_") / f"nside{NSIDE}" + + +def chunk_path(store: Path, ipix: int) -> Path: + return store_dir(store) / f"star_chunk-{ipix:06d}.fits" + + +def chunks_for_disc(ra_deg: float, dec_deg: float, radius_deg: float) -> list[int]: + """Every pixel that touches the disc, as sorted ids. + + ``inclusive=True`` makes this a conservative superset — the guarantee ``cut`` + relies on is that no star inside the disc lives in a pixel this omits. + """ + vec = hp.ang2vec(ra_deg, dec_deg, lonlat=True) + return sorted(int(i) for i in hp.query_disc( + NSIDE, vec, np.radians(radius_deg), inclusive=True, fact=4, nest=NEST)) + + +def pixel_cone(ipix: int) -> tuple[float, float, float]: + """(ra, dec, radius_arcmin) of a cone that CONTAINS pixel ``ipix``. + + Vizier speaks cones, HEALPix speaks quadrilaterals, so the query is the + pixel's bounding cone: its center, and the largest center-to-boundary + distance plus a margin. The cone spills over the pixel edges, which costs a + little duplication between neighbours and buys the containment ``cut`` + depends on. The duplicates are removed on read, by ``ID_COL``. + """ + ra_c, dec_c = hp.pix2ang(NSIDE, ipix, nest=NEST, lonlat=True) + ra_b, dec_b = hp.vec2ang(hp.boundaries(NSIDE, ipix, step=8, nest=NEST).T, + lonlat=True) + center = SkyCoord(ra_c * u.deg, dec_c * u.deg) + radius = center.separation(SkyCoord(ra_b * u.deg, dec_b * u.deg)).deg.max() + return float(ra_c), float(dec_c), float((radius + MARGIN_DEG) * 60.0) + + +def write_atomic(table: Table, path: Path) -> None: + """Publish ``table`` at ``path`` all-or-nothing. + + The store's only cache test is ``Path.exists``, so a write killed part-way + (timeout, OOM, node failure) would otherwise leave a truncated FITS that + every later run trusts forever. The temp keeps the ``.fits`` suffix because + astropy picks its writer from the extension, and is dot-prefixed and + PID-tagged so it stays out of the ``star_chunk-*`` globs and two concurrent + writers cannot collide. + """ + tmp = path.parent / f".tmp-{os.getpid()}-{path.name}" + try: + table.write(tmp, overwrite=True) + os.replace(tmp, path) + finally: + if tmp.exists(): + tmp.unlink() + + +def read_chunks(store: Path, ipixels: list[int]) -> Table: + """Load and deduplicate the given chunks. + + A missing chunk is fatal (see the module docstring): it means the fetch + footprint did not cover this exposure, and an under-masked exposure is worse + than a failed job. + """ + missing = [i for i in ipixels if not chunk_path(store, i).exists()] + if missing: + raise SystemExit( + f"star chunk(s) {missing} not in {store_dir(store)}. The campaign's " + f"star_catalogue fetch did not cover this exposure — re-run it " + f"(and check that its tile list contains this exposure's tiles).") + + table = vstack([Table.read(chunk_path(store, i)) for i in ipixels], + metadata_conflicts="silent") + _, keep = np.unique(np.asarray(table[ID_COL]), return_index=True) + return table[np.sort(keep)] + + +# --- exposure footprint ----------------------------------------------------- + + +def _wcs(header) -> WCS: + """Build the WCS by hand, from the linear terms only. + + Same construction as ``scripts/python/create_star_cat.py``: it sidesteps + distortion-convention incompatibilities between headers and astropy, and a + focal-plane footprint needs nothing finer. + """ + w = WCS(naxis=2) + w.wcs.ctype = [header["CTYPE1"], header["CTYPE2"]] + try: + w.wcs.cunit = [header["CUNIT1"], header["CUNIT2"]] + except KeyError: + w.wcs.cunit = ["deg", "deg"] + w.wcs.crpix = [header["CRPIX1"], header["CRPIX2"]] + w.wcs.crval = [header["CRVAL1"], header["CRVAL2"]] + w.wcs.cd = [[header["CD1_1"], header["CD1_2"]], + [header["CD2_1"], header["CD2_2"]]] + return w + + +def focal_plane_disc(image: Path, n_ccd: int = 40) -> tuple[float, float, float]: + """(ra, dec, radius_deg) of the disc covering all CCDs of one exposure.""" + centers, radii = [], [] + for ext in range(1, n_ccd + 1): + h = fits.getheader(image, ext) + w = _wcs(h) + (ra_c, dec_c), (ra_0, dec_0) = w.all_pix2world( + [[h["NAXIS1"] / 2.0, h["NAXIS2"] / 2.0], [0, 0]], 1) + centers.append(SkyCoord(ra_c * u.deg, dec_c * u.deg)) + radii.append(centers[-1].separation( + SkyCoord(ra_0 * u.deg, dec_0 * u.deg)).deg) + + ras = np.array([c.ra.deg for c in centers]) + decs = np.array([c.dec.deg for c in centers]) + center = SkyCoord(ras.mean() * u.deg, decs.mean() * u.deg) + seps = center.separation(SkyCoord(ras * u.deg, decs * u.deg)).deg + return (float(center.ra.deg), float(center.dec.deg), + float(np.max(seps + np.array(radii)))) + + +def exposure_image(images_dir: Path) -> Path: + """The one multi-extension exposure image in a get_images output dir. + + That dir is a symlink farm holding ``image-.fitsfz`` plus its weight and + flag; only the image carries the 40 CCD WCSs. + """ + found = sorted(p for p in Path(images_dir).iterdir() if "image" in p.name) + if not found: + raise SystemExit(f"no image file in {images_dir}") + return found[0] + + +# --- fetch ------------------------------------------------------------------ + + +def campaign_chunks(tile_ids: list[str]) -> list[int]: + """Every chunk the campaign's exposures can need, from the tile list alone.""" + from shapepipe.utilities.cfis import get_tile_coord_from_nixy + + needed: set[int] = set() + for tile_id in tile_ids: + nix, niy = tile_id.split(".") + ra, dec = get_tile_coord_from_nixy(nix, niy) + needed.update(chunks_for_disc(ra.degree, dec.degree, PAD_DEG)) + return sorted(needed) + + +def fetch(args: argparse.Namespace) -> None: + from shapepipe.utilities.vizier import query_vizier + + tile_ids = [ln.strip() for ln in Path(args.tile_list).read_text().splitlines() + if ln.strip()] + needed = campaign_chunks(tile_ids) + out_dir = store_dir(args.store) + out_dir.mkdir(parents=True, exist_ok=True) + todo = [i for i in needed if not chunk_path(args.store, i).exists()] + print(f"star chunks: {len(needed)} needed for {len(tile_ids)} tiles, " + f"{len(todo)} to fetch -> {out_dir}", file=sys.stderr) + + def one(ipix: int) -> int: + ra, dec, radius_arcmin = pixel_cone(ipix) + table = query_vizier(ra, dec, radius_arcmin, CAT_ID) + write_atomic(table, chunk_path(args.store, ipix)) + print(f"chunk {ipix}: {len(table)} rows " + f"(ra={ra:.4f} dec={dec:.4f} r={radius_arcmin:.1f}')", + file=sys.stderr) + return len(table) + + # A handful of concurrent queries, never one per exposure: the same modest + # concurrency the per-exposure rule reached through --local-cores, now an + # explicit number instead of an accident of the head node's CPU count. + if todo: + with ThreadPoolExecutor(max_workers=args.workers) as pool: + list(pool.map(one, todo)) + + write_manifest(args, tile_ids, needed, len(todo)) + + +def write_manifest(args, tile_ids, needed, n_fetched) -> None: + """The rule's declared output. + + Not a ``completeness.py`` verdict: this rule runs no ``shapepipe_run`` and + has no per-runner count floors, so there is nothing to compose and no + separate ``log:`` — under ``set -euo pipefail`` the job either completes or + aborts at the failing query, and snakemake's captured stderr is the evidence. + The manifest keeps the workflow's "one rule, one manifest" currency: written + last, and only when the content changed, so an unchanged campaign leaves the + mtime where it was rather than churning the `mtime` rerun-trigger. + """ + body = json.dumps({ + "stage": "star_catalogue", + "level": "campaign", + "status": "complete", + "catalogue": CAT_ID, + "nside": NSIDE, + "nest": NEST, + "pad_deg": PAD_DEG, + "store": str(store_dir(args.store)), + "n_tiles": len(tile_ids), + "n_chunks": len(needed), + "n_fetched": n_fetched, + "chunks": needed, + }, indent=2, sort_keys=True) + path = Path(args.manifest) + path.parent.mkdir(parents=True, exist_ok=True) + if not path.exists() or path.read_text() != body: + path.write_text(body) + + +# --- cut -------------------------------------------------------------------- + + +def cut(args: argparse.Namespace) -> None: + image = exposure_image(args.images) + ra, dec, radius = focal_plane_disc(image) + ipixels = chunks_for_disc(ra, dec, radius) + table = read_chunks(args.store, ipixels) + + center = SkyCoord(ra * u.deg, dec * u.deg) + stars = SkyCoord(np.asarray(table["RAJ2000"]) * u.deg, + np.asarray(table["DEJ2000"]) * u.deg) + inside = table[center.separation(stars).deg <= radius] + + print(f"{image.name}: ra={ra:.4f} dec={dec:.4f} r={radius:.4f} deg, " + f"{len(ipixels)} chunks -> {len(inside)} stars", file=sys.stderr) + out = Path(args.out) + out.parent.mkdir(parents=True, exist_ok=True) + write_atomic(inside, out) + + +# --- CLI -------------------------------------------------------------------- + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + sub = p.add_subparsers(dest="cmd", required=True) + + f = sub.add_parser("fetch", help="fetch the campaign footprint's chunks") + f.add_argument("--tile-list", required=True, type=Path) + f.add_argument("--store", required=True, type=Path) + f.add_argument("--manifest", required=True, type=Path) + f.add_argument("--workers", type=int, default=4) + f.set_defaults(func=fetch) + + c = sub.add_parser("cut", help="cut one exposure's catalogue from the store") + c.add_argument("--images", required=True, type=Path, + help="a get_images output dir holding image-.fitsfz") + c.add_argument("--store", required=True, type=Path) + c.add_argument("--out", required=True, type=Path) + c.set_defaults(func=cut) + + args = p.parse_args() + args.func(args) + + +if __name__ == "__main__": + main() From f2cf65b6a61a2d4a043fdd2df1abf612ca343fe1 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 29 Aug 2026 16:29:09 -0400 Subject: [PATCH 34/41] =?UTF-8?q?feat(container):=20sp=20container=20?= =?UTF-8?q?=E2=80=94=20per-user=20image=20layers=20and=20resolution?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The image every job runs inside is a fixed /project .sif, shared by everyone and refreshed by hand. `sp container` gives each user their own copy and one resolution order that the CLI and the workflow both obey: a writable sandbox if they built one, else their cached SIF, else the `container:` path in workflow/config.yaml. An empty cache lands on that shared .sif, so nothing changes until a user opts in. pull and sandbox stage to a sibling path and swap it in, so an in-flight job never sees a half-written image and a failed rebuild leaves the working one intact. status reads the image's OCI labels and places its revision against this checkout's HEAD. exec runs a one-off command with the profile's binds. The script is stdlib-only: it runs on the bare host, outside the container. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011Vg5ZoFShkKRdkk9bpbpXy --- profiles/nibi/config.yaml | 7 +- workflow/README.md | 45 +++- workflow/Snakefile | 11 +- workflow/bin/sp | 14 +- workflow/scripts/container.py | 432 ++++++++++++++++++++++++++++++++++ 5 files changed, 503 insertions(+), 6 deletions(-) create mode 100644 workflow/scripts/container.py diff --git a/profiles/nibi/config.yaml b/profiles/nibi/config.yaml index 8bc71a9d1..8987ab9bc 100644 --- a/profiles/nibi/config.yaml +++ b/profiles/nibi/config.yaml @@ -8,7 +8,12 @@ # # Launch via `workflow/bin/sp` (loads apptainer/1.4.5, uses the /project venv). # software-deployment-method wraps every job's shell in `apptainer exec` — the -# user never types apptainer; the container is set in the Snakefile (local .sif). +# user never types apptainer. WHICH image is resolved in the Snakefile through +# workflow/scripts/container.py: this user's writable sandbox if they built one, +# else their cached SIF (`sp container pull`), else the `container:` path in +# workflow/config.yaml — the shared /project .sif, and the default whenever the +# cache is empty. `sp container status` prints which layer is live. `sp container +# pull` needs the network: login node or salloc, never a batch job. executor: slurm diff --git a/workflow/README.md b/workflow/README.md index 1d219617e..52e8d58c3 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -30,6 +30,7 @@ uv pip install 'snakemake>=9,<10' 'snakemake-executor-plugin-slurm>=2.7,<3' workflow/bin/sp run # bring products on disk up to date with the tile list workflow/bin/sp report # emit run_report.json now (mid-run is fine) workflow/bin/sp cancel # scancel this workflow's jobs +workflow/bin/sp container status # which image the jobs will run ``` Installed and pinned versions on nibi (`/project/def-mjhudson/cdaley/snakemake-env`, @@ -38,10 +39,49 @@ Pin range: `snakemake>=9,<10`, `snakemake-executor-plugin-slurm>=2.7,<3`. The v8→v9 breaks matter here: `--use-singularity` became `--sdm`, executors became plugins, and full `rerun-triggers` became the default. -Anything other than `run`, `report`, `cancel` passes straight through to +Anything other than `run`, `report`, `container`, `cancel` passes straight through to snakemake with the workflow's profile and state dir — the escape hatch for `sp --unlock`, `sp --dag`, `sp exp_psf ...`. +## The container image + +`sp container` owns which image the jobs run inside. Two layers, and the second +only exists if you ask for one: + +* your **cached SIF** (`~/.cache/shapepipe/shapepipe.sif`, `SP_CACHE_DIR` or + `SP_CONTAINER` to move it) — a pristine pull of the published image, private + to you, so nobody else's refresh moves the ground under your running jobs; +* an optional **sandbox** (`~/.cache/shapepipe/sandbox/`, `SP_SANDBOX`) — the + same image unpacked writable, so a `pip install` into it sticks. The escape + hatch for work needing a package the image does not carry yet. + +The Snakefile's `container:` is that resolution, in one order shared by the CLI +and the workflow: **sandbox → cached SIF → the `container:` path in +`config.yaml`**. With an empty cache — the normal case — that lands on the +shared `/project` `.sif` the workflow has always used, so this changes nothing +until you opt in. + +```bash +sp container status # layers present, active one, revision vs HEAD +sp container pull # ghcr.io/cosmostat/shapepipe:develop-runtime +sp container pull --tag docker://... # some other image +sp container sandbox # unpack the SIF writable (opt-in) +sp container exec --writable pip install +sp container exec python -c 'import shapepipe' +sp container resolve # just the path the workflow will run +``` + +`status` reads the image's OCI labels and places its +`org.opencontainers.image.revision` against this checkout's HEAD: +in-sync / behind / ahead / diverged, or unknown when the image carries no label +or the commit was never fetched here. + +**`pull` needs the network.** Compute nodes on Alliance clusters generally have +none, so run it on a login node or inside an `salloc` allocation — never from a +batch job. `pull` and `sandbox` both stage to a sibling path and swap it in, so +an in-flight job never sees a half-written image and a failed rebuild leaves the +one you had intact. + ## Execution: two static invocations The exposure job set is data-derived from the tiles' `find_exposures` output, @@ -91,7 +131,7 @@ profile-only pass. workflow/ Snakefile parse-time index load; global container:; onsuccess/onerror report hooks config.yaml the run: tile list, paths, container, chunk count - bin/sp committed launcher (module load + /project venv + run/report/cancel) + bin/sp committed launcher (module load + /project venv + run/report/container/cancel) rules/ prepare.smk tile get_images/uncompress/find_exposures exposure.smk per-exposure: get_images, star_cat, split, mask, psf (no temp()) @@ -102,6 +142,7 @@ workflow/ build_forest.py per-tile exposure symlink forest (group-compatible shell) completeness.py the ported count-floor table (shared by sp_rule + run_report) run_report.py standalone report (NOT a DAG node; run_report hooks call it) + container.py image layers + the resolution order behind `sp container` (stdlib-only) clean_exposure.py ONE exposure's store + manifests + logs -> tombstone (the clean_exposure rule) profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; keep-going ``` diff --git a/workflow/Snakefile b/workflow/Snakefile index 352761501..2d9f712de 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -44,8 +44,15 @@ from snakemake.exceptions import WorkflowError configfile: str(Path(workflow.snakefile).parent / "config.yaml") # Every job's shell runs inside this container (apptainer software-deployment in -# the profile); the user never types apptainer. -container: config["container"] +# the profile); the user never types apptainer. WHICH image is the one resolution +# order `sp container` exposes — this user's writable sandbox if they built one, +# else their cached SIF if they pulled one, else `container:` above. An empty +# cache therefore lands on exactly the shared /project image the workflow has +# always run, and a package installed into a sandbox reaches the jobs too. +sys.path.insert(0, str(Path(workflow.snakefile).parent / "scripts")) +import container as _container # noqa: E402 + +container: _container.resolve_image()[0] # --- paths ----------------------------------------------------------------- RUN_DIR = Path(config["run_dir"]) diff --git a/workflow/bin/sp b/workflow/bin/sp index ec563f032..72e80b783 100755 --- a/workflow/bin/sp +++ b/workflow/bin/sp @@ -1,7 +1,7 @@ #!/usr/bin/env bash # sp — the committed launcher for the ShapePipe Snakemake workflow (PRD #848 D1). # -# Two verbs, nothing else: +# Three verbs, nothing else: # # sp run [ARGS...] bring the products on disk up to date with the tile list. # Two snakemake invocations over one Snakefile: @@ -9,6 +9,12 @@ # 2. COMPUTE snakemake all <- its PARSE builds the index # ARGS (--jobs, -n, --forcerun, ...) pass through to BOTH. # sp report [ARGS...] emit run_report.json now (mid-run is fine). +# sp container VERB manage the image every job runs inside: pull, status, +# sandbox, exec, resolve. `sp container --help` documents +# the two layers and the resolution order (sandbox -> your +# cached SIF -> the `container:` path in config.yaml). +# `pull` needs the network: run it on a login node or in an +# salloc allocation, never from a batch job. # # Anything else is passed straight through to snakemake with the same profile and # state dir (the escape hatch: `sp --unlock`, `sp exp_psf ...`, `sp --dag`). @@ -83,6 +89,12 @@ case "$cmd" in python "$SCRIPTS/run_report.py" --run-dir "$RUN_DIR" --index "$INDEX_DB" \ --status manual "$@" ;; + container) + # The image layer, outside snakemake entirely: the script is stdlib-only so + # it runs on the bare host, and the apptainer module is already loaded above. + shift + python "$SCRIPTS/container.py" "$@" + ;; cancel) # Kept only because it is two lines: scancel this workflow's jobs by name # before an --unlock. Not part of the design surface. diff --git a/workflow/scripts/container.py b/workflow/scripts/container.py new file mode 100644 index 000000000..0479b0f46 --- /dev/null +++ b/workflow/scripts/container.py @@ -0,0 +1,432 @@ +#!/usr/bin/env python3 +"""Manage this user's copy of the ShapePipe container image. + +Two layers, and the second only exists when you ask for one: + +* the **SIF** (``~/.cache/shapepipe/shapepipe.sif``) -- a pristine, read-only + copy of the published image, pulled into your own cache. Per-user by + construction: one file, one owner, nobody else's refresh moves the ground + under a running job. +* an optional **sandbox** (``~/.cache/shapepipe/sandbox/``) -- the same image + unpacked into a writable directory, so a ``pip install`` inside it sticks. + The escape hatch for work that needs a package the image does not carry yet. + +Resolution order, shared by this CLI and by the workflow: **sandbox if it +exists, else the cached SIF if it exists, else the ``container:`` path in +workflow/config.yaml**. That last one is the current shared /project image, so +a checkout with an empty cache behaves exactly as it did before this verb +existed, and a package installed into your sandbox is there for your workflow +jobs too. + +Subcommands, exposed as ``sp container ``:: + + sp container pull # fetch the tag into the cache + sp container status # what is here, and how current it is + sp container sandbox # unpack the SIF into a writable dir + sp container exec # run something inside it + sp container exec --writable # ... with writes that persist + +``pull`` needs the network. Compute nodes on Alliance clusters generally have +none, so run it on a login node or inside an ``salloc`` allocation -- never +from a batch job. + +Deliberately **stdlib-only**: it runs on the bare host, outside the container, +where the science stack is not installed, and so must import without it. +""" + +import argparse +import os +import re +import shutil +import subprocess +import sys +from pathlib import Path + +# The published image. CI pushes one tag per branch, sanitized; `-runtime` is +# the slim variant the workflow runs. +CONTAINER_URI = "docker://ghcr.io/cosmostat/shapepipe:develop-runtime" + +# The single source of truth for the fallback image: the workflow's own +# `container:` key, which is also what the Snakefile reads. Written down once, +# here, so the CLI and the workflow cannot disagree about the default. +CONFIG_FILE = Path(__file__).resolve().parents[1] / "config.yaml" +CONFIG_KEY = "container" + +# ~/.cache/shapepipe by default; SP_CACHE_DIR moves the whole cache (e.g. onto +# a filesystem with room), XDG_CACHE_HOME moves it with everything else. +CACHE_DIR = Path( + os.environ.get("SP_CACHE_DIR") + or Path(os.environ.get("XDG_CACHE_HOME", "~/.cache")) / "shapepipe" +).expanduser() + +# This user's pristine image. Override with ``SP_CONTAINER`` (absolute path). +DEFAULT_SIF = CACHE_DIR / "shapepipe.sif" + +# The optional writable unpacking of it. Override with ``SP_SANDBOX``. +DEFAULT_SANDBOX = CACHE_DIR / "sandbox" + +# Bind mounts for `exec`, matching the nibi profile's apptainer-args (the two +# cluster filesystems this workflow reads and writes, plus the home that holds +# ~/.ssl/cadcproxy.pem). Override wholesale with ``SP_APPTAINER_BINDS``. +DEFAULT_BINDS = "/project,/scratch,/home" + + +def configured_default(): + """Return the ``container:`` path from workflow/config.yaml, or ``None``. + + A deliberately minimal scalar read (the same one bin/sp does in sed): this + module is stdlib-only, so there is no yaml to import. + """ + try: + text = CONFIG_FILE.read_text() + except OSError: + return None + match = re.search(rf"^{CONFIG_KEY}:[ \t]*(\S+)", text, re.MULTILINE) + return match.group(1) if match else None + + +def local_sif(): + """Return this user's cached image path (may not exist yet).""" + override = os.environ.get("SP_CONTAINER") + return (Path(override) if override else DEFAULT_SIF).expanduser() + + +def local_sandbox(): + """Return this user's writable sandbox directory (may not exist).""" + override = os.environ.get("SP_SANDBOX") + return (Path(override) if override else DEFAULT_SANDBOX).expanduser() + + +def resolve_image(): + """Return ``(path, kind)`` for the image everything should run. + + ``kind`` is ``"sandbox"``, ``"sif"``, ``"configured"`` (the shared + /project image named in config.yaml -- the default when the cache is + empty) or ``"none"``. + """ + sandbox = local_sandbox() + if sandbox.is_dir(): + return str(sandbox), "sandbox" + sif = local_sif() + if sif.exists(): + return str(sif), "sif" + default = configured_default() + if default: + return default, "configured" + return "", "none" + + +def image_labels(image): + """Return the image's OCI labels as a dict, or ``{}`` if unreadable. + + Never raises: a missing file, a missing ``apptainer`` or a corrupt image + all mean "we don't know", which every caller treats as non-fatal. + """ + path = Path(image) + if not path.exists() or shutil.which("apptainer") is None: + return {} + try: + out = subprocess.run( + ["apptainer", "inspect", "--labels", str(path)], + capture_output=True, + text=True, + timeout=60, + ) + except (OSError, subprocess.SubprocessError): + return {} + if out.returncode != 0: + return {} + labels = {} + for line in out.stdout.splitlines(): + key, sep, value = line.partition(":") + if sep: + labels[key.strip()] = value.strip() + return labels + + +def image_revision(image): + """Return the commit the image was built from, or ``None``.""" + return image_labels(image).get("org.opencontainers.image.revision") + + +def _require_apptainer(): + """Exit unless ``apptainer`` is on PATH (bin/sp loads the module).""" + if shutil.which("apptainer") is None: + sys.exit("apptainer is not on PATH (bin/sp loads apptainer/1.4.5)") + + +def _git(*args, cwd=None): + """Run git, returning stripped stdout or ``None`` on any failure.""" + try: + out = subprocess.run( + ["git", *args], capture_output=True, text=True, cwd=cwd, timeout=30 + ) + except (OSError, subprocess.SubprocessError): + return None + return out.stdout.strip() if out.returncode == 0 else None + + +def compare_revision(revision, repo=None): + """Place an image revision relative to this checkout's HEAD. + + One of ``"in-sync"``, ``"behind"`` (the image predates HEAD), ``"ahead"``, + ``"diverged"``, or ``"unknown"`` (no label, no git, or a commit this clone + has never fetched). + """ + if not revision: + return "unknown" + repo = repo or Path(__file__).resolve().parents[2] + head = _git("rev-parse", "HEAD", cwd=repo) + if head is None: + return "unknown" + if head == revision: + return "in-sync" + if _git("cat-file", "-e", f"{revision}^{{commit}}", cwd=repo) is None: + return "unknown" + if _git("merge-base", "--is-ancestor", revision, head, cwd=repo) is not None: + return "behind" + if _git("merge-base", "--is-ancestor", head, revision, cwd=repo) is not None: + return "ahead" + return "diverged" + + +def cmd_pull(args): + """Pull ``--tag`` into the cache, atomically.""" + _require_apptainer() + sif = local_sif() + sif.parent.mkdir(parents=True, exist_ok=True) + # Pull to a sibling temp name and rename: an atomic rename within one + # directory, so an in-flight job sees either the whole old image or the + # whole new one. Pulling in place leaves the file half-written for the many + # minutes the pull takes. Jobs already running hold the old inode open. + tmp = sif.with_name(sif.name + f".pull.{os.getpid()}") + print(f"pulling {args.tag}\n -> {sif}") + try: + subprocess.run( + ["apptainer", "pull", "--force", "--name", str(tmp), args.tag], check=True + ) + os.replace(tmp, sif) + except subprocess.CalledProcessError as exc: + tmp.unlink(missing_ok=True) + sys.exit(f"pull failed ({exc.returncode}); {sif} is unchanged") + except KeyboardInterrupt: + tmp.unlink(missing_ok=True) + raise + labels = image_labels(sif) + print(f"revision: {labels.get('org.opencontainers.image.revision', 'unknown')}") + print(f"version: {labels.get('org.opencontainers.image.version', 'unknown')}") + return 0 + + +def cmd_sandbox(args): + """Unpack the image into a writable directory -- the opt-in escape hatch.""" + _require_apptainer() + sandbox = local_sandbox() + if sandbox.exists() and not args.force: + sys.exit( + f"sandbox already exists at {sandbox}\n" + "pass --force to discard it and rebuild from a clean image" + ) + source = args.source + if not source: + image, kind = resolve_image() + if kind == "sandbox": + # Rebuilding from the sandbox itself would just re-copy the drift. + image = str(local_sif()) if local_sif().exists() else ( + configured_default() or CONTAINER_URI + ) + source = image or CONTAINER_URI + sandbox.parent.mkdir(parents=True, exist_ok=True) + print(f"building sandbox from {source}\n -> {sandbox}") + # Build beside the target and swap it in, as `pull` does -- and for a + # sharper reason. A half-written .sif fails loudly, but a half-unpacked + # sandbox *directory* is still a directory, so resolve_image() would elect + # it and every job would silently run a broken tree. Staging also means a + # --force rebuild that fails leaves the sandbox you already had intact. + # + # `--fix-perms` so the tree can be deleted again later. No `--fakeroot`: an + # unprivileged build from an existing image goes through user namespaces, + # which is what the Alliance clusters provide. + staging = sandbox.with_name(f"{sandbox.name}.build.{os.getpid()}") + shutil.rmtree(staging, ignore_errors=True) + try: + subprocess.run( + ["apptainer", "build", "--sandbox", "--fix-perms", str(staging), source], + check=True, + ) + except subprocess.CalledProcessError as exc: + shutil.rmtree(staging, ignore_errors=True) + sys.exit(f"sandbox build failed ({exc.returncode}); {sandbox} is unchanged") + except (KeyboardInterrupt, OSError): + shutil.rmtree(staging, ignore_errors=True) + raise + + if sandbox.exists(): + print(f"replacing {sandbox}") + shutil.rmtree(sandbox, ignore_errors=True) + if sandbox.exists(): + shutil.rmtree(staging, ignore_errors=True) + sys.exit(f"could not remove {sandbox}; remove it by hand and retry") + os.replace(staging, sandbox) + print( + "\nthis sandbox now takes precedence over the SIF everywhere, including " + "workflow jobs.\ninstall into it with: sp container exec --writable pip " + "install \nreset to a clean image with: sp container pull && " + "sp container sandbox --force" + ) + return 0 + + +def cmd_status(args): + """Report which image layer is live, its revision, and how current it is.""" + sif = local_sif() + sandbox = local_sandbox() + active, kind = resolve_image() + + if sif.exists(): + print(f"SIF: {sif} ({sif.stat().st_size / 1e9:.1f} GB)") + else: + print(f"SIF: absent ({sif})") + if sandbox.is_dir(): + print(f"sandbox: {sandbox} (writable; may carry local modifications)") + else: + print("sandbox: none") + print(f"configured: {configured_default() or 'unset'} ({CONFIG_FILE})") + + if kind == "none": + print("\nactive: NONE -- no cached image and no container: in config.yaml") + print(f"run: sp container pull --tag {CONTAINER_URI}") + return 1 + + print(f"\nactive: {active} ({kind})") + if kind == "configured" and not Path(active).exists(): + print(" WARNING: that path does not exist on this host") + return 1 + + labels = image_labels(active) + revision = labels.get("org.opencontainers.image.revision") + source = "" + if revision is None and kind == "sandbox" and sif.exists(): + # Some sandbox trees do not carry the original labels through. The SIF + # beside it is the best remaining evidence of what it was built from -- + # a guess, so it is labelled as one rather than printed as fact. + revision = image_revision(sif) + if revision: + source = " (inferred from the SIF beside it, not read from the sandbox)" + print(f"revision: {revision or 'unknown'}{source}") + print(f"version: {labels.get('org.opencontainers.image.version', 'unknown')}") + if kind == "sandbox": + print( + " (that revision is what the sandbox was built from; " + "anything\n installed into it since is in no label)" + ) + verdict = compare_revision(revision) + explain = { + "in-sync": "matches this checkout's HEAD", + "behind": "older than this checkout's HEAD -- pull to refresh", + "ahead": "newer than this checkout's HEAD", + "diverged": "on a different branch from this checkout", + "unknown": "cannot compare (no label, or a commit this clone lacks)", + }[verdict] + print(f"checkout: {verdict} ({explain})") + return 0 + + +def cmd_exec(args): + """Run a command inside the resolved image -- the one-off path.""" + _require_apptainer() + command = [a for a in args.command if a != "--"] + if not command: + sys.exit("nothing to run; pass a command after `exec`") + binds = args.bind or os.environ.get("SP_APPTAINER_BINDS", DEFAULT_BINDS) + + if args.writable: + # A SIF is a read-only filesystem, so `--writable` against one fails + # obscurely; only a sandbox takes writes. + sandbox = local_sandbox() + if not sandbox.is_dir(): + sys.exit( + f"--writable needs a sandbox, and there is none at {sandbox}\n" + "build one with: sp container sandbox" + ) + image, extra = str(sandbox), ["--writable"] + else: + image, kind = resolve_image() + if kind == "none": + sys.exit("no image resolved; run: sp container pull") + extra = [] + + cmd = ["apptainer", "exec", *extra, "--cleanenv", "--bind", binds, image, *command] + return subprocess.run(cmd).returncode + + +def cmd_resolve(args): + """Print just the resolved image path -- what the Snakefile consumes.""" + image, kind = resolve_image() + if kind == "none": + sys.exit("no image resolved; run: sp container pull") + print(image) + return 0 + + +def build_parser(): + parser = argparse.ArgumentParser( + prog="sp container", + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + sub = parser.add_subparsers(dest="subcommand", required=True) + + p_pull = sub.add_parser( + "pull", + help="fetch the image into the cache (login node or salloc: compute " + "nodes have no network)", + ) + p_pull.add_argument( + "--tag", default=CONTAINER_URI, help=f"image to pull (default: {CONTAINER_URI})" + ) + p_pull.set_defaults(func=cmd_pull) + + p_status = sub.add_parser( + "status", help="report which image layer is live and how current it is" + ) + p_status.set_defaults(func=cmd_status) + + p_sandbox = sub.add_parser( + "sandbox", help="unpack the image into a writable directory (opt-in)" + ) + p_sandbox.add_argument( + "--source", help="image to unpack (default: the cached SIF, or the config path)" + ) + p_sandbox.add_argument( + "--force", + action="store_true", + help="discard an existing sandbox and rebuild from a clean image", + ) + p_sandbox.set_defaults(func=cmd_sandbox) + + p_exec = sub.add_parser("exec", help="run a command inside the resolved image") + p_exec.add_argument("--bind", help=f"bind mounts (default: {DEFAULT_BINDS})") + p_exec.add_argument( + "--writable", + action="store_true", + help="run against the sandbox so writes (e.g. pip install) persist", + ) + p_exec.add_argument("command", nargs=argparse.REMAINDER) + p_exec.set_defaults(func=cmd_exec) + + p_resolve = sub.add_parser( + "resolve", help="print the resolved image path (what the workflow runs)" + ) + p_resolve.set_defaults(func=cmd_resolve) + + return parser + + +def main(argv=None): + args = build_parser().parse_args(argv) + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) From 1ed2537b91bfe27707363dff4b4308112eb8c209 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 29 Aug 2026 16:30:26 -0400 Subject: [PATCH 35/41] Fuse the short rules into per-unit group jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three group labels, one sbatch per unit instead of one per rule: tile_prep (the whole prepare chain), exp_short (exp_split + exp_mask) and tile_finish (tile_merge_cats + tile_make_cat). Every member's median is under three minutes, well below the 15-minute floor Alliance policy asks us to bundle away, and at DR6 scale the ungrouped form is tens of thousands of submissions for minutes of work each. Heavy rules stay standalone (exp_psf, tile_detect, tile_vignets, tile_ngmix), and exp_get_images stays out of exp_short because the exp_star_cat localrule sits between it and exp_mask — including it would make the group both a dependency and a dependent of that localrule. tile_merge_headers is short but stranded between a localrule and tile_detect, so it stays one job. Group resources compose per toposort level (GroupResources.basic_layered): non -additive resources max across levels, runtime sums. Attempt-scaled mem_mb and per-rule retries survive — GroupJob.attempt re-sets attempt on every member and GroupJob.restart_times is the max over them. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011Vg5ZoFShkKRdkk9bpbpXy --- profiles/nibi/config.yaml | 10 ++++++---- workflow/rules/exposure.smk | 17 +++++++++++++++++ workflow/rules/prepare.smk | 22 ++++++++++++++++++++++ workflow/rules/tile.smk | 17 ++++++++++++++++- 4 files changed, 61 insertions(+), 5 deletions(-) diff --git a/profiles/nibi/config.yaml b/profiles/nibi/config.yaml index 8987ab9bc..65746786f 100644 --- a/profiles/nibi/config.yaml +++ b/profiles/nibi/config.yaml @@ -86,7 +86,9 @@ rerun-triggers: [mtime, params, code, software-env] # ngmix thread count. The RULES own threads and resources; this profile only # sets defaults for rules that state nothing (default-resources above). # -# `group:` fusion of the short rules (uncompress, merges) into their chunky -# neighbours (PRD D-profile) is NOT set here — it requires labels in the rule -# files themselves, which is out of scope for this profile-only pass. Deferred -# to whichever slice next touches workflow/rules/*.smk. +# `group:` fusion of the short rules (PRD D-profile) lives in the rule files, not +# here: the labels are `tile_prep` (prepare.smk), `exp_short` (exp_split + +# exp_mask) and `tile_finish` (tile_merge_cats + tile_make_cat), each documented +# in its file's docstring. Grouping composes resources per toposort level (max +# mem/threads, summed runtime for a linear chain) and leaves the attempt scaling +# above intact, which is the other reason set-resources must stay out of here. diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index c6fea1cae..e41920137 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -19,6 +19,21 @@ by the accumulating index. A temp() here would delete an exposure the moment this invocation's readers finished and cascade destructive reruns across spatial neighbours the next time a tile is appended. +GROUPING (``group: "exp_short"``) covers exp_split and exp_mask, and only them — +one sbatch per exposure for two jobs whose medians are 1:28 and 1:54, well under +the 15-minute floor Alliance policy asks us to bundle away. The composition +rules are in prepare.smk's docstring; this chain is linear too, so the group +asks max(mem_mb) = 8000*attempt, max(threads) = 8, sum(runtime) = 240 min. + +The two rules NOT in it are structural, not taste: + * exp_psf is heavy (16 GB, 4 h) and never fuses with a short rule; + * exp_get_images cannot join, because ``exp_star_cat`` — a LOCALRULE, and so + ungroupable — sits between it and exp_mask. Pulling get_images in would make + the group both a dependency and a dependent of exp_star_cat, i.e. a cycle. + Starting the group at exp_split leaves star_cat's inputs entirely upstream + of it, so the group has one clean external edge. +Different exposures share no DAG edge, so this is one group job per exposure. + NUMBER_LIST is set only for exp_split (its numbering scheme IS the exposure id); never for get_images / exp_mask / exp_psf, whose per-CCD or download numbering would make the #746 startup validation turn tolerated per-CCD attrition into a @@ -231,6 +246,7 @@ rule exp_star_cat: # Split the multi-HDU exposure into single-CCD files (+ headers-*.npy, which the # tiles' merge_headers reads). rule exp_split: + group: "exp_short" input: rules.exp_get_images.output.manifest output: @@ -248,6 +264,7 @@ rule exp_split: sp_shell("exp_split", "config_exp_Sp.ini") rule exp_mask: + group: "exp_short" input: # Both inputs are real INPUT_DIRs of config_exp_Ma.ini: the split CCDs # and this exposure's own star_cat_exp farm. diff --git a/workflow/rules/prepare.smk b/workflow/rules/prepare.smk index 08ac0d66f..96a0ae261 100644 --- a/workflow/rules/prepare.smk +++ b/workflow/rules/prepare.smk @@ -8,8 +8,27 @@ data-derived tile->exposure edge that invocation 2's parse aggregates into the index. Nibi compute nodes have internet, so downloads run in-DAG (no login-node tier). +All three rules carry ``group: "tile_prep"``, so one tile's whole chain is ONE +sbatch instead of three (medians 0:41 / ~0:40 / 0:15 — all far under the +15-minute floor Alliance policy asks us to bundle away, and at DR6 scale three +submissions per tile is a scheduler load out of all proportion to the work). +Group membership is per connected DAG component and distinct tiles share no +edge, so this is exactly one group job per tile, never a cross-tile bundle. Rules stay group-compatible: shell only, no mid-chain localrules, no pipe outputs. +Group resource composition (snakemake 9.23, ``GroupResources.basic_layered`` in +snakemake/resources.py): jobs are laid out per toposort level; within a level +non-additive resources (mem_mb, cpus) SUM — split into layers when a global +constraint is exceeded, the group's width being the widest layer — while the +additive resource ``runtime`` is maxed within a layer and SUMMED across layers. +This chain is strictly linear, one job per level, so the group asks for +max(mem_mb) = 8000*attempt, max(threads) = 4 and sum(runtime) = 150 min. +Attempt scaling survives grouping: ``GroupJob.attempt``'s setter clears the +cached group resources and re-sets ``attempt`` on every member (jobs.py), and +``GroupJob.restart_times`` is the max over members — so tile_get_images' +``retries: 2`` still governs. A retry re-runs the whole group, which is safe +because every rule ``rm -rf``s its own run dir at start. + Star catalogues for masking are NOT a prepare-phase concern and not pre-run input: the compute DAG fetches the campaign footprint's stars once (``star_catalogue``) and cuts them per exposure (``exp_star_cat``), both in @@ -22,6 +41,7 @@ because it has no mask rule yet — see tile.smk. # config_tile_Git.ini simply has no NUMBER_LIST, so this is now a property of the # config, not of an injection step. rule tile_get_images: + group: "tile_prep" output: manifest = f"{TILE_DIR}/manifests/tile_get_images.json" log: @@ -38,6 +58,7 @@ rule tile_get_images: sp_shell("tile_get_images", "config_tile_Git.ini") rule tile_uncompress: + group: "tile_prep" input: rules.tile_get_images.output.manifest output: @@ -55,6 +76,7 @@ rule tile_uncompress: sp_shell("tile_uncompress", "config_tile_Uz.ini") rule tile_find_exposures: + group: "tile_prep" input: rules.tile_uncompress.output.manifest output: diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 5f8dfacd0..32c5165f6 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -12,7 +12,20 @@ shard level is not cosmetic: ``exp_utils.get_exp_output_files`` hardwires ``///output/run_sp_*`` into its glob, so a flat forest makes every tile gather stage fail "No split_exp_runner output found". -All rules are group-compatible (shell only, no mid-chain localrules). +All rules are group-compatible (shell only, no mid-chain localrules), but only +the tail pair is actually grouped: ``group: "tile_finish"`` on tile_merge_cats +(median 0:15) and tile_make_cat (1:34) — one sbatch per tile for two short jobs +of identical shape (16 GB, 8 threads, 120 min each; the group asks max mem_mb = +16000*attempt, max threads = 8, sum runtime = 240, per the composition rules in +prepare.smk's docstring). Distinct tiles share no edge, so it is one group job +per tile. + +Nothing else on the tile side may join a group. merge_cats' upstream is +tile_ngmix (hours) and make_cat is the terminus, so this pair is the whole +connected short region. tile_merge_headers (0:38) is short but STRANDED: its +upstream is the tile_exp_forest localrule and its only downstream is tile_detect +(16 GB), so any group containing it either fuses a localrule — impossible — or +pulls a heavy rule in. It stays one job per tile, deliberately. Note there is no `tile_mask` rule: the committed config chain is the "sx_nomask" tile_detect variant (config_tile_Sx.ini reads Git + Uz + Mh, no mask @@ -261,6 +274,7 @@ def ngmix_chunkdirs(wc): # The gather: merge the N chunk catalogues. N_SPLIT_MAX comes from the workflow's # own chunk count via $NGMIX_N_CHUNKS (env-expanded by the module). rule tile_merge_cats: + group: "tile_finish" input: manifests = ngmix_manifests, chunkdirs = ngmix_chunkdirs, @@ -285,6 +299,7 @@ rule tile_merge_cats: # No protected(): the full default rerun-triggers govern, and protected() only # ever forced people through a `--forcerun` detour. rule tile_make_cat: + group: "tile_finish" input: ms = rules.tile_merge_cats.output.manifest, store = rules.tile_vignets.output.store, From aee48dc5364ded9518f8918ba41b381eda31a6a4 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 29 Aug 2026 16:42:58 -0400 Subject: [PATCH 36/41] Derive merge_sep_cats chunk paths from the run-directory name MergeSep.process derived chunk k's input with re.sub("1", str(k), path, 1) -- the first literal "1" ANYWHERE in the path. Under the sharded store (.../tiles/21/210.282/...) that lands in a parent directory, so every chunk after the first was looked for somewhere that does not exist, and the workflow config had to carry a relative INPUT_DIR to keep the first "1" inside "Ng1u". chunk_path() now substitutes inside the last "run_*" path component only, which is where the chunk number actually lives, and raises if there is none. The bash pipeline's relative paths are unchanged; the workflow config becomes a fixed $SP_RUN path like every other config. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011Vg5ZoFShkKRdkk9bpbpXy --- .../merge_sep_cats_package/merge_sep_cats.py | 46 +++++++++++++++++-- tests/unit/test_merge_sep_cats_paths.py | 40 ++++++++++++++++ .../config/cfis/config_merge_sep_cats.ini | 9 +--- 3 files changed, 84 insertions(+), 11 deletions(-) create mode 100644 tests/unit/test_merge_sep_cats_paths.py diff --git a/src/shapepipe/modules/merge_sep_cats_package/merge_sep_cats.py b/src/shapepipe/modules/merge_sep_cats_package/merge_sep_cats.py index 5ecdb6fb0..b0be88944 100644 --- a/src/shapepipe/modules/merge_sep_cats_package/merge_sep_cats.py +++ b/src/shapepipe/modules/merge_sep_cats_package/merge_sep_cats.py @@ -7,7 +7,6 @@ """ import os -import re import warnings import numpy as np @@ -16,6 +15,48 @@ from shapepipe.pipeline import file_io +def chunk_path(input_file, n): + """Chunk Path. + + Derive chunk ``n``'s input path from chunk 1's. + + The separate catalogues live in ShapePipe run directories whose names differ + only in the chunk number (``run_sp_tile_ngmix_Ng1u`` -> + ``run_sp_tile_ngmix_Ng2u``). The substitution is confined to that + run-directory component: replacing the first "1" found anywhere in the path + breaks for absolute paths whose parent directories carry digits, e.g. a + sharded store ``.../tiles/21/210.282/output/run_..._Ng1u/...``. + + Parameters + ---------- + input_file : str + Path to chunk 1's catalogue + n : int + Chunk number + + Returns + ------- + str + Path to chunk ``n``'s catalogue + + Raises + ------ + ValueError + If no run-directory component of the path carries a chunk number + + """ + parts = input_file.split(os.sep) + for idx in reversed(range(len(parts))): + if parts[idx].startswith("run_") and "1" in parts[idx]: + parts[idx] = parts[idx].replace("1", str(n), 1) + return os.sep.join(parts) + + raise ValueError( + f"Cannot derive chunk {n}'s path from '{input_file}': no 'run_*' " + + "directory component contains a chunk number '1'" + ) + + class MergeSep(object): """Merge Sep. @@ -79,8 +120,7 @@ def process(self): input_path_n = [] input_path_n.append(input_file) for n in range(2, self._n_split_max + 1): - res = re.sub("1", str(n), input_file, 1) - input_path_n.append(res) + input_path_n.append(chunk_path(input_file, n)) # Open first catalogue, read number of extensions and columns cat0 = file_io.FITSCatalogue(input_file, SEx_catalogue=True) diff --git a/tests/unit/test_merge_sep_cats_paths.py b/tests/unit/test_merge_sep_cats_paths.py new file mode 100644 index 000000000..754bc76dd --- /dev/null +++ b/tests/unit/test_merge_sep_cats_paths.py @@ -0,0 +1,40 @@ +"""Chunk-path derivation in the merge_sep_cats module.""" + +import pytest + +from shapepipe.modules.merge_sep_cats_package.merge_sep_cats import chunk_path + + +REL = "./output/run_sp_tile_ngmix_Ng1u/ngmix_runner/output/ngmix-210-282.fits" +ABS = ( + "/scratch/run/tiles/21/210.282/output/run_sp_tile_ngmix_Ng1u" + "/ngmix_runner/output/ngmix-210-282.fits" +) + + +def test_relative_path_unchanged_behaviour(): + """The bash pipeline's relative INPUT_DIR still resolves as before.""" + assert chunk_path(REL, 3) == ( + "./output/run_sp_tile_ngmix_Ng3u/ngmix_runner/output/ngmix-210-282.fits" + ) + + +def test_sharded_absolute_path(): + """Digits in the sharded parent dirs and in the file number are untouched.""" + assert chunk_path(ABS, 2) == ( + "/scratch/run/tiles/21/210.282/output/run_sp_tile_ngmix_Ng2u" + "/ngmix_runner/output/ngmix-210-282.fits" + ) + + +def test_double_digit_chunk(): + assert "Ng12u" in chunk_path(ABS, 12) + + +def test_chunk_one_is_identity(): + assert chunk_path(ABS, 1) == ABS + + +def test_no_run_directory_raises(): + with pytest.raises(ValueError, match="run_"): + chunk_path("/scratch/run/tiles/21/210.282/ngmix-210-282.fits", 2) diff --git a/workflow/config/cfis/config_merge_sep_cats.ini b/workflow/config/cfis/config_merge_sep_cats.ini index 750fe1124..9dda5cf0f 100644 --- a/workflow/config/cfis/config_merge_sep_cats.ini +++ b/workflow/config/cfis/config_merge_sep_cats.ini @@ -42,14 +42,7 @@ NUMBER_LIST = $SP_UNIT_NUM # derives the other chunks' paths itself from N_SPLIT_MAX and this pattern # (chunk dirs are run_sp_tile_ngmix_Ngu, k=1..N_SPLIT_MAX), all under the # same fixed output dir. -# -# RELATIVE ON PURPOSE (do not "fix" this to $SP_RUN/...): merge_sep_cats derives -# chunk k's path with re.sub("1", str(k), input_file, 1) -- it replaces the FIRST -# "1" in the path. An absolute path under the sharded store -# (.../tiles/21/210.282/...) has digits before "Ng1u", so chunk 2 would be looked -# for in a nonexistent directory. Every rule runs shapepipe_run with cwd=$SP_RUN, -# so "./output/..." resolves and "Ng1u" carries the first "1". -INPUT_DIR = ./output/run_sp_tile_ngmix_Ng1u/ngmix_runner/output +INPUT_DIR = $SP_RUN/output/run_sp_tile_ngmix_Ng1u/ngmix_runner/output # Output directory OUTPUT_DIR = $SP_RUN/output From d5c208e3a26cbac88e13824757ee68a48eb6af37 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 29 Aug 2026 16:43:19 -0400 Subject: [PATCH 37/41] Fail loudly on an unset SP_PHASE and an unresolvable container Three silent degradations, all at parse time: * SP_PHASE unset. A bare `snakemake` parsed as a passthrough, built `rule all` from an index it never built, and exited 0 on an empty DAG. The Snakefile now requires SP_PHASE, and bin/sp's sm() supplies "passthrough" for the escape hatch (`sp --unlock`, `sp --dag`, `sp exp_psf ...`) so the three phases are named rather than inferred from absence. Submitted jobs re-parse under `sbatch --export=ALL`, so they carry the launching phase. * A container: key lost from config.yaml with an empty cache resolved to "", which passes dry-run and only fails on a node. Now a WorkflowError. * SP_CONTAINER pointing at a nonexistent path was silently ignored and the job ran the shared image instead of the one the user named. Now a ContainerError, reported (not raised) by `sp container status`, which is the verb you run when something is wrong. Also: `sp container exec` reads the nibi profile's apptainer-args instead of a hand-rolled --cleanenv, so a one-off gets the same PYTHONPATH pin, --home and binds that jobs do; and status's "behind" hint names the tag `pull` would fetch, which is develop-runtime regardless of the branch you are on. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011Vg5ZoFShkKRdkk9bpbpXy --- workflow/Snakefile | 42 +++++++++++++++---- workflow/bin/sp | 11 ++++- workflow/scripts/container.py | 78 ++++++++++++++++++++++++++++++++--- 3 files changed, 118 insertions(+), 13 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 2d9f712de..c89ff6faf 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -52,7 +52,21 @@ configfile: str(Path(workflow.snakefile).parent / "config.yaml") sys.path.insert(0, str(Path(workflow.snakefile).parent / "scripts")) import container as _container # noqa: E402 -container: _container.resolve_image()[0] +# Loud at PARSE time, both ways: a broken SP_CONTAINER override, or nothing to +# resolve at all (empty cache AND no `container:` key). The empty string the +# resolver returns for "none" is a valid-looking container directive that passes +# dry-run and only fails once jobs reach a node. +try: + _image, _kind = _container.resolve_image() +except _container.ContainerError as _exc: + raise WorkflowError(str(_exc)) +if _kind == "none": + raise WorkflowError( + f"No container image resolved: no sandbox, no cached SIF, and no " + f"'{_container.CONFIG_KEY}:' key in {_container.CONFIG_FILE}. " + f"Run `sp container pull`, or restore the key.") + +container: _image # --- paths ----------------------------------------------------------------- RUN_DIR = Path(config["run_dir"]) @@ -76,12 +90,26 @@ sys.path.insert(0, str(SCRIPTS)) import build_index # noqa: E402 from completeness import STAGE_DIR # noqa: E402 -# SP_PHASE is set by bin/sp on the two invocations of `sp run` and NOWHERE else. -# It gates the two parse-time side effects — the index build and the report -# hooks — so that a passthrough invocation (`sp --unlock`, `sp --dag`, `sp -# exp_psf ...`) never mutates durable state or dies on a threshold it was not -# asked about. Unset means "read the index, build nothing". +# SP_PHASE is set by bin/sp and NOWHERE else: `prepare`/`compute` on the two +# invocations of `sp run`, `passthrough` on the escape hatch (`sp --unlock`, `sp +# --dag`, `sp exp_psf ...`). It gates the two parse-time side effects — the index +# build and the report hooks — so that a passthrough parse never mutates durable +# state or dies on a threshold it was not asked about. +# +# Requiring it to be SET is the point of the check below: a bare `snakemake` +# would otherwise parse as a passthrough, find `rule all` gated on an index it +# never built, and exit 0 on an empty DAG. Jobs re-parse this file under the +# slurm executor and inherit the head process's environment, so a submitted job +# always carries the launching phase. PHASE = os.environ.get("SP_PHASE", "") +if not PHASE: + raise WorkflowError( + "SP_PHASE is not set — run the workflow through workflow/bin/sp " + "(`sp run`), which exports it. A bare snakemake invocation would " + "silently build an empty DAG.") +if PHASE not in ("prepare", "compute", "passthrough"): + raise WorkflowError( + f"SP_PHASE={PHASE!r} is not one of prepare, compute, passthrough.") with open(config["tile_list"]) as f: TILES = [ln.strip() for ln in f if ln.strip()] @@ -93,7 +121,7 @@ with open(config["tile_list"]) as f: # chunk number is not ours alone: it names the run dir # (`run_sp_tile_ngmix_Ngu`, from the template's RUN_NAME), and # merge_sep_cats derives chunks 2..N from chunk 1's path by substituting the -# first "1" — which only works for bare integers. +# chunk number in that run-directory name — which only works for bare integers. scattergather: ngmix=int(config.get("ngmix_chunks", 8)) diff --git a/workflow/bin/sp b/workflow/bin/sp index 72e80b783..d5f7bfb5a 100755 --- a/workflow/bin/sp +++ b/workflow/bin/sp @@ -53,7 +53,16 @@ export SP_MISSING_THRESHOLD="${SP_MISSING_THRESHOLD:-0.0}" # --snakefile pins the workflow to this checkout: sp must work from any cwd # (an sbatch head job starts in the submission directory, not the repo). -sm() { snakemake --snakefile "$HERE/Snakefile" --profile "$PROFILE" --directory "$STATE_DIR" "$@"; } +# +# SP_PHASE is REQUIRED by the Snakefile (a bare `snakemake` would build an empty +# DAG and exit 0). `sp run` sets prepare/compute on its two invocations; every +# other verb and the escape hatch fall through to `passthrough`, which parses the +# index without building it and schedules no side effects. +sm() { + SP_PHASE="${SP_PHASE:-passthrough}" \ + snakemake --snakefile "$HERE/Snakefile" --profile "$PROFILE" \ + --directory "$STATE_DIR" "$@" +} cmd="${1:-}" case "$cmd" in diff --git a/workflow/scripts/container.py b/workflow/scripts/container.py index 0479b0f46..bbd7e0bb9 100644 --- a/workflow/scripts/container.py +++ b/workflow/scripts/container.py @@ -37,11 +37,21 @@ import argparse import os import re +import shlex import shutil import subprocess import sys from pathlib import Path + +class ContainerError(Exception): + """A misconfiguration the caller must fix (bad override, no image at all). + + Raised rather than ``sys.exit``ed so the Snakefile, which imports this + module at parse time, can turn it into a WorkflowError instead of a + SystemExit. ``main`` below turns it back into a one-line CLI error. + """ + # The published image. CI pushes one tag per branch, sanitized; `-runtime` is # the slim variant the workflow runs. CONTAINER_URI = "docker://ghcr.io/cosmostat/shapepipe:develop-runtime" @@ -52,6 +62,12 @@ CONFIG_FILE = Path(__file__).resolve().parents[1] / "config.yaml" CONFIG_KEY = "container" +# The profile whose `apptainer-args:` every workflow job runs under. `exec` reads +# it at runtime rather than restating it, so a one-off `sp container exec` and a +# job see the same environment (the PYTHONPATH pin above all: a divergence there +# means the one-off imports a different src/ than the workflow does). +PROFILE_FILE = Path(__file__).resolve().parents[2] / "profiles" / "nibi" / "config.yaml" + # ~/.cache/shapepipe by default; SP_CACHE_DIR moves the whole cache (e.g. onto # a filesystem with room), XDG_CACHE_HOME moves it with everything else. CACHE_DIR = Path( @@ -85,6 +101,20 @@ def configured_default(): return match.group(1) if match else None +def profile_apptainer_args(): + """Return the profile's ``apptainer-args`` as a token list, or ``[]``. + + Minimal scalar read again (stdlib-only); a missing or unreadable profile + yields ``[]``, which callers replace with their own defaults. + """ + try: + text = PROFILE_FILE.read_text() + except OSError: + return [] + match = re.search(r'^apptainer-args:[ \t]*"(.*)"[ \t]*$', text, re.MULTILINE) + return shlex.split(match.group(1)) if match else [] + + def local_sif(): """Return this user's cached image path (may not exist yet).""" override = os.environ.get("SP_CONTAINER") @@ -110,6 +140,14 @@ def resolve_image(): sif = local_sif() if sif.exists(): return str(sif), "sif" + # An override that resolves to nothing is a typo, never an intention: falling + # through to the shared image would run the job against something other than + # what the user named. + if os.environ.get("SP_CONTAINER"): + raise ContainerError( + f"SP_CONTAINER={os.environ['SP_CONTAINER']} does not exist " + f"(resolved to {sif}). Unset it, or point it at an image that does." + ) default = configured_default() if default: return default, "configured" @@ -281,7 +319,13 @@ def cmd_status(args): """Report which image layer is live, its revision, and how current it is.""" sif = local_sif() sandbox = local_sandbox() - active, kind = resolve_image() + # status is the verb you run WHEN something is wrong, so a broken override is + # reported here rather than raised. + try: + active, kind = resolve_image() + except ContainerError as exc: + print(f"active: NONE -- {exc}") + return 1 if sif.exists(): print(f"SIF: {sif} ({sif.stat().st_size / 1e9:.1f} GB)") @@ -323,7 +367,11 @@ def cmd_status(args): verdict = compare_revision(revision) explain = { "in-sync": "matches this checkout's HEAD", - "behind": "older than this checkout's HEAD -- pull to refresh", + "behind": ( + "older than this checkout's HEAD -- `sp container pull` fetches " + f"{CONTAINER_URI}, which is not built from this branch unless you " + "pass --tag" + ), "ahead": "newer than this checkout's HEAD", "diverged": "on a different branch from this checkout", "unknown": "cannot compare (no label, or a commit this clone lacks)", @@ -338,7 +386,24 @@ def cmd_exec(args): command = [a for a in args.command if a != "--"] if not command: sys.exit("nothing to run; pass a command after `exec`") - binds = args.bind or os.environ.get("SP_APPTAINER_BINDS", DEFAULT_BINDS) + + # Same environment the workflow's jobs get: the profile's apptainer-args + # verbatim (--cleanenv, the PYTHONPATH pin, --home, its binds). Explicit + # binds still win, and a profile that cannot be read falls back to the old + # standalone defaults. + env_args = profile_apptainer_args() + explicit_binds = args.bind or os.environ.get("SP_APPTAINER_BINDS") + if not env_args: + print( + f"warning: could not read apptainer-args from {PROFILE_FILE}; " + "falling back to --cleanenv and the default binds, which may not " + "match what jobs run under", + file=sys.stderr, + ) + env_args = ["--cleanenv"] + explicit_binds = explicit_binds or DEFAULT_BINDS + if explicit_binds: + env_args += ["--bind", explicit_binds] if args.writable: # A SIF is a read-only filesystem, so `--writable` against one fails @@ -356,7 +421,7 @@ def cmd_exec(args): sys.exit("no image resolved; run: sp container pull") extra = [] - cmd = ["apptainer", "exec", *extra, "--cleanenv", "--bind", binds, image, *command] + cmd = ["apptainer", "exec", *extra, *env_args, image, *command] return subprocess.run(cmd).returncode @@ -425,7 +490,10 @@ def build_parser(): def main(argv=None): args = build_parser().parse_args(argv) - return args.func(args) + try: + return args.func(args) + except ContainerError as exc: + sys.exit(str(exc)) if __name__ == "__main__": From 15c4e6c66b612bcf8ef8e97b8eb8dde7cdafef56 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 29 Aug 2026 16:43:19 -0400 Subject: [PATCH 38/41] Group tile_exp_forest with tile_merge_headers The docstring called tile_exp_forest a localrule and closed the question on that premise; it is not one (Snakefile's localrules list does not name it), so the two short jobs at the head of the tile chain can fuse: one sbatch per tile asking max mem_mb = 8000*attempt, max threads = 4, sum runtime = 140. tile_detect also consumes the forest, but a consumer outside the group is an ordinary edge on the group job and does not pull its 16 GB in. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011Vg5ZoFShkKRdkk9bpbpXy --- workflow/rules/tile.smk | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 32c5165f6..8fc230c29 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -12,20 +12,22 @@ shard level is not cosmetic: ``exp_utils.get_exp_output_files`` hardwires ``///output/run_sp_*`` into its glob, so a flat forest makes every tile gather stage fail "No split_exp_runner output found". -All rules are group-compatible (shell only, no mid-chain localrules), but only -the tail pair is actually grouped: ``group: "tile_finish"`` on tile_merge_cats -(median 0:15) and tile_make_cat (1:34) — one sbatch per tile for two short jobs -of identical shape (16 GB, 8 threads, 120 min each; the group asks max mem_mb = -16000*attempt, max threads = 8, sum runtime = 240, per the composition rules in -prepare.smk's docstring). Distinct tiles share no edge, so it is one group job -per tile. +All rules are group-compatible (shell only, no mid-chain localrules), and the two +short regions are grouped, per the composition rules in prepare.smk's docstring. +Distinct tiles share no edge, so each is one group job per tile: -Nothing else on the tile side may join a group. merge_cats' upstream is -tile_ngmix (hours) and make_cat is the terminus, so this pair is the whole -connected short region. tile_merge_headers (0:38) is short but STRANDED: its -upstream is the tile_exp_forest localrule and its only downstream is tile_detect -(16 GB), so any group containing it either fuses a localrule — impossible — or -pulls a heavy rule in. It stays one job per tile, deliberately. +* ``group: "tile_gather"`` — tile_exp_forest (2 GB, 20 min) and + tile_merge_headers (median 0:38; 8 GB, 4 threads, 120 min). The group asks max + mem_mb = 8000*attempt, max threads = 4, sum runtime = 140. tile_detect also + consumes the forest, but a consumer OUTSIDE the group is just an ordinary DAG + edge on the group job — it does not pull tile_detect (16 GB) in. +* ``group: "tile_finish"`` — tile_merge_cats (median 0:15) and tile_make_cat + (1:34), two short jobs of identical shape (16 GB, 8 threads, 120 min each; + max mem_mb = 16000*attempt, max threads = 8, sum runtime = 240). + +Nothing else joins either. The heavy middle (tile_detect, tile_vignets, +tile_ngmix) is where the wall time is, and fusing a short rule onto one of those +would reserve its footprint for the short job too. Note there is no `tile_mask` rule: the committed config chain is the "sx_nomask" tile_detect variant (config_tile_Sx.ini reads Git + Uz + Mh, no mask @@ -119,6 +121,7 @@ def tile_exp_all(wc): return tile_exp_split(wc) + tile_exp_mask(wc) + tile_ex # Its output stays a directory() (it has no ShapePipe run dir and no manifest — # it is not a shapepipe_run at all). rule tile_exp_forest: + group: "tile_gather" input: tile_exp_all output: @@ -142,6 +145,7 @@ rule tile_exp_forest: # (log_exp_headers--.sqlite, which Sx / PiViVi / ngmix consume). # Reads headers-*.npy through the forest -> the split manifests are the edge. rule tile_merge_headers: + group: "tile_gather" input: forest = rules.tile_exp_forest.output.forest, split = tile_exp_split, From 109739db43c2640656603705776395421142cea4 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 29 Aug 2026 16:35:49 -0400 Subject: [PATCH 39/41] fix(psf): deterministic star train/validation split; align tile STAR_THRESH with the 80% convention The 80/20 star split drew from unseeded np.random, so the PSF star sample differed between identical runs; it is now a permutation seeded from the unit's file number (SEED_FROM_POSITION philosophy). The tile PSF-interp configs kept the pre-split STAR_THRESH=20 that fdc86553 raised to 22 in the validation config; align the science path on 22. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Jrs8TeTCccRMC45QVGKPeY (cherry picked from commit 1da350ced18b425bcfc635f56d080697e0a94e71) --- example/cfis/config_tile_PiViVi_canfar_sx.ini | 2 +- example/cfis/config_tile_PiViVi_canfar_uc.ini | 2 +- .../modules/setools_package/setools.py | 19 ++++++++++++------- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/example/cfis/config_tile_PiViVi_canfar_sx.ini b/example/cfis/config_tile_PiViVi_canfar_sx.ini index ca7efd9ec..991153143 100644 --- a/example/cfis/config_tile_PiViVi_canfar_sx.ini +++ b/example/cfis/config_tile_PiViVi_canfar_sx.ini @@ -79,7 +79,7 @@ POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD GET_SHAPES = True # Number of stars threshold -STAR_THRESH = 20 +STAR_THRESH = 22 # chi^2 threshold CHI2_THRESH = 2 diff --git a/example/cfis/config_tile_PiViVi_canfar_uc.ini b/example/cfis/config_tile_PiViVi_canfar_uc.ini index d59af1c4a..b33361b6d 100644 --- a/example/cfis/config_tile_PiViVi_canfar_uc.ini +++ b/example/cfis/config_tile_PiViVi_canfar_uc.ini @@ -77,7 +77,7 @@ POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD GET_SHAPES = True # Number of stars threshold -STAR_THRESH = 20 +STAR_THRESH = 22 # chi^2 threshold CHI2_THRESH = 2 diff --git a/src/shapepipe/modules/setools_package/setools.py b/src/shapepipe/modules/setools_package/setools.py index c430b52b1..33d8efa8b 100644 --- a/src/shapepipe/modules/setools_package/setools.py +++ b/src/shapepipe/modules/setools_package/setools.py @@ -658,13 +658,18 @@ def _make_rand_split(self): cat_size = len(np.where(mask)[0]) n_keep = int(np.ceil(cat_size * ratio)) - mask_ratio = [] - mask_left = list(range(0, cat_size)) - while len(mask_ratio) != n_keep: - idx = np.random.randint(0, len(mask_left)) - mask_ratio.append(mask_left.pop(idx)) - mask_ratio = np.array(mask_ratio) - mask_left = np.array(mask_left) + # Deterministic split, seeded from the unit's file number: the + # train/validation assignment is a pure function of the input + # catalogue, so the PSF star sample (and everything downstream + # of the PSF model) is reproducible run-to-run. An unseeded + # np.random here made the shear catalogue non-reproducible + # upstream of ngmix's own position seeding. + seed = int( + re.sub(r"\D", "", self._file_number_string) or 0 + ) % (2 ** 32) + perm = np.random.RandomState(seed).permutation(cat_size) + mask_ratio = perm[:n_keep] + mask_left = np.sort(perm[n_keep:]) self.rand_split[key]["mask"] = mask self.rand_split[key][f"ratio_{int(ratio * 100)}"] = mask_ratio self.rand_split[key][f"ratio_{100 - int(ratio * 100)}"] = mask_left From 90782098f17b138823d3635df9879f477f06db69 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 29 Aug 2026 18:53:37 -0400 Subject: [PATCH 40/41] fix(configs): the workflow's own PiViVi config takes STAR_THRESH 22 too #873 aligned the tile PSF-interp threshold with the 80/20 split convention in example/cfis, the only place those configs exist on develop. This branch carries a committed fork of that chain under workflow/config/cfis (#848 D2), so the science path the Snakemake run actually reads kept the stale 20 after the cherry-pick. Mirror it. The general hazard: every science-config change landing against example/cfis must be mirrored here until the two chains are unified. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01VUNNyG8tw6Mdbvo4SVNrhp --- workflow/config/cfis/config_tile_PiViVi.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/config/cfis/config_tile_PiViVi.ini b/workflow/config/cfis/config_tile_PiViVi.ini index 246ee18f9..22294213f 100644 --- a/workflow/config/cfis/config_tile_PiViVi.ini +++ b/workflow/config/cfis/config_tile_PiViVi.ini @@ -83,7 +83,7 @@ POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD GET_SHAPES = True # Number of stars threshold -STAR_THRESH = 20 +STAR_THRESH = 22 # chi^2 threshold CHI2_THRESH = 2 From 43987f6a8828835c1f2a0729cd0957f7d9b1df32 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 29 Aug 2026 19:01:40 -0400 Subject: [PATCH 41/41] feat(orchestration): the persistent root D5 always described MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit D5 has always specified two roots — a persistent one for the durable, low-volume products and a scratch one for bulk intermediates — and the implementation has had one, because def-mjhudson /project was hard-full at 27/27 TiB when the campaign started. Space has since been reclaimed, so the design can land. `products_dir:` is the persistent root. It defaults to `run_dir`, so a fixture or a smoke test still needs exactly one path; when set, it takes the tree that mirrors the scratch one shard for shard and holds the final catalogues, the index, missing.json and the report. The star-catalogue sky store moves there too: it is the one product in this workflow bought with something local compute cannot re-buy — ~1.5k catalogue-server queries at DR6 scale. The final catalogue is the case that matters most, and not only because it is the science product. `tile_finished()` keys on its path to cut a finished tile's exposure edges, so on scratch a purge would not merely lose catalogues: it would make every finished tile re-declare inputs against exposure stores reclamation had already deleted, which is the rerun avalanche the cut exists to prevent, arriving by a different door. Snakemake's own state stays on scratch — the one durable-looking thing that does. D5's parenthetical listed it under the persistent root; that was wrong on its own terms. It is one small, hot metadata file per output, rewritten every job, >170k files of churn at DR6 against a backed-up filesystem with a 1M-inode group quota, and it is reconstructible: losing it costs a re-parse, not a rerun. The comment in bin/sp now says so instead of blaming the quota that first exposed it. Also refreshed: the profile's claim that this branch never touches src/, which four commits ago stopped being true. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01VUNNyG8tw6Mdbvo4SVNrhp --- profiles/nibi/config.yaml | 11 ++++-- workflow/Snakefile | 24 +++++++++++- workflow/bin/sp | 12 ++++-- workflow/config.yaml | 79 +++++++++++++++++++++++---------------- workflow/rules/tile.smk | 2 +- 5 files changed, 87 insertions(+), 41 deletions(-) diff --git a/profiles/nibi/config.yaml b/profiles/nibi/config.yaml index 65746786f..7b75b458f 100644 --- a/profiles/nibi/config.yaml +++ b/profiles/nibi/config.yaml @@ -37,12 +37,15 @@ software-deployment-method: [apptainer] # OMP_NUM_THREADS=1 caps OpenBLAS fork-explosion (verified: pool 32->1) # MALLOC_ARENA_MAX=2 bounded allocator (im_sims/nibi lesson) # --home /home/cdaley wins over the SDM's --home ; restores cadcproxy.pem for vos/vcp -# PYTHONPATH SETTLED CALL 3 — P0 pins THIS branch's src/ (identical to -# develop@97e16d50: orchestration commits never touch src/). +# PYTHONPATH SETTLED CALL 3 — pins THIS branch's src/, which is +# develop@97e16d50 plus the four commits that genuinely need +# module code: ngmix chunk fields + position-seeded RNG, +# merge_sep_cats chunk paths, the vizier star-cat helpers, +# and the cherry-picked #873 (seeded setools split). # NOT shapepipe-prod (drifted to a PR branch mid-run, and the # live p3-batch1 job reads it) and not the sif default -# (frozen pre-#843). Production later rebuilds the sif at the -# validated commit and DROPS this --env line. +# (frozen pre-#843, and pre-#873). Production later rebuilds +# the sif at the validated commit and DROPS this --env line. apptainer-args: "--cleanenv --env OMP_NUM_THREADS=1 --env MALLOC_ARENA_MAX=2 --env PYTHONPATH=/project/def-mjhudson/cdaley/shapepipe-snakemake/src --home /home/cdaley --bind /project --bind /scratch" latency-wait: 60 # NFS: wait for outputs to appear after a job diff --git a/workflow/Snakefile b/workflow/Snakefile index c89ff6faf..0f2541334 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -69,7 +69,16 @@ if _kind == "none": container: _image # --- paths ----------------------------------------------------------------- +# TWO ROOTS (D5). RUN_DIR is the scratch root: bulk intermediates, sized so a +# batch finishes inside the purge window. PRODUCTS_DIR is the persistent root +# for the durable, low-volume products — the final catalogues, the index, the +# report. Snakemake's own state is the one durable-looking thing that stays on +# scratch (bin/sp explains why: tens of thousands of small, hot metadata writes +# against a backed-up, file-count-limited filesystem). RUN_DIR = Path(config["run_dir"]) +# Defaults to RUN_DIR so a scratch-only run (a fixture, a smoke test) needs no +# second path: one root, exactly the pre-D5 layout. +PRODUCTS_DIR = Path(config.get("products_dir") or RUN_DIR) # Run-independent root for the mask star catalogues: the HEALPix chunk store the # star_catalogue rule fills, and the per-exposure cuts exp_star_cat makes from it # (config.yaml explains the placement). @@ -191,6 +200,9 @@ wildcard_constraints: # is built by these helpers, so a mismatched (shard, id) pair is never requested. TILE_DIR = str(RUN_DIR / "tiles" / "{shard}" / "{tile}") EXP_DIR = str(RUN_DIR / "exp" / "{shard}" / "{exp}") +# The persistent root mirrors the scratch one, shard for shard, so the two trees +# read as the same campaign seen from two filesystems. +PROD_TILE_DIR = str(PRODUCTS_DIR / "tiles" / "{shard}" / "{tile}") def tile_dir(tile): return f"{RUN_DIR}/tiles/{tile[:2]}/{tile}" @@ -208,7 +220,17 @@ def forest_dir(tile): return f"{tile_dir(tile)}/exp_forest" def final_cat(tile): - return f"{tile_dir(tile)}/final_cat-{tile}.fits" + """The campaign's science product, and its tile-finished marker. + + It lives on the PERSISTENT root, not the scratch one, and both halves of + that sentence are load-bearing. As a product: it is what the campaign is + for, and a 60-day purge must not eat it. As a marker: tile_finished() keys + on this path to cut a finished tile's exposure edges (D5), so if a purge + could remove it, finished tiles would re-declare inputs against exposure + stores that reclamation deleted — the rerun avalanche the cut exists to + prevent, arriving by way of the purge instead. + """ + return f"{PRODUCTS_DIR}/tiles/{tile[:2]}/{tile}/final_cat-{tile}.fits" def unit_num(unit): """$SP_UNIT_NUM: ShapePipe's image-number convention, dot -> dash, leading diff --git a/workflow/bin/sp b/workflow/bin/sp index d5f7bfb5a..921b95d8d 100755 --- a/workflow/bin/sp +++ b/workflow/bin/sp @@ -42,9 +42,15 @@ cfg() { sed -n "s/^$1:[[:space:]]*//p" "$CONFIG" | head -1; } RUN_DIR="$(cfg run_dir)"; INDEX_DB="$(cfg index_db)" # Snakemake state (.snakemake: metadata, locks, incomplete markers) lives NEXT TO -# THE RUN on /scratch, never on /project (a hard 27/27 TiB group quota killed a -# metadata write mid-run, live). --directory only moves state: all data paths are -# absolute, and the Snakefile resolves its own configfile. +# THE RUN on /scratch, never on the persistent root — the one exception to D5's +# "durable, low-volume products go on /project". It is neither: one small, hot +# metadata file per output, rewritten on every job, which at DR6 is >170k files +# of churn against a backed-up filesystem with a 1M-inode group quota. It is also +# reconstructible — losing it costs a re-parse, not a re-run. (The lesson arrived +# the hard way: a hard 27/27 TiB group quota killed a metadata write mid-run, +# live. The quota has since eased; the placement is right on its own merits.) +# --directory only moves state: all data paths are absolute, and the Snakefile +# resolves its own configfile. STATE_DIR="${SP_STATE_DIR:-${RUN_DIR}-state}"; mkdir -p "$STATE_DIR" # SP_MISSING_THRESHOLD gates the compute parse's index build: the fraction of diff --git a/workflow/config.yaml b/workflow/config.yaml index c81d4e45d..e67b6867e 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -6,21 +6,43 @@ # The tile list that scopes this run (one "IDra.IDdec" per line). # The campaign grows by appending to this file — parse-time config, so -# completed work is never invalidated. Current contents: the 186/187 quad -# (18 exposures, complete and validated against the bash baseline), plus the -# first 30 tiles of the p3-batch1 list (100 unique exposures — chosen so the -# worst-case un-cleaned exposure high-water ~660G fits the scratch quota with -# margin; the remaining 20 batch1 tiles are the next append). (The 210/211 -# quad used earlier is unusable: its tile images are symlinks into anaennis' -# moved processed_tiles tree — ~8.5k of the 10.3k staged tiles are broken.) -tile_list: /scratch/cdaley/shapepipe-output/smk-g3/tiles4.txt +# completed work is never invalidated. Current contents: the 34-tile smk-g4 +# benchmark set, identical to the tile list smk-g3 finished (the 186/187 quad +# plus the first 30 p3-batch1 tiles), so the two campaigns' catalogues are +# comparable object for object. It lives on the persistent root because it +# defines the campaign and must outlive the scratch purge. +# (The 210/211 quad used earlier is unusable: its tile images are symlinks into +# anaennis' moved processed_tiles tree — ~8.5k of the 10.3k staged tiles are +# broken.) +tile_list: /project/def-mjhudson/cdaley/sp-products/smk-g4/tiles34.txt # The container every job runs inside (apptainer software-deployment in the profile). container: /project/def-mjhudson/cdaley/containers/shapepipe-develop-runtime.sif -# Where products land ($SP_RUN root). The sharded per-unit stores live under here: +# THE TWO ROOTS (D5). +# +# run_dir is the SCRATCH root and the $SP_RUN every config interpolates: bulk +# intermediates, sized so a batch finishes inside the 60-day purge window. The +# sharded per-unit stores live under it: # /tiles/<2-char prefix>// and /exp/// -run_dir: /scratch/cdaley/shapepipe-output/smk-g3 +run_dir: /scratch/cdaley/shapepipe-output/smk-g4 + +# products_dir is the PERSISTENT root: the durable, low-volume products — the +# final catalogues (/tiles///final_cat-.fits, +# mirroring the scratch tree shard for shard), the index, the report. ~32-46 MB +# per tile, so a full DR6 campaign is a few hundred GB. +# +# The final catalogue is also the tile-finished MARKER that cuts a finished +# tile's exposure edges (D5), which is the second reason it cannot sit on +# scratch: a purge would not merely lose a product, it would make every finished +# tile re-declare inputs against exposure stores reclamation already deleted. +# +# Unset means "one root": products land under run_dir, exactly the pre-D5 +# layout, which is what a fixture or smoke test wants. +# +# Snakemake's own state is the one durable-looking thing that stays on scratch +# (-state; bin/sp explains why). +products_dir: /project/def-mjhudson/cdaley/sp-products/smk-g4 # There is no config_src knob: the config chain is workflow/config/cfis, resolved # relative to the Snakefile. The configs interpolate $SP_RUN / $SP_UNIT_NUM / @@ -38,8 +60,12 @@ run_dir: /scratch/cdaley/shapepipe-output/smk-g3 # `exp_star_cat` makes from those chunks, with no network at all. # Network therefore scales with SKY AREA, not exposure count: exposures overlap # ~7-10 deep, so a full-UNIONS footprint is ~1.5k queries against ~25k exposures. -# Design intent is /project — this is a durable science product, not scratch -# work — blocked only by the quota below. +# +# On the PERSISTENT root: the sky store is a durable science product bought with +# ~1.5k catalogue-server queries at DR6 scale, and re-buying it after a scratch +# purge is the one cost in this workflow that cannot be paid with local compute. +# (It sat on scratch through smk-g3 only because def-mjhudson /project was then +# hard-full at 27/27 TiB.) # # THE STORE IS NOT KEYED BY SCRIPT VERSION. A semantic change to # workflow/scripts/star_cats.py (padding, catalogue ID, column set) does rerun @@ -47,32 +73,21 @@ run_dir: /scratch/cdaley/shapepipe-output/smk-g3 # is skipped and only re-cut. Clear the store by hand when the change must reach # the data. Changing NSIDE or the catalogue ID is the exception: those name the # directory, so a change there fetches into a new one beside the old. -# -# On /scratch because def-mjhudson /project is hard-full (27/27 TiB). The 60-day -# purge therefore applies. A purged per-exposure cut is free to remake, and -# exp_star_cat declares its ccd-0 farm link as an output so a purge leaves that -# link dangling and the rule reruns rather than trusting a manifest that outlived -# its data. A purged CHUNK costs its query again: exp_star_cat fails loudly on a -# missing chunk, and re-running `star_catalogue` refetches it. -star_cats: /scratch/cdaley/shapepipe-output/star-cat-cache +star_cats: /project/def-mjhudson/cdaley/sp-products/star-cat-cache -# The run index. Design intent (finding 15) is durable products on /project — -# BLOCKED for now: def-mjhudson /project is hard-full (27/27 TiB), so index and -# report live with the run on /scratch until space is reclaimed. Mind the -# 60-day purge for anything that must survive. -index_db: /scratch/cdaley/shapepipe-output/smk-g3/index/run_index.sqlite +# The run index, and — sharing its directory — missing.json and run_report.json. +# On the persistent root with the catalogues (D5): the index is the record of +# which tile reads which exposure, so it is what a post-purge reconstruction +# would otherwise have to rebuild from tile headers. +index_db: /project/def-mjhudson/cdaley/sp-products/smk-g4/index/run_index.sqlite # Rolling exposure-store reclamation (D5). When true, the COMPUTE DAG grows one # `clean_exposure` job per exposure. It fires once every campaign tile that reads # that exposure has its vignets, deletes the exposure's store AND its manifests, # and leaves `cleaned.json`, which absorbs the manifests — `sp report` reads them -# back out of the tombstone and reports the exposure as `cleaned`. On here: -# the quad is validated against the bash baseline (validation verdict -# p0night1), so its exposure stores' inspection value is spent, and the -# campaign is scaling into the p3-batch1 tiles — 181 exposures un-cleaned -# would be ~1.2 TB against a 1 TiB scratch quota, so rolling reclamation is -# load-bearing from here on. Flipping it on reclaims retroactively — the -# missing tombstones schedule exactly the outstanding clean jobs. +# back out of the tombstone and reports the exposure as `cleaned`. On for smk-g4: +# reclamation is part of what this campaign measures, and at 34 tiles from +# scratch the un-cleaned exposure high-water is the run's binding disk constraint. clean: true # Tiles that may NOT pin an exposure store (default: empty). diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 8fc230c29..2739a1932 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -309,7 +309,7 @@ rule tile_make_cat: store = rules.tile_vignets.output.store, output: manifest = f"{TILE_DIR}/manifests/tile_make_cat.json", - final_cat = f"{TILE_DIR}/final_cat-{{tile}}.fits", + final_cat = f"{PROD_TILE_DIR}/final_cat-{{tile}}.fits", log: f"{TILE_DIR}/logs/tile_make_cat.json" params: