research: point-source CPU phase 4c — MAX_CONTAINING_SIZE headroom A/B (PyAutoArray#583)

hpc/batch_cpu/submit_breakdown_point_source_image_solver_config_sweep_mcs_ral_cpu_fp64

@@ -0,0 +1,139 @@
+#!/bin/bash -l
+#
+# 2026-09 point-source CPU speed-up campaign, phase 4c: the MAX_CONTAINING_SIZE
+# headroom A/B (PyAutoArray#583) on RAL CPU.
+#
+# WHAT THIS MEASURES. scripts/point_source_image/likelihood_breakdown/
+# solver_config_sweep.py --mcs-headroom: the production FitPositionsImagePairAllSolved
+# likelihood at the production geometry (+-9.9" / 0.2 / 1e-3) with the library's
+# MAX_CONTAINING_SIZE (15, the control) against 18 / 20 / 24, each patched for
+# tracing only (ArrayTriangles __defaults__ + module constant; fresh closure +
+# jax.clear_caches() per route), interleaved 20 rounds x 20 calls, bootstrap 90 %
+# CI, vmap 1/4/16, compile_s, XLA memory, FLOPs, the 200 prior + 200 stress
+# completeness draws, the NumPy uncapped containing counts for EVERY refinement
+# step on the prior / stress / stream draws, and
+# gates.mcs_no_unexplained_change (a row may differ from the control only on a
+# draw whose uncapped count exceeds the control's cap at some step).
+#
+# LIBRARY. Unchanged: every library is the shared mirror (/mnt/ral/jnightin/PyAuto,
+# via activate.sh). The job refuses to run unless every library imports from the
+# mirror and PyAutoArray includes #580 (4383ea81, the phase-4b step-0 route), and
+# asserts the JSON source_revisions against the mirror HEADs afterwards.
+#
+# THREADS. OMP/MKL/OPENBLAS pinned to 1 (BLAS family only); the JAX CPU backend
+# sizes its Eigen pool from the 8 CPUs of --cpus-per-task=8, as phases 1-4b.
+# XLA_FLAGS are the phase-1 row's, byte for byte.
+#
+# NODE. Pinned to the Xeon Platinum 8490H node of phases 4a/4b (euclid-ral-compute-10-4,
+# jobs 356367 / 357321). Check `sinfo -p ral -N -o "%N %T %C"` first: `idle*`
+# nodes are unreachable.
+
+# WALL-BASIS:
+#   RAL job 357321 (this cell in phase-4b form, --step0-route with 4 routes, 8490H
+#   euclid-ral-compute-10-4 under load ~200, 8 CPUs, 200 + 200 draws, 20 x 20) ran
+#   480 s. Phase 4c also runs 4 routes (control + 3 caps) plus the NumPy uncapped
+#   counts on 200 stress + 16 stream draws, so the basis is scaled to 600 s.
+#   Headroom 3 covers a quieter or busier node.
+#   cell: point_source_image/solver_config_sweep/simple  device: cpu  precision: fp64
+#   source: measured-wall  wall: 600  ref: ral-job-357321-point_source_image-solver_config_sweep  headroom: 3.0
+
+#SBATCH -J point_source_mcs_p4c
+#SBATCH --partition=ral
+#SBATCH --nodelist=euclid-ral-compute-10-4
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=8
+#SBATCH --mem=32gb
+#SBATCH --time=1:30:00
+#SBATCH -o output/output.%A.out
+#SBATCH -e error/error.%A.err
+#SBATCH --mail-type=END,FAIL
+#SBATCH --mail-user=james.w.nightingale@durham.ac.uk
+
+export AP_ROOT="$(cd "${SLURM_SUBMIT_DIR:-$(dirname "$0")}/../.." && pwd)"
+echo "AP_ROOT=$AP_ROOT"
+source $AP_ROOT/activate.sh
+
+export OMP_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export OPENBLAS_NUM_THREADS=1
+export NPROC=${SLURM_CPUS_PER_TASK:-8}
+export JAX_PLATFORMS=cpu
+export JAX_ENABLE_X64=1
+export XLA_FLAGS="--xla_disable_hlo_passes=constant_folding --xla_gpu_autotune_level=0 --xla_gpu_enable_triton_gemm=false"
+export NUMBA_CACHE_DIR=$AP_ROOT/output/numba_cache_${SLURM_JOB_ID:-local}
+mkdir -p "$NUMBA_CACHE_DIR" "$MPLCONFIGDIR"
+# No persistent compile cache: compile_s is a gate and must be cold per route.
+unset JAX_COMPILATION_CACHE_DIR
+
+CONFIG_NAME=hpc_ral_cpu_fp64
+RESULTS_DIR=$AP_ROOT/results/breakdown/point_source_image
+mkdir -p "$RESULTS_DIR"
+
+echo "=========================================="
+date
+hostname
+lscpu | grep 'Model name' || true
+echo "partition:      ${SLURM_JOB_PARTITION:-unset}"
+echo "cpus-per-task:  ${SLURM_CPUS_PER_TASK:-unset}"
+echo "sched_affinity: $(python3 -c 'import os; print(len(os.sched_getaffinity(0)))')"
+echo "loadavg:        $(cat /proc/loadavg)"
+echo "Cell:           point_source_image/likelihood_breakdown/solver_config_sweep --mcs-headroom"
+echo "Thread env:     OMP=$OMP_NUM_THREADS MKL=$MKL_NUM_THREADS OPENBLAS=$OPENBLAS_NUM_THREADS NPROC=$NPROC"
+echo "XLA_FLAGS:      $XLA_FLAGS"
+echo "PYTHONPATH:     $PYTHONPATH"
+
+echo "=== source revisions ==="
+for r in PyAutoNerves PyAutoFit PyAutoArray PyAutoGalaxy PyAutoLens; do
+    printf '%-14s %s\n' "$r (mirror)" "$(git -C /mnt/ral/jnightin/PyAuto/$r rev-parse HEAD 2>&1)"
+done
+echo "AP_ROOT rev:   $(git -C $AP_ROOT rev-parse HEAD 2>&1)"
+git -C /mnt/ral/jnightin/PyAuto/PyAutoArray merge-base --is-ancestor 4383ea81 HEAD \
+    || { echo "FATAL: mirror PyAutoArray lacks #580 (4383ea81); HPCPullPyAuto first" >&2; exit 2; }
+
+cd $AP_ROOT
+
+echo "=== import provenance ==="
+python3 - <<'PYEOF' || exit 3
+import autoarray, autofit, autogalaxy, autolens, jax, jaxlib, numpy
+for m in (autoarray, autofit, autogalaxy, autolens):
+    print(f"{m.__name__:<11} {m.__version__:<12} {m.__file__}")
+    if not m.__file__.startswith("/mnt/ral/jnightin/PyAuto/"):
+        raise SystemExit(f"REFUSING: {m.__name__} from {m.__file__}, not the shared mirror")
+from autoarray.structures.triangles import array as a
+print("MAX_CONTAINING_SIZE", a.MAX_CONTAINING_SIZE, "_STEP0_CONTAINMENT", a._STEP0_CONTAINMENT)
+if a.MAX_CONTAINING_SIZE != 15:
+    raise SystemExit(f"REFUSING: library MAX_CONTAINING_SIZE {a.MAX_CONTAINING_SIZE}, control must be 15")
+print("jax", jax.__version__, "jaxlib", jaxlib.__version__, "numpy", numpy.__version__, jax.devices())
+PYEOF
+
+echo "=========================================="
+echo "RUN ${CONFIG_NAME} -> ${RESULTS_DIR}"
+date
+echo "loadavg: $(cat /proc/loadavg)"
+PYTHONUNBUFFERED=1 python3 scripts/point_source_image/likelihood_breakdown/solver_config_sweep.py \
+    --config-name "$CONFIG_NAME" --output-dir "$RESULTS_DIR" \
+    --mcs-headroom \
+    --n-draws 200 --n-stress 200 --rounds 20 --calls 20
+
+echo "=== source_revisions check (JSON vs imported HEADs) ==="
+python3 - "$RESULTS_DIR/solver_config_sweep_mcs_${CONFIG_NAME}.json" <<'PYEOF'
+import json, subprocess, sys
+path = sys.argv[1]
+d = json.load(open(path))
+revs = d["source_revisions"]
+def head(p):
+    return subprocess.check_output(["git", "-C", p, "rev-parse", "HEAD"], text=True).strip()
+for r in ("PyAutoNerves", "PyAutoFit", "PyAutoArray", "PyAutoGalaxy", "PyAutoLens"):
+    h = head(f"/mnt/ral/jnightin/PyAuto/{r}")
+    if revs[r] != h:
+        raise SystemExit(f"MISMATCH {path}: {r} JSON {revs[r]} != {h}")
+subprocess.check_call(["git", "-C", "/mnt/ral/jnightin/PyAuto/PyAutoArray", "merge-base",
+                       "--is-ancestor", "4383ea81", revs["PyAutoArray"]])
+print("autoarray_imported_from", d.get("autoarray_imported_from"))
+print("OK", path, "all_gates_pass", d.get("all_gates_pass"))
+PYEOF
+
+echo "=========================================="
+echo "loadavg: $(cat /proc/loadavg)"
+echo "Finished."
+date

hpc/batch_gpu/submit_breakdown_point_source_image_solver_config_sweep_mcs_a100_fp64

@@ -0,0 +1,144 @@
+#!/bin/bash -l
+#
+# 2026-09 point-source CPU speed-up campaign, phase 4c: the MAX_CONTAINING_SIZE
+# headroom A/B (PyAutoArray#583): the A100 no-regression leg for the chosen cap
+# (N = 20). The A100 is launch-bound for this likelihood (phases 3 / 4b), so ~1x
+# is the expectation.
+#
+# WHAT THIS MEASURES. scripts/point_source_image/likelihood_breakdown/
+# solver_config_sweep.py --mcs-headroom --configs control,mcs20: the production
+# FitPositionsImagePairAllSolved likelihood at the production geometry
+# (+-9.9" / 0.2 / 1e-3) with the library's MAX_CONTAINING_SIZE (15, the control)
+# against 20, patched for tracing only (ArrayTriangles __defaults__ + module
+# constant; fresh closure + jax.clear_caches() per row), interleaved 20 rounds x
+# 20 calls, bootstrap 90 % CI, vmap 1/4/16, compile_s, XLA memory, FLOPs, the
+# 200 prior + 200 stress completeness draws, the NumPy uncapped containing counts
+# for every refinement step, and gates.mcs_no_unexplained_change.
+#
+# LIBRARY. Unchanged: the row measures the control-vs-MCS-20 patch, so it needs
+# no library branch. Every library is the shared mirror (/mnt/ral/jnightin/PyAuto,
+# via activate.sh); the job refuses to run unless every library imports from the
+# mirror, the mirror PyAutoArray includes #580 (4383ea81) and its
+# MAX_CONTAINING_SIZE is still 15, and asserts the JSON source_revisions
+# afterwards.
+#
+# BACKEND. JAX_PLATFORMS=cuda (NOT "cuda,cpu": a list can silently fall back to
+# CPU) and JAX_PLATFORM_NAME=cuda; the job asserts jax.default_backend()=="gpu".
+# The harness's fiducial gate is backend-aware (A100 7.743201200876806).
+# XLA_FLAGS are byte-identical to the CPU submit's.
+
+# WALL-BASIS:
+#   RAL job 357322 (this cell in phase-4b form, --step0-route with 4 routes, A100
+#   80 GB euclid-ral-gpu-1, 200 + 200 draws, 20 x 20) ran 432 s (7:32 job wall);
+#   the A100 is launch-bound, so the completeness draws dominate. Phase 4c runs 2
+#   rows (control + mcs20) plus the NumPy uncapped counts, so 432 s stays the basis.
+#   cell: point_source_image/solver_config_sweep/simple  device: gpu  precision: fp64
+#   source: measured-wall  wall: 432  ref: ral-job-357322-point_source_image-solver_config_sweep  headroom: 3.0
+
+#SBATCH -J point_source_mcs_p4c_gpu
+#SBATCH --partition=gpu
+#SBATCH --gres=gpu:1
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=8
+#SBATCH --mem=32gb
+#SBATCH --time=3:00:00
+#SBATCH -o output/output.%A.out
+#SBATCH -e error/error.%A.err
+#SBATCH --mail-type=END,FAIL
+#SBATCH --mail-user=james.w.nightingale@durham.ac.uk
+
+export AP_ROOT="$(cd "${SLURM_SUBMIT_DIR:-$(dirname "$0")}/../.." && pwd)"
+echo "AP_ROOT=$AP_ROOT"
+source $AP_ROOT/activate.sh
+
+export OMP_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export OPENBLAS_NUM_THREADS=1
+export NPROC=${SLURM_CPUS_PER_TASK:-8}
+export JAX_PLATFORM_NAME=cuda
+export JAX_PLATFORMS=cuda
+export XLA_PYTHON_CLIENT_PREALLOCATE=false
+export JAX_ENABLE_X64=1
+export XLA_FLAGS="--xla_disable_hlo_passes=constant_folding --xla_gpu_autotune_level=0 --xla_gpu_enable_triton_gemm=false"
+export NUMBA_CACHE_DIR=$AP_ROOT/output/numba_cache_${SLURM_JOB_ID:-local}
+mkdir -p "$NUMBA_CACHE_DIR" "$MPLCONFIGDIR"
+# No persistent compile cache: compile_s is a gate and must be cold per row.
+unset JAX_COMPILATION_CACHE_DIR
+
+CONFIG_NAME=hpc_ral_a100_fp64
+RESULTS_DIR=$AP_ROOT/results/breakdown/point_source_image
+mkdir -p "$RESULTS_DIR"
+
+echo "=========================================="
+date
+hostname
+lscpu | grep 'Model name' || true
+nvidia-smi --query-gpu=name,driver_version,memory.total,utilization.gpu --format=csv || true
+echo "gpus:           ${CUDA_VISIBLE_DEVICES:-unset}"
+echo "partition:      ${SLURM_JOB_PARTITION:-unset}"
+echo "cpus-per-task:  ${SLURM_CPUS_PER_TASK:-unset}"
+echo "sched_affinity: $(python3 -c 'import os; print(len(os.sched_getaffinity(0)))')"
+echo "loadavg:        $(cat /proc/loadavg)"
+echo "Cell:           point_source_image/likelihood_breakdown/solver_config_sweep --mcs-headroom --configs control,mcs20"
+echo "Thread env:     OMP=$OMP_NUM_THREADS MKL=$MKL_NUM_THREADS OPENBLAS=$OPENBLAS_NUM_THREADS NPROC=$NPROC"
+echo "XLA_FLAGS:      $XLA_FLAGS"
+echo "PYTHONPATH:     $PYTHONPATH"
+
+echo "=== source revisions ==="
+for r in PyAutoNerves PyAutoFit PyAutoArray PyAutoGalaxy PyAutoLens; do
+    printf '%-14s %s\n' "$r (mirror)" "$(git -C /mnt/ral/jnightin/PyAuto/$r rev-parse HEAD 2>&1)"
+done
+echo "AP_ROOT rev:   $(git -C $AP_ROOT rev-parse HEAD 2>&1)"
+git -C /mnt/ral/jnightin/PyAuto/PyAutoArray merge-base --is-ancestor 4383ea81 HEAD \
+    || { echo "FATAL: mirror PyAutoArray lacks #580 (4383ea81); HPCPullPyAuto first" >&2; exit 2; }
+
+cd $AP_ROOT
+
+echo "=== import provenance ==="
+python3 - <<'PYEOF' || exit 3
+import autoarray, autofit, autogalaxy, autolens, jax, jaxlib, numpy
+for m in (autoarray, autofit, autogalaxy, autolens):
+    print(f"{m.__name__:<11} {m.__version__:<12} {m.__file__}")
+    if not m.__file__.startswith("/mnt/ral/jnightin/PyAuto/"):
+        raise SystemExit(f"REFUSING: {m.__name__} from {m.__file__}, not the shared mirror")
+from autoarray.structures.triangles import array as a
+print("MAX_CONTAINING_SIZE", a.MAX_CONTAINING_SIZE, "_STEP0_CONTAINMENT", a._STEP0_CONTAINMENT)
+if a.MAX_CONTAINING_SIZE != 15:
+    raise SystemExit(f"REFUSING: library MAX_CONTAINING_SIZE {a.MAX_CONTAINING_SIZE}, control must be 15")
+print("jax", jax.__version__, "jaxlib", jaxlib.__version__, "numpy", numpy.__version__, jax.devices())
+if jax.default_backend() != "gpu":
+    raise SystemExit(f"REFUSING: JAX backend is {jax.default_backend()}, not gpu")
+PYEOF
+
+echo "=========================================="
+echo "RUN ${CONFIG_NAME} -> ${RESULTS_DIR}"
+date
+echo "loadavg: $(cat /proc/loadavg)"
+PYTHONUNBUFFERED=1 python3 scripts/point_source_image/likelihood_breakdown/solver_config_sweep.py \
+    --config-name "$CONFIG_NAME" --output-dir "$RESULTS_DIR" \
+    --mcs-headroom --configs control,mcs20 \
+    --n-draws 200 --n-stress 200 --rounds 20 --calls 20
+
+echo "=== source_revisions check (JSON vs imported HEADs) ==="
+python3 - "$RESULTS_DIR/solver_config_sweep_mcs_${CONFIG_NAME}.json" <<'PYEOF'
+import json, subprocess, sys
+path = sys.argv[1]
+d = json.load(open(path))
+revs = d["source_revisions"]
+def head(p):
+    return subprocess.check_output(["git", "-C", p, "rev-parse", "HEAD"], text=True).strip()
+for r in ("PyAutoNerves", "PyAutoFit", "PyAutoArray", "PyAutoGalaxy", "PyAutoLens"):
+    h = head(f"/mnt/ral/jnightin/PyAuto/{r}")
+    if revs[r] != h:
+        raise SystemExit(f"MISMATCH {path}: {r} JSON {revs[r]} != {h}")
+subprocess.check_call(["git", "-C", "/mnt/ral/jnightin/PyAuto/PyAutoArray", "merge-base",
+                       "--is-ancestor", "4383ea81", revs["PyAutoArray"]])
+print("autoarray_imported_from", d.get("autoarray_imported_from"))
+print("OK", path, "all_gates_pass", d.get("all_gates_pass"))
+PYEOF
+
+echo "=========================================="
+echo "loadavg: $(cat /proc/loadavg)"
+nvidia-smi --query-gpu=name,memory.used --format=csv || true
+echo "Finished."
+date