results: A100 A/B of the compacted ConstantSplit assembly (PyAutoArray#537, 12x)

hpc/batch_gpu/submit_breakdown_imaging_delaunay_nn_a100_hst_fp64_assembly

@@ -0,0 +1,108 @@
+#!/bin/bash -l
+#
+# A100 likelihood-breakdown profile: imaging/delaunay_nn x hst x fp64 — PyAutoArray#536
+# A/B leg: **feature**.
+#
+# Verification run for the ConstantSplit split-stencil assembly compaction
+# (PyAutoArray issue #536 / branch feature/delaunay-nn-constant-split-assembly:
+# the JAX path of pixel_splitted_regularization_matrix_from scatters only the
+# first SPLIT_REG_COMPACT_WIDTH = 12 columns of every (4P, K) stencil row
+# instead of the full padded (4P, K, K) outer product, supplements the
+# SPLIT_REG_WIDE_ROW_BUDGET = 256 widest rows at full width, and NaNs the
+# matrix out if more rows exceed the compact width than the budget holds).
+# Both legs of the A/B run in the same session on the same node class with the
+# same everything except the PyAutoArray checkout prepended to PYTHONPATH:
+#
+#   control  -> 180c8a40 (the branch's merge base = PyAutoArray main)
+#   feature  -> feature/delaunay-nn-constant-split-assembly (04152bb7)
+#
+# The predecessor A/B on this cell (#532/#533, merge base d7c96762) is recorded
+# in results/notes/delaunay_nn_launch_latency.md; its legs are kept as
+# submit_..._launch_latency{,_control} and are comparable to these rows only
+# insofar as the merge base moved by that one merge.
+#
+# The shared /mnt/ral/jnightin/PyAuto/PyAutoArray install is NOT touched — long
+# science runs (subhalo validation, Euclid DR1) share it. The private checkouts
+# live under /mnt/ral/jnightin/PyAuto_wt/delaunay-nn-constant-split-assembly/
+# and are prepended to the PYTHONPATH that $AP_ROOT/activate.sh exports, so only
+# PyAutoArray is overridden and Nerves/Fit/Galaxy/Lens stay on the shared tree.
+#
+# WITNESS. The target row is "Regularization matrix (H, ConstantSplit
+# assembly)" in the --split-setup table (JSON regularization_split /
+# regularization_split_vmap): 10.0 ms per call at vmap 16 on the #533 shipped
+# leg, and the witness is < 3 ms per call. The params->H prefix
+# (regularization_matrix_prefix_s) carries it: 16.4 ms per call at vmap 16 on
+# that leg, witness < 11 ms. Judge on the params->H prefix and the assembly
+# row; the H *step* row is a prefix difference and shifts with attribution
+# (the caveat in results/notes/delaunay_walk_early_exit.md).
+#
+#   --split-setup    four-way split of the inversion-setup block (+ the
+#                    split-point Sibson cut of the params->H interval)
+#   --vmap-batch 16  batched re-timing at the Nautilus reference n_batch
+
+#SBATCH -J bd_delnn_asm
+#SBATCH --partition=gpu
+#SBATCH --gres=gpu:1
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=4
+#SBATCH --mem=64gb
+#SBATCH --time=12:00:00
+#SBATCH -o output/output.%A.out
+#SBATCH -e error/error.%A.err
+
+export AP_ROOT="$(cd "${SLURM_SUBMIT_DIR:-$(dirname "$0")}/../.." && pwd)"
+echo "AP_ROOT=$AP_ROOT"
+source $AP_ROOT/activate.sh
+
+# --- PyAutoArray A/B override (private checkout, shared install untouched) ---
+AA_ROOT=/mnt/ral/jnightin/PyAuto_wt/delaunay-nn-constant-split-assembly/PyAutoArray_feature
+export PYTHONPATH=$AA_ROOT:$PYTHONPATH
+
+export JAX_PLATFORM_NAME=cuda
+export JAX_PLATFORMS=cuda,cpu
+export XLA_PYTHON_CLIENT_PREALLOCATE=false
+export JAX_ENABLE_X64=True
+export XLA_FLAGS="--xla_disable_hlo_passes=constant_folding"
+export NUMBA_CACHE_DIR=/tmp/numba_cache
+export MPLCONFIGDIR=/tmp/matplotlib
+
+# PYAUTO_SIBSON_QUERY_CHUNK is deliberately left UNSET on both A/B legs, as in
+# the #533 run whose rows these are compared against: the library default
+# (4096, chosen by that run's sweep) is what production uses. The script
+# records the effective SIBSON_QUERY_CHUNK under "sibson" in the result JSON.
+
+nvidia-smi
+
+# MIG backstop. A no-op since the fleet's MIG mode was confirmed Disabled
+# (2026-09-05); kept because a silent removal re-opens the trap. Only bounces
+# when the job was dispatched with --requeue.
+source "$AP_ROOT/hpc/batch_gpu/_gpu_preflight.sh"
+
+echo "=========================================="
+date
+echo "Cell:           likelihood_breakdown/imaging/delaunay_nn"
+echo "Instrument:     hst"
+echo "Precision:      fp64"
+echo "A/B leg:        assembly (feature/delaunay-nn-constant-split-assembly, 04152bb7)"
+echo "Flags:          --split-setup --vmap-batch 16 --config-name hpc_a100_fp64_assembly"
+echo "Sibson chunk:   PYAUTO_SIBSON_QUERY_CHUNK=${PYAUTO_SIBSON_QUERY_CHUNK:-unset (library default)}"
+
+echo "=== source revisions ==="
+printf '%-14s %s\n' "PyAutoArray(AB)" "$(git -C $AA_ROOT rev-parse HEAD 2>&1)"
+for r in PyAutoNerves PyAutoFit PyAutoGalaxy PyAutoLens; do
+    printf '%-14s %s\n' "$r" "$(git -C /mnt/ral/jnightin/PyAuto/$r rev-parse HEAD 2>&1)"
+done
+echo "AP_ROOT rev:   $(git -C $AP_ROOT rev-parse HEAD 2>&1)"
+
+cd $AP_ROOT
+
+echo "=== import provenance ==="
+python3 -c "import autoarray, autolens; print('autoarray.__file__ =', autoarray.__file__); print('autoarray.__version__ =', autoarray.__version__); print('autolens.__file__ =', autolens.__file__)"
+
+python3 -u scripts/imaging/likelihood_breakdown/delaunay_nn.py \
+    --config-name hpc_a100_fp64_assembly \
+    --split-setup \
+    --vmap-batch 16
+
+echo "Finished."
+date

hpc/batch_gpu/submit_breakdown_imaging_delaunay_nn_a100_hst_fp64_assembly_control

@@ -0,0 +1,108 @@
+#!/bin/bash -l
+#
+# A100 likelihood-breakdown profile: imaging/delaunay_nn x hst x fp64 — PyAutoArray#536
+# A/B leg: **control**.
+#
+# Verification run for the ConstantSplit split-stencil assembly compaction
+# (PyAutoArray issue #536 / branch feature/delaunay-nn-constant-split-assembly:
+# the JAX path of pixel_splitted_regularization_matrix_from scatters only the
+# first SPLIT_REG_COMPACT_WIDTH = 12 columns of every (4P, K) stencil row
+# instead of the full padded (4P, K, K) outer product, supplements the
+# SPLIT_REG_WIDE_ROW_BUDGET = 256 widest rows at full width, and NaNs the
+# matrix out if more rows exceed the compact width than the budget holds).
+# Both legs of the A/B run in the same session on the same node class with the
+# same everything except the PyAutoArray checkout prepended to PYTHONPATH:
+#
+#   control  -> 180c8a40 (the branch's merge base = PyAutoArray main)
+#   feature  -> feature/delaunay-nn-constant-split-assembly (04152bb7)
+#
+# The predecessor A/B on this cell (#532/#533, merge base d7c96762) is recorded
+# in results/notes/delaunay_nn_launch_latency.md; its legs are kept as
+# submit_..._launch_latency{,_control} and are comparable to these rows only
+# insofar as the merge base moved by that one merge.
+#
+# The shared /mnt/ral/jnightin/PyAuto/PyAutoArray install is NOT touched — long
+# science runs (subhalo validation, Euclid DR1) share it. The private checkouts
+# live under /mnt/ral/jnightin/PyAuto_wt/delaunay-nn-constant-split-assembly/
+# and are prepended to the PYTHONPATH that $AP_ROOT/activate.sh exports, so only
+# PyAutoArray is overridden and Nerves/Fit/Galaxy/Lens stay on the shared tree.
+#
+# WITNESS. The target row is "Regularization matrix (H, ConstantSplit
+# assembly)" in the --split-setup table (JSON regularization_split /
+# regularization_split_vmap): 10.0 ms per call at vmap 16 on the #533 shipped
+# leg, and the witness is < 3 ms per call. The params->H prefix
+# (regularization_matrix_prefix_s) carries it: 16.4 ms per call at vmap 16 on
+# that leg, witness < 11 ms. Judge on the params->H prefix and the assembly
+# row; the H *step* row is a prefix difference and shifts with attribution
+# (the caveat in results/notes/delaunay_walk_early_exit.md).
+#
+#   --split-setup    four-way split of the inversion-setup block (+ the
+#                    split-point Sibson cut of the params->H interval)
+#   --vmap-batch 16  batched re-timing at the Nautilus reference n_batch
+
+#SBATCH -J bd_delnn_asm_ctl
+#SBATCH --partition=gpu
+#SBATCH --gres=gpu:1
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=4
+#SBATCH --mem=64gb
+#SBATCH --time=12:00:00
+#SBATCH -o output/output.%A.out
+#SBATCH -e error/error.%A.err
+
+export AP_ROOT="$(cd "${SLURM_SUBMIT_DIR:-$(dirname "$0")}/../.." && pwd)"
+echo "AP_ROOT=$AP_ROOT"
+source $AP_ROOT/activate.sh
+
+# --- PyAutoArray A/B override (private checkout, shared install untouched) ---
+AA_ROOT=/mnt/ral/jnightin/PyAuto_wt/delaunay-nn-constant-split-assembly/PyAutoArray_control
+export PYTHONPATH=$AA_ROOT:$PYTHONPATH
+
+export JAX_PLATFORM_NAME=cuda
+export JAX_PLATFORMS=cuda,cpu
+export XLA_PYTHON_CLIENT_PREALLOCATE=false
+export JAX_ENABLE_X64=True
+export XLA_FLAGS="--xla_disable_hlo_passes=constant_folding"
+export NUMBA_CACHE_DIR=/tmp/numba_cache
+export MPLCONFIGDIR=/tmp/matplotlib
+
+# PYAUTO_SIBSON_QUERY_CHUNK is deliberately left UNSET on both A/B legs, as in
+# the #533 run whose rows these are compared against: the library default
+# (4096, chosen by that run's sweep) is what production uses. The script
+# records the effective SIBSON_QUERY_CHUNK under "sibson" in the result JSON.
+
+nvidia-smi
+
+# MIG backstop. A no-op since the fleet's MIG mode was confirmed Disabled
+# (2026-09-05); kept because a silent removal re-opens the trap. Only bounces
+# when the job was dispatched with --requeue.
+source "$AP_ROOT/hpc/batch_gpu/_gpu_preflight.sh"
+
+echo "=========================================="
+date
+echo "Cell:           likelihood_breakdown/imaging/delaunay_nn"
+echo "Instrument:     hst"
+echo "Precision:      fp64"
+echo "A/B leg:        control (merge base 180c8a40)"
+echo "Flags:          --split-setup --vmap-batch 16 --config-name hpc_a100_fp64_assembly_control"
+echo "Sibson chunk:   PYAUTO_SIBSON_QUERY_CHUNK=${PYAUTO_SIBSON_QUERY_CHUNK:-unset (library default)}"
+
+echo "=== source revisions ==="
+printf '%-14s %s\n' "PyAutoArray(AB)" "$(git -C $AA_ROOT rev-parse HEAD 2>&1)"
+for r in PyAutoNerves PyAutoFit PyAutoGalaxy PyAutoLens; do
+    printf '%-14s %s\n' "$r" "$(git -C /mnt/ral/jnightin/PyAuto/$r rev-parse HEAD 2>&1)"
+done
+echo "AP_ROOT rev:   $(git -C $AP_ROOT rev-parse HEAD 2>&1)"
+
+cd $AP_ROOT
+
+echo "=== import provenance ==="
+python3 -c "import autoarray, autolens; print('autoarray.__file__ =', autoarray.__file__); print('autoarray.__version__ =', autoarray.__version__); print('autolens.__file__ =', autolens.__file__)"
+
+python3 -u scripts/imaging/likelihood_breakdown/delaunay_nn.py \
+    --config-name hpc_a100_fp64_assembly_control \
+    --split-setup \
+    --vmap-batch 16
+
+echo "Finished."
+date

hpc/batch_gpu/submit_delaunay_nn_assembly_bench_a100

@@ -0,0 +1,65 @@
+#!/bin/bash -l
+#
+# A100 direct bench of the DelaunayNN ConstantSplit assembly — PyAutoArray#536.
+#
+# The companion to the breakdown/runtime A/B legs: instead of timing the
+# assembly as a difference of two compiled prefixes, this times the assembly
+# function itself on the real (4P, K) HST split stencil tables committed at
+# results/delaunay_nn/tables_hst_1500.npz. `V0` is the pre-#536 full padded
+# (4P, K, K) scatter and `V6_library` is the shipped library path with its
+# default compaction, so the V0-vs-V6_library row is the direct statement of
+# what the change bought, free of any prefix-attribution caveat.
+#
+# Runs on the **feature** private checkout (the library default constants are
+# what V6_library exercises); V0 is reproduced from the same code by forcing
+# compact_width = K, so both rows come from one checkout and no control
+# checkout is needed here.
+#
+# Cheap: no dataset, no imaging fit, minutes not hours.
+
+#SBATCH -J delnn_asm_bench
+#SBATCH --partition=gpu
+#SBATCH --gres=gpu:1
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=4
+#SBATCH --mem=64gb
+#SBATCH --time=1:00:00
+#SBATCH -o output/output.%A.out
+#SBATCH -e error/error.%A.err
+
+export AP_ROOT="$(cd "${SLURM_SUBMIT_DIR:-$(dirname "$0")}/../.." && pwd)"
+echo "AP_ROOT=$AP_ROOT"
+source $AP_ROOT/activate.sh
+
+AA_ROOT=/mnt/ral/jnightin/PyAuto_wt/delaunay-nn-constant-split-assembly/PyAutoArray_feature
+export PYTHONPATH=$AA_ROOT:$PYTHONPATH
+
+export JAX_PLATFORM_NAME=cuda
+export JAX_PLATFORMS=cuda,cpu
+export XLA_PYTHON_CLIENT_PREALLOCATE=false
+export JAX_ENABLE_X64=True
+export XLA_FLAGS="--xla_disable_hlo_passes=constant_folding"
+export NUMBA_CACHE_DIR=/tmp/numba_cache
+export MPLCONFIGDIR=/tmp/matplotlib
+
+nvidia-smi
+
+# MIG backstop; no-op since 2026-09-05, only bounces under --requeue.
+source "$AP_ROOT/hpc/batch_gpu/_gpu_preflight.sh"
+
+echo "=========================================="
+date
+printf 'PyAutoArray(AB) %s\n' "$(git -C $AA_ROOT rev-parse HEAD 2>&1)"
+echo "AP_ROOT rev:   $(git -C $AP_ROOT rev-parse HEAD 2>&1)"
+
+cd $AP_ROOT
+python3 -c "import autoarray; print('autoarray.__file__ =', autoarray.__file__)"
+
+python3 -u scripts/misc/delaunay_nn/assembly_bench.py \
+    --batch 16 \
+    --repeats 10 \
+    --label "A100 fp64, PyAutoArray#536 A/B session" \
+    --out results/delaunay_nn/assembly_bench_a100_fp64.json
+
+echo "Finished."
+date

hpc/batch_gpu/submit_runtime_imaging_delaunay_nn_a100_hst_fp64_assembly

@@ -0,0 +1,64 @@
+#!/bin/bash -l
+#
+# A100 likelihood-runtime profile: imaging/delaunay_nn x hst x fp64 — PyAutoArray#536
+# A/B leg: **feature**. Single-JIT whole-likelihood cost, the end-to-end companion
+# to submit_breakdown_imaging_delaunay_nn_a100_hst_fp64_assembly.
+#
+# Same private-checkout PYTHONPATH override as the breakdown A/B legs; the
+# shared /mnt/ral/jnightin/PyAuto install is untouched. Phase A (--vmap-probe)
+# is skipped: only the full single-JIT + vmap timing is wanted here, and this
+# cell's probe is known to over-recommend batch 64 (cuFFT batched-plan failure,
+# job 342280) — with no probe JSON present the script falls back to the
+# VMAP_BATCH table (16), which is the number both legs must share.
+#
+#   control  -> 180c8a40 (the branch's merge base = PyAutoArray main)
+#   feature  -> feature/delaunay-nn-constant-split-assembly (04152bb7)
+#
+# PYAUTO_SIBSON_QUERY_CHUNK is left unset on both legs (the library default
+# 4096 is what production uses); the result JSON records the effective chunk.
+
+#SBATCH -J rt_delnn_asm
+#SBATCH --partition=gpu
+#SBATCH --gres=gpu:1
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=4
+#SBATCH --mem=64gb
+#SBATCH --time=3:00:00
+#SBATCH -o output/output.%A.out
+#SBATCH -e error/error.%A.err
+
+export AP_ROOT="$(cd "${SLURM_SUBMIT_DIR:-$(dirname "$0")}/../.." && pwd)"
+echo "AP_ROOT=$AP_ROOT"
+source $AP_ROOT/activate.sh
+
+AA_ROOT=/mnt/ral/jnightin/PyAuto_wt/delaunay-nn-constant-split-assembly/PyAutoArray_feature
+export PYTHONPATH=$AA_ROOT:$PYTHONPATH
+
+export JAX_PLATFORM_NAME=cuda
+export JAX_PLATFORMS=cuda,cpu
+export XLA_PYTHON_CLIENT_PREALLOCATE=false
+export JAX_ENABLE_X64=True
+export NUMBA_CACHE_DIR=/tmp/numba_cache
+export MPLCONFIGDIR=/tmp/matplotlib
+
+nvidia-smi
+
+# MIG backstop; no-op since 2026-09-05, only bounces under --requeue.
+source "$AP_ROOT/hpc/batch_gpu/_gpu_preflight.sh"
+
+echo "=========================================="
+date
+echo "Cell:           likelihood_runtime/imaging/delaunay_nn"
+echo "A/B leg:        assembly (feature/delaunay-nn-constant-split-assembly, 04152bb7)"
+echo "Sibson chunk:   PYAUTO_SIBSON_QUERY_CHUNK=${PYAUTO_SIBSON_QUERY_CHUNK:-unset (library default)}"
+printf 'PyAutoArray(AB) %s\n' "$(git -C $AA_ROOT rev-parse HEAD 2>&1)"
+
+cd $AP_ROOT
+python3 -c "import autoarray; print('autoarray.__file__ =', autoarray.__file__)"
+
+python3 -u scripts/imaging/likelihood_runtime/delaunay_nn.py \
+    --instrument hst \
+    --config-name hpc_a100_fp64_assembly
+
+echo "Finished."
+date