diff --git a/hpc/batch_gpu/submit_breakdown_imaging_delaunay_nn_a100_hst_fp64_chunk_sweep b/hpc/batch_gpu/submit_breakdown_imaging_delaunay_nn_a100_hst_fp64_chunk_sweep new file mode 100644 index 000000000..4641e0888 --- /dev/null +++ b/hpc/batch_gpu/submit_breakdown_imaging_delaunay_nn_a100_hst_fp64_chunk_sweep @@ -0,0 +1,138 @@ +#!/bin/bash -l +# +# A100 likelihood-breakdown SWEEP: imaging/delaunay_nn x hst x fp64 — PyAutoArray#532 +# One array task per SIBSON_QUERY_CHUNK value, on the **feature** checkout. +# +# WHAT THIS MEASURES +# The Sibson passes push their queries through a `jax.lax.map` over +# SIBSON_QUERY_CHUNK-sized blocks, so the chunk sets how many *sequential* +# trips the two passes take: at chunk 256 the HST cell runs 95 of them (71 +# data + 24 split) and each trip pays the A100 kernel-launch floor. The chunk +# was only ever a memory guard on the (chunk, 3, 2) cavity intermediates. +# This sweep says what the largest safe value is: per-call time from the +# result JSON, peak VRAM from the nvidia-smi sampler below. +# +# DECISION RULE (plan A.3): ship the largest of 512/1024/2048/4096 whose peak +# VRAM at vmap 16 stays under ~50 % of the 80 GB and which is fastest per +# call; if 4096 wins on both, ship 2048 unless the margin is > 15 %, to keep +# headroom for larger cells. The chosen default lands as a follow-up commit on +# the library branch before its PR is merged. +# +# ARMS — 4 array tasks, chunk = 512 / 1024 / 2048 / 4096 (index -> CHUNKS below). +# PYAUTO_SIBSON_QUERY_CHUNK is PyAutoArray's env override of +# SIBSON_QUERY_CHUNK, read once at import; the breakdown script records the +# effective value (and the raw env var) under "sibson" in every result JSON, +# so a sweep row is self-describing and cannot be mistaken for another row. +# +# Each task writes delaunay_nn_hpc_a100_fp64_chunk.{json,png} — +# --config-name is the existing suffix mechanism in _profile_cli.resolve_output_paths +# (basename = _), so the four rows never clobber each other +# or the A/B legs' rows. +# +# PEAK VRAM +# `nvidia-smi --query-gpu=memory.used --format=csv -l 2` runs in the +# background for the life of the Python process and writes into this task's +# SLURM log; the CSV header repeats, so read the peak with +# +# grep -o '[0-9]\+ MiB' output/output._.out | sort -n | tail -1 +# +# It is sampled, not a high-water mark from the allocator: XLA_PYTHON_CLIENT_- +# PREALLOCATE=false means the reading tracks real allocation, but a spike +# shorter than the 2 s period can be missed. Read it as a floor on the peak. +# +# WALL +# The #531 A/B legs of this exact cell (breakdown, --split-setup --vmap-batch 16) +# ran 2:29 and 2:28 wall on euclid-ral-gpu-2 (jobs 342317/342318, +# results/notes/delaunay_walk_early_exit.md). This adds one more prefix compile +# unbatched and one under vmap, and a larger chunk trades launches for bigger +# kernels, so 4:00:00 per task is ~50x the measured wall. + +#SBATCH -J bd_delnn_chunksweep +#SBATCH --partition=gpu +#SBATCH --gres=gpu:1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=4 +#SBATCH --mem=64gb +#SBATCH --time=4:00:00 +#SBATCH --array=0-3 +#SBATCH -o output/output.%A_%a.out +#SBATCH -e error/error.%A_%a.err + +export AP_ROOT="$(cd "${SLURM_SUBMIT_DIR:-$(dirname "$0")}/../.." && pwd)" +echo "AP_ROOT=$AP_ROOT" +source $AP_ROOT/activate.sh + +# --- PyAutoArray override (private feature checkout, shared install untouched) --- +AA_ROOT=/mnt/ral/jnightin/PyAuto_wt/delaunay-nn-launch-latency/PyAutoArray_feature +export PYTHONPATH=$AA_ROOT:$PYTHONPATH + +export JAX_PLATFORM_NAME=cuda +export JAX_PLATFORMS=cuda,cpu +export XLA_PYTHON_CLIENT_PREALLOCATE=false +export JAX_ENABLE_X64=True +export XLA_FLAGS="--xla_disable_hlo_passes=constant_folding" +export NUMBA_CACHE_DIR=/tmp/numba_cache +export MPLCONFIGDIR=/tmp/matplotlib + +CHUNKS=(512 1024 2048 4096) +CHUNK=${CHUNKS[$SLURM_ARRAY_TASK_ID]} +if [ -z "$CHUNK" ]; then + echo "FATAL: no chunk for array task ${SLURM_ARRAY_TASK_ID} (expected 0-3)." >&2 + exit 1 +fi +export PYAUTO_SIBSON_QUERY_CHUNK=$CHUNK +CONFIG_NAME="hpc_a100_fp64_chunk${CHUNK}" + +nvidia-smi + +# MIG backstop. A no-op since the fleet's MIG mode was confirmed Disabled +# (2026-09-05); kept because a silent removal re-opens the trap. Only bounces +# when the job was dispatched with --requeue. +source "$AP_ROOT/hpc/batch_gpu/_gpu_preflight.sh" + +echo "==========================================" +date +echo "Task: $SLURM_ARRAY_TASK_ID / 3" +echo "Cell: likelihood_breakdown/imaging/delaunay_nn" +echo "Instrument: hst" +echo "Precision: fp64" +echo "A/B leg: launch_latency (feature/delaunay-nn-launch-latency)" +echo "Sibson chunk: PYAUTO_SIBSON_QUERY_CHUNK=$PYAUTO_SIBSON_QUERY_CHUNK" +echo "Flags: --split-setup --vmap-batch 16 --config-name $CONFIG_NAME" + +echo "=== source revisions ===" +printf '%-14s %s\n' "PyAutoArray(AB)" "$(git -C $AA_ROOT rev-parse HEAD 2>&1)" +for r in PyAutoNerves PyAutoFit PyAutoGalaxy PyAutoLens; do + printf '%-14s %s\n' "$r" "$(git -C /mnt/ral/jnightin/PyAuto/$r rev-parse HEAD 2>&1)" +done +echo "AP_ROOT rev: $(git -C $AP_ROOT rev-parse HEAD 2>&1)" + +cd $AP_ROOT + +echo "=== import provenance ===" +python3 -c "import autoarray, autolens; print('autoarray.__file__ =', autoarray.__file__); print('autoarray.__version__ =', autoarray.__version__); print('autolens.__file__ =', autolens.__file__)" +python3 -c "from autoarray.inversion.mesh.interpolator.sibson import SIBSON_QUERY_CHUNK; print('SIBSON_QUERY_CHUNK (effective) =', SIBSON_QUERY_CHUNK)" + +# --- background peak-VRAM sampler (killed after the run) --- +echo "=== nvidia-smi memory.used sampler (every 2 s) ===" +nvidia-smi --query-gpu=memory.used --format=csv -l 2 & +VRAM_SAMPLER_PID=$! +trap 'kill "$VRAM_SAMPLER_PID" 2>/dev/null' EXIT + +python3 -u scripts/imaging/likelihood_breakdown/delaunay_nn.py \ + --config-name $CONFIG_NAME \ + --split-setup \ + --vmap-batch 16 + +kill "$VRAM_SAMPLER_PID" 2>/dev/null +trap - EXIT + +# The SLURM log path is relative to the submit dir (hpc/batch_gpu), not to +# $AP_ROOT which this script cd'd into. Trailing lines may still be buffered, +# so re-read the log after the job for the authoritative peak. +_LOG_DIR="${SLURM_SUBMIT_DIR:-$AP_ROOT/hpc/batch_gpu}/output" +echo "Peak sampled VRAM this task (so far):" +grep -o '[0-9]\+ MiB' "$_LOG_DIR/output.${SLURM_ARRAY_JOB_ID}_${SLURM_ARRAY_TASK_ID}.out" 2>/dev/null | sort -n | tail -1 + +echo "Finished." +date diff --git a/hpc/batch_gpu/submit_breakdown_imaging_delaunay_nn_a100_hst_fp64_launch_latency b/hpc/batch_gpu/submit_breakdown_imaging_delaunay_nn_a100_hst_fp64_launch_latency new file mode 100644 index 000000000..79ed19b26 --- /dev/null +++ b/hpc/batch_gpu/submit_breakdown_imaging_delaunay_nn_a100_hst_fp64_launch_latency @@ -0,0 +1,114 @@ +#!/bin/bash -l +# +# A100 likelihood-breakdown profile: imaging/delaunay_nn x hst x fp64 — PyAutoArray#532 +# A/B leg: **launch_latency** (feature). +# +# Verification run for the DelaunayNN Sibson launch-latency change +# (PyAutoArray issue #532 / branch feature/delaunay-nn-launch-latency: candidate +# loop unrolled inside the cavity walk, data grid + split points located and +# interpolated in one concatenated Sibson pass, SIBSON_QUERY_CHUNK returned to +# being only a memory guard). Both legs of the A/B run in the same session on +# the same node class with the same everything except the PyAutoArray checkout +# prepended to PYTHONPATH: +# +# control -> d7c96762 (the branch's merge base = PyAutoArray main, #531 merged) +# feature -> feature/delaunay-nn-launch-latency +# +# The previous A/B on this cell (#531, merge base bcd15cd9) is recorded in +# results/notes/delaunay_walk_early_exit.md; its feature leg is kept as +# submit_breakdown_imaging_delaunay_nn_a100_hst_fp64_walk_early_exit and is not +# comparable to these two rows (different merge base, and this profiling +# checkout adds the split-point Sibson cut). +# +# The shared /mnt/ral/jnightin/PyAuto/PyAutoArray install is NOT touched — long +# science runs (subhalo validation, Euclid DR1) share it. The private checkouts +# live under /mnt/ral/jnightin/PyAuto_wt/delaunay-nn-launch-latency/ and are +# prepended to the PYTHONPATH that $AP_ROOT/activate.sh exports, so only +# PyAutoArray is overridden and Nerves/Fit/Galaxy/Lens stay on the shared tree. +# +# WITNESS. Compare the **params->H prefix** (regularization_matrix_prefix_s, +# equivalently Tri+interp + H) across the two legs — 144.79 ms unbatched / +# 24.4 ms per call at vmap 16 on the control (#531's post-merge DelaunayNN +# numbers); the task's witness is <= 60 ms unbatched and <= 11 ms at vmap 16. +# Do NOT judge on the differenced rows: the feature moves the split-point walk +# across a prefix boundary exactly as #531 did, so the individual rows shift +# even where the aggregate does not. +# +# The four-way --split-setup table now also carries the params->H interval cut +# at the split-point Sibson prefix ("Split-point Sibson" / +# "Regularization matrix (H, ConstantSplit assembly)", JSON +# split_sibson_prefix_s + regularization_split), which is what says how much of +# the residual is the second Sibson pass and how much is the 33-wide +# ConstantSplit assembly. Both legs must run the *same* profiling checkout for +# those rows to exist on both sides. +# +# --split-setup four-way split of the inversion-setup block (+ the +# split-point Sibson cut of the params->H interval) +# --vmap-batch 16 batched re-timing at the Nautilus reference n_batch + +#SBATCH -J bd_delnn_launch +#SBATCH --partition=gpu +#SBATCH --gres=gpu:1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=4 +#SBATCH --mem=64gb +#SBATCH --time=12:00:00 +#SBATCH -o output/output.%A.out +#SBATCH -e error/error.%A.err + +export AP_ROOT="$(cd "${SLURM_SUBMIT_DIR:-$(dirname "$0")}/../.." && pwd)" +echo "AP_ROOT=$AP_ROOT" +source $AP_ROOT/activate.sh + +# --- PyAutoArray A/B override (private checkout, shared install untouched) --- +AA_ROOT=/mnt/ral/jnightin/PyAuto_wt/delaunay-nn-launch-latency/PyAutoArray_feature +export PYTHONPATH=$AA_ROOT:$PYTHONPATH + +export JAX_PLATFORM_NAME=cuda +export JAX_PLATFORMS=cuda,cpu +export XLA_PYTHON_CLIENT_PREALLOCATE=false +export JAX_ENABLE_X64=True +export XLA_FLAGS="--xla_disable_hlo_passes=constant_folding" +export NUMBA_CACHE_DIR=/tmp/numba_cache +export MPLCONFIGDIR=/tmp/matplotlib + +# PYAUTO_SIBSON_QUERY_CHUNK is deliberately left UNSET on both A/B legs: the +# chunk sweep (submit_..._chunk_sweep) is where it varies, and a leg that set +# it would not be comparable to one that did not. The script records the +# effective SIBSON_QUERY_CHUNK either way under "sibson" in the result JSON. + +nvidia-smi + +# MIG backstop. A no-op since the fleet's MIG mode was confirmed Disabled +# (2026-09-05); kept because a silent removal re-opens the trap. Only bounces +# when the job was dispatched with --requeue. +source "$AP_ROOT/hpc/batch_gpu/_gpu_preflight.sh" + +echo "==========================================" +date +echo "Cell: likelihood_breakdown/imaging/delaunay_nn" +echo "Instrument: hst" +echo "Precision: fp64" +echo "A/B leg: launch_latency (feature/delaunay-nn-launch-latency)" +echo "Flags: --split-setup --vmap-batch 16 --config-name hpc_a100_fp64_launch_latency" +echo "Sibson chunk: PYAUTO_SIBSON_QUERY_CHUNK=${PYAUTO_SIBSON_QUERY_CHUNK:-unset (library default)}" + +echo "=== source revisions ===" +printf '%-14s %s\n' "PyAutoArray(AB)" "$(git -C $AA_ROOT rev-parse HEAD 2>&1)" +for r in PyAutoNerves PyAutoFit PyAutoGalaxy PyAutoLens; do + printf '%-14s %s\n' "$r" "$(git -C /mnt/ral/jnightin/PyAuto/$r rev-parse HEAD 2>&1)" +done +echo "AP_ROOT rev: $(git -C $AP_ROOT rev-parse HEAD 2>&1)" + +cd $AP_ROOT + +echo "=== import provenance ===" +python3 -c "import autoarray, autolens; print('autoarray.__file__ =', autoarray.__file__); print('autoarray.__version__ =', autoarray.__version__); print('autolens.__file__ =', autolens.__file__)" + +python3 -u scripts/imaging/likelihood_breakdown/delaunay_nn.py \ + --config-name hpc_a100_fp64_launch_latency \ + --split-setup \ + --vmap-batch 16 + +echo "Finished." +date diff --git a/hpc/batch_gpu/submit_breakdown_imaging_delaunay_nn_a100_hst_fp64_launch_latency_control b/hpc/batch_gpu/submit_breakdown_imaging_delaunay_nn_a100_hst_fp64_launch_latency_control new file mode 100644 index 000000000..67d84b133 --- /dev/null +++ b/hpc/batch_gpu/submit_breakdown_imaging_delaunay_nn_a100_hst_fp64_launch_latency_control @@ -0,0 +1,113 @@ +#!/bin/bash -l +# +# A100 likelihood-breakdown profile: imaging/delaunay_nn x hst x fp64 — PyAutoArray#532 +# A/B leg: **control**. +# +# Verification run for the DelaunayNN Sibson launch-latency change +# (PyAutoArray issue #532 / branch feature/delaunay-nn-launch-latency: candidate +# loop unrolled inside the cavity walk, data grid + split points located and +# interpolated in one concatenated Sibson pass, SIBSON_QUERY_CHUNK returned to +# being only a memory guard). Both legs of the A/B run in the same session on +# the same node class with the same everything except the PyAutoArray checkout +# prepended to PYTHONPATH: +# +# control -> d7c96762 (the branch's merge base = PyAutoArray main, #531 merged) +# feature -> feature/delaunay-nn-launch-latency +# +# This file replaces the #531 control leg of the same cell (merge base +# bcd15cd9); that A/B is recorded in results/notes/delaunay_walk_early_exit.md +# and its feature leg is kept as +# submit_breakdown_imaging_delaunay_nn_a100_hst_fp64_walk_early_exit. +# +# The shared /mnt/ral/jnightin/PyAuto/PyAutoArray install is NOT touched — long +# science runs (subhalo validation, Euclid DR1) share it. The private checkouts +# live under /mnt/ral/jnightin/PyAuto_wt/delaunay-nn-launch-latency/ and are +# prepended to the PYTHONPATH that $AP_ROOT/activate.sh exports, so only +# PyAutoArray is overridden and Nerves/Fit/Galaxy/Lens stay on the shared tree. +# +# WITNESS. Compare the **params->H prefix** (regularization_matrix_prefix_s, +# equivalently Tri+interp + H) across the two legs — 144.79 ms unbatched / +# 24.4 ms per call at vmap 16 on the control (#531's post-merge DelaunayNN +# numbers); the task's witness is <= 60 ms unbatched and <= 11 ms at vmap 16. +# Do NOT judge on the differenced rows: the feature moves the split-point walk +# across a prefix boundary exactly as #531 did, so the individual rows shift +# even where the aggregate does not. +# +# The four-way --split-setup table now also carries the params->H interval cut +# at the split-point Sibson prefix ("Split-point Sibson" / +# "Regularization matrix (H, ConstantSplit assembly)", JSON +# split_sibson_prefix_s + regularization_split), which is what says how much of +# the residual is the second Sibson pass and how much is the 33-wide +# ConstantSplit assembly. Both legs must run the *same* profiling checkout for +# those rows to exist on both sides. +# +# --split-setup four-way split of the inversion-setup block (+ the +# split-point Sibson cut of the params->H interval) +# --vmap-batch 16 batched re-timing at the Nautilus reference n_batch + +#SBATCH -J bd_delnn_ctrl +#SBATCH --partition=gpu +#SBATCH --gres=gpu:1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=4 +#SBATCH --mem=64gb +#SBATCH --time=12:00:00 +#SBATCH -o output/output.%A.out +#SBATCH -e error/error.%A.err + +export AP_ROOT="$(cd "${SLURM_SUBMIT_DIR:-$(dirname "$0")}/../.." && pwd)" +echo "AP_ROOT=$AP_ROOT" +source $AP_ROOT/activate.sh + +# --- PyAutoArray A/B override (private checkout, shared install untouched) --- +AA_ROOT=/mnt/ral/jnightin/PyAuto_wt/delaunay-nn-launch-latency/PyAutoArray_control +export PYTHONPATH=$AA_ROOT:$PYTHONPATH + +export JAX_PLATFORM_NAME=cuda +export JAX_PLATFORMS=cuda,cpu +export XLA_PYTHON_CLIENT_PREALLOCATE=false +export JAX_ENABLE_X64=True +export XLA_FLAGS="--xla_disable_hlo_passes=constant_folding" +export NUMBA_CACHE_DIR=/tmp/numba_cache +export MPLCONFIGDIR=/tmp/matplotlib + +# PYAUTO_SIBSON_QUERY_CHUNK is deliberately left UNSET on both A/B legs: the +# chunk sweep (submit_..._chunk_sweep) is where it varies, and a leg that set +# it would not be comparable to one that did not. The script records the +# effective SIBSON_QUERY_CHUNK either way under "sibson" in the result JSON. + +nvidia-smi + +# MIG backstop. A no-op since the fleet's MIG mode was confirmed Disabled +# (2026-09-05); kept because a silent removal re-opens the trap. Only bounces +# when the job was dispatched with --requeue. +source "$AP_ROOT/hpc/batch_gpu/_gpu_preflight.sh" + +echo "==========================================" +date +echo "Cell: likelihood_breakdown/imaging/delaunay_nn" +echo "Instrument: hst" +echo "Precision: fp64" +echo "A/B leg: control (merge base d7c96762)" +echo "Flags: --split-setup --vmap-batch 16 --config-name hpc_a100_fp64_launch_latency_control" +echo "Sibson chunk: PYAUTO_SIBSON_QUERY_CHUNK=${PYAUTO_SIBSON_QUERY_CHUNK:-unset (library default)}" + +echo "=== source revisions ===" +printf '%-14s %s\n' "PyAutoArray(AB)" "$(git -C $AA_ROOT rev-parse HEAD 2>&1)" +for r in PyAutoNerves PyAutoFit PyAutoGalaxy PyAutoLens; do + printf '%-14s %s\n' "$r" "$(git -C /mnt/ral/jnightin/PyAuto/$r rev-parse HEAD 2>&1)" +done +echo "AP_ROOT rev: $(git -C $AP_ROOT rev-parse HEAD 2>&1)" + +cd $AP_ROOT + +echo "=== import provenance ===" +python3 -c "import autoarray, autolens; print('autoarray.__file__ =', autoarray.__file__); print('autoarray.__version__ =', autoarray.__version__); print('autolens.__file__ =', autolens.__file__)" + +python3 -u scripts/imaging/likelihood_breakdown/delaunay_nn.py \ + --config-name hpc_a100_fp64_launch_latency_control \ + --split-setup \ + --vmap-batch 16 + +echo "Finished." +date diff --git a/hpc/batch_gpu/submit_runtime_imaging_delaunay_nn_a100_hst_fp64_control b/hpc/batch_gpu/submit_runtime_imaging_delaunay_nn_a100_hst_fp64_control new file mode 100644 index 000000000..8fae1c198 --- /dev/null +++ b/hpc/batch_gpu/submit_runtime_imaging_delaunay_nn_a100_hst_fp64_control @@ -0,0 +1,64 @@ +#!/bin/bash -l +# +# A100 likelihood-runtime profile: imaging/delaunay_nn x hst x fp64 — PyAutoArray#532 +# A/B leg: **control**. Single-JIT whole-likelihood cost, the end-to-end companion +# to submit_breakdown_imaging_delaunay_nn_a100_hst_fp64_control. +# +# Same private-checkout PYTHONPATH override as the breakdown A/B legs; the +# shared /mnt/ral/jnightin/PyAuto install is untouched. Phase A (--vmap-probe) +# is skipped: only the full single-JIT + vmap timing is wanted here, and this +# cell's probe is known to over-recommend batch 64 (cuFFT batched-plan failure, +# job 342280) — with no probe JSON present the script falls back to the +# VMAP_BATCH table (16), which is the number both legs must share. +# +# control -> d7c96762 (the branch's merge base = PyAutoArray main) +# feature -> feature/delaunay-nn-launch-latency +# +# PYAUTO_SIBSON_QUERY_CHUNK is left unset on both legs (the chunk sweep is a +# separate submit); the result JSON records the effective chunk either way. + +#SBATCH -J rt_delaunay_nn_control +#SBATCH --partition=gpu +#SBATCH --gres=gpu:1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=4 +#SBATCH --mem=64gb +#SBATCH --time=3:00:00 +#SBATCH -o output/output.%A.out +#SBATCH -e error/error.%A.err + +export AP_ROOT="$(cd "${SLURM_SUBMIT_DIR:-$(dirname "$0")}/../.." && pwd)" +echo "AP_ROOT=$AP_ROOT" +source $AP_ROOT/activate.sh + +AA_ROOT=/mnt/ral/jnightin/PyAuto_wt/delaunay-nn-launch-latency/PyAutoArray_control +export PYTHONPATH=$AA_ROOT:$PYTHONPATH + +export JAX_PLATFORM_NAME=cuda +export JAX_PLATFORMS=cuda,cpu +export XLA_PYTHON_CLIENT_PREALLOCATE=false +export JAX_ENABLE_X64=True +export NUMBA_CACHE_DIR=/tmp/numba_cache +export MPLCONFIGDIR=/tmp/matplotlib + +nvidia-smi + +# MIG backstop; no-op since 2026-09-05, only bounces under --requeue. +source "$AP_ROOT/hpc/batch_gpu/_gpu_preflight.sh" + +echo "==========================================" +date +echo "Cell: likelihood_runtime/imaging/delaunay_nn" +echo "A/B leg: control (merge base d7c96762)" +echo "Sibson chunk: PYAUTO_SIBSON_QUERY_CHUNK=${PYAUTO_SIBSON_QUERY_CHUNK:-unset (library default)}" +printf 'PyAutoArray(AB) %s\n' "$(git -C $AA_ROOT rev-parse HEAD 2>&1)" + +cd $AP_ROOT +python3 -c "import autoarray; print('autoarray.__file__ =', autoarray.__file__)" + +python3 -u scripts/imaging/likelihood_runtime/delaunay_nn.py \ + --instrument hst \ + --config-name hpc_a100_fp64_launch_latency_control + +echo "Finished." +date diff --git a/hpc/batch_gpu/submit_runtime_imaging_delaunay_nn_a100_hst_fp64_launch_latency b/hpc/batch_gpu/submit_runtime_imaging_delaunay_nn_a100_hst_fp64_launch_latency new file mode 100644 index 000000000..f167a582c --- /dev/null +++ b/hpc/batch_gpu/submit_runtime_imaging_delaunay_nn_a100_hst_fp64_launch_latency @@ -0,0 +1,64 @@ +#!/bin/bash -l +# +# A100 likelihood-runtime profile: imaging/delaunay_nn x hst x fp64 — PyAutoArray#532 +# A/B leg: **launch_latency** (feature). Single-JIT whole-likelihood cost, the end-to-end companion +# to submit_breakdown_imaging_delaunay_nn_a100_hst_fp64_launch_latency. +# +# Same private-checkout PYTHONPATH override as the breakdown A/B legs; the +# shared /mnt/ral/jnightin/PyAuto install is untouched. Phase A (--vmap-probe) +# is skipped: only the full single-JIT + vmap timing is wanted here, and this +# cell's probe is known to over-recommend batch 64 (cuFFT batched-plan failure, +# job 342280) — with no probe JSON present the script falls back to the +# VMAP_BATCH table (16), which is the number both legs must share. +# +# control -> d7c96762 (the branch's merge base = PyAutoArray main) +# feature -> feature/delaunay-nn-launch-latency +# +# PYAUTO_SIBSON_QUERY_CHUNK is left unset on both legs (the chunk sweep is a +# separate submit); the result JSON records the effective chunk either way. + +#SBATCH -J rt_delaunay_nn_launch +#SBATCH --partition=gpu +#SBATCH --gres=gpu:1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=4 +#SBATCH --mem=64gb +#SBATCH --time=3:00:00 +#SBATCH -o output/output.%A.out +#SBATCH -e error/error.%A.err + +export AP_ROOT="$(cd "${SLURM_SUBMIT_DIR:-$(dirname "$0")}/../.." && pwd)" +echo "AP_ROOT=$AP_ROOT" +source $AP_ROOT/activate.sh + +AA_ROOT=/mnt/ral/jnightin/PyAuto_wt/delaunay-nn-launch-latency/PyAutoArray_feature +export PYTHONPATH=$AA_ROOT:$PYTHONPATH + +export JAX_PLATFORM_NAME=cuda +export JAX_PLATFORMS=cuda,cpu +export XLA_PYTHON_CLIENT_PREALLOCATE=false +export JAX_ENABLE_X64=True +export NUMBA_CACHE_DIR=/tmp/numba_cache +export MPLCONFIGDIR=/tmp/matplotlib + +nvidia-smi + +# MIG backstop; no-op since 2026-09-05, only bounces under --requeue. +source "$AP_ROOT/hpc/batch_gpu/_gpu_preflight.sh" + +echo "==========================================" +date +echo "Cell: likelihood_runtime/imaging/delaunay_nn" +echo "A/B leg: launch_latency (feature/delaunay-nn-launch-latency)" +echo "Sibson chunk: PYAUTO_SIBSON_QUERY_CHUNK=${PYAUTO_SIBSON_QUERY_CHUNK:-unset (library default)}" +printf 'PyAutoArray(AB) %s\n' "$(git -C $AA_ROOT rev-parse HEAD 2>&1)" + +cd $AP_ROOT +python3 -c "import autoarray; print('autoarray.__file__ =', autoarray.__file__)" + +python3 -u scripts/imaging/likelihood_runtime/delaunay_nn.py \ + --instrument hst \ + --config-name hpc_a100_fp64_launch_latency + +echo "Finished." +date diff --git a/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_chunk1024.json b/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_chunk1024.json new file mode 100644 index 000000000..4be1fca6a --- /dev/null +++ b/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_chunk1024.json @@ -0,0 +1,75 @@ +{ + "autolens_version": "2026.8.17.1", + "device": { + "backend": "gpu", + "device": "cuda:0", + "xla_flags": "--xla_disable_hlo_passes=constant_folding --xla_gpu_autotune_level=0", + "omp_num_threads": null, + "cpu_count": 124, + "nvidia_smi": "NVIDIA A100 80GB PCIe, 41495 MiB, 81920 MiB" + }, + "instrument": "hst", + "configuration": { + "pixel_scale_arcsec": 0.05, + "mask_radius_arcsec": 3.5, + "image_pixels_masked": 15361, + "over_sampled_pixels": 17980, + "delaunay_vertices": 1500, + "edge_zeroed_pixels": 0, + "inversion_path": "dense" + }, + "sibson": { + "PYAUTO_SIBSON_QUERY_CHUNK": "1024", + "query_chunk": 1024, + "max_cavity_triangles": 32, + "max_neighbors": 32, + "mesh_query_chunk": 1024, + "mesh_max_cavity_triangles": 32, + "mesh_max_neighbors": 32 + }, + "steps": { + "Ray-trace data grid": 0.00015423318836838007, + "Ray-trace mesh grid": 0.00025799749419093133, + "Lens light images (pre-PSF)": 0.00013769229408353568, + "Blurred image (PSF convolution)": 0.0008483751909807325, + "Profile-subtracted image": 0.00014305920340120792, + "Inversion setup (steps 5-8 combined)": 0.038230485492385925, + "Data vector (D)": 0.0003124712035059929, + "Curvature matrix (F)": 0.004834010917693377, + "Regularization matrix (H)": 0.01020407958421856, + "Regularized reconstruction": 0.030783028388395906, + "Mapped recon + log evidence": 0.0022159656975418328 + }, + "total_step_by_step": 0.08812139865476638, + "regularization_matrix_prefix_s": 0.03532371879555285, + "interpolator_prefix_s": 0.02511963921133429, + "split_sibson_prefix_s": 0.025481182103976607, + "regularization_split": { + "Split-point Sibson": 0.0003615428926423171, + "Regularization matrix (H, ConstantSplit assembly)": 0.009842536691576243 + }, + "setup_split": { + "Border relocation": 0.0010811569867655636, + "Triangulation + interpolation": 0.024038482224568725, + "Mapping matrix": -4.216590896248887e-05, + "Blurred mapping matrix (PSF)": 0.008481317502446472 + }, + "vmap_batch": 16, + "steps_vmap_per_call": { + "Inversion setup (steps 5-8 combined)": 0.01577441432455089, + "Regularization matrix (H)": 0.01250513866834808 + }, + "setup_split_vmap": { + "Border relocation": 6.533263076562435e-05, + "Triangulation + interpolation": 0.006996601943683345, + "Mapping matrix": 8.30817807582207e-05, + "Blurred mapping matrix (PSF)": 0.008348123551695608 + }, + "regularization_matrix_prefix_vmap_per_call_s": 0.01956707324279705, + "interpolator_prefix_vmap_per_call_s": 0.007061934574448969, + "split_sibson_prefix_vmap_per_call_s": 0.007009946556354407, + "regularization_split_vmap": { + "Split-point Sibson": -5.198801809456165e-05, + "Regularization matrix (H, ConstantSplit assembly)": 0.012557126686442644 + } +} \ No newline at end of file diff --git a/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_chunk1024.png b/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_chunk1024.png new file mode 100644 index 000000000..2b3be5183 Binary files /dev/null and b/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_chunk1024.png differ diff --git a/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_chunk2048.json b/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_chunk2048.json new file mode 100644 index 000000000..052f0406a --- /dev/null +++ b/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_chunk2048.json @@ -0,0 +1,75 @@ +{ + "autolens_version": "2026.8.17.1", + "device": { + "backend": "gpu", + "device": "cuda:0", + "xla_flags": "--xla_disable_hlo_passes=constant_folding --xla_gpu_autotune_level=0", + "omp_num_threads": null, + "cpu_count": 124, + "nvidia_smi": "NVIDIA A100 80GB PCIe, 41495 MiB, 81920 MiB" + }, + "instrument": "hst", + "configuration": { + "pixel_scale_arcsec": 0.05, + "mask_radius_arcsec": 3.5, + "image_pixels_masked": 15361, + "over_sampled_pixels": 17980, + "delaunay_vertices": 1500, + "edge_zeroed_pixels": 0, + "inversion_path": "dense" + }, + "sibson": { + "PYAUTO_SIBSON_QUERY_CHUNK": "2048", + "query_chunk": 2048, + "max_cavity_triangles": 32, + "max_neighbors": 32, + "mesh_query_chunk": 2048, + "mesh_max_cavity_triangles": 32, + "mesh_max_neighbors": 32 + }, + "steps": { + "Ray-trace data grid": 0.00017858201172202826, + "Ray-trace mesh grid": 0.00016824111808091402, + "Lens light images (pre-PSF)": 0.00014161630533635617, + "Blurred image (PSF convolution)": 0.0008371551986783743, + "Profile-subtracted image": 0.00015230320859700442, + "Inversion setup (steps 5-8 combined)": 0.030364763713441788, + "Data vector (D)": 0.00034742509014904497, + "Curvature matrix (F)": 0.004843506799079478, + "Regularization matrix (H)": 0.01020721672102809, + "Regularized reconstruction": 0.030865628807805478, + "Mapped recon + log evidence": 0.002280585188418627 + }, + "total_step_by_step": 0.08038702416233719, + "regularization_matrix_prefix_s": 0.02721772431395948, + "interpolator_prefix_s": 0.01701050759293139, + "split_sibson_prefix_s": 0.020696922019124032, + "regularization_split": { + "Split-point Sibson": 0.0036864144261926426, + "Regularization matrix (H, ConstantSplit assembly)": 0.006520802294835447 + }, + "setup_split": { + "Border relocation": 0.0011690844781696796, + "Triangulation + interpolation": 0.01584142311476171, + "Mapping matrix": 0.003715232200920581, + "Blurred mapping matrix (PSF)": 0.008650905406102541 + }, + "vmap_batch": 16, + "steps_vmap_per_call": { + "Inversion setup (steps 5-8 combined)": 0.017888259218307213, + "Regularization matrix (H)": 0.01252487623714842 + }, + "setup_split_vmap": { + "Border relocation": 6.430382491089404e-05, + "Triangulation + interpolation": 0.006537624831253197, + "Mapping matrix": 0.00011037081858376093, + "Blurred mapping matrix (PSF)": 0.008406378912332002 + }, + "regularization_matrix_prefix_vmap_per_call_s": 0.01912680489331251, + "interpolator_prefix_vmap_per_call_s": 0.006601928656164091, + "split_sibson_prefix_vmap_per_call_s": 0.00906878200621577, + "regularization_split_vmap": { + "Split-point Sibson": 0.0024668533500516792, + "Regularization matrix (H, ConstantSplit assembly)": 0.010058022887096741 + } +} \ No newline at end of file diff --git a/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_chunk2048.png b/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_chunk2048.png new file mode 100644 index 000000000..2859e3655 Binary files /dev/null and b/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_chunk2048.png differ diff --git a/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_chunk4096.json b/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_chunk4096.json new file mode 100644 index 000000000..5a93d01c4 --- /dev/null +++ b/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_chunk4096.json @@ -0,0 +1,75 @@ +{ + "autolens_version": "2026.8.17.1", + "device": { + "backend": "gpu", + "device": "cuda:0", + "xla_flags": "--xla_disable_hlo_passes=constant_folding --xla_gpu_autotune_level=0", + "omp_num_threads": null, + "cpu_count": 124, + "nvidia_smi": "NVIDIA A100 80GB PCIe, 41495 MiB, 81920 MiB" + }, + "instrument": "hst", + "configuration": { + "pixel_scale_arcsec": 0.05, + "mask_radius_arcsec": 3.5, + "image_pixels_masked": 15361, + "over_sampled_pixels": 17980, + "delaunay_vertices": 1500, + "edge_zeroed_pixels": 0, + "inversion_path": "dense" + }, + "sibson": { + "PYAUTO_SIBSON_QUERY_CHUNK": "4096", + "query_chunk": 4096, + "max_cavity_triangles": 32, + "max_neighbors": 32, + "mesh_query_chunk": 4096, + "mesh_max_cavity_triangles": 32, + "mesh_max_neighbors": 32 + }, + "steps": { + "Ray-trace data grid": 0.00017484501004219055, + "Ray-trace mesh grid": 0.0001391771947965026, + "Lens light images (pre-PSF)": 0.00014347019605338575, + "Blurred image (PSF convolution)": 0.0008634342113509775, + "Profile-subtracted image": 0.00011680941097438335, + "Inversion setup (steps 5-8 combined)": 0.027436923095956446, + "Data vector (D)": 0.0003299400908872485, + "Curvature matrix (F)": 0.004923265404067933, + "Regularization matrix (H)": 0.010306526208296418, + "Regularized reconstruction": 0.030681254900991918, + "Mapped recon + log evidence": 0.002266249293461442 + }, + "total_step_by_step": 0.07738189501687885, + "regularization_matrix_prefix_s": 0.02466363471467048, + "interpolator_prefix_s": 0.014357108506374061, + "split_sibson_prefix_s": 0.014411408104933798, + "regularization_split": { + "Split-point Sibson": 5.4299598559737206e-05, + "Regularization matrix (H, ConstantSplit assembly)": 0.010252226609736681 + }, + "setup_split": { + "Border relocation": 0.0010606890078634023, + "Triangulation + interpolation": 0.013296419498510658, + "Mapping matrix": 0.0038190915016457425, + "Blurred mapping matrix (PSF)": 0.00458048840519041 + }, + "vmap_batch": 16, + "steps_vmap_per_call": { + "Inversion setup (steps 5-8 combined)": 0.016444710818177555, + "Regularization matrix (H)": 0.010053458980110008 + }, + "setup_split_vmap": { + "Border relocation": 6.394876836566255e-05, + "Triangulation + interpolation": 0.006333736907981802, + "Mapping matrix": 0.00012005826720269414, + "Blurred mapping matrix (PSF)": 0.008361651781888212 + }, + "regularization_matrix_prefix_vmap_per_call_s": 0.016451144656457473, + "interpolator_prefix_vmap_per_call_s": 0.006397685676347464, + "split_sibson_prefix_vmap_per_call_s": 0.006423418663325719, + "regularization_split_vmap": { + "Split-point Sibson": 2.5732986978254865e-05, + "Regularization matrix (H, ConstantSplit assembly)": 0.010027725993131753 + } +} \ No newline at end of file diff --git a/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_chunk4096.png b/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_chunk4096.png new file mode 100644 index 000000000..6eedb6cc1 Binary files /dev/null and b/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_chunk4096.png differ diff --git a/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_chunk512.json b/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_chunk512.json new file mode 100644 index 000000000..5bba3ac4f --- /dev/null +++ b/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_chunk512.json @@ -0,0 +1,75 @@ +{ + "autolens_version": "2026.8.17.1", + "device": { + "backend": "gpu", + "device": "cuda:0", + "xla_flags": "--xla_disable_hlo_passes=constant_folding --xla_gpu_autotune_level=0", + "omp_num_threads": null, + "cpu_count": 124, + "nvidia_smi": "NVIDIA A100 80GB PCIe, 41495 MiB, 81920 MiB" + }, + "instrument": "hst", + "configuration": { + "pixel_scale_arcsec": 0.05, + "mask_radius_arcsec": 3.5, + "image_pixels_masked": 15361, + "over_sampled_pixels": 17980, + "delaunay_vertices": 1500, + "edge_zeroed_pixels": 0, + "inversion_path": "dense" + }, + "sibson": { + "PYAUTO_SIBSON_QUERY_CHUNK": "512", + "query_chunk": 512, + "max_cavity_triangles": 32, + "max_neighbors": 32, + "mesh_query_chunk": 512, + "mesh_max_cavity_triangles": 32, + "mesh_max_neighbors": 32 + }, + "steps": { + "Ray-trace data grid": 0.00016558608040213586, + "Ray-trace mesh grid": 0.00016404699999839067, + "Lens light images (pre-PSF)": 0.0001410800963640213, + "Blurred image (PSF convolution)": 0.0008301343070343137, + "Profile-subtracted image": 0.00013937831390649081, + "Inversion setup (steps 5-8 combined)": 0.054924222803674635, + "Data vector (D)": 0.00032864811364561317, + "Curvature matrix (F)": 0.0048281439114362, + "Regularization matrix (H)": 0.010850800108164547, + "Regularized reconstruction": 0.030726613593287766, + "Mapped recon + log evidence": 0.0022821352118626236 + }, + "total_step_by_step": 0.10538078953977674, + "regularization_matrix_prefix_s": 0.052204915205948055, + "interpolator_prefix_s": 0.04135411509778351, + "split_sibson_prefix_s": 0.041616532602347434, + "regularization_split": { + "Split-point Sibson": 0.00026241750456392626, + "Regularization matrix (H, ConstantSplit assembly)": 0.010588382603600621 + }, + "setup_split": { + "Border relocation": 0.0010920888977125286, + "Triangulation + interpolation": 0.04026202620007098, + "Mapping matrix": 0.0004652242874726639, + "Blurred mapping matrix (PSF)": 0.008294324530288576 + }, + "vmap_batch": 16, + "steps_vmap_per_call": { + "Inversion setup (steps 5-8 combined)": 0.016959620868146886, + "Regularization matrix (H)": 0.009933905967045575 + }, + "setup_split_vmap": { + "Border relocation": 6.434451352106407e-05, + "Triangulation + interpolation": 0.008244804812420625, + "Mapping matrix": 3.903353645000676e-05, + "Blurred mapping matrix (PSF)": 0.008420113136526197 + }, + "regularization_matrix_prefix_vmap_per_call_s": 0.018243055292987264, + "interpolator_prefix_vmap_per_call_s": 0.008309149325941689, + "split_sibson_prefix_vmap_per_call_s": 0.0082290498437942, + "regularization_split_vmap": { + "Split-point Sibson": -8.009948214748874e-05, + "Regularization matrix (H, ConstantSplit assembly)": 0.010014005449193064 + } +} \ No newline at end of file diff --git a/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_chunk512.png b/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_chunk512.png new file mode 100644 index 000000000..6a122fe01 Binary files /dev/null and b/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_chunk512.png differ diff --git a/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_launch_latency.json b/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_launch_latency.json new file mode 100644 index 000000000..2c515652b --- /dev/null +++ b/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_launch_latency.json @@ -0,0 +1,75 @@ +{ + "autolens_version": "2026.8.17.1", + "device": { + "backend": "gpu", + "device": "cuda:0", + "xla_flags": "--xla_disable_hlo_passes=constant_folding --xla_gpu_autotune_level=0", + "omp_num_threads": null, + "cpu_count": 124, + "nvidia_smi": "NVIDIA A100 80GB PCIe, 41495 MiB, 81920 MiB" + }, + "instrument": "hst", + "configuration": { + "pixel_scale_arcsec": 0.05, + "mask_radius_arcsec": 3.5, + "image_pixels_masked": 15361, + "over_sampled_pixels": 17980, + "delaunay_vertices": 1500, + "edge_zeroed_pixels": 0, + "inversion_path": "dense" + }, + "sibson": { + "PYAUTO_SIBSON_QUERY_CHUNK": null, + "query_chunk": 4096, + "max_cavity_triangles": 32, + "max_neighbors": 32, + "mesh_query_chunk": 4096, + "mesh_max_cavity_triangles": 32, + "mesh_max_neighbors": 32 + }, + "steps": { + "Ray-trace data grid": 0.00016866999212652444, + "Ray-trace mesh grid": 0.00015950507950037717, + "Lens light images (pre-PSF)": 0.00013269931077957152, + "Blurred image (PSF convolution)": 0.000807152409106493, + "Profile-subtracted image": 0.00012276130728423597, + "Inversion setup (steps 5-8 combined)": 0.02748184888623655, + "Data vector (D)": 0.0003359609981998801, + "Curvature matrix (F)": 0.004811750981025398, + "Regularization matrix (H)": 0.013768400601111352, + "Regularized reconstruction": 0.03073617338668555, + "Mapped recon + log evidence": 0.002190651698037982 + }, + "total_step_by_step": 0.08071557465009391, + "regularization_matrix_prefix_s": 0.028212246601469815, + "interpolator_prefix_s": 0.014443846000358463, + "split_sibson_prefix_s": 0.014242498995736241, + "regularization_split": { + "Split-point Sibson": -0.00020134700462222203, + "Regularization matrix (H, ConstantSplit assembly)": 0.013969747605733574 + }, + "setup_split": { + "Border relocation": 0.0010788040002807976, + "Triangulation + interpolation": 0.013365042000077666, + "Mapping matrix": 0.0001024774042889462, + "Blurred mapping matrix (PSF)": 0.008352953987196089 + }, + "vmap_batch": 16, + "steps_vmap_per_call": { + "Inversion setup (steps 5-8 combined)": 0.017673922318499534, + "Regularization matrix (H)": 0.00749125627335161 + }, + "setup_split_vmap": { + "Border relocation": 6.434513779822737e-05, + "Triangulation + interpolation": 0.008879176044138147, + "Mapping matrix": -0.0024034101443248792, + "Blurred mapping matrix (PSF)": 0.010807657918485349 + }, + "regularization_matrix_prefix_vmap_per_call_s": 0.016434777455287986, + "interpolator_prefix_vmap_per_call_s": 0.008943521181936375, + "split_sibson_prefix_vmap_per_call_s": 0.006433532337541692, + "regularization_split_vmap": { + "Split-point Sibson": -0.002509988844394683, + "Regularization matrix (H, ConstantSplit assembly)": 0.010001245117746294 + } +} \ No newline at end of file diff --git a/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_launch_latency.png b/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_launch_latency.png new file mode 100644 index 000000000..92d1997f2 Binary files /dev/null and b/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_launch_latency.png differ diff --git a/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_launch_latency_chunk256.json b/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_launch_latency_chunk256.json new file mode 100644 index 000000000..646f84877 --- /dev/null +++ b/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_launch_latency_chunk256.json @@ -0,0 +1,75 @@ +{ + "autolens_version": "2026.8.17.1", + "device": { + "backend": "gpu", + "device": "cuda:0", + "xla_flags": "--xla_disable_hlo_passes=constant_folding --xla_gpu_autotune_level=0", + "omp_num_threads": null, + "cpu_count": 124, + "nvidia_smi": "NVIDIA A100 80GB PCIe, 41495 MiB, 81920 MiB" + }, + "instrument": "hst", + "configuration": { + "pixel_scale_arcsec": 0.05, + "mask_radius_arcsec": 3.5, + "image_pixels_masked": 15361, + "over_sampled_pixels": 17980, + "delaunay_vertices": 1500, + "edge_zeroed_pixels": 0, + "inversion_path": "dense" + }, + "sibson": { + "PYAUTO_SIBSON_QUERY_CHUNK": null, + "query_chunk": 256, + "max_cavity_triangles": 32, + "max_neighbors": 32, + "mesh_query_chunk": 256, + "mesh_max_cavity_triangles": 32, + "mesh_max_neighbors": 32 + }, + "steps": { + "Ray-trace data grid": 0.00018331699538975954, + "Ray-trace mesh grid": 0.00019325988832861186, + "Lens light images (pre-PSF)": 0.0001313793007284403, + "Blurred image (PSF convolution)": 0.0008148874156177043, + "Profile-subtracted image": 0.00012932929676026105, + "Inversion setup (steps 5-8 combined)": 0.08826995380222798, + "Data vector (D)": 0.00032921519596129656, + "Curvature matrix (F)": 0.004849728802219034, + "Regularization matrix (H)": 0.014725465490482745, + "Regularized reconstruction": 0.030718375695869327, + "Mapped recon + log evidence": 0.0022180565865710378 + }, + "total_step_by_step": 0.1425629684701562, + "regularization_matrix_prefix_s": 0.08828681579325348, + "interpolator_prefix_s": 0.07356135030277074, + "split_sibson_prefix_s": 0.0745569997932762, + "regularization_split": { + "Split-point Sibson": 0.000995649490505457, + "Regularization matrix (H, ConstantSplit assembly)": 0.013729815999977288 + }, + "setup_split": { + "Border relocation": 0.001080550905317068, + "Triangulation + interpolation": 0.07248079939745367, + "Mapping matrix": 0.0012596811167895766, + "Blurred mapping matrix (PSF)": 0.007429173169657591 + }, + "vmap_batch": 16, + "steps_vmap_per_call": { + "Inversion setup (steps 5-8 combined)": 0.019239204018958844, + "Regularization matrix (H)": 0.01258424959378317 + }, + "setup_split_vmap": { + "Border relocation": 8.873737388057635e-05, + "Triangulation + interpolation": 0.01039904342032969, + "Mapping matrix": 0.002579292599693873, + "Blurred mapping matrix (PSF)": 0.008347076112113426 + }, + "regularization_matrix_prefix_vmap_per_call_s": 0.023072030387993436, + "interpolator_prefix_vmap_per_call_s": 0.010487780794210266, + "split_sibson_prefix_vmap_per_call_s": 0.012919878780667204, + "regularization_split_vmap": { + "Split-point Sibson": 0.002432097986456938, + "Regularization matrix (H, ConstantSplit assembly)": 0.010152151607326233 + } +} \ No newline at end of file diff --git a/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_launch_latency_chunk256.png b/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_launch_latency_chunk256.png new file mode 100644 index 000000000..08c980990 Binary files /dev/null and b/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_launch_latency_chunk256.png differ diff --git a/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_launch_latency_control.json b/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_launch_latency_control.json new file mode 100644 index 000000000..b5cac64a0 --- /dev/null +++ b/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_launch_latency_control.json @@ -0,0 +1,75 @@ +{ + "autolens_version": "2026.8.17.1", + "device": { + "backend": "gpu", + "device": "cuda:0", + "xla_flags": "--xla_disable_hlo_passes=constant_folding --xla_gpu_autotune_level=0", + "omp_num_threads": null, + "cpu_count": 124, + "nvidia_smi": "NVIDIA A100 80GB PCIe, 41495 MiB, 81920 MiB" + }, + "instrument": "hst", + "configuration": { + "pixel_scale_arcsec": 0.05, + "mask_radius_arcsec": 3.5, + "image_pixels_masked": 15361, + "over_sampled_pixels": 17980, + "delaunay_vertices": 1500, + "edge_zeroed_pixels": 0, + "inversion_path": "dense" + }, + "sibson": { + "PYAUTO_SIBSON_QUERY_CHUNK": null, + "query_chunk": 256, + "max_cavity_triangles": 32, + "max_neighbors": 32, + "mesh_query_chunk": 256, + "mesh_max_cavity_triangles": 32, + "mesh_max_neighbors": 32 + }, + "steps": { + "Ray-trace data grid": 0.00016944108065217734, + "Ray-trace mesh grid": 0.00016055398155003786, + "Lens light images (pre-PSF)": 0.00013826219365000724, + "Blurred image (PSF convolution)": 0.0008539632894098758, + "Profile-subtracted image": 0.00013835930731147527, + "Inversion setup (steps 5-8 combined)": 0.11155231418088078, + "Data vector (D)": 0.00034814300015568734, + "Curvature matrix (F)": 0.004803768103010952, + "Regularization matrix (H)": 0.04510317989625037, + "Regularized reconstruction": 0.030819932091981174, + "Mapped recon + log evidence": 0.0022691522957757116 + }, + "total_step_by_step": 0.19635706942062825, + "regularization_matrix_prefix_s": 0.1439034678041935, + "interpolator_prefix_s": 0.09880028790794312, + "split_sibson_prefix_s": 0.13415851339232177, + "regularization_split": { + "Split-point Sibson": 0.03535822548437864, + "Regularization matrix (H, ConstantSplit assembly)": 0.009744954411871731 + }, + "setup_split": { + "Border relocation": 0.0015263534151017666, + "Triangulation + interpolation": 0.09727393449284136, + "Mapping matrix": -0.0009093439904972883, + "Blurred mapping matrix (PSF)": 0.008942755893804138 + }, + "vmap_batch": 16, + "steps_vmap_per_call": { + "Inversion setup (steps 5-8 combined)": 0.02036440764350118, + "Regularization matrix (H)": 0.012622409319737926 + }, + "setup_split_vmap": { + "Border relocation": 9.202954388456419e-05, + "Triangulation + interpolation": 0.011602595300064421, + "Mapping matrix": 7.437069871230051e-05, + "Blurred mapping matrix (PSF)": 0.01070463745854795 + }, + "regularization_matrix_prefix_vmap_per_call_s": 0.024317034163686913, + "interpolator_prefix_vmap_per_call_s": 0.011694624843948986, + "split_sibson_prefix_vmap_per_call_s": 0.014432050731556956, + "regularization_split_vmap": { + "Split-point Sibson": 0.0027374258876079697, + "Regularization matrix (H, ConstantSplit assembly)": 0.009884983432129957 + } +} \ No newline at end of file diff --git a/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_launch_latency_control.png b/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_launch_latency_control.png new file mode 100644 index 000000000..d063bd8e1 Binary files /dev/null and b/results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_launch_latency_control.png differ diff --git a/results/notes/delaunay_nn_launch_latency.md b/results/notes/delaunay_nn_launch_latency.md new file mode 100644 index 000000000..edb8b6e91 --- /dev/null +++ b/results/notes/delaunay_nn_launch_latency.md @@ -0,0 +1,277 @@ +# DelaunayNN Sibson — launch-latency cuts + query-chunk sweep (A100, 2026-09-08) + +PyAutoArray issue #532 / PR #533, branch `feature/delaunay-nn-launch-latency`, against its +merge base `d7c96762` (PyAutoArray `main`, #531 merged). Two commits are measured here: +`9e9a7d50` (Phase A at the old chunk 256) and `e5d31d74` (Phase A **as shipped**, with the +new `SIBSON_QUERY_CHUNK = 4096` default this A/B chose). + +## What changed + +Three cuts to `autoarray/inversion/mesh/interpolator/sibson.py`, none of which change a +number: + +1. **The 3-trip candidate-edge `fori_loop` inside the cavity walk is unrolled at trace + time** when `_sibson_unroll_candidates()` is true — `PYAUTO_SIBSON_UNROLL_CANDIDATES` + wins if set, otherwise `jax.default_backend() != "cpu"`. Same `add_candidate` calls for + edges 0, 1, 2 in that order, so the same cavity insertion order, the same stencil column + order and the same floating-point summation order. It removes ~28 % of the kernel + launches per chunk (1,244 → 892). CPU stays rolled (measured ~8-9 % faster there). +2. **`jax_delaunay_nn` locates and interpolates the data grid and the 6,000 `ConstantSplit` + cross points in one concatenated Sibson pass**, computing the circumcircles once — the + #531 pattern, now applied to the Sibson path. +3. **`SIBSON_QUERY_CHUNK` is a memory guard again**, with `PYAUTO_SIBSON_QUERY_CHUNK` as an + import-time override so it can be swept without editing source. The sweep below set the + new default. + +## Verdict + +On the A100, the HST / Hilbert-1500 / MGE-60 / `ConstantSplit` **DelaunayNN** imaging +likelihood's **params→H prefix — the aggregate invariant to the attribution shift below — +falls 143.90 → 28.21 ms unbatched (−115.7 ms, 5.10×)** and 24.32 → 16.44 ms per call at +`vmap` 16 (−7.9 ms, 1.48×). Whole-likelihood single-JIT runtime goes **201.32 → 76.32 ms +(2.64×)** and the step-by-step sum 196.36 → 80.72 ms; at `vmap` 16 the whole likelihood +goes 58.02 → 52.62 ms per call (1.10×). The `29144.581944` pin held **exactly on every leg +and every chunk**. + +**The witness is half met.** Its unbatched half (params→H ≤ 60 ms) is met at every chunk +≥ 512 — 28.21 ms at the shipped default, 24.66 ms on the sweep's own 4096 row. Its +batched half (≤ 11 ms per call at `vmap` 16) is **not** met: 16.44 ms. The new +split-Sibson stage says why, and it is not the Sibson loops — see +"Where the remaining per-call cost is". + +## Provenance + +| Job | Cell | Leg | Node | Start (2026-09-08) | Elapsed | State | +|---|---|---|---|---|---:|---| +| 342321 | `likelihood_breakdown/imaging/delaunay_nn` | control `d7c96762` | `euclid-ral-gpu-2` | 02:16:25 | 3:00 | COMPLETED | +| 342322 | `likelihood_breakdown/imaging/delaunay_nn` | feature `9e9a7d50`, chunk 256 | `euclid-ral-gpu-2` | 02:19:27 | 2:49 | COMPLETED | +| 342323_0 | `likelihood_breakdown/imaging/delaunay_nn` | feature `9e9a7d50`, chunk 512 | `euclid-ral-gpu-2` | 02:22:17 | 2:36 | COMPLETED | +| 342323_1 | `likelihood_breakdown/imaging/delaunay_nn` | feature `9e9a7d50`, chunk 1024 | `euclid-ral-gpu-2` | 02:24:54 | 2:26 | COMPLETED | +| 342323_2 | `likelihood_breakdown/imaging/delaunay_nn` | feature `9e9a7d50`, chunk 2048 | `euclid-ral-gpu-2` | 02:27:20 | 2:37 | COMPLETED | +| 342323_3 | `likelihood_breakdown/imaging/delaunay_nn` | feature `9e9a7d50`, chunk 4096 | `euclid-ral-gpu-2` | 02:29:58 | 2:33 | COMPLETED | +| 342324 | `likelihood_runtime/imaging/delaunay_nn` | control `d7c96762` | `euclid-ral-gpu-2` | 02:32:32 | 1:22 | COMPLETED | +| 342325 | `likelihood_runtime/imaging/delaunay_nn` | feature `9e9a7d50`, chunk 256 | `euclid-ral-gpu-2` | 02:33:55 | 1:18 | COMPLETED | +| **342329** | `likelihood_breakdown/imaging/delaunay_nn` | **shipped `e5d31d74`, chunk 4096** | `euclid-ral-gpu-2` | 02:40:12 | 2:41 | COMPLETED | +| **342330** | `likelihood_runtime/imaging/delaunay_nn` | **shipped `e5d31d74`, chunk 4096** | `euclid-ral-gpu-2` | 02:42:54 | 1:25 | COMPLETED | + +- Breakdown flags `--split-setup --vmap-batch 16`; runtime cell full timing only (no + `--vmap-probe`), `vmap` batch 16 on both legs. `PyAutoLens 2026.8.17.1`, fp64 + (`JAX_ENABLE_X64=True`), dense inversion path, NVIDIA A100 80GB PCIe. +- Recorded `xla_flags` on every row: `--xla_disable_hlo_passes=constant_folding + --xla_gpu_autotune_level=0` — identical across all ten job units, so these rows are + comparable with the 2026-09-07 rows in `delaunay_walk_early_exit.md` and **not** with the + 2026-07-10 rows in `preopt_breakdown_baseline.md`. +- **All ten job units ran on the same node inside one 28-minute window**, so this is a + same-session A/B, not a comparison against a recorded baseline. +- `backend = gpu`, `device = cuda:0` on every leg, so the feature legs took the *unrolled* + candidate branch (the gate is `jax.default_backend() != "cpu"`). + +### The A/B differs in PyAutoArray and nothing else + +| Repo | Revision | +|---|---| +| PyAutoArray — control leg | `d7c96762801237526d383ce3c53da6cf847875c1` | +| PyAutoArray — feature leg (chunk 256, sweep) | `9e9a7d5020f3467a113af596d9df5edb4ecf0ff2` | +| PyAutoArray — feature leg (shipped) | `e5d31d74f938b2707e11b971683104c441789647` | +| PyAutoNerves | `efe7c04a79f8389bdcccb88b4d6f8781b9b27142` | +| PyAutoFit | `2680b32d6633cc9c1818e86d881d34bff7dc293b` | +| PyAutoGalaxy | `6d216c151c914b2ce79af18fbc0348a6fc5425d7` | +| PyAutoLens | `9468e3e1bc10405c622e7408c04c0760dcb26882` | +| autolens_profiling (`AP_ROOT`) | `99f4b533b52cda974f62c59e8b2995ccb941474b` + the uncommitted split-Sibson stage and these five submits | + +The **shared** `/mnt/ral/jnightin/PyAuto` install was deliberately not touched — the +subhalo-validation and Euclid DR1 CPU runs (`342299`, `342311`, `342314`) were live on it +throughout. Every leg instead prepends a private PyAutoArray checkout to the `PYTHONPATH` +that `activate.sh` exports: +`/mnt/ral/jnightin/PyAuto_wt/delaunay-nn-launch-latency/PyAutoArray_{control,feature}`, and +each job prints `autoarray.__file__` and the checkout's `git rev-parse HEAD` into its SLURM +log. Submits run from `/mnt/ral/jnightin/autolens_profiling_wt/delaunay-nn-launch-latency` +(detached at `99f4b53`): +`hpc/batch_gpu/submit_breakdown_imaging_delaunay_nn_a100_hst_fp64_{launch_latency_control,launch_latency,chunk_sweep}` +and `submit_runtime_imaging_delaunay_nn_a100_hst_fp64_{control,launch_latency}`. + +### Pins + +`EXPECTED_LOG_EVIDENCE_HST = 29144.581944` passed **unchanged on every leg** — both +breakdown A/B legs, all four sweep chunks, the shipped re-measure, and `pinned_drift: []` +on all four runtime rows. The chunk is bit-neutral by construction and the sweep proves it +end-to-end: five different `lax.map` block sizes, one log evidence. + +## Control vs the shipped feature + +All values ms per likelihood call. `vmap/16` is `jax.jit(jax.vmap(fn))` over a params +pytree broadcast to batch 16, reported as batch time / 16; only the combined +inversion-setup block, the `--split-setup` prefixes and the params→H prefix are re-timed +under `vmap`, so the remaining rows read `—` there. + +| Step | control | shipped | Δ | control vmap/16 | shipped vmap/16 | +|---|---:|---:|---:|---:|---:| +| Ray-trace data grid | 0.169 | 0.169 | −0.001 | — | — | +| Ray-trace mesh grid | 0.161 | 0.160 | −0.001 | — | — | +| Lens light images (pre-PSF) | 0.138 | 0.133 | −0.006 | — | — | +| Blurred image (PSF convolution) | 0.854 | 0.807 | −0.047 | — | — | +| Profile-subtracted image | 0.138 | 0.123 | −0.016 | — | — | +| **Inversion setup (steps 5–8 combined)** | **111.552** | **27.482** | **−84.071** | **20.364** | **17.674** | +| Data vector (D) | 0.348 | 0.336 | −0.012 | — | — | +| Curvature matrix (F) | 4.804 | 4.812 | +0.008 | — | — | +| **Regularization matrix (H)** | **45.103** | **13.768** | **−31.335** | **12.622** | **7.491** | +| Regularized reconstruction | 30.820 | 30.736 | −0.084 | — | — | +| Mapped recon + log evidence | 2.269 | 2.191 | −0.078 | — | — | +| **Total step-by-step (unbatched)** | **196.357** | **80.716** | **−115.641** | — | — | + +### Four-way setup split and the prefixes + +| Piece | control | shipped | Δ | control vmap/16 | shipped vmap/16 | +|---|---:|---:|---:|---:|---:| +| Border relocation | 1.526 | 1.079 | −0.447 | 0.092 | 0.064 | +| **Triangulation + interpolation** | **97.274** | **13.365** | **−83.909** | **11.603** | **8.879** | +| Mapping matrix | −0.909 | 0.102 | +1.011 | 0.074 | −2.403 | +| Blurred mapping matrix (PSF) | 8.943 | 8.353 | −0.590 | 10.705 | 10.808 | +| *Interpolator prefix (params→step 6)* | *98.800* | *14.444* | *−84.356* | *11.695* | *8.944* | +| *Split-Sibson prefix (params→split mappings)* | *134.159* | *14.242* | *−119.917* | *14.432* | *6.434* | +| **params→H prefix (the witness)** | **143.903** | **28.212** | **−115.691 (5.10×)** | **24.317** | **16.435 (1.48×)** | + +The ±2 ms sign flips in the "Mapping matrix" rows are the by-difference artifact documented +in `delaunay_nn_breakdown.md`; they are below the run-to-run noise of two independently +compiled prefixes. The shipped leg's interpolator-prefix `vmap` number (8.944 ms) is the +*first* prefix measured in that run and carries warm-up: the split-Sibson prefix in the +same run — a strict superset — reads 6.434 ms, and the sweep's own 4096 row reads 6.398 ms. +Read the Sibson share at `vmap` 16 as **≈ 6.4 ms per call**. + +### Split-point Sibson vs ConstantSplit assembly (the new stage) + +| Piece | control | feature @256 | shipped @4096 | control vmap/16 | feature @256 vmap/16 | shipped vmap/16 | +|---|---:|---:|---:|---:|---:|---:| +| Split-point Sibson | 35.358 | 0.996 | −0.201 | 2.737 | 2.432 | −2.510 | +| H, ConstantSplit assembly | 9.745 | 13.730 | 13.970 | 9.885 | 10.152 | 10.001 | + +**The 35.4 → ~0 ms collapse of "Split-point Sibson" is the attribution shift, not a +saving.** Before the change a prefix stopping at step 6 never asked for the split points, +so XLA dead-code-eliminated the second Sibson pass out of it and its whole cost surfaced in +this row's subtraction. The feature runs both query sets in *one* concatenated pass inside +the step-6 prefix, so the row empties (and can go slightly negative — a prefix that must +*materialise* the split mappings/sizes/weights as outputs against one that fuses them +straight into `reg_split_from`). Only `regularization_matrix_prefix_s` — the params→H +prefix — says whether work went away, and it fell 5.10×. This is exactly what #531 did to +the Delaunay H row, one boundary further in. + +## The chunk sweep + +`PYAUTO_SIBSON_QUERY_CHUNK` swept on the feature checkout, `--split-setup --vmap-batch 16`, +peak VRAM from a 2 s `nvidia-smi --query-gpu=memory.used` sampler running for the life of +the Python process (a floor on the true peak: a sub-2 s spike can be missed). + +| chunk | params→H unbatched | params→H per call @vmap 16 | Sibson share @vmap 16 | ConstantSplit assembly @vmap 16 | peak sampled VRAM | pin | +|---:|---:|---:|---:|---:|---:|---| +| 256 (`9e9a7d50`, job 342322) | 88.287 | 23.072 | 10.488 | 10.152 | 41,495 MiB | PASS | +| 512 (342323_0) | 52.205 | 18.243 | 8.309 | 10.014 | 41,503 MiB | PASS | +| 1024 (342323_1) | 35.324 | 19.567 | 7.062 | 12.557 | 41,503 MiB | PASS | +| 2048 (342323_2) | 27.218 | 19.127 | 6.602 | 10.058 | 41,503 MiB | PASS | +| **4096 (342323_3)** | **24.664** | **16.451** | **6.398** | 10.028 | 41,503 MiB | PASS | +| 4096 shipped (`e5d31d74`, 342329) | 28.212 | 16.435 | 6.434 | 10.001 | not sampled | PASS | +| *control 256 (342321)* | *143.903* | *24.317* | *11.695* | *9.885* | *41,495 MiB* | *PASS* | + +"Sibson share @vmap 16" is the interpolator prefix (or, where the run's first prefix carries +warm-up, the split-Sibson prefix, which is a strict superset of it and reads the same). + +The **256 row is the same-chunk A/B**: it isolates the unroll + single concatenated pass +from the chunk change (143.903 → 88.287 ms unbatched, 1.63×). Everything below it is the +chunk doing the rest of the work. + +### The default: 4096 + +The pre-registered rule (plan A.3) was "the largest of 512/1024/2048/4096 whose peak VRAM at +`vmap` 16 stays under ~50 % of the 80 GB and which is fastest per call; if 4096 wins on +both, ship 2048 unless the margin is > 15 %". + +4096 is fastest on both readings. **The VRAM clause turned out uninformative and was +therefore not used to break the tie.** The ~41.5 GiB plateau (50.7 % of 81,920 MiB) is +identical at every chunk *and* on the control leg, and the GPU read 0 MiB with no other +process at the start of every job, so the plateau is this cell's `vmap`-16 dense inversion +block — not the cavity intermediates. A 16× change of chunk moves it by 8 MiB. The guard +arithmetic says why there is room: `(C, 3, 2)` fp64 intermediates at ~25 kB per query per +lane over 4096 queries × 16 lanes is ~1.6 GB, ~2 % of an 80 GB card. + +With the VRAM leg vacuous, the decision rests on speed alone: 4096 is 14.0 % faster per +call at `vmap` 16 than 2048 (16.451 vs 19.127) and 9.4 % faster unbatched — a margin the +rule's "ship 2048 unless > 15 %" clause would have sent to 2048 had the VRAM clause been +live. It is not, so **4096 ships** (`e5d31d74`), with `PYAUTO_SIBSON_QUERY_CHUNK` kept as +the documented escape hatch for a smaller GPU or a much larger cell. The constant's comment +now carries this table, the rule and the arithmetic. + +## Whole-likelihood runtime cell (`likelihood_runtime/imaging/delaunay_nn`) + +| | control | feature @256 | shipped @4096 | Δ (control → shipped) | +|---|---:|---:|---:|---:| +| Full pipeline (single JIT) | 201.320 | 136.082 | 76.322 | **−125.0 (2.64×)** | +| `vmap` batch 16, per call | 58.022 | 54.209 | 52.618 | −5.4 (1.10×) | + +This is what a production evaluation pays. The unbatched saving (−125.0 ms) exceeds the +params→H prefix drop (−115.7 ms) by the fusion slack expected when the whole pipeline is +one program. The batched saving is small for the same reason as #531: at batch 16 the dense +linear algebra dominates. + +## Where the remaining per-call cost is + +At `vmap` 16 and the shipped chunk, the 16.44 ms params→H prefix decomposes as: + +| piece | per call @vmap 16 | comparison | +|---|---:|---| +| Sibson (locate + both interpolation passes) | ~6.4 ms | Delaunay's whole interpolator prefix: 5.07 ms | +| split-point Sibson residual after concatenation | ~0 ms | (folded into the row above) | +| **H, ConstantSplit assembly** | **~10.0 ms** | **Delaunay's H row: 0.07 ms** | + +The assembly row reads 9.9–10.2 ms at *every* chunk and on the control — it is untouched by +Phase A and cannot be touched by any change to the Sibson chunking. **After Phase A roughly +60 % of the per-call params→H prefix is the 33-wide split-stencil regularization assembly, +not the Sibson loops.** That is why the ≤ 11 ms batched witness could not be met: 10 ms of +it was never Sibson's to give. + +Unbatched the split is the other way round — 14.44 ms interpolator prefix against 13.97 ms +assembly out of 28.21 ms — because the assembly's 33-wide block build amortises under +`vmap` while the Sibson loops do not. + +## Phase B recommendation — the numbers, not the decision + +Phase B as planned is the cavity `fori_loop` → `lax.while_loop` early exit. It targets the +Sibson share only: + +- **Its headroom at `vmap` 16 is ~6.4 ms per call.** The observed cavity occupancy on the + production audit is 11 of the 32 fixed trips (`max_cavity=11`, `max_neighbors=13` in the + workspace_test gate), so the early exit removes at most the tail of a 32-trip loop that + is already latency-amortised at chunk 4096. The plan's own estimate was ~20 % of the + remaining Sibson cost, i.e. **≈ 1.3 ms per call at `vmap` 16** and ≈ 3 ms unbatched. +- **The ConstantSplit assembly is ~10.0 ms per call at `vmap` 16** — 1.6× the entire Sibson + share and 143× Delaunay's equivalent H row (0.07 ms). It is a 33-wide split-stencil + block-diagonal build (`reg_split_from`), and nothing in Phase A or Phase B touches it. +- For scale: the whole likelihood at `vmap` 16 is 52.6 ms per call, so Phase B's ~1.3 ms is + ~2.5 % of a production evaluation and the assembly's 10.0 ms is ~19 %. + +So the bigger per-call lever is a **ConstantSplit-assembly rework**, not the cavity early +exit — by roughly 8× on the batched number that production pays. Phase B remains the +cheaper, lower-risk change (one loop, `stop_gradient` boundary already scoped, no numerical +reordering) and is the only one of the two with a written plan; the assembly rework has no +prompt yet and would need its own investigation of where the 33-wide build spends its time. +Both are worth filing; which goes first is a call about risk appetite, not about which is +larger. + +Phase C (loop-free k-ring cavity, changes fp summation order) is now the smallest of the +three levers on the batched number and should be re-costed against these figures before it +is planned. + +## Artifacts + +- `results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_launch_latency_control.{json,png}` +- `results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_launch_latency.{json,png}` (shipped `e5d31d74`, chunk 4096) +- `results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_launch_latency_chunk256.{json,png}` (`9e9a7d50`, the same-chunk A/B) +- `results/breakdown/imaging/delaunay_nn_hpc_a100_fp64_chunk{512,1024,2048,4096}.{json,png}` +- `results/runtime/imaging/delaunay_nn/delaunay_nn_hpc_a100_fp64_launch_latency_control.json` +- `results/runtime/imaging/delaunay_nn/delaunay_nn_hpc_a100_fp64_launch_latency.json` (shipped) +- `results/runtime/imaging/delaunay_nn/delaunay_nn_hpc_a100_fp64_launch_latency_chunk256.json` +- `hpc/batch_gpu/submit_breakdown_imaging_delaunay_nn_a100_hst_fp64_{launch_latency_control,launch_latency,chunk_sweep}` +- `hpc/batch_gpu/submit_runtime_imaging_delaunay_nn_a100_hst_fp64_{control,launch_latency}` + +The 2026-09-07 `..._walk_early_exit` rows in `delaunay_walk_early_exit.md` are left +untouched; the control leg here supersedes them for any comparison against this branch (its +merge base moved from `bcd15cd9` to `d7c96762`, and this profiling checkout adds the +split-point Sibson cut, so the row sets are not interchangeable). diff --git a/results/runtime/imaging/delaunay_nn/delaunay_nn_hpc_a100_fp64_launch_latency.json b/results/runtime/imaging/delaunay_nn/delaunay_nn_hpc_a100_fp64_launch_latency.json new file mode 100644 index 000000000..d71a3d854 --- /dev/null +++ b/results/runtime/imaging/delaunay_nn/delaunay_nn_hpc_a100_fp64_launch_latency.json @@ -0,0 +1,30 @@ +{ + "autolens_version": "2026.8.17.1", + "device": { + "backend": "gpu", + "device": "cuda:0", + "xla_flags": "--xla_disable_hlo_passes=constant_folding --xla_gpu_autotune_level=0", + "omp_num_threads": null, + "cpu_count": 124, + "nvidia_smi": "NVIDIA A100 80GB PCIe, 35331 MiB, 81920 MiB" + }, + "instrument": "hst", + "configuration": { + "pixel_scale_arcsec": 0.05, + "mask_radius_arcsec": 3.5, + "image_pixels_masked": 15361, + "over_sampled_pixels": 17980, + "delaunay_vertices": 1500, + "edge_zeroed_pixels": 0, + "inversion_path": "dense" + }, + "full_pipeline_single_jit": 0.07632201469969005, + "vmap": { + "batch_size": 16, + "batch_time": 0.8418838044861332, + "per_call": 0.052617737780383325, + "speedup_vs_single_jit": 1.5 + }, + "pinned_expected": 29144.581943885652, + "pinned_drift": [] +} \ No newline at end of file diff --git a/results/runtime/imaging/delaunay_nn/delaunay_nn_hpc_a100_fp64_launch_latency_chunk256.json b/results/runtime/imaging/delaunay_nn/delaunay_nn_hpc_a100_fp64_launch_latency_chunk256.json new file mode 100644 index 000000000..5d367b14d --- /dev/null +++ b/results/runtime/imaging/delaunay_nn/delaunay_nn_hpc_a100_fp64_launch_latency_chunk256.json @@ -0,0 +1,30 @@ +{ + "autolens_version": "2026.8.17.1", + "device": { + "backend": "gpu", + "device": "cuda:0", + "xla_flags": "--xla_disable_hlo_passes=constant_folding --xla_gpu_autotune_level=0", + "omp_num_threads": null, + "cpu_count": 124, + "nvidia_smi": "NVIDIA A100 80GB PCIe, 35331 MiB, 81920 MiB" + }, + "instrument": "hst", + "configuration": { + "pixel_scale_arcsec": 0.05, + "mask_radius_arcsec": 3.5, + "image_pixels_masked": 15361, + "over_sampled_pixels": 17980, + "delaunay_vertices": 1500, + "edge_zeroed_pixels": 0, + "inversion_path": "dense" + }, + "full_pipeline_single_jit": 0.13608224969357252, + "vmap": { + "batch_size": 16, + "batch_time": 0.8673493437934667, + "per_call": 0.05420933398709167, + "speedup_vs_single_jit": 2.5 + }, + "pinned_expected": 29144.581943885652, + "pinned_drift": [] +} \ No newline at end of file diff --git a/results/runtime/imaging/delaunay_nn/delaunay_nn_hpc_a100_fp64_launch_latency_control.json b/results/runtime/imaging/delaunay_nn/delaunay_nn_hpc_a100_fp64_launch_latency_control.json new file mode 100644 index 000000000..54f01a579 --- /dev/null +++ b/results/runtime/imaging/delaunay_nn/delaunay_nn_hpc_a100_fp64_launch_latency_control.json @@ -0,0 +1,30 @@ +{ + "autolens_version": "2026.8.17.1", + "device": { + "backend": "gpu", + "device": "cuda:0", + "xla_flags": "--xla_disable_hlo_passes=constant_folding --xla_gpu_autotune_level=0", + "omp_num_threads": null, + "cpu_count": 124, + "nvidia_smi": "NVIDIA A100 80GB PCIe, 35333 MiB, 81920 MiB" + }, + "instrument": "hst", + "configuration": { + "pixel_scale_arcsec": 0.05, + "mask_radius_arcsec": 3.5, + "image_pixels_masked": 15361, + "over_sampled_pixels": 17980, + "delaunay_vertices": 1500, + "edge_zeroed_pixels": 0, + "inversion_path": "dense" + }, + "full_pipeline_single_jit": 0.20131972269155085, + "vmap": { + "batch_size": 16, + "batch_time": 0.9283518825890497, + "per_call": 0.058021992661815604, + "speedup_vs_single_jit": 3.5 + }, + "pinned_expected": 29144.581943885652, + "pinned_drift": [] +} \ No newline at end of file diff --git a/scripts/imaging/likelihood_breakdown/delaunay_nn.py b/scripts/imaging/likelihood_breakdown/delaunay_nn.py index b33670ce6..1eb9a89ec 100644 --- a/scripts/imaging/likelihood_breakdown/delaunay_nn.py +++ b/scripts/imaging/likelihood_breakdown/delaunay_nn.py @@ -28,8 +28,12 @@ SIBSON_MAX_CAVITY_TRIANGLES = 32 SIBSON_MAX_NEIGHBORS = 32 - SIBSON_QUERY_CHUNK = 256 + SIBSON_QUERY_CHUNK = 4096 +The first two are correctness caps; ``SIBSON_QUERY_CHUNK`` is a memory guard +only (it is bit-neutral — the same log evidence at every chunk) and its default +was raised from 256 to 4096 by PyAutoArray#533, with the import-time override +``PYAUTO_SIBSON_QUERY_CHUNK`` for a smaller GPU or a much larger cell. ``autoarray/inversion/mesh/mesh/delaunay_nn.py`` binds them onto the mesh class as ``max_cavity_triangles`` / ``max_neighbors`` / ``query_chunk``. The workspace mass-model audit (101 traced Hilbert meshes) observed maxima of 25 cavity @@ -90,6 +94,67 @@ therefore returns the step-6 outputs alongside H, making it a strict superset of ``_setup_prefix_fn(6)``. +Split-point Sibson vs ConstantSplit assembly (``--split-setup``) +---------------------------------------------------------------- + +Under ``--split-setup`` a third prefix is compiled *between* those two. It +stops right after ``InterpolatorDelaunayNN._mappings_sizes_weights_split`` — +returning the split-point mappings, sizes **and** weights alongside the step-6 +outputs, the same strict-superset trick prefix 11 uses — and two more rows come +out of the same prefix-difference arithmetic:: + + Split-point Sibson + = t(params -> split mappings/sizes/weights) - t(params -> interpolator) + Regularization matrix (H, ConstantSplit assembly) + = t(params -> H) - t(params -> split mappings/sizes/weights) + +The first row is the **second Sibson pass alone** (the ~6,000 split queries, +about a quarter of the ~24,000 queries the concatenated Sibson pass makes); the +second is the 33-wide ConstantSplit assembly alone (``reg_split_from`` plus the +block-diagonal build), with no Sibson work left in it. Both attribute the same +interval the "Regularization matrix (H)" row reports and sum to it, so they are +printed beside the four-way ``--split-setup`` table rather than added to the +step list: the step list, ``steps``, ``setup_split`` and every other +pre-existing JSON field are unchanged, so older result JSONs stay comparable +row-for-row. The absolute prefix is recorded as ``split_sibson_prefix_s`` next +to ``interpolator_prefix_s`` and ``regularization_matrix_prefix_s``. + +The DCE trap bites this prefix exactly as it bites prefix 11, and for the same +reason: return only the step-6 outputs and XLA eliminates the split walk, so +the row reads ~0 and the split cost reappears in the H row. The prefix +therefore returns all three split arrays, the ones ``ConstantSplit`` actually +consumes. + +**Judge an A/B on the prefixes, not on the differenced rows.** A change that +moves work across a prefix boundary (the #531 concatenated locate did exactly +that) moves the rows without moving the total; ``regularization_matrix_prefix_s`` +is the params->H number that is comparable across legs, with +``split_sibson_prefix_s`` and ``interpolator_prefix_s`` saying where inside it +the time went. + +That is not hypothetical for this cell: PyAutoArray#532 gives ``jax_delaunay_nn`` +one concatenated Sibson pass over data grid + split points, exactly as #531 did +for the locate. Where XLA cannot narrow the concatenated pass back down, the +split-side work moves *inside* ``_setup_prefix_fn(6)`` — into the "Triangulation ++ interpolation" row — and "Split-point Sibson" collapses towards ~0 on the +feature leg while the interpolator prefix grows. That is an attribution shift, +not a saving; only the params->H prefix says whether the work went away. + +Provenance +---------- + +The Sibson static caps in force are recorded in the result JSON under +``sibson``: ``SIBSON_MAX_CAVITY_TRIANGLES`` / ``SIBSON_MAX_NEIGHBORS`` / +``SIBSON_QUERY_CHUNK`` as imported, the mesh's own ``query_chunk``, and the +raw ``PYAUTO_SIBSON_QUERY_CHUNK`` env override PyAutoArray reads once at +import. The chunk sets how many sequential ``lax.map`` trips the two Sibson +passes take, so it rescales every Sibson timing in the run the way a stray +``XLA_FLAGS`` rescales everything else (autolens_profiling#59) — a sweep row +that does not carry its chunk is comparable to nothing. The chunk sweep names +its rows through ``--config-name`` (e.g. +``--config-name hpc_a100_fp64_chunk1024`` writes +``delaunay_nn_hpc_a100_fp64_chunk1024.json``). + Before 2026-09 this row timed ``jnp.array(inversion.regularization_matrix)`` — a host-to-device copy of the 19.5 MB matrix the *eager NumPy* ``FitImaging`` had already computed. On the A100 that read as ~14.4 ms of PCIe traffic and @@ -137,6 +202,7 @@ def _profiling_root() -> _Path: _sys.path.insert(0, _misc_dir) +import os import subprocess import sys import time @@ -280,6 +346,35 @@ def jit_profile(func, label, *args, n_repeats=10): return compiled, result +def sibson_provenance(mesh=None) -> dict: + """Record the Sibson static caps in force for this run. + + Recorded the way ``device_info_dict`` records ``xla_flags``: the query + chunk sets the number of sequential ``lax.map`` trips the Sibson passes + make, so changing it rescales every Sibson timing in the result. A sweep + row that does not carry its chunk cannot be compared to another row. + + ``PYAUTO_SIBSON_QUERY_CHUNK`` is PyAutoArray's env override, read once at + import; the imported ``SIBSON_QUERY_CHUNK`` is therefore the *effective* + value and is recorded whether or not the override is set. ``mesh`` (when + given) contributes the mesh instance's own ``query_chunk``, which is what + the interpolator actually passes down. + """ + from autoarray.inversion.mesh.interpolator import sibson as _sibson_mod + + info = { + "PYAUTO_SIBSON_QUERY_CHUNK": os.environ.get("PYAUTO_SIBSON_QUERY_CHUNK") or None, + "query_chunk": int(_sibson_mod.SIBSON_QUERY_CHUNK), + "max_cavity_triangles": int(_sibson_mod.SIBSON_MAX_CAVITY_TRIANGLES), + "max_neighbors": int(_sibson_mod.SIBSON_MAX_NEIGHBORS), + } + if mesh is not None: + info["mesh_query_chunk"] = int(mesh.query_chunk) + info["mesh_max_cavity_triangles"] = int(mesh.max_cavity_triangles) + info["mesh_max_neighbors"] = int(mesh.max_neighbors) + return info + + timer = Timer() likelihood_steps = [] # (label, per_call_seconds) for the final summary @@ -714,6 +809,16 @@ def profile_subtract(data, blurred_image): print(f" mapper.pixels (source): {mapper.pixels}") print(f" pix_indexes shape: {mapper.pix_indexes_for_sub_slim_index.shape}") +# Sibson caps / query chunk in force — printed here (the first Sibson call of +# the run) and again in the summary header, and carried in the result JSON. +sibson_info = sibson_provenance(mesh=pixelization_obj.mesh) +print( + f" Sibson caps: cavity={sibson_info['max_cavity_triangles']}, " + f"neighbors={sibson_info['max_neighbors']}, " + f"query_chunk={sibson_info['query_chunk']} " + f"(PYAUTO_SIBSON_QUERY_CHUNK={sibson_info['PYAUTO_SIBSON_QUERY_CHUNK'] or 'unset'})" +) + # --------------------------------------------------------------------------- # Steps 7-13: Extract matrices from FitImaging inversion for consistency # --------------------------------------------------------------------------- @@ -820,11 +925,21 @@ def _setup_prefix_fn(upto): """Return a ``params_tree -> `` function for the given stage. ``upto`` is the pipeline step the prefix stops at: 5 border relocation, - 6 interpolator+mapper, 7 mapping matrix, 8 blurred mapping matrix, and + 6 interpolator+mapper, ``_SPLIT_SIBSON_STAGE`` the interpolator's + split-point Sibson pass, 7 mapping matrix, 8 blurred mapping matrix, and 11 the ConstantSplit regularization matrix H (the step-11 row's prefix; it branches off after the mapper, before the mapping matrix, because that is exactly what the inversion's H depends on). + ``_SPLIT_SIBSON_STAGE`` sits between 6 and 11 and is timed under + ``--split-setup``: it stops right after + ``InterpolatorDelaunayNN._mappings_sizes_weights_split``, so + ``t(split) - t(6)`` is the split-side Sibson pass alone and + ``t(11) - t(split)`` is the ConstantSplit assembly alone (module + docstring, "Split-point Sibson vs ConstantSplit assembly"). Like prefix + 11 it is a strict superset of prefix 6 and returns all three split + arrays — drop them and XLA eliminates the split walk it is there to time. + Prefix 11 returns the step-6 outputs **as well as** H. That is load-bearing, not cosmetic: H is built from the interpolator's split-point walk (``_mappings_sizes_weights_split``) and does not consume the per-query @@ -865,6 +980,21 @@ def fn(pt): m.pix_indexes_for_sub_slim_index, m.pix_weights_for_sub_slim_index, ) + if upto == _SPLIT_SIBSON_STAGE: + # Stop right after the split-point Sibson pass. All three arrays + # are returned (mappings, sizes, weights — exactly what + # ``ConstantSplit.regularization_matrix_from`` consumes) on top of + # the step-6 outputs, making this a strict superset of prefix 6 + # and a strict subset of prefix 11. Returning fewer of them lets + # XLA dead-code-eliminate the walk this stage exists to measure. + split_mappings, split_sizes, split_weights = interp._mappings_sizes_weights_split + return ( + m.pix_indexes_for_sub_slim_index, + m.pix_weights_for_sub_slim_index, + split_mappings, + split_sizes, + split_weights, + ) if upto == 11: # H is the mapper's own regularization block: the same call the # inversion makes (``AbstractRegularization.regularization_matrix_from`` @@ -896,13 +1026,23 @@ def fn(pt): 7: "Mapping matrix", 8: "Blurred mapping matrix (PSF)", } +# Stage key of the split-point Sibson prefix. Deliberately not an int: it is +# not a pipeline step, it is a cut *inside* the params->H interval between +# steps 6 and 11, and the four-way setup split must keep differencing over +# (5, 6, 7, 8) only. +_SPLIT_SIBSON_STAGE = "6s" +_REG_SPLIT_LABELS = ( + "Split-point Sibson", + "Regularization matrix (H, ConstantSplit assembly)", +) _split_setup = "--split-setup" in sys.argv -_prefix_per_call: dict[int, float] = {} +_prefix_per_call: dict[int | str, float] = {} _setup_split: dict | None = None +_reg_split: dict | None = None if _split_setup: print("\n--- Inversion setup four-way split (--split-setup) ---") - _prefix_stages = (5, 6, 7, 8) + _prefix_stages = (5, 6, _SPLIT_SIBSON_STAGE, 7, 8) else: print("\n--- Interpolator prefix (needed for the step-11 H attribution) ---") _prefix_stages = (6,) @@ -922,6 +1062,10 @@ def fn(pt): " prefix per-call: " + ", ".join(f"5..{u}={_prefix_per_call[u] * 1000:.2f} ms" for u in (5, 6, 7, 8)) ) + print( + " split-point Sibson prefix (5..6 + split walk): " + f"{_prefix_per_call[_SPLIT_SIBSON_STAGE] * 1000:.2f} ms" + ) for _label, _dt in _setup_split.items(): print(f" {_label}: {_dt * 1000:8.2f} ms") _combined = dict(likelihood_steps)["Inversion setup (steps 5-8 combined)"] @@ -1018,6 +1162,28 @@ def compute_curvature_matrix(blurred_mapping_matrix, noise_map): print(f" params->H prefix per-call: {reg_matrix_prefix_per_call * 1000:9.3f} ms") print(f" params->interpolator prefix: {_prefix_per_call[6] * 1000:9.3f} ms") print(f" attributed H row (difference): {reg_matrix_attributed * 1000:9.3f} ms") + +# Split the same interval in two with the intermediate prefix (--split-setup +# only): split-side Sibson pass vs the ConstantSplit assembly that consumes it. +# These sum to the H row above; the step list is untouched. +_split_sibson_prefix = _prefix_per_call.get(_SPLIT_SIBSON_STAGE) +if _split_sibson_prefix is not None: + _reg_split = { + _REG_SPLIT_LABELS[0]: _split_sibson_prefix - _prefix_per_call[6], + _REG_SPLIT_LABELS[1]: reg_matrix_prefix_per_call - _split_sibson_prefix, + } + print(f" params->split-Sibson prefix: {_split_sibson_prefix * 1000:9.3f} ms") + for _lab, _dt in _reg_split.items(): + print(f" {_lab + ':':<52}{_dt * 1000:9.3f} ms") + if min(_reg_split.values()) < 0.0: + # Same caveat as the H row: the split prefix has to *materialise* the + # split mappings/sizes/weights as outputs, while prefix 11 fuses them + # straight into ``reg_split_from`` — so the assembly row can come out + # slightly negative even though prefix 11 strictly contains this one. + print( + " NOTE: negative attribution — XLA fused work across the prefix " + "boundary; read the row as ~0 and the prefixes as the bound." + ) if reg_matrix_attributed < 0.0: print( " NOTE: negative attribution — XLA fused work across the prefix " @@ -1042,6 +1208,8 @@ def compute_curvature_matrix(blurred_mapping_matrix, noise_map): _vmap_steps: dict[str, float] | None = None _vmap_split: dict[str, float] | None = None +_vmap_reg_split: dict[str, float] | None = None +_vmap_split_sibson_prefix: float | None = None _vmap_error: str | None = None if _vmap_batch is not None: @@ -1071,8 +1239,11 @@ def _vmap_profile(func, label, n_repeats=10): return per_call try: - _vmap_prefix_per_call: dict[int, float] = {} - for _upto in (5, 6, 7, 8, 11): + _vmap_prefix_per_call: dict[int | str, float] = {} + _vmap_prefix_stages = ( + (5, 6, _SPLIT_SIBSON_STAGE, 7, 8, 11) if _split_setup else (5, 6, 7, 8, 11) + ) + for _upto in _vmap_prefix_stages: _vmap_prefix_per_call[_upto] = _vmap_profile( _setup_prefix_fn(_upto), f"setup_prefix_{_upto}" ) @@ -1090,10 +1261,18 @@ def _vmap_profile(func, label, n_repeats=10): } _vmap_h_prefix = _vmap_prefix_per_call[11] _vmap_interp_prefix = _vmap_prefix_per_call[6] + _vmap_split_sibson_prefix = _vmap_prefix_per_call.get(_SPLIT_SIBSON_STAGE) + if _vmap_split_sibson_prefix is not None: + _vmap_reg_split = { + _REG_SPLIT_LABELS[0]: _vmap_split_sibson_prefix - _vmap_interp_prefix, + _REG_SPLIT_LABELS[1]: _vmap_h_prefix - _vmap_split_sibson_prefix, + } except Exception: # noqa: BLE001 — a vmap failure must not lose the unbatched run _vmap_error = _traceback.format_exc() _vmap_h_prefix = None _vmap_interp_prefix = None + _vmap_split_sibson_prefix = None + _vmap_reg_split = None print(" VMAP FAILED — unbatched results are unaffected. Traceback:") print(_vmap_error) else: @@ -1320,6 +1499,11 @@ def compute_log_evidence( print(f" Over-sampled pixels: {n_over_sampled_pixels}") print(f" DelaunayNN vertices: {n_source_pixels}") print(f" Edge zeroed pixels: {edge_pixels_total}") +print( + f" Sibson query chunk: {sibson_info['query_chunk']} " + f"(PYAUTO_SIBSON_QUERY_CHUNK={sibson_info['PYAUTO_SIBSON_QUERY_CHUNK'] or 'unset'}; " + f"caps {sibson_info['max_cavity_triangles']}/{sibson_info['max_neighbors']})" +) print("-" * 70) max_label = max(len(label) for label, _ in likelihood_steps) @@ -1341,10 +1525,17 @@ def compute_log_evidence( print(f" Regularization matrix (H) — params->H prefix: {reg_matrix_prefix_per_call:.6f} s") print(f" Regularization matrix (H) — interp. prefix: {_prefix_per_call[6]:.6f} s") +if _split_sibson_prefix is not None: + print(f" Regularization matrix (H) — split-Sibson pfx: {_split_sibson_prefix:.6f} s") + +_have_reg_split = _reg_split is not None or _vmap_reg_split is not None if _setup_split is not None or _vmap_split is not None: print("-" * 70) - _split_label_width = max(len(k) for k in _prefix_labels.values()) + _split_row_labels = list(_prefix_labels.values()) + if _have_reg_split: + _split_row_labels += list(_REG_SPLIT_LABELS) + _split_label_width = max(len(k) for k in _split_row_labels) print( f" inversion-setup split{'':<{_split_label_width - 21}} " f"{'unbatched':>14}" @@ -1355,6 +1546,15 @@ def compute_log_evidence( _unb = f"{_setup_split[_lab]:12.6f} s" if _setup_split is not None else f"{'—':>14}" _bat = f" {_vmap_split[_lab]:20.6f} s" if _vmap_split is not None else "" print(f" {_lab:<{_split_label_width}} {_unb}{_bat}") + if _have_reg_split: + # The same params->H interval the "Regularization matrix (H)" step row + # reports, cut at the split-point Sibson prefix. These two sum to that + # row; they are not extra steps and are not added to the step total. + print(" params->H interval (cut at the split-point Sibson prefix)") + for _lab in _REG_SPLIT_LABELS: + _unb = f"{_reg_split[_lab]:12.6f} s" if _reg_split is not None else f"{'—':>14}" + _bat = f" {_vmap_reg_split[_lab]:20.6f} s" if _vmap_reg_split is not None else "" + print(f" {_lab:<{_split_label_width}} {_unb}{_bat}") if _vmap_error is not None: print("-" * 70) print(f" vmap batch {_vmap_batch}: FAILED (traceback in the result JSON).") @@ -1374,6 +1574,9 @@ def compute_log_evidence( "edge_zeroed_pixels": int(edge_pixels_total), "inversion_path": "sparse" if _cli.use_sparse_operator else "dense", }, + # Sibson caps / query chunk provenance (see the module docstring): the + # chunk rescales every Sibson timing, so a sweep row carries its own. + "sibson": sibson_info, "steps": {label: per_call for label, per_call in likelihood_steps}, "total_step_by_step": step_total, # Absolute prefix times behind the attributed "Regularization matrix (H)" @@ -1382,6 +1585,15 @@ def compute_log_evidence( "interpolator_prefix_s": float(_prefix_per_call[6]), } +if _split_sibson_prefix is not None: + # Third prefix, between the interpolator and H: stops after the + # interpolator's split-point Sibson pass. ``regularization_split`` cuts the + # params->H interval in two with it (split-side Sibson / ConstantSplit + # assembly); ``steps`` and ``setup_split`` are unchanged. + breakdown_summary["split_sibson_prefix_s"] = float(_split_sibson_prefix) +if _reg_split is not None: + breakdown_summary["regularization_split"] = {k: float(v) for k, v in _reg_split.items()} + if _setup_split is not None: breakdown_summary["setup_split"] = {k: float(v) for k, v in _setup_split.items()} @@ -1394,6 +1606,14 @@ def compute_log_evidence( breakdown_summary["setup_split_vmap"] = {k: float(v) for k, v in _vmap_split.items()} breakdown_summary["regularization_matrix_prefix_vmap_per_call_s"] = float(_vmap_h_prefix) breakdown_summary["interpolator_prefix_vmap_per_call_s"] = float(_vmap_interp_prefix) + if _vmap_split_sibson_prefix is not None: + breakdown_summary["split_sibson_prefix_vmap_per_call_s"] = float( + _vmap_split_sibson_prefix + ) + if _vmap_reg_split is not None: + breakdown_summary["regularization_split_vmap"] = { + k: float(v) for k, v in _vmap_reg_split.items() + } dict_path, chart_path = resolve_output_paths( _cli, diff --git a/scripts/imaging/likelihood_runtime/delaunay_nn.py b/scripts/imaging/likelihood_runtime/delaunay_nn.py index f282016f0..e605472a1 100644 --- a/scripts/imaging/likelihood_runtime/delaunay_nn.py +++ b/scripts/imaging/likelihood_runtime/delaunay_nn.py @@ -15,9 +15,11 @@ Sibson weights need fixed-shape arrays under ``jax.jit``: PyAutoArray pins ``SIBSON_MAX_CAVITY_TRIANGLES = 32``, ``SIBSON_MAX_NEIGHBORS = 32`` and -``SIBSON_QUERY_CHUNK = 256`` in -``autoarray/inversion/mesh/interpolator/sibson.py``, so every mapper row is 32 -wide here against Delaunay's 3. The per-step attribution of that cost is the +``SIBSON_QUERY_CHUNK = 4096`` in +``autoarray/inversion/mesh/interpolator/sibson.py`` (the chunk default was +raised from 256 to 4096 by PyAutoArray#533; ``PYAUTO_SIBSON_QUERY_CHUNK`` +overrides it at import time), so every mapper row is 32 wide here against +Delaunay's 3. The per-step attribution of that cost is the breakdown cell's job (``likelihood_breakdown/imaging/delaunay_nn.py``); this script answers only "what does one likelihood cost end to end".