diff --git a/hpc/batch_gpu/submit_breakdown_interferometer_mge_a100_sma_fp64_real_scatter b/hpc/batch_gpu/submit_breakdown_interferometer_mge_a100_sma_fp64_real_scatter new file mode 100644 index 000000000..f5dee5395 --- /dev/null +++ b/hpc/batch_gpu/submit_breakdown_interferometer_mge_a100_sma_fp64_real_scatter @@ -0,0 +1,109 @@ +#!/bin/bash -l +# +# A100 likelihood-breakdown profile: interferometer/mge x sma x fp64, LEVER 3 (real scatter). +# PyAutoArray#577 scatters the real mapping matrix into the native stack and casts to +# complex128 afterwards in TransformerNUFFT.transform_mapping_matrix; the complex128 +# scatter was a fixed ~0.85 s on the A100 (autolens_profiling#308, step 3). Same arms +# as submit_breakdown_interferometer_mge_a100_sma_fp64 (DENSE library path + W~ arm), +# run twice in one job on one node: +# 1. CONTROL: PyAutoArray origin/main (cast-then-scatter), $PYAUTO_BRANCH_LIBS/PyAutoArray_main +# 2. BRANCH: PyAutoArray feature/interferometer-transform-real-scatter, $PYAUTO_BRANCH_LIBS/PyAutoArray +# Both legs use the same private PyAutoGalaxy / PyAutoLens (origin/main) copies. +# +# BRANCH LIBRARIES: a PRIVATE copy under $PYAUTO_BRANCH_LIBS (default below), prepended to +# PYTHONPATH after activate.sh. The shared mirror /mnt/ral/jnightin/PyAuto is not touched. +# Each leg fails unless autoarray/autogalaxy/autolens import from that copy. +# +# Output (distinct names, so the #308 mge_hpc_a100_fp64.json stays; the dashboard +# regex does not read them): +# results/breakdown/interferometer/sma/mge_hpc_a100_fp64_real_scatter.{json,png} +# results/breakdown/interferometer/sma/mge_hpc_a100_fp64_real_scatter_main_control.{json,png} + +#SBATCH -J breakdown_interferometer_mge_sma_fp64_real_scatter +#SBATCH --partition=gpu +#SBATCH --gres=gpu:1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=4 +#SBATCH --mem=64gb +#SBATCH --time=1:00:00 +#SBATCH -o output/output.%A.out +#SBATCH -e error/error.%A.err +#SBATCH --mail-type=END,FAIL +#SBATCH --mail-user=james.w.nightingale@durham.ac.uk + +# Derive the repo root from where this was submitted (hpc/batch_gpu/../..): it runs from +# a RAL worktree of the feature branch, not only the canonical RAL checkout. +export AP_ROOT="$(cd "${SLURM_SUBMIT_DIR:-$(dirname "$0")}/../.." && pwd)" +export PROJECT_PATH="$AP_ROOT" +export PYAUTO_HPC_BASE="${PYAUTO_HPC_BASE:-/mnt/ral/jnightin/PyAuto}" +export PYAUTO_BRANCH_LIBS="${PYAUTO_BRANCH_LIBS:-/mnt/ral/jnightin/PyAuto_branch/interferometer-transform-real-scatter}" +echo "AP_ROOT=$AP_ROOT" +source $AP_ROOT/activate.sh +BASE_PYTHONPATH="$PYTHONPATH" + +export JAX_PLATFORM_NAME=cuda +export JAX_PLATFORMS=cuda,cpu +export XLA_PYTHON_CLIENT_PREALLOCATE=false +export JAX_ENABLE_X64=True +export XLA_FLAGS="--xla_disable_hlo_passes=constant_folding" + +nvidia-smi + +cd $AP_ROOT +OUT_DIR=$AP_ROOT/results/breakdown/interferometer/sma +STATUS=0 + +run_leg() { + local array_dir="$1" suffix="$2" + export PYAUTO_ARRAY_DIR="$PYAUTO_BRANCH_LIBS/$array_dir" + export PYTHONPATH="$PYAUTO_ARRAY_DIR:$PYAUTO_BRANCH_LIBS/PyAutoGalaxy:$PYAUTO_BRANCH_LIBS/PyAutoLens:$BASE_PYTHONPATH" + local tmp_out="$AP_ROOT/hpc/batch_gpu/output/real_scatter_${SLURM_JOB_ID:-local}_${suffix}" + mkdir -p "$tmp_out" + + echo "==========================================" + date + echo "Cell: likelihood_breakdown/interferometer/mge" + echo "Instrument: sma" + echo "Precision: fp64" + echo "Inversion path: dense (mapping) + W~ arm" + echo "Leg: $suffix ($array_dir)" + + echo "=== source revisions ===" + printf '%-14s %s (branch copy)\n' PyAutoArray "$(git -C $PYAUTO_ARRAY_DIR rev-parse HEAD 2>&1)" + for r in PyAutoGalaxy PyAutoLens; do + printf '%-14s %s (branch copy)\n' "$r" "$(git -C $PYAUTO_BRANCH_LIBS/$r rev-parse HEAD 2>&1)" + done + for r in PyAutoNerves PyAutoFit; do + printf '%-14s %s (shared mirror)\n' "$r" "$(git -C $PYAUTO_HPC_BASE/$r rev-parse HEAD 2>&1)" + done + echo "AP_ROOT rev: $(git -C $AP_ROOT rev-parse HEAD 2>&1)" + + echo "=== import provenance (must be the branch copy) ===" + python3 -c " +import os, autoarray, autogalaxy, autolens +base = os.environ['PYAUTO_BRANCH_LIBS'] +assert autoarray.__file__.startswith(os.environ['PYAUTO_ARRAY_DIR'] + '/'), autoarray.__file__ +for m in (autoarray, autogalaxy, autolens): + print(m.__name__ + '.__file__ =', m.__file__) + assert m.__file__.startswith(base), f'{m.__name__} not imported from {base}' +" || return 1 + + python3 -u scripts/interferometer/likelihood_breakdown/mge.py \ + --config-name hpc_a100_fp64 \ + --instrument sma \ + --w-tilde \ + --vmap-batch 4 \ + --output-dir "$tmp_out" || return 1 + + for ext in json png; do + mv "$tmp_out/mge_hpc_a100_fp64.$ext" "$OUT_DIR/mge_hpc_a100_fp64_${suffix}.$ext" || return 1 + done + rmdir "$tmp_out" +} + +run_leg PyAutoArray_main real_scatter_main_control || STATUS=1 +run_leg PyAutoArray real_scatter || STATUS=1 + +echo "Finished (status $STATUS)." +date +exit $STATUS diff --git a/results/breakdown/interferometer/sma/mge_hpc_a100_fp64_real_scatter.json b/results/breakdown/interferometer/sma/mge_hpc_a100_fp64_real_scatter.json new file mode 100644 index 000000000..27e1a2c5e --- /dev/null +++ b/results/breakdown/interferometer/sma/mge_hpc_a100_fp64_real_scatter.json @@ -0,0 +1,309 @@ +{ + "stage": "complete", + "autolens_version": "2026.8.17.1", + "device": { + "backend": "gpu", + "device": "cuda:0", + "hostname": "euclid-ral-gpu-1", + "xla_flags": "--xla_disable_hlo_passes=constant_folding --xla_gpu_autotune_level=0 --xla_gpu_enable_triton_gemm=false", + "omp_num_threads": null, + "cpu_count": 124, + "jax_compilation_cache_dir": "/mnt/ral/jnightin/.cache/pyauto_jax", + "autotune_cache_entries_at_start": 204, + "cache_fresh": false, + "nvidia_smi": "NVIDIA A100 80GB PCIe, 2091 MiB, 81920 MiB" + }, + "instrument": "sma", + "model": "mge", + "transformer": "TransformerNUFFT", + "use_mixed_precision": false, + "configuration": { + "pixel_scale_arcsec": 0.1, + "mask_radius_arcsec": 3.5, + "real_space_shape": [ + 256, + 256 + ], + "image_pixels_masked": 3852, + "visibilities": 190, + "linear_gaussians": 20, + "lens_light": null, + "inversion_path": "dense", + "inversion_class": "InversionInterferometerMapping", + "dense_inversion_class": "InversionInterferometerMapping", + "transformer": "TransformerNUFFT", + "chunk_size": null, + "chunk_size_note": "TransformerNUFFT.transform_mapping_matrix ignores chunk_size: step 3 is one nufft2d2 over every column and every visibility.", + "nnls_solver": "pdip", + "nnls_preconditioning": "raw", + "nufftax_version": "0.6.1", + "transform": "library", + "transform_column_batch": null, + "transform_vis_chunk": null, + "w_tilde_arm": true, + "library_sparse_arm": false, + "n_repeats": 10, + "thread_env": { + "n_threads": null, + "vars": [ + "OPENBLAS_NUM_THREADS", + "MKL_NUM_THREADS", + "OMP_NUM_THREADS", + "VECLIB_MAXIMUM_THREADS", + "NUMEXPR_NUM_THREADS" + ], + "set_to": null, + "preexisting": {}, + "overridden": {}, + "note": "Not pinned (legacy variant): recorded as found." + }, + "host_load_avg_start": [ + 1.0, + 0.37, + 0.14 + ], + "host_load_avg_end": [ + 1.2, + 0.48, + 0.18 + ] + }, + "reference_mode": "eager_numpy", + "library_path": { + "status": "ok", + "where": "eager FitInterferometer reference" + }, + "full_pipeline_status": { + "status": "ok" + }, + "transformed_mapping_matrix_chunked_vs_library_max_rel_diff": null, + "log_likelihood_reference": -3153.942384509246, + "figure_of_merit_reference": -3153.948230379729, + "figure_of_merit_step_by_step": -3153.94823037973, + "log_likelihood_step_by_step": -3153.942384509246, + "figure_of_merit_full_pipeline": -3153.94823037973, + "figure_of_merit_abs_diff_nats": 9.094947017729282e-13, + "log_likelihood_abs_diff_nats": 0.0, + "full_pipeline_abs_diff_nats": 9.094947017729282e-13, + "log_likelihood_atol_nats": 0.001, + "diagonal_floor_bias_nats": 0.0058458704829718044, + "diagonal_floor_note": "log_likelihood - figure_of_merit: fast_chi_squared uses F with the no-regularization diagonal floor (0.001), so the figure of merit the sampler sees sits 0.5 * floor * |s|^2 below the residual-form log L.", + "setup_cross_check": { + "mapping_matrix_max_rel_diff_vs_eager": 1.4599552037340833e-14, + "transformed_mapping_matrix_max_rel_diff_vs_eager": 6.376873006251392e-17 + }, + "log_det_terms": { + "value": 0.0, + "timed": false, + "note": "Structurally zero: an MGE-only inversion has no regularization." + }, + "nnls": { + "iterations": 15, + "converged": true, + "max_iter": 50, + "solver": "pdip", + "preconditioning": "raw" + }, + "steps": { + "Ray-trace grids": 0.0004672404000302777, + "Mapping matrix (20 Gaussians)": 0.0003364969001268037, + "Transformed mapping matrix (NUFFT)": 0.0005087248006020672, + "Data vector (D)": 0.00014704819914186374, + "Curvature matrix (F)": 0.00018495770054869354, + "Reconstruction (PDIP NNLS)": 0.0022696095998981035, + "Fast chi-squared + figure of merit": 0.00012597449967870488 + }, + "total_step_by_step": 0.0040400521000265146, + "source_revisions": { + "autolens_profiling": "5e105d4cf812ac86c5aa73638b41547094058120", + "PyAutoNerves": "1fa613aa8d89a79613009fb66f5b4e2d21deb57a", + "PyAutoFit": "dd9fbe0aab60c4a57702b1444ca031b317c91331", + "PyAutoArray": "f9d3bc671d3c36dc41bf9f1f22b61e1e61e84a07", + "PyAutoGalaxy": "0e4b89cff42882a6cf5bbed6c9686f7212d7e24c", + "PyAutoLens": "4487eb471baa35841e90d3db0911799239594bce" + }, + "full_pipeline_single_jit": 0.003394538200518582, + "steps_sub_rows": { + "Ray-trace (standalone, eager tracer)": 0.00022902920027263464, + "transform_mapping_matrix (standalone)": 0.0007975454005645588, + "Mapped visibilities + residual chi-squared (log_likelihood)": 0.00015384010039269923, + "Dense chain from mapping matrix (steps 3-8 fused)": 0.00277797149901744 + }, + "setup_split": { + "Ray-trace grids": 0.0004672404000302777, + "Mapping matrix (20 Gaussians)": 0.0003364969001268037, + "Transformed mapping matrix (NUFFT)": 0.0005087248006020672 + }, + "setup_prefix_per_call_s": { + "1": 0.0004672404000302777, + "2": 0.0008037373001570814, + "3": 0.0013124621007591486 + }, + "jit_phases": { + "setup_prefix_1": { + "lower_s": 0.10094172500248533, + "compile_s": 0.49636943999212235, + "first_call_s": 0.002068086003419012, + "steady_per_call_s": 0.0004672404000302777 + }, + "setup_prefix_2": { + "lower_s": 0.25877309300994966, + "compile_s": 0.09995866600365844, + "first_call_s": 0.004721862991573289, + "steady_per_call_s": 0.0008037373001570814 + }, + "setup_prefix_3": { + "lower_s": 0.4559228269936284, + "compile_s": 2.3583277680008905, + "first_call_s": 0.008015304003492929, + "steady_per_call_s": 0.0013124621007591486 + }, + "ray_trace_standalone": { + "lower_s": 0.06847341501270421, + "compile_s": 0.42112189601175487, + "first_call_s": 0.001133996993303299, + "steady_per_call_s": 0.00022902920027263464 + }, + "transform_mapping_matrix_standalone": { + "lower_s": 0.04230592999374494, + "compile_s": 0.409305178996874, + "first_call_s": 0.004353187992819585, + "steady_per_call_s": 0.0007975454005645588 + }, + "data_vector": { + "lower_s": 0.011182726011611521, + "compile_s": 0.05135813199740369, + "first_call_s": 0.0007562810060335323, + "steady_per_call_s": 0.00014704819914186374 + }, + "curvature_matrix": { + "lower_s": 0.01704829599475488, + "compile_s": 0.05622486199717969, + "first_call_s": 0.03742657900147606, + "steady_per_call_s": 0.00018495770054869354 + }, + "reconstruction": { + "lower_s": 0.07299336101277731, + "compile_s": 0.33881302799272817, + "first_call_s": 0.013371780005400069, + "steady_per_call_s": 0.0022696095998981035 + }, + "figure_of_merit": { + "lower_s": 0.012049034994561225, + "compile_s": 0.08521031300188042, + "first_call_s": 0.0010127980058314279, + "steady_per_call_s": 0.00012597449967870488 + }, + "log_likelihood_residual_visibilities": { + "lower_s": 0.009625354010495357, + "compile_s": 0.10284215000865515, + "first_call_s": 0.0008721699996385723, + "steady_per_call_s": 0.00015384010039269923 + }, + "dense_chain_from_mapping_matrix": { + "lower_s": 0.07550147999427281, + "compile_s": 0.8048413630021969, + "first_call_s": 0.008042522997129709, + "steady_per_call_s": 0.00277797149901744 + }, + "full_pipeline": { + "lower_s": 0.7602524710091529, + "compile_s": 2.65932311200595, + "first_call_s": 0.015421414995216765, + "steady_per_call_s": 0.003394538200518582 + }, + "w_tilde_curvature": { + "lower_s": 0.07950064199394546, + "compile_s": 0.15091606200439855, + "first_call_s": 0.0055944930063560605, + "steady_per_call_s": 0.00032817610044730825 + }, + "w_tilde_data_vector": { + "lower_s": 0.006158664997201413, + "compile_s": 0.10735630700946786, + "first_call_s": 0.0007066719990689307, + "steady_per_call_s": 0.00012881349975941704 + }, + "w_tilde_reconstruction": { + "lower_s": 0.0003061559982597828, + "compile_s": 9.34000127017498e-06, + "first_call_s": 0.0026421779912197962, + "steady_per_call_s": 0.002311281199217774 + }, + "w_tilde_figure_of_merit": { + "lower_s": 0.0061031760124024, + "compile_s": 0.048269107996020466, + "first_call_s": 0.0007780710002407432, + "steady_per_call_s": 0.0001332325002294965 + }, + "w_tilde_chain_from_mapping_matrix": { + "lower_s": 0.0412423520028824, + "compile_s": 0.5278875100048026, + "first_call_s": 0.008941802007029764, + "steady_per_call_s": 0.0024323277990333735 + }, + "setup_prefix_1_vmap4": { + "first_call_s": 0.5478371100034565, + "batch_per_call_s": 0.00045085449964972214, + "steady_per_call_s": 0.00011271362491243053 + }, + "setup_prefix_2_vmap4": { + "first_call_s": 0.5143930930062197, + "batch_per_call_s": 0.0008817203008220531, + "steady_per_call_s": 0.0002204300752055133 + }, + "setup_prefix_3_vmap4": { + "first_call_s": 2.9522122939961264, + "batch_per_call_s": 0.0026681378003559075, + "steady_per_call_s": 0.0006670344500889769 + }, + "full_pipeline_vmap4": { + "first_call_s": 3.941707543999655, + "batch_per_call_s": 0.005373813299229369, + "steady_per_call_s": 0.0013434533248073421 + } + }, + "peak_rss_mb": 1312.0625, + "w_tilde": { + "operator_build_s": 1.0876574669964612, + "operator_build_nufft_chunk_size": null, + "operator_shape": [ + 70, + 70 + ], + "curvature_max_rel_diff_vs_dense": 2.7493413814337997e-14, + "data_vector_max_rel_diff_vs_dense": 2.36318493749997e-16, + "figure_of_merit": -3153.94823037973, + "figure_of_merit_chain": -3153.94823037973, + "figure_of_merit_abs_diff_vs_fit_nats": 9.094947017729282e-13, + "figure_of_merit_abs_diff_vs_step_by_step_nats": 0.0, + "log_likelihood_unfloored": -3153.942384509246, + "log_likelihood_abs_diff_vs_fit_nats": 0.0, + "operator_build_peak_rss_mb": 1159.2421875, + "total_step_by_step": 0.0029015032996539958, + "chain_from_mapping_matrix_s": 0.0024323277990333735, + "dense_chain_from_mapping_matrix_s": 0.00277797149901744, + "dense_transform_plus_curvature_s": 0.0009825031011132523, + "w_tilde_curvature_s": 0.00032817610044730825, + "speedup_curvature_vs_dense_transform_plus_curvature": 2.993828922258775, + "speedup_chain_vs_dense_chain": 1.1421040782913503, + "note": "Measurement-only: an MGE-only fit never takes this path in the library (inversion/factory.py switches the sparse operator off when every linear object is an AbstractLinearObjFuncList). The chain row replaces steps 3-8 from the same mapping matrix; its per-call cost has no N_vis term." + }, + "w_tilde_steps": { + "Curvature matrix F~ = B\u1d40 W~ B": 0.00032817610044730825, + "Data vector D~ = B\u1d40 d~": 0.00012881349975941704, + "Reconstruction (PDIP NNLS)": 0.002311281199217774, + "Fast chi-squared + figure of merit (constants hoisted)": 0.0001332325002294965 + }, + "vmap_batch": 4, + "steps_vmap_per_call": { + "Full pipeline": 0.0013434533248073421 + }, + "setup_split_vmap": { + "Ray-trace grids": 0.00011271362491243053, + "Mapping matrix (20 Gaussians)": 0.00010771645029308275, + "Transformed mapping matrix (NUFFT)": 0.0004466043748834636 + }, + "pinned_expected": -3153.948230379729, + "pinned_drift": [] +} \ No newline at end of file diff --git a/results/breakdown/interferometer/sma/mge_hpc_a100_fp64_real_scatter.png b/results/breakdown/interferometer/sma/mge_hpc_a100_fp64_real_scatter.png new file mode 100644 index 000000000..e2fc917b2 Binary files /dev/null and b/results/breakdown/interferometer/sma/mge_hpc_a100_fp64_real_scatter.png differ diff --git a/results/breakdown/interferometer/sma/mge_hpc_a100_fp64_real_scatter_main_control.json b/results/breakdown/interferometer/sma/mge_hpc_a100_fp64_real_scatter_main_control.json new file mode 100644 index 000000000..7bc44f2cd --- /dev/null +++ b/results/breakdown/interferometer/sma/mge_hpc_a100_fp64_real_scatter_main_control.json @@ -0,0 +1,309 @@ +{ + "stage": "complete", + "autolens_version": "2026.8.17.1", + "device": { + "backend": "gpu", + "device": "cuda:0", + "hostname": "euclid-ral-gpu-1", + "xla_flags": "--xla_disable_hlo_passes=constant_folding --xla_gpu_autotune_level=0 --xla_gpu_enable_triton_gemm=false", + "omp_num_threads": null, + "cpu_count": 124, + "jax_compilation_cache_dir": "/mnt/ral/jnightin/.cache/pyauto_jax", + "autotune_cache_entries_at_start": 204, + "cache_fresh": false, + "nvidia_smi": "NVIDIA A100 80GB PCIe, 2091 MiB, 81920 MiB" + }, + "instrument": "sma", + "model": "mge", + "transformer": "TransformerNUFFT", + "use_mixed_precision": false, + "configuration": { + "pixel_scale_arcsec": 0.1, + "mask_radius_arcsec": 3.5, + "real_space_shape": [ + 256, + 256 + ], + "image_pixels_masked": 3852, + "visibilities": 190, + "linear_gaussians": 20, + "lens_light": null, + "inversion_path": "dense", + "inversion_class": "InversionInterferometerMapping", + "dense_inversion_class": "InversionInterferometerMapping", + "transformer": "TransformerNUFFT", + "chunk_size": null, + "chunk_size_note": "TransformerNUFFT.transform_mapping_matrix ignores chunk_size: step 3 is one nufft2d2 over every column and every visibility.", + "nnls_solver": "pdip", + "nnls_preconditioning": "raw", + "nufftax_version": "0.6.1", + "transform": "library", + "transform_column_batch": null, + "transform_vis_chunk": null, + "w_tilde_arm": true, + "library_sparse_arm": false, + "n_repeats": 10, + "thread_env": { + "n_threads": null, + "vars": [ + "OPENBLAS_NUM_THREADS", + "MKL_NUM_THREADS", + "OMP_NUM_THREADS", + "VECLIB_MAXIMUM_THREADS", + "NUMEXPR_NUM_THREADS" + ], + "set_to": null, + "preexisting": {}, + "overridden": {}, + "note": "Not pinned (legacy variant): recorded as found." + }, + "host_load_avg_start": [ + 0.3, + 0.07, + 0.02 + ], + "host_load_avg_end": [ + 1.0, + 0.36, + 0.13 + ] + }, + "reference_mode": "eager_numpy", + "library_path": { + "status": "ok", + "where": "eager FitInterferometer reference" + }, + "full_pipeline_status": { + "status": "ok" + }, + "transformed_mapping_matrix_chunked_vs_library_max_rel_diff": null, + "log_likelihood_reference": -3153.942384509246, + "figure_of_merit_reference": -3153.948230379729, + "figure_of_merit_step_by_step": -3153.94823037973, + "log_likelihood_step_by_step": -3153.942384509246, + "figure_of_merit_full_pipeline": -3153.94823037973, + "figure_of_merit_abs_diff_nats": 9.094947017729282e-13, + "log_likelihood_abs_diff_nats": 0.0, + "full_pipeline_abs_diff_nats": 9.094947017729282e-13, + "log_likelihood_atol_nats": 0.001, + "diagonal_floor_bias_nats": 0.0058458704829718044, + "diagonal_floor_note": "log_likelihood - figure_of_merit: fast_chi_squared uses F with the no-regularization diagonal floor (0.001), so the figure of merit the sampler sees sits 0.5 * floor * |s|^2 below the residual-form log L.", + "setup_cross_check": { + "mapping_matrix_max_rel_diff_vs_eager": 1.4599552037340833e-14, + "transformed_mapping_matrix_max_rel_diff_vs_eager": 6.376873006251392e-17 + }, + "log_det_terms": { + "value": 0.0, + "timed": false, + "note": "Structurally zero: an MGE-only inversion has no regularization." + }, + "nnls": { + "iterations": 15, + "converged": true, + "max_iter": 50, + "solver": "pdip", + "preconditioning": "raw" + }, + "steps": { + "Ray-trace grids": 0.00045880660036345944, + "Mapping matrix (20 Gaussians)": 0.0002708125990466215, + "Transformed mapping matrix (NUFFT)": 0.851492780901026, + "Data vector (D)": 0.00012399749975884334, + "Curvature matrix (F)": 0.0001862796998466365, + "Reconstruction (PDIP NNLS)": 0.002279294599429704, + "Fast chi-squared + figure of merit": 0.00014390730066224933 + }, + "total_step_by_step": 0.8549558792001335, + "source_revisions": { + "autolens_profiling": "5e105d4cf812ac86c5aa73638b41547094058120", + "PyAutoNerves": "1fa613aa8d89a79613009fb66f5b4e2d21deb57a", + "PyAutoFit": "dd9fbe0aab60c4a57702b1444ca031b317c91331", + "PyAutoArray": "1bf641e418c638ab9b9db85743a6c879dcaf2b85", + "PyAutoGalaxy": "0e4b89cff42882a6cf5bbed6c9686f7212d7e24c", + "PyAutoLens": "4487eb471baa35841e90d3db0911799239594bce" + }, + "full_pipeline_single_jit": 0.8684562675000052, + "steps_sub_rows": { + "Ray-trace (standalone, eager tracer)": 0.00014652310055680574, + "transform_mapping_matrix (standalone)": 0.8565189123997697, + "Mapped visibilities + residual chi-squared (log_likelihood)": 0.00015539710002485663, + "Dense chain from mapping matrix (steps 3-8 fused)": 0.8657640049990732 + }, + "setup_split": { + "Ray-trace grids": 0.00045880660036345944, + "Mapping matrix (20 Gaussians)": 0.0002708125990466215, + "Transformed mapping matrix (NUFFT)": 0.851492780901026 + }, + "setup_prefix_per_call_s": { + "1": 0.00045880660036345944, + "2": 0.0007296191994100809, + "3": 0.852222400100436 + }, + "jit_phases": { + "setup_prefix_1": { + "lower_s": 0.10171850498591084, + "compile_s": 0.49212419099058025, + "first_call_s": 0.0020998950058128685, + "steady_per_call_s": 0.00045880660036345944 + }, + "setup_prefix_2": { + "lower_s": 0.25790916199912317, + "compile_s": 0.09295938999275677, + "first_call_s": 0.004541905000223778, + "steady_per_call_s": 0.0007296191994100809 + }, + "setup_prefix_3": { + "lower_s": 0.45644683099817485, + "compile_s": 0.179421197011834, + "first_call_s": 0.8676638780016219, + "steady_per_call_s": 0.852222400100436 + }, + "ray_trace_standalone": { + "lower_s": 0.07064297900069505, + "compile_s": 0.42303982299927156, + "first_call_s": 0.0011973960063187405, + "steady_per_call_s": 0.00014652310055680574 + }, + "transform_mapping_matrix_standalone": { + "lower_s": 0.042584587004967034, + "compile_s": 0.9248853479948593, + "first_call_s": 0.8601834480068646, + "steady_per_call_s": 0.8565189123997697 + }, + "data_vector": { + "lower_s": 0.012463029008358717, + "compile_s": 0.05396149000443984, + "first_call_s": 0.0007802009931765497, + "steady_per_call_s": 0.00012399749975884334 + }, + "curvature_matrix": { + "lower_s": 0.016075696010375395, + "compile_s": 0.05777538400434423, + "first_call_s": 0.19478765399253462, + "steady_per_call_s": 0.0001862796998466365 + }, + "reconstruction": { + "lower_s": 0.12100225299946032, + "compile_s": 0.34964548698917497, + "first_call_s": 0.25436961599916685, + "steady_per_call_s": 0.002279294599429704 + }, + "figure_of_merit": { + "lower_s": 0.012329940989729948, + "compile_s": 0.0864146090025315, + "first_call_s": 0.001025086996378377, + "steady_per_call_s": 0.00014390730066224933 + }, + "log_likelihood_residual_visibilities": { + "lower_s": 0.010906359006185085, + "compile_s": 0.10078020498622209, + "first_call_s": 0.0009174379956675693, + "steady_per_call_s": 0.00015539710002485663 + }, + "dense_chain_from_mapping_matrix": { + "lower_s": 0.07418305600003805, + "compile_s": 1.3171029029908823, + "first_call_s": 0.8674424009950599, + "steady_per_call_s": 0.8657640049990732 + }, + "full_pipeline": { + "lower_s": 0.9052041549875867, + "compile_s": 3.3965679399989313, + "first_call_s": 0.8694847349979682, + "steady_per_call_s": 0.8684562675000052 + }, + "w_tilde_curvature": { + "lower_s": 0.10636623899335973, + "compile_s": 0.15056932599691208, + "first_call_s": 0.012388581002596766, + "steady_per_call_s": 0.00032038409990491344 + }, + "w_tilde_data_vector": { + "lower_s": 0.006486122001660988, + "compile_s": 0.11069166699599009, + "first_call_s": 0.0007907200051704422, + "steady_per_call_s": 0.00011536469974089414 + }, + "w_tilde_reconstruction": { + "lower_s": 0.00036459499096963555, + "compile_s": 1.1049996828660369e-05, + "first_call_s": 0.0027695760072674602, + "steady_per_call_s": 0.0022860805009258913 + }, + "w_tilde_figure_of_merit": { + "lower_s": 0.006254363994230516, + "compile_s": 0.048317608001525514, + "first_call_s": 0.0007720509893260896, + "steady_per_call_s": 0.00011999249982181936 + }, + "w_tilde_chain_from_mapping_matrix": { + "lower_s": 0.041221424005925655, + "compile_s": 0.5323582169949077, + "first_call_s": 0.009071961001609452, + "steady_per_call_s": 0.002401753098820336 + }, + "setup_prefix_1_vmap4": { + "first_call_s": 0.5513790560071357, + "batch_per_call_s": 0.0004518004992860369, + "steady_per_call_s": 0.00011295012482150923 + }, + "setup_prefix_2_vmap4": { + "first_call_s": 0.5207873860053951, + "batch_per_call_s": 0.0008705025000381283, + "steady_per_call_s": 0.00021762562500953208 + }, + "setup_prefix_3_vmap4": { + "first_call_s": 1.4985183169919765, + "batch_per_call_s": 0.900627132000227, + "steady_per_call_s": 0.22515678300005676 + }, + "full_pipeline_vmap4": { + "first_call_s": 5.479659164993791, + "batch_per_call_s": 0.9009468501011725, + "steady_per_call_s": 0.22523671252529312 + } + }, + "peak_rss_mb": 1315.86328125, + "w_tilde": { + "operator_build_s": 1.1583863529958762, + "operator_build_nufft_chunk_size": null, + "operator_shape": [ + 70, + 70 + ], + "curvature_max_rel_diff_vs_dense": 2.7493413814337997e-14, + "data_vector_max_rel_diff_vs_dense": 2.36318493749997e-16, + "figure_of_merit": -3153.94823037973, + "figure_of_merit_chain": -3153.94823037973, + "figure_of_merit_abs_diff_vs_fit_nats": 9.094947017729282e-13, + "figure_of_merit_abs_diff_vs_step_by_step_nats": 0.0, + "log_likelihood_unfloored": -3153.942384509246, + "log_likelihood_abs_diff_vs_fit_nats": 0.0, + "operator_build_peak_rss_mb": 1160.61328125, + "total_step_by_step": 0.0028418218003935184, + "chain_from_mapping_matrix_s": 0.002401753098820336, + "dense_chain_from_mapping_matrix_s": 0.8657640049990732, + "dense_transform_plus_curvature_s": 0.8567051920996164, + "w_tilde_curvature_s": 0.00032038409990491344, + "speedup_curvature_vs_dense_transform_plus_curvature": 2673.9940975656323, + "speedup_chain_vs_dense_chain": 360.4716927082591, + "note": "Measurement-only: an MGE-only fit never takes this path in the library (inversion/factory.py switches the sparse operator off when every linear object is an AbstractLinearObjFuncList). The chain row replaces steps 3-8 from the same mapping matrix; its per-call cost has no N_vis term." + }, + "w_tilde_steps": { + "Curvature matrix F~ = B\u1d40 W~ B": 0.00032038409990491344, + "Data vector D~ = B\u1d40 d~": 0.00011536469974089414, + "Reconstruction (PDIP NNLS)": 0.0022860805009258913, + "Fast chi-squared + figure of merit (constants hoisted)": 0.00011999249982181936 + }, + "vmap_batch": 4, + "steps_vmap_per_call": { + "Full pipeline": 0.22523671252529312 + }, + "setup_split_vmap": { + "Ray-trace grids": 0.00011295012482150923, + "Mapping matrix (20 Gaussians)": 0.00010467550018802285, + "Transformed mapping matrix (NUFFT)": 0.22493915737504722 + }, + "pinned_expected": -3153.948230379729, + "pinned_drift": [] +} \ No newline at end of file diff --git a/results/breakdown/interferometer/sma/mge_hpc_a100_fp64_real_scatter_main_control.png b/results/breakdown/interferometer/sma/mge_hpc_a100_fp64_real_scatter_main_control.png new file mode 100644 index 000000000..4297f0d25 Binary files /dev/null and b/results/breakdown/interferometer/sma/mge_hpc_a100_fp64_real_scatter_main_control.png differ diff --git a/results/notes/interferometer_mge_breakdown_2026_09.md b/results/notes/interferometer_mge_breakdown_2026_09.md index e130d2f7a..af3f56a98 100644 --- a/results/notes/interferometer_mge_breakdown_2026_09.md +++ b/results/notes/interferometer_mge_breakdown_2026_09.md @@ -355,3 +355,74 @@ These rows are now in `scripts/misc/vram/config.py` under `VMAP_BATCH_SPARSE`. T arms had finished. The partial JSON is not committed. The A100 alma_high leg covers the witness. - **CPU jvla** was not attempted. - **A100 alma_high** was not re-run in round 2; its JSON is round 1. + +## Lever 3: real scatter (2026-09-26) + +PyAutoArray#577 (branch `feature/interferometer-transform-real-scatter`, PyAutoArray `f9d3bc67`) +changes `TransformerNUFFT.transform_mapping_matrix` in both its JAX and NumPy branches. It now +scatters the mapping matrix into the `(n_src, N_y, N_x)` native stack in the matrix's own real +dtype, flips it, and casts to complex128 just before `nufft2d2`. Before, it cast first and then +scattered in complex128. A real-to-complex cast is exact, so the output is bit-identical. A new +library test asserts `np.array_equal` against the old formula for NumPy and under `jax.jit` on a +circular mask. + +### A100 sma, dense library path, fp64 + +RAL job 356364 ran two legs on the same node (`euclid-ral-gpu-1`), using the arms of the #308 +submit (dense library path, `--w-tilde`, `--vmap-batch 4`): +- Control: PyAutoArray `origin/main`, `1bf641e4`. +- Branch: `f9d3bc67`. + +Both legs used the same private PyAutoGalaxy `0e4b89cf` and PyAutoLens `4487eb47` (origin/main) +from `/mnt/ral/jnightin/PyAuto_branch/interferometer-transform-real-scatter/`, and the mirror's +PyAutoFit `dd9fbe0a` and PyAutoNerves `1fa613aa`. The submit is +`submit_breakdown_interferometer_mge_a100_sma_fp64_real_scatter`, and each leg asserts +`__file__`. The JSONs are `sma/mge_hpc_a100_fp64_real_scatter{,_main_control}.json`. The dashboard +regex does not read them, so the #308 `sma/mge_hpc_a100_fp64.json` row is unchanged. + +| Per call, A100 | #308 baseline (gpu-2) | control, main (gpu-1) | branch, real scatter (gpu-1) | +|---|---:|---:|---:| +| Step 3, transformed mapping matrix | 852.7 ms | 851.5 ms | **0.51 ms** | +| Total, step by step | 856.2 ms | 855.0 ms | **4.04 ms** | +| Full pipeline, single jit | 854.6 ms | 868.5 ms | **3.39 ms** | +| Full pipeline, vmap batch 4 (per call) | 231.4 ms | 225.2 ms | **1.34 ms** | +| log L (step by step) | -3153.942384509246 | -3153.942384509246 | -3153.942384509246 | + +The witness passes. Step 3 falls from 852.7 ms to 0.51 ms, a factor of about 1700, and log L is +bit-identical: the difference is 0.0 nats against both the baseline and the control. The pinned +check passed, and the `.err` has no float32 truncation. + +The whole A100 sma likelihood drops from about 0.85 s to about 4 ms. That flips the sma row of +the GPU-vs-CPU verdict (lever 4) from "GPU ~10x slower" to "GPU ~20x faster" than the laptop +CPU's ~84 ms. With the floor gone, the dense sma chain (4.0 ms) is now close to the W~ chain +(2.9 ms in this job). + +### CPU, no regression + +These are laptop JAX CPU fp64 runs. The laptop was heavily shared during them (load average +5-14), so each result is several runs in ABBA order. + +| CPU | main `1bf641e4` | branch `f9d3bc67` | +|---|---|---| +| `transform_mapping_matrix` micro (jit, min of 3), sma, 6 runs each | median 95.6 ms (80.7-100.7) | median 88.8 ms (75.0-124.1) | +| `transform_mapping_matrix` micro (jit, min of 3), alma 1M vis, 5 runs each | 15.03, 14.33, 13.92, 15.33, 15.11 s | 18.65¹, 13.05, 13.77, 14.40, 14.07 s | +| sma breakdown, full pipeline single jit, 3 runs | 100.2, 77.6, 81.1 ms | 85.1, 79.4, 75.8 ms | +| sma breakdown, step 3, 3 runs | 85.9, 72.4, 82.4 ms | 106.5, 82.9, 169.8 ms | +| log L | -3153.942384509247 | -3153.942384509247 | + +¹ The first run was taken while the laptop was under peak load (14). The four ABBA repeats that +followed put the branch at or below main. + +The isolated step-3 timing on CPU is dominated by load noise; the 169.8 ms run is one such +outlier. The single-jit full pipeline and the micro-timing medians show no CPU regression. The +committed CPU row (`mge_breakdown_sma_v2026.8.17.1.json`, 82.0 ms at step 3) was not +overwritten. + +### Not run + +- **A100 alma.** The alma dense library path still OOMs, so this change cannot show there. The + #308 alma step 3 of ~915 ms comes from the chunked-transform arm (`--transform-chunk`), and + that arm is the cell script's own per-column prototype (`_transform_script_chunked`). It still + scatters in complex128 itself and does not call `transform_mapping_matrix`, so a re-run would + measure unchanged code. Porting the real scatter into that prototype is a script change, left + for any future dense-alma work.