PhotoPipe + ShapePipe

check.txt

@@ -1 +1,3 @@
-All checks passed!
+scripts/fill_photoz_bands.py:22:1: I001 [*] Import block is un-sorted or un-formatted
+Found 1 error.
+[*] 1 fixable with the `--fix` option.

cosmo_val/cat_config.yaml

@@ -908,7 +908,56 @@ SP_v1.4.8:
     dec_col: Dec
     e1_col: e1
     e2_col: e2
-    path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits
+SP_v1.4.6.3_uncal:
+   pipeline: SP
+   subdir: /n17data/UNIONS/WL/v1.4.x
+   shear:
+      path: v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits
+      covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt
+      ra_col: RA
+      dec_col: Dec
+      e1_col: e1_uncal
+      e2_col: e2_uncal
+      w_col: w_des
+      R: 1.0
+   psf:
+      path: unions_shapepipe_psf_2024_v1.4.a.fits
+      hdu: 1
+   patch_number: 100
+
+SP_v1.4.6.3_uncal_w_iv:
+   pipeline: SP
+   subdir: /n17data/UNIONS/WL/v1.4.x
+   shear:
+      path: v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits
+      covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt
+      ra_col: RA
+      dec_col: Dec
+      e1_col: e1_uncal
+      e2_col: e2_uncal
+      w_col: w_iv
+      R: 1.0
+   psf:
+      path: unions_shapepipe_psf_2024_v1.4.a.fits
+      hdu: 1
+   patch_number: 100
+
+SP_v1.4.6.3_uncal_w_1:
+   pipeline: SP
+   subdir: /n17data/UNIONS/WL/v1.4.x
+   shear:
+      path: v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits
+      covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt
+      ra_col: RA
+      dec_col: Dec
+      e1_col: e1_uncal
+      e2_col: e2_uncal
+      w_col: one
+      R: 1.0
+   psf:
+      path: unions_shapepipe_psf_2024_v1.4.a.fits
+      hdu: 1
+   patch_number: 100
 SP_v1.4.11.2:
   subdir: /n17data/UNIONS/WL/v1.4.x
   pipeline: SP
@@ -980,7 +1029,98 @@ SP_v1.4.11.3:
     e2_star_col: HSM_G2_STAR
   shear:
     R: 1.0
-    path: /n17data/UNIONS/WL/v1.4.x/v1.4.11.3/unions_shapepipe_cut_struc_2024_v1.4.11.3.fits
+    covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt
+    path: v1.4.11.3/unions_shapepipe_cut_struc_2024_v1.4.11.3.fits
+    redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt
+    w_col: w_des
+    e1_col: e1
+    e1_col_corrected: e1_leak_corrected
+    e1_PSF_col: e1_PSF
+    e2_col: e2
+    e2_col_corrected: e2_leak_corrected
+    e2_PSF_col: e2_PSF
+  star:
+    ra_col: RA
+    dec_col: Dec
+    e1_col: e1
+    e2_col: e2
+    path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits
+SP_v1.4.12.3:
+  subdir: /n17data/UNIONS/WL/v1.4.x
+  pipeline: SP
+  colour: lightblue
+  getdist_colour: 0.0, 0.5, 1.0
+  ls: dashdot
+  marker: ^
+  cov_th:
+    A: 2405.3892055695346
+    n_e: 6.128201234871523
+    n_psf: 0.752316232272063
+    sigma_e: 0.379587601488189
+  mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.6_nside_8192.fits
+  psf:
+    PSF_flag: FLAG_PSF_HSM
+    PSF_size: SIGMA_PSF_HSM
+    square_size: true
+    star_flag: FLAG_STAR_HSM
+    star_size: SIGMA_STAR_HSM
+    hdu: 1
+    path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits
+    ra_col: RA
+    dec_col: Dec
+    e1_PSF_col: E1_PSF_HSM
+    e1_star_col: E1_STAR_HSM
+    e2_PSF_col: E2_PSF_HSM
+    e2_star_col: E2_STAR_HSM
+  shear:
+    R: 1.0
+    covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt
+    path: v1.4.12.3/unions_shapepipe_cut_struc_2024_v1.4.12.3.fits
+    redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt
+    w_col: w_des
+    e1_col: e1
+    e1_col_corrected: e1_leak_corrected
+    e1_PSF_col: e1_PSF
+    e2_col: e2
+    e2_col_corrected: e2_leak_corrected
+    e2_PSF_col: e2_PSF
+  star:
+    ra_col: RA
+    dec_col: Dec
+    e1_col: e1
+    e2_col: e2
+    path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits
+SP_v1.4.13.3:
+  subdir: /n17data/UNIONS/WL/v1.4.x
+  pipeline: SP
+  colour: cyan
+  getdist_colour: 0.0, 0.5, 1.0
+  ls: dashdot
+  marker: h
+  cov_th:
+    A: 2405.3892055695346
+    n_e: 6.128201234871523
+    n_psf: 0.752316232272063
+    sigma_e: 0.379587601488189
+  mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.6_nside_8192.fits
+  psf:
+    PSF_flag: FLAG_PSF_HSM
+    PSF_size: SIGMA_PSF_HSM
+    square_size: true
+    star_flag: FLAG_STAR_HSM
+    star_size: SIGMA_STAR_HSM
+    hdu: 1
+    path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits
+    ra_col: RA
+    dec_col: Dec
+    e1_PSF_col: E1_PSF_HSM
+    e1_star_col: E1_STAR_HSM
+    e2_PSF_col: E2_PSF_HSM
+    e2_star_col: E2_STAR_HSM
+  shear:
+    R: 1.0
+    covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt
+    path: v1.4.13.3/unions_shapepipe_cut_struc_2024_v1.4.13.3.fits
     redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt
     w_col: w_des
     e1_col: e1
@@ -1038,11 +1178,12 @@ SP_v1.4.11.3_ecut07:
     e1_col: e1
     e2_col: e2
     path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits
-SP_v1.4.6_uncal:
+SP_v1.4.6.3_uncal:
   pipeline: SP
   subdir: /n17data/UNIONS/WL/v1.4.x
   shear:
-    path: v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits
+    path: v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits
+    covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt
     ra_col: RA
     dec_col: Dec
     e1_col: e1_uncal
@@ -1053,11 +1194,12 @@ SP_v1.4.6_uncal:
     path: unions_shapepipe_psf_2024_v1.4.a.fits
     hdu: 1
   patch_number: 100
-SP_v1.4.6_uncal_w_iv:
+SP_v1.4.6.3_uncal_w_iv:
   pipeline: SP
   subdir: /n17data/UNIONS/WL/v1.4.x
   shear:
-    path: v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits
+    path: v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits
+    covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt
     ra_col: RA
     dec_col: Dec
     e1_col: e1_uncal
@@ -1068,11 +1210,12 @@ SP_v1.4.6_uncal_w_iv:
     path: unions_shapepipe_psf_2024_v1.4.a.fits
     hdu: 1
   patch_number: 100
-SP_v1.4.6_uncal_w_1:
+SP_v1.4.6.3_uncal_w_1:
   pipeline: SP
   subdir: /n17data/UNIONS/WL/v1.4.x
   shear:
-    path: v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits
+    path: v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits
+    covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt
     ra_col: RA
     dec_col: Dec
     e1_col: e1_uncal
@@ -1128,6 +1271,7 @@ SP_v1.4.8_uncal:
     path: unions_shapepipe_psf_2024_v1.4.a.fits
     hdu: 1
   patch_number: 100
+>>>>>>> upstream/develop
 SP_v1.4_LFmask_8k:
   subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask
   pipeline: SP

format.txt

@@ -1,2 +1,3 @@
-Would reformat: src/sp_validation/tests/test_masks.py
-1 file would be reformatted, 225 files already formatted
+Would reformat: scripts/check_filled_fields.py
+Would reformat: scripts/fill_photoz_bands.py
+2 files would be reformatted, 231 files already formatted

report.md

@@ -1,10 +1,15 @@
 ### `ruff check .`
 
-✅ clean
+```
+scripts/fill_photoz_bands.py:22:1: I001 [*] Import block is un-sorted or un-formatted
+Found 1 error.
+[*] 1 fixable with the `--fix` option.
+```
 
 ### `ruff format --check .`
 
 ```
-Would reformat: src/sp_validation/tests/test_masks.py
-1 file would be reformatted, 225 files already formatted
+Would reformat: scripts/check_filled_fields.py
+Would reformat: scripts/fill_photoz_bands.py
+2 files would be reformatted, 231 files already formatted
 ```

scripts/check_filled_fields.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+"""Check how many entries in selected HDF5 fields are filled (value != -199).
+
+Reads in chunks with a progress bar and periodic fill-fraction updates.
+"""
+
+import h5py
+import numpy as np
+from tqdm import tqdm
+
+HDF5_FILE = "unions_shapepipe_comprehensive_struc_ugriz_2024_v1.6.c.DR6.hdf5"
+EMPTY_VALUE = -199
+CHUNK_SIZE = 5_000_000  # rows per chunk
+REPORT_EVERY = 10  # print running fractions every N chunks
+
+FIELDS = [
+    "Z_B",
+    "Z_B_MIN",
+    "Z_B_MAX",
+    "T_B",
+    "MAG_GAAP_0p7_u",
+    "MAG_GAAP_1p0_u",
+    "MAG_GAAP_0p7_g",
+    "MAG_GAAP_1p0_g",
+    "MAG_GAAP_0p7_r",
+    "MAG_GAAP_1p0_r",
+    "MAG_GAAP_0p7_i",
+    "MAG_GAAP_1p0_i",
+    "MAG_GAAP_0p7_z",
+    "MAG_GAAP_1p0_z",
+    "MAG_GAAP_0p7_z2",
+    "MAG_GAAP_1p0_z2",
+]
+
+with h5py.File(HDF5_FILE, "r") as f:
+    data = f["data"]
+    n_total = data.shape[0]
+    n_chunks = (n_total + CHUNK_SIZE - 1) // CHUNK_SIZE
+    print(f"Total entries : {n_total:,}")
+    print(f"Chunk size    : {CHUNK_SIZE:,}  ({n_chunks} chunks)\n")
+
+    counts = {field: 0 for field in FIELDS}
+
+    with tqdm(total=n_total, unit="rows", unit_scale=True, desc="Reading") as pbar:
+        for chunk_idx in range(n_chunks):
+            start = chunk_idx * CHUNK_SIZE
+            end = min(start + CHUNK_SIZE, n_total)
+
+            for field in FIELDS:
+                counts[field] += int(np.sum(data[field, start:end] != EMPTY_VALUE))
+
+            pbar.update(end - start)
+
+            # Periodic running-fraction report
+            if (chunk_idx + 1) % REPORT_EVERY == 0 or (chunk_idx + 1) == n_chunks:
+                rows_done = end
+                tqdm.write(
+                    f"\n  --- after {rows_done:,} rows ({100 * rows_done / n_total:.1f}%) ---"
+                )
+                tqdm.write(f"  {'Field':<22} {'Filled %':>9}")
+                for field in FIELDS:
+                    pct = 100.0 * counts[field] / rows_done
+                    tqdm.write(f"  {field:<22} {pct:>8.2f}%")
+
+# Final summary
+print(f"\n{'=' * 56}")
+print(f"FINAL SUMMARY  (total rows: {n_total:,})")
+print(f"{'Field':<22} {'Filled':>12} {'Empty':>12} {'Filled %':>10}")
+print("-" * 60)
+for field in FIELDS:
+    n_filled = counts[field]
+    n_empty = n_total - n_filled
+    pct = 100.0 * n_filled / n_total
+    print(f"{field:<22} {n_filled:>12,} {n_empty:>12,} {pct:>9.2f}%")