diff --git a/check.txt b/check.txt index 1f5f344d..6589f995 100644 --- a/check.txt +++ b/check.txt @@ -1 +1,3 @@ -All checks passed! +scripts/fill_photoz_bands.py:22:1: I001 [*] Import block is un-sorted or un-formatted +Found 1 error. +[*] 1 fixable with the `--fix` option. diff --git a/cosmo_val/cat_config.yaml b/cosmo_val/cat_config.yaml index cc1326df..b8d1a50a 100644 --- a/cosmo_val/cat_config.yaml +++ b/cosmo_val/cat_config.yaml @@ -908,7 +908,56 @@ SP_v1.4.8: dec_col: Dec e1_col: e1 e2_col: e2 - path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits +SP_v1.4.6.3_uncal: + pipeline: SP + subdir: /n17data/UNIONS/WL/v1.4.x + shear: + path: v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + ra_col: RA + dec_col: Dec + e1_col: e1_uncal + e2_col: e2_uncal + w_col: w_des + R: 1.0 + psf: + path: unions_shapepipe_psf_2024_v1.4.a.fits + hdu: 1 + patch_number: 100 + +SP_v1.4.6.3_uncal_w_iv: + pipeline: SP + subdir: /n17data/UNIONS/WL/v1.4.x + shear: + path: v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + ra_col: RA + dec_col: Dec + e1_col: e1_uncal + e2_col: e2_uncal + w_col: w_iv + R: 1.0 + psf: + path: unions_shapepipe_psf_2024_v1.4.a.fits + hdu: 1 + patch_number: 100 + +SP_v1.4.6.3_uncal_w_1: + pipeline: SP + subdir: /n17data/UNIONS/WL/v1.4.x + shear: + path: v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + ra_col: RA + dec_col: Dec + e1_col: e1_uncal + e2_col: e2_uncal + w_col: one + R: 1.0 + psf: + path: unions_shapepipe_psf_2024_v1.4.a.fits + hdu: 1 + patch_number: 100 SP_v1.4.11.2: subdir: /n17data/UNIONS/WL/v1.4.x pipeline: SP @@ -980,7 +1029,98 @@ SP_v1.4.11.3: e2_star_col: HSM_G2_STAR shear: R: 1.0 - path: /n17data/UNIONS/WL/v1.4.x/v1.4.11.3/unions_shapepipe_cut_struc_2024_v1.4.11.3.fits + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: v1.4.11.3/unions_shapepipe_cut_struc_2024_v1.4.11.3.fits + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w_des + e1_col: e1 + e1_col_corrected: e1_leak_corrected + e1_PSF_col: e1_PSF + e2_col: e2 + e2_col_corrected: e2_leak_corrected + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits +SP_v1.4.12.3: + subdir: /n17data/UNIONS/WL/v1.4.x + pipeline: SP + colour: lightblue + getdist_colour: 0.0, 0.5, 1.0 + ls: dashdot + marker: ^ + cov_th: + A: 2405.3892055695346 + n_e: 6.128201234871523 + n_psf: 0.752316232272063 + sigma_e: 0.379587601488189 + mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.6_nside_8192.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: v1.4.12.3/unions_shapepipe_cut_struc_2024_v1.4.12.3.fits + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w_des + e1_col: e1 + e1_col_corrected: e1_leak_corrected + e1_PSF_col: e1_PSF + e2_col: e2 + e2_col_corrected: e2_leak_corrected + e2_PSF_col: e2_PSF + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits +SP_v1.4.13.3: + subdir: /n17data/UNIONS/WL/v1.4.x + pipeline: SP + colour: cyan + getdist_colour: 0.0, 0.5, 1.0 + ls: dashdot + marker: h + cov_th: + A: 2405.3892055695346 + n_e: 6.128201234871523 + n_psf: 0.752316232272063 + sigma_e: 0.379587601488189 + mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.6_nside_8192.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: v1.4.13.3/unions_shapepipe_cut_struc_2024_v1.4.13.3.fits redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt w_col: w_des e1_col: e1 @@ -1038,11 +1178,12 @@ SP_v1.4.11.3_ecut07: e1_col: e1 e2_col: e2 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_star_2024_v1.4.a.fits -SP_v1.4.6_uncal: +SP_v1.4.6.3_uncal: pipeline: SP subdir: /n17data/UNIONS/WL/v1.4.x shear: - path: v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits + path: v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt ra_col: RA dec_col: Dec e1_col: e1_uncal @@ -1053,11 +1194,12 @@ SP_v1.4.6_uncal: path: unions_shapepipe_psf_2024_v1.4.a.fits hdu: 1 patch_number: 100 -SP_v1.4.6_uncal_w_iv: +SP_v1.4.6.3_uncal_w_iv: pipeline: SP subdir: /n17data/UNIONS/WL/v1.4.x shear: - path: v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits + path: v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt ra_col: RA dec_col: Dec e1_col: e1_uncal @@ -1068,11 +1210,12 @@ SP_v1.4.6_uncal_w_iv: path: unions_shapepipe_psf_2024_v1.4.a.fits hdu: 1 patch_number: 100 -SP_v1.4.6_uncal_w_1: +SP_v1.4.6.3_uncal_w_1: pipeline: SP subdir: /n17data/UNIONS/WL/v1.4.x shear: - path: v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits + path: v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt ra_col: RA dec_col: Dec e1_col: e1_uncal @@ -1128,6 +1271,7 @@ SP_v1.4.8_uncal: path: unions_shapepipe_psf_2024_v1.4.a.fits hdu: 1 patch_number: 100 +>>>>>>> upstream/develop SP_v1.4_LFmask_8k: subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/SP_LFmask pipeline: SP diff --git a/format.txt b/format.txt index a0b12d74..9e4589eb 100644 --- a/format.txt +++ b/format.txt @@ -1,2 +1,3 @@ -Would reformat: src/sp_validation/tests/test_masks.py -1 file would be reformatted, 225 files already formatted +Would reformat: scripts/check_filled_fields.py +Would reformat: scripts/fill_photoz_bands.py +2 files would be reformatted, 231 files already formatted diff --git a/report.md b/report.md index e681a67c..de9975ce 100644 --- a/report.md +++ b/report.md @@ -1,10 +1,15 @@ ### `ruff check .` -✅ clean +``` +scripts/fill_photoz_bands.py:22:1: I001 [*] Import block is un-sorted or un-formatted +Found 1 error. +[*] 1 fixable with the `--fix` option. +``` ### `ruff format --check .` ``` -Would reformat: src/sp_validation/tests/test_masks.py -1 file would be reformatted, 225 files already formatted +Would reformat: scripts/check_filled_fields.py +Would reformat: scripts/fill_photoz_bands.py +2 files would be reformatted, 231 files already formatted ``` diff --git a/scripts/check_filled_fields.py b/scripts/check_filled_fields.py new file mode 100644 index 00000000..675ad116 --- /dev/null +++ b/scripts/check_filled_fields.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Check how many entries in selected HDF5 fields are filled (value != -199). + +Reads in chunks with a progress bar and periodic fill-fraction updates. +""" + +import h5py +import numpy as np +from tqdm import tqdm + +HDF5_FILE = "unions_shapepipe_comprehensive_struc_ugriz_2024_v1.6.c.DR6.hdf5" +EMPTY_VALUE = -199 +CHUNK_SIZE = 5_000_000 # rows per chunk +REPORT_EVERY = 10 # print running fractions every N chunks + +FIELDS = [ + "Z_B", + "Z_B_MIN", + "Z_B_MAX", + "T_B", + "MAG_GAAP_0p7_u", + "MAG_GAAP_1p0_u", + "MAG_GAAP_0p7_g", + "MAG_GAAP_1p0_g", + "MAG_GAAP_0p7_r", + "MAG_GAAP_1p0_r", + "MAG_GAAP_0p7_i", + "MAG_GAAP_1p0_i", + "MAG_GAAP_0p7_z", + "MAG_GAAP_1p0_z", + "MAG_GAAP_0p7_z2", + "MAG_GAAP_1p0_z2", +] + +with h5py.File(HDF5_FILE, "r") as f: + data = f["data"] + n_total = data.shape[0] + n_chunks = (n_total + CHUNK_SIZE - 1) // CHUNK_SIZE + print(f"Total entries : {n_total:,}") + print(f"Chunk size : {CHUNK_SIZE:,} ({n_chunks} chunks)\n") + + counts = {field: 0 for field in FIELDS} + + with tqdm(total=n_total, unit="rows", unit_scale=True, desc="Reading") as pbar: + for chunk_idx in range(n_chunks): + start = chunk_idx * CHUNK_SIZE + end = min(start + CHUNK_SIZE, n_total) + + for field in FIELDS: + counts[field] += int(np.sum(data[field, start:end] != EMPTY_VALUE)) + + pbar.update(end - start) + + # Periodic running-fraction report + if (chunk_idx + 1) % REPORT_EVERY == 0 or (chunk_idx + 1) == n_chunks: + rows_done = end + tqdm.write( + f"\n --- after {rows_done:,} rows ({100 * rows_done / n_total:.1f}%) ---" + ) + tqdm.write(f" {'Field':<22} {'Filled %':>9}") + for field in FIELDS: + pct = 100.0 * counts[field] / rows_done + tqdm.write(f" {field:<22} {pct:>8.2f}%") + +# Final summary +print(f"\n{'=' * 56}") +print(f"FINAL SUMMARY (total rows: {n_total:,})") +print(f"{'Field':<22} {'Filled':>12} {'Empty':>12} {'Filled %':>10}") +print("-" * 60) +for field in FIELDS: + n_filled = counts[field] + n_empty = n_total - n_filled + pct = 100.0 * n_filled / n_total + print(f"{field:<22} {n_filled:>12,} {n_empty:>12,} {pct:>9.2f}%") diff --git a/scripts/fill_photoz_bands.py b/scripts/fill_photoz_bands.py new file mode 100755 index 00000000..442012a9 --- /dev/null +++ b/scripts/fill_photoz_bands.py @@ -0,0 +1,592 @@ +#!/usr/bin/env python + +"""fill_photoz_bands.py + +Add PhotoPipe photo-z and multi-band magnitude fields to a ShapePipe +comprehensive HDF5 catalogue and fill them from PhotoPipe FITS tiles. + +Two phases, both resumable via a checkpoint file: + + Phase 1 — Create output: copy the input HDF5 and append the new + columns initialised to EMPTY_VALUE (-199). + Phase 2 — Fill tiles: for each PhotoPipe FITS tile found in fits_dir, + write the field values into the output file. + +Skips missing FITS files. Warns (does not error) on missing PhotoPipe +keys. Supports interrupt + restart at any point. + +:Authors: Martin Kilbinger + +""" + +import json +import os +import sys +import warnings +from collections import defaultdict +from timeit import default_timer as timer + +import h5py +import numpy as np +import tqdm +from astropy.io import fits +from cs_util import args as cs_args +from cs_util import logging + +FITS_HDU = 1 +EMPTY_VALUE = -199 +COPY_CHUNK = 2_000_000 # rows per chunk when copying input → output +SCAN_CHUNK = 5_000_000 # rows per chunk when scanning TILE_ID +MAX_CONSEC_FAILS = 10 # abort if this many tiles in a row fail + +REQUESTED_KEYS = [ + "Z_B", + "Z_B_MIN", + "Z_B_MAX", + "T_B", + "MAG_GAAP_u", + "MAGERR_GAAP_u", + "MAG_GAAP_0p7_u", + "MAGERR_GAAP_0p7_u", + "MAG_GAAP_1p0_u", + "MAGERR_GAAP_1p0_u", + "FLAG_GAAP_u", + "MAG_LIM_u", + "FLUX_GAAP_u", + "FLUXERR_GAAP_u", + "EXTINCTION_u", + "MAG_GAAP_g", + "MAGERR_GAAP_g", + "MAG_GAAP_0p7_g", + "MAGERR_GAAP_0p7_g", + "MAG_GAAP_1p0_g", + "MAGERR_GAAP_1p0_g", + "FLAG_GAAP_g", + "MAG_LIM_g", + "FLUX_GAAP_g", + "FLUXERR_GAAP_g", + "EXTINCTION_g", + "MAG_GAAP_r", + "MAGERR_GAAP_r", + "MAG_GAAP_0p7_r", + "MAGERR_GAAP_0p7_r", + "MAG_GAAP_1p0_r", + "MAGERR_GAAP_1p0_r", + "FLAG_GAAP_r", + "MAG_LIM_r", + "FLUX_GAAP_r", + "FLUXERR_GAAP_r", + "EXTINCTION_r", + "MAG_GAAP_i", + "MAGERR_GAAP_i", + "MAG_GAAP_0p7_i", + "MAGERR_GAAP_0p7_i", + "MAG_GAAP_1p0_i", + "MAGERR_GAAP_1p0_i", + "FLAG_GAAP_i", + "MAG_LIM_i", + "FLUX_GAAP_i", + "FLUXERR_GAAP_i", + "EXTINCTION_i", + "MAG_GAAP_z", + "MAGERR_GAAP_z", + "MAG_GAAP_0p7_z", + "MAGERR_GAAP_0p7_z", + "MAG_GAAP_1p0_z", + "MAGERR_GAAP_1p0_z", + "FLAG_GAAP_z", + "MAG_LIM_z", + "FLUX_GAAP_z", + "FLUXERR_GAAP_z", + "EXTINCTION_z", + "MAG_GAAP_z2", + "MAGERR_GAAP_z2", + "MAG_GAAP_0p7_z2", + "MAGERR_GAAP_0p7_z2", + "MAG_GAAP_1p0_z2", + "MAGERR_GAAP_1p0_z2", + "FLAG_GAAP_z2", + "MAG_LIM_z2", + "FLUX_GAAP_z2", + "FLUXERR_GAAP_z2", + "EXTINCTION_z2", + "EXTINCTION", + "ODDS", + "CHI_SQUARED_BPZ", + "M_0", + "BPZ_FILT", + "BPZ_NONDETFILT", + "BPZ_FLAGFILT", +] + + +def params_default(): + """Params Default. + + Return default parameter values and additional information + about type and command line options. + + Returns + ------- + tuple + parameter dict, short_options dict, types dict, help_strings dict + + """ + params = { + "input": "unions_shapepipe_comprehensive_struc_2024_v1.5.c.hdf5", + "output": "unions_shapepipe_comprehensive_struc_ugriz_2024_v1.5.c.hdf5", + "fits_dir": "UNIONS_DR6", + "checkpoint": "fill_photoz_bands_checkpoint.json", + "verbose": False, + } + + short_options = { + "input": "-i", + "output": "-o", + "fits_dir": "-d", + "checkpoint": "-c", + } + + types = {} + + help_strings = { + "input": "input HDF5 catalogue (no PhotoPipe fields), default={}", + "output": "output HDF5 catalogue (PhotoPipe fields added and filled), default={}", + "fits_dir": "directory with PhotoPipe FITS tiles, default={}", + "checkpoint": "checkpoint JSON file for resume support, default={}", + } + + return params, short_options, types, help_strings + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def detect_dataset_name(hf): + """Return the first dataset name in an HDF5 file. + + Tries common names first: dat, dat_comb, data. + + Parameters + ---------- + hf : h5py.File + + Returns + ------- + str + Dataset name. + + Raises + ------ + KeyError + If no dataset is found. + + """ + for name in ("dat", "dat_comb", "data"): + if name in hf: + return name + # Fall back to first key + keys = list(hf.keys()) + if not keys: + raise KeyError("HDF5 file contains no datasets.") + return keys[0] + + +def strip_dtype(dtype): + """Strip h5py metadata from a structured numpy dtype descriptor. + + h5py sometimes adds encoding metadata to dtype descriptions, e.g. + ``(' 1)[0] + 1 + blocks = np.split(sorted_idx, gaps) + fits_offset = 0 + for block in blocks: + b_min, b_max = int(block[0]), int(block[-1]) + n_block = b_max - b_min + 1 + chunk = dset[b_min : b_max + 1] + for key in valid_keys: + chunk[key] = fits_data[key][fits_offset : fits_offset + n_block] + dset[b_min : b_max + 1] = chunk + fits_offset += n_block + + +# --------------------------------------------------------------------------- +# Phase 1: create output file +# --------------------------------------------------------------------------- + + +def create_output_file(input_path, output_path, dataset_name, verbose=False): + """Create output HDF5 by copying input and appending empty PhotoPipe fields. + + Parameters + ---------- + input_path : str + output_path : str + dataset_name : str + Dataset name in the input file (used for output too). + verbose : bool + + """ + print(f"Phase 1: creating output file '{output_path}'") + t0 = timer() + + with h5py.File(input_path, "r") as hf_in: + dset_in = hf_in[dataset_name] + n_total = dset_in.shape[0] + dtype_out = build_output_dtype(dset_in.dtype, REQUESTED_KEYS) + new_keys = [k for k in REQUESTED_KEYS if k not in set(dset_in.dtype.names)] + + print(f" Input rows : {n_total:,}") + print(f" Input fields : {len(dset_in.dtype.names)}") + print(f" New fields : {new_keys}") + size_mb = n_total * dtype_out.itemsize / 1_048_576 + print(f" Output size : ~{size_mb:,.0f} MB") + + with h5py.File(output_path, "w") as hf_out: + dset_out = hf_out.create_dataset( + dataset_name, + shape=(n_total,), + dtype=dtype_out, + ) + + # Copy input fields chunk by chunk + input_fields = dset_in.dtype.names + with tqdm.tqdm( + total=n_total, unit="rows", unit_scale=True, desc=" Copying" + ) as pbar: + for start in range(0, n_total, COPY_CHUNK): + end = min(start + COPY_CHUNK, n_total) + chunk_in = dset_in[start:end] + chunk_out = np.empty(end - start, dtype=dtype_out) + for field in input_fields: + chunk_out[field] = chunk_in[field] + for key in new_keys: + chunk_out[key] = EMPTY_VALUE + dset_out[start:end] = chunk_out + pbar.update(end - start) + + # Copy all other top-level datasets/groups unchanged + for key in hf_in.keys(): + if key != dataset_name: + hf_in.copy(key, hf_out) + if verbose: + print(f" Copied group/dataset '{key}' unchanged.") + + elapsed = timer() - t0 + print(f" Done in {elapsed:.1f}s\n") + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +def main(argv=None): + """Main. + + Main program. + + """ + params, short_options, types, help_strings = params_default() + + options = cs_args.parse_options(params, short_options, types, help_strings) + params.update(options) + + logging.log_command(argv) + + verbose = params["verbose"] + + if not os.path.exists(params["input"]): + print(f"ERROR: input file not found: {params['input']}", file=sys.stderr) + return 1 + if not os.path.isdir(params["fits_dir"]): + print(f"ERROR: FITS directory not found: {params['fits_dir']}", file=sys.stderr) + return 1 + + # ------------------------------------------------------------------ + # Load checkpoint + # ------------------------------------------------------------------ + if os.path.exists(params["checkpoint"]): + with open(params["checkpoint"]) as f: + checkpoint = json.load(f) + done_tiles = set(checkpoint.get("done_tiles", [])) + print(f"Resuming: {len(done_tiles)} tiles already completed.") + else: + done_tiles = set() + checkpoint = {} + + # ------------------------------------------------------------------ + # Detect input dataset name (cache in checkpoint) + # ------------------------------------------------------------------ + if "dataset_name" in checkpoint: + dataset_name = checkpoint["dataset_name"] + else: + with h5py.File(params["input"], "r") as hf: + dataset_name = detect_dataset_name(hf) + checkpoint["dataset_name"] = dataset_name + with open(params["checkpoint"], "w") as cf: + json.dump(checkpoint, cf) + + if verbose: + print(f"Input dataset: '{dataset_name}'") + + # ------------------------------------------------------------------ + # Phase 1: create output file if needed + # ------------------------------------------------------------------ + if not checkpoint.get("output_created", False): + if os.path.exists(params["output"]): + print( + f"WARNING: output file '{params['output']}' exists but checkpoint " + "does not mark it as complete. Overwriting." + ) + create_output_file( + params["input"], params["output"], dataset_name, verbose=verbose + ) + checkpoint["output_created"] = True + with open(params["checkpoint"], "w") as cf: + json.dump(checkpoint, cf) + else: + if verbose: + print("Phase 1 already done (output file exists in checkpoint).") + + # ------------------------------------------------------------------ + # Phase 2: fill tiles + # ------------------------------------------------------------------ + t0 = timer() + with h5py.File(params["output"], "r+") as hf: + dset = hf[dataset_name] + n_total = dset.shape[0] + print(f"Phase 2: filling tiles in '{params['output']}'") + print(f" {n_total:,} rows, dataset '{dataset_name}'") + + hdf5_fields = set(dset.dtype.names) + + # Check all requested keys are present + for key in REQUESTED_KEYS: + if key not in hdf5_fields: + warnings.warn( + f"Key '{key}' missing from output dataset — was Phase 1 complete?" + ) + + # Build TILE_ID → output row indices (always scanned; too large for checkpoint) + print(" Building tile→index map (scans all rows)...") + tile_index_map_lists = defaultdict(list) + with tqdm.tqdm( + total=n_total, unit="rows", unit_scale=True, desc=" Scanning TILE_ID" + ) as pbar: + for start in range(0, n_total, SCAN_CHUNK): + end = min(start + SCAN_CHUNK, n_total) + tile_chunk = dset[start:end]["TILE_ID"] + for local_i, tid in enumerate(tile_chunk): + tile_index_map_lists[tid].append(start + local_i) + pbar.update(end - start) + + tile_index_map = { + tid: np.array(idxs, dtype=np.int64) + for tid, idxs in tile_index_map_lists.items() + } + print(f" Map built: {len(tile_index_map)} unique tiles.") + + unique_tiles = sorted(tile_index_map.keys()) + n_tiles = len(unique_tiles) + + valid_keys = None # determined from first available FITS tile + n_skipped_missing = 0 + n_skipped_done = 0 + n_skipped_size = 0 + n_errors = 0 + n_consec_fails = 0 + n_processed = 0 + + print(f"\n Processing {n_tiles} tiles ({len(done_tiles)} already done)...\n") + pbar = tqdm.tqdm(unique_tiles, total=n_tiles, unit="tile") + + for tile_id in pbar: + if n_consec_fails >= MAX_CONSEC_FAILS: + checkpoint["done_tiles"] = list(done_tiles) + with open(params["checkpoint"], "w") as cf: + json.dump(checkpoint, cf) + print( + f"\nERROR: {n_consec_fails} tiles failed in a row, " + "likely a systematic problem; aborting.", + file=sys.stderr, + ) + sys.exit(1) + + tile_str = tile_id.decode() if isinstance(tile_id, bytes) else tile_id + + if tile_str in done_tiles: + n_skipped_done += 1 + continue + + fits_path = tile_id_to_fits_path(tile_id, params["fits_dir"]) + if not os.path.exists(fits_path): + n_skipped_missing += 1 + done_tiles.add(tile_str) + pbar.set_postfix({"done": n_processed, "missing": n_skipped_missing}) + continue + + try: + with fits.open(fits_path, memmap=True) as hdu_list: + fits_data = hdu_list[FITS_HDU].data + + # Validate keys on first successfully opened tile + if valid_keys is None: + valid_keys, missing_keys = check_fits_keys( + fits_data.dtype.names, REQUESTED_KEYS + ) + valid_keys = [k for k in valid_keys if k in hdf5_fields] + if missing_keys: + warnings.warn( + f"Keys absent from PhotoPipe FITS (skipped): {missing_keys}" + ) + tqdm.tqdm.write(f"\n Keys to fill: {valid_keys}\n") + + hdf5_indices = tile_index_map[tile_id] + n_hdf5 = len(hdf5_indices) + n_fits = len(fits_data) + + if n_hdf5 != n_fits: + warnings.warn( + f"Tile {tile_str}: HDF5 has {n_hdf5} rows, FITS has " + f"{n_fits} — size mismatch, skipping." + ) + n_skipped_size += 1 + n_consec_fails += 1 + pbar.set_postfix( + {"done": n_processed, "size_err": n_skipped_size} + ) + continue + + write_tile_to_hdf5(dset, hdf5_indices, fits_data, valid_keys) + + except Exception as e: + warnings.warn(f"Tile {tile_str}: error ({e}), skipping.") + n_errors += 1 + n_consec_fails += 1 + continue + + n_processed += 1 + n_consec_fails = 0 + done_tiles.add(tile_str) + + if n_processed % 50 == 0: + checkpoint["done_tiles"] = list(done_tiles) + with open(params["checkpoint"], "w") as cf: + json.dump(checkpoint, cf) + + pbar.set_postfix({"done": n_processed, "missing": n_skipped_missing}) + + # Final checkpoint flush + checkpoint["done_tiles"] = list(done_tiles) + with open(params["checkpoint"], "w") as cf: + json.dump(checkpoint, cf) + + elapsed = timer() - t0 + print(f"\nDone in {elapsed:.1f}s") + print(f" Tiles processed : {n_processed}") + print(f" Tiles skipped (done) : {n_skipped_done}") + print(f" FITS files missing : {n_skipped_missing}") + print(f" Size mismatches : {n_skipped_size}") + print(f" Tiles failed (error) : {n_errors}") + if n_processed == 0 and (n_errors > 0 or n_skipped_size > 0): + print( + "WARNING: no tiles were filled; all available tiles failed.", + file=sys.stderr, + ) + + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv))