From aa4c73b6865d3df3c47885a2779c3560a57b976b Mon Sep 17 00:00:00 2001 From: oczoske Date: Tue, 5 May 2026 13:12:16 +0200 Subject: [PATCH 001/130] Remove offset hack --- scopesim/effects/spectral_trace_list.py | 25 +++++++++++-------- scopesim/effects/spectral_trace_list_utils.py | 11 +++++--- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/scopesim/effects/spectral_trace_list.py b/scopesim/effects/spectral_trace_list.py index d455fc088..0fdeb3a78 100644 --- a/scopesim/effects/spectral_trace_list.py +++ b/scopesim/effects/spectral_trace_list.py @@ -245,16 +245,21 @@ def apply_to(self, obj, **kwargs): obj.cube = obj.make_hdu() # Check whether an offset slit is used. If so, recompute spectral traces. - offset_x = obj.cube.header["CRVAL1D"] - offset_y = obj.cube.header["CRVAL2D"] - if (offset_x != self.meta["offset_x"] or - offset_y != self.meta["offset_y"]): - logger.debug("Recomputing spectral traces for offset (%.1g, %.1g)", - offset_x, offset_y) - self.meta["offset_x"] = offset_x - self.meta["offset_y"] = offset_y - self.make_spectral_traces() - self.update_meta() + # TODO: this needs to be removed. If transforms are available that + # can be applied to the actual slit position, they ought be used + # without modification. If they are not available but need to be + # hacked, then that hack needs to be defined via the yaml and only + # be used in a particular case, not in general. + #offset_x = obj.cube.header["CRVAL1D"] + #offset_y = obj.cube.header["CRVAL2D"] + #if (offset_x != self.meta["offset_x"] or + # offset_y != self.meta["offset_y"]): + # logger.debug("Recomputing spectral traces for offset (%.1g, %.1g)", + # offset_x, offset_y) + # self.meta["offset_x"] = offset_x + # self.meta["offset_y"] = offset_y + # self.make_spectral_traces() + # self.update_meta() spt = self.spectral_traces[obj.trace_id] obj.hdu = spt.map_spectra_to_focal_plane(obj) diff --git a/scopesim/effects/spectral_trace_list_utils.py b/scopesim/effects/spectral_trace_list_utils.py index 7541f9cbf..343718894 100644 --- a/scopesim/effects/spectral_trace_list_utils.py +++ b/scopesim/effects/spectral_trace_list_utils.py @@ -122,17 +122,20 @@ def compute_interpolation_functions(self): `xi` (spatial coordinate along the slit, in arcsec) and `lam` (wavelength, in um). - The interpolation functions include a shift in the focal-plane - coordinates, determined from the CRVAL of the source FOV. + #The interpolation functions include a shift in the focal-plane + #coordinates, determined from the CRVAL of the source FOV. """ - x_arr = self.table[self.meta["x_colname"]] + self.meta["offset_x"] - y_arr = self.table[self.meta["y_colname"]] + self.meta["offset_y"] + x_arr = self.table[self.meta["x_colname"]] + y_arr = self.table[self.meta["y_colname"]] + xi_arr = self.table[self.meta["s_colname"]] lam_arr = self.table[self.meta["wave_colname"]] self.wave_min = quantify(np.min(lam_arr), u.um).value self.wave_max = quantify(np.max(lam_arr), u.um).value + # TODO There should be an option to include pre- and + # posttransforms. But how should they be defined? self.xy2xi = Transform2D.fit(x_arr, y_arr, xi_arr) self.xy2lam = Transform2D.fit(x_arr, y_arr, lam_arr) self.xilam2x = Transform2D.fit(xi_arr, lam_arr, x_arr) From 43dda59f9255efd397bf6f001c889593d11ea1e5 Mon Sep 17 00:00:00 2001 From: oczoske Date: Thu, 7 May 2026 16:15:03 +0200 Subject: [PATCH 002/130] Different approach to offset slits --- scopesim/effects/apertures.py | 8 ++++++++ scopesim/effects/spectral_trace_list.py | 19 ++++++++++++++++--- scopesim/effects/spectral_trace_list_utils.py | 8 ++++++-- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/scopesim/effects/apertures.py b/scopesim/effects/apertures.py index fd6984a5b..d704265bb 100644 --- a/scopesim/effects/apertures.py +++ b/scopesim/effects/apertures.py @@ -129,6 +129,14 @@ def apply_to(self, obj, **kwargs): vol["meta"]["xi_min"] = min(x) * u.arcsec vol["meta"]["xi_max"] = max(x) * u.arcsec + if "offset_type" in self.meta: + print("Yes, got it: ", self.meta["offset_type"]) + vol["meta"]["offset_type"] = self.meta["offset_type"] + vol["meta"]["offset_x"] = self.meta["offset_x"] + vol["meta"]["offset_y"] = self.meta["offset_y"] + else: + vol["meta"]["offset_type"] = "none" + print("What is offset_type?") return obj # Outdated. Remove when removing all old FOVManager code from effects diff --git a/scopesim/effects/spectral_trace_list.py b/scopesim/effects/spectral_trace_list.py index 0fdeb3a78..3ec77f55e 100644 --- a/scopesim/effects/spectral_trace_list.py +++ b/scopesim/effects/spectral_trace_list.py @@ -16,7 +16,8 @@ from .effects import Effect from .ter_curves import FilterCurve -from .spectral_trace_list_utils import SpectralTrace, make_image_interpolations +from .spectral_trace_list_utils import (SpectralTrace, make_image_interpolations, + det_offset) from ..optics.image_plane_utils import header_from_list_of_xy from ..optics.fov import FieldOfView from ..optics.fov_volume_list import FovVolumeList @@ -125,8 +126,6 @@ def __init__(self, **kwargs): "x_colname": "x", "y_colname": "y", "s_colname": "s", - "offset_x": 0, # [mm] in detector plane - "offset_y": 0, # [mm] in detector plane "wave_colname": "wavelength", "center_on_wave_mid": False, "dwave": 0.002, # [um] for finding the best fit dispersion @@ -223,6 +222,7 @@ def apply_to(self, obj, **kwargs): aperture_id=vol["aperture_id"]) for ex_vol in extracted_vols: + print("Extracted volume:", ex_vol["meta"]["offset_type"]) ex_vol["meta"].update(vol) ex_vol["meta"].pop("wave_min") ex_vol["meta"].pop("wave_max") @@ -262,8 +262,21 @@ def apply_to(self, obj, **kwargs): # self.update_meta() spt = self.spectral_traces[obj.trace_id] + remove_pre = False + if ("offset_type" in obj.meta and + obj.meta["offset_type"] == "detector"): + spt.xy2lam.pretransform_x = (det_offset, + {"offset": obj.meta["offset_x"]}) + spt.xy2lam.pretransform_y = (det_offset, + {"offset": obj.meta["offset_y"]}) + remove_pre = True + obj.hdu = spt.map_spectra_to_focal_plane(obj) + if remove_pre: + spt.xy2lam.pretransform_y = None + spt.xy2xi.pretransform_x = None + logger.debug("%s done", self.display_name) return obj diff --git a/scopesim/effects/spectral_trace_list_utils.py b/scopesim/effects/spectral_trace_list_utils.py index 343718894..1fc8e999f 100644 --- a/scopesim/effects/spectral_trace_list_utils.py +++ b/scopesim/effects/spectral_trace_list_utils.py @@ -49,8 +49,6 @@ class SpectralTrace: "x_colname": "x", "y_colname": "y", "s_colname": "s", - "offset_x": 0, - "offset_y": 0, "wave_colname": "wavelength", "dwave": 0.002, "aperture_id": 0, @@ -114,6 +112,7 @@ def fov_grid(self): return {"wave_min": wave_min, "wave_max": wave_max, "trace_id": self.trace_id, "aperture_id": aperture_id} + def compute_interpolation_functions(self): """ Compute various interpolation functions between slit and focal plane. @@ -134,6 +133,7 @@ def compute_interpolation_functions(self): self.wave_min = quantify(np.min(lam_arr), u.um).value self.wave_max = quantify(np.max(lam_arr), u.um).value + # TODO There should be an option to include pre- and # posttransforms. But how should they be defined? self.xy2xi = Transform2D.fit(x_arr, y_arr, xi_arr) @@ -1045,3 +1045,7 @@ def get_affine_parameters(coords): shears = (np.average(shears, axis=0) * rad2deg) - (90 + rotations) return rotations, shears + +def det_offset(x, offset): + """Apply offset to x""" + return x + offset From a5bd2b2dee5317d1a068f63bef4db88a9f5707d0 Mon Sep 17 00:00:00 2001 From: oczoske Date: Sat, 16 May 2026 12:36:06 +0200 Subject: [PATCH 003/130] Remove stray print that caused test failures --- scopesim/effects/spectral_trace_list.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scopesim/effects/spectral_trace_list.py b/scopesim/effects/spectral_trace_list.py index 3ec77f55e..a28973f15 100644 --- a/scopesim/effects/spectral_trace_list.py +++ b/scopesim/effects/spectral_trace_list.py @@ -222,7 +222,6 @@ def apply_to(self, obj, **kwargs): aperture_id=vol["aperture_id"]) for ex_vol in extracted_vols: - print("Extracted volume:", ex_vol["meta"]["offset_type"]) ex_vol["meta"].update(vol) ex_vol["meta"].pop("wave_min") ex_vol["meta"].pop("wave_max") From 418acadcd43270dd07a87cfcb4a6015c3d70e9cb Mon Sep 17 00:00:00 2001 From: oczoske Date: Sat, 16 May 2026 18:24:18 +0200 Subject: [PATCH 004/130] Start MicadoIFUSpectralTraceList --- scopesim/effects/__init__.py | 1 + scopesim/effects/micado_ifu_trace_list.py | 63 +++++++++++++++++++++++ scopesim/effects/spectral_trace_list.py | 2 + 3 files changed, 66 insertions(+) create mode 100644 scopesim/effects/micado_ifu_trace_list.py diff --git a/scopesim/effects/__init__.py b/scopesim/effects/__init__.py index da03ffc0f..170b41924 100644 --- a/scopesim/effects/__init__.py +++ b/scopesim/effects/__init__.py @@ -8,6 +8,7 @@ from .spectral_trace_list import * from .spectral_efficiency import * from .metis_lms_trace_list import * +from .micado_ifu_trace_list import * from .mosaic_trace_list import * from .surface_list import * from .ter_curves import * diff --git a/scopesim/effects/micado_ifu_trace_list.py b/scopesim/effects/micado_ifu_trace_list.py new file mode 100644 index 000000000..826399e1f --- /dev/null +++ b/scopesim/effects/micado_ifu_trace_list.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +"""SpectralTraceList and SpectralTrace for the MICADO IFU.""" + +import copy +import warnings + +from tqdm.auto import tqdm +import numpy as np +from scipy.interpolate import RectBivariateSpline + +from astropy.io import fits +from astropy.io import ascii as ioascii +from astropy.table import Table +from astropy.wcs import WCS +from astropy import units as u + +from ..utils import from_currsys, find_file, quantify, get_logger +from .spectral_trace_list import SpectralTraceList +from .spectral_trace_list_utils import SpectralTrace +from .spectral_trace_list_utils import Transform2D +from .spectral_trace_list_utils import make_image_interpolations +from .apertures import ApertureMask +from .ter_curves import TERCurve +from ..optics.fov import FieldOfView, FieldOfView3D +from ..optics.fov_volume_list import FovVolumeList + +logger = get_logger(__name__) + + +class MicadoIFUSpectralTraceList(SpectralTraceList): + """SpectralTraceList for the MICADO IFU.""" + + _class_params = { + "naxis1": 112, + "nslice": 32, + "slicewidth": 0.012, # arcsec + } + + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + # field of view of the instrument + + def make_spectral_traces(self): + """Make a spectral trace for each combination of order and aperture""" + self.ext_data = self._file[0].header["EDATA"] + self.ext_cat = self._file[0].header["ECAT"] + self.catalog = Table(self._file[self.ext_cat].data) + self.slicelist = self._file["Aperture List"].data + spec_traces = {} + for row in self.catalog: + if row["image_plane_id"] == -99: + continue + params = {col: row[col] for col in row.colnames} + params.update(self.meta) + hdu = self._file[row["extension_id"]] + for apid in self.slicelist["id"]: + specid = f"{row['description']}_{apid:02d}" + params["aperture_id"] = apid + spec_traces[specid] = SpectralTrace(hdu, **params) + + self.spectral_traces = spec_traces diff --git a/scopesim/effects/spectral_trace_list.py b/scopesim/effects/spectral_trace_list.py index a28973f15..48e5abefa 100644 --- a/scopesim/effects/spectral_trace_list.py +++ b/scopesim/effects/spectral_trace_list.py @@ -148,6 +148,8 @@ def make_spectral_traces(self): self.catalog = Table(self._file[self.ext_cat].data) spec_traces = {} for row in self.catalog: + if row["image_plane_id"] == -99: + continue params = {col: row[col] for col in row.colnames} params.update(self.meta) hdu = self._file[row["extension_id"]] From 31ce32c66b5ef4d46cb1332db56857d872e9c55e Mon Sep 17 00:00:00 2001 From: oczoske Date: Mon, 18 May 2026 14:23:47 +0200 Subject: [PATCH 005/130] Split apply_to into separate methods for setup and observe --- scopesim/effects/metis_lms_trace_list.py | 173 +++++++++++----------- scopesim/effects/spectral_trace_list.py | 175 +++++++++++++---------- 2 files changed, 187 insertions(+), 161 deletions(-) diff --git a/scopesim/effects/metis_lms_trace_list.py b/scopesim/effects/metis_lms_trace_list.py index bf8c0d88f..e6770a900 100644 --- a/scopesim/effects/metis_lms_trace_list.py +++ b/scopesim/effects/metis_lms_trace_list.py @@ -53,93 +53,98 @@ def __init__(self, **kwargs): self.slicelist["top"].max() - self.slicelist["bottom"].min()]) - def apply_to(self, obj, **kwargs): - """See parent docstring.""" - if isinstance(obj, FovVolumeList): - # Create a single volume that covers the aperture and - # the maximum wavelength range of LMS - volumes = [spectral_trace.fov_grid() - for spectral_trace in self.spectral_traces.values()] - - wave_min = min(vol["wave_min"] for vol in volumes) - wave_max = max(vol["wave_max"] for vol in volumes) - extracted_vols = obj.extract(axes=["wave"], - edges=([[wave_min, wave_max]])) - obj.volumes = extracted_vols - - if isinstance(obj, FieldOfView): - # Application to field of view - logger.debug("Executing %s, FoV", self.meta['name']) - if obj.hdu is not None and obj.hdu.header["NAXIS"] == 3: - obj.cube = obj.hdu - elif obj.hdu is None and obj.cube is None: - obj.cube = obj.make_hdu() - - fovcube = obj.cube.data - n_z, n_y, n_x = fovcube.shape - fovwcs = WCS(obj.cube.header) - # Make this linear to avoid jump at RA 0 deg - fovwcs.wcs.ctype = ["LINEAR", "LINEAR", fovwcs.wcs.ctype[2]] - fovwcs_spat = fovwcs.sub(2) - ny_slice = self.meta["slice_samples"] - - # Spatial pixel coordinates - xslice, yslice = np.meshgrid(np.arange(n_x), - np.arange(ny_slice)) - - fovimage = np.zeros((obj.detector_header["NAXIS2"], - obj.detector_header["NAXIS1"]), - dtype=np.float32) - - for sptid, spt in tqdm(self.spectral_traces.items(), - desc=" Spectral Traces", position=2): - ymin = spt.meta["fov"]["y_min"] - ymax = spt.meta["fov"]["y_max"] - - slicewcs = fovwcs.deepcopy() - - slicewcs.wcs.ctype = ["LINEAR", "LINEAR", - slicewcs.wcs.ctype[2]] - slicewcs.wcs.crpix[1] = (ny_slice + 1) / 2 - slicewcs.wcs.crval[1] = (ymin + ymax) / 2 / 3600 - slicewcs.wcs.cdelt[1] = (ymax - ymin) / ny_slice / 3600 - slicewcs_spat = slicewcs.sub(2) - - # World coordinates for the slice - xworld, yworld = slicewcs_spat.all_pix2world(xslice, - yslice, 0) - # FOV pixel coordinates for the slice - xfov, yfov = fovwcs_spat.all_world2pix(xworld, yworld, 0) - - slicecube = np.zeros((n_z, ny_slice, n_x)) - for islice in range(n_z): - ifov = RectBivariateSpline(np.arange(n_y), - np.arange(n_x), - fovcube[islice], kx=1, ky=1) - slicecube[islice] = ifov(yfov, xfov, grid=False) - - slicefov = FieldOfView3D(obj.header, - [obj.meta["wave_min"], - obj.meta["wave_max"]]) - slicefov.detector_header = obj.detector_header - slicefov.meta["xi_min"] = obj.meta["xi_min"] - slicefov.meta["xi_max"] = obj.meta["xi_max"] - slicefov.meta["trace_id"] = sptid - slicefov.cube = fits.ImageHDU(header=slicewcs.to_header(), - data=slicecube) - # slicefov.cube.writeto(f"slicefov_{sptid}.fits", overwrite=True) - slicefov.hdu = spt.map_spectra_to_focal_plane(slicefov) - if slicefov.hdu is not None: - sxmin = slicefov.hdu.header["XMIN"] - sxmax = slicefov.hdu.header["XMAX"] - symin = slicefov.hdu.header["YMIN"] - symax = slicefov.hdu.header["YMAX"] - fovimage[symin:symax, sxmin:sxmax] += slicefov.hdu.data - - obj.hdu = fits.ImageHDU(data=fovimage, header=obj.detector_header) + # Overrides for the two apply_to() methods + def apply_to_fovvolumelist(self, obj): + """Setup + Create a single volume that covers the aperture and + the maximum wavelength range of LMS. + """ + logger.debug("Executing %s: Setup", self.meta['name']) + volumes = [spectral_trace.fov_grid() + for spectral_trace in self.spectral_traces.values()] + + wave_min = min(vol["wave_min"] for vol in volumes) + wave_max = max(vol["wave_max"] for vol in volumes) + extracted_vols = obj.extract(axes=["wave"], + edges=([[wave_min, wave_max]])) + obj.volumes = extracted_vols + return obj + + def apply_to_fov(self, obj): + """Application to field of view""" + + logger.debug("Executing %s, FoV", self.meta['name']) + if obj.hdu is not None and obj.hdu.header["NAXIS"] == 3: + obj.cube = obj.hdu + elif obj.hdu is None and obj.cube is None: + obj.cube = obj.make_hdu() + + fovcube = obj.cube.data + n_z, n_y, n_x = fovcube.shape + fovwcs = WCS(obj.cube.header) + # Make this linear to avoid jump at RA 0 deg + fovwcs.wcs.ctype = ["LINEAR", "LINEAR", fovwcs.wcs.ctype[2]] + fovwcs_spat = fovwcs.sub(2) + ny_slice = self.meta["slice_samples"] + + # Spatial pixel coordinates + xslice, yslice = np.meshgrid(np.arange(n_x), + np.arange(ny_slice)) + + fovimage = np.zeros((obj.detector_header["NAXIS2"], + obj.detector_header["NAXIS1"]), + dtype=np.float32) + + for sptid, spt in tqdm(self.spectral_traces.items(), + desc=" Spectral Traces", position=2): + ymin = spt.meta["fov"]["y_min"] + ymax = spt.meta["fov"]["y_max"] + + slicewcs = fovwcs.deepcopy() + + slicewcs.wcs.ctype = ["LINEAR", "LINEAR", + slicewcs.wcs.ctype[2]] + slicewcs.wcs.crpix[1] = (ny_slice + 1) / 2 + slicewcs.wcs.crval[1] = (ymin + ymax) / 2 / 3600 + slicewcs.wcs.cdelt[1] = (ymax - ymin) / ny_slice / 3600 + slicewcs_spat = slicewcs.sub(2) + + # World coordinates for the slice + xworld, yworld = slicewcs_spat.all_pix2world(xslice, + yslice, 0) + # FOV pixel coordinates for the slice + xfov, yfov = fovwcs_spat.all_world2pix(xworld, yworld, 0) + + slicecube = np.zeros((n_z, ny_slice, n_x)) + for islice in range(n_z): + ifov = RectBivariateSpline(np.arange(n_y), + np.arange(n_x), + fovcube[islice], kx=1, ky=1) + slicecube[islice] = ifov(yfov, xfov, grid=False) + + slicefov = FieldOfView3D(obj.header, + [obj.meta["wave_min"], + obj.meta["wave_max"]]) + slicefov.detector_header = obj.detector_header + slicefov.meta["xi_min"] = obj.meta["xi_min"] + slicefov.meta["xi_max"] = obj.meta["xi_max"] + slicefov.meta["trace_id"] = sptid + slicefov.cube = fits.ImageHDU(header=slicewcs.to_header(), + data=slicecube) + # slicefov.cube.writeto(f"slicefov_{sptid}.fits", overwrite=True) + slicefov.hdu = spt.map_spectra_to_focal_plane(slicefov) + if slicefov.hdu is not None: + sxmin = slicefov.hdu.header["XMIN"] + sxmax = slicefov.hdu.header["XMAX"] + symin = slicefov.hdu.header["YMIN"] + symax = slicefov.hdu.header["YMAX"] + fovimage[symin:symax, sxmin:sxmax] += slicefov.hdu.data + + obj.hdu = fits.ImageHDU(data=fovimage, header=obj.detector_header) return obj + def make_spectral_traces(self): """Compute the transformations by interpolation.""" # nslice = len(self._file["Aperture List"].data) diff --git a/scopesim/effects/spectral_trace_list.py b/scopesim/effects/spectral_trace_list.py index 48e5abefa..c4d3fc925 100644 --- a/scopesim/effects/spectral_trace_list.py +++ b/scopesim/effects/spectral_trace_list.py @@ -184,6 +184,7 @@ def update_meta(self): self.meta["y_min"] = min(ylim) self.meta["y_max"] = max(ylim) + def apply_to(self, obj, **kwargs): """ Interface between ``FieldOfView`` and ``SpectralTraceList``. @@ -198,89 +199,109 @@ def apply_to(self, obj, **kwargs): list, identified by meta["trace_id"]. """ if isinstance(obj, FovVolumeList): - logger.debug("%s applied to %s", self.display_name, - obj.__class__.__name__) - # Setup of FieldOfView object - # volumes = [spectral_trace.fov_grid() - # for spectral_trace in self.spectral_traces.values()] - - new_vols_list = [] - - # for vol in volumes: - for spt in self.spectral_traces.values(): - vol = spt.fov_grid() - wave_edges = [vol["wave_min"], vol["wave_max"]] - if "x_min" in vol: - x_edges = [vol["x_min"], vol["x_max"]] - y_edges = [vol["y_min"], vol["y_max"]] - extracted_vols = obj.extract( - axes=["wave", "x", "y"], - edges=(wave_edges, x_edges, y_edges), - aperture_id=vol["aperture_id"]) - else: - extracted_vols = obj.extract( - axes=["wave"], - edges=(wave_edges, ), - aperture_id=vol["aperture_id"]) - - for ex_vol in extracted_vols: - ex_vol["meta"].update(vol) - ex_vol["meta"].pop("wave_min") - ex_vol["meta"].pop("wave_max") - new_vols_list.extend(extracted_vols) - - obj.volumes = new_vols_list + obj = self.apply_to_fovvolumelist(obj) if isinstance(obj, FieldOfView): - logger.debug("%s applied to %s", self.display_name, - obj.__class__.__name__) - # Application to field of view - if obj.hdu is not None and obj.hdu.header["NAXIS"] == 3: - obj.cube = obj.hdu - elif obj.hdu is not None and obj.hdu.header["NAXIS"] == 2: - # todo: catch the case of obj.hdu.header["NAXIS"] == 2 - # for MAAT - pass - elif obj.hdu is None and obj.cube is None: - logger.info("Making cube") - obj.cube = obj.make_hdu() - - # Check whether an offset slit is used. If so, recompute spectral traces. - # TODO: this needs to be removed. If transforms are available that - # can be applied to the actual slit position, they ought be used - # without modification. If they are not available but need to be - # hacked, then that hack needs to be defined via the yaml and only - # be used in a particular case, not in general. - #offset_x = obj.cube.header["CRVAL1D"] - #offset_y = obj.cube.header["CRVAL2D"] - #if (offset_x != self.meta["offset_x"] or - # offset_y != self.meta["offset_y"]): - # logger.debug("Recomputing spectral traces for offset (%.1g, %.1g)", - # offset_x, offset_y) - # self.meta["offset_x"] = offset_x - # self.meta["offset_y"] = offset_y - # self.make_spectral_traces() - # self.update_meta() - - spt = self.spectral_traces[obj.trace_id] - remove_pre = False - if ("offset_type" in obj.meta and - obj.meta["offset_type"] == "detector"): - spt.xy2lam.pretransform_x = (det_offset, - {"offset": obj.meta["offset_x"]}) - spt.xy2lam.pretransform_y = (det_offset, - {"offset": obj.meta["offset_y"]}) - remove_pre = True - - obj.hdu = spt.map_spectra_to_focal_plane(obj) - - if remove_pre: - spt.xy2lam.pretransform_y = None - spt.xy2xi.pretransform_x = None + obj = self.apply_to_fov(obj) logger.debug("%s done", self.display_name) return obj + def apply_to_fovvolumelist(self, obj): + """ + Setup of SpectralTraceList + + During setup of the required FieldOfView objects, the + SpectralTraceList is asked for the source space volumes that + it requires (spatial limits and wavelength limits). + """ + logger.debug("%s applied to %s", self.display_name, + obj.__class__.__name__) + + new_vols_list = [] + + for spt in self.spectral_traces.values(): + vol = spt.fov_grid() + wave_edges = [vol["wave_min"], vol["wave_max"]] + if "x_min" in vol: + x_edges = [vol["x_min"], vol["x_max"]] + y_edges = [vol["y_min"], vol["y_max"]] + extracted_vols = obj.extract( + axes=["wave", "x", "y"], + edges=(wave_edges, x_edges, y_edges), + aperture_id=vol["aperture_id"]) + else: + extracted_vols = obj.extract( + axes=["wave"], + edges=(wave_edges, ), + aperture_id=vol["aperture_id"]) + + for ex_vol in extracted_vols: + ex_vol["meta"].update(vol) + ex_vol["meta"].pop("wave_min") + ex_vol["meta"].pop("wave_max") + new_vols_list.extend(extracted_vols) + + obj.volumes = new_vols_list + return obj + + + def apply_to_fov(self, obj): + """Apply to FieldOfView + + During "observation" the method is passed a single FieldOfView + object and applies the mapping to the image plane to it. + The FieldOfView object is associated to one SpectralTrace from the + list, identified by meta["trace_id"]. + """ + logger.debug("%s applied to %s %s", self.display_name, + obj.__class__.__name__, obj.trace_id) + + if obj.hdu is not None and obj.hdu.header["NAXIS"] == 3: + obj.cube = obj.hdu + elif obj.hdu is not None and obj.hdu.header["NAXIS"] == 2: + # todo: catch the case of obj.hdu.header["NAXIS"] == 2 + # for MAAT + pass + elif obj.hdu is None and obj.cube is None: + logger.info("Making cube") + obj.cube = obj.make_hdu() + + # Check whether an offset slit is used. If so, recompute spectral traces. + # TODO: this needs to be removed. If transforms are available that + # can be applied to the actual slit position, they ought be used + # without modification. If they are not available but need to be + # hacked, then that hack needs to be defined via the yaml and only + # be used in a particular case, not in general. + #offset_x = obj.cube.header["CRVAL1D"] + #offset_y = obj.cube.header["CRVAL2D"] + #if (offset_x != self.meta["offset_x"] or + # offset_y != self.meta["offset_y"]): + # logger.debug("Recomputing spectral traces for offset (%.1g, %.1g)", + # offset_x, offset_y) + # self.meta["offset_x"] = offset_x + # self.meta["offset_y"] = offset_y + # self.make_spectral_traces() + # self.update_meta() + + spt = self.spectral_traces[obj.trace_id] + remove_pre = False + if ("offset_type" in obj.meta and + obj.meta["offset_type"] == "detector"): + spt.xy2lam.pretransform_x = (det_offset, + {"offset": obj.meta["offset_x"]}) + spt.xy2lam.pretransform_y = (det_offset, + {"offset": obj.meta["offset_y"]}) + remove_pre = True + + obj.hdu = spt.map_spectra_to_focal_plane(obj) + + if remove_pre: + spt.xy2lam.pretransform_y = None + spt.xy2xi.pretransform_x = None + + return obj + @property def footprint(self): """Return the footprint of the entire SpectralTraceList.""" From 3375ec53ec2ab2df8bf42fa770d1772a4b6b9ca8 Mon Sep 17 00:00:00 2001 From: oczoske Date: Mon, 18 May 2026 19:01:40 +0200 Subject: [PATCH 006/130] Remove prints --- scopesim/effects/apertures.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scopesim/effects/apertures.py b/scopesim/effects/apertures.py index d704265bb..b8ca2ce2c 100644 --- a/scopesim/effects/apertures.py +++ b/scopesim/effects/apertures.py @@ -118,6 +118,7 @@ def apply_to(self, obj, **kwargs): """See parent docstring.""" if isinstance(obj, FovVolumeList): logger.debug("Executing %s, FoV setup", self.meta['name']) + x = quantity_from_table("x", self.table, u.arcsec).to_value(u.arcsec) y = quantity_from_table("y", self.table, @@ -130,13 +131,11 @@ def apply_to(self, obj, **kwargs): vol["meta"]["xi_max"] = max(x) * u.arcsec if "offset_type" in self.meta: - print("Yes, got it: ", self.meta["offset_type"]) vol["meta"]["offset_type"] = self.meta["offset_type"] vol["meta"]["offset_x"] = self.meta["offset_x"] vol["meta"]["offset_y"] = self.meta["offset_y"] else: vol["meta"]["offset_type"] = "none" - print("What is offset_type?") return obj # Outdated. Remove when removing all old FOVManager code from effects From 385cade20f40b9f977aae01704975ded02e5f330 Mon Sep 17 00:00:00 2001 From: oczoske Date: Mon, 18 May 2026 19:02:00 +0200 Subject: [PATCH 007/130] Set ImagePlane to 32bit --- scopesim/optics/image_plane.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scopesim/optics/image_plane.py b/scopesim/optics/image_plane.py index 794c03fbb..b58ac50cc 100644 --- a/scopesim/optics/image_plane.py +++ b/scopesim/optics/image_plane.py @@ -68,7 +68,7 @@ def __init__(self, header, cmds=None, **kwargs): raise ValueError(f"header must have a valid image-plane WCS: " f"{dict(header)}") - image = zeros_from_header(header) + image = zeros_from_header(header, dtype=np.float32) self.hdu = fits.ImageHDU(data=image, header=header) self.hdu.header["BUNIT"] = "ph s-1" # photons per second (per pixel) From b2de962a16f2f55e50d97b60110aab46703948e7 Mon Sep 17 00:00:00 2001 From: oczoske Date: Mon, 18 May 2026 19:02:20 +0200 Subject: [PATCH 008/130] MICADO IFU works --- scopesim/effects/micado_ifu_trace_list.py | 153 +++++++++++++++++++++- 1 file changed, 149 insertions(+), 4 deletions(-) diff --git a/scopesim/effects/micado_ifu_trace_list.py b/scopesim/effects/micado_ifu_trace_list.py index 826399e1f..c8236e793 100644 --- a/scopesim/effects/micado_ifu_trace_list.py +++ b/scopesim/effects/micado_ifu_trace_list.py @@ -34,6 +34,8 @@ class MicadoIFUSpectralTraceList(SpectralTraceList): "naxis1": 112, "nslice": 32, "slicewidth": 0.012, # arcsec + "wave_min": 1.5, + "wave_max": 2.5, # todo: do not hardcode } @@ -42,6 +44,104 @@ def __init__(self, **kwargs): # field of view of the instrument + + def apply_to_fovvolumelist(self, obj): + """ + Setup + + Create a single volume that covers the aperture and the maximum + wavelength range of the MICADO IFU. + """ + logger.debug("Executing %s: Setup", self.meta['name']) + volumes = [spectral_trace.fov_grid() + for spectral_trace in self.spectral_traces.values()] + + wave_min = min(vol["wave_min"] for vol in volumes) + wave_max = max(vol["wave_max"] for vol in volumes) + extracted_vols = obj.extract(axes=["wave"], + edges=([[wave_min, wave_max]])) + obj.volumes = extracted_vols + self.wave_min = wave_min + self.wave_max = wave_max + return obj + + def apply_to_fov(self, obj): + """Apply to FieldOfView""" + + logger.debug("Executing %s, FoV", self.meta['name']) + if obj.hdu is not None and obj.hdu.header["NAXIS"] == 3: + obj.cube = obj.hdu + elif obj.hdu is None and obj.cube is None: + obj.cube = obj.make_hdu() + + fovcube = obj.cube.data + n_z, n_y, n_x = fovcube.shape + fovwcs = WCS(obj.cube.header) + # Make this linear to avoid jump at RA 0 deg + fovwcs.wcs.ctype = ["LINEAR", "LINEAR", fovwcs.wcs.ctype[2]] + fovwcs_spat = fovwcs.sub(2) + ny_slice = self.meta["slice_samples"] + + # Spatial pixel coordinates + xslice, yslice = np.meshgrid(np.arange(n_x), + np.arange(ny_slice)) + + fovimage = np.zeros((obj.detector_header["NAXIS2"], + obj.detector_header["NAXIS1"]), + dtype=np.float32) + + for sptid, spt in tqdm(self.spectral_traces.items(), + desc=" Spectral Traces", position=2): + ymin = spt.meta["fov"]["y_min"] + ymax = spt.meta["fov"]["y_max"] + + + slicewcs = fovwcs.deepcopy() + + slicewcs.wcs.ctype = ["LINEAR", "LINEAR", + slicewcs.wcs.ctype[2]] + slicewcs.wcs.crpix[1] = (ny_slice + 1) / 2 + slicewcs.wcs.crval[1] = (ymin + ymax) / 2 / 3600 + slicewcs.wcs.cdelt[1] = (ymax - ymin) / ny_slice / 3600 + slicewcs_spat = slicewcs.sub(2) + + # World coordinates for the slice + xworld, yworld = slicewcs_spat.all_pix2world(xslice, + yslice, 0) + # FOV pixel coordinates for the slice + xfov, yfov = fovwcs_spat.all_world2pix(xworld, yworld, 0) + + slicecube = np.zeros((n_z, ny_slice, n_x)) + for islice in range(n_z): + ifov = RectBivariateSpline(np.arange(n_y), + np.arange(n_x), + fovcube[islice], kx=1, ky=1) + slicecube[islice] = ifov(yfov, xfov, grid=False) + + slicefov = FieldOfView3D(obj.header, + [obj.meta["wave_min"], + obj.meta["wave_max"]]) + slicefov.detector_header = obj.detector_header + slicefov.meta["xi_min"] = spt.meta["fov"]["xi_min"] + slicefov.meta["xi_max"] = spt.meta["fov"]["xi_max"] + slicefov.meta["trace_id"] = sptid + slicefov.cube = fits.ImageHDU(header=slicewcs.to_header(), + data=slicecube) + # slicefov.cube.writeto(f"slicefov_{sptid}.fits", overwrite=True) + slicefov.hdu = spt.map_spectra_to_focal_plane(slicefov) + if slicefov.hdu is not None: + sxmin = slicefov.hdu.header["XMIN"] + sxmax = slicefov.hdu.header["XMAX"] + symin = slicefov.hdu.header["YMIN"] + symax = slicefov.hdu.header["YMAX"] + fovimage[symin:symax, sxmin:sxmax] += slicefov.hdu.data + + obj.hdu = fits.ImageHDU(data=fovimage, header=obj.detector_header) + return obj + + + + def make_spectral_traces(self): """Make a spectral trace for each combination of order and aperture""" self.ext_data = self._file[0].header["EDATA"] @@ -54,10 +154,55 @@ def make_spectral_traces(self): continue params = {col: row[col] for col in row.colnames} params.update(self.meta) - hdu = self._file[row["extension_id"]] - for apid in self.slicelist["id"]: + trace_hdu = self._file[row["extension_id"]] + + for spslice, apid in enumerate(self.slicelist["id"]): specid = f"{row['description']}_{apid:02d}" - params["aperture_id"] = apid - spec_traces[specid] = SpectralTrace(hdu, **params) + thistrace = MicadoIFUSpectralTrace(trace_hdu, self.slicelist, spslice, + **params) + thistrace.meta["trace_id"] = specid + spec_traces[specid] = thistrace self.spectral_traces = spec_traces + + + +class MicadoIFUSpectralTrace(SpectralTrace): + """SpectralTrace for the MICADO IFU""" + + _class_params = { + "naxis1": 112, + "nslice": 32, + "slicewidth": 0.012, # arcsec + } + + + def __init__(self, trace_tbl, aplist, spslice, cmds=None, **kwargs): + super().__init__(trace_tbl, **kwargs) + # Provisional + self.aplist = aplist + self.meta["slice"] = spslice + self.meta["fov"] = self.fov_grid() + + def fov_grid(self): + """ + Provide information on the source space volume required by the effect. + """ + aperture = self.aplist[self.meta["slice"]] + x_min = aperture["left"] + x_max = aperture["right"] + y_min = aperture["bottom"] + y_max = aperture["top"] + xi_min = aperture["xi1"] + xi_max = aperture["xi2"] + + # ..todo: just a hack - xi and x are the same except xi is a quantity + xi_min = quantify(xi_min, u.arcsec) + xi_max = quantify(xi_max, u.arcsec) + + return {"x_min": x_min, "x_max": x_max, + "y_min": y_min, "y_max": y_max, + "xi_min": xi_min, "xi_max": xi_max, + "wave_min": self.meta["wave_min"], + "wave_max": self.meta["wave_max"], + "trace_id": self.trace_id} From b63315127a42954c33c43baebbbe1457781ddf04 Mon Sep 17 00:00:00 2001 From: oczoske Date: Tue, 19 May 2026 17:45:51 +0200 Subject: [PATCH 009/130] Staggered ifu slices --- scopesim/effects/micado_ifu_trace_list.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/scopesim/effects/micado_ifu_trace_list.py b/scopesim/effects/micado_ifu_trace_list.py index c8236e793..d01ef0035 100644 --- a/scopesim/effects/micado_ifu_trace_list.py +++ b/scopesim/effects/micado_ifu_trace_list.py @@ -18,7 +18,7 @@ from .spectral_trace_list import SpectralTraceList from .spectral_trace_list_utils import SpectralTrace from .spectral_trace_list_utils import Transform2D -from .spectral_trace_list_utils import make_image_interpolations +from .spectral_trace_list_utils import make_image_interpolations, det_offset from .apertures import ApertureMask from .ter_curves import TERCurve from ..optics.fov import FieldOfView, FieldOfView3D @@ -42,8 +42,6 @@ class MicadoIFUSpectralTraceList(SpectralTraceList): def __init__(self, **kwargs): super().__init__(**kwargs) - # field of view of the instrument - def apply_to_fovvolumelist(self, obj): """ @@ -174,16 +172,26 @@ class MicadoIFUSpectralTrace(SpectralTrace): "naxis1": 112, "nslice": 32, "slicewidth": 0.012, # arcsec + "pixsize": 0.015, # mm } def __init__(self, trace_tbl, aplist, spslice, cmds=None, **kwargs): super().__init__(trace_tbl, **kwargs) + + self._set_dispersion(self.wave_min, self.wave_max) # Provisional self.aplist = aplist self.meta["slice"] = spslice self.meta["fov"] = self.fov_grid() + lam = np.linspace(self.wave_min, self.wave_max, 1001) * u.um + off_y = aplist['offset'][spslice] + dlam = np.median(self.dlam_per_pix(lam)) / self.meta["pixsize"] * off_y + self.xy2lam.posttransform = (det_offset, {"offset": dlam}) + self.xilam2x.pretransform_y = (det_offset, {"offset": dlam}) + self.xilam2y.pretransform_y = (det_offset, {"offset": dlam}) + def fov_grid(self): """ Provide information on the source space volume required by the effect. From 8b01497d3d3adec269eeffabadeaa5896c023bbc Mon Sep 17 00:00:00 2001 From: oczoske Date: Wed, 20 May 2026 13:19:03 +0200 Subject: [PATCH 010/130] Flipped sign of offset --- scopesim/effects/micado_ifu_trace_list.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scopesim/effects/micado_ifu_trace_list.py b/scopesim/effects/micado_ifu_trace_list.py index d01ef0035..cd2595b44 100644 --- a/scopesim/effects/micado_ifu_trace_list.py +++ b/scopesim/effects/micado_ifu_trace_list.py @@ -41,7 +41,8 @@ class MicadoIFUSpectralTraceList(SpectralTraceList): def __init__(self, **kwargs): super().__init__(**kwargs) - + self.wave_min = self.meta["wave_min"] + self.wave_max = self.meta["wave_max"] def apply_to_fovvolumelist(self, obj): """ @@ -188,9 +189,9 @@ def __init__(self, trace_tbl, aplist, spslice, cmds=None, **kwargs): lam = np.linspace(self.wave_min, self.wave_max, 1001) * u.um off_y = aplist['offset'][spslice] dlam = np.median(self.dlam_per_pix(lam)) / self.meta["pixsize"] * off_y - self.xy2lam.posttransform = (det_offset, {"offset": dlam}) - self.xilam2x.pretransform_y = (det_offset, {"offset": dlam}) - self.xilam2y.pretransform_y = (det_offset, {"offset": dlam}) + self.xy2lam.posttransform = (det_offset, {"offset": -dlam}) + self.xilam2x.pretransform_y = (det_offset, {"offset": -dlam}) + self.xilam2y.pretransform_y = (det_offset, {"offset": -dlam}) def fov_grid(self): """ From 10a7bdb9609160ee56d6f998fe9e7dcf52b18e2b Mon Sep 17 00:00:00 2001 From: oczoske Date: Mon, 25 May 2026 06:26:30 +0200 Subject: [PATCH 011/130] offset more systematic --- scopesim/effects/spectral_trace_list.py | 40 +++++++++++++++++++++---- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/scopesim/effects/spectral_trace_list.py b/scopesim/effects/spectral_trace_list.py index c4d3fc925..4422e1835 100644 --- a/scopesim/effects/spectral_trace_list.py +++ b/scopesim/effects/spectral_trace_list.py @@ -288,10 +288,17 @@ def apply_to_fov(self, obj): remove_pre = False if ("offset_type" in obj.meta and obj.meta["offset_type"] == "detector"): + self.meta["offset_type"] = "detector" + self.meta["offset_x"] = obj.meta["offset_x"] + self.meta["offset_y"] = obj.meta["offset_y"] spt.xy2lam.pretransform_x = (det_offset, {"offset": obj.meta["offset_x"]}) spt.xy2lam.pretransform_y = (det_offset, {"offset": obj.meta["offset_y"]}) + spt.xilam2x.posttransform = (det_offset, + {"offset": -obj.meta["offset_x"]}) + spt.xilam2y.posttransform = (det_offset, + {"offset": -obj.meta["offset_y"]}) remove_pre = True obj.hdu = spt.map_spectra_to_focal_plane(obj) @@ -299,6 +306,8 @@ def apply_to_fov(self, obj): if remove_pre: spt.xy2lam.pretransform_y = None spt.xy2xi.pretransform_x = None + spt.xilam2x.posttransform = None + spt.xilam2y.posttransform = None return obj @@ -401,15 +410,36 @@ def rectify_traces(self, hdulist, xi_min=None, xi_max=None, interps=None, for i, trace_id in tqdm(enumerate(self.spectral_traces, start=1), desc=" Traces", total=len(self.spectral_traces)): - hdu = self[trace_id].rectify(hdulist, - interps=interps, - bin_width=bin_width, - xi_min=xi_min, xi_max=xi_max, - wave_min=wave_min, wave_max=wave_max) + spt = self[trace_id] + remove_pre = False + if ("offset_type" in self.meta and + self.meta["offset_type"] == "detector"): + spt.xy2lam.pretransform_x = (det_offset, + {"offset": self.meta["offset_x"]}) + spt.xy2lam.pretransform_y = (det_offset, + {"offset": self.meta["offset_y"]}) + spt.xilam2x.posttransform = (det_offset, + {"offset": -self.meta["offset_x"]}) + spt.xilam2y.posttransform = (det_offset, + {"offset": -self.meta["offset_y"]}) + remove_pre = True + + + hdu = spt.rectify(hdulist, + interps=interps, + bin_width=bin_width, + xi_min=xi_min, xi_max=xi_max, + wave_min=wave_min, wave_max=wave_max) if hdu is not None: # ..todo: rectify does not do that yet outhdul.append(hdu) outhdul[0].header[f"EXTNAME{i}"] = trace_id + if remove_pre: + spt.xy2lam.pretransform_y = None + spt.xy2xi.pretransform_x = None + spt.xilam2x.posttransform = None + spt.xilam2y.posttransform = None + outhdul[0].header.update(inhdul[0].header) return outhdul From 2ce224d6014b33b73ea00c945574c2a703d558d7 Mon Sep 17 00:00:00 2001 From: Kieran Leschinski Date: Wed, 19 Aug 2026 15:35:43 +0200 Subject: [PATCH 012/130] Fix transposed axes in make_image_interpolations RectBivariateSpline was constructed with x = arange(NAXIS1) and y = arange(NAXIS2) against data of shape (NAXIS2, NAXIS1), which raises ValueError for any non-square image. It went unnoticed because current detector images are square and the caller in SpectralTrace.rectify compensates by passing (j, i) swapped. Build the spline with axes matching the data layout instead; the (j, i) call convention used by rectify and existing tests is now correct by construction and documented. Results on square images are unchanged. Adds a non-square regression test. --- scopesim/effects/spectral_trace_list_utils.py | 11 ++++++++--- .../tests_effects/test_SpectralTraceListUtils.py | 10 ++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/scopesim/effects/spectral_trace_list_utils.py b/scopesim/effects/spectral_trace_list_utils.py index d523aee72..da583fdc0 100644 --- a/scopesim/effects/spectral_trace_list_utils.py +++ b/scopesim/effects/spectral_trace_list_utils.py @@ -1022,13 +1022,18 @@ def _xiy2xlam_fit(layout, params): def make_image_interpolations(hdulist, **kwargs): - """Create 2D interpolation functions for images.""" + """Create 2D interpolation functions for images. + + The interpolation functions are called with (j, i) pixel coordinates, + i.e. the first argument corresponds to the row (NAXIS2) axis of the + image, the second to the column (NAXIS1) axis. + """ interps = [] for hdu in hdulist: if isinstance(hdu, fits.ImageHDU): interps.append( - RectBivariateSpline(np.arange(hdu.header["NAXIS1"]), - np.arange(hdu.header["NAXIS2"]), + RectBivariateSpline(np.arange(hdu.header["NAXIS2"]), + np.arange(hdu.header["NAXIS1"]), hdu.data, **kwargs) ) return interps diff --git a/scopesim/tests/tests_effects/test_SpectralTraceListUtils.py b/scopesim/tests/tests_effects/test_SpectralTraceListUtils.py index ca2f19d2b..e1bcbb463 100644 --- a/scopesim/tests/tests_effects/test_SpectralTraceListUtils.py +++ b/scopesim/tests/tests_effects/test_SpectralTraceListUtils.py @@ -202,3 +202,13 @@ def test_interpolation_is_accurate(self): interps = make_image_interpolations(hdul, kx=1, ky=1) imginterp = interps[0](yy, xx, grid=False) assert np.allclose(imginterp, img) + + def test_works_with_non_square_image(self): + # The axes passed to RectBivariateSpline were transposed, which + # raised ValueError for any image with NAXIS1 != NAXIS2 + xx, yy = np.meshgrid(np.arange(80), np.arange(50)) + img = np.sin(xx/6) * np.cos(yy/7) # shape (50, 80) + hdul = fits.HDUList(fits.ImageHDU(data=img)) + interps = make_image_interpolations(hdul, kx=1, ky=1) + imginterp = interps[0](yy, xx, grid=False) + assert np.allclose(imginterp, img) From fdd4822a815e1d09ca649aec737a610bcc95e5bf Mon Sep 17 00:00:00 2001 From: teutoburg Date: Sun, 30 Aug 2026 15:07:38 +0200 Subject: [PATCH 013/130] Refactor `TERCurve.apply_to()` For detailed explanation see #1002, this is basically the same but for `TERCurve`. The new `.__call__()` in this case is exclusively for cubes, because 1D spectra work differently. But those could be simplified as well, and as a result the `combine_two_spectra` function was no longer needed. Also finally remove the long-deprecated `.fov_grid()` methods that were still here. Everything passes without them. Well, except of course `test_current_filter_has_fov_grid_method`... --- scopesim/effects/ter_curves.py | 153 ++++++++---------- scopesim/effects/ter_curves_utils.py | 49 ------ scopesim/tests/tests_effects/test_TERCurve.py | 3 - 3 files changed, 68 insertions(+), 137 deletions(-) diff --git a/scopesim/effects/ter_curves.py b/scopesim/effects/ter_curves.py index 4d5e1f8b3..1b32b6674 100644 --- a/scopesim/effects/ter_curves.py +++ b/scopesim/effects/ter_curves.py @@ -4,26 +4,38 @@ import warnings from typing import ClassVar from collections.abc import Collection, Iterable -from pooch import retrieve import numpy as np +from numpy.typing import ArrayLike, NDArray from scipy import integrate from astropy import units as u from astropy.io import fits from astropy.table import Table +from synphot import SpectralElement, SourceSpectrum import skycalc_ipy from .effects import Effect -from .ter_curves_utils import (add_edge_zeros, combine_two_spectra, - apply_throughput_to_cube, download_svo_filter, - download_svo_filter_list) +from .ter_curves_utils import ( + add_edge_zeros, + download_svo_filter, + download_svo_filter_list, +) from ..optics.fov_volume_list import FovVolumeList from ..optics.surface import SpectralSurface from ..source.source import Source -from ..source.source_fields import (CubeSourceField, SpectrumSourceField, - BackgroundSourceField) -from ..utils import (from_currsys, quantify, check_keys, find_file, - figure_factory, get_logger) +from ..source.source_fields import ( + CubeSourceField, + SpectrumSourceField, + BackgroundSourceField, +) +from ..utils import ( + from_currsys, + quantify, + check_keys, + find_file, + figure_factory, + get_logger, +) from ..server.download_utils import create_retriever logger = get_logger(__name__) @@ -109,53 +121,64 @@ def __init__(self, filename=None, **kwargs): self.surface.table = data self.surface.table.meta.update(self.meta) + def __call__(self, data: ArrayLike, waveset: u.Quantity[u.um]) -> NDArray: + return data * self.throughput(waveset).value[:, None, None] + + def _apply_to_fvl(self, fvl: FovVolumeList) -> None: + logger.debug("Apply %s, FoV setup", self.display_name) + + wave = self.surface.throughput.waveset + thru = self.surface.throughput(wave) + valid_waves = np.argwhere(thru > 0) + wave_min = wave[max(0, valid_waves[0][0] - 1)] + wave_max = wave[min(len(wave) - 1, valid_waves[-1][0] + 1)] + + fvl.shrink("wave", [wave_min.to_value(u.um), wave_max.to_value(u.um)]) + + def _apply_to_src(self, src: Source) -> None: + logger.debug("Apply %s, Source spectra", self.display_name) + + self.meta = from_currsys(self.meta, self.cmds) + wave_min = quantify(self.meta["wave_min"], u.um).to(u.AA) + wave_max = quantify(self.meta["wave_max"], u.um).to(u.AA) + + thru = self.throughput + + # apply transmission to source spectra + for fld in src.fields: + if isinstance(fld, CubeSourceField): + fld.field.data = self(fld.field.data, fld.waveset) + elif isinstance(fld, SpectrumSourceField): + fld.spectra = { + isp: spec * thru # This works because synphot! + for isp, spec in fld.spectra.items() + } + else: + # Rather log than raise here, can still move on + logger.error( + "Source field is neither Cube nor has spectra, this " + "should not happen, but moving on.") + + # add the effect background to the source background field + if self.background_source is not None: + for bgs in self.background_source: + src.append(bgs) + def apply_to(self, obj, **kwargs): if isinstance(obj, Source): - self.meta = from_currsys(self.meta, self.cmds) - wave_min = quantify(self.meta["wave_min"], u.um).to(u.AA) - wave_max = quantify(self.meta["wave_max"], u.um).to(u.AA) - - thru = self.throughput - - # apply transmission to source spectra - for fld in obj.fields: - if isinstance(fld, CubeSourceField): - fld.field = apply_throughput_to_cube( - fld.field, thru, fld.waveset) - elif isinstance(fld, SpectrumSourceField): - fld.spectra = { - isp: combine_two_spectra(spec, thru, "multiply", - wave_min, wave_max) - for isp, spec in fld.spectra.items() - } - else: - # Rather log than raise here, can still move on - logger.error("Source field is neither Cube nor has " - "spectra, this shouldn't occur...") - - # add the effect background to the source background field - if self.background_source is not None: - for bgs in self.background_source: - obj.append(bgs) + self._apply_to_src(obj) if isinstance(obj, FovVolumeList): - wave = self.surface.throughput.waveset - thru = self.surface.throughput(wave) - valid_waves = np.argwhere(thru > 0) - wave_min = wave[max(0, valid_waves[0][0] - 1)] - wave_max = wave[min(len(wave) - 1, valid_waves[-1][0] + 1)] - - obj.shrink("wave", [wave_min.to_value(u.um), - wave_max.to_value(u.um)]) + self._apply_to_fvl(obj) return obj @property - def emission(self): + def emission(self) -> SourceSpectrum: return self.surface.emission @property - def throughput(self): + def throughput(self) -> SpectralElement: return self.surface.throughput @property @@ -166,23 +189,16 @@ def background_source(self): bkg_hdr = fits.Header() bkg_hdr.update({ + "ROLE": "background", # not used yet "BG_SRC": True, "BG_SURF": self.display_name, "SPEC_REF": 0, - # Seem those are not needed... - # "CUNIT1": "ARCSEC", - # "CUNIT2": "ARCSEC", - # "CDELT1": 0, - # "CDELT2": 0, "BUNIT": "photlam arcsec-2", "SOLIDANG": "arcsec-2", }) bkg_fld = BackgroundSourceField(field=None, spectra=bkg_spec, header=bkg_hdr) bkg_src = Source(field=bkg_fld) - # Before BackgroundSourceField: - # bkg_hdu = fits.ImageHDU(header=bkg_hdr) - # bkg_src = Source(image_hdu=bkg_hdu, spectra=flux) self._background_source = bkg_src return [self._background_source] @@ -604,34 +620,6 @@ def __init__(self, cmds=None, **kwargs): # TODO: maybe use actually masked table here? self.table["transmission"][mask] = 0 - def fov_grid(self, which="waveset", **kwargs): - warnings.warn("The fov_grid method is deprecated and will be removed " - "in a future release.", DeprecationWarning, stacklevel=2) - if which == "waveset": - self.meta.update(kwargs) - self.meta = from_currsys(self.meta, self.cmds) - # ..todo:: replace the 101 with a variable in !SIM - wave = np.linspace(self.meta["wave_min"], - self.meta["wave_max"], 101) - wave = quantify(wave, u.um) - throughput = self.surface.transmission(wave) - min_thru = self.meta["minimum_throughput"] - valid_waves = np.where(throughput.value > min_thru)[0] - if len(valid_waves) > 0: - wave_edges = [min(wave[valid_waves].value), - max(wave[valid_waves].value)] * u.um - else: - raise ValueError("No transmission found above the threshold {}" - " in this wavelength range {}. Did you open " - "the shutter?" - "".format(self.meta["minimum_throughput"], - [self.meta["wave_min"], - self.meta["wave_max"]])) - else: - wave_edges = [] - - return wave_edges - @property def fwhm(self): wave = self.surface.wavelength @@ -789,11 +777,6 @@ def surface(self): def throughput(self): return self.current_filter.throughput - def fov_grid(self, which="waveset", **kwargs): - warnings.warn("The fov_grid method is deprecated and will be removed " - "in a future release.", DeprecationWarning, stacklevel=2) - return self.current_filter.fov_grid(which=which, **kwargs) - def change_filter(self, filtername=None): """Change the current filter.""" if filtername in self.filters.keys(): diff --git a/scopesim/effects/ter_curves_utils.py b/scopesim/effects/ter_curves_utils.py index cabc6d796..827bf742a 100644 --- a/scopesim/effects/ter_curves_utils.py +++ b/scopesim/effects/ter_curves_utils.py @@ -389,55 +389,6 @@ def apply_throughput_to_cube( return cube -def combine_two_spectra(spec_a, spec_b, action, wave_min, wave_max): - """ - Combine transmission and/or emission spectrum with a common waverange. - - Spec_A is the source spectrum - Spec_B is either the transmission or emission that should be applied - - Parameters - ---------- - spec_a : synphot.SourceSpectrum - spec_b : synphot.SpectralElement, synphot.SourceSpectrum - action: {"multiply", "add"} - wave_min, wave_max : quantity - [Angstrom] - - Returns - ------- - new_source : synphot.SourceSpectrum - - """ - if spec_a.waveset is None: - wave_val = spec_b.waveset.value - else: - wave_val = spec_a.waveset.value - mask = (wave_val > wave_min.value) * (wave_val < wave_max.value) - - wave = ([wave_min.value] + list(wave_val[mask]) + [wave_max.value]) * u.AA - if "mult" in action.lower(): - spec_c = spec_a(wave) * spec_b(wave) - # Diagnostic plots - not for general use - # from matplotlib import pyplot as plt - # plt.plot(wave, spec_a(wave), label="spec_a") - # plt.plot(wave, spec_b(wave), label="spec_b") - # plt.plot(wave, spec_c, label="spec_c") - # plt.xlim(2.9e4, 4.2e4) - # plt.legend() - # plt.show() - elif "add" in action.lower(): - spec_c = spec_a(wave) + spec_b(wave) - else: - raise ValueError(f"action {action} unknown") - - new_source = SourceSpectrum(Empirical1D, points=wave, lookup_table=spec_c) - new_source.meta.update(spec_b.meta) - new_source.meta.update(spec_a.meta) - - return new_source - - def add_edge_zeros(tbl, wave_colname): if isinstance(tbl, Table): vals = np.zeros(len(tbl.colnames)) diff --git a/scopesim/tests/tests_effects/test_TERCurve.py b/scopesim/tests/tests_effects/test_TERCurve.py index 8e9a1847b..aadff5cc3 100644 --- a/scopesim/tests/tests_effects/test_TERCurve.py +++ b/scopesim/tests/tests_effects/test_TERCurve.py @@ -112,9 +112,6 @@ def test_loads_all_filters(self, fwheel): def test_current_filter_is_filter(self, fwheel): assert isinstance(fwheel.current_filter, tc.FilterCurve) - def test_current_filter_has_fov_grid_method(self, fwheel): - assert hasattr(fwheel.current_filter, "fov_grid") - def test_change_to_known_filter(self, fwheel): fwheel.change_filter('Ks') assert fwheel.current_filter.meta["name"] == "Ks" From 94998b3d3330cb3e56eeafc799be2b1689f92eac Mon Sep 17 00:00:00 2001 From: teutoburg Date: Sun, 30 Aug 2026 20:42:18 +0200 Subject: [PATCH 014/130] Refactor `.apply_to()` in electronic effects See #1002 for details and explanation. - Add `Detector.data` property setter to eleminate external access of `Detector._hdu` attribute. - Turn various effect attributes which need currsys/meta resolving into properties. This allows to encapsulate the resolving and make calls to the attributes much cleaner. **If we ever make effects dataclasses with explicit attribute fields rather than kwargs, this will make that process much smoother, because accessing a dataclass field looks identical to accessing a property, we just remove the property definitions then**. - Anything that the `.__call__()` needs, which is taken from the `obj` (in this case usually `Detector`), is resolved before and passed to `.__call__()` as an argument. That means the latter doesn't need to know about `obj`, which is cleaner and matches the PSF case. - I think there are now a few new `np.random.default_rng` cases. This was taken from a branch I started several months ago and forgot about in the much more recent discussion about those. Might not be final. This isn't completely exhaustive for all the effects here, but I tried to cover the "standard" ones... --- scopesim/detector/detector.py | 7 + scopesim/effects/electronic/electrons.py | 71 +++-- scopesim/effects/electronic/exposure.py | 4 +- scopesim/effects/electronic/noise.py | 261 +++++++++++------- scopesim/effects/electronic/pixels.py | 107 ++++--- scopesim/effects/rotation.py | 2 +- .../tests_effects/test_BasicReadoutNoise.py | 21 +- 7 files changed, 292 insertions(+), 181 deletions(-) diff --git a/scopesim/detector/detector.py b/scopesim/detector/detector.py index 2b0e43ed8..b6708d365 100644 --- a/scopesim/detector/detector.py +++ b/scopesim/detector/detector.py @@ -1,4 +1,7 @@ +# -*- coding: utf-8 -*- + import numpy as np +from numpy.typing import ArrayLike from astropy.io.fits import ImageHDU from astropy.wcs import WCS @@ -72,6 +75,10 @@ def data(self): """Return data from internal HDU.""" return self._hdu.data + @data.setter + def data(self, data: ArrayLike): + self._hdu.data = data + @property def image(self): """Return data from internal HDU.""" diff --git a/scopesim/effects/electronic/electrons.py b/scopesim/effects/electronic/electrons.py index c860362bd..1135eb283 100644 --- a/scopesim/effects/electronic/electrons.py +++ b/scopesim/effects/electronic/electrons.py @@ -9,17 +9,18 @@ """ from typing import ClassVar +from numbers import Real # matches int, float and all the numpy scalars import numpy as np +from numpy.typing import ArrayLike, NDArray from scipy.signal import oaconvolve from .. import Effect from ...detector import Detector from ...utils import figure_factory, check_keys -from ...utils import from_currsys, real_colname, get_logger +from ...utils import from_currsys, real_colname from . import logger -logger = get_logger(__name__) class LinearityCurve(Effect): """ @@ -62,21 +63,29 @@ def __init__(self, **kwargs): check_keys(self.meta, self.required_keys, action="error") + def __call__(self, data: ArrayLike, ndit: int = 1) -> NDArray: + incident = self.incident * ndit + measured = self.measured * ndit + return np.interp(data, incident, measured) + + @property + def incident(self) -> NDArray: + if self.table is not None: + return self.table["incident"] + return np.asarray(from_currsys(self.meta["incident"], self.cmds)) + + @property + def measured(self) -> NDArray: + if self.table is not None: + return self.table["measured"] + return np.asarray(from_currsys(self.meta["measured"], self.cmds)) + def apply_to(self, obj, **kwargs): if not isinstance(obj, Detector): return obj ndit = from_currsys(self.meta["ndit"], self.cmds) - if self.table is not None: - incident = self.table["incident"] * ndit - measured = self.table["measured"] * ndit - else: - incident = np.asarray(from_currsys(self.meta["incident"], - self.cmds)) * ndit - measured = np.asarray(from_currsys(self.meta["measured"], - self.cmds)) * ndit - obj._hdu.data = np.interp(obj._hdu.data, incident, measured) - + obj.data = self(obj.data, ndit) return obj def plot(self, **kwargs): @@ -298,28 +307,34 @@ def _should_apply(self) -> bool: return True - def apply_to(self, obj, **kwargs): - if not isinstance(obj, Detector): - return obj - - new_dtype = self.meta["dtype"] + def __call__(self, data: ArrayLike, gain: Real) -> NDArray: + return data / gain + def _get_gain(self, det_meta) -> Real: # Apply the gain value (copy from DarkCurrent) # Note that this does not cater for the case where the gain is given # as a plain dictionary. Should we implement that? if hasattr(self.cmds["!DET.gain"], "dic"): - dtcr_id = obj.meta[real_colname("id", obj.meta)] + dtcr_id = det_meta[real_colname("id", det_meta)] gain = self.cmds["!DET.gain"].dic[dtcr_id] logger.info(f"Detector {dtcr_id}: applying gain {gain}") - elif isinstance(self.cmds["!DET.gain"], (float, int)): + return gain + if isinstance(self.cmds["!DET.gain"], Real): gain = self.cmds["!DET.gain"] logger.info(f"Applying gain {gain}") - else: - raise ValueError(".meta['gain'] must be either " - f"dict or float, but is {self.cmds['!DET.gain']}") + return gain + raise ValueError( + f"{self.__class__.__name__}.meta['gain'] must be either " + f"dict or float, but is {self.cmds['!DET.gain']}") + + def apply_to(self, obj, **kwargs): + if not isinstance(obj, Detector): + return obj + + new_dtype = self.meta["dtype"] # Apply gain - obj._hdu.data /= gain + obj.data = self(obj.data, self._get_gain(obj.meta)) # Type-conversion wraps around input values that are higher or lower than # the respective maximum and minimum values of the new data type. Before @@ -328,14 +343,14 @@ def apply_to(self, obj, **kwargs): if np.issubdtype(new_dtype, np.integer): minval = np.iinfo(new_dtype).min maxval = np.iinfo(new_dtype).max - minvals_mask = obj._hdu.data < minval - maxvals_mask = obj._hdu.data > maxval + minvals_mask = obj.data < minval + maxvals_mask = obj.data > maxval if minvals_mask.any(): - obj._hdu.data[minvals_mask] = minval + obj.data[minvals_mask] = minval logger.warning( f"Effect ADConversion: {minvals_mask.sum()} negative pixels") if maxvals_mask.any(): - obj._hdu.data[maxvals_mask] = maxval + obj.data[maxvals_mask] = maxval logger.warning( f"Effect ADConversion: {maxvals_mask.sum()} saturated pixels") @@ -343,6 +358,6 @@ def apply_to(self, obj, **kwargs): # set to the modified data. It should be fine to simply re-assign the # data attribute, but just in case it's not... logger.debug("Applying digitization to dtype %s.", new_dtype) - obj._hdu.data = obj._hdu.data.astype(new_dtype) + obj.data = obj.data.astype(new_dtype) return obj diff --git a/scopesim/effects/electronic/exposure.py b/scopesim/effects/electronic/exposure.py index 8ff88e1e7..f131387dd 100644 --- a/scopesim/effects/electronic/exposure.py +++ b/scopesim/effects/electronic/exposure.py @@ -217,7 +217,7 @@ def apply_to(self, obj, **kwargs): logger.debug("Exposure: DIT = %s s, NDIT = %s", dit, ndit) if self.current_mode == "average": - obj._hdu.data /= ndit + obj.data = obj.data / ndit return obj @@ -276,6 +276,6 @@ def apply_to(self, obj, **kwargs): "readout call." ) - obj._hdu.data *= dit * ndit + obj.data = obj.data * dit * ndit return obj diff --git a/scopesim/effects/electronic/noise.py b/scopesim/effects/electronic/noise.py index 0698c09f5..a021143b5 100644 --- a/scopesim/effects/electronic/noise.py +++ b/scopesim/effects/electronic/noise.py @@ -2,9 +2,11 @@ """Any kinds of electronic or photonic noise.""" from typing import ClassVar -from numbers import Real +from collections.abc import Mapping +from numbers import Real # matches int, float and all the numpy scalars import numpy as np +from numpy.typing import ArrayLike, NDArray from astropy.io import fits from .. import Effect @@ -24,14 +26,18 @@ def __init__(self, **kwargs): self.meta.update(kwargs) check_keys(self.meta, self.required_keys, action="error") + def __call__(self, data: ArrayLike) -> NDArray: + return data + self.bias_level + + @property + def bias_level(self) -> float: + return from_currsys(self.meta["bias"], self.cmds) + def apply_to(self, obj, **kwargs): if not isinstance(obj, Detector): return obj - biaslevel = from_currsys(self.meta["bias"], self.cmds) - # Can't do in-place because of Quantization data type conflicts. - obj._hdu.data = obj._hdu.data + biaslevel - + obj.data = self(obj.data) return obj @@ -55,32 +61,89 @@ def __init__(self, **kwargs): check_keys(self.meta, self.required_keys, action="error") - def apply_to(self, det, **kwargs): - if not isinstance(det, Detector): - return det - - self.meta["random_seed"] = from_currsys(self.meta["random_seed"], - self.cmds) - if self.meta["random_seed"] is not None: - np.random.seed(self.meta["random_seed"]) - + def __call__(self, data: ArrayLike, ndit: int = 1) -> NDArray: self.meta = from_currsys(self.meta, self.cmds) - ron_keys = ["noise_std", "n_channels", "channel_fraction", - "line_fraction", "pedestal_fraction", "read_fraction"] - ron_kwargs = {key: self.meta[key] for key in ron_keys} - ron_kwargs["image_shape"] = det._hdu.data.shape - ron_frame = _make_ron_frame(**ron_kwargs) + ron_frame = self._make_ron_frame(data.shape) + stacked_ron_frame = np.zeros_like(ron_frame) - for i in range(self.meta["ndit"]): + for i in range(ndit): dx = np.random.randint(0, ron_frame.shape[1]) dy = np.random.randint(0, ron_frame.shape[0]) stacked_ron_frame += np.roll(ron_frame, (dy, dx), axis=(0, 1)) # TODO: this .T is ugly. Work out where things are getting switched and remove it! - # Can't do in-place because of Quantization data type conflicts. - det._hdu.data = det._hdu.data + stacked_ron_frame.T + return data + stacked_ron_frame.T + + @property + def noise_std(self) -> int: + return from_currsys(self.meta["noise_std"], self.cmds) + + @property + def n_channels(self) -> int: + return from_currsys(self.meta["n_channels"], self.cmds) + + @property + def random_seed(self) -> int: + return from_currsys(self.meta["random_seed"], self.cmds) + + @staticmethod + def _pseudo_random_field( + rng, + scale: float = 1., + size: tuple[int, ...] = (1024, 1024), + ) -> NDArray: + n = 256 + image = np.zeros(size) + batch = rng.normal(loc=0, scale=scale, size=(2*n, 2*n)) + for y in range(0, size[1], n): + for x in range(0, size[0], n): + i, j = rng.integers(n, size=2) + dx, dy = min(size[0]-x, n), min(size[1]-y, n) + image[x:x+dx, y:y+dy] = batch[i:i+dx, j:j+dy] + + return image + + def _make_ron_frame(self, shape: tuple[int, ...]) -> NDArray: + # TODO: Add yaml-settable seed here + rng = np.random.default_rng(self.random_seed) + + channel_fraction = self.meta["channel_fraction"] + line_fraction = self.meta["line_fraction"] + pedestal_fraction = self.meta["pedestal_fraction"] + read_fraction = self.meta["read_fraction"] + + pixel_std = self.noise_std * (pedestal_fraction + read_fraction)**0.5 + if shape < (1024, 1024): + pixel = rng.normal(loc=0, scale=pixel_std, size=shape) + line = rng.normal( + loc=0, + scale=self.noise_std * line_fraction**0.5, + size=shape[1], + ) + else: + # TODO: Why bother with this pseudo random function? + pixel = self._pseudo_random_field(rng, scale=pixel_std, size=shape) + line = pixel[0] + + channel = np.repeat( + rng.normal( + loc=0, + scale=self.noise_std * channel_fraction**0.5, + size=self.n_channels, + ), + max(1, shape[0] // self.n_channels) + 1, + axis=0, + ) + return (pixel + line).T + channel[:shape[0]] + + def apply_to(self, det, **kwargs): + if not isinstance(det, Detector): + return det + + ndit = from_currsys(self.meta["ndit"], self.cmds) + det._hdu.data = self(det._hdu.data, ndit) return det def plot(self, det, **kwargs): @@ -109,21 +172,25 @@ def __init__(self, **kwargs): check_keys(self.meta, self.required_keys, action="error") + def __call__(self, data: ArrayLike, ndit: int = 1) -> NDArray: + rng = np.random.default_rng(self.random_seed) + scale = self.noise_std * np.sqrt(float(ndit)) + return data + rng.normal(loc=0, scale=scale, size=data.shape) + + @property + def noise_std(self) -> float: + return from_currsys(self.meta["noise_std"], self.cmds) + + @property + def random_seed(self) -> int: + return from_currsys(self.meta["random_seed"], self.cmds) + def apply_to(self, det, **kwargs): if not isinstance(det, Detector): return det ndit = from_currsys(self.meta["ndit"], self.cmds) - ron = from_currsys(self.meta["noise_std"], self.cmds) - noise_std = ron * np.sqrt(float(ndit)) - - random_seed = from_currsys(self.meta["random_seed"], self.cmds) - if random_seed is not None: - np.random.seed(random_seed) - # Can't do in-place because of Quantization data type conflicts. - det._hdu.data = det._hdu.data + np.random.normal( - loc=0, scale=noise_std, size=det._hdu.data.shape) - + det._hdu.data = self(det._hdu.data, ndit) return det def plot(self, det): @@ -191,7 +258,7 @@ def apply_to(self, obj, **kwargs): dtcr_id = obj.meta[id_key] if id_key is not None else None prnu_std_meta = from_currsys(self.meta["prnu_std"], self.cmds) - if isinstance(prnu_std_meta, dict): + if isinstance(prnu_std_meta, Mapping): prnu_std = float(from_currsys(prnu_std_meta[dtcr_id], self.cmds)) elif isinstance(prnu_std_meta, Real): prnu_std = float(prnu_std_meta) @@ -201,7 +268,7 @@ def apply_to(self, obj, **kwargs): f"or a dict keyed by detector ID, got {type(prnu_std_meta)}" ) - shape = obj._hdu.data.shape + shape = obj.data.shape if dtcr_id not in self._gain_maps: rng = np.random.default_rng(random_seed) self._gain_maps[dtcr_id] = rng.normal( @@ -211,7 +278,7 @@ def apply_to(self, obj, **kwargs): if self._gain_maps[dtcr_id].shape != shape: raise ValueError("gain map shape mismatch") - obj._hdu.data = obj._hdu.data * self._gain_maps[dtcr_id] + obj.data = obj.data * self._gain_maps[dtcr_id] return obj def plot(self, det_id=None): @@ -236,13 +303,36 @@ def __init__(self, **kwargs): self.meta["random_seed"] = "!SIM.random.seed" self.meta.update(kwargs) + def __call__(self, data: ArrayLike) -> NDArray: + rng = np.random.default_rng(self.random_seed) + + # Check if there are negative values in the data. + values_negative = data < 0 + if values_negative.any(): + logger.warning( + "Effect ShotNoise: %d negative pixels", values_negative.sum()) + data[values_negative] = 0 + + # Apply a Poisson distribution to the low values. + values_low = data < 1e7 + data[values_low] = rng.poisson(data[values_low]) + + # Apply a normal distribution to the high values. + values_high = ~values_low + data[values_high] = rng.normal(data[values_high], np.sqrt(data[values_high])) + + return data + + @property + def random_seed(self) -> int: + return from_currsys(self.meta["random_seed"], self.cmds) + def apply_to(self, det, **kwargs): if not isinstance(det, Detector): return det - self.meta["random_seed"] = from_currsys(self.meta["random_seed"], - self.cmds) - rng = np.random.default_rng(self.meta["random_seed"]) + self.meta["random_seed"] = from_currsys( + self.meta["random_seed"], self.cmds) # numpy has a problem with generating Poisson distributions above # certain values. E.g. on linux, numpy.random.poisson(1e20) raises @@ -264,25 +354,8 @@ def apply_to(self, det, **kwargs): # - numpy.nan are implicitly passed through the normal distribution; # because the Poisson distribution cannot handle them. - data = det._hdu.data - - # Check if there are negative values in the data. - values_negative = data < 0 - if values_negative.any(): - logger.warning( - "Effect ShotNoise: %d negative pixels", values_negative.sum()) - data[values_negative] = 0 - - # Apply a Poisson distribution to the low values. - values_low = data < 1e7 - data[values_low] = rng.poisson(data[values_low]) - - # Apply a normal distribution to the high values. - values_high = ~values_low - data[values_high] = rng.normal(data[values_high], np.sqrt(data[values_high])) - new_imagehdu = fits.ImageHDU( - data=data, + data=self(det._hdu.data), header=det._hdu.header, ) @@ -314,25 +387,41 @@ def __init__(self, **kwargs): super().__init__(**kwargs) check_keys(self.meta, self.required_keys, action="error") + def __call__( + self, + data: np.ndarray, + dark_level: float, + dit: float, + ndit: int, + ) -> np.ndarray: + return data + dark_level * dit * ndit + + @property + def random_seed(self) -> int: + return from_currsys(self.meta["random_seed"], self.cmds) + + def _get_dark_level(self, det_meta) -> float: + dark_level = float(from_currsys(self.meta["value"], self.cmds)) + if isinstance(dark_level, Real): + return dark_level + if isinstance(dark_level, Mapping): + dtcr_id = det_meta[real_colname("id", det_meta)] + return from_currsys(dark_level[dtcr_id], self.cmds) + raise ValueError( + f"<{self.__class__.__name__}>.meta['value'] must be either " + f"dict-like or scalar number, but is {dark_level}." + ) + def apply_to(self, obj, **kwargs): if not isinstance(obj, Detector): return obj - if isinstance(from_currsys(self.meta["value"], self.cmds), dict): - dtcr_id = obj.meta[real_colname("id", obj.meta)] - dark = from_currsys(self.meta["value"][dtcr_id], self.cmds) - elif isinstance(from_currsys(self.meta["value"], self.cmds), Real): - dark = float(from_currsys(self.meta["value"], self.cmds)) - else: - raise ValueError(".meta['value'] must be either " - f"dict or float, but is {self.meta['value']}") - + # Dark level needs detector meta so can't go into __call__() + dark_level = self._get_dark_level(obj.meta) dit = from_currsys(self.meta["dit"], self.cmds) ndit = from_currsys(self.meta["ndit"], self.cmds) - # Can't do in-place because of Quantization data type conflicts. - obj._hdu.data = obj._hdu.data + dark * dit * ndit - + obj.data = self(obj.data, dark_level, dit, ndit) return obj def plot(self, det, **kwargs): @@ -348,39 +437,3 @@ def plot(self, det, **kwargs): ax.plot(times, levels, **kwargs) ax.set_xlabel("time") ax.set_ylabel("dark level") - - -def _pseudo_random_field(scale=1, size=(1024, 1024)): - n = 256 - image = np.zeros(size) - batch = np.random.normal(loc=0, scale=scale, size=(2*n, 2*n)) - for y in range(0, size[1], n): - for x in range(0, size[0], n): - i, j = np.random.randint(n, size=2) - dx, dy = min(size[0]-x, n), min(size[1]-y, n) - image[x:x+dx, y:y+dy] = batch[i:i+dx, j:j+dy] - - return image - - -def _make_ron_frame(image_shape, noise_std, n_channels, channel_fraction, - line_fraction, pedestal_fraction, read_fraction): - shape = image_shape - w_chan = max(1, shape[0] // n_channels) - - pixel_std = noise_std * (pedestal_fraction + read_fraction)**0.5 - line_std = noise_std * line_fraction**0.5 - if shape < (1024, 1024): - pixel = np.random.normal(loc=0, scale=pixel_std, size=shape) - line = np.random.normal(loc=0, scale=line_std, size=shape[1]) - else: - pixel = _pseudo_random_field(scale=pixel_std, size=shape) - line = pixel[0] - - channel_std = noise_std * channel_fraction**0.5 - channel = np.repeat(np.random.normal(loc=0, scale=channel_std, - size=n_channels), w_chan + 1, axis=0) - - ron_frame = (pixel + line).T + channel[:shape[0]] - - return ron_frame diff --git a/scopesim/effects/electronic/pixels.py b/scopesim/effects/electronic/pixels.py index 5c8c8c8cc..2a238c1ac 100644 --- a/scopesim/effects/electronic/pixels.py +++ b/scopesim/effects/electronic/pixels.py @@ -3,6 +3,8 @@ from typing import ClassVar +from numpy.typing import ArrayLike, NDArray + from .. import Effect from ...detector import Detector from ...utils import from_currsys, figure_factory, check_keys, real_colname @@ -46,6 +48,27 @@ def __init__(self, **kwargs): raise ValueError( "Parameter 'border' must have exactly four values.") + def __call__(self, data: ArrayLike, border: dict[int, int]) -> NDArray: + if border[0] > 0: + data[:border[0], :] = 0 + if border[1] > 0: + data[:, :border[1]] = 0 + if border[2] > 0: + data[-border[2]:, :] = 0 + if border[3] > 0: + data[:, -border[3]:] = 0 + return data + + def _get_border(self, det_meta) -> dict[int, int]: + if hasattr(self.meta["border"], "dic"): + dtcr_id = det_meta[real_colname("id", det_meta)] + return self.meta["border"].dic[dtcr_id] + if isinstance(self.meta["border"], list): + return self.meta["border"] + raise ValueError( + f"{self.__class__.__name__}.meta['border'] must be either " + f"dict or list, but is {self.meta['border']}") + def apply_to(self, obj, **kwargs): """Mask border pixels.""" if not isinstance(obj, Detector): @@ -54,24 +77,7 @@ def apply_to(self, obj, **kwargs): return obj logger.info(f"Applying border {from_currsys(self.meta['border'])}") - if hasattr(self.meta["border"], "dic"): - dtcr_id = obj.meta[real_colname("id", obj.meta)] - border = self.meta["border"].dic[dtcr_id] - elif isinstance(self.meta["border"], list): - border = self.meta["border"] - else: - raise ValueError( - ".meta['border'] must be either " - f"dict or list, but is {self.meta['border']}") - - if border[0] > 0: - obj.data[:border[0], :] = 0 - if border[1] > 0: - obj.data[:, :border[1]] = 0 - if border[2] > 0: - obj.data[-border[2]:, :] = 0 - if border[3] > 0: - obj.data[:, -border[3]:] = 0 + obj.data = self(obj.data, self._get_border(obj.meta)) return obj def plot(self, det, **kwargs): @@ -90,44 +96,61 @@ def __str__(self) -> str: return msg -class BinnedImage(Effect): - required_keys = {"bin_size"} +class BinnedImageBase(Effect): + """Base class for binning effects.""" + z_order: ClassVar[tuple[int, ...]] = (870,) def __init__(self, **kwargs): super().__init__(**kwargs) check_keys(self.meta, self.required_keys, action="error") + def __call__(self, data: ArrayLike) -> NDArray: + raise NotImplementedError("Subclasses must implement this.") + def apply_to(self, det, **kwargs): if not isinstance(det, Detector): return det - bs = from_currsys(self.meta["bin_size"], self.cmds) - image = det._hdu.data - h, w = image.shape - new_image = image.reshape((h//bs, bs, w//bs, bs)) - det._hdu.data = new_image.sum(axis=3).sum(axis=1) - + det._hdu.data = self(det._hdu.data) return det -class UnequalBinnedImage(Effect): - required_keys = {"binx","biny"} - z_order: ClassVar[tuple[int, ...]] = (870,) +class BinnedImage(BinnedImageBase): + required_keys = {"bin_size"} - def __init__(self, **kwargs): - super().__init__(**kwargs) - check_keys(self.meta, self.required_keys, action="error") + def __call__(self, data: ArrayLike) -> NDArray: + height, width = data.shape + binned_data = data.reshape(( + height//self.bin_size, + self.bin_size, + width//self.bin_size, + self.bin_size + )).sum(axis=3).sum(axis=1) + return binned_data - def apply_to(self, det, **kwargs): - if not isinstance(det, Detector): - return det + @property + def bin_size(self) -> int: + return from_currsys(self.meta["bin_size"], self.cmds) - bx = from_currsys(self.meta["binx"], self.cmds) - by = from_currsys(self.meta["biny"], self.cmds) - image = det._hdu.data - h, w = image.shape - new_image = image.reshape((h//bx, bx, w//by, by)) - det._hdu.data = new_image.sum(axis=3).sum(axis=1) - return det +class UnequalBinnedImage(BinnedImageBase): + required_keys = {"binx","biny"} + + def __call__(self, data: ArrayLike) -> NDArray: + height, width = data.shape + binned_data = data.reshape(( + height//self.binx, + self.binx, + width//self.biny, + self.binx + )).sum(axis=3).sum(axis=1) + return binned_data + + @property + def binx(self) -> int: + return from_currsys(self.meta["binx"], self.cmds) + + @property + def biny(self) -> int: + return from_currsys(self.meta["biny"], self.cmds) diff --git a/scopesim/effects/rotation.py b/scopesim/effects/rotation.py index 7b208bfcc..d1d48b578 100644 --- a/scopesim/effects/rotation.py +++ b/scopesim/effects/rotation.py @@ -40,5 +40,5 @@ def apply_to(self, obj, **kwargs): return obj rotations = from_currsys(self.meta["rotations"], self.cmds) - obj._hdu.data = np.rot90(obj._hdu.data, rotations) + obj.data = np.rot90(obj.data, rotations) return obj diff --git a/scopesim/tests/tests_effects/test_BasicReadoutNoise.py b/scopesim/tests/tests_effects/test_BasicReadoutNoise.py index 89bc4029c..b8bb69e33 100644 --- a/scopesim/tests/tests_effects/test_BasicReadoutNoise.py +++ b/scopesim/tests/tests_effects/test_BasicReadoutNoise.py @@ -2,7 +2,7 @@ from pytest import approx import numpy as np -from scopesim.effects.electronic.noise import PoorMansHxRGReadoutNoise, _make_ron_frame +from scopesim.effects.electronic.noise import PoorMansHxRGReadoutNoise from scopesim.tests.mocks.py_objects.detector_objects import _basic_detector @@ -39,12 +39,25 @@ def test_noise_reduces_with_square_root_of_ndit(self, ndit, noise_std): class TestMakeRonFrame: @pytest.mark.parametrize("n", (1, 4, 9, 25)) def test_stdev_increase_with_square_of_n(self, n): - frames = np.array([_make_ron_frame((256, 256), 10, 2, 0.1, 0.2, 0.3, 0.4) + ron = PoorMansHxRGReadoutNoise(noise_std=10, n_channels=2, ndit=1) + ron.meta.update({ + "channel_fraction": 0.1, + "line_fraction": 0.2, + "pedestal_fraction": 0.3, + "read_fraction": 0.4, + }) + frames = np.array([ron._make_ron_frame((256, 256)) for _ in range(n)]) assert np.std(np.sum(frames, axis=0)) == approx(10*n**0.5, rel=0.3) @pytest.mark.parametrize("shape", [(3, 7), (7, 3)]) def test_makes_frame_sizes_for_non_integer_n_channels(self, shape): - n_channels = 2 - frame = _make_ron_frame(shape, 5, n_channels, 0.25, 0.25, 0.25, 0.25) + ron = PoorMansHxRGReadoutNoise(noise_std=5, n_channels=2, ndit=1) + ron.meta.update({ + "channel_fraction": 0.25, + "line_fraction": 0.25, + "pedestal_fraction": 0.25, + "read_fraction": 0.25, + }) + frame = ron._make_ron_frame(shape) assert frame.shape > (0, 0) From 779f1462e901d5a3d376d276e1af2a5723766086 Mon Sep 17 00:00:00 2001 From: teutoburg Date: Mon, 31 Aug 2026 23:39:02 +0200 Subject: [PATCH 015/130] Eliminate remaining `Detector._hdu` calls Somehow forgotten in the previous commit... --- scopesim/effects/electronic/electrons.py | 3 +-- scopesim/effects/electronic/noise.py | 11 +++-------- scopesim/effects/electronic/pixels.py | 2 +- scopesim/effects/mosaic_trace_list.py | 4 ++-- .../tests/tests_effects/test_ADConversion.py | 18 +++++++++--------- .../tests_effects/test_BasicReadoutNoise.py | 4 ++-- .../test_InterPixelCapacitance.py | 4 ++-- .../tests/tests_effects/test_LinearityCurve.py | 6 +++--- scopesim/tests/tests_effects/test_exposure.py | 14 +++++++------- .../tests_effects/test_mosaic_trace_list.py | 6 +++--- scopesim/tests/tests_effects/test_prnu.py | 12 ++++++------ .../tests/tests_effects/test_shot_noise.py | 18 +++++++++--------- 12 files changed, 48 insertions(+), 54 deletions(-) diff --git a/scopesim/effects/electronic/electrons.py b/scopesim/effects/electronic/electrons.py index 1135eb283..b28f55b05 100644 --- a/scopesim/effects/electronic/electrons.py +++ b/scopesim/effects/electronic/electrons.py @@ -189,8 +189,7 @@ def apply_to(self, det, **kwargs): det.__class__.__name__) return det - newdata = oaconvolve(det._hdu.data, self.kernel, mode="same") - det._hdu.data = newdata + det.data = oaconvolve(det.data, self.kernel, mode="same") return det def update(self, **kwargs): diff --git a/scopesim/effects/electronic/noise.py b/scopesim/effects/electronic/noise.py index a021143b5..717fec584 100644 --- a/scopesim/effects/electronic/noise.py +++ b/scopesim/effects/electronic/noise.py @@ -143,7 +143,7 @@ def apply_to(self, det, **kwargs): return det ndit = from_currsys(self.meta["ndit"], self.cmds) - det._hdu.data = self(det._hdu.data, ndit) + det.data = self(det.data, ndit) return det def plot(self, det, **kwargs): @@ -190,7 +190,7 @@ def apply_to(self, det, **kwargs): return det ndit = from_currsys(self.meta["ndit"], self.cmds) - det._hdu.data = self(det._hdu.data, ndit) + det.data = self(det.data, ndit) return det def plot(self, det): @@ -354,12 +354,7 @@ def apply_to(self, det, **kwargs): # - numpy.nan are implicitly passed through the normal distribution; # because the Poisson distribution cannot handle them. - new_imagehdu = fits.ImageHDU( - data=self(det._hdu.data), - header=det._hdu.header, - ) - - det._hdu = new_imagehdu + det._hdu = fits.ImageHDU(header=det.header, data=self(det.data)) return det def plot(self, det): diff --git a/scopesim/effects/electronic/pixels.py b/scopesim/effects/electronic/pixels.py index 2a238c1ac..93f746700 100644 --- a/scopesim/effects/electronic/pixels.py +++ b/scopesim/effects/electronic/pixels.py @@ -112,7 +112,7 @@ def apply_to(self, det, **kwargs): if not isinstance(det, Detector): return det - det._hdu.data = self(det._hdu.data) + det.data = self(det.data) return det diff --git a/scopesim/effects/mosaic_trace_list.py b/scopesim/effects/mosaic_trace_list.py index 8b23e3ad3..542755ebf 100644 --- a/scopesim/effects/mosaic_trace_list.py +++ b/scopesim/effects/mosaic_trace_list.py @@ -253,8 +253,8 @@ def apply_to(self, det, **kwargs): type(det)) return det - image = det._hdu.data - detwcs = WCS(det._hdu.header, key="D") + image = det.data + detwcs = WCS(det.header, key="D") output_format = from_currsys(self.meta["format"], self.cmds) diff --git a/scopesim/tests/tests_effects/test_ADConversion.py b/scopesim/tests/tests_effects/test_ADConversion.py index daece3751..eba0ad615 100644 --- a/scopesim/tests/tests_effects/test_ADConversion.py +++ b/scopesim/tests/tests_effects/test_ADConversion.py @@ -32,9 +32,9 @@ def fixture_mock_detector(): def fixture_detector_with_data(mock_detector): """Instantiate a Detector with some data""" det = mock_detector - width = det._hdu.data.shape[1] - det._hdu.data[:] = 1.2 - det._hdu.data[:, width//2] = 1.99 + width = det.data.shape[1] + det.data[:] = 1.2 + det.data[:, width//2] = 1.99 return det @@ -73,11 +73,11 @@ def test_converts_pixels_to_integer_with_custom(self, detector_with_data): def test_applies_gain(self, mock_detector, gain): det = mock_detector data = np.random.randn(100, 100) * 100. + 1000. - det._hdu.data = 1. * data + det.data = 1. * data adconverter = ADConversion(gain=gain) adconverter.cmds = {"!DET.gain": gain, "!OBS.ndit": 1} adconverter.apply_to(det) - assert np.all((data/gain).astype(int) == det._hdu.data) + assert np.all((data/gain).astype(int) == det.data) @pytest.mark.usefixtures("patch_mock_path_micado") def test_applies_gain_list_to_detector_list(self): @@ -104,10 +104,10 @@ def test_caps_data_overflow(self, mock_detector, newtype, maxval, minval): data = medval * np.ones((100, 100), dtype=np.float64) data[50, 50] = 1.e12 data[60, 60] = -100000. - det._hdu.data = data + det.data = data adconverter = ADConversion(gain=gain, dtype=newtype) adconverter.cmds = {"!DET.gain": gain, "!OBS.ndit": 1} adconverter.apply_to(det) - assert(np.median(det._hdu.data) == newtype(medval/gain)) - assert(np.max(det._hdu.data) == maxval) - assert(np.min(det._hdu.data) == minval) + assert(np.median(det.data) == newtype(medval/gain)) + assert(np.max(det.data) == maxval) + assert(np.min(det.data) == minval) diff --git a/scopesim/tests/tests_effects/test_BasicReadoutNoise.py b/scopesim/tests/tests_effects/test_BasicReadoutNoise.py index b8bb69e33..e119dfa29 100644 --- a/scopesim/tests/tests_effects/test_BasicReadoutNoise.py +++ b/scopesim/tests/tests_effects/test_BasicReadoutNoise.py @@ -23,7 +23,7 @@ def test_creates_noise_std_as_needed(self, noise_std): ron = PoorMansHxRGReadoutNoise(noise_std=noise_std, n_channels=64, ndit=1) dtcr = ron.apply_to(dtcr) - assert np.std(dtcr._hdu.data) == approx(noise_std, rel=0.05) + assert np.std(dtcr.data) == approx(noise_std, rel=0.05) @pytest.mark.parametrize("noise_std", [1, 9, 100]) @pytest.mark.parametrize("ndit", [1, 9, 100]) @@ -31,7 +31,7 @@ def test_noise_reduces_with_square_root_of_ndit(self, ndit, noise_std): dtcr = _basic_detector(width=256) ron = PoorMansHxRGReadoutNoise(noise_std=noise_std, n_channels=64, ndit=ndit) dtcr = ron.apply_to(dtcr) - noise_real = np.std(dtcr._hdu.data) + noise_real = np.std(dtcr.data) assert noise_real == approx(noise_std * ndit**0.5, rel=0.1) diff --git a/scopesim/tests/tests_effects/test_InterPixelCapacitance.py b/scopesim/tests/tests_effects/test_InterPixelCapacitance.py index fb6adb5ee..bed34660b 100644 --- a/scopesim/tests/tests_effects/test_InterPixelCapacitance.py +++ b/scopesim/tests/tests_effects/test_InterPixelCapacitance.py @@ -87,7 +87,7 @@ def fixture_detector(): """Instantiate a random detector object""" hdu = fits.ImageHDU(data=np.random.randn(214, 333)) det = Detector(hdu.header) - det._hdu.data = hdu.data + det.data = hdu.data return det class TestApply: @@ -102,7 +102,7 @@ def test_convolve_unit_source(self): [0., 1., 0.], [0., 0., 0.]])) det = Detector(hdu.header) - det._hdu.data = hdu.data + det.data = hdu.data ipc = IPC(kernel=np.random.rand(3, 3)) newdet = ipc.apply_to(det) np.testing.assert_allclose(newdet.hdu.data, ipc.kernel) diff --git a/scopesim/tests/tests_effects/test_LinearityCurve.py b/scopesim/tests/tests_effects/test_LinearityCurve.py index a0612000d..e29a39830 100644 --- a/scopesim/tests/tests_effects/test_LinearityCurve.py +++ b/scopesim/tests/tests_effects/test_LinearityCurve.py @@ -46,11 +46,11 @@ class TestApplyTo: (60, 60), (1e5, 60)]) def test_only_applied_to_detector(self, in_flux, out_flux, basic_lincurve): dtcr = _basic_detector() - dtcr._hdu.data[0, 0] = in_flux + dtcr.data[0, 0] = in_flux new_dtcr = basic_lincurve.apply_to(dtcr) - assert new_dtcr._hdu.data[0, 0] == out_flux - assert np.min(new_dtcr._hdu.data) == 0 + assert new_dtcr.data[0, 0] == out_flux + assert np.min(new_dtcr.data) == 0 def test_bypasses_non_detector_objects(self, basic_lincurve): new_dict = basic_lincurve.apply_to({"gigawatts": 1.21}) diff --git a/scopesim/tests/tests_effects/test_exposure.py b/scopesim/tests/tests_effects/test_exposure.py index 18a1f091d..e3203c353 100644 --- a/scopesim/tests/tests_effects/test_exposure.py +++ b/scopesim/tests/tests_effects/test_exposure.py @@ -39,7 +39,7 @@ def fixture_exposureoutput(): @pytest.fixture(name="detector", scope="function") def fixture_detector(): det = Detector(_image_hdu_square().header) - det._hdu.data[:] = 1.e5 + det.data[:] = 1.e5 return det class TestExposureIntegration: @@ -47,9 +47,9 @@ def test_initialises_correctly(self, exposureintegration): assert isinstance(exposureintegration, ExposureIntegration) def test_integrates_correctly(self, exposureintegration, detector): - orig = 1. * detector._hdu.data + orig = 1. * detector.data assert isinstance(exposureintegration.apply_to(detector), Detector) - assert np.allclose(detector._hdu.data, 4 * orig) + assert np.allclose(detector.data, 4 * orig) def test_fails_without_dit(self): with pytest.raises(ValueError): @@ -91,17 +91,17 @@ def test_cannot_set_to_unknown_mode(self, exposureoutput): (2., 5), (3, 36)]) def test_applies_average(self, dit, ndit, detector): - det_mean = detector._hdu.data.mean() + det_mean = detector.data.mean() exposureoutput = ExposureOutput("average", dit=dit, ndit=ndit) result = exposureoutput.apply_to(detector) - assert np.isclose(result._hdu.data.mean(), det_mean / ndit) + assert np.isclose(result.data.mean(), det_mean / ndit) @pytest.mark.parametrize("dit, ndit", [(1., 1), (2., 5), (3, 36)]) def test_applies_sum(self, dit, ndit, detector): - det_mean = detector._hdu.data.mean() + det_mean = detector.data.mean() exposureoutput = ExposureOutput("sum", dit=dit, ndit=ndit) result = exposureoutput.apply_to(detector) - assert np.isclose(result._hdu.data.mean(), det_mean) + assert np.isclose(result.data.mean(), det_mean) diff --git a/scopesim/tests/tests_effects/test_mosaic_trace_list.py b/scopesim/tests/tests_effects/test_mosaic_trace_list.py index da3207e76..39433af43 100644 --- a/scopesim/tests/tests_effects/test_mosaic_trace_list.py +++ b/scopesim/tests/tests_effects/test_mosaic_trace_list.py @@ -104,7 +104,7 @@ def fixture_detector(): xsize, ysize = 50, 4096 hdr = header_from_list_of_xy([-xsize/2, xsize/2], [-ysize/2, ysize/2], 1, "D") dtcr = Detector(hdr) - dtcr._hdu.data[rows, :] = 1 + dtcr.data[rows, :] = 1 return dtcr @pytest.fixture(name="apply_collapse1d", scope="class") @@ -159,7 +159,7 @@ def test_collapse1d_detector_has_bintable(self, apply_collapse1d): assert isinstance(apply_collapse1d._hdu, fits.BinTableHDU) def test_collapse1d_gives_correct_result(self, apply_collapse1d): - assert np.all(apply_collapse1d._hdu.data['spectrum'] == 7) + assert np.all(apply_collapse1d.data['spectrum'] == 7) @pytest.mark.usefixtures("patch_mock_path_mosaic") @@ -181,7 +181,7 @@ def test_converttotable_detector_has_bintable(self, apply_converttotable): assert isinstance(apply_converttotable._hdu, fits.BinTableHDU) def test_converttotable_gives_correct_result(self, apply_converttotable): - for row in apply_converttotable._hdu.data: + for row in apply_converttotable.data: assert np.all(row['spectrum'] == 1) @pytest.mark.usefixtures("patch_mock_path_mosaic") diff --git a/scopesim/tests/tests_effects/test_prnu.py b/scopesim/tests/tests_effects/test_prnu.py index 2c7bbcbd6..4d74810f5 100644 --- a/scopesim/tests/tests_effects/test_prnu.py +++ b/scopesim/tests/tests_effects/test_prnu.py @@ -11,7 +11,7 @@ def make_detector(value=1000, size=10): hdr = header_from_list_of_xy([-size/2, size/2], [-size/2, size/2], 1, "D") dtcr = Detector(hdr) - dtcr._hdu.data[:] = value + dtcr.data[:] = value return dtcr @@ -21,7 +21,7 @@ def test_output_std_matches_prnu_std(self): prnu_std = 0.05 dtcr = make_detector(value=1000, size=100) PixelResponseNonUniformity(prnu_std=prnu_std, prnu_seed=42).apply_to(dtcr) - rel_std = dtcr._hdu.data.std() / dtcr._hdu.data.mean() + rel_std = dtcr.data.std() / dtcr.data.mean() assert abs(rel_std - prnu_std) < 0.01 def test_gain_map_is_reused(self): @@ -31,24 +31,24 @@ def test_gain_map_is_reused(self): prnu = PixelResponseNonUniformity(prnu_std=0.01, prnu_seed=42) prnu.apply_to(dtcr1) prnu.apply_to(dtcr2) - np.testing.assert_array_equal(dtcr1._hdu.data, dtcr2._hdu.data) + np.testing.assert_array_equal(dtcr1.data, dtcr2.data) def test_dict_prnu_std(self): """Dict mode: different amplitude per detector ID.""" hdr = header_from_list_of_xy([-5, 5], [-5, 5], 1, "D") dtcr = Detector(hdr) dtcr.meta["id"] = "H2RG" - dtcr._hdu.data[:] = 1000 + dtcr.data[:] = 1000 prnu = PixelResponseNonUniformity( prnu_std={"H2RG": 0.005, "GeoSnap": 0.020}, prnu_seed=42) prnu.apply_to(dtcr) - assert dtcr._hdu.data.std() > 0 + assert dtcr.data.std() > 0 def test_multiplicative_zero_signal(self): """Zero signal should remain zero — confirms multiplicative (not additive).""" dtcr = make_detector(value=0) PixelResponseNonUniformity(prnu_std=0.01, prnu_seed=42).apply_to(dtcr) - assert dtcr._hdu.data.sum() == 0 + assert dtcr.data.sum() == 0 def test_non_detector_passthrough(self): """Non-Detector objects should be returned unchanged.""" diff --git a/scopesim/tests/tests_effects/test_shot_noise.py b/scopesim/tests/tests_effects/test_shot_noise.py index ccdca344f..c7cde5b55 100644 --- a/scopesim/tests/tests_effects/test_shot_noise.py +++ b/scopesim/tests/tests_effects/test_shot_noise.py @@ -21,10 +21,10 @@ def test_applies_dark_current_with_level_of_dit(self): value_too_high = 1e20 hdr = header_from_list_of_xy([-hw, hw], [-hw, hw], 1, "D") dtcr = Detector(hdr) - dtcr._hdu.data[0][0] = value_normal - dtcr._hdu.data[0][1] = -1 - dtcr._hdu.data[1][0] = value_too_high - dtcr._hdu.data[1][1] = np.nan + dtcr.data[0][0] = value_normal + dtcr.data[0][1] = -1 + dtcr.data[1][0] = value_too_high + dtcr.data[1][1] = np.nan sn = ShotNoise() # To prevent that data[0][0] does not change by chance. @@ -33,10 +33,10 @@ def test_applies_dark_current_with_level_of_dit(self): dtcr = sn.apply_to(dtcr) # Ensure that the values have changed. - assert dtcr._hdu.data[0][0] != value_normal - assert dtcr._hdu.data[1][0] != value_too_high + assert dtcr.data[0][0] != value_normal + assert dtcr.data[1][0] != value_too_high # Sensibility checks on the values. - assert 1 < dtcr._hdu.data[0][0] < value_normal * 2 - assert np.isnan(dtcr._hdu.data[0][1]) or dtcr._hdu.data[0][1] == 0 - assert np.isnan(dtcr._hdu.data[1][1]) + assert 1 < dtcr.data[0][0] < value_normal * 2 + assert np.isnan(dtcr.data[0][1]) or dtcr.data[0][1] == 0 + assert np.isnan(dtcr.data[1][1]) From a522f1ee8da5a40fcea6eba7468987f1bfb3aaec Mon Sep 17 00:00:00 2001 From: Kieran Leschinski Date: Fri, 4 Sep 2026 18:55:50 +0200 Subject: [PATCH 016/130] Snap the placement origin, not the centre, in overlay_image overlay_image computed the array origin at which to place small_im as ceil(coords) - small_im.shape // 2. That is only correct when coords sits on the pixel lattice small_im's shape implies: half-integer for even shapes, integer for odd ones. For anything in between it is off by up to a whole pixel, and the direction of the error flips with the parity of the shape. Chunked FOVs hit this whenever the detector footprint is not a whole number of pixels across, as MICADO's is not (189.32 mm / 0.015 mm = 12621.33 px in y). The last chunk of each axis is then a fraction of a chunk wide, so it has a different size and a pixel lattice offset from its neighbours', and it got displaced the opposite way from them. The seam between the two picked up a duplicated or a dropped row/column and the image content shifted by a pixel across it. Snap the origin instead, which is parity-independent. Half-integer origins - a genuine sub-pixel shift, which sub_pixel=True would have to handle properly - round towards the lower index, matching what the old expression did for even shapes. --- scopesim/optics/image_plane_utils.py | 33 +++++++++++--- .../tests_optics/test_ImagePlaneUtils.py | 44 +++++++++++++++++++ 2 files changed, 70 insertions(+), 7 deletions(-) diff --git a/scopesim/optics/image_plane_utils.py b/scopesim/optics/image_plane_utils.py index 62f717315..940bcfb44 100644 --- a/scopesim/optics/image_plane_utils.py +++ b/scopesim/optics/image_plane_utils.py @@ -437,13 +437,32 @@ def overlay_image(small_im, big_im, coords, mask=None, sub_pixel=False): coords = np.array([*coords, (big_im.shape[0] - 1) / 2]) # FIXME: this would not be necessary if we used WCS instead of manual 2pix - # Round to 1e-4 pix before ceil: WCS deg-space round-trips leave - # float dust of order 1e-8..1e-6 pix on integer-valued coords, which - # ceil would amplify to a full-pixel shift; genuine sub-pixel intent - # cannot be finer than 0.5 pix here (sub_pixel is not implemented), - # and the half-integer even-shape convention is preserved exactly. - coords = np.ceil(np.asarray(coords).round(4)).astype(np.intp) - idx = coords.astype(int)[::-1] - np.array(small_im.shape) // 2 + # Round to 1e-4 pix first: WCS deg-space round-trips leave float dust of + # order 1e-8..1e-6 pix on integer-valued coords, which the snap below + # would otherwise amplify to a full-pixel shift; genuine sub-pixel intent + # cannot be finer than 0.5 pix here (sub_pixel is not implemented). + coords = np.asarray(coords, dtype=float).round(4) + + # `idx` is the origin, in array order, at which small_im must be placed to + # centre it on `coords` (both are 0-based pixel-centre coordinates, as + # returned by WCS.wcs_world2pix(..., 0)). + # + # Snap that origin, NOT `coords` itself. The old expression + # np.ceil(coords)[::-1] - np.array(small_im.shape) // 2 + # is off by up to a whole pixel whenever `coords` is not on the lattice + # small_im implies (half-integer for even shapes, integer for odd ones), + # and the direction of that error flips with the parity of the shape. Two + # adjacent FOVs of different sizes therefore get displaced opposite ways + # and their shared edge picks up a duplicated or a dropped row/column. + # Snapping the origin instead keeps every placement consistent, whatever + # the shape. + # + # Half-integer origins (a real sub-pixel shift, which sub_pixel=True would + # have to handle properly) are rounded towards the lower index, matching + # what the old expression did for even shapes. + idx = np.ceil( + coords[::-1] - (np.array(small_im.shape) - 1) / 2 - 0.5 + ).astype(np.intp) # Image ranges idx1 = np.maximum(0, idx) diff --git a/scopesim/tests/tests_optics/test_ImagePlaneUtils.py b/scopesim/tests/tests_optics/test_ImagePlaneUtils.py index 8658dd423..dd95ae12f 100644 --- a/scopesim/tests/tests_optics/test_ImagePlaneUtils.py +++ b/scopesim/tests/tests_optics/test_ImagePlaneUtils.py @@ -229,6 +229,50 @@ def test_overlay_images_works_for_3d_cubes_at_border(self): plt.show() +class TestOverlayImageSnapping: + """The origin, not the centre, must be snapped to the pixel grid. + + ``ceil(coords) - shape // 2`` snapped the centre instead, which is off by + up to a whole pixel whenever ``coords`` is not on the lattice implied by + ``small_im.shape``, in a direction that flips with the parity of that + shape. Adjacent FOVs of different sizes were therefore displaced opposite + ways, and their shared edge gained a duplicated or a dropped row/column. + """ + + @pytest.mark.parametrize("size", [1, 2, 3, 4, 7, 8]) + def test_places_image_on_its_own_lattice_without_shift(self, size): + big = np.zeros((32, 32)) + small = np.ones((size, size)) + # centre coordinate that the shape's own lattice implies + centre = 10 + (size - 1) / 2 + imp_utils.overlay_image(small, big, (centre, centre)) + rows = np.where(big.any(axis=1))[0] + assert rows[0] == 10 + assert rows[-1] == 10 + size - 1 + + @pytest.mark.parametrize("size", [1, 2, 3, 4, 7, 8]) + def test_snaps_sub_pixel_offset_the_same_way_for_any_shape(self, size): + # A 0.3 pix offset must never move the image by a whole pixel, whatever + # the parity of the shape. + big = np.zeros((32, 32)) + small = np.ones((size, size)) + centre = 10 + (size - 1) / 2 + 0.3 + imp_utils.overlay_image(small, big, (centre, centre)) + rows = np.where(big.any(axis=1))[0] + assert rows[0] == 10 + assert rows[-1] == 10 + size - 1 + + def test_adjacent_differently_sized_images_tile_exactly(self): + # Mimics chunked FOVs, where the last chunk is smaller than the rest. + big = np.zeros((1, 20)) + origin = 0 + for size in (8, 8, 4): + small = np.ones((1, size)) + imp_utils.overlay_image(small, big, (origin + (size - 1) / 2, 0)) + origin += size + assert (big == 1).all() + + class TestRescaleImageHDU: @pytest.mark.parametrize("pixel_scale", [0.3, 0.5, 1, 2, 3]) def test_rescales_a_2D_imagehdu(self, pixel_scale): From e59648436c92aba269248549768517443599ea85 Mon Sep 17 00:00:00 2001 From: Kieran Leschinski Date: Fri, 4 Sep 2026 18:56:02 +0200 Subject: [PATCH 017/130] Derive the FOV image cutout WCS from the slice it actually took extract_area_from_imagehdu slices the source array on the source's own pixel grid, growing the slice outwards to whole pixels with floor/ceil so it covers the FOV footprint. It then built the accompanying WCS with create_wcs_from_points([xy0s, xy1s]) - the world corners from *before* that growing, rounded independently. The two disagreed in both size and phase. NAXISn could be a pixel smaller than data.shape, which fits.ImageHDU silently corrects on construction, leaving CRPIXn describing an array of the wrong size; and the WCS put the cutout on the FOV's pixel lattice rather than the source's, which for chunked FOVs is offset from it by a fraction of a pixel. Projecting such a cutout mis-registered it by up to a whole pixel, by differing amounts for neighbouring FOVs, so the checkerboard phase flipped across the chunk seams and a row/column was duplicated or dropped there. The FIXME this replaces described the size half of the problem. Build the cutout WCS by shifting the input WCS's CRPIX by the slice origin instead. The cutout's pixels are input pixels, so this is exact: the cutout keeps the input's pixel lattice and NAXISn always agrees with data.shape. The added integration test reassembles a checkerboard from chunked FOVs with chunk_size=8, over four detector footprints including two that are not a whole number of pixels across. All four fail without this commit and the one before it. --- scopesim/optics/fov.py | 39 ++++--- .../test_source_fov_imageplane_play_nicely.py | 100 ++++++++++++++++++ scopesim/tests/tests_optics/test_fov_utls.py | 71 +++++++++++++ 3 files changed, 189 insertions(+), 21 deletions(-) diff --git a/scopesim/optics/fov.py b/scopesim/optics/fov.py index 503214f53..d6d97f247 100644 --- a/scopesim/optics/fov.py +++ b/scopesim/optics/fov.py @@ -340,27 +340,24 @@ def extract_area_from_imagehdu(self, imagehdu, corners): xy1p += (xy0p == xy1p) logger.debug("xy0p: %s; xy1p: %s", xy0p, xy1p) - new_wcs, new_naxis = imp_utils.create_wcs_from_points( - np.array([xy0s, xy1s]).round(11), pixel_scale=hdr["CDELT1"]) - - # TODO: Come back at some point and figure out if the failing tests - # here are relevant or can be ignored... - # FIXME: Commented out for now because it appears too often IRL... - # try: - # roundtrip = new_wcs.wcs_world2pix( - # np.array([xy0s, xy1s - .5*image_wcs.wcs.cdelt]), 0).round(5) - # np.testing.assert_array_equal(roundtrip[0], [0, 0]) - # np.testing.assert_array_equal(roundtrip[1], new_naxis - [1, 1]) - # except AssertionError: - # logger.exception("WCS roundtrip assertion failed.") - # FIXME: Related to the above, this sometimes fails: - # np.testing.assert_equal(xy1p - xy0p, new_naxis) - # This occurs when the floor and ceil in xy0s and xy1s produce an - # off-by-one error. Using .round instead for both would solve things - # in those cases, but breaks in other cases. Find a proper solution! - # Note: This is not super fatal, because the resulting projections - # will trim off that extra pixel later on, but this should still - # be addressed. + # Describe the cutout with the input WCS, shifted by the slice origin. + # The cutout's pixels ARE input pixels, so this is exact: the cutout + # stays on the input's pixel lattice and NAXISn always agrees with + # data.shape. + # + # Deriving the WCS from [xy0s, xy1s] instead (via + # create_wcs_from_points) got both wrong. Those are the world corners + # *before* the floor/ceil above grew the slice outwards to whole input + # pixels, so the resulting WCS described a differently sized image + # (NAXISn disagreed with data.shape by a pixel, silently corrected by + # fits.ImageHDU, which left CRPIXn describing the wrong array) placed + # on the FOV's lattice rather than the input's. Projecting that cutout + # then mis-registered it by up to a whole pixel, and by differing + # amounts for neighbouring FOVs, so the seam between two FOVs picked + # up a duplicated or a dropped row/column. + new_wcs = image_wcs.deepcopy() + new_wcs.wcs.crpix = image_wcs.wcs.crpix - xy0p + new_naxis = xy1p - xy0p new_hdr = new_wcs.to_header() new_hdr.update({"NAXIS1": new_naxis[0], "NAXIS2": new_naxis[1]}) diff --git a/scopesim/tests/tests_integrations/test_source_fov_imageplane_play_nicely.py b/scopesim/tests/tests_integrations/test_source_fov_imageplane_play_nicely.py index 82209d5ab..f87ea5ec2 100644 --- a/scopesim/tests/tests_integrations/test_source_fov_imageplane_play_nicely.py +++ b/scopesim/tests/tests_integrations/test_source_fov_imageplane_play_nicely.py @@ -8,6 +8,9 @@ from astropy import units as u +from astropy.wcs import WCS + +from scopesim.optics import image_plane_utils as imp_utils from scopesim.optics.fov import FieldOfView2D from scopesim.optics.image_plane import ImagePlane @@ -64,3 +67,100 @@ def test_can_extract_the_source_in_a_fov(self, fov_hdr, comb_src, plt.subplot(133) plt.imshow(imp.hdu.data, origin="lower", norm=LogNorm()) plt.show() + +def _checkerboard_source(naxis, pixel_scale, crpix=None): + """0/1 checkerboard image source, one board square per pixel.""" + from astropy.io import fits + from scopesim.source.source import Source + from scopesim.source import source_templates as st + + nx, ny = naxis + wcs = WCS(naxis=2) + wcs.wcs.ctype = ["LINEAR", "LINEAR"] + wcs.wcs.cunit = ["deg", "deg"] + wcs.wcs.cdelt = 2 * [pixel_scale / 3600] + wcs.wcs.crval = [0, 0] + wcs.wcs.crpix = crpix or [(nx + 1) / 2, (ny + 1) / 2] + + hdr = wcs.to_header() + hdr["BUNIT"] = "" + board = (np.indices((ny, nx)).sum(axis=0) % 2).astype(float) + hdu = fits.ImageHDU(header=hdr, data=board) + hdu.header["SPEC_REF"] = 0 + hdu.header["SPECMAG"] = 0 + hdu.header["SPECUNIT"] = "mag" + return Source(image_hdu=hdu, spectra=[st.vega_spectrum()]), board + + +def _chunked_fov_headers(x_mm, y_mm, pixel_size, pixel_scale, chunk): + """Reproduce FOVManager's chunking of a detector footprint.""" + plate_scale = pixel_scale / pixel_size + lims = [v / pixel_size * pixel_scale + for v in (x_mm[0], x_mm[1], y_mm[0], y_mm[1])] + + def edges(vmin, vmax): + out = [vmin] + for val in np.arange(vmin, vmax, chunk * pixel_scale): + if vmin < val < vmax and val > out[-1]: + out.append(float(val)) + return out + [vmax] + + for x0, x1 in zip(*(lambda e: (e[:-1], e[1:]))(edges(lims[0], lims[1]))): + for y0, y1 in zip(*(lambda e: (e[:-1], e[1:]))(edges(lims[2], lims[3]))): + skyhdr = imp_utils.header_from_list_of_xy( + [x0 / 3600, x1 / 3600], [y0 / 3600, y1 / 3600], + pixel_scale=pixel_scale / 3600) + dethdr, _ = imp_utils.det_wcs_from_sky_wcs( + WCS(skyhdr), pixel_scale, plate_scale) + skyhdr.update(dethdr.to_header()) + yield skyhdr + + +class TestChunkedFOVsTileTheImagePlane: + """A checkerboard must survive being cut into chunked FOVs and reassembled. + + ``chunk_size`` is 8 here so the whole thing runs in well under a second, + but the geometry is MICADO's: the detector footprint is not a whole number + of pixels across (MICADO's y axis is 189.32 mm / 0.015 mm = 12621.33 px), + so the last chunk in each axis is a fraction of a chunk wide and lands on + a pixel lattice offset from its neighbours'. The projection used to snap + those chunks inconsistently, which put a duplicated or a dropped + row/column at the seam and flipped the checkerboard phase across it. + """ + + PIXEL_SIZE = 0.01 # mm / pix + PIXEL_SCALE = 0.2 # arcsec / pix + CHUNK = 8 + + @pytest.mark.parametrize("half_mm, exact_extent", [ + (0.12, 24.0), # whole pixels, whole chunks + (0.105, 21.0), # whole pixels, fractional last chunk + (0.0866, 17.32), # fractional pixels -- MICADO's y axis + (0.0855, 17.10), + ]) + def test_checkerboard_survives_chunking(self, half_mm, exact_extent): + x_mm = y_mm = (-half_mm, half_mm) + assert (x_mm[1] - x_mm[0]) / self.PIXEL_SIZE == approx(exact_extent) + + imphdr = imp_utils.header_from_list_of_xy( + list(x_mm), list(y_mm), pixel_scale=self.PIXEL_SIZE, wcs_suffix="D") + imp = ImagePlane(imphdr) + + source, board = _checkerboard_source( + (imphdr["NAXIS1"], imphdr["NAXIS2"]), self.PIXEL_SCALE, + crpix=[imphdr["CRPIX1D"], imphdr["CRPIX2D"]]) + + fov_hdrs = list(_chunked_fov_headers( + x_mm, y_mm, self.PIXEL_SIZE, self.PIXEL_SCALE, self.CHUNK)) + assert len(fov_hdrs) == 9 + + for hdr in fov_hdrs: + fov = FieldOfView2D(hdr, waverange=[0.5, 2.5] * u.um, + area=1 * u.m**2) + fov.extract_from(source) + fov.view() + imp.add(fov.hdu, wcs_suffix="D") + + # no gaps and no double-counting at the chunk seams + got = imp.data / imp.data.max() + np.testing.assert_allclose(got, board, atol=1e-6) diff --git a/scopesim/tests/tests_optics/test_fov_utls.py b/scopesim/tests/tests_optics/test_fov_utls.py index ba08540dd..5a782a9b6 100644 --- a/scopesim/tests/tests_optics/test_fov_utls.py +++ b/scopesim/tests/tests_optics/test_fov_utls.py @@ -6,6 +6,8 @@ from synphot import Empirical1D, SourceSpectrum from synphot.units import PHOTLAM from astropy import units as u +from astropy.io import fits +from astropy.wcs import WCS from scopesim.optics.fov import FieldOfView, extract_range_from_spectrum from scopesim.optics import image_plane_utils as imp_utils @@ -128,3 +130,72 @@ def test_logs_msg_for_waverang_overlap_mismatch( extract_range_from_spectrum(spec, waverange) assert msg in caplog.text + +class TestExtractAreaKeepsTheInputPixelGrid: + """The cutout is made of input pixels, so its WCS must say so. + + The cutout WCS used to be re-derived from the world corners *before* the + slice was grown outwards to whole input pixels. That gave a WCS for a + differently sized image (``NAXISn`` disagreed with ``data.shape``, silently + corrected by ``fits.ImageHDU``, which left ``CRPIXn`` describing the wrong + array) sitting on the FOV's pixel lattice rather than the input's. The + cutout was then mis-registered by up to a whole pixel when projected, and + by differing amounts for neighbouring FOVs, so their shared edge gained a + duplicated or a dropped row/column. + """ + + @staticmethod + def _image_hdu(naxis=32, pixel_scale=0.1, crval=(0.0, 0.0)): + """Image with a distinct value in every pixel, in deg.""" + skywcs = WCS(naxis=2) + skywcs.wcs.ctype = ["LINEAR", "LINEAR"] + skywcs.wcs.cunit = ["deg", "deg"] + skywcs.wcs.cdelt = 2 * [pixel_scale / 3600] + skywcs.wcs.crval = list(crval) + skywcs.wcs.crpix = 2 * [(naxis + 1) / 2] + hdr = skywcs.to_header() + hdr["BUNIT"] = "" + data = np.arange(naxis * naxis, dtype=float).reshape(naxis, naxis) + return fits.ImageHDU(header=hdr, data=data) + + @staticmethod + def _fov(x_min, x_max, y_min, y_max, pixel_scale=0.1): + skyhdr = imp_utils.header_from_list_of_xy( + [x_min / 3600, x_max / 3600], [y_min / 3600, y_max / 3600], + pixel_scale=pixel_scale / 3600) + dethdr, _ = imp_utils.det_wcs_from_sky_wcs( + WCS(skyhdr), pixel_scale, pixel_scale / 0.01) + skyhdr.update(dethdr.to_header()) + return FieldOfView(skyhdr, [1.0, 2.0]) + + # FOVs deliberately offset from the image's pixel edges by a fraction of a + # pixel, which is what chunking produces whenever the detector extent is + # not a whole number of pixels. + @pytest.mark.parametrize("offset", [0.0, 0.16, 0.34, 0.5, 0.68, 0.84]) + def test_naxis_matches_data_shape(self, offset): + hdu = self._image_hdu() + fov = self._fov(-0.8 + offset * 0.1, 0.0 + offset * 0.1, + -0.8 + offset * 0.1, 0.0 + offset * 0.1) + cut = fov.extract_area_from_imagehdu(hdu, fov.get_corners("deg")[0]) + assert (cut.header["NAXIS2"], cut.header["NAXIS1"]) == cut.data.shape + + @pytest.mark.parametrize("offset", [0.0, 0.16, 0.34, 0.5, 0.68, 0.84]) + def test_cutout_pixels_keep_their_world_coordinates(self, offset): + hdu = self._image_hdu() + fov = self._fov(-0.8 + offset * 0.1, 0.0 + offset * 0.1, + -0.8 + offset * 0.1, 0.0 + offset * 0.1) + cut = fov.extract_area_from_imagehdu(hdu, fov.get_corners("deg")[0]) + + # every cutout value must sit at the world coordinate its value had in + # the input image + in_wcs = WCS(hdu.header, naxis=2) + cut_wcs = WCS(cut.header, naxis=2) + ny, nx = cut.data.shape + yy, xx = np.indices((ny, nx)) + world = cut_wcs.wcs_pix2world( + np.column_stack([xx.ravel(), yy.ravel()]), 0) + back = in_wcs.wcs_world2pix(world, 0).round(6) + expected = hdu.data[back[:, 1].astype(int), back[:, 0].astype(int)] + np.testing.assert_array_equal(cut.data.ravel(), expected) + # ... and land on whole input pixels, not between them + np.testing.assert_allclose(back, back.round(), atol=1e-6) From bb100a8b43a8fa437ad8003961ceb5b63d5e3995 Mon Sep 17 00:00:00 2001 From: Kieran Leschinski Date: Fri, 4 Sep 2026 20:08:07 +0200 Subject: [PATCH 018/130] Share one pixel lattice between an image plane and its chunks This is the last of the three off-by-one causes and the most invasive of them: it changes CRVALn for any region that is not a whole number of pixels across. It is separable from the two commits before it - drop it and the dropped row and the phase flips stay fixed, but MICADO keeps one duplicated row at the last chunk boundary in y. create_wcs_from_points rounds the requested extent to a whole NAXISn and then centres the grid on the region, folding that region's own rounding remainder into CRVAL. Each region therefore gets its own sub-pixel lattice. Chunked FOVs are all carved from one origin in whole-pixel steps - the image plane, the full-size chunks and the smaller last chunk all measure from the same detector edge - so they ought to share a lattice, and instead ended up offset from one another by fractions of a pixel. The projection then has to snap them to whole pixels, and the last chunk could land a whole pixel off its neighbours. MICADO's detector plane is 189.32 mm / 0.015 mm = 12621.33 px tall, so it hits this on every run: with chunk_size 2048 the last chunk in y is 333.33 px and its lattice is offset from the six full chunks'. When NAXISn truncates the region, anchor the lower pixel edge at points.min and let the leftover fraction fall off the far edge; regions carved from the same origin then coincide. When NAXISn covers the region there is nothing to truncate, so keep centring - that is what a region smaller than one pixel needs, where NAXISn was clamped up to 1. Verified on the full MICADO array (12621 x 12888 image plane, 4 mas checkerboard, chunk_size 2048), checking that the board's row and column phase alternates strictly across the whole image plane: origin/main rows: 10407 bad, cols: 10408 bad, plus a dropped row at y = 12288 first two commits rows: 166 bad, cols: 0 bad with this commit rows: 0 bad, cols: 0 bad --- scopesim/optics/image_plane_utils.py | 36 ++++++++++++++- .../tests_optics/test_ImagePlaneUtils.py | 46 +++++++++++++++++++ 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/scopesim/optics/image_plane_utils.py b/scopesim/optics/image_plane_utils.py index 940bcfb44..b6feb9230 100644 --- a/scopesim/optics/image_plane_utils.py +++ b/scopesim/optics/image_plane_utils.py @@ -214,7 +214,41 @@ def create_wcs_from_points( offset[zeroaxis] = 0 crpix = (naxis + 1) / 2 - crval = (points.min(axis=0) + points.max(axis=0) + offset) / 2 + + # Where to put the pixel lattice when NAXISn cannot cover the requested + # region exactly. + # + # If the grid truncates the region (naxis < extent, i.e. the region is not + # a whole number of pixels across), anchor the lower pixel edge at + # points.min and let the leftover fraction fall off the far edge. Every + # region carved out of the same origin in whole-pixel steps then shares one + # lattice - which is what chunked FOVs are: the image plane, the full-size + # chunks and the smaller last chunk all measure from the same detector + # edge. Centring each of them on its own midpoint instead, and absorbing + # its own rounding remainder into CRVAL, put them on lattices offset from + # one another by fractions of a pixel, so the projection had to snap them + # and the last chunk could land a whole pixel off its neighbours. + # + # MICADO's detector plane is 189.32 mm / 0.015 mm = 12621.33 px tall, so it + # hits this on every run. + # + # If the grid covers the region (naxis >= extent) there is nothing to + # truncate, so keep centring it on the region. That is the right thing for + # a region smaller than a single pixel, where naxis was clamped up to 1. + _pxs = float(pixel_scale.value if isinstance(pixel_scale, u.Quantity) + else pixel_scale) + _pts = points.value if isinstance(points, u.Quantity) else np.asarray(points) + _naxis = np.asarray(naxis, dtype=float) + _extent = np.atleast_1d(np.asarray(extent, dtype=float)) + + crval = np.where( + _naxis < _extent, + _pts.min(axis=0) + (_naxis / 2) * _pxs, # anchored + (_pts.min(axis=0) + _pts.max(axis=0) + + np.asarray(offset, dtype=float)) / 2, # centred + ) + if isinstance(points, u.Quantity): + crval = crval * points.unit # Cannot do `in "DX"` here because that would also match the empty string. linsuff = {"D", "X"} diff --git a/scopesim/tests/tests_optics/test_ImagePlaneUtils.py b/scopesim/tests/tests_optics/test_ImagePlaneUtils.py index dd95ae12f..d5c65c3e4 100644 --- a/scopesim/tests/tests_optics/test_ImagePlaneUtils.py +++ b/scopesim/tests/tests_optics/test_ImagePlaneUtils.py @@ -229,6 +229,52 @@ def test_overlay_images_works_for_3d_cubes_at_border(self): plt.show() +class TestLatticeIsSharedBetweenChunks: + """Regions carved from one origin in whole-pixel steps share a lattice. + + This is what chunked FOVs are: the image plane, the full-size chunks and + the smaller last chunk are all measured from the same detector edge. When + the region is not a whole number of pixels across, CRVAL used to absorb + each region's own rounding remainder, putting them on lattices offset from + one another by fractions of a pixel. + """ + + @pytest.mark.parametrize("extent_px", [24.0, 21.0, 17.32, 17.1, 12621 + 1/3]) + @pytest.mark.parametrize("chunk", [4, 8]) + def test_chunks_land_on_the_parent_lattice(self, extent_px, chunk): + pixel_scale = 0.1 + lo = -extent_px / 2 * pixel_scale + hi = lo + extent_px * pixel_scale + + parent, parent_naxis = imp_utils.create_wcs_from_points( + np.array([[lo, lo], [hi, hi]]), pixel_scale) + + edges = [lo] + for val in np.arange(lo, hi, chunk * pixel_scale): + if lo < val < hi and val > edges[-1]: + edges.append(float(val)) + edges.append(hi) + + for x0, x1 in zip(edges[:-1], edges[1:]): + child, child_naxis = imp_utils.create_wcs_from_points( + np.array([[x0, x0], [x1, x1]]), pixel_scale) + # the child's pixel 0 must sit on a whole-pixel offset of the + # parent's grid, otherwise the projection has to snap it + world = child.wcs_pix2world([[0, 0]], 0) + pix = parent.wcs_world2pix(world, 0)[0] + np.testing.assert_allclose(pix, np.round(pix), atol=1e-6) + + @pytest.mark.parametrize("pnts, pixel_scale", [ + (np.array([[0, 0]]), 5), # single point + (np.array([[-1, -1], [-1, 1], [1, -1], [1, 1]]), 5), # sub-pixel + ]) + def test_regions_smaller_than_a_pixel_stay_centred(self, pnts, pixel_scale): + wcs, naxis = imp_utils.create_wcs_from_points(pnts, pixel_scale) + np.testing.assert_array_equal(naxis, [1, 1]) + np.testing.assert_array_equal(wcs.wcs.crval, + pnts.mean(axis=0)) + + class TestOverlayImageSnapping: """The origin, not the centre, must be snapped to the pixel grid. From 3fd301db000d990d75219b9c378ce3607def6fb4 Mon Sep 17 00:00:00 2001 From: Kieran Leschinski Date: Fri, 4 Sep 2026 21:57:06 +0200 Subject: [PATCH 019/130] Generate chunk edges without np.arange's platform-dependent endpoint FOVManager._get_splits used np.arange(v_min, v_max, step) for the chunk boundaries. That returns ceil((v_max - v_min) / step) elements, which is one too many whenever the division rounds just above an integer - as it does for a span that is an exact multiple of step: (1.2000000000000002 - -1.2000000000000002) / 0.8 == 3.0000000000000004 Whether the surplus element lands exactly on v_max or one ulp below it depends on how the platform rounds v_min + i * step. It lands on v_max on x86, where FovVolumeList.split then discards it (v_max <= value). On arm64 macOS it lands below v_max, which is a perfectly legal split point, so split accepts it and carves off a volume ~1e-15 px wide. create_wcs_from_points rounds that up to a whole pixel, so it becomes a spurious one-pixel FOV, off the pixel lattice, adding flux at the edge of the array. A detector footprint that is an exact multiple of chunk_size therefore got 4 extra degenerate FOVs on macOS and none on Linux or Windows. Emulating the arm64 rounding locally, a 24 px footprint at chunk_size 8 gives: np.arange, x86 rounding 9 volumes, none degenerate np.arange, arm64 rounding 16 volumes, 4 of width 0 chunk_edges 9 volumes, none degenerate Compute the edges from the chunk count instead and return only interior points, so no edge can coincide with either bound on any platform. step still need not divide the span or be an integer, so this is not a range(). The two test helpers added in the previous commits duplicated the arange expression and so inherited the same fragility - that is what failed CI on macos-latest, on exactly the exact-multiple parametrisations. They now call chunk_edges rather than reimplementing it. Behaviour on x86 is unchanged: the full MICADO image plane (12621 x 12888) comes out bit-identical to the previous commit. --- scopesim/optics/fov_manager.py | 33 +++++++++-- .../test_source_fov_imageplane_play_nicely.py | 15 +++-- .../tests/tests_optics/test_FOVManager.py | 59 ++++++++++++++++++- .../tests_optics/test_ImagePlaneUtils.py | 8 +-- 4 files changed, 96 insertions(+), 19 deletions(-) diff --git a/scopesim/optics/fov_manager.py b/scopesim/optics/fov_manager.py index 14157fb3b..5dc3d64f4 100644 --- a/scopesim/optics/fov_manager.py +++ b/scopesim/optics/fov_manager.py @@ -61,6 +61,32 @@ logger = get_logger(__name__) +def chunk_edges(vmin, vmax, step): + """Return the interior points that cut ``[vmin, vmax]`` into `step` chunks. + + Only interior points are returned, so no edge can coincide with `vmin` or + `vmax`. `step` need not divide the span, and need not be an integer (see + HAWKI/test_hawki/test_full_package_hawki.py), so this cannot be a plain + ``range``. + + It cannot be ``np.arange(vmin, vmax, step)`` either. That returns + ``ceil((vmax - vmin) / step)`` elements, which is one too many when the + division rounds just above an integer - as it does for a span that is an + exact multiple of `step`, e.g. ``(1.2000000000000002 - -1.2000000000000002) + / 0.8 == 3.0000000000000004``. Whether the surplus element then lands + exactly on `vmax` or one ulp below it depends on how the platform rounds + ``vmin + i * step``: it lands on `vmax` on x86, but below it on arm64 + macOS. One ulp below `vmax` is a perfectly legal split point, so + ``FovVolumeList.split`` accepts it and carves off a volume ~1e-15 px wide, + which ``create_wcs_from_points`` then rounds up to a whole pixel - a + spurious one-pixel FOV, off the pixel lattice, double-counting flux at the + edge of the array. + + """ + n_chunks = max(1, int(np.ceil((vmax - vmin) / step - 1e-9))) + return vmin + step * np.arange(1, n_chunks) + + class FOVManager: """ A class to manage the (monochromatic) image windows covering the target. @@ -122,11 +148,8 @@ def _get_splits(self, pixel_scale): if vol_pix_area > max_seg_size: step = chunk_size * pixel_scale - # These are not always integers, unlike in the tests. - # See for example HAWKI/test_hawki/test_full_package_hawki.py. - # The np.arange can therefore not be changed to just a range. - yield (np.arange(vol["x_min"], vol["x_max"], step), - np.arange(vol["y_min"], vol["y_max"], step)) + yield (chunk_edges(vol["x_min"], vol["x_max"], step), + chunk_edges(vol["y_min"], vol["y_max"], step)) def generate_fovs_list(self) -> Iterator[FieldOfView]: """ diff --git a/scopesim/tests/tests_integrations/test_source_fov_imageplane_play_nicely.py b/scopesim/tests/tests_integrations/test_source_fov_imageplane_play_nicely.py index f87ea5ec2..8f7bd3899 100644 --- a/scopesim/tests/tests_integrations/test_source_fov_imageplane_play_nicely.py +++ b/scopesim/tests/tests_integrations/test_source_fov_imageplane_play_nicely.py @@ -12,6 +12,7 @@ from scopesim.optics import image_plane_utils as imp_utils from scopesim.optics.fov import FieldOfView2D +from scopesim.optics.fov_manager import chunk_edges from scopesim.optics.image_plane import ImagePlane from scopesim.tests.mocks.py_objects import source_objects as src @@ -99,14 +100,12 @@ def _chunked_fov_headers(x_mm, y_mm, pixel_size, pixel_scale, chunk): for v in (x_mm[0], x_mm[1], y_mm[0], y_mm[1])] def edges(vmin, vmax): - out = [vmin] - for val in np.arange(vmin, vmax, chunk * pixel_scale): - if vmin < val < vmax and val > out[-1]: - out.append(float(val)) - return out + [vmax] - - for x0, x1 in zip(*(lambda e: (e[:-1], e[1:]))(edges(lims[0], lims[1]))): - for y0, y1 in zip(*(lambda e: (e[:-1], e[1:]))(edges(lims[2], lims[3]))): + # the real chunk-edge helper, not a copy of it + return [vmin, *chunk_edges(vmin, vmax, chunk * pixel_scale), vmax] + + xe, ye = edges(lims[0], lims[1]), edges(lims[2], lims[3]) + for x0, x1 in zip(xe[:-1], xe[1:]): + for y0, y1 in zip(ye[:-1], ye[1:]): skyhdr = imp_utils.header_from_list_of_xy( [x0 / 3600, x1 / 3600], [y0 / 3600, y1 / 3600], pixel_scale=pixel_scale / 3600) diff --git a/scopesim/tests/tests_optics/test_FOVManager.py b/scopesim/tests/tests_optics/test_FOVManager.py index e1a055e71..f5f094ddf 100644 --- a/scopesim/tests/tests_optics/test_FOVManager.py +++ b/scopesim/tests/tests_optics/test_FOVManager.py @@ -3,7 +3,7 @@ import numpy as np from astropy import units as u -from scopesim.optics.fov_manager import FOVManager +from scopesim.optics.fov_manager import FOVManager, chunk_edges from scopesim.tests.mocks.py_objects import effects_objects as eo from scopesim.utils import from_currsys @@ -56,3 +56,60 @@ def test_returns_n_fovs_for_smaller_chunk_size(self, chunk_size, n_fovs): assert len(fovs) == 4 assert fov_skycorners.min(axis=0)[0] == approx(-1024 / 3600) # [deg] 2k detector / pixel_scale assert fovs[0].waverange[0] == 0.6 * u.um # filter blue edge + + +class TestChunkEdges: + """Chunk edges must be interior and platform-independent. + + np.arange(vmin, vmax, step) returns one element too many when + (vmax - vmin) / step rounds just above an integer, and whether that + element lands on vmax or one ulp below it depends on how the platform + rounds vmin + i * step. One ulp below vmax is a legal split point, so + FovVolumeList.split would carve off a ~1e-15 px volume that + create_wcs_from_points then rounds up into a spurious one-pixel FOV. + """ + + @pytest.mark.parametrize("span_px, chunk, n_chunks", [ + (24.0, 8, 3), # exact multiple -- the fragile case + (24.0, 4, 6), # exact multiple + (16.0, 8, 2), + (21.0, 8, 3), # fractional last chunk + (17.32, 8, 3), # fractional pixels, fractional last chunk + (12621 + 1 / 3, 2048, 7), # MICADO's y axis + (5.0, 8, 1), # smaller than one chunk + ]) + def test_edges_are_interior_and_count_is_right(self, span_px, chunk, + n_chunks): + pixel_scale = 0.1 + vmin = -span_px / 2 * pixel_scale + vmax = vmin + span_px * pixel_scale + edges = chunk_edges(vmin, vmax, chunk * pixel_scale) + + assert len(edges) == n_chunks - 1 + assert (edges > vmin).all() + assert (edges < vmax).all() + + @pytest.mark.parametrize("span_px, chunk", [(24.0, 8), (24.0, 4), + (16.0, 8), (32.0, 16)]) + def test_exact_multiples_give_no_sliver_chunk(self, span_px, chunk): + # A span that is an exact multiple of the chunk must come out as + # equal chunks, with no degenerate remainder however the float + # arithmetic rounds. + pixel_scale = 0.1 + vmin = -span_px / 2 * pixel_scale + vmax = vmin + span_px * pixel_scale + edges = chunk_edges(vmin, vmax, chunk * pixel_scale) + + bounds = np.array([vmin, *edges, vmax]) + widths = np.diff(bounds) / pixel_scale + assert len(widths) == span_px / chunk + np.testing.assert_allclose(widths, chunk, atol=1e-9) + + def test_a_split_one_ulp_inside_the_upper_bound_is_not_emitted(self): + # np.arange does emit such a value on arm64 macOS; assert directly + # that chunk_edges never does, on any platform. + vmin, vmax, step = -1.2000000000000002, 1.2000000000000002, 0.8 + edges = chunk_edges(vmin, vmax, step) + assert (edges < np.nextafter(vmax, -np.inf)).all() + # ... which np.arange does not guarantee + assert len(np.arange(vmin, vmax, step)) == len(edges) + 2 diff --git a/scopesim/tests/tests_optics/test_ImagePlaneUtils.py b/scopesim/tests/tests_optics/test_ImagePlaneUtils.py index d5c65c3e4..1832ec8e1 100644 --- a/scopesim/tests/tests_optics/test_ImagePlaneUtils.py +++ b/scopesim/tests/tests_optics/test_ImagePlaneUtils.py @@ -7,6 +7,7 @@ import matplotlib.pyplot as plt from scopesim.optics import image_plane_utils as imp_utils +from scopesim.optics.fov_manager import chunk_edges from scopesim.tests.mocks.py_objects import imagehdu_objects as imo @@ -249,11 +250,8 @@ def test_chunks_land_on_the_parent_lattice(self, extent_px, chunk): parent, parent_naxis = imp_utils.create_wcs_from_points( np.array([[lo, lo], [hi, hi]]), pixel_scale) - edges = [lo] - for val in np.arange(lo, hi, chunk * pixel_scale): - if lo < val < hi and val > edges[-1]: - edges.append(float(val)) - edges.append(hi) + # the real chunk-edge helper, not a copy of it + edges = [lo, *chunk_edges(lo, hi, chunk * pixel_scale), hi] for x0, x1 in zip(edges[:-1], edges[1:]): child, child_naxis = imp_utils.create_wcs_from_points( From 0070eafe3648c4f790cd8ab6e20f3b0f40a4b46a Mon Sep 17 00:00:00 2001 From: teutoburg Date: Sat, 5 Sep 2026 19:24:42 +0200 Subject: [PATCH 020/130] Comments and cleanup --- scopesim/effects/detector_list.py | 6 ++++-- scopesim/optics/fov_manager.py | 5 ++--- scopesim/optics/image_plane_utils.py | 4 ---- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/scopesim/effects/detector_list.py b/scopesim/effects/detector_list.py index 61fd4fcba..821229394 100644 --- a/scopesim/effects/detector_list.py +++ b/scopesim/effects/detector_list.py @@ -497,9 +497,11 @@ def _get_fov_limits(self, pixel_scale=None): def image_plane_header(self): """Create and return the Image Plane Header.""" # FIXME: Heavy property..... - points = self._get_corner_points().value new_wcs, naxis = create_wcs_from_points( - points, self.pixel_size.to_value(u.mm), "D") + self._get_corner_points(), + self.pixel_size.to(u.mm), + wcs_suffix="D", + ) hdr = fits.Header() hdr["NAXIS"] = 3 diff --git a/scopesim/optics/fov_manager.py b/scopesim/optics/fov_manager.py index 5dc3d64f4..af062eac8 100644 --- a/scopesim/optics/fov_manager.py +++ b/scopesim/optics/fov_manager.py @@ -170,7 +170,7 @@ def generate_fovs_list(self) -> Iterator[FieldOfView]: for effect in self.effects: self.volumes_list = effect.apply_to(self.volumes_list, **params) - # ..todo: add catch to split volumes larger than chunk_size + # TODO: add catch to split volumes larger than chunk_size pixel_scale = from_currsys(self.meta["pixel_scale"], self.cmds) plate_scale = from_currsys(self.meta["plate_scale"], self.cmds) @@ -182,7 +182,6 @@ def generate_fovs_list(self) -> Iterator[FieldOfView]: for vol in self.volumes_list: xs_min, xs_max = vol["x_min"] / 3600., vol["x_max"] / 3600. ys_min, ys_max = vol["y_min"] / 3600., vol["y_max"] / 3600. - waverange = (vol["wave_min"], vol["wave_max"]) skyhdr = ipu.header_from_list_of_xy([xs_min, xs_max], [ys_min, ys_max], pixel_scale=pixel_scale / 3600.) @@ -210,7 +209,7 @@ def generate_fovs_list(self) -> Iterator[FieldOfView]: new_fov = fovcls( skyhdr, - waverange, + (vol["wave_min"], vol["wave_max"]), detector_header=dethdr, cmds=self.cmds, **vol["meta"], diff --git a/scopesim/optics/image_plane_utils.py b/scopesim/optics/image_plane_utils.py index b6feb9230..7cb70d2bc 100644 --- a/scopesim/optics/image_plane_utils.py +++ b/scopesim/optics/image_plane_utils.py @@ -1221,8 +1221,6 @@ def det_wcs_from_sky_wcs( Shape of the image (``NAXIS1``, ``NAXIS2``). """ - # TODO: Using astropy units for now to avoid deg vs. arcsec confusion. - # Once Scopesim is consistent there, remove astropy units. pixel_scale <<= u.arcsec / u.pixel plate_scale <<= u.arcsec / u.mm logger.debug("Pixel scale: %s", pixel_scale) @@ -1268,8 +1266,6 @@ def sky_wcs_from_det_wcs(det_wcs: WCS, Shape of the image (``NAXIS1``, ``NAXIS2``). """ - # TODO: Using astropy units for now to avoid deg vs. arcsec confusion. - # Once Scopesim is consistent there, remove astropy units. pixel_scale <<= u.arcsec / u.pixel plate_scale <<= u.arcsec / u.mm logger.debug("Pixel scale: %s", pixel_scale) From ca8b8eb1efff6f15de81869d7b2315b8b6a3a04d Mon Sep 17 00:00:00 2001 From: teutoburg Date: Sun, 6 Sep 2026 02:23:49 +0200 Subject: [PATCH 021/130] Careful when cherry-picking... --- scopesim/effects/detector_list.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scopesim/effects/detector_list.py b/scopesim/effects/detector_list.py index 821229394..f3a1a5f09 100644 --- a/scopesim/effects/detector_list.py +++ b/scopesim/effects/detector_list.py @@ -499,7 +499,7 @@ def image_plane_header(self): # FIXME: Heavy property..... new_wcs, naxis = create_wcs_from_points( self._get_corner_points(), - self.pixel_size.to(u.mm), + self.pixel_size.to(u.mm) / u.pixel, wcs_suffix="D", ) From f37016765dec4af64b7a5c571c2f19da0823e86d Mon Sep 17 00:00:00 2001 From: teutoburg Date: Sun, 30 Aug 2026 23:05:14 +0200 Subject: [PATCH 022/130] Remove `apply_throughput_to_cube` This was only used in two places, and one of them has now been refactored to use something else, so no point in keeping the separate function, which in the end was just a one-liner anyway. --- scopesim/effects/spectral_efficiency.py | 13 ++++++----- scopesim/effects/ter_curves_utils.py | 30 ------------------------- 2 files changed, 8 insertions(+), 35 deletions(-) diff --git a/scopesim/effects/spectral_efficiency.py b/scopesim/effects/spectral_efficiency.py index 35fe76fc3..fde4c8297 100644 --- a/scopesim/effects/spectral_efficiency.py +++ b/scopesim/effects/spectral_efficiency.py @@ -11,7 +11,6 @@ from .effects import Effect from .ter_curves import TERCurve -from .ter_curves_utils import apply_throughput_to_cube from ..utils import figure_factory, get_logger @@ -91,9 +90,12 @@ def get_efficiencies(self): wavelength = tbl['wavelength'].quantity efficiency = tbl['efficiency'].value params.pop("filename", None) # don't pass filename to TERCurve! - effic_curve = TERCurve(array_dict={"wavelength":wavelength, - "transmission":efficiency}, - **params) + effic_curve = TERCurve( + array_dict={ + "wavelength": wavelength, + "transmission": efficiency, + }, + **params) efficiencies[name] = effic_curve hdul.close() @@ -111,7 +113,8 @@ def apply_to(self, obj, **kwargs): swcs = WCS(obj.hdu.header).spectral with u.set_enabled_equivalencies(u.spectral()): wave = swcs.pixel_to_world(np.arange(swcs.pixel_shape[0])) << u.um - obj.hdu = apply_throughput_to_cube(obj.hdu, effic.throughput, wave) + thru = effic.throughput(wave) + obj.hdu.data = obj.hdu.data * thru.value[:, None, None] return obj def plot(self): diff --git a/scopesim/effects/ter_curves_utils.py b/scopesim/effects/ter_curves_utils.py index 827bf742a..8dcf62320 100644 --- a/scopesim/effects/ter_curves_utils.py +++ b/scopesim/effects/ter_curves_utils.py @@ -359,36 +359,6 @@ def scale_spectrum(spectrum, filter_name, amplitude): return spectrum -def apply_throughput_to_cube( - cube: fits.ImageHDU, - thru: SpectralElement | SourceSpectrum, - wave_cube: u.Quantity, -) -> fits.ImageHDU: - """ - Apply throughput curve to a spectroscopic cube. - - Parameters - ---------- - cube : fits.ImageHDU - Three-dimensional image, dimension 0 (in python convention) is the - spectral dimension. - thru : SpectralElement | SourceSpectrum - Throughput curve, spectrum or filter. - wave_cube : u.Quantity["length"] - Wavelength axis of the cube. - - Returns - ------- - cube : fits.ImageHDU - Header unchanged, data multiplied with wavelength-dependent throughput. - - """ - # Note: wave_cube used to be converted to AA, but synphot understands um - # or whatever just as well... - cube.data *= thru(wave_cube).value[:, None, None] - return cube - - def add_edge_zeros(tbl, wave_colname): if isinstance(tbl, Table): vals = np.zeros(len(tbl.colnames)) From 070f4b9781ddcb6d2ca45c7ecb1c6c27883396bb Mon Sep 17 00:00:00 2001 From: teutoburg Date: Sun, 30 Aug 2026 23:46:28 +0200 Subject: [PATCH 023/130] mv `scale_spectrum` to `surface.py`, rm unused functions It was only used there, and the module is small enough that keeping it elsewhere doesn't make much sense. Also removes one cross-import between optics and effects. There is still `get_filter`, but maybe we can remove that as well, and the whole scale function actually, if we used the existing functionality in SpeXtra / synphot directly... The functions `zero_mag_flux` and `get_zero_mag_spectrum` were not used anywhere except in their own unit tests, and are definetely redundant with SpeXtra / synphot, so were removed entirely. --- scopesim/effects/ter_curves_utils.py | 147 +----------------- scopesim/optics/surface.py | 121 ++++++++++++-- scopesim/tests/tests_effects/test_Surface.py | 27 +++- .../tests/tests_effects/test_SurfaceList.py | 3 +- .../tests_effects/test_ter_curves_utils.py | 51 +----- 5 files changed, 138 insertions(+), 211 deletions(-) diff --git a/scopesim/effects/ter_curves_utils.py b/scopesim/effects/ter_curves_utils.py index 8dcf62320..b7454e2c2 100644 --- a/scopesim/effects/ter_curves_utils.py +++ b/scopesim/effects/ter_curves_utils.py @@ -8,12 +8,8 @@ from astropy.table import Table from astropy.io.votable import parse_single_table from astropy.io import ascii as ioascii -from astropy.io import fits -from synphot import SpectralElement, SourceSpectrum, Empirical1D, Observation -from synphot.units import PHOTLAM +from synphot import SpectralElement, Empirical1D -from scopesim.source.source_templates import vega_spectrum, st_spectrum, \ - ab_spectrum from ..utils import find_file, quantity_from_table, get_logger from ..server.download_utils import create_client, handle_download @@ -218,147 +214,6 @@ def get_filter(filter_name): return filt -def get_zero_mag_spectrum(system_name: str = "AB") -> SourceSpectrum: - """ - Return a synphot spectrum of 0 mag in given photometric system. - - Parameters - ---------- - system_name : {"AB", "Vega", "ST", "HST"} - Name of the photometric system. - - Raises - ------ - ValueError - Raised if `system_name` is invalid. - - Returns - ------- - spec : synphot.SourceSpectrum - Output spectrum. - - """ - if system_name.lower() == "vega": - return vega_spectrum() - if system_name.lower() == "ab": - return ab_spectrum() - if system_name.lower() in ("st", "hst"): - return st_spectrum() - raise ValueError(f"system_name {system_name} is unknown") - - -def zero_mag_flux(filter_name, photometric_system, return_filter=False): - """ - Return the zero magnitude photon flux for a filter. - - Acceptable filter names are those given in - ``scopesim.effects.ter_curves_utils.FILTER_DEFAULTS`` or a string with an - appropriate name for a filter in the Spanish-VO filter-service. Such - strings must use the naming convention: observatory/instrument.filter. E.g: - "paranal/HAWKI.Ks", or "Gemini/GMOS-N.CaT". - - Parameters - ---------- - filter_name : str - Name of the filter - see above. - - photometric_system : {"vega", "AB", "ST"} - Name of the photometric system - - return_filter : bool, optional - If True, also returns the filter curve object - - Returns - ------- - flux : float - [PHOTLAM] - filt : ``synphot.SpectralElement`` - If `return_filter` is True - - """ - filt = get_filter(filter_name) - spec = get_zero_mag_spectrum(photometric_system) - - obs = Observation(spec, filt) - flux = obs.effstim(flux_unit=PHOTLAM) - - if return_filter: - return flux, filt - return flux - - -def scale_spectrum(spectrum, filter_name, amplitude): - """ - Scale a SourceSpectrum to a value in a filter. - - Parameters - ---------- - spectrum : synphot.SourceSpectrum - - filter_name : str - Name of a filter from - - a local instrument package (available in ``rc.__search_path__``) - - a generic filter name (see ``ter_curves_utils.FILTER_DEFAULTS``) - - a spanish-vo filter service reference (e.g. "Paranal/HAWKI.Ks") - - amplitude : astropy.Quantity, float - The value that the spectrum should have in the given filter. Acceptable - astropy quantities are: - - u.mag : Vega magnitudes - - u.ABmag : AB magnitudes - - u.STmag : HST magnitudes - - u.Jy : Jansky per filter bandpass - Additionally the "FLAM" and "FNU" units from ``synphot.units`` can - be used when passing the quantity for `amplitude`. - - Returns - ------- - spectrum : synphot.SourceSpectrum - Input spectrum scaled to the given amplitude in the given filter. - - Examples - -------- - :: - - >>> from scopesim.source.source_templates import vega_spectrum - >>> from scopesim.effects.ter_curves_utils as ter_utils - >>> - >>> spec = vega_spectrum() - >>> vega_185 = ter_utils.scale_spectrum(spec, "Ks", -1.85 * u.mag) - >>> ab_0 = ter_utils.scale_spectrum(spec, "Ks", 0 * u.ABmag) - >>> jy_3630 = ter_utils.scale_spectrum(spec, "Ks", 3630 * u.Jy) - - """ - if isinstance(amplitude, u.Quantity): - if amplitude.unit.physical_type == "spectral flux density": - if amplitude.unit != u.ABmag: - amplitude = amplitude.to(u.ABmag) - ref_spec = ab_spectrum(amplitude.value) - - elif amplitude.unit.physical_type == "spectral flux density wav": - if amplitude.unit != u.STmag: - amplitude = amplitude.to(u.STmag) - ref_spec = st_spectrum(amplitude.value) - - elif amplitude.unit == u.mag: - ref_spec = vega_spectrum(amplitude.value) - - else: - raise ValueError(f"Units of amplitude must be one of " - f"[u.mag, u.ABmag, u.STmag, u.Jy]: {amplitude}") - else: - ref_spec = vega_spectrum(amplitude) - - filt = get_filter(filter_name) - ref_flux = Observation(ref_spec, filt).effstim(flux_unit=PHOTLAM) - - real_flux = Observation(spectrum, filt).effstim(flux_unit=PHOTLAM) - scale_factor = ref_flux / real_flux - spectrum *= scale_factor.value - - return spectrum - - def add_edge_zeros(tbl, wave_colname): if isinstance(tbl, Table): vals = np.zeros(len(tbl.colnames)) diff --git a/scopesim/optics/surface.py b/scopesim/optics/surface.py index fb7f70963..f60c9f0e4 100644 --- a/scopesim/optics/surface.py +++ b/scopesim/optics/surface.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + from pathlib import Path from dataclasses import dataclass from typing import Any @@ -8,14 +10,29 @@ from astropy.io import ascii as ioascii from astropy.table import Table -from synphot import SpectralElement -from synphot.models import Empirical1D +from synphot import SpectralElement, SourceSpectrum, Empirical1D, Observation +from synphot.units import PHOTLAM + +from ..utils import ( + get_meta_quantity, + quantify, + from_currsys, + convert_table_comments_to_dict, + find_file, + get_logger, +) +from ..source.source_templates import ( + vega_spectrum, + st_spectrum, + ab_spectrum, +) +from ..effects.ter_curves_utils import get_filter +from .surface_utils import ( + make_emission_from_emissivity, + make_emission_from_array, + extract_type_from_unit, +) -from ..effects import ter_curves_utils as ter_utils -from .surface_utils import (make_emission_from_emissivity, - make_emission_from_array, extract_type_from_unit) -from ..utils import (get_meta_quantity, quantify, from_currsys, - convert_table_comments_to_dict, find_file, get_logger) logger = get_logger(__name__) @@ -88,15 +105,15 @@ def wavelength(self): return self._get_array("wavelength") @property - def throughput(self): + def throughput(self) -> SpectralElement: return self._get_ter_property(self.meta.get("action", "transmission")) @property - def transmission(self): + def transmission(self) -> SpectralElement: return self._get_ter_property("transmission") @property - def emissivity(self): + def emissivity(self) -> SourceSpectrum: return self._get_ter_property("emissivity") @property @@ -144,7 +161,7 @@ def emission(self): filter_name = dic["filter_name"] if "filename_format" in dic: filter_name = dic["filename_format"].format(filter_name) - flux = ter_utils.scale_spectrum(flux, filter_name, amplitude) + flux = scale_spectrum(flux, filter_name, amplitude) return flux @@ -197,9 +214,12 @@ def _get_ter_property(self, ter_property, fmt="synphot"): fill_value = 0. if ter_property == "transmission" else np.nan if value_arr is not None and wave is not None and fmt == "synphot": - response_curve = SpectralElement(Empirical1D, points=wave, - lookup_table=value_arr, - fill_value=fill_value) + response_curve = SpectralElement( + Empirical1D, + points=wave, + lookup_table=value_arr, + fill_value=fill_value, + ) elif fmt == "array": response_curve = value_arr else: @@ -299,3 +319,76 @@ def __str__(self): msg = f"SpectralSurface [{cols}] \"{name}\"" return msg + + +# TODO: This feels WET with some SpeXtra / synphot functionality... +def scale_spectrum(spectrum, filter_name, amplitude): + """ + Scale a SourceSpectrum to a value in a filter. + + Parameters + ---------- + spectrum : synphot.SourceSpectrum + + filter_name : str + Name of a filter from + - a local instrument package (available in ``rc.__search_path__``) + - a generic filter name (see ``ter_curves_utils.FILTER_DEFAULTS``) + - a spanish-vo filter service reference (e.g. "Paranal/HAWKI.Ks") + + amplitude : astropy.Quantity, float + The value that the spectrum should have in the given filter. Acceptable + astropy quantities are: + - u.mag : Vega magnitudes + - u.ABmag : AB magnitudes + - u.STmag : HST magnitudes + - u.Jy : Jansky per filter bandpass + Additionally the "FLAM" and "FNU" units from ``synphot.units`` can + be used when passing the quantity for `amplitude`. + + Returns + ------- + spectrum : synphot.SourceSpectrum + Input spectrum scaled to the given amplitude in the given filter. + + Examples + -------- + :: + + >>> from scopesim.source.source_templates import vega_spectrum + >>> from scopesim.optics.surface import scale_spectrum + >>> + >>> spec = vega_spectrum() + >>> vega_185 = scale_spectrum(spec, "Ks", -1.85 * u.mag) + >>> ab_0 = scale_spectrum(spec, "Ks", 0 * u.ABmag) + >>> jy_3630 = scale_spectrum(spec, "Ks", 3630 * u.Jy) + + """ + if isinstance(amplitude, u.Quantity): + if amplitude.unit.physical_type == "spectral flux density": + if amplitude.unit != u.ABmag: + amplitude = amplitude.to(u.ABmag) + ref_spec = ab_spectrum(amplitude.value) + + elif amplitude.unit.physical_type == "spectral flux density wav": + if amplitude.unit != u.STmag: + amplitude = amplitude.to(u.STmag) + ref_spec = st_spectrum(amplitude.value) + + elif amplitude.unit == u.mag: + ref_spec = vega_spectrum(amplitude.value) + + else: + raise ValueError(f"Units of amplitude must be one of " + f"[u.mag, u.ABmag, u.STmag, u.Jy]: {amplitude}") + else: + ref_spec = vega_spectrum(amplitude) + + filt = get_filter(filter_name) + ref_flux = Observation(ref_spec, filt).effstim(flux_unit=PHOTLAM) + + real_flux = Observation(spectrum, filt).effstim(flux_unit=PHOTLAM) + scale_factor = ref_flux / real_flux + spectrum *= scale_factor.value + + return spectrum diff --git a/scopesim/tests/tests_effects/test_Surface.py b/scopesim/tests/tests_effects/test_Surface.py index d1e4823b8..141ffe28e 100644 --- a/scopesim/tests/tests_effects/test_Surface.py +++ b/scopesim/tests/tests_effects/test_Surface.py @@ -1,4 +1,6 @@ +# -*- coding: utf-8 -*- import pytest +from pytest import approx import numpy as np from astropy import units as u @@ -6,7 +8,8 @@ from synphot import SpectralElement, SourceSpectrum from scopesim.effects import TERCurve -from scopesim.optics.surface import SpectralSurface +from scopesim.optics.surface import SpectralSurface, scale_spectrum +from scopesim.source.source_templates import vega_spectrum PLOTS = False @@ -51,3 +54,25 @@ def test_returns_surface_object_for_arrays(self): wave = surf.surface.wavelength plt.plot(wave, surf.surface.emission(wave)) plt.show() + + +@pytest.mark.webtest +@pytest.mark.usefixtures("no_file_error") +class TestScaleSpectrum: + def test_scales_vega_spectrum_to_vega_ab_or_jansky(self): + spec = vega_spectrum() + vega_185 = scale_spectrum(spec, "Ks", -1.85 * u.mag) + ab_0 = scale_spectrum(spec, "Ks", 0 * u.ABmag) + jy_3630 = scale_spectrum(spec, "Ks", 3630 * u.Jy) + + wave = np.linspace(1.8, 2.5, 1000) * u.um + assert vega_185(wave).value == approx(ab_0(wave).value, rel=1e-2) + assert vega_185(wave).value == approx(jy_3630(wave).value, rel=1e-2) + + if PLOTS: + plt.plot(wave, spec(wave), "b") + plt.plot(wave, vega_185(wave), "r:") + plt.plot(wave, ab_0(wave), "g--") + plt.plot(wave, jy_3630(wave), "y-.") + plt.semilogy() + plt.show() diff --git a/scopesim/tests/tests_effects/test_SurfaceList.py b/scopesim/tests/tests_effects/test_SurfaceList.py index 8128dd161..465cffdc7 100644 --- a/scopesim/tests/tests_effects/test_SurfaceList.py +++ b/scopesim/tests/tests_effects/test_SurfaceList.py @@ -1,4 +1,5 @@ -""" Tests for the the new SurfaceList object """ +# -*- coding: utf-8 -*- +"""Tests for the the SurfaceList object.""" import pytest from pytest import approx diff --git a/scopesim/tests/tests_effects/test_ter_curves_utils.py b/scopesim/tests/tests_effects/test_ter_curves_utils.py index 555b0c41f..13206ee15 100644 --- a/scopesim/tests/tests_effects/test_ter_curves_utils.py +++ b/scopesim/tests/tests_effects/test_ter_curves_utils.py @@ -1,9 +1,9 @@ +# -*- coding: utf-8 -*- + import pytest -import scopesim.source.source_templates from pytest import approx import numpy as np -from matplotlib import pyplot as plt from astropy import units as u @@ -28,53 +28,6 @@ def test_returns_specific_from_svo(self): assert np.max(ks(wave)) > 0.75 -@pytest.mark.webtest -@pytest.mark.usefixtures("no_file_error") -@pytest.mark.parametrize("filter_name, phot_system, flux", - [("V", "vega", 995), - ("z", "AB", 602), - ("Ks", "vega", 43)]) -def test_zero_mag_flux(filter_name, phot_system, flux): - """ - Photon fluxes are in line with: - http://www.astronomy.ohio-state.edu/~martini/usefuldata.html - """ - new_flux = ter_utils.zero_mag_flux(filter_name, phot_system) - assert flux == approx(new_flux.value, rel=0.08) - - -@pytest.mark.webtest -@pytest.mark.usefixtures("no_file_error") -def test_compare_br_gamma(): - filt = "Ks" - flux_vega = ter_utils.zero_mag_flux(filt, "vega").value - flux_ab = ter_utils.zero_mag_flux(filt, "AB").value - delta_mag = 2.5 * np.log10(flux_ab/flux_vega) - assert delta_mag == approx(1.85, rel=0.01) - - -@pytest.mark.webtest -@pytest.mark.usefixtures("no_file_error") -class TestScaleSpectrum: - def test_scales_vega_spectrum_to_vega_ab_or_jansky(self): - spec = scopesim.source.source_templates.vega_spectrum() - vega_185 = ter_utils.scale_spectrum(spec, "Ks", -1.85 * u.mag) - ab_0 = ter_utils.scale_spectrum(spec, "Ks", 0 * u.ABmag) - jy_3630 = ter_utils.scale_spectrum(spec, "Ks", 3630 * u.Jy) - - wave = np.linspace(1.8, 2.5, 1000) * u.um - assert vega_185(wave).value == approx(ab_0(wave).value, rel=1e-2) - assert vega_185(wave).value == approx(jy_3630(wave).value, rel=1e-2) - - if PLOTS: - plt.plot(wave, spec(wave), "b") - plt.plot(wave, vega_185(wave), "r:") - plt.plot(wave, ab_0(wave), "g--") - plt.plot(wave, jy_3630(wave), "y-.") - plt.semilogy() - plt.show() - - @pytest.mark.webtest @pytest.mark.usefixtures("no_file_error") class TestGetFilterEffectiveWavelength: From 146a466b92da3aee2ced7fe10eac45434f8116cb Mon Sep 17 00:00:00 2001 From: teutoburg Date: Sun, 6 Sep 2026 17:26:30 +0200 Subject: [PATCH 024/130] Tiny cleanup in `OpticalTrain.readout()` - Remove two local vars only needed once. - Consistent naming of detector manager instance and list. --- scopesim/optics/optical_train.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/scopesim/optics/optical_train.py b/scopesim/optics/optical_train.py index 00823c7d8..45cfe947e 100644 --- a/scopesim/optics/optical_train.py +++ b/scopesim/optics/optical_train.py @@ -473,11 +473,12 @@ def readout(self, filename=None, reset=True, **kwargs): self.cmds[f"!OBS.{key}"] = value hduls = [] - for i, detector_array in enumerate(self.detector_managers): - array_effects = self.optics_manager.detector_array_effects - dtcr_effects = self.optics_manager.detector_effects - hdul = detector_array.readout( - self.image_planes, array_effects, dtcr_effects) + for i, detector_manager in enumerate(self.detector_managers): + hdul = detector_manager.readout( + self.image_planes, + self.optics_manager.detector_array_effects, + self.optics_manager.detector_effects, + ) fits_effects = self.optics_manager.get_all(ExtraFitsKeywords) if len(fits_effects) > 0: From 056b1af841133dd95d0ea76e5755c559eb68c18b Mon Sep 17 00:00:00 2001 From: teutoburg Date: Mon, 7 Sep 2026 01:04:20 +0200 Subject: [PATCH 025/130] More cleanup --- scopesim/effects/electronic/noise.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scopesim/effects/electronic/noise.py b/scopesim/effects/electronic/noise.py index 717fec584..00b6049ff 100644 --- a/scopesim/effects/electronic/noise.py +++ b/scopesim/effects/electronic/noise.py @@ -391,10 +391,6 @@ def __call__( ) -> np.ndarray: return data + dark_level * dit * ndit - @property - def random_seed(self) -> int: - return from_currsys(self.meta["random_seed"], self.cmds) - def _get_dark_level(self, det_meta) -> float: dark_level = float(from_currsys(self.meta["value"], self.cmds)) if isinstance(dark_level, Real): From bb32c7d44a24d4f37c6c13f8df0e9b2f6c2bfa69 Mon Sep 17 00:00:00 2001 From: "astarvienna-bot[bot]" <217837468+myciapp[bot]@users.noreply.github.com> Date: Sun, 6 Sep 2026 23:19:13 +0000 Subject: [PATCH 026/130] Bumping version from 0.12.0b2.dev5 to 0.12.0b3.dev0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index da766c6da..75e53403b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ScopeSim" -version = "0.12.0b2.dev5" +version = "0.12.0b3.dev0" description = "Generalised telescope observation simulator" license = {text = "GPL-3.0-or-later"} authors = [ From 81b83624803b64614c5888db7f599f5644da0b66 Mon Sep 17 00:00:00 2001 From: teutoburg Date: Mon, 7 Sep 2026 01:04:56 +0200 Subject: [PATCH 027/130] Add `Detector.det_id` convenience property We needed the line `det_meta[real_colname("id", det_meta)]` a few times now, so it's worth making that simpler. Also means not the whole meta, but rather just a simple int can be passed to the resolve methods. --- scopesim/detector/detector.py | 26 +++++++++++++++++++++--- scopesim/detector/detector_manager.py | 16 ++++++++------- scopesim/effects/electronic/electrons.py | 11 +++++----- scopesim/effects/electronic/noise.py | 24 ++++++++++------------ 4 files changed, 48 insertions(+), 29 deletions(-) diff --git a/scopesim/detector/detector.py b/scopesim/detector/detector.py index b6708d365..eb1ff3a98 100644 --- a/scopesim/detector/detector.py +++ b/scopesim/detector/detector.py @@ -7,9 +7,17 @@ from astropy.wcs import WCS from ..optics import ImagePlane -from ..optics.image_plane_utils import (add_imagehdu_to_imagehdu, - sky_wcs_from_det_wcs) -from ..utils import get_logger, from_currsys, stringify_dict, zeros_from_header +from ..optics.image_plane_utils import ( + add_imagehdu_to_imagehdu, + sky_wcs_from_det_wcs, +) +from ..utils import ( + get_logger, + from_currsys, + stringify_dict, + zeros_from_header, + real_colname, +) logger = get_logger(__name__) @@ -65,6 +73,18 @@ def hdu(self): return self._hdu + @property + def det_id(self) -> int | None: + """Return detector ID. + + Avoid calling it just "id", because ``id(self)`` is something else. + + If not found in `self.meta`, return None. + """ + if key := real_colname("id", self.meta): + return self.meta[key] + return None + @property def header(self): """Return header from internal HDU.""" diff --git a/scopesim/detector/detector_manager.py b/scopesim/detector/detector_manager.py index de6fd24a7..9c0f80b86 100644 --- a/scopesim/detector/detector_manager.py +++ b/scopesim/detector/detector_manager.py @@ -40,8 +40,13 @@ def __init__(self, detector_list=None, cmds=None, **kwargs): self._latest_exposure: HDUList | None = None - def readout(self, image_planes, array_effects=None, dtcr_effects=None, - **kwargs) -> HDUList: + def readout( + self, + image_planes, + array_effects: list[Effect] | None = None, + dtcr_effects: list[Effect] | None = None, + **kwargs + ) -> HDUList: """ Read out the detector array into a FITS HDU List. @@ -74,9 +79,6 @@ def readout(self, image_planes, array_effects=None, dtcr_effects=None, Output FITS HDU List. """ - # .. note:: Detector is what used to be called Chip - # DetectorManager is the old Detector - self._array_effects = array_effects or [] self._dtcr_effects = dtcr_effects or [] self.meta.update(kwargs) @@ -100,7 +102,7 @@ def readout(self, image_planes, array_effects=None, dtcr_effects=None, detector = effect.apply_to(detector) # 6. add necessary header keywords - # .. todo: add keywords + # TODO: add keywords # FIXME: Why is this applied twice ??? for effect in self._array_effects: @@ -148,7 +150,7 @@ def _make_primary_hdu(self): return prihdu def _make_effects_hdu(self): - # .. todo:: decide what goes into the effects table of meta data + # TODO: decide what goes into the effects table of meta data # effects = self._array_effects + self._dtcr_effects return TableHDU() diff --git a/scopesim/effects/electronic/electrons.py b/scopesim/effects/electronic/electrons.py index b28f55b05..b279df63e 100644 --- a/scopesim/effects/electronic/electrons.py +++ b/scopesim/effects/electronic/electrons.py @@ -18,7 +18,7 @@ from .. import Effect from ...detector import Detector from ...utils import figure_factory, check_keys -from ...utils import from_currsys, real_colname +from ...utils import from_currsys from . import logger @@ -309,14 +309,13 @@ def _should_apply(self) -> bool: def __call__(self, data: ArrayLike, gain: Real) -> NDArray: return data / gain - def _get_gain(self, det_meta) -> Real: + def _get_gain(self, det_id: int) -> Real: # Apply the gain value (copy from DarkCurrent) # Note that this does not cater for the case where the gain is given # as a plain dictionary. Should we implement that? if hasattr(self.cmds["!DET.gain"], "dic"): - dtcr_id = det_meta[real_colname("id", det_meta)] - gain = self.cmds["!DET.gain"].dic[dtcr_id] - logger.info(f"Detector {dtcr_id}: applying gain {gain}") + gain = self.cmds["!DET.gain"].dic[det_id] + logger.info(f"Detector {det_id}: applying gain {gain}") return gain if isinstance(self.cmds["!DET.gain"], Real): gain = self.cmds["!DET.gain"] @@ -333,7 +332,7 @@ def apply_to(self, obj, **kwargs): new_dtype = self.meta["dtype"] # Apply gain - obj.data = self(obj.data, self._get_gain(obj.meta)) + obj.data = self(obj.data, self._get_gain(obj.det_id)) # Type-conversion wraps around input values that are higher or lower than # the respective maximum and minimum values of the new data type. Before diff --git a/scopesim/effects/electronic/noise.py b/scopesim/effects/electronic/noise.py index 00b6049ff..c149ed5f7 100644 --- a/scopesim/effects/electronic/noise.py +++ b/scopesim/effects/electronic/noise.py @@ -11,7 +11,7 @@ from .. import Effect from ...detector import Detector -from ...utils import from_currsys, figure_factory, check_keys, real_colname +from ...utils import from_currsys, figure_factory, check_keys from . import logger @@ -247,19 +247,18 @@ class PixelResponseNonUniformity(Effect): def __init__(self, **kwargs): super().__init__(**kwargs) self.meta.update(kwargs) - self._gain_maps = {} # keyed by dtcr_id + self._gain_maps = {} # keyed by det_id def apply_to(self, obj, **kwargs): if not isinstance(obj, Detector): return obj random_seed = from_currsys(self.meta.get("prnu_seed"), self.cmds) - id_key = real_colname("id", obj.meta) - dtcr_id = obj.meta[id_key] if id_key is not None else None + det_id = obj.det_id prnu_std_meta = from_currsys(self.meta["prnu_std"], self.cmds) if isinstance(prnu_std_meta, Mapping): - prnu_std = float(from_currsys(prnu_std_meta[dtcr_id], self.cmds)) + prnu_std = float(from_currsys(prnu_std_meta[det_id], self.cmds)) elif isinstance(prnu_std_meta, Real): prnu_std = float(prnu_std_meta) else: @@ -269,16 +268,16 @@ def apply_to(self, obj, **kwargs): ) shape = obj.data.shape - if dtcr_id not in self._gain_maps: + if det_id not in self._gain_maps: rng = np.random.default_rng(random_seed) - self._gain_maps[dtcr_id] = rng.normal( + self._gain_maps[det_id] = rng.normal( loc=1.0, scale=prnu_std, size=shape, ) - if self._gain_maps[dtcr_id].shape != shape: + if self._gain_maps[det_id].shape != shape: raise ValueError("gain map shape mismatch") - obj.data = obj.data * self._gain_maps[dtcr_id] + obj.data = obj.data * self._gain_maps[det_id] return obj def plot(self, det_id=None): @@ -391,13 +390,12 @@ def __call__( ) -> np.ndarray: return data + dark_level * dit * ndit - def _get_dark_level(self, det_meta) -> float: + def _get_dark_level(self, det_id: int) -> float: dark_level = float(from_currsys(self.meta["value"], self.cmds)) if isinstance(dark_level, Real): return dark_level if isinstance(dark_level, Mapping): - dtcr_id = det_meta[real_colname("id", det_meta)] - return from_currsys(dark_level[dtcr_id], self.cmds) + return from_currsys(dark_level[det_id], self.cmds) raise ValueError( f"<{self.__class__.__name__}>.meta['value'] must be either " f"dict-like or scalar number, but is {dark_level}." @@ -408,7 +406,7 @@ def apply_to(self, obj, **kwargs): return obj # Dark level needs detector meta so can't go into __call__() - dark_level = self._get_dark_level(obj.meta) + dark_level = self._get_dark_level(obj.det_id) dit = from_currsys(self.meta["dit"], self.cmds) ndit = from_currsys(self.meta["ndit"], self.cmds) From c9954a731d877e40bf2dad39796e882f6d2220ce Mon Sep 17 00:00:00 2001 From: teutoburg Date: Mon, 7 Sep 2026 13:27:03 +0200 Subject: [PATCH 028/130] Hotfix after #1006 --- scopesim/effects/electronic/pixels.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scopesim/effects/electronic/pixels.py b/scopesim/effects/electronic/pixels.py index 93f746700..f97d1eb2c 100644 --- a/scopesim/effects/electronic/pixels.py +++ b/scopesim/effects/electronic/pixels.py @@ -140,9 +140,9 @@ class UnequalBinnedImage(BinnedImageBase): def __call__(self, data: ArrayLike) -> NDArray: height, width = data.shape binned_data = data.reshape(( - height//self.binx, - self.binx, - width//self.biny, + height//self.biny, + self.biny, + width//self.binx, self.binx )).sum(axis=3).sum(axis=1) return binned_data From 71428519e2c78f2a9f332d4b21d0baac47eb2bf5 Mon Sep 17 00:00:00 2001 From: jbrink Date: Tue, 1 Sep 2026 14:27:03 +0200 Subject: [PATCH 029/130] Fixes to FOV and implane-utils for correct flat_lamp scaling --- scopesim/optics/fov.py | 28 +++++++++++++++++++++------- scopesim/optics/image_plane_utils.py | 6 ++++-- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/scopesim/optics/fov.py b/scopesim/optics/fov.py index 503214f53..f229c2f10 100644 --- a/scopesim/optics/fov.py +++ b/scopesim/optics/fov.py @@ -605,15 +605,17 @@ def _make_scaled_hdu(self, field: ImageSourceField) -> fits.ImageHDU: """Deal with BUNIT and pixel area.""" # Note: Do not scale source data - make a copy first. field_hdu = field.field.copy() # .field is the HDU (yeah...) - field_hdu.data /= self.pixel_area.value + field_hdu.data /= field.pixel_area.value # TODO: Check if this scaling is actually correct. How does this # work with the add_imagehdu_to_imagehdu below? Isn't that # supposed to conserve flux? Test carefully!! if field.is_bunit_spatially_differential: + logger.debug("differential bunit...") # Field is in (PHOTLAM) arcsec-2, need to scale by pixarea - field_hdu.data *= field.pixel_area.value + field_hdu.data *= self.pixel_area.value else: + logger.debug("binned bunit...") # Pixel area doesn't cancel out, need to convert new_bunit = field.bunit / u.arcsec**2 field_hdu.header["BUNIT"] = new_bunit.to_string("fits") @@ -751,6 +753,7 @@ def _make_cubefields(self): image = np.sum(field.data, axis=0) * PHOTLAM/u.arcsec**2 image = (image * self.area * spectral_bin_width).to(u.ph/u.s/u.arcsec**2) + logger.debug("2D FOV make_cubefields: image.mean() = %f", image.mean().value) # FIXME: This might create a 2D ImageHDU with a 3D header, not ideal... yield fits.ImageHDU(data=image, header=field.header) @@ -763,6 +766,7 @@ def _make_imagefields(self, fov_waveset, bin_widths): * yield image to be added to canvas image """ for field in self._get_image_fields(): + logger.debug("2D FOV make_imagefields: field.data.sum() = %f", field.data.sum()) field_hdu = self._make_scaled_hdu(field) # Reevaluate spectrum onto FOV waveset @@ -773,6 +777,7 @@ def _make_imagefields(self, fov_waveset, bin_widths): # Rescale 2D flux weight map by integrated flux from spectrum field_hdu.data *= flux # ph s-1 arcsec-2 + logger.debug("2D FOV make_imagefields: field_hdu.data.mean() = %f ph s-1 arcsec-2", field_hdu.data.mean()) yield field_hdu def _make_tablefields(self, fov_waveset, bin_widths, use_photlam=False): @@ -851,11 +856,13 @@ def make_hdu(self): canvas_image_hdu = imp_utils.add_imagehdu_to_imagehdu( tmp_hdu, canvas_image_hdu, - conserve_flux=True, + conserve_flux=False, spline_order=self.spline_order) + logger.debug("2D FOV make_hdu: canvas_image_hdu.data.mean() = %f ph s-1 arcsec-2", canvas_image_hdu.data.mean()) # TODO: Move this further along at some point... canvas_image_hdu.data *= self.pixel_area.value # eliminate arcsec-2 + logger.debug("After scaling by pixel area: canvas_image_hdu.data.mean() = %f ph s-1 pix-1", canvas_image_hdu.data.mean()) for flux, weight, x, y in self._make_tablefields( fov_waveset, bin_widths): @@ -932,7 +939,8 @@ def _make_cubefields(self, fov_waveset): field_data = field_interp(fov_waveset.value) # Pixel scale conversion - field_data *= field.pixel_area / self.pixel_area + # field_data *= field.pixel_area / self.pixel_area ## not needed in flux density cubes?? + logger.debug("3D FOV make_cubefields: field_data.mean() = %f PHOTLAM arcsec-2", field_data.mean()) field_hdu = fits.ImageHDU(data=field_data, header=field.header) yield field_hdu @@ -957,12 +965,14 @@ def _make_imagefields(self, fov_waveset, spline_order): canvas_image_hdu = imp_utils.add_imagehdu_to_imagehdu( field_hdu, canvas_image_hdu, - spline_order=spline_order) + spline_order=spline_order, + conserve_flux=False, + ) spec = field.spectrum(fov_waveset) - # 2D * 1D -> 3D field_cube = canvas_image_hdu.data[None, :, :] * spec[:, None, None] + logger.debug("3D FOV make_imagefields: field_cube.mean() = %f", field_cube.mean().value) yield field_cube.value def _make_tablefields(self, fov_waveset): @@ -1113,7 +1123,9 @@ def make_hdu(self): canvas_cube_hdu = imp_utils.add_imagehdu_to_imagehdu( field_hdu, canvas_cube_hdu, - spline_order=self.spline_order) + spline_order=self.spline_order, + conserve_flux=False, + ) canvas_cube_hdu.data = sum(self._make_imagefields( fov_waveset, self.spline_order), @@ -1132,6 +1144,8 @@ def make_hdu(self): canvas_cube_hdu.data *= 1e4 # ph/s/AA/arcsec2 --> ph/s/um/arcsec2 canvas_cube_hdu.header["BUNIT"] = "ph s-1 um-1 arcsec-2" + logger.debug("3D FOV make_hdu: canvas_cube_hdu.data.mean() = %f", canvas_cube_hdu.data.mean()) + return canvas_cube_hdu # [ph s-1 um-1 (arcsec-2)] def plot_data(self): diff --git a/scopesim/optics/image_plane_utils.py b/scopesim/optics/image_plane_utils.py index 62f717315..e59fadc90 100644 --- a/scopesim/optics/image_plane_utils.py +++ b/scopesim/optics/image_plane_utils.py @@ -593,7 +593,9 @@ def rescale_imagehdu(imagehdu: fits.ImageHDU, pixel_scale: float | u.Quantity, new_im = np.nan_to_num(new_im, copy=False) sum_new = np.sum(new_im) if sum_new != 0: - new_im *= sum_orig / sum_new + flux_factor = sum_orig / sum_new + logger.debug("flux factor = %f", flux_factor) + new_im *= flux_factor elif sum_orig != 0: logger.warning( "rescale_imagehdu: all input flux (%g) was lost in resampling; " @@ -838,7 +840,7 @@ def add_imagehdu_to_imagehdu(image_hdu: fits.ImageHDU, spline_order=spline_order, conserve_flux=conserve_flux) # TODO: Perhaps add separately formatted WCS logger? - # logger.debug("fromrescale %s", WCS(new_hdu.header, key=canvas_wcs.wcs.alt)) + logger.debug("fromrescale %s", WCS(new_hdu.header, key=canvas_wcs.wcs.alt)) new_hdu = reorient_imagehdu(new_hdu, wcs_suffix=canvas_wcs.wcs.alt, spline_order=spline_order, From d5fd5295fcd7fd7dab9a46b91ca6a6d4469ce5ca Mon Sep 17 00:00:00 2001 From: teutoburg Date: Fri, 4 Sep 2026 12:34:23 +0200 Subject: [PATCH 030/130] Stick with binned flux for 2D images --- scopesim/optics/fov.py | 66 ++++++++++++++++++++++++++++-------------- 1 file changed, 44 insertions(+), 22 deletions(-) diff --git a/scopesim/optics/fov.py b/scopesim/optics/fov.py index f229c2f10..e995fab0e 100644 --- a/scopesim/optics/fov.py +++ b/scopesim/optics/fov.py @@ -605,6 +605,7 @@ def _make_scaled_hdu(self, field: ImageSourceField) -> fits.ImageHDU: """Deal with BUNIT and pixel area.""" # Note: Do not scale source data - make a copy first. field_hdu = field.field.copy() # .field is the HDU (yeah...) + logger.debug("scaling by %f", field.pixel_area.value) field_hdu.data /= field.pixel_area.value # TODO: Check if this scaling is actually correct. How does this @@ -613,12 +614,14 @@ def _make_scaled_hdu(self, field: ImageSourceField) -> fits.ImageHDU: if field.is_bunit_spatially_differential: logger.debug("differential bunit...") # Field is in (PHOTLAM) arcsec-2, need to scale by pixarea + logger.debug("scaling by %f", self.pixel_area.value) field_hdu.data *= self.pixel_area.value else: logger.debug("binned bunit...") # Pixel area doesn't cancel out, need to convert new_bunit = field.bunit / u.arcsec**2 field_hdu.header["BUNIT"] = new_bunit.to_string("fits") + logger.debug("BUNIT after scaling: %s", field_hdu.header["BUNIT"]) return field_hdu @@ -746,14 +749,16 @@ def _make_cubefields(self): # cube_fields come in with units of photlam/arcsec2, # need to convert to ph/s # We need to the voxel volume (spectral and solid angle) for that. - spectral_bin_width = (field.header["CDELT3"] * - u.Unit(field.header["CUNIT3"]) - ).to(u.Angstrom) + spectral_bin_width = ( + field.header["CDELT3"] * + u.Unit(field.header["CUNIT3"]) + ).to(u.Angstrom) # First collapse to image, then convert units image = np.sum(field.data, axis=0) * PHOTLAM/u.arcsec**2 - image = (image * self.area * - spectral_bin_width).to(u.ph/u.s/u.arcsec**2) - logger.debug("2D FOV make_cubefields: image.mean() = %f", image.mean().value) + image = (image * self.area * field.pixel_area * + spectral_bin_width).to(u.ph/u.s) + logger.debug("2D FOV make_cubefields: image.mean() = %f %s", + image.mean().value, image.unit) # FIXME: This might create a 2D ImageHDU with a 3D header, not ideal... yield fits.ImageHDU(data=image, header=field.header) @@ -766,8 +771,18 @@ def _make_imagefields(self, fov_waveset, bin_widths): * yield image to be added to canvas image """ for field in self._get_image_fields(): - logger.debug("2D FOV make_imagefields: field.data.sum() = %f", field.data.sum()) - field_hdu = self._make_scaled_hdu(field) + logger.debug("2D FOV make_imagefields: field.data.sum() = %f %s", + field.data.sum(), field.bunit) + + field_hdu = field.field.copy() + + if field.is_bunit_spatially_differential: + logger.warning("Differential BUNIT in 2D image is discouraged.") + logger.debug("scaling by %f", field.pixel_area.value) + field_hdu.data *= field.pixel_area.value + new_bunit = field.bunit / field.pixel_area.unit + field_hdu.header["BUNIT"] = new_bunit.to_string("fits") + logger.debug("BUNIT after scaling: %s", field_hdu.header["BUNIT"]) # Reevaluate spectrum onto FOV waveset spec = field.spectrum(fov_waveset) # PHOTLAM @@ -776,8 +791,10 @@ def _make_imagefields(self, fov_waveset, bin_widths): flux = (spec * bin_widths * self.area).to_value(u.ph / u.s).sum() # Rescale 2D flux weight map by integrated flux from spectrum - field_hdu.data *= flux # ph s-1 arcsec-2 - logger.debug("2D FOV make_imagefields: field_hdu.data.mean() = %f ph s-1 arcsec-2", field_hdu.data.mean()) + field_hdu.data *= flux # ph s-1 + logger.info( + "2D FOV make_imagefields: field_hdu.data.mean() = %f ph/s", + field_hdu.data.mean()) yield field_hdu def _make_tablefields(self, fov_waveset, bin_widths, use_photlam=False): @@ -856,13 +873,14 @@ def make_hdu(self): canvas_image_hdu = imp_utils.add_imagehdu_to_imagehdu( tmp_hdu, canvas_image_hdu, - conserve_flux=False, + conserve_flux=True, # binned flux needs this spline_order=self.spline_order) - logger.debug("2D FOV make_hdu: canvas_image_hdu.data.mean() = %f ph s-1 arcsec-2", canvas_image_hdu.data.mean()) - - # TODO: Move this further along at some point... - canvas_image_hdu.data *= self.pixel_area.value # eliminate arcsec-2 - logger.debug("After scaling by pixel area: canvas_image_hdu.data.mean() = %f ph s-1 pix-1", canvas_image_hdu.data.mean()) + logger.info( + "2D FOV make_hdu: canvas_image_hdu.data.mean() = %f ph/s/pix", + canvas_image_hdu.data.mean()) + logger.info( + "2D FOV make_hdu: canvas_image_hdu.data.max() = %f ph/s/pix", + canvas_image_hdu.data.max()) for flux, weight, x, y in self._make_tablefields( fov_waveset, bin_widths): @@ -939,8 +957,10 @@ def _make_cubefields(self, fov_waveset): field_data = field_interp(fov_waveset.value) # Pixel scale conversion - # field_data *= field.pixel_area / self.pixel_area ## not needed in flux density cubes?? - logger.debug("3D FOV make_cubefields: field_data.mean() = %f PHOTLAM arcsec-2", field_data.mean()) + field_data *= field.pixel_area / self.pixel_area + logger.debug( + "3D FOV make_cubefields: field_data.mean() = %f " + "PHOTLAM arcsec-2", field_data.mean()) field_hdu = fits.ImageHDU(data=field_data, header=field.header) yield field_hdu @@ -966,13 +986,14 @@ def _make_imagefields(self, fov_waveset, spline_order): field_hdu, canvas_image_hdu, spline_order=spline_order, - conserve_flux=False, + conserve_flux=True, ) spec = field.spectrum(fov_waveset) # 2D * 1D -> 3D field_cube = canvas_image_hdu.data[None, :, :] * spec[:, None, None] - logger.debug("3D FOV make_imagefields: field_cube.mean() = %f", field_cube.mean().value) + logger.debug("3D FOV make_imagefields: field_cube.mean() = %f", + field_cube.mean().value) yield field_cube.value def _make_tablefields(self, fov_waveset): @@ -1124,7 +1145,7 @@ def make_hdu(self): field_hdu, canvas_cube_hdu, spline_order=self.spline_order, - conserve_flux=False, + conserve_flux=True, ) canvas_cube_hdu.data = sum(self._make_imagefields( @@ -1144,7 +1165,8 @@ def make_hdu(self): canvas_cube_hdu.data *= 1e4 # ph/s/AA/arcsec2 --> ph/s/um/arcsec2 canvas_cube_hdu.header["BUNIT"] = "ph s-1 um-1 arcsec-2" - logger.debug("3D FOV make_hdu: canvas_cube_hdu.data.mean() = %f", canvas_cube_hdu.data.mean()) + logger.debug("3D FOV make_hdu: canvas_cube_hdu.data.mean() = %f", + canvas_cube_hdu.data.mean()) return canvas_cube_hdu # [ph s-1 um-1 (arcsec-2)] From ad3592f47927919392d945d4d645f59751fe197a Mon Sep 17 00:00:00 2001 From: oczoske Date: Wed, 9 Sep 2026 16:22:25 +0200 Subject: [PATCH 031/130] Apply throughput in-place for speed --- scopesim/effects/ter_curves.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scopesim/effects/ter_curves.py b/scopesim/effects/ter_curves.py index 1b32b6674..53641a043 100644 --- a/scopesim/effects/ter_curves.py +++ b/scopesim/effects/ter_curves.py @@ -147,7 +147,9 @@ def _apply_to_src(self, src: Source) -> None: # apply transmission to source spectra for fld in src.fields: if isinstance(fld, CubeSourceField): - fld.field.data = self(fld.field.data, fld.waveset) + ## in-place is much faster than self() + #fld.field.data = self(fld.field.data, fld.waveset) + fld.field.data *= thru(fld.waveset).value[:, None, None] elif isinstance(fld, SpectrumSourceField): fld.spectra = { isp: spec * thru # This works because synphot! From 549b04d1219b51810dc1069007249b6141daa77e Mon Sep 17 00:00:00 2001 From: jbrink Date: Wed, 9 Sep 2026 16:43:33 +0200 Subject: [PATCH 032/130] SPEC scaling now also fixed - moved all scaling to conserve-flux functionality in image_plane_utils and added a parameter to identify differential flux units (as used in cube sources). --- scopesim/optics/fov.py | 20 +++++++++++--------- scopesim/optics/image_plane_utils.py | 16 ++++++++++++---- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/scopesim/optics/fov.py b/scopesim/optics/fov.py index e995fab0e..141d72731 100644 --- a/scopesim/optics/fov.py +++ b/scopesim/optics/fov.py @@ -606,7 +606,6 @@ def _make_scaled_hdu(self, field: ImageSourceField) -> fits.ImageHDU: # Note: Do not scale source data - make a copy first. field_hdu = field.field.copy() # .field is the HDU (yeah...) logger.debug("scaling by %f", field.pixel_area.value) - field_hdu.data /= field.pixel_area.value # TODO: Check if this scaling is actually correct. How does this # work with the add_imagehdu_to_imagehdu below? Isn't that @@ -614,10 +613,11 @@ def _make_scaled_hdu(self, field: ImageSourceField) -> fits.ImageHDU: if field.is_bunit_spatially_differential: logger.debug("differential bunit...") # Field is in (PHOTLAM) arcsec-2, need to scale by pixarea - logger.debug("scaling by %f", self.pixel_area.value) - field_hdu.data *= self.pixel_area.value + # logger.debug("scaling by %f", self.pixel_area.value) + # field_hdu.data *= self.pixel_area.value else: logger.debug("binned bunit...") + field_hdu.data /= field.pixel_area.value # Pixel area doesn't cancel out, need to convert new_bunit = field.bunit / u.arcsec**2 field_hdu.header["BUNIT"] = new_bunit.to_string("fits") @@ -957,10 +957,9 @@ def _make_cubefields(self, fov_waveset): field_data = field_interp(fov_waveset.value) # Pixel scale conversion - field_data *= field.pixel_area / self.pixel_area + # field_data *= field.pixel_area / self.pixel_area logger.debug( - "3D FOV make_cubefields: field_data.mean() = %f " - "PHOTLAM arcsec-2", field_data.mean()) + "3D FOV make_cubefields: field_data.mean() = %f [%s]", field_data.mean(), field.bunit) field_hdu = fits.ImageHDU(data=field_data, header=field.header) yield field_hdu @@ -987,13 +986,15 @@ def _make_imagefields(self, fov_waveset, spline_order): canvas_image_hdu, spline_order=spline_order, conserve_flux=True, + differential=True, ) spec = field.spectrum(fov_waveset) # 2D * 1D -> 3D field_cube = canvas_image_hdu.data[None, :, :] * spec[:, None, None] - logger.debug("3D FOV make_imagefields: field_cube.mean() = %f", - field_cube.mean().value) + # logger.debug("3D FOV make_imagefields: field_cube.mean() = %f", + # field_cube.mean().value) + logger.debug(f"3D FOV make_imagefields: {field_cube.mean() = }") yield field_cube.value def _make_tablefields(self, fov_waveset): @@ -1145,7 +1146,8 @@ def make_hdu(self): field_hdu, canvas_cube_hdu, spline_order=self.spline_order, - conserve_flux=True, + conserve_flux=False, + differential=True, ) canvas_cube_hdu.data = sum(self._make_imagefields( diff --git a/scopesim/optics/image_plane_utils.py b/scopesim/optics/image_plane_utils.py index e59fadc90..307b0e0fd 100644 --- a/scopesim/optics/image_plane_utils.py +++ b/scopesim/optics/image_plane_utils.py @@ -485,7 +485,8 @@ def overlay_image(small_im, big_im, coords, mask=None, sub_pixel=False): def rescale_imagehdu(imagehdu: fits.ImageHDU, pixel_scale: float | u.Quantity, wcs_suffix: str = "", conserve_flux: bool = True, - spline_order: int = 1) -> fits.ImageHDU: + spline_order: int = 1, + differential: bool = False) -> fits.ImageHDU: """ Scale the .data array by the ratio of pixel_scale [deg] and CDELTn. @@ -540,6 +541,8 @@ def rescale_imagehdu(imagehdu: fits.ImageHDU, pixel_scale: float | u.Quantity, return imagehdu sum_orig = np.sum(imagehdu.data) + if differential: + sum_orig *= primary_wcs.wcs.cdelt[0]*primary_wcs.wcs.cdelt[1] # Perform the rescaling. Axes need to be inverted because python. zoom_np = zoom[::-1] @@ -592,9 +595,12 @@ def rescale_imagehdu(imagehdu: fits.ImageHDU, pixel_scale: float | u.Quantity, if conserve_flux: new_im = np.nan_to_num(new_im, copy=False) sum_new = np.sum(new_im) + if differential: + sum_new *= (pixel_scale.value**2) + if sum_new != 0: flux_factor = sum_orig / sum_new - logger.debug("flux factor = %f", flux_factor) + logger.info("flux factor = %f", flux_factor) new_im *= flux_factor elif sum_orig != 0: logger.warning( @@ -785,7 +791,8 @@ def add_imagehdu_to_imagehdu(image_hdu: fits.ImageHDU, canvas_hdu: fits.ImageHDU, spline_order: int = 1, wcs_suffix: str = "", - conserve_flux: bool = True) -> fits.ImageHDU: + conserve_flux: bool = True, + differential: bool = False) -> fits.ImageHDU: """ Re-project one ``fits.ImageHDU`` onto another ``fits.ImageHDU``. @@ -838,7 +845,8 @@ def add_imagehdu_to_imagehdu(image_hdu: fits.ImageHDU, new_hdu = rescale_imagehdu(image_hdu, pixel_scale=canvas_pixel_scale / conv_fac, wcs_suffix=canvas_wcs.wcs.alt, spline_order=spline_order, - conserve_flux=conserve_flux) + conserve_flux=conserve_flux, + differential=differential) # TODO: Perhaps add separately formatted WCS logger? logger.debug("fromrescale %s", WCS(new_hdu.header, key=canvas_wcs.wcs.alt)) new_hdu = reorient_imagehdu(new_hdu, From 4c120070ba3e88f6723df3ca66d7f22c3462404b Mon Sep 17 00:00:00 2001 From: jbrink Date: Wed, 9 Sep 2026 16:46:05 +0200 Subject: [PATCH 033/130] Removed stray f-string from debug logging. --- scopesim/optics/fov.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scopesim/optics/fov.py b/scopesim/optics/fov.py index 141d72731..da8489256 100644 --- a/scopesim/optics/fov.py +++ b/scopesim/optics/fov.py @@ -992,9 +992,8 @@ def _make_imagefields(self, fov_waveset, spline_order): spec = field.spectrum(fov_waveset) # 2D * 1D -> 3D field_cube = canvas_image_hdu.data[None, :, :] * spec[:, None, None] - # logger.debug("3D FOV make_imagefields: field_cube.mean() = %f", - # field_cube.mean().value) - logger.debug(f"3D FOV make_imagefields: {field_cube.mean() = }") + logger.debug("3D FOV make_imagefields: field_cube.mean() = %f", + field_cube.mean().value) yield field_cube.value def _make_tablefields(self, fov_waveset): From 14aeaae23d5b3fee65d8d2a561e479b9d6baa36c Mon Sep 17 00:00:00 2001 From: oczoske Date: Wed, 9 Sep 2026 18:01:47 +0200 Subject: [PATCH 034/130] Test for cube source --- scopesim/tests/tests_effects/test_TERCurve.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/scopesim/tests/tests_effects/test_TERCurve.py b/scopesim/tests/tests_effects/test_TERCurve.py index aadff5cc3..67c7ed5a5 100644 --- a/scopesim/tests/tests_effects/test_TERCurve.py +++ b/scopesim/tests/tests_effects/test_TERCurve.py @@ -1,8 +1,12 @@ +"""Tests for TERCurve class""" + import pytest import numpy as np +import numpy.testing as npt from matplotlib import pyplot as plt from astropy import units as u +from astropy.wcs import WCS from scopesim.effects import ter_curves as tc from scopesim.tests.mocks.py_objects import source_objects as so @@ -20,6 +24,8 @@ def _filter_wheel(mock_path_micado): "filename_format": fname, "current_filter": "Br-gamma"}) +# pylint: disable=missing-class-docstring, +# pylint: disable=missing-function-docstring class TestTERCurveApplyTo: def test_adds_bg_to_source_if_source_has_no_bg(self): @@ -47,6 +53,26 @@ def test_adds_bg_to_source_if_source_has_no_bg(self): plt.semilogy(wave, flux, "r") plt.show() + def test_applies_to_cube_source(self): + src = so._cube_source() + wcs = WCS(src.cube_fields[0].header).spectral + nlam = src.cube_fields[0].data.shape[0] + lam = wcs.all_pix2world(np.arange(nlam), 0)[0] * u.m + + eff = eo._filter_surface() + thru = eff.throughput(lam) + + # Save spectrum before application + orig = src.cube_fields[0].data[:, 25, 25] * 1. + + # Apply effect + src = eff.apply_to(src) + + # Spectrum after application + new = src.cube_fields[0].data[:, 25, 25] + + npt.assert_allclose(new, orig * thru) + class TestTERCurvePlot: def test_plots_only_transmission(self): From 828a28438c3cf96500f03379c088c9ab69bc6ffb Mon Sep 17 00:00:00 2001 From: teutoburg Date: Wed, 9 Sep 2026 19:54:48 +0200 Subject: [PATCH 035/130] Add `Detector.__str__()` method --- scopesim/detector/detector.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scopesim/detector/detector.py b/scopesim/detector/detector.py index eb1ff3a98..0bceceeb4 100644 --- a/scopesim/detector/detector.py +++ b/scopesim/detector/detector.py @@ -103,3 +103,8 @@ def data(self, data: ArrayLike): def image(self): """Return data from internal HDU.""" return self.data + + def __str__(self) -> str: + if self.det_id is not None: + return f"Detector {self.det_id:>2d}" + return "Detector" From 38d69ac9c43070c744497d9794c7bc72352e72c1 Mon Sep 17 00:00:00 2001 From: teutoburg Date: Wed, 9 Sep 2026 19:55:22 +0200 Subject: [PATCH 036/130] Use integer detector ID in tests Like in the real case. --- scopesim/tests/tests_effects/test_prnu.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scopesim/tests/tests_effects/test_prnu.py b/scopesim/tests/tests_effects/test_prnu.py index 4d74810f5..021155fe3 100644 --- a/scopesim/tests/tests_effects/test_prnu.py +++ b/scopesim/tests/tests_effects/test_prnu.py @@ -37,10 +37,10 @@ def test_dict_prnu_std(self): """Dict mode: different amplitude per detector ID.""" hdr = header_from_list_of_xy([-5, 5], [-5, 5], 1, "D") dtcr = Detector(hdr) - dtcr.meta["id"] = "H2RG" + dtcr.meta["id"] = 0 dtcr.data[:] = 1000 prnu = PixelResponseNonUniformity( - prnu_std={"H2RG": 0.005, "GeoSnap": 0.020}, prnu_seed=42) + prnu_std={0: 0.005, 1: 0.020}, prnu_seed=42) prnu.apply_to(dtcr) assert dtcr.data.std() > 0 From b729dae6f575de491528098a5e3e4ac66431e3e6 Mon Sep 17 00:00:00 2001 From: jbrink Date: Thu, 10 Sep 2026 10:16:38 +0200 Subject: [PATCH 037/130] Fixed conserve_flux=True parameter in 3D Fov cube case - cancels out, so does not change the results. Debug logging changes. --- scopesim/optics/fov.py | 4 ++-- scopesim/optics/image_plane_utils.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/scopesim/optics/fov.py b/scopesim/optics/fov.py index da8489256..58b1b433f 100644 --- a/scopesim/optics/fov.py +++ b/scopesim/optics/fov.py @@ -605,7 +605,6 @@ def _make_scaled_hdu(self, field: ImageSourceField) -> fits.ImageHDU: """Deal with BUNIT and pixel area.""" # Note: Do not scale source data - make a copy first. field_hdu = field.field.copy() # .field is the HDU (yeah...) - logger.debug("scaling by %f", field.pixel_area.value) # TODO: Check if this scaling is actually correct. How does this # work with the add_imagehdu_to_imagehdu below? Isn't that @@ -617,6 +616,7 @@ def _make_scaled_hdu(self, field: ImageSourceField) -> fits.ImageHDU: # field_hdu.data *= self.pixel_area.value else: logger.debug("binned bunit...") + logger.debug("scaling by %f", field.pixel_area.value) field_hdu.data /= field.pixel_area.value # Pixel area doesn't cancel out, need to convert new_bunit = field.bunit / u.arcsec**2 @@ -1145,7 +1145,7 @@ def make_hdu(self): field_hdu, canvas_cube_hdu, spline_order=self.spline_order, - conserve_flux=False, + conserve_flux=True, differential=True, ) diff --git a/scopesim/optics/image_plane_utils.py b/scopesim/optics/image_plane_utils.py index 307b0e0fd..439358fbd 100644 --- a/scopesim/optics/image_plane_utils.py +++ b/scopesim/optics/image_plane_utils.py @@ -541,6 +541,8 @@ def rescale_imagehdu(imagehdu: fits.ImageHDU, pixel_scale: float | u.Quantity, return imagehdu sum_orig = np.sum(imagehdu.data) + + # scale by pixel area if the flux is differential (i.e. per unit area) if differential: sum_orig *= primary_wcs.wcs.cdelt[0]*primary_wcs.wcs.cdelt[1] @@ -600,7 +602,7 @@ def rescale_imagehdu(imagehdu: fits.ImageHDU, pixel_scale: float | u.Quantity, if sum_new != 0: flux_factor = sum_orig / sum_new - logger.info("flux factor = %f", flux_factor) + logger.debug("flux factor = %f", flux_factor) new_im *= flux_factor elif sum_orig != 0: logger.warning( From fd9d080eb491dc71d8840c279ebb75af472f547c Mon Sep 17 00:00:00 2001 From: teutoburg Date: Wed, 9 Sep 2026 19:37:18 +0200 Subject: [PATCH 038/130] Formatting --- scopesim/effects/electronic/dmps.py | 15 ++++++++------ scopesim/effects/electronic/exposure.py | 14 +++++++------ scopesim/optics/optical_train.py | 27 ++++++++++++++++--------- 3 files changed, 35 insertions(+), 21 deletions(-) diff --git a/scopesim/effects/electronic/dmps.py b/scopesim/effects/electronic/dmps.py index 1f1bcf3fd..296703a88 100644 --- a/scopesim/effects/electronic/dmps.py +++ b/scopesim/effects/electronic/dmps.py @@ -73,9 +73,11 @@ def __init__(self, **kwargs): def apply_to(self, obj, **kwargs): logger.debug("Entering DetectorModePropertiesSetter.apply_to") - mode_name = kwargs.get("detector_readout_mode", - from_currsys("!OBS.detector_readout_mode", - self.cmds)) + mode_name = kwargs.get( + "detector_readout_mode", + from_currsys("!OBS.detector_readout_mode", self.cmds), + ) + if isinstance(obj, ImagePlane) and mode_name == "auto": mode_name = self.select_mode(obj, **kwargs) logger.info("Detector mode set to %s", mode_name) @@ -101,9 +103,10 @@ def select_mode(self, obj, **kwargs): saturation level (peak to full_well). """ immax = np.max(obj.data) - fillfrac = kwargs.get("fill_frac", - from_currsys("!OBS.auto_exposure.fill_frac", - self.cmds)) + fillfrac = kwargs.get( + "fill_frac", + from_currsys("!OBS.auto_exposure.fill_frac", self.cmds), + ) goodmodes = [] goodron = [] diff --git a/scopesim/effects/electronic/exposure.py b/scopesim/effects/electronic/exposure.py index f131387dd..0b20b698e 100644 --- a/scopesim/effects/electronic/exposure.py +++ b/scopesim/effects/electronic/exposure.py @@ -70,10 +70,10 @@ def _log_dit_ndit(dit: float, ndit: int) -> None: logger.info("Total exposure time: %.3f s", dit * ndit) def estimate_dit_ndit( - self, - exptime: float, - image_plane_max: float, - **kwargs + self, + exptime: float, + image_plane_max: float, + **kwargs ) -> tuple[float, int]: """ Automatically determine DIT and NDIT from exposure time. @@ -136,8 +136,10 @@ def apply_to(self, obj, **kwargs): # in both cases, `obj` is an ImagePlane. return obj - exptime = kwargs.pop("exptime", - from_currsys("!OBS.exptime", self.cmds)) + exptime = kwargs.pop( + "exptime", + from_currsys("!OBS.exptime", self.cmds), + ) mindit = from_currsys(self.meta["mindit"], self.cmds) # TODO: Remove this silly try-except once currsys works properly... diff --git a/scopesim/optics/optical_train.py b/scopesim/optics/optical_train.py index 45cfe947e..996aa9ecf 100644 --- a/scopesim/optics/optical_train.py +++ b/scopesim/optics/optical_train.py @@ -184,12 +184,19 @@ def update(self, **kwargs): self.optics_manager.update(**kwargs) opt_man = self.optics_manager - self.fov_manager = FOVManager(opt_man.fov_setup_effects, cmds=self.cmds, - **kwargs) - self.image_planes = [ImagePlane(hdr, self.cmds, **kwargs) - for hdr in opt_man.image_plane_headers] - self.detector_managers = [DetectorManager(det_list, cmds=self.cmds, **kwargs) - for det_list in opt_man.detector_setup_effects] + self.fov_manager = FOVManager( + opt_man.fov_setup_effects, + cmds=self.cmds, + **kwargs + ) + self.image_planes = [ + ImagePlane(hdr, self.cmds, **kwargs) + for hdr in opt_man.image_plane_headers + ] + self.detector_managers = [ + DetectorManager(det_list, cmds=self.cmds, **kwargs) + for det_list in opt_man.detector_setup_effects + ] # Move everything from CurrObs to CurrSys, so CurrObs is clean for # .observe and .readout. This is necessary because the setup and @@ -488,9 +495,11 @@ def readout(self, filename=None, reset=True, **kwargs): try: hdul = self.write_header(hdul) except Exception: - logger.exception("Header update failed, data will be " - "saved with incomplete header. See stack " - "trace for details.") + logger.exception( + "Header update failed, data will be saved with " + "incomplete header. See stack trace for details." + ) + if filename is not None and isinstance(filename, str): fname = filename From d81fe40d22a33a2e6302bde000a6343e6fb93a9d Mon Sep 17 00:00:00 2001 From: teutoburg Date: Thu, 10 Sep 2026 15:20:09 +0200 Subject: [PATCH 039/130] Formatting --- scopesim/optics/fov.py | 49 ++++++----------- scopesim/optics/image_plane_utils.py | 82 ++++++++++++++++++---------- 2 files changed, 70 insertions(+), 61 deletions(-) diff --git a/scopesim/optics/fov.py b/scopesim/optics/fov.py index 1c314c007..514ef7280 100644 --- a/scopesim/optics/fov.py +++ b/scopesim/optics/fov.py @@ -2,7 +2,6 @@ """Defines FieldOfView class.""" from warnings import warn -from copy import deepcopy from itertools import chain from collections.abc import Iterable, Generator @@ -32,7 +31,6 @@ get_logger, array_minmax, close_loop, - unit_includes_per_physical_type, figure_factory, image_plotter, cube_plotter, @@ -603,14 +601,8 @@ def _make_scaled_hdu(self, field: ImageSourceField) -> fits.ImageHDU: # Note: Do not scale source data - make a copy first. field_hdu = field.field.copy() # .field is the HDU (yeah...) - # TODO: Check if this scaling is actually correct. How does this - # work with the add_imagehdu_to_imagehdu below? Isn't that - # supposed to conserve flux? Test carefully!! if field.is_bunit_spatially_differential: logger.debug("differential bunit...") - # Field is in (PHOTLAM) arcsec-2, need to scale by pixarea - # logger.debug("scaling by %f", self.pixel_area.value) - # field_hdu.data *= self.pixel_area.value else: logger.debug("binned bunit...") logger.debug("scaling by %f", field.pixel_area.value) @@ -864,20 +856,16 @@ def make_hdu(self): data=np.zeros((self.header["NAXIS2"], self.header["NAXIS1"])), header=self.header) - for tmp_hdu in chain(self._make_cubefields(), - self._make_imagefields( - fov_waveset, bin_widths)): + for tmp_hdu in chain( + self._make_cubefields(), + self._make_imagefields(fov_waveset, bin_widths) + ): canvas_image_hdu = imp_utils.add_imagehdu_to_imagehdu( tmp_hdu, canvas_image_hdu, conserve_flux=True, # binned flux needs this - spline_order=self.spline_order) - logger.info( - "2D FOV make_hdu: canvas_image_hdu.data.mean() = %f ph/s/pix", - canvas_image_hdu.data.mean()) - logger.info( - "2D FOV make_hdu: canvas_image_hdu.data.max() = %f ph/s/pix", - canvas_image_hdu.data.max()) + spline_order=self.spline_order, + ) for flux, weight, x, y in self._make_tablefields( fov_waveset, bin_widths): @@ -947,16 +935,16 @@ def _make_cubefields(self, fov_waveset): # Assumption is that ImageHDUs have units of PHOTLAM arcsec-2 # TODO: Deal with this bounds_error in a more elegant way - field_interp = interp1d(field.waveset.to(u.um).value, - field.data, axis=0, kind="linear", - bounds_error=False, fill_value=0) + field_interp = interp1d( + field.waveset.to(u.um).value, + field.data, + axis=0, + kind="linear", + bounds_error=False, + fill_value=0, + ) field_data = field_interp(fov_waveset.value) - - # Pixel scale conversion - # field_data *= field.pixel_area / self.pixel_area - logger.debug( - "3D FOV make_cubefields: field_data.mean() = %f [%s]", field_data.mean(), field.bunit) field_hdu = fits.ImageHDU(data=field_data, header=field.header) yield field_hdu @@ -989,8 +977,6 @@ def _make_imagefields(self, fov_waveset, spline_order): spec = field.spectrum(fov_waveset) # 2D * 1D -> 3D field_cube = canvas_image_hdu.data[None, :, :] * spec[:, None, None] - logger.debug("3D FOV make_imagefields: field_cube.mean() = %f", - field_cube.mean().value) yield field_cube.value def _make_tablefields(self, fov_waveset): @@ -1146,9 +1132,10 @@ def make_hdu(self): differential=True, ) - canvas_cube_hdu.data = sum(self._make_imagefields( - fov_waveset, self.spline_order), - start=canvas_cube_hdu.data) + canvas_cube_hdu.data = sum( + self._make_imagefields(fov_waveset, self.spline_order), + start=canvas_cube_hdu.data, + ) for flux, x, y in self._make_tablefields(fov_waveset): # To prevent adding array values in this manner. diff --git a/scopesim/optics/image_plane_utils.py b/scopesim/optics/image_plane_utils.py index 2372b8191..7304d41d7 100644 --- a/scopesim/optics/image_plane_utils.py +++ b/scopesim/optics/image_plane_utils.py @@ -536,10 +536,14 @@ def overlay_image(small_im, big_im, coords, mask=None, sub_pixel=False): return big_im -def rescale_imagehdu(imagehdu: fits.ImageHDU, pixel_scale: float | u.Quantity, - wcs_suffix: str = "", conserve_flux: bool = True, - spline_order: int = 1, - differential: bool = False) -> fits.ImageHDU: +def rescale_imagehdu( + imagehdu: fits.ImageHDU, + pixel_scale: float | u.Quantity, + wcs_suffix: str = "", + conserve_flux: bool = True, + spline_order: int = 1, + differential: bool = False, +) -> fits.ImageHDU: """ Scale the .data array by the ratio of pixel_scale [deg] and CDELTn. @@ -597,7 +601,7 @@ def rescale_imagehdu(imagehdu: fits.ImageHDU, pixel_scale: float | u.Quantity, # scale by pixel area if the flux is differential (i.e. per unit area) if differential: - sum_orig *= primary_wcs.wcs.cdelt[0]*primary_wcs.wcs.cdelt[1] + sum_orig *= primary_wcs.wcs.cdelt[0] * primary_wcs.wcs.cdelt[1] # Perform the rescaling. Axes need to be inverted because python. zoom_np = zoom[::-1] @@ -709,9 +713,12 @@ def rescale_imagehdu(imagehdu: fits.ImageHDU, pixel_scale: float | u.Quantity, return imagehdu -def reorient_imagehdu(imagehdu: fits.ImageHDU, wcs_suffix: str = "", - conserve_flux: bool = True, - spline_order: int = 1) -> fits.ImageHDU: +def reorient_imagehdu( + imagehdu: fits.ImageHDU, + wcs_suffix: str = "", + conserve_flux: bool = True, + spline_order: int = 1, +) -> fits.ImageHDU: """ Apply an affine transformation to the image, as given in its header. @@ -769,9 +776,15 @@ def reorient_imagehdu(imagehdu: fits.ImageHDU, wcs_suffix: str = "", return imagehdu -def affine_map(input, matrix=None, rotation_angle: float = 0., - shear_angle: float = 0., scale_factor=None, - reshape: bool = True, spline_order: int = 3): +def affine_map( + input, + matrix=None, + rotation_angle: float = 0., + shear_angle: float = 0., + scale_factor=None, + reshape: bool = True, + spline_order: int = 3, +): """ Apply an affine transformation matrix to an image around its centre. @@ -842,12 +855,14 @@ def affine_map(input, matrix=None, rotation_angle: float = 0., return output -def add_imagehdu_to_imagehdu(image_hdu: fits.ImageHDU, - canvas_hdu: fits.ImageHDU, - spline_order: int = 1, - wcs_suffix: str = "", - conserve_flux: bool = True, - differential: bool = False) -> fits.ImageHDU: +def add_imagehdu_to_imagehdu( + image_hdu: fits.ImageHDU, + canvas_hdu: fits.ImageHDU, + spline_order: int = 1, + wcs_suffix: str = "", + conserve_flux: bool = True, + differential: bool = False, +) -> fits.ImageHDU: """ Re-project one ``fits.ImageHDU`` onto another ``fits.ImageHDU``. @@ -897,17 +912,22 @@ def add_imagehdu_to_imagehdu(image_hdu: fits.ImageHDU, canvas_pixel_scale = float(canvas_wcs.wcs.cdelt[0]) conv_fac = u.Unit(image_hdu.header[f"CUNIT1{wcs_suffix}"].lower()).to(canvas_wcs.wcs.cunit[0]) - new_hdu = rescale_imagehdu(image_hdu, pixel_scale=canvas_pixel_scale / conv_fac, - wcs_suffix=canvas_wcs.wcs.alt, - spline_order=spline_order, - conserve_flux=conserve_flux, - differential=differential) + new_hdu = rescale_imagehdu( + image_hdu, + pixel_scale=canvas_pixel_scale / conv_fac, + wcs_suffix=canvas_wcs.wcs.alt, + spline_order=spline_order, + conserve_flux=conserve_flux, + differential=differential, + ) # TODO: Perhaps add separately formatted WCS logger? logger.debug("fromrescale %s", WCS(new_hdu.header, key=canvas_wcs.wcs.alt)) - new_hdu = reorient_imagehdu(new_hdu, - wcs_suffix=canvas_wcs.wcs.alt, - spline_order=spline_order, - conserve_flux=conserve_flux) + new_hdu = reorient_imagehdu( + new_hdu, + wcs_suffix=canvas_wcs.wcs.alt, + spline_order=spline_order, + conserve_flux=conserve_flux, + ) img_center = np.array([[new_hdu.header[f"NAXIS{i+1}"] for i in range(new_hdu.header["NAXIS"])]]) @@ -1250,10 +1270,12 @@ def det_wcs_from_sky_wcs( return create_wcs_from_points(corners, pixel_size, "D") -def sky_wcs_from_det_wcs(det_wcs: WCS, - pixel_scale: float, - plate_scale: float, - naxis=None) -> tuple[WCS, np.ndarray]: +def sky_wcs_from_det_wcs( + det_wcs: WCS, + pixel_scale: float, + plate_scale: float, + naxis=None, +) -> tuple[WCS, np.ndarray]: """ Create celestial WCS from detector WCS using pixel and plate scales. From 16f2b019a9c70a2045b95fbcc36e662ff5180835 Mon Sep 17 00:00:00 2001 From: teutoburg Date: Wed, 9 Sep 2026 19:56:04 +0200 Subject: [PATCH 040/130] Allow readout ID, fix Path support --- scopesim/optics/optical_train.py | 48 +++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/scopesim/optics/optical_train.py b/scopesim/optics/optical_train.py index 996aa9ecf..19245c9e2 100644 --- a/scopesim/optics/optical_train.py +++ b/scopesim/optics/optical_train.py @@ -3,11 +3,13 @@ import copy from datetime import datetime +from pathlib import Path import numpy as np from scipy.interpolate import interp1d from astropy import units as u from astropy.wcs import WCS +from astropy.io.fits import HDUList from tqdm.auto import tqdm @@ -426,23 +428,39 @@ def prepare_source(self, source): return source @top_level_catch - def readout(self, filename=None, reset=True, **kwargs): + def readout( + self, + filename: Path | str | None = None, + reset: bool = True, + roid: int = 0, + **kwargs + ) -> list[HDUList]: """ Produce detector readouts for the observed image. Parameters ---------- - filename : str, optional - Where to save the FITS file - kwargs + filename : Path | str | None, optional + Full path or local name for to store the resulting FITS file. + If None (the default), the function still returns the result, but + will not automatically save it to disk. + reset : bool, optional + If true (the default), do not keep any parameters set during any + previous readout. This is usually what you want. + roid : int, optional + Readout ID, used for seeding randomness. If you run multiple + readouts on the same observation and want the random effects to + produce different results, set this parameter with e.g. the loop + counter. Simple integers are fine (0, 1, 2, ...), because this gets + wrangled into a proper random seed downstream. If you need to + reproduce a specific readout in a sequence, set e.g. `roid=4`. + **kwargs : + Any other kwargs passed to the detector effects. Returns ------- - hdu : fits.HDUList - - Notes - ----- - - Apply detector plane (0D, 2D) effects - z_order = 500..599 + list[HDUList] + List of HDUList objects. """ if reset: @@ -500,11 +518,15 @@ def readout(self, filename=None, reset=True, **kwargs): "incomplete header. See stack trace for details." ) + if filename is not None: + if len(self.detector_managers) == 1: + fname = filename + else: + if isinstance(filename, str): + fname = f"{i}_{filename}" + if isinstance(filename, Path): + fname = filename.parent / f"{i}_{filename.name}" - if filename is not None and isinstance(filename, str): - fname = filename - if len(self.detector_managers) > 1: - fname = f"{i}_{filename}" hdul.writeto(fname, overwrite=True) hduls.append(hdul) From 120e7dc7d8e601ce013ddb10c474a14e9abafaee Mon Sep 17 00:00:00 2001 From: teutoburg Date: Thu, 10 Sep 2026 14:38:31 +0200 Subject: [PATCH 041/130] Resolve random seed in `.observe()` Concretize seed from system entropy to keep constant during the observe run. For multiple observations, this is cleared above. --- scopesim/optics/optical_train.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/scopesim/optics/optical_train.py b/scopesim/optics/optical_train.py index 996aa9ecf..385c20e5e 100644 --- a/scopesim/optics/optical_train.py +++ b/scopesim/optics/optical_train.py @@ -245,6 +245,17 @@ def observe(self, orig_source=None, update=True, **kwargs): if update: self.update(**kwargs) + if self.cmds["!SIM.random.seed"] is None: + # Concretize seed from system entropy to keep constant during the + # observe run. For multiple observations, this is cleared above. + self.cmds["!SIM.random.seed"] = np.random.SeedSequence().entropy + elif self.cmds["!SIM.random.seed"] in {"None", "none", "NONE"}: + logger.warning( + "Got random seed '%s' (a string, not None!), use NULL in yaml", + self.cmds["!SIM.random.seed"] + ) + self.cmds["!SIM.random.seed"] = np.random.SeedSequence().entropy + # self.set_focus(**kwargs) # put focus back on current instrument package # Make a copy of the Source and prepare for observation (convert to From 9e7985536a863c6c7fbc9c26b3f7880657daa6d9 Mon Sep 17 00:00:00 2001 From: teutoburg Date: Thu, 10 Sep 2026 16:39:33 +0200 Subject: [PATCH 042/130] Make sure cmds is cleared in between observations --- scopesim/optics/optical_train.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scopesim/optics/optical_train.py b/scopesim/optics/optical_train.py index 385c20e5e..8393a2e91 100644 --- a/scopesim/optics/optical_train.py +++ b/scopesim/optics/optical_train.py @@ -169,9 +169,9 @@ def load(self, user_commands): self.yaml_dicts = self.cmds.yaml_dicts self.optics_manager = OpticsManager(self.yaml_dicts, self.cmds) - self.update() + self.update(migrate_cmds=True) - def update(self, **kwargs): + def update(self, migrate_cmds=False, **kwargs): """ Update the user-defined parameters and remake main internal classes. @@ -209,7 +209,8 @@ def update(self, **kwargs): # self.cmds.maps[0].clear() # HACK: recursive_update is needed to avoid overwriting emtpy !ABCs # TODO: or is it?? - self.cmds.maps[1].dic = recursive_update(self.cmds.maps[1].dic, self.cmds.maps[0].dic) + if migrate_cmds: + self.cmds.maps[1].dic = recursive_update(self.cmds.maps[1].dic, self.cmds.maps[0].dic) self.cmds.maps[0].dic.clear() @top_level_catch From 30b8d36401300bd3479e40c1a6c2260d5d9bde6a Mon Sep 17 00:00:00 2001 From: oczoske Date: Thu, 10 Sep 2026 22:33:53 +0200 Subject: [PATCH 043/130] Set INHERIT keyword and fix unit to FITS standard --- scopesim/effects/mosaic_trace_list.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scopesim/effects/mosaic_trace_list.py b/scopesim/effects/mosaic_trace_list.py index 542755ebf..6f3083304 100644 --- a/scopesim/effects/mosaic_trace_list.py +++ b/scopesim/effects/mosaic_trace_list.py @@ -271,8 +271,9 @@ def apply_to(self, det, **kwargs): det._hdu = fits.BinTableHDU.from_columns([ fits.Column(name="wavelength", format="D", array=lam, unit="um"), - fits.Column(name="spectrum", format="D", array=spec, unit="ADU"), + fits.Column(name="spectrum", format="D", array=spec, unit="adu"), ]) + det._hdu.header["INHERIT"] = True elif output_format == "table": ntrace = len(self.spectral_traces) @@ -298,8 +299,9 @@ def apply_to(self, det, **kwargs): tab = Table(data=[idarr, xarr, yarr, lamarr, specarr], names=["id", "x", "y", "wavelength", "spectrum"], - units=["", "arcsec", "arcsec", "um", "ADU"]) + units=["", "arcsec", "arcsec", "um", "adu"]) det._hdu = fits.BinTableHDU(data=tab) + det._hdu.header["INHERIT"] = True elif output_format == "image": pass From c29df4e454e0a4eadd435449d944e8bc632e0bc8 Mon Sep 17 00:00:00 2001 From: oczoske Date: Fri, 11 Sep 2026 18:15:42 +0200 Subject: [PATCH 044/130] Fix indexing in FoV extraction --- scopesim/optics/fov.py | 15 +++++---- .../tests/tests_optics/test_FieldOfView.py | 31 +++++-------------- 2 files changed, 17 insertions(+), 29 deletions(-) diff --git a/scopesim/optics/fov.py b/scopesim/optics/fov.py index 514ef7280..960baa107 100644 --- a/scopesim/optics/fov.py +++ b/scopesim/optics/fov.py @@ -330,12 +330,12 @@ def extract_area_from_imagehdu(self, imagehdu, corners): xyp.sort(axis=0) logger.debug("xyp:\n%s", xyp) - xy0p = np.max(((0, 0), np.floor(xyp[0]).astype(int)), axis=0) - xy1p = np.min(((naxis1, naxis2), np.ceil(xyp[1]).astype(int)), axis=0) - logger.debug("xy0p: %s; xy1p: %s", xy0p, xy1p) - - # Add 1 if the same - xy1p += (xy0p == xy1p) + xy0p = np.max(((0, 0), np.floor(xyp[0]).astype(np.intp)), axis=0) + # xyp are inclusive bounds; python (in)famously excludes upper bound, + # hence xy1p needs to be increased by one + xy1p = np.min(((naxis1, naxis2), + 1 + np.ceil(xyp[1]).astype(np.intp)), + axis=0) logger.debug("xy0p: %s; xy1p: %s", xy0p, xy1p) # Describe the cutout with the input WCS, shifted by the slice origin. @@ -356,6 +356,9 @@ def extract_area_from_imagehdu(self, imagehdu, corners): new_wcs = image_wcs.deepcopy() new_wcs.wcs.crpix = image_wcs.wcs.crpix - xy0p new_naxis = xy1p - xy0p + logger.debug("orig image wcs: %s", image_wcs) + logger.debug("new cutout wcs: %s", new_wcs) + logger.debug("new cutout naxis: %s", new_naxis) new_hdr = new_wcs.to_header() new_hdr.update({"NAXIS1": new_naxis[0], "NAXIS2": new_naxis[1]}) diff --git a/scopesim/tests/tests_optics/test_FieldOfView.py b/scopesim/tests/tests_optics/test_FieldOfView.py index 49577b070..de961eb0f 100644 --- a/scopesim/tests/tests_optics/test_FieldOfView.py +++ b/scopesim/tests/tests_optics/test_FieldOfView.py @@ -1,3 +1,6 @@ +# -*- coding: utf-8 -*- +"""Test for FieldOfView classes""" + import pytest from pytest import approx import numpy as np @@ -5,7 +8,6 @@ from astropy.io import fits from astropy.table import Table from matplotlib import pyplot as plt -from matplotlib.colors import LogNorm from scopesim.tests.mocks.py_objects import header_objects as ho from scopesim.tests.mocks.py_objects import source_objects as so @@ -69,9 +71,6 @@ def test_initialises_with_header_and_waverange(self): class TestExtractFrom: - # @pytest.mark.xfail(reason=("is_field_in_fov drops table if anything is " - # "outside fov volume, therefore no point source " - # "is extracted...")) def test_extract_point_sources_from_table(self): src = so._table_source() src.fields[0].field["x"] = [-15, -5, 0, 0] * u.arcsec @@ -88,7 +87,7 @@ def test_extract_2d_image_from_hduimage(self): fov = _fov_190_210_um() fov.extract_from(src) - assert fov.fields[0].data.shape == (51, 25) + assert fov.fields[0].data.shape == (51, 26) assert len(fov.fields[0].spectra[0].waveset) == 11 assert fov.fields[0].spectra[0].waveset[0].value == approx(19000) @@ -107,11 +106,8 @@ def test_extract_3d_cube_that_is_offset_relative_to_fov(self): fov = _fov_197_202_um() fov.extract_from(src) - assert fov.fields[0].field.shape == (3, 51, 25) + assert fov.fields[0].field.shape == (3, 51, 26) - # @pytest.mark.xfail(reason=("is_field_in_fov drops table if anything is " - # "outside fov volume, therefore no point source " - # "is extracted...")) def test_extract_one_of_each_type_from_source_object(self): src_table = so._table_source() # 4 sources, put two outside of FOV src_table.fields[0].field["x"] = [-15, -5, 0, 0] * u.arcsec @@ -124,7 +120,7 @@ def test_extract_one_of_each_type_from_source_object(self): fov.extract_from(src) assert fov.fields[0].field.shape == (3, 51, 51) - assert fov.fields[1].field.shape == (51, 25) + assert fov.fields[1].field.shape == (51, 26) assert len(fov.fields[2].field) == 2 # assert len(fov.spectra) == 3 @@ -145,17 +141,6 @@ def test_ignores_fields_outside_fov_boundary(self): assert len(fov.fields) == 0 - @pytest.mark.skip(reason="SPEC_REF is obsolete, just rm this test?") - def test_all_spectra_are_referenced_correctly(self): - src = so._image_source() + so._cube_source() + so._table_source() - fov = _fov_190_210_um() - fov.extract_from(src) - # check the same spectrum object is referenced by both lists - assert fov.fields[0].header["SPEC_REF"] == \ - src.fields[0].header["SPEC_REF"] - assert all(fov.fields[2][i]["ref"] == src.fields[2][i]["ref"] - for i in range(4)) - def test_contains_all_fields_inside_fov(self): src = so._image_source() + so._cube_source() + so._table_source() the_fov = FieldOfView(ho._basic_fov_header(), (1, 2) * u.um, @@ -294,7 +279,7 @@ def test_makes_cube_from_all_types_of_source_object(self): if PLOTS: im = cube.data[0, :, :] - plt.imshow(im, origin="lower", norm=LogNorm(), vmin=1e-8) + plt.imshow(im, origin="lower", norm="log", vmin=1e-8) plt.show() @pytest.mark.parametrize("src", [so._table_source(), @@ -434,7 +419,7 @@ def test_makes_image_from_all_types_of_source_object(self): if PLOTS: im = image.data - plt.imshow(im, origin="lower", norm=LogNorm(), vmin=1e-8) + plt.imshow(im, origin="lower", norm="log", vmin=1e-8) plt.show() From a64d9b0f884e90e2c54801c955ace57c1653ce09 Mon Sep 17 00:00:00 2001 From: jbrink Date: Fri, 11 Sep 2026 10:06:39 +0200 Subject: [PATCH 045/130] Added tests to catch the through-slit imaging issue --- .../test_source_fov_imageplane_play_nicely.py | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/scopesim/tests/tests_integrations/test_source_fov_imageplane_play_nicely.py b/scopesim/tests/tests_integrations/test_source_fov_imageplane_play_nicely.py index 8f7bd3899..3816a62a1 100644 --- a/scopesim/tests/tests_integrations/test_source_fov_imageplane_play_nicely.py +++ b/scopesim/tests/tests_integrations/test_source_fov_imageplane_play_nicely.py @@ -163,3 +163,110 @@ def test_checkerboard_survives_chunking(self, half_mm, exact_extent): # no gaps and no double-counting at the chunk seams got = imp.data / imp.data.max() np.testing.assert_allclose(got, board, atol=1e-6) + + +def _flat_source(naxis=64, pixel_scale=0.064): + """Uniformly illuminated image source, one everywhere, flat spectrum.""" + from astropy.io import fits + from synphot import Empirical1D, SourceSpectrum + from scopesim.source.source import Source + + wcs = WCS(naxis=2) + wcs.wcs.ctype = ["LINEAR", "LINEAR"] + wcs.wcs.cunit = ["arcsec", "arcsec"] + wcs.wcs.cdelt = 2 * [pixel_scale] + wcs.wcs.crval = [0, 0] + wcs.wcs.crpix = 2 * [(naxis + 1) / 2] + + hdu = fits.ImageHDU(data=np.ones((naxis, naxis)), header=wcs.to_header()) + spec = SourceSpectrum(Empirical1D, points=[0.5, 2.5] * u.um, + lookup_table=[1, 1] * u.Unit("ph s-1 m-2 um-1")) + return Source(image_hdu=hdu, spectra=[spec]) + + +def _slit_fov(x, y, pixel_scale, pixel_size): + """FOV covering a slit of x, y [arcsec], built like FOVManager does.""" + skyhdr = imp_utils.header_from_list_of_xy( + [v / 3600 for v in x], [v / 3600 for v in y], + pixel_scale=pixel_scale / 3600) + dethdr, _ = imp_utils.det_wcs_from_sky_wcs( + WCS(skyhdr), pixel_scale, pixel_scale / pixel_size) + skyhdr.update(dethdr.to_header()) + return FieldOfView2D(skyhdr, waverange=[1.0, 2.0] * u.um, area=1 * u.m**2) + + +class TestFlatSourceThroughASlitFOV: + """A flat lamp seen through a slit must fill the slit, uniformly. + + A slit FOV is typically *thinner than one source pixel*: MICADO's Short + slit is 0.016" tall while a flat-lamp source is sampled at 0.064"/pix. The + cutout is grown outwards to whole source pixels, so it covers more sky than + the slit does; if its WCS is then written on the *source's* pixel lattice + rather than on the FOV's, ``overlay_image`` crops the surplus away and the + flux through the slit comes out wrong -- for the MICADO case, exactly half + of the slit was left dark. This is what commit 9740ebd6 reverted in + ``FieldOfView.extract_area_from_imagehdu``. + + Note that these tests and ``TestChunkedFOVsTileTheImagePlane`` above are the + two sides of the same trade-off: the cutout WCS can describe either the + source's pixel lattice or the FOV's, and ``extract_area_from_imagehdu`` + currently has to pick one. Resolving it properly needs sub-pixel-accurate + cropping, i.e. weighting the partial edge pixels of the cutout. + """ + + SRC_PIXEL_SCALE = 0.064 # arcsec / pix, the flat lamp + FOV_PIXEL_SCALE = 0.004 # arcsec / pix, MICADO SPEC + PIXEL_SIZE = 0.015 # mm / pix + + # slit extents [arcsec], all thinner than or comparable to a source pixel + SLITS = [ + ((-1.5, 1.5), (-0.008, 0.008)), # MICADO Short slit, centred + ((-1.5, 1.5), (0.010, 0.026)), # ... offset within one source pixel + ((-1.5, 1.5), (-0.032, 0.032)), # exactly one source pixel tall + ((-1.5, 1.5), (-0.08, 0.08)), # 2.5 source pixels tall + ((-1.5, 1.5), (0.020, 0.060)), # straddling a source pixel edge + ((-2.0, 2.0), (-0.008, 0.008)), # longer slit + ] + + def _observe(self, x, y): + fov = _slit_fov(x, y, self.FOV_PIXEL_SCALE, self.PIXEL_SIZE) + fov.extract_from(_flat_source(pixel_scale=self.SRC_PIXEL_SCALE)) + fov.view() + return fov + + @pytest.mark.parametrize(("x", "y"), SLITS) + def test_flux_through_slit_matches_slit_area(self, x, y): + fov = self._observe(x, y) + + # each source pixel carries 1 ph/s here (flat 1 ph/s/m2/um over the + # 1 um wide FOV waverange, collecting area 1 m2), so the flux through + # the slit is just the slit area in source pixels + slit_area = (x[1] - x[0]) * (y[1] - y[0]) + expected = slit_area / self.SRC_PIXEL_SCALE**2 + + assert fov.hdu.data.sum() == approx(expected, rel=1e-6) + + # ... and it survives the projection onto the image plane + x_mm = [v * self.PIXEL_SIZE / self.FOV_PIXEL_SCALE for v in x] + y_mm = [v * self.PIXEL_SIZE / self.FOV_PIXEL_SCALE for v in y] + imp = ImagePlane(imp_utils.header_from_list_of_xy( + x_mm, y_mm, pixel_scale=self.PIXEL_SIZE, wcs_suffix="D")) + imp.add(fov.hdu, wcs_suffix="D") + + assert imp.data.sum() == approx(expected, rel=1e-6) + + @pytest.mark.parametrize(("x", "y"), SLITS) + def test_illumination_is_uniform_across_the_slit(self, x, y): + fov = self._observe(x, y) + data = fov.hdu.data + + if PLOTS: + plt.plot(data.mean(axis=1)) + plt.xlabel("pixel across slit") + plt.ylabel("flux [ph s-1 pix-1]") + plt.show() + + # a flat lamp illuminates the whole slit evenly: no dark rows at the + # edges, no step in the profile across the slit + assert data.min() > 0 + np.testing.assert_allclose(data, data.max(), rtol=1e-6) From 9c7b0dd2d93791bcdfe6a86c4c384d81cfd47cac Mon Sep 17 00:00:00 2001 From: "Fabian H." <73600109+teutoburg@users.noreply.github.com> Date: Sat, 12 Sep 2026 12:50:23 +0200 Subject: [PATCH 046/130] Skip all tests in test_rst_utils.py This functionality hasn't been used in years, and the tests sometimes cause re-runs because of some Tcl/Tkinter weirdness. We can always re-enable the tests if we use it again... --- scopesim/tests/tests_reports/test_rst_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scopesim/tests/tests_reports/test_rst_utils.py b/scopesim/tests/tests_reports/test_rst_utils.py index aca00adef..fee66a7ef 100644 --- a/scopesim/tests/tests_reports/test_rst_utils.py +++ b/scopesim/tests/tests_reports/test_rst_utils.py @@ -10,6 +10,7 @@ from scopesim.tests.mocks.py_objects import report_objects as ro from scopesim.tests.mocks.py_objects import effects_objects as eo +pytestmark = pytest.mark.skip # whole file CLEAN_UP = True PLOTS = False From 719360d7baac9c146a5bf7c8051b31497f9567db Mon Sep 17 00:00:00 2001 From: "Fabian H." <73600109+teutoburg@users.noreply.github.com> Date: Sat, 12 Sep 2026 13:27:45 +0200 Subject: [PATCH 047/130] Set mpl backend for tests Attempt to reduce the annoying Tcl/Tkinter fails seen on Windows. --- scopesim/tests/conftest.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scopesim/tests/conftest.py b/scopesim/tests/conftest.py index 0be2a072f..705fd5aa3 100644 --- a/scopesim/tests/conftest.py +++ b/scopesim/tests/conftest.py @@ -6,6 +6,7 @@ import pytest from unittest.mock import patch +import matplotlib import scopesim as sim from astar_utils import UniqueList @@ -15,6 +16,8 @@ sim.rc.__currsys__["!SIM.file.error_on_missing_file"] = True +matplotlib.use("agg") # Don't render plots in normal testing + @pytest.fixture(scope="function", autouse=True) def configure_logging(): From 5752b8f885d3083d73e6ed0cdd2c473262aad333 Mon Sep 17 00:00:00 2001 From: Kieran Leschinski Date: Fri, 4 Sep 2026 22:07:33 +0200 Subject: [PATCH 048/130] Remove unused `fov_grid()` methods Some were never called, others even raised an error. In `TERCurve`, the check was ported to the `apply_to()`. Originally clauded by @astronomyk, now rebased and reworded by @teutoburg. --- pyproject.toml | 5 +++- scopesim/effects/apertures.py | 19 ------------ scopesim/effects/detector_list.py | 25 ---------------- scopesim/effects/effects.py | 24 ++++++++------- scopesim/effects/shifts.py | 29 ++----------------- scopesim/effects/surface_list.py | 26 ----------------- scopesim/effects/ter_curves.py | 8 +++++ .../test_AtmosphericDispersion.py | 24 +++++++-------- .../test_AtmosphericDispersionCorrection.py | 8 +++-- scopesim/tests/tests_effects/test_ConstPSF.py | 1 - scopesim/tests/tests_effects/test_TERCurve.py | 15 ++++++++++ .../tests/tests_effects/test_slitwheel.py | 3 -- 12 files changed, 58 insertions(+), 129 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 75e53403b..4597b8c6e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -107,7 +107,10 @@ filterwarnings = [ "ignore:invalid value encountered in divide:RuntimeWarning", "ignore:Cannot merge meta key.*:astropy.utils.metadata.MergeConflictWarning", "ignore::astropy.wcs.wcs.FITSFixedWarning", - "default:The fov_grid*:DeprecationWarning", + # Only SpectralTraceList still calls the deprecated fov_grid internally. + # Delete this line once that call moves to an internal method; every other + # fov_grid DeprecationWarning is now an error. + "default:The fov_grid*:DeprecationWarning:scopesim.effects.spectral_trace_list", # Pytest Notebook stuff "ignore:Proactor*:RuntimeWarning", ] diff --git a/scopesim/effects/apertures.py b/scopesim/effects/apertures.py index fd6984a5b..47137a145 100644 --- a/scopesim/effects/apertures.py +++ b/scopesim/effects/apertures.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- """Effects related to field masks, including spectroscopic slits.""" -import warnings from typing import ClassVar import yaml @@ -131,18 +130,6 @@ def apply_to(self, obj, **kwargs): return obj - # Outdated. Remove when removing all old FOVManager code from effects - def fov_grid(self, which="edges", **kwargs): - """Return a header with the sky coordinates.""" - warnings.warn("The fov_grid method is deprecated and will be removed " - "in a future release.", DeprecationWarning, stacklevel=2) - if which == "edges": - self.meta.update(kwargs) - return self.header - elif which == "masks": - self.meta.update(kwargs) - return self.mask - @property def hdu(self): return fits.ImageHDU(data=self.mask, header=self.header) @@ -458,12 +445,6 @@ def apply_to(self, obj, **kwargs): """Use apply_to of current_slit.""" return self.current_slit.apply_to(obj, **kwargs) - def fov_grid(self, which="edges", **kwargs): - """See parent docstring.""" - warnings.warn("The fov_grid method is deprecated and will be removed " - "in a future release.", DeprecationWarning, stacklevel=2) - return self.current_slit.fov_grid(which=which, **kwargs) - def change_slit(self, slitname=None): """Change the current slit.""" if not slitname or slitname in self.slits.keys(): diff --git a/scopesim/effects/detector_list.py b/scopesim/effects/detector_list.py index f3a1a5f09..83979273e 100644 --- a/scopesim/effects/detector_list.py +++ b/scopesim/effects/detector_list.py @@ -242,31 +242,6 @@ def _get_corner_points(self): return points * u.mm - def fov_grid(self, which="edges", **kwargs): - """Return an ApertureMask object. kwargs are "pixel_scale" [arcsec].""" - # This has really been taken care of by apply_to now - # TODO v1.0: finally rm this completely - raise AttributeError("The DetectorList.fov_grid() method has been " - "removed in v0.10.0.") - # aperture_mask = None - # if which == "edges": - # self.meta.update(kwargs) - # self.meta = from_currsys(self.meta, self.cmds) - - # hdr = self.image_plane_header - # xy_mm = calc_footprint(hdr, "D") - # pixel_size = hdr["CDELT1D"] # mm - # pixel_scale = self.meta["pixel_scale"] # ["] - - # # x["] = x[mm] * ["] / [mm] - # xy_sky = xy_mm * pixel_scale / pixel_size - - # aperture_mask = ApertureMask(array_dict={"x": xy_sky[:, 0], - # "y": xy_sky[:, 1]}, - # pixel_scale=pixel_scale) - - # return aperture_mask - @property def image_plane_header(self): """Create and return the Image Plane Header.""" diff --git a/scopesim/effects/effects.py b/scopesim/effects/effects.py index 30b4bd38f..da47304af 100644 --- a/scopesim/effects/effects.py +++ b/scopesim/effects/effects.py @@ -11,20 +11,19 @@ from ..reports.rst_utils import table_to_rst -# FIXME: This docstring is out-of-date for several reasons: -# - Effects can act on objects other than Source (eg FOV, IMP, DET) -# - fov_grid is outdated - - # @dataclass(kw_only=True, eq=False) class Effect: """ Base class for representing the effects (artifacts) in an optical system. The ``Effect`` class is conceived to independently apply the changes that - an optical component (or series thereof) has on an incoming 3D description - of an on-sky object. In other words, **an Effect object should receive a - derivative of a ``Source`` object, alter it somehow, and return it**. + an optical component (or series thereof) has on an incoming description of + an on-sky object. **An Effect object should receive an object, alter it + somehow, and return an instance of the same class.** Besides ``Source``, + that object may be a ``FovVolumeList``, a ``FieldOfView``, an + ``ImagePlane`` or a ``Detector``; ``apply_to`` is expected to dispatch on + the type it is given and to pass anything it does not handle straight + through. The interface for the Effect base-class has been kept very general so that it can easily be sub-classed as data for new effects becomes available. @@ -32,9 +31,12 @@ class Effect: attributes: * ``self.meta`` - a dictionary to contain metadata. - * ``self.apply_to(obj, **kwargs)`` - a method which accepts a - Source-derivative and returns an instance of the same class as ``obj`` - * ``self.fov_grid(which="", **kwargs)`` + * ``self.apply_to(obj, **kwargs)`` - a method which accepts one of the + objects above and returns an instance of the same class as ``obj`` + + An Effect that needs to constrain the source-space volume the simulation + covers does so in the ``FovVolumeList`` branch of ``apply_to``, by calling + ``shrink``, ``extract`` or ``split`` on the list it is passed. Parameters diff --git a/scopesim/effects/shifts.py b/scopesim/effects/shifts.py index 1fc9d79bf..492550b71 100644 --- a/scopesim/effects/shifts.py +++ b/scopesim/effects/shifts.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- """TBA.""" -import warnings from typing import ClassVar import numpy as np @@ -27,17 +26,6 @@ def apply_to(self, obj, **kwargs): """See parent docstring.""" return obj - def fov_grid(self, which="shifts", **kwargs): - """See parent docstring.""" - warnings.warn("The fov_grid method is deprecated and will be removed " - "in a future release.", DeprecationWarning, stacklevel=2) - if which == "shifts": - col_names = ["wavelength", "dx", "dy"] - waves, dx, dy = [self.get_table(**kwargs)[col] - for col in col_names] - return waves, dx, dy - return None - def get_table(self, **kwargs): if self.table is None: names = ["wavelength", "dx", "dy"] @@ -70,7 +58,7 @@ class AtmosphericDispersion(Shift3D): Used to generate the wavelength bins based on shifts due to the atmosphere. Doesn't contain an ``apply_to`` function, but provides information through - the ``fov_grid`` function. + the ``get_table`` function. Required Parameters ------------------- @@ -133,7 +121,7 @@ def __init__(self, **kwargs): def get_table(self, **kwargs): """ - Called by the fov_grid method of Shift3D. + Compute the wavelength-dependent refraction shifts. Returns ------- @@ -248,19 +236,6 @@ def apply_to(self, fov, **kwargs): return fov - def fov_grid(self, which="shifts", **kwargs): - """See parent docstring.""" - warnings.warn("The fov_grid method is deprecated and will be removed " - "in a future release.", DeprecationWarning, stacklevel=2) - kwargs.update(self.meta) - if "quick_adc" in self.meta: - ad = AtmosphericDispersion(**self.meta) - waves, dx, dy = ad.fov_grid() - dx *= -(1 - self.meta["efficiency"]) - dy *= -(1 - self.meta["efficiency"]) - return waves, dx, dy - return None - def plot(self): return None diff --git a/scopesim/effects/surface_list.py b/scopesim/effects/surface_list.py index 70d3e0006..1db0cf948 100644 --- a/scopesim/effects/surface_list.py +++ b/scopesim/effects/surface_list.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- """TBA.""" -import warnings from collections import OrderedDict from copy import deepcopy from typing import ClassVar @@ -39,31 +38,6 @@ def __init__(self, **kwargs): surf_kwargs["filename"] = from_currsys(surf_kwargs["filename"], self.cmds) self.surfaces[surf_kwargs["name"]] = SpectralSurface(**surf_kwargs) - def fov_grid(self, which="waveset", **kwargs): - warnings.warn("The fov_grid method is deprecated and will be removed " - "in a future release.", DeprecationWarning, stacklevel=2) - wave_edges = [] - if which == "waveset": - self.meta.update(kwargs) - self.meta = from_currsys(self.meta, self.cmds) - wave_min = quantify(self.meta["wave_min"], u.um) - wave_max = quantify(self.meta["wave_max"], u.um) - # ..todo:: add 1001 to default.yaml somewhere - wave = np.linspace(wave_min, wave_max, 1001) - throughput = self.throughput(wave) - threshold = self.meta["minimum_throughput"] - valid_waves = np.where(throughput >= threshold)[0] - - if not len(valid_waves): - msg = ("No transmission found above the threshold " - f"{self.meta['minimum_throughput']} in this wavelength " - f"range {[self.meta['wave_min'], self.meta['wave_max']]}." - " Did you open the shutter?") - raise ValueError(msg) - - wave_edges = [min(wave[valid_waves]), max(wave[valid_waves])] - return wave_edges - @property def throughput(self): if self._throughput is None: diff --git a/scopesim/effects/ter_curves.py b/scopesim/effects/ter_curves.py index 53641a043..591301832 100644 --- a/scopesim/effects/ter_curves.py +++ b/scopesim/effects/ter_curves.py @@ -130,6 +130,14 @@ def _apply_to_fvl(self, fvl: FovVolumeList) -> None: wave = self.surface.throughput.waveset thru = self.surface.throughput(wave) valid_waves = np.argwhere(thru > 0) + + if not len(valid_waves): + raise ValueError( + f"No transmission found in {self.display_name} over the " + f"wavelength range [{wave.min().to_value(u.um):.3f}, " + f"{wave.max().to_value(u.um):.3f}] um. " + "Did you open the shutter?") + wave_min = wave[max(0, valid_waves[0][0] - 1)] wave_max = wave[min(len(wave) - 1, valid_waves[-1][0] + 1)] diff --git a/scopesim/tests/tests_effects/test_AtmosphericDispersion.py b/scopesim/tests/tests_effects/test_AtmosphericDispersion.py index 392d9e7a4..a578674a1 100644 --- a/scopesim/tests/tests_effects/test_AtmosphericDispersion.py +++ b/scopesim/tests/tests_effects/test_AtmosphericDispersion.py @@ -44,21 +44,17 @@ def test_throws_error_if_one_keyword_missing(self, atmo_yaml_dict): isinstance(AtmosphericDispersion(), AtmosphericDispersion) -class TestFovGrid: - def test_returns_list_of_3_arrays_with_correct_which(self, atmo_yaml_dict): +class TestGetTable: + def test_returns_wavelength_dx_dy_columns(self, atmo_yaml_dict): atmo_disp = AtmosphericDispersion(**atmo_yaml_dict["properties"]) - response = atmo_disp.fov_grid() - assert len(response) == 3 - assert all([isinstance(x, np.ndarray) for x in response]) - - def test_returns_non_with_wrong_which_keyword(self, atmo_yaml_dict): - atmo_disp = AtmosphericDispersion(**atmo_yaml_dict["properties"]) - response = atmo_disp.fov_grid(which="bogus") - assert response is None + tbl = atmo_disp.get_table() + assert tbl.colnames == ["wavelength", "dx", "dy"] + assert all(isinstance(tbl[col], np.ndarray) for col in tbl.colnames) def test_returns_similar_values_to_lasilla_website(self, atmo_yaml_dict): atmo_disp = AtmosphericDispersion(**atmo_yaml_dict["properties"]) - waves, dx, dy = atmo_disp.fov_grid() + tbl = atmo_disp.get_table() + waves, dx, dy = tbl["wavelength"], tbl["dx"], tbl["dy"] assert dy[0] - dy[-1] == approx(0.53, rel=1e-2) assert all(dx == 0) assert waves[0] == 0.5 and waves[-1] == 2.5 @@ -66,14 +62,16 @@ def test_returns_similar_values_to_lasilla_website(self, atmo_yaml_dict): def test_returns_same_results_when_turned_90_degrees(self, atmo_yaml_dict): atmo_yaml_dict["properties"]["pupil_angle"] = 90 atmo_disp = AtmosphericDispersion(**atmo_yaml_dict["properties"]) - waves, dx, dy = atmo_disp.fov_grid() + tbl = atmo_disp.get_table() + dx, dy = tbl["dx"], tbl["dy"] assert dx[0] - dx[-1] == approx(0.53, rel=1e-2) assert all([y == approx(0) for y in dy]) def test_returns_same_results_when_turned_30_degrees(self, atmo_yaml_dict): atmo_yaml_dict["properties"]["pupil_angle"] = 30 atmo_disp = AtmosphericDispersion(**atmo_yaml_dict["properties"]) - waves, dx, dy = atmo_disp.fov_grid() + tbl = atmo_disp.get_table() + dx, dy = tbl["dx"], tbl["dy"] dr = ((dx[0] - dx[-1])**2 + (dy[0] - dy[-1])**2)**0.5 assert dr == approx(0.53, rel=1e-2) diff --git a/scopesim/tests/tests_effects/test_AtmosphericDispersionCorrection.py b/scopesim/tests/tests_effects/test_AtmosphericDispersionCorrection.py index 4ff560b9f..55b83bb70 100644 --- a/scopesim/tests/tests_effects/test_AtmosphericDispersionCorrection.py +++ b/scopesim/tests/tests_effects/test_AtmosphericDispersionCorrection.py @@ -101,9 +101,11 @@ def test_shifts_between_adc_and_ad_are_opposite(self, atmo_params, waves, ad = AD(**atmo_params) adc = ADC(**atmo_params) - ad_shifts = ad.fov_grid() - ad_x_shift = np.interp(fov_wave_mid, ad_shifts[0], ad_shifts[1]) - ad_y_shift = np.interp(fov_wave_mid, ad_shifts[0], ad_shifts[2]) + ad_shifts = ad.get_table() + ad_x_shift = np.interp(fov_wave_mid, ad_shifts["wavelength"], + ad_shifts["dx"]) + ad_y_shift = np.interp(fov_wave_mid, ad_shifts["wavelength"], + ad_shifts["dy"]) adc.apply_to(fov) new_crpix_d = np.array([fov.header["CRPIX1D"], fov.header["CRPIX2D"]]) diff --git a/scopesim/tests/tests_effects/test_ConstPSF.py b/scopesim/tests/tests_effects/test_ConstPSF.py index d6270dd1c..27706b5d9 100644 --- a/scopesim/tests/tests_effects/test_ConstPSF.py +++ b/scopesim/tests/tests_effects/test_ConstPSF.py @@ -12,7 +12,6 @@ # - psf : returns self, as array returns a layer based on defaults # 4. should have the following methods: # - (?) apply_to(obj) -# - fov_grid(which="waveset") # - get_kernel(obj) """ diff --git a/scopesim/tests/tests_effects/test_TERCurve.py b/scopesim/tests/tests_effects/test_TERCurve.py index 67c7ed5a5..cf508a3c3 100644 --- a/scopesim/tests/tests_effects/test_TERCurve.py +++ b/scopesim/tests/tests_effects/test_TERCurve.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """Tests for TERCurve class""" import pytest @@ -7,8 +8,10 @@ from matplotlib import pyplot as plt from astropy import units as u from astropy.wcs import WCS +from astropy.table import Table from scopesim.effects import ter_curves as tc +from scopesim.optics.fov_volume_list import FovVolumeList from scopesim.tests.mocks.py_objects import source_objects as so from scopesim.tests.mocks.py_objects import effects_objects as eo @@ -73,6 +76,18 @@ def test_applies_to_cube_source(self): npt.assert_allclose(new, orig * thru) + def test_raises_helpful_error_when_nothing_transmits(self): + """A fully opaque surface used to die on a bare IndexError inside the + volume setup. The error must name the effect and the waverange.""" + tbl = Table(data=[[0.5, 1.0, 2.0, 3.0], [0.0, 0.0, 0.0, 0.0]], + names=["wavelength", "transmission"]) + tbl["wavelength"].unit = "um" + eff = tc.TERCurve(table=tbl, wave_min=0.5, wave_max=3.0, + name="closed shutter") + + with pytest.raises(ValueError, match="Did you open the shutter"): + eff.apply_to(FovVolumeList()) + class TestTERCurvePlot: def test_plots_only_transmission(self): diff --git a/scopesim/tests/tests_effects/test_slitwheel.py b/scopesim/tests/tests_effects/test_slitwheel.py index b46f10f19..707e3dac8 100644 --- a/scopesim/tests/tests_effects/test_slitwheel.py +++ b/scopesim/tests/tests_effects/test_slitwheel.py @@ -27,9 +27,6 @@ def test_has_aperture_masks(self, swheel): def test_current_slit_is_aperture_mask(self, swheel): assert isinstance(swheel.current_slit, ApertureMask) - def test_current_slit_has_fov_grid_method(self, swheel): - assert hasattr(swheel.current_slit, "fov_grid") - def test_change_to_known_slit(self, swheel): swheel.change_slit('A') assert swheel.current_slit.meta['name'] == 'A' From eecba478c37c838544e48f5351e6056259adf369 Mon Sep 17 00:00:00 2001 From: teutoburg Date: Sat, 12 Sep 2026 13:03:37 +0200 Subject: [PATCH 049/130] Remove old comments --- scopesim/source/source.py | 67 +-------------------------------------- 1 file changed, 1 insertion(+), 66 deletions(-) diff --git a/scopesim/source/source.py b/scopesim/source/source.py index b738cb92d..181aa63fb 100644 --- a/scopesim/source/source.py +++ b/scopesim/source/source.py @@ -1,24 +1,5 @@ +# -*- coding: utf-8 -*- """ -# old functionality to implement: -# - provide x, y, lam, spectra, weight, ref -# - overridden + : number, Source, SourceSpectrum -# - overridden * : number, SpectralElement -# - write to and read from file -# - shift all fields -# - rotate around the centre -# - photons_in_range returns the photons per spectrum in a wavelength range -# - image_in_range returns an image of the source for a wavelength range -# -# old functionality which will be removed: -# - project_onto_chip -# - apply_optical_train -# -# old structure --> new structure: -# - all data held in 6 arrays -# --> new dicts for fields, spectrum -# field can be a Table or an ImageHDU -# spectrum is a SourceSpectrum -# # Use cases: # image + spectrum # images + spectra @@ -242,27 +223,6 @@ def _from_imagehdu_and_spectra(self, image_hdu, **kwargs): assert not self.fields, "Constructor method must act on empty instance!" - if not image_hdu.header.get("BG_SRC"): - pass - # FIXME: This caused more problems than it solved! - # Find out if there's a good reason to mess with this, - # otherwise just remove... - - # image_hdu.header["CRVAL1"] = 0 - # image_hdu.header["CRVAL2"] = 0 - # image_hdu.header["CRPIX1"] = image_hdu.header["NAXIS1"] / 2 - # image_hdu.header["CRPIX2"] = image_hdu.header["NAXIS2"] / 2 - # #image_hdu.header["CRPIX1"] = (image_hdu.header["NAXIS1"] + 1) / 2 - # #image_hdu.header["CRPIX2"] = (image_hdu.header["NAXIS2"] + 1) / 2 - # # .. todo:: find where the actual problem is with negative CDELTs - # # .. todo:: --> abs(pixel_scale) in header_from_list_of_xy - # if image_hdu.header["CDELT1"] < 0: - # image_hdu.header["CDELT1"] *= -1 - # image_hdu.data = image_hdu.data[:, ::-1] - # if image_hdu.header["CDELT2"] < 0: - # image_hdu.header["CDELT2"] *= -1 - # image_hdu.data = image_hdu.data[::-1, :] - if isinstance(image_hdu, fits.PrimaryHDU): image_hdu = fits.ImageHDU(data=image_hdu.data, header=image_hdu.header) @@ -270,7 +230,6 @@ def _from_imagehdu_and_spectra(self, image_hdu, if not spectra: raise ValueError("No spectrum was provided.") - # image_hdu.header["SPEC_REF"] = len(self.spectra) assert len(spectra) == 1, f"_from_imagehdu_and_spectra needs single spectrum, ref was {image_hdu.header.get('SPEC_REF')}" image_hdu.header["SPEC_REF"] = 0 @@ -455,7 +414,6 @@ def bandpass(self, bandpass): self._bandpass = bandpass - # ..todo: rewrite this method def image_in_range(self, wave_min, wave_max, pixel_scale=1*u.arcsec, layers=None, area=None, spline_order=1, sub_pixel=False): if layers is None: @@ -481,17 +439,6 @@ def image_in_range(self, wave_min, wave_max, pixel_scale=1*u.arcsec, flux = self.photons_in_range(wave_min, wave_max, area, ref) # [ph s-1] or [ph s-1 m-2] come out of photons_in_range - # ## ..todo: CATCH UNITS HERE. DEAL WITH THEM PROPERLY - # Currently assuming that all images are scaled appropriately - # and that they have SPEC_REF - - # else: - # field = scale_imagehdu(field, area=area, - # solid_angle=pixel_scale**2, - # waverange=(wave_min, wave_max)) - # # [ph s-1] or [ph s-1 m-2] come out of photons_in_range - # flux = 1 - image = field.data * flux hdu = fits.ImageHDU(header=field.header, data=image) hdu_or_table = hdu @@ -537,18 +484,6 @@ def fluxes(self, wave_min, wave_max, **kwargs): def image(self, wave_min, wave_max, **kwargs): return self.image_in_range(wave_min, wave_max, **kwargs) - # @classmethod - # def load(cls, filename): - # """Load :class:'.Source' object from filename""" - # with open(filename, "rb") as fp1: - # src = pickle.load(fp1) - # return src - - # def dump(self, filename): - # """Save to filename as a pickle""" - # with open(filename, "wb") as fp1: - # pickle.dump(self, fp1) - def shift(self, dx: float = 0, dy: float = 0, layers=None) -> None: """ Shift the position of one or more fields w.r.t. the optical axis. From 1519e9c45080d5095ddec8323dcf7edb713687c3 Mon Sep 17 00:00:00 2001 From: teutoburg Date: Sun, 13 Sep 2026 18:23:57 +0200 Subject: [PATCH 050/130] SimCADO corpse removal All of this was only used in tests, not in any live part of ScopeSim. Some things were still used to setup or assert tests on live code, so I moved those to `/tests`. --- scopesim/optics/image_plane.py | 7 +- scopesim/optics/image_plane_utils.py | 340 +-------- scopesim/source/source.py | 74 -- scopesim/source/source_templates.py | 5 +- scopesim/source/source_utils.py | 90 +-- ...t_flux_is_conserved_through_full_system.py | 21 +- .../tests/tests_optics/test_ImagePlane.py | 698 ++++-------------- .../tests_optics/test_ImagePlaneUtils.py | 21 - scopesim/tests/tests_source/source_helpers.py | 95 +++ .../tests/tests_source/test_source_Source.py | 176 +---- 10 files changed, 267 insertions(+), 1260 deletions(-) create mode 100644 scopesim/tests/tests_source/source_helpers.py diff --git a/scopesim/optics/image_plane.py b/scopesim/optics/image_plane.py index e2b101f48..7b94adaf0 100644 --- a/scopesim/optics/image_plane.py +++ b/scopesim/optics/image_plane.py @@ -1,15 +1,10 @@ # -*- coding: utf-8 -*- """Contains ``ImagePlane`` class.""" -from warnings import warn - -import numpy as np - from astropy.io import fits -from astropy.table import Table from astropy.wcs import WCS -from .image_plane_utils import add_table_to_imagehdu, add_imagehdu_to_imagehdu +from .image_plane_utils import add_imagehdu_to_imagehdu from ..utils import ( from_currsys, diff --git a/scopesim/optics/image_plane_utils.py b/scopesim/optics/image_plane_utils.py index 7304d41d7..15f91daf5 100644 --- a/scopesim/optics/image_plane_utils.py +++ b/scopesim/optics/image_plane_utils.py @@ -1,160 +1,19 @@ # -*- coding: utf-8 -*- -from itertools import product -from collections.abc import Iterable - import numpy as np from numpy.typing import ArrayLike, NDArray from astropy import units as u from astropy.wcs import WCS, find_all_wcs from astropy.io import fits -from astropy.table import Table from astropy.utils.exceptions import AstropyWarning from scipy import ndimage as ndi -from ..utils import (unit_from_table, quantity_from_table, has_needed_keywords, - get_logger) +from ..utils import get_logger logger = get_logger(__name__) -def get_canvas_header(hdu_or_table_list, pixel_scale=1 * u.arcsec): - """ - Generate a fits.Header with a WCS that covers everything in the FOV. - - Parameters - ---------- - hdu_or_table_list : list - A list of Tables and/or ImageHDU py_objects - - pixel_scale : astropy.Quantity - [arcsec] The pixel scale of the projection. Default in 1 arcsec - - Returns - ------- - header : fits.Header - A Header containing a WCS and NAXISn values to build an ImageHDU - - """ - size_warning = ("Header dimension are {adverb} large: {num_pix}. " - "Any image made from this header will use more that " - ">{size} in memory") - - def _get_headers(hdus_or_tables): - tables = [] - for hdu_or_table in hdus_or_tables: - if isinstance(hdu_or_table, fits.ImageHDU): - yield hdu_or_table.header - elif isinstance(hdu_or_table, fits.Header): - yield hdu_or_table - elif isinstance(hdu_or_table, Table): - tables.append(hdu_or_table) - else: - raise TypeError( - "hdu_or_table_list may only contain fits.ImageHDU, Table " - f"or fits.Header, found {type(hdu_or_table)}.") - if tables: - yield _make_bounding_header_for_tables(*tables, - pixel_scale=pixel_scale) - - headers = list(_get_headers(hdu_or_table_list)) - - if not headers: - logger.warning("No tables or ImageHDUs were passed") - return None - - hdr = _make_bounding_header_from_headers(*headers, pixel_scale=pixel_scale) - - num_pix = hdr["NAXIS1"] * hdr["NAXIS2"] - if num_pix > 2 ** 28: - raise MemoryError(size_warning.format(adverb="too", num_pix=num_pix, - size="8 GB")) - if num_pix > 2 ** 25: # 2 * 4096**2 - logger.warning(size_warning.format(adverb="", num_pix=num_pix, - size="256 MB")) - return hdr - - -def _make_bounding_header_from_headers(*headers, pixel_scale=1*u.arcsec): - """ - Return a Header with WCS and NAXISn keywords bounding all input ImageHDUs. - - Parameters - ---------- - headers : list of fits.ImageHDU - pixel_scale : u.Quantity - [arcsec] - - Returns - ------- - hdr : fits.Header - - """ - wcs_suffix = "D" if pixel_scale.unit.physical_type == "length" else "" - unit = u.Unit(_get_unit_from_headers(*headers, wcs_suffix=wcs_suffix)) - - if unit.physical_type == "angle": - unit = "deg" - pixel_scale = pixel_scale.to_value(u.deg) - else: - pixel_scale = pixel_scale.to_value(unit) - - extents = [calc_footprint(header, wcs_suffix, unit) for header in headers] - pnts = np.vstack(extents) - - hdr = header_from_list_of_xy(pnts[:, 0], pnts[:, 1], - pixel_scale, wcs_suffix) - hdr["NAXIS1"] += 1 - hdr["NAXIS2"] += 1 - hdr[f"CRVAL1{wcs_suffix}"] -= 0.5 * pixel_scale - hdr[f"CRVAL2{wcs_suffix}"] -= 0.5 * pixel_scale - - return hdr - - -def _make_bounding_header_for_tables(*tables, pixel_scale=1*u.arcsec): - """ - Return a Header with WCS and NAXISn keywords bounding all input Tables. - - Parameters - ---------- - tables : list of astropy.Tables - [arcsec] Must contain columns: "x", "y" - - pixel_scale : u.Quantity - [arcsec] - - Returns - ------- - hdr : fits.Header - - """ - wcs_suffix = "D" if pixel_scale.unit.physical_type == "length" else "" - # FIXME: Convert to deg here? If yes, remove the arcsec=True below... - # Note: this could all be a lot simpler if we have consistent units, i.e. - # don't need to convert mm -> mm and arcsec -> arcsec (or deg) - new_unit = u.mm if wcs_suffix == "D" else u.arcsec # u.deg - tbl_unit = u.mm if wcs_suffix == "D" else u.arcsec - x_name = "x_mm" if wcs_suffix == "D" else "x" - y_name = "y_mm" if wcs_suffix == "D" else "y" - - pixel_scale = pixel_scale.to(new_unit) - - extents = [] - for table in tables: - extent = calc_table_footprint(table, x_name, y_name, - tbl_unit, new_unit, - padding=pixel_scale) - extents.append(extent) - pnts = np.vstack(extents) - - # TODO: check if this could just use create_wcs_from_points - hdr = header_from_list_of_xy(pnts[:, 0], pnts[:, 1], pixel_scale.value, - wcs_suffix, arcsec=wcs_suffix != "D") - return hdr - - def create_wcs_from_points( points: ArrayLike, pixel_scale: float, @@ -313,95 +172,6 @@ def header_from_list_of_xy(x, y, pixel_scale, wcs_suffix="", arcsec=False): return hdr -def add_table_to_imagehdu(table: Table, canvas_hdu: fits.ImageHDU, - sub_pixel: bool = True, - wcs_suffix: str = "") -> fits.ImageHDU: - """ - Add files from an astropy.Table to the image of an fits.ImageHDU. - - Parameters - ---------- - table : astropy.Table - Must contain the columns "x_mm", "y_mm", "flux" with the units in the - column attribute .unit, or in the table.meta dictionary as - "_unit". Default units are ``mm`` and ``ph / s / pix`` - - canvas_hdu : fits.ImageHDU - The ImageHDU onto which the table files should be projected. - This must include a valid WCS - - sub_pixel : bool, optional - Default is True. If True, sub-pixel shifts of files will be taken into - account when projecting onto the canvas pixel grid. This takes about 5x - longer than ignoring the sub-pixel shifts - - wcs_suffix : str, optional - - Returns - ------- - canvas_hdu : fits.ImageHDU - - """ - s = wcs_suffix - if not has_needed_keywords(canvas_hdu.header, s): - raise ValueError(f"canvas_hdu must include an appropriate WCS: {s}") - - f = quantity_from_table("flux", table, default_unit=u.Unit("ph s-1")) - if s == "D": - x = quantity_from_table("x_mm", table, default_unit=u.mm).to(u.mm) - y = quantity_from_table("y_mm", table, default_unit=u.mm).to(u.mm) - else: - arcsec = u.arcsec - canvas_unit = u.Unit(canvas_hdu.header[f"CUNIT1{s}"]) - x = quantity_from_table("x", table, default_unit=arcsec.to(canvas_unit)) - y = quantity_from_table("y", table, default_unit=arcsec.to(canvas_unit)) - x *= arcsec.to(canvas_unit) - y *= arcsec.to(canvas_unit) - - xpix, ypix = val2pix(canvas_hdu.header, x.value, y.value, s) - - naxis1 = canvas_hdu.header["NAXIS1"] - naxis2 = canvas_hdu.header["NAXIS2"] - # Occasionally 0 is returned as ~ -1e-11 - eps = -1e-7 - mask = (xpix >= eps) * (xpix < naxis1) * (ypix >= eps) * (ypix < naxis2) - - if sub_pixel: - canvas_hdu = _add_subpixel_sources_to_canvas( - canvas_hdu, xpix, ypix, f, mask) - else: - canvas_hdu = _add_intpixel_sources_to_canvas( - canvas_hdu, xpix, ypix, f, mask) - - return canvas_hdu - - -def _add_intpixel_sources_to_canvas(canvas_hdu, xpix, ypix, flux, mask): - canvas_hdu.header["comment"] = f"Adding {len(flux)} int-pixel files" - for xpx, ypx, flx, msk in zip(xpix.astype(int), ypix.astype(int), - flux, mask): - # To prevent adding array values in this manner. - assert not isinstance(xpx, Iterable), "xpx should be integer" - canvas_hdu.data[ypx, xpx] += flx.value * msk - - return canvas_hdu - - -def _add_subpixel_sources_to_canvas(canvas_hdu, xpix, ypix, flux, mask): - canvas_hdu.header["comment"] = f"Adding {len(flux)} sub-pixel files" - canvas_shape = canvas_hdu.data.shape - for xpx, ypx, flx, msk in zip(xpix, ypix, flux, mask): - if msk: - xx, yy, fracs = sub_pixel_fractions(xpx, ypx) - for x, y, frac in zip(xx, yy, fracs): - if y < canvas_shape[0] and x < canvas_shape[1]: - # To prevent adding array values in this manner. - assert not isinstance(x, Iterable), "x should be integer" - canvas_hdu.data[y, x] += frac * flx.value - - return canvas_hdu - - def sub_pixel_fractions(x, y): """ Make a list of pixel coordinates and weights to reflect sub-pixel shifts. @@ -955,6 +725,7 @@ def add_imagehdu_to_imagehdu( return canvas_hdu +# TODO: Use proper WCS functions instead and rm this on-foot implementation! def pix2val(header, x, y, wcs_suffix=""): """ Return the real coordinates [deg, mm] for coordinates from a Header WCS. @@ -995,6 +766,7 @@ def pix2val(header, x, y, wcs_suffix=""): return a, b +# TODO: Use proper WCS functions instead and rm this on-foot implementation! def val2pix(header, a, b, wcs_suffix=""): """ Return the pixel coordinates for real coordinates [deg, mm] from a WCS. @@ -1105,103 +877,6 @@ def calc_footprint(header, wcs_suffix="", new_unit: str = None): return xy1 -def calc_table_footprint( - table: Table, - x_name: str, - y_name: str, - tbl_unit: str, - new_unit: str, - padding: u.Quantity | None = None, -) -> NDArray: - """ - Equivalent to ``calc_footprint()``, but for tables instead of images. - - Parameters - ---------- - table : astropy.table.Table - Table containing data. - x_name : str - Name of the column in `table` to use as x-coordinates. - y_name : str - Name of the column in `table` to use as y-coordinates. - tbl_unit : str - Default unit to use for x and y if no units are found in `table`. - new_unit : str - Unit to convert x and y to, can be identical to `tbl_unit`. - padding : astropy.units.Quantity, optional - Constant value to subtract from minima and add to maxima. If used, must - be Quantity with same physical type as x and y. If None (default), no - padding is added. - - Returns - ------- - extent : (4, 2) array - Array containing corner points (clockwise from bottom left). Format and - order are equivalent to the output of - ``astropy.wcs.WCS.calc_footprint()``. - - """ - if padding is not None: - padding = padding.to_value(new_unit) - else: - padding = 0. - - x_convf = unit_from_table(x_name, table, tbl_unit).to(new_unit) - y_convf = unit_from_table(y_name, table, tbl_unit).to(new_unit) - - x_col = table[x_name] * x_convf - y_col = table[y_name] * y_convf - - x_min = x_col.min() - padding - x_max = x_col.max() + padding - y_min = y_col.min() - padding - y_max = y_col.max() + padding - - extent = np.array([[x_min, y_min], - [x_min, y_max], - [x_max, y_max], - [x_max, y_min]]) - - return extent - - -def split_header(hdr, chunk_size, wcs_suffix=""): - """ - Split a header into many smaller parts of the chunk_size. - - Parameters - ---------- - hdr - chunk_size - wcs_suffix - - Returns - ------- - hdr_list - """ - # TODO: test that this works - s = wcs_suffix - naxis1, naxis2 = hdr["NAXIS1"+s], hdr["NAXIS2"+s] - x0_pix, y0_pix = hdr["CRPIX1"+s], hdr["CRPIX2"+s] # pix - x0_sky, y0_sky = hdr["CRVAL1"+s], hdr["CRVAL2"+s] # deg - x_delt, y_delt = hdr["CDELT1"+s], hdr["CDELT2"+s] # deg / pix - - hdr_list = [] - for x1_pix in range(0, naxis1, chunk_size): - for y1_pix in range(0, naxis2, chunk_size): - x1_sky = x0_sky + (x1_pix - x0_pix) * x_delt - y1_sky = y0_sky + (y1_pix - y0_pix) * y_delt - x2_sky = x1_sky + x_delt * min(chunk_size, naxis1 - x1_pix) - y2_sky = y1_sky + y_delt * min(chunk_size, naxis2 - y1_pix) - - hdr_sky = header_from_list_of_xy([x1_sky, x2_sky], - [y1_sky, y2_sky], - pixel_scale=x_delt, wcs_suffix=s) - hdr_list.append(hdr_sky) - - return hdr_list - - def _fix_360(arr): """Fix the "full circle overflow" that occurs with deg.""" if isinstance(arr, u.Quantity): @@ -1214,15 +889,6 @@ def _fix_360(arr): return arr -def _get_unit_from_headers(*headers, wcs_suffix: str = "") -> str: - unit = headers[0][f"CUNIT1{wcs_suffix}"].lower() - assert all(header[f"CUNIT{i}{wcs_suffix}"].lower() == unit - for header, i in product(headers, range(1, 3))), \ - [(i, header[f"CUNIT{i}{wcs_suffix}"]) - for header, i in product(headers, range(1, 3))] - return unit - - def det_wcs_from_sky_wcs( sky_wcs: WCS, pixel_scale: float, diff --git a/scopesim/source/source.py b/scopesim/source/source.py index 181aa63fb..8d264b061 100644 --- a/scopesim/source/source.py +++ b/scopesim/source/source.py @@ -25,18 +25,14 @@ from synphot import SpectralElement, SourceSpectrum -from ..optics.image_plane import ImagePlane -from ..optics import image_plane_utils as imp_utils from .source_utils import ( validate_source_input, convert_to_list_of_spectra, - photons_in_range, ) from .source_fields import ( SourceField, TableSourceField, SpectrumSourceField, - HDUSourceField, ImageSourceField, CubeSourceField, ) @@ -44,7 +40,6 @@ find_file, is_fits, get_fits_type, - quantity_from_table, figure_factory, get_logger, ) @@ -414,75 +409,6 @@ def bandpass(self, bandpass): self._bandpass = bandpass - def image_in_range(self, wave_min, wave_max, pixel_scale=1*u.arcsec, - layers=None, area=None, spline_order=1, sub_pixel=False): - if layers is None: - layers = range(len(self.fields)) - fields = [self.fields[ii].field for ii in layers] - - hdr = imp_utils.get_canvas_header(fields, pixel_scale=pixel_scale) - im_plane = ImagePlane(hdr) - - for field in fields: - if isinstance(field, Table): - fluxes = self.photons_in_range(wave_min, wave_max, area, - field["ref"]) * field["weight"] - x = quantity_from_table("x", field, u.arcsec) - y = quantity_from_table("y", field, u.arcsec) - tbl = Table(names=["x", "y", "flux"], data=[x, y, fluxes]) - tbl.meta.update(field.meta) - hdu_or_table = tbl - - elif isinstance(field, fits.ImageHDU): - if field.header["SPEC_REF"] != "": - ref = [field.header["SPEC_REF"]] - flux = self.photons_in_range(wave_min, wave_max, area, ref) - # [ph s-1] or [ph s-1 m-2] come out of photons_in_range - - image = field.data * flux - hdu = fits.ImageHDU(header=field.header, data=image) - hdu_or_table = hdu - else: - continue - - im_plane.add(hdu_or_table, sub_pixel=sub_pixel, - spline_order=spline_order) - - return im_plane - - def photons_in_range(self, wave_min, wave_max, area=None, indices=None): - """ - - Parameters - ---------- - wave_min : float, u.Quantity - [um] - wave_max : float, u.Quantity - [um] - area : float, u.Quantity, optional - [m2] - indices : list of integers, optional - - Returns - ------- - counts : u.Quantity list - [ph / s / m2] if area is None - [ph / s] if area is passed - - """ - if indices is None: - indices = self.spectra.keys() - - spectra = [self.spectra[ii] for ii in indices] - counts = photons_in_range(spectra, wave_min, wave_max, area=area, - bandpass=self.bandpass) - return counts - - def fluxes(self, wave_min, wave_max, **kwargs): - return self.photons_in_range(wave_min, wave_max, **kwargs) - - def image(self, wave_min, wave_max, **kwargs): - return self.image_in_range(wave_min, wave_max, **kwargs) def shift(self, dx: float = 0, dy: float = 0, layers=None) -> None: """ diff --git a/scopesim/source/source_templates.py b/scopesim/source/source_templates.py index 0a88afbc6..7b4b04757 100644 --- a/scopesim/source/source_templates.py +++ b/scopesim/source/source_templates.py @@ -1,11 +1,11 @@ +# -*- coding: utf-8 -*- + from pathlib import Path import numpy as np - from astropy import units as u from astropy.table import Table from astropy.io import fits -from astropy.utils.decorators import deprecated_renamed_argument from synphot import SourceSpectrum, ConstFlux1D, Empirical1D from synphot.units import PHOTLAM @@ -30,7 +30,6 @@ def empty_sky(flux=0): return sky -@deprecated_renamed_argument("mag", "flux", "0.1.5") def star(x=0, y=0, flux=0): """ Source object for a single star in either vega, AB magnitudes, or Jansky diff --git a/scopesim/source/source_utils.py b/scopesim/source/source_utils.py index caf43a218..d969149be 100644 --- a/scopesim/source/source_utils.py +++ b/scopesim/source/source_utils.py @@ -4,11 +4,10 @@ from pathlib import Path import numpy as np -from scipy import integrate from astropy import wcs, units as u from astropy.io import fits from astropy.table import Table -from synphot import SourceSpectrum, Empirical1D, SpectralElement +from synphot import SourceSpectrum, Empirical1D from ..utils import find_file, get_logger, convert_table_comments_to_dict @@ -111,93 +110,6 @@ def _get_list(): return list(_get_list()) -# FIXME: typing: This should work with the more general (and true, since -# conversion is done anyway) Quantity["length"] (or "area" resp.), but -# doing so currently causes a NameError. Not sure what's going on. -def photons_in_range( - spectra: SourceSpectrum, - wave_min: u.Quantity[u.um] | float, - wave_max: u.Quantity[u.um] | float, - area: u.Quantity[u.m**2] | float | None = None, - bandpass: SpectralElement | None = None, -) -> u.Quantity[u.ph * u.s**-1 * u.m**-2] | u.Quantity[u.ph * u.s**-1]: - """ - Integrate photons from spectrum in given wavelength range. - - Parameters - ---------- - spectra : SourceSpectrum - Input spectrum. - wave_min : u.Quantity["length"] or float - Minimum wavelength. If float, assumes um. - wave_max : u.Quantity["length"] or float - Maximum wavelength. If float, assumes um. - area : u.Quantity["area"] or float, optional - Area to multiply with. If float, assumes m**2. The default is None. - bandpass : SpectralElement, optional - Filter to take into account, if any. The default is None. - - Returns - ------- - counts : astropy.units.Quantity - Either in ph/s/m**2 or just ph/s (if area was given). - - """ - # Note: Assuming um if given as float. - wave_min = (wave_min << u.um << u.Angstrom).value - wave_max = (wave_max << u.um << u.Angstrom).value - # Note: There appear to be some float shenanigans going on here, but - # rounding produces an error in the spectrum evaluation. Not sure what's - # going on, maybe it's fine as-is. - - counts = [] - for spec in spectra: - waveset = spec.waveset.value - mask = (waveset > wave_min) * (waveset < wave_max) - wave = np.array([wave_min, *waveset[mask], wave_max]) - flux = spec(wave).value - - # flux [ph s-1 cm-2] == flux [ph s-1 cm-2 AA-1] * wave [AA] - if isinstance(bandpass, SpectralElement): - bandpass.model.bounds_error = True - counts.append(integrate.trapezoid(bandpass(wave).value * flux, wave)) - else: - counts.append(integrate.trapezoid(flux, wave)) - - # counts = flux [ph s-1 cm-2] - counts = (counts * u.ph * u.s**-1 * u.cm**-2).to(u.ph * u.s**-1 * u.m**-2) - if area is not None: - counts *= (area << u.m**2) - - return counts - - -def scale_imagehdu(imagehdu, waverange, area=None): - """ - ..todo: implement this - - For the moment, all imagehdu must be accompanied by a spectrum in PHOTLAM - - Future functionality will include scaling here of: - ph s-1 - ph s-1 m-2 - ph s-1 m-2 - ph s-1 m-2 um-1 - ph s-1 m-2 um-1 arcsec-2 - J s-1 m-2 Hz-1 - J s-1 m-2 Hz-1 arcsec-2 - ABMAG - ABMAG arcsec-2 - VEGAMAG - VEGAMAG arcsec-2 - """ - if "SPEC_REF" not in imagehdu.header: - raise ValueError("For this version, an ImageHDU must be associated " - "with a spectrum. This will change in the future.") - - return imagehdu - - def make_img_wcs_header( pixel_scale: float, image_size: tuple[int, int], diff --git a/scopesim/tests/tests_integrations/test_flux_is_conserved_through_full_system.py b/scopesim/tests/tests_integrations/test_flux_is_conserved_through_full_system.py index 65690fe0d..6c7542b5c 100644 --- a/scopesim/tests/tests_integrations/test_flux_is_conserved_through_full_system.py +++ b/scopesim/tests/tests_integrations/test_flux_is_conserved_through_full_system.py @@ -1,18 +1,21 @@ +# -*- coding: utf-8 -*- import pytest from pytest import approx from unittest.mock import patch import numpy as np +import matplotlib.pyplot as plt from scopesim.optics.optical_train import OpticalTrain from scopesim.commands import UserCommands from scopesim.tests.mocks.py_objects.source_objects import ( - _image_source, _single_table_source, _table_source_overlapping) - -import matplotlib.pyplot as plt -from matplotlib.colors import LogNorm + _image_source, + _single_table_source, + _table_source_overlapping, +) +from scopesim.tests.tests_source.source_helpers import src_photons_in_range PLOTS = False @@ -48,11 +51,11 @@ def test_flux_is_conserved_for_no_bg_emission(self, cmds, tbl_src): opt.observe(tbl_src) im = opt.image_planes[0].image bg_flux = np.pi / 4 * np.prod(im.shape) - src_flux = tbl_src.photons_in_range(1, 2, 1)[0].value + src_flux = src_photons_in_range(tbl_src, 1, 2, 1)[0].value if PLOTS: implane = opt.image_planes[0] - plt.imshow(implane.image.T, origin="lower", norm=LogNorm()) + plt.imshow(implane.image.T, origin="lower", norm="log") plt.colorbar() plt.show() @@ -70,11 +73,11 @@ def test_flux_is_conserved_for_yes_bg_emission(self, non_unity_cmds, tbl_src): opt = OpticalTrain(non_unity_cmds) opt.observe(tbl_src) im = opt.image_planes[0].image - src_flux = tbl_src.photons_in_range(1, 2, 1)[0].value + src_flux = src_photons_in_range(tbl_src, 1, 2, 1)[0].value if PLOTS: implane = opt.image_planes[0] - plt.imshow(implane.image.T, origin="lower", norm=LogNorm()) + plt.imshow(implane.image.T, origin="lower", norm="log") plt.colorbar() plt.show() @@ -133,7 +136,7 @@ def test_stacked_stars(self): if PLOTS: fig, axes = plt.subplots(nrows=1, ncols=1, figsize=(12, 12)) - axes.imshow(np.sqrt(im_h), norm=LogNorm(), cmap="inferno") + axes.imshow(np.sqrt(im_h), norm="log", cmap="inferno") axes.set_title('H-band MORFEO "PSF Generic" ') plt.show() diff --git a/scopesim/tests/tests_optics/test_ImagePlane.py b/scopesim/tests/tests_optics/test_ImagePlane.py index 1a8a81190..c941e903e 100644 --- a/scopesim/tests/tests_optics/test_ImagePlane.py +++ b/scopesim/tests/tests_optics/test_ImagePlane.py @@ -1,9 +1,11 @@ +# -*- coding: utf-8 -*- """Tests for ImagePlane and some ImagePlaneUtils""" # pylint: disable=missing-class-docstring # pylint: disable=missing-function-docstring from copy import deepcopy +from itertools import product import pytest from pytest import approx @@ -11,22 +13,108 @@ import numpy as np from astropy.io import fits from astropy import units as u -from astropy.table import Table from astropy import wcs - import matplotlib.pyplot as plt -from matplotlib.colors import LogNorm import scopesim.optics.image_plane as opt_imp import scopesim.optics.image_plane_utils as imp_utils -from scopesim.tests.mocks.py_objects.imagehdu_objects import \ - _image_hdu_square, _image_hdu_rect, _image_hdu_three_wcs,\ - _image_hdu_3d_data +from scopesim.tests.mocks.py_objects.imagehdu_objects import ( + _image_hdu_square, + _image_hdu_rect, + _image_hdu_three_wcs, + _image_hdu_3d_data, +) PLOTS = False +# The three functions below were oringinally stored in image_plane_utils, +# but ultimately only used here, so I moved them here as helpers. +# TODO: Check if the same functionality can be achieved in another way. +# ScopeSim internally uses something else (imageplane headers from detectors), +# so tests should ideally mock that instead... +def get_canvas_header(hdu_or_table_list, pixel_scale=1*u.arcsec): + """ + Generate a fits.Header with a WCS that covers everything in the FOV. + + Parameters + ---------- + hdu_or_table_list : list + A list of Tables and/or ImageHDU py_objects + + pixel_scale : astropy.Quantity + [arcsec] The pixel scale of the projection. Default in 1 arcsec + + Returns + ------- + header : fits.Header + A Header containing a WCS and NAXISn values to build an ImageHDU + + """ + def _get_headers(hdus_or_tables): + for hdu_or_table in hdus_or_tables: + if isinstance(hdu_or_table, fits.ImageHDU): + yield hdu_or_table.header + elif isinstance(hdu_or_table, fits.Header): + yield hdu_or_table + else: + raise TypeError( + "hdu_or_table_list may only contain fits.ImageHDU " + f"or fits.Header, found {type(hdu_or_table)}.") + + headers = list(_get_headers(hdu_or_table_list)) + + hdr = _make_bounding_header_from_headers(*headers, pixel_scale=pixel_scale) + return hdr + + +def _make_bounding_header_from_headers(*headers, pixel_scale): + """ + Return a Header with WCS and NAXISn keywords bounding all input ImageHDUs. + + Parameters + ---------- + headers : list of fits.ImageHDU + pixel_scale : u.Quantity + [arcsec] + + Returns + ------- + hdr : fits.Header + + """ + def _get_unit_from_headers(*headers, wcs_suffix: str) -> str: + unit = headers[0][f"CUNIT1{wcs_suffix}"].lower() + assert all(header[f"CUNIT{i}{wcs_suffix}"].lower() == unit + for header, i in product(headers, range(1, 3))), \ + [(i, header[f"CUNIT{i}{wcs_suffix}"]) + for header, i in product(headers, range(1, 3))] + return unit + + wcs_suffix = "D" if pixel_scale.unit.physical_type == "length" else "" + unit = u.Unit(_get_unit_from_headers(*headers, wcs_suffix=wcs_suffix)) + + if unit.physical_type == "angle": + unit = "deg" + pixel_scale = pixel_scale.to_value(u.deg) + else: + pixel_scale = pixel_scale.to_value(unit) + + extents = [imp_utils.calc_footprint(header, wcs_suffix, unit) + for header in headers] + pnts = np.vstack(extents) + + hdr = imp_utils.header_from_list_of_xy( + pnts[:, 0], pnts[:, 1], pixel_scale, wcs_suffix) + hdr["NAXIS1"] += 1 + hdr["NAXIS2"] += 1 + hdr[f"CRVAL1{wcs_suffix}"] -= 0.5 * pixel_scale + hdr[f"CRVAL2{wcs_suffix}"] -= 0.5 * pixel_scale + + return hdr + + @pytest.fixture(scope="function", name="image_hdu_rect") def fixture_image_hdu_rect(): return _image_hdu_rect() @@ -57,361 +145,14 @@ def fixture_image_hdu_3d_data(): return _image_hdu_3d_data() -@pytest.fixture(scope="function", name="input_table") -def fixture_input_table(): - x = [-10, -10, 0, 10, 10] * u.arcsec - y = [-10, 10, 0, -10, 10] * u.arcsec - f = [1, 3, 1, 1, 5] - tbl = Table(names=["x", "y", "flux"], data=[x, y, f]) - - return tbl - - -@pytest.fixture(scope="function", name="input_table_mm") -def fixture_input_table_mm(): - x = [-10, -10, 0, 10, 10] * u.mm - y = [-10, 10, 0, -10, 10] * u.mm - f = [1, 3, 1, 1, 5] - tbl = Table(names=["x_mm", "y_mm", "flux"], data=[x, y, f]) - - return tbl - - -class TestCombineTableBoundaries: - def test_all_three_tables_are_inside_header_wcs(self, input_table): - tbl1 = deepcopy(input_table) - tbl2 = deepcopy(input_table) - tbl3 = deepcopy(input_table) - - tbl2["x"] -= 25 - tbl3["y"] -= 60 - - hdr = imp_utils._make_bounding_header_for_tables(tbl1, tbl2, tbl3) - as2deg = u.arcsec.to(u.deg) - for tbl in [tbl1, tbl2, tbl3]: - x, y = imp_utils.val2pix(hdr, tbl["x"]*as2deg, tbl["y"]*as2deg) - for xi, yi in zip(x, y): - assert 0 <= xi < hdr["NAXIS1"] - assert 0 <= yi < hdr["NAXIS2"] - - if PLOTS: - xy = imp_utils.calc_footprint(hdr) - x, y = xy[:, 0], xy[:, 1] - x, y = imp_utils.val2pix(hdr, x, y) - x0, y0 = imp_utils.val2pix(hdr, 0, 0) - - plt.plot(x, y, "b") - plt.plot(x0, y0, "ro") - for tbl in [tbl1, tbl2, tbl3]: - x, y = imp_utils.val2pix(hdr, tbl["x"] / 3600., - tbl["y"] / 3600.) - plt.plot(x, y, "k.") - - plt.show() - - def test_all_three_mm_tables_are_inside_header_wcs(self, input_table_mm): - tbl1 = deepcopy(input_table_mm) - tbl2 = deepcopy(input_table_mm) - tbl3 = deepcopy(input_table_mm) - - tbl2["x_mm"] += 50 - tbl3["y_mm"] += 25 - - hdr = imp_utils._make_bounding_header_for_tables(tbl1, tbl2, tbl3, - pixel_scale=100*u.um) - for tbl in [tbl1, tbl2, tbl3]: - x, y = imp_utils.val2pix(hdr, tbl["x_mm"], tbl["y_mm"], "D") - for xi, yi in zip(x, y): - assert 0 <= xi < hdr["NAXIS1"] - assert 0 <= yi < hdr["NAXIS2"] - - if PLOTS: - xy = imp_utils.calc_footprint(hdr, "D") - x, y = xy[:, 0], xy[:, 1] - x, y = imp_utils.val2pix(hdr, x, y, "D") - x0, y0 = imp_utils.val2pix(hdr, 0, 0, "D") - - plt.plot(x, y, "b") - plt.plot(x0, y0, "ro") - for tbl in [tbl1, tbl2, tbl3]: - x, y = imp_utils.val2pix(hdr, tbl["x_mm"], tbl["y_mm"], "D") - plt.plot(x, y, "k.") - - plt.show() - - -class TestCombineImageHDUBoundaries: - def test_all_two_imagehdus_are_inside_header_wcs(self, image_hdu_square, - image_hdu_rect): - - image_hdu_rect.header["CRVAL1"] -= 70 # * u.arcsec.to(u.deg) - image_hdu_square.header["CRVAL2"] += 70 # * u.arcsec.to(u.deg) - - hdr = imp_utils._make_bounding_header_from_headers( - image_hdu_square.header, image_hdu_rect.header) - for imhdr in [image_hdu_square.header, image_hdu_rect.header]: - xy = imp_utils.calc_footprint(imhdr) - x, y = xy[:, 0], xy[:, 1] - x *= u.arcsec.to(u.deg) - y *= u.arcsec.to(u.deg) - x, y = imp_utils.val2pix(hdr, x, y) - for xi, yi in zip(x, y): - assert 0 <= xi < hdr["NAXIS1"] - assert 0 <= yi < hdr["NAXIS2"] - - if PLOTS: - for imhdr in [image_hdu_square.header, image_hdu_rect.header, hdr]: - xy = imp_utils.calc_footprint(imhdr) - x, y = xy[:, 0], xy[:, 1] - xp, yp = imp_utils.val2pix(imhdr, x, y) - plt.plot(x, y, "r-") - - plt.plot(0, 0, "ro") - plt.gca().set_aspect(1) - plt.show() - - def test_all_two_mm_imagehdus_are_inside_header_wcs(self, - image_hdu_square_mm, - image_hdu_rect_mm): - - image_hdu_rect_mm.header["CRVAL1D"] -= 40 - image_hdu_square_mm.header["CRVAL2D"] += 80 - - hdr = imp_utils._make_bounding_header_from_headers( - image_hdu_square_mm.header, image_hdu_rect_mm.header, - pixel_scale=1*u.mm) - for imhdr in [image_hdu_square_mm.header, image_hdu_rect_mm.header]: - xy = imp_utils.calc_footprint(imhdr, "D") - x, y = xy[:, 0], xy[:, 1] - x, y = imp_utils.val2pix(hdr, x, y, "D") - for xi, yi in zip(x, y): - assert 0 <= xi < hdr["NAXIS1"] - assert 0 <= yi < hdr["NAXIS2"] - - if PLOTS: - for imhdr in [image_hdu_square_mm.header, - image_hdu_rect_mm.header, hdr]: - xy = imp_utils.calc_footprint(imhdr, "D") - x, y = xy[:, 0], xy[:, 1] - xp, yp = imp_utils.val2pix(imhdr, x, y, "D") - plt.plot(x, y, "r-") - - plt.plot(0, 0, "ro") - plt.gca().set_aspect(1) - plt.show() - - -class TestGetCanvasHeader: - def test_all_5_objects_are_inside_header_wcs(self, image_hdu_square, - image_hdu_rect, input_table): - - tbl1 = deepcopy(input_table) - tbl2 = deepcopy(input_table) - tbl3 = deepcopy(input_table) - - tbl2["x"] -= 150 - tbl3["y"] -= 100 - - image_hdu_rect.header["CRVAL1"] += 100 * u.arcsec.to(u.deg) - image_hdu_square.header["CRVAL1"] += 0 * u.arcsec.to(u.deg) - image_hdu_square.header["CRVAL2"] += 100 * u.arcsec.to(u.deg) - - hdr = imp_utils.get_canvas_header([image_hdu_square, tbl1, tbl2, tbl3, - image_hdu_rect], pixel_scale=1*u.arcsec) - - as2deg = u.arcsec.to(u.deg) - - for im in [image_hdu_square.header, image_hdu_rect.header]: - xy = imp_utils.calc_footprint(im) - x, y = xy[:, 0], xy[:, 1] - x, y = imp_utils.val2pix(hdr, x*as2deg, y*as2deg) - for xi, yi in zip(x, y): - assert 0 <= xi < hdr["NAXIS1"] - assert 0 <= yi < hdr["NAXIS2"] - - for tbl in [tbl1, tbl2, tbl3]: - x, y = imp_utils.val2pix(hdr, tbl["x"]*as2deg, tbl["y"]*as2deg) - for xi, yi in zip(x, y): - assert 0 <= xi < hdr["NAXIS1"] - assert 0 <= yi < hdr["NAXIS2"] - - if PLOTS: - - xy = imp_utils.calc_footprint(hdr) - x, y = xy[:, 0], xy[:, 1] - x, y = imp_utils.val2pix(hdr, x, y) - plt.plot(x, y, "b") - x0, y0 = imp_utils.val2pix(hdr, 0, 0) - plt.plot(x0, y0, "ro") - - for tbl in [tbl1, tbl2, tbl3]: - x, y = imp_utils.val2pix(hdr, tbl["x"]*as2deg, tbl["y"]*as2deg) - plt.plot(x, y, "k.") - - for im in [image_hdu_square.header, image_hdu_rect.header]: - xy = imp_utils.calc_footprint(im) - x, y = xy[:, 0], xy[:, 1] - x, y = imp_utils.val2pix(hdr, x, y) - plt.plot(x, y, "r-") - - plt.gca().set_aspect(1) - plt.show() - - def test_all_5_objects_are_inside_mm_header_wcs(self, image_hdu_square_mm, - image_hdu_rect_mm, - input_table_mm): - - tbl1 = deepcopy(input_table_mm) - tbl2 = deepcopy(input_table_mm) - tbl3 = deepcopy(input_table_mm) - - tbl2["x_mm"] -= 150 - tbl3["y_mm"] -= 100 - - image_hdu_rect_mm.header["CRVAL1D"] += 100 - image_hdu_square_mm.header["CRVAL1D"] += 0 - image_hdu_square_mm.header["CRVAL2D"] += 100 - - hdr = imp_utils.get_canvas_header([image_hdu_square_mm, tbl1, tbl2, - tbl3, image_hdu_rect_mm], - pixel_scale=1*u.mm) - - for im in [image_hdu_square_mm.header, image_hdu_rect_mm.header]: - xy = imp_utils.calc_footprint(im, "D") - x, y = xy[:, 0], xy[:, 1] - x, y = imp_utils.val2pix(hdr, x, y, "D") - for xi, yi in zip(x, y): - assert 0 <= xi < hdr["NAXIS1"] - assert 0 <= yi < hdr["NAXIS2"] - - for tbl in [tbl1, tbl2, tbl3]: - x, y = imp_utils.val2pix(hdr, tbl["x_mm"], tbl["y_mm"], "D") - for xi, yi in zip(x, y): - assert 0 <= xi < hdr["NAXIS1"] - assert 0 <= yi < hdr["NAXIS2"] - - if PLOTS: - - xy = imp_utils.calc_footprint(hdr, "D") - x, y = xy[:, 0], xy[:, 1] - x, y = imp_utils.val2pix(hdr, x, y, "D") - plt.plot(x, y, "b") - x0, y0 = imp_utils.val2pix(hdr, 0, 0, "D") - plt.plot(x0, y0, "ro") - - for tbl in [tbl1, tbl2, tbl3]: - x, y = imp_utils.val2pix(hdr, tbl["x_mm"], tbl["y_mm"], "D") - plt.plot(x, y, "k.") - - for im in [image_hdu_square_mm.header, image_hdu_rect_mm.header]: - xy = imp_utils.calc_footprint(im, "D") - x, y = xy[:, 0], xy[:, 1] - x, y = imp_utils.val2pix(hdr, x, y, "D") - plt.plot(x, y, "r-") - - plt.gca().set_aspect(1) - plt.show() - - -class TestAddTableToImageHDU: - def test_points_are_added_to_small_canvas(self, input_table): - tbl1 = deepcopy(input_table) - hdr = imp_utils.get_canvas_header([tbl1]) - - im = np.zeros((hdr["NAXIS2"], hdr["NAXIS1"])) - canvas_hdu = fits.ImageHDU(header=hdr, data=im) - canvas_hdu = imp_utils.add_table_to_imagehdu(tbl1, canvas_hdu) - - assert np.sum(canvas_hdu.data) == np.sum(tbl1["flux"]) - - if PLOTS: - # "top left is green, top right is yellow" - plt.imshow(canvas_hdu.data, origin="lower") - plt.show() - - def test_mm_points_are_added_to_small_canvas(self, input_table_mm): - tbl1 = deepcopy(input_table_mm) - hdr = imp_utils.get_canvas_header([tbl1], pixel_scale=2*u.mm) - - im = np.zeros((hdr["NAXIS2"], hdr["NAXIS1"])) - canvas_hdu = fits.ImageHDU(header=hdr, data=im) - canvas_hdu = imp_utils.add_table_to_imagehdu(tbl1, canvas_hdu, - wcs_suffix="D") - - assert np.sum(canvas_hdu.data) == np.sum(tbl1["flux"]) - - if PLOTS: - # "top left is green, top right is yellow" - plt.imshow(canvas_hdu.data, origin="lower") - plt.show() - - # add a paramaterised test here to check border cases - def test_points_are_added_to_massive_canvas(self, input_table): - tbl1 = deepcopy(input_table) - tbl2 = deepcopy(input_table) - tbl3 = deepcopy(input_table) - - tbl1["y"] += 50 - tbl2["x"] += 20 - tbl3["x"] -= 25 - tbl3["y"] -= 25 - - hdr = imp_utils.get_canvas_header([tbl1, tbl2, tbl3], - pixel_scale=1*u.arcsec) - - im = np.zeros((hdr["NAXIS2"], hdr["NAXIS1"])) - canvas_hdu = fits.ImageHDU(header=hdr, data=im) - - for tbl in [tbl1, tbl2, tbl3]: - canvas_hdu = imp_utils.add_table_to_imagehdu(tbl, canvas_hdu, True) - - total_flux = np.sum([tbl1["flux"], tbl2["flux"], tbl3["flux"]]) - assert np.sum(canvas_hdu.data) == total_flux - - if PLOTS: - x, y = imp_utils.val2pix(hdr, 0, 0) - plt.plot(x, y, "ro") - "top left is green, top right is yellow" - plt.imshow(canvas_hdu.data, origin="lower") - plt.show() - - def test_mm_points_are_added_to_massive_canvas(self, input_table_mm): - tbl1 = deepcopy(input_table_mm) - tbl2 = deepcopy(input_table_mm) - tbl3 = deepcopy(input_table_mm) - - tbl1["y_mm"] += 50 - tbl2["x_mm"] += 20 - tbl3["x_mm"] -= 25 - tbl3["y_mm"] -= 25 - - hdr = imp_utils.get_canvas_header([tbl1, tbl2, tbl3], - pixel_scale=1*u.mm) - im = np.zeros((hdr["NAXIS2"], hdr["NAXIS1"])) - canvas_hdu = fits.ImageHDU(header=hdr, data=im) - - for tbl in [tbl1, tbl2, tbl3]: - canvas_hdu = imp_utils.add_table_to_imagehdu(tbl, canvas_hdu, - True, wcs_suffix="D") - - total_flux = np.sum([tbl1["flux"], tbl1["flux"], tbl1["flux"]]) - assert np.sum(canvas_hdu.data) == total_flux - - if PLOTS: - x, y = imp_utils.val2pix(hdr, 0, 0, "D") - plt.plot(x, y, "ro") - # "top left is green, top right is yellow" - plt.imshow(canvas_hdu.data, origin="lower") - plt.show() - - class TestAddImageHDUToImageHDU: - def test_image_is_added_to_small_canvas(self, image_hdu_rect, - image_hdu_square): + def test_image_is_added_to_small_canvas( + self, image_hdu_rect, image_hdu_square, + ): im_hdu = image_hdu_rect im_hdu.header["CRVAL1"] -= 150*u.arcsec.to(u.deg) im_hdu.header["CRVAL2"] += 40*u.arcsec.to(u.deg) - hdr = imp_utils.get_canvas_header([im_hdu, image_hdu_square]) + hdr = get_canvas_header([im_hdu, image_hdu_square]) im = np.zeros((hdr["NAXIS2"], hdr["NAXIS1"])) canvas_hdu = fits.ImageHDU(header=hdr, data=im) @@ -436,18 +177,19 @@ def test_image_is_added_to_small_canvas(self, image_hdu_rect, plt.show() - def test_mm_image_is_added_to_small_canvas(self, image_hdu_rect_mm, - image_hdu_square_mm): + def test_mm_image_is_added_to_small_canvas( + self, image_hdu_rect_mm, image_hdu_square_mm, + ): im_hdu = image_hdu_rect_mm im_hdu.header["CRVAL1D"] -= 150 im_hdu.header["CRVAL2D"] += 40 - hdr = imp_utils.get_canvas_header([im_hdu, image_hdu_square_mm], - pixel_scale=1*u.mm) + hdr = get_canvas_header([im_hdu, image_hdu_square_mm], 1*u.mm) im = np.zeros((hdr["NAXIS2"], hdr["NAXIS1"])) canvas_hdu = fits.ImageHDU(header=hdr, data=im) - canvas_hdu = imp_utils.add_imagehdu_to_imagehdu(im_hdu, canvas_hdu, - wcs_suffix="D") + canvas_hdu = imp_utils.add_imagehdu_to_imagehdu( + im_hdu, canvas_hdu, wcs_suffix="D", + ) assert np.sum(canvas_hdu.data) == approx(np.sum(im_hdu.data)) @@ -466,32 +208,21 @@ def test_mm_image_is_added_to_small_canvas(self, image_hdu_rect_mm, plt.show() - def test_image_and_tables_on_large_canvas(self, input_table, image_hdu_rect, - image_hdu_square): - tbl1 = deepcopy(input_table) - tbl2 = deepcopy(input_table) - - tbl1["y"] -= 100 - tbl2["x"] += 100 - tbl2["y"] += 100 - + def test_images_on_large_canvas(self, image_hdu_rect, image_hdu_square): im_hdu = image_hdu_rect im_hdu.header["CRVAL1"] -= 150 # *u.arcsec.to(u.deg) im_hdu.header["CRVAL2"] += 20 # *u.arcsec.to(u.deg) - total_flux = (np.sum(tbl1["flux"]) + np.sum(tbl2["flux"]) + - im_hdu.data.sum() + image_hdu_square.data.sum()) + total_flux = im_hdu.data.sum() + image_hdu_square.data.sum() - hdr = imp_utils.get_canvas_header([im_hdu, image_hdu_square, - tbl1, tbl2], pixel_scale=3*u.arcsec) + hdr = get_canvas_header([im_hdu, image_hdu_square], 3*u.arcsec) im = np.zeros((hdr["NAXIS2"], hdr["NAXIS1"])) canvas_hdu = fits.ImageHDU(header=hdr, data=im) canvas_hdu = imp_utils.add_imagehdu_to_imagehdu(im_hdu, canvas_hdu) - canvas_hdu = imp_utils.add_imagehdu_to_imagehdu(image_hdu_square, - canvas_hdu) - canvas_hdu = imp_utils.add_table_to_imagehdu(tbl1, canvas_hdu) - canvas_hdu = imp_utils.add_table_to_imagehdu(tbl2, canvas_hdu) + canvas_hdu = imp_utils.add_imagehdu_to_imagehdu( + image_hdu_square, canvas_hdu, + ) assert np.sum(canvas_hdu.data) == approx(total_flux, rel=1e-2) @@ -511,39 +242,28 @@ def test_image_and_tables_on_large_canvas(self, input_table, image_hdu_rect, plt.show() - def test_mm_image_and_tables_on_large_canvas(self, input_table_mm, - image_hdu_rect_mm, - image_hdu_square_mm): + def test_mm_images_on_large_canvas( + self, image_hdu_rect_mm, image_hdu_square_mm, + ): image_hdu_rect = image_hdu_rect_mm image_hdu_square = image_hdu_square_mm - tbl1 = deepcopy(input_table_mm) - tbl2 = deepcopy(input_table_mm) - - tbl1["y_mm"] -= 100 - tbl2["x_mm"] += 100 - tbl2["y_mm"] += 100 im_hdu = image_hdu_rect im_hdu.header["CRVAL1D"] -= 150 im_hdu.header["CRVAL2D"] += 20 - hdr = imp_utils.get_canvas_header([im_hdu, image_hdu_square, - tbl1, tbl2], pixel_scale=3*u.mm) + hdr = get_canvas_header([im_hdu, image_hdu_square], 3*u.mm) im = np.zeros((hdr["NAXIS2"], hdr["NAXIS1"])) canvas_hdu = fits.ImageHDU(header=hdr, data=im) - canvas_hdu = imp_utils.add_table_to_imagehdu(tbl1, canvas_hdu, - wcs_suffix="D") - canvas_hdu = imp_utils.add_table_to_imagehdu(tbl2, canvas_hdu, - wcs_suffix="D") - canvas_hdu = imp_utils.add_imagehdu_to_imagehdu(im_hdu, canvas_hdu, - wcs_suffix="D") - canvas_hdu = imp_utils.add_imagehdu_to_imagehdu(image_hdu_square, - canvas_hdu, - wcs_suffix="D") - - total_flux = np.sum(tbl1["flux"]) + np.sum(tbl2["flux"]) + \ - np.sum(im_hdu.data) + np.sum(image_hdu_square.data) + canvas_hdu = imp_utils.add_imagehdu_to_imagehdu( + im_hdu, canvas_hdu, wcs_suffix="D", + ) + canvas_hdu = imp_utils.add_imagehdu_to_imagehdu( + image_hdu_square, canvas_hdu, wcs_suffix="D", + ) + + total_flux = np.sum(im_hdu.data) + np.sum(image_hdu_square.data) assert np.sum(canvas_hdu.data) == approx(total_flux, rel=5e-3) if PLOTS: @@ -564,11 +284,10 @@ def test_mm_image_and_tables_on_large_canvas(self, input_table_mm, class TestImagePlaneAdd: - def test_simple_add_imagehdu_conserves_flux(self, image_hdu_square, - image_hdu_rect): - fields = [image_hdu_rect, image_hdu_square] - hdr = imp_utils.get_canvas_header(pixel_scale=1 * u.arcsec, - hdu_or_table_list=fields) + def test_simple_add_imagehdu_conserves_flux( + self, image_hdu_square, image_hdu_rect, + ): + hdr = get_canvas_header([image_hdu_rect, image_hdu_square]) orig_sum = image_hdu_rect.data.sum() @@ -598,11 +317,10 @@ def test_add_many_tables_and_imagehdus(self, image_hdu_rect, image_hdu_square): im_hdu.header["CRVAL1"] -= 150*u.arcsec.to(u.deg) im_hdu.header["CRVAL2"] += 20*u.arcsec.to(u.deg) - fields = [im_hdu, image_hdu_square] - hdr = imp_utils.get_canvas_header(fields, pixel_scale=1 * u.arcsec) + hdr = get_canvas_header([im_hdu, image_hdu_square]) implane = opt_imp.ImagePlane(hdr) - implane.add(fields) + implane.add([im_hdu, image_hdu_square]) total_flux = np.sum(im_hdu.data) + np.sum(image_hdu_square.data) assert np.sum(implane.data) == approx(total_flux) @@ -618,11 +336,12 @@ def test_add_many_tables_and_imagehdus(self, image_hdu_rect, image_hdu_square): plt.plot(x0, y0, "ro") plt.gca().set_aspect(1) - plt.imshow(implane.data, origin="lower", norm=LogNorm()) + plt.imshow(implane.data, origin="lower", norm="log") plt.show() - def test_add_many_mm_tables_and_imagehdus( - self, image_hdu_rect_mm, image_hdu_square_mm): + def test_add_many_mm_imagehdus( + self, image_hdu_rect_mm, image_hdu_square_mm, + ): image_hdu_rect = image_hdu_rect_mm image_hdu_square = image_hdu_square_mm @@ -630,10 +349,9 @@ def test_add_many_mm_tables_and_imagehdus( im_hdu.header["CRVAL1D"] -= 150 # mm im_hdu.header["CRVAL2D"] += 20 - fields = [im_hdu, image_hdu_square] - hdr = imp_utils.get_canvas_header(fields, pixel_scale=1*u.mm) + hdr = get_canvas_header([im_hdu, image_hdu_square], 1*u.mm) implane = opt_imp.ImagePlane(hdr) - implane.add(fields, wcs_suffix="D") + implane.add([im_hdu, image_hdu_square], wcs_suffix="D") total_flux = np.sum(im_hdu.data) + np.sum(image_hdu_square.data) assert np.sum(implane.data) == approx(total_flux) @@ -649,7 +367,7 @@ def test_add_many_mm_tables_and_imagehdus( plt.plot(x0, y0, "ro") plt.gca().set_aspect(1) - plt.imshow(implane.data, origin="lower", norm=LogNorm()) + plt.imshow(implane.data, origin="lower", norm="log") plt.show() @@ -725,7 +443,7 @@ def test_rescale_works_on_3d_imageplane(self, image_hdu_3d_data): ############################################################################### -# ..todo: When you have time, reintegrate these tests, There are some good ones +# TODO: When you have time, reintegrate these tests, There are some good ones class TestGetSpatialExtentOfHeader: def test_returns_right_sky_coords_from_known_coords(self, image_hdu_square): @@ -742,141 +460,24 @@ def test_returns_right_sky_coords_from_known_coords(self, image_hdu_square): class TestMakeImagePlaneHeader: - def test_header_contains_future_naxis_pixel_sizes(self, image_hdu_square, - image_hdu_rect): - hdr = imp_utils.get_canvas_header([image_hdu_square, image_hdu_rect]) + def test_header_contains_future_naxis_pixel_sizes( + self, image_hdu_square, image_hdu_rect, + ): + hdr = get_canvas_header([image_hdu_square, image_hdu_rect]) assert hdr["NAXIS1"] == 100 + 1 assert hdr["NAXIS2"] == 200 + 1 @pytest.mark.parametrize("offset", -np.random.randint(200, 1001, 10)) - def test_header_contains_spread_out_regions(self, offset, image_hdu_square, - image_hdu_rect): + def test_header_contains_spread_out_regions( + self, offset, image_hdu_square, image_hdu_rect, + ): image_hdu_rect.header["CRVAL1"] += offset # *u.arcsec.to(u.deg) - hdr = imp_utils.get_canvas_header([image_hdu_square, image_hdu_rect]) + hdr = get_canvas_header([image_hdu_square, image_hdu_rect]) image_width = image_hdu_square.header["NAXIS1"] // 2 + \ image_hdu_rect.header["NAXIS1"] // 2 + abs(offset) + 1 assert hdr["NAXIS1"] == image_width - @pytest.mark.parametrize("offset", [0, 10, 25, 50]) - def test_header_has_correct_size_based_on_table_extremes(self, offset, - input_table): - tbl1 = input_table - tbl2 = deepcopy(input_table) - tbl3 = deepcopy(input_table) - tbl2["x"] += offset - tbl3["y"] += offset - hdr = imp_utils.get_canvas_header([tbl1, tbl2, tbl3], - pixel_scale=0.1*u.arcsec) - - assert hdr["NAXIS1"] == np.max(tbl1["x"] + tbl2["x"]) * 10 + 3 # 4 - assert hdr["NAXIS2"] == np.max(tbl1["y"] + tbl3["y"]) * 10 + 3 # 4 - - # @pytest.mark.parametrize("pix_scl", [5, 1, 0.5, 0.1]) - # def test_header_has_correct_size_with_tbl_and_image_input(self, input_table, - # image_hdu_square, - # pix_scl): - # input_table["x"] += 100 - # hdr = imp_utils.get_canvas_header([image_hdu_square, input_table], - # pixel_scale=pix_scl * u.arcsec) - # assert hdr["NAXIS1"] == approx(hdr["CRPIX1"] + - # np.max(input_table["x"]) / pix_scl + 1, - # abs=0.5) - - def test_header_does_not_contain_negative_naxis_keywords(self): - pass - -# -# class TestGetCornerSkyCoordsFromTable: -# def test_table_with_column_units_returns_right_values(self): -# x, y = [1] * u.arcmin, [1] * u.arcmin -# tbl = Table(names=["x", "y"], data=[x, y]) -# xsky, ysky = imp_utils.get_corner_sky_coords_from_table(tbl) -# -# assert xsky[0]*u.deg == x[0].to(u.deg) -# assert ysky[0]*u.deg == y[0].to(u.deg) -# -# def test_table_with_meta_units_returns_right_values(self): -# x, y = [1], [1] -# tbl = Table(names=["x", "y"], data=[x, y]) -# tbl.meta.update({"x_unit": u.arcmin, "y_unit": u.arcmin}) -# xsky, ysky = impl_utils.get_corner_sky_coords_from_table(tbl) -# -# assert xsky[0] == x[0]*u.arcmin.to(u.deg) -# assert ysky[0] == y[0]*u.arcmin.to(u.deg) -# -# def test_table_with_default_units_returns_right_values(self): -# x, y = [60], [60] # because default unit is arcsec -# tbl = Table(names=["x", "y"], data=[x, y]) -# xsky, ysky = impl_utils.get_corner_sky_coords_from_table(tbl) -# -# assert pytest.approx(xsky[0] == x[0] * u.arcmin.to(u.deg)) -# assert pytest.approx(ysky[0] == y[0] * u.arcmin.to(u.deg)) -# -# -# class TestGetCornerSkyCoords: -# def test_returns_coords_for_combination_of_table_and_header(self, -# image_hdu_square): -# x, y = [-100, 70] * u.arcsec, [0, 0] * u.arcsec -# tbl = Table(names=["x", "y"], data=[x, y]) -# xsky, ysky = impl_utils.get_corner_sky_coords([tbl, image_hdu_square]) -# -# assert np.all((xsky == x.to_value(u.deg))) -# - - -class TestAddTableToImageHDU2: - @pytest.mark.parametrize("xpix, ypix, value", - [(51, 51, 2), - (48, 51, 3), - (48, 48, 4), - (51, 48, 5)]) - def test_integer_pixel_fluxes_are_added_correctly(self, xpix, ypix, value, - image_hdu_square): - # Given the weird behaviour on pixel boundaries - x, y = [1.5, -1.5, -1.5, 1.5]*u.arcsec, [1.5, 1.5, -1.5, -1.5]*u.arcsec - flux = [1, 2, 3, 4] * u.Unit("ph s-1") - tbl = Table(names=["x", "y", "flux"], data=[x, y, flux]) - - hdu = imp_utils.add_table_to_imagehdu(tbl, image_hdu_square, - sub_pixel=False) - - assert hdu.data[ypix, xpix] == value - - @pytest.mark.parametrize("x, y, flux, xpix, ypix, value", - [([0], [0], [1], 50, 50, .25), - ([0.7], [0.7], [1], 50, 50, 0.64), - ([-0.7], [-0.7], [1], 48, 48, 0.04), - ([5.5], [-4.7], [1], 55, 45, 0.8), - ([5.5], [-4.7], [1], 55, 44, 0.2)]) - def test_sub_pixel_fluxes_are_added_correctly(self, x, y, flux, xpix, ypix, - value, image_hdu_square): - # Given the weird behaviour on pixel boundaries - tbl = Table(names=["x", "y", "flux"], - data=[x*u.arcsec, y*u.arcsec, flux*u.Unit("ph s-1")]) - hdu = imp_utils.add_table_to_imagehdu(tbl, image_hdu_square, - sub_pixel=True) - - if PLOTS: - plt.imshow(hdu.data[45:55, 45:55], origin="lower") - plt.colorbar() - plt.show() - - assert hdu.data[ypix, xpix] == approx(value + 1) - - @pytest.mark.parametrize("x, y, flux", - [([100, -100], [0, 0], [10, 10])]) - def test_source_outside_canvas_are_ignored(self, x, y, flux, - image_hdu_square): - # Given the weird behaviour on pixel boundaries - tbl = Table(names=["x", "y", "flux"], - data=[x * u.arcsec, y * u.arcsec, - flux * u.Unit("ph s-1")]) - hdu = imp_utils.add_table_to_imagehdu(tbl, image_hdu_square, - sub_pixel=True) - - assert np.sum(hdu.data) == np.prod(hdu.data.shape) - class TestAddImagehduToImageHDU: @pytest.mark.parametrize("angle", [0, 30, 45, 89]) @@ -915,11 +516,10 @@ def test_throws_error_when_initialised_with_nothing(self): with pytest.raises(TypeError): opt_imp.ImagePlane() - def test_initialises_with_header_with_hdu(self, image_hdu_square, - image_hdu_rect): - hdr = imp_utils.get_canvas_header(pixel_scale=0.1 * u.arcsec, - hdu_or_table_list=[image_hdu_rect, - image_hdu_square]) + def test_initialises_with_header_with_hdu( + self, image_hdu_square, image_hdu_rect, + ): + hdr = get_canvas_header([image_hdu_rect, image_hdu_square]) implane = opt_imp.ImagePlane(hdr) assert isinstance(implane, opt_imp.ImagePlane) assert isinstance(implane.hdu, fits.ImageHDU) diff --git a/scopesim/tests/tests_optics/test_ImagePlaneUtils.py b/scopesim/tests/tests_optics/test_ImagePlaneUtils.py index 1832ec8e1..c0116087a 100644 --- a/scopesim/tests/tests_optics/test_ImagePlaneUtils.py +++ b/scopesim/tests/tests_optics/test_ImagePlaneUtils.py @@ -14,27 +14,6 @@ PLOTS = False -class TestSplitHeader: - def test_is_the_header_split_into_the_right_amount_of_chunks(self): - hdr = imp_utils.header_from_list_of_xy([-1.024, 1.024], - [-1.024, 1.024], 0.002) - hdrs = imp_utils.split_header(hdr, 128) - area_sum = np.sum([hdr["NAXIS1"] * hdr["NAXIS2"] for hdr in hdrs]) - assert len(hdrs) == 64 - assert area_sum == hdr["NAXIS1"] * hdr["NAXIS2"] - - @pytest.mark.parametrize("x, y, pix", [(0.19, 0.2, 0.01), - (2.19, 1.55, 0.01), - (1.29, 1.2, 0.02), - (2.55, 3.75, 1)]) - def test_final_header_is_smaller_for_odd_size(self, x, y, pix): - hdr = imp_utils.header_from_list_of_xy([-1., x], - [-1., y], pix) - hdrs = imp_utils.split_header(hdr, 100) - area_sum = np.sum([hdr["NAXIS1"] * hdr["NAXIS2"] for hdr in hdrs]) - assert area_sum == hdr["NAXIS1"] * hdr["NAXIS2"] - - class TestAddImageHDUtoImageHDU: def big_small_hdus(self, big_wh=(20, 10), big_offsets=(0, 0), small_wh=(6, 3), small_offsets=(0, 0), pixel_scale=0.1): diff --git a/scopesim/tests/tests_source/source_helpers.py b/scopesim/tests/tests_source/source_helpers.py new file mode 100644 index 000000000..1a92fb99a --- /dev/null +++ b/scopesim/tests/tests_source/source_helpers.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- +"""Aux function for source testing.""" + +import numpy as np +from scipy import integrate +from astropy import units as u +from synphot import SourceSpectrum, SpectralElement + + +# This used to be in source_utils, but was ultimately only used in a few tests, +# so moved here as a simple helper function. +def photons_in_range( + spectra: SourceSpectrum, + wave_min: u.Quantity[u.um] | float, + wave_max: u.Quantity[u.um] | float, + area: u.Quantity[u.m**2] | float | None = None, + bandpass: SpectralElement | None = None, +) -> u.Quantity[u.ph * u.s**-1 * u.m**-2] | u.Quantity[u.ph * u.s**-1]: + """ + Integrate photons from spectrum in given wavelength range. + + Parameters + ---------- + spectra : SourceSpectrum + Input spectrum. + wave_min : u.Quantity["length"] or float + Minimum wavelength. If float, assumes um. + wave_max : u.Quantity["length"] or float + Maximum wavelength. If float, assumes um. + area : u.Quantity["area"] or float, optional + Area to multiply with. If float, assumes m**2. The default is None. + bandpass : SpectralElement, optional + Filter to take into account, if any. The default is None. + + Returns + ------- + counts : astropy.units.Quantity + Either in ph/s/m**2 or just ph/s (if area was given). + + """ + # Note: Assuming um if given as float. + wave_min = (wave_min << u.um << u.Angstrom).value + wave_max = (wave_max << u.um << u.Angstrom).value + # Note: There appear to be some float shenanigans going on here, but + # rounding produces an error in the spectrum evaluation. Not sure what's + # going on, maybe it's fine as-is. + + counts = [] + for spec in spectra: + waveset = spec.waveset.value + mask = (waveset > wave_min) * (waveset < wave_max) + wave = np.array([wave_min, *waveset[mask], wave_max]) + flux = spec(wave).value + + # flux [ph s-1 cm-2] == flux [ph s-1 cm-2 AA-1] * wave [AA] + if isinstance(bandpass, SpectralElement): + bandpass.model.bounds_error = True + counts.append(integrate.trapezoid(bandpass(wave).value * flux, wave)) + else: + counts.append(integrate.trapezoid(flux, wave)) + + # counts = flux [ph s-1 cm-2] + counts = (counts * u.ph * u.s**-1 * u.cm**-2).to(u.ph * u.s**-1 * u.m**-2) + if area is not None: + counts *= (area << u.m**2) + + return counts + + +# This used to be a method of Source, but was ultimately only used in two tests, +# so moved here as a simple helper function. +def src_photons_in_range(src, wave_min, wave_max, area=None): + """ + + Parameters + ---------- + wave_min : float, u.Quantity + [um] + wave_max : float, u.Quantity + [um] + area : float, u.Quantity, optional + [m2] + + Returns + ------- + counts : u.Quantity list + [ph / s / m2] if area is None + [ph / s] if area is passed + + """ + indices = src.spectra.keys() + spectra = [src.spectra[ii] for ii in indices] + counts = photons_in_range(spectra, wave_min, wave_max, area=area, + bandpass=src.bandpass) + return counts diff --git a/scopesim/tests/tests_source/test_source_Source.py b/scopesim/tests/tests_source/test_source_Source.py index feea28e44..2f5b06fbe 100644 --- a/scopesim/tests/tests_source/test_source_Source.py +++ b/scopesim/tests/tests_source/test_source_Source.py @@ -1,8 +1,10 @@ -# actually for Source +# -*- coding: utf-8 -*- + import pytest from pytest import approx import numpy as np +import matplotlib.pyplot as plt from astropy.io import fits from astropy.io import ascii as ioascii @@ -10,7 +12,7 @@ from astropy import units as u from astropy import wcs -from synphot import SourceSpectrum, SpectralElement +from synphot import SourceSpectrum from synphot.models import Empirical1D from synphot.units import PHOTLAM @@ -18,14 +20,10 @@ from scopesim.source.source import Source from scopesim.source.source_fields import CubeSourceField, TableSourceField -from scopesim.optics.image_plane import ImagePlane from scopesim.utils import convert_table_comments_to_dict from scopesim.tests.mocks.py_objects import source_objects as so -import matplotlib.pyplot as plt -from matplotlib.colors import LogNorm - PLOTS = False @@ -261,83 +259,6 @@ def test_empty_source_is_the_additive_identity(self, image_source): assert len(new_source_2.fields) == len(new_source_2._meta_dicts) -class TestSourceImageInRange: - @pytest.mark.skip(reason=( - "This was the only place where the 'add table to ImagePlane' branch " - "was still used. Figure out what actually gets tested here and then " - "try to do it in another way." - )) - def test_returns_an_image_plane_object(self, table_source): - im = table_source.image_in_range(1*u.um, 2*u.um) - assert isinstance(im, ImagePlane) - - @pytest.mark.skip(reason=( - "This was the only place where the 'add table to ImagePlane' branch " - "was still used. Figure out what actually gets tested here and then " - "try to do it in another way." - )) - def test_flux_from_table_on_image_is_as_expected(self, table_source): - ph = table_source.photons_in_range(1*u.um, 2*u.um) - ref = table_source.fields[0].field["ref"] - weight = table_source.fields[0].field["weight"] - counts = np.sum([ph.value[r] * w for r, w in zip(ref, weight)]) - - im = table_source.image_in_range(1*u.um, 2*u.um) - assert np.sum(im.image) == approx(counts) - - @pytest.mark.parametrize("pix_scl", [0.1, 0.2, 0.4, 1.0]) - def test_flux_from_imagehdu_is_as_expected(self, image_source, pix_scl): - ph = image_source.photons_in_range(1*u.um, 2*u.um)[0].value - im_sum = ph * np.sum(image_source.fields[0].data) - im = image_source.image_in_range(1*u.um, 2*u.um, pix_scl*u.arcsec) - assert np.sum(im.image) == approx(im_sum) - - def test_combines_more_that_one_field_into_image(self, image_source, - table_source): - ph = table_source.photons_in_range(1 * u.um, 2 * u.um) - fld = table_source.fields[0] - tbl_sum = u.Quantity([ph[fld.field["ref"][ii]] * fld.field["weight"][ii] - for ii in range(len(fld.field))]) - tbl_sum = np.sum(tbl_sum.value) - - ph = image_source.photons_in_range(1 * u.um, 2 * u.um)[0] - im_sum = np.sum(image_source.fields[0].data) * ph.value - - comb_src = image_source - im_plane = comb_src.image_in_range(1*u.um, 2*u.um, 0.3*u.arcsec) - - assert np.sum(im_plane.image) == approx(im_sum) - - if PLOTS: - plt.imshow(im_plane.image.T, origin="lower", norm=LogNorm()) - plt.show() - - -class TestSourcePhotonsInRange: - def test_correct_photons_are_returned_for_table_source(self, table_source): - ph = table_source.photons_in_range(1, 2) - assert np.allclose(ph.value, [4., 2., 2.]) - - def test_correct_photons_are_returned_for_image_source(self, image_source): - ph = image_source.photons_in_range(1, 2) - assert np.allclose(ph.value, [2.]) - - def test_correct_photons_are_returned_for_no_spectra(self, image_source): - image_source.fields[0].spectra = {} - ph = image_source.photons_in_range(1, 2) - assert len(ph) == 0 - - @pytest.mark.parametrize("area, expected", [(None, 2), (1, 2), (10, 20)]) - def test_photons_increase_with_area(self, area, expected, image_source): - ph = image_source.photons_in_range(1, 2, area=area) - assert ph[0].value == approx(expected) - - def test_photons_returned_only_for_indices(self, table_source): - ph = table_source.photons_in_range(1, 2, indices=[0, 2]) - assert len(ph) == 2 - assert np.allclose(ph.value, [4, 2]) - - @pytest.mark.xfail class TestSourceShift: def test_that_it_does_what_it_should(self): @@ -350,65 +271,6 @@ def test_that_it_does_what_it_should(self): assert False -class TestPhotonsInRange: - @pytest.mark.parametrize("ii, n_ph", - [(0, 50), - (1, 8e12)]) - def test_returns_correct_number_of_photons_for_one_spectrum(self, ii, n_ph, - input_spectra): - spec = input_spectra[ii] - counts = source_utils.photons_in_range([spec], 1, 2) - assert np.isclose(counts.value, n_ph, rtol=2e-3) - - def test_returns_ones_for_unity_spectrum(self): - flux = np.ones(11) * u.Unit("ph s-1 m-2 um-1") - wave = np.linspace(1, 2, 11) * u.um - spec = SourceSpectrum(Empirical1D, points=wave, lookup_table=flux) - counts = source_utils.photons_in_range([spec], 1 * u.um, 2 * u.um) - assert counts.value == approx(1) - - @pytest.mark.parametrize("area, expected_units", - [(1*u.m**2, u.ph / u.s), - (None, u.ph / u.s / u.m**2)]) - def test_returns_correct_units_with_without_area_argument(self, area, - expected_units): - flux = np.ones(11) * u.Unit("ph s-1 m-2 um-1") - wave = np.linspace(1, 2, 11) * u.um - spec = SourceSpectrum(Empirical1D, points=wave, lookup_table=flux) - counts = source_utils.photons_in_range([spec], 1 * u.um, 2 * u.um, - area=area) - assert counts.unit == expected_units - - def test_returns_correct_half_flux_with_bandpass(self): - flux = np.ones(11) * u.Unit("ph s-1 m-2 um-1") - wave = np.linspace(0.5, 2.5, 11) * u.um - spec = SourceSpectrum(Empirical1D, points=wave, lookup_table=flux) - bandpass = SpectralElement(Empirical1D, - points=np.linspace(1, 2, 13)*u.um, - lookup_table=0.5 * np.ones(13)) - counts = source_utils.photons_in_range([spec], 1 * u.um, 2 * u.um, - bandpass=bandpass) - assert counts.value == approx(0.5) - - @pytest.mark.parametrize("flux, area, expected", - [(np.linspace(0, 1, 11), 1E4*u.cm**2, 0.25), - (np.linspace(0, 1, 11)**2, None, 0.13625), - (np.linspace(0, 1, 11)**0.5, 100, 34.931988)]) - def test_with_bandpass_and_area_returns_correct_value(self, flux, area, - expected): - flux *= u.Unit("ph s-1 m-2 um-1") - spec = SourceSpectrum(Empirical1D, - points=np.linspace(0.5, 2.5, 11) * u.um, - lookup_table=flux) - bandpass = SpectralElement(Empirical1D, - points=np.linspace(1, 2, 13)*u.um, - lookup_table=0.5 * np.ones(13)) - counts = source_utils.photons_in_range([spec], 1 * u.um, 2 * u.um, - bandpass=bandpass, - area=area) - assert counts.value == approx(expected) - - class TestSpectraListConverter: def test_works_for_arrays(self): spec = source_utils.convert_to_list_of_spectra( @@ -465,33 +327,3 @@ def test_throws_for_invalid_ref(): # Minimal Table tbl = Table(data=[[0, 1]], names=["ref"]) TableSourceField(tbl, {0: None}) - - -# -# class TestScaleImageHDU: -# def test_scaling_properly_for_si_photlam_in_header(self): -# hdu = fits.ImageHDU(data=np.ones((10,10))) -# hdu.header["CDELT1"] = 0.1 * u.arcsec.to(u.deg) -# hdu.header["CDELT2"] = 0.1 * u.arcsec.to(u.deg) -# hdu.header["BUNIT"] = "ph s-1 m-2 um-1" -# waverange = (1, 2)*u.um -# scaled_hdu = source2_utils.scale_imagehdu(hdu, waverange) -# assert scaled_hdu -# -# def test_scaling_properly_for_cgs_photlam_per_arcsec2_in_header(self): -# pass -# -# def test_scaling_properly_for_photlam_per_arcsec2_no_area_in_header(self): -# pass -# -# def test_scaling_properly_for_Jansky_in_header(self): -# pass -# -# def test_scaling_properly_for_Jansky_per_arcsec_in_header(self): -# pass -# -# def test_scaling_properly_for_photlam_and_bscale_in_header(self): -# pass -# -# def test_scaling_properly_for_photlam_and_bscale_bzero_in_header(self): -# pass From 019abf13822219f9c620c1595ac11de82b9ca48c Mon Sep 17 00:00:00 2001 From: teutoburg Date: Sun, 13 Sep 2026 22:16:51 +0200 Subject: [PATCH 051/130] Some more cleanup and removals --- docs/source/_ext/scopesim_sphinx_ext.py | 13 ++++++++--- scopesim/effects/effects_utils.py | 13 ++--------- scopesim/effects/ter_curves_utils.py | 1 + .../tests/tests_effects/test_effect_utils.py | 23 ++++--------------- 4 files changed, 17 insertions(+), 33 deletions(-) diff --git a/docs/source/_ext/scopesim_sphinx_ext.py b/docs/source/_ext/scopesim_sphinx_ext.py index 4ab146d1e..ea7957f8f 100644 --- a/docs/source/_ext/scopesim_sphinx_ext.py +++ b/docs/source/_ext/scopesim_sphinx_ext.py @@ -1,6 +1,13 @@ +# -*- coding: utf-8 -*- + +# TODO: What does this (rather clunky) Sphinx extension do that isn't covered +# by autodoc/autosummary?? If it's redundant, just get rid of it tbh. + import os import numpy as np -from scopesim.effects import effects_utils as eu + +# This function is only used here. If this extension is removed, rm the func. +from scopesim.effects.effects_utils import scopesim_effect_classes def setup(app): @@ -8,7 +15,7 @@ def setup(app): "../effects_docstrings")) os.makedirs(output_dir, exist_ok=True) - efs_dict = eu.scopesim_effect_classes() + efs_dict = scopesim_effect_classes() eff_types = np.unique([eff.split(".")[0] for eff in efs_dict]) eff_type_strs = {eff_type: f"{eff_type}\n{'='*len(eff_type)}\n\n" for eff_type in eff_types} @@ -37,7 +44,7 @@ def setup(app): :glob: * - + """ with open(os.path.join(output_dir, "index.rst"), "w") as f: diff --git a/scopesim/effects/effects_utils.py b/scopesim/effects/effects_utils.py index f9522ded1..f96dceeb9 100644 --- a/scopesim/effects/effects_utils.py +++ b/scopesim/effects/effects_utils.py @@ -1,10 +1,9 @@ +# -*- coding: utf-8 -*- """TBA.""" import inspect from collections.abc import Iterable -from astropy.table import Table - from .. import effects as efs from ..utils import get_logger @@ -51,15 +50,7 @@ def is_spectroscope(effects): return any(isinstance(eff, spec_classes) for eff in effects) -def empty_surface_list(**kwargs): - tbl = Table(names=["name", "outer", "inner", "angle", - "temperature", "action", "filename"], - data=[["test"], [0.], [0.], [0.], [0.], ["none"], ["none"]], - meta={"outer_unit": "m", "inner_unit": "m", - "angle_unit": "deg", "temperature_unit": "deg_C"}) - return efs.SurfaceList(table=tbl[:0], **kwargs) - - +# This is only used for the questionable sphinx extension. def scopesim_effect_classes(base_effect=efs.Effect): members = inspect.getmembers(efs) efs_dict = {".".join([cls.__module__, cls.__name__]).replace("scopesim.effects.", ""): cls diff --git a/scopesim/effects/ter_curves_utils.py b/scopesim/effects/ter_curves_utils.py index b7454e2c2..dd2f2f1c5 100644 --- a/scopesim/effects/ter_curves_utils.py +++ b/scopesim/effects/ter_curves_utils.py @@ -49,6 +49,7 @@ PATH_SVO_DATA = PATH_HERE.parent / "data" / "svo" +# This function is only used in the AnisoCADO PSF... def get_filter_effective_wavelength(filter_name): # TODO: This is technically stored in the SVO XML file as WavelengthEff... # (actually WavelengthMean, by definition of formula ...) diff --git a/scopesim/tests/tests_effects/test_effect_utils.py b/scopesim/tests/tests_effects/test_effect_utils.py index 6a2779116..8e2611cea 100644 --- a/scopesim/tests/tests_effects/test_effect_utils.py +++ b/scopesim/tests/tests_effects/test_effect_utils.py @@ -1,10 +1,10 @@ +# -*- coding: utf-8 -*- + import pytest from scopesim import effects as efs -from scopesim.effects import effects_utils as eu, GaussianDiffractionPSF, \ - SurfaceList -from scopesim.tests.mocks.py_objects.effects_objects import _surf_list, \ - _surf_list_empty, _filter_surface +from scopesim.effects import effects_utils as eu, GaussianDiffractionPSF + from scopesim.tests.mocks.py_objects.yaml_objects import _atmo_yaml_dict @@ -13,21 +13,6 @@ def atmo_yaml_dict(): return _atmo_yaml_dict() -@pytest.fixture(scope="function") -def surf_list(): - return _surf_list() - - -@pytest.fixture(scope="function") -def surf_list_empty(): - return _surf_list_empty() - - -@pytest.fixture(scope="function") -def filter_surface(): - return _filter_surface() - - class TestMakeEffect: def test_it_creates_an_effects_object(self, atmo_yaml_dict): effdic = atmo_yaml_dict["effects"][0] From 5d69ce98e52430460d3efe25e7639ee1e091049a Mon Sep 17 00:00:00 2001 From: oczoske Date: Mon, 14 Sep 2026 17:08:00 +0200 Subject: [PATCH 052/130] Ensure positive CDELT and pixel area --- scopesim/optics/fov.py | 1 + scopesim/utils.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/scopesim/optics/fov.py b/scopesim/optics/fov.py index 960baa107..4f37d75c1 100644 --- a/scopesim/optics/fov.py +++ b/scopesim/optics/fov.py @@ -355,6 +355,7 @@ def extract_area_from_imagehdu(self, imagehdu, corners): # up a duplicated or a dropped row/column. new_wcs = image_wcs.deepcopy() new_wcs.wcs.crpix = image_wcs.wcs.crpix - xy0p + new_wcs.wcs.cdelt = np.abs(new_wcs.wcs.cdelt) new_naxis = xy1p - xy0p logger.debug("orig image wcs: %s", image_wcs) logger.debug("new cutout wcs: %s", new_wcs) diff --git a/scopesim/utils.py b/scopesim/utils.py index a22c19ae7..62d506a52 100644 --- a/scopesim/utils.py +++ b/scopesim/utils.py @@ -556,6 +556,8 @@ def unit_includes_per_physical_type(unit, physical_type): def pixel_area(header: fits.Header) -> u.Quantity[u.arcsec**2]: """Calculate area covered by one pixel in arcsec**2 from header. + The pixel area is always positive. + .. versionadded:: 0.11.1 """ @@ -571,7 +573,7 @@ def pixel_area(header: fits.Header) -> u.Quantity[u.arcsec**2]: area = (header["CDELT1"] * u.Unit(header["CUNIT1"]) * header["CDELT2"] * u.Unit(header["CUNIT2"])) - return area.to(u.arcsec**2) + return np.abs(area.to(u.arcsec**2)) def has_needed_keywords(header, suffix=""): From cf8915d2d2f20b766e58bb0355bf5c3c268aced0 Mon Sep 17 00:00:00 2001 From: teutoburg Date: Mon, 7 Sep 2026 01:30:56 +0200 Subject: [PATCH 053/130] Improve randomness, refactor electronic effects - Add `ElectronicEffect` base class - Use composite random seed - Use `Detector.det_id` property everywhere --- scopesim/effects/electronic/__init__.py | 30 +- scopesim/effects/electronic/electrons.py | 62 +- scopesim/effects/electronic/noise.py | 550 +++++++++--------- scopesim/effects/electronic/pixels.py | 44 +- scopesim/optics/optical_train.py | 2 +- .../tests_effects/test_BasicReadoutNoise.py | 6 +- 6 files changed, 345 insertions(+), 349 deletions(-) diff --git a/scopesim/effects/electronic/__init__.py b/scopesim/effects/electronic/__init__.py index 5d10e0d42..abc4912e7 100644 --- a/scopesim/effects/electronic/__init__.py +++ b/scopesim/effects/electronic/__init__.py @@ -19,9 +19,37 @@ - InterPixelCapacitance - apply IPC kernel to detector readout """ -from ...utils import get_logger +from ...utils import get_logger, check_keys +from ...detector import Detector +from .. import Effect + logger = get_logger(__name__) + +class ElectronicEffect(Effect): + """Base class for electronic effects. + + This will eventually replace the 800-range zorder (or parts of it). + """ + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.meta.update(kwargs) + check_keys(self.meta, self.required_keys, action="error") + + def _apply_to_det(self, det: Detector) -> None: + """Subclasses can override if more params needed in call.""" + logger.debug("Apply %s to %s", self.display_name, det) + det.data = self(det.data) + + def apply_to(self, obj, **kwargs): + """See parent docstring.""" + if isinstance(obj, Detector): + self._apply_to_det(obj) + + return obj + + from .electrons import LinearityCurve, ADConversion, InterPixelCapacitance from .noise import (Bias, PoorMansHxRGReadoutNoise, BasicReadoutNoise, ShotNoise, DarkCurrent, PixelResponseNonUniformity) diff --git a/scopesim/effects/electronic/electrons.py b/scopesim/effects/electronic/electrons.py index b279df63e..c2bee25b2 100644 --- a/scopesim/effects/electronic/electrons.py +++ b/scopesim/effects/electronic/electrons.py @@ -15,14 +15,11 @@ from numpy.typing import ArrayLike, NDArray from scipy.signal import oaconvolve -from .. import Effect -from ...detector import Detector -from ...utils import figure_factory, check_keys -from ...utils import from_currsys -from . import logger +from ...utils import from_currsys, figure_factory +from . import ElectronicEffect, Detector, logger -class LinearityCurve(Effect): +class LinearityCurve(ElectronicEffect): """ Detector linearity effect. @@ -57,12 +54,6 @@ class LinearityCurve(Effect): report_plot_include: ClassVar[bool] = True report_table_include: ClassVar[bool] = False - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.meta.update(kwargs) - - check_keys(self.meta, self.required_keys, action="error") - def __call__(self, data: ArrayLike, ndit: int = 1) -> NDArray: incident = self.incident * ndit measured = self.measured * ndit @@ -80,13 +71,12 @@ def measured(self) -> NDArray: return self.table["measured"] return np.asarray(from_currsys(self.meta["measured"], self.cmds)) - def apply_to(self, obj, **kwargs): - if not isinstance(obj, Detector): - return obj + def _apply_to_det(self, det: Detector) -> None: + """Subclasses can override if more params needed in call.""" + logger.debug("Apply %s to %s", self.display_name, det) ndit = from_currsys(self.meta["ndit"], self.cmds) - obj.data = self(obj.data, ndit) - return obj + det.data = self(det.data, ndit) def plot(self, **kwargs): fig, ax = figure_factory() @@ -102,7 +92,7 @@ def plot(self, **kwargs): return fig -class InterPixelCapacitance(Effect): +class InterPixelCapacitance(ElectronicEffect): r"""Inter-pixel capacitance effect. The effect models cross-talk due to inter-pixel capacitance with @@ -156,8 +146,6 @@ class InterPixelCapacitance(Effect): def __init__(self, **kwargs): super().__init__(**kwargs) - - self.meta.update(kwargs) self.kernel = self._build_kernel(kwargs) def _build_kernel(self, params): @@ -183,14 +171,10 @@ def _build_kernel(self, params): ]) return kernel - def apply_to(self, det, **kwargs): - if not isinstance(det, Detector): - logger.debug("%s applied to %s", self.display_name, - det.__class__.__name__) - return det - + def _apply_to_det(self, det: Detector) -> None: + """Subclasses can override if more params needed in call.""" + logger.debug("Apply %s to %s", self.display_name, det) det.data = oaconvolve(det.data, self.kernel, mode="same") - return det def update(self, **kwargs): """Update the IPC kernel. @@ -228,7 +212,7 @@ def __str__(self): return msg -class ADConversion(Effect): +class ADConversion(ElectronicEffect): """Analogue-Digital Conversion effect. The effect applies the gain factor (electrons/ADU) to the detector readouts @@ -293,7 +277,7 @@ def _should_apply(self) -> bool: logger.warning("Cannot access cmds for ADConversion effect.") return True - # ..todo: need to deal with this case more realistically + # TODO: need to deal with this case more realistically # Is this still necessary? #if self.cmds.get("!OBS.autoexpset", False): # logger.info("DIT, NDIT determined by AutoExposure -> " @@ -325,14 +309,14 @@ def _get_gain(self, det_id: int) -> Real: f"{self.__class__.__name__}.meta['gain'] must be either " f"dict or float, but is {self.cmds['!DET.gain']}") - def apply_to(self, obj, **kwargs): - if not isinstance(obj, Detector): - return obj + def _apply_to_det(self, det: Detector) -> None: + """Subclasses can override if more params needed in call.""" + logger.debug("Apply %s to %s", self.display_name, det) new_dtype = self.meta["dtype"] # Apply gain - obj.data = self(obj.data, self._get_gain(obj.det_id)) + det.data = self(det.data, self._get_gain(det.det_id)) # Type-conversion wraps around input values that are higher or lower than # the respective maximum and minimum values of the new data type. Before @@ -341,14 +325,14 @@ def apply_to(self, obj, **kwargs): if np.issubdtype(new_dtype, np.integer): minval = np.iinfo(new_dtype).min maxval = np.iinfo(new_dtype).max - minvals_mask = obj.data < minval - maxvals_mask = obj.data > maxval + minvals_mask = det.data < minval + maxvals_mask = det.data > maxval if minvals_mask.any(): - obj.data[minvals_mask] = minval + det.data[minvals_mask] = minval logger.warning( f"Effect ADConversion: {minvals_mask.sum()} negative pixels") if maxvals_mask.any(): - obj.data[maxvals_mask] = maxval + det.data[maxvals_mask] = maxval logger.warning( f"Effect ADConversion: {maxvals_mask.sum()} saturated pixels") @@ -356,6 +340,4 @@ def apply_to(self, obj, **kwargs): # set to the modified data. It should be fine to simply re-assign the # data attribute, but just in case it's not... logger.debug("Applying digitization to dtype %s.", new_dtype) - obj.data = obj.data.astype(new_dtype) - - return obj + det.data = det.data.astype(new_dtype) diff --git a/scopesim/effects/electronic/noise.py b/scopesim/effects/electronic/noise.py index c149ed5f7..69d22a118 100644 --- a/scopesim/effects/electronic/noise.py +++ b/scopesim/effects/electronic/noise.py @@ -1,31 +1,33 @@ # -*- coding: utf-8 -*- """Any kinds of electronic or photonic noise.""" +from hashlib import sha256 from typing import ClassVar from collections.abc import Mapping from numbers import Real # matches int, float and all the numpy scalars import numpy as np from numpy.typing import ArrayLike, NDArray -from astropy.io import fits -from .. import Effect -from ...detector import Detector -from ...utils import from_currsys, figure_factory, check_keys -from . import logger +from ...utils import from_currsys, figure_factory +from . import ElectronicEffect, Detector, logger -class Bias(Effect): +# TODO: Potential refactoring in multiple effects here: +# The various __call__ now get a det_id, which is what the _get_... need +# for their lookup. Either pass the rng to __call__ instead of det_id, +# or call the _get_.. in there. Needs some consideration of outside +# callers, would they rather pass which one? And how about det_id = None? +# Also could refactor the _get_... (also elsewhere) to a abstract value +# plus the concrete key. + + +class Bias(ElectronicEffect): """Adds a constant bias level to readout.""" required_keys = {"bias"} z_order: ClassVar[tuple[int, ...]] = (855,) - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.meta.update(kwargs) - check_keys(self.meta, self.required_keys, action="error") - def __call__(self, data: ArrayLike) -> NDArray: return data + self.bias_level @@ -33,15 +35,169 @@ def __call__(self, data: ArrayLike) -> NDArray: def bias_level(self) -> float: return from_currsys(self.meta["bias"], self.cmds) - def apply_to(self, obj, **kwargs): - if not isinstance(obj, Detector): - return obj - obj.data = self(obj.data) - return obj +class DarkCurrent(ElectronicEffect): + """ + required: dit, ndit, value + """ + + required_keys = {"value", "dit", "ndit"} + z_order: ClassVar[tuple[int, ...]] = (830,) + + def __call__( + self, + data: ArrayLike, + dark_level: float, + dit: float, + ndit: int, + ) -> NDArray: + return data + dark_level * dit * ndit + + def _get_dark_level(self, det_id: int) -> float: + dark_level = float(from_currsys(self.meta["value"], self.cmds)) + if isinstance(dark_level, Real): + return dark_level + if isinstance(dark_level, Mapping): + return from_currsys(dark_level[det_id], self.cmds) + raise TypeError( + f"<{self.__class__.__name__}>.meta['value'] must be either " + f"dict-like or scalar number, but is {dark_level}." + ) + + def _apply_to_det(self, det: Detector) -> None: + logger.debug("Apply %s to %s", self.display_name, det) + + # Dark level needs detector meta so can't go into __call__() + dark_level = self._get_dark_level(det.det_id) + dit = from_currsys(self.meta["dit"], self.cmds) + ndit = from_currsys(self.meta["ndit"], self.cmds) + + det.data = self(det.data, dark_level, dit, ndit) + + def plot(self, det, **kwargs): + """Plot effect.""" + dit = from_currsys(self.meta["dit"], self.cmds) + ndit = from_currsys(self.meta["ndit"], self.cmds) + total_time = dit * ndit + times = np.linspace(0, 2*total_time, 10) + dtcr = self.apply_to(det) + dark_level = dtcr.data[0, 0] / total_time # just read one pixel + levels = dark_level * times + fig, ax = figure_factory() + ax.plot(times, levels, **kwargs) + ax.set_xlabel("time") + ax.set_ylabel("dark level") + return ax + +# TODO: Add tests for seed resolving, start with code from #975, but for +# basic_instrument, and work from there. +class RandomEffect(ElectronicEffect): + """Mixin class for Effects that need random seeds.""" -class PoorMansHxRGReadoutNoise(Effect): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.meta["random_seed"] = "!SIM.random.seed" # Default + + @property + def root_seed(self) -> int: + """Resolve root seed from cmds, or use value from effect kwargs.""" + return from_currsys(self.meta["random_seed"], self.cmds) + + @classmethod + def cls_seed(cls) -> int: + """Generate reproducible seed from hash of class name.""" + digest = sha256(cls.__name__.encode("utf-8")).digest() + return int.from_bytes(digest[:16], "little") + + @property + def readout_seed(self) -> int: + """Readout ID, or 0 if not found in cmds.""" + return self.cmds.get("!OBS.roid", 0) if self.cmds is not None else 0 + + @property + def random_seed(self) -> int | None: + """Composite random seed, or None.""" + seed = [self.readout_seed, self.cls_seed(), self.root_seed] + # TODO: Consider removing this if root seed is resolved upstream! + if None in seed: + return None + return seed + + def create_rng(self, det_id: int | None = None) -> np.random.Generator: + """ + Instantiate np.random.Generator using composite seed. + + Parameters + ---------- + det_id : int | None, optional + Detector ID. If None (the default), detector ID is not included in + the composite seed, meaning results will look identical on all + detectors passed to the effect in the same readout. + + Returns + ------- + np.random.Generator + New Generator instance. + + """ + if det_id is not None: + seed = [det_id, *self.random_seed] + else: + seed = self.random_seed + return np.random.default_rng(seed) + + def plot(self, det): + """Plot effect image.""" + detector = self.apply_to(det) + fig, ax = figure_factory() + ax.imshow(detector.data) + return ax + + def plot_hist(self, det, **kwargs): + """Plot effect histogram.""" + detector = self.apply_to(det) + fig, ax = figure_factory() + ax.hist(detector.data.flatten()) + return ax + + +class BasicReadoutNoise(RandomEffect): + """Readout noise computed as: ron * sqrt(NDIT).""" + + required_keys = {"noise_std", "ndit"} + z_order: ClassVar[tuple[int, ...]] = (811,) + + def __call__( + self, + data: ArrayLike, + ndit: int = 1, + det_id: int | None = None, + ) -> NDArray: + rng = self.create_rng(det_id) + return data + self._create_noise_frame(data.shape, rng, ndit) + + @property + def noise_std(self) -> float: + return from_currsys(self.meta["noise_std"], self.cmds) + + def _create_noise_frame( + self, + shape: tuple[int, ...], + rng: np.random.Generator, + ndit: int, + ) -> NDArray: + scale = self.noise_std * np.sqrt(float(ndit)) + return rng.normal(loc=0, scale=scale, size=shape) + + def _apply_to_det(self, det: Detector) -> None: + logger.debug("Apply %s to %s", self.display_name, det) + + ndit = from_currsys(self.meta["ndit"], self.cmds) + det.data = self(det.data, ndit) + + +class PoorMansHxRGReadoutNoise(BasicReadoutNoise): required_keys = {"noise_std", "n_channels", "ndit"} z_order: ClassVar[tuple[int, ...]] = (811,) report_plot_include: ClassVar[bool] = False @@ -54,60 +210,34 @@ def __init__(self, **kwargs): "read_fraction": 0.4, "line_fraction": 0.25, "channel_fraction": 0.05, - "random_seed": "!SIM.random.seed", } self.meta.update(params) self.meta.update(kwargs) - check_keys(self.meta, self.required_keys, action="error") - - def __call__(self, data: ArrayLike, ndit: int = 1) -> NDArray: - self.meta = from_currsys(self.meta, self.cmds) - - ron_frame = self._make_ron_frame(data.shape) - - stacked_ron_frame = np.zeros_like(ron_frame) - for i in range(ndit): - dx = np.random.randint(0, ron_frame.shape[1]) - dy = np.random.randint(0, ron_frame.shape[0]) - stacked_ron_frame += np.roll(ron_frame, (dy, dx), axis=(0, 1)) - - # TODO: this .T is ugly. Work out where things are getting switched and remove it! - return data + stacked_ron_frame.T - - @property - def noise_std(self) -> int: - return from_currsys(self.meta["noise_std"], self.cmds) - @property def n_channels(self) -> int: return from_currsys(self.meta["n_channels"], self.cmds) - @property - def random_seed(self) -> int: - return from_currsys(self.meta["random_seed"], self.cmds) - - @staticmethod - def _pseudo_random_field( - rng, - scale: float = 1., - size: tuple[int, ...] = (1024, 1024), + def _create_noise_frame( + self, + shape: tuple[int, ...], + rng: np.random.Generator, + ndit: int, ) -> NDArray: - n = 256 - image = np.zeros(size) - batch = rng.normal(loc=0, scale=scale, size=(2*n, 2*n)) - for y in range(0, size[1], n): - for x in range(0, size[0], n): - i, j = rng.integers(n, size=2) - dx, dy = min(size[0]-x, n), min(size[1]-y, n) - image[x:x+dx, y:y+dy] = batch[i:i+dx, j:j+dy] + ron_frame = self._make_ron_frame(rng, shape) + stacked_ron_frame = np.zeros_like(ron_frame) - return image + for i in range(ndit): + stacked_ron_frame += np.roll( + ron_frame, + rng.integers((0, 0), ron_frame.shape), + axis=(0, 1), + ) - def _make_ron_frame(self, shape: tuple[int, ...]) -> NDArray: - # TODO: Add yaml-settable seed here - rng = np.random.default_rng(self.random_seed) + return stacked_ron_frame + def _make_ron_frame(self, rng, shape: tuple[int, ...]) -> NDArray: + self.meta = from_currsys(self.meta, self.cmds) # TODO: Is this needed? channel_fraction = self.meta["channel_fraction"] line_fraction = self.meta["line_fraction"] pedestal_fraction = self.meta["pedestal_fraction"] @@ -136,78 +266,27 @@ def _make_ron_frame(self, shape: tuple[int, ...]) -> NDArray: axis=0, ) - return (pixel + line).T + channel[:shape[0]] - - def apply_to(self, det, **kwargs): - if not isinstance(det, Detector): - return det - - ndit = from_currsys(self.meta["ndit"], self.cmds) - det.data = self(det.data, ndit) - return det - - def plot(self, det, **kwargs): - """Plot effect image.""" - dtcr = self.apply_to(det) - fig, ax = figure_factory() - ax.imshow(dtcr.data, origin="lower") - - def plot_hist(self, det, **kwargs): - """Plot effect histogram.""" - dtcr = self.apply_to(det) - fig, ax = figure_factory() - ax.hist(dtcr.data.flatten()) - - -class BasicReadoutNoise(Effect): - """Readout noise computed as: ron * sqrt(NDIT).""" - - required_keys = {"noise_std", "ndit"} - z_order: ClassVar[tuple[int, ...]] = (811,) - - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.meta["random_seed"] = "!SIM.random.seed" - self.meta.update(kwargs) + return (pixel + line) + channel[:shape[0], None] - check_keys(self.meta, self.required_keys, action="error") - - def __call__(self, data: ArrayLike, ndit: int = 1) -> NDArray: - rng = np.random.default_rng(self.random_seed) - scale = self.noise_std * np.sqrt(float(ndit)) - return data + rng.normal(loc=0, scale=scale, size=data.shape) - - @property - def noise_std(self) -> float: - return from_currsys(self.meta["noise_std"], self.cmds) - - @property - def random_seed(self) -> int: - return from_currsys(self.meta["random_seed"], self.cmds) - - def apply_to(self, det, **kwargs): - if not isinstance(det, Detector): - return det - - ndit = from_currsys(self.meta["ndit"], self.cmds) - det.data = self(det.data, ndit) - return det - - def plot(self, det): - """Plot effect image.""" - dtcr = self.apply_to(det) - fig, ax = figure_factory() - ax.imshow(dtcr.data) + @staticmethod + def _pseudo_random_field( + rng, + scale: float = 1., + size: tuple[int, ...] = (1024, 1024), + ) -> NDArray: + n = 256 + image = np.zeros(size) + batch = rng.normal(loc=0, scale=scale, size=(2*n, 2*n)) + for y in range(0, size[1], n): + for x in range(0, size[0], n): + i, j = rng.integers(n, size=2) + dx, dy = min(size[0]-x, n), min(size[1]-y, n) + image[x:x+dx, y:y+dy] = batch[i:i+dx, j:j+dy] - def plot_hist(self, det, **kwargs): - """Plot effect histogram.""" - dtcr = self.apply_to(det) - fig, ax = figure_factory() - ax.hist(dtcr.data.flatten()) + return image -# TODO: Is this really a "noise" effect? Sounds more like "electrons" tbh. -class PixelResponseNonUniformity(Effect): +class PixelResponseNonUniformity(RandomEffect): """Pixel Response Non-Uniformity (PRNU). Models the fixed pattern of per-pixel gain variations across the detector @@ -223,8 +302,6 @@ class PixelResponseNonUniformity(Effect): prnu_std : float or dict Standard deviation of the per-pixel gain distribution. - prnu_seed : int, fixed - include: "!DET.include_prnu" Example @@ -236,49 +313,61 @@ class PixelResponseNonUniformity(Effect): class: PixelResponseNonUniformity kwargs: prnu_std: 0.001 - prnu_seed: 42 include: "!DET.include_prnu" """ - required_keys: ClassVar[set] = set() + required_keys: ClassVar[set] = {"prnu_std"} z_order: ClassVar[tuple[int, ...]] = (805,) def __init__(self, **kwargs): super().__init__(**kwargs) - self.meta.update(kwargs) self._gain_maps = {} # keyed by det_id - def apply_to(self, obj, **kwargs): - if not isinstance(obj, Detector): - return obj - - random_seed = from_currsys(self.meta.get("prnu_seed"), self.cmds) - det_id = obj.det_id - - prnu_std_meta = from_currsys(self.meta["prnu_std"], self.cmds) - if isinstance(prnu_std_meta, Mapping): - prnu_std = float(from_currsys(prnu_std_meta[det_id], self.cmds)) - elif isinstance(prnu_std_meta, Real): - prnu_std = float(prnu_std_meta) - else: - raise TypeError( - ".meta['prnu_std'] must be a float " - f"or a dict keyed by detector ID, got {type(prnu_std_meta)}" - ) - - shape = obj.data.shape + def __call__( + self, + data: ArrayLike, + prnu_std: float, + det_id: int | None = None, + ) -> NDArray: if det_id not in self._gain_maps: - rng = np.random.default_rng(random_seed) + # TODO: Should not use roid here??? + rng = self.create_rng(det_id) self._gain_maps[det_id] = rng.normal( - loc=1.0, scale=prnu_std, size=shape, + loc=1.0, scale=prnu_std, size=data.shape, ) - if self._gain_maps[det_id].shape != shape: - raise ValueError("gain map shape mismatch") + return data * self._gain_maps[det_id] - obj.data = obj.data * self._gain_maps[det_id] - return obj + @property + def random_seed(self) -> int | None: + """Composite random seed, or None. + + Override from base class to remove ROID, because PRNU should be + identical in one observation. + """ + seed = [self.cls_seed(), self.root_seed] + # TODO: Consider removing this if root seed is resolved upstream! + if None in seed: + return None + return seed + + def _get_prnu_std(self, det_id: int) -> float: + prnu_std = from_currsys(self.meta["prnu_std"], self.cmds) + if isinstance(prnu_std, Real): + return prnu_std + if isinstance(prnu_std, Mapping): + return from_currsys(prnu_std[det_id], self.cmds) + raise TypeError( + f"<{self.__class__.__name__}>.meta['value'] must be either " + f"dict-like or scalar number, but is {prnu_std}." + ) + + def _apply_to_det(self, det: Detector) -> None: + logger.debug("Apply %s to %s", self.display_name, det) + + prnu_std = self._get_prnu_std(det.det_id) + det.data = self(det.data, prnu_std) def plot(self, det_id=None): """Plot effect.""" @@ -291,19 +380,38 @@ def plot(self, det_id=None): im = ax.imshow(gain_map, origin="lower", aspect="auto", vmin=1 - dev, vmax=1 + dev) fig.colorbar(im, ax=ax, label="per-pixel gain") - return fig + return fig, ax -class ShotNoise(Effect): - z_order: ClassVar[tuple[int, ...]] = (820,) +class ShotNoise(RandomEffect): + """Poissonian photon noise. - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.meta["random_seed"] = "!SIM.random.seed" - self.meta.update(kwargs) + Notes + ----- - def __call__(self, data: ArrayLike) -> NDArray: - rng = np.random.default_rng(self.random_seed) + Numpy has a problem with generating Poisson distributions above certain + values. E.g. on linux, numpy.random.poisson(1e20) raises ValueError: lam + value too large. The value might be smaller on other (operating) systems. + + The poisson and normal distribution are basically the same + above ~100 counts: + poisson(x) ~= normal(mu=x, sigma=x**0.5) + + Therefore a limit of 1e7 is used, above which the Poisson distribution is + approximated with a normal distribution. + + Also, the normal distribution takes only 60% as long as the Poisson + distribution for large arrays. + + Special values should be handled with care: + - Negative values are mapped to 0; there cannot be negative flux. + - numpy.nan are implicitly passed through the normal distribution; + because the Poisson distribution cannot handle them. + """ + z_order: ClassVar[tuple[int, ...]] = (820,) + + def __call__(self, data: ArrayLike, det_id: int | None = None) -> NDArray: + rng = self.create_rng(det_id) # Check if there are negative values in the data. values_negative = data < 0 @@ -318,111 +426,9 @@ def __call__(self, data: ArrayLike) -> NDArray: # Apply a normal distribution to the high values. values_high = ~values_low - data[values_high] = rng.normal(data[values_high], np.sqrt(data[values_high])) - - return data - - @property - def random_seed(self) -> int: - return from_currsys(self.meta["random_seed"], self.cmds) - - def apply_to(self, det, **kwargs): - if not isinstance(det, Detector): - return det - - self.meta["random_seed"] = from_currsys( - self.meta["random_seed"], self.cmds) - - # numpy has a problem with generating Poisson distributions above - # certain values. E.g. on linux, numpy.random.poisson(1e20) raises - # ValueError: lam value too large - # The value might be smaller on other (operating) systems. - # - # The poisson and normal distribution are basically the same - # above ~100 counts: - # poisson(x) ~= normal(mu=x, sigma=x**0.5) - # - # Therefore a limit of 1e7 is used, above which the Poisson - # distribution is approximated with a normal distribution. - # - # Also, the normal distribution takes only 60% as long as the - # poisson distribution for large arrays. - # - # Special values should be handled with care: - # - Negative values are mapped to 0; there cannot be negative flux. - # - numpy.nan are implicitly passed through the normal distribution; - # because the Poisson distribution cannot handle them. - - det._hdu = fits.ImageHDU(header=det.header, data=self(det.data)) - return det - - def plot(self, det): - """Plot effect image.""" - dtcr = self.apply_to(det) - fig, ax = figure_factory() - ax.imshow(dtcr.data) - - def plot_hist(self, det, **kwargs): - """Plot effect histogram.""" - dtcr = self.apply_to(det) - fig, ax = figure_factory() - ax.hist(dtcr.data.flatten()) - - -class DarkCurrent(Effect): - """ - required: dit, ndit, value - """ - - required_keys = {"value", "dit", "ndit"} - z_order: ClassVar[tuple[int, ...]] = (830,) - - def __init__(self, **kwargs): - super().__init__(**kwargs) - check_keys(self.meta, self.required_keys, action="error") - - def __call__( - self, - data: np.ndarray, - dark_level: float, - dit: float, - ndit: int, - ) -> np.ndarray: - return data + dark_level * dit * ndit - - def _get_dark_level(self, det_id: int) -> float: - dark_level = float(from_currsys(self.meta["value"], self.cmds)) - if isinstance(dark_level, Real): - return dark_level - if isinstance(dark_level, Mapping): - return from_currsys(dark_level[det_id], self.cmds) - raise ValueError( - f"<{self.__class__.__name__}>.meta['value'] must be either " - f"dict-like or scalar number, but is {dark_level}." + data[values_high] = rng.normal( + loc=data[values_high], + scale=np.sqrt(data[values_high]), ) - def apply_to(self, obj, **kwargs): - if not isinstance(obj, Detector): - return obj - - # Dark level needs detector meta so can't go into __call__() - dark_level = self._get_dark_level(obj.det_id) - dit = from_currsys(self.meta["dit"], self.cmds) - ndit = from_currsys(self.meta["ndit"], self.cmds) - - obj.data = self(obj.data, dark_level, dit, ndit) - return obj - - def plot(self, det, **kwargs): - """Plot effect.""" - dit = from_currsys(self.meta["dit"], self.cmds) - ndit = from_currsys(self.meta["ndit"], self.cmds) - total_time = dit * ndit - times = np.linspace(0, 2*total_time, 10) - dtcr = self.apply_to(det) - dark_level = dtcr.data[0, 0] / total_time # just read one pixel - levels = dark_level * times - fig, ax = figure_factory() - ax.plot(times, levels, **kwargs) - ax.set_xlabel("time") - ax.set_ylabel("dark level") + return data diff --git a/scopesim/effects/electronic/pixels.py b/scopesim/effects/electronic/pixels.py index f97d1eb2c..e26b26e96 100644 --- a/scopesim/effects/electronic/pixels.py +++ b/scopesim/effects/electronic/pixels.py @@ -5,13 +5,11 @@ from numpy.typing import ArrayLike, NDArray -from .. import Effect -from ...detector import Detector -from ...utils import from_currsys, figure_factory, check_keys, real_colname -from .. import logger +from ...utils import from_currsys, figure_factory +from . import ElectronicEffect, Detector, logger -class ReferencePixelBorder(Effect): +class ReferencePixelBorder(ElectronicEffect): """Remove signal from reference pixels. Detectors often have a number of rows and columns around the edges masked. @@ -59,26 +57,20 @@ def __call__(self, data: ArrayLike, border: dict[int, int]) -> NDArray: data[:, -border[3]:] = 0 return data - def _get_border(self, det_meta) -> dict[int, int]: + def _get_border(self, det_id: int) -> dict[int, int]: if hasattr(self.meta["border"], "dic"): - dtcr_id = det_meta[real_colname("id", det_meta)] - return self.meta["border"].dic[dtcr_id] + return self.meta["border"].dic[det_id] if isinstance(self.meta["border"], list): return self.meta["border"] - raise ValueError( + raise TypeError( f"{self.__class__.__name__}.meta['border'] must be either " f"dict or list, but is {self.meta['border']}") - def apply_to(self, obj, **kwargs): - """Mask border pixels.""" - if not isinstance(obj, Detector): - logger.warning( - "ReferencePixelBorder: got non-detector object: %s", type(obj)) - return obj - + def _apply_to_det(self, det: Detector) -> None: + """Subclasses can override if more params needed in call.""" + logger.debug("Apply %s to %s", self.display_name, det) logger.info(f"Applying border {from_currsys(self.meta['border'])}") - obj.data = self(obj.data, self._get_border(obj.meta)) - return obj + det.data = self(det.data, self._get_border(det.det_id)) def plot(self, det, **kwargs): """Show the masked detector image.""" @@ -96,25 +88,11 @@ def __str__(self) -> str: return msg -class BinnedImageBase(Effect): +class BinnedImageBase(ElectronicEffect): """Base class for binning effects.""" z_order: ClassVar[tuple[int, ...]] = (870,) - def __init__(self, **kwargs): - super().__init__(**kwargs) - check_keys(self.meta, self.required_keys, action="error") - - def __call__(self, data: ArrayLike) -> NDArray: - raise NotImplementedError("Subclasses must implement this.") - - def apply_to(self, det, **kwargs): - if not isinstance(det, Detector): - return det - - det.data = self(det.data) - return det - class BinnedImage(BinnedImageBase): required_keys = {"bin_size"} diff --git a/scopesim/optics/optical_train.py b/scopesim/optics/optical_train.py index 8121670de..3046de244 100644 --- a/scopesim/optics/optical_train.py +++ b/scopesim/optics/optical_train.py @@ -486,7 +486,7 @@ def readout( # self.cmds["!OBS.dit"] = None # self.cmds["!OBS.ndit"] = None # TODO: This is still hacky but seems to work for now... - params = {"exptime": None} + params = {"exptime": None, "roid": roid} params.update(kwargs) if params["exptime"] is not None and params.get("dit") is None: params.update( diff --git a/scopesim/tests/tests_effects/test_BasicReadoutNoise.py b/scopesim/tests/tests_effects/test_BasicReadoutNoise.py index e119dfa29..1b0ac3e30 100644 --- a/scopesim/tests/tests_effects/test_BasicReadoutNoise.py +++ b/scopesim/tests/tests_effects/test_BasicReadoutNoise.py @@ -46,7 +46,8 @@ def test_stdev_increase_with_square_of_n(self, n): "pedestal_fraction": 0.3, "read_fraction": 0.4, }) - frames = np.array([ron._make_ron_frame((256, 256)) + rng = np.random.default_rng(42) + frames = np.array([ron._make_ron_frame(rng, (256, 256)) for _ in range(n)]) assert np.std(np.sum(frames, axis=0)) == approx(10*n**0.5, rel=0.3) @@ -59,5 +60,6 @@ def test_makes_frame_sizes_for_non_integer_n_channels(self, shape): "pedestal_fraction": 0.25, "read_fraction": 0.25, }) - frame = ron._make_ron_frame(shape) + rng = np.random.default_rng(42) + frame = ron._make_ron_frame(rng, shape) assert frame.shape > (0, 0) From 5a6f25bd6ac5759d0fbf913c4e48bd7cf87591f3 Mon Sep 17 00:00:00 2001 From: oczoske Date: Thu, 17 Sep 2026 12:39:38 +0200 Subject: [PATCH 054/130] Include dark current in AutoExposure --- scopesim/effects/electronic/exposure.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/scopesim/effects/electronic/exposure.py b/scopesim/effects/electronic/exposure.py index 0b20b698e..342e9ac67 100644 --- a/scopesim/effects/electronic/exposure.py +++ b/scopesim/effects/electronic/exposure.py @@ -102,8 +102,17 @@ def estimate_dit_ndit( "fill_frac", from_currsys(self.meta["fill_frac"], self.cmds) ) + try: + dark_current = kwargs.get( + "dark_current", + from_currsys(self.meta["dark_current"], self.cmds) + ) + except KeyError: + logger.warning("No dark current found for %s", + self.display_name) + dark_current = 0 - dit_nosat = fill_frac * full_well / image_plane_max + dit_nosat = fill_frac * full_well / (image_plane_max + dark_current) logger.info("Required DIT without saturation: %.3f s", dit_nosat) # np.ceil so that dit is at most what is required for fill_frac From bb15056aa6158008cf3dabf801e351367faa0a94 Mon Sep 17 00:00:00 2001 From: oczoske Date: Thu, 17 Sep 2026 13:42:41 +0200 Subject: [PATCH 055/130] A few more tests --- .../tests/tests_effects/test_AutoExposure.py | 45 ++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/scopesim/tests/tests_effects/test_AutoExposure.py b/scopesim/tests/tests_effects/test_AutoExposure.py index 2876de7bf..d836d78dc 100644 --- a/scopesim/tests/tests_effects/test_AutoExposure.py +++ b/scopesim/tests/tests_effects/test_AutoExposure.py @@ -32,6 +32,7 @@ def fixture_autoexposure(): """Instantiate an AutoExposure object""" return AutoExposure(fill_frac=0.75, full_well=1e5, + dark_current=0, mindit=0.011, exptime=None) @@ -40,6 +41,7 @@ class TestAutoExposure: def test_initialises_correctly(self): autoexposure = AutoExposure(fill_frac=0.75, full_well=1e5, + dark_current=0, mindit=0.011) assert isinstance(autoexposure, AutoExposure) @@ -102,7 +104,7 @@ def test_fill_frac_acts_correctly(self, imageplane): assert out_ndit_1 == fill_2 / fill_1 * out_ndit_2 @pytest.mark.xfail(reason=("This now fails because a user-specified DIT " - "and NDIT will now just go thourgh unmodified. " + "and NDIT will now just go through unmodified. " "Decide what to do with this test later...")) def test_exptime_specified_by_dit_ndit(self, autoexposure, imageplane): """ @@ -139,3 +141,44 @@ def test_exptime_at_least_mindit(self, imageplane): assert dit == mindit assert ndit == 1 + + def test_takes_dark_current_into_account(self, imageplane): + exptime = 3600 + autoexp_1 = AutoExposure(fill_frac=0.75, + full_well=1e6, + mindit=1.3, + dark_current=0) + autoexp_1.cmds["!OBS.exptime"] = exptime + autoexp_1.apply_to(imageplane) + dit1 = autoexp_1.cmds["!OBS.dit"] + + autoexp_2 = AutoExposure(fill_frac=0.75, + full_well=1e6, + mindit=1.3, + dark_current=1e4) + autoexp_2.cmds["!OBS.exptime"] = exptime + autoexp_2.apply_to(imageplane) + dit2 = autoexp_2.cmds["!OBS.dit"] + + assert dit2 < dit1 + + def test_no_dark_gives_zero_dark(self, imageplane): + exptime = 3600 + # no dark provided + autoexp_1 = AutoExposure(fill_frac=0.75, + full_well=1e6, + mindit=1.3) + autoexp_1.cmds["!OBS.exptime"] = exptime + autoexp_1.apply_to(imageplane) + dit1 = autoexp_1.cmds["!OBS.dit"] + + # zero dark explicitly provided + autoexp_2 = AutoExposure(fill_frac=0.75, + full_well=1e6, + mindit=1.3, + dark_current=0.) + autoexp_2.cmds["!OBS.exptime"] = exptime + autoexp_2.apply_to(imageplane) + dit2 = autoexp_2.cmds["!OBS.dit"] + + assert dit2 == dit1 From 3aabed0be597f05ac1ef980ee01821805d9346bf Mon Sep 17 00:00:00 2001 From: oczoske Date: Thu, 17 Sep 2026 14:00:43 +0200 Subject: [PATCH 056/130] test for warning issued --- scopesim/tests/tests_effects/test_AutoExposure.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/scopesim/tests/tests_effects/test_AutoExposure.py b/scopesim/tests/tests_effects/test_AutoExposure.py index d836d78dc..179d1419d 100644 --- a/scopesim/tests/tests_effects/test_AutoExposure.py +++ b/scopesim/tests/tests_effects/test_AutoExposure.py @@ -1,7 +1,8 @@ """Tests for Effect AutoExposure.""" +import logging + import pytest -from unittest.mock import patch from scopesim import UserCommands from scopesim.optics.image_plane import ImagePlane @@ -9,9 +10,10 @@ from scopesim.tests.mocks.py_objects.imagehdu_objects import _image_hdu_square -# pylint: disable=no-self-use, missing-class-docstring +# pylint: disable=missing-class-docstring # pylint: disable=missing-function-docstring +LOGGER = logging.getLogger(__name__) def _patched_cmds(exptime=1, dit=None, ndit=None): return UserCommands(properties={"!OBS.exptime": exptime, @@ -162,14 +164,16 @@ def test_takes_dark_current_into_account(self, imageplane): assert dit2 < dit1 - def test_no_dark_gives_zero_dark(self, imageplane): + def test_no_dark_gives_zero_dark(self, imageplane, caplog): exptime = 3600 # no dark provided autoexp_1 = AutoExposure(fill_frac=0.75, full_well=1e6, mindit=1.3) autoexp_1.cmds["!OBS.exptime"] = exptime + caplog.set_level(logging.WARNING) autoexp_1.apply_to(imageplane) + assert "No dark current found for" in caplog.text dit1 = autoexp_1.cmds["!OBS.dit"] # zero dark explicitly provided From 1239de591a717691aca3c5eb5f08df5b6000e619 Mon Sep 17 00:00:00 2001 From: oczoske Date: Thu, 17 Sep 2026 17:21:48 +0200 Subject: [PATCH 057/130] Cosmetics --- scopesim/effects/micado_ifu_trace_list.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/scopesim/effects/micado_ifu_trace_list.py b/scopesim/effects/micado_ifu_trace_list.py index cd2595b44..390407a06 100644 --- a/scopesim/effects/micado_ifu_trace_list.py +++ b/scopesim/effects/micado_ifu_trace_list.py @@ -94,12 +94,12 @@ def apply_to_fov(self, obj): ymin = spt.meta["fov"]["y_min"] ymax = spt.meta["fov"]["y_max"] - slicewcs = fovwcs.deepcopy() slicewcs.wcs.ctype = ["LINEAR", "LINEAR", slicewcs.wcs.ctype[2]] slicewcs.wcs.crpix[1] = (ny_slice + 1) / 2 + # todo: here we silently assume that ymin, ymax are in degrees slicewcs.wcs.crval[1] = (ymin + ymax) / 2 / 3600 slicewcs.wcs.cdelt[1] = (ymax - ymin) / ny_slice / 3600 slicewcs_spat = slicewcs.sub(2) @@ -112,6 +112,7 @@ def apply_to_fov(self, obj): slicecube = np.zeros((n_z, ny_slice, n_x)) for islice in range(n_z): + # todo: replace by explicit bilinear interpolation ifov = RectBivariateSpline(np.arange(n_y), np.arange(n_x), fovcube[islice], kx=1, ky=1) @@ -139,8 +140,6 @@ def apply_to_fov(self, obj): return obj - - def make_spectral_traces(self): """Make a spectral trace for each combination of order and aperture""" self.ext_data = self._file[0].header["EDATA"] @@ -157,15 +156,17 @@ def make_spectral_traces(self): for spslice, apid in enumerate(self.slicelist["id"]): specid = f"{row['description']}_{apid:02d}" - thistrace = MicadoIFUSpectralTrace(trace_hdu, self.slicelist, spslice, - **params) + thistrace = MicadoIFUSpectralTrace( + trace_hdu, + self.slicelist, + spslice, + **params) thistrace.meta["trace_id"] = specid spec_traces[specid] = thistrace self.spectral_traces = spec_traces - class MicadoIFUSpectralTrace(SpectralTrace): """SpectralTrace for the MICADO IFU""" @@ -187,8 +188,8 @@ def __init__(self, trace_tbl, aplist, spslice, cmds=None, **kwargs): self.meta["fov"] = self.fov_grid() lam = np.linspace(self.wave_min, self.wave_max, 1001) * u.um - off_y = aplist['offset'][spslice] - dlam = np.median(self.dlam_per_pix(lam)) / self.meta["pixsize"] * off_y + off_y = aplist['offset'][spslice] # offset on detector in mm + dlam = np.median(self.dlam_per_pix(lam)) * off_y / self.meta["pixsize"] self.xy2lam.posttransform = (det_offset, {"offset": -dlam}) self.xilam2x.pretransform_y = (det_offset, {"offset": -dlam}) self.xilam2y.pretransform_y = (det_offset, {"offset": -dlam}) From 4af9ce564d8bc2da15bae6d3f03416883b527524 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 18 Sep 2026 01:31:22 +0000 Subject: [PATCH 058/130] Bump soupsieve from 2.8.4 to 2.9 Bumps [soupsieve](https://github.com/facelessuser/soupsieve) from 2.8.4 to 2.9. - [Release notes](https://github.com/facelessuser/soupsieve/releases) - [Commits](https://github.com/facelessuser/soupsieve/compare/2.8.4...2.9) --- updated-dependencies: - dependency-name: soupsieve dependency-version: '2.9' dependency-type: indirect ... Signed-off-by: dependabot[bot] --- poetry.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index ebb77fc36..b7aff9557 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3797,14 +3797,14 @@ files = [ [[package]] name = "soupsieve" -version = "2.8.4" +version = "2.9.2" description = "A modern CSS selector implementation for Beautiful Soup." optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main", "dev", "docs"] files = [ - {file = "soupsieve-2.8.4-py3-none-any.whl", hash = "sha256:e7e6b0769c8f51ed59acab6e994b00621096cfb1c640a7509295987388fbaf65"}, - {file = "soupsieve-2.8.4.tar.gz", hash = "sha256:e121fd02e975c695e4e9e8774a5ee35d74714b59307868dcc5319ad2d9e3328e"}, + {file = "soupsieve-2.9.2-py3-none-any.whl", hash = "sha256:8089a26fd974ca7a1f30276d3d8492ab266ab15af581642dfe8aa162e0c1c823"}, + {file = "soupsieve-2.9.2.tar.gz", hash = "sha256:4a55d8cf158a9c2e587fa4922f1bbb91d68ac829e2d6f25403a85747c71daf74"}, ] [[package]] From d828994f5c53ea54f833625b4b0f75cdf5ab5f62 Mon Sep 17 00:00:00 2001 From: "astarvienna-bot[bot]" <217837468+myciapp[bot]@users.noreply.github.com> Date: Fri, 18 Sep 2026 09:16:45 +0000 Subject: [PATCH 059/130] Bumping version from 0.12.0b3.dev0 to 0.12.0b3.dev1 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4597b8c6e..3b247f257 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ScopeSim" -version = "0.12.0b3.dev0" +version = "0.12.0b3.dev1" description = "Generalised telescope observation simulator" license = {text = "GPL-3.0-or-later"} authors = [ From 0c30b870888a42c7fbe52df0a652419363a352f0 Mon Sep 17 00:00:00 2001 From: oczoske Date: Fri, 18 Sep 2026 12:43:21 +0200 Subject: [PATCH 060/130] import numpy --- scopesim/optics/image_plane.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scopesim/optics/image_plane.py b/scopesim/optics/image_plane.py index 9cda002b4..3ecba4d4f 100644 --- a/scopesim/optics/image_plane.py +++ b/scopesim/optics/image_plane.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- """Contains ``ImagePlane`` class.""" +import numpy as np from astropy.io import fits from astropy.wcs import WCS From 26c780f0811eac56b25101205a4fee6330a89eee Mon Sep 17 00:00:00 2001 From: oczoske Date: Fri, 18 Sep 2026 17:47:39 +0200 Subject: [PATCH 061/130] Defuse trace outside fov --- scopesim/effects/micado_ifu_trace_list.py | 6 +++++- scopesim/effects/spectral_trace_list_utils.py | 9 +++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/scopesim/effects/micado_ifu_trace_list.py b/scopesim/effects/micado_ifu_trace_list.py index 390407a06..f10342501 100644 --- a/scopesim/effects/micado_ifu_trace_list.py +++ b/scopesim/effects/micado_ifu_trace_list.py @@ -128,7 +128,11 @@ def apply_to_fov(self, obj): slicefov.cube = fits.ImageHDU(header=slicewcs.to_header(), data=slicecube) # slicefov.cube.writeto(f"slicefov_{sptid}.fits", overwrite=True) - slicefov.hdu = spt.map_spectra_to_focal_plane(slicefov) + try: + slicefov.hdu = spt.map_spectra_to_focal_plane(slicefov) + except ValueError as err: + logger.debug(err) + if slicefov.hdu is not None: sxmin = slicefov.hdu.header["XMIN"] sxmax = slicefov.hdu.header["XMAX"] diff --git a/scopesim/effects/spectral_trace_list_utils.py b/scopesim/effects/spectral_trace_list_utils.py index 2c00c98ad..b3a4d7576 100644 --- a/scopesim/effects/spectral_trace_list_utils.py +++ b/scopesim/effects/spectral_trace_list_utils.py @@ -175,6 +175,7 @@ def map_spectra_to_focal_plane(self, fov) -> fits.ImageHDU: where this image lies in the focal plane. """ logger.debug("Mapping %s", fov.trace_id) + # Initialise the image based on the footprint of the spectral # trace and the focal plane WCS wave_min = fov.meta["wave_min"].value # [um] @@ -188,8 +189,10 @@ def map_spectra_to_focal_plane(self, fov) -> fits.ImageHDU: xi_max=xi_max, ) - if xlim_mm is None: - raise ValueError("xlim_mm is None") + if xlim_mm is None or ylim_mm is None: + # Cases where there is no overlap betwen slit/trace and + # fov are acceptable. + return None fov_header = fov.header det_header = fov.detector_header @@ -531,6 +534,7 @@ def footprint(self, wave_min=None, wave_max=None, xi_min=None, xi_max=None): # Requested wavelenth range is entirely outside definition range: # no footprint if wave_min > np.max(wave_val) or wave_max < np.min(wave_val): + logger.info("Trace not in wavelength range") return None, None # Restrict to overlap of requested range and definition range @@ -559,6 +563,7 @@ def footprint(self, wave_min=None, wave_max=None, xi_min=None, xi_max=None): # Requested slit range is entirely outside definition range: # no footprint if xi_min > np.max(xi_val) or xi_max < np.min(xi_val): + logger.info("Slit outside range (spatial)") return None, None # Restrict to overlap of requested range and definition range From 00b4a93fc73bd96b0d98d6b4f750cdc74a48d014 Mon Sep 17 00:00:00 2001 From: oczoske Date: Fri, 18 Sep 2026 18:03:13 +0200 Subject: [PATCH 062/130] comments forgotten in #922 --- scopesim/effects/micado_ifu_trace_list.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scopesim/effects/micado_ifu_trace_list.py b/scopesim/effects/micado_ifu_trace_list.py index f10342501..42eaea14e 100644 --- a/scopesim/effects/micado_ifu_trace_list.py +++ b/scopesim/effects/micado_ifu_trace_list.py @@ -99,7 +99,8 @@ def apply_to_fov(self, obj): slicewcs.wcs.ctype = ["LINEAR", "LINEAR", slicewcs.wcs.ctype[2]] slicewcs.wcs.crpix[1] = (ny_slice + 1) / 2 - # todo: here we silently assume that ymin, ymax are in degrees + # TODO: here we silently assume that ymin, ymax are in arcsec + # and wcs in degrees slicewcs.wcs.crval[1] = (ymin + ymax) / 2 / 3600 slicewcs.wcs.cdelt[1] = (ymax - ymin) / ny_slice / 3600 slicewcs_spat = slicewcs.sub(2) @@ -112,7 +113,7 @@ def apply_to_fov(self, obj): slicecube = np.zeros((n_z, ny_slice, n_x)) for islice in range(n_z): - # todo: replace by explicit bilinear interpolation + # TODO: replace by explicit bilinear interpolation ifov = RectBivariateSpline(np.arange(n_y), np.arange(n_x), fovcube[islice], kx=1, ky=1) @@ -127,7 +128,7 @@ def apply_to_fov(self, obj): slicefov.meta["trace_id"] = sptid slicefov.cube = fits.ImageHDU(header=slicewcs.to_header(), data=slicecube) - # slicefov.cube.writeto(f"slicefov_{sptid}.fits", overwrite=True) + try: slicefov.hdu = spt.map_spectra_to_focal_plane(slicefov) except ValueError as err: @@ -143,7 +144,6 @@ def apply_to_fov(self, obj): obj.hdu = fits.ImageHDU(data=fovimage, header=obj.detector_header) return obj - def make_spectral_traces(self): """Make a spectral trace for each combination of order and aperture""" self.ext_data = self._file[0].header["EDATA"] From 8c8bdc3998d02002b52a52a2b3b56220d949fea4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 18 Sep 2026 20:55:58 +0000 Subject: [PATCH 063/130] Bump anyio from 3.7.1 to 4.14.2 Bumps [anyio](https://github.com/agronholm/anyio) from 3.7.1 to 4.14.2. - [Release notes](https://github.com/agronholm/anyio/releases) - [Commits](https://github.com/agronholm/anyio/compare/3.7.1...4.14.2) --- updated-dependencies: - dependency-name: anyio dependency-version: 4.14.2 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- poetry.lock | 28 +++++++--------------------- 1 file changed, 7 insertions(+), 21 deletions(-) diff --git a/poetry.lock b/poetry.lock index b7aff9557..eaa19d9ef 100644 --- a/poetry.lock +++ b/poetry.lock @@ -50,24 +50,22 @@ numpy = ">=2.2.6,<3.0.0" [[package]] name = "anyio" -version = "3.7.1" -description = "High level compatibility layer for multiple asynchronous event loop implementations" +version = "4.15.1" +description = "High-level concurrency and networking framework on top of asyncio or Trio" optional = false -python-versions = ">=3.7" +python-versions = ">=3.10" groups = ["main", "dev"] files = [ - {file = "anyio-3.7.1-py3-none-any.whl", hash = "sha256:91dee416e570e92c64041bd18b900d1d6fa78dff7048769ce5ac5ddad004fbb5"}, - {file = "anyio-3.7.1.tar.gz", hash = "sha256:44a3c9aba0f5defa43261a8b3efb97891f2bd7d804e0e1f56419befa1adfc780"}, + {file = "anyio-4.15.1-py3-none-any.whl", hash = "sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101"}, + {file = "anyio-4.15.1.tar.gz", hash = "sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94"}, ] [package.dependencies] idna = ">=2.8" -sniffio = ">=1.1" +typing_extensions = {version = ">=4.16.0", markers = "python_version < \"3.15\""} [package.extras] -doc = ["Sphinx", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme (>=1.2.2)", "sphinxcontrib-jquery"] -test = ["anyio[trio]", "coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "mock (>=4) ; python_version < \"3.8\"", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17) ; python_version < \"3.12\" and platform_python_implementation == \"CPython\" and platform_system != \"Windows\""] -trio = ["trio (<0.22)"] +trio = ["trio (>=0.32.0)"] [[package]] name = "appnope" @@ -3771,18 +3769,6 @@ pyyaml = ">=6.0.3,<7.0.0" [package.extras] syn = ["synphot (>=1.7.0,<2.0.0)"] -[[package]] -name = "sniffio" -version = "1.3.0" -description = "Sniff out which async library your code is running under" -optional = false -python-versions = ">=3.7" -groups = ["main", "dev"] -files = [ - {file = "sniffio-1.3.0-py3-none-any.whl", hash = "sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384"}, - {file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"}, -] - [[package]] name = "snowballstemmer" version = "2.2.0" From 00bc44329f472902005470ea038f7a1909f0fadb Mon Sep 17 00:00:00 2001 From: "astarvienna-bot[bot]" <217837468+myciapp[bot]@users.noreply.github.com> Date: Fri, 18 Sep 2026 21:23:18 +0000 Subject: [PATCH 064/130] Bumping version from 0.12.0b3.dev1 to 0.12.0b3.dev2 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3b247f257..81028dc13 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ScopeSim" -version = "0.12.0b3.dev1" +version = "0.12.0b3.dev2" description = "Generalised telescope observation simulator" license = {text = "GPL-3.0-or-later"} authors = [ From 19450cfa85877584fb713e6a16908da625cf0d28 Mon Sep 17 00:00:00 2001 From: teutoburg Date: Sat, 19 Sep 2026 22:01:32 +0200 Subject: [PATCH 065/130] Bumping version from 0.12.0b3.dev2 to 0.12.0rc0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 81028dc13..4651c5530 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ScopeSim" -version = "0.12.0b3.dev2" +version = "0.12.0rc0" description = "Generalised telescope observation simulator" license = {text = "GPL-3.0-or-later"} authors = [ From 5cafd646f17439ada8dd887dbf763c4f6be27de5 Mon Sep 17 00:00:00 2001 From: "astarvienna-bot[bot]" <217837468+myciapp[bot]@users.noreply.github.com> Date: Sat, 19 Sep 2026 20:02:10 +0000 Subject: [PATCH 066/130] Replaced next version placeholders with 0.12.0 --- scopesim/effects/mosaic_trace_list.py | 2 +- scopesim/effects/psfs/discrete.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scopesim/effects/mosaic_trace_list.py b/scopesim/effects/mosaic_trace_list.py index 6f3083304..34e6bb66e 100644 --- a/scopesim/effects/mosaic_trace_list.py +++ b/scopesim/effects/mosaic_trace_list.py @@ -239,7 +239,7 @@ class MosaicOutputFormat(MosaicSpectralTraceList): "image": The output is a (pseudo-)detector image where the spectrum for each fibre occupies a row. - .. versionadded:: PLACEHOLDER_NEXT_RELEASE_VERSION + .. versionadded:: 0.12.0 """ required_keys = {"filename", "format"} diff --git a/scopesim/effects/psfs/discrete.py b/scopesim/effects/psfs/discrete.py index 3e85adf9c..64d852efc 100644 --- a/scopesim/effects/psfs/discrete.py +++ b/scopesim/effects/psfs/discrete.py @@ -172,7 +172,7 @@ class FieldConstantPSF(DiscretePSF): Fixed handling of background level and rounded edges to avoid visible "squares" in the image. - .. versionchanged:: PLACEHOLDER_NEXT_RELEASE_VERSION + .. versionchanged:: 0.12.0 Interpolation spline order can now be set via "!SIM.psf.interp_order". From 3ace6b3d7473a804048c85ddd623b2e0dbb6ba68 Mon Sep 17 00:00:00 2001 From: teutoburg Date: Sat, 19 Sep 2026 22:12:56 +0200 Subject: [PATCH 067/130] Bumping version from 0.12.0rc0 to 0.12.0rc1 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4651c5530..74e457216 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ScopeSim" -version = "0.12.0rc0" +version = "0.12.0rc1" description = "Generalised telescope observation simulator" license = {text = "GPL-3.0-or-later"} authors = [ From e0e8357be9a7a3573f117917bb28d664dc5daf93 Mon Sep 17 00:00:00 2001 From: "Fabian H." <73600109+teutoburg@users.noreply.github.com> Date: Mon, 21 Sep 2026 12:56:00 +0200 Subject: [PATCH 068/130] Remove no longer needed FOV logging --- scopesim/optics/fov.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/scopesim/optics/fov.py b/scopesim/optics/fov.py index 4f37d75c1..6ed2d2e74 100644 --- a/scopesim/optics/fov.py +++ b/scopesim/optics/fov.py @@ -764,9 +764,6 @@ def _make_imagefields(self, fov_waveset, bin_widths): * yield image to be added to canvas image """ for field in self._get_image_fields(): - logger.debug("2D FOV make_imagefields: field.data.sum() = %f %s", - field.data.sum(), field.bunit) - field_hdu = field.field.copy() if field.is_bunit_spatially_differential: @@ -785,9 +782,6 @@ def _make_imagefields(self, fov_waveset, bin_widths): # Rescale 2D flux weight map by integrated flux from spectrum field_hdu.data *= flux # ph s-1 - logger.info( - "2D FOV make_imagefields: field_hdu.data.mean() = %f ph/s", - field_hdu.data.mean()) yield field_hdu def _make_tablefields(self, fov_waveset, bin_widths, use_photlam=False): From ebd042b11db8b6a1767f364c8705d602c52a9c82 Mon Sep 17 00:00:00 2001 From: "Fabian H." <73600109+teutoburg@users.noreply.github.com> Date: Mon, 21 Sep 2026 13:09:00 +0200 Subject: [PATCH 069/130] Add 'prerelease' option to versioning in workflow --- .github/workflows/prepare_release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/prepare_release.yml b/.github/workflows/prepare_release.yml index 705405e4f..ba82cfc6f 100644 --- a/.github/workflows/prepare_release.yml +++ b/.github/workflows/prepare_release.yml @@ -10,6 +10,7 @@ on: - patch - minor - major + - prerelease - prepatch - preminor - premajor From b535b1384440b2664766dd525fd49d80e9868334 Mon Sep 17 00:00:00 2001 From: "astarvienna-bot[bot]" <217837468+myciapp[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:10:04 +0000 Subject: [PATCH 070/130] Bumping version from 0.12.0rc1 to 0.12.0rc2 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 74e457216..7942931bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ScopeSim" -version = "0.12.0rc1" +version = "0.12.0rc2" description = "Generalised telescope observation simulator" license = {text = "GPL-3.0-or-later"} authors = [ From 338356ce0fb5e4ed5002cef5dd5df3d74cbd6a9e Mon Sep 17 00:00:00 2001 From: "astarvienna-bot[bot]" <217837468+myciapp[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:12:51 +0000 Subject: [PATCH 071/130] Bumping version from 0.12.0rc2 to 0.12.0rc3.dev0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7942931bc..ff545c329 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ScopeSim" -version = "0.12.0rc2" +version = "0.12.0rc3.dev0" description = "Generalised telescope observation simulator" license = {text = "GPL-3.0-or-later"} authors = [ From 7f5d53ec0908a79d7a3a70084335864f88479643 Mon Sep 17 00:00:00 2001 From: "Fabian H." <73600109+teutoburg@users.noreply.github.com> Date: Wed, 23 Sep 2026 20:04:59 +0200 Subject: [PATCH 072/130] Add release notes for v0.12.0 to CHANGELOG --- CHANGELOG.md | 97 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 013c8fa8c..e2df097a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,100 @@ +# Version 0.12.0 +**2026-09-24** + +Large release with many important bug fixes centered around flux conservation and pixel shifts. + +> [!IMPORTANT] +> The minimum required Python version for this package is now **3.12** (see Dependency Changes). + +## Feature highlights +### General +- Improved handling of random seeds. + Nosie can now vary per readout (needs different `roid`) and is fully reproducible. + For instruments (and modes) with multiple detectors, the random seed varies per detector. +- PSF interpolation was made more efficient in some cases, and the interpolation order can now be specified. + +### MICADO +- Calibration mode added (see IRDB). +- IFU and offset slits implemented. + +### MOSAIC +- Tabular output format for the mIFU mode added. + +> [!WARNING] +> We expect backwards incompatibility, so we strongly recommend updating your IRDB packages as well. + +## What's Changed +### API Changes +* Reduce `OpticalTrain.write_header()` to the bare minimum by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/935 +* PSF interpolation order by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/950 +* Fully remove deprecated functionality by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/963 +* AtmoLibraryTERCurve works with parameters other than pwv by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/972 +* Make laser parameters configurable; utility function for tunable laser by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/970 +### Bugs fixed +* Catch wrong inputs to `UserCommands` by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/934 +* Dark value can be given as number without floating point by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/941 +* Remove reflectance factor from integrating sphere background by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/962 +* Fix scalar Quantity pattern matching by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/964 +* Fix flux loss, aliasing, and placement errors in image-plane resampling and overlay by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/958 +* Fix NameError in `SpectralTrace.plot` when `plot_trace_id` is used without `plot_outline` by @astronomyk in https://github.com/AstarVienna/ScopeSim/pull/981 +* Fix `XiLamImage` overwriting the cunit of its primary WCS by @astronomyk in https://github.com/AstarVienna/ScopeSim/pull/982 +* Skip empty FOVs by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1001 +* Fix transposed axes in `make_image_interpolations` by @astronomyk in https://github.com/AstarVienna/ScopeSim/pull/984 +* Fix 2D image rescaling by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1016 +* Fix yet another off-by-one bug by @astronomyk in https://github.com/AstarVienna/ScopeSim/pull/1013 +* SPEC scaling now also fixed by @janusbrink in https://github.com/AstarVienna/ScopeSim/pull/1018 +* Set INHERIT keyword and fix unit to FITS standard by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/1022 +* Fix indexing in FoV extraction by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1024 +### Changes to or addition of Effects +* MOSAIC mIFU: tabular output by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/968 +* Refactor `PSF.apply_to()` by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1002 +* Refactor `TERCurve.apply_to()` by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1004 +* Refactor `.apply_to()` in electronic effects by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1006 +* Apply throughput in-place for speed by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/1017 +* `AutoExposure` to take dark current into account by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/1031 +* Improve randomness, refactor electronic effects by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1030 +* Define slit transforms for image slicer IFUs and offset slits by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/922 +### New Features or Improvements +* Use optimal inverse transform for rectification by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/961 +* Skip PSF convolution for exactly-uniform fields by @astronomyk in https://github.com/AstarVienna/ScopeSim/pull/973 +* More useful error when download fails by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/990 +* Allow readout ID, fix Path support by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1020 +* Resolve random seed in `.observe()` by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1021 +### Dependency Changes +* Bump the dev-dependencies group with 3 updates by @dependabot[bot] in https://github.com/AstarVienna/ScopeSim/pull/940 +* Switch from httpx to httpxyz by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/924 +* Bump various dependencies by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/946 +* Bump astar-utils from 0.5.3 to 0.6.0 by @dependabot[bot] in https://github.com/AstarVienna/ScopeSim/pull/951 +* Bump actions/checkout from 6 to 7 by @dependabot[bot] in https://github.com/AstarVienna/ScopeSim/pull/947 +* Bump various dependencies by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/954 +* Bump actions/setup-python from 6 to 7 by @dependabot[bot] in https://github.com/AstarVienna/ScopeSim/pull/960 +* Drop support for Python 3.10 and 3.11 by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/969 +* Bump matplotlib and skycalc-ipy by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/977 +* Bump tornado from 6.5.7 to 6.5.8 by @dependabot[bot] in https://github.com/AstarVienna/ScopeSim/pull/1011 +* Bump tqdm from 4.68.4 to 4.70.0 by @dependabot[bot] in https://github.com/AstarVienna/ScopeSim/pull/1009 +* Bump the dev-dependencies group across 1 directory with 4 updates by @dependabot[bot] in https://github.com/AstarVienna/ScopeSim/pull/1008 +* Bump soupsieve from 2.8.4 to 2.9 by @dependabot[bot] in https://github.com/AstarVienna/ScopeSim/pull/1032 +* Bump anyio from 3.7.1 to 4.14.2 by @dependabot[bot] in https://github.com/AstarVienna/ScopeSim/pull/1033 +### Other Changes +* Add prepare-release workflow from elsewhere by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/971 +* Remove obsolete logic to deal with unversioned versions by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/979 +* Make `Transform2D` per-call transform overrides non-persistent by @astronomyk in https://github.com/AstarVienna/ScopeSim/pull/983 +* Cache the detector layout file in `MetisLMSSpectralTraceList` to avoid reading the file N*28 times for every `OpticalTrain` usage by @astronomyk in https://github.com/AstarVienna/ScopeSim/pull/987 +* Improve handling of spectral trace mapping and rectification by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/999 +* Cleanup PSF by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1003 +* Use correct NumPy type hints by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1005 +* Mark test_python_image_coords as webtest, because face is downloaded. by @hugobuddel in https://github.com/AstarVienna/ScopeSim/pull/1012 +* Cleanup in `TER_curve_utils` etc. by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1014 +* Add `Detector.det_id` convenience property by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1015 +* Formatting and cleanup by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1019 +* Set mpl backend for tests by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1026 +* Skip all tests in test_rst_utils.py by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1025 +* Remove unused `fov_grid()` methods by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1027 +* SimCADO corpse removal by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1029 + +**Full Changelog**: https://github.com/AstarVienna/ScopeSim/compare/v0.11.4...v0.12.0 + + # Version 0.11.4 **2026-05-20** From bd9ddcf7bb8ccc197132409e12d2df8d88f98049 Mon Sep 17 00:00:00 2001 From: oczoske Date: Thu, 24 Sep 2026 09:01:38 +0200 Subject: [PATCH 073/130] Update CHANGELOG.md --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2df097a6..d6b92f39f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ Large release with many important bug fixes centered around flux conservation an ## Feature highlights ### General - Improved handling of random seeds. - Nosie can now vary per readout (needs different `roid`) and is fully reproducible. + Noise can now vary per readout (needs different `roid`) and is fully reproducible. For instruments (and modes) with multiple detectors, the random seed varies per detector. - PSF interpolation was made more efficient in some cases, and the interpolation order can now be specified. @@ -29,7 +29,6 @@ Large release with many important bug fixes centered around flux conservation an * PSF interpolation order by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/950 * Fully remove deprecated functionality by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/963 * AtmoLibraryTERCurve works with parameters other than pwv by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/972 -* Make laser parameters configurable; utility function for tunable laser by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/970 ### Bugs fixed * Catch wrong inputs to `UserCommands` by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/934 * Dark value can be given as number without floating point by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/941 @@ -47,6 +46,7 @@ Large release with many important bug fixes centered around flux conservation an * Fix indexing in FoV extraction by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1024 ### Changes to or addition of Effects * MOSAIC mIFU: tabular output by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/968 +* METIS WCU: Make laser parameters configurable; utility function for tunable laser by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/970 * Refactor `PSF.apply_to()` by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1002 * Refactor `TERCurve.apply_to()` by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1004 * Refactor `.apply_to()` in electronic effects by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1006 From 80c93cbfb7054efbfb6ab2ef6d2ed18cb5574a3f Mon Sep 17 00:00:00 2001 From: "astarvienna-bot[bot]" <217837468+myciapp[bot]@users.noreply.github.com> Date: Thu, 24 Sep 2026 12:34:47 +0000 Subject: [PATCH 074/130] Bumping version from 0.12.0rc3.dev0 to 0.12.0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ff545c329..4b22f7aa7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ScopeSim" -version = "0.12.0rc3.dev0" +version = "0.12.0" description = "Generalised telescope observation simulator" license = {text = "GPL-3.0-or-later"} authors = [ From 4ed185bd98606122aa5d1bdc4552c76ca1f0ee84 Mon Sep 17 00:00:00 2001 From: "astarvienna-bot[bot]" <217837468+myciapp[bot]@users.noreply.github.com> Date: Thu, 24 Sep 2026 12:35:36 +0000 Subject: [PATCH 075/130] Bumping version from 0.12.0 to 0.12.1.dev0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4b22f7aa7..61752913c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ScopeSim" -version = "0.12.0" +version = "0.12.1.dev0" description = "Generalised telescope observation simulator" license = {text = "GPL-3.0-or-later"} authors = [ From a8f524fb33a147d8373223db1bce754e980e92fd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 15:44:47 +0000 Subject: [PATCH 076/130] Bump tqdm from 4.70.0 to 4.70.1 in the main-dependencies group Bumps the main-dependencies group with 1 update: [tqdm](https://github.com/tqdm/tqdm). Updates `tqdm` from 4.70.0 to 4.70.1 - [Release notes](https://github.com/tqdm/tqdm/releases) - [Commits](https://github.com/tqdm/tqdm/compare/v4.70.0...v4.70.1) --- updated-dependencies: - dependency-name: tqdm dependency-version: 4.70.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: main-dependencies ... Signed-off-by: dependabot[bot] --- poetry.lock | 26 +++++++++++++------------- pyproject.toml | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/poetry.lock b/poetry.lock index eaa19d9ef..284ea6bed 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.5.1 and should not be changed by hand. [[package]] name = "accessible-pygments" @@ -1247,7 +1247,7 @@ comm = ">=0.1.1" debugpy = ">=1.6.5" ipython = ">=7.23.1" jupyter-client = ">=8.9.0" -jupyter-core = ">=5.1,<6.0.dev0 || >=6.1.dev0" +jupyter-core = ">=5.1,<6.0 || >=6.1.dev0" matplotlib-inline = ">=0.1" nest-asyncio2 = ">=1.7.0" packaging = ">=22" @@ -1579,7 +1579,7 @@ files = [ ipykernel = ">=6.14" ipython = "*" jupyter-client = ">=7.0.0" -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +jupyter-core = ">=4.12,<5.0 || >=5.1.dev0" prompt-toolkit = ">=3.0.30" pygments = "*" pyzmq = ">=17" @@ -1668,14 +1668,14 @@ anyio = ">=3.1.0" argon2-cffi = ">=21.1" jinja2 = ">=3.0.3" jupyter-client = ">=7.4.4" -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +jupyter-core = ">=4.12,<5.0 || >=5.1.dev0" jupyter-events = ">=0.11.0" jupyter-server-terminals = ">=0.4.4" nbconvert = ">=6.4.4" nbformat = ">=5.3.0" packaging = ">=22.0" prometheus-client = ">=0.9" -pywinpty = {version = ">=2.0.1,<3.0.4 || >3.0.4", markers = "os_name == \"nt\""} +pywinpty = {version = ">=2.0.1,!=3.0.4", markers = "os_name == \"nt\""} pyzmq = ">=24" send2trash = ">=1.8.2" terminado = ">=0.8.3" @@ -1746,7 +1746,7 @@ files = [ [package.dependencies] async-lru = ">=1.0.0" httpx = ">=0.25.0,<1" -ipykernel = ">=6.5.0,<6.30.0 || >6.30.0" +ipykernel = ">=6.5.0,!=6.30.0" jinja2 = ">=3.0.3" jupyter-builder = ">=1.0.2" jupyter-core = "*" @@ -2482,7 +2482,7 @@ files = [ [package.dependencies] fastjsonschema = ">=2.15" jsonschema = ">=2.6" -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +jupyter-core = ">=4.12,<5.0 || >=5.1.dev0" traitlets = ">=5.1" [package.extras] @@ -2504,9 +2504,9 @@ files = [ [package.dependencies] docutils = ">=0.18.1" jinja2 = "*" -nbconvert = ">=5.3,<5.4 || >5.4" +nbconvert = ">=5.3,!=5.4" nbformat = "*" -sphinx = ">=1.8,<8.2.0 || >8.2.0,<8.2.1 || >8.2.1" +sphinx = ">=1.8,!=8.2.0,!=8.2.1" traitlets = ">=5" [[package]] @@ -4232,14 +4232,14 @@ files = [ [[package]] name = "tqdm" -version = "4.70.0" +version = "4.70.1" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "tqdm-4.70.0-py3-none-any.whl", hash = "sha256:7f585706bfddbdebf89daac705b2dfcc16890130727d3197ca62c732b4310953"}, - {file = "tqdm-4.70.0.tar.gz", hash = "sha256:55b0b0dbd97462d06ebee91e4dac24ed4d4702be82b24f07e6c1d27e08cea220"}, + {file = "tqdm-4.70.1-py3-none-any.whl", hash = "sha256:c293e525e6fef9c20e8728fd4612df02a0aa31bb5fe91ecd93e123b1b7bffa73"}, + {file = "tqdm-4.70.1.tar.gz", hash = "sha256:cefd0eca11b2a37a3aee776544d4f4ae913f02688135b5556b8788dfa474afc4"}, ] [package.dependencies] @@ -4412,4 +4412,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">=3.12,<3.15" -content-hash = "ed6ef2a6cb8814ee6ab96c983711908f3102cbfdeb28136410bf675ad2da745c" +content-hash = "7c06b18379d2badb1c8603f9d29d48487af53f925335fbc9a95aa031614ebc6f" diff --git a/pyproject.toml b/pyproject.toml index 61752913c..c3865c647 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ dependencies = [ "lxml (>=6.1.1,<7.0.0)", "pyyaml (>=6.0.3,<7.0.0)", "more-itertools (>=10.2.0,<11.0.0)", - "tqdm (>=4.70.0,<5.0.0)", + "tqdm (>=4.70.1,<5.0.0)", "packaging (>=26.0,<28.0)", "synphot (>=1.7.0,<2.0.0)", From 720022edef580f14797a93d1250f162d75416c55 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 15:46:20 +0000 Subject: [PATCH 077/130] Bump sphinxcontrib-mermaid Bumps the dev-dependencies group with 1 update: [sphinxcontrib-mermaid](https://github.com/mgaitan/sphinxcontrib-mermaid). Updates `sphinxcontrib-mermaid` from 2.1.0 to 2.1.1 - [Changelog](https://github.com/mgaitan/sphinxcontrib-mermaid/blob/master/CHANGELOG.md) - [Commits](https://github.com/mgaitan/sphinxcontrib-mermaid/compare/2.1.0...2.1.1) --- updated-dependencies: - dependency-name: sphinxcontrib-mermaid dependency-version: 2.1.1 dependency-type: direct:development update-type: version-update:semver-patch dependency-group: dev-dependencies ... Signed-off-by: dependabot[bot] --- poetry.lock | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/poetry.lock b/poetry.lock index eaa19d9ef..babc24283 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.5.1 and should not be changed by hand. [[package]] name = "accessible-pygments" @@ -1247,7 +1247,7 @@ comm = ">=0.1.1" debugpy = ">=1.6.5" ipython = ">=7.23.1" jupyter-client = ">=8.9.0" -jupyter-core = ">=5.1,<6.0.dev0 || >=6.1.dev0" +jupyter-core = ">=5.1,<6.0 || >=6.1.dev0" matplotlib-inline = ">=0.1" nest-asyncio2 = ">=1.7.0" packaging = ">=22" @@ -1579,7 +1579,7 @@ files = [ ipykernel = ">=6.14" ipython = "*" jupyter-client = ">=7.0.0" -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +jupyter-core = ">=4.12,<5.0 || >=5.1.dev0" prompt-toolkit = ">=3.0.30" pygments = "*" pyzmq = ">=17" @@ -1668,14 +1668,14 @@ anyio = ">=3.1.0" argon2-cffi = ">=21.1" jinja2 = ">=3.0.3" jupyter-client = ">=7.4.4" -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +jupyter-core = ">=4.12,<5.0 || >=5.1.dev0" jupyter-events = ">=0.11.0" jupyter-server-terminals = ">=0.4.4" nbconvert = ">=6.4.4" nbformat = ">=5.3.0" packaging = ">=22.0" prometheus-client = ">=0.9" -pywinpty = {version = ">=2.0.1,<3.0.4 || >3.0.4", markers = "os_name == \"nt\""} +pywinpty = {version = ">=2.0.1,!=3.0.4", markers = "os_name == \"nt\""} pyzmq = ">=24" send2trash = ">=1.8.2" terminado = ">=0.8.3" @@ -1746,7 +1746,7 @@ files = [ [package.dependencies] async-lru = ">=1.0.0" httpx = ">=0.25.0,<1" -ipykernel = ">=6.5.0,<6.30.0 || >6.30.0" +ipykernel = ">=6.5.0,!=6.30.0" jinja2 = ">=3.0.3" jupyter-builder = ">=1.0.2" jupyter-core = "*" @@ -2482,7 +2482,7 @@ files = [ [package.dependencies] fastjsonschema = ">=2.15" jsonschema = ">=2.6" -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +jupyter-core = ">=4.12,<5.0 || >=5.1.dev0" traitlets = ">=5.1" [package.extras] @@ -2504,9 +2504,9 @@ files = [ [package.dependencies] docutils = ">=0.18.1" jinja2 = "*" -nbconvert = ">=5.3,<5.4 || >5.4" +nbconvert = ">=5.3,!=5.4" nbformat = "*" -sphinx = ">=1.8,<8.2.0 || >8.2.0,<8.2.1 || >8.2.1" +sphinx = ">=1.8,!=8.2.0,!=8.2.1" traitlets = ">=5" [[package]] @@ -3948,14 +3948,14 @@ test = ["flake8", "mypy", "pytest"] [[package]] name = "sphinxcontrib-mermaid" -version = "2.1.0" +version = "2.1.1" description = "Mermaid diagrams in your Sphinx-powered docs" optional = false python-versions = ">=3.10" groups = ["docs"] files = [ - {file = "sphinxcontrib_mermaid-2.1.0-py3-none-any.whl", hash = "sha256:417cd144ec4b28852f46ba653f02ce8e538881c812111671a4c30344e87f2112"}, - {file = "sphinxcontrib_mermaid-2.1.0.tar.gz", hash = "sha256:13c5f9ac395cb6abf403eca34e228dc9fb3a30c9d960dbf3e40e9a8cef969549"}, + {file = "sphinxcontrib_mermaid-2.1.1-py3-none-any.whl", hash = "sha256:3b9ac1a8e0b5154dec3c0acf9b5c54dd9ba58540e416f20858c300814c25648f"}, + {file = "sphinxcontrib_mermaid-2.1.1.tar.gz", hash = "sha256:f8d7f1b2f0ad69b7ebdd5cbf532fb4ee107893b5373bc2d0e9634089c0b2cfee"}, ] [package.dependencies] From 99c06335070c99201528b75b52b8a39b64b86a0d Mon Sep 17 00:00:00 2001 From: "astarvienna-bot[bot]" <217837468+myciapp[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 19:25:45 +0000 Subject: [PATCH 078/130] Bumping version from 0.12.1.dev0 to 0.12.1.dev1 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 61752913c..141acea6d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ScopeSim" -version = "0.12.1.dev0" +version = "0.12.1.dev1" description = "Generalised telescope observation simulator" license = {text = "GPL-3.0-or-later"} authors = [ From a256ed3d7709e6aafa1bca305a00c96e5b175cf5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 19:35:25 +0000 Subject: [PATCH 079/130] Bump urllib3 from 2.7.0 to 2.8.0 Bumps [urllib3](https://github.com/urllib3/urllib3) from 2.7.0 to 2.8.0. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/2.7.0...2.8.0) --- updated-dependencies: - dependency-name: urllib3 dependency-version: 2.8.0 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index babc24283..5eab9023c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -4308,14 +4308,14 @@ dev = ["flake8", "flake8-annotations", "flake8-bandit", "flake8-bugbear", "flake [[package]] name = "urllib3" -version = "2.7.0" +version = "2.8.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.10" groups = ["main", "dev", "docs"] files = [ - {file = "urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897"}, - {file = "urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c"}, + {file = "urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3"}, + {file = "urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63"}, ] [package.extras] From 4cda19bd343a15b0c19d34f1a905233cb55bd7ea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 19:39:34 +0000 Subject: [PATCH 080/130] Bump tornado from 6.5.8 to 6.5.9 Bumps [tornado](https://github.com/tornadoweb/tornado) from 6.5.8 to 6.5.9. - [Changelog](https://github.com/tornadoweb/tornado/blob/master/docs/releases.rst) - [Commits](https://github.com/tornadoweb/tornado/compare/v6.5.8...v6.5.9) --- updated-dependencies: - dependency-name: tornado dependency-version: 6.5.9 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- poetry.lock | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/poetry.lock b/poetry.lock index babc24283..1ad9fe772 100644 --- a/poetry.lock +++ b/poetry.lock @@ -4212,22 +4212,22 @@ test = ["flake8", "isort", "pytest"] [[package]] name = "tornado" -version = "6.5.8" +version = "6.5.9" description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." optional = false python-versions = ">=3.9" groups = ["dev", "docs", "test"] files = [ - {file = "tornado-6.5.8-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:cc6aa787d7cfab7c3d35189dc7a56fbd2399a569624c730c6b55b3d6531d0403"}, - {file = "tornado-6.5.8-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9715b5eb79735b2bcd454ce216a9275b7c0470e64ea1bf5742f78b2f72b26eeb"}, - {file = "tornado-6.5.8-cp39-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:547d63f450d570c14fe0e8db2cfb14c9bbd1c2503b4a6612586267955aa47b58"}, - {file = "tornado-6.5.8-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7e2360a0ffbe145eca8af0b19cb7203d79b1a98dd4cccdd6b368f6f49c2e3808"}, - {file = "tornado-6.5.8-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:5d242290bdf7ab3151bc1065fdd75c0dcc21cbc7b49f22a4c56329c2d6566d22"}, - {file = "tornado-6.5.8-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:7b94ff0e128fe0542f3bd331fb44d06260fc4ac16881545159f34ef08aad4195"}, - {file = "tornado-6.5.8-cp39-abi3-win32.whl", hash = "sha256:67832909c4779c64942380cb5f044a5c6163d00831472d80e25e115de9917836"}, - {file = "tornado-6.5.8-cp39-abi3-win_amd64.whl", hash = "sha256:11881db6b7c168494be2c2d12e65931451bdf7ee718535418ae1d8855dd5a0ee"}, - {file = "tornado-6.5.8-cp39-abi3-win_arm64.whl", hash = "sha256:68a7468c7e289f8514d7d664101753903217eff1bb6822c6b5994a0b5f5bcb26"}, - {file = "tornado-6.5.8.tar.gz", hash = "sha256:9452e1b208a8bd771e2cb1f2ff564985b9b214bdebbe622793e1799e0a6bd23f"}, + {file = "tornado-6.5.9-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:dea3bee433b73d6312d993ce89ad5f2b8d3ed9fde56e79ba5342b9edb78d342a"}, + {file = "tornado-6.5.9-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:22bed49cdb55292d1d58f46008ba2cc3020e5b84995ef6ae749676c8b5309514"}, + {file = "tornado-6.5.9-cp39-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0cc0876a58eee9143476f24901b1b8eb4d52443283ade2b6bca650dd1da04f84"}, + {file = "tornado-6.5.9-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc0d703cc7ec92b47ebfe236c97cd765c1f2e45744a2e156e6677e54c948274d"}, + {file = "tornado-6.5.9-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:93f9ab0226ef625b94cacb7981a5af104f1735ea02f92282abe5c389109fb31d"}, + {file = "tornado-6.5.9-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5801bbf94dfde8d21087b6aa0b17f3a5dab6110dfe44e7e4492df5de393ac15b"}, + {file = "tornado-6.5.9-cp39-abi3-win32.whl", hash = "sha256:dca9f377e80efe5dd4d52a2776b67492a81a39b2d931de169b6b36d0c1da444c"}, + {file = "tornado-6.5.9-cp39-abi3-win_amd64.whl", hash = "sha256:f810730ac71eadf009e6e5cb6d534d096887310f5649cff19d7e0d19c0bf4ee7"}, + {file = "tornado-6.5.9-cp39-abi3-win_arm64.whl", hash = "sha256:b5f4d92798337260c6d9f9f0e400c7539dd5e55cf93479a3f8508c9aa09c2b3b"}, + {file = "tornado-6.5.9.tar.gz", hash = "sha256:4d868544ebdf2fc155239a74397f65ebfea9b4d3635296c96b5dfb0701c354f5"}, ] [[package]] From 3fe382130f1a65f14e4d5e43e7fe6d6bdecb9497 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 19:42:24 +0000 Subject: [PATCH 081/130] Bump notebook from 7.6.1 to 7.6.3 Bumps [notebook](https://github.com/jupyter/notebook) from 7.6.1 to 7.6.3. - [Release notes](https://github.com/jupyter/notebook/releases) - [Changelog](https://github.com/jupyter/notebook/blob/@jupyter-notebook/tree@7.6.3/CHANGELOG.md) - [Commits](https://github.com/jupyter/notebook/compare/@jupyter-notebook/tree@7.6.1...@jupyter-notebook/tree@7.6.3) --- updated-dependencies: - dependency-name: notebook dependency-version: 7.6.3 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- poetry.lock | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/poetry.lock b/poetry.lock index babc24283..3baf85d80 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1733,14 +1733,14 @@ test = ["bash-kernel", "pytest"] [[package]] name = "jupyterlab" -version = "4.6.3" +version = "4.6.4" description = "JupyterLab computational environment" optional = false python-versions = ">=3.10" groups = ["dev"] files = [ - {file = "jupyterlab-4.6.3-py3-none-any.whl", hash = "sha256:0a1ebc6567186f1eabd99536e94df7ed9e96d1e7c5ddf3e4406ae16e88abacb7"}, - {file = "jupyterlab-4.6.3.tar.gz", hash = "sha256:2e3db6e3a12495ebd188276e985bf5ac502fbde3d1e8628819920210008de498"}, + {file = "jupyterlab-4.6.4-py3-none-any.whl", hash = "sha256:15b13f991d3985129c797eb84d9949eeb8b6615e14b444868e642411f2c418b2"}, + {file = "jupyterlab-4.6.4.tar.gz", hash = "sha256:404f49b081819378524886c9db66dba57a5565981eff885830df1baba3a17df5"}, ] [package.dependencies] @@ -2535,20 +2535,20 @@ files = [ [[package]] name = "notebook" -version = "7.6.1" +version = "7.6.3" description = "Jupyter Notebook - A web-based notebook environment for interactive computing" optional = false python-versions = ">=3.10" groups = ["dev"] files = [ - {file = "notebook-7.6.1-py3-none-any.whl", hash = "sha256:6ea1e4c926f0dc490444ddcc335797a3dacda470a805d01031872fb119988ba2"}, - {file = "notebook-7.6.1.tar.gz", hash = "sha256:0b45fd1010668dd4808c40914d957706dbf044a677d28764dc881b74dfcede82"}, + {file = "notebook-7.6.3-py3-none-any.whl", hash = "sha256:ad7e0eb765fba836cd4a2ab0c7a3a26cde1d91665fbf6f533b6ae7b2de6d88d2"}, + {file = "notebook-7.6.3.tar.gz", hash = "sha256:e2c08e469c0ae20bb0b3214f0ab77e79653317a2f8e5b34c10361c66874a5b50"}, ] [package.dependencies] jupyter-builder = ">=1.0.2,<2" jupyter-server = ">=2.19.0,<3" -jupyterlab = ">=4.6.2,<4.7" +jupyterlab = ">=4.6.4,<4.7" jupyterlab-server = ">=2.28.0,<3" notebook-shim = ">=0.2,<0.3" tornado = ">=6.2.0" From dea4b10d5e78d15841b7f3c54740935cb5908d1a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 19:42:54 +0000 Subject: [PATCH 082/130] Bump jupyterlab from 4.6.3 to 4.6.4 Bumps [jupyterlab](https://github.com/jupyterlab/jupyterlab) from 4.6.3 to 4.6.4. - [Release notes](https://github.com/jupyterlab/jupyterlab/releases) - [Changelog](https://github.com/jupyterlab/jupyterlab/blob/main/RELEASE.md) - [Commits](https://github.com/jupyterlab/jupyterlab/compare/@jupyterlab/lsp@4.6.3...@jupyterlab/lsp@4.6.4) --- updated-dependencies: - dependency-name: jupyterlab dependency-version: 4.6.4 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index babc24283..e7db5b54e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1733,14 +1733,14 @@ test = ["bash-kernel", "pytest"] [[package]] name = "jupyterlab" -version = "4.6.3" +version = "4.6.4" description = "JupyterLab computational environment" optional = false python-versions = ">=3.10" groups = ["dev"] files = [ - {file = "jupyterlab-4.6.3-py3-none-any.whl", hash = "sha256:0a1ebc6567186f1eabd99536e94df7ed9e96d1e7c5ddf3e4406ae16e88abacb7"}, - {file = "jupyterlab-4.6.3.tar.gz", hash = "sha256:2e3db6e3a12495ebd188276e985bf5ac502fbde3d1e8628819920210008de498"}, + {file = "jupyterlab-4.6.4-py3-none-any.whl", hash = "sha256:15b13f991d3985129c797eb84d9949eeb8b6615e14b444868e642411f2c418b2"}, + {file = "jupyterlab-4.6.4.tar.gz", hash = "sha256:404f49b081819378524886c9db66dba57a5565981eff885830df1baba3a17df5"}, ] [package.dependencies] From 43228458cb1ce971d8434fde11ea6f9b88f49f2f Mon Sep 17 00:00:00 2001 From: "astarvienna-bot[bot]" <217837468+myciapp[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 19:56:17 +0000 Subject: [PATCH 083/130] Bumping version from 0.12.1.dev1 to 0.12.1.dev2 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 141acea6d..f31788528 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ScopeSim" -version = "0.12.1.dev1" +version = "0.12.1.dev2" description = "Generalised telescope observation simulator" license = {text = "GPL-3.0-or-later"} authors = [ From 8200eaa3e2de285469f12c60c4db36bfa24cbbf9 Mon Sep 17 00:00:00 2001 From: "astarvienna-bot[bot]" <217837468+myciapp[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 22:37:18 +0000 Subject: [PATCH 084/130] Bumping version from 0.12.1.dev2 to 0.12.1.dev3 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f31788528..9189bd408 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ScopeSim" -version = "0.12.1.dev2" +version = "0.12.1.dev3" description = "Generalised telescope observation simulator" license = {text = "GPL-3.0-or-later"} authors = [ From 91d452fb55494b865db6fd018a09eae33c68e251 Mon Sep 17 00:00:00 2001 From: "astarvienna-bot[bot]" <217837468+myciapp[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 22:42:58 +0000 Subject: [PATCH 085/130] Bumping version from 0.12.1.dev3 to 0.12.1.dev4 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b51c3ba4e..efa2773ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ScopeSim" -version = "0.12.1.dev3" +version = "0.12.1.dev4" description = "Generalised telescope observation simulator" license = {text = "GPL-3.0-or-later"} authors = [ From d8f2c096beb84e47e5d12295678f2c703f1736e2 Mon Sep 17 00:00:00 2001 From: "astarvienna-bot[bot]" <217837468+myciapp[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 22:50:44 +0000 Subject: [PATCH 086/130] Bumping version from 0.12.1.dev4 to 0.12.1.dev5 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index efa2773ca..ff0d93783 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ScopeSim" -version = "0.12.1.dev4" +version = "0.12.1.dev5" description = "Generalised telescope observation simulator" license = {text = "GPL-3.0-or-later"} authors = [ From 2d24516855e35bdffa5acd0485dbacd41855c727 Mon Sep 17 00:00:00 2001 From: "astarvienna-bot[bot]" <217837468+myciapp[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 23:51:20 +0000 Subject: [PATCH 087/130] Bumping version from 0.12.1.dev5 to 0.12.1.dev6 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ff0d93783..f6ecd946f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ScopeSim" -version = "0.12.1.dev5" +version = "0.12.1.dev6" description = "Generalised telescope observation simulator" license = {text = "GPL-3.0-or-later"} authors = [ From dbbba2942ff69af9eef3f83bb7422fc595d43dec Mon Sep 17 00:00:00 2001 From: Yashvi-Sharma Date: Thu, 15 Oct 2026 17:49:04 -0700 Subject: [PATCH 088/130] tested zimager mode, fixed minor typos, small change in dichroictree effect to set the arm id as FOV attribute so that image_plane_id can correctly get propagated for fov to plane mapping --- scopesim/effects/ter_curves.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scopesim/effects/ter_curves.py b/scopesim/effects/ter_curves.py index 340af7eab..123f08090 100644 --- a/scopesim/effects/ter_curves.py +++ b/scopesim/effects/ter_curves.py @@ -1621,6 +1621,7 @@ def apply_to(self, obj, **kwargs): if isinstance(obj, FieldOfView): logger.debug("Executing %s, observe", self.meta['name']) obj_id = obj.meta.get(id_colname, None) + setattr(obj, id_colname, obj_id) row = self.table[self.table[id_colname] == obj_id] if len(row) != 1: From 38bf6c59b2b3bc793175629ca04192417895e261 Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Fri, 16 Oct 2026 00:21:54 -0700 Subject: [PATCH 089/130] "Add test coverage and refactor FWHM logic for MoffatPSF and AOEnhanceablePSF handling. Improve caching logic, update observation-dependent interpolation, and ensure proper fixture utilization in tests." --- scopesim/effects/psfs/analytical.py | 48 ++++---- .../tests/tests_effects/test_MoffatPSF.py | 107 +++++++++++++++++- 2 files changed, 131 insertions(+), 24 deletions(-) diff --git a/scopesim/effects/psfs/analytical.py b/scopesim/effects/psfs/analytical.py index dc39f5207..e40b37c99 100644 --- a/scopesim/effects/psfs/analytical.py +++ b/scopesim/effects/psfs/analytical.py @@ -258,19 +258,7 @@ def __init__(self, **kwargs): self._fwhm_interp = None def fwhm(self, wavelengths): - fwhm = self.meta["fwhm"] - if isinstance(fwhm, dict) and {"pivot_wave", "pivot_wave_unit"}.issubset(fwhm): - target, location, time, _ = get_observation_info_from_cmds(self.cmds) - key = tuple(fwhm.items()), target, location, time - elif isinstance(fwhm, dict): - key = tuple(fwhm.items()) - else: - key = fwhm - - if self._fwhm_interp is None or self._fwhm_interp[1] != key: - self._fwhm_interp = self.get_fwhm_interp(), key - - return self._fwhm_interp[0](wavelengths) + return self.get_fwhm_interp()(wavelengths) @property def alpha(self): @@ -282,33 +270,47 @@ def get_fwhm_interp(self): Overwrite this function for subclassing if FWHM input options change. """ fwhm = self.meta["fwhm"] + if isinstance(fwhm, dict) and {"pivot_wave", "pivot_wave_unit"}.issubset(fwhm): + target, location, time, _ = get_observation_info_from_cmds(self.cmds) + zenith_angle = get_zenith_angle(target, location, time) + # The natural seeing law depends on the observation only through zenith angle. + key = tuple(fwhm.items()), zenith_angle + elif isinstance(fwhm, dict): + key = tuple(fwhm.items()) + else: + key = fwhm + + if self._fwhm_interp is not None and self._fwhm_interp[1] == key: + return self._fwhm_interp[0] + if isinstance(fwhm, dict): logger.info("dict supplied for FWHM") if check_keys(fwhm, {"seeing", "seeing_unit", "pivot_wave", "pivot_wave_unit"}, action="warn"): logger.info("seeing and pivot supplied, using natural scale seeing law") - target, location, time, _ = get_observation_info_from_cmds(self.cmds) - zenith_angle = get_zenith_angle(target, location, time) seeing = (fwhm["seeing"] * u.Unit(fwhm["seeing_unit"])).to(u.arcsec) pivot = (fwhm["pivot_wave"] * u.Unit(fwhm["pivot_wave_unit"])).to(u.um) - return partial(self.natural_scale, seeing=seeing, pivot=pivot, zenith_angle=zenith_angle*u.deg) + interp = partial(self.natural_scale, seeing=seeing, pivot=pivot, zenith_angle=zenith_angle*u.deg) elif check_keys(fwhm, {"seeing", "seeing_unit"}, action="error"): logger.info("only seeing supplied, FWHM is wavelength independent") seeing = (fwhm["seeing"] * u.Unit(fwhm["seeing_unit"])).to_value(u.arcsec) - return lambda wavelengths: np.full(wavelengths.shape, seeing, dtype=float) * u.arcsec + interp = lambda wavelengths: np.full(wavelengths.shape, seeing, dtype=float) * u.arcsec - if isinstance(fwhm, (int, float)): + elif isinstance(fwhm, (int, float)): logger.info("float value supplied, assuming arcsec") - return lambda wavelengths: np.full(wavelengths.shape, fwhm, dtype=float) * u.arcsec + interp = lambda wavelengths: np.full(wavelengths.shape, fwhm, dtype=float) * u.arcsec - if isinstance(fwhm, str): + elif isinstance(fwhm, str): logger.info("filename supplied for FWHM") self.table = DataContainer(filename=find_file(from_currsys(fwhm, self.cmds))).table - interp = self.fwhm_from_table(self.table) - return lambda wavelengths: interp(wavelengths) * u.arcsec + table_interp = self.fwhm_from_table(self.table) + interp = lambda wavelengths: table_interp(wavelengths) * u.arcsec + else: + raise TypeError("fwhm kwarg must be of type dict or float or str") - raise TypeError("fwhm kwarg must be of type dict or float or str") + self._fwhm_interp = interp, key + return interp def get_kernel(self, fov): pixel_scale = fov.header["CDELT1"] * u.deg.to(u.arcsec) diff --git a/scopesim/tests/tests_effects/test_MoffatPSF.py b/scopesim/tests/tests_effects/test_MoffatPSF.py index 1119df8dd..17e01a245 100644 --- a/scopesim/tests/tests_effects/test_MoffatPSF.py +++ b/scopesim/tests/tests_effects/test_MoffatPSF.py @@ -1,10 +1,14 @@ import logging +from unittest.mock import Mock import numpy as np import pytest from astropy import units as u +from astropy.coordinates import EarthLocation, SkyCoord +from astropy.table import Table +from astropy.time import Time -from scopesim.effects.psfs.analytical import MoffatPSF +from scopesim.effects.psfs.analytical import AOEnhanceablePSF, MoffatPSF class SimpleFov: @@ -87,3 +91,104 @@ def test_legacy_renormalization_can_be_requested(): kernel = psf.get_kernel(SimpleFov()) assert np.sum(kernel) == pytest.approx(1.0) + + +@pytest.fixture +def seeing_law(): + return {"seeing": 0.7, "seeing_unit": "arcsec", "pivot_wave": 500, "pivot_wave_unit": "nm"} + + +@pytest.fixture +def observing_conditions(monkeypatch): + location = EarthLocation.from_geodetic(-155 * u.deg, 20 * u.deg, 4000 * u.m) + conditions = {"time": Time("2026-01-01T00:00:00"), "alt": 60 * u.deg} + + def observation_info(cmds): + time = conditions["time"] + target = SkyCoord(alt=conditions["alt"], az=0 * u.deg, frame="altaz", location=location, obstime=time) + return target, location, time, None + + monkeypatch.setattr("scopesim.effects.psfs.analytical.get_observation_info_from_cmds", observation_info) + return conditions + + +@pytest.mark.parametrize("psf_class", [MoffatPSF, AOEnhanceablePSF]) +def test_observation_configuration_is_only_required_when_evaluating(psf_class, seeing_law, monkeypatch): + observation_info = Mock(side_effect=KeyError("!OBS.airmass")) + monkeypatch.setattr("scopesim.effects.psfs.analytical.get_observation_info_from_cmds", observation_info) + kwargs = {"alpha": 4.765, "fwhm": seeing_law} + if psf_class is AOEnhanceablePSF: + kwargs.update(ao_table="unused.dat", enable_ao=False, is_absolute=False) + psf = psf_class(**kwargs) + observation_info.assert_not_called() + with pytest.raises(KeyError, match="airmass"): + psf.fwhm(np.array([0.5]) * u.um) + + +@pytest.mark.parametrize("fwhm", [0.7, {"seeing": 700, "seeing_unit": "mas"}]) +def test_constant_seeing_is_evaluated_without_observation_commands(fwhm, monkeypatch): + observation_info = Mock(side_effect=KeyError("!OBS.airmass")) + monkeypatch.setattr("scopesim.effects.psfs.analytical.get_observation_info_from_cmds", observation_info) + psf = MoffatPSF(alpha=4.765, fwhm=fwhm) + np.testing.assert_allclose(psf.fwhm(np.array([0.5, 1.0]) * u.um).to_value(u.arcsec), [0.7, 0.7]) + observation_info.assert_not_called() + + +def test_changing_observation_times_reuses_same_seeing_law(seeing_law, observing_conditions): + psf = MoffatPSF(alpha=4.765, fwhm=seeing_law) + wavelengths = np.array([0.5, 1.0]) * u.um + first = psf.fwhm(wavelengths) + cached_interp = psf._fwhm_interp[0] + observing_conditions["time"] = observing_conditions["time"] + 1 * u.hour + second = psf.fwhm(wavelengths) + np.testing.assert_allclose(second.to_value(u.arcsec), first.to_value(u.arcsec)) + assert psf._fwhm_interp[0] is cached_interp + + +def test_zenith_angle_change_updates_seeing_law(seeing_law, observing_conditions): + psf = MoffatPSF(alpha=4.765, fwhm=seeing_law) + wavelengths = np.array([0.5]) * u.um + first = psf.fwhm(wavelengths) + cached_interp = psf._fwhm_interp[0] + observing_conditions["alt"] = 30 * u.deg + second = psf.fwhm(wavelengths) + expected_ratio = (np.cos(30 * u.deg) / np.cos(60 * u.deg))**0.6 + np.testing.assert_allclose(second.to_value(u.arcsec) / first.to_value(u.arcsec), expected_ratio) + assert psf._fwhm_interp[0] is not cached_interp + + +@pytest.mark.parametrize("parameter, value, expected_ratio", [("seeing", 0.9, 0.9 / 0.7), ("pivot_wave", 1000, 2**0.2)]) +def test_seeing_parameters_update_cached_law(parameter, value, expected_ratio, seeing_law, observing_conditions): + psf = MoffatPSF(alpha=4.765, fwhm=seeing_law) + wavelengths = np.array([1.0]) * u.um + first = psf.fwhm(wavelengths) + psf.meta["fwhm"][parameter] = value + second = psf.fwhm(wavelengths) + np.testing.assert_allclose(second.to_value(u.arcsec) / first.to_value(u.arcsec), expected_ratio) + + +def test_tabulated_fwhm_without_observation_commands(tmp_path, monkeypatch): + observation_info = Mock(side_effect=KeyError("!OBS.airmass")) + monkeypatch.setattr("scopesim.effects.psfs.analytical.get_observation_info_from_cmds", observation_info) + filename = tmp_path / "fwhm.ecsv" + table = Table({"wavelength": np.array([0.5, 1.0, 1.5, 2.0]) * u.um, "fwhm": [0.5, 0.6, 0.7, 0.8]}) + table.write(filename, format="ascii.ecsv") + psf = MoffatPSF(alpha=4.765, fwhm=str(filename)) + np.testing.assert_allclose(psf.fwhm(np.array([0.5, 1.0]) * u.um).to_value(u.arcsec), [0.5, 0.6]) + observation_info.assert_not_called() + + +def test_ao_toggle_preserves_natural_and_absolute_widths(tmp_path): + filename = tmp_path / "ao.ecsv" + table = Table({"wavelength": np.array([0.5, 1.0, 1.5, 2.0]) * u.um, "fwhm": [0.5, 0.5, 0.5, 0.5]}) + table.write(filename, format="ascii.ecsv") + psf = AOEnhanceablePSF(alpha=4.765, fwhm=0.7, ao_table=str(filename), enable_ao=False, is_absolute=False) + wavelengths = np.array([1.0]) * u.um + np.testing.assert_allclose(psf.fwhm(wavelengths).to_value(u.arcsec), [0.7]) + psf.meta["enable_ao"] = True + np.testing.assert_allclose(psf.fwhm(wavelengths).to_value(u.arcsec), [0.35]) + psf.meta["is_absolute"] = True + np.testing.assert_allclose(psf.fwhm(wavelengths).to_value(u.arcsec), [0.5]) + psf.meta["fwhm"] = 0.9 + psf.meta["enable_ao"] = False + np.testing.assert_allclose(psf.fwhm(wavelengths).to_value(u.arcsec), [0.9]) From efa913a22e0b239f5a80cd94ee27218519c54751 Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Fri, 16 Oct 2026 01:23:12 -0700 Subject: [PATCH 090/130] **Add tests and refactor branch throughput logic for imaging sources** - Introduce unit tests for dichroic effects and selector wheel handling with independent branch attenuation. - Refactor `apply_to` methods in `SelectorWheel` and `DichroicTree` to support source-specific FoV contexts. - Ensure wavelength-dependent source spectra are properly preserved and processed through branch effects. - Enhance `extract_source` to apply pre-extraction branch filtering. - Simplify downstream emission handling in the optical train with branch-specific throughput. --- scopesim/effects/selector_wheel.py | 11 +- scopesim/effects/ter_curves.py | 14 ++- scopesim/optics/optical_train.py | 24 +++- .../tests_effects/test_imaging_branches.py | 115 ++++++++++++++++++ 4 files changed, 153 insertions(+), 11 deletions(-) create mode 100644 scopesim/tests/tests_effects/test_imaging_branches.py diff --git a/scopesim/effects/selector_wheel.py b/scopesim/effects/selector_wheel.py index 7ee03f242..41c9970e7 100644 --- a/scopesim/effects/selector_wheel.py +++ b/scopesim/effects/selector_wheel.py @@ -19,6 +19,7 @@ from ..optics.fov import FieldOfView from ..optics.image_plane import ImagePlane from ..detector.detector import Detector +from ..source.source import Source logger = get_logger(__name__) @@ -78,8 +79,14 @@ def __init__(self, **kwargs): self.z_order = self._resolve_z_order() - def apply_to(self, obj, **kwargs): - """Based on the selector_key's value in obj.meta, apply the corresponding effect from the wheel.""" + def apply_to(self, obj, fov=None, **kwargs): + """Select from obj.meta, or fov.meta when applying branch effects to a Source.""" + if isinstance(obj, Source) and fov is not None: + selector_value = fov.meta[self.meta["selector_key"]] + effect_to_apply = self.get_effect(selector_value) + if effect_to_apply is not None: + obj = effect_to_apply.apply_to(obj, fov=fov, **kwargs) + if isinstance(obj, FieldOfView): if self.meta['selector_key'] not in obj.meta.keys(): raise ValueError(f"Selector key {self.meta['selector_key']} not found in FieldOfView meta.") diff --git a/scopesim/effects/ter_curves.py b/scopesim/effects/ter_curves.py index 123f08090..e48cae5d2 100644 --- a/scopesim/effects/ter_curves.py +++ b/scopesim/effects/ter_curves.py @@ -4,6 +4,7 @@ import warnings from typing import ClassVar from collections.abc import Collection, Iterable +from numbers import Integral import numpy as np from scipy import integrate from astropy import units as u @@ -1557,6 +1558,7 @@ class DichroicTree(Effect): ``` This effect creates a volume of FoVs, one for each row in the dichroic tree table. + For imaging, pass a Source with its fov context to apply the branch throughput before rendering. """ required_keys = {"tree_filename", "dichroic_names", "filename_format"} z_order: ClassVar[tuple[int, ...]] = (117, 217, 617) @@ -1564,6 +1566,9 @@ class DichroicTree(Effect): def __init__(self, cmds=None, **kwargs): check_keys(kwargs, self.required_keys, action="error") super().__init__(cmds=cmds, **kwargs) + if "z_order" in self.meta: + z_order = self.meta["z_order"] + self.z_order = (int(z_order),) if isinstance(z_order, Integral) else tuple(z_order) # Load all the dichroic TER curves self.dichroics = {} @@ -1580,7 +1585,7 @@ def __init__(self, cmds=None, **kwargs): tree = DataContainer(filename=tree_path) self.table = tree.table - def apply_to(self, obj, **kwargs): + def apply_to(self, obj, fov=None, **kwargs): """ To the object, loop through each row of the table and apply the dichroics (defined by columns) with the corresponding action defined in the table cell. @@ -1618,10 +1623,11 @@ def apply_to(self, obj, **kwargs): obj.volumes = new_vols # 2. During observe - if isinstance(obj, FieldOfView): + if isinstance(obj, FieldOfView) or (isinstance(obj, Source) and fov is not None): logger.debug("Executing %s, observe", self.meta['name']) - obj_id = obj.meta.get(id_colname, None) - setattr(obj, id_colname, obj_id) + branch_fov = obj if isinstance(obj, FieldOfView) else fov + obj_id = branch_fov.meta[id_colname] + setattr(branch_fov, id_colname, obj_id) row = self.table[self.table[id_colname] == obj_id] if len(row) != 1: diff --git a/scopesim/optics/optical_train.py b/scopesim/optics/optical_train.py index 8ef81cd9c..e49e71c5f 100644 --- a/scopesim/optics/optical_train.py +++ b/scopesim/optics/optical_train.py @@ -23,7 +23,7 @@ from .image_plane import ImagePlane from ..commands.user_commands import UserCommands from ..detector import DetectorManager -from ..effects import ExtraFitsKeywords +from ..effects import DichroicTree, ExtraFitsKeywords, SelectorWheel from ..utils import from_currsys, top_level_catch, get_logger from ..source.source_templates import empty_sky from ..source.source_fields import SpectrumSourceField @@ -37,7 +37,12 @@ N_PROCESSES = mp.cpu_count() - 1 USE_MULTIPROCESSING = False -def extract_source(fov, source): +def extract_source(fov, source, source_effects=()): + # Branch spectra must be filtered before extraction clips them to the FOV wavelength range. + if source_effects: + source = source.make_copy() + for effect in source_effects: + source = effect.apply_to(source, fov=fov) fov.extract_from(source) return fov @@ -538,7 +543,16 @@ def observe(self, orig_source=None, update=True, **kwargs): source = self.prepare_source(source) # [1D - transmission curves] - for effect in self.optics_manager.source_effects: + source_effects = self.optics_manager.source_effects + branch_effects = [] + if not self.fov_manager.is_spectroscope: + for ii, effect in enumerate(source_effects): + if isinstance(effect, (DichroicTree, SelectorWheel)): + # Keep downstream attenuation and emission in optical order for each branch. + source_effects, branch_effects = source_effects[:ii], source_effects[ii:] + break + + for effect in source_effects: source = effect.apply_to(source) # [3D - Atmospheric shifts, PSF, NCPAs, Grating shift/distortion] @@ -552,7 +566,7 @@ def observe(self, orig_source=None, update=True, **kwargs): with mp.Pool(processes=N_PROCESSES) as pool: fovs = pool.starmap(extract_source, - zip(fovs, [source] * len(fovs))) + zip(fovs, [source] * len(fovs), [branch_effects] * len(fovs))) with mp.Pool(processes=N_PROCESSES) as pool: fovs = pool.starmap(view_fov, @@ -568,7 +582,7 @@ def observe(self, orig_source=None, update=True, **kwargs): fovs = self.fov_manager.fovs nobar = len(fovs) <= 1 for fov in tqdm(fovs, desc=" FOVs", position=0, disable=nobar): - fov.extract_from(source) + fov = extract_source(fov, source, branch_effects) fov.view() foveffs = self.optics_manager.fov_effects diff --git a/scopesim/tests/tests_effects/test_imaging_branches.py b/scopesim/tests/tests_effects/test_imaging_branches.py new file mode 100644 index 000000000..ce308f392 --- /dev/null +++ b/scopesim/tests/tests_effects/test_imaging_branches.py @@ -0,0 +1,115 @@ +"""Branch throughput must be applied before integrating an imaging spectrum.""" + +from types import SimpleNamespace + +import numpy as np +import pytest + +from scopesim.commands import UserCommands +from scopesim.effects import DichroicTree, SelectorWheel, TERCurve +from scopesim.optics.image_plane import ImagePlane +from scopesim.optics.optical_train import OpticalTrain, extract_source +from scopesim.source.source import Source +from scopesim.tests.mocks.py_objects.fov_objects import _centre_fov +from scopesim.tests.mocks.py_objects.source_objects import _table_source + + +@pytest.fixture +def tree(tmp_path): + (tmp_path / 'tree.dat').write_text('image_plane_id first second\n0 R X\n1 T R\n2 T T\n') + for name, transmission in [('first', 0.75), ('second', 0.6)]: + curve = '# wavelength_unit: um\nwavelength transmission reflection emissivity\n' + curve += f'0.5 {transmission} {1 - transmission} 0\n2.5 {transmission} {1 - transmission} 0\n' + (tmp_path / f'{name}.dat').write_text(curve) + return DichroicTree(name='tree', tree_filename=str(tmp_path / 'tree.dat'), dichroic_names=['first', 'second'], filename_format=str(tmp_path / '{}.dat')) + + +@pytest.fixture +def selector(): + wheel = [] + for arm, transmission in enumerate([0.25, 0.75]): + curve = {'wavelength': [0.5, 2.5], 'transmission': [transmission, transmission], 'emissivity': [0, 0]} + wheel.append({'selector_value': arm, 'effect_class': 'QuantumEfficiencyCurve', 'effect_kwargs': {'array_dict': curve, 'wavelength_unit': 'um'}}) + return SelectorWheel(selector_key='image_plane_id', wheel=wheel, allowed_missing_selector_values=[2]) + + +@pytest.fixture +def source(): + source = _table_source() + background = TERCurve(array_dict={'wavelength': [0.5, 2.5], 'transmission': [1, 1], 'emission': [1e-8, 1e-8]}, wavelength_unit='um', emission_unit='photlam') + source.append(background.background_source[0]) + return source + + +@pytest.mark.parametrize('effect_name, transmissions', [('tree', [0.25, 0.3, 0.45]), ('selector', [0.25, 0.75, 1.0])]) +def test_branch_attenuates_coloured_sources_and_background_independently(request, effect_name, transmissions, source): + effect = request.getfixturevalue(effect_name) + reference = extract_source(_centre_fov(n=10), source).make_hdu().data + for arm, transmission in enumerate(transmissions): + fov = _centre_fov(n=10) + fov.meta['image_plane_id'] = arm + extract_source(fov, source, [effect]) + np.testing.assert_allclose(fov.make_hdu().data, transmission * reference) + # Rendering each branch must leave the shared source, including its sky, intact. + np.testing.assert_array_equal(extract_source(_centre_fov(n=10), source).make_hdu().data, reference) + + +@pytest.mark.parametrize('effect_name', ['tree', 'selector']) +def test_source_without_fov_context_is_unchanged(request, effect_name, source): + effect = request.getfixturevalue(effect_name) + reference = extract_source(_centre_fov(n=10), source).make_hdu().data + effect.apply_to(source) + np.testing.assert_array_equal(extract_source(_centre_fov(n=10), source).make_hdu().data, reference) + + +@pytest.mark.parametrize('effect_name', ['tree', 'selector']) +def test_missing_branch_id_fails(request, effect_name, source): + effect = request.getfixturevalue(effect_name) + fov = _centre_fov(n=10) + fov.meta.clear() + with pytest.raises(KeyError, match='image_plane_id'): + effect.apply_to(source, fov=fov) + + +def test_observe_keeps_downstream_emission_after_branch_split(tree, source): + common = TERCurve(array_dict={'wavelength': [0.5, 2.5], 'transmission': [0.8, 0.8], 'emissivity': [0, 0]}, wavelength_unit='um') + downstream = TERCurve(array_dict={'wavelength': [0.5, 2.5], 'transmission': [0.5, 0.5], 'emission': [2e-8, 2e-8]}, wavelength_unit='um', emission_unit='photlam') + train = OpticalTrain() + train.cmds = UserCommands() + fovs = [_centre_fov(n=10) for arm in range(3)] + for arm, fov in enumerate(fovs): + fov.meta['image_plane_id'] = arm + train.fov_manager = SimpleNamespace(fovs=fovs, is_spectroscope=False) + train.optics_manager = SimpleNamespace(source_effects=[common, tree, downstream], fov_effects=[], image_plane_effects=[]) + train.image_planes = [ImagePlane(fov.header) for fov in fovs] + + prepared = train.prepare_source(source.make_copy()) + reference = extract_source(_centre_fov(n=10), prepared).make_hdu().data + emission = extract_source(_centre_fov(n=10), downstream.background_source[0]).make_hdu().data + train.observe(source, update=False) + for arm, transmission in enumerate([0.25, 0.3, 0.45]): + expected = reference * 0.8 * transmission * 0.5 + emission + np.testing.assert_allclose(train._last_fovs[arm].hdu.data, expected) + assert train._last_fovs[arm].image_plane_id == arm + + +@pytest.mark.parametrize('spectrum_id, expected', [(0, 0.5), (1, 0.5 + 1 / 24), (2, 0.5 - 1 / 24)]) +def test_wavelength_dependent_branch_preserves_source_colour(spectrum_id, expected): + spectrum = _table_source().fields[0].spectra[spectrum_id] + source = Source(x=[0], y=[0], ref=[0], weight=[1], spectra=[spectrum]) + curve = {'wavelength': [0.5, 2.5], 'transmission': [0, 1], 'emissivity': [0, 0]} + selector = SelectorWheel(selector_key='image_plane_id', wheel=[{'selector_value': 0, 'effect_class': 'QuantumEfficiencyCurve', 'effect_kwargs': {'array_dict': curve, 'wavelength_unit': 'um'}}]) + reference = extract_source(_centre_fov(n=10), source).make_hdu().data.sum() + fov = _centre_fov(n=10) + fov.meta['image_plane_id'] = 0 + measured = extract_source(fov, source, [selector]).make_hdu().data.sum() / reference + # Linear spectra and throughput give these analytic integrals over 1--2 um. + # The input spectra are sampled every 0.02 um. + np.testing.assert_allclose(measured, expected, rtol=0, atol=1e-4) + + +@pytest.mark.parametrize('z_order, expected', [([117, 217, 517], (117, 217, 517)), (517, (517,))]) +def test_dichroic_source_stage_is_opt_in(tree, z_order, expected): + assert tree.z_order == (117, 217, 617) + configured = DichroicTree(z_order=z_order, **{key: tree.meta[key] for key in tree.required_keys}) + assert configured.z_order == expected From 2a68fbc0809592aaa4c49f796dee393916bc8605 Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Fri, 16 Oct 2026 20:43:12 -0700 Subject: [PATCH 091/130] Support independent detector-arm gains, readout modes and FITS metadata --- scopesim/effects/detector_list.py | 11 ++-- scopesim/effects/electronic/dmps.py | 8 ++- scopesim/effects/electronic/electrons.py | 13 ++--- scopesim/effects/fits_headers.py | 9 ++++ .../test_detector_arm_configuration.py | 54 +++++++++++++++++++ 5 files changed, 82 insertions(+), 13 deletions(-) create mode 100644 scopesim/tests/tests_effects/test_detector_arm_configuration.py diff --git a/scopesim/effects/detector_list.py b/scopesim/effects/detector_list.py index 6e197f75f..da9532a5f 100644 --- a/scopesim/effects/detector_list.py +++ b/scopesim/effects/detector_list.py @@ -355,16 +355,17 @@ def detector_headers(self, ids=None): hdr["PC2_1"] = -sang hdr["PC2_2"] = cang - # hdr["GAIN"] = row["gain"] - if "id" in row: - hdr["DET_ID"] = row["id"] - hdr["EXTNAME"] = f"DET_{row['id']}" - row_dict = { f"HIERARCH {col.upper()}" if len(col) > 8 else col: row[col] for col in row.colnames } hdr.update(row_dict) + if "id" in row.colnames: + # Keep routing identifiers numeric when detector metadata is + # serialized to FITS (NumPy integers otherwise become strings). + hdr["ID"] = int(row["id"]) + hdr["DET_ID"] = int(row["id"]) + hdr["EXTNAME"] = f"DET_{row['id']}" hdrs.append(hdr) return hdrs diff --git a/scopesim/effects/electronic/dmps.py b/scopesim/effects/electronic/dmps.py index 1f1bcf3fd..f0f7dd994 100644 --- a/scopesim/effects/electronic/dmps.py +++ b/scopesim/effects/electronic/dmps.py @@ -23,6 +23,9 @@ class DetectorModePropertiesSetter(Effect): mode_properties : dict A dictionary containing the DET parameters to be changed for each mode. See below for an example yaml entry. + mode_key : str + Command selecting the mode, by default ``!OBS.detector_readout_mode``. + Separate keys allow independent detector arms to select their modes. Examples -------- @@ -70,11 +73,12 @@ def __init__(self, **kwargs): check_keys(self.meta, self.required_keys, action="error") self.mode_properties = kwargs["mode_properties"] + self.mode_key = kwargs.get("mode_key", "!OBS.detector_readout_mode") def apply_to(self, obj, **kwargs): logger.debug("Entering DetectorModePropertiesSetter.apply_to") mode_name = kwargs.get("detector_readout_mode", - from_currsys("!OBS.detector_readout_mode", + from_currsys(self.mode_key, self.cmds)) if isinstance(obj, ImagePlane) and mode_name == "auto": mode_name = self.select_mode(obj, **kwargs) @@ -82,7 +86,7 @@ def apply_to(self, obj, **kwargs): self.meta["detector_readout_mode"] = mode_name props_dict = self.mode_properties[mode_name] - self.cmds["!OBS.detector_readout_mode"] = mode_name + self.cmds[self.mode_key] = mode_name for key, value in props_dict.items(): self.cmds[key] = value logger.debug("DMPS setting %s: %s", key, value) diff --git a/scopesim/effects/electronic/electrons.py b/scopesim/effects/electronic/electrons.py index c860362bd..33d39d689 100644 --- a/scopesim/effects/electronic/electrons.py +++ b/scopesim/effects/electronic/electrons.py @@ -271,7 +271,7 @@ def __init__(self, **kwargs): super().__init__(**kwargs) params = { "dtype": "uint16", - "gain": 1. # default, usually overridden from yaml + "gain": "!DET.gain" } self.meta.update(params) self.meta.update(kwargs) @@ -307,16 +307,17 @@ def apply_to(self, obj, **kwargs): # Apply the gain value (copy from DarkCurrent) # Note that this does not cater for the case where the gain is given # as a plain dictionary. Should we implement that? - if hasattr(self.cmds["!DET.gain"], "dic"): + configured_gain = from_currsys(self.meta["gain"], self.cmds) + if hasattr(configured_gain, "dic"): dtcr_id = obj.meta[real_colname("id", obj.meta)] - gain = self.cmds["!DET.gain"].dic[dtcr_id] + gain = configured_gain.dic[dtcr_id] logger.info(f"Detector {dtcr_id}: applying gain {gain}") - elif isinstance(self.cmds["!DET.gain"], (float, int)): - gain = self.cmds["!DET.gain"] + elif isinstance(configured_gain, (float, int)): + gain = configured_gain logger.info(f"Applying gain {gain}") else: raise ValueError(".meta['gain'] must be either " - f"dict or float, but is {self.cmds['!DET.gain']}") + f"dict or float, but is {configured_gain}") # Apply gain obj._hdu.data /= gain diff --git a/scopesim/effects/fits_headers.py b/scopesim/effects/fits_headers.py index 9110f3331..987117edf 100644 --- a/scopesim/effects/fits_headers.py +++ b/scopesim/effects/fits_headers.py @@ -153,6 +153,9 @@ class ExtraFitsKeywords(Effect): header['HIERARCH ESO DET DIT'] = (5, '[s] exposure length') header['EXTNAME'] = "DET2.DATA" + A group may additionally specify ``detector_id``. It then applies only to + HDULists containing that ``DET_ID``, including their primary headers. + Resolved and un-resolved keywords --------------------------------- ScopeSim uses bang-strings to resolve global parameters. @@ -285,6 +288,12 @@ def apply_to(self, hdul, **kwargs): return hdul for dic in self.dict_list: + # Each detector array produces its own HDUList. A detector selector + # also permits arm-specific metadata in that array's primary HDU. + if "detector_id" in dic: + detector_ids = [hdu.header["DET_ID"] for hdu in hdul if "DET_ID" in hdu.header] + if dic["detector_id"] not in detector_ids: + continue resolved = flatten_dict(dic.get("keywords", {}), resolve=True, optics_manager=opt_train) unresolved = flatten_dict(dic.get("unresolved_keywords", {})) diff --git a/scopesim/tests/tests_effects/test_detector_arm_configuration.py b/scopesim/tests/tests_effects/test_detector_arm_configuration.py new file mode 100644 index 000000000..be090afde --- /dev/null +++ b/scopesim/tests/tests_effects/test_detector_arm_configuration.py @@ -0,0 +1,54 @@ +import numpy as np +import pytest +from scopesim.detector import Detector +from scopesim.optics.image_plane_utils import header_from_list_of_xy +from astropy.io import fits +from scopesim.commands import UserCommands +from scopesim.effects import DetectorList, ExtraFitsKeywords, DetectorModePropertiesSetter +from scopesim.effects.electronic import ADConversion + + +def detector(nx=120, ny=80): + header = header_from_list_of_xy([-.015*nx/2, .015*nx/2], [-.015*ny/2, .015*ny/2], .015, 'D') + return Detector(header) + + + +def test_gain_uses_selected_arm_and_missing_gain_fails(): + det = detector() + det.data[:] = 12 + ADConversion(gain='!DET.gain_uvb', dtype='float32', cmds={'!DET.gain_uvb': .6}).apply_to(det) + np.testing.assert_allclose(det.data, 20) + with pytest.raises(ValueError, match='gain_uvb'): + ADConversion(gain='!DET.gain_uvb', dtype='float32', cmds={}).apply_to(det) + + + +def test_detector_specific_primary_headers(): + eff = ExtraFitsKeywords(header_dict=[{'detector_id': 1, 'ext_number': [0, 1], 'keywords': {'ARM': 'VIS'}}]) + for detector_id in [0, 1, 2]: + hdul = fits.HDUList([fits.PrimaryHDU(), fits.ImageHDU(header=fits.Header({'DET_ID': detector_id}))]) + eff.apply_to(hdul) + assert ('ARM' in hdul[0].header) == (detector_id == 1) + + + +def test_detector_layout_sets_real_id_header(): + effect = DetectorList(array_dict={'id': [2], 'x_cen': [0.], 'y_cen': [0.], 'x_size': [16], 'y_size': [8], 'pixel_size': [.018], 'angle': [0.], 'gain': [2.29]}, x_cen_unit='mm', y_cen_unit='mm', x_size_unit='pixel', y_size_unit='pixel', pixel_size_unit='mm', angle_unit='deg', gain_unit='electron/adu') + header = effect.detector_headers()[0] + assert header['DET_ID'] == 2 + assert header['EXTNAME'] == 'DET_2' + det = Detector(header) + assert det.meta['ID'] == 2 + assert det.hdu.header['DET_ID'] == 2 + + + +def test_independent_detector_readout_modes(): + cmds = UserCommands(properties={'!DET.mode_blue': 'slow', '!DET.mode_red': 'fast'}) + for arm in ['blue', 'red']: + modes = {'slow': {f'!DET.gain_{arm}': .6}, 'fast': {f'!DET.gain_{arm}': 1.8}} + DetectorModePropertiesSetter(mode_properties=modes, mode_key=f'!DET.mode_{arm}', cmds=cmds).apply_to(None) + assert cmds['!DET.gain_blue'] == .6 + assert cmds['!DET.gain_red'] == 1.8 + From 0ac61b83d91b41cf1588624190f3e1234b09a7c0 Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Fri, 16 Oct 2026 20:43:12 -0700 Subject: [PATCH 092/130] Add charge binning with focal-plane WCS and repeatable detector readouts --- scopesim/detector/detector.py | 10 ++-- scopesim/effects/electronic/__init__.py | 2 +- scopesim/effects/electronic/pixels.py | 49 +++++++++++++++ scopesim/optics/optical_train.py | 3 +- .../tests/tests_effects/test_OnChipBinning.py | 59 +++++++++++++++++++ 5 files changed, 115 insertions(+), 8 deletions(-) create mode 100644 scopesim/tests/tests_effects/test_OnChipBinning.py diff --git a/scopesim/detector/detector.py b/scopesim/detector/detector.py index 2b0e43ed8..4db025f53 100644 --- a/scopesim/detector/detector.py +++ b/scopesim/detector/detector.py @@ -1,5 +1,3 @@ -import numpy as np - from astropy.io.fits import ImageHDU from astropy.wcs import WCS @@ -14,6 +12,7 @@ class Detector: def __init__(self, header, cmds=None, **kwargs): + self._native_header = header.copy() self._hdu = ImageHDU(header=header, data=zeros_from_header(header)) self.meta = {} self.meta.update(header) @@ -40,10 +39,9 @@ def extract_from(self, image_plane, spline_order=1, reset=True): self.header.update(sky_wcs.to_header()) def reset(self): - """Reset internal HDU data to all-zeros-array.""" - # The detector might have been converted to integers by the - # Quantization effect, so it is not possible to use zeros_like. - self._hdu.data = np.zeros(self._hdu.data.shape) + """Reset charge and restore the configured, unbinned detector geometry.""" + self._hdu = ImageHDU(header=self._native_header, data=zeros_from_header(self._native_header)) + self.meta.update(self._native_header) @property def hdu(self): diff --git a/scopesim/effects/electronic/__init__.py b/scopesim/effects/electronic/__init__.py index 80a5ea6db..64db13abf 100644 --- a/scopesim/effects/electronic/__init__.py +++ b/scopesim/effects/electronic/__init__.py @@ -26,7 +26,7 @@ from .noise import (Bias, PoorMansHxRGReadoutNoise, BasicReadoutNoise, PixelResponseNonUniformity, ShotNoise, DarkCurrent) from .exposure import AutoExposure, ExposureIntegration, ExposureOutput -from .pixels import ReferencePixelBorder, BinnedImage, UnequalBinnedImage +from .pixels import ReferencePixelBorder, BinnedImage, UnequalBinnedImage, OnChipBinning from .dmps import DetectorModePropertiesSetter diff --git a/scopesim/effects/electronic/pixels.py b/scopesim/effects/electronic/pixels.py index 5c8c8c8cc..eeb72f9f0 100644 --- a/scopesim/effects/electronic/pixels.py +++ b/scopesim/effects/electronic/pixels.py @@ -131,3 +131,52 @@ def apply_to(self, det, **kwargs): det._hdu.data = new_image.sum(axis=3).sum(axis=1) return det + + +class OnChipBinning(Effect): + """Sum charge in FITS x/y pixel blocks before adding read noise. + + Unlike a software sum after readout, each output pixel receives one read + noise draw. Configure this effect between charge integration and read noise. + WCS reference pixels refer to the centers of the new pixel blocks. + """ + + required_keys = {"binx", "biny"} + z_order = (815,) + + def __init__(self, **kwargs): + super().__init__(**kwargs) + check_keys(self.meta, self.required_keys, action="error") + + def apply_to(self, det, **kwargs): + if not isinstance(det, Detector): + return det + bx = from_currsys(self.meta["binx"], self.cmds) + by = from_currsys(self.meta["biny"], self.cmds) + if not isinstance(bx, int) or not isinstance(by, int) or min(bx, by) < 1: + raise ValueError("On-chip bin factors must be positive integers") + if "allowed_bins" in self.meta and [bx, by] not in self.meta["allowed_bins"]: + raise ValueError(f"Unsupported on-chip binning: {bx} x {by}") + ny, nx = det.data.shape + det._hdu.data = det.data.reshape(ny // by, by, nx // bx, bx).sum(axis=(1, 3)) + header = det.header + for suffix in [""] + list("ABCDEFGHIJKLMNOPQRSTUVWXYZ"): + if f"CRPIX1{suffix}" not in header: + continue + for axis, factor in [(1, bx), (2, by)]: + header[f"CRPIX{axis}{suffix}"] = (header[f"CRPIX{axis}{suffix}"] - 0.5) / factor + 0.5 + if f"CD1_1{suffix}" in header: + for i in (1, 2): + for j, factor in [(1, bx), (2, by)]: + header[f"CD{i}_{j}{suffix}"] *= factor + else: + header[f"CDELT1{suffix}"] *= bx + header[f"CDELT2{suffix}"] *= by + if f"PC1_2{suffix}" in header: + header[f"PC1_2{suffix}"] *= by / bx + if f"PC2_1{suffix}" in header: + header[f"PC2_1{suffix}"] *= bx / by + header["BINX"] = bx + header["BINY"] = by + det.meta.update(header) + return det diff --git a/scopesim/optics/optical_train.py b/scopesim/optics/optical_train.py index 8ef81cd9c..ad3504013 100644 --- a/scopesim/optics/optical_train.py +++ b/scopesim/optics/optical_train.py @@ -226,6 +226,7 @@ def trace_detector_coordinates(self, *, xi=0 * u.arcsec, """ from ..effects import ( BinnedImage, + OnChipBinning, Rotate90CCD, SelectorWheel, SpectralTraceList, @@ -333,7 +334,7 @@ def image_origin_on_canvas(shape_xy, image_wcs, canvas_wcs): raise NotImplementedError( "Trace detector coordinates do not yet support " "post-extraction detector binning.") - if isinstance(selected_effect, UnequalBinnedImage): + if isinstance(selected_effect, (UnequalBinnedImage, OnChipBinning)): binx = from_currsys( selected_effect.meta["binx"], self.cmds) biny = from_currsys( diff --git a/scopesim/tests/tests_effects/test_OnChipBinning.py b/scopesim/tests/tests_effects/test_OnChipBinning.py new file mode 100644 index 000000000..477c7457d --- /dev/null +++ b/scopesim/tests/tests_effects/test_OnChipBinning.py @@ -0,0 +1,59 @@ +import numpy as np +import pytest +from scopesim.detector import Detector +from scopesim.optics.image_plane_utils import header_from_list_of_xy +from astropy.wcs import WCS +from scopesim.effects.electronic import OnChipBinning, BasicReadoutNoise + + +def detector(nx=120, ny=80): + header = header_from_list_of_xy([-.015*nx/2, .015*nx/2], [-.015*ny/2, .015*ny/2], .015, 'D') + return Detector(header) + + + +@pytest.mark.parametrize('bx,by', [(1, 1), (1, 2), (2, 2)]) +def test_binning_charge_and_focal_coordinates(bx, by): + det = detector() + det.data[:] = 7 + original = WCS(det.header, key='D') + OnChipBinning(binx=bx, biny=by).apply_to(det) + assert det.data.shape == (80//by, 120//bx) + assert np.all(det.data == 7*bx*by) + assert det.data.sum() == 7*120*80 + binned = WCS(det.hdu.header, key='D') + expected = original.all_pix2world((np.array([10, 20])+.5)*bx-.5, (np.array([12, 23])+.5)*by-.5, 0) + np.testing.assert_allclose(binned.all_pix2world([10, 20], [12, 23], 0), expected, atol=1e-12) + det.reset() + assert det.data.shape == (80, 120) + assert det.header['CDELT2D'] == .015 + OnChipBinning(binx=bx, biny=by).apply_to(det) + assert det.data.shape == (80//by, 120//bx) + + + +def test_binning_rotated_cd_matrix(): + det = detector() + det.header['CD1_1D'], det.header['CD1_2D'] = .012, -.009 + det.header['CD2_1D'], det.header['CD2_2D'] = .009, .012 + wcs = WCS(det.header, key='D') + OnChipBinning(binx=1, biny=2).apply_to(det) + actual = WCS(det.header, key='D').all_pix2world(9, 15, 0) + np.testing.assert_allclose(actual, wcs.all_pix2world(9, 30.5, 0), atol=1e-12) + + + +def test_binning_rejects_unsupported_modes(): + with pytest.raises(ValueError, match='Unsupported'): + OnChipBinning(binx=2, biny=1, allowed_bins=[[1, 1], [1, 2], [2, 2]]).apply_to(detector()) + with pytest.raises(ValueError): + OnChipBinning(binx=1, biny=3).apply_to(detector()) + + + +def test_ccd_bin_does_not_multiply_amplifier_noise(): + det = detector(600, 400) + OnChipBinning(binx=2, biny=2).apply_to(det) + BasicReadoutNoise(noise_std=3, ndit=1, random_seed=17).apply_to(det) + assert det.data.std() == pytest.approx(3, rel=.02) + From ef55346d5bd2d8e19fc50bd5602bc25ca7a01069 Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Fri, 16 Oct 2026 20:43:12 -0700 Subject: [PATCH 093/130] Interpolate measured read noise by DIT and record total exposure time --- scopesim/effects/electronic/__init__.py | 2 +- scopesim/effects/electronic/exposure.py | 1 + scopesim/effects/electronic/noise.py | 17 ++++++++++++ .../test_ReadoutNoiseFromTable.py | 26 +++++++++++++++++++ 4 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 scopesim/tests/tests_effects/test_ReadoutNoiseFromTable.py diff --git a/scopesim/effects/electronic/__init__.py b/scopesim/effects/electronic/__init__.py index 64db13abf..1e551139a 100644 --- a/scopesim/effects/electronic/__init__.py +++ b/scopesim/effects/electronic/__init__.py @@ -24,7 +24,7 @@ from .electrons import LinearityCurve, ADConversion, InterPixelCapacitance from .noise import (Bias, PoorMansHxRGReadoutNoise, BasicReadoutNoise, - PixelResponseNonUniformity, ShotNoise, DarkCurrent) + PixelResponseNonUniformity, ShotNoise, DarkCurrent, ReadoutNoiseFromTable) from .exposure import AutoExposure, ExposureIntegration, ExposureOutput from .pixels import ReferencePixelBorder, BinnedImage, UnequalBinnedImage, OnChipBinning from .dmps import DetectorModePropertiesSetter diff --git a/scopesim/effects/electronic/exposure.py b/scopesim/effects/electronic/exposure.py index 8ff88e1e7..05da1a208 100644 --- a/scopesim/effects/electronic/exposure.py +++ b/scopesim/effects/electronic/exposure.py @@ -277,5 +277,6 @@ def apply_to(self, obj, **kwargs): ) obj._hdu.data *= dit * ndit + obj.header["EXPTIME"] = (dit * ndit, "Total accumulated integration time [s]") return obj diff --git a/scopesim/effects/electronic/noise.py b/scopesim/effects/electronic/noise.py index 635b5f238..ecfebf54b 100644 --- a/scopesim/effects/electronic/noise.py +++ b/scopesim/effects/electronic/noise.py @@ -139,6 +139,23 @@ def plot_hist(self, det, **kwargs): ax.hist(dtcr.data.flatten()) +class ReadoutNoiseFromTable(BasicReadoutNoise): + """Interpolate measured read noise in electrons as a function of DIT [s].""" + + required_keys = {"filename", "dit", "ndit"} + + def apply_to(self, det, **kwargs): + if not isinstance(det, Detector): + return det + dit = from_currsys(self.meta["dit"], self.cmds) + times = self.table["dit"] + if not times[0] <= dit <= times[-1]: + raise ValueError(f"DIT {dit} s is outside the measured read-noise table") + self.meta["noise_std"] = np.interp(dit, times, self.table["read_noise"]) + det.header["RDNOISE"] = (self.meta["noise_std"], "Read noise per integration [electron]") + return super().apply_to(det, **kwargs) + + # TODO: Is this really a "noise" effect? Sounds more like "electrons" tbh. class PixelResponseNonUniformity(Effect): """Pixel Response Non-Uniformity (PRNU). diff --git a/scopesim/tests/tests_effects/test_ReadoutNoiseFromTable.py b/scopesim/tests/tests_effects/test_ReadoutNoiseFromTable.py new file mode 100644 index 000000000..2ed1fdb8f --- /dev/null +++ b/scopesim/tests/tests_effects/test_ReadoutNoiseFromTable.py @@ -0,0 +1,26 @@ +import numpy as np +import pytest +from scopesim.detector import Detector +from scopesim.optics.image_plane_utils import header_from_list_of_xy +from scopesim.effects.electronic import ReadoutNoiseFromTable, ExposureOutput + + +def detector(nx=120, ny=80): + header = header_from_list_of_xy([-.015*nx/2, .015*nx/2], [-.015*ny/2, .015*ny/2], .015, 'D') + return Detector(header) + + + +@pytest.mark.parametrize('ndit', [1, 4, 9]) +def test_nir_averaging_preserves_signal_and_reduces_noise(tmp_path, ndit): + noise_file = tmp_path / 'noise.dat' + noise_file.write_text('dit read_noise\n10 12\n100 6\n') + det = detector(500, 400) + det.data[:] = 30*ndit + ReadoutNoiseFromTable(filename=str(noise_file), dit=55, ndit=ndit, random_seed=21).apply_to(det) + ExposureOutput(dit=55, ndit=ndit, mode='average').apply_to(det) + assert det.data.mean() == pytest.approx(30, abs=.1) + assert det.data.std() == pytest.approx(9/np.sqrt(ndit), rel=.02) + with pytest.raises(ValueError, match='outside'): + ReadoutNoiseFromTable(filename=str(noise_file), dit=101, ndit=ndit).apply_to(det) + From fe052a1da00c3e016b1e5200fba43753033e3986 Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Fri, 16 Oct 2026 20:43:12 -0700 Subject: [PATCH 094/130] Accept calibrated slit dispersion and optical line spread in spectral traces --- scopesim/effects/spectral_trace_list.py | 7 +- scopesim/effects/spectral_trace_list_utils.py | 65 +++++++++++++++---- .../test_calibrated_spectral_trace.py | 44 +++++++++++++ 3 files changed, 98 insertions(+), 18 deletions(-) create mode 100644 scopesim/tests/tests_effects/test_calibrated_spectral_trace.py diff --git a/scopesim/effects/spectral_trace_list.py b/scopesim/effects/spectral_trace_list.py index 7d4c6a7ce..489790e33 100644 --- a/scopesim/effects/spectral_trace_list.py +++ b/scopesim/effects/spectral_trace_list.py @@ -259,11 +259,8 @@ def apply_to(self, obj, **kwargs): # self.update_meta() spt = self.spectral_traces[obj.trace_id] - try: - # If footprint is outside FOV, this will assign None to the hdu - obj.hdu = spt.map_spectra_to_focal_plane(obj) - except ValueError as err: # xlim_mm is None - logger.error(err) + # If the footprint is outside the FOV, the mapper returns None. + obj.hdu = spt.map_spectra_to_focal_plane(obj) obj.image_plane_id = spt.meta["image_plane_id"] logger.debug("%s done", self.display_name) diff --git a/scopesim/effects/spectral_trace_list_utils.py b/scopesim/effects/spectral_trace_list_utils.py index ad1dd802e..af1feae68 100644 --- a/scopesim/effects/spectral_trace_list_utils.py +++ b/scopesim/effects/spectral_trace_list_utils.py @@ -16,6 +16,7 @@ from scipy.interpolate import RectBivariateSpline from scipy.interpolate import interp1d +from scipy.ndimage import gaussian_filter1d from astropy.table import Table, vstack from astropy.modeling import fitting @@ -105,6 +106,7 @@ class SpectralTrace: "pixel_size": None, "description": "", "trace_flux_jacobian": "!SIM.spectral.trace_flux_jacobian", + "fit_degree": 4, } def __init__(self, trace_tbl, cmds=None, **kwargs): @@ -130,6 +132,9 @@ def __init__(self, trace_tbl, cmds=None, **kwargs): ("PLTSCALE", "plate_scale"), ("DETPAD", "detector_pad"), ("DETANG", "detector_angle"), + ("FITDEG", "fit_degree"), + ("LSFRES", "optical_resolving_power"), + ("SLITCOL", "slit_dispersion_colname"), ): if header_key in trace_tbl.header: self.meta[meta_key] = trace_tbl.header[header_key] @@ -194,12 +199,19 @@ def compute_interpolation_functions(self): self.wave_min = quantify(np.min(lam_arr), u.um).value self.wave_max = quantify(np.max(lam_arr), u.um).value - self.xy2xi = Transform2D.fit(x_arr, y_arr, xi_arr) - self.xy2lam = Transform2D.fit(x_arr, y_arr, lam_arr) - self.xilam2x = Transform2D.fit(xi_arr, lam_arr, x_arr) - self.xilam2y = Transform2D.fit(xi_arr, lam_arr, y_arr) - self._xiy2x = Transform2D.fit(xi_arr, y_arr, x_arr) - self._xiy2lam = Transform2D.fit(xi_arr, y_arr, lam_arr) + degree = self.meta["fit_degree"] + self.xy2xi = Transform2D.fit(x_arr, y_arr, xi_arr, degree=degree) + self.xy2lam = Transform2D.fit(x_arr, y_arr, lam_arr, degree=degree) + self.xilam2x = Transform2D.fit(xi_arr, lam_arr, x_arr, degree=degree) + self.xilam2y = Transform2D.fit(xi_arr, lam_arr, y_arr, degree=degree) + self._xiy2x = Transform2D.fit(xi_arr, y_arr, x_arr, degree=degree) + self._xiy2lam = Transform2D.fit(xi_arr, y_arr, lam_arr, degree=degree) + if "slit_dispersion_colname" in self.meta: + center = xi_arr == 0 + derivative = self.table[self.meta["slit_dispersion_colname"]].quantity.to_value(u.um / u.arcsec) + # FOVs are restricted to this trace; allow unit-conversion roundoff + # at the wavelength endpoints when interpolating the derivative. + self.slit_dispersion = interp1d(lam_arr[center], derivative[center], bounds_error=False, fill_value="extrapolate") if self.dispersion_axis == "unknown": dlam_dx, dlam_dy = self.xy2lam.gradient() @@ -293,11 +305,13 @@ def map_spectra_to_focal_plane(self, fov): image = np.zeros((sub_naxis2, sub_naxis1), dtype=np.float32) self._set_dispersion(wave_min, wave_max, pixsize=pixsize) - try: - xilam = XiLamImage(fov, self.dlam_per_pix) - self._xilamimg = xilam # ..todo: remove or make available with a debug flag? - except ValueError: - logger.warning(" ---> %s gave ValueError", self.trace_id) + slit_dispersion = None + if "slit_dispersion_colname" in self.meta: + slit_dispersion = self.slit_dispersion + xilam = XiLamImage(fov, self.dlam_per_pix, slit_dispersion=slit_dispersion) + if "optical_resolving_power" in self.meta: + xilam.convolve_lsf(self.meta["optical_resolving_power"]) + self._xilamimg = xilam npix_xi, npix_lam = xilam.npix_xi, xilam.npix_lam xilam_wcs = xilam.wcs @@ -848,9 +862,13 @@ class XiLamImage(): dlam_per_pix : a 1-D interpolation function from wavelength (in um) to dispersion (in um/pixel); alternatively a number giving an average dispersion + slit_dispersion : callable, optional + Physical cross-slit derivative d(lambda)/d(eta) in um/arcsec as a + function of wavelength in um. When supplied, this replaces the + legacy coupling between sky-grid sampling and detector dispersion. """ - def __init__(self, fov, dlam_per_pix): + def __init__(self, fov, dlam_per_pix, slit_dispersion=None): # ..todo: we assume that we always have a cube. We use SpecCADO's # add_cube_layer method cube_wcs = WCS(fov.cube.header, key=" ") @@ -883,8 +901,11 @@ def __init__(self, fov, dlam_per_pix): logger.warning("Using scalar dlam_per_pix = %.2g", dlam_per_pix_val) + dlam_per_arcsec = dlam_per_pix_val / d_eta if slit_dispersion is None else slit_dispersion(self.lam) for i, eta in enumerate(cube_eta): - lam0 = self.lam + dlam_per_pix_val * eta / d_eta + # A positive calibrated derivative moves an input line to a larger + # apparent wavelength: sample the source at lambda_out - shift. + lam0 = self.lam + dlam_per_arcsec * eta if slit_dispersion is None else self.lam - dlam_per_arcsec * eta # lam0 is the target wavelength. We need to check that this # overlaps with the wavelength range covered by the cube @@ -929,6 +950,24 @@ def __init__(self, fov, dlam_per_pix): kx=spline_order[0], ky=spline_order[1]) + def convolve_lsf(self, resolving_power): + """Apply an optical Gaussian LSF at constant R, preserving photon density. + + This is the optical blur alone; slit illumination is integrated above. + Convolving lambda * I_lambda on a uniform log(lambda) grid preserves + integrated flux rather than treating I_lambda as a log-bin count. + """ + if resolving_power <= 0: + raise ValueError("Optical resolving power must be positive") + log_wave = np.log(self.lam) + n = int(np.ceil((log_wave[-1] - log_wave[0]) / np.min(np.diff(log_wave)))) + 1 + log_grid = np.linspace(log_wave[0], log_wave[-1], n) + sigma = 1 / (resolving_power * np.sqrt(8 * np.log(2)) * (log_grid[1] - log_grid[0])) + density = interp1d(log_wave, self.image * self.lam, axis=1)(log_grid) + blurred = gaussian_filter1d(density, sigma, axis=1, mode="nearest") + self.image = interp1d(log_grid, blurred, axis=1)(log_wave) / self.lam + self.interp = RectBivariateSpline(self.xi, self.lam, self.image, kx=1, ky=1) + class Transform2D(): """ diff --git a/scopesim/tests/tests_effects/test_calibrated_spectral_trace.py b/scopesim/tests/tests_effects/test_calibrated_spectral_trace.py new file mode 100644 index 000000000..858a6c2e9 --- /dev/null +++ b/scopesim/tests/tests_effects/test_calibrated_spectral_trace.py @@ -0,0 +1,44 @@ +from types import SimpleNamespace +import numpy as np +import pytest +from astropy import units as u +from astropy.io import fits +from scopesim.effects.spectral_trace_list_utils import XiLamImage + + +def line_cube(step, offset=0): + wave = np.linspace(.995, 1.005, 2001) + eta = np.arange(-.5+step/2, .5, step) + spatial = np.exp(-.5*((eta-offset)/.12)**2) + profile = np.exp(-.5*((wave-1)/.00002)**2) + cube = profile[:, None, None]*spatial[None, :, None]*np.ones((1, 1, 3)) + header = fits.Header({'CTYPE1': 'LINEAR', 'CTYPE2': 'LINEAR', 'CTYPE3': 'WAVE', 'CUNIT1': 'arcsec', 'CUNIT2': 'arcsec', 'CUNIT3': 'um', 'CDELT1': .1, 'CDELT2': step, 'CDELT3': wave[1]-wave[0], 'CRPIX1': 1., 'CRPIX2': 1., 'CRPIX3': 1., 'CRVAL1': 0., 'CRVAL2': eta[0], 'CRVAL3': wave[0]}) + return SimpleNamespace(cube=fits.ImageHDU(cube, header), meta={'xi_min': 0*u.arcsec}) + + + +def test_slit_dispersion_is_independent_of_numerical_sky_grid(): + spectra = [XiLamImage(line_cube(step), lambda w: .00001, slit_dispersion=lambda w: .0002) for step in [.1, .05, .025]] + # Finite spatial quadrature changes, while the physical slit dispersion stays fixed. + for image in spectra[:-1]: + np.testing.assert_allclose(image.image, spectra[-1].image, atol=.001, rtol=.02) + + + +def test_calibrated_slit_shift_has_forward_model_sign(): + image = XiLamImage(line_cube(.025, offset=.1), lambda w: .00001, slit_dispersion=lambda w: .002) + centroid = np.sum(image.image[1]*image.lam)/image.image[1].sum() + assert centroid == pytest.approx(1.0002, abs=2e-6) + + + +def test_optical_lsf_preserves_photons_and_adds_variance(): + image = XiLamImage(line_cube(.05), lambda w: 0., slit_dispersion=lambda w: 0.) + before = np.trapezoid(image.image[1], image.lam) + image.convolve_lsf(10000) + after = np.trapezoid(image.image[1], image.lam) + assert after == pytest.approx(before, rel=.001) + center = np.sum(image.lam*image.image[1])/image.image[1].sum() + variance = np.sum((image.lam-center)**2*image.image[1])/image.image[1].sum() + assert variance == pytest.approx(.00002**2 + (1/10000/np.sqrt(8*np.log(2)))**2, rel=.02) + From c76a4c2102b79611a4f84aeafaf900b18b1b0166 Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Fri, 16 Oct 2026 20:43:12 -0700 Subject: [PATCH 095/130] Load registered detector-background maps from named FITS extensions --- scopesim/effects/illumination.py | 11 +++++-- .../test_registered_background.py | 33 +++++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 scopesim/tests/tests_effects/test_registered_background.py diff --git a/scopesim/effects/illumination.py b/scopesim/effects/illumination.py index d4ff12bfe..65b5b8181 100644 --- a/scopesim/effects/illumination.py +++ b/scopesim/effects/illumination.py @@ -538,6 +538,7 @@ class ImagePlaneBackground(Effect): trace list. The supplied value or model should already include downstream throughput and detector QE. For tapered QE maps, use an average or otherwise representative positional QE when deriving this non-dispersed background. + A FITS ``filename`` and ``extname`` can supply an already registered map. """ z_order: ClassVar[tuple[int, ...]] = (760,) @@ -562,7 +563,7 @@ def apply_to(self, obj, **kwargs): return obj shape = obj.hdu.data.shape - if self._map is None or shape != self._map_shape: + if self._file is not None or self._map is None or shape != self._map_shape: self._map = self._make_map(shape) self._map_shape = shape @@ -570,7 +571,13 @@ def apply_to(self, obj, **kwargs): return obj def _make_map(self, shape): - if self._model is None: + if self._file is not None: + # The calibration image must be registered to this image plane. + extname = str(from_currsys(self.meta["extname"], self.cmds)) + background = self._file[extname].data + if background.shape != shape: + raise ValueError("Background calibration shape differs from the image plane") + elif self._model is None: value = from_currsys(self.meta["value"], self.cmds) background = np.full(shape, value, dtype=np.float32) else: diff --git a/scopesim/tests/tests_effects/test_registered_background.py b/scopesim/tests/tests_effects/test_registered_background.py new file mode 100644 index 000000000..7c5964159 --- /dev/null +++ b/scopesim/tests/tests_effects/test_registered_background.py @@ -0,0 +1,33 @@ +import numpy as np +import pytest +from scopesim.detector import Detector +from scopesim.optics.image_plane_utils import header_from_list_of_xy +from astropy.io import fits +from scopesim.effects import ImagePlaneBackground +from scopesim.optics.image_plane import ImagePlane + + +def detector(nx=120, ny=80): + header = header_from_list_of_xy([-.015*nx/2, .015*nx/2], [-.015*ny/2, .015*ny/2], .015, 'D') + return Detector(header) + + + +def test_background_uses_current_named_calibration(tmp_path): + path = tmp_path / 'background.fits' + fits.HDUList([fits.PrimaryHDU(), fits.ImageHDU(np.full((80, 120), .15), name='0.9'), fits.ImageHDU(np.full((80, 120), .04), name='0.9JH')]).writeto(path) + cmds = {'!INST.slit': '0.9'} + effect = ImagePlaneBackground(filename=str(path), extname='!INST.slit', cmds=cmds) + plane = ImagePlane(detector().header) + effect.apply_to(plane) + np.testing.assert_allclose(plane.data, .15) + cmds['!INST.slit'] = '0.9JH' + plane.data[:] = 0 + effect.apply_to(plane) + np.testing.assert_allclose(plane.data, .04) + with pytest.raises(ValueError, match='shape'): + effect.apply_to(ImagePlane(detector(60, 40).header)) + cmds['!INST.slit'] = 'missing' + with pytest.raises(KeyError): + effect.apply_to(plane) + From 93237d8531e06fd081580db7f87ff715c53f0eaf Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Fri, 16 Oct 2026 20:43:12 -0700 Subject: [PATCH 096/130] Honor wavelength units in cached SkyCalc spectra --- scopesim/effects/ter_curves.py | 5 +++- .../tests/tests_effects/test_local_skycalc.py | 25 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 scopesim/tests/tests_effects/test_local_skycalc.py diff --git a/scopesim/effects/ter_curves.py b/scopesim/effects/ter_curves.py index 340af7eab..d60da9ae9 100644 --- a/scopesim/effects/ter_curves.py +++ b/scopesim/effects/ter_curves.py @@ -520,9 +520,12 @@ def load_skycalc_table(self): else: path = find_file(use_local_file) - fits_tbl = fits.getdata(path, ext=1) + with fits.open(path) as sky_file: + fits_tbl = sky_file[1].data + wave_unit = u.Unit(sky_file[1].columns["lam"].unit) fits_hdr = fits.getheader(path, ext=0) tbl = Table(fits_tbl) + tbl["lam"] = tbl["lam"] * wave_unit.to(u.um) tbl["lam"].unit = u.um for colname in tbl.colnames: if "flux" in colname: diff --git a/scopesim/tests/tests_effects/test_local_skycalc.py b/scopesim/tests/tests_effects/test_local_skycalc.py new file mode 100644 index 000000000..fd27af994 --- /dev/null +++ b/scopesim/tests/tests_effects/test_local_skycalc.py @@ -0,0 +1,25 @@ +import numpy as np +import pytest +from astropy import units as u +from astropy.table import Table +from scopesim.effects import SkycalcTERCurve + + +@pytest.mark.parametrize('wave_unit', [u.nm, u.um]) +def test_local_skycalc_respects_wavelength_unit(tmp_path, wave_unit): + wave = np.array([.5, 1., 2.]) * u.um + table = Table([wave.to(wave_unit), [.5, .8, .9], [100., 200., 300.]], names=['lam', 'trans', 'flux']) + path = tmp_path / 'sky.fits' + table.write(path) + sky = SkycalcTERCurve(use_local_skycalc_file=str(path)) + np.testing.assert_allclose(sky.skycalc_table['wavelength'], wave.value) + assert sky.surface.transmission(1*u.um).value == pytest.approx(.8) + + + +def test_skycalc_missing_wavelength_unit_fails(tmp_path): + path = tmp_path / 'sky.fits' + Table([[500., 1000., 2000.], [.5, .8, .9], [100., 200., 300.]], names=['lam', 'trans', 'flux']).write(path) + with pytest.raises(TypeError, match='Unit'): + SkycalcTERCurve(use_local_skycalc_file=str(path)) + From b6f31f6aa37add3f237c56c3619580d6889ecd25 Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Fri, 16 Oct 2026 20:43:12 -0700 Subject: [PATCH 097/130] Interpolate atmospheric shifts plane by plane without spectral mixing --- scopesim/effects/atmo_dispersion.py | 39 +++++-------------- .../tests/tests_effects/test_shift_planes.py | 15 +++++++ 2 files changed, 24 insertions(+), 30 deletions(-) create mode 100644 scopesim/tests/tests_effects/test_shift_planes.py diff --git a/scopesim/effects/atmo_dispersion.py b/scopesim/effects/atmo_dispersion.py index 268cd7fee..40455ae01 100644 --- a/scopesim/effects/atmo_dispersion.py +++ b/scopesim/effects/atmo_dispersion.py @@ -131,7 +131,7 @@ def roll_shift(data, dx, dy): def map_coords_shift(data, dx, dy, chunk_w: int=8, order: int=3): """ Shift each wavelength plane data[w, :, :] by (dx[w], dy[w]) using - scipy.ndimage.map_coordinates with chunked processing. + scipy.ndimage.map_coordinates independently on each spatial plane. Parameters ---------- @@ -142,7 +142,7 @@ def map_coords_shift(data, dx, dy, chunk_w: int=8, order: int=3): dy : np.ndarray, shape (nwave,) Shift along y (axis=0) in pixels. Positive moves content toward +y. chunk_w : int - Number of wavelength planes to process per chunk. Default 8. + Retained for call compatibility; planes are now processed separately. order : int Spline interpolation order. Default 3. @@ -156,35 +156,14 @@ def map_coords_shift(data, dx, dy, chunk_w: int=8, order: int=3): if dx.shape != (nw,) or dy.shape != (nw,): raise ValueError("dx and dy must have shape (nwave,)") - # Base grids for a single chunk (will be broadcast) - yi = np.arange(ny, dtype=np.float32)[None, :, None] # (1, ny, 1) - xi = np.arange(nx, dtype=np.float32)[None, None, :] # (1, 1, nx) - + yi, xi = np.indices((ny, nx), dtype=np.float32) out = np.empty_like(data) - prefilter = (order > 1) - - for w0 in range(0, nw, chunk_w): - w1 = min(w0 + chunk_w, nw) - k = w1 - w0 - - # wave index grid for this chunk: (k, 1, 1) - wi = np.arange(w0, w1, dtype=np.float32)[:, None, None] - - # To shift content by +dx/+dy, sample input at (x - dx, y - dy) - xcoords = xi - dx[w0:w1][:, None, None] # (k, 1, nx) -> broadcast to (k, ny, nx) - ycoords = yi - dy[w0:w1][:, None, None] # (k, ny, 1) -> broadcast to (k, ny, nx) - - # Broadcast to full chunk volume - xcoords = np.broadcast_to(xcoords, (k, ny, nx)) - ycoords = np.broadcast_to(ycoords, (k, ny, nx)) - wcoords = np.broadcast_to(wi, (k, ny, nx)) - - # Axis order in coords must match axis order of data: (wave, y, x) - coords = np.array([wcoords, ycoords, xcoords], dtype=np.float32) # (3, k, ny, nx) - - out[w0:w1, :, :] = map_coordinates( - data, coords, order=order, mode='nearest', prefilter=prefilter - ) + # Atmospheric refraction shifts sky positions without mixing wavelengths. + # Filtering the full cube once per small output chunk scales quadratically + # with the spectral grid; spatial-plane interpolation is linear in nw. + for wave in range(nw): + coords = [yi - dy[wave], xi - dx[wave]] + out[wave] = map_coordinates(data[wave], coords, order=order, mode='nearest', prefilter=order > 1) return out diff --git a/scopesim/tests/tests_effects/test_shift_planes.py b/scopesim/tests/tests_effects/test_shift_planes.py new file mode 100644 index 000000000..25bca3181 --- /dev/null +++ b/scopesim/tests/tests_effects/test_shift_planes.py @@ -0,0 +1,15 @@ +import numpy as np +from scipy.ndimage import map_coordinates +from scopesim.effects.atmo_dispersion import ShiftFoV3D + + +def test_dispersion_does_not_mix_wavelengths(): + data = np.zeros((19, 15, 17)) + data[:, 7, 8] = np.arange(19) + dx, dy = np.linspace(-.2, .2, 19), np.linspace(.4, .6, 19) + shifted = ShiftFoV3D.map_coords_shift(data, dx, dy) + yy, xx = np.indices(data.shape[1:]) + for wave in range(19): + expected = map_coordinates(data[wave], [yy-dy[wave], xx-dx[wave]], order=3, mode='nearest') + np.testing.assert_allclose(shifted[wave], expected, atol=2e-5) + From c0e0dfdc42df8972be4f4458abdf7d757bd54273 Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Fri, 16 Oct 2026 20:46:51 -0700 Subject: [PATCH 098/130] Require explicit wavelength units for legacy unitless SkyCalc files --- scopesim/effects/ter_curves.py | 4 +++- scopesim/tests/tests_effects/test_OnChipBinning.py | 1 - scopesim/tests/tests_effects/test_ReadoutNoiseFromTable.py | 1 - scopesim/tests/tests_effects/test_SkycalcTERCurve.py | 4 +++- .../tests/tests_effects/test_calibrated_spectral_trace.py | 1 - .../tests/tests_effects/test_detector_arm_configuration.py | 1 - scopesim/tests/tests_effects/test_local_skycalc.py | 1 - scopesim/tests/tests_effects/test_registered_background.py | 1 - scopesim/tests/tests_effects/test_shift_planes.py | 1 - 9 files changed, 6 insertions(+), 9 deletions(-) diff --git a/scopesim/effects/ter_curves.py b/scopesim/effects/ter_curves.py index d60da9ae9..e5eb929b8 100644 --- a/scopesim/effects/ter_curves.py +++ b/scopesim/effects/ter_curves.py @@ -522,7 +522,9 @@ def load_skycalc_table(self): path = find_file(use_local_file) with fits.open(path) as sky_file: fits_tbl = sky_file[1].data - wave_unit = u.Unit(sky_file[1].columns["lam"].unit) + # Older cached files require an explicit unit if TUNIT is absent. + unit = self.meta["local_skycalc_wavelength_unit"] if "local_skycalc_wavelength_unit" in self.meta else sky_file[1].columns["lam"].unit + wave_unit = u.Unit(unit) fits_hdr = fits.getheader(path, ext=0) tbl = Table(fits_tbl) tbl["lam"] = tbl["lam"] * wave_unit.to(u.um) diff --git a/scopesim/tests/tests_effects/test_OnChipBinning.py b/scopesim/tests/tests_effects/test_OnChipBinning.py index 477c7457d..06ed0a62d 100644 --- a/scopesim/tests/tests_effects/test_OnChipBinning.py +++ b/scopesim/tests/tests_effects/test_OnChipBinning.py @@ -56,4 +56,3 @@ def test_ccd_bin_does_not_multiply_amplifier_noise(): OnChipBinning(binx=2, biny=2).apply_to(det) BasicReadoutNoise(noise_std=3, ndit=1, random_seed=17).apply_to(det) assert det.data.std() == pytest.approx(3, rel=.02) - diff --git a/scopesim/tests/tests_effects/test_ReadoutNoiseFromTable.py b/scopesim/tests/tests_effects/test_ReadoutNoiseFromTable.py index 2ed1fdb8f..ef372b486 100644 --- a/scopesim/tests/tests_effects/test_ReadoutNoiseFromTable.py +++ b/scopesim/tests/tests_effects/test_ReadoutNoiseFromTable.py @@ -23,4 +23,3 @@ def test_nir_averaging_preserves_signal_and_reduces_noise(tmp_path, ndit): assert det.data.std() == pytest.approx(9/np.sqrt(ndit), rel=.02) with pytest.raises(ValueError, match='outside'): ReadoutNoiseFromTable(filename=str(noise_file), dit=101, ndit=ndit).apply_to(det) - diff --git a/scopesim/tests/tests_effects/test_SkycalcTERCurve.py b/scopesim/tests/tests_effects/test_SkycalcTERCurve.py index 689621b10..b87995c81 100644 --- a/scopesim/tests/tests_effects/test_SkycalcTERCurve.py +++ b/scopesim/tests/tests_effects/test_SkycalcTERCurve.py @@ -59,7 +59,9 @@ def test_initialises_with_non_skycalc_keys(self): def test_initialise_with_local_skycalc_file(self, mock_path): sky_ter = SkycalcTERCurve( - use_local_skycalc_file=str(mock_path / "skycalc_override.fits")) + use_local_skycalc_file=str(mock_path / "skycalc_override.fits"), + local_skycalc_wavelength_unit=u.um) + # This legacy fixture predates wavelength units in its FITS columns. assert sky_ter.skycalc_table is not None diff --git a/scopesim/tests/tests_effects/test_calibrated_spectral_trace.py b/scopesim/tests/tests_effects/test_calibrated_spectral_trace.py index 858a6c2e9..f6b8a8eee 100644 --- a/scopesim/tests/tests_effects/test_calibrated_spectral_trace.py +++ b/scopesim/tests/tests_effects/test_calibrated_spectral_trace.py @@ -41,4 +41,3 @@ def test_optical_lsf_preserves_photons_and_adds_variance(): center = np.sum(image.lam*image.image[1])/image.image[1].sum() variance = np.sum((image.lam-center)**2*image.image[1])/image.image[1].sum() assert variance == pytest.approx(.00002**2 + (1/10000/np.sqrt(8*np.log(2)))**2, rel=.02) - diff --git a/scopesim/tests/tests_effects/test_detector_arm_configuration.py b/scopesim/tests/tests_effects/test_detector_arm_configuration.py index be090afde..2b81134c4 100644 --- a/scopesim/tests/tests_effects/test_detector_arm_configuration.py +++ b/scopesim/tests/tests_effects/test_detector_arm_configuration.py @@ -51,4 +51,3 @@ def test_independent_detector_readout_modes(): DetectorModePropertiesSetter(mode_properties=modes, mode_key=f'!DET.mode_{arm}', cmds=cmds).apply_to(None) assert cmds['!DET.gain_blue'] == .6 assert cmds['!DET.gain_red'] == 1.8 - diff --git a/scopesim/tests/tests_effects/test_local_skycalc.py b/scopesim/tests/tests_effects/test_local_skycalc.py index fd27af994..8baab5ba1 100644 --- a/scopesim/tests/tests_effects/test_local_skycalc.py +++ b/scopesim/tests/tests_effects/test_local_skycalc.py @@ -22,4 +22,3 @@ def test_skycalc_missing_wavelength_unit_fails(tmp_path): Table([[500., 1000., 2000.], [.5, .8, .9], [100., 200., 300.]], names=['lam', 'trans', 'flux']).write(path) with pytest.raises(TypeError, match='Unit'): SkycalcTERCurve(use_local_skycalc_file=str(path)) - diff --git a/scopesim/tests/tests_effects/test_registered_background.py b/scopesim/tests/tests_effects/test_registered_background.py index 7c5964159..54606b9a5 100644 --- a/scopesim/tests/tests_effects/test_registered_background.py +++ b/scopesim/tests/tests_effects/test_registered_background.py @@ -30,4 +30,3 @@ def test_background_uses_current_named_calibration(tmp_path): cmds['!INST.slit'] = 'missing' with pytest.raises(KeyError): effect.apply_to(plane) - diff --git a/scopesim/tests/tests_effects/test_shift_planes.py b/scopesim/tests/tests_effects/test_shift_planes.py index 25bca3181..5f4bcabe1 100644 --- a/scopesim/tests/tests_effects/test_shift_planes.py +++ b/scopesim/tests/tests_effects/test_shift_planes.py @@ -12,4 +12,3 @@ def test_dispersion_does_not_mix_wavelengths(): for wave in range(19): expected = map_coordinates(data[wave], [yy-dy[wave], xx-dx[wave]], order=3, mode='nearest') np.testing.assert_allclose(shifted[wave], expected, atol=2e-5) - From 7ffffb87851c1532eb89880387c1ba49db8f7872 Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Sat, 17 Oct 2026 02:45:32 -0700 Subject: [PATCH 099/130] Use physical slit coordinates and detector-area Jacobians --- scopesim/effects/apertures.py | 4 ++ scopesim/effects/metis_lms_trace_list.py | 2 + scopesim/effects/spectral_trace_list.py | 13 +++++++ scopesim/effects/spectral_trace_list_utils.py | 31 ++++++++-------- .../test_SpectralTraceListUtils.py | 2 +- .../test_calibrated_spectral_trace.py | 37 ++++++++++++++++++- 6 files changed, 71 insertions(+), 18 deletions(-) diff --git a/scopesim/effects/apertures.py b/scopesim/effects/apertures.py index aca858141..199914541 100644 --- a/scopesim/effects/apertures.py +++ b/scopesim/effects/apertures.py @@ -128,6 +128,8 @@ def apply_to(self, obj, **kwargs): for vol in obj.volumes: vol["meta"]["xi_min"] = min(x) * u.arcsec vol["meta"]["xi_max"] = max(x) * u.arcsec + vol["meta"]["eta_min"] = min(y) * u.arcsec + vol["meta"]["eta_max"] = max(y) * u.arcsec vol["meta"]["slit_name"] = self.meta["name"] if "offset_type" in self.meta: @@ -305,6 +307,8 @@ def apply_to(self, obj, **kwargs): # ..todo: HUGE HACK - Get rid of this! vol["meta"]["xi_min"] = row["left"] * u.arcsec vol["meta"]["xi_max"] = row["right"] * u.arcsec + vol["meta"]["eta_min"] = row["bottom"] * u.arcsec + vol["meta"]["eta_max"] = row["top"] * u.arcsec vol["conserve_image"] = row["conserve_image"] vol["shape"] = row["shape"] vol["angle"] = row["angle"] diff --git a/scopesim/effects/metis_lms_trace_list.py b/scopesim/effects/metis_lms_trace_list.py index 92780daca..d1b650e0a 100644 --- a/scopesim/effects/metis_lms_trace_list.py +++ b/scopesim/effects/metis_lms_trace_list.py @@ -144,6 +144,8 @@ def apply_to_fov(self, obj): slicefov.detector_header = obj.detector_header slicefov.meta["xi_min"] = obj.meta["xi_min"] slicefov.meta["xi_max"] = obj.meta["xi_max"] + slicefov.meta["eta_min"] = ymin * u.arcsec + slicefov.meta["eta_max"] = ymax * u.arcsec slicefov.meta["trace_id"] = sptid slicefov.cube = fits.ImageHDU(header=slicewcs.to_header(), data=slicecube) diff --git a/scopesim/effects/spectral_trace_list.py b/scopesim/effects/spectral_trace_list.py index 6597ebb20..e0bdad1f7 100644 --- a/scopesim/effects/spectral_trace_list.py +++ b/scopesim/effects/spectral_trace_list.py @@ -690,6 +690,7 @@ def __init__(self, cmds=None, **kwargs): self.cmds = cmds trace_param_filename = kwargs.pop("filename") + self.physical_slit_dispersion = kwargs.pop("physical_slit_dispersion", False) save_generated_hdulist = kwargs.pop("save_generated_hdulist", False) generated_hdulist_filename = kwargs.pop( "generated_hdulist_filename", "analytical_echelle_traces.fits") @@ -881,7 +882,19 @@ def on_detector(wave, order): 'x': xval, 'y': yval}) + if self.physical_slit_dispersion: + # The analytical prescription assumes equal slit magnification + # along and across the slit. Use its native, unrotated echelle + # mapping; neither the sky raster nor detector rotation changes + # this physical wavelength displacement. Preserve the legacy sign. + native_angle = row['plate_scale'] * u.Unit(trace_params.meta['plate_scale_unit']) * pix_size + raw_pixel = ss.wavelength_to_x_pixel(w, order) + delta_wave = ss.x_pixel_to_wavelength(raw_pixel + .5, order) - ss.x_pixel_to_wavelength(raw_pixel - .5, order) + order_table['dlam_deta'] = (-delta_wave / native_angle).to(u.um / u.arcsec) + trace_hdu = fits.BinTableHDU(order_table) + if self.physical_slit_dispersion: + trace_hdu.header['SLITCOL'] = 'dlam_deta' trace_hdu.header['DISPDIR'] = row['dispdir'] trace_hdu.header["EXTNAME"] = f'{prefix}_{order:d}' trace_hdu.header["DESIGNR"] = ( diff --git a/scopesim/effects/spectral_trace_list_utils.py b/scopesim/effects/spectral_trace_list_utils.py index 9bbbbfc39..a3a10b44d 100644 --- a/scopesim/effects/spectral_trace_list_utils.py +++ b/scopesim/effects/spectral_trace_list_utils.py @@ -394,11 +394,7 @@ def map_spectra_to_focal_plane(self, fov): def _trace_flux_scale(self, x_mm, y_mm, pixsize, pixscale, det_header): """Return the local [arcsec um] per detector pixel flux scale.""" - try: - use_jacobian = bool(from_currsys( - self.meta["trace_flux_jacobian"], self.cmds)) - except ValueError: - use_jacobian = False + use_jacobian = from_currsys(self.meta["trace_flux_jacobian"], self.cmds) if not use_jacobian: dlam_by_dx, dlam_by_dy = self.xy2lam.gradient() @@ -415,13 +411,10 @@ def _trace_flux_scale(self, x_mm, y_mm, pixsize, pixscale, det_header): @staticmethod def _detector_pixel_area(det_header): """Return detector WCS pixel area in mm2.""" - try: - pixel_scale_matrix = WCS(det_header, key="D").pixel_scale_matrix - return np.abs(np.linalg.det(pixel_scale_matrix)) - except Exception: # pragma: no cover - FITS fallback for odd headers - cdelt1 = det_header["CDELT1D"] * u.Unit(det_header["CUNIT1D"]) - cdelt2 = det_header["CDELT2D"] * u.Unit(det_header["CUNIT2D"]) - return np.abs((cdelt1 * cdelt2).to_value(u.mm**2)) + detector_wcs = WCS(det_header, key="D") + pixel_area = np.abs(np.linalg.det(detector_wcs.pixel_scale_matrix)) + pixel_area *= u.Unit(detector_wcs.wcs.cunit[0]) * u.Unit(detector_wcs.wcs.cunit[1]) + return pixel_area.to_value(u.mm**2) def rectify( self, @@ -889,8 +882,16 @@ def __init__(self, fov, dlam_per_pix, slit_dispersion=None): (n_lam, n_eta, n_xi) = fov.cube.data.shape # arrays of cube coordinates - cube_xi = d_xi * np.arange(n_xi) + fov.meta["xi_min"].value - cube_eta = d_eta * (np.arange(n_eta) - (n_eta - 1) / 2) + cube_xi = cube_wcs.all_pix2world(np.arange(n_xi), np.zeros(n_xi), np.zeros(n_xi), 0)[0] + cube_eta = cube_wcs.all_pix2world(np.zeros(n_eta), np.arange(n_eta), np.zeros(n_eta), 0)[1] + if cube_wcs.has_celestial: + # Slit coordinates are offsets about zero; RA wraps at 360 degrees. + cube_xi = (cube_xi + 180) % 360 - 180 + cube_xi *= u.Unit(cube_wcs.wcs.cunit[0]).to(u.arcsec) + cube_eta *= u.Unit(cube_wcs.wcs.cunit[1]).to(u.arcsec) + # Each IFU slice has its own optical axis, independent of sky position. + eta_center = (fov.meta["eta_min"] + fov.meta["eta_max"]) / 2 + cube_eta -= eta_center.to_value(u.arcsec) cube_lam = wcs_lam.all_pix2world(np.arange(n_lam), 0)[0] cube_lam *= u.Unit(wcs_lam.wcs.cunit[0]).to(u.um) @@ -924,7 +925,7 @@ def __init__(self, fov, dlam_per_pix, slit_dispersion=None): # Default WCS with xi in arcsec self.wcs = WCS(naxis=2) self.wcs.wcs.crpix = [1, 1] - self.wcs.wcs.crval = [self.lam[0], fov.meta["xi_min"].value] + self.wcs.wcs.crval = [self.lam[0], cube_xi[0]] self.wcs.wcs.pc = [[1, 0], [0, 1]] self.wcs.wcs.cdelt = [d_lam, d_xi] self.wcs.wcs.ctype = ["LINEAR", "LINEAR"] diff --git a/scopesim/tests/tests_effects/test_SpectralTraceListUtils.py b/scopesim/tests/tests_effects/test_SpectralTraceListUtils.py index dfaa2d7f6..6bd4fa101 100644 --- a/scopesim/tests/tests_effects/test_SpectralTraceListUtils.py +++ b/scopesim/tests/tests_effects/test_SpectralTraceListUtils.py @@ -303,7 +303,7 @@ def __init__(self, n_lam=20, n_eta=3, n_xi=11): hdr["CUNIT1"], hdr["CUNIT2"], hdr["CUNIT3"] = "arcsec", "arcsec", "um" self.cube = fits.ImageHDU( data=np.ones((n_lam, n_eta, n_xi)), header=hdr) - self.meta = {"xi_min": -0.5 * u.arcsec, "xi_max": 0.5 * u.arcsec} + self.meta = {"xi_min": -0.5 * u.arcsec, "xi_max": 0.5 * u.arcsec, "eta_min": -0.5 * u.arcsec, "eta_max": 0.5 * u.arcsec} class TestXiLamImage: diff --git a/scopesim/tests/tests_effects/test_calibrated_spectral_trace.py b/scopesim/tests/tests_effects/test_calibrated_spectral_trace.py index f6b8a8eee..60d4496c6 100644 --- a/scopesim/tests/tests_effects/test_calibrated_spectral_trace.py +++ b/scopesim/tests/tests_effects/test_calibrated_spectral_trace.py @@ -3,7 +3,7 @@ import pytest from astropy import units as u from astropy.io import fits -from scopesim.effects.spectral_trace_list_utils import XiLamImage +from scopesim.effects.spectral_trace_list_utils import XiLamImage, SpectralTrace def line_cube(step, offset=0): @@ -13,7 +13,7 @@ def line_cube(step, offset=0): profile = np.exp(-.5*((wave-1)/.00002)**2) cube = profile[:, None, None]*spatial[None, :, None]*np.ones((1, 1, 3)) header = fits.Header({'CTYPE1': 'LINEAR', 'CTYPE2': 'LINEAR', 'CTYPE3': 'WAVE', 'CUNIT1': 'arcsec', 'CUNIT2': 'arcsec', 'CUNIT3': 'um', 'CDELT1': .1, 'CDELT2': step, 'CDELT3': wave[1]-wave[0], 'CRPIX1': 1., 'CRPIX2': 1., 'CRPIX3': 1., 'CRVAL1': 0., 'CRVAL2': eta[0], 'CRVAL3': wave[0]}) - return SimpleNamespace(cube=fits.ImageHDU(cube, header), meta={'xi_min': 0*u.arcsec}) + return SimpleNamespace(cube=fits.ImageHDU(cube, header), meta={'xi_min': 0*u.arcsec, 'eta_min': -.5*u.arcsec, 'eta_max': .5*u.arcsec}) @@ -41,3 +41,36 @@ def test_optical_lsf_preserves_photons_and_adds_variance(): center = np.sum(image.lam*image.image[1])/image.image[1].sum() variance = np.sum((image.lam-center)**2*image.image[1])/image.image[1].sum() assert variance == pytest.approx(.00002**2 + (1/10000/np.sqrt(8*np.log(2)))**2, rel=.02) + + +def test_slit_coordinates_follow_cube_wcs(): + fov = line_cube(.025) + fov.cube.header['CRVAL1'] = 2. + fov.cube.header['CRPIX1'] = 2. + fov.cube.header['CRVAL2'] += .1 + image = XiLamImage(fov, lambda wave: .00001, slit_dispersion=lambda wave: .002) + np.testing.assert_allclose(image.xi, [1.9, 2., 2.1]) + centroid = np.sum(image.image[1] * image.lam) / image.image[1].sum() + assert centroid == pytest.approx(1.0002, abs=2e-6) + + +def test_translating_an_ifu_slice_preserves_its_local_spectrum(): + fov = line_cube(.025) + centered = XiLamImage(fov, lambda wave: .00001, slit_dispersion=lambda wave: .002) + fov.cube.header['CRVAL2'] += 2. + fov.meta['eta_min'] += 2*u.arcsec + fov.meta['eta_max'] += 2*u.arcsec + translated = XiLamImage(fov, lambda wave: .00001, slit_dispersion=lambda wave: .002) + np.testing.assert_allclose(translated.image, centered.image, atol=1e-7) + + +def test_missing_aperture_geometry_fails(): + fov = line_cube(.025) + del fov.meta['eta_min'] + with pytest.raises(KeyError, match='eta_min'): + XiLamImage(fov, lambda wave: .00001, slit_dispersion=lambda wave: .002) + + +def test_detector_jacobian_uses_physical_pixel_area(): + header = fits.Header({'CTYPE1D': 'LINEAR', 'CTYPE2D': 'LINEAR', 'CUNIT1D': 'm', 'CUNIT2D': 'm', 'CDELT1D': .000015, 'CDELT2D': .000030, 'CRPIX1D': 1., 'CRPIX2D': 1., 'CRVAL1D': 0., 'CRVAL2D': 0.}) + assert SpectralTrace._detector_pixel_area(header) == pytest.approx(.015*.030) From fa2598b724bf5f90d211f41b8518a6cb9e1f0967 Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Sat, 17 Oct 2026 02:46:41 -0700 Subject: [PATCH 100/130] Avoid numeric-array command recursion and key seeing caches by zenith angle --- scopesim/effects/psfs/analytical.py | 4 +++- scopesim/tests/test_utils_functions.py | 9 ++++++++ .../tests/tests_effects/test_MoffatPSF.py | 23 +++++++++++++++++++ scopesim/utils.py | 5 +++- 4 files changed, 39 insertions(+), 2 deletions(-) diff --git a/scopesim/effects/psfs/analytical.py b/scopesim/effects/psfs/analytical.py index dc39f5207..419fec803 100644 --- a/scopesim/effects/psfs/analytical.py +++ b/scopesim/effects/psfs/analytical.py @@ -261,7 +261,9 @@ def fwhm(self, wavelengths): fwhm = self.meta["fwhm"] if isinstance(fwhm, dict) and {"pivot_wave", "pivot_wave_unit"}.issubset(fwhm): target, location, time, _ = get_observation_info_from_cmds(self.cmds) - key = tuple(fwhm.items()), target, location, time + # The seeing law depends on zenith angle, not the coordinate frame + # instance (SkyCoord equality raises for different observation times). + key = tuple(fwhm.items()), get_zenith_angle(target, location, time) elif isinstance(fwhm, dict): key = tuple(fwhm.items()) else: diff --git a/scopesim/tests/test_utils_functions.py b/scopesim/tests/test_utils_functions.py index a1df0f1b3..1469b32d9 100644 --- a/scopesim/tests/test_utils_functions.py +++ b/scopesim/tests/test_utils_functions.py @@ -199,6 +199,15 @@ def test_converts_list(self): def test_converts_numpy_array(self): assert utils.from_currsys(np.array(["!SIM.random.seed"]*2))[1] is None + @pytest.mark.parametrize("dtype", [np.float32, np.float64, np.int32]) + def test_numeric_calibration_array_keeps_values_and_precision(self, dtype): + values = np.arange(12, dtype=dtype).reshape(3, 4) + resolved = utils.from_currsys(values) + np.testing.assert_array_equal(resolved, values) + assert resolved.dtype == dtype + resolved[0, 0] = 20 + assert values[0, 0] == 0 + def test_converts_dict(self): assert utils.from_currsys({"seed": "!SIM.random.seed"})["seed"] is None diff --git a/scopesim/tests/tests_effects/test_MoffatPSF.py b/scopesim/tests/tests_effects/test_MoffatPSF.py index 1119df8dd..a0625d29e 100644 --- a/scopesim/tests/tests_effects/test_MoffatPSF.py +++ b/scopesim/tests/tests_effects/test_MoffatPSF.py @@ -7,6 +7,29 @@ from scopesim.effects.psfs.analytical import MoffatPSF +def test_seeing_cache_uses_physical_zenith_angle(monkeypatch): + from astropy.coordinates import SkyCoord, AltAz, EarthLocation + from astropy.time import Time + from scopesim.effects.psfs import analytical + + location = EarthLocation.from_geodetic(0*u.deg, 30*u.deg) + altitude = 60*u.deg + times = iter(Time(60310 + np.arange(10)/86400, format='mjd')) + + def observing_info(cmds): + time = next(times) + target = SkyCoord(alt=altitude, az=0*u.deg, frame=AltAz(obstime=time, location=location)) + return target, location, time, 'dark' + + monkeypatch.setattr(analytical, 'get_observation_info_from_cmds', observing_info) + psf = MoffatPSF(alpha=4.765, fwhm={'seeing': .7, 'seeing_unit': 'arcsec', 'pivot_wave': 500, 'pivot_wave_unit': 'nm'}) + first = psf.fwhm(np.array([.5])*u.um) + np.testing.assert_allclose(psf.fwhm(np.array([.5])*u.um), first) + altitude = 30*u.deg + expected = .7 / np.sin(altitude.to_value(u.rad))**.6 + assert psf.fwhm(np.array([.5])*u.um)[0].to_value(u.arcsec) == pytest.approx(expected) + + class SimpleFov: header = {"CDELT1": 0.1 / 3600} meta = {"wave_min": 1.0 * u.um, "wave_max": 1.1 * u.um} diff --git a/scopesim/utils.py b/scopesim/utils.py index 36b2e5080..a6c60801d 100644 --- a/scopesim/utils.py +++ b/scopesim/utils.py @@ -610,6 +610,9 @@ def from_currsys(item, cmds=None): item.meta = item_meta if isinstance(item, np.ndarray) and not isinstance(item, u.Quantity): + # Numeric calibration arrays cannot contain command references. + if item.dtype.kind in "biufc": + return item.copy() item = np.array([from_currsys(x, cmds) for x in item]) if isinstance(item, list): @@ -1261,4 +1264,4 @@ def _get_next_moon_cached(moontype="full", today=None, location_key=None): nextm = times[mask][np.argmax(moonalts[mask])] else: raise ValueError(f"Invalid moon type: {moontype}, should be 'full', 'new' or 'half'.") - return nextm \ No newline at end of file + return nextm From a23b80c8c2b201ef48bc89183ec70ada15693731 Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Sat, 17 Oct 2026 02:50:05 -0700 Subject: [PATCH 101/130] Integrate native spectra into FOV bins without early line resampling --- scopesim/optics/fov.py | 63 ++++++++++++++----- scopesim/optics/optical_train.py | 52 ++------------- .../test_spectral_bin_integration.py | 54 ++++++++++++++++ 3 files changed, 105 insertions(+), 64 deletions(-) create mode 100644 scopesim/tests/tests_optics/test_spectral_bin_integration.py diff --git a/scopesim/optics/fov.py b/scopesim/optics/fov.py index d38f4abe5..c9b5d59b8 100644 --- a/scopesim/optics/fov.py +++ b/scopesim/optics/fov.py @@ -7,7 +7,7 @@ from weakref import WeakKeyDictionary import numpy as np -from scipy.interpolate import interp1d +from scipy.integrate import cumulative_trapezoid from astropy import units as u from astropy.io import fits @@ -490,10 +490,11 @@ def waveset(self): for cube in cube_fields]) _waveset = cube_fields[naxis3_max].waveset elif specfields := self._get_fields(SpectrumSourceField): - _waveset = np.concatenate([ + _waveset = np.concatenate([self.waverange << u.um] + [ spec.waveset.to(u.um) for field in specfields for spec in field.spectra.values() + if spec.waveset is not None ]) else: _waveset = self.waverange << u.um @@ -928,7 +929,7 @@ def _make_cubefields(self, fov_waveset): Find Cube fields. * rescale and reorient cubes - * interp1d smaller cubes with waveset + * integrate native spectral samples into the FOV wavelength bins * yield cubes to be added to cavas cube """ for field in self._get_cube_fields(): @@ -936,17 +937,9 @@ def _make_cubefields(self, fov_waveset): # Assumption is that ImageHDUs have units of PHOTLAM arcsec-2 # TODO: Deal with this bounds_error in a more elegant way - field_interp = interp1d( - field.waveset.to(u.um).value, - field.data, - axis=0, - kind="linear", - bounds_error=False, - fill_value=0, - ) - - field_data = field_interp(fov_waveset.value) + field_data = rebin_spectral_density(field.waveset.to_value(u.um), field.data, spectral_bin_edges(fov_waveset.to_value(u.um))) field_hdu = fits.ImageHDU(data=field_data, header=field.header) + field_hdu.header.update(CRPIX3=1., CRVAL3=fov_waveset[0].to_value(u.um), CDELT3=np.diff(fov_waveset[:2])[0].to_value(u.um), CUNIT3='um', CTYPE3='WAVE') yield field_hdu def _make_imagefields(self, fov_waveset, spline_order): @@ -975,7 +968,7 @@ def _make_imagefields(self, fov_waveset, spline_order): differential=True, ) - spec = field.spectrum(fov_waveset) + spec = average_spectrum(field.spectrum, fov_waveset) # 2D * 1D -> 3D field_cube = canvas_image_hdu.data[None, :, :] * spec[:, None, None] yield field_cube.value @@ -998,7 +991,7 @@ def _make_tablefields(self, fov_waveset): # if this is the case and if yes, remove the condition here!! # Fluxes (yes that's the correct plural) in PHOTLAM fluxes = { - ref: spec(fov_waveset) + ref: average_spectrum(spec, fov_waveset) for ref, spec in field.spectra.items() if ref in field.field["ref"] } @@ -1025,7 +1018,7 @@ def _make_backfields(self, fov_waveset): # Remember, cube output needs PHOTLAM arcsec-2 ! # So if SOLIDANG or BUNIT or whatever is not in arcsec-2, # the spectrum shoule be scaled accordingly! - spec = field.spectrum(fov_waveset) + spec = average_spectrum(field.spectrum, fov_waveset) yield spec[:, None, None].value def make_hdu(self): @@ -1189,6 +1182,8 @@ def extract_range_from_spectrum(spectrum, waverange): wave_min, wave_max = quantify(waverange, u.um).to(u.AA).value spec_waveset = _spectrum_waveset_aa_value(spectrum) + if spec_waveset is None: + return spectrum mask = (spec_waveset > wave_min) & (spec_waveset < wave_max) # FIXME: Why did I comment this out in 2023? Seems useful to have... @@ -1214,6 +1209,42 @@ def _spectrum_waveset_aa_value(spectrum): """Return cached spectrum waveset values in Angstrom.""" spec_waveset = _SPECTRUM_WAVESET_AA_CACHE.get(spectrum) if spec_waveset is None: + if spectrum.waveset is None: + return None spec_waveset = spectrum.waveset.to(u.AA).value _SPECTRUM_WAVESET_AA_CACHE[spectrum] = spec_waveset return spec_waveset + + +def spectral_bin_edges(centers): + """Edges of bins whose centers are the supplied wavelength samples.""" + half_steps = np.diff(centers) / 2 + return np.r_[centers[0] - half_steps[0], centers[:-1] + half_steps, centers[-1] + half_steps[-1]] + + +def rebin_spectral_density(wave, density, edges): + """Integrate a piecewise-linear density, with zero flux beyond its support. + + Wavelength is the first axis; density may also carry spatial dimensions. + The result remains a density, averaged over each requested bin. + """ + if wave[1] < wave[0]: + wave, density = wave[::-1], density[::-1] + integral = cumulative_trapezoid(density, wave, axis=0, initial=0) + clipped = np.clip(edges, wave[0], wave[-1]) + index = np.clip(np.searchsorted(wave, clipped, side='right') - 1, 0, len(wave) - 2) + dimensions = (len(edges),) + (1,)*(density.ndim - 1) + offset = (clipped - wave[index]).reshape(dimensions) + step = (wave[index + 1] - wave[index]).reshape(dimensions) + at_edges = integral[index] + density[index]*offset + (density[index + 1] - density[index])*offset**2/(2*step) + widths = np.diff(edges).reshape((len(edges) - 1,) + (1,)*(density.ndim - 1)) + return np.diff(at_edges, axis=0) / widths + + +def average_spectrum(spectrum, wavelengths): + """Average PHOTLAM over numerical bins without discarding native line samples.""" + edges = spectral_bin_edges(wavelengths.to_value(u.AA)) + native = _spectrum_waveset_aa_value(spectrum) + wave = edges if native is None else np.union1d(edges, native[(native > edges[0]) & (native < edges[-1])]) + flux = spectrum(wave*u.AA).to_value(PHOTLAM) + return rebin_spectral_density(wave, flux, edges) * PHOTLAM diff --git a/scopesim/optics/optical_train.py b/scopesim/optics/optical_train.py index 3630585ce..d98963f3e 100644 --- a/scopesim/optics/optical_train.py +++ b/scopesim/optics/optical_train.py @@ -6,7 +6,6 @@ from pathlib import Path import numpy as np -from scipy.interpolate import interp1d from astropy import units as u from astropy.table import QTable from astropy.wcs import WCS @@ -14,7 +13,6 @@ from tqdm.auto import tqdm -from synphot import SourceSpectrum, Empirical1D from synphot.units import PHOTLAM from astar_utils.nested_mapping import RecursiveNestedMapping, is_bangkey, recursive_update @@ -27,7 +25,6 @@ from ..effects import ExtraFitsKeywords from ..utils import from_currsys, top_level_catch, get_logger from ..source.source_templates import empty_sky -from ..source.source_fields import SpectrumSourceField from .. import __version__ @@ -644,22 +641,6 @@ def prepare_source(self, source): # TODO: this is not sufficiently general # TODO: Maybe move this to source_fields?? - for field in source.fields: - if not isinstance(field, SpectrumSourceField): - continue - for ispec, spec in field.spectra.items(): - # Put on fov wavegrid - wave_min = min(fov.meta["wave_min"] for fov in self.fov_manager.fovs) - wave_max = max(fov.meta["wave_max"] for fov in self.fov_manager.fovs) - wave_unit = u.Unit(from_currsys("!SIM.spectral.wave_unit", self.cmds)) - dwave = from_currsys("!SIM.spectral.spectral_bin_width", self.cmds) # Not a quantity - fov_waveset = np.arange(wave_min.value, wave_max.value, dwave) * wave_unit - fov_waveset = fov_waveset.to(u.um) - - field.spectra[ispec] = SourceSpectrum(Empirical1D, - points=fov_waveset, - lookup_table=spec(fov_waveset)) - for cube in source.cube_fields: header, data, wave = cube.header, cube.data, cube.waveset @@ -694,35 +675,10 @@ def prepare_source(self, source): cube.header["CUNIT1"] = "deg" cube.header["CUNIT2"] = "deg" - # Put on fov wavegrid - wave_min = min(fov.meta["wave_min"] for fov in self.fov_manager.fovs) - wave_max = max(fov.meta["wave_max"] for fov in self.fov_manager.fovs) - wave_unit = u.Unit(from_currsys("!SIM.spectral.wave_unit", self.cmds)) - dwave = from_currsys("!SIM.spectral.spectral_bin_width", self.cmds) # Not a quantity - fov_waveset = np.arange(wave_min.value, wave_max.value, dwave) * wave_unit - fov_waveset = fov_waveset.to(u.um) - - if (wave.to(u.um).min() > fov_waveset.max() or - wave.to(u.um).max() < fov_waveset.min()): - logger.warning("Source and FOV wavesets disjoint.") - - # Interpolate into new data cube. - # This is done layer by layer for memory reasons. - new_data = np.zeros((fov_waveset.shape[0], data.shape[1], data.shape[2]), - dtype=np.float32) - for j in range(data.shape[1]): - cube_interp = interp1d(wave.to_value(u.um), data[:, j, :], - axis=0, kind="linear", - bounds_error=False, fill_value=0) - new_data[:, j, :] = cube_interp(fov_waveset.value) - - cube.data = new_data - cube.header["CTYPE3"] = "WAVE" - cube.header["CRPIX3"] = 1 - cube.header["CRVAL3"] = wave_min.value - cube.header["CDELT3"] = dwave - cube.header["CUNIT3"] = wave_unit.name - + # Keep the native wavelength samples until the spatial FOV is cut. + # Rebinning the entire source here loses narrow lines and expands + # resolved cubes over the full instrument band unnecessarily. + cube.data = data cube.wcs = WCS(cube.field) return source diff --git a/scopesim/tests/tests_optics/test_spectral_bin_integration.py b/scopesim/tests/tests_optics/test_spectral_bin_integration.py new file mode 100644 index 000000000..f5f826a13 --- /dev/null +++ b/scopesim/tests/tests_optics/test_spectral_bin_integration.py @@ -0,0 +1,54 @@ +import numpy as np +import pytest +from astropy import units as u +from synphot import SourceSpectrum, Empirical1D +from synphot.units import PHOTLAM +from scopesim.optics.fov import average_spectrum, spectral_bin_edges, rebin_spectral_density +from scopesim.optics.optical_train import OpticalTrain +from scopesim.source.source import Source +from scopesim.source.source_templates import ab_spectrum + + +@pytest.mark.parametrize('resolving_power', [20000, 200000]) +@pytest.mark.parametrize('phase', [0, .2, .5, .8]) +def test_narrow_line_photons_survive_coarse_grid_phase(resolving_power, phase): + center = 5000 + phase*.1 + sigma = center / resolving_power / np.sqrt(8*np.log(2)) + native = center + np.linspace(-8, 8, 193)*sigma + flux = np.exp(-.5*((native - center)/sigma)**2) / (sigma*np.sqrt(2*np.pi)) + spectrum = SourceSpectrum(Empirical1D, points=native*u.AA, lookup_table=flux*PHOTLAM) + wave = np.arange(4998, 5002, .1)*u.AA + density = average_spectrum(spectrum, wave) + photons = (density * np.diff(spectral_bin_edges(wave.to_value(u.AA)))*u.AA).sum() + assert photons.to_value(u.ph/u.s/u.cm**2) == pytest.approx(1, rel=.001) + + +def test_flat_ab_continuum_bin_integral(): + spectrum = ab_spectrum(mag=20) + wave = np.linspace(5000, 5100, 101)*u.AA + edges = spectral_bin_edges(wave.to_value(u.AA)) + # Constant f_nu gives photon density proportional to 1/lambda. + scale = spectrum(5000*u.AA).to_value(PHOTLAM)*5000 + expected = scale*np.log(edges[1:]/edges[:-1])/np.diff(edges) + np.testing.assert_allclose(average_spectrum(spectrum, wave).to_value(PHOTLAM), expected, rtol=1e-6) + + +@pytest.mark.parametrize('descending', [False, True]) +def test_cube_rebinning_preserves_flux_spatial_weights_and_support(descending): + wave = np.array([1., 2., 3.]) + weights = np.array([[1., 2.], [3., 4.]]) + density = wave[:, None, None]*weights + if descending: + wave, density = wave[::-1], density[::-1] + rebinned = rebin_spectral_density(wave, density, np.array([.5, 1.5, 2.5, 3.5])) + np.testing.assert_allclose(rebinned, np.array([.625, 2., 1.375])[:, None, None]*weights) + np.testing.assert_allclose(rebinned.sum(axis=0), 4*weights) + + +def test_prepare_source_keeps_native_line_sampling(): + wave = np.linspace(4999.9, 5000.1, 101)*u.AA + spectrum = SourceSpectrum(Empirical1D, points=wave, lookup_table=np.exp(-((wave.value - 5000)/.01)**2)*PHOTLAM) + source = Source(x=[0.], y=[0.], ref=[0], weight=[1.], spectra=[spectrum]) + train = OpticalTrain() + prepared = train.prepare_source(source) + np.testing.assert_array_equal(prepared.spectra[0].waveset, spectrum.waveset) From b3c9107379a69393c2eb94978163b5f738dc2547 Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Sat, 17 Oct 2026 03:01:44 -0700 Subject: [PATCH 102/130] Preserve per-arm exposure metadata after upstream readout changes --- scopesim/effects/electronic/exposure.py | 2 ++ scopesim/optics/optical_train.py | 12 +++++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/scopesim/effects/electronic/exposure.py b/scopesim/effects/electronic/exposure.py index 9871819c0..f11237375 100644 --- a/scopesim/effects/electronic/exposure.py +++ b/scopesim/effects/electronic/exposure.py @@ -288,6 +288,8 @@ def apply_to(self, obj, **kwargs): ) obj.header["EXPTIME"] = (dit * ndit, "Total accumulated integration time [s]") + obj.header['DIT'] = (dit, 'Integration time per exposure [s]') + obj.header['NDIT'] = ndit obj.data = obj.data * dit * ndit return obj diff --git a/scopesim/optics/optical_train.py b/scopesim/optics/optical_train.py index d98963f3e..80e13ced6 100644 --- a/scopesim/optics/optical_train.py +++ b/scopesim/optics/optical_train.py @@ -767,7 +767,7 @@ def readout( hdul = effect.apply_to(hdul, optical_train=self) else: try: - hdul = self.write_header(hdul, detector_array) + hdul = self.write_header(hdul, detector_manager) except Exception: logger.exception( "Header update failed, data will be saved with " @@ -817,10 +817,12 @@ def write_header(self, hdulist, detector_array): # Image HDUs for hdu in hdulist[1:]: - hdu.header["EXPTIME"] = from_currsys("!OBS.exptime", self.cmds), "[s]" - hdu.header["DIT"] = from_currsys("!OBS.dit", self.cmds), "[s]" - hdu.header["NDIT"] = from_currsys("!OBS.ndit", self.cmds) - hdu.header["DETECTOR"] = detector_array._detector_list.meta["detector"] + # Arm-specific integration effects already recorded their actual + # exposure. Global command values are only for minimal setups. + for key in ('EXPTIME', 'DIT', 'NDIT'): + if key not in hdu.header: + hdu.header[key] = from_currsys(f'!OBS.{key.lower()}', self.cmds) + hdu.header['DETECTOR'] = detector_array._detector_list.display_name return hdulist def shutdown(self): From 0701bb57c69faa50aa3ab70e71210d9c853e924a Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Sat, 17 Oct 2026 03:02:56 -0700 Subject: [PATCH 103/130] Integrate effective PSFs before clipping physical slit apertures --- scopesim/effects/psfs/analytical.py | 92 +++++++++--- scopesim/effects/spectral_efficiency.py | 19 +-- scopesim/effects/ter_curves.py | 10 +- scopesim/optics/fov.py | 140 ++++++++++-------- scopesim/optics/fov_manager.py | 10 ++ scopesim/optics/image_plane_utils.py | 33 +++++ scopesim/optics/optical_train.py | 34 ++++- .../tests/tests_effects/test_MoffatPSF.py | 2 +- .../tests/tests_optics/test_FieldOfView.py | 6 +- .../test_slit_spatial_integration.py | 130 ++++++++++++++++ 10 files changed, 369 insertions(+), 107 deletions(-) create mode 100644 scopesim/tests/tests_optics/test_slit_spatial_integration.py diff --git a/scopesim/effects/psfs/analytical.py b/scopesim/effects/psfs/analytical.py index 419fec803..46deac4e5 100644 --- a/scopesim/effects/psfs/analytical.py +++ b/scopesim/effects/psfs/analytical.py @@ -2,17 +2,18 @@ """Contains simple Vibration, NCPA, Seeing and Diffraction PSFs.""" from typing import ClassVar, Any -from functools import partial +from functools import partial, lru_cache import numpy as np from astropy import units as u from astropy.convolution import Gaussian2DKernel -from astropy.modeling.models import Moffat2D from scipy.interpolate import make_interp_spline +from scipy.special import gammaln, stdtr +from astropy.wcs import WCS from .. import DataContainer from ...optics import ImagePlane -from ...optics.fov import FieldOfView +from ...optics.fov import FieldOfView, FieldOfView3D from ...utils import (from_currsys, quantify, quantity_from_table, figure_factory, check_keys, get_logger, find_file, get_zenith_angle, get_observation_info_from_cmds) from . import PSF, PoorMansFOV @@ -256,6 +257,7 @@ class MoffatPSF(AnalyticalPSF): def __init__(self, **kwargs): super().__init__(**kwargs) self._fwhm_interp = None + self._kernel_cache = {} def fwhm(self, wavelengths): fwhm = self.meta["fwhm"] @@ -313,8 +315,9 @@ def get_fwhm_interp(self): raise TypeError("fwhm kwarg must be of type dict or float or str") def get_kernel(self, fov): - pixel_scale = fov.header["CDELT1"] * u.deg.to(u.arcsec) + pixel_scale = abs(fov.header["CDELT1"]) * u.deg.to(u.arcsec) pixel_scale = quantify(pixel_scale, u.arcsec) + ystep = abs(fov.header["CDELT2"] / fov.header["CDELT1"]) # Sample the wavelength-dependent seeing law with a bounded number of # representative points. Kernel sizing is driven by the broadest PSF @@ -326,14 +329,17 @@ def get_kernel(self, fov): target = self._target_enclosed_energy() max_ksize = self._max_kernel_size() - ksize = self._minimum_kernel_size(np.max(fwhms)) + ksize = self._minimum_kernel_size(np.max(fwhms) / min(1., ystep)) if max_ksize is not None: ksize = min(ksize, max_ksize) - kernel, norm = self._make_moffat_kernel(gammas, alpha, ksize) + key = (tuple(gammas), alpha, ystep, target, max_ksize, ksize, self.meta["rounded_edges"], self._renormalize_clipped_kernel()) + if key in self._kernel_cache: + return self._kernel_cache[key].copy() + kernel, norm = self._make_moffat_kernel(gammas, alpha, ksize, ystep) while norm < target and (max_ksize is None or ksize < max_ksize): ksize = self._next_kernel_size(ksize, max_ksize) - kernel, norm = self._make_moffat_kernel(gammas, alpha, ksize) + kernel, norm = self._make_moffat_kernel(gammas, alpha, ksize, ystep) if norm < target: logger.warning( @@ -354,8 +360,36 @@ def get_kernel(self, fov): if self._renormalize_clipped_kernel() and norm > 0: kernel /= norm + if len(self._kernel_cache) >= 256: + self._kernel_cache.pop(next(iter(self._kernel_cache))) + self._kernel_cache[key] = kernel return kernel + def point_source_image(self, fov, x, y): + """Pixel-integrated effective PSF at an actual sky position [arcsec].""" + dx, dy = abs(fov.header['CDELT1'])*3600, abs(fov.header['CDELT2'])*3600 + xp, yp = WCS(fov.header, naxis=2).all_world2pix(x/3600, y/3600, 0) + xx = (np.arange(fov.header['NAXIS1']) - xp)[None, :]*dx + yy = (np.arange(fov.header['NAXIS2']) - yp)[:, None]*dy + wave = self._sample_kernel_wavelengths(fov.waveset) + gammas = self.fwhm2gamma(self.fwhm(wave), self.alpha).to_value(u.arcsec) + return moffat_cell_integrals(xx, yy, dx, dy, gammas, self.alpha) + + def _apply_convolution(self, obj): + if isinstance(obj, FieldOfView3D) and obj.hdu is None: + obj.hdu = obj.make_hdu(spatial_psf=self) + else: + super()._apply_convolution(obj) + + def padded_header(self, fov): + """Spatial support needed to illuminate this FOV through the PSF.""" + kernel = self.get_kernel(fov) + header = fov.header.copy() + for axis, padding in ((1, kernel.shape[1]//2), (2, kernel.shape[0]//2)): + header[f'NAXIS{axis}'] += 2*padding + header[f'CRPIX{axis}'] += padding + return header + def _sample_kernel_wavelengths(self, waveset: u.Quantity) -> u.Quantity: max_samples = max(2, int(from_currsys( self.meta.get("max_kernel_wavelength_samples", 16), self.cmds))) @@ -389,22 +423,9 @@ def _minimum_kernel_size(self, max_fwhm_pix: float) -> int: kx = float(from_currsys(self.meta.get("kernel_size", 4.0), self.cmds)) return self._ensure_odd_int(kx * max_fwhm_pix) - def _make_moffat_kernel(self, gammas: np.ndarray, alpha:float, ksize: int) -> tuple[np.ndarray, float]: - amplitude = (alpha - 1) / (np.pi * gammas**2) - x, y = np.meshgrid( - np.arange(ksize) - ksize // 2, - np.arange(ksize) - ksize // 2, - ) - cube = Moffat2D.evaluate( - x=x[None, ...], - y=y[None, ...], - amplitude=amplitude[:, None, None], - x_0=0, - y_0=0, - gamma=gammas[:, None, None], - alpha=alpha, - ) - kernel = np.mean(cube, axis=0) + def _make_moffat_kernel(self, gammas, alpha, ksize, ystep=1.): + positions = np.arange(ksize) - ksize//2 + kernel = moffat_cell_integrals(positions[None, :], positions[:, None]*ystep, 1., ystep, gammas, alpha) if from_currsys(self.meta.get("rounded_edges", False), self.cmds): kernel = self._round_kernel_edges(kernel) return kernel, float(np.sum(kernel)) @@ -453,6 +474,31 @@ def fwhm_from_table(table): return lambda wavelengths: interp(wavelengths.to_value(u.um)) +@lru_cache(maxsize=16) +def _legendre_rule(order): + return np.polynomial.legendre.leggauss(order) + + +def moffat_cell_integrals(x, y, dx, dy, gammas, alpha): + """Integrate normalized Moffat profiles over rectangles, averaging wavelengths. + + Integrate the conditional Student-t profile analytically along y and use + Gauss-Legendre quadrature along x. This resolves narrow profiles without + oversampling a spectral cube. All coordinates share one angular unit. + """ + nodes, weights = _legendre_rule(max(8, int(np.ceil(8*dx/np.min(gammas))))) + xx = x[..., None] + dx*nodes/2 + lower, upper = y[..., None] - dy/2, y[..., None] + dy/2 + integral = np.zeros(np.broadcast_shapes(x.shape, y.shape)) + normalization = np.exp(gammaln(alpha - .5) - gammaln(alpha - 1)) / np.sqrt(np.pi) + for gamma in gammas: + marginal = normalization/gamma * (1 + (xx/gamma)**2)**(.5 - alpha) + scale = np.sqrt((2*alpha - 1)/(gamma**2 + xx**2)) + probability = stdtr(2*alpha - 1, upper*scale) - stdtr(2*alpha - 1, lower*scale) + integral += np.sum(marginal*probability*weights, axis=-1)*dx/2 + return integral/len(gammas) + + class AOEnhanceablePSF(MoffatPSF): """ Wavelength dependent Moffat PSF but with added AO scaling. diff --git a/scopesim/effects/spectral_efficiency.py b/scopesim/effects/spectral_efficiency.py index 472389889..bb860cc98 100644 --- a/scopesim/effects/spectral_efficiency.py +++ b/scopesim/effects/spectral_efficiency.py @@ -112,12 +112,7 @@ def apply_to(self, obj, **kwargs): logger.warning("No grating efficiency for trace %s", trace_id) return obj - swcs = WCS(obj.hdu.header).spectral - with u.set_enabled_equivalencies(u.spectral()): - wave = swcs.pixel_to_world(np.arange(swcs.pixel_shape[0])) << u.um - thru = effic.throughput(wave) - obj.hdu.data = obj.hdu.data * thru.value[:, None, None] - return obj + return effic.apply_to(obj) def plot(self): """Plot the grating efficiencies.""" @@ -197,9 +192,12 @@ def apply_to(self, obj, **kwargs): """Interface between FieldOfView and SpectralEfficiency.""" trace_id = obj.trace_id - swcs = WCS(obj.hdu.header).spectral - with u.set_enabled_equivalencies(u.spectral()): - wave = swcs.pixel_to_world(np.arange(swcs.pixel_shape[0])) << u.um + if obj.hdu is None: + wave = obj.cube_waveset + else: + swcs = WCS(obj.hdu.header).spectral + with u.set_enabled_equivalencies(u.spectral()): + wave = swcs.pixel_to_world(np.arange(swcs.pixel_shape[0])) << u.um efficiency = self.efficiency_generator(trace_id, wave) params = {"description": trace_id} @@ -207,8 +205,7 @@ def apply_to(self, obj, **kwargs): effic_curve = TERCurve(array_dict={"wavelength": wave, "transmission": efficiency}, **params) self.efficiencies[trace_id] = effic_curve - obj.hdu.data *= effic_curve.throughput(wave).value[:, None, None] - return obj + return effic_curve.apply_to(obj) def plot(self): """Plot the grating efficiencies.""" diff --git a/scopesim/effects/ter_curves.py b/scopesim/effects/ter_curves.py index c0d68eb2e..e35d2129f 100644 --- a/scopesim/effects/ter_curves.py +++ b/scopesim/effects/ter_curves.py @@ -232,7 +232,15 @@ def apply_to(self, obj, **kwargs): ## If obj is a FieldOfView, apply the throughput if isinstance(obj, FieldOfView): thru = self.throughput - if obj.hdu is not None and obj.hdu.header['NAXIS'] == 3: + if obj.hdu is None: + # Retain native spectral knots until spatial rendering, so + # narrow lines survive multiplication by the throughput. + for field in obj.fields: + if isinstance(field, CubeSourceField): + field.data *= thru(field.waveset).value[:, None, None] + elif isinstance(field, SpectrumSourceField): + field.spectra = {ref: spec * thru for ref, spec in field.spectra.items()} + elif obj.hdu.header['NAXIS'] == 3: swcs = WCS(obj.hdu.header).spectral with u.set_enabled_equivalencies(u.spectral()): wave = swcs.pixel_to_world(np.arange(swcs.pixel_shape[0])) << u.um diff --git a/scopesim/optics/fov.py b/scopesim/optics/fov.py index c9b5d59b8..2c3577d00 100644 --- a/scopesim/optics/fov.py +++ b/scopesim/optics/fov.py @@ -8,6 +8,7 @@ import numpy as np from scipy.integrate import cumulative_trapezoid +from scipy.signal import fftconvolve from astropy import units as u from astropy.io import fits @@ -134,7 +135,7 @@ def pixel_area(self): self.meta["pixel_area"] = pixel_area(self.header) return self.meta["pixel_area"] - def extract_from(self, src) -> None: + def extract_from(self, src, spatial_header=None) -> None: """ Extract relevent fields from source object. @@ -156,16 +157,20 @@ def extract_from(self, src) -> None: """ assert isinstance(src, Source), f"expected Source: {type(src)}" + self.fields = [] + self.hdu = None - fields_in_fov = list(self.get_fields_in_fov(src.fields)) + corners_arcsec, _ = self.get_corners("arcsec") + if spatial_header is not None: + corners_arcsec = imp_utils.calc_footprint(spatial_header, new_unit='arcsec') + corners_deg = corners_arcsec / 3600 + fields_in_fov = list(self.get_fields_in_fov(src.fields, corners_arcsec)) if not fields_in_fov: logger.warning("No fields in FOV.") else: logger.debug("%d fields in FOV", len(fields_in_fov)) - corners_arcsec, _ = self.get_corners("arcsec") - corners_deg, _ = self.get_corners("deg") minmax = array_minmax(corners_arcsec) * u.arcsec for field in fields_in_fov: @@ -251,9 +256,10 @@ def flatten(self): image = np.sum(self.hdu.data, axis=0) self.hdu.data = image - def get_fields_in_fov(self, fields: Iterable[SourceField]) -> Generator: + def get_fields_in_fov(self, fields: Iterable[SourceField], fov_corners=None) -> Generator: """Return True if Source.field footprint is inside FOV footprint.""" - fov_corners, _ = self.get_corners("arcsec") + if fov_corners is None: + fov_corners, _ = self.get_corners("arcsec") for field in fields: field_corners = field.get_corners("arcsec") @@ -358,7 +364,6 @@ def extract_area_from_imagehdu(self, imagehdu, corners): # up a duplicated or a dropped row/column. new_wcs = image_wcs.deepcopy() new_wcs.wcs.crpix = image_wcs.wcs.crpix - xy0p - new_wcs.wcs.cdelt = np.abs(new_wcs.wcs.cdelt) new_naxis = xy1p - xy0p logger.debug("orig image wcs: %s", image_wcs) logger.debug("new cutout wcs: %s", new_wcs) @@ -404,8 +409,12 @@ def _extract_volume_from_cube(self, cubehdu, new_hdr, xy0p, xy1p): wdel = hdr["CDELT3"] * u.Unit(hdr.get("CUNIT3", "AA")) fov_wmin, fov_wmax = self.waverange # need to go [+/-] half a bin - mask = ((hdu_waves > fov_wmin - 0.5 * wdel) * - (hdu_waves <= fov_wmax + 0.5 * wdel)) + margin = abs(wdel)/2 + if isinstance(self, FieldOfView3D): + # Spectroscopic inputs remain densities until bin integration; + # retain neighbours on both sides of the output bin edges. + margin = abs(wdel) + np.diff(self.cube_waveset[:2])[0]/2 + mask = (hdu_waves > fov_wmin - margin) & (hdu_waves <= fov_wmax + margin) # OC [2021-12-14] if fov range is not covered by the source return nothing if not np.any(mask): @@ -420,12 +429,11 @@ def _extract_volume_from_cube(self, cubehdu, new_hdr, xy0p, xy1p): f0 = (abs(hdu_waves[i0p] - fov_wmin + 0.5 * wdel) % wdel) / wdel # blue edge f1 = (abs(hdu_waves[i1p] - fov_wmax - 0.5 * wdel) % wdel) / wdel # red edge - data = cubehdu.data[i0p:i1p+1, - xy0p[1]:xy1p[1], - xy0p[0]:xy1p[0]] - data[0, :, :] *= f0.to(1).round(11).value # decompose dimensionless - if i1p > i0p: - data[-1, :, :] *= f1.to(1).round(11).value + data = cubehdu.data[i0p:i1p+1, xy0p[1]:xy1p[1], xy0p[0]:xy1p[0]].copy() + if not isinstance(self, FieldOfView3D): + data[0] *= f0.to_value(1).round(11) + if i1p > i0p: + data[-1] *= f1.to_value(1).round(11) # w0, w1 : the closest cube wavelengths outside the fov edge wavelengths # fov_waves : the fov edge wavelengths @@ -817,8 +825,8 @@ def _make_tablefields(self, fov_waveset, bin_widths, use_photlam=False): else: # Note: these had x/ypix+0.5 until a06ab75 # TODO: could these be something more numpythonic grid-ish? - x = np.array(xpix).astype(int) - y = np.array(ypix).astype(int) # quickest way to round + x = np.floor(xpix + .5).astype(int) + y = np.floor(ypix + .5).astype(int) flux = np.array([fluxes[int(ref)] for ref in field.field["ref"]]) yield flux, np.array(field.field["weight"]), x, y @@ -876,14 +884,11 @@ def make_hdu(self): # enabled, it is therefore not necessary to deploy the fix # below in the else-branch. assert not isinstance(x, Iterable), "x must be an integer" - try: + if 0 <= x < canvas_image_hdu.data.shape[1] and 0 <= y < canvas_image_hdu.data.shape[0]: canvas_image_hdu.data[y, x] += flux * weight - except IndexError: - continue else: # Mask out any stars that were pushed out of the fov by rounding - mask = ((x < canvas_image_hdu.data.shape[1]) * - (y < canvas_image_hdu.data.shape[0])) + mask = (x >= 0) & (y >= 0) & (x < canvas_image_hdu.data.shape[1]) & (y < canvas_image_hdu.data.shape[0]) # This used to contain this line: # canvas_image_hdu.data[y[mask], x[mask]] += flux[mask] * weight[mask] @@ -924,6 +929,13 @@ class FieldOfView3D(FieldOfView): """ + @property + def cube_waveset(self): + wave_unit = u.Unit(from_currsys('!SIM.spectral.wave_unit', self.cmds)) + step = from_currsys('!SIM.spectral.spectral_bin_width', self.cmds) + start, stop = self.meta['wave_min'].value, self.meta['wave_max'].value + return (start + np.arange(int((stop - start)/step))*step)*wave_unit + def _make_cubefields(self, fov_waveset): """ Find Cube fields. @@ -942,7 +954,7 @@ def _make_cubefields(self, fov_waveset): field_hdu.header.update(CRPIX3=1., CRVAL3=fov_waveset[0].to_value(u.um), CDELT3=np.diff(fov_waveset[:2])[0].to_value(u.um), CUNIT3='um', CTYPE3='WAVE') yield field_hdu - def _make_imagefields(self, fov_waveset, spline_order): + def _make_imagefields(self, fov_waveset, spline_order, spatial_psf=None): """ Find Image fields. @@ -957,20 +969,19 @@ def _make_imagefields(self, fov_waveset, spline_order): # ImageHDUs have photons/second/pixel. field_hdu = self._make_scaled_hdu(field) - canvas_image_hdu = fits.ImageHDU( - data=zeros_from_header(self.header, ndims=2), - header=self.header) - canvas_image_hdu = imp_utils.add_imagehdu_to_imagehdu( - field_hdu, - canvas_image_hdu, - spline_order=spline_order, - conserve_flux=True, - differential=True, - ) + if spatial_psf is None: + image = imp_utils.project_spatial_density(field_hdu, self.header) + else: + padded = spatial_psf.padded_header(self) + image = imp_utils.project_spatial_density(field_hdu, padded) + image = fftconvolve(image, spatial_psf.get_kernel(self), mode='same') + px = (padded['NAXIS1'] - self.header['NAXIS1'])//2 + py = (padded['NAXIS2'] - self.header['NAXIS2'])//2 + image = image[py:py + self.header['NAXIS2'], px:px + self.header['NAXIS1']] spec = average_spectrum(field.spectrum, fov_waveset) # 2D * 1D -> 3D - field_cube = canvas_image_hdu.data[None, :, :] * spec[:, None, None] + field_cube = image[None, :, :] * spec[:, None, None] yield field_cube.value def _make_tablefields(self, fov_waveset): @@ -1009,7 +1020,7 @@ def _make_tablefields(self, fov_waveset): for i, j, k in zip(xs, ys, fracs): yield flux_vector * k, i, j else: - yield flux_vector, int(xpix), int(ypix) + yield flux_vector, int(np.floor(xpix + .5)), int(np.floor(ypix + .5)) def _make_backfields(self, fov_waveset): for field in self._get_background_fields(): @@ -1021,7 +1032,7 @@ def _make_backfields(self, fov_waveset): spec = average_spectrum(field.spectrum, fov_waveset) yield spec[:, None, None].value - def make_hdu(self): + def make_hdu(self, spatial_psf=None): """ TBA. @@ -1073,21 +1084,7 @@ def make_hdu(self): # waverange needs to be resampled (I guess) to spectral_bin_width, # but self.waverange can only access the fields. # -> Perhaps change that?? - wave_unit = u.Unit(from_currsys("!SIM.spectral.wave_unit", self.cmds)) - fov_waveset = np.arange( - self.meta["wave_min"].value, self.meta["wave_max"].value, - from_currsys("!SIM.spectral.spectral_bin_width", self.cmds)) * wave_unit - - # Note: There is an edge case where some floating point madness - # results in the arange ticking over by another bin by just - # .000000000005 (yeah...), which creates and off-by-one error - # further down the line. - # TODO: Find a better way to solve this, perhaps with linspace... - n_wave = int( - (self.meta["wave_max"].value - self.meta["wave_min"].value) / - from_currsys("!SIM.spectral.spectral_bin_width", self.cmds) - ) - fov_waveset = fov_waveset[:n_wave].to(u.um) + fov_waveset = self.cube_waveset.to(u.um) # make canvas cube based on waveset of largest cube and NAXIS1,2 # from fov.header @@ -1118,23 +1115,35 @@ def make_hdu(self): assert canvas_cube_hdu.header["NAXIS3"] == self.detector_header["NAXIS3"] for field_hdu in self._make_cubefields(fov_waveset): - canvas_cube_hdu = imp_utils.add_imagehdu_to_imagehdu( - field_hdu, - canvas_cube_hdu, - spline_order=self.spline_order, - conserve_flux=True, - differential=True, - ) + if spatial_psf is None: + canvas_cube_hdu.data += imp_utils.project_spatial_density(field_hdu, self.header) + else: + padded = spatial_psf.padded_header(self) + kernel = spatial_psf.get_kernel(self)[None, :, :] + px = (padded['NAXIS1'] - self.header['NAXIS1'])//2 + py = (padded['NAXIS2'] - self.header['NAXIS2'])//2 + # Bound the temporary padded cube, even for long input spectra. + for start in range(0, len(fov_waveset), 64): + chunk = fits.ImageHDU(data=field_hdu.data[start:start + 64], header=field_hdu.header) + data = imp_utils.project_spatial_density(chunk, padded) + data = fftconvolve(data, kernel, mode='same', axes=(-2, -1)) + canvas_cube_hdu.data[start:start + 64] += data[:, py:py + self.header['NAXIS2'], px:px + self.header['NAXIS1']] canvas_cube_hdu.data = sum( - self._make_imagefields(fov_waveset, self.spline_order), + self._make_imagefields(fov_waveset, self.spline_order, spatial_psf), start=canvas_cube_hdu.data, ) - for flux, x, y in self._make_tablefields(fov_waveset): - # To prevent adding array values in this manner. - assert not isinstance(x, Iterable), "x should be integer" - canvas_cube_hdu.data[:, y, x] += flux + if spatial_psf is None: + for flux, x, y in self._make_tablefields(fov_waveset): + if 0 <= x < self.header['NAXIS1'] and 0 <= y < self.header['NAXIS2']: + canvas_cube_hdu.data[:, y, x] += flux + else: + for field in self._get_table_fields(): + spectra = {ref: average_spectrum(spec, fov_waveset).value for ref, spec in field.spectra.items()} + for row in field.field: + image = spatial_psf.point_source_image(self, row['x'], row['y'])/self.pixel_area.value + canvas_cube_hdu.data += spectra[row['ref']][:, None, None]*image[None, :, :]*row['weight'] canvas_cube_hdu.data = sum(self._make_backfields(fov_waveset), start=canvas_cube_hdu.data) @@ -1209,9 +1218,10 @@ def _spectrum_waveset_aa_value(spectrum): """Return cached spectrum waveset values in Angstrom.""" spec_waveset = _SPECTRUM_WAVESET_AA_CACHE.get(spectrum) if spec_waveset is None: - if spectrum.waveset is None: + waveset = spectrum.waveset + if waveset is None: return None - spec_waveset = spectrum.waveset.to(u.AA).value + spec_waveset = waveset.to_value(u.AA) _SPECTRUM_WAVESET_AA_CACHE[spectrum] = spec_waveset return spec_waveset diff --git a/scopesim/optics/fov_manager.py b/scopesim/optics/fov_manager.py index 4e7a894fe..bd4db5b2b 100644 --- a/scopesim/optics/fov_manager.py +++ b/scopesim/optics/fov_manager.py @@ -215,6 +215,16 @@ def generate_fovs_list(self) -> Iterator[FieldOfView]: [ys_min, ys_max], pixel_scale=pixel_scale / 3600.) + if self.is_spectroscope and "eta_min" in vol["meta"]: + # Tile the physical slit exactly. Detector sampling sets the + # approximate integration step, never the admitted slit area. + for axis, lower, upper in ((1, xs_min, xs_max), (2, ys_min, ys_max)): + count = max(1, int(np.ceil((upper - lower)*3600/pixel_scale))) + skyhdr[f'NAXIS{axis}'] = count + skyhdr[f'CDELT{axis}'] = (upper - lower)/count + skyhdr[f'CRPIX{axis}'] = (count + 1)/2 + skyhdr[f'CRVAL{axis}'] = (lower + upper)/2 + fov_dethdr, _ = ipu.det_wcs_from_sky_wcs( WCS(skyhdr), pixel_scale, plate_scale) skyhdr.update(fov_dethdr.to_header()) diff --git a/scopesim/optics/image_plane_utils.py b/scopesim/optics/image_plane_utils.py index 15f91daf5..d64c995a7 100644 --- a/scopesim/optics/image_plane_utils.py +++ b/scopesim/optics/image_plane_utils.py @@ -7,6 +7,7 @@ from astropy.io import fits from astropy.utils.exceptions import AstropyWarning from scipy import ndimage as ndi +from scipy.sparse import csr_array from ..utils import get_logger @@ -625,6 +626,38 @@ def affine_map( return output +def project_spatial_density(image_hdu, canvas_header): + """Average a surface brightness image/cube over destination pixel areas. + + Overlaps are exact for aligned rectangular pixels, including signed and + unequal scales. Rotated inputs retain the existing affine interpolation + before this area integration. No global flux renormalization is applied + after clipping to the destination footprint. + """ + source = reorient_imagehdu(image_hdu.copy()) + source_wcs = WCS(source.header, naxis=2) + canvas_wcs = WCS(canvas_header, naxis=2) + nx, ny = source.data.shape[-1], source.data.shape[-2] + x = np.arange(canvas_header['NAXIS1'] + 1) - .5 + y = np.arange(canvas_header['NAXIS2'] + 1) - .5 + xedges, _ = source_wcs.all_world2pix(*canvas_wcs.all_pix2world(x, np.zeros_like(x), 0), 0) + _, yedges = source_wcs.all_world2pix(*canvas_wcs.all_pix2world(np.zeros_like(y), y, 0), 0) + + weights = [] + for count, edges in ((nx, xedges), (ny, yedges)): + lower = np.minimum(edges[:-1], edges[1:])[:, None] + upper = np.maximum(edges[:-1], edges[1:])[:, None] + input_lower = np.arange(count)[None, :] - .5 + overlap = np.maximum(0, np.minimum(upper, input_lower + 1) - np.maximum(lower, input_lower)) + weights.append(csr_array(overlap/(upper - lower))) + wx, wy = weights + planes = source.data.reshape(-1, ny, nx) + rows = wy @ planes.transpose(1, 0, 2).reshape(ny, -1) + rows = rows.reshape(wy.shape[0], len(planes), nx).transpose(1, 0, 2) + mapped = (wx @ rows.reshape(-1, nx).T).T + return mapped.reshape(source.data.shape[:-2] + (wy.shape[0], wx.shape[0])) + + def add_imagehdu_to_imagehdu( image_hdu: fits.ImageHDU, canvas_hdu: fits.ImageHDU, diff --git a/scopesim/optics/optical_train.py b/scopesim/optics/optical_train.py index 80e13ced6..a6cfad1d2 100644 --- a/scopesim/optics/optical_train.py +++ b/scopesim/optics/optical_train.py @@ -20,9 +20,11 @@ from .optics_manager import OpticsManager from .fov_manager import FOVManager from .image_plane import ImagePlane +from .fov import FieldOfView3D from ..commands.user_commands import UserCommands from ..detector import DetectorManager -from ..effects import ExtraFitsKeywords +from ..effects import ExtraFitsKeywords, SelectorWheel, MoffatPSF, TERCurve, SpectralEfficiency, EchelleSpectralEfficiency +from ..effects.ter_curves import FilterWheelBase, TaperedQuantumEfficiency from ..utils import from_currsys, top_level_catch, get_logger from ..source.source_templates import empty_sky from .. import __version__ @@ -586,10 +588,28 @@ def observe(self, orig_source=None, update=True, **kwargs): fovs = self.fov_manager.fovs nobar = len(fovs) <= 1 for fov in tqdm(fovs, desc=" FOVs", position=0, disable=nobar): - fov.extract_from(source) - - fov.view() - foveffs = self.optics_manager.fov_effects + foveffs = [] + for effect in self.optics_manager.fov_effects: + if isinstance(effect, SelectorWheel): + effect = effect.get_effect(fov.meta[effect.meta['selector_key']]) + if isinstance(effect, FilterWheelBase): + effect = effect.current_filter + if effect is not None: + foveffs.append(effect) + + # Render at the PSF stage when preceding effects only multiply + # spectra. Point positions then stay continuous, and resolved + # images are convolved before clipping to the aperture. + psf_indices = [i for i, effect in enumerate(foveffs) if isinstance(effect, MoffatPSF)] + defer_render = isinstance(fov, FieldOfView3D) and len(psf_indices) == 1 + if defer_render: + psf_index = psf_indices[0] + defer_render = all(isinstance(effect, (TERCurve, SpectralEfficiency, EchelleSpectralEfficiency, TaperedQuantumEfficiency)) for effect in foveffs[:psf_index]) + if defer_render: + fov.extract_from(source, spatial_header=foveffs[psf_index].padded_header(fov)) + else: + fov.extract_from(source) + fov.view() nobar = len(foveffs) <= 1 for effect in tqdm(foveffs, disable=nobar, desc=" FOV effects", position=1):#, leave=False): @@ -608,6 +628,10 @@ def observe(self, orig_source=None, update=True, **kwargs): self.image_planes[fov.image_plane_id].header.update( WCS(fov.hdu).to_header()) + # Mapping is complete. Keep the rendered FOV and trace's + # rectified spectrum, without retaining every input cube. + fov.cube = None + # ..todo: finish off the multiple image plane stuff # END OF MULTIPROCESSING diff --git a/scopesim/tests/tests_effects/test_MoffatPSF.py b/scopesim/tests/tests_effects/test_MoffatPSF.py index a0625d29e..2ecf2a3d9 100644 --- a/scopesim/tests/tests_effects/test_MoffatPSF.py +++ b/scopesim/tests/tests_effects/test_MoffatPSF.py @@ -31,7 +31,7 @@ def observing_info(cmds): class SimpleFov: - header = {"CDELT1": 0.1 / 3600} + header = {"CDELT1": 0.1 / 3600, "CDELT2": 0.1 / 3600} meta = {"wave_min": 1.0 * u.um, "wave_max": 1.1 * u.um} waveset = np.linspace(1.0, 1.1, 5) * u.um diff --git a/scopesim/tests/tests_optics/test_FieldOfView.py b/scopesim/tests/tests_optics/test_FieldOfView.py index de961eb0f..bca296ad8 100644 --- a/scopesim/tests/tests_optics/test_FieldOfView.py +++ b/scopesim/tests/tests_optics/test_FieldOfView.py @@ -453,14 +453,18 @@ def test_make_spectrum_from_image(self): def test_make_spectrum_from_cube(self): src_cube = so._cube_source() # 10x10" @ 0.2"/pix, [0.5, 2.5]m @ 0.02µm fov = _fov_197_202_um(ndim=1) + original = src_cube.fields[0].data.copy() fov.extract_from(src_cube) spec = fov.make_hdu() - in_sum = np.sum(src_cube.fields[0].data[74:77, :, :]) * 1e-8 + # 1.98 and 2.00 um cells are complete; the 2.02 um cell is half + # inside this FOV. Cropping must not change the input cube in place. + in_sum = (original[74:76].sum() + original[76].sum()/2)*1e-8 out_sum = np.sum(spec(fov.waveset).value) assert in_sum == approx(out_sum) + np.testing.assert_array_equal(src_cube.fields[0].data, original) def test_makes_spectrum_from_all_types_of_source_object(self): src_table = so._table_source() diff --git a/scopesim/tests/tests_optics/test_slit_spatial_integration.py b/scopesim/tests/tests_optics/test_slit_spatial_integration.py new file mode 100644 index 000000000..8027faf12 --- /dev/null +++ b/scopesim/tests/tests_optics/test_slit_spatial_integration.py @@ -0,0 +1,130 @@ +from types import SimpleNamespace +import numpy as np +import pytest +from astropy import units as u +from astropy.wcs import WCS +from astropy.io import fits +from synphot import SourceSpectrum, ConstFlux1D +from synphot.units import PHOTLAM +from scipy.special import stdtr +from scopesim import Source, UserCommands +from scopesim.effects import TERCurve, DetectorWindow +from scopesim.effects.psfs.analytical import MoffatPSF +from scopesim.optics.fov import FieldOfView3D +from scopesim.optics.fov_manager import FOVManager +from scopesim.optics.image_plane_utils import header_from_list_of_xy, det_wcs_from_sky_wcs, project_spatial_density + + +def slit_fov(width, step=.1570545): + nx, ny = int(np.ceil(11/step)), int(np.ceil(width/step)) + header = header_from_list_of_xy([-5.5/3600, 5.5/3600], [-width/7200, width/7200], step/3600) + header.update(NAXIS1=nx, NAXIS2=ny, CRPIX1=(nx+1)/2, CRPIX2=(ny+1)/2, CRVAL1=0., CRVAL2=0., CDELT1=11/nx/3600, CDELT2=width/ny/3600) + return SimpleNamespace(header=header, waveset=np.linspace(.5, .6, 16)*u.um) + + +@pytest.mark.parametrize('width', [.30, .33, .70]) +@pytest.mark.parametrize('fwhm', [.10, .14, .7]) +@pytest.mark.parametrize('offset', [0, .13, .4]) +def test_integrated_point_psf_matches_analytic_slit_loss(width, fwhm, offset): + alpha = 4.765 + psf = MoffatPSF(alpha=alpha, fwhm=fwhm, rounded_edges=False) + image = psf.point_source_image(slit_fov(width), .037, offset) + gamma = fwhm/(2*np.sqrt(2**(1/alpha) - 1)) + scale = np.sqrt(2*alpha - 2)/gamma + expected = stdtr(2*alpha - 2, (width/2 - offset)*scale) - stdtr(2*alpha - 2, (-width/2 - offset)*scale) + # The analytic comparison has an infinite slit length; the model has 11 arcsec. + assert image.sum() == pytest.approx(expected, abs=2e-5) + + +def test_off_slit_source_remains_in_psf_extraction_support(): + fov = slit_fov(.33) + psf = MoffatPSF(alpha=4.765, fwhm=.7) + padded = psf.padded_header(fov) + x, y = WCS(padded).all_world2pix(0, .4/3600, 0) + assert -.5 < x < padded['NAXIS1'] - .5 + assert -.5 < y < padded['NAXIS2'] - .5 + assert psf.point_source_image(fov, 0, .4).sum() > .1 + + +def test_kernel_cache_responds_to_seeing_and_pixel_scale(): + fov = slit_fov(.33) + psf = MoffatPSF(alpha=4.765, fwhm=.7, rounded_edges=False) + first = psf.get_kernel(fov) + np.testing.assert_array_equal(psf.get_kernel(fov), first) + psf.meta['fwhm'] = .2 + narrower = psf.get_kernel(fov) + assert narrower.max() > first.max() + fov.header['CDELT2'] /= 2 + finer = psf.get_kernel(fov) + assert finer.max() < narrower.max() + + +def physical_fov(width=.33): + header = slit_fov(width).header + detector_wcs, _ = det_wcs_from_sky_wcs(WCS(header), .1570545, 10.) + header.update(detector_wcs.to_header()) + cmds = UserCommands(properties={'!SIM.spectral.spectral_bin_width': .01}) + return FieldOfView3D(header, [.5, .6], cmds=cmds, area=1*u.cm**2) + + +def test_throughput_then_psf_admits_off_slit_point_light(): + fov = physical_fov() + psf = MoffatPSF(alpha=4.765, fwhm=.7, rounded_edges=False) + source = Source(x=[.037], y=[.4], ref=[0], weight=[1.], spectra=[SourceSpectrum(ConstFlux1D, amplitude=1*PHOTLAM)]) + fov.extract_from(source, spatial_header=psf.padded_header(fov)) + transmission = TERCurve(array_dict={'wavelength': [.4, .7], 'transmission': [.5, .5]}, wavelength_unit='um') + transmission.apply_to(fov) + psf.apply_to(fov) + expected = psf.point_source_image(fov, .037, .4).sum()*.5 + actual = fov.hdu.data.sum(axis=(1, 2))*fov.pixel_area.value/1e4 + np.testing.assert_allclose(actual, expected, rtol=1e-6) + + +@pytest.mark.parametrize('sign', [-1, 1]) +def test_surface_brightness_uses_signed_wcs_and_pixel_overlaps(sign): + header = header_from_list_of_xy([-.8/3600, .8/3600], [-.6/3600, .6/3600], .1/3600) + header['CDELT1'] *= sign + field = fits.ImageHDU(data=np.full((header['NAXIS2'], header['NAXIS1']), 7.), header=header) + target = slit_fov(.33).header + projected = project_spatial_density(field, target) + # Only the small input's length illuminates this 11-arcsec slit. + expected_area = header['NAXIS1']*.1*.33 + assert projected.sum()*abs(target['CDELT1']*target['CDELT2'])*3600**2 == pytest.approx(7*expected_area, rel=1e-6) + + +def test_resolved_image_and_cube_have_same_psf_and_spectral_normalization(): + header = header_from_list_of_xy([-1/3600, 1/3600], [-1/3600, 1/3600], .05/3600) + yy, xx = np.indices((header['NAXIS2'], header['NAXIS1'])) + density = np.exp(-((xx - 20)**2 + (yy - 26)**2)/8) + image = fits.ImageHDU(data=density*.05**2, header=header) + spectrum = SourceSpectrum(ConstFlux1D, amplitude=1*PHOTLAM) + image_source = Source(image_hdu=image, spectra=[spectrum]) + cube_header = header.copy() + cube_header.update(CRPIX3=1., CRVAL3=.49, CDELT3=.01, CUNIT3='um', CTYPE3='WAVE', BUNIT='photlam arcsec-2') + cube = fits.ImageHDU(data=np.broadcast_to(density, (13, *density.shape)).copy(), header=cube_header) + cube_source = Source(cube=cube) + psf = MoffatPSF(alpha=4.765, fwhm=.7, rounded_edges=False) + rendered = [] + for source in (image_source, cube_source): + fov = physical_fov() + fov.extract_from(source, spatial_header=psf.padded_header(fov)) + psf.apply_to(fov) + rendered.append(fov.hdu.data) + np.testing.assert_allclose(rendered[0], rendered[1], rtol=2e-6, atol=1e-10) + + +@pytest.mark.parametrize('width', [.30, .33, .70]) +def test_fov_manager_tiles_physical_slit_width(width): + class SlitVolume: + def apply_to(self, volumes, **kwargs): + for volume in volumes: + volume.update(x_min=-5.5, x_max=5.5, y_min=-width/2, y_max=width/2) + volume['meta'].update(eta_min=-width/2*u.arcsec, eta_max=width/2*u.arcsec, image_plane_id=0) + return volumes + + detector = DetectorWindow(pixel_size=.015, x=0, y=0, width=100, height=100, units='pixel', image_plane_id=0, pixel_scale=.1570545, plate_scale=10.) + manager = FOVManager([SlitVolume(), detector], pixel_scale=.1570545, plate_scale=10., preload_fovs=False) + manager.is_spectroscope = True + fov = next(manager.generate_fovs_list()) + assert fov.header['NAXIS2']*fov.header['CDELT2']*3600 == pytest.approx(width) + assert fov.header['NAXIS1']*fov.header['CDELT1']*3600 == pytest.approx(11.) From dcb05181ebcd5deb8bc69a7ba9f5fd59f8996d67 Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Sat, 17 Oct 2026 03:11:04 -0700 Subject: [PATCH 104/130] Reuse spatial shift operators across wavelength planes --- scopesim/effects/atmo_dispersion.py | 51 +++++++++++++------ .../tests_effects/test_spatial_cube_shifts.py | 30 +++++++++++ 2 files changed, 66 insertions(+), 15 deletions(-) create mode 100644 scopesim/tests/tests_effects/test_spatial_cube_shifts.py diff --git a/scopesim/effects/atmo_dispersion.py b/scopesim/effects/atmo_dispersion.py index 40455ae01..0d7cb82fa 100644 --- a/scopesim/effects/atmo_dispersion.py +++ b/scopesim/effects/atmo_dispersion.py @@ -1,5 +1,5 @@ from typing import ClassVar, Any -from functools import partial +from functools import partial, lru_cache import numpy as np from astropy import units as u from astropy.coordinates import SkyCoord, EarthLocation @@ -9,6 +9,7 @@ from astropy.coordinates import SkyCoord, HADec from scipy.interpolate import make_interp_spline, RegularGridInterpolator from scipy.ndimage import map_coordinates +from scipy.sparse import csr_array from .effects import Effect from ..optics.fov import FieldOfView3D @@ -122,13 +123,16 @@ def roll_shift(data, dx, dy): for w in range(nw): out[w] = np.roll(data[w], (dy[w], dx[w]), axis=(0, 1)) # zero out the rolled in part, select the correct slice based on sign of dxint, dyint - xslice = slice(0, dx[w]) if dx[w] > 0 else slice(nx-dx[w], nx) - yslice = slice(0, dy[w]) if dy[w] > 0 else slice(ny-dy[w], ny) - out[w][yslice, xslice] = 0 + if dx[w] != 0: + xslice = slice(0, dx[w]) if dx[w] > 0 else slice(nx + dx[w], nx) + out[w, :, xslice] = 0 + if dy[w] != 0: + yslice = slice(0, dy[w]) if dy[w] > 0 else slice(ny + dy[w], ny) + out[w, yslice, :] = 0 return out @staticmethod - def map_coords_shift(data, dx, dy, chunk_w: int=8, order: int=3): + def map_coords_shift(data, dx, dy, chunk_w: int=64, order: int=3): """ Shift each wavelength plane data[w, :, :] by (dx[w], dy[w]) using scipy.ndimage.map_coordinates independently on each spatial plane. @@ -142,7 +146,7 @@ def map_coords_shift(data, dx, dy, chunk_w: int=8, order: int=3): dy : np.ndarray, shape (nwave,) Shift along y (axis=0) in pixels. Positive moves content toward +y. chunk_w : int - Retained for call compatibility; planes are now processed separately. + Number of independent wavelength planes processed together. order : int Spline interpolation order. Default 3. @@ -156,15 +160,32 @@ def map_coords_shift(data, dx, dy, chunk_w: int=8, order: int=3): if dx.shape != (nw,) or dy.shape != (nw,): raise ValueError("dx and dy must have shape (nwave,)") - yi, xi = np.indices((ny, nx), dtype=np.float32) - out = np.empty_like(data) - # Atmospheric refraction shifts sky positions without mixing wavelengths. - # Filtering the full cube once per small output chunk scales quadratically - # with the spectral grid; spatial-plane interpolation is linear in nw. - for wave in range(nw): - coords = [yi - dy[wave], xi - dx[wave]] - out[wave] = map_coordinates(data[wave], coords, order=order, mode='nearest', prefilter=order > 1) - return out + # The same rounded displacement recurs in many wavelength planes. + # Reuse its spatial interpolation operator, without ever interpolating + # along wavelength or filtering the same profile thousands of times. + shifted = data + for axis, displacement, length in ((2, dx, nx), (1, dy, ny)): + out = np.empty(data.shape, dtype=float) + for offset in np.unique(displacement): + operator = _spatial_shift_operator(length, float(offset), order) + planes = np.flatnonzero(displacement == offset) + for start in range(0, len(planes), chunk_w): + selected = planes[start:start + chunk_w] + block = np.moveaxis(shifted[selected], axis, 0) + result = operator @ block.reshape(length, -1) + out[selected] = np.moveaxis(result.reshape(block.shape), 0, axis) + shifted = out + return shifted.astype(data.dtype, copy=False) + + +@lru_cache(maxsize=128) +def _spatial_shift_operator(length, shift, order): + # Apply scipy's interpolation to unit basis profiles once. Cubic spline + # coefficients decay exponentially; omit only weights below 1e-13. + row, column = np.indices((length, length), dtype=float) + response = map_coordinates(np.eye(length), [row, column - shift], order=order, mode='nearest', prefilter=order > 1) + response[abs(response) < 1e-13] = 0 + return csr_array(response.T) class ADShift(ShiftFoV3D): diff --git a/scopesim/tests/tests_effects/test_spatial_cube_shifts.py b/scopesim/tests/tests_effects/test_spatial_cube_shifts.py new file mode 100644 index 000000000..e8ef5a18b --- /dev/null +++ b/scopesim/tests/tests_effects/test_spatial_cube_shifts.py @@ -0,0 +1,30 @@ +import numpy as np +import pytest +from scipy.ndimage import map_coordinates +from scopesim.effects.atmo_dispersion import ShiftFoV3D + + +@pytest.mark.parametrize('shape', [(13, 5, 7), (9, 1, 4), (9, 8, 1)]) +def test_chunked_cubic_shift_matches_independent_spatial_planes(shape): + data = np.random.default_rng(127).uniform(size=shape) + dx = np.linspace(-2.3, 3.7, shape[0], dtype=np.float32) + dy = np.linspace(1.8, -.4, shape[0], dtype=np.float32) + yy, xx = np.indices(shape[1:], dtype=np.float32) + expected = np.stack([map_coordinates(plane, [yy - sy, xx - sx], order=3, mode='nearest') for plane, sx, sy in zip(data, dx, dy)]) + actual = ShiftFoV3D.map_coords_shift(data, dx, dy, chunk_w=4) + np.testing.assert_allclose(actual, expected, rtol=2e-6, atol=1e-6) + + +def test_spatial_shift_does_not_broaden_a_spectral_line(): + cube = np.zeros((11, 5, 7)) + cube[5, 2, 3] = 1. + shifted = ShiftFoV3D.map_coords_shift(cube, np.full(11, .3), np.full(11, -.2)) + np.testing.assert_array_equal(shifted[:5], 0.) + np.testing.assert_array_equal(shifted[6:], 0.) + + +@pytest.mark.parametrize('dx,dy', [(1, 0), (-1, 0), (0, 1), (0, -1), (2, -2)]) +def test_integer_roll_does_not_wrap_light_at_edges(dx, dy): + cube = np.ones((1, 5, 7)) + shifted = ShiftFoV3D.roll_shift(cube, [dx], [dy]) + assert shifted.sum() == (7 - abs(dx))*(5 - abs(dy)) From f9dccd8f6cfbffceb1ab92290e3380f66577cc93 Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Sat, 17 Oct 2026 03:11:26 -0700 Subject: [PATCH 105/130] Preserve spectral edge illumination and reuse the rectified wavelength grid --- scopesim/effects/spectral_trace_list_utils.py | 88 +++++++++---------- scopesim/optics/fov.py | 13 +-- scopesim/optics/optical_train.py | 6 +- .../test_calibrated_spectral_trace.py | 11 +++ .../test_slit_spatial_integration.py | 16 ++++ 5 files changed, 81 insertions(+), 53 deletions(-) diff --git a/scopesim/effects/spectral_trace_list_utils.py b/scopesim/effects/spectral_trace_list_utils.py index a3a10b44d..3e7fbc71b 100644 --- a/scopesim/effects/spectral_trace_list_utils.py +++ b/scopesim/effects/spectral_trace_list_utils.py @@ -14,9 +14,8 @@ import numpy as np -from scipy.interpolate import RectBivariateSpline -from scipy.interpolate import interp1d -from scipy.ndimage import gaussian_filter1d +from scipy.interpolate import interp1d, RectBivariateSpline +from scipy.ndimage import gaussian_filter1d, map_coordinates from astropy.table import Table, vstack from astropy.modeling import fitting @@ -273,6 +272,24 @@ def _focal_plane_grid(self, fov): y_mm = np.linspace(ymin_mm, ymax_mm, sub_naxis2, endpoint=False, dtype=np.float32) return xmin, ymin, x_mm, y_mm, det_wcs + def spectral_padding(self, fov): + """Input wavelength support for slit displacement and optical blur.""" + lower, upper = fov.meta['wave_min'].to_value(u.um), fov.meta['wave_max'].to_value(u.um) + wave = np.unique(u.Quantity(self.table[self.meta['wave_colname']]).to_value(u.um)) + wave = np.r_[lower, wave[(wave > lower) & (wave < upper)], upper] + width = (fov.meta['eta_max'] - fov.meta['eta_min']).to_value(u.arcsec) + if 'slit_dispersion_colname' in self.meta: + derivative = self.slit_dispersion(wave) + else: + pixsize = (fov.detector_header['CDELT1D']*u.Unit(fov.detector_header['CUNIT1D'])).to_value(u.mm) + self._set_dispersion(lower, upper, pixsize=pixsize) + step = abs(fov.header['CDELT2'])*3600 + derivative = self.dlam_per_pix(wave)/step + padding = np.max(abs(derivative))*width/2 + if 'optical_resolving_power' in self.meta: + padding += 5*upper/(self.meta['optical_resolving_power']*np.sqrt(8*np.log(2))) + return padding*u.um + def map_spectra_to_focal_plane(self, fov): """ Apply the spectral trace mapping to a spectral cube. @@ -314,9 +331,6 @@ def map_spectra_to_focal_plane(self, fov): xilam.convolve_lsf(self.meta["optical_resolving_power"]) self._xilamimg = xilam - npix_xi, npix_lam = xilam.npix_xi, xilam.npix_lam - xilam_wcs = xilam.wcs - # focal-plane coordinate images ximg_fpa, yimg_fpa = np.meshgrid(x_mm, y_mm) @@ -324,33 +338,11 @@ def map_spectra_to_focal_plane(self, fov): xi_fpa = self.xy2xi(ximg_fpa, yimg_fpa).astype(np.float32) lam_fpa = self.xy2lam(ximg_fpa, yimg_fpa).astype(np.float32) - # mask everything outside the wavelength range - mask = (xi_fpa >= xi_min) & (xi_fpa <= xi_max) - xi_fpa *= mask - lam_fpa *= mask - - # mask everything outside the wavelength range - # mask = (lam_fpa >= wave_min) & (lam_fpa <= wave_max) - # xi_fpa *= mask - # lam_fpa *= mask - - # Convert to pixel images - # These are the pixel coordinates in the image corresponding to - # xi, lambda - # It is much quicker to do the linear transformation by hand - # than to use the astropy.wcs functions for conversion. - i_img = ((lam_fpa - xilam_wcs.wcs.crval[0]) - / xilam_wcs.wcs.cdelt[0]).astype(int) - j_img = ((xi_fpa - xilam_wcs.wcs.crval[1]) - / xilam_wcs.wcs.cdelt[1]).astype(int) - - # truncate images to remove pixel coordinates outside the image - ijmask = ((i_img >= 0) * (i_img < npix_lam) - * (j_img >= 0) * (j_img < npix_xi)) - - # do the actual interpolation - # image is in [ph/s/um/arcsec] - image = xilam.interp(xi_fpa, lam_fpa, grid=False) * ijmask + # The physical slit ends include the outer half-cells. A truncated + # integer index incorrectly admits pixels below the lower boundary. + inside = (xi_fpa >= xi_min) & (xi_fpa < xi_max) + inside &= (lam_fpa >= wave_min) & (lam_fpa < wave_max) + image = xilam.interp(xi_fpa, lam_fpa, grid=False)*inside # Scale to ph / s / pixel image *= self._trace_flux_scale( @@ -914,12 +906,16 @@ def __init__(self, fov, dlam_per_pix, slit_dispersion=None): # lam0 is the target wavelength. We need to check that this # overlaps with the wavelength range covered by the cube if lam0.min() < cube_lam.max() and lam0.max() > cube_lam.min(): - plane = fov.cube.data[:, i, :].T - plane_interp = RectBivariateSpline(cube_xi, cube_lam, plane, - kx=1, ky=1) - self.image += plane_interp(cube_xi, lam0) + # Only wavelength changes here; xi is already on its native + # grid. Building a 2D spline for every slit cell is redundant. + index = np.searchsorted(cube_lam, lam0, side='right') - 1 + index = np.clip(index, 0, n_lam - 2) + fraction = (lam0 - cube_lam[index])/d_lam + valid = (lam0 >= cube_lam[0]) & (lam0 <= cube_lam[-1]) + plane = fov.cube.data[:, i, :] + self.image += ((1 - fraction[:, None])*plane[index] + fraction[:, None]*plane[index + 1]).T*valid - self.image *= d_eta # ph/s/um/arcsec2 --> ph/s/um/arcsec + self.image *= abs(d_eta) # ph/s/um/arcsec2 --> ph/s/um/arcsec # WCS for the xi-lambda image, i.e. the rectified 2D spectrum # Default WCS with xi in arcsec @@ -945,14 +941,13 @@ def __init__(self, fov, dlam_per_pix, slit_dispersion=None): self.xi = self.wcs.all_pix2world(self.lam[0], np.arange(n_xi), 0)[1] self.npix_xi = n_xi self.npix_lam = n_lam - # ..todo: cubic spline introduces negative values, linear does not. - # Alternative might be to cubic-spline interpolate on sqrt(image), - # with subsequent squaring of the result. This would require - # wrapping RectBivariateSpline in a new (sub)class. - spline_order = (1, 1) - self.interp = RectBivariateSpline(self.xi, self.lam, self.image, - kx=spline_order[0], - ky=spline_order[1]) + + def interp(self, xi, wavelength, grid=True): + """Bilinear density interpolation on the existing uniform WCS grid.""" + xi, wavelength = np.meshgrid(xi, wavelength, indexing='ij') if grid else np.broadcast_arrays(xi, wavelength) + row = (xi - self.xi[0])/self.wcs.wcs.cdelt[1] + column = (wavelength - self.lam[0])/self.wcs.wcs.cdelt[0] + return map_coordinates(self.image, [row, column], order=1, mode='nearest', prefilter=False) def convolve_lsf(self, resolving_power): """Apply an optical Gaussian LSF at constant R, preserving photon density. @@ -970,7 +965,6 @@ def convolve_lsf(self, resolving_power): density = interp1d(log_wave, self.image * self.lam, axis=1)(log_grid) blurred = gaussian_filter1d(density, sigma, axis=1, mode="nearest") self.image = interp1d(log_grid, blurred, axis=1)(log_wave) / self.lam - self.interp = RectBivariateSpline(self.xi, self.lam, self.image, kx=1, ky=1) class Transform2D(): diff --git a/scopesim/optics/fov.py b/scopesim/optics/fov.py index 2c3577d00..892e2fd20 100644 --- a/scopesim/optics/fov.py +++ b/scopesim/optics/fov.py @@ -76,6 +76,7 @@ def __init__(self, header, waverange, detector_header=None, cmds=None, **kwargs) "wave_max": quantify(waverange[1], u.um), "wave_bin_n": 1, "wave_bin_type": "linear", + 'spectral_padding': 0*u.um, "area": 0 * u.m**2, "pixel_area": None, # [arcsec] @@ -172,6 +173,7 @@ def extract_from(self, src, spatial_header=None) -> None: logger.debug("%d fields in FOV", len(fields_in_fov)) minmax = array_minmax(corners_arcsec) * u.arcsec + source_range = self.waverange + np.array([-1, 1])*self.meta['spectral_padding'] for field in fields_in_fov: if isinstance(field, TableSourceField): @@ -183,7 +185,7 @@ def extract_from(self, src, spatial_header=None) -> None: new_fld = TableSourceField( field=extracted, spectra={ - ref: extract_range_from_spectrum(spec, self.waverange) + ref: extract_range_from_spectrum(spec, source_range) for ref, spec in field.spectra.items() if ref in extracted["ref"] }, @@ -197,7 +199,7 @@ def extract_from(self, src, spatial_header=None) -> None: new_fld = ImageSourceField( field=extracted, spectra={ - ref: extract_range_from_spectrum(spec, self.waverange) + ref: extract_range_from_spectrum(spec, source_range) for ref, spec in field.spectra.items() }, ) @@ -215,7 +217,7 @@ def extract_from(self, src, spatial_header=None) -> None: field=None, header=field.header, spectra={ - ref: extract_range_from_spectrum(spec, self.waverange) + ref: extract_range_from_spectrum(spec, source_range) for ref, spec in field.spectra.items() }, ) @@ -407,7 +409,7 @@ def _extract_volume_from_cube(self, cubehdu, new_hdr, xy0p, xy1p): hdu_waves = hdu_waves.quantity.to(u.um) wdel = hdr["CDELT3"] * u.Unit(hdr.get("CUNIT3", "AA")) - fov_wmin, fov_wmax = self.waverange + fov_wmin, fov_wmax = self.waverange + np.array([-1, 1])*self.meta['spectral_padding'] # need to go [+/-] half a bin margin = abs(wdel)/2 if isinstance(self, FieldOfView3D): @@ -934,7 +936,8 @@ def cube_waveset(self): wave_unit = u.Unit(from_currsys('!SIM.spectral.wave_unit', self.cmds)) step = from_currsys('!SIM.spectral.spectral_bin_width', self.cmds) start, stop = self.meta['wave_min'].value, self.meta['wave_max'].value - return (start + np.arange(int((stop - start)/step))*step)*wave_unit + padding = int(np.ceil(self.meta['spectral_padding'].to_value(wave_unit)/step)) + return (start + np.arange(-padding, int((stop - start)/step) + padding)*step)*wave_unit def _make_cubefields(self, fov_waveset): """ diff --git a/scopesim/optics/optical_train.py b/scopesim/optics/optical_train.py index a6cfad1d2..eda4ba665 100644 --- a/scopesim/optics/optical_train.py +++ b/scopesim/optics/optical_train.py @@ -23,7 +23,7 @@ from .fov import FieldOfView3D from ..commands.user_commands import UserCommands from ..detector import DetectorManager -from ..effects import ExtraFitsKeywords, SelectorWheel, MoffatPSF, TERCurve, SpectralEfficiency, EchelleSpectralEfficiency +from ..effects import ExtraFitsKeywords, SelectorWheel, MoffatPSF, TERCurve, SpectralEfficiency, EchelleSpectralEfficiency, SpectralTraceList from ..effects.ter_curves import FilterWheelBase, TaperedQuantumEfficiency from ..utils import from_currsys, top_level_catch, get_logger from ..source.source_templates import empty_sky @@ -597,6 +597,10 @@ def observe(self, orig_source=None, update=True, **kwargs): if effect is not None: foveffs.append(effect) + for effect in foveffs: + if isinstance(effect, SpectralTraceList): + fov.meta['spectral_padding'] = effect.spectral_traces[fov.trace_id].spectral_padding(fov) + # Render at the PSF stage when preceding effects only multiply # spectra. Point positions then stay continuous, and resolved # images are convolved before clipping to the aperture. diff --git a/scopesim/tests/tests_effects/test_calibrated_spectral_trace.py b/scopesim/tests/tests_effects/test_calibrated_spectral_trace.py index 60d4496c6..00114bcee 100644 --- a/scopesim/tests/tests_effects/test_calibrated_spectral_trace.py +++ b/scopesim/tests/tests_effects/test_calibrated_spectral_trace.py @@ -74,3 +74,14 @@ def test_missing_aperture_geometry_fails(): def test_detector_jacobian_uses_physical_pixel_area(): header = fits.Header({'CTYPE1D': 'LINEAR', 'CTYPE2D': 'LINEAR', 'CUNIT1D': 'm', 'CUNIT2D': 'm', 'CDELT1D': .000015, 'CDELT2D': .000030, 'CRPIX1D': 1., 'CRPIX2D': 1., 'CRVAL1D': 0., 'CRVAL2D': 0.}) assert SpectralTrace._detector_pixel_area(header) == pytest.approx(.015*.030) + + +def test_rectified_interpolation_reproduces_a_linear_density_field(): + fov = line_cube(.1) + wave = .995 + np.arange(2001)*fov.cube.header['CDELT3'] + xi = np.arange(3)*.1 + fov.cube.data[:] = 2 + 3*xi[None, None, :] + 5*wave[:, None, None] + image = XiLamImage(fov, lambda w: 0., slit_dispersion=lambda w: 0.) + positions, wavelengths = np.array([.03, .17]), np.array([.998, 1.003]) + expected = 2 + 3*positions + 5*wavelengths + np.testing.assert_allclose(image.interp(positions, wavelengths, grid=False), expected, rtol=2e-6) diff --git a/scopesim/tests/tests_optics/test_slit_spatial_integration.py b/scopesim/tests/tests_optics/test_slit_spatial_integration.py index 8027faf12..2a12e4ba5 100644 --- a/scopesim/tests/tests_optics/test_slit_spatial_integration.py +++ b/scopesim/tests/tests_optics/test_slit_spatial_integration.py @@ -128,3 +128,19 @@ def apply_to(self, volumes, **kwargs): fov = next(manager.generate_fovs_list()) assert fov.header['NAXIS2']*fov.header['CDELT2']*3600 == pytest.approx(width) assert fov.header['NAXIS1']*fov.header['CDELT1']*3600 == pytest.approx(11.) + + +@pytest.mark.parametrize('edge', ['left', 'right']) +def test_subpixel_deposition_does_not_wrap_or_index_beyond_slit(edge): + fov = physical_fov() + fov.meta['sub_pixel'] = True + xpixel = -.25 if edge == 'left' else fov.header['NAXIS1'] - .75 + ypixel = (fov.header['NAXIS2'] - 1)/2 + x, y = WCS(fov.header, naxis=2).all_pix2world(xpixel, ypixel, 0) + source = Source(x=[x*3600], y=[y*3600], ref=[0], weight=[1.], spectra=[SourceSpectrum(ConstFlux1D, amplitude=1*PHOTLAM)]) + fov.extract_from(source) + cube = fov.make_hdu() + admitted = cube.data.sum(axis=(1, 2))*fov.pixel_area.value/1e4 + np.testing.assert_allclose(admitted, .75, rtol=1e-6) + opposite = -1 if edge == 'left' else 0 + np.testing.assert_array_equal(cube.data[:, :, opposite], 0.) From c4f0ec177d9be0e321bd08e3f5f27636df0f247e Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Sat, 17 Oct 2026 03:11:37 -0700 Subject: [PATCH 106/130] Share detector binning geometry and advance readout noise realizations --- scopesim/effects/electronic/pixels.py | 108 ++++++++---------- scopesim/optics/optical_train.py | 59 +++------- .../tests/tests_effects/test_OnChipBinning.py | 19 ++- .../tests/tests_optics/test_OpticalTrain.py | 50 +++++--- 4 files changed, 120 insertions(+), 116 deletions(-) diff --git a/scopesim/effects/electronic/pixels.py b/scopesim/effects/electronic/pixels.py index cefff7924..efb966cf2 100644 --- a/scopesim/effects/electronic/pixels.py +++ b/scopesim/effects/electronic/pixels.py @@ -5,7 +5,7 @@ from numpy.typing import ArrayLike, NDArray -from ...utils import from_currsys, figure_factory, check_keys +from ...utils import from_currsys, figure_factory from . import ElectronicEffect, Detector, logger @@ -93,19 +93,55 @@ class BinnedImageBase(ElectronicEffect): z_order: ClassVar[tuple[int, ...]] = (870,) + def __call__(self, data: ArrayLike) -> NDArray: + bx, by = self.bin_factors + ny, nx = data.shape + return data.reshape(ny//by, by, nx//bx, bx).sum(axis=(1, 3)) + + def bin_header(self, native_header): + """WCS of the summed pixel blocks, also used by trace coordinates.""" + bx, by = self.bin_factors + if not isinstance(bx, int) or not isinstance(by, int) or min(bx, by) < 1: + raise ValueError('Bin factors must be positive integers') + if 'allowed_bins' in self.meta and [bx, by] not in self.meta['allowed_bins']: + raise ValueError(f'Unsupported binning: {bx} x {by}') + header = native_header.copy() + for axis, factor in ((1, bx), (2, by)): + if header[f'NAXIS{axis}'] % factor: + raise ValueError('Binning must divide the full detector dimensions') + header[f'NAXIS{axis}'] //= factor + for suffix in [''] + list('ABCDEFGHIJKLMNOPQRSTUVWXYZ'): + if f'CRPIX1{suffix}' not in header: + continue + for axis, factor in ((1, bx), (2, by)): + header[f'CRPIX{axis}{suffix}'] = (header[f'CRPIX{axis}{suffix}'] - .5)/factor + .5 + if f'CD1_1{suffix}' in header: + for i in (1, 2): + for j, factor in ((1, bx), (2, by)): + header[f'CD{i}_{j}{suffix}'] *= factor + else: + header[f'CDELT1{suffix}'] *= bx + header[f'CDELT2{suffix}'] *= by + if f'PC1_2{suffix}' in header: + header[f'PC1_2{suffix}'] *= by/bx + if f'PC2_1{suffix}' in header: + header[f'PC2_1{suffix}'] *= bx/by + header['BINX'], header['BINY'] = bx, by + return header + + def _apply_to_det(self, det): + header = self.bin_header(det.header) + det.data = self(det.data) + det.header.update(header) + det.meta.update(header) + class BinnedImage(BinnedImageBase): required_keys = {"bin_size"} - def __call__(self, data: ArrayLike) -> NDArray: - height, width = data.shape - binned_data = data.reshape(( - height//self.bin_size, - self.bin_size, - width//self.bin_size, - self.bin_size - )).sum(axis=3).sum(axis=1) - return binned_data + @property + def bin_factors(self): + return self.bin_size, self.bin_size @property def bin_size(self) -> int: @@ -115,15 +151,9 @@ def bin_size(self) -> int: class UnequalBinnedImage(BinnedImageBase): required_keys = {"binx","biny"} - def __call__(self, data: ArrayLike) -> NDArray: - height, width = data.shape - binned_data = data.reshape(( - height//self.biny, - self.biny, - width//self.binx, - self.binx - )).sum(axis=3).sum(axis=1) - return binned_data + @property + def bin_factors(self): + return self.binx, self.biny @property def binx(self) -> int: @@ -134,7 +164,7 @@ def biny(self) -> int: return from_currsys(self.meta["biny"], self.cmds) -class OnChipBinning(ElectronicEffect): +class OnChipBinning(UnequalBinnedImage): """Sum charge in FITS x/y pixel blocks before adding read noise. Unlike a software sum after readout, each output pixel receives one read @@ -142,42 +172,4 @@ class OnChipBinning(ElectronicEffect): WCS reference pixels refer to the centers of the new pixel blocks. """ - required_keys = {"binx", "biny"} z_order = (815,) - - def __init__(self, **kwargs): - super().__init__(**kwargs) - check_keys(self.meta, self.required_keys, action="error") - - def apply_to(self, det, **kwargs): - if not isinstance(det, Detector): - return det - bx = from_currsys(self.meta["binx"], self.cmds) - by = from_currsys(self.meta["biny"], self.cmds) - if not isinstance(bx, int) or not isinstance(by, int) or min(bx, by) < 1: - raise ValueError("On-chip bin factors must be positive integers") - if "allowed_bins" in self.meta and [bx, by] not in self.meta["allowed_bins"]: - raise ValueError(f"Unsupported on-chip binning: {bx} x {by}") - ny, nx = det.data.shape - det._hdu.data = det.data.reshape(ny // by, by, nx // bx, bx).sum(axis=(1, 3)) - header = det.header - for suffix in [""] + list("ABCDEFGHIJKLMNOPQRSTUVWXYZ"): - if f"CRPIX1{suffix}" not in header: - continue - for axis, factor in [(1, bx), (2, by)]: - header[f"CRPIX{axis}{suffix}"] = (header[f"CRPIX{axis}{suffix}"] - 0.5) / factor + 0.5 - if f"CD1_1{suffix}" in header: - for i in (1, 2): - for j, factor in [(1, bx), (2, by)]: - header[f"CD{i}_{j}{suffix}"] *= factor - else: - header[f"CDELT1{suffix}"] *= bx - header[f"CDELT2{suffix}"] *= by - if f"PC1_2{suffix}" in header: - header[f"PC1_2{suffix}"] *= by / bx - if f"PC2_1{suffix}" in header: - header[f"PC2_1{suffix}"] *= bx / by - header["BINX"] = bx - header["BINY"] = by - det.meta.update(header) - return det diff --git a/scopesim/optics/optical_train.py b/scopesim/optics/optical_train.py index eda4ba665..8e3312207 100644 --- a/scopesim/optics/optical_train.py +++ b/scopesim/optics/optical_train.py @@ -121,6 +121,7 @@ def __init__(self, cmds=None): self.detector_managers = [] self.yaml_dicts = None self._last_source = None + self._readout_id = 0 if cmds is not None: self.load(cmds) @@ -233,13 +234,11 @@ def trace_detector_coordinates(self, *, xi=0 * u.arcsec, explicit definition of which output image owns each trace coordinate. """ from ..effects import ( - BinnedImage, - OnChipBinning, Rotate90CCD, SelectorWheel, SpectralTraceList, - UnequalBinnedImage, ) + from ..effects.electronic.pixels import BinnedImageBase trace_lists = self.optics_manager.get_all(SpectralTraceList) if len(trace_lists) != 1: @@ -326,31 +325,14 @@ def image_origin_on_canvas(shape_xy, image_wcs, canvas_wcs): "detectors on one image plane.") detector = detector_manager[0] detector_id = detector.header["ID"] + readout_header = detector._native_header.copy() for effect in self.optics_manager.detector_effects: - selected_effect = ( - effect.wheel_effects.get(detector_id) - if isinstance(effect, SelectorWheel) - else effect - ) + selected_effect = effect.get_effect(detector_id) if isinstance(effect, SelectorWheel) else effect if selected_effect is None: continue - if isinstance(selected_effect, BinnedImage): - bin_size = from_currsys( - selected_effect.meta["bin_size"], self.cmds) - if bin_size != 1: - raise NotImplementedError( - "Trace detector coordinates do not yet support " - "post-extraction detector binning.") - if isinstance(selected_effect, (UnequalBinnedImage, OnChipBinning)): - binx = from_currsys( - selected_effect.meta["binx"], self.cmds) - biny = from_currsys( - selected_effect.meta["biny"], self.cmds) - if binx != 1 or biny != 1: - raise NotImplementedError( - "Trace detector coordinates do not yet support " - "post-extraction detector binning.") + if isinstance(selected_effect, BinnedImageBase): + readout_header = selected_effect.bin_header(readout_header) if isinstance(selected_effect, Rotate90CCD): rotations = from_currsys( selected_effect.meta["rotations"], self.cmds) @@ -359,7 +341,7 @@ def image_origin_on_canvas(shape_xy, image_wcs, canvas_wcs): "Trace detector coordinates do not yet support " "post-extraction detector rotation.") - detector_wcs = WCS(detector.header, key="D") + detector_wcs = WCS(detector._native_header, key="D") image_plane_wcs = image_plane._det_wcs if ( not np.allclose(image_plane_wcs.wcs.pc, np.eye(2)) @@ -429,13 +411,7 @@ def image_origin_on_canvas(shape_xy, image_wcs, canvas_wcs): f"Trace {trace_name!r} has overlapping FOV wavelength " "ranges.") - ( - _, - _, - x_grid_mm, - y_grid_mm, - subimage_wcs, - ) = trace._focal_plane_grid(fov) + _, _, x_grid_mm, y_grid_mm, subimage_wcs = trace._focal_plane_grid(fov) if ( not np.allclose(subimage_wcs.wcs.pc, np.eye(2)) or not np.allclose( @@ -477,6 +453,8 @@ def image_origin_on_canvas(shape_xy, image_wcs, canvas_wcs): f"Requested wavelengths for trace {trace_name!r} " "do not map through a configured FOV.") + focal_x, focal_y = detector_wcs.all_pix2world(x_pix, y_pix, 0) + x_pix, y_pix = WCS(readout_header, key='D').all_world2pix(focal_x, focal_y, 0) point_count = wave_grid.size columns["readout_index"].extend( np.full(point_count, readout_index)) @@ -534,6 +512,7 @@ def observe(self, orig_source=None, update=True, **kwargs): """ if update: self.update(**kwargs) + self._readout_id = 0 if self.cmds["!SIM.random.seed"] is None: # Concretize seed from system entropy to keep constant during the @@ -716,7 +695,7 @@ def readout( self, filename: Path | str | None = None, reset: bool = True, - roid: int = 0, + roid: int | None = None, **kwargs ) -> list[HDUList]: """ @@ -731,13 +710,10 @@ def readout( reset : bool, optional If true (the default), do not keep any parameters set during any previous readout. This is usually what you want. - roid : int, optional - Readout ID, used for seeding randomness. If you run multiple - readouts on the same observation and want the random effects to - produce different results, set this parameter with e.g. the loop - counter. Simple integers are fine (0, 1, 2, ...), because this gets - wrangled into a proper random seed downstream. If you need to - reproduce a specific readout in a sequence, set e.g. `roid=4`. + roid : int or None, optional + Readout ID for reproducible noise. By default, successive calls + use 0, 1, 2, ... within each observation. Supply an explicit ID to + replay that noise realization with the same root seed. **kwargs : Any other kwargs passed to the detector effects. @@ -749,6 +725,9 @@ def readout( """ if reset: self.cmds.clear() + if roid is None: + roid = self._readout_id + self._readout_id += 1 # Hack to make sure AutoExposure and Quantization work properly. # Should probably be removed once #428 is fixed properly. diff --git a/scopesim/tests/tests_effects/test_OnChipBinning.py b/scopesim/tests/tests_effects/test_OnChipBinning.py index 06ed0a62d..ec3347f70 100644 --- a/scopesim/tests/tests_effects/test_OnChipBinning.py +++ b/scopesim/tests/tests_effects/test_OnChipBinning.py @@ -3,7 +3,7 @@ from scopesim.detector import Detector from scopesim.optics.image_plane_utils import header_from_list_of_xy from astropy.wcs import WCS -from scopesim.effects.electronic import OnChipBinning, BasicReadoutNoise +from scopesim.effects.electronic import OnChipBinning, UnequalBinnedImage, BasicReadoutNoise def detector(nx=120, ny=80): @@ -56,3 +56,20 @@ def test_ccd_bin_does_not_multiply_amplifier_noise(): OnChipBinning(binx=2, biny=2).apply_to(det) BasicReadoutNoise(noise_std=3, ndit=1, random_seed=17).apply_to(det) assert det.data.std() == pytest.approx(3, rel=.02) + + +def test_software_sum_accumulates_independent_read_noise(): + det = detector(600, 400) + BasicReadoutNoise(noise_std=3, ndit=1, random_seed=17).apply_to(det) + UnequalBinnedImage(binx=2, biny=2).apply_to(det) + assert det.data.std() == pytest.approx(6, rel=.02) + assert det.header['CDELT1D'] == pytest.approx(.03) + assert det.header['CDELT2D'] == pytest.approx(.03) + + +def test_header_projection_matches_applied_binning(): + det = detector() + binning = OnChipBinning(binx=1, biny=2) + expected = binning.bin_header(det.header) + binning.apply_to(det) + assert det.header == expected diff --git a/scopesim/tests/tests_optics/test_OpticalTrain.py b/scopesim/tests/tests_optics/test_OpticalTrain.py index 581b5e29f..0a8bb22cc 100644 --- a/scopesim/tests/tests_optics/test_OpticalTrain.py +++ b/scopesim/tests/tests_optics/test_OpticalTrain.py @@ -23,6 +23,8 @@ Effect, SpectralTraceList, UnequalBinnedImage, + OnChipBinning, + BasicReadoutNoise, ) from scopesim.utils import find_file @@ -205,8 +207,8 @@ def test_maps_live_trace_to_zero_origin_detector_pixels(self): assert coordinates["trace_id"][0] == "linear" assert coordinates["readout_index"][0] == 0 - assert coordinates["detector_x"][0].to_value(u.pixel) == approx(49) - assert coordinates["detector_y"][0].to_value(u.pixel) == approx(39) + assert coordinates['detector_x'][0].to_value(u.pixel) == approx(49.5) + assert coordinates['detector_y'][0].to_value(u.pixel) == approx(39.5) def test_includes_renderer_endpoint_sampling(self): train, _ = spectral_geometry_train() @@ -214,12 +216,9 @@ def test_includes_renderer_endpoint_sampling(self): coordinates = train.trace_detector_coordinates( wavelengths={"linear": [1.25] * u.um}) - # The rendered subimage samples -0.65..0.65 mm at 13 inclusive - # positions. The trace's -0.25 mm point therefore lands here, rather - # than at the ideal detector-WCS coordinate x=47. - expected_x = 43 + (-0.25 + 0.65) / (0.65 + 0.65) * 12 - assert coordinates["detector_x"][0].to_value(u.pixel) == approx( - expected_x) + # The upstream renderer now uses native detector centers, excluding + # the next pixel beyond the subimage's last column. + assert coordinates['detector_x'][0].to_value(u.pixel) == approx(47.) def test_returns_all_trace_wavelengths_and_requested_slit_positions(self): train, _ = spectral_geometry_train() @@ -247,8 +246,8 @@ def test_uses_changed_live_trace_transform(self): wavelengths={"linear": [1.5] * u.um}) trace = trace_list.spectral_traces["linear"] - trace.meta["offset_y"] = 0.1 - trace.compute_interpolation_functions() + trace.xilam2y.posttransform = (lambda y: y + .1, {}) + trace.xy2lam.pretransform_y = (lambda y: y - .1, {}) after = train.trace_detector_coordinates( wavelengths={"linear": [1.5] * u.um}) @@ -261,14 +260,19 @@ def test_rejects_multiple_detectors_on_one_image_plane(self): with pytest.raises(NotImplementedError, match="multiple detectors"): train.trace_detector_coordinates() - def test_rejects_post_extraction_binning(self): + @pytest.mark.parametrize('binning_class', [UnequalBinnedImage, OnChipBinning]) + def test_binned_coordinates_match_before_and_after_readout_geometry(self, binning_class): train, _ = spectral_geometry_train() - train.optics_manager.detector_effects = [ - UnequalBinnedImage(binx=2, biny=1), - ] - - with pytest.raises(NotImplementedError, match="binning"): - train.trace_detector_coordinates() + native = train.trace_detector_coordinates() + binning = binning_class(binx=2, biny=1) + train.optics_manager.detector_effects = [binning] + before = train.trace_detector_coordinates() + np.testing.assert_allclose(before['detector_x'], (native['detector_x'] + .5*u.pixel)/2 - .5*u.pixel) + np.testing.assert_allclose(before['detector_y'], native['detector_y']) + binning.apply_to(train.detector_managers[0][0]) + after = train.trace_detector_coordinates() + np.testing.assert_array_equal(before['detector_x'], after['detector_x']) + np.testing.assert_array_equal(before['detector_y'], after['detector_y']) @pytest.mark.slow @@ -381,6 +385,18 @@ def test_works_with_a_pointer_to_fits_imagehdu(self, cmds): @pytest.mark.usefixtures("patch_mock_path") class TestReadout: + def test_successive_noise_realizations_can_be_replayed(self, unity_cmds, unity_src): + commands = deepcopy(unity_cmds) + commands['!SIM.random.seed'] = 17 + opt = OpticalTrain(commands) + opt.optics_manager.add_effect(BasicReadoutNoise(noise_std=3., ndit=1, cmds=opt.cmds)) + opt.observe(unity_src) + first = opt.readout()[0][1].data + second = opt.readout()[0][1].data + replay = opt.readout(roid=0)[0][1].data + assert not np.array_equal(first, second) + np.testing.assert_array_equal(first, replay) + def test_readout_works_when_source_observed(self, unity_cmds, unity_src): opt = OpticalTrain(unity_cmds) From 3264a6e6e594cd69be06d98cdba975858ad1c640 Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Sat, 17 Oct 2026 03:11:41 -0700 Subject: [PATCH 107/130] Align regression checks with upstream QE and rotated trace geometry --- .../tests_effects/test_SpectralTraceList.py | 6 ++++-- .../tests_effects/test_SpectralTraceListUtils.py | 6 +++--- .../tests/tests_optics/test_OpticsManager.py | 16 ++++++++-------- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/scopesim/tests/tests_effects/test_SpectralTraceList.py b/scopesim/tests/tests_effects/test_SpectralTraceList.py index 8461857a2..f1ba54a70 100644 --- a/scopesim/tests/tests_effects/test_SpectralTraceList.py +++ b/scopesim/tests/tests_effects/test_SpectralTraceList.py @@ -224,7 +224,9 @@ def test_detector_angle_clips_to_detector_not_padding(self, tmp_path): ) unrotated_trace = next(iter(unrotated.spectral_traces.values())) - assert trace.wave_min == pytest.approx(unrotated_trace.wave_min) - assert trace.wave_max == pytest.approx(unrotated_trace.wave_max) + # Rotation changes which wavelengths reach the rectangular detector; + # clipping must occur after rotation, not at the old unrotated edges. + assert trace.wave_min < trace.wave_max + assert not np.allclose([trace.wave_min, trace.wave_max], [unrotated_trace.wave_min, unrotated_trace.wave_max]) assert not np.allclose(trace.table["x"], unrotated_trace.table["x"]) assert not np.allclose(trace.table["y"], unrotated_trace.table["y"]) diff --git a/scopesim/tests/tests_effects/test_SpectralTraceListUtils.py b/scopesim/tests/tests_effects/test_SpectralTraceListUtils.py index 6bd4fa101..04be6beef 100644 --- a/scopesim/tests/tests_effects/test_SpectralTraceListUtils.py +++ b/scopesim/tests/tests_effects/test_SpectralTraceListUtils.py @@ -196,7 +196,7 @@ def test_call_gives_correct_result(self, quadratic): # matrix and explicit function td2d = Transform2D(quadratic['matrix']) - assert td2d(x, y) == quadratic['function'](x, y) + assert td2d(x, y) == pytest.approx(quadratic['function'](x, y)) def test_gradient_gives_correct_result(self, quadratic): x = np.random.randn() @@ -205,8 +205,8 @@ def test_gradient_gives_correct_result(self, quadratic): tf2d = Transform2D(quadratic['matrix']) tf2d_grad = tf2d.gradient() - assert tf2d_grad[0](x, y) == quadratic['gradient'][0](x, y) - assert tf2d_grad[1](x, y) == quadratic['gradient'][1](x, y) + assert tf2d_grad[0](x, y) == pytest.approx(quadratic['gradient'][0](x, y)) + assert tf2d_grad[1](x, y) == pytest.approx(quadratic['gradient'][1](x, y)) def test_grid_true_gives_correct_shape(self, tf2d): n_x, n_y = 12, 3 diff --git a/scopesim/tests/tests_optics/test_OpticsManager.py b/scopesim/tests/tests_optics/test_OpticsManager.py index 1c6b77cac..cd055f5d2 100644 --- a/scopesim/tests/tests_optics/test_OpticsManager.py +++ b/scopesim/tests/tests_optics/test_OpticsManager.py @@ -59,7 +59,7 @@ def test_effect_order_table_uses_runtime_phase_order(self, detector_setup_rows = tbl[tbl["phase"] == "detector_setup"] assert source_rows["effect"].tolist() == ["detector_qe_curve"] - assert fov_rows["effect"].tolist() == ["detector_qe_curve"] + assert len(fov_rows) == 0 # Ordinary QE is applied once, at the source. assert detector_setup_rows["effect"].tolist() == [ "micado_detector_geometry"] @@ -76,7 +76,7 @@ def test_effect_order_table_summarises_selector_wheels(self): "selector_key": "aperture_id", "wheel": [{ "selector_value": [1, 2], - "effect_class": "TERCurve", + "effect_class": "SpectralQuantumEfficiency", "effect_kwargs": {"filename": "TER_blank.dat"}, }], }, @@ -86,14 +86,14 @@ def test_effect_order_table_summarises_selector_wheels(self): opt_man = opt_mgr.OpticsManager([yaml_dict]) tbl = opt_man.effect_order_table() - source_row = tbl[tbl["phase"] == "source"][0] fov_row = tbl[tbl["phase"] == "fov"][0] - assert source_row["effect"] == "selected_qe" - assert source_row["class"] == "SelectorWheel" - assert source_row["selector_key"] == "aperture_id" - assert source_row["selector_values"] == "1, 2" - assert source_row["selected_classes"] == "TERCurve" + assert len(tbl[tbl['phase'] == 'source']) == 0 + assert fov_row['effect'] == 'selected_qe' + assert fov_row['class'] == 'SelectorWheel' + assert fov_row['selector_key'] == 'aperture_id' + assert fov_row['selector_values'] == '1, 2' + assert fov_row['selected_classes'] == 'SpectralQuantumEfficiency' assert fov_row["phase_z_orders"] == "610" From 1f528b4a9b595d4191df8ce0f38a3b3e43dd60ca Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Sat, 17 Oct 2026 03:41:54 -0700 Subject: [PATCH 108/130] Inspect trace geometry with additive image-plane backgrounds --- scopesim/optics/optical_train.py | 9 +++++---- .../tests/tests_optics/test_OpticalTrain.py | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/scopesim/optics/optical_train.py b/scopesim/optics/optical_train.py index 8e3312207..117a37930 100644 --- a/scopesim/optics/optical_train.py +++ b/scopesim/optics/optical_train.py @@ -234,6 +234,7 @@ def trace_detector_coordinates(self, *, xi=0 * u.arcsec, explicit definition of which output image owns each trace coordinate. """ from ..effects import ( + ImagePlaneBackground, Rotate90CCD, SelectorWheel, SpectralTraceList, @@ -247,10 +248,10 @@ def trace_detector_coordinates(self, *, xi=0 * u.arcsec, "SpectralTraceList.") trace_list = trace_lists[0] - if self.optics_manager.image_plane_effects: - raise NotImplementedError( - "Trace detector coordinates do not yet support image-plane " - "effects.") + for effect in self.optics_manager.image_plane_effects: + selected = effect.wheel_effects.values() if isinstance(effect, SelectorWheel) else [effect] + if any(not isinstance(item, ImagePlaneBackground) for item in selected): + raise NotImplementedError('Trace detector coordinates only support additive image-plane backgrounds.') if self.optics_manager.detector_array_effects: raise NotImplementedError( "Trace detector coordinates do not yet support detector-array " diff --git a/scopesim/tests/tests_optics/test_OpticalTrain.py b/scopesim/tests/tests_optics/test_OpticalTrain.py index 0a8bb22cc..4f8e9c003 100644 --- a/scopesim/tests/tests_optics/test_OpticalTrain.py +++ b/scopesim/tests/tests_optics/test_OpticalTrain.py @@ -25,6 +25,8 @@ UnequalBinnedImage, OnChipBinning, BasicReadoutNoise, + ImagePlaneBackground, + SelectorWheel, ) from scopesim.utils import find_file @@ -199,6 +201,22 @@ def test_ignore_effects_works(self, cmds_with_ignore): class TestTraceDetectorCoordinates: + @pytest.mark.parametrize('selected', [False, True]) + def test_additive_background_preserves_trace_coordinates(self, selected): + train, _ = spectral_geometry_train() + before = train.trace_detector_coordinates() + background = SelectorWheel(selector_key='image_plane_id', wheel=[{'selector_value': [0], 'effect_class': 'ImagePlaneBackground', 'effect_kwargs': {'value': 1.}}]) if selected else ImagePlaneBackground(value=1.) + train.optics_manager.image_plane_effects = [background] + after = train.trace_detector_coordinates() + np.testing.assert_array_equal(before['detector_x'], after['detector_x']) + np.testing.assert_array_equal(before['detector_y'], after['detector_y']) + + def test_unsupported_image_plane_transform_is_not_silently_ignored(self): + train, _ = spectral_geometry_train() + train.optics_manager.image_plane_effects = [Effect()] + with pytest.raises(NotImplementedError, match='additive'): + train.trace_detector_coordinates() + def test_maps_live_trace_to_zero_origin_detector_pixels(self): train, _ = spectral_geometry_train() From 7e30187eed857aaae85c3f35929e02b0f6a1f9b8 Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Sat, 17 Oct 2026 03:42:45 -0700 Subject: [PATCH 109/130] Resolve detector readout modes during trace-coordinate inspection --- scopesim/optics/optical_train.py | 13 +++++++++---- scopesim/tests/tests_optics/test_OpticalTrain.py | 12 ++++++++++++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/scopesim/optics/optical_train.py b/scopesim/optics/optical_train.py index 117a37930..9b7a72d8f 100644 --- a/scopesim/optics/optical_train.py +++ b/scopesim/optics/optical_train.py @@ -234,6 +234,7 @@ def trace_detector_coordinates(self, *, xi=0 * u.arcsec, explicit definition of which output image owns each trace coordinate. """ from ..effects import ( + DetectorModePropertiesSetter, ImagePlaneBackground, Rotate90CCD, SelectorWheel, @@ -252,10 +253,14 @@ def trace_detector_coordinates(self, *, xi=0 * u.arcsec, selected = effect.wheel_effects.values() if isinstance(effect, SelectorWheel) else [effect] if any(not isinstance(item, ImagePlaneBackground) for item in selected): raise NotImplementedError('Trace detector coordinates only support additive image-plane backgrounds.') - if self.optics_manager.detector_array_effects: - raise NotImplementedError( - "Trace detector coordinates do not yet support detector-array " - "effects.") + for effect in self.optics_manager.detector_array_effects: + if not isinstance(effect, DetectorModePropertiesSetter): + raise NotImplementedError('Trace detector coordinates do not yet support this detector-array effect.') + # Mode setters update commands, not image coordinates. Resolve an + # explicit mode before reading any binning command it can set. + if from_currsys(effect.mode_key, self.cmds) == 'auto': + raise NotImplementedError('Select an explicit detector readout mode before inspecting coordinates.') + effect.apply_to(None) fovs_by_trace = {} for fov in self.fov_manager.fovs: diff --git a/scopesim/tests/tests_optics/test_OpticalTrain.py b/scopesim/tests/tests_optics/test_OpticalTrain.py index 4f8e9c003..b083dc83b 100644 --- a/scopesim/tests/tests_optics/test_OpticalTrain.py +++ b/scopesim/tests/tests_optics/test_OpticalTrain.py @@ -26,6 +26,7 @@ OnChipBinning, BasicReadoutNoise, ImagePlaneBackground, + DetectorModePropertiesSetter, SelectorWheel, ) from scopesim.utils import find_file @@ -201,6 +202,17 @@ def test_ignore_effects_works(self, cmds_with_ignore): class TestTraceDetectorCoordinates: + def test_explicit_detector_mode_resolves_its_binning_before_mapping(self): + train, _ = spectral_geometry_train() + before = train.trace_detector_coordinates() + train.cmds = UserCommands(properties={'!OBS.detector_readout_mode': 'slow'}) + setter = DetectorModePropertiesSetter(mode_properties={'slow': {'!DET.binx': 2}}, cmds=train.cmds) + train.optics_manager.detector_array_effects = [setter] + train.optics_manager.detector_effects = [OnChipBinning(binx='!DET.binx', biny=1, cmds=train.cmds)] + after = train.trace_detector_coordinates() + np.testing.assert_allclose(after['detector_x'], (before['detector_x'] + .5*u.pixel)/2 - .5*u.pixel) + np.testing.assert_array_equal(after['detector_y'], before['detector_y']) + @pytest.mark.parametrize('selected', [False, True]) def test_additive_background_preserves_trace_coordinates(self, selected): train, _ = spectral_geometry_train() From ce5c8cd5f5926b6e3d02ef677e68d2870f0cdbb5 Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Sat, 17 Oct 2026 03:54:50 -0700 Subject: [PATCH 110/130] Retain upstream fibre and METIS observation geometry --- scopesim/effects/metis_lms_trace_list.py | 2 +- scopesim/optics/fov_manager.py | 19 ++++++++++++------- scopesim/optics/optical_train.py | 4 +++- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/scopesim/effects/metis_lms_trace_list.py b/scopesim/effects/metis_lms_trace_list.py index d1b650e0a..b6d8f82b8 100644 --- a/scopesim/effects/metis_lms_trace_list.py +++ b/scopesim/effects/metis_lms_trace_list.py @@ -395,7 +395,7 @@ def get_waverange(self, det_mm_lims): lam0 = from_currsys(self.meta["wavelen"], self.cmds) xi0 = 0. - ymid = self.xilam2y(xi0, lam0)[0] # estimate y level of trace + ymid = self.xilam2y(xi0, lam0) # scalar input gives a scalar trace coordinate waverange = self.xy2lam(np.array([xmin, xmax]), np.array([ymid, ymid]), grid=False) diff --git a/scopesim/optics/fov_manager.py b/scopesim/optics/fov_manager.py index bd4db5b2b..261f48f86 100644 --- a/scopesim/optics/fov_manager.py +++ b/scopesim/optics/fov_manager.py @@ -189,7 +189,10 @@ def generate_fovs_list(self) -> Iterator[FieldOfView]: for vol in self.volumes_list: image_plane_id = vol["meta"].get("image_plane_id") det_eff = None - if image_plane_id is not None: + if image_plane_id is None and len(detector_effects) == 1: + # Single-plane fibre/imaging packages need no arm-routing key. + det_eff, = detector_effects + elif image_plane_id is not None: for candidate in detector_effects: if (from_currsys(candidate.meta["image_plane_id"], self.cmds) == image_plane_id): @@ -201,10 +204,9 @@ def generate_fovs_list(self) -> Iterator[FieldOfView]: "plate_scale": self.meta["plate_scale"], } if det_eff is not None: - scale_meta["pixel_scale"] = det_eff.meta.get( - "pixel_scale", scale_meta["pixel_scale"]) - scale_meta["plate_scale"] = det_eff.meta.get( - "plate_scale", scale_meta["plate_scale"]) + for key in ('pixel_scale', 'plate_scale'): + if det_eff.meta[key] != f'!INST.{key}': + scale_meta[key] = det_eff.meta[key] scales = from_currsys(scale_meta, self.cmds) pixel_scale = scales["pixel_scale"] plate_scale = scales["plate_scale"] @@ -228,11 +230,14 @@ def generate_fovs_list(self) -> Iterator[FieldOfView]: fov_dethdr, _ = ipu.det_wcs_from_sky_wcs( WCS(skyhdr), pixel_scale, plate_scale) skyhdr.update(fov_dethdr.to_header()) - dethdr = fov_dethdr + dethdr = fov_dethdr.to_header() + dethdr.update(NAXIS=2, NAXIS1=skyhdr['NAXIS1'], NAXIS2=skyhdr['NAXIS2']) # useful for spectroscopy mode where slit dimensions is not the same # as detector dimensions - if decouple and det_eff is not None: + if decouple: + if det_eff is None: + raise ValueError('A decoupled FOV needs an image-plane ID or one unambiguous detector layout.') dethdr = det_eff.image_plane_header # TODO: Why is this .image_plane_header and not # .detector_headers()[0] or something? diff --git a/scopesim/optics/optical_train.py b/scopesim/optics/optical_train.py index 9b7a72d8f..15af845ed 100644 --- a/scopesim/optics/optical_train.py +++ b/scopesim/optics/optical_train.py @@ -583,7 +583,9 @@ def observe(self, orig_source=None, update=True, **kwargs): foveffs.append(effect) for effect in foveffs: - if isinstance(effect, SpectralTraceList): + # Slit shear needs spectral support before extraction. + # Fibre bundles have no single FOV trace and map separately. + if isinstance(effect, SpectralTraceList) and isinstance(fov, FieldOfView3D) and fov.trace_id is not None: fov.meta['spectral_padding'] = effect.spectral_traces[fov.trace_id].spectral_padding(fov) # Render at the PSF stage when preceding effects only multiply From 06f64b092eab01ebe4da789149d1a3be514c17b4 Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Sat, 17 Oct 2026 04:04:19 -0700 Subject: [PATCH 111/130] Inherit shared trace parameters in METIS LMS slices --- scopesim/effects/metis_lms_trace_list.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scopesim/effects/metis_lms_trace_list.py b/scopesim/effects/metis_lms_trace_list.py index b6d8f82b8..64f6f460d 100644 --- a/scopesim/effects/metis_lms_trace_list.py +++ b/scopesim/effects/metis_lms_trace_list.py @@ -324,6 +324,7 @@ class MetisLMSSpectralTrace(SpectralTrace): """SpectralTrace for the METIS LM spectrograph.""" _class_params = { + **SpectralTrace._class_params, "naxis1": 122, "nslice": 28, "slicewidth": 0.0207, # arcsec From acfa5a71edad17952bd15d1c4c672a4c3bbc0f06 Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Sun, 18 Oct 2026 00:17:43 -0700 Subject: [PATCH 112/130] Fix fixed-groove cross-disperser angles against camera projection --- scopesim/optics/echelle.py | 25 +++++---- .../tests_effects/test_SpectralTraceList.py | 5 +- scopesim/tests/tests_optics/test_echelle.py | 55 +++++++++++++++++++ 3 files changed, 73 insertions(+), 12 deletions(-) create mode 100644 scopesim/tests/tests_optics/test_echelle.py diff --git a/scopesim/optics/echelle.py b/scopesim/optics/echelle.py index 4c7614e63..115422883 100644 --- a/scopesim/optics/echelle.py +++ b/scopesim/optics/echelle.py @@ -191,16 +191,21 @@ def estimate_xdisp_angle_with_groove_len( if not isinstance(d, u.Quantity): d = d * u.mm - # assume we want a dispersion at the midpoint that matches the range to the detector length - t = detector_length / focal_length - k = (l_max - l_min) / d - x = k * (1 + np.sqrt(1 + t ** 2)) / 2 / t - beta_center_guess = np.arccos(np.sqrt(x)) - beta_half_range = np.arcsin(k / 2 / np.sqrt(x)) - beta_max = beta_center_guess + beta_half_range - beta_min = beta_center_guess - beta_half_range - alpha = np.arcsin(l_max / d - np.sin(beta_max)) - return alpha.to(u.rad), ((beta_min + beta_max) / 2).to(u.rad) + # The camera maps beta_center +/- half_angle to +/- detector_length/2. + # Subtracting the two grating equations eliminates the incidence angle: + # (l_max - l_min)/d = 2*cos(beta_center)*sin(half_angle). + if l_max <= l_min or detector_length <= 0 * u.mm or focal_length <= 0 * u.mm or d <= 0 * u.mm: + raise ValueError("Wavelength range, detector length, focal length and groove spacing must be positive.") + half_angle = np.arctan(detector_length / (2 * focal_length)) + cos_center = (l_max - l_min) / (2 * d * np.sin(half_angle)) + if not 0 < cos_center <= 1: + raise ValueError("The wavelength range cannot fit this detector with the specified groove spacing.") + beta_center = np.arccos(cos_center) + sin_alpha = l_min / d - np.sin(beta_center - half_angle) + if beta_center + half_angle >= 90 * u.deg or not -1 <= sin_alpha <= 1: + raise ValueError("The requested detector span requires a non-propagating grating angle.") + alpha = np.arcsin(sin_alpha) + return alpha.to(u.rad), beta_center.to(u.rad) @staticmethod def estimate_xdisp_freq_and_angle(l_min: float | u.Quantity, diff --git a/scopesim/tests/tests_effects/test_SpectralTraceList.py b/scopesim/tests/tests_effects/test_SpectralTraceList.py index 8461857a2..536f601aa 100644 --- a/scopesim/tests/tests_effects/test_SpectralTraceList.py +++ b/scopesim/tests/tests_effects/test_SpectralTraceList.py @@ -224,7 +224,8 @@ def test_detector_angle_clips_to_detector_not_padding(self, tmp_path): ) unrotated_trace = next(iter(unrotated.spectral_traces.values())) - assert trace.wave_min == pytest.approx(unrotated_trace.wave_min) - assert trace.wave_max == pytest.approx(unrotated_trace.wave_max) + # Rotation changes which wavelengths intersect the detector rectangle. + assert trace.wave_min < trace.wave_max + assert not np.allclose([trace.wave_min, trace.wave_max], [unrotated_trace.wave_min, unrotated_trace.wave_max]) assert not np.allclose(trace.table["x"], unrotated_trace.table["x"]) assert not np.allclose(trace.table["y"], unrotated_trace.table["y"]) diff --git a/scopesim/tests/tests_optics/test_echelle.py b/scopesim/tests/tests_optics/test_echelle.py new file mode 100644 index 000000000..6c2080d60 --- /dev/null +++ b/scopesim/tests/tests_optics/test_echelle.py @@ -0,0 +1,55 @@ +"""Planar grating geometry checked through the camera's forward projection.""" + +import numpy as np +import pytest +from astropy import units as u + +from scopesim.optics.echelle import GratingSetup, spectrograph_factory + + +# Wavelength endpoints [nm], cross-disperser density [mm^-1], detector height [pix]. +CHANNELS = [(310, 430, 2000, 4096), (430, 650, 1100, 4096), (650, 980, 625, 4096), + (960, 1340, 350, 2048), (1470, 1830, 300, 2048), (1990, 2450, 270, 2048)] + + +@pytest.mark.parametrize("low,high,density,height", CHANNELS) +def test_fixed_groove_endpoints_fill_camera(low, high, density, height): + instrument = spectrograph_factory(low, high, 225, 18000, 65.6, 40, 50, 1/24, 4.46, + 2048, height, .015, xdisp_groove_length=1/density, dispersion_focal_len=270) + wavelength = np.array([low, high]) * u.nm + pixels = instrument.wavelength_to_y_pixel(wavelength) + # Algebraic inverse/forward evaluation; 1e-7 pixel allows floating-point cancellation. + np.testing.assert_allclose(pixels.value, [0, height], atol=1e-7, rtol=0) + recovered = instrument.y_pixel_to_wavelength(pixels, 1) + np.testing.assert_allclose(recovered.to_value(u.nm), wavelength.value, atol=1e-9, rtol=0) + + +def test_units_and_signed_incidence(): + numeric = GratingSetup.estimate_xdisp_angle_with_groove_len(1470, 1830, 30.72, 225, 1/300) + physical = GratingSetup.estimate_xdisp_angle_with_groove_len(1.47*u.um, 1.83*u.um, 3.072*u.cm, .225*u.m, (1/300)*u.mm) + assert physical[0] < 0 * u.rad + for plain, quantity in zip(numeric, physical): + assert plain.unit == u.rad + assert quantity.unit == u.rad + assert plain.to_value(u.rad) == pytest.approx(quantity.to_value(u.rad), abs=1e-12) + + +@pytest.mark.parametrize("inputs", [(310, 430, 1, 225, .0005), (3000, 3010, 61.44, 225, .0001), + (430, 310, 61.44, 225, .0005), (310, 430, 0, 225, .0005)]) +def test_impossible_geometry_fails(inputs): + with pytest.raises(ValueError): + GratingSetup.estimate_xdisp_angle_with_groove_len(*inputs) + + +def test_explicit_angle_keeps_configured_geometry(): + instrument = spectrograph_factory(960, 1340, 225, 18000, 65.6, 57, 80, 1/24, 4.46, + 2048, 2048, .015, xdisp_groove_length=1/350, xdisp_beta_center=12) + assert instrument.cross_disperser.alpha.to_value(u.deg) == pytest.approx(12) + assert instrument.cross_disperser.beta_center.to_value(u.deg) == pytest.approx(12) + + +def test_free_frequency_endpoints_fill_camera(): + instrument = spectrograph_factory(960, 1340, 225, 18000, 65.6, 57, 80, 1/24, 4.46, + 2048, 2048, .015, xdisp_groove_length=np.nan) + pixels = instrument.wavelength_to_y_pixel(np.array([960, 1340]) * u.nm) + np.testing.assert_allclose(pixels.value, [0, 2048], atol=1e-7, rtol=0) From aefbe60de1e3af42529bec1ea71eb89db7eb9d9a Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Sun, 18 Oct 2026 00:56:16 -0700 Subject: [PATCH 113/130] Expose configured echelle placement for full-FSR inspection --- scopesim/effects/spectral_trace_list.py | 33 +++++-------------- scopesim/optics/echelle.py | 32 ++++++++++++------ .../tests_effects/test_SpectralTraceList.py | 7 ++++ scopesim/tests/tests_optics/test_echelle.py | 9 +++++ 4 files changed, 46 insertions(+), 35 deletions(-) diff --git a/scopesim/effects/spectral_trace_list.py b/scopesim/effects/spectral_trace_list.py index 7d4c6a7ce..97bc3c697 100644 --- a/scopesim/effects/spectral_trace_list.py +++ b/scopesim/effects/spectral_trace_list.py @@ -648,6 +648,7 @@ def _generate_trace_hdulist(self, trace_params): hdul[0].header["EDATA"] = 2 trace_ids, ap_ids, im_ids = [], [], [] + self._spectrographs = {} for row in trace_params.table: prefix = row["prefix"] min_order = row['m0'] - row['n'] + 1 @@ -712,6 +713,9 @@ def _generate_trace_hdulist(self, trace_params): ).to_value(u.deg) cang = np.cos(np.deg2rad(detector_angle)) sang = np.sin(np.deg2rad(detector_angle)) + ss._detector_cos, ss._detector_sin = cang, sang + ss._slit_offset_pix = slit_offset_pix + self._spectrographs[prefix] = ss detector_x_padding = max( 0.0, ( @@ -727,17 +731,6 @@ def _generate_trace_hdulist(self, trace_params): raw_x_min = 0.5 - detector_x_padding - slit_x_padding raw_x_max = disp_npix - 0.5 + detector_x_padding + slit_x_padding - def raw_detector_pixels(wave, order): - x = ss.wavelength_to_x_pixel(wave, order) - y = ss.wavelength_to_y_pixel(wave) - x = u.Quantity(x, copy=False).to_value( - u.dimensionless_unscaled) - y = u.Quantity(y, copy=False).to_value( - u.dimensionless_unscaled) - xpix = np.broadcast_to(x, (slit_offset_pix.size, wave.size)) - ypix = y[None, :] + slit_offset_pix[:, None] - return xpix, ypix - rotated_x_min = rotated_y_min = np.inf rotated_x_max = rotated_y_max = -np.inf # The analytical echelle layout is relative, not yet detector @@ -749,27 +742,17 @@ def raw_detector_pixels(wave, order): raw_x = np.linspace( raw_x_min, raw_x_max, num=max(disp_npix, 2)) wave = ss.x_pixel_to_wavelength(raw_x, order) - xpix, ypix = raw_detector_pixels(wave, order) - xrot = cang * xpix - sang * ypix - yrot = sang * xpix + cang * ypix + xrot, yrot = ss._rotated_detector_pixels(wave, order) rotated_x_min = min(rotated_x_min, float(np.nanmin(xrot))) rotated_x_max = max(rotated_x_max, float(np.nanmax(xrot))) rotated_y_min = min(rotated_y_min, float(np.nanmin(yrot))) rotated_y_max = max(rotated_y_max, float(np.nanmax(yrot))) rotated_x_center = rotated_x_min + (rotated_x_max - rotated_x_min) / 2 rotated_y_center = rotated_y_min + (rotated_y_max - rotated_y_min) / 2 - - def rotated_detector_pixels(wave, order): - xpix, ypix = raw_detector_pixels(wave, order) - xrot = cang * xpix - sang * ypix - yrot = sang * xpix + cang * ypix - return ( - xrot - rotated_x_center + disp_npix / 2, - yrot - rotated_y_center + xdisp_npix / 2, - ) + ss._detector_center = (rotated_x_center, rotated_y_center) def on_detector(wave, order): - xpix, ypix = rotated_detector_pixels(wave, order) + xpix, ypix = ss._detector_pixels(wave, order) return ( (xpix >= 0) & (xpix <= disp_npix) @@ -808,7 +791,7 @@ def on_detector(wave, order): continue s = np.tile(slit_pos, wave.size).reshape(wave.size, slit_pos.size).T.ravel() w = np.tile(wave, slit_offset_pix.size) - xpix, ypix = rotated_detector_pixels(wave, order) + xpix, ypix = ss._detector_pixels(wave, order) xval = (xpix.ravel() - disp_npix / 2) * pix_size.to(u.mm) yval = (ypix.ravel() - xdisp_npix / 2) * pix_size.to(u.mm) diff --git a/scopesim/optics/echelle.py b/scopesim/optics/echelle.py index 115422883..00d6eb9d5 100644 --- a/scopesim/optics/echelle.py +++ b/scopesim/optics/echelle.py @@ -563,17 +563,13 @@ def order_mask(self, wave, fsr_edge: bool = False): def edge_wave(self, fsr=True): """ :param fsr: True to return the FSR edge wavelengths, False to return the detector edges - :return: wavelengths at the edges of the detector (or FSR) for each order, always include central pixel wavelength + :return: the two endpoint wavelengths for each order """ - pix = np.array([0, self.detector.n_pix_x // 2, self.detector.n_pix_x-1]) + .5 - fiducial_waves = self.x_pixel_to_wavelength(pix, self.orders[:, None]) - if not fsr: - return fiducial_waves[:, [0, -1]] - - central_fsr = fiducial_waves[:, 1] / self.orders - fsr_edges = (u.Quantity([-central_fsr / 2, central_fsr / 2]) + fiducial_waves[:, 1]).T - - return fsr_edges + if fsr: + center = self.x_pixel_to_wavelength(self.detector.n_pix_x/2, self.orders) + return center[:, None] * (1 + np.array([-.5, .5])/self.orders[:, None]) + pixels = np.array([.5, self.detector.n_pix_x - .5]) + return self.x_pixel_to_wavelength(pixels, self.orders[:, None]) def central_wave(self, order): """ @@ -621,6 +617,22 @@ def x_pixel_wavelengths(self, pixel_phase=.5): def wavelength_to_y_pixel(self, wave): return self.y_pixel_for_beta(self.cross_disperser.beta(wave, 1)) + def _rotated_detector_pixels(self, wave, order): + x = self.wavelength_to_x_pixel(wave, order).to_value(u.dimensionless_unscaled) + y = self.wavelength_to_y_pixel(wave).to_value(u.dimensionless_unscaled) + xpix = np.broadcast_to(x, (self._slit_offset_pix.size, wave.size)) + ypix = y[None, :] + self._slit_offset_pix[:, None] + return self._detector_cos*xpix - self._detector_sin*ypix, self._detector_sin*xpix + self._detector_cos*ypix + + def _detector_pixels(self, wave, order): + """The trace generator's placed coordinates for all configured slit nodes. + + Unlike the clipped trace table, this mapping can also inspect the full + FSR outside the detector. It uses the same rotation and placement. + """ + x, y = self._rotated_detector_pixels(wave, order) + return x - self._detector_center[0] + self.detector.n_pix_x/2, y - self._detector_center[1] + self.detector.n_pix_y/2 + def y_pixel_to_wavelength(self, pixel, m): """ :param pixel: pixel diff --git a/scopesim/tests/tests_effects/test_SpectralTraceList.py b/scopesim/tests/tests_effects/test_SpectralTraceList.py index 536f601aa..b0261e7b0 100644 --- a/scopesim/tests/tests_effects/test_SpectralTraceList.py +++ b/scopesim/tests/tests_effects/test_SpectralTraceList.py @@ -197,6 +197,13 @@ def test_detector_angle_clips_to_detector_not_padding(self, tmp_path): assert trace.meta["detector_angle"] == 25 assert trace.meta["detector_pad"] == 10 + model = spt._spectrographs['b'] + order = int(trace.meta['trace_id'].split('_')[1]) + wave = u.Quantity(trace.table['wavelength'][:n_wave]) + model_x, model_y = model._detector_pixels(wave, order) + np.testing.assert_allclose(model_x, xpix, atol=1e-8, rtol=0) + np.testing.assert_allclose(model_y, ypix, atol=1e-8, rtol=0) + padded_file = tmp_path / "echelle_trace_parameters_more_padding.dat" _write_echelle_trace_params( padded_file, diff --git a/scopesim/tests/tests_optics/test_echelle.py b/scopesim/tests/tests_optics/test_echelle.py index 6c2080d60..34d0cd260 100644 --- a/scopesim/tests/tests_optics/test_echelle.py +++ b/scopesim/tests/tests_optics/test_echelle.py @@ -53,3 +53,12 @@ def test_free_frequency_endpoints_fill_camera(): 2048, 2048, .015, xdisp_groove_length=np.nan) pixels = instrument.wavelength_to_y_pixel(np.array([960, 1340]) * u.nm) np.testing.assert_allclose(pixels.value, [0, 2048], atol=1e-7, rtol=0) + + +def test_fsr_is_centered_on_the_optical_axis(): + instrument = spectrograph_factory(1990, 2450, 225, 18000, 65.6, 31, 38, 1/24, 4.46, + 2048, 2048, .015, xdisp_groove_length=1/270, dispersion_focal_len=270) + edges = instrument.edge_wave(fsr=True) + center = instrument.central_wave(instrument.orders) + np.testing.assert_allclose(edges.mean(axis=1).to_value(u.nm), center.to_value(u.nm), rtol=1e-12) + np.testing.assert_allclose(np.diff(edges, axis=1).ravel().to_value(u.nm), instrument.fsr(instrument.orders).to_value(u.nm), rtol=1e-12) From 87d6f581e8912e5c3cb5ac58c78fef3f946e1fe1 Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Sun, 18 Oct 2026 01:33:13 -0700 Subject: [PATCH 114/130] Center analytical echelle traces on complete FSR and slit envelopes --- scopesim/effects/spectral_trace_list.py | 57 ++++++------------- .../tests_effects/test_SpectralTraceList.py | 44 ++++++++++++++ 2 files changed, 62 insertions(+), 39 deletions(-) diff --git a/scopesim/effects/spectral_trace_list.py b/scopesim/effects/spectral_trace_list.py index 97bc3c697..e83ede470 100644 --- a/scopesim/effects/spectral_trace_list.py +++ b/scopesim/effects/spectral_trace_list.py @@ -716,53 +716,32 @@ def _generate_trace_hdulist(self, trace_params): ss._detector_cos, ss._detector_sin = cang, sang ss._slit_offset_pix = slit_offset_pix self._spectrographs[prefix] = ss - detector_x_padding = max( - 0.0, - ( - disp_npix * abs(cang) - + xdisp_npix * abs(sang) - - disp_npix - ) / 2, - ) - slit_x_padding = ( - abs(np.tan(np.deg2rad(detector_angle))) - * float(np.nanmax(np.abs(slit_offset_pix))) - ) - raw_x_min = 0.5 - detector_x_padding - slit_x_padding - raw_x_max = disp_npix - 0.5 + detector_x_padding + slit_x_padding - rotated_x_min = rotated_y_min = np.inf rotated_x_max = rotated_y_max = -np.inf - # The analytical echelle layout is relative, not yet detector - # placed. Rotate the footprint first, then place that rotated - # footprint on the detector. Detector padding is not part of the - # optical trace geometry; downstream detector/FOV code is - # responsible for clipping padded display or extraction regions. - for i, order in enumerate(ss.orders): - raw_x = np.linspace( - raw_x_min, raw_x_max, num=max(disp_npix, 2)) - wave = ss.x_pixel_to_wavelength(raw_x, order) + # Place the complete FSR/slit envelope, including portions outside + # the detector. Rotation and translation must not redefine the FSR. + for order, edges in zip(ss.orders, ss.edge_wave(fsr=True)): + wave = np.linspace(edges[0], edges[1], disp_npix + 1) xrot, yrot = ss._rotated_detector_pixels(wave, order) - rotated_x_min = min(rotated_x_min, float(np.nanmin(xrot))) - rotated_x_max = max(rotated_x_max, float(np.nanmax(xrot))) - rotated_y_min = min(rotated_y_min, float(np.nanmin(yrot))) - rotated_y_max = max(rotated_y_max, float(np.nanmax(yrot))) - rotated_x_center = rotated_x_min + (rotated_x_max - rotated_x_min) / 2 - rotated_y_center = rotated_y_min + (rotated_y_max - rotated_y_min) / 2 + rotated_x_min = min(rotated_x_min, xrot.min()) + rotated_x_max = max(rotated_x_max, xrot.max()) + rotated_y_min = min(rotated_y_min, yrot.min()) + rotated_y_max = max(rotated_y_max, yrot.max()) + rotated_x_center = (rotated_x_min + rotated_x_max) / 2 + rotated_y_center = (rotated_y_min + rotated_y_max) / 2 ss._detector_center = (rotated_x_center, rotated_y_center) + # Project the placed detector rectangle back onto the unrotated + # model x axis. This includes every slit node: its offset is in y. + raw_x_center = cang*rotated_x_center + sang*rotated_y_center + raw_x_halfspan = (abs(cang)*disp_npix + abs(sang)*xdisp_npix) / 2 + candidate_raw_x = np.linspace(raw_x_center - raw_x_halfspan, raw_x_center + raw_x_halfspan, disp_npix + 1) + def on_detector(wave, order): xpix, ypix = ss._detector_pixels(wave, order) - return ( - (xpix >= 0) - & (xpix <= disp_npix) - & (ypix >= 0) - & (ypix <= xdisp_npix) - ) + return (xpix >= 0) & (xpix <= disp_npix) & (ypix >= 0) & (ypix <= xdisp_npix) - for i, order in enumerate(ss.orders): - candidate_raw_x = np.linspace( - raw_x_min, raw_x_max, num=max(disp_npix, 2)) + for order in ss.orders: candidate_wave = ss.x_pixel_to_wavelength(candidate_raw_x, order) valid_wave = np.any( on_detector(candidate_wave, order), axis=0) diff --git a/scopesim/tests/tests_effects/test_SpectralTraceList.py b/scopesim/tests/tests_effects/test_SpectralTraceList.py index b0261e7b0..662b7ab4a 100644 --- a/scopesim/tests/tests_effects/test_SpectralTraceList.py +++ b/scopesim/tests/tests_effects/test_SpectralTraceList.py @@ -6,6 +6,7 @@ import numpy as np from astropy import units as u from astropy.io import fits +from astropy.table import Table from scopesim.effects.spectral_trace_list import SpectralTraceList, \ @@ -137,6 +138,49 @@ def _echelle_cmds(): class TestEchelleSpectralTraceList: + @pytest.mark.parametrize('angle', [-25, 0, 25]) + def test_centers_full_fsr_and_keeps_translated_detector_intersections(self, tmp_path, angle): + params_file = tmp_path / 'echelle_trace_parameters.dat' + _write_echelle_trace_params(params_file, detector_angle=angle) + params = Table.read(params_file, format='ascii') + for key, value in {'m0': 62, 'n': 6, 'min_wave': 960, 'max_wave': 1340, 'n_disp': 2048, 'n_xdisp': 1024, 'disp_freq': 24., 'xdisp_freq': 350.}.items(): + params[key][0] = value + params['xbeta_center'] = [10.9] + params.write(params_file, format='ascii', overwrite=True) + effect = EchelleSpectralTraceList(cmds=_echelle_cmds(), filename=str(params_file), wave_colname='wavelength', s_colname='s') + model = effect._spectrographs['b'] + nx, ny = model.detector.n_pix_x, model.detector.n_pix_y + bounds = [] + raw_grid = np.linspace(-2*nx, 3*nx, 50001) + for order, edges in zip(model.orders, model.edge_wave(fsr=True)): + wave = np.linspace(edges[0], edges[1], 4*nx + 1) + x, y = model._detector_pixels(wave, order) + bounds.append([x.min(), x.max(), y.min(), y.max()]) + # Translation changes placement, not dispersion or slit length. + xrot, yrot = model._rotated_detector_pixels(wave, order) + np.testing.assert_allclose(np.diff(x), np.diff(xrot), atol=1e-9, rtol=0) + np.testing.assert_allclose(np.diff(y), np.diff(yrot), atol=1e-9, rtol=0) + slit_length = np.hypot(x[2] - x[0], y[2] - y[0]) + np.testing.assert_allclose(slit_length, model._slit_offset_pix[2] - model._slit_offset_pix[0], atol=1e-9, rtol=0) + + # A deliberately oversized wavelength search independently checks + # that the translated clipping search has not lost either edge. + wave = model.x_pixel_to_wavelength(raw_grid, order) + x, y = model._detector_pixels(wave, order) + inside = np.any((x >= 0) & (x <= nx) & (y >= 0) & (y <= ny), axis=0) + trace_id = f'b_{order}' + if not inside.any(): + assert trace_id not in effect.spectral_traces + continue + trace = effect.spectral_traces[trace_id] + actual = model.wavelength_to_x_pixel(u.Quantity(trace.table['wavelength']), order).value + expected = raw_grid[inside][[0, -1]] + # The generator locates each edge on a roughly one-pixel grid. + np.testing.assert_allclose([actual.min(), actual.max()], expected, atol=3, rtol=0) + bounds = np.array(bounds) + envelope_center = [(bounds[:, 0].min() + bounds[:, 1].max())/2, (bounds[:, 2].min() + bounds[:, 3].max())/2] + np.testing.assert_allclose(envelope_center, [nx/2, ny/2], atol=.01, rtol=0) + def test_does_not_write_generated_hdulist_by_default(self, tmp_path): params_file = tmp_path / "echelle_trace_parameters.dat" _write_echelle_trace_params(params_file) From 4c64fe557a74f0b8a4d3acc54ad69054de014220 Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Sun, 18 Oct 2026 03:47:52 -0700 Subject: [PATCH 115/130] Support fixed detector positions in analytical echelle traces Apply optional physical X/Y positions after rotation and before clipping. Skip automatic placement when positions are supplied, and preserve the original placement and FSR convention when they are absent. Reject incomplete positions without substituting zeros. Validation: 36 targeted ScopeSim tests pass; workspace checks reproduce all original row-2 and main trace tables exactly without placement columns. --- scopesim/effects/spectral_trace_list.py | 62 +++++++++++++------ scopesim/optics/echelle.py | 12 ++-- .../tests_effects/test_SpectralTraceList.py | 58 ++++++++++++++--- scopesim/tests/tests_optics/test_echelle.py | 6 +- 4 files changed, 104 insertions(+), 34 deletions(-) diff --git a/scopesim/effects/spectral_trace_list.py b/scopesim/effects/spectral_trace_list.py index e83ede470..8def5ec7d 100644 --- a/scopesim/effects/spectral_trace_list.py +++ b/scopesim/effects/spectral_trace_list.py @@ -614,6 +614,13 @@ class EchelleSpectralTraceList(SpectralTraceList): nIR 2 0 40 24 970 2500 20000 64.2 225 225 4.7 0.7 10.0 10 0.015 4096 4096 45 175 10 x ---------------------------------------------------------------- + Optional ``detector_offset_x`` and ``detector_offset_y`` columns specify + the rotated analytical camera origin relative to detector center, positive + right/up. Both columns and their ``*_unit`` metadata are required together; + values must have length units. Explicit zeros place that origin at detector + center. Offsets are applied after rotation, before detector clipping, with + no automatic recentering. Omitting both columns retains automatic placement. + The calculated traces are stored in the same HDUList format as required by SpectralTraceList, and supplied to the parent class through hdulist kwarg. @@ -716,27 +723,44 @@ def _generate_trace_hdulist(self, trace_params): ss._detector_cos, ss._detector_sin = cang, sang ss._slit_offset_pix = slit_offset_pix self._spectrographs[prefix] = ss - rotated_x_min = rotated_y_min = np.inf - rotated_x_max = rotated_y_max = -np.inf - # Place the complete FSR/slit envelope, including portions outside - # the detector. Rotation and translation must not redefine the FSR. - for order, edges in zip(ss.orders, ss.edge_wave(fsr=True)): - wave = np.linspace(edges[0], edges[1], disp_npix + 1) - xrot, yrot = ss._rotated_detector_pixels(wave, order) - rotated_x_min = min(rotated_x_min, xrot.min()) - rotated_x_max = max(rotated_x_max, xrot.max()) - rotated_y_min = min(rotated_y_min, yrot.min()) - rotated_y_max = max(rotated_y_max, yrot.max()) - rotated_x_center = (rotated_x_min + rotated_x_max) / 2 - rotated_y_center = (rotated_y_min + rotated_y_max) / 2 + if 'detector_offset_x' in trace_params.table.colnames or 'detector_offset_y' in trace_params.table.colnames: + # Multiplying an ASCII table's masked cell by a Unit yields + # zero, so reject missing values before attaching units. + if np.ma.is_masked(row['detector_offset_x']) or np.ma.is_masked(row['detector_offset_y']): + raise ValueError('Both detector offsets must have values.') + offset_x = row['detector_offset_x'] * u.Unit(trace_params.meta['detector_offset_x_unit']) + offset_y = row['detector_offset_y'] * u.Unit(trace_params.meta['detector_offset_y_unit']) + # Offsets locate the rotated camera origin relative to detector + # center. The raw mapping includes the unrotated pixel origin. + rotated_x_center = cang*disp_npix/2 - sang*xdisp_npix/2 - (offset_x/pix_size).to_value(u.dimensionless_unscaled) + rotated_y_center = sang*disp_npix/2 + cang*xdisp_npix/2 - (offset_y/pix_size).to_value(u.dimensionless_unscaled) + + # Project the translated detector rectangle back onto model x; + # slit offsets are in model y and cancel in this projection. + raw_x_center = cang*rotated_x_center + sang*rotated_y_center + raw_x_halfspan = (abs(cang)*disp_npix + abs(sang)*xdisp_npix)/2 + candidate_raw_x = np.linspace(raw_x_center - raw_x_halfspan, raw_x_center + raw_x_halfspan, disp_npix + 1) + else: + # Preserve the original automatic placement and sampling for + # parameter files that do not specify a detector position. + detector_x_padding = max(0., (disp_npix*abs(cang) + xdisp_npix*abs(sang) - disp_npix)/2) + slit_x_padding = abs(np.tan(np.deg2rad(detector_angle))) * np.max(np.abs(slit_offset_pix)) + raw_x_min = .5 - detector_x_padding - slit_x_padding + raw_x_max = disp_npix - .5 + detector_x_padding + slit_x_padding + candidate_raw_x = np.linspace(raw_x_min, raw_x_max, num=max(disp_npix, 2)) + rotated_x_min = rotated_y_min = np.inf + rotated_x_max = rotated_y_max = -np.inf + for order in ss.orders: + wave = ss.x_pixel_to_wavelength(candidate_raw_x, order) + xrot, yrot = ss._rotated_detector_pixels(wave, order) + rotated_x_min = min(rotated_x_min, xrot.min()) + rotated_x_max = max(rotated_x_max, xrot.max()) + rotated_y_min = min(rotated_y_min, yrot.min()) + rotated_y_max = max(rotated_y_max, yrot.max()) + rotated_x_center = rotated_x_min + (rotated_x_max - rotated_x_min)/2 + rotated_y_center = rotated_y_min + (rotated_y_max - rotated_y_min)/2 ss._detector_center = (rotated_x_center, rotated_y_center) - # Project the placed detector rectangle back onto the unrotated - # model x axis. This includes every slit node: its offset is in y. - raw_x_center = cang*rotated_x_center + sang*rotated_y_center - raw_x_halfspan = (abs(cang)*disp_npix + abs(sang)*xdisp_npix) / 2 - candidate_raw_x = np.linspace(raw_x_center - raw_x_halfspan, raw_x_center + raw_x_halfspan, disp_npix + 1) - def on_detector(wave, order): xpix, ypix = ss._detector_pixels(wave, order) return (xpix >= 0) & (xpix <= disp_npix) & (ypix >= 0) & (ypix <= xdisp_npix) diff --git a/scopesim/optics/echelle.py b/scopesim/optics/echelle.py index 00d6eb9d5..3dc1d4362 100644 --- a/scopesim/optics/echelle.py +++ b/scopesim/optics/echelle.py @@ -565,11 +565,13 @@ def edge_wave(self, fsr=True): :param fsr: True to return the FSR edge wavelengths, False to return the detector edges :return: the two endpoint wavelengths for each order """ - if fsr: - center = self.x_pixel_to_wavelength(self.detector.n_pix_x/2, self.orders) - return center[:, None] * (1 + np.array([-.5, .5])/self.orders[:, None]) - pixels = np.array([.5, self.detector.n_pix_x - .5]) - return self.x_pixel_to_wavelength(pixels, self.orders[:, None]) + pixels = np.array([0, self.detector.n_pix_x//2, self.detector.n_pix_x - 1]) + .5 + fiducial_waves = self.x_pixel_to_wavelength(pixels, self.orders[:, None]) + if not fsr: + return fiducial_waves[:, [0, -1]] + # Retain the original central-pixel wavelength convention for the FSR. + central_fsr = fiducial_waves[:, 1] / self.orders + return (u.Quantity([-central_fsr/2, central_fsr/2]) + fiducial_waves[:, 1]).T def central_wave(self, order): """ diff --git a/scopesim/tests/tests_effects/test_SpectralTraceList.py b/scopesim/tests/tests_effects/test_SpectralTraceList.py index 662b7ab4a..8686336af 100644 --- a/scopesim/tests/tests_effects/test_SpectralTraceList.py +++ b/scopesim/tests/tests_effects/test_SpectralTraceList.py @@ -139,23 +139,32 @@ def _echelle_cmds(): class TestEchelleSpectralTraceList: @pytest.mark.parametrize('angle', [-25, 0, 25]) - def test_centers_full_fsr_and_keeps_translated_detector_intersections(self, tmp_path, angle): + @pytest.mark.parametrize('offset', [(0., 0.), (1.5, -1.2)]) + def test_fixed_placement_and_detector_intersections(self, tmp_path, angle, offset): params_file = tmp_path / 'echelle_trace_parameters.dat' _write_echelle_trace_params(params_file, detector_angle=angle) params = Table.read(params_file, format='ascii') - for key, value in {'m0': 62, 'n': 6, 'min_wave': 960, 'max_wave': 1340, 'n_disp': 2048, 'n_xdisp': 1024, 'disp_freq': 24., 'xdisp_freq': 350.}.items(): + for key, value in {'m0': 75, 'n': 6, 'min_wave': 960, 'max_wave': 1340, 'n_disp': 2048, 'n_xdisp': 1024, 'disp_freq': 24., 'xdisp_freq': 350.}.items(): params[key][0] = value params['xbeta_center'] = [10.9] + params['detector_offset_x'], params['detector_offset_y'] = [offset[0]], [offset[1]] + params.meta['comments'].extend(['detector_offset_x_unit : mm', 'detector_offset_y_unit : mm']) params.write(params_file, format='ascii', overwrite=True) effect = EchelleSpectralTraceList(cmds=_echelle_cmds(), filename=str(params_file), wave_colname='wavelength', s_colname='s') model = effect._spectrographs['b'] nx, ny = model.detector.n_pix_x, model.detector.n_pix_y - bounds = [] raw_grid = np.linspace(-2*nx, 3*nx, 50001) for order, edges in zip(model.orders, model.edge_wave(fsr=True)): wave = np.linspace(edges[0], edges[1], 4*nx + 1) x, y = model._detector_pixels(wave, order) - bounds.append([x.min(), x.max(), y.min(), y.max()]) + # Rotate about the camera origin, then translate in detector axes. + xraw = model.wavelength_to_x_pixel(wave, order).value - nx/2 + yraw = model.wavelength_to_y_pixel(wave).value[None, :] - ny/2 + model._slit_offset_pix[:, None] + c, s = np.cos(np.deg2rad(angle)), np.sin(np.deg2rad(angle)) + expected_x = c*xraw - s*yraw + nx/2 + offset[0]/.015 + expected_y = s*xraw + c*yraw + ny/2 + offset[1]/.015 + np.testing.assert_allclose(x, expected_x, atol=1e-9, rtol=0) + np.testing.assert_allclose(y, expected_y, atol=1e-9, rtol=0) # Translation changes placement, not dispersion or slit length. xrot, yrot = model._rotated_detector_pixels(wave, order) np.testing.assert_allclose(np.diff(x), np.diff(xrot), atol=1e-9, rtol=0) @@ -177,9 +186,44 @@ def test_centers_full_fsr_and_keeps_translated_detector_intersections(self, tmp_ expected = raw_grid[inside][[0, -1]] # The generator locates each edge on a roughly one-pixel grid. np.testing.assert_allclose([actual.min(), actual.max()], expected, atol=3, rtol=0) - bounds = np.array(bounds) - envelope_center = [(bounds[:, 0].min() + bounds[:, 1].max())/2, (bounds[:, 2].min() + bounds[:, 3].max())/2] - np.testing.assert_allclose(envelope_center, [nx/2, ny/2], atol=.01, rtol=0) + + @pytest.mark.parametrize('missing', ['detector_offset_x', 'detector_offset_y', 'detector_offset_x_unit', 'detector_offset_y_unit']) + def test_incomplete_placement_fails(self, tmp_path, missing): + path = tmp_path / 'echelle_trace_parameters.dat' + _write_echelle_trace_params(path) + params = Table.read(path, format='ascii') + for axis in ['x', 'y']: + key = f'detector_offset_{axis}' + if missing != key: + params[key] = [0.] + if missing != f'{key}_unit': + params.meta['comments'].append(f'{key}_unit : mm') + params.write(path, format='ascii', overwrite=True) + with pytest.raises(KeyError, match=missing): + EchelleSpectralTraceList(cmds=_echelle_cmds(), filename=str(path)) + + @pytest.mark.parametrize('missing_value', ['detector_offset_x', 'detector_offset_y']) + def test_blank_placement_value_does_not_become_zero(self, tmp_path, missing_value): + path = tmp_path / 'echelle_trace_parameters.dat' + _write_echelle_trace_params(path) + params = Table.read(path, format='ascii') + for axis in ['x', 'y']: + key = f'detector_offset_{axis}' + params[key] = np.ma.array([0.], mask=[key == missing_value]) + params.meta['comments'].append(f'{key}_unit : mm') + params.write(path, format='ascii', overwrite=True) + with pytest.raises(ValueError, match='Both detector offsets must have values'): + EchelleSpectralTraceList(cmds=_echelle_cmds(), filename=str(path)) + + def test_placement_requires_length_units(self, tmp_path): + path = tmp_path / 'echelle_trace_parameters.dat' + _write_echelle_trace_params(path) + params = Table.read(path, format='ascii') + params['detector_offset_x'], params['detector_offset_y'] = [0.], [0.] + params.meta['comments'].extend(['detector_offset_x_unit : arcsec', 'detector_offset_y_unit : mm']) + params.write(path, format='ascii', overwrite=True) + with pytest.raises(u.UnitConversionError): + EchelleSpectralTraceList(cmds=_echelle_cmds(), filename=str(path)) def test_does_not_write_generated_hdulist_by_default(self, tmp_path): params_file = tmp_path / "echelle_trace_parameters.dat" diff --git a/scopesim/tests/tests_optics/test_echelle.py b/scopesim/tests/tests_optics/test_echelle.py index 34d0cd260..ef8a4fbc4 100644 --- a/scopesim/tests/tests_optics/test_echelle.py +++ b/scopesim/tests/tests_optics/test_echelle.py @@ -55,10 +55,10 @@ def test_free_frequency_endpoints_fill_camera(): np.testing.assert_allclose(pixels.value, [0, 2048], atol=1e-7, rtol=0) -def test_fsr_is_centered_on_the_optical_axis(): +def test_fsr_retains_the_central_pixel_wavelength_convention(): instrument = spectrograph_factory(1990, 2450, 225, 18000, 65.6, 31, 38, 1/24, 4.46, 2048, 2048, .015, xdisp_groove_length=1/270, dispersion_focal_len=270) edges = instrument.edge_wave(fsr=True) - center = instrument.central_wave(instrument.orders) + center = instrument.x_pixel_to_wavelength(instrument.detector.n_pix_x//2 + .5, instrument.orders) np.testing.assert_allclose(edges.mean(axis=1).to_value(u.nm), center.to_value(u.nm), rtol=1e-12) - np.testing.assert_allclose(np.diff(edges, axis=1).ravel().to_value(u.nm), instrument.fsr(instrument.orders).to_value(u.nm), rtol=1e-12) + np.testing.assert_allclose(np.diff(edges, axis=1).ravel().to_value(u.nm), (center/instrument.orders).to_value(u.nm), rtol=1e-12) From ee4e4e67486cc5fcdb969c261199749876ed6e18 Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Sun, 18 Oct 2026 04:10:58 -0700 Subject: [PATCH 116/130] Key natural-seeing cache by zenith angle instead of coordinate frames Repeated FWHM requests with dark-sky timing can produce AltAz coordinates with different obstimes, whose equality raises in Astropy. Use the numeric zenith angle and seeing parameters as the cache identity while retaining the existing seeing law and public interfaces. Validation: 11 PSF tests pass, covering advancing nighttime timestamps, cache reuse, changing seeing/pivot/altitude and RA-Dec observation times, relative AO, constant widths and table interpolation. The original failing performance-notebook cell now completes. --- scopesim/effects/psfs/analytical.py | 4 +- .../tests/tests_effects/test_MoffatPSF.py | 102 +++++++++++++++++- 2 files changed, 103 insertions(+), 3 deletions(-) diff --git a/scopesim/effects/psfs/analytical.py b/scopesim/effects/psfs/analytical.py index dc39f5207..7481aa02e 100644 --- a/scopesim/effects/psfs/analytical.py +++ b/scopesim/effects/psfs/analytical.py @@ -261,7 +261,9 @@ def fwhm(self, wavelengths): fwhm = self.meta["fwhm"] if isinstance(fwhm, dict) and {"pivot_wave", "pivot_wave_unit"}.issubset(fwhm): target, location, time, _ = get_observation_info_from_cmds(self.cmds) - key = tuple(fwhm.items()), target, location, time + # Seeing depends on zenith angle, not the AltAz frame timestamp. + # SkyCoord equality raises when otherwise identical frames differ. + key = tuple(fwhm.items()), get_zenith_angle(target, location, time) elif isinstance(fwhm, dict): key = tuple(fwhm.items()) else: diff --git a/scopesim/tests/tests_effects/test_MoffatPSF.py b/scopesim/tests/tests_effects/test_MoffatPSF.py index 1119df8dd..652e6f047 100644 --- a/scopesim/tests/tests_effects/test_MoffatPSF.py +++ b/scopesim/tests/tests_effects/test_MoffatPSF.py @@ -3,8 +3,13 @@ import numpy as np import pytest from astropy import units as u +from astropy.coordinates import AltAz, EarthLocation, SkyCoord +from astropy.table import Table +from astropy.time import Time +from astropy.utils import iers -from scopesim.effects.psfs.analytical import MoffatPSF +from scopesim.commands import UserCommands +from scopesim.effects.psfs.analytical import AOEnhanceablePSF, MoffatPSF class SimpleFov: @@ -16,7 +21,11 @@ class SimpleFov: def test_constant_fwhm_does_not_require_observation_commands(): psf = MoffatPSF(alpha=4.765, fwhm=0.7) - assert isinstance(psf, MoffatPSF) + wave = np.array([500., 1000.])*u.nm + np.testing.assert_allclose(psf.fwhm(wave).to_value(u.arcsec), [.7, .7]) + cached = psf._fwhm_interp[0] + np.testing.assert_allclose(psf.fwhm(wave.to(u.um)).to_value(u.arcsec), [.7, .7]) + assert psf._fwhm_interp[0] is cached def test_kernel_grows_to_flux_accuracy_without_renormalizing(): @@ -87,3 +96,92 @@ def test_legacy_renormalization_can_be_requested(): kernel = psf.get_kernel(SimpleFov()) assert np.sum(kernel) == pytest.approx(1.0) + + +@pytest.fixture +def seeing_cmds(monkeypatch): + monkeypatch.setattr(iers.conf, 'auto_download', False) + # The regression uses historical timestamps, independent of table freshness. + monkeypatch.setattr(iers.conf, 'auto_max_age', None) + monkeypatch.setattr(Time, 'now', lambda: Time('2026-10-18T10:50:25.709309')) + return UserCommands(properties={ + '!ATMO.longitude': -155.47833, '!ATMO.latitude': 19.82833, '!ATMO.altitude': 4160., + '!OBS.alt': 65.38002267134289, '!OBS.az': 0., '!OBS.airmass': None, + '!OBS.ra': None, '!OBS.dec': None, '!OBS.brightness': 'dark', + }) + + +@pytest.fixture +def natural_psf(seeing_cmds): + return MoffatPSF(cmds=seeing_cmds, alpha=4.765, + fwhm={'seeing': .65, 'seeing_unit': 'arcsec', 'pivot_wave': 500., 'pivot_wave_unit': 'nm'}) + + +def test_nighttime_timestamp_change_reuses_the_same_seeing(natural_psf, monkeypatch): + wave = np.array([500., 1000.])*u.nm + first = natural_psf.fwhm(wave) + cached = natural_psf._fwhm_interp[0] + # These are the two times in the performance notebook's original traceback. + monkeypatch.setattr(Time, 'now', lambda: Time('2026-10-18T10:50:25.735796')) + second = natural_psf.fwhm(wave.to(u.um)) + np.testing.assert_allclose(first.to_value(u.arcsec), [.68825435, .59916022], atol=1e-8, rtol=0) + np.testing.assert_array_equal(second, first) + assert natural_psf._fwhm_interp[0] is cached + + +def test_seeing_and_pivot_changes_refresh_the_cache(natural_psf): + wave = np.array([500., 1000.])*u.nm + first = natural_psf.fwhm(wave) + natural_psf.meta['fwhm']['seeing'] = 1.3 + np.testing.assert_allclose(natural_psf.fwhm(wave).to_value(u.arcsec), 2*first.to_value(u.arcsec)) + natural_psf.meta['fwhm']['pivot_wave'] = 1000. + np.testing.assert_allclose(natural_psf.fwhm(wave).to_value(u.arcsec), 2*2**.2*first.to_value(u.arcsec)) + + +def test_altitude_change_refreshes_the_seeing(natural_psf): + wave = np.array([500.])*u.nm + natural_psf.cmds['!OBS.alt'] = 90. + np.testing.assert_allclose(natural_psf.fwhm(wave).to_value(u.arcsec), [.65]) + natural_psf.cmds['!OBS.alt'] = 30. + np.testing.assert_allclose(natural_psf.fwhm(wave).to_value(u.arcsec), [.65*2**.6]) + + +def test_radec_observation_time_changes_the_seeing(natural_psf): + natural_psf.cmds['!OBS.ra'], natural_psf.cmds['!OBS.dec'] = 30., 20. + target = SkyCoord(ra=30*u.deg, dec=20*u.deg, distance=1*u.AU) + location = EarthLocation.from_geodetic(-155.47833*u.deg, 19.82833*u.deg, 4160*u.m) + widths = [] + for time in ['2026-10-18T10:50:25', '2026-10-18T12:50:25']: + natural_psf.cmds['!OBS.mjdobs'] = time + alt = target.transform_to(AltAz(obstime=Time(time), location=location)).alt + assert alt > 0*u.deg + width = natural_psf.fwhm(np.array([500.])*u.nm).to_value(u.arcsec)[0] + np.testing.assert_allclose(width, .65/np.sin(alt)**.6) + widths.append(width) + assert not np.isclose(widths[0], widths[1]) + + +def test_relative_ao_uses_the_same_natural_seeing_cache(seeing_cmds, tmp_path, monkeypatch): + path = tmp_path / 'ao.dat' + Table({'wavelength': [.4, .5, 1., 2.5], 'fwhm': [.5, .5, .5, .5]}, meta={'comments': ['wavelength_unit : um']}).write(path, format='ascii') + psf = AOEnhanceablePSF(cmds=seeing_cmds, ao_table=str(path), is_absolute=False, enable_ao=True, alpha=4.765, + fwhm={'seeing': .65, 'seeing_unit': 'arcsec', 'pivot_wave': 500., 'pivot_wave_unit': 'nm'}) + wave = np.array([500., 1000.])*u.nm + first = psf.fwhm(wave) + cached = psf._fwhm_interp[0] + monkeypatch.setattr(Time, 'now', lambda: Time('2026-10-18T10:50:25.735796')) + np.testing.assert_array_equal(psf.fwhm(wave), first) + assert psf._fwhm_interp[0] is cached + psf.meta['enable_ao'] = False + np.testing.assert_allclose(psf.fwhm(wave).to_value(u.arcsec), 2*first.to_value(u.arcsec)) + + +def test_table_fwhm_reuses_interpolator_without_observation_commands(tmp_path): + path = tmp_path / 'fwhm.dat' + Table({'wavelength': [.4, .5, 1., 2.5], 'fwhm': [.4, .5, 1., 2.5]}, meta={'comments': ['wavelength_unit : um']}).write(path, format='ascii') + psf = MoffatPSF(alpha=4.765, fwhm=str(path)) + wave = np.array([500., 1000.])*u.nm + np.testing.assert_allclose(psf.fwhm(wave).to_value(u.arcsec), [.5, 1.]) + cached = psf._fwhm_interp[0] + np.testing.assert_allclose(psf.fwhm(wave.to(u.um)).to_value(u.arcsec), [.5, 1.]) + assert psf._fwhm_interp[0] is cached From 307faba50cedefd0e908924cf809b14b16242f5a Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Sun, 18 Oct 2026 06:10:38 -0700 Subject: [PATCH 117/130] Honor explicit command keys in detector readout --- scopesim/optics/optical_train.py | 4 ++-- scopesim/tests/tests_optics/test_OpticalTrain.py | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/scopesim/optics/optical_train.py b/scopesim/optics/optical_train.py index 15af845ed..65e7956e4 100644 --- a/scopesim/optics/optical_train.py +++ b/scopesim/optics/optical_train.py @@ -746,7 +746,7 @@ def readout( # self.cmds["!OBS.ndit"] = None # TODO: This is still hacky but seems to work for now... params = {"exptime": None, "roid": roid} - params.update(kwargs) + params.update({key.removeprefix('!OBS.'): value for key, value in kwargs.items()}) if params["exptime"] is not None and params.get("dit") is None: params.update( dit=None, @@ -761,8 +761,8 @@ def readout( for key, value in params.items(): if is_bangkey(key): - # skip if already explicitly !ABC.xyz logger.debug("%s: %s given in kwargs", key, value) + self.cmds[key] = value continue # otherwise fallback to !OBS logger.debug("%s: %s given in kwargs, put in !OBS", key, value) diff --git a/scopesim/tests/tests_optics/test_OpticalTrain.py b/scopesim/tests/tests_optics/test_OpticalTrain.py index b083dc83b..1585e350d 100644 --- a/scopesim/tests/tests_optics/test_OpticalTrain.py +++ b/scopesim/tests/tests_optics/test_OpticalTrain.py @@ -415,6 +415,16 @@ def test_works_with_a_pointer_to_fits_imagehdu(self, cmds): @pytest.mark.usefixtures("patch_mock_path") class TestReadout: + def test_explicit_readout_keys_and_reset(self, unity_cmds): + opt = OpticalTrain(deepcopy(unity_cmds)) + for seconds in (3600., 300., 3600.): + opt.readout(**{'!OBS.dit': seconds, '!OBS.ndit': 1, '!OBS.dit_blue': seconds / 2}) + assert opt.cmds['!OBS.dit'] == seconds + assert opt.cmds['!OBS.dit_blue'] == seconds / 2 + opt.readout(dit=17., ndit=2) + assert opt.cmds['!OBS.dit'] == 17. + assert opt.cmds['!OBS.ndit'] == 2 + def test_successive_noise_realizations_can_be_replayed(self, unity_cmds, unity_src): commands = deepcopy(unity_cmds) commands['!SIM.random.seed'] = 17 From f2c9c01cd2e2fbe79d86b08234664f3e10f95204 Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Sun, 18 Oct 2026 06:19:42 -0700 Subject: [PATCH 118/130] Derive slit dispersion from the two-collimator optical prescription --- scopesim/effects/spectral_efficiency.py | 6 +++- scopesim/effects/spectral_trace_list.py | 23 ++++++++----- scopesim/optics/echelle.py | 38 ++++++++++++++++++++- scopesim/tests/tests_optics/test_echelle.py | 28 +++++++++++++++ 4 files changed, 84 insertions(+), 11 deletions(-) diff --git a/scopesim/effects/spectral_efficiency.py b/scopesim/effects/spectral_efficiency.py index bb860cc98..62a34bb6d 100644 --- a/scopesim/effects/spectral_efficiency.py +++ b/scopesim/effects/spectral_efficiency.py @@ -168,13 +168,17 @@ def _generate_efficiency_curve_func(self) -> Callable: xdisp_groove_length = u.Unit(trace_params.meta["xdisp_freq_unit"]) / row['xdisp_freq'] echelle_groove_length = u.Unit(trace_params.meta["disp_freq_unit"]) / row['disp_freq'] pix_per_res_elem = row['fwhm'] + prescription = {} + if 'collimator1_focal_length' in trace_params.colnames: + for key in ('collimator1_focal_length', 'collimator2_focal_length', 'entrance_slit_scale'): + prescription[key] = row[key] * u.Unit(trace_params.meta[key + '_unit']) spectrographs[prefix] = echelle.spectrograph_factory(min_wave, max_wave, focal_len, design_res, echelle_angle, min_order, max_order, echelle_groove_length, pix_per_res_elem, disp_npix, xdisp_npix, pix_size, xdisp_groove_length=xdisp_groove_length, xdisp_beta_center=xdisp_beta_center, - dispersion_focal_len=dispersion_focal_len) + dispersion_focal_len=dispersion_focal_len, **prescription) self._spectrographs = spectrographs def efficiency_curve(trace_id, wavelength): diff --git a/scopesim/effects/spectral_trace_list.py b/scopesim/effects/spectral_trace_list.py index 082b4bff0..6f3062628 100644 --- a/scopesim/effects/spectral_trace_list.py +++ b/scopesim/effects/spectral_trace_list.py @@ -685,6 +685,12 @@ class EchelleSpectralTraceList(SpectralTraceList): center. Offsets are applied after rotation, before detector clipping, with no automatic recentering. Omitting both columns retains automatic placement. + With ``physical_slit_dispersion=True``, supply ``collimator1_focal_length``, + ``collimator2_focal_length`` and ``entrance_slit_scale`` columns with their + ``*_unit`` metadata. The entrance scale is length per on-sky arcsecond. + These determine the illumination angle and asymmetric pupil relay; + the along-slit plate scale is not used for across-slit dispersion. + The calculated traces are stored in the same HDUList format as required by SpectralTraceList, and supplied to the parent class through hdulist kwarg. @@ -749,13 +755,19 @@ def _generate_trace_hdulist(self, trace_params): xdisp_groove_length = u.Unit(trace_params.meta["xdisp_freq_unit"]) / row['xdisp_freq'] echelle_groove_length = u.Unit(trace_params.meta["disp_freq_unit"]) / row['disp_freq'] pix_per_res_elem = row['fwhm'] + prescription = {} + if 'collimator1_focal_length' in trace_params.table.colnames: + for key in ('collimator1_focal_length', 'collimator2_focal_length', 'entrance_slit_scale'): + prescription[key] = row[key] * u.Unit(trace_params.meta[key + '_unit']) + nominal_width = row['nominal_slit_width'] * u.Unit(trace_params.meta['nominal_slit_width_unit']) + pix_per_res_elem = (nominal_width * prescription['entrance_slit_scale'] * focal_len / prescription['collimator2_focal_length'] / pix_size).to_value(u.dimensionless_unscaled) ss = echelle.spectrograph_factory(min_wave, max_wave, focal_len, design_res, echelle_angle, min_order, max_order, echelle_groove_length, pix_per_res_elem, disp_npix, xdisp_npix, pix_size, xdisp_groove_length=xdisp_groove_length, xdisp_beta_center=xdisp_beta_center, - dispersion_focal_len=dispersion_focal_len) + dispersion_focal_len=dispersion_focal_len, **prescription) _warn_if_echelle_design_res_inconsistent( prefix, design_res, @@ -869,14 +881,7 @@ def on_detector(wave, order): 'y': yval}) if self.physical_slit_dispersion: - # The analytical prescription assumes equal slit magnification - # along and across the slit. Use its native, unrotated echelle - # mapping; neither the sky raster nor detector rotation changes - # this physical wavelength displacement. Preserve the legacy sign. - native_angle = row['plate_scale'] * u.Unit(trace_params.meta['plate_scale_unit']) * pix_size - raw_pixel = ss.wavelength_to_x_pixel(w, order) - delta_wave = ss.x_pixel_to_wavelength(raw_pixel + .5, order) - ss.x_pixel_to_wavelength(raw_pixel - .5, order) - order_table['dlam_deta'] = (-delta_wave / native_angle).to(u.um / u.arcsec) + order_table['dlam_deta'] = ss.slit_dispersion(w, order) trace_hdu = fits.BinTableHDU(order_table) if self.physical_slit_dispersion: diff --git a/scopesim/optics/echelle.py b/scopesim/optics/echelle.py index 3dc1d4362..709b2fd2e 100644 --- a/scopesim/optics/echelle.py +++ b/scopesim/optics/echelle.py @@ -11,7 +11,10 @@ def spectrograph_factory(min_wave: float|u.Quantity, max_wave: float|u.Quantity, echelle_groove_length: float|u.Quantity, pix_per_res_elem: float, disp_npix: int, xdisp_npix: int, pix_size: float|u.Quantity, xdisp_groove_length: float|u.Quantity = 0.0, xdisp_beta_center: float|u.Quantity = 0.0, - dispersion_focal_len: float|u.Quantity = None): + dispersion_focal_len: float|u.Quantity = None, + collimator1_focal_length: u.Quantity = None, + collimator2_focal_length: u.Quantity = None, + entrance_slit_scale: u.Quantity = None): """ Parameters @@ -47,6 +50,11 @@ def spectrograph_factory(min_wave: float|u.Quantity, max_wave: float|u.Quantity, dispersion_focal_len: float|u.Quantity Effective focal length for the echelle/main-dispersion detector coordinate. If float, assumed to be in mm. Defaults to focal_len. + collimator1_focal_length, collimator2_focal_length: u.Quantity + Physical collimator lengths. When supplied together, derive the + dispersion focal length from focal_len times their ratio. + entrance_slit_scale: u.Quantity + Entrance-slit length per on-sky angle, required with the collimators. Returns ------- @@ -98,6 +106,9 @@ def spectrograph_factory(min_wave: float|u.Quantity, max_wave: float|u.Quantity, pixels_per_res_elem=pix_per_res_elem, focal_length=focal_len, dispersion_focal_length=dispersion_focal_len, + collimator1_focal_length=collimator1_focal_length, + collimator2_focal_length=collimator2_focal_length, + entrance_slit_scale=entrance_slit_scale, grating=echelle_grating, detector=Detector(disp_npix, xdisp_npix, pix_size), cross_disperser=cross_disperser) @@ -369,6 +380,9 @@ def __init__( detector: Detector, cross_disperser: GratingSetup = None, dispersion_focal_length: u.Quantity = None, + collimator1_focal_length: u.Quantity = None, + collimator2_focal_length: u.Quantity = None, + entrance_slit_scale: u.Quantity = None, ): """ @@ -403,6 +417,12 @@ def __init__( if dispersion_focal_length is None else dispersion_focal_length.to(u.mm) ) + if collimator1_focal_length is not None: + self.collimator1_focal_length = collimator1_focal_length.to(u.mm) + self.collimator2_focal_length = collimator2_focal_length.to(u.mm) + self.entrance_slit_scale = entrance_slit_scale.to(u.mm / u.arcsec) + # Angular dispersion is amplified by the asymmetric pupil relay. + self.dispersion_focal_length = self.focal_length * self.collimator1_focal_length / self.collimator2_focal_length self.pixel_scale = np.arctan( self.detector.pixel_size / self.dispersion_focal_length) self.beta_central_pixel = self.grating.beta_center @@ -437,6 +457,22 @@ def set_beta_center(self, beta, littrow: bool = False): if littrow: self.grating.alpha = beta + def slit_dispersion(self, wavelength, order): + """Apparent wavelength shift per across-slit arcsecond at fixed incident wavelength. + + m*lambda = d*(sin(alpha) + sin(beta)); increasing entrance-slit eta + increases alpha and reduces the wavelength inferred at nominal alpha. + The first collimator sets this angle; the pupil relay is already in + the camera's dispersion focal length and must not enter a second time. + """ + derivative = -self.grating.d * np.cos(self.grating.alpha) * self.entrance_slit_scale / (order * self.collimator1_focal_length) + return np.ones(np.shape(wavelength)) * derivative.to(u.um / u.arcsec) + + def projected_slit_width(self, wavelength, order, slit_width): + """Unrotated echelle-coordinate slit width, including local anamorphism.""" + half_shift = self.slit_dispersion(wavelength, order) * slit_width / 2 + return abs(self.wavelength_to_x_pixel(wavelength + half_shift, order) - self.wavelength_to_x_pixel(wavelength - half_shift, order)) * u.pixel + @property def orders(self): try: diff --git a/scopesim/tests/tests_optics/test_echelle.py b/scopesim/tests/tests_optics/test_echelle.py index ef8a4fbc4..446374ea2 100644 --- a/scopesim/tests/tests_optics/test_echelle.py +++ b/scopesim/tests/tests_optics/test_echelle.py @@ -12,6 +12,34 @@ (960, 1340, 350, 2048), (1470, 1830, 300, 2048), (1990, 2450, 270, 2048)] +def test_asymmetric_pupil_slit_dispersion_and_anamorphism(): + instrument = spectrograph_factory(960, 1340, 225, 18000, 65.6, 58, 80, 1/24, 4.46, + 2048, 2048, .015, xdisp_groove_length=1/350, xdisp_beta_center=10.9, + collimator1_focal_length=1500*u.mm, collimator2_focal_length=1250*u.mm, entrance_slit_scale=.5306*u.mm/u.arcsec) + assert instrument.dispersion_focal_length == 270*u.mm + center = instrument.central_wave(65) + derivative = instrument.slit_dispersion(center, 65) + assert derivative < 0*u.um/u.arcsec + for slit, expected in ((.7, 17800), (.33, 37800)): + resolution = (center / abs(derivative * slit*u.arcsec)).to_value(u.dimensionless_unscaled) + assert resolution == pytest.approx(expected, rel=.01) + width = instrument.projected_slit_width(center, 65, .7*u.arcsec) + assert width.to_value(u.pixel) == pytest.approx(4.45704, rel=1e-5) + wavelengths = center * np.array([.99, 1., 1.01]) + widths = instrument.projected_slit_width(wavelengths, 65, .7*u.arcsec) + assert np.all(np.diff(widths) > 0*u.pixel) + eta = .0825*u.arcsec + dalpha = (eta * instrument.entrance_slit_scale / instrument.collimator1_focal_length).to_value(u.dimensionless_unscaled) * u.rad + exact = instrument.grating.d / 65 * (np.sin(instrument.grating.alpha) - np.sin(instrument.grating.alpha + dalpha)) + assert (derivative * eta).to_value(u.nm) == pytest.approx(exact.to_value(u.nm), rel=1e-4) + + +def test_incomplete_physical_prescription_fails(): + with pytest.raises(AttributeError): + spectrograph_factory(960, 1340, 225, 18000, 65.6, 58, 80, 1/24, 4.46, 2048, 2048, .015, + xdisp_groove_length=1/350, collimator1_focal_length=1500*u.mm) + + @pytest.mark.parametrize("low,high,density,height", CHANNELS) def test_fixed_groove_endpoints_fill_camera(low, high, density, height): instrument = spectrograph_factory(low, high, 225, 18000, 65.6, 40, 50, 1/24, 4.46, From e9f4b77acf5e4b57053c82533300580222cac76a Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Sun, 18 Oct 2026 06:55:50 -0700 Subject: [PATCH 119/130] Integrate finite slit and detector cells with cached physical geometry --- scopesim/effects/psfs/analytical.py | 24 ++++ scopesim/effects/spectral_trace_list_utils.py | 133 ++++++++++++------ scopesim/optics/fov.py | 2 + scopesim/optics/optical_train.py | 11 +- .../test_finite_spectral_cells.py | 36 +++++ 5 files changed, 156 insertions(+), 50 deletions(-) create mode 100644 scopesim/tests/tests_effects/test_finite_spectral_cells.py diff --git a/scopesim/effects/psfs/analytical.py b/scopesim/effects/psfs/analytical.py index 46deac4e5..2f13811a9 100644 --- a/scopesim/effects/psfs/analytical.py +++ b/scopesim/effects/psfs/analytical.py @@ -377,6 +377,30 @@ def point_source_image(self, fov, x, y): def _apply_convolution(self, obj): if isinstance(obj, FieldOfView3D) and obj.hdu is None: + if obj.trace_id is not None: + # Finite cross-slit cells carry a uniform illumination density. + # Resolve the effective PSF before shearing those cells in + # wavelength. Refine only as needed to resolve this effective PSF. + wave = self._sample_kernel_wavelengths(obj.waveset) + width = (obj.meta['eta_max'] - obj.meta['eta_min']).to_value(u.arcsec) + fwhm = self.fwhm(wave).to_value(u.arcsec).min() + obj.meta['effective_psf_fwhm'] = fwhm*u.arcsec + step = fwhm / 10 + rows = max(obj.header['NAXIS2'], int(np.ceil(width / step))) + obj.header['NAXIS2'] = rows + obj.header['CDELT2'] = np.copysign(width / rows / 3600, obj.header['CDELT2']) + obj.header['CRPIX2'] = (rows + 1) / 2 + obj.header['CRVAL2'] = ((obj.meta['eta_min'] + obj.meta['eta_max']) / 2).to_value(u.deg) + length = (obj.meta['xi_max'] - obj.meta['xi_min']).to_value(u.arcsec) + columns = max(obj.header['NAXIS1'], int(np.ceil(6*length/fwhm))) + obj.header['NAXIS1'] = columns + obj.header['CDELT1'] = np.copysign(length/columns/3600, obj.header['CDELT1']) + obj.header['CRPIX1'] = (columns+1)/2 + obj.header['CRVAL1'] = ((obj.meta['xi_min'] + obj.meta['xi_max'])/2).to_value(u.deg) + obj.meta['pixel_area'] = None + obj._volume = None + if 'slit_spectral_width' in obj.meta: + obj.meta['slit_spectral_width'] *= min(1., fwhm / width) obj.hdu = obj.make_hdu(spatial_psf=self) else: super()._apply_convolution(obj) diff --git a/scopesim/effects/spectral_trace_list_utils.py b/scopesim/effects/spectral_trace_list_utils.py index 3e7fbc71b..54f5626a7 100644 --- a/scopesim/effects/spectral_trace_list_utils.py +++ b/scopesim/effects/spectral_trace_list_utils.py @@ -16,6 +16,7 @@ from scipy.interpolate import interp1d, RectBivariateSpline from scipy.ndimage import gaussian_filter1d, map_coordinates +from scipy.sparse import csr_matrix from astropy.table import Table, vstack from astropy.modeling import fitting @@ -90,6 +91,7 @@ class SpectralTrace: - x, y : [mm] """ + _pixel_quadrature_order = 4 _class_params = { "x_colname": "x", "y_colname": "y", @@ -146,6 +148,8 @@ def __init__(self, trace_tbl, cmds=None, **kwargs): # Declaration of other attributes self._xilamimg = None + self._slit_cache = {} + self._pixel_cache = {} self.dlam_per_pix = None self.fit_inverse = True @@ -268,8 +272,8 @@ def _focal_plane_grid(self, fov): xmin_mm, ymin_mm = fpa_wcsd.all_pix2world(xmin, ymin, 0) xmax_mm, ymax_mm = fpa_wcsd.all_pix2world(xmax, ymax, 0) - x_mm = np.linspace(xmin_mm, xmax_mm, sub_naxis1, endpoint=False, dtype=np.float32) - y_mm = np.linspace(ymin_mm, ymax_mm, sub_naxis2, endpoint=False, dtype=np.float32) + x_mm = np.linspace(xmin_mm, xmax_mm, sub_naxis1, endpoint=False) + y_mm = np.linspace(ymin_mm, ymax_mm, sub_naxis2, endpoint=False) return xmin, ymin, x_mm, y_mm, det_wcs def spectral_padding(self, fov): @@ -286,6 +290,9 @@ def spectral_padding(self, fov): step = abs(fov.header['CDELT2'])*3600 derivative = self.dlam_per_pix(wave)/step padding = np.max(abs(derivative))*width/2 + # Resolve the physical slit profile rather than using one fixed + # wavelength step for all orders. The PSF may request a finer step. + fov.meta['slit_spectral_width'] = np.min(abs(derivative))*width*u.um if 'optical_resolving_power' in self.meta: padding += 5*upper/(self.meta['optical_resolving_power']*np.sqrt(8*np.log(2))) return padding*u.um @@ -323,30 +330,50 @@ def map_spectra_to_focal_plane(self, fov): image = np.zeros((sub_naxis2, sub_naxis1), dtype=np.float32) self._set_dispersion(wave_min, wave_max, pixsize=pixsize) + quadrature_order = self._pixel_quadrature_order + if 'effective_psf_fwhm' in fov.meta: + middle = (wave_min + wave_max)/2 + dx_dxi, _ = self.xilam2x.gradient() + dy_dxi, _ = self.xilam2y.gradient() + along_step = pixsize/np.hypot(dx_dxi(0., middle), dy_dxi(0., middle)) + across_step = abs(self.dlam_per_pix(middle)/self.slit_dispersion(middle)) if 'slit_dispersion_colname' in self.meta else pixscale + psf_width = fov.meta['effective_psf_fwhm'].to_value(u.arcsec) + quadrature_order = max(quadrature_order, int(np.ceil(8*max(along_step, across_step)/psf_width))) slit_dispersion = None if "slit_dispersion_colname" in self.meta: slit_dispersion = self.slit_dispersion - xilam = XiLamImage(fov, self.dlam_per_pix, slit_dispersion=slit_dispersion) + xilam = XiLamImage(fov, self.dlam_per_pix, slit_dispersion=slit_dispersion, _cache=self._slit_cache) if "optical_resolving_power" in self.meta: xilam.convolve_lsf(self.meta["optical_resolving_power"]) self._xilamimg = xilam - # focal-plane coordinate images - ximg_fpa, yimg_fpa = np.meshgrid(x_mm, y_mm) - - # Image mapping (xi, lambda) on the focal plane - xi_fpa = self.xy2xi(ximg_fpa, yimg_fpa).astype(np.float32) - lam_fpa = self.xy2lam(ximg_fpa, yimg_fpa).astype(np.float32) - - # The physical slit ends include the outer half-cells. A truncated - # integer index incorrectly admits pixels below the lower boundary. - inside = (xi_fpa >= xi_min) & (xi_fpa < xi_max) - inside &= (lam_fpa >= wave_min) & (lam_fpa < wave_max) - image = xilam.interp(xi_fpa, lam_fpa, grid=False)*inside - - # Scale to ph / s / pixel - image *= self._trace_flux_scale( - ximg_fpa, yimg_fpa, pixsize, pixscale, det_header) + # Integrate the density over each detector pixel. The inverse optical + # map is locally affine within a pixel; its exact center and Jacobian + # are cached, while quadrature samples are streamed to bound memory. + xx, yy = np.meshgrid(np.arange(sub_naxis1), np.arange(sub_naxis2)) + ximg_fpa, yimg_fpa = det_wcs.all_pix2world(xx, yy, 0) + lam_fpa = self.xy2lam(ximg_fpa, yimg_fpa) + jacobian = from_currsys(self.meta['trace_flux_jacobian'], self.cmds) + key = (det_header.tostring(), xmin, ymin, image.shape, wave_min, wave_max, xi_min, xi_max, pixscale, jacobian) + if self._pixel_cache.get('key') != key: + self._pixel_cache.clear() + matrix = det_wcs.pixel_scale_matrix + dxidx, dxidy = self.xy2xi.gradient() + dlamdx, dlamdy = self.xy2lam.gradient() + ax, ay = dxidx(ximg_fpa, yimg_fpa), dxidy(ximg_fpa, yimg_fpa) + bx, by = dlamdx(ximg_fpa, yimg_fpa), dlamdy(ximg_fpa, yimg_fpa) + self._pixel_cache.update(key=key, xi=self.xy2xi(ximg_fpa, yimg_fpa), lam=lam_fpa, + xi_x=ax*matrix[0, 0] + ay*matrix[1, 0], xi_y=ax*matrix[0, 1] + ay*matrix[1, 1], + lam_x=bx*matrix[0, 0] + by*matrix[1, 0], lam_y=bx*matrix[0, 1] + by*matrix[1, 1], + scale=self._trace_flux_scale(ximg_fpa, yimg_fpa, pixsize, pixscale, det_header)) + geometry = self._pixel_cache + nodes, weights = np.polynomial.legendre.leggauss(quadrature_order) + for nx, wx in zip(nodes/2, weights/2): + for ny, wy in zip(nodes/2, weights/2): + xi = geometry['xi'] + nx*geometry['xi_x'] + ny*geometry['xi_y'] + lam = geometry['lam'] + nx*geometry['lam_x'] + ny*geometry['lam_y'] + inside = (xi >= xi_min) & (xi < xi_max) & (lam >= wave_min) & (lam < wave_max) + image[inside] += xilam.interp(xi[inside], lam[inside], grid=False) * geometry['scale'][inside] * wx * wy detector_qe = fov.meta.get("detector_qe") if detector_qe is not None: @@ -837,7 +864,41 @@ def __str__(self): return msg -class XiLamImage(): +def _uniform_sum_cdf(position, width_a, width_b): + """CDF of two centered uniform intervals, including a zero-width interval.""" + wide, narrow = np.maximum(width_a, width_b), np.minimum(width_a, width_b) + probability = np.clip(.5 + position / wide, 0, 1) + finite = narrow > 0 + a, b, x = wide[finite], narrow[finite], position[:, finite] + tail = np.clip((a + b)/2 - abs(x), 0, b)**2 / (2*a*b) + probability[:, finite] = np.where(abs(x) <= (a - b)/2, .5 + x/a, np.where(x < 0, tail, 1 - tail)) + return probability + + +def _slit_cell_operator(wavelength, derivative, eta, deta): + """Conservative overlap of wavelength bins sheared across one finite slit cell. + + Wavelength and its derivative use um and um/arcsec; eta uses arcsec. + Locally affine dispersion maps each rectangular (lambda, eta) cell to a + trapezoidal wavelength profile. Integrating its CDF at output edges keeps + the input photon integral, including lines narrower than a wavelength bin. + """ + step = wavelength[1] - wavelength[0] + center = np.arange(len(wavelength)) + derivative * eta / step + spectral_width = 1 + eta * np.gradient(derivative, wavelength) + slit_width = abs(derivative * deta / step) + assert np.all(spectral_width > 0), 'Slit displacement must preserve wavelength ordering.' + span = spectral_width + slit_width + first = np.floor(center - span/2 + .5).astype(int) + output = first[None, :] + np.arange(int(np.ceil(span.max())) + 1)[:, None] + lower = output - .5 - center + probability = _uniform_sum_cdf(lower + 1, spectral_width, slit_width) - _uniform_sum_cdf(lower, spectral_width, slit_width) + source = np.broadcast_to(np.arange(len(wavelength)), output.shape) + valid = (output >= 0) & (output < len(wavelength)) & (probability > 0) + return csr_matrix((probability[valid], (output[valid], source[valid])), shape=(len(wavelength), len(wavelength))) + + +class XiLamImage: """ Class to compute a rectified 2D spectrum. @@ -856,7 +917,7 @@ class XiLamImage(): legacy coupling between sky-grid sampling and detector dispersion. """ - def __init__(self, fov, dlam_per_pix, slit_dispersion=None): + def __init__(self, fov, dlam_per_pix, slit_dispersion=None, _cache=None): # ..todo: we assume that we always have a cube. We use SpecCADO's # add_cube_layer method cube_wcs = WCS(fov.cube.header, key=" ") @@ -897,25 +958,17 @@ def __init__(self, fov, dlam_per_pix, slit_dispersion=None): logger.warning("Using scalar dlam_per_pix = %.2g", dlam_per_pix_val) - dlam_per_arcsec = dlam_per_pix_val / d_eta if slit_dispersion is None else slit_dispersion(self.lam) - for i, eta in enumerate(cube_eta): - # A positive calibrated derivative moves an input line to a larger - # apparent wavelength: sample the source at lambda_out - shift. - lam0 = self.lam + dlam_per_arcsec * eta if slit_dispersion is None else self.lam - dlam_per_arcsec * eta - - # lam0 is the target wavelength. We need to check that this - # overlaps with the wavelength range covered by the cube - if lam0.min() < cube_lam.max() and lam0.max() > cube_lam.min(): - # Only wavelength changes here; xi is already on its native - # grid. Building a 2D spline for every slit cell is redundant. - index = np.searchsorted(cube_lam, lam0, side='right') - 1 - index = np.clip(index, 0, n_lam - 2) - fraction = (lam0 - cube_lam[index])/d_lam - valid = (lam0 >= cube_lam[0]) & (lam0 <= cube_lam[-1]) - plane = fov.cube.data[:, i, :] - self.image += ((1 - fraction[:, None])*plane[index] + fraction[:, None]*plane[index + 1]).T*valid - - self.image *= abs(d_eta) # ph/s/um/arcsec2 --> ph/s/um/arcsec + derivative = -dlam_per_pix_val / d_eta if slit_dispersion is None else slit_dispersion(self.lam) + derivative = np.broadcast_to(derivative, self.lam.shape) + key = (cube_lam[0], d_lam, n_lam, cube_eta[0], d_eta, n_eta, derivative.tobytes()) + if _cache is None: + _cache = {} + if _cache.get('key') != key: + _cache.clear() + _cache['operators'] = [_slit_cell_operator(cube_lam, derivative, eta, d_eta) for eta in cube_eta] + _cache['key'] = key + for i, operator in enumerate(_cache['operators']): + self.image += (operator @ fov.cube.data[:, i, :]).T * abs(d_eta) # WCS for the xi-lambda image, i.e. the rectified 2D spectrum # Default WCS with xi in arcsec diff --git a/scopesim/optics/fov.py b/scopesim/optics/fov.py index 892e2fd20..e155b7111 100644 --- a/scopesim/optics/fov.py +++ b/scopesim/optics/fov.py @@ -935,6 +935,8 @@ class FieldOfView3D(FieldOfView): def cube_waveset(self): wave_unit = u.Unit(from_currsys('!SIM.spectral.wave_unit', self.cmds)) step = from_currsys('!SIM.spectral.spectral_bin_width', self.cmds) + if 'slit_spectral_width' in self.meta: + step = min(step, self.meta['slit_spectral_width'].to_value(wave_unit)/20) start, stop = self.meta['wave_min'].value, self.meta['wave_max'].value padding = int(np.ceil(self.meta['spectral_padding'].to_value(wave_unit)/step)) return (start + np.arange(-padding, int((stop - start)/step) + padding)*step)*wave_unit diff --git a/scopesim/optics/optical_train.py b/scopesim/optics/optical_train.py index 65e7956e4..fe2d9445a 100644 --- a/scopesim/optics/optical_train.py +++ b/scopesim/optics/optical_train.py @@ -432,16 +432,7 @@ def image_origin_on_canvas(shape_xy, image_wcs, canvas_wcs): x_mm = trace.xilam2x(xi_flat[in_fov], wave_flat[in_fov]) y_mm = trace.xilam2y(xi_flat[in_fov], wave_flat[in_fov]) - subimage_x = ( - (x_mm - x_grid_mm[0]) - / (x_grid_mm[-1] - x_grid_mm[0]) - * (len(x_grid_mm) - 1) - ) - subimage_y = ( - (y_mm - y_grid_mm[0]) - / (y_grid_mm[-1] - y_grid_mm[0]) - * (len(y_grid_mm) - 1) - ) + subimage_x, subimage_y = subimage_wcs.all_world2pix(x_mm, y_mm, 0) subimage_shape = np.array( [len(x_grid_mm), len(y_grid_mm)]) diff --git a/scopesim/tests/tests_effects/test_finite_spectral_cells.py b/scopesim/tests/tests_effects/test_finite_spectral_cells.py new file mode 100644 index 000000000..4f74407bc --- /dev/null +++ b/scopesim/tests/tests_effects/test_finite_spectral_cells.py @@ -0,0 +1,36 @@ +"""Photon integrals and line moments across finite slit and wavelength cells.""" +import numpy as np +import pytest +from scopesim.effects.spectral_trace_list_utils import _slit_cell_operator + + +@pytest.mark.parametrize('phase', [0., .25, .5, .75]) +@pytest.mark.parametrize('slope', [0., .02]) +def test_slit_remap_preserves_photons_and_illumination_centroid(phase, slope): + wave = np.linspace(.99, 1.01, 2001) + derivative = -.001 + slope * (wave - 1.) + step = wave[1] - wave[0] + eta, deta = .0825 + phase*step/abs(derivative[1000]), .03 + photons = np.zeros(len(wave)) + photons[1000] = 17. + output = _slit_cell_operator(wave, derivative, eta, deta) @ photons + assert output.sum() == pytest.approx(17., rel=1e-12) + center = np.sum(output * wave) / output.sum() + assert abs(center - (1. + eta*derivative[1000])) < .01*step + + +def test_zero_slit_dispersion_is_identity(): + wave = np.linspace(1., 1.1, 101) + photons = np.arange(len(wave)) + 1. + actual = _slit_cell_operator(wave, np.zeros_like(wave), .1, .1) @ photons + np.testing.assert_allclose(actual, photons, rtol=1e-12) + + +def test_finite_slit_cell_width_has_uniform_variance(): + wave = np.linspace(.99, 1.01, 10001) + derivative = np.full_like(wave, -.001) + photons = np.zeros(len(wave)) + photons[5000] = 1. + output = _slit_cell_operator(wave, derivative, 0., .1) @ photons + variance = np.sum(output * (wave - 1.)**2) + assert variance == pytest.approx((.001*.1)**2 / 12, rel=.002) From cafce81d0ad5e47fab86c30357a34259222d6252 Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Sun, 18 Oct 2026 07:23:07 -0700 Subject: [PATCH 120/130] Restore photon-conserving cube-to-spectrum collapse and representation tests --- scopesim/optics/fov.py | 12 +- .../tests/tests_optics/test_FieldOfView.py | 208 ++++-------------- 2 files changed, 50 insertions(+), 170 deletions(-) diff --git a/scopesim/optics/fov.py b/scopesim/optics/fov.py index e155b7111..af6e36207 100644 --- a/scopesim/optics/fov.py +++ b/scopesim/optics/fov.py @@ -652,14 +652,10 @@ def _make_cubefields(self): * yield scaled flux to be added to canvas flux """ for field in self._get_cube_fields(): - fluxes = field.data.sum(axis=(1, 2)) - fov_waveset_fluxes = np.interp(self.waveset, field.waveset, fluxes) - - # .lower() is needed because astropy doesn't recognize PHOTLAM - field_unit = field.header.get("BUNIT", PHOTLAM).lower() - flux_scale_factor = u.Unit(field_unit).to(PHOTLAM) - - yield fov_waveset_fluxes * flux_scale_factor + fluxes = field.data.sum(axis=(1, 2))*u.Unit(field.header['BUNIT'].lower()) + if field.is_bunit_spatially_differential: + fluxes *= field.pixel_area + yield np.interp(self.waveset, field.waveset, fluxes.to_value(PHOTLAM)) def _make_imagefields(self): """ diff --git a/scopesim/tests/tests_optics/test_FieldOfView.py b/scopesim/tests/tests_optics/test_FieldOfView.py index bca296ad8..bab799ad9 100644 --- a/scopesim/tests/tests_optics/test_FieldOfView.py +++ b/scopesim/tests/tests_optics/test_FieldOfView.py @@ -164,123 +164,6 @@ def test_handles_nans(self): class TestMakeCube: - @pytest.mark.xfail(reason="cube flux is broken") - def test_makes_cube_from_table(self): - src_table = so._table_source() # 10x10" @ 0.2"/pix, [0.5, 2.5]m @ 0.02µm - fov = _fov_190_210_um(ndim=3) - fov.extract_from(src_table) - - cube = fov.make_hdu() - - in_sum = 0 - waveset = fov.spectra[0].waveset - for x, y, ref, weight in src_table.fields[0]: - flux = src_table.spectra[ref](waveset).to(u.ph/u.s/u.m**2/u.um).value - in_sum += np.sum(flux) * weight - - out_sum = np.sum(cube.data) - - assert out_sum == approx(in_sum, rel=0.01) - - if PLOTS: - plt.imshow(cube.data[0, :, :], origin="lower") - plt.show() - - @pytest.mark.xfail(reason="cube flux is broken") - def test_makes_cube_from_imagehdu(self): - src_image = so._image_source() # 10x10" @ 0.2"/pix, [0.5, 2.5]m @ 0.02µm - fov = _fov_190_210_um(ndim=3) - fov.extract_from(src_image) - - cube = fov.make_hdu() - - waveset = np.linspace(1.9, 2.1, np.shape(cube)[0]) * u.um - spec = fov.spectra[0](waveset).to(u.ph/u.s/u.m**2/u.um).value - in_sum = np.sum(src_image.fields[0].data) * np.sum(spec) - out_sum = np.sum(cube.data) - - assert out_sum == approx(in_sum, rel=0.01) - - if PLOTS: - plt.imshow(cube.data[0, :, :], origin="lower") - plt.show() - - @pytest.mark.xfail(reason="cube flux is broken") - def test_makes_cube_from_other_cube_imagehdu(self): - import scopesim as sim - sim.rc.__currsys__["!SIM.spectral.spectral_bin_width"] = 0.01 - src_cube = so._cube_source() # 10x10" @ 0.2"/pix, [0.5, 2.5]m @ 0.02µm - fov = _fov_197_202_um(ndim=3) - fov.extract_from(src_cube) - - cube = fov.make_hdu() - - # layer 74 to 77 are extracted by FOV - in_sum = np.sum(src_cube.fields[0].data[74:77, :, :]) - - out_sum = np.sum(cube.data) - - assert out_sum == approx(in_sum) - - if PLOTS: - plt.imshow(cube.data[0, :, :], origin="lower") - plt.show() - - @pytest.mark.xfail(reason="cube flux is broken") - def test_makes_cube_from_two_similar_cube_imagehdus(self): - src_cube = so._cube_source() + so._cube_source(dx=1) # 2 cubes 10x10" @ 0.2"/pix, [0.5, 2.5]m @ 0.02µm - fov = _fov_197_202_um(ndim=3) - fov.extract_from(src_cube) - - cube = fov.make_hdu() - - # layer 74 to 77 are extracted by FOV - bin_widths = np.array([0.01, 0.02, 0.01])[:, None, None] * 1e4 # um -> AA - in_sum = 2 * np.sum(src_cube.fields[0].data[74:77, :, :] * bin_widths) - - out_sum = np.sum(cube.data) - - assert out_sum == approx(in_sum, rel=1e-3) - - if PLOTS: - plt.imshow(cube.data[0, :, :], origin="lower") - plt.show() - - @pytest.mark.xfail(reason="cube flux is broken") - def test_makes_cube_from_all_types_of_source_object(self): - src_all = so._table_source() + \ - so._image_source(dx=-4, dy=-4) + \ - so._cube_source(weight=1e-8, dx=4) - - fov = _fov_190_210_um(ndim=3) - fov.extract_from(src_all) - - cube = fov.make_hdu() - - # sum up the expected flux in the output cube - # bin_width * half width edge bin * PHOTLAM -> SI - waveset = fov.spectra[0].waveset - - table_sum = 0 - for x, y, ref, weight in src_all.fields[0]: - flux = src_all.spectra[ref](waveset).value - table_sum += np.sum(flux) * weight - - ref = src_all.fields[1].header["SPEC_REF"] - spec = fov.spectra[ref](waveset).value - image_sum = np.sum(src_all.fields[1].data) * np.sum(spec) - - cube_sum = np.sum(src_all.fields[2].data[70:81, :, :]) * 0.02 * 0.95 - - in_sum = table_sum + image_sum + cube_sum - out_sum = np.sum(cube.data) - - assert out_sum == approx(in_sum, rel=0.02) - - if PLOTS: - im = cube.data[0, :, :] - plt.imshow(im, origin="lower", norm="log", vmin=1e-8) - plt.show() @pytest.mark.parametrize("src", [so._table_source(), so._image_source(), @@ -498,49 +381,50 @@ def test_makes_spectrum_from_all_types_of_source_object(self): plt.show() -@pytest.mark.skip(reason="needs split FOVs after subclass refactor") class TestMakeSpectrumImageCubeAllPlayNicely: - def test_make_cube_and_make_spectrum_return_the_same_fluxes(self): - src_all = so._table_source() + \ - so._image_source(dx=-4, dy=-4) + \ - so._cube_source(weight=1e-8, dx=4) - - fov = _fov_190_210_um(ndim=3) - fov.extract_from(src_all) - - # if photlam, units of ph / s / cm2 / AA, else units of ph / s / voxel - cube = fov.make_hdu() - # cube_waves = get_cube_waveset(cube.header) - cube_spectrum = cube.data.sum(axis=2).sum(axis=1) - - # always units of ph / s / cm-2 / AA-1 - waves = fov.waveset - spectrum = fov.make_hdu()(waves).value - - bin_width = 0.02 # um - photlam_to_si = 1e8 # cm-2 AA-1 --> m-2 um-1 - edge_compensation = 0.91 - - spectrum *= bin_width * photlam_to_si * edge_compensation - - if PLOTS: - plt.plot(waves, spectrum, "k") - # plt.plot(cube_waves, cube_spectrum, "r") - plt.show() - - assert np.sum(cube.data) == approx(np.sum(spectrum), rel=0.001) - - def test_make_cube_and_make_image_return_the_same_fluxes(self): - src_all = so._cube_source(weight=1e-8, dx=4) - # so._image_source(dx=-4, dy=-4) # - # so._table_source()# + \ - - fov = _fov_190_210_um() - fov.extract_from(src_all) - - # if photlam, units of ph / s / cm2 / AA, else units of ph / s / voxel - cube_sum = np.sum(fov.make_hdu().data) - # if photlam, units of ph / s / cm2 / AA, else units of ph / s / pixel - image_sum = np.sum(fov.make_hdu().data) - - assert cube_sum == approx(image_sum, rel=0.05) + @pytest.mark.parametrize("kind", ["table", "image", "cube", "two_cubes", "mixed"]) + @pytest.mark.parametrize("spectral_step", [.01, .005]) + def test_photon_integrals_agree_between_representations(self, kind, spectral_step): + from synphot import SourceSpectrum, Empirical1D + from scopesim.commands import UserCommands + from scopesim.optics.optical_train import OpticalTrain + + # A line fully inside the band isolates representation and grid errors + # from partial end bins. Its integral is the native trapezoidal integral. + wave = np.linspace(1.5, 2.5, 101)*u.um + profile = np.exp(-.5*((wave.to_value(u.um)-2.)/.02)**2) + spectrum = SourceSpectrum(Empirical1D, points=wave, lookup_table=profile*u.ph/u.s/u.m**2/u.um) + table = so._table_source() + table.fields[0].spectra = {ref: spectrum for ref in table.fields[0].spectra} + image = so._image_source() + image.fields[0].spectra = {0: spectrum} + cube = so._cube_source() + cube.fields[0].data = cube.fields[0].data[-1][None, :, :]/4 * profile[:, None, None] + cube.fields[0].header['CRVAL3'] = 2. + cube.fields[0].header['CDELT3'] = .01 + cube.fields[0].header['CUNIT3'] = 'um' + sources = {'table': table, 'image': image, 'cube': cube, 'two_cubes': cube+cube, 'mixed': table+image+cube} + source = sources[kind] + spatial_weight = {'table': table.fields[0].field['weight'].sum(), 'image': image.fields[0].data.sum(), 'cube': cube.fields[0].data[50].sum()} + spatial_weight['two_cubes'] = 2*spatial_weight['cube'] + spatial_weight['mixed'] = spatial_weight['table']+spatial_weight['image']+spatial_weight['cube'] + expected = spatial_weight[kind]*np.trapezoid(profile, wave.to_value(u.um)) + source = OpticalTrain().prepare_source(source) + + commands = UserCommands(properties={'!SIM.spectral.spectral_bin_width': spectral_step}) + rates = [] + for cls in (FieldOfView1D, FieldOfView2D, FieldOfView3D): + fov = cls(ho._fov_header(), [1.9, 2.1]*u.um, area=1*u.m**2, cmds=commands) + fov.extract_from(source) + product = fov.make_hdu() + if cls is FieldOfView1D: + samples = product(fov.waveset).to(u.ph/u.s/u.m**2/u.um) + rate = np.trapezoid(samples, fov.waveset.to(u.um))*fov.area + rates.append(rate.to_value(u.ph/u.s)) + elif cls is FieldOfView2D: + rates.append(product.data.sum()) + else: + rate = product.data.sum()*u.Unit(product.header['BUNIT'])*fov.pixel_area*product.header['CDELT3']*u.um + rates.append(rate.to_value(u.ph/u.s)) + # The line tails outside the band contain less than 10^-6 of its flux. + assert rates == approx([expected]*3, rel=2e-5) From 4c10ecdfcae3d6e436c916477acbcdf26d5fb497 Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Sun, 18 Oct 2026 10:45:03 -0700 Subject: [PATCH 121/130] Check slit anamorphism against finite grating-edge propagation --- scopesim/tests/tests_optics/test_echelle.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scopesim/tests/tests_optics/test_echelle.py b/scopesim/tests/tests_optics/test_echelle.py index 446374ea2..2370f03dd 100644 --- a/scopesim/tests/tests_optics/test_echelle.py +++ b/scopesim/tests/tests_optics/test_echelle.py @@ -28,6 +28,16 @@ def test_asymmetric_pupil_slit_dispersion_and_anamorphism(): wavelengths = center * np.array([.99, 1., 1.01]) widths = instrument.projected_slit_width(wavelengths, 65, .7*u.arcsec) assert np.all(np.diff(widths) > 0*u.pixel) + half_angle = (.35*u.arcsec * instrument.entrance_slit_scale / instrument.collimator1_focal_length).value * u.rad + beta_low = np.arcsin(65*wavelengths/instrument.grating.d - np.sin(instrument.grating.alpha + half_angle)) + beta_high = np.arcsin(65*wavelengths/instrument.grating.d - np.sin(instrument.grating.alpha - half_angle)) + edge_width = abs(instrument.x_pixel_for_beta(beta_high) - instrument.x_pixel_for_beta(beta_low)) + # Independent finite slit edges retain the same anamorphism as dispersion. + np.testing.assert_allclose(widths.to_value(u.pixel), edge_width.value, rtol=2e-7) + step = center*1e-7 + dispersion = (instrument.wavelength_to_x_pixel(wavelengths+step, 65)-instrument.wavelength_to_x_pixel(wavelengths-step, 65))/(2*step) + measured_delta = widths.to_value(u.pixel)/dispersion + np.testing.assert_allclose(measured_delta.to_value(u.nm), abs(derivative*.7*u.arcsec).to_value(u.nm), rtol=2e-7) eta = .0825*u.arcsec dalpha = (eta * instrument.entrance_slit_scale / instrument.collimator1_focal_length).to_value(u.dimensionless_unscaled) * u.rad exact = instrument.grating.d / 65 * (np.sin(instrument.grating.alpha) - np.sin(instrument.grating.alpha + dalpha)) From 0b15a616bb785c5604cb81f009e6d677c8a9bdc2 Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Sun, 18 Oct 2026 12:18:38 -0700 Subject: [PATCH 122/130] Reuse finite-cell operators and bound full-frame projection cost --- scopesim/effects/atmo_dispersion.py | 25 +++++--- scopesim/effects/psfs/analytical.py | 16 ++++- scopesim/effects/spectral_trace_list_utils.py | 39 ++++++----- scopesim/optics/fov.py | 64 ++++--------------- scopesim/optics/optical_train.py | 3 +- .../test_AtmosphericDispersion.py | 15 +++++ 6 files changed, 81 insertions(+), 81 deletions(-) diff --git a/scopesim/effects/atmo_dispersion.py b/scopesim/effects/atmo_dispersion.py index 0d7cb82fa..6a14fee04 100644 --- a/scopesim/effects/atmo_dispersion.py +++ b/scopesim/effects/atmo_dispersion.py @@ -120,15 +120,22 @@ def roll_shift(data, dx, dy): raise ValueError("dx and dy must have shape (nwave,)") out = np.empty_like(data) - for w in range(nw): - out[w] = np.roll(data[w], (dy[w], dx[w]), axis=(0, 1)) - # zero out the rolled in part, select the correct slice based on sign of dxint, dyint - if dx[w] != 0: - xslice = slice(0, dx[w]) if dx[w] > 0 else slice(nx + dx[w], nx) - out[w, :, xslice] = 0 - if dy[w] != 0: - yslice = slice(0, dy[w]) if dy[w] > 0 else slice(ny + dy[w], ny) - out[w, yslice, :] = 0 + boundaries = np.r_[0, np.flatnonzero((np.diff(dx) != 0) | (np.diff(dy) != 0))+1, nw] + for start, stop in zip(boundaries[:-1], boundaries[1:]): + xshift, yshift = dx[start], dy[start] + if abs(xshift) >= nx or abs(yshift) >= ny: + out[start:stop] = 0 + continue + if xshift == 0 and yshift == 0: + out[start:stop] = data[start:stop] + continue + out[start:stop] = np.roll(data[start:stop], (yshift, xshift), axis=(1, 2)) + if xshift != 0: + xslice = slice(0, xshift) if xshift > 0 else slice(nx + xshift, nx) + out[start:stop, :, xslice] = 0 + if yshift != 0: + yslice = slice(0, yshift) if yshift > 0 else slice(ny + yshift, ny) + out[start:stop, yslice, :] = 0 return out @staticmethod diff --git a/scopesim/effects/psfs/analytical.py b/scopesim/effects/psfs/analytical.py index 2f13811a9..9278dced2 100644 --- a/scopesim/effects/psfs/analytical.py +++ b/scopesim/effects/psfs/analytical.py @@ -405,11 +405,21 @@ def _apply_convolution(self, obj): else: super()._apply_convolution(obj) - def padded_header(self, fov): + def padded_header(self, fov, point_sources=False): """Spatial support needed to illuminate this FOV through the PSF.""" - kernel = self.get_kernel(fov) + if point_sources: + # Points use analytic cell integrals, so constructing a convolution + # kernel merely to determine source-selection bounds is redundant. + wave = self._sample_kernel_wavelengths(fov.waveset) + gamma = self.fwhm2gamma(self.fwhm(wave), self.alpha).to_value(u.arcsec).max() + radius = gamma*np.sqrt((1-self._target_enclosed_energy())**(1/(1-self.alpha))-1) + padding_x = int(np.ceil(radius/(abs(fov.header['CDELT1'])*3600))) + padding_y = int(np.ceil(radius/(abs(fov.header['CDELT2'])*3600))) + else: + kernel = self.get_kernel(fov) + padding_y, padding_x = kernel.shape[0]//2, kernel.shape[1]//2 header = fov.header.copy() - for axis, padding in ((1, kernel.shape[1]//2), (2, kernel.shape[0]//2)): + for axis, padding in ((1, padding_x), (2, padding_y)): header[f'NAXIS{axis}'] += 2*padding header[f'CRPIX{axis}'] += padding return header diff --git a/scopesim/effects/spectral_trace_list_utils.py b/scopesim/effects/spectral_trace_list_utils.py index 54f5626a7..4b40573e0 100644 --- a/scopesim/effects/spectral_trace_list_utils.py +++ b/scopesim/effects/spectral_trace_list_utils.py @@ -330,15 +330,6 @@ def map_spectra_to_focal_plane(self, fov): image = np.zeros((sub_naxis2, sub_naxis1), dtype=np.float32) self._set_dispersion(wave_min, wave_max, pixsize=pixsize) - quadrature_order = self._pixel_quadrature_order - if 'effective_psf_fwhm' in fov.meta: - middle = (wave_min + wave_max)/2 - dx_dxi, _ = self.xilam2x.gradient() - dy_dxi, _ = self.xilam2y.gradient() - along_step = pixsize/np.hypot(dx_dxi(0., middle), dy_dxi(0., middle)) - across_step = abs(self.dlam_per_pix(middle)/self.slit_dispersion(middle)) if 'slit_dispersion_colname' in self.meta else pixscale - psf_width = fov.meta['effective_psf_fwhm'].to_value(u.arcsec) - quadrature_order = max(quadrature_order, int(np.ceil(8*max(along_step, across_step)/psf_width))) slit_dispersion = None if "slit_dispersion_colname" in self.meta: slit_dispersion = self.slit_dispersion @@ -367,9 +358,21 @@ def map_spectra_to_focal_plane(self, fov): lam_x=bx*matrix[0, 0] + by*matrix[1, 0], lam_y=bx*matrix[0, 1] + by*matrix[1, 1], scale=self._trace_flux_scale(ximg_fpa, yimg_fpa, pixsize, pixscale, det_header)) geometry = self._pixel_cache - nodes, weights = np.polynomial.legendre.leggauss(quadrature_order) - for nx, wx in zip(nodes/2, weights/2): - for ny, wy in zip(nodes/2, weights/2): + quadrature = np.full(2, self._pixel_quadrature_order) + if 'effective_psf_fwhm' in fov.meta: + middle = (wave_min + wave_max)/2 + xm, ym = self.xilam2x(0., middle), self.xilam2y(0., middle) + dxidx, dxidy = self.xy2xi.gradient() + dlamdx, dlamdy = self.xy2lam.gradient() + angular_dispersion = self.slit_dispersion(middle) if slit_dispersion is not None else -self.dlam_per_pix(middle)/pixscale + angular_map = np.array([[dxidx(xm, ym), dxidy(xm, ym)], [dlamdx(xm, ym)/angular_dispersion, dlamdy(xm, ym)/angular_dispersion]]) + angular_steps = np.linalg.norm(angular_map @ det_wcs.pixel_scale_matrix, axis=0) + psf_width = fov.meta['effective_psf_fwhm'].to_value(u.arcsec) + quadrature = np.maximum(quadrature, np.ceil(8*angular_steps/psf_width).astype(int)) + xnodes, xweights = np.polynomial.legendre.leggauss(quadrature[0]) + ynodes, yweights = np.polynomial.legendre.leggauss(quadrature[1]) + for nx, wx in zip(xnodes/2, xweights/2): + for ny, wy in zip(ynodes/2, yweights/2): xi = geometry['xi'] + nx*geometry['xi_x'] + ny*geometry['xi_y'] lam = geometry['lam'] + nx*geometry['lam_x'] + ny*geometry['lam_y'] inside = (xi >= xi_min) & (xi < xi_max) & (lam >= wave_min) & (lam < wave_max) @@ -949,7 +952,6 @@ def __init__(self, fov, dlam_per_pix, slit_dispersion=None, _cache=None): cube_lam *= u.Unit(wcs_lam.wcs.cunit[0]).to(u.um) # Initialise the array to hold the xi-lambda image - self.image = np.zeros((n_xi, n_lam), dtype=np.float32) self.lam = cube_lam try: dlam_per_pix_val = dlam_per_pix(np.asarray(self.lam)) @@ -965,10 +967,15 @@ def __init__(self, fov, dlam_per_pix, slit_dispersion=None, _cache=None): _cache = {} if _cache.get('key') != key: _cache.clear() - _cache['operators'] = [_slit_cell_operator(cube_lam, derivative, eta, d_eta) for eta in cube_eta] + operators = [_slit_cell_operator(cube_lam, derivative, eta, d_eta).tocoo() for eta in cube_eta] + rows = np.concatenate([operator.row for operator in operators]) + columns = np.concatenate([operator.col*n_eta+i for i, operator in enumerate(operators)]) + weights = np.concatenate([operator.data for operator in operators])*abs(d_eta) + # Interleaved wavelength/slit cells match the contiguous cube layout, + # avoiding a separate strided dense copy for every across-slit row. + _cache['operator'] = csr_matrix((weights, (rows, columns)), shape=(n_lam, n_lam*n_eta), dtype=np.float32) _cache['key'] = key - for i, operator in enumerate(_cache['operators']): - self.image += (operator @ fov.cube.data[:, i, :]).T * abs(d_eta) + self.image = (_cache['operator'] @ fov.cube.data.reshape(n_lam*n_eta, n_xi)).T.astype(np.float32, copy=False) # WCS for the xi-lambda image, i.e. the rectified 2D spectrum # Default WCS with xi in arcsec diff --git a/scopesim/optics/fov.py b/scopesim/optics/fov.py index af6e36207..1afed3d03 100644 --- a/scopesim/optics/fov.py +++ b/scopesim/optics/fov.py @@ -1034,50 +1034,12 @@ def _make_backfields(self, fov_waveset): yield spec[:, None, None].value def make_hdu(self, spatial_psf=None): - """ - TBA. - - Used for IFUs, slit spectrographs, and coherent MOSs (e.g.KMOS) - - Returned cube units are ``ph s-1 voxel-1`` - - .. note:: ``self.make_cube()`` does NOT store anything in ``self.cube`` - - self.cube and self.make_cube() are deliberately kept seperately - so that self.cube will not be accidently overwritten by a rogue - call from an Effect object. - - All Effect objects should specifically test whether - ``self.cube is None`` before assigning a new cube it - - The cube is made with these steps: - - 1. Make waveset and canvas cube:: - - if at least one cube: - set waveset to equal largest cube waveset - else: - make waveset from self.meta values - make canvas cube based on waveset of largest cube and NAXIS1,2 from fov.header - - 2. Find Cube fields (see ``FieldOfView._make_cube_cubefields()``). - 3. Find Image fields (see ``FieldOfView._make_cube_imagefields()``). - 4. Find Table fields (see ``FieldOfView._make_cube_tablefields()``). - - ``PHOTLAM = ph / (cm2 * s * AA)``. - Original source fields are in units of: - - - tables: (PHOTLAM in spectrum) - - images: arcsec-2 (PHOTLAM in spectrum) - - cubes: PHOTLAM arcsec-2 - - .. warning:: Input Images and Cubes should have units of PHOTLAM arcsec-2 - - Returns - ------- - canvas_cube_hdu : fits.ImageHDU - [ph s-1 um-1 arcsec-2] # as needed by SpectralTrace + """Render spectral surface-brightness density in ph/s/um/arcsec2. + Native spectra are integrated into the cube wavelength cells. Point + sources use cell-integrated PSF illumination; resolved fields are + convolved before the physical aperture is clipped. Collecting area is + applied here. Slit and detector cell integrals are applied by the trace. """ # 1. Make waveset and canvas cube (area, bin_width are applied at end) # TODO: Why is this not self.waveset? What's different? @@ -1092,7 +1054,7 @@ def make_hdu(self, spatial_psf=None): canvas_cube_hdu = fits.ImageHDU( data=np.zeros((len(fov_waveset), self.header["NAXIS2"], - self.header["NAXIS1"])), + self.header["NAXIS1"]), dtype=np.float32), header=self.header) # set BUNIT initially to PHOTLAM / arcsec**2 canvas_cube_hdu.header["BUNIT"] = "ph cm-2 s-1 AA-1 arcsec-2" @@ -1130,10 +1092,8 @@ def make_hdu(self, spatial_psf=None): data = fftconvolve(data, kernel, mode='same', axes=(-2, -1)) canvas_cube_hdu.data[start:start + 64] += data[:, py:py + self.header['NAXIS2'], px:px + self.header['NAXIS1']] - canvas_cube_hdu.data = sum( - self._make_imagefields(fov_waveset, self.spline_order, spatial_psf), - start=canvas_cube_hdu.data, - ) + for image in self._make_imagefields(fov_waveset, self.spline_order, spatial_psf): + canvas_cube_hdu.data += image if spatial_psf is None: for flux, x, y in self._make_tablefields(fov_waveset): @@ -1141,13 +1101,13 @@ def make_hdu(self, spatial_psf=None): canvas_cube_hdu.data[:, y, x] += flux else: for field in self._get_table_fields(): - spectra = {ref: average_spectrum(spec, fov_waveset).value for ref, spec in field.spectra.items()} + spectra = {ref: average_spectrum(spec, fov_waveset).value.astype(np.float32) for ref, spec in field.spectra.items()} for row in field.field: - image = spatial_psf.point_source_image(self, row['x'], row['y'])/self.pixel_area.value + image = (spatial_psf.point_source_image(self, row['x'], row['y'])/self.pixel_area.value).astype(np.float32) canvas_cube_hdu.data += spectra[row['ref']][:, None, None]*image[None, :, :]*row['weight'] - canvas_cube_hdu.data = sum(self._make_backfields(fov_waveset), - start=canvas_cube_hdu.data) + for background in self._make_backfields(fov_waveset): + canvas_cube_hdu.data += background # SpectralTrace wants ph/s/um/arcsec2 --> get rid of m2, leave um canvas_cube_hdu.data *= self.area.to(u.cm ** 2).value diff --git a/scopesim/optics/optical_train.py b/scopesim/optics/optical_train.py index fe2d9445a..bfacc785e 100644 --- a/scopesim/optics/optical_train.py +++ b/scopesim/optics/optical_train.py @@ -588,7 +588,8 @@ def observe(self, orig_source=None, update=True, **kwargs): psf_index = psf_indices[0] defer_render = all(isinstance(effect, (TERCurve, SpectralEfficiency, EchelleSpectralEfficiency, TaperedQuantumEfficiency)) for effect in foveffs[:psf_index]) if defer_render: - fov.extract_from(source, spatial_header=foveffs[psf_index].padded_header(fov)) + point_sources = not (source.image_fields or source.cube_fields) + fov.extract_from(source, spatial_header=foveffs[psf_index].padded_header(fov, point_sources=point_sources)) else: fov.extract_from(source) fov.view() diff --git a/scopesim/tests/tests_effects/test_AtmosphericDispersion.py b/scopesim/tests/tests_effects/test_AtmosphericDispersion.py index a578674a1..a72cc531f 100644 --- a/scopesim/tests/tests_effects/test_AtmosphericDispersion.py +++ b/scopesim/tests/tests_effects/test_AtmosphericDispersion.py @@ -3,12 +3,27 @@ import numpy as np +from scopesim.effects.atmo_dispersion import ADCShift + from scopesim.effects.shifts import AtmosphericDispersion, \ get_pixel_border_waves_from_atmo_disp from ..mocks.py_objects.yaml_objects import _atmo_yaml_dict +def test_repeated_integer_adc_shifts_preserve_positions_and_clip_edges(): + cube = np.zeros((10, 5, 7)) + cube[:, 2, 3] = 1 + dx = np.array([-1.2, -1.2, 0., 0., .9, .9, 8., -8., 0., 0.]) + dy = np.array([1., 1., 0., 0., -1., -1., 0., 0., 6., -6.]) + shifted = ADCShift.roll_shift(cube, dx, dy) + expected = np.zeros_like(cube) + expected[:2, 3, 2] = 1 + expected[2:4, 2, 3] = 1 + expected[4:6, 1, 4] = 1 + np.testing.assert_array_equal(shifted, expected) + + @pytest.fixture(scope="function") def atmo_yaml_dict(): atmo_dict = _atmo_yaml_dict() From 089852a62afd7456c2aba350538a7ec76049d813 Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Sun, 18 Oct 2026 12:43:26 -0700 Subject: [PATCH 123/130] fix(psf): invalidate sampled kernels when the FOV grid changes --- scopesim/effects/psfs/discrete.py | 14 ++++++++++++-- scopesim/tests/tests_effects/test_ConstPSF.py | 19 +++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/scopesim/effects/psfs/discrete.py b/scopesim/effects/psfs/discrete.py index 64d852efc..53b334b2a 100644 --- a/scopesim/effects/psfs/discrete.py +++ b/scopesim/effects/psfs/discrete.py @@ -192,13 +192,12 @@ def __init__(self, **kwargs): self.current_ext = None self.current_data = None self.kernel = None + self._kernel_grid_key = None def get_kernel(self, fov): """Find nearest wavelength and build PSF kernel from file.""" idx = _nearest_index(fov.wavelength, self._waveset) ext = self.kernel_indices[idx] - if ext == self.current_layer_id: - return self.kernel try: spline_order = from_currsys(self.meta["interp_order"], cmds=self.cmds) @@ -207,10 +206,19 @@ def get_kernel(self, fov): logger.warning("PSF parameter interp_order not found, using 1 (linear interpolation).") spline_order = 1 + # A reference PSF layer can serve FOVs with different wavelength and + # spatial grids. Reusing its sampled cube solely by layer is invalid. + key = (ext, spline_order, fov.hdu.header['NAXIS'], fov.header['NAXIS1'], fov.header['NAXIS2'], fov.header['CDELT1']) + if fov.hdu.header['NAXIS'] == 3: + key += tuple(fov.hdu.header[name] for name in ('NAXIS1', 'NAXIS2', 'CDELT1', 'CUNIT1', 'NAXIS3', 'CDELT3', 'CRPIX3', 'CRVAL3', 'CUNIT3')) + key += (self.meta.get('interp_psf', True), self.cmds.get('!SIM.computing.psf_maxsize')) + if key == self._kernel_grid_key: + return self.kernel if fov.hdu.header["NAXIS"] == 3: self.current_layer_id = ext self.make_psf_cube(fov, spline_order) + self._kernel_grid_key = key return self.kernel self.kernel = self._file[ext].data @@ -240,6 +248,7 @@ def get_kernel(self, fov): self.kernel = _cutout_kernel( self.kernel, fov.header, kernel_header=hdr) + self._kernel_grid_key = key return np.atleast_2d(self.kernel.squeeze()) def make_psf_cube(self, fov, spline_order=1): @@ -265,6 +274,7 @@ def make_psf_cube(self, fov, spline_order=1): - fov.hdu.header["CRPIX3"]) + fov.hdu.header["CRVAL3"] ) + lam *= u.Unit(fov.hdu.header['CUNIT3']).to(u.um) if not self.meta.get("interp_psf", True): lam = np.array([lam[len(lam)//2]]) diff --git a/scopesim/tests/tests_effects/test_ConstPSF.py b/scopesim/tests/tests_effects/test_ConstPSF.py index 27706b5d9..53abcdbf7 100644 --- a/scopesim/tests/tests_effects/test_ConstPSF.py +++ b/scopesim/tests/tests_effects/test_ConstPSF.py @@ -16,9 +16,12 @@ """ import pytest +from types import SimpleNamespace import numpy as np from numpy import testing as npt +from astropy import units as u +from astropy.io import fits import matplotlib.pyplot as plt @@ -76,6 +79,22 @@ def test_initialised_with_psf_name_and_file_format(self, mock_path): class TestGetKernel: + def test_spectral_kernel_follows_current_grid_and_units(self, const_psf): + const_psf.cmds = {'!SIM.computing.psf_maxsize': 21} + header = fits.Header({'CDELT1': .004/3600, 'CDELT2': .004/3600, 'CUNIT1': 'deg', 'CUNIT2': 'deg', 'CDELT3': .01, 'CRPIX3': 1., 'CRVAL3': 1.55, 'CUNIT3': 'um'}) + first = fits.ImageHDU(np.zeros((3, 15, 15)), header) + fov = SimpleNamespace(header=first.header, hdu=first, wavelength=1.6*u.um) + initial = const_psf.get_kernel(fov).copy() + second = fits.ImageHDU(np.zeros((5, 15, 15)), header) + fov.header, fov.hdu = second.header, second + expanded = const_psf.get_kernel(fov).copy() + assert expanded.shape[0] == 5 + npt.assert_allclose(expanded[:3], initial) + second.header['CUNIT3'] = 'nm' + second.header['CDELT3'] *= 1000 + second.header['CRVAL3'] *= 1000 + npt.assert_allclose(const_psf.get_kernel(fov), expanded, rtol=1e-6) + @pytest.mark.parametrize("waves, max_pixel", [([1.1, 1.3], 1 / 3.), ([1.5, 1.7], 1 / 3.), From 8a43ec43b6bcd6160d44aa7547a18e9629d5079a Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Sun, 18 Oct 2026 12:45:12 -0700 Subject: [PATCH 124/130] fix(numerics): retain double precision through spectral projection --- scopesim/effects/spectral_trace_list_utils.py | 11 ++++---- scopesim/optics/fov.py | 27 +++++++++---------- scopesim/optics/image_plane.py | 2 +- scopesim/optics/optical_train.py | 2 +- .../test_calibrated_spectral_trace.py | 12 +++++++++ .../tests/tests_optics/test_ImagePlane.py | 9 +++++++ .../test_spectral_bin_integration.py | 12 +++++++++ 7 files changed, 54 insertions(+), 21 deletions(-) diff --git a/scopesim/effects/spectral_trace_list_utils.py b/scopesim/effects/spectral_trace_list_utils.py index 4b40573e0..317e68ca9 100644 --- a/scopesim/effects/spectral_trace_list_utils.py +++ b/scopesim/effects/spectral_trace_list_utils.py @@ -327,7 +327,7 @@ def map_spectra_to_focal_plane(self, fov): sub_naxis2 = len(y_mm) xmax = xmin + sub_naxis1 ymax = ymin + sub_naxis2 - image = np.zeros((sub_naxis2, sub_naxis1), dtype=np.float32) + image = np.zeros((sub_naxis2, sub_naxis1), dtype=np.float64) self._set_dispersion(wave_min, wave_max, pixsize=pixsize) slit_dispersion = None @@ -381,8 +381,8 @@ def map_spectra_to_focal_plane(self, fov): detector_qe = fov.meta.get("detector_qe") if detector_qe is not None: xpix_img, ypix_img = np.meshgrid( - np.arange(xmin, xmax, dtype=np.float32), - np.arange(ymin, ymax, dtype=np.float32), + np.arange(xmin, xmax, dtype=np.float64), + np.arange(ymin, ymax, dtype=np.float64), ) image = apply_detector_qe_to_trace_image( image, @@ -973,9 +973,10 @@ def __init__(self, fov, dlam_per_pix, slit_dispersion=None, _cache=None): weights = np.concatenate([operator.data for operator in operators])*abs(d_eta) # Interleaved wavelength/slit cells match the contiguous cube layout, # avoiding a separate strided dense copy for every across-slit row. - _cache['operator'] = csr_matrix((weights, (rows, columns)), shape=(n_lam, n_lam*n_eta), dtype=np.float32) + _cache['operator'] = csr_matrix((weights, (rows, columns)), shape=(n_lam, n_lam*n_eta)) _cache['key'] = key - self.image = (_cache['operator'] @ fov.cube.data.reshape(n_lam*n_eta, n_xi)).T.astype(np.float32, copy=False) + cells = fov.cube.data.reshape(n_lam*n_eta, n_xi) + self.image = (_cache['operator'] @ cells).T # WCS for the xi-lambda image, i.e. the rectified 2D spectrum # Default WCS with xi in arcsec diff --git a/scopesim/optics/fov.py b/scopesim/optics/fov.py index 1afed3d03..610fac9c4 100644 --- a/scopesim/optics/fov.py +++ b/scopesim/optics/fov.py @@ -255,7 +255,7 @@ def view(self, sub_pixel: bool | None = None): def flatten(self): """If cube, collapse along first axis.""" if self.hdu and self.hdu.header["NAXIS"] == 3: - image = np.sum(self.hdu.data, axis=0) + image = np.sum(self.hdu.data, axis=0, dtype=np.float64) self.hdu.data = image def get_fields_in_fov(self, fields: Iterable[SourceField], fov_corners=None) -> Generator: @@ -652,7 +652,7 @@ def _make_cubefields(self): * yield scaled flux to be added to canvas flux """ for field in self._get_cube_fields(): - fluxes = field.data.sum(axis=(1, 2))*u.Unit(field.header['BUNIT'].lower()) + fluxes = field.data.sum(axis=(1, 2), dtype=np.float64)*u.Unit(field.header['BUNIT'].lower()) if field.is_bunit_spatially_differential: fluxes *= field.pixel_area yield np.interp(self.waveset, field.waveset, fluxes.to_value(PHOTLAM)) @@ -666,7 +666,7 @@ def _make_imagefields(self): * yield spectrum multiply by sum to be added to canvas flux """ for field in self._get_image_fields(): - weight = np.sum(field.data) # Shouldn't that be 1 by convention? + weight = np.sum(field.data, dtype=np.float64) # Shouldn't that be 1 by convention? yield field.spectrum(self.waveset).value * weight def _make_tablefields(self): @@ -757,7 +757,7 @@ def _make_cubefields(self): u.Unit(field.header["CUNIT3"]) ).to(u.Angstrom) # First collapse to image, then convert units - image = np.sum(field.data, axis=0) * PHOTLAM/u.arcsec**2 + image = np.sum(field.data, axis=0, dtype=np.float64) * PHOTLAM/u.arcsec**2 image = (image * self.area * field.pixel_area * spectral_bin_width).to(u.ph/u.s) logger.debug("2D FOV make_cubefields: image.mean() = %f %s", @@ -1049,13 +1049,10 @@ def make_hdu(self, spatial_psf=None): # -> Perhaps change that?? fov_waveset = self.cube_waveset.to(u.um) - # make canvas cube based on waveset of largest cube and NAXIS1,2 - # from fov.header - canvas_cube_hdu = fits.ImageHDU( - data=np.zeros((len(fov_waveset), - self.header["NAXIS2"], - self.header["NAXIS1"]), dtype=np.float32), - header=self.header) + # Retain double precision when combining faint sources with sky and + # thermal backgrounds; subtraction can expose storage-rounding error. + shape = (len(fov_waveset), self.header['NAXIS2'], self.header['NAXIS1']) + canvas_cube_hdu = fits.ImageHDU(data=np.zeros(shape, dtype=np.float64), header=self.header) # set BUNIT initially to PHOTLAM / arcsec**2 canvas_cube_hdu.header["BUNIT"] = "ph cm-2 s-1 AA-1 arcsec-2" @@ -1101,10 +1098,12 @@ def make_hdu(self, spatial_psf=None): canvas_cube_hdu.data[:, y, x] += flux else: for field in self._get_table_fields(): - spectra = {ref: average_spectrum(spec, fov_waveset).value.astype(np.float32) for ref, spec in field.spectra.items()} + spectra = {ref: average_spectrum(spec, fov_waveset).value for ref, spec in field.spectra.items()} for row in field.field: - image = (spatial_psf.point_source_image(self, row['x'], row['y'])/self.pixel_area.value).astype(np.float32) - canvas_cube_hdu.data += spectra[row['ref']][:, None, None]*image[None, :, :]*row['weight'] + image = spatial_psf.point_source_image(self, row['x'], row['y'])/self.pixel_area.value + for start in range(0, len(fov_waveset), 64): + spectrum = spectra[row['ref']][start:start + 64] + canvas_cube_hdu.data[start:start + 64] += spectrum[:, None, None]*image[None, :, :]*row['weight'] for background in self._make_backfields(fov_waveset): canvas_cube_hdu.data += background diff --git a/scopesim/optics/image_plane.py b/scopesim/optics/image_plane.py index 3ecba4d4f..f65c86f73 100644 --- a/scopesim/optics/image_plane.py +++ b/scopesim/optics/image_plane.py @@ -64,7 +64,7 @@ def __init__(self, header, cmds=None, **kwargs): raise ValueError(f"header must have a valid image-plane WCS: " f"{dict(header)}") - image = zeros_from_header(header, dtype=np.float32) + image = zeros_from_header(header, dtype=np.float64) self.hdu = fits.ImageHDU(data=image, header=header) self.hdu.header["BUNIT"] = "ph s-1" # photons per second (per pixel) diff --git a/scopesim/optics/optical_train.py b/scopesim/optics/optical_train.py index bfacc785e..2910909f6 100644 --- a/scopesim/optics/optical_train.py +++ b/scopesim/optics/optical_train.py @@ -653,7 +653,7 @@ def prepare_source(self, source): # Need to check whether BUNIT is per arcsec2 or per pixel inunit = u.Unit(header["BUNIT"]) - data = data.astype(np.float32) * inunit + data = data.astype(np.float64, copy=False) * inunit factor = 1 for base, power in zip(inunit.bases, inunit.powers): if (base**power).is_equivalent(u.arcsec**(-2)): diff --git a/scopesim/tests/tests_effects/test_calibrated_spectral_trace.py b/scopesim/tests/tests_effects/test_calibrated_spectral_trace.py index 00114bcee..86e4d36fe 100644 --- a/scopesim/tests/tests_effects/test_calibrated_spectral_trace.py +++ b/scopesim/tests/tests_effects/test_calibrated_spectral_trace.py @@ -6,6 +6,18 @@ from scopesim.effects.spectral_trace_list_utils import XiLamImage, SpectralTrace +def test_single_precision_cube_is_integrated_in_double_precision(): + fov = line_cube(.05) + fov.cube.data = np.zeros(fov.cube.data.shape, dtype=np.float32) + fov.cube.data[1000, :, :] = 1. + fov.cube.data[1000, 0, :] = 1e8 + image = XiLamImage(fov, lambda wave: .00001, slit_dispersion=lambda wave: .002) + # Integration across eta must retain the 19 faint cells beside the bright + # cell. The remapped line lies well inside the spectral boundaries. + expected = (1e8 + 19)*.05 + np.testing.assert_allclose(image.image.sum(axis=1), expected, rtol=1e-12) + + def line_cube(step, offset=0): wave = np.linspace(.995, 1.005, 2001) eta = np.arange(-.5+step/2, .5, step) diff --git a/scopesim/tests/tests_optics/test_ImagePlane.py b/scopesim/tests/tests_optics/test_ImagePlane.py index c941e903e..2866f3451 100644 --- a/scopesim/tests/tests_optics/test_ImagePlane.py +++ b/scopesim/tests/tests_optics/test_ImagePlane.py @@ -284,6 +284,15 @@ def test_mm_images_on_large_canvas( class TestImagePlaneAdd: + def test_faint_signal_is_retained_on_bright_background(self, image_hdu_square): + plane = opt_imp.ImagePlane(image_hdu_square.header) + background = fits.ImageHDU(np.full(image_hdu_square.data.shape, 1e8), image_hdu_square.header) + signal = fits.ImageHDU(np.ones(image_hdu_square.data.shape), image_hdu_square.header) + plane.add(background) + before = plane.data.copy() + plane.add(signal) + np.testing.assert_allclose(plane.data - before, 1., atol=1e-7) + def test_simple_add_imagehdu_conserves_flux( self, image_hdu_square, image_hdu_rect, ): diff --git a/scopesim/tests/tests_optics/test_spectral_bin_integration.py b/scopesim/tests/tests_optics/test_spectral_bin_integration.py index f5f826a13..71c216db9 100644 --- a/scopesim/tests/tests_optics/test_spectral_bin_integration.py +++ b/scopesim/tests/tests_optics/test_spectral_bin_integration.py @@ -52,3 +52,15 @@ def test_prepare_source_keeps_native_line_sampling(): train = OpticalTrain() prepared = train.prepare_source(source) np.testing.assert_array_equal(prepared.spectra[0].waveset, spectrum.waveset) + + +def test_prepare_cube_retains_faint_contrast_in_double_precision(): + from scopesim.tests.mocks.py_objects.source_objects import _cube_source + source = _cube_source() + field = source.cube_fields[0] + field.data.fill(1e8) + field.data[:, 0, 0] += 1. + prepared = OpticalTrain().prepare_source(source) + data = prepared.cube_fields[0].data + contrast = (data[:, 0, 0] - data[:, 0, 1])/data[:, 0, 1] + np.testing.assert_allclose(contrast, 1e-8, rtol=1e-6) From a0495f72a52b01a4bc52bc013b0bd43a4f1bc4be Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Sun, 18 Oct 2026 13:16:05 -0700 Subject: [PATCH 125/130] fix(cache): bound reuse to evaluated inputs and one observation --- scopesim/effects/psfs/analytical.py | 30 +++++++----- scopesim/effects/psfs/discrete.py | 15 +----- scopesim/effects/spectral_trace_list_utils.py | 38 +++++++-------- scopesim/optics/fov.py | 26 ++++------ scopesim/optics/optical_train.py | 13 ++++- scopesim/tests/tests_effects/test_ConstPSF.py | 15 ++++++ .../tests/tests_effects/test_MoffatPSF.py | 43 +++++++++++++++++ .../test_calibrated_spectral_trace.py | 48 +++++++++++++++++++ .../test_spectral_bin_integration.py | 12 +++++ 9 files changed, 175 insertions(+), 65 deletions(-) diff --git a/scopesim/effects/psfs/analytical.py b/scopesim/effects/psfs/analytical.py index 9278dced2..2685e6c43 100644 --- a/scopesim/effects/psfs/analytical.py +++ b/scopesim/effects/psfs/analytical.py @@ -3,6 +3,7 @@ from typing import ClassVar, Any from functools import partial, lru_cache +from pathlib import Path import numpy as np from astropy import units as u @@ -20,6 +21,12 @@ logger = get_logger(__name__) + +def _psf_table_key(filename, cmds): + path = Path(find_file(from_currsys(filename, cmds))) + return str(path), path.read_bytes() + + class AnalyticalPSF(PSF): """Base class for analytical PSFs.""" @@ -257,7 +264,6 @@ class MoffatPSF(AnalyticalPSF): def __init__(self, **kwargs): super().__init__(**kwargs) self._fwhm_interp = None - self._kernel_cache = {} def fwhm(self, wavelengths): fwhm = self.meta["fwhm"] @@ -268,6 +274,8 @@ def fwhm(self, wavelengths): key = tuple(fwhm.items()), get_zenith_angle(target, location, time) elif isinstance(fwhm, dict): key = tuple(fwhm.items()) + elif isinstance(fwhm, str): + key = _psf_table_key(fwhm, self.cmds) else: key = fwhm @@ -333,9 +341,6 @@ def get_kernel(self, fov): if max_ksize is not None: ksize = min(ksize, max_ksize) - key = (tuple(gammas), alpha, ystep, target, max_ksize, ksize, self.meta["rounded_edges"], self._renormalize_clipped_kernel()) - if key in self._kernel_cache: - return self._kernel_cache[key].copy() kernel, norm = self._make_moffat_kernel(gammas, alpha, ksize, ystep) while norm < target and (max_ksize is None or ksize < max_ksize): ksize = self._next_kernel_size(ksize, max_ksize) @@ -360,9 +365,6 @@ def get_kernel(self, fov): if self._renormalize_clipped_kernel() and norm > 0: kernel /= norm - if len(self._kernel_cache) >= 256: - self._kernel_cache.pop(next(iter(self._kernel_cache))) - self._kernel_cache[key] = kernel return kernel def point_source_image(self, fov, x, y): @@ -510,7 +512,9 @@ def fwhm_from_table(table): @lru_cache(maxsize=16) def _legendre_rule(order): - return np.polynomial.legendre.leggauss(order) + nodes, weights = np.polynomial.legendre.leggauss(order) + nodes.flags.writeable = weights.flags.writeable = False + return nodes, weights def moffat_cell_integrals(x, y, dx, dy, gammas, alpha): @@ -587,9 +591,10 @@ def natural_alpha(self): @property def ao_alpha(self): - if self._ao_alpha is None or self._ao_alpha[1] != self.meta['ao_table']: + key = _psf_table_key(self.meta['ao_table'], self.cmds), self.meta['alpha'] + if self._ao_alpha is None or self._ao_alpha[1] != key: _, alpha = self.ao_table_data() - self._ao_alpha = alpha, self.meta['ao_table'] + self._ao_alpha = alpha, key return self._ao_alpha[0] @property @@ -612,9 +617,10 @@ def ao_table_data(self): @property def ao_scale(self): - if self._ao_interp is None or self._ao_interp[1] != self.meta['ao_table']: + key = _psf_table_key(self.meta['ao_table'], self.cmds) + if self._ao_interp is None or self._ao_interp[1] != key: ao_table, _ = self.ao_table_data() - self._ao_interp = self.fwhm_from_table(ao_table), self.meta['ao_table'] + self._ao_interp = self.fwhm_from_table(ao_table), key return self._ao_interp[0] def ao_fwhm(self, wavelengths): diff --git a/scopesim/effects/psfs/discrete.py b/scopesim/effects/psfs/discrete.py index 53b334b2a..8b5c33355 100644 --- a/scopesim/effects/psfs/discrete.py +++ b/scopesim/effects/psfs/discrete.py @@ -192,7 +192,6 @@ def __init__(self, **kwargs): self.current_ext = None self.current_data = None self.kernel = None - self._kernel_grid_key = None def get_kernel(self, fov): """Find nearest wavelength and build PSF kernel from file.""" @@ -206,22 +205,13 @@ def get_kernel(self, fov): logger.warning("PSF parameter interp_order not found, using 1 (linear interpolation).") spline_order = 1 - # A reference PSF layer can serve FOVs with different wavelength and - # spatial grids. Reusing its sampled cube solely by layer is invalid. - key = (ext, spline_order, fov.hdu.header['NAXIS'], fov.header['NAXIS1'], fov.header['NAXIS2'], fov.header['CDELT1']) - if fov.hdu.header['NAXIS'] == 3: - key += tuple(fov.hdu.header[name] for name in ('NAXIS1', 'NAXIS2', 'CDELT1', 'CUNIT1', 'NAXIS3', 'CDELT3', 'CRPIX3', 'CRVAL3', 'CUNIT3')) - key += (self.meta.get('interp_psf', True), self.cmds.get('!SIM.computing.psf_maxsize')) - if key == self._kernel_grid_key: - return self.kernel - if fov.hdu.header["NAXIS"] == 3: self.current_layer_id = ext self.make_psf_cube(fov, spline_order) - self._kernel_grid_key = key return self.kernel - self.kernel = self._file[ext].data + # Keep the loaded reference independent of callers modifying a kernel. + self.kernel = self._file[ext].data.copy() self.current_layer_id = ext hdr = self._file[ext].header @@ -248,7 +238,6 @@ def get_kernel(self, fov): self.kernel = _cutout_kernel( self.kernel, fov.header, kernel_header=hdr) - self._kernel_grid_key = key return np.atleast_2d(self.kernel.squeeze()) def make_psf_cube(self, fov, spline_order=1): diff --git a/scopesim/effects/spectral_trace_list_utils.py b/scopesim/effects/spectral_trace_list_utils.py index 317e68ca9..09e0af973 100644 --- a/scopesim/effects/spectral_trace_list_utils.py +++ b/scopesim/effects/spectral_trace_list_utils.py @@ -149,7 +149,6 @@ def __init__(self, trace_tbl, cmds=None, **kwargs): # Declaration of other attributes self._xilamimg = None self._slit_cache = {} - self._pixel_cache = {} self.dlam_per_pix = None self.fit_inverse = True @@ -339,25 +338,20 @@ def map_spectra_to_focal_plane(self, fov): self._xilamimg = xilam # Integrate the density over each detector pixel. The inverse optical - # map is locally affine within a pixel; its exact center and Jacobian - # are cached, while quadrature samples are streamed to bound memory. + # map is locally affine within a pixel. Recompute centers and gradients: + # configured transforms can be edited in place between observations. xx, yy = np.meshgrid(np.arange(sub_naxis1), np.arange(sub_naxis2)) ximg_fpa, yimg_fpa = det_wcs.all_pix2world(xx, yy, 0) lam_fpa = self.xy2lam(ximg_fpa, yimg_fpa) - jacobian = from_currsys(self.meta['trace_flux_jacobian'], self.cmds) - key = (det_header.tostring(), xmin, ymin, image.shape, wave_min, wave_max, xi_min, xi_max, pixscale, jacobian) - if self._pixel_cache.get('key') != key: - self._pixel_cache.clear() - matrix = det_wcs.pixel_scale_matrix - dxidx, dxidy = self.xy2xi.gradient() - dlamdx, dlamdy = self.xy2lam.gradient() - ax, ay = dxidx(ximg_fpa, yimg_fpa), dxidy(ximg_fpa, yimg_fpa) - bx, by = dlamdx(ximg_fpa, yimg_fpa), dlamdy(ximg_fpa, yimg_fpa) - self._pixel_cache.update(key=key, xi=self.xy2xi(ximg_fpa, yimg_fpa), lam=lam_fpa, - xi_x=ax*matrix[0, 0] + ay*matrix[1, 0], xi_y=ax*matrix[0, 1] + ay*matrix[1, 1], - lam_x=bx*matrix[0, 0] + by*matrix[1, 0], lam_y=bx*matrix[0, 1] + by*matrix[1, 1], - scale=self._trace_flux_scale(ximg_fpa, yimg_fpa, pixsize, pixscale, det_header)) - geometry = self._pixel_cache + matrix = det_wcs.pixel_scale_matrix + dxidx, dxidy = self.xy2xi.gradient() + dlamdx, dlamdy = self.xy2lam.gradient() + ax, ay = dxidx(ximg_fpa, yimg_fpa), dxidy(ximg_fpa, yimg_fpa) + bx, by = dlamdx(ximg_fpa, yimg_fpa), dlamdy(ximg_fpa, yimg_fpa) + xi_fpa = self.xy2xi(ximg_fpa, yimg_fpa) + xi_x, xi_y = ax*matrix[0, 0] + ay*matrix[1, 0], ax*matrix[0, 1] + ay*matrix[1, 1] + lam_x, lam_y = bx*matrix[0, 0] + by*matrix[1, 0], bx*matrix[0, 1] + by*matrix[1, 1] + scale = self._trace_flux_scale(ximg_fpa, yimg_fpa, pixsize, pixscale, det_header) quadrature = np.full(2, self._pixel_quadrature_order) if 'effective_psf_fwhm' in fov.meta: middle = (wave_min + wave_max)/2 @@ -373,10 +367,10 @@ def map_spectra_to_focal_plane(self, fov): ynodes, yweights = np.polynomial.legendre.leggauss(quadrature[1]) for nx, wx in zip(xnodes/2, xweights/2): for ny, wy in zip(ynodes/2, yweights/2): - xi = geometry['xi'] + nx*geometry['xi_x'] + ny*geometry['xi_y'] - lam = geometry['lam'] + nx*geometry['lam_x'] + ny*geometry['lam_y'] + xi = xi_fpa + nx*xi_x + ny*xi_y + lam = lam_fpa + nx*lam_x + ny*lam_y inside = (xi >= xi_min) & (xi < xi_max) & (lam >= wave_min) & (lam < wave_max) - image[inside] += xilam.interp(xi[inside], lam[inside], grid=False) * geometry['scale'][inside] * wx * wy + image[inside] += xilam.interp(xi[inside], lam[inside], grid=False) * scale[inside] * wx * wy detector_qe = fov.meta.get("detector_qe") if detector_qe is not None: @@ -962,7 +956,9 @@ def __init__(self, fov, dlam_per_pix, slit_dispersion=None, _cache=None): derivative = -dlam_per_pix_val / d_eta if slit_dispersion is None else slit_dispersion(self.lam) derivative = np.broadcast_to(derivative, self.lam.shape) - key = (cube_lam[0], d_lam, n_lam, cube_eta[0], d_eta, n_eta, derivative.tobytes()) + # The operator depends only on these evaluated physical arrays. Include + # every element, so direct WCS/table/callable changes cannot alias it. + key = (cube_lam.tobytes(), cube_eta.tobytes(), d_eta, derivative.tobytes()) if _cache is None: _cache = {} if _cache.get('key') != key: diff --git a/scopesim/optics/fov.py b/scopesim/optics/fov.py index 610fac9c4..7311b2970 100644 --- a/scopesim/optics/fov.py +++ b/scopesim/optics/fov.py @@ -4,7 +4,6 @@ from warnings import warn from itertools import chain from collections.abc import Iterable, Generator -from weakref import WeakKeyDictionary import numpy as np from scipy.integrate import cumulative_trapezoid @@ -44,7 +43,6 @@ logger = get_logger(__name__) -_SPECTRUM_WAVESET_AA_CACHE = WeakKeyDictionary() class FieldOfView: @@ -136,7 +134,7 @@ def pixel_area(self): self.meta["pixel_area"] = pixel_area(self.header) return self.meta["pixel_area"] - def extract_from(self, src, spatial_header=None) -> None: + def extract_from(self, src, spatial_header=None, _spectral_wavesets=None) -> None: """ Extract relevent fields from source object. @@ -185,7 +183,7 @@ def extract_from(self, src, spatial_header=None) -> None: new_fld = TableSourceField( field=extracted, spectra={ - ref: extract_range_from_spectrum(spec, source_range) + ref: extract_range_from_spectrum(spec, source_range, _waveset=None if _spectral_wavesets is None else _spectral_wavesets[spec]) for ref, spec in field.spectra.items() if ref in extracted["ref"] }, @@ -199,7 +197,7 @@ def extract_from(self, src, spatial_header=None) -> None: new_fld = ImageSourceField( field=extracted, spectra={ - ref: extract_range_from_spectrum(spec, source_range) + ref: extract_range_from_spectrum(spec, source_range, _waveset=None if _spectral_wavesets is None else _spectral_wavesets[spec]) for ref, spec in field.spectra.items() }, ) @@ -217,7 +215,7 @@ def extract_from(self, src, spatial_header=None) -> None: field=None, header=field.header, spectra={ - ref: extract_range_from_spectrum(spec, source_range) + ref: extract_range_from_spectrum(spec, source_range, _waveset=None if _spectral_wavesets is None else _spectral_wavesets[spec]) for ref, spec in field.spectra.items() }, ) @@ -1145,12 +1143,12 @@ def replace_nans(field, cmds) -> None: "set in '!SIM.computing.nan_fill_value'.") -def extract_range_from_spectrum(spectrum, waverange): +def extract_range_from_spectrum(spectrum, waverange, _waveset=None): assert isinstance(spectrum, SourceSpectrum), ( f"spectrum must be of type synphot.SourceSpectrum: {type(spectrum)}") wave_min, wave_max = quantify(waverange, u.um).to(u.AA).value - spec_waveset = _spectrum_waveset_aa_value(spectrum) + spec_waveset = _spectrum_waveset_aa_value(spectrum) if _waveset is None else _waveset if spec_waveset is None: return spectrum mask = (spec_waveset > wave_min) & (spec_waveset < wave_max) @@ -1175,15 +1173,9 @@ def extract_range_from_spectrum(spectrum, waverange): def _spectrum_waveset_aa_value(spectrum): - """Return cached spectrum waveset values in Angstrom.""" - spec_waveset = _SPECTRUM_WAVESET_AA_CACHE.get(spectrum) - if spec_waveset is None: - waveset = spectrum.waveset - if waveset is None: - return None - spec_waveset = waveset.to_value(u.AA) - _SPECTRUM_WAVESET_AA_CACHE[spectrum] = spec_waveset - return spec_waveset + # A spectrum can change its sampling in place, including through redshift. + waveset = spectrum.waveset + return None if waveset is None else waveset.to_value(u.AA) def spectral_bin_edges(centers): diff --git a/scopesim/optics/optical_train.py b/scopesim/optics/optical_train.py index 2910909f6..ec211795c 100644 --- a/scopesim/optics/optical_train.py +++ b/scopesim/optics/optical_train.py @@ -537,6 +537,15 @@ def observe(self, orig_source=None, update=True, **kwargs): for effect in self.optics_manager.source_effects: source = effect.apply_to(source) + # This source is a private copy, after all source effects. FOV extraction + # copies native-sampled spectra before FOV effects can modify them. Evaluate + # native sampling once for this observation; retain no cache on the + # source, spectrum, FOV or train, and never reuse it on the next call. + spectral_wavesets = {} + for spectrum in source.spectra.values(): + waveset = spectrum.waveset + spectral_wavesets[spectrum] = None if waveset is None else waveset.to_value(u.AA) + # [3D - Atmospheric shifts, PSF, NCPAs, Grating shift/distortion] # START OF MULTIPROCESSING @@ -589,9 +598,9 @@ def observe(self, orig_source=None, update=True, **kwargs): defer_render = all(isinstance(effect, (TERCurve, SpectralEfficiency, EchelleSpectralEfficiency, TaperedQuantumEfficiency)) for effect in foveffs[:psf_index]) if defer_render: point_sources = not (source.image_fields or source.cube_fields) - fov.extract_from(source, spatial_header=foveffs[psf_index].padded_header(fov, point_sources=point_sources)) + fov.extract_from(source, spatial_header=foveffs[psf_index].padded_header(fov, point_sources=point_sources), _spectral_wavesets=spectral_wavesets) else: - fov.extract_from(source) + fov.extract_from(source, _spectral_wavesets=spectral_wavesets) fov.view() nobar = len(foveffs) <= 1 for effect in tqdm(foveffs, disable=nobar, diff --git a/scopesim/tests/tests_effects/test_ConstPSF.py b/scopesim/tests/tests_effects/test_ConstPSF.py index 53abcdbf7..cfbbd337f 100644 --- a/scopesim/tests/tests_effects/test_ConstPSF.py +++ b/scopesim/tests/tests_effects/test_ConstPSF.py @@ -79,6 +79,21 @@ def test_initialised_with_psf_name_and_file_format(self, mock_path): class TestGetKernel: + def test_returned_kernel_does_not_alias_reference_or_next_call(self, const_psf, centre_fov): + centre_fov.view() + initial = const_psf.get_kernel(centre_fov) + expected = initial.copy() + initial.fill(0.) + npt.assert_array_equal(const_psf.get_kernel(centre_fov), expected) + + def test_reference_image_edits_reach_next_kernel(self, const_psf, centre_fov): + centre_fov.view() + first = const_psf.get_kernel(centre_fov).copy() + ext = const_psf.current_layer_id + const_psf._file[ext].data[:] = np.roll(const_psf._file[ext].data, 3, axis=0) + second = const_psf.get_kernel(centre_fov) + assert not np.allclose(first, second) + def test_spectral_kernel_follows_current_grid_and_units(self, const_psf): const_psf.cmds = {'!SIM.computing.psf_maxsize': 21} header = fits.Header({'CDELT1': .004/3600, 'CDELT2': .004/3600, 'CUNIT1': 'deg', 'CUNIT2': 'deg', 'CDELT3': .01, 'CRPIX3': 1., 'CRVAL3': 1.55, 'CUNIT3': 'um'}) diff --git a/scopesim/tests/tests_effects/test_MoffatPSF.py b/scopesim/tests/tests_effects/test_MoffatPSF.py index 00dc6d16b..6fe78db4b 100644 --- a/scopesim/tests/tests_effects/test_MoffatPSF.py +++ b/scopesim/tests/tests_effects/test_MoffatPSF.py @@ -208,3 +208,46 @@ def test_table_fwhm_reuses_interpolator_without_observation_commands(tmp_path): cached = psf._fwhm_interp[0] np.testing.assert_allclose(psf.fwhm(wave.to(u.um)).to_value(u.arcsec), [.5, 1.]) assert psf._fwhm_interp[0] is cached + + +def test_returned_kernel_mutation_does_not_change_later_psf(): + psf = MoffatPSF(alpha=4.765, fwhm=.7) + first = psf.get_kernel(SimpleFov()) + expected = first.copy() + first.fill(0.) + np.testing.assert_array_equal(psf.get_kernel(SimpleFov()), expected) + + +def test_overwriting_fwhm_table_refreshes_interpolator(tmp_path): + path = tmp_path/'fwhm.dat' + table = Table({'wavelength': [.4, .5, 1., 2.5], 'fwhm': [.4, .5, 1., 2.5]}, meta={'comments': ['wavelength_unit : um']}) + table.write(path, format='ascii') + psf = MoffatPSF(alpha=4.765, fwhm=str(path)) + wave = np.array([.5, 1.])*u.um + first = psf.fwhm(wave) + table['fwhm'] *= 2 + table.write(path, format='ascii', overwrite=True) + np.testing.assert_allclose(psf.fwhm(wave), 2*first) + + +def test_ao_cache_tracks_resolved_path_contents_and_alpha(tmp_path): + first, second = tmp_path/'first.dat', tmp_path/'second.dat' + table = Table({'wavelength': [.4, .5, 1., 2.5], 'fwhm': [.3, .3, .3, .3]}, meta={'comments': ['wavelength_unit : um']}) + table.write(first, format='ascii') + table['fwhm'] *= 2 + table.write(second, format='ascii') + cmds = UserCommands(properties={'!INST.ao_file': str(first)}) + psf = AOEnhanceablePSF(cmds=cmds, ao_table='!INST.ao_file', is_absolute=True, enable_ao=True, alpha=4.765) + # PSF construction resolves command references. Also exercise a reference + # assigned directly to the configured metadata after construction. + psf.meta['ao_table'] = '!INST.ao_file' + wave = np.array([.5, 1.])*u.um + np.testing.assert_allclose(psf.fwhm(wave).value, .3) + assert psf.alpha == 4.765 + cmds['!INST.ao_file'] = str(second) + np.testing.assert_allclose(psf.fwhm(wave).value, .6) + table['fwhm'] /= 3 + table.write(second, format='ascii', overwrite=True) + np.testing.assert_allclose(psf.fwhm(wave).value, .2) + psf.meta['alpha'] = 3. + assert psf.alpha == 3. diff --git a/scopesim/tests/tests_effects/test_calibrated_spectral_trace.py b/scopesim/tests/tests_effects/test_calibrated_spectral_trace.py index 86e4d36fe..e263f9ca6 100644 --- a/scopesim/tests/tests_effects/test_calibrated_spectral_trace.py +++ b/scopesim/tests/tests_effects/test_calibrated_spectral_trace.py @@ -4,6 +4,8 @@ from astropy import units as u from astropy.io import fits from scopesim.effects.spectral_trace_list_utils import XiLamImage, SpectralTrace +from astropy.table import Table +from scopesim.optics.image_plane_utils import header_from_list_of_xy def test_single_precision_cube_is_integrated_in_double_precision(): @@ -97,3 +99,49 @@ def test_rectified_interpolation_reproduces_a_linear_density_field(): positions, wavelengths = np.array([.03, .17]), np.array([.998, 1.003]) expected = 2 + 3*positions + 5*wavelengths np.testing.assert_allclose(image.interp(positions, wavelengths, grid=False), expected, rtol=2e-6) + + +@pytest.mark.parametrize('change', ['wavelength_origin', 'wavelength_step', 'wavelength_pc', 'slit_origin', 'slit_step', 'slit_pc', 'slice_center', 'dispersion', 'source', 'units']) +def test_reused_slit_operator_matches_fresh_after_direct_changes(change): + fov = line_cube(.1) + cache = {} + slope = [.002] + dispersion = lambda wave: slope[0]*wave + XiLamImage(fov, lambda wave: .00001, slit_dispersion=dispersion, _cache=cache) + if change == 'wavelength_origin': fov.cube.header['CRVAL3'] += .001 + if change == 'wavelength_step': fov.cube.header['CDELT3'] *= 1.1 + if change == 'wavelength_pc': fov.cube.header['PC3_3'] = 1.1 + if change == 'slit_origin': fov.cube.header['CRVAL2'] += .04 + if change == 'slit_step': fov.cube.header['CDELT2'] *= 1.1 + if change == 'slit_pc': fov.cube.header['PC2_2'] = 1.1 + if change == 'slice_center': fov.meta['eta_min'] += .04*u.arcsec + if change == 'dispersion': slope[0] *= 1.3 + if change == 'source': fov.cube.data *= 2. + if change == 'units': + fov.cube.header['CUNIT2'], fov.cube.header['CUNIT3'] = 'deg', 'nm' + for keyword in ('CDELT2', 'CRVAL2'): fov.cube.header[keyword] /= 3600 + for keyword in ('CDELT3', 'CRVAL3'): fov.cube.header[keyword] *= 1000 + reused = XiLamImage(fov, lambda wave: .00001, slit_dispersion=dispersion, _cache=cache) + fresh = XiLamImage(fov, lambda wave: .00001, slit_dispersion=dispersion) + np.testing.assert_array_equal(reused.image, fresh.image) + + +@pytest.mark.parametrize('coordinate', ['xy2lam', 'xy2xi']) +def test_direct_optical_transform_edits_reach_next_projection(coordinate): + xi, wave = np.meshgrid(np.linspace(-1., 1., 5), np.linspace(.99, 1.01, 9)) + table = Table({'wavelength': wave.ravel()*u.um, 's': xi.ravel()*u.arcsec, 'x': (wave.ravel()-1)*1000*u.mm, 'y': xi.ravel()*.15*u.mm}) + reused, fresh = [SpectralTrace(table.copy(), trace_flux_jacobian=True) for _ in range(2)] + fov = line_cube(.1) + fov.cube.data *= np.array([1., 2., 4.])[None, None, :] + fov.header = fov.cube.header + fov.detector_header = header_from_list_of_xy([-10., 10.], [-2., 2.], .015, 'D') + fov.trace_id = 'linear' + fov.meta.update(wave_min=.995*u.um, wave_max=1.005*u.um, xi_min=-.05*u.arcsec, xi_max=.25*u.arcsec) + before = reused.map_spectra_to_focal_plane(fov).data.copy() + for trace in (reused, fresh): + if coordinate == 'xy2lam': trace.xy2lam.matrix[0, 0] += .0002 + if coordinate == 'xy2xi': trace.xy2xi.matrix[0, 0] += .04 + actual = reused.map_spectra_to_focal_plane(fov).data + expected = fresh.map_spectra_to_focal_plane(fov).data + np.testing.assert_array_equal(actual, expected) + assert np.max(abs(actual-before)) > .001*before.max() diff --git a/scopesim/tests/tests_optics/test_spectral_bin_integration.py b/scopesim/tests/tests_optics/test_spectral_bin_integration.py index 71c216db9..a037bb5b5 100644 --- a/scopesim/tests/tests_optics/test_spectral_bin_integration.py +++ b/scopesim/tests/tests_optics/test_spectral_bin_integration.py @@ -33,6 +33,18 @@ def test_flat_ab_continuum_bin_integral(): np.testing.assert_allclose(average_spectrum(spectrum, wave).to_value(PHOTLAM), expected, rtol=1e-6) +def test_mutating_spectrum_redshift_updates_native_sampling(): + points = np.array([4999., 4999.99, 5000., 5000.01, 5001.])*u.AA + spectrum = SourceSpectrum(Empirical1D, points=points, lookup_table=[0., 0., 1., 0., 0.]) + average_spectrum(spectrum, np.arange(4998., 5002., .1)*u.AA) + spectrum.z = .01 + fresh = SourceSpectrum(Empirical1D, points=points, lookup_table=[0., 0., 1., 0., 0.], z=.01) + wave = np.arange(5048., 5052., .1)*u.AA + actual = average_spectrum(spectrum, wave) + np.testing.assert_allclose(actual, average_spectrum(fresh, wave), rtol=1e-12) + assert actual.sum() > 0 + + @pytest.mark.parametrize('descending', [False, True]) def test_cube_rebinning_preserves_flux_spatial_weights_and_support(descending): wave = np.array([1., 2., 3.]) From 8727eafed5056e477278130ca4ffcdfa43e9f101 Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Mon, 19 Oct 2026 12:32:43 -0700 Subject: [PATCH 126/130] fix(atmosphere): displace incoming illumination before slit clipping --- scopesim/effects/atmo_dispersion.py | 48 +++-- scopesim/effects/psfs/analytical.py | 4 +- scopesim/optics/fov.py | 22 ++- scopesim/optics/optical_train.py | 27 ++- .../test_adc_slit_illumination.py | 166 ++++++++++++++++++ 5 files changed, 245 insertions(+), 22 deletions(-) create mode 100644 scopesim/tests/tests_optics/test_adc_slit_illumination.py diff --git a/scopesim/effects/atmo_dispersion.py b/scopesim/effects/atmo_dispersion.py index 6a14fee04..7ceeea515 100644 --- a/scopesim/effects/atmo_dispersion.py +++ b/scopesim/effects/atmo_dispersion.py @@ -62,15 +62,17 @@ class ShiftFoV3D(Effect): def __init__(self, **kwargs): super().__init__(**kwargs) + self.meta.setdefault("method", None) - if self.meta.get("method", None) not in [None, "roll", "map_coords"]: + if self.meta["method"] not in [None, "roll", "map_coords"]: raise ValueError("Method must be 'roll' or 'map_coords' if provided.") def get_shifts(self, obj): """ Shifts calculated from obj (FoV3D) properties or from init data. Overwrite this function when sub-classing. - Should return interp spline values for dx and dy at data wavelength bins in arcsec + Return dx and dy at cube wavelength bins in arcsec. The FOV can be + unrendered when its incoming illumination needs spatial padding. """ if self.data is not None: if isinstance(self.data, Table): @@ -82,14 +84,24 @@ def get_shifts(self, obj): logger.warning("No shifts as a function of wavelength provided in init.") raise NotImplementedError - swcs = WCS(obj.hdu.header).spectral - with u.set_enabled_equivalencies(u.spectral()): - wave = swcs.pixel_to_world(np.arange(swcs.pixel_shape[0])) << u.um - return dx(wave), dy(wave) + wave = _cube_wavelengths(obj) + return dx(wave), dy(wave) + + def _padded_header(self, header, shifts): + """Spatial support needed before applying this displacement.""" + header = header.copy() + # Cubic interpolation needs neighboring samples as well as the shift. + guard = 0 if self.meta['method'] == 'roll' else 4 + for axis, displacement in enumerate(shifts, start=1): + cells = (displacement/(abs(header[f'CDELT{axis}'])*u.deg)).decompose().value + padding = int(np.ceil(np.max(np.abs(cells)))) + guard if np.any(cells) else 0 + header[f'NAXIS{axis}'] += 2*padding + header[f'CRPIX{axis}'] += padding + return header def apply_to(self, obj, **kwargs): if isinstance(obj, FieldOfView3D) and obj.hdu is not None and obj.hdu.header['NAXIS'] == 3: - dxs, dys = self.get_shifts(obj) + dxs, dys = kwargs['_shifts'] if '_shifts' in kwargs else self.get_shifts(obj) # get bin widths in spatial axis and divide to get dxs and dys in steps dxs = np.round((dxs / (obj.hdu.header["CDELT1"]*u.deg)).decompose().value, 1) @@ -243,7 +255,7 @@ def apply_to(self, obj, **kwargs): def get_shifts(self, obj: FieldOfView3D): shift_arcsec = self._get_shifts_arcsec(obj) - pos_angle_y = field_rotation_pa_y(obj.hdu.header) + pos_angle_y = field_rotation_pa_y(obj.header if obj.hdu is None else obj.hdu.header) par_angle = get_parallactic_angle(self.target, self.location, self.time) theta = np.deg2rad(par_angle - pos_angle_y) # angle of zenith direction from +y-axis counter-clockwise logger.info(f'Direction of zenith, measured counter-clockwise from +y axis is {int(par_angle - pos_angle_y)}.') @@ -256,9 +268,7 @@ def _get_shifts_arcsec(self, obj: FieldOfView3D): wave_ref = from_currsys(self.meta["wave_ref_um"], self.cmds) * u.um n_ref = self.n(wave_ref) - swcs = WCS(obj.hdu.header).spectral - with u.set_enabled_equivalencies(u.spectral()): - wave = swcs.pixel_to_world(np.arange(swcs.pixel_shape[0])) << u.um + wave = _cube_wavelengths(obj) n_wave = self.n(wave) delta_n = n_wave - n_ref return 206265 * delta_n * np.tan(self.zenith_angle.to(u.rad).value) * u.arcsec @@ -316,9 +326,7 @@ def get_shifts(self, obj: FieldOfView3D): """ Computes an AD correction residual for a given zenith angle. """ - swcs = WCS(obj.hdu.header).spectral - with u.set_enabled_equivalencies(u.spectral()): - wave = swcs.pixel_to_world(np.arange(swcs.pixel_shape[0])) << u.um + wave = _cube_wavelengths(obj) shifts = np.zeros_like(wave.value) * u.arcsec if self.data is not None and isinstance(self.data, Table): @@ -361,7 +369,7 @@ def get_shifts(self, obj: FieldOfView3D): shifts += ad_shift ad.zenith_angle = self.zenith_angle - pos_angle_y = field_rotation_pa_y(obj.hdu.header) + pos_angle_y = field_rotation_pa_y(obj.header if obj.hdu is None else obj.hdu.header) par_angle = get_parallactic_angle(self.target, self.location, self.time) theta = np.deg2rad(par_angle - pos_angle_y) # angle of zenith direction from +y-axis counter-clockwise logger.info(f'Direction of zenith, measured counter-clockwise from +y axis is {int(par_angle - pos_angle_y)}.') @@ -408,12 +416,20 @@ def _get_ad_shift(self, ad_kwargs): ########################### AD utils ############################### +def _cube_wavelengths(obj): + if obj.hdu is None: + return obj.cube_waveset.to(u.um) + swcs = WCS(obj.hdu.header).spectral + with u.set_enabled_equivalencies(u.spectral()): + return swcs.pixel_to_world(np.arange(swcs.pixel_shape[0])).to(u.um) + + def field_rotation_pa_y(header) -> float: """ Returns PA of detector +Y axis on sky, in deg (N->E convention). 0 deg = Y points north, 90 deg = Y points east """ - w = WCS(header).dropaxis(-1) + w = WCS(header, naxis=2) # local anchor (image center) x0, y0 = w.wcs.crpix lon0, lat0 = w.pixel_to_world(x0, y0) diff --git a/scopesim/effects/psfs/analytical.py b/scopesim/effects/psfs/analytical.py index 2685e6c43..1e8bb0ff4 100644 --- a/scopesim/effects/psfs/analytical.py +++ b/scopesim/effects/psfs/analytical.py @@ -377,7 +377,7 @@ def point_source_image(self, fov, x, y): gammas = self.fwhm2gamma(self.fwhm(wave), self.alpha).to_value(u.arcsec) return moffat_cell_integrals(xx, yy, dx, dy, gammas, self.alpha) - def _apply_convolution(self, obj): + def _apply_convolution(self, obj, spatial_shifts=()): if isinstance(obj, FieldOfView3D) and obj.hdu is None: if obj.trace_id is not None: # Finite cross-slit cells carry a uniform illumination density. @@ -403,7 +403,7 @@ def _apply_convolution(self, obj): obj._volume = None if 'slit_spectral_width' in obj.meta: obj.meta['slit_spectral_width'] *= min(1., fwhm / width) - obj.hdu = obj.make_hdu(spatial_psf=self) + obj.hdu = obj.make_hdu(spatial_psf=self, spatial_shifts=spatial_shifts) else: super()._apply_convolution(obj) diff --git a/scopesim/optics/fov.py b/scopesim/optics/fov.py index 7311b2970..c9765abd9 100644 --- a/scopesim/optics/fov.py +++ b/scopesim/optics/fov.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- """Defines FieldOfView class.""" +from copy import copy from warnings import warn from itertools import chain from collections.abc import Iterable, Generator @@ -1031,7 +1032,7 @@ def _make_backfields(self, fov_waveset): spec = average_spectrum(field.spectrum, fov_waveset) yield spec[:, None, None].value - def make_hdu(self, spatial_psf=None): + def make_hdu(self, spatial_psf=None, spatial_shifts=()): """Render spectral surface-brightness density in ph/s/um/arcsec2. Native spectra are integrated into the cube wavelength cells. Point @@ -1039,6 +1040,25 @@ def make_hdu(self, spatial_psf=None): convolved before the physical aperture is clipped. Collecting area is applied here. Slit and detector cell integrals are applied by the trace. """ + if spatial_shifts: + # Keep incoming illumination outside the aperture until every + # atmospheric displacement is applied. The physical slit metadata + # and cell sizes stay unchanged; only temporary spatial support grows. + shifts = [(effect, effect.get_shifts(self)) for effect in spatial_shifts] + padded = copy(self) + padded.header = self.header.copy() + for effect, displacement in shifts: + padded.header = effect._padded_header(padded.header, displacement) + padded.hdu = padded.make_hdu(spatial_psf=spatial_psf) + for effect, displacement in shifts: + effect.apply_to(padded, _shifts=displacement) + px = (padded.header['NAXIS1'] - self.header['NAXIS1'])//2 + py = (padded.header['NAXIS2'] - self.header['NAXIS2'])//2 + header = padded.hdu.header.copy() + header.update(CRPIX1=self.header['CRPIX1'], CRPIX2=self.header['CRPIX2']) + data = padded.hdu.data[:, py:py + self.header['NAXIS2'], px:px + self.header['NAXIS1']].copy() + return fits.ImageHDU(data=data, header=header) + # 1. Make waveset and canvas cube (area, bin_width are applied at end) # TODO: Why is this not self.waveset? What's different? # -> For non-cube input but cube output (e.g. 2D image in spec mode), diff --git a/scopesim/optics/optical_train.py b/scopesim/optics/optical_train.py index ec211795c..9aa230d28 100644 --- a/scopesim/optics/optical_train.py +++ b/scopesim/optics/optical_train.py @@ -24,7 +24,8 @@ from ..commands.user_commands import UserCommands from ..detector import DetectorManager from ..effects import ExtraFitsKeywords, SelectorWheel, MoffatPSF, TERCurve, SpectralEfficiency, EchelleSpectralEfficiency, SpectralTraceList -from ..effects.ter_curves import FilterWheelBase, TaperedQuantumEfficiency +from ..effects.ter_curves import DichroicTree, FilterWheelBase, TaperedQuantumEfficiency +from ..effects.atmo_dispersion import ShiftFoV3D from ..utils import from_currsys, top_level_catch, get_logger from ..source.source_templates import empty_sky from .. import __version__ @@ -573,6 +574,10 @@ def observe(self, orig_source=None, update=True, **kwargs): fovs = self.fov_manager.fovs nobar = len(fovs) <= 1 for fov in tqdm(fovs, desc=" FOVs", position=0, disable=nobar): + # FOVManager can reuse a FOV whose previous observation is + # already mapped to detector coordinates. Prepare new sky + # illumination from its physical header, not that old HDU. + fov.hdu = None foveffs = [] for effect in self.optics_manager.fov_effects: if isinstance(effect, SelectorWheel): @@ -596,16 +601,32 @@ def observe(self, orig_source=None, update=True, **kwargs): if defer_render: psf_index = psf_indices[0] defer_render = all(isinstance(effect, (TERCurve, SpectralEfficiency, EchelleSpectralEfficiency, TaperedQuantumEfficiency)) for effect in foveffs[:psf_index]) + spatial_shifts = [] if defer_render: + for effect in foveffs[psf_index + 1:]: + if isinstance(effect, ShiftFoV3D): + spatial_shifts.append(effect) + elif not isinstance(effect, (TERCurve, DichroicTree, SpectralEfficiency, EchelleSpectralEfficiency, TaperedQuantumEfficiency)): + break + # Wavelength-only multipliers commute with a displacement + # within each spectral plane; stop before other spatial effects. point_sources = not (source.image_fields or source.cube_fields) - fov.extract_from(source, spatial_header=foveffs[psf_index].padded_header(fov, point_sources=point_sources), _spectral_wavesets=spectral_wavesets) + spatial_header = foveffs[psf_index].padded_header(fov, point_sources=point_sources) + for effect in spatial_shifts: + spatial_header = effect._padded_header(spatial_header, effect.get_shifts(fov)) + fov.extract_from(source, spatial_header=spatial_header, _spectral_wavesets=spectral_wavesets) else: fov.extract_from(source, _spectral_wavesets=spectral_wavesets) fov.view() nobar = len(foveffs) <= 1 for effect in tqdm(foveffs, disable=nobar, desc=" FOV effects", position=1):#, leave=False): - fov = effect.apply_to(fov) + if effect in spatial_shifts: + continue # Applied to incoming illumination at the PSF stage. + if defer_render and effect is foveffs[psf_index] and spatial_shifts: + effect._apply_convolution(fov, spatial_shifts=spatial_shifts) + else: + fov = effect.apply_to(fov) if fov.hdu is None: logger.info(" Skipping empty FOV") diff --git a/scopesim/tests/tests_optics/test_adc_slit_illumination.py b/scopesim/tests/tests_optics/test_adc_slit_illumination.py new file mode 100644 index 000000000..20df6706a --- /dev/null +++ b/scopesim/tests/tests_optics/test_adc_slit_illumination.py @@ -0,0 +1,166 @@ +import numpy as np +import pytest +from astropy import units as u +from astropy.io import fits +from synphot import SourceSpectrum, ConstFlux1D +from synphot.units import PHOTLAM + +from scopesim import Source +from scopesim.effects.atmo_dispersion import ShiftFoV3D, _cube_wavelengths +from scopesim.effects.psfs.analytical import MoffatPSF +from scopesim.source.source_fields import BackgroundSourceField +from scopesim.optics.image_plane_utils import header_from_list_of_xy +from scopesim.tests.tests_optics.test_slit_spatial_integration import physical_fov + + +class Displacement(ShiftFoV3D): + def __init__(self, dx, dy, **kwargs): + super().__init__(**kwargs) + self.dx, self.dy = dx*u.arcsec, dy*u.arcsec + + def get_shifts(self, obj): + wave = _cube_wavelengths(obj) + return np.broadcast_to(self.dx, wave.shape, subok=True), np.broadcast_to(self.dy, wave.shape, subok=True) + + +def render(fov, source, psf, shifts): + support = psf.padded_header(fov, point_sources=not (source.image_fields or source.cube_fields)) + for effect in shifts: + support = effect._padded_header(support, effect.get_shifts(fov)) + fov.extract_from(source, spatial_header=support) + psf._apply_convolution(fov, spatial_shifts=shifts) + return fov.hdu.data*fov.pixel_area.value/1e4 + + +@pytest.mark.parametrize('width', [.33, .7, .9]) +@pytest.mark.parametrize('position,steps', [((0., 0.), (0, 1)), ((.037, .4), (0, -2)), ((5.6, .1), (-2, 1)), ((7., 0.), (-15, 0))]) +@pytest.mark.parametrize('sign', [-1, 1]) +def test_shifted_point_uses_incoming_psf_before_slit_and_ends(width, position, steps, sign): + fov = physical_fov(width) + fov.header['CDELT2'] *= sign + original = fov.header.copy() + cell = np.array([fov.header['CDELT1'], fov.header['CDELT2']])*3600 + displacement = np.array(steps)*cell + shift = Displacement(*displacement, method='roll') + psf = MoffatPSF(alpha=4.765, fwhm=.7) + source = Source(x=[position[0]], y=[position[1]], ref=[0], weight=[1.], spectra=[SourceSpectrum(ConstFlux1D, amplitude=1*PHOTLAM)]) + actual = render(fov, source, psf, [shift]) + expected = psf.point_source_image(fov, *(position + displacement)) + np.testing.assert_allclose(actual, np.broadcast_to(expected, actual.shape), rtol=2e-6, atol=1e-12) + for key in ('NAXIS1', 'NAXIS2', 'CRPIX1', 'CRPIX2', 'CDELT1', 'CDELT2', 'CRPIX1D', 'CRPIX2D'): + assert fov.header[key] == original[key] + assert fov.hdu.header[key] == original[key] + + +@pytest.mark.parametrize('method', ['roll', 'map_coords']) +def test_uniform_sky_is_not_lost_at_shifted_slit_edges(method): + fov = physical_fov(.7) + source = Source() + source.fields.append(BackgroundSourceField(field=None, header=fits.Header(), spectra={0: SourceSpectrum(ConstFlux1D, amplitude=1*PHOTLAM)})) + psf = MoffatPSF(alpha=4.765, fwhm=.7) + shift = Displacement(.23, -.17, method=method) + actual = render(fov, source, psf, [shift]) + np.testing.assert_allclose(actual, fov.pixel_area.value, rtol=1e-12) + + +def test_wavelength_dependent_shift_keeps_spectral_planes_independent(): + fov = physical_fov(.7) + count = len(fov.cube_waveset) + cell_y = fov.header['CDELT2']*3600 + displacement = np.resize([-1., 0., 1.], count)*cell_y + shift = Displacement(0., displacement, method='roll') + psf = MoffatPSF(alpha=4.765, fwhm=.7) + source = Source(x=[0], y=[.2], ref=[0], weight=[1.], spectra=[SourceSpectrum(ConstFlux1D, amplitude=1*PHOTLAM)]) + actual = render(fov, source, psf, [shift]) + expected = np.stack([psf.point_source_image(fov, 0., .2 + dy) for dy in displacement]) + np.testing.assert_allclose(actual, expected, rtol=1e-6) + + +def test_two_displacements_keep_light_that_reenters_slit(): + fov = physical_fov(.7) + step = fov.header['CDELT2']*3600 + shifts = [Displacement(0., 4*step, method='roll'), Displacement(0., -4*step, method='roll')] + psf = MoffatPSF(alpha=4.765, fwhm=.7) + source = Source(x=[0], y=[0], ref=[0], weight=[1.], spectra=[SourceSpectrum(ConstFlux1D, amplitude=1*PHOTLAM)]) + actual = render(fov, source, psf, shifts) + expected = psf.point_source_image(fov, 0., 0.) + np.testing.assert_allclose(actual, np.broadcast_to(expected, actual.shape), rtol=1e-6) + + +def test_shifted_resolved_image_and_cube_agree(): + header = header_from_list_of_xy([-1/3600, 1/3600], [-1/3600, 1/3600], .05/3600) + yy, xx = np.indices((header['NAXIS2'], header['NAXIS1'])) + density = np.exp(-((xx-20)**2 + (yy-28)**2)/8) + image_source = Source(image_hdu=fits.ImageHDU(data=density*.05**2, header=header), spectra=[SourceSpectrum(ConstFlux1D, amplitude=1*PHOTLAM)]) + cube_header = header.copy() + cube_header.update(CRPIX3=1., CRVAL3=.49, CDELT3=.01, CUNIT3='um', CTYPE3='WAVE', BUNIT='photlam arcsec-2') + cube_source = Source(cube=fits.ImageHDU(data=np.broadcast_to(density, (13, *density.shape)).copy(), header=cube_header)) + psf = MoffatPSF(alpha=4.765, fwhm=.7) + shift = Displacement(.15, -.2, method='roll') + images = [render(physical_fov(.7), source, psf, [shift]) for source in (image_source, cube_source)] + np.testing.assert_allclose(images[0], images[1], rtol=2e-6, atol=1e-10) + fov = physical_fov(.7) + cell = np.array([fov.header['CDELT1'], fov.header['CDELT2']])*3600 + displacement = np.rint(np.round(np.array([.15, -.2])/cell, 1))*cell + moved = image_source.make_copy() + moved.shift(*displacement) + expected = render(fov, moved, psf, []) + np.testing.assert_allclose(images[0], expected, rtol=2e-6, atol=1e-10) + + +@pytest.mark.parametrize('method', ['roll', 'map_coords']) +def test_zero_shift_reproduces_unshifted_render(method): + psf = MoffatPSF(alpha=4.765, fwhm=.7) + source = Source(x=[.1], y=[.2], ref=[0], weight=[1.], spectra=[SourceSpectrum(ConstFlux1D, amplitude=1*PHOTLAM)]) + unshifted = render(physical_fov(.7), source, psf, []) + shift = Displacement(0., 0., method=method) + fov = physical_fov(.7) + assert shift._padded_header(fov.header, shift.get_shifts(fov)) == fov.header + shifted = render(fov, source, psf, [shift]) + np.testing.assert_allclose(unshifted, shifted, rtol=1e-14, atol=1e-14) + + +@pytest.mark.parametrize('position', [(0., 0.), (.1, .4), (5.6, 0.)]) +def test_fractional_shift_agrees_with_pre_slit_psf_flux_and_centroid(position): + fov = physical_fov(.7) + cell = np.array([fov.header['CDELT1'], fov.header['CDELT2']])*3600 + shift = Displacement(*(np.array([.3, -.4])*cell), method='map_coords') + psf = MoffatPSF(alpha=4.765, fwhm=.7) + source = Source(x=[position[0]], y=[position[1]], ref=[0], weight=[1.], spectra=[SourceSpectrum(ConstFlux1D, amplitude=1*PHOTLAM)]) + actual = render(fov, source, psf, [shift])[0] + expected = psf.point_source_image(fov, *(position + np.array([.3, -.4])*cell)) + assert actual.sum() == pytest.approx(expected.sum(), rel=1e-3) + yy, xx = np.indices(actual.shape) + for coordinate in (xx, yy): + measured = np.sum(actual*coordinate)/actual.sum() + reference = np.sum(expected*coordinate)/expected.sum() + assert measured == pytest.approx(reference, abs=.01) + + +@pytest.mark.parametrize('intervening_throughput', [False, True]) +def test_observe_applies_shift_before_slit_through_wavelength_throughput(intervening_throughput): + from types import SimpleNamespace + from scopesim import OpticalTrain + from scopesim.effects import TERCurve + + psf = MoffatPSF(alpha=4.765, fwhm=.7) + images = [] + for move_source in (False, True): + fov = physical_fov(.7) + displacement = -2*fov.header['CDELT2']*3600 + source_y = .4 + displacement if move_source else .4 + source = Source(x=[0], y=[source_y], ref=[0], weight=[1.], spectra=[SourceSpectrum(ConstFlux1D, amplitude=1*PHOTLAM)]) + effects = [psf] + if intervening_throughput: + effects.append(TERCurve(array_dict={'wavelength': [.4, .7], 'transmission': [.2, .8]}, wavelength_unit='um')) + if not move_source: + effects.append(Displacement(0., displacement, method='roll')) + train = OpticalTrain() + train.cmds = fov.cmds + train.fov_manager = SimpleNamespace(fovs=[fov]) + train.optics_manager = SimpleNamespace(source_effects=[], fov_effects=effects, image_plane_effects=[]) + train.image_planes = [SimpleNamespace(add=lambda hdu, **kwargs: images.append(hdu.data.copy()))] + train.observe(source, update=False) + train.observe(source, update=False) + for image in images[1:]: + np.testing.assert_allclose(images[0], image, rtol=2e-6, atol=1e-12) From aef3a53debb21d6edd92e20e3702e54f5fb57d05 Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Mon, 19 Oct 2026 12:36:58 -0700 Subject: [PATCH 127/130] perf(atmosphere): skip zero shifts on the final illumination grid --- scopesim/effects/atmo_dispersion.py | 8 ++++++- scopesim/optics/fov.py | 4 +++- .../test_adc_slit_illumination.py | 24 +++++++++++++++++++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/scopesim/effects/atmo_dispersion.py b/scopesim/effects/atmo_dispersion.py index 7ceeea515..3fd85fd18 100644 --- a/scopesim/effects/atmo_dispersion.py +++ b/scopesim/effects/atmo_dispersion.py @@ -87,13 +87,17 @@ def get_shifts(self, obj): wave = _cube_wavelengths(obj) return dx(wave), dy(wave) - def _padded_header(self, header, shifts): + def _padded_header(self, header, shifts, rounded=False): """Spatial support needed before applying this displacement.""" header = header.copy() # Cubic interpolation needs neighboring samples as well as the shift. guard = 0 if self.meta['method'] == 'roll' else 4 for axis, displacement in enumerate(shifts, start=1): cells = (displacement/(abs(header[f'CDELT{axis}'])*u.deg)).decompose().value + if rounded: + cells = np.round(cells, 1) + if self.meta['method'] == 'roll': + cells = np.round(np.asarray(cells, dtype=np.float32)) padding = int(np.ceil(np.max(np.abs(cells)))) + guard if np.any(cells) else 0 header[f'NAXIS{axis}'] += 2*padding header[f'CRPIX{axis}'] += padding @@ -184,6 +188,8 @@ def map_coords_shift(data, dx, dy, chunk_w: int=64, order: int=3): # along wavelength or filtering the same profile thousands of times. shifted = data for axis, displacement, length in ((2, dx, nx), (1, dy, ny)): + if not np.any(displacement): + continue out = np.empty(data.shape, dtype=float) for offset in np.unique(displacement): operator = _spatial_shift_operator(length, float(offset), order) diff --git a/scopesim/optics/fov.py b/scopesim/optics/fov.py index c9765abd9..3f56c4e28 100644 --- a/scopesim/optics/fov.py +++ b/scopesim/optics/fov.py @@ -1048,7 +1048,9 @@ def make_hdu(self, spatial_psf=None, spatial_shifts=()): padded = copy(self) padded.header = self.header.copy() for effect, displacement in shifts: - padded.header = effect._padded_header(padded.header, displacement) + # This is the final PSF grid. Earlier source selection keeps + # conservative support because its coarser grid can still change. + padded.header = effect._padded_header(padded.header, displacement, rounded=True) padded.hdu = padded.make_hdu(spatial_psf=spatial_psf) for effect, displacement in shifts: effect.apply_to(padded, _shifts=displacement) diff --git a/scopesim/tests/tests_optics/test_adc_slit_illumination.py b/scopesim/tests/tests_optics/test_adc_slit_illumination.py index 20df6706a..4dae1e4d3 100644 --- a/scopesim/tests/tests_optics/test_adc_slit_illumination.py +++ b/scopesim/tests/tests_optics/test_adc_slit_illumination.py @@ -164,3 +164,27 @@ def test_observe_applies_shift_before_slit_through_wavelength_throughput(interve train.observe(source, update=False) for image in images[1:]: np.testing.assert_allclose(images[0], image, rtol=2e-6, atol=1e-12) + + +@pytest.mark.parametrize('method', ['roll', 'map_coords']) +@pytest.mark.parametrize('fraction', [1e-12, .04, .54, .56]) +def test_final_padding_uses_applied_displacement_but_source_support_is_conservative(method, fraction): + fov = physical_fov(.7) + cell_y = fov.header['CDELT2']*3600 + shift = Displacement(1e-14, fraction*cell_y, method=method) + shifts = shift.get_shifts(fov) + source_support = shift._padded_header(fov.header, shifts) + final_support = shift._padded_header(fov.header, shifts, rounded=True) + assert source_support['NAXIS2'] > fov.header['NAXIS2'] + assert final_support['NAXIS1'] == fov.header['NAXIS1'] + applied = np.round(fraction, 1) + if method == 'roll': + applied = np.round(np.float32(applied)) + padding = int(np.ceil(abs(applied))) + (4 if method == 'map_coords' else 0) if applied else 0 + assert final_support['NAXIS2'] == fov.header['NAXIS2'] + 2*padding + if applied == 0: + psf = MoffatPSF(alpha=4.765, fwhm=.7) + source = Source(x=[.1], y=[.2], ref=[0], weight=[1.], spectra=[SourceSpectrum(ConstFlux1D, amplitude=1*PHOTLAM)]) + actual = render(fov, source, psf, [shift]) + expected = render(physical_fov(.7), source, psf, []) + np.testing.assert_array_equal(actual, expected) From e2cefad8838f9db7ccffd9b131169c48b79f04a1 Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Tue, 20 Oct 2026 12:55:39 -0700 Subject: [PATCH 128/130] feat: support configured detector operating points and avalanche noise --- scopesim/effects/electronic/__init__.py | 4 +- scopesim/effects/electronic/exposure.py | 30 +++++++ scopesim/effects/electronic/noise.py | 46 ++++++++-- scopesim/effects/illumination.py | 1 + scopesim/effects/selector_wheel.py | 6 +- scopesim/effects/spectral_efficiency.py | 4 +- scopesim/effects/spectral_trace_list.py | 4 +- .../test_detector_operating_points.py | 85 +++++++++++++++++++ 8 files changed, 168 insertions(+), 12 deletions(-) create mode 100644 scopesim/tests/tests_effects/test_detector_operating_points.py diff --git a/scopesim/effects/electronic/__init__.py b/scopesim/effects/electronic/__init__.py index e086896cf..991d1aa33 100644 --- a/scopesim/effects/electronic/__init__.py +++ b/scopesim/effects/electronic/__init__.py @@ -52,8 +52,8 @@ def apply_to(self, obj, **kwargs): from .electrons import LinearityCurve, ADConversion, InterPixelCapacitance from .noise import (Bias, PoorMansHxRGReadoutNoise, BasicReadoutNoise, - PixelResponseNonUniformity, ShotNoise, DarkCurrent, ReadoutNoiseFromTable) -from .exposure import AutoExposure, ExposureIntegration, ExposureOutput + PixelResponseNonUniformity, ShotNoise, AvalancheShotNoise, DarkCurrent, ReadoutNoiseFromTable) +from .exposure import AutoExposure, ExposureIntegration, ExposureOutput, ExposureLimit from .pixels import ReferencePixelBorder, BinnedImage, UnequalBinnedImage, OnChipBinning from .dmps import DetectorModePropertiesSetter diff --git a/scopesim/effects/electronic/exposure.py b/scopesim/effects/electronic/exposure.py index f11237375..328eefb90 100644 --- a/scopesim/effects/electronic/exposure.py +++ b/scopesim/effects/electronic/exposure.py @@ -2,6 +2,7 @@ """Exposure actions.""" from typing import ClassVar +from warnings import warn import numpy as np @@ -293,3 +294,32 @@ def apply_to(self, obj, **kwargs): obj.data = obj.data * dit * ndit return obj + + + +class ExposureLimit(Effect): + """Warn about expected charge per integration without changing the exposure. + + Apply after integration, dark current and on-chip binning, before noise. + The configured usable well refers to a charge packet at this readout stage. + Software binning is deliberately later and does not change this limit. + """ + + required_keys = {"usable_well", "ndit"} + z_order = (819,) + + def __init__(self, **kwargs): + super().__init__(**kwargs) + check_keys(self.meta, self.required_keys, action="error") + + def apply_to(self, obj, **kwargs): + if isinstance(obj, Detector): + well = from_currsys(self.meta["usable_well"], self.cmds) + ndit = from_currsys(self.meta["ndit"], self.cmds) + peak = obj.data.max() / ndit + obj.header["PEAKDIT"] = (peak, "Expected peak charge per integration [electron]") + obj.meta["WELLDEP"] = well + obj.header["WELLDEP"] = (well, "Usable charge per integration [input electron]") + if peak > well: + warn(f"Detector {obj.det_id}: peak {peak:.1f} e per integration exceeds usable well {well:.1f} e; DIT/NDIT are unchanged.", UserWarning, stacklevel=2) + return obj diff --git a/scopesim/effects/electronic/noise.py b/scopesim/effects/electronic/noise.py index 8c6aef4fa..e9d9198a0 100644 --- a/scopesim/effects/electronic/noise.py +++ b/scopesim/effects/electronic/noise.py @@ -141,10 +141,9 @@ def create_rng(self, det_id: int | None = None) -> np.random.Generator: New Generator instance. """ - if det_id is not None: - seed = [det_id, *self.random_seed] - else: - seed = self.random_seed + seed = self.random_seed + if det_id is not None and seed is not None: + seed = [det_id, *seed] return np.random.default_rng(seed) def plot(self, det): @@ -194,7 +193,7 @@ def _apply_to_det(self, det: Detector) -> None: logger.debug("Apply %s to %s", self.display_name, det) ndit = from_currsys(self.meta["ndit"], self.cmds) - det.data = self(det.data, ndit) + det.data = self(det.data, ndit, det_id=det.det_id) class PoorMansHxRGReadoutNoise(BasicReadoutNoise): @@ -427,6 +426,9 @@ class ShotNoise(RandomEffect): """ z_order: ClassVar[tuple[int, ...]] = (820,) + def _apply_to_det(self, det): + det.data = self(det.data, det_id=det.det_id) + def __call__(self, data: ArrayLike, det_id: int | None = None) -> NDArray: rng = self.create_rng(det_id) @@ -449,3 +451,37 @@ def __call__(self, data: ArrayLike, det_id: int | None = None) -> NDArray: ) return data + + + +class AvalancheShotNoise(ShotNoise): + """Input-referred photon multiplication noise, with independent dark charge. + + Conditional on N detected photons, a gamma variate of mean N and variance + N(F-1) gives total photon variance F*mu. F is a variance multiplier, not + an RMS factor. The adopted input-referred dark term remains Poisson. + Apply after ExposureIntegration and DarkCurrent, before read noise. + """ + + required_keys = {"excess_noise_factor", "dark_current", "dit", "ndit"} + + @property + def excess_noise_factor(self): + return from_currsys(self.meta["excess_noise_factor"], self.cmds) + + def __call__(self, data, dark_charge=0., det_id=None): + factor = self.excess_noise_factor + if factor < 1: + raise ValueError("The avalanche variance multiplier must be at least one.") + if factor == 1: + return super().__call__(data, det_id=det_id) + rng = self.create_rng(det_id) + photons = rng.poisson(data - dark_charge) + signal = rng.gamma(photons / (factor - 1), factor - 1) + return signal + rng.poisson(dark_charge, size=data.shape) + + def _apply_to_det(self, det): + dark = from_currsys(self.meta["dark_current"], self.cmds) + dit = from_currsys(self.meta["dit"], self.cmds) + ndit = from_currsys(self.meta["ndit"], self.cmds) + det.data = self(det.data, dark * dit * ndit, det_id=det.det_id) diff --git a/scopesim/effects/illumination.py b/scopesim/effects/illumination.py index 65b5b8181..5179aa7db 100644 --- a/scopesim/effects/illumination.py +++ b/scopesim/effects/illumination.py @@ -212,6 +212,7 @@ def _effect_from_spec(spec, cmds=None): def _detector_qe_from_spec(spec, filename=None, cmds=None): + spec = from_currsys(spec, cmds) if spec is not None: if isinstance(spec, str): return SpectralQuantumEfficiency(filename=spec, cmds=cmds) diff --git a/scopesim/effects/selector_wheel.py b/scopesim/effects/selector_wheel.py index 7ee03f242..333635bb1 100644 --- a/scopesim/effects/selector_wheel.py +++ b/scopesim/effects/selector_wheel.py @@ -13,7 +13,7 @@ import importlib from numbers import Integral -from ..utils import (check_keys, get_logger, real_colname) +from ..utils import from_currsys, check_keys, get_logger, real_colname from .effects import Effect from ..optics.fov_volume_list import FovVolumeList from ..optics.fov import FieldOfView @@ -62,8 +62,10 @@ def __init__(self, **kwargs): self.wheel_effects = {} for wheel_entry in self.meta["wheel"]: selector_value = wheel_entry["selector_value"] # can be single value or list of values - effect_class_name = wheel_entry["effect_class"] + effect_class_name = from_currsys(wheel_entry["effect_class"], self.cmds) effect_kwargs = wheel_entry.get("effect_kwargs", {}) + if isinstance(effect_kwargs, str): + effect_kwargs = from_currsys(effect_kwargs, self.cmds) # Dynamically import the effect class effect_module = importlib.import_module("scopesim.effects") diff --git a/scopesim/effects/spectral_efficiency.py b/scopesim/effects/spectral_efficiency.py index 62a34bb6d..b551b72fc 100644 --- a/scopesim/effects/spectral_efficiency.py +++ b/scopesim/effects/spectral_efficiency.py @@ -11,7 +11,7 @@ from .effects import Effect from .ter_curves import TERCurve -from ..utils import figure_factory, get_logger, check_keys +from ..utils import figure_factory, get_logger, check_keys, from_currsys from .data_container import DataContainer from ..optics import echelle @@ -144,7 +144,7 @@ def __init__(self, **kwargs): self.efficiency_generator = self._generate_efficiency_curve_func() def _generate_efficiency_curve_func(self) -> Callable: - trace_params = self.table + trace_params = from_currsys(self.table, self.cmds) spectrographs = {} for row in trace_params: prefix = row["prefix"] # note trance ids are assumed to be prefix_{order} diff --git a/scopesim/effects/spectral_trace_list.py b/scopesim/effects/spectral_trace_list.py index 6f3062628..b10cbb49d 100644 --- a/scopesim/effects/spectral_trace_list.py +++ b/scopesim/effects/spectral_trace_list.py @@ -704,10 +704,12 @@ def __init__(self, cmds=None, **kwargs): trace_param_filename = kwargs.pop("filename") self.physical_slit_dispersion = kwargs.pop("physical_slit_dispersion", False) + self.slit_samples = kwargs.pop("slit_samples", 3) save_generated_hdulist = kwargs.pop("save_generated_hdulist", False) generated_hdulist_filename = kwargs.pop( "generated_hdulist_filename", "analytical_echelle_traces.fits") trace_params = DataContainer(filename=trace_param_filename) + trace_params.table = from_currsys(trace_params.table, self.cmds) hdulist = self._generate_trace_hdulist(trace_params) if save_generated_hdulist: output_path = Path(generated_hdulist_filename).expanduser() @@ -778,7 +780,7 @@ def _generate_trace_hdulist(self, trace_params): ) slit_edge = (row['slitlength'] / 2) * u.Unit(trace_params.meta["slitlength_unit"]) - slit_pos = np.linspace(-slit_edge, slit_edge, num=3) + slit_pos = np.linspace(-slit_edge, slit_edge, num=self.slit_samples) if "plate_scale" in trace_params.table.colnames: plate_scale = row["plate_scale"] * u.Unit( trace_params.meta["plate_scale_unit"]) diff --git a/scopesim/tests/tests_effects/test_detector_operating_points.py b/scopesim/tests/tests_effects/test_detector_operating_points.py new file mode 100644 index 000000000..1763f1d26 --- /dev/null +++ b/scopesim/tests/tests_effects/test_detector_operating_points.py @@ -0,0 +1,85 @@ +import warnings + +import numpy as np +import pytest + +from scopesim import UserCommands +from scopesim.detector import Detector +from scopesim.effects.electronic import AvalancheShotNoise, DarkCurrent, ExposureIntegration, ExposureLimit, OnChipBinning, BasicReadoutNoise +from scopesim.optics.image_plane_utils import header_from_list_of_xy + + +@pytest.mark.parametrize('factor', [1., 1.2]) +@pytest.mark.parametrize('photons,dark', [(0., 4.), (.2, .01), (50., 3.)]) +def test_avalanche_preserves_physical_mean_and_only_amplifies_photon_variance(factor, photons, dark): + cmds = UserCommands(properties={'!SIM.random.seed': 7143}) + effect = AvalancheShotNoise(excess_noise_factor=factor, dark_current=dark, dit=1., ndit=1, cmds=cmds) + data = effect(np.full(400000, photons + dark), dark_charge=dark, det_id=2) + mean, variance = photons + dark, factor * photons + dark + assert data.mean() == pytest.approx(mean, abs=5*np.sqrt(variance/data.size)) + assert data.var() == pytest.approx(variance, rel=.025) + + +def test_independent_integrations_and_read_noise_after_avalanche(): + cmds = UserCommands(properties={'!SIM.random.seed': 736}) + count, photons, dark, ron, factor = 12, 7., .4, .5, 1.2 + effect = AvalancheShotNoise(excess_noise_factor=factor, dark_current=dark, dit=1., ndit=count, cmds=cmds) + data = effect(np.full(400000, count*(photons+dark)), dark_charge=count*dark, det_id=3) + data = BasicReadoutNoise(noise_std=ron, ndit=count, cmds=cmds)(data, ndit=count, det_id=3) + assert data.mean() == pytest.approx(count*(photons+dark), abs=.1) + assert data.var() == pytest.approx(count*(factor*photons + dark + ron**2), rel=.02) + cmds['!OBS.roid'] = 2 + other = effect(np.full(400000, count*(photons+dark)), dark_charge=count*dark, det_id=3) + assert not np.array_equal(data, other) + + +def test_exposure_limit_warns_after_charge_binning_without_changing_counts_or_commands(): + cmds = UserCommands(properties={'!OBS.dit': 10., '!OBS.ndit': 4}) + header = header_from_list_of_xy([-2, 2], [-2, 2], 1., 'D') + header['ID'] = 0 + detector = Detector(header, cmds=cmds) + detector.data[:] = 400. + limit = ExposureLimit(usable_well=15000., ndit='!OBS.ndit', cmds=cmds) + ExposureIntegration(dit='!OBS.dit', ndit='!OBS.ndit', cmds=cmds).apply_to(detector) + DarkCurrent(value=0., dit='!OBS.dit', ndit='!OBS.ndit', cmds=cmds).apply_to(detector) + with warnings.catch_warnings(record=True) as caught: + limit.apply_to(detector) + assert not caught # Summed 16000 e is only 4000 e per integration. + OnChipBinning(binx=2, biny=2).apply_to(detector) + before = detector.data.copy() + with pytest.warns(UserWarning, match='peak 16000.0 e.*DIT/NDIT are unchanged'): + limit.apply_to(detector) + np.testing.assert_array_equal(detector.data, before) + assert detector.header['PEAKDIT'] == 16000 + assert cmds['!OBS.dit'] == 10 and cmds['!OBS.ndit'] == 4 + + + +def test_avalanche_supports_unseeded_readouts(): + cmds = UserCommands(properties={'!SIM.random.seed': None}) + effect = AvalancheShotNoise(excess_noise_factor=1.2, dark_current=0., dit=1., ndit=1, cmds=cmds) + first = effect(np.full(1000, 10.), det_id=3) + second = effect(np.full(1000, 10.), det_id=3) + assert not np.array_equal(first, second) + + +def test_detector_ids_produce_independent_read_noise(): + cmds = UserCommands(properties={'!SIM.random.seed': 7143}) + effect = BasicReadoutNoise(noise_std=2., ndit=1, cmds=cmds) + header = header_from_list_of_xy([-50, 50], [-50, 50], 1., 'D') + header['ID'] = 1 + first = Detector(header.copy(), cmds=cmds) + header['ID'] = 2 + second = Detector(header.copy(), cmds=cmds) + effect.apply_to(first) + effect.apply_to(second) + assert abs(np.corrcoef(first.data.ravel(), second.data.ravel())[0, 1]) < .05 + + +def test_selector_detector_properties_keep_exposure_references_live(): + from scopesim.effects import SelectorWheel + from scopesim.utils import from_currsys + cmds = UserCommands(properties={'!DET.noise_class': 'BasicReadoutNoise', '!DET.noise_kwargs': {'noise_std': 2., 'ndit': '!OBS.ndit'}, '!OBS.ndit': 1}) + effect = SelectorWheel(selector_key='detector_id', wheel=[{'selector_value': 0, 'effect_class': '!DET.noise_class', 'effect_kwargs': '!DET.noise_kwargs'}], cmds=cmds) + cmds['!OBS.ndit'] = 7 + assert from_currsys(effect.get_effect(0).meta['ndit'], cmds) == 7 From 2780412e5c6a9d7f3c7664ca8ef76f5c9278c6d5 Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Tue, 20 Oct 2026 23:38:24 -0700 Subject: [PATCH 129/130] Preserve exposure metadata when converting MOSAIC detector outputs --- scopesim/effects/mosaic_trace_list.py | 4 ++++ .../tests_effects/test_mosaic_trace_list.py | 23 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/scopesim/effects/mosaic_trace_list.py b/scopesim/effects/mosaic_trace_list.py index 34e6bb66e..c74c9d214 100644 --- a/scopesim/effects/mosaic_trace_list.py +++ b/scopesim/effects/mosaic_trace_list.py @@ -255,6 +255,9 @@ def apply_to(self, det, **kwargs): image = det.data detwcs = WCS(det.header, key="D") + # ExposureIntegration has already recorded the actual accumulated time. + # New table HDUs need those cards, but not the detector-image WCS. + exposure_header = fits.Header(card for card in det.header.cards if card.keyword in ("DIT", "NDIT", "EXPTIME")) output_format = from_currsys(self.meta["format"], self.cmds) @@ -309,6 +312,7 @@ def apply_to(self, det, **kwargs): else: logger.error("Unknown output format: %s", output_format) + det.header.update(exposure_header) return det diff --git a/scopesim/tests/tests_effects/test_mosaic_trace_list.py b/scopesim/tests/tests_effects/test_mosaic_trace_list.py index 39433af43..f5bef8e8d 100644 --- a/scopesim/tests/tests_effects/test_mosaic_trace_list.py +++ b/scopesim/tests/tests_effects/test_mosaic_trace_list.py @@ -12,6 +12,7 @@ MosaicSpectralTraceList, MosaicSpectralTrace, MosaicOutputFormat) +from scopesim.effects.electronic.exposure import ExposureIntegration from scopesim.optics.image_plane_utils import header_from_list_of_xy from scopesim.detector import Detector @@ -201,3 +202,25 @@ def test_image_returns_detector(self, apply_image): def test_image_detector_has_image(self, apply_image): assert isinstance(apply_image._hdu, fits.ImageHDU) + + +@pytest.mark.usefixtures("patch_mock_path_mosaic") +@pytest.mark.parametrize("output_format", ["collapse1d", "table", "image"]) +def test_output_format_preserves_accumulated_counts_and_exposure(output_format): + header = header_from_list_of_xy([-25, 25], [-2048, 2048], 1, "D") + detector = Detector(header) + rows = [2023, 2031, 2039, 2047, 2055, 2063, 2071] + detector.data[rows, :] = 1 + rate_sum = detector.data.sum() + ExposureIntegration(dit=17., ndit=3).apply_to(detector) + output = MosaicOutputFormat(filename="TRACE_MOS-LR-J.fits", format=output_format).apply_to(detector) + assert output.header["DIT"] == 17. + assert output.header["NDIT"] == 3 + assert output.header["EXPTIME"] == 51. + if output_format == "image": + counts = output.data.sum() + assert "CTYPE1D" in output.header + else: + counts = output.data["spectrum"].sum() + assert "CTYPE1D" not in output.header + assert counts == pytest.approx(rate_sum * 51.) From b9ba36e6961368e6bdfcec263e3aae83e3686f9f Mon Sep 17 00:00:00 2001 From: Jeb Bailey Date: Wed, 21 Oct 2026 20:11:57 -0700 Subject: [PATCH 130/130] Keep ADC slit regression fixture explicit about spectroscopic mode --- scopesim/tests/tests_optics/test_adc_slit_illumination.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scopesim/tests/tests_optics/test_adc_slit_illumination.py b/scopesim/tests/tests_optics/test_adc_slit_illumination.py index 4dae1e4d3..3df011292 100644 --- a/scopesim/tests/tests_optics/test_adc_slit_illumination.py +++ b/scopesim/tests/tests_optics/test_adc_slit_illumination.py @@ -157,7 +157,7 @@ def test_observe_applies_shift_before_slit_through_wavelength_throughput(interve effects.append(Displacement(0., displacement, method='roll')) train = OpticalTrain() train.cmds = fov.cmds - train.fov_manager = SimpleNamespace(fovs=[fov]) + train.fov_manager = SimpleNamespace(fovs=[fov], is_spectroscope=True) train.optics_manager = SimpleNamespace(source_effects=[], fov_effects=effects, image_plane_effects=[]) train.image_planes = [SimpleNamespace(add=lambda hdu, **kwargs: images.append(hdu.data.copy()))] train.observe(source, update=False)