From aa4c73b6865d3df3c47885a2779c3560a57b976b Mon Sep 17 00:00:00 2001 From: oczoske Date: Tue, 5 May 2026 13:12:16 +0200 Subject: [PATCH 01/87] Remove offset hack --- scopesim/effects/spectral_trace_list.py | 25 +++++++++++-------- scopesim/effects/spectral_trace_list_utils.py | 11 +++++--- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/scopesim/effects/spectral_trace_list.py b/scopesim/effects/spectral_trace_list.py index d455fc088..0fdeb3a78 100644 --- a/scopesim/effects/spectral_trace_list.py +++ b/scopesim/effects/spectral_trace_list.py @@ -245,16 +245,21 @@ def apply_to(self, obj, **kwargs): obj.cube = obj.make_hdu() # Check whether an offset slit is used. If so, recompute spectral traces. - offset_x = obj.cube.header["CRVAL1D"] - offset_y = obj.cube.header["CRVAL2D"] - if (offset_x != self.meta["offset_x"] or - offset_y != self.meta["offset_y"]): - logger.debug("Recomputing spectral traces for offset (%.1g, %.1g)", - offset_x, offset_y) - self.meta["offset_x"] = offset_x - self.meta["offset_y"] = offset_y - self.make_spectral_traces() - self.update_meta() + # TODO: this needs to be removed. If transforms are available that + # can be applied to the actual slit position, they ought be used + # without modification. If they are not available but need to be + # hacked, then that hack needs to be defined via the yaml and only + # be used in a particular case, not in general. + #offset_x = obj.cube.header["CRVAL1D"] + #offset_y = obj.cube.header["CRVAL2D"] + #if (offset_x != self.meta["offset_x"] or + # offset_y != self.meta["offset_y"]): + # logger.debug("Recomputing spectral traces for offset (%.1g, %.1g)", + # offset_x, offset_y) + # self.meta["offset_x"] = offset_x + # self.meta["offset_y"] = offset_y + # self.make_spectral_traces() + # self.update_meta() spt = self.spectral_traces[obj.trace_id] obj.hdu = spt.map_spectra_to_focal_plane(obj) diff --git a/scopesim/effects/spectral_trace_list_utils.py b/scopesim/effects/spectral_trace_list_utils.py index 7541f9cbf..343718894 100644 --- a/scopesim/effects/spectral_trace_list_utils.py +++ b/scopesim/effects/spectral_trace_list_utils.py @@ -122,17 +122,20 @@ def compute_interpolation_functions(self): `xi` (spatial coordinate along the slit, in arcsec) and `lam` (wavelength, in um). - The interpolation functions include a shift in the focal-plane - coordinates, determined from the CRVAL of the source FOV. + #The interpolation functions include a shift in the focal-plane + #coordinates, determined from the CRVAL of the source FOV. """ - x_arr = self.table[self.meta["x_colname"]] + self.meta["offset_x"] - y_arr = self.table[self.meta["y_colname"]] + self.meta["offset_y"] + x_arr = self.table[self.meta["x_colname"]] + y_arr = self.table[self.meta["y_colname"]] + xi_arr = self.table[self.meta["s_colname"]] lam_arr = self.table[self.meta["wave_colname"]] self.wave_min = quantify(np.min(lam_arr), u.um).value self.wave_max = quantify(np.max(lam_arr), u.um).value + # TODO There should be an option to include pre- and + # posttransforms. But how should they be defined? self.xy2xi = Transform2D.fit(x_arr, y_arr, xi_arr) self.xy2lam = Transform2D.fit(x_arr, y_arr, lam_arr) self.xilam2x = Transform2D.fit(xi_arr, lam_arr, x_arr) From 43dda59f9255efd397bf6f001c889593d11ea1e5 Mon Sep 17 00:00:00 2001 From: oczoske Date: Thu, 7 May 2026 16:15:03 +0200 Subject: [PATCH 02/87] Different approach to offset slits --- scopesim/effects/apertures.py | 8 ++++++++ scopesim/effects/spectral_trace_list.py | 19 ++++++++++++++++--- scopesim/effects/spectral_trace_list_utils.py | 8 ++++++-- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/scopesim/effects/apertures.py b/scopesim/effects/apertures.py index fd6984a5b..d704265bb 100644 --- a/scopesim/effects/apertures.py +++ b/scopesim/effects/apertures.py @@ -129,6 +129,14 @@ def apply_to(self, obj, **kwargs): vol["meta"]["xi_min"] = min(x) * u.arcsec vol["meta"]["xi_max"] = max(x) * u.arcsec + if "offset_type" in self.meta: + print("Yes, got it: ", self.meta["offset_type"]) + vol["meta"]["offset_type"] = self.meta["offset_type"] + vol["meta"]["offset_x"] = self.meta["offset_x"] + vol["meta"]["offset_y"] = self.meta["offset_y"] + else: + vol["meta"]["offset_type"] = "none" + print("What is offset_type?") return obj # Outdated. Remove when removing all old FOVManager code from effects diff --git a/scopesim/effects/spectral_trace_list.py b/scopesim/effects/spectral_trace_list.py index 0fdeb3a78..3ec77f55e 100644 --- a/scopesim/effects/spectral_trace_list.py +++ b/scopesim/effects/spectral_trace_list.py @@ -16,7 +16,8 @@ from .effects import Effect from .ter_curves import FilterCurve -from .spectral_trace_list_utils import SpectralTrace, make_image_interpolations +from .spectral_trace_list_utils import (SpectralTrace, make_image_interpolations, + det_offset) from ..optics.image_plane_utils import header_from_list_of_xy from ..optics.fov import FieldOfView from ..optics.fov_volume_list import FovVolumeList @@ -125,8 +126,6 @@ def __init__(self, **kwargs): "x_colname": "x", "y_colname": "y", "s_colname": "s", - "offset_x": 0, # [mm] in detector plane - "offset_y": 0, # [mm] in detector plane "wave_colname": "wavelength", "center_on_wave_mid": False, "dwave": 0.002, # [um] for finding the best fit dispersion @@ -223,6 +222,7 @@ def apply_to(self, obj, **kwargs): aperture_id=vol["aperture_id"]) for ex_vol in extracted_vols: + print("Extracted volume:", ex_vol["meta"]["offset_type"]) ex_vol["meta"].update(vol) ex_vol["meta"].pop("wave_min") ex_vol["meta"].pop("wave_max") @@ -262,8 +262,21 @@ def apply_to(self, obj, **kwargs): # self.update_meta() spt = self.spectral_traces[obj.trace_id] + remove_pre = False + if ("offset_type" in obj.meta and + obj.meta["offset_type"] == "detector"): + spt.xy2lam.pretransform_x = (det_offset, + {"offset": obj.meta["offset_x"]}) + spt.xy2lam.pretransform_y = (det_offset, + {"offset": obj.meta["offset_y"]}) + remove_pre = True + obj.hdu = spt.map_spectra_to_focal_plane(obj) + if remove_pre: + spt.xy2lam.pretransform_y = None + spt.xy2xi.pretransform_x = None + logger.debug("%s done", self.display_name) return obj diff --git a/scopesim/effects/spectral_trace_list_utils.py b/scopesim/effects/spectral_trace_list_utils.py index 343718894..1fc8e999f 100644 --- a/scopesim/effects/spectral_trace_list_utils.py +++ b/scopesim/effects/spectral_trace_list_utils.py @@ -49,8 +49,6 @@ class SpectralTrace: "x_colname": "x", "y_colname": "y", "s_colname": "s", - "offset_x": 0, - "offset_y": 0, "wave_colname": "wavelength", "dwave": 0.002, "aperture_id": 0, @@ -114,6 +112,7 @@ def fov_grid(self): return {"wave_min": wave_min, "wave_max": wave_max, "trace_id": self.trace_id, "aperture_id": aperture_id} + def compute_interpolation_functions(self): """ Compute various interpolation functions between slit and focal plane. @@ -134,6 +133,7 @@ def compute_interpolation_functions(self): self.wave_min = quantify(np.min(lam_arr), u.um).value self.wave_max = quantify(np.max(lam_arr), u.um).value + # TODO There should be an option to include pre- and # posttransforms. But how should they be defined? self.xy2xi = Transform2D.fit(x_arr, y_arr, xi_arr) @@ -1045,3 +1045,7 @@ def get_affine_parameters(coords): shears = (np.average(shears, axis=0) * rad2deg) - (90 + rotations) return rotations, shears + +def det_offset(x, offset): + """Apply offset to x""" + return x + offset From a5bd2b2dee5317d1a068f63bef4db88a9f5707d0 Mon Sep 17 00:00:00 2001 From: oczoske Date: Sat, 16 May 2026 12:36:06 +0200 Subject: [PATCH 03/87] Remove stray print that caused test failures --- scopesim/effects/spectral_trace_list.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scopesim/effects/spectral_trace_list.py b/scopesim/effects/spectral_trace_list.py index 3ec77f55e..a28973f15 100644 --- a/scopesim/effects/spectral_trace_list.py +++ b/scopesim/effects/spectral_trace_list.py @@ -222,7 +222,6 @@ def apply_to(self, obj, **kwargs): aperture_id=vol["aperture_id"]) for ex_vol in extracted_vols: - print("Extracted volume:", ex_vol["meta"]["offset_type"]) ex_vol["meta"].update(vol) ex_vol["meta"].pop("wave_min") ex_vol["meta"].pop("wave_max") From 418acadcd43270dd07a87cfcb4a6015c3d70e9cb Mon Sep 17 00:00:00 2001 From: oczoske Date: Sat, 16 May 2026 18:24:18 +0200 Subject: [PATCH 04/87] Start MicadoIFUSpectralTraceList --- scopesim/effects/__init__.py | 1 + scopesim/effects/micado_ifu_trace_list.py | 63 +++++++++++++++++++++++ scopesim/effects/spectral_trace_list.py | 2 + 3 files changed, 66 insertions(+) create mode 100644 scopesim/effects/micado_ifu_trace_list.py diff --git a/scopesim/effects/__init__.py b/scopesim/effects/__init__.py index da03ffc0f..170b41924 100644 --- a/scopesim/effects/__init__.py +++ b/scopesim/effects/__init__.py @@ -8,6 +8,7 @@ from .spectral_trace_list import * from .spectral_efficiency import * from .metis_lms_trace_list import * +from .micado_ifu_trace_list import * from .mosaic_trace_list import * from .surface_list import * from .ter_curves import * diff --git a/scopesim/effects/micado_ifu_trace_list.py b/scopesim/effects/micado_ifu_trace_list.py new file mode 100644 index 000000000..826399e1f --- /dev/null +++ b/scopesim/effects/micado_ifu_trace_list.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +"""SpectralTraceList and SpectralTrace for the MICADO IFU.""" + +import copy +import warnings + +from tqdm.auto import tqdm +import numpy as np +from scipy.interpolate import RectBivariateSpline + +from astropy.io import fits +from astropy.io import ascii as ioascii +from astropy.table import Table +from astropy.wcs import WCS +from astropy import units as u + +from ..utils import from_currsys, find_file, quantify, get_logger +from .spectral_trace_list import SpectralTraceList +from .spectral_trace_list_utils import SpectralTrace +from .spectral_trace_list_utils import Transform2D +from .spectral_trace_list_utils import make_image_interpolations +from .apertures import ApertureMask +from .ter_curves import TERCurve +from ..optics.fov import FieldOfView, FieldOfView3D +from ..optics.fov_volume_list import FovVolumeList + +logger = get_logger(__name__) + + +class MicadoIFUSpectralTraceList(SpectralTraceList): + """SpectralTraceList for the MICADO IFU.""" + + _class_params = { + "naxis1": 112, + "nslice": 32, + "slicewidth": 0.012, # arcsec + } + + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + # field of view of the instrument + + def make_spectral_traces(self): + """Make a spectral trace for each combination of order and aperture""" + self.ext_data = self._file[0].header["EDATA"] + self.ext_cat = self._file[0].header["ECAT"] + self.catalog = Table(self._file[self.ext_cat].data) + self.slicelist = self._file["Aperture List"].data + spec_traces = {} + for row in self.catalog: + if row["image_plane_id"] == -99: + continue + params = {col: row[col] for col in row.colnames} + params.update(self.meta) + hdu = self._file[row["extension_id"]] + for apid in self.slicelist["id"]: + specid = f"{row['description']}_{apid:02d}" + params["aperture_id"] = apid + spec_traces[specid] = SpectralTrace(hdu, **params) + + self.spectral_traces = spec_traces diff --git a/scopesim/effects/spectral_trace_list.py b/scopesim/effects/spectral_trace_list.py index a28973f15..48e5abefa 100644 --- a/scopesim/effects/spectral_trace_list.py +++ b/scopesim/effects/spectral_trace_list.py @@ -148,6 +148,8 @@ def make_spectral_traces(self): self.catalog = Table(self._file[self.ext_cat].data) spec_traces = {} for row in self.catalog: + if row["image_plane_id"] == -99: + continue params = {col: row[col] for col in row.colnames} params.update(self.meta) hdu = self._file[row["extension_id"]] From 31ce32c66b5ef4d46cb1332db56857d872e9c55e Mon Sep 17 00:00:00 2001 From: oczoske Date: Mon, 18 May 2026 14:23:47 +0200 Subject: [PATCH 05/87] Split apply_to into separate methods for setup and observe --- scopesim/effects/metis_lms_trace_list.py | 173 +++++++++++----------- scopesim/effects/spectral_trace_list.py | 175 +++++++++++++---------- 2 files changed, 187 insertions(+), 161 deletions(-) diff --git a/scopesim/effects/metis_lms_trace_list.py b/scopesim/effects/metis_lms_trace_list.py index bf8c0d88f..e6770a900 100644 --- a/scopesim/effects/metis_lms_trace_list.py +++ b/scopesim/effects/metis_lms_trace_list.py @@ -53,93 +53,98 @@ def __init__(self, **kwargs): self.slicelist["top"].max() - self.slicelist["bottom"].min()]) - def apply_to(self, obj, **kwargs): - """See parent docstring.""" - if isinstance(obj, FovVolumeList): - # Create a single volume that covers the aperture and - # the maximum wavelength range of LMS - volumes = [spectral_trace.fov_grid() - for spectral_trace in self.spectral_traces.values()] - - wave_min = min(vol["wave_min"] for vol in volumes) - wave_max = max(vol["wave_max"] for vol in volumes) - extracted_vols = obj.extract(axes=["wave"], - edges=([[wave_min, wave_max]])) - obj.volumes = extracted_vols - - if isinstance(obj, FieldOfView): - # Application to field of view - logger.debug("Executing %s, FoV", self.meta['name']) - if obj.hdu is not None and obj.hdu.header["NAXIS"] == 3: - obj.cube = obj.hdu - elif obj.hdu is None and obj.cube is None: - obj.cube = obj.make_hdu() - - fovcube = obj.cube.data - n_z, n_y, n_x = fovcube.shape - fovwcs = WCS(obj.cube.header) - # Make this linear to avoid jump at RA 0 deg - fovwcs.wcs.ctype = ["LINEAR", "LINEAR", fovwcs.wcs.ctype[2]] - fovwcs_spat = fovwcs.sub(2) - ny_slice = self.meta["slice_samples"] - - # Spatial pixel coordinates - xslice, yslice = np.meshgrid(np.arange(n_x), - np.arange(ny_slice)) - - fovimage = np.zeros((obj.detector_header["NAXIS2"], - obj.detector_header["NAXIS1"]), - dtype=np.float32) - - for sptid, spt in tqdm(self.spectral_traces.items(), - desc=" Spectral Traces", position=2): - ymin = spt.meta["fov"]["y_min"] - ymax = spt.meta["fov"]["y_max"] - - slicewcs = fovwcs.deepcopy() - - slicewcs.wcs.ctype = ["LINEAR", "LINEAR", - slicewcs.wcs.ctype[2]] - slicewcs.wcs.crpix[1] = (ny_slice + 1) / 2 - slicewcs.wcs.crval[1] = (ymin + ymax) / 2 / 3600 - slicewcs.wcs.cdelt[1] = (ymax - ymin) / ny_slice / 3600 - slicewcs_spat = slicewcs.sub(2) - - # World coordinates for the slice - xworld, yworld = slicewcs_spat.all_pix2world(xslice, - yslice, 0) - # FOV pixel coordinates for the slice - xfov, yfov = fovwcs_spat.all_world2pix(xworld, yworld, 0) - - slicecube = np.zeros((n_z, ny_slice, n_x)) - for islice in range(n_z): - ifov = RectBivariateSpline(np.arange(n_y), - np.arange(n_x), - fovcube[islice], kx=1, ky=1) - slicecube[islice] = ifov(yfov, xfov, grid=False) - - slicefov = FieldOfView3D(obj.header, - [obj.meta["wave_min"], - obj.meta["wave_max"]]) - slicefov.detector_header = obj.detector_header - slicefov.meta["xi_min"] = obj.meta["xi_min"] - slicefov.meta["xi_max"] = obj.meta["xi_max"] - slicefov.meta["trace_id"] = sptid - slicefov.cube = fits.ImageHDU(header=slicewcs.to_header(), - data=slicecube) - # slicefov.cube.writeto(f"slicefov_{sptid}.fits", overwrite=True) - slicefov.hdu = spt.map_spectra_to_focal_plane(slicefov) - if slicefov.hdu is not None: - sxmin = slicefov.hdu.header["XMIN"] - sxmax = slicefov.hdu.header["XMAX"] - symin = slicefov.hdu.header["YMIN"] - symax = slicefov.hdu.header["YMAX"] - fovimage[symin:symax, sxmin:sxmax] += slicefov.hdu.data - - obj.hdu = fits.ImageHDU(data=fovimage, header=obj.detector_header) + # Overrides for the two apply_to() methods + def apply_to_fovvolumelist(self, obj): + """Setup + Create a single volume that covers the aperture and + the maximum wavelength range of LMS. + """ + logger.debug("Executing %s: Setup", self.meta['name']) + volumes = [spectral_trace.fov_grid() + for spectral_trace in self.spectral_traces.values()] + + wave_min = min(vol["wave_min"] for vol in volumes) + wave_max = max(vol["wave_max"] for vol in volumes) + extracted_vols = obj.extract(axes=["wave"], + edges=([[wave_min, wave_max]])) + obj.volumes = extracted_vols + return obj + + def apply_to_fov(self, obj): + """Application to field of view""" + + logger.debug("Executing %s, FoV", self.meta['name']) + if obj.hdu is not None and obj.hdu.header["NAXIS"] == 3: + obj.cube = obj.hdu + elif obj.hdu is None and obj.cube is None: + obj.cube = obj.make_hdu() + + fovcube = obj.cube.data + n_z, n_y, n_x = fovcube.shape + fovwcs = WCS(obj.cube.header) + # Make this linear to avoid jump at RA 0 deg + fovwcs.wcs.ctype = ["LINEAR", "LINEAR", fovwcs.wcs.ctype[2]] + fovwcs_spat = fovwcs.sub(2) + ny_slice = self.meta["slice_samples"] + + # Spatial pixel coordinates + xslice, yslice = np.meshgrid(np.arange(n_x), + np.arange(ny_slice)) + + fovimage = np.zeros((obj.detector_header["NAXIS2"], + obj.detector_header["NAXIS1"]), + dtype=np.float32) + + for sptid, spt in tqdm(self.spectral_traces.items(), + desc=" Spectral Traces", position=2): + ymin = spt.meta["fov"]["y_min"] + ymax = spt.meta["fov"]["y_max"] + + slicewcs = fovwcs.deepcopy() + + slicewcs.wcs.ctype = ["LINEAR", "LINEAR", + slicewcs.wcs.ctype[2]] + slicewcs.wcs.crpix[1] = (ny_slice + 1) / 2 + slicewcs.wcs.crval[1] = (ymin + ymax) / 2 / 3600 + slicewcs.wcs.cdelt[1] = (ymax - ymin) / ny_slice / 3600 + slicewcs_spat = slicewcs.sub(2) + + # World coordinates for the slice + xworld, yworld = slicewcs_spat.all_pix2world(xslice, + yslice, 0) + # FOV pixel coordinates for the slice + xfov, yfov = fovwcs_spat.all_world2pix(xworld, yworld, 0) + + slicecube = np.zeros((n_z, ny_slice, n_x)) + for islice in range(n_z): + ifov = RectBivariateSpline(np.arange(n_y), + np.arange(n_x), + fovcube[islice], kx=1, ky=1) + slicecube[islice] = ifov(yfov, xfov, grid=False) + + slicefov = FieldOfView3D(obj.header, + [obj.meta["wave_min"], + obj.meta["wave_max"]]) + slicefov.detector_header = obj.detector_header + slicefov.meta["xi_min"] = obj.meta["xi_min"] + slicefov.meta["xi_max"] = obj.meta["xi_max"] + slicefov.meta["trace_id"] = sptid + slicefov.cube = fits.ImageHDU(header=slicewcs.to_header(), + data=slicecube) + # slicefov.cube.writeto(f"slicefov_{sptid}.fits", overwrite=True) + slicefov.hdu = spt.map_spectra_to_focal_plane(slicefov) + if slicefov.hdu is not None: + sxmin = slicefov.hdu.header["XMIN"] + sxmax = slicefov.hdu.header["XMAX"] + symin = slicefov.hdu.header["YMIN"] + symax = slicefov.hdu.header["YMAX"] + fovimage[symin:symax, sxmin:sxmax] += slicefov.hdu.data + + obj.hdu = fits.ImageHDU(data=fovimage, header=obj.detector_header) return obj + def make_spectral_traces(self): """Compute the transformations by interpolation.""" # nslice = len(self._file["Aperture List"].data) diff --git a/scopesim/effects/spectral_trace_list.py b/scopesim/effects/spectral_trace_list.py index 48e5abefa..c4d3fc925 100644 --- a/scopesim/effects/spectral_trace_list.py +++ b/scopesim/effects/spectral_trace_list.py @@ -184,6 +184,7 @@ def update_meta(self): self.meta["y_min"] = min(ylim) self.meta["y_max"] = max(ylim) + def apply_to(self, obj, **kwargs): """ Interface between ``FieldOfView`` and ``SpectralTraceList``. @@ -198,89 +199,109 @@ def apply_to(self, obj, **kwargs): list, identified by meta["trace_id"]. """ if isinstance(obj, FovVolumeList): - logger.debug("%s applied to %s", self.display_name, - obj.__class__.__name__) - # Setup of FieldOfView object - # volumes = [spectral_trace.fov_grid() - # for spectral_trace in self.spectral_traces.values()] - - new_vols_list = [] - - # for vol in volumes: - for spt in self.spectral_traces.values(): - vol = spt.fov_grid() - wave_edges = [vol["wave_min"], vol["wave_max"]] - if "x_min" in vol: - x_edges = [vol["x_min"], vol["x_max"]] - y_edges = [vol["y_min"], vol["y_max"]] - extracted_vols = obj.extract( - axes=["wave", "x", "y"], - edges=(wave_edges, x_edges, y_edges), - aperture_id=vol["aperture_id"]) - else: - extracted_vols = obj.extract( - axes=["wave"], - edges=(wave_edges, ), - aperture_id=vol["aperture_id"]) - - for ex_vol in extracted_vols: - ex_vol["meta"].update(vol) - ex_vol["meta"].pop("wave_min") - ex_vol["meta"].pop("wave_max") - new_vols_list.extend(extracted_vols) - - obj.volumes = new_vols_list + obj = self.apply_to_fovvolumelist(obj) if isinstance(obj, FieldOfView): - logger.debug("%s applied to %s", self.display_name, - obj.__class__.__name__) - # Application to field of view - if obj.hdu is not None and obj.hdu.header["NAXIS"] == 3: - obj.cube = obj.hdu - elif obj.hdu is not None and obj.hdu.header["NAXIS"] == 2: - # todo: catch the case of obj.hdu.header["NAXIS"] == 2 - # for MAAT - pass - elif obj.hdu is None and obj.cube is None: - logger.info("Making cube") - obj.cube = obj.make_hdu() - - # Check whether an offset slit is used. If so, recompute spectral traces. - # TODO: this needs to be removed. If transforms are available that - # can be applied to the actual slit position, they ought be used - # without modification. If they are not available but need to be - # hacked, then that hack needs to be defined via the yaml and only - # be used in a particular case, not in general. - #offset_x = obj.cube.header["CRVAL1D"] - #offset_y = obj.cube.header["CRVAL2D"] - #if (offset_x != self.meta["offset_x"] or - # offset_y != self.meta["offset_y"]): - # logger.debug("Recomputing spectral traces for offset (%.1g, %.1g)", - # offset_x, offset_y) - # self.meta["offset_x"] = offset_x - # self.meta["offset_y"] = offset_y - # self.make_spectral_traces() - # self.update_meta() - - spt = self.spectral_traces[obj.trace_id] - remove_pre = False - if ("offset_type" in obj.meta and - obj.meta["offset_type"] == "detector"): - spt.xy2lam.pretransform_x = (det_offset, - {"offset": obj.meta["offset_x"]}) - spt.xy2lam.pretransform_y = (det_offset, - {"offset": obj.meta["offset_y"]}) - remove_pre = True - - obj.hdu = spt.map_spectra_to_focal_plane(obj) - - if remove_pre: - spt.xy2lam.pretransform_y = None - spt.xy2xi.pretransform_x = None + obj = self.apply_to_fov(obj) logger.debug("%s done", self.display_name) return obj + def apply_to_fovvolumelist(self, obj): + """ + Setup of SpectralTraceList + + During setup of the required FieldOfView objects, the + SpectralTraceList is asked for the source space volumes that + it requires (spatial limits and wavelength limits). + """ + logger.debug("%s applied to %s", self.display_name, + obj.__class__.__name__) + + new_vols_list = [] + + for spt in self.spectral_traces.values(): + vol = spt.fov_grid() + wave_edges = [vol["wave_min"], vol["wave_max"]] + if "x_min" in vol: + x_edges = [vol["x_min"], vol["x_max"]] + y_edges = [vol["y_min"], vol["y_max"]] + extracted_vols = obj.extract( + axes=["wave", "x", "y"], + edges=(wave_edges, x_edges, y_edges), + aperture_id=vol["aperture_id"]) + else: + extracted_vols = obj.extract( + axes=["wave"], + edges=(wave_edges, ), + aperture_id=vol["aperture_id"]) + + for ex_vol in extracted_vols: + ex_vol["meta"].update(vol) + ex_vol["meta"].pop("wave_min") + ex_vol["meta"].pop("wave_max") + new_vols_list.extend(extracted_vols) + + obj.volumes = new_vols_list + return obj + + + def apply_to_fov(self, obj): + """Apply to FieldOfView + + During "observation" the method is passed a single FieldOfView + object and applies the mapping to the image plane to it. + The FieldOfView object is associated to one SpectralTrace from the + list, identified by meta["trace_id"]. + """ + logger.debug("%s applied to %s %s", self.display_name, + obj.__class__.__name__, obj.trace_id) + + if obj.hdu is not None and obj.hdu.header["NAXIS"] == 3: + obj.cube = obj.hdu + elif obj.hdu is not None and obj.hdu.header["NAXIS"] == 2: + # todo: catch the case of obj.hdu.header["NAXIS"] == 2 + # for MAAT + pass + elif obj.hdu is None and obj.cube is None: + logger.info("Making cube") + obj.cube = obj.make_hdu() + + # Check whether an offset slit is used. If so, recompute spectral traces. + # TODO: this needs to be removed. If transforms are available that + # can be applied to the actual slit position, they ought be used + # without modification. If they are not available but need to be + # hacked, then that hack needs to be defined via the yaml and only + # be used in a particular case, not in general. + #offset_x = obj.cube.header["CRVAL1D"] + #offset_y = obj.cube.header["CRVAL2D"] + #if (offset_x != self.meta["offset_x"] or + # offset_y != self.meta["offset_y"]): + # logger.debug("Recomputing spectral traces for offset (%.1g, %.1g)", + # offset_x, offset_y) + # self.meta["offset_x"] = offset_x + # self.meta["offset_y"] = offset_y + # self.make_spectral_traces() + # self.update_meta() + + spt = self.spectral_traces[obj.trace_id] + remove_pre = False + if ("offset_type" in obj.meta and + obj.meta["offset_type"] == "detector"): + spt.xy2lam.pretransform_x = (det_offset, + {"offset": obj.meta["offset_x"]}) + spt.xy2lam.pretransform_y = (det_offset, + {"offset": obj.meta["offset_y"]}) + remove_pre = True + + obj.hdu = spt.map_spectra_to_focal_plane(obj) + + if remove_pre: + spt.xy2lam.pretransform_y = None + spt.xy2xi.pretransform_x = None + + return obj + @property def footprint(self): """Return the footprint of the entire SpectralTraceList.""" From 3375ec53ec2ab2df8bf42fa770d1772a4b6b9ca8 Mon Sep 17 00:00:00 2001 From: oczoske Date: Mon, 18 May 2026 19:01:40 +0200 Subject: [PATCH 06/87] Remove prints --- scopesim/effects/apertures.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scopesim/effects/apertures.py b/scopesim/effects/apertures.py index d704265bb..b8ca2ce2c 100644 --- a/scopesim/effects/apertures.py +++ b/scopesim/effects/apertures.py @@ -118,6 +118,7 @@ def apply_to(self, obj, **kwargs): """See parent docstring.""" if isinstance(obj, FovVolumeList): logger.debug("Executing %s, FoV setup", self.meta['name']) + x = quantity_from_table("x", self.table, u.arcsec).to_value(u.arcsec) y = quantity_from_table("y", self.table, @@ -130,13 +131,11 @@ def apply_to(self, obj, **kwargs): vol["meta"]["xi_max"] = max(x) * u.arcsec if "offset_type" in self.meta: - print("Yes, got it: ", self.meta["offset_type"]) vol["meta"]["offset_type"] = self.meta["offset_type"] vol["meta"]["offset_x"] = self.meta["offset_x"] vol["meta"]["offset_y"] = self.meta["offset_y"] else: vol["meta"]["offset_type"] = "none" - print("What is offset_type?") return obj # Outdated. Remove when removing all old FOVManager code from effects From 385cade20f40b9f977aae01704975ded02e5f330 Mon Sep 17 00:00:00 2001 From: oczoske Date: Mon, 18 May 2026 19:02:00 +0200 Subject: [PATCH 07/87] Set ImagePlane to 32bit --- scopesim/optics/image_plane.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scopesim/optics/image_plane.py b/scopesim/optics/image_plane.py index 794c03fbb..b58ac50cc 100644 --- a/scopesim/optics/image_plane.py +++ b/scopesim/optics/image_plane.py @@ -68,7 +68,7 @@ def __init__(self, header, cmds=None, **kwargs): raise ValueError(f"header must have a valid image-plane WCS: " f"{dict(header)}") - image = zeros_from_header(header) + image = zeros_from_header(header, dtype=np.float32) self.hdu = fits.ImageHDU(data=image, header=header) self.hdu.header["BUNIT"] = "ph s-1" # photons per second (per pixel) From b2de962a16f2f55e50d97b60110aab46703948e7 Mon Sep 17 00:00:00 2001 From: oczoske Date: Mon, 18 May 2026 19:02:20 +0200 Subject: [PATCH 08/87] MICADO IFU works --- scopesim/effects/micado_ifu_trace_list.py | 153 +++++++++++++++++++++- 1 file changed, 149 insertions(+), 4 deletions(-) diff --git a/scopesim/effects/micado_ifu_trace_list.py b/scopesim/effects/micado_ifu_trace_list.py index 826399e1f..c8236e793 100644 --- a/scopesim/effects/micado_ifu_trace_list.py +++ b/scopesim/effects/micado_ifu_trace_list.py @@ -34,6 +34,8 @@ class MicadoIFUSpectralTraceList(SpectralTraceList): "naxis1": 112, "nslice": 32, "slicewidth": 0.012, # arcsec + "wave_min": 1.5, + "wave_max": 2.5, # todo: do not hardcode } @@ -42,6 +44,104 @@ def __init__(self, **kwargs): # field of view of the instrument + + def apply_to_fovvolumelist(self, obj): + """ + Setup + + Create a single volume that covers the aperture and the maximum + wavelength range of the MICADO IFU. + """ + logger.debug("Executing %s: Setup", self.meta['name']) + volumes = [spectral_trace.fov_grid() + for spectral_trace in self.spectral_traces.values()] + + wave_min = min(vol["wave_min"] for vol in volumes) + wave_max = max(vol["wave_max"] for vol in volumes) + extracted_vols = obj.extract(axes=["wave"], + edges=([[wave_min, wave_max]])) + obj.volumes = extracted_vols + self.wave_min = wave_min + self.wave_max = wave_max + return obj + + def apply_to_fov(self, obj): + """Apply to FieldOfView""" + + logger.debug("Executing %s, FoV", self.meta['name']) + if obj.hdu is not None and obj.hdu.header["NAXIS"] == 3: + obj.cube = obj.hdu + elif obj.hdu is None and obj.cube is None: + obj.cube = obj.make_hdu() + + fovcube = obj.cube.data + n_z, n_y, n_x = fovcube.shape + fovwcs = WCS(obj.cube.header) + # Make this linear to avoid jump at RA 0 deg + fovwcs.wcs.ctype = ["LINEAR", "LINEAR", fovwcs.wcs.ctype[2]] + fovwcs_spat = fovwcs.sub(2) + ny_slice = self.meta["slice_samples"] + + # Spatial pixel coordinates + xslice, yslice = np.meshgrid(np.arange(n_x), + np.arange(ny_slice)) + + fovimage = np.zeros((obj.detector_header["NAXIS2"], + obj.detector_header["NAXIS1"]), + dtype=np.float32) + + for sptid, spt in tqdm(self.spectral_traces.items(), + desc=" Spectral Traces", position=2): + ymin = spt.meta["fov"]["y_min"] + ymax = spt.meta["fov"]["y_max"] + + + slicewcs = fovwcs.deepcopy() + + slicewcs.wcs.ctype = ["LINEAR", "LINEAR", + slicewcs.wcs.ctype[2]] + slicewcs.wcs.crpix[1] = (ny_slice + 1) / 2 + slicewcs.wcs.crval[1] = (ymin + ymax) / 2 / 3600 + slicewcs.wcs.cdelt[1] = (ymax - ymin) / ny_slice / 3600 + slicewcs_spat = slicewcs.sub(2) + + # World coordinates for the slice + xworld, yworld = slicewcs_spat.all_pix2world(xslice, + yslice, 0) + # FOV pixel coordinates for the slice + xfov, yfov = fovwcs_spat.all_world2pix(xworld, yworld, 0) + + slicecube = np.zeros((n_z, ny_slice, n_x)) + for islice in range(n_z): + ifov = RectBivariateSpline(np.arange(n_y), + np.arange(n_x), + fovcube[islice], kx=1, ky=1) + slicecube[islice] = ifov(yfov, xfov, grid=False) + + slicefov = FieldOfView3D(obj.header, + [obj.meta["wave_min"], + obj.meta["wave_max"]]) + slicefov.detector_header = obj.detector_header + slicefov.meta["xi_min"] = spt.meta["fov"]["xi_min"] + slicefov.meta["xi_max"] = spt.meta["fov"]["xi_max"] + slicefov.meta["trace_id"] = sptid + slicefov.cube = fits.ImageHDU(header=slicewcs.to_header(), + data=slicecube) + # slicefov.cube.writeto(f"slicefov_{sptid}.fits", overwrite=True) + slicefov.hdu = spt.map_spectra_to_focal_plane(slicefov) + if slicefov.hdu is not None: + sxmin = slicefov.hdu.header["XMIN"] + sxmax = slicefov.hdu.header["XMAX"] + symin = slicefov.hdu.header["YMIN"] + symax = slicefov.hdu.header["YMAX"] + fovimage[symin:symax, sxmin:sxmax] += slicefov.hdu.data + + obj.hdu = fits.ImageHDU(data=fovimage, header=obj.detector_header) + return obj + + + + def make_spectral_traces(self): """Make a spectral trace for each combination of order and aperture""" self.ext_data = self._file[0].header["EDATA"] @@ -54,10 +154,55 @@ def make_spectral_traces(self): continue params = {col: row[col] for col in row.colnames} params.update(self.meta) - hdu = self._file[row["extension_id"]] - for apid in self.slicelist["id"]: + trace_hdu = self._file[row["extension_id"]] + + for spslice, apid in enumerate(self.slicelist["id"]): specid = f"{row['description']}_{apid:02d}" - params["aperture_id"] = apid - spec_traces[specid] = SpectralTrace(hdu, **params) + thistrace = MicadoIFUSpectralTrace(trace_hdu, self.slicelist, spslice, + **params) + thistrace.meta["trace_id"] = specid + spec_traces[specid] = thistrace self.spectral_traces = spec_traces + + + +class MicadoIFUSpectralTrace(SpectralTrace): + """SpectralTrace for the MICADO IFU""" + + _class_params = { + "naxis1": 112, + "nslice": 32, + "slicewidth": 0.012, # arcsec + } + + + def __init__(self, trace_tbl, aplist, spslice, cmds=None, **kwargs): + super().__init__(trace_tbl, **kwargs) + # Provisional + self.aplist = aplist + self.meta["slice"] = spslice + self.meta["fov"] = self.fov_grid() + + def fov_grid(self): + """ + Provide information on the source space volume required by the effect. + """ + aperture = self.aplist[self.meta["slice"]] + x_min = aperture["left"] + x_max = aperture["right"] + y_min = aperture["bottom"] + y_max = aperture["top"] + xi_min = aperture["xi1"] + xi_max = aperture["xi2"] + + # ..todo: just a hack - xi and x are the same except xi is a quantity + xi_min = quantify(xi_min, u.arcsec) + xi_max = quantify(xi_max, u.arcsec) + + return {"x_min": x_min, "x_max": x_max, + "y_min": y_min, "y_max": y_max, + "xi_min": xi_min, "xi_max": xi_max, + "wave_min": self.meta["wave_min"], + "wave_max": self.meta["wave_max"], + "trace_id": self.trace_id} From b63315127a42954c33c43baebbbe1457781ddf04 Mon Sep 17 00:00:00 2001 From: oczoske Date: Tue, 19 May 2026 17:45:51 +0200 Subject: [PATCH 09/87] Staggered ifu slices --- scopesim/effects/micado_ifu_trace_list.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/scopesim/effects/micado_ifu_trace_list.py b/scopesim/effects/micado_ifu_trace_list.py index c8236e793..d01ef0035 100644 --- a/scopesim/effects/micado_ifu_trace_list.py +++ b/scopesim/effects/micado_ifu_trace_list.py @@ -18,7 +18,7 @@ from .spectral_trace_list import SpectralTraceList from .spectral_trace_list_utils import SpectralTrace from .spectral_trace_list_utils import Transform2D -from .spectral_trace_list_utils import make_image_interpolations +from .spectral_trace_list_utils import make_image_interpolations, det_offset from .apertures import ApertureMask from .ter_curves import TERCurve from ..optics.fov import FieldOfView, FieldOfView3D @@ -42,8 +42,6 @@ class MicadoIFUSpectralTraceList(SpectralTraceList): def __init__(self, **kwargs): super().__init__(**kwargs) - # field of view of the instrument - def apply_to_fovvolumelist(self, obj): """ @@ -174,16 +172,26 @@ class MicadoIFUSpectralTrace(SpectralTrace): "naxis1": 112, "nslice": 32, "slicewidth": 0.012, # arcsec + "pixsize": 0.015, # mm } def __init__(self, trace_tbl, aplist, spslice, cmds=None, **kwargs): super().__init__(trace_tbl, **kwargs) + + self._set_dispersion(self.wave_min, self.wave_max) # Provisional self.aplist = aplist self.meta["slice"] = spslice self.meta["fov"] = self.fov_grid() + lam = np.linspace(self.wave_min, self.wave_max, 1001) * u.um + off_y = aplist['offset'][spslice] + dlam = np.median(self.dlam_per_pix(lam)) / self.meta["pixsize"] * off_y + self.xy2lam.posttransform = (det_offset, {"offset": dlam}) + self.xilam2x.pretransform_y = (det_offset, {"offset": dlam}) + self.xilam2y.pretransform_y = (det_offset, {"offset": dlam}) + def fov_grid(self): """ Provide information on the source space volume required by the effect. From 8b01497d3d3adec269eeffabadeaa5896c023bbc Mon Sep 17 00:00:00 2001 From: oczoske Date: Wed, 20 May 2026 13:19:03 +0200 Subject: [PATCH 10/87] Flipped sign of offset --- scopesim/effects/micado_ifu_trace_list.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scopesim/effects/micado_ifu_trace_list.py b/scopesim/effects/micado_ifu_trace_list.py index d01ef0035..cd2595b44 100644 --- a/scopesim/effects/micado_ifu_trace_list.py +++ b/scopesim/effects/micado_ifu_trace_list.py @@ -41,7 +41,8 @@ class MicadoIFUSpectralTraceList(SpectralTraceList): def __init__(self, **kwargs): super().__init__(**kwargs) - + self.wave_min = self.meta["wave_min"] + self.wave_max = self.meta["wave_max"] def apply_to_fovvolumelist(self, obj): """ @@ -188,9 +189,9 @@ def __init__(self, trace_tbl, aplist, spslice, cmds=None, **kwargs): lam = np.linspace(self.wave_min, self.wave_max, 1001) * u.um off_y = aplist['offset'][spslice] dlam = np.median(self.dlam_per_pix(lam)) / self.meta["pixsize"] * off_y - self.xy2lam.posttransform = (det_offset, {"offset": dlam}) - self.xilam2x.pretransform_y = (det_offset, {"offset": dlam}) - self.xilam2y.pretransform_y = (det_offset, {"offset": dlam}) + self.xy2lam.posttransform = (det_offset, {"offset": -dlam}) + self.xilam2x.pretransform_y = (det_offset, {"offset": -dlam}) + self.xilam2y.pretransform_y = (det_offset, {"offset": -dlam}) def fov_grid(self): """ From 10a7bdb9609160ee56d6f998fe9e7dcf52b18e2b Mon Sep 17 00:00:00 2001 From: oczoske Date: Mon, 25 May 2026 06:26:30 +0200 Subject: [PATCH 11/87] offset more systematic --- scopesim/effects/spectral_trace_list.py | 40 +++++++++++++++++++++---- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/scopesim/effects/spectral_trace_list.py b/scopesim/effects/spectral_trace_list.py index c4d3fc925..4422e1835 100644 --- a/scopesim/effects/spectral_trace_list.py +++ b/scopesim/effects/spectral_trace_list.py @@ -288,10 +288,17 @@ def apply_to_fov(self, obj): remove_pre = False if ("offset_type" in obj.meta and obj.meta["offset_type"] == "detector"): + self.meta["offset_type"] = "detector" + self.meta["offset_x"] = obj.meta["offset_x"] + self.meta["offset_y"] = obj.meta["offset_y"] spt.xy2lam.pretransform_x = (det_offset, {"offset": obj.meta["offset_x"]}) spt.xy2lam.pretransform_y = (det_offset, {"offset": obj.meta["offset_y"]}) + spt.xilam2x.posttransform = (det_offset, + {"offset": -obj.meta["offset_x"]}) + spt.xilam2y.posttransform = (det_offset, + {"offset": -obj.meta["offset_y"]}) remove_pre = True obj.hdu = spt.map_spectra_to_focal_plane(obj) @@ -299,6 +306,8 @@ def apply_to_fov(self, obj): if remove_pre: spt.xy2lam.pretransform_y = None spt.xy2xi.pretransform_x = None + spt.xilam2x.posttransform = None + spt.xilam2y.posttransform = None return obj @@ -401,15 +410,36 @@ def rectify_traces(self, hdulist, xi_min=None, xi_max=None, interps=None, for i, trace_id in tqdm(enumerate(self.spectral_traces, start=1), desc=" Traces", total=len(self.spectral_traces)): - hdu = self[trace_id].rectify(hdulist, - interps=interps, - bin_width=bin_width, - xi_min=xi_min, xi_max=xi_max, - wave_min=wave_min, wave_max=wave_max) + spt = self[trace_id] + remove_pre = False + if ("offset_type" in self.meta and + self.meta["offset_type"] == "detector"): + spt.xy2lam.pretransform_x = (det_offset, + {"offset": self.meta["offset_x"]}) + spt.xy2lam.pretransform_y = (det_offset, + {"offset": self.meta["offset_y"]}) + spt.xilam2x.posttransform = (det_offset, + {"offset": -self.meta["offset_x"]}) + spt.xilam2y.posttransform = (det_offset, + {"offset": -self.meta["offset_y"]}) + remove_pre = True + + + hdu = spt.rectify(hdulist, + interps=interps, + bin_width=bin_width, + xi_min=xi_min, xi_max=xi_max, + wave_min=wave_min, wave_max=wave_max) if hdu is not None: # ..todo: rectify does not do that yet outhdul.append(hdu) outhdul[0].header[f"EXTNAME{i}"] = trace_id + if remove_pre: + spt.xy2lam.pretransform_y = None + spt.xy2xi.pretransform_x = None + spt.xilam2x.posttransform = None + spt.xilam2y.posttransform = None + outhdul[0].header.update(inhdul[0].header) return outhdul From 2ce224d6014b33b73ea00c945574c2a703d558d7 Mon Sep 17 00:00:00 2001 From: Kieran Leschinski Date: Wed, 19 Aug 2026 15:35:43 +0200 Subject: [PATCH 12/87] Fix transposed axes in make_image_interpolations RectBivariateSpline was constructed with x = arange(NAXIS1) and y = arange(NAXIS2) against data of shape (NAXIS2, NAXIS1), which raises ValueError for any non-square image. It went unnoticed because current detector images are square and the caller in SpectralTrace.rectify compensates by passing (j, i) swapped. Build the spline with axes matching the data layout instead; the (j, i) call convention used by rectify and existing tests is now correct by construction and documented. Results on square images are unchanged. Adds a non-square regression test. --- scopesim/effects/spectral_trace_list_utils.py | 11 ++++++++--- .../tests_effects/test_SpectralTraceListUtils.py | 10 ++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/scopesim/effects/spectral_trace_list_utils.py b/scopesim/effects/spectral_trace_list_utils.py index d523aee72..da583fdc0 100644 --- a/scopesim/effects/spectral_trace_list_utils.py +++ b/scopesim/effects/spectral_trace_list_utils.py @@ -1022,13 +1022,18 @@ def _xiy2xlam_fit(layout, params): def make_image_interpolations(hdulist, **kwargs): - """Create 2D interpolation functions for images.""" + """Create 2D interpolation functions for images. + + The interpolation functions are called with (j, i) pixel coordinates, + i.e. the first argument corresponds to the row (NAXIS2) axis of the + image, the second to the column (NAXIS1) axis. + """ interps = [] for hdu in hdulist: if isinstance(hdu, fits.ImageHDU): interps.append( - RectBivariateSpline(np.arange(hdu.header["NAXIS1"]), - np.arange(hdu.header["NAXIS2"]), + RectBivariateSpline(np.arange(hdu.header["NAXIS2"]), + np.arange(hdu.header["NAXIS1"]), hdu.data, **kwargs) ) return interps diff --git a/scopesim/tests/tests_effects/test_SpectralTraceListUtils.py b/scopesim/tests/tests_effects/test_SpectralTraceListUtils.py index ca2f19d2b..e1bcbb463 100644 --- a/scopesim/tests/tests_effects/test_SpectralTraceListUtils.py +++ b/scopesim/tests/tests_effects/test_SpectralTraceListUtils.py @@ -202,3 +202,13 @@ def test_interpolation_is_accurate(self): interps = make_image_interpolations(hdul, kx=1, ky=1) imginterp = interps[0](yy, xx, grid=False) assert np.allclose(imginterp, img) + + def test_works_with_non_square_image(self): + # The axes passed to RectBivariateSpline were transposed, which + # raised ValueError for any image with NAXIS1 != NAXIS2 + xx, yy = np.meshgrid(np.arange(80), np.arange(50)) + img = np.sin(xx/6) * np.cos(yy/7) # shape (50, 80) + hdul = fits.HDUList(fits.ImageHDU(data=img)) + interps = make_image_interpolations(hdul, kx=1, ky=1) + imginterp = interps[0](yy, xx, grid=False) + assert np.allclose(imginterp, img) From fdd4822a815e1d09ca649aec737a610bcc95e5bf Mon Sep 17 00:00:00 2001 From: teutoburg Date: Sun, 30 Aug 2026 15:07:38 +0200 Subject: [PATCH 13/87] Refactor `TERCurve.apply_to()` For detailed explanation see #1002, this is basically the same but for `TERCurve`. The new `.__call__()` in this case is exclusively for cubes, because 1D spectra work differently. But those could be simplified as well, and as a result the `combine_two_spectra` function was no longer needed. Also finally remove the long-deprecated `.fov_grid()` methods that were still here. Everything passes without them. Well, except of course `test_current_filter_has_fov_grid_method`... --- scopesim/effects/ter_curves.py | 153 ++++++++---------- scopesim/effects/ter_curves_utils.py | 49 ------ scopesim/tests/tests_effects/test_TERCurve.py | 3 - 3 files changed, 68 insertions(+), 137 deletions(-) diff --git a/scopesim/effects/ter_curves.py b/scopesim/effects/ter_curves.py index 4d5e1f8b3..1b32b6674 100644 --- a/scopesim/effects/ter_curves.py +++ b/scopesim/effects/ter_curves.py @@ -4,26 +4,38 @@ import warnings from typing import ClassVar from collections.abc import Collection, Iterable -from pooch import retrieve import numpy as np +from numpy.typing import ArrayLike, NDArray from scipy import integrate from astropy import units as u from astropy.io import fits from astropy.table import Table +from synphot import SpectralElement, SourceSpectrum import skycalc_ipy from .effects import Effect -from .ter_curves_utils import (add_edge_zeros, combine_two_spectra, - apply_throughput_to_cube, download_svo_filter, - download_svo_filter_list) +from .ter_curves_utils import ( + add_edge_zeros, + download_svo_filter, + download_svo_filter_list, +) from ..optics.fov_volume_list import FovVolumeList from ..optics.surface import SpectralSurface from ..source.source import Source -from ..source.source_fields import (CubeSourceField, SpectrumSourceField, - BackgroundSourceField) -from ..utils import (from_currsys, quantify, check_keys, find_file, - figure_factory, get_logger) +from ..source.source_fields import ( + CubeSourceField, + SpectrumSourceField, + BackgroundSourceField, +) +from ..utils import ( + from_currsys, + quantify, + check_keys, + find_file, + figure_factory, + get_logger, +) from ..server.download_utils import create_retriever logger = get_logger(__name__) @@ -109,53 +121,64 @@ def __init__(self, filename=None, **kwargs): self.surface.table = data self.surface.table.meta.update(self.meta) + def __call__(self, data: ArrayLike, waveset: u.Quantity[u.um]) -> NDArray: + return data * self.throughput(waveset).value[:, None, None] + + def _apply_to_fvl(self, fvl: FovVolumeList) -> None: + logger.debug("Apply %s, FoV setup", self.display_name) + + wave = self.surface.throughput.waveset + thru = self.surface.throughput(wave) + valid_waves = np.argwhere(thru > 0) + wave_min = wave[max(0, valid_waves[0][0] - 1)] + wave_max = wave[min(len(wave) - 1, valid_waves[-1][0] + 1)] + + fvl.shrink("wave", [wave_min.to_value(u.um), wave_max.to_value(u.um)]) + + def _apply_to_src(self, src: Source) -> None: + logger.debug("Apply %s, Source spectra", self.display_name) + + self.meta = from_currsys(self.meta, self.cmds) + wave_min = quantify(self.meta["wave_min"], u.um).to(u.AA) + wave_max = quantify(self.meta["wave_max"], u.um).to(u.AA) + + thru = self.throughput + + # apply transmission to source spectra + for fld in src.fields: + if isinstance(fld, CubeSourceField): + fld.field.data = self(fld.field.data, fld.waveset) + elif isinstance(fld, SpectrumSourceField): + fld.spectra = { + isp: spec * thru # This works because synphot! + for isp, spec in fld.spectra.items() + } + else: + # Rather log than raise here, can still move on + logger.error( + "Source field is neither Cube nor has spectra, this " + "should not happen, but moving on.") + + # add the effect background to the source background field + if self.background_source is not None: + for bgs in self.background_source: + src.append(bgs) + def apply_to(self, obj, **kwargs): if isinstance(obj, Source): - self.meta = from_currsys(self.meta, self.cmds) - wave_min = quantify(self.meta["wave_min"], u.um).to(u.AA) - wave_max = quantify(self.meta["wave_max"], u.um).to(u.AA) - - thru = self.throughput - - # apply transmission to source spectra - for fld in obj.fields: - if isinstance(fld, CubeSourceField): - fld.field = apply_throughput_to_cube( - fld.field, thru, fld.waveset) - elif isinstance(fld, SpectrumSourceField): - fld.spectra = { - isp: combine_two_spectra(spec, thru, "multiply", - wave_min, wave_max) - for isp, spec in fld.spectra.items() - } - else: - # Rather log than raise here, can still move on - logger.error("Source field is neither Cube nor has " - "spectra, this shouldn't occur...") - - # add the effect background to the source background field - if self.background_source is not None: - for bgs in self.background_source: - obj.append(bgs) + self._apply_to_src(obj) if isinstance(obj, FovVolumeList): - wave = self.surface.throughput.waveset - thru = self.surface.throughput(wave) - valid_waves = np.argwhere(thru > 0) - wave_min = wave[max(0, valid_waves[0][0] - 1)] - wave_max = wave[min(len(wave) - 1, valid_waves[-1][0] + 1)] - - obj.shrink("wave", [wave_min.to_value(u.um), - wave_max.to_value(u.um)]) + self._apply_to_fvl(obj) return obj @property - def emission(self): + def emission(self) -> SourceSpectrum: return self.surface.emission @property - def throughput(self): + def throughput(self) -> SpectralElement: return self.surface.throughput @property @@ -166,23 +189,16 @@ def background_source(self): bkg_hdr = fits.Header() bkg_hdr.update({ + "ROLE": "background", # not used yet "BG_SRC": True, "BG_SURF": self.display_name, "SPEC_REF": 0, - # Seem those are not needed... - # "CUNIT1": "ARCSEC", - # "CUNIT2": "ARCSEC", - # "CDELT1": 0, - # "CDELT2": 0, "BUNIT": "photlam arcsec-2", "SOLIDANG": "arcsec-2", }) bkg_fld = BackgroundSourceField(field=None, spectra=bkg_spec, header=bkg_hdr) bkg_src = Source(field=bkg_fld) - # Before BackgroundSourceField: - # bkg_hdu = fits.ImageHDU(header=bkg_hdr) - # bkg_src = Source(image_hdu=bkg_hdu, spectra=flux) self._background_source = bkg_src return [self._background_source] @@ -604,34 +620,6 @@ def __init__(self, cmds=None, **kwargs): # TODO: maybe use actually masked table here? self.table["transmission"][mask] = 0 - def fov_grid(self, which="waveset", **kwargs): - warnings.warn("The fov_grid method is deprecated and will be removed " - "in a future release.", DeprecationWarning, stacklevel=2) - if which == "waveset": - self.meta.update(kwargs) - self.meta = from_currsys(self.meta, self.cmds) - # ..todo:: replace the 101 with a variable in !SIM - wave = np.linspace(self.meta["wave_min"], - self.meta["wave_max"], 101) - wave = quantify(wave, u.um) - throughput = self.surface.transmission(wave) - min_thru = self.meta["minimum_throughput"] - valid_waves = np.where(throughput.value > min_thru)[0] - if len(valid_waves) > 0: - wave_edges = [min(wave[valid_waves].value), - max(wave[valid_waves].value)] * u.um - else: - raise ValueError("No transmission found above the threshold {}" - " in this wavelength range {}. Did you open " - "the shutter?" - "".format(self.meta["minimum_throughput"], - [self.meta["wave_min"], - self.meta["wave_max"]])) - else: - wave_edges = [] - - return wave_edges - @property def fwhm(self): wave = self.surface.wavelength @@ -789,11 +777,6 @@ def surface(self): def throughput(self): return self.current_filter.throughput - def fov_grid(self, which="waveset", **kwargs): - warnings.warn("The fov_grid method is deprecated and will be removed " - "in a future release.", DeprecationWarning, stacklevel=2) - return self.current_filter.fov_grid(which=which, **kwargs) - def change_filter(self, filtername=None): """Change the current filter.""" if filtername in self.filters.keys(): diff --git a/scopesim/effects/ter_curves_utils.py b/scopesim/effects/ter_curves_utils.py index cabc6d796..827bf742a 100644 --- a/scopesim/effects/ter_curves_utils.py +++ b/scopesim/effects/ter_curves_utils.py @@ -389,55 +389,6 @@ def apply_throughput_to_cube( return cube -def combine_two_spectra(spec_a, spec_b, action, wave_min, wave_max): - """ - Combine transmission and/or emission spectrum with a common waverange. - - Spec_A is the source spectrum - Spec_B is either the transmission or emission that should be applied - - Parameters - ---------- - spec_a : synphot.SourceSpectrum - spec_b : synphot.SpectralElement, synphot.SourceSpectrum - action: {"multiply", "add"} - wave_min, wave_max : quantity - [Angstrom] - - Returns - ------- - new_source : synphot.SourceSpectrum - - """ - if spec_a.waveset is None: - wave_val = spec_b.waveset.value - else: - wave_val = spec_a.waveset.value - mask = (wave_val > wave_min.value) * (wave_val < wave_max.value) - - wave = ([wave_min.value] + list(wave_val[mask]) + [wave_max.value]) * u.AA - if "mult" in action.lower(): - spec_c = spec_a(wave) * spec_b(wave) - # Diagnostic plots - not for general use - # from matplotlib import pyplot as plt - # plt.plot(wave, spec_a(wave), label="spec_a") - # plt.plot(wave, spec_b(wave), label="spec_b") - # plt.plot(wave, spec_c, label="spec_c") - # plt.xlim(2.9e4, 4.2e4) - # plt.legend() - # plt.show() - elif "add" in action.lower(): - spec_c = spec_a(wave) + spec_b(wave) - else: - raise ValueError(f"action {action} unknown") - - new_source = SourceSpectrum(Empirical1D, points=wave, lookup_table=spec_c) - new_source.meta.update(spec_b.meta) - new_source.meta.update(spec_a.meta) - - return new_source - - def add_edge_zeros(tbl, wave_colname): if isinstance(tbl, Table): vals = np.zeros(len(tbl.colnames)) diff --git a/scopesim/tests/tests_effects/test_TERCurve.py b/scopesim/tests/tests_effects/test_TERCurve.py index 8e9a1847b..aadff5cc3 100644 --- a/scopesim/tests/tests_effects/test_TERCurve.py +++ b/scopesim/tests/tests_effects/test_TERCurve.py @@ -112,9 +112,6 @@ def test_loads_all_filters(self, fwheel): def test_current_filter_is_filter(self, fwheel): assert isinstance(fwheel.current_filter, tc.FilterCurve) - def test_current_filter_has_fov_grid_method(self, fwheel): - assert hasattr(fwheel.current_filter, "fov_grid") - def test_change_to_known_filter(self, fwheel): fwheel.change_filter('Ks') assert fwheel.current_filter.meta["name"] == "Ks" From 94998b3d3330cb3e56eeafc799be2b1689f92eac Mon Sep 17 00:00:00 2001 From: teutoburg Date: Sun, 30 Aug 2026 20:42:18 +0200 Subject: [PATCH 14/87] Refactor `.apply_to()` in electronic effects See #1002 for details and explanation. - Add `Detector.data` property setter to eleminate external access of `Detector._hdu` attribute. - Turn various effect attributes which need currsys/meta resolving into properties. This allows to encapsulate the resolving and make calls to the attributes much cleaner. **If we ever make effects dataclasses with explicit attribute fields rather than kwargs, this will make that process much smoother, because accessing a dataclass field looks identical to accessing a property, we just remove the property definitions then**. - Anything that the `.__call__()` needs, which is taken from the `obj` (in this case usually `Detector`), is resolved before and passed to `.__call__()` as an argument. That means the latter doesn't need to know about `obj`, which is cleaner and matches the PSF case. - I think there are now a few new `np.random.default_rng` cases. This was taken from a branch I started several months ago and forgot about in the much more recent discussion about those. Might not be final. This isn't completely exhaustive for all the effects here, but I tried to cover the "standard" ones... --- scopesim/detector/detector.py | 7 + scopesim/effects/electronic/electrons.py | 71 +++-- scopesim/effects/electronic/exposure.py | 4 +- scopesim/effects/electronic/noise.py | 261 +++++++++++------- scopesim/effects/electronic/pixels.py | 107 ++++--- scopesim/effects/rotation.py | 2 +- .../tests_effects/test_BasicReadoutNoise.py | 21 +- 7 files changed, 292 insertions(+), 181 deletions(-) diff --git a/scopesim/detector/detector.py b/scopesim/detector/detector.py index 2b0e43ed8..b6708d365 100644 --- a/scopesim/detector/detector.py +++ b/scopesim/detector/detector.py @@ -1,4 +1,7 @@ +# -*- coding: utf-8 -*- + import numpy as np +from numpy.typing import ArrayLike from astropy.io.fits import ImageHDU from astropy.wcs import WCS @@ -72,6 +75,10 @@ def data(self): """Return data from internal HDU.""" return self._hdu.data + @data.setter + def data(self, data: ArrayLike): + self._hdu.data = data + @property def image(self): """Return data from internal HDU.""" diff --git a/scopesim/effects/electronic/electrons.py b/scopesim/effects/electronic/electrons.py index c860362bd..1135eb283 100644 --- a/scopesim/effects/electronic/electrons.py +++ b/scopesim/effects/electronic/electrons.py @@ -9,17 +9,18 @@ """ from typing import ClassVar +from numbers import Real # matches int, float and all the numpy scalars import numpy as np +from numpy.typing import ArrayLike, NDArray from scipy.signal import oaconvolve from .. import Effect from ...detector import Detector from ...utils import figure_factory, check_keys -from ...utils import from_currsys, real_colname, get_logger +from ...utils import from_currsys, real_colname from . import logger -logger = get_logger(__name__) class LinearityCurve(Effect): """ @@ -62,21 +63,29 @@ def __init__(self, **kwargs): check_keys(self.meta, self.required_keys, action="error") + def __call__(self, data: ArrayLike, ndit: int = 1) -> NDArray: + incident = self.incident * ndit + measured = self.measured * ndit + return np.interp(data, incident, measured) + + @property + def incident(self) -> NDArray: + if self.table is not None: + return self.table["incident"] + return np.asarray(from_currsys(self.meta["incident"], self.cmds)) + + @property + def measured(self) -> NDArray: + if self.table is not None: + return self.table["measured"] + return np.asarray(from_currsys(self.meta["measured"], self.cmds)) + def apply_to(self, obj, **kwargs): if not isinstance(obj, Detector): return obj ndit = from_currsys(self.meta["ndit"], self.cmds) - if self.table is not None: - incident = self.table["incident"] * ndit - measured = self.table["measured"] * ndit - else: - incident = np.asarray(from_currsys(self.meta["incident"], - self.cmds)) * ndit - measured = np.asarray(from_currsys(self.meta["measured"], - self.cmds)) * ndit - obj._hdu.data = np.interp(obj._hdu.data, incident, measured) - + obj.data = self(obj.data, ndit) return obj def plot(self, **kwargs): @@ -298,28 +307,34 @@ def _should_apply(self) -> bool: return True - def apply_to(self, obj, **kwargs): - if not isinstance(obj, Detector): - return obj - - new_dtype = self.meta["dtype"] + def __call__(self, data: ArrayLike, gain: Real) -> NDArray: + return data / gain + def _get_gain(self, det_meta) -> Real: # Apply the gain value (copy from DarkCurrent) # Note that this does not cater for the case where the gain is given # as a plain dictionary. Should we implement that? if hasattr(self.cmds["!DET.gain"], "dic"): - dtcr_id = obj.meta[real_colname("id", obj.meta)] + dtcr_id = det_meta[real_colname("id", det_meta)] gain = self.cmds["!DET.gain"].dic[dtcr_id] logger.info(f"Detector {dtcr_id}: applying gain {gain}") - elif isinstance(self.cmds["!DET.gain"], (float, int)): + return gain + if isinstance(self.cmds["!DET.gain"], Real): gain = self.cmds["!DET.gain"] logger.info(f"Applying gain {gain}") - else: - raise ValueError(".meta['gain'] must be either " - f"dict or float, but is {self.cmds['!DET.gain']}") + return gain + raise ValueError( + f"{self.__class__.__name__}.meta['gain'] must be either " + f"dict or float, but is {self.cmds['!DET.gain']}") + + def apply_to(self, obj, **kwargs): + if not isinstance(obj, Detector): + return obj + + new_dtype = self.meta["dtype"] # Apply gain - obj._hdu.data /= gain + obj.data = self(obj.data, self._get_gain(obj.meta)) # Type-conversion wraps around input values that are higher or lower than # the respective maximum and minimum values of the new data type. Before @@ -328,14 +343,14 @@ def apply_to(self, obj, **kwargs): if np.issubdtype(new_dtype, np.integer): minval = np.iinfo(new_dtype).min maxval = np.iinfo(new_dtype).max - minvals_mask = obj._hdu.data < minval - maxvals_mask = obj._hdu.data > maxval + minvals_mask = obj.data < minval + maxvals_mask = obj.data > maxval if minvals_mask.any(): - obj._hdu.data[minvals_mask] = minval + obj.data[minvals_mask] = minval logger.warning( f"Effect ADConversion: {minvals_mask.sum()} negative pixels") if maxvals_mask.any(): - obj._hdu.data[maxvals_mask] = maxval + obj.data[maxvals_mask] = maxval logger.warning( f"Effect ADConversion: {maxvals_mask.sum()} saturated pixels") @@ -343,6 +358,6 @@ def apply_to(self, obj, **kwargs): # set to the modified data. It should be fine to simply re-assign the # data attribute, but just in case it's not... logger.debug("Applying digitization to dtype %s.", new_dtype) - obj._hdu.data = obj._hdu.data.astype(new_dtype) + obj.data = obj.data.astype(new_dtype) return obj diff --git a/scopesim/effects/electronic/exposure.py b/scopesim/effects/electronic/exposure.py index 8ff88e1e7..f131387dd 100644 --- a/scopesim/effects/electronic/exposure.py +++ b/scopesim/effects/electronic/exposure.py @@ -217,7 +217,7 @@ def apply_to(self, obj, **kwargs): logger.debug("Exposure: DIT = %s s, NDIT = %s", dit, ndit) if self.current_mode == "average": - obj._hdu.data /= ndit + obj.data = obj.data / ndit return obj @@ -276,6 +276,6 @@ def apply_to(self, obj, **kwargs): "readout call." ) - obj._hdu.data *= dit * ndit + obj.data = obj.data * dit * ndit return obj diff --git a/scopesim/effects/electronic/noise.py b/scopesim/effects/electronic/noise.py index 0698c09f5..a021143b5 100644 --- a/scopesim/effects/electronic/noise.py +++ b/scopesim/effects/electronic/noise.py @@ -2,9 +2,11 @@ """Any kinds of electronic or photonic noise.""" from typing import ClassVar -from numbers import Real +from collections.abc import Mapping +from numbers import Real # matches int, float and all the numpy scalars import numpy as np +from numpy.typing import ArrayLike, NDArray from astropy.io import fits from .. import Effect @@ -24,14 +26,18 @@ def __init__(self, **kwargs): self.meta.update(kwargs) check_keys(self.meta, self.required_keys, action="error") + def __call__(self, data: ArrayLike) -> NDArray: + return data + self.bias_level + + @property + def bias_level(self) -> float: + return from_currsys(self.meta["bias"], self.cmds) + def apply_to(self, obj, **kwargs): if not isinstance(obj, Detector): return obj - biaslevel = from_currsys(self.meta["bias"], self.cmds) - # Can't do in-place because of Quantization data type conflicts. - obj._hdu.data = obj._hdu.data + biaslevel - + obj.data = self(obj.data) return obj @@ -55,32 +61,89 @@ def __init__(self, **kwargs): check_keys(self.meta, self.required_keys, action="error") - def apply_to(self, det, **kwargs): - if not isinstance(det, Detector): - return det - - self.meta["random_seed"] = from_currsys(self.meta["random_seed"], - self.cmds) - if self.meta["random_seed"] is not None: - np.random.seed(self.meta["random_seed"]) - + def __call__(self, data: ArrayLike, ndit: int = 1) -> NDArray: self.meta = from_currsys(self.meta, self.cmds) - ron_keys = ["noise_std", "n_channels", "channel_fraction", - "line_fraction", "pedestal_fraction", "read_fraction"] - ron_kwargs = {key: self.meta[key] for key in ron_keys} - ron_kwargs["image_shape"] = det._hdu.data.shape - ron_frame = _make_ron_frame(**ron_kwargs) + ron_frame = self._make_ron_frame(data.shape) + stacked_ron_frame = np.zeros_like(ron_frame) - for i in range(self.meta["ndit"]): + for i in range(ndit): dx = np.random.randint(0, ron_frame.shape[1]) dy = np.random.randint(0, ron_frame.shape[0]) stacked_ron_frame += np.roll(ron_frame, (dy, dx), axis=(0, 1)) # TODO: this .T is ugly. Work out where things are getting switched and remove it! - # Can't do in-place because of Quantization data type conflicts. - det._hdu.data = det._hdu.data + stacked_ron_frame.T + return data + stacked_ron_frame.T + + @property + def noise_std(self) -> int: + return from_currsys(self.meta["noise_std"], self.cmds) + + @property + def n_channels(self) -> int: + return from_currsys(self.meta["n_channels"], self.cmds) + + @property + def random_seed(self) -> int: + return from_currsys(self.meta["random_seed"], self.cmds) + + @staticmethod + def _pseudo_random_field( + rng, + scale: float = 1., + size: tuple[int, ...] = (1024, 1024), + ) -> NDArray: + n = 256 + image = np.zeros(size) + batch = rng.normal(loc=0, scale=scale, size=(2*n, 2*n)) + for y in range(0, size[1], n): + for x in range(0, size[0], n): + i, j = rng.integers(n, size=2) + dx, dy = min(size[0]-x, n), min(size[1]-y, n) + image[x:x+dx, y:y+dy] = batch[i:i+dx, j:j+dy] + + return image + + def _make_ron_frame(self, shape: tuple[int, ...]) -> NDArray: + # TODO: Add yaml-settable seed here + rng = np.random.default_rng(self.random_seed) + + channel_fraction = self.meta["channel_fraction"] + line_fraction = self.meta["line_fraction"] + pedestal_fraction = self.meta["pedestal_fraction"] + read_fraction = self.meta["read_fraction"] + + pixel_std = self.noise_std * (pedestal_fraction + read_fraction)**0.5 + if shape < (1024, 1024): + pixel = rng.normal(loc=0, scale=pixel_std, size=shape) + line = rng.normal( + loc=0, + scale=self.noise_std * line_fraction**0.5, + size=shape[1], + ) + else: + # TODO: Why bother with this pseudo random function? + pixel = self._pseudo_random_field(rng, scale=pixel_std, size=shape) + line = pixel[0] + + channel = np.repeat( + rng.normal( + loc=0, + scale=self.noise_std * channel_fraction**0.5, + size=self.n_channels, + ), + max(1, shape[0] // self.n_channels) + 1, + axis=0, + ) + return (pixel + line).T + channel[:shape[0]] + + def apply_to(self, det, **kwargs): + if not isinstance(det, Detector): + return det + + ndit = from_currsys(self.meta["ndit"], self.cmds) + det._hdu.data = self(det._hdu.data, ndit) return det def plot(self, det, **kwargs): @@ -109,21 +172,25 @@ def __init__(self, **kwargs): check_keys(self.meta, self.required_keys, action="error") + def __call__(self, data: ArrayLike, ndit: int = 1) -> NDArray: + rng = np.random.default_rng(self.random_seed) + scale = self.noise_std * np.sqrt(float(ndit)) + return data + rng.normal(loc=0, scale=scale, size=data.shape) + + @property + def noise_std(self) -> float: + return from_currsys(self.meta["noise_std"], self.cmds) + + @property + def random_seed(self) -> int: + return from_currsys(self.meta["random_seed"], self.cmds) + def apply_to(self, det, **kwargs): if not isinstance(det, Detector): return det ndit = from_currsys(self.meta["ndit"], self.cmds) - ron = from_currsys(self.meta["noise_std"], self.cmds) - noise_std = ron * np.sqrt(float(ndit)) - - random_seed = from_currsys(self.meta["random_seed"], self.cmds) - if random_seed is not None: - np.random.seed(random_seed) - # Can't do in-place because of Quantization data type conflicts. - det._hdu.data = det._hdu.data + np.random.normal( - loc=0, scale=noise_std, size=det._hdu.data.shape) - + det._hdu.data = self(det._hdu.data, ndit) return det def plot(self, det): @@ -191,7 +258,7 @@ def apply_to(self, obj, **kwargs): dtcr_id = obj.meta[id_key] if id_key is not None else None prnu_std_meta = from_currsys(self.meta["prnu_std"], self.cmds) - if isinstance(prnu_std_meta, dict): + if isinstance(prnu_std_meta, Mapping): prnu_std = float(from_currsys(prnu_std_meta[dtcr_id], self.cmds)) elif isinstance(prnu_std_meta, Real): prnu_std = float(prnu_std_meta) @@ -201,7 +268,7 @@ def apply_to(self, obj, **kwargs): f"or a dict keyed by detector ID, got {type(prnu_std_meta)}" ) - shape = obj._hdu.data.shape + shape = obj.data.shape if dtcr_id not in self._gain_maps: rng = np.random.default_rng(random_seed) self._gain_maps[dtcr_id] = rng.normal( @@ -211,7 +278,7 @@ def apply_to(self, obj, **kwargs): if self._gain_maps[dtcr_id].shape != shape: raise ValueError("gain map shape mismatch") - obj._hdu.data = obj._hdu.data * self._gain_maps[dtcr_id] + obj.data = obj.data * self._gain_maps[dtcr_id] return obj def plot(self, det_id=None): @@ -236,13 +303,36 @@ def __init__(self, **kwargs): self.meta["random_seed"] = "!SIM.random.seed" self.meta.update(kwargs) + def __call__(self, data: ArrayLike) -> NDArray: + rng = np.random.default_rng(self.random_seed) + + # Check if there are negative values in the data. + values_negative = data < 0 + if values_negative.any(): + logger.warning( + "Effect ShotNoise: %d negative pixels", values_negative.sum()) + data[values_negative] = 0 + + # Apply a Poisson distribution to the low values. + values_low = data < 1e7 + data[values_low] = rng.poisson(data[values_low]) + + # Apply a normal distribution to the high values. + values_high = ~values_low + data[values_high] = rng.normal(data[values_high], np.sqrt(data[values_high])) + + return data + + @property + def random_seed(self) -> int: + return from_currsys(self.meta["random_seed"], self.cmds) + def apply_to(self, det, **kwargs): if not isinstance(det, Detector): return det - self.meta["random_seed"] = from_currsys(self.meta["random_seed"], - self.cmds) - rng = np.random.default_rng(self.meta["random_seed"]) + self.meta["random_seed"] = from_currsys( + self.meta["random_seed"], self.cmds) # numpy has a problem with generating Poisson distributions above # certain values. E.g. on linux, numpy.random.poisson(1e20) raises @@ -264,25 +354,8 @@ def apply_to(self, det, **kwargs): # - numpy.nan are implicitly passed through the normal distribution; # because the Poisson distribution cannot handle them. - data = det._hdu.data - - # Check if there are negative values in the data. - values_negative = data < 0 - if values_negative.any(): - logger.warning( - "Effect ShotNoise: %d negative pixels", values_negative.sum()) - data[values_negative] = 0 - - # Apply a Poisson distribution to the low values. - values_low = data < 1e7 - data[values_low] = rng.poisson(data[values_low]) - - # Apply a normal distribution to the high values. - values_high = ~values_low - data[values_high] = rng.normal(data[values_high], np.sqrt(data[values_high])) - new_imagehdu = fits.ImageHDU( - data=data, + data=self(det._hdu.data), header=det._hdu.header, ) @@ -314,25 +387,41 @@ def __init__(self, **kwargs): super().__init__(**kwargs) check_keys(self.meta, self.required_keys, action="error") + def __call__( + self, + data: np.ndarray, + dark_level: float, + dit: float, + ndit: int, + ) -> np.ndarray: + return data + dark_level * dit * ndit + + @property + def random_seed(self) -> int: + return from_currsys(self.meta["random_seed"], self.cmds) + + def _get_dark_level(self, det_meta) -> float: + dark_level = float(from_currsys(self.meta["value"], self.cmds)) + if isinstance(dark_level, Real): + return dark_level + if isinstance(dark_level, Mapping): + dtcr_id = det_meta[real_colname("id", det_meta)] + return from_currsys(dark_level[dtcr_id], self.cmds) + raise ValueError( + f"<{self.__class__.__name__}>.meta['value'] must be either " + f"dict-like or scalar number, but is {dark_level}." + ) + def apply_to(self, obj, **kwargs): if not isinstance(obj, Detector): return obj - if isinstance(from_currsys(self.meta["value"], self.cmds), dict): - dtcr_id = obj.meta[real_colname("id", obj.meta)] - dark = from_currsys(self.meta["value"][dtcr_id], self.cmds) - elif isinstance(from_currsys(self.meta["value"], self.cmds), Real): - dark = float(from_currsys(self.meta["value"], self.cmds)) - else: - raise ValueError(".meta['value'] must be either " - f"dict or float, but is {self.meta['value']}") - + # Dark level needs detector meta so can't go into __call__() + dark_level = self._get_dark_level(obj.meta) dit = from_currsys(self.meta["dit"], self.cmds) ndit = from_currsys(self.meta["ndit"], self.cmds) - # Can't do in-place because of Quantization data type conflicts. - obj._hdu.data = obj._hdu.data + dark * dit * ndit - + obj.data = self(obj.data, dark_level, dit, ndit) return obj def plot(self, det, **kwargs): @@ -348,39 +437,3 @@ def plot(self, det, **kwargs): ax.plot(times, levels, **kwargs) ax.set_xlabel("time") ax.set_ylabel("dark level") - - -def _pseudo_random_field(scale=1, size=(1024, 1024)): - n = 256 - image = np.zeros(size) - batch = np.random.normal(loc=0, scale=scale, size=(2*n, 2*n)) - for y in range(0, size[1], n): - for x in range(0, size[0], n): - i, j = np.random.randint(n, size=2) - dx, dy = min(size[0]-x, n), min(size[1]-y, n) - image[x:x+dx, y:y+dy] = batch[i:i+dx, j:j+dy] - - return image - - -def _make_ron_frame(image_shape, noise_std, n_channels, channel_fraction, - line_fraction, pedestal_fraction, read_fraction): - shape = image_shape - w_chan = max(1, shape[0] // n_channels) - - pixel_std = noise_std * (pedestal_fraction + read_fraction)**0.5 - line_std = noise_std * line_fraction**0.5 - if shape < (1024, 1024): - pixel = np.random.normal(loc=0, scale=pixel_std, size=shape) - line = np.random.normal(loc=0, scale=line_std, size=shape[1]) - else: - pixel = _pseudo_random_field(scale=pixel_std, size=shape) - line = pixel[0] - - channel_std = noise_std * channel_fraction**0.5 - channel = np.repeat(np.random.normal(loc=0, scale=channel_std, - size=n_channels), w_chan + 1, axis=0) - - ron_frame = (pixel + line).T + channel[:shape[0]] - - return ron_frame diff --git a/scopesim/effects/electronic/pixels.py b/scopesim/effects/electronic/pixels.py index 5c8c8c8cc..2a238c1ac 100644 --- a/scopesim/effects/electronic/pixels.py +++ b/scopesim/effects/electronic/pixels.py @@ -3,6 +3,8 @@ from typing import ClassVar +from numpy.typing import ArrayLike, NDArray + from .. import Effect from ...detector import Detector from ...utils import from_currsys, figure_factory, check_keys, real_colname @@ -46,6 +48,27 @@ def __init__(self, **kwargs): raise ValueError( "Parameter 'border' must have exactly four values.") + def __call__(self, data: ArrayLike, border: dict[int, int]) -> NDArray: + if border[0] > 0: + data[:border[0], :] = 0 + if border[1] > 0: + data[:, :border[1]] = 0 + if border[2] > 0: + data[-border[2]:, :] = 0 + if border[3] > 0: + data[:, -border[3]:] = 0 + return data + + def _get_border(self, det_meta) -> dict[int, int]: + if hasattr(self.meta["border"], "dic"): + dtcr_id = det_meta[real_colname("id", det_meta)] + return self.meta["border"].dic[dtcr_id] + if isinstance(self.meta["border"], list): + return self.meta["border"] + raise ValueError( + f"{self.__class__.__name__}.meta['border'] must be either " + f"dict or list, but is {self.meta['border']}") + def apply_to(self, obj, **kwargs): """Mask border pixels.""" if not isinstance(obj, Detector): @@ -54,24 +77,7 @@ def apply_to(self, obj, **kwargs): return obj logger.info(f"Applying border {from_currsys(self.meta['border'])}") - if hasattr(self.meta["border"], "dic"): - dtcr_id = obj.meta[real_colname("id", obj.meta)] - border = self.meta["border"].dic[dtcr_id] - elif isinstance(self.meta["border"], list): - border = self.meta["border"] - else: - raise ValueError( - ".meta['border'] must be either " - f"dict or list, but is {self.meta['border']}") - - if border[0] > 0: - obj.data[:border[0], :] = 0 - if border[1] > 0: - obj.data[:, :border[1]] = 0 - if border[2] > 0: - obj.data[-border[2]:, :] = 0 - if border[3] > 0: - obj.data[:, -border[3]:] = 0 + obj.data = self(obj.data, self._get_border(obj.meta)) return obj def plot(self, det, **kwargs): @@ -90,44 +96,61 @@ def __str__(self) -> str: return msg -class BinnedImage(Effect): - required_keys = {"bin_size"} +class BinnedImageBase(Effect): + """Base class for binning effects.""" + z_order: ClassVar[tuple[int, ...]] = (870,) def __init__(self, **kwargs): super().__init__(**kwargs) check_keys(self.meta, self.required_keys, action="error") + def __call__(self, data: ArrayLike) -> NDArray: + raise NotImplementedError("Subclasses must implement this.") + def apply_to(self, det, **kwargs): if not isinstance(det, Detector): return det - bs = from_currsys(self.meta["bin_size"], self.cmds) - image = det._hdu.data - h, w = image.shape - new_image = image.reshape((h//bs, bs, w//bs, bs)) - det._hdu.data = new_image.sum(axis=3).sum(axis=1) - + det._hdu.data = self(det._hdu.data) return det -class UnequalBinnedImage(Effect): - required_keys = {"binx","biny"} - z_order: ClassVar[tuple[int, ...]] = (870,) +class BinnedImage(BinnedImageBase): + required_keys = {"bin_size"} - def __init__(self, **kwargs): - super().__init__(**kwargs) - check_keys(self.meta, self.required_keys, action="error") + def __call__(self, data: ArrayLike) -> NDArray: + height, width = data.shape + binned_data = data.reshape(( + height//self.bin_size, + self.bin_size, + width//self.bin_size, + self.bin_size + )).sum(axis=3).sum(axis=1) + return binned_data - def apply_to(self, det, **kwargs): - if not isinstance(det, Detector): - return det + @property + def bin_size(self) -> int: + return from_currsys(self.meta["bin_size"], self.cmds) - bx = from_currsys(self.meta["binx"], self.cmds) - by = from_currsys(self.meta["biny"], self.cmds) - image = det._hdu.data - h, w = image.shape - new_image = image.reshape((h//bx, bx, w//by, by)) - det._hdu.data = new_image.sum(axis=3).sum(axis=1) - return det +class UnequalBinnedImage(BinnedImageBase): + required_keys = {"binx","biny"} + + def __call__(self, data: ArrayLike) -> NDArray: + height, width = data.shape + binned_data = data.reshape(( + height//self.binx, + self.binx, + width//self.biny, + self.binx + )).sum(axis=3).sum(axis=1) + return binned_data + + @property + def binx(self) -> int: + return from_currsys(self.meta["binx"], self.cmds) + + @property + def biny(self) -> int: + return from_currsys(self.meta["biny"], self.cmds) diff --git a/scopesim/effects/rotation.py b/scopesim/effects/rotation.py index 7b208bfcc..d1d48b578 100644 --- a/scopesim/effects/rotation.py +++ b/scopesim/effects/rotation.py @@ -40,5 +40,5 @@ def apply_to(self, obj, **kwargs): return obj rotations = from_currsys(self.meta["rotations"], self.cmds) - obj._hdu.data = np.rot90(obj._hdu.data, rotations) + obj.data = np.rot90(obj.data, rotations) return obj diff --git a/scopesim/tests/tests_effects/test_BasicReadoutNoise.py b/scopesim/tests/tests_effects/test_BasicReadoutNoise.py index 89bc4029c..b8bb69e33 100644 --- a/scopesim/tests/tests_effects/test_BasicReadoutNoise.py +++ b/scopesim/tests/tests_effects/test_BasicReadoutNoise.py @@ -2,7 +2,7 @@ from pytest import approx import numpy as np -from scopesim.effects.electronic.noise import PoorMansHxRGReadoutNoise, _make_ron_frame +from scopesim.effects.electronic.noise import PoorMansHxRGReadoutNoise from scopesim.tests.mocks.py_objects.detector_objects import _basic_detector @@ -39,12 +39,25 @@ def test_noise_reduces_with_square_root_of_ndit(self, ndit, noise_std): class TestMakeRonFrame: @pytest.mark.parametrize("n", (1, 4, 9, 25)) def test_stdev_increase_with_square_of_n(self, n): - frames = np.array([_make_ron_frame((256, 256), 10, 2, 0.1, 0.2, 0.3, 0.4) + ron = PoorMansHxRGReadoutNoise(noise_std=10, n_channels=2, ndit=1) + ron.meta.update({ + "channel_fraction": 0.1, + "line_fraction": 0.2, + "pedestal_fraction": 0.3, + "read_fraction": 0.4, + }) + frames = np.array([ron._make_ron_frame((256, 256)) for _ in range(n)]) assert np.std(np.sum(frames, axis=0)) == approx(10*n**0.5, rel=0.3) @pytest.mark.parametrize("shape", [(3, 7), (7, 3)]) def test_makes_frame_sizes_for_non_integer_n_channels(self, shape): - n_channels = 2 - frame = _make_ron_frame(shape, 5, n_channels, 0.25, 0.25, 0.25, 0.25) + ron = PoorMansHxRGReadoutNoise(noise_std=5, n_channels=2, ndit=1) + ron.meta.update({ + "channel_fraction": 0.25, + "line_fraction": 0.25, + "pedestal_fraction": 0.25, + "read_fraction": 0.25, + }) + frame = ron._make_ron_frame(shape) assert frame.shape > (0, 0) From 779f1462e901d5a3d376d276e1af2a5723766086 Mon Sep 17 00:00:00 2001 From: teutoburg Date: Mon, 31 Aug 2026 23:39:02 +0200 Subject: [PATCH 15/87] Eliminate remaining `Detector._hdu` calls Somehow forgotten in the previous commit... --- scopesim/effects/electronic/electrons.py | 3 +-- scopesim/effects/electronic/noise.py | 11 +++-------- scopesim/effects/electronic/pixels.py | 2 +- scopesim/effects/mosaic_trace_list.py | 4 ++-- .../tests/tests_effects/test_ADConversion.py | 18 +++++++++--------- .../tests_effects/test_BasicReadoutNoise.py | 4 ++-- .../test_InterPixelCapacitance.py | 4 ++-- .../tests/tests_effects/test_LinearityCurve.py | 6 +++--- scopesim/tests/tests_effects/test_exposure.py | 14 +++++++------- .../tests_effects/test_mosaic_trace_list.py | 6 +++--- scopesim/tests/tests_effects/test_prnu.py | 12 ++++++------ .../tests/tests_effects/test_shot_noise.py | 18 +++++++++--------- 12 files changed, 48 insertions(+), 54 deletions(-) diff --git a/scopesim/effects/electronic/electrons.py b/scopesim/effects/electronic/electrons.py index 1135eb283..b28f55b05 100644 --- a/scopesim/effects/electronic/electrons.py +++ b/scopesim/effects/electronic/electrons.py @@ -189,8 +189,7 @@ def apply_to(self, det, **kwargs): det.__class__.__name__) return det - newdata = oaconvolve(det._hdu.data, self.kernel, mode="same") - det._hdu.data = newdata + det.data = oaconvolve(det.data, self.kernel, mode="same") return det def update(self, **kwargs): diff --git a/scopesim/effects/electronic/noise.py b/scopesim/effects/electronic/noise.py index a021143b5..717fec584 100644 --- a/scopesim/effects/electronic/noise.py +++ b/scopesim/effects/electronic/noise.py @@ -143,7 +143,7 @@ def apply_to(self, det, **kwargs): return det ndit = from_currsys(self.meta["ndit"], self.cmds) - det._hdu.data = self(det._hdu.data, ndit) + det.data = self(det.data, ndit) return det def plot(self, det, **kwargs): @@ -190,7 +190,7 @@ def apply_to(self, det, **kwargs): return det ndit = from_currsys(self.meta["ndit"], self.cmds) - det._hdu.data = self(det._hdu.data, ndit) + det.data = self(det.data, ndit) return det def plot(self, det): @@ -354,12 +354,7 @@ def apply_to(self, det, **kwargs): # - numpy.nan are implicitly passed through the normal distribution; # because the Poisson distribution cannot handle them. - new_imagehdu = fits.ImageHDU( - data=self(det._hdu.data), - header=det._hdu.header, - ) - - det._hdu = new_imagehdu + det._hdu = fits.ImageHDU(header=det.header, data=self(det.data)) return det def plot(self, det): diff --git a/scopesim/effects/electronic/pixels.py b/scopesim/effects/electronic/pixels.py index 2a238c1ac..93f746700 100644 --- a/scopesim/effects/electronic/pixels.py +++ b/scopesim/effects/electronic/pixels.py @@ -112,7 +112,7 @@ def apply_to(self, det, **kwargs): if not isinstance(det, Detector): return det - det._hdu.data = self(det._hdu.data) + det.data = self(det.data) return det diff --git a/scopesim/effects/mosaic_trace_list.py b/scopesim/effects/mosaic_trace_list.py index 8b23e3ad3..542755ebf 100644 --- a/scopesim/effects/mosaic_trace_list.py +++ b/scopesim/effects/mosaic_trace_list.py @@ -253,8 +253,8 @@ def apply_to(self, det, **kwargs): type(det)) return det - image = det._hdu.data - detwcs = WCS(det._hdu.header, key="D") + image = det.data + detwcs = WCS(det.header, key="D") output_format = from_currsys(self.meta["format"], self.cmds) diff --git a/scopesim/tests/tests_effects/test_ADConversion.py b/scopesim/tests/tests_effects/test_ADConversion.py index daece3751..eba0ad615 100644 --- a/scopesim/tests/tests_effects/test_ADConversion.py +++ b/scopesim/tests/tests_effects/test_ADConversion.py @@ -32,9 +32,9 @@ def fixture_mock_detector(): def fixture_detector_with_data(mock_detector): """Instantiate a Detector with some data""" det = mock_detector - width = det._hdu.data.shape[1] - det._hdu.data[:] = 1.2 - det._hdu.data[:, width//2] = 1.99 + width = det.data.shape[1] + det.data[:] = 1.2 + det.data[:, width//2] = 1.99 return det @@ -73,11 +73,11 @@ def test_converts_pixels_to_integer_with_custom(self, detector_with_data): def test_applies_gain(self, mock_detector, gain): det = mock_detector data = np.random.randn(100, 100) * 100. + 1000. - det._hdu.data = 1. * data + det.data = 1. * data adconverter = ADConversion(gain=gain) adconverter.cmds = {"!DET.gain": gain, "!OBS.ndit": 1} adconverter.apply_to(det) - assert np.all((data/gain).astype(int) == det._hdu.data) + assert np.all((data/gain).astype(int) == det.data) @pytest.mark.usefixtures("patch_mock_path_micado") def test_applies_gain_list_to_detector_list(self): @@ -104,10 +104,10 @@ def test_caps_data_overflow(self, mock_detector, newtype, maxval, minval): data = medval * np.ones((100, 100), dtype=np.float64) data[50, 50] = 1.e12 data[60, 60] = -100000. - det._hdu.data = data + det.data = data adconverter = ADConversion(gain=gain, dtype=newtype) adconverter.cmds = {"!DET.gain": gain, "!OBS.ndit": 1} adconverter.apply_to(det) - assert(np.median(det._hdu.data) == newtype(medval/gain)) - assert(np.max(det._hdu.data) == maxval) - assert(np.min(det._hdu.data) == minval) + assert(np.median(det.data) == newtype(medval/gain)) + assert(np.max(det.data) == maxval) + assert(np.min(det.data) == minval) diff --git a/scopesim/tests/tests_effects/test_BasicReadoutNoise.py b/scopesim/tests/tests_effects/test_BasicReadoutNoise.py index b8bb69e33..e119dfa29 100644 --- a/scopesim/tests/tests_effects/test_BasicReadoutNoise.py +++ b/scopesim/tests/tests_effects/test_BasicReadoutNoise.py @@ -23,7 +23,7 @@ def test_creates_noise_std_as_needed(self, noise_std): ron = PoorMansHxRGReadoutNoise(noise_std=noise_std, n_channels=64, ndit=1) dtcr = ron.apply_to(dtcr) - assert np.std(dtcr._hdu.data) == approx(noise_std, rel=0.05) + assert np.std(dtcr.data) == approx(noise_std, rel=0.05) @pytest.mark.parametrize("noise_std", [1, 9, 100]) @pytest.mark.parametrize("ndit", [1, 9, 100]) @@ -31,7 +31,7 @@ def test_noise_reduces_with_square_root_of_ndit(self, ndit, noise_std): dtcr = _basic_detector(width=256) ron = PoorMansHxRGReadoutNoise(noise_std=noise_std, n_channels=64, ndit=ndit) dtcr = ron.apply_to(dtcr) - noise_real = np.std(dtcr._hdu.data) + noise_real = np.std(dtcr.data) assert noise_real == approx(noise_std * ndit**0.5, rel=0.1) diff --git a/scopesim/tests/tests_effects/test_InterPixelCapacitance.py b/scopesim/tests/tests_effects/test_InterPixelCapacitance.py index fb6adb5ee..bed34660b 100644 --- a/scopesim/tests/tests_effects/test_InterPixelCapacitance.py +++ b/scopesim/tests/tests_effects/test_InterPixelCapacitance.py @@ -87,7 +87,7 @@ def fixture_detector(): """Instantiate a random detector object""" hdu = fits.ImageHDU(data=np.random.randn(214, 333)) det = Detector(hdu.header) - det._hdu.data = hdu.data + det.data = hdu.data return det class TestApply: @@ -102,7 +102,7 @@ def test_convolve_unit_source(self): [0., 1., 0.], [0., 0., 0.]])) det = Detector(hdu.header) - det._hdu.data = hdu.data + det.data = hdu.data ipc = IPC(kernel=np.random.rand(3, 3)) newdet = ipc.apply_to(det) np.testing.assert_allclose(newdet.hdu.data, ipc.kernel) diff --git a/scopesim/tests/tests_effects/test_LinearityCurve.py b/scopesim/tests/tests_effects/test_LinearityCurve.py index a0612000d..e29a39830 100644 --- a/scopesim/tests/tests_effects/test_LinearityCurve.py +++ b/scopesim/tests/tests_effects/test_LinearityCurve.py @@ -46,11 +46,11 @@ class TestApplyTo: (60, 60), (1e5, 60)]) def test_only_applied_to_detector(self, in_flux, out_flux, basic_lincurve): dtcr = _basic_detector() - dtcr._hdu.data[0, 0] = in_flux + dtcr.data[0, 0] = in_flux new_dtcr = basic_lincurve.apply_to(dtcr) - assert new_dtcr._hdu.data[0, 0] == out_flux - assert np.min(new_dtcr._hdu.data) == 0 + assert new_dtcr.data[0, 0] == out_flux + assert np.min(new_dtcr.data) == 0 def test_bypasses_non_detector_objects(self, basic_lincurve): new_dict = basic_lincurve.apply_to({"gigawatts": 1.21}) diff --git a/scopesim/tests/tests_effects/test_exposure.py b/scopesim/tests/tests_effects/test_exposure.py index 18a1f091d..e3203c353 100644 --- a/scopesim/tests/tests_effects/test_exposure.py +++ b/scopesim/tests/tests_effects/test_exposure.py @@ -39,7 +39,7 @@ def fixture_exposureoutput(): @pytest.fixture(name="detector", scope="function") def fixture_detector(): det = Detector(_image_hdu_square().header) - det._hdu.data[:] = 1.e5 + det.data[:] = 1.e5 return det class TestExposureIntegration: @@ -47,9 +47,9 @@ def test_initialises_correctly(self, exposureintegration): assert isinstance(exposureintegration, ExposureIntegration) def test_integrates_correctly(self, exposureintegration, detector): - orig = 1. * detector._hdu.data + orig = 1. * detector.data assert isinstance(exposureintegration.apply_to(detector), Detector) - assert np.allclose(detector._hdu.data, 4 * orig) + assert np.allclose(detector.data, 4 * orig) def test_fails_without_dit(self): with pytest.raises(ValueError): @@ -91,17 +91,17 @@ def test_cannot_set_to_unknown_mode(self, exposureoutput): (2., 5), (3, 36)]) def test_applies_average(self, dit, ndit, detector): - det_mean = detector._hdu.data.mean() + det_mean = detector.data.mean() exposureoutput = ExposureOutput("average", dit=dit, ndit=ndit) result = exposureoutput.apply_to(detector) - assert np.isclose(result._hdu.data.mean(), det_mean / ndit) + assert np.isclose(result.data.mean(), det_mean / ndit) @pytest.mark.parametrize("dit, ndit", [(1., 1), (2., 5), (3, 36)]) def test_applies_sum(self, dit, ndit, detector): - det_mean = detector._hdu.data.mean() + det_mean = detector.data.mean() exposureoutput = ExposureOutput("sum", dit=dit, ndit=ndit) result = exposureoutput.apply_to(detector) - assert np.isclose(result._hdu.data.mean(), det_mean) + assert np.isclose(result.data.mean(), det_mean) diff --git a/scopesim/tests/tests_effects/test_mosaic_trace_list.py b/scopesim/tests/tests_effects/test_mosaic_trace_list.py index da3207e76..39433af43 100644 --- a/scopesim/tests/tests_effects/test_mosaic_trace_list.py +++ b/scopesim/tests/tests_effects/test_mosaic_trace_list.py @@ -104,7 +104,7 @@ def fixture_detector(): xsize, ysize = 50, 4096 hdr = header_from_list_of_xy([-xsize/2, xsize/2], [-ysize/2, ysize/2], 1, "D") dtcr = Detector(hdr) - dtcr._hdu.data[rows, :] = 1 + dtcr.data[rows, :] = 1 return dtcr @pytest.fixture(name="apply_collapse1d", scope="class") @@ -159,7 +159,7 @@ def test_collapse1d_detector_has_bintable(self, apply_collapse1d): assert isinstance(apply_collapse1d._hdu, fits.BinTableHDU) def test_collapse1d_gives_correct_result(self, apply_collapse1d): - assert np.all(apply_collapse1d._hdu.data['spectrum'] == 7) + assert np.all(apply_collapse1d.data['spectrum'] == 7) @pytest.mark.usefixtures("patch_mock_path_mosaic") @@ -181,7 +181,7 @@ def test_converttotable_detector_has_bintable(self, apply_converttotable): assert isinstance(apply_converttotable._hdu, fits.BinTableHDU) def test_converttotable_gives_correct_result(self, apply_converttotable): - for row in apply_converttotable._hdu.data: + for row in apply_converttotable.data: assert np.all(row['spectrum'] == 1) @pytest.mark.usefixtures("patch_mock_path_mosaic") diff --git a/scopesim/tests/tests_effects/test_prnu.py b/scopesim/tests/tests_effects/test_prnu.py index 2c7bbcbd6..4d74810f5 100644 --- a/scopesim/tests/tests_effects/test_prnu.py +++ b/scopesim/tests/tests_effects/test_prnu.py @@ -11,7 +11,7 @@ def make_detector(value=1000, size=10): hdr = header_from_list_of_xy([-size/2, size/2], [-size/2, size/2], 1, "D") dtcr = Detector(hdr) - dtcr._hdu.data[:] = value + dtcr.data[:] = value return dtcr @@ -21,7 +21,7 @@ def test_output_std_matches_prnu_std(self): prnu_std = 0.05 dtcr = make_detector(value=1000, size=100) PixelResponseNonUniformity(prnu_std=prnu_std, prnu_seed=42).apply_to(dtcr) - rel_std = dtcr._hdu.data.std() / dtcr._hdu.data.mean() + rel_std = dtcr.data.std() / dtcr.data.mean() assert abs(rel_std - prnu_std) < 0.01 def test_gain_map_is_reused(self): @@ -31,24 +31,24 @@ def test_gain_map_is_reused(self): prnu = PixelResponseNonUniformity(prnu_std=0.01, prnu_seed=42) prnu.apply_to(dtcr1) prnu.apply_to(dtcr2) - np.testing.assert_array_equal(dtcr1._hdu.data, dtcr2._hdu.data) + np.testing.assert_array_equal(dtcr1.data, dtcr2.data) def test_dict_prnu_std(self): """Dict mode: different amplitude per detector ID.""" hdr = header_from_list_of_xy([-5, 5], [-5, 5], 1, "D") dtcr = Detector(hdr) dtcr.meta["id"] = "H2RG" - dtcr._hdu.data[:] = 1000 + dtcr.data[:] = 1000 prnu = PixelResponseNonUniformity( prnu_std={"H2RG": 0.005, "GeoSnap": 0.020}, prnu_seed=42) prnu.apply_to(dtcr) - assert dtcr._hdu.data.std() > 0 + assert dtcr.data.std() > 0 def test_multiplicative_zero_signal(self): """Zero signal should remain zero — confirms multiplicative (not additive).""" dtcr = make_detector(value=0) PixelResponseNonUniformity(prnu_std=0.01, prnu_seed=42).apply_to(dtcr) - assert dtcr._hdu.data.sum() == 0 + assert dtcr.data.sum() == 0 def test_non_detector_passthrough(self): """Non-Detector objects should be returned unchanged.""" diff --git a/scopesim/tests/tests_effects/test_shot_noise.py b/scopesim/tests/tests_effects/test_shot_noise.py index ccdca344f..c7cde5b55 100644 --- a/scopesim/tests/tests_effects/test_shot_noise.py +++ b/scopesim/tests/tests_effects/test_shot_noise.py @@ -21,10 +21,10 @@ def test_applies_dark_current_with_level_of_dit(self): value_too_high = 1e20 hdr = header_from_list_of_xy([-hw, hw], [-hw, hw], 1, "D") dtcr = Detector(hdr) - dtcr._hdu.data[0][0] = value_normal - dtcr._hdu.data[0][1] = -1 - dtcr._hdu.data[1][0] = value_too_high - dtcr._hdu.data[1][1] = np.nan + dtcr.data[0][0] = value_normal + dtcr.data[0][1] = -1 + dtcr.data[1][0] = value_too_high + dtcr.data[1][1] = np.nan sn = ShotNoise() # To prevent that data[0][0] does not change by chance. @@ -33,10 +33,10 @@ def test_applies_dark_current_with_level_of_dit(self): dtcr = sn.apply_to(dtcr) # Ensure that the values have changed. - assert dtcr._hdu.data[0][0] != value_normal - assert dtcr._hdu.data[1][0] != value_too_high + assert dtcr.data[0][0] != value_normal + assert dtcr.data[1][0] != value_too_high # Sensibility checks on the values. - assert 1 < dtcr._hdu.data[0][0] < value_normal * 2 - assert np.isnan(dtcr._hdu.data[0][1]) or dtcr._hdu.data[0][1] == 0 - assert np.isnan(dtcr._hdu.data[1][1]) + assert 1 < dtcr.data[0][0] < value_normal * 2 + assert np.isnan(dtcr.data[0][1]) or dtcr.data[0][1] == 0 + assert np.isnan(dtcr.data[1][1]) From a522f1ee8da5a40fcea6eba7468987f1bfb3aaec Mon Sep 17 00:00:00 2001 From: Kieran Leschinski Date: Fri, 4 Sep 2026 18:55:50 +0200 Subject: [PATCH 16/87] Snap the placement origin, not the centre, in overlay_image overlay_image computed the array origin at which to place small_im as ceil(coords) - small_im.shape // 2. That is only correct when coords sits on the pixel lattice small_im's shape implies: half-integer for even shapes, integer for odd ones. For anything in between it is off by up to a whole pixel, and the direction of the error flips with the parity of the shape. Chunked FOVs hit this whenever the detector footprint is not a whole number of pixels across, as MICADO's is not (189.32 mm / 0.015 mm = 12621.33 px in y). The last chunk of each axis is then a fraction of a chunk wide, so it has a different size and a pixel lattice offset from its neighbours', and it got displaced the opposite way from them. The seam between the two picked up a duplicated or a dropped row/column and the image content shifted by a pixel across it. Snap the origin instead, which is parity-independent. Half-integer origins - a genuine sub-pixel shift, which sub_pixel=True would have to handle properly - round towards the lower index, matching what the old expression did for even shapes. --- scopesim/optics/image_plane_utils.py | 33 +++++++++++--- .../tests_optics/test_ImagePlaneUtils.py | 44 +++++++++++++++++++ 2 files changed, 70 insertions(+), 7 deletions(-) diff --git a/scopesim/optics/image_plane_utils.py b/scopesim/optics/image_plane_utils.py index 62f717315..940bcfb44 100644 --- a/scopesim/optics/image_plane_utils.py +++ b/scopesim/optics/image_plane_utils.py @@ -437,13 +437,32 @@ def overlay_image(small_im, big_im, coords, mask=None, sub_pixel=False): coords = np.array([*coords, (big_im.shape[0] - 1) / 2]) # FIXME: this would not be necessary if we used WCS instead of manual 2pix - # Round to 1e-4 pix before ceil: WCS deg-space round-trips leave - # float dust of order 1e-8..1e-6 pix on integer-valued coords, which - # ceil would amplify to a full-pixel shift; genuine sub-pixel intent - # cannot be finer than 0.5 pix here (sub_pixel is not implemented), - # and the half-integer even-shape convention is preserved exactly. - coords = np.ceil(np.asarray(coords).round(4)).astype(np.intp) - idx = coords.astype(int)[::-1] - np.array(small_im.shape) // 2 + # Round to 1e-4 pix first: WCS deg-space round-trips leave float dust of + # order 1e-8..1e-6 pix on integer-valued coords, which the snap below + # would otherwise amplify to a full-pixel shift; genuine sub-pixel intent + # cannot be finer than 0.5 pix here (sub_pixel is not implemented). + coords = np.asarray(coords, dtype=float).round(4) + + # `idx` is the origin, in array order, at which small_im must be placed to + # centre it on `coords` (both are 0-based pixel-centre coordinates, as + # returned by WCS.wcs_world2pix(..., 0)). + # + # Snap that origin, NOT `coords` itself. The old expression + # np.ceil(coords)[::-1] - np.array(small_im.shape) // 2 + # is off by up to a whole pixel whenever `coords` is not on the lattice + # small_im implies (half-integer for even shapes, integer for odd ones), + # and the direction of that error flips with the parity of the shape. Two + # adjacent FOVs of different sizes therefore get displaced opposite ways + # and their shared edge picks up a duplicated or a dropped row/column. + # Snapping the origin instead keeps every placement consistent, whatever + # the shape. + # + # Half-integer origins (a real sub-pixel shift, which sub_pixel=True would + # have to handle properly) are rounded towards the lower index, matching + # what the old expression did for even shapes. + idx = np.ceil( + coords[::-1] - (np.array(small_im.shape) - 1) / 2 - 0.5 + ).astype(np.intp) # Image ranges idx1 = np.maximum(0, idx) diff --git a/scopesim/tests/tests_optics/test_ImagePlaneUtils.py b/scopesim/tests/tests_optics/test_ImagePlaneUtils.py index 8658dd423..dd95ae12f 100644 --- a/scopesim/tests/tests_optics/test_ImagePlaneUtils.py +++ b/scopesim/tests/tests_optics/test_ImagePlaneUtils.py @@ -229,6 +229,50 @@ def test_overlay_images_works_for_3d_cubes_at_border(self): plt.show() +class TestOverlayImageSnapping: + """The origin, not the centre, must be snapped to the pixel grid. + + ``ceil(coords) - shape // 2`` snapped the centre instead, which is off by + up to a whole pixel whenever ``coords`` is not on the lattice implied by + ``small_im.shape``, in a direction that flips with the parity of that + shape. Adjacent FOVs of different sizes were therefore displaced opposite + ways, and their shared edge gained a duplicated or a dropped row/column. + """ + + @pytest.mark.parametrize("size", [1, 2, 3, 4, 7, 8]) + def test_places_image_on_its_own_lattice_without_shift(self, size): + big = np.zeros((32, 32)) + small = np.ones((size, size)) + # centre coordinate that the shape's own lattice implies + centre = 10 + (size - 1) / 2 + imp_utils.overlay_image(small, big, (centre, centre)) + rows = np.where(big.any(axis=1))[0] + assert rows[0] == 10 + assert rows[-1] == 10 + size - 1 + + @pytest.mark.parametrize("size", [1, 2, 3, 4, 7, 8]) + def test_snaps_sub_pixel_offset_the_same_way_for_any_shape(self, size): + # A 0.3 pix offset must never move the image by a whole pixel, whatever + # the parity of the shape. + big = np.zeros((32, 32)) + small = np.ones((size, size)) + centre = 10 + (size - 1) / 2 + 0.3 + imp_utils.overlay_image(small, big, (centre, centre)) + rows = np.where(big.any(axis=1))[0] + assert rows[0] == 10 + assert rows[-1] == 10 + size - 1 + + def test_adjacent_differently_sized_images_tile_exactly(self): + # Mimics chunked FOVs, where the last chunk is smaller than the rest. + big = np.zeros((1, 20)) + origin = 0 + for size in (8, 8, 4): + small = np.ones((1, size)) + imp_utils.overlay_image(small, big, (origin + (size - 1) / 2, 0)) + origin += size + assert (big == 1).all() + + class TestRescaleImageHDU: @pytest.mark.parametrize("pixel_scale", [0.3, 0.5, 1, 2, 3]) def test_rescales_a_2D_imagehdu(self, pixel_scale): From e59648436c92aba269248549768517443599ea85 Mon Sep 17 00:00:00 2001 From: Kieran Leschinski Date: Fri, 4 Sep 2026 18:56:02 +0200 Subject: [PATCH 17/87] Derive the FOV image cutout WCS from the slice it actually took extract_area_from_imagehdu slices the source array on the source's own pixel grid, growing the slice outwards to whole pixels with floor/ceil so it covers the FOV footprint. It then built the accompanying WCS with create_wcs_from_points([xy0s, xy1s]) - the world corners from *before* that growing, rounded independently. The two disagreed in both size and phase. NAXISn could be a pixel smaller than data.shape, which fits.ImageHDU silently corrects on construction, leaving CRPIXn describing an array of the wrong size; and the WCS put the cutout on the FOV's pixel lattice rather than the source's, which for chunked FOVs is offset from it by a fraction of a pixel. Projecting such a cutout mis-registered it by up to a whole pixel, by differing amounts for neighbouring FOVs, so the checkerboard phase flipped across the chunk seams and a row/column was duplicated or dropped there. The FIXME this replaces described the size half of the problem. Build the cutout WCS by shifting the input WCS's CRPIX by the slice origin instead. The cutout's pixels are input pixels, so this is exact: the cutout keeps the input's pixel lattice and NAXISn always agrees with data.shape. The added integration test reassembles a checkerboard from chunked FOVs with chunk_size=8, over four detector footprints including two that are not a whole number of pixels across. All four fail without this commit and the one before it. --- scopesim/optics/fov.py | 39 ++++--- .../test_source_fov_imageplane_play_nicely.py | 100 ++++++++++++++++++ scopesim/tests/tests_optics/test_fov_utls.py | 71 +++++++++++++ 3 files changed, 189 insertions(+), 21 deletions(-) diff --git a/scopesim/optics/fov.py b/scopesim/optics/fov.py index 503214f53..d6d97f247 100644 --- a/scopesim/optics/fov.py +++ b/scopesim/optics/fov.py @@ -340,27 +340,24 @@ def extract_area_from_imagehdu(self, imagehdu, corners): xy1p += (xy0p == xy1p) logger.debug("xy0p: %s; xy1p: %s", xy0p, xy1p) - new_wcs, new_naxis = imp_utils.create_wcs_from_points( - np.array([xy0s, xy1s]).round(11), pixel_scale=hdr["CDELT1"]) - - # TODO: Come back at some point and figure out if the failing tests - # here are relevant or can be ignored... - # FIXME: Commented out for now because it appears too often IRL... - # try: - # roundtrip = new_wcs.wcs_world2pix( - # np.array([xy0s, xy1s - .5*image_wcs.wcs.cdelt]), 0).round(5) - # np.testing.assert_array_equal(roundtrip[0], [0, 0]) - # np.testing.assert_array_equal(roundtrip[1], new_naxis - [1, 1]) - # except AssertionError: - # logger.exception("WCS roundtrip assertion failed.") - # FIXME: Related to the above, this sometimes fails: - # np.testing.assert_equal(xy1p - xy0p, new_naxis) - # This occurs when the floor and ceil in xy0s and xy1s produce an - # off-by-one error. Using .round instead for both would solve things - # in those cases, but breaks in other cases. Find a proper solution! - # Note: This is not super fatal, because the resulting projections - # will trim off that extra pixel later on, but this should still - # be addressed. + # Describe the cutout with the input WCS, shifted by the slice origin. + # The cutout's pixels ARE input pixels, so this is exact: the cutout + # stays on the input's pixel lattice and NAXISn always agrees with + # data.shape. + # + # Deriving the WCS from [xy0s, xy1s] instead (via + # create_wcs_from_points) got both wrong. Those are the world corners + # *before* the floor/ceil above grew the slice outwards to whole input + # pixels, so the resulting WCS described a differently sized image + # (NAXISn disagreed with data.shape by a pixel, silently corrected by + # fits.ImageHDU, which left CRPIXn describing the wrong array) placed + # on the FOV's lattice rather than the input's. Projecting that cutout + # then mis-registered it by up to a whole pixel, and by differing + # amounts for neighbouring FOVs, so the seam between two FOVs picked + # up a duplicated or a dropped row/column. + new_wcs = image_wcs.deepcopy() + new_wcs.wcs.crpix = image_wcs.wcs.crpix - xy0p + new_naxis = xy1p - xy0p new_hdr = new_wcs.to_header() new_hdr.update({"NAXIS1": new_naxis[0], "NAXIS2": new_naxis[1]}) diff --git a/scopesim/tests/tests_integrations/test_source_fov_imageplane_play_nicely.py b/scopesim/tests/tests_integrations/test_source_fov_imageplane_play_nicely.py index 82209d5ab..f87ea5ec2 100644 --- a/scopesim/tests/tests_integrations/test_source_fov_imageplane_play_nicely.py +++ b/scopesim/tests/tests_integrations/test_source_fov_imageplane_play_nicely.py @@ -8,6 +8,9 @@ from astropy import units as u +from astropy.wcs import WCS + +from scopesim.optics import image_plane_utils as imp_utils from scopesim.optics.fov import FieldOfView2D from scopesim.optics.image_plane import ImagePlane @@ -64,3 +67,100 @@ def test_can_extract_the_source_in_a_fov(self, fov_hdr, comb_src, plt.subplot(133) plt.imshow(imp.hdu.data, origin="lower", norm=LogNorm()) plt.show() + +def _checkerboard_source(naxis, pixel_scale, crpix=None): + """0/1 checkerboard image source, one board square per pixel.""" + from astropy.io import fits + from scopesim.source.source import Source + from scopesim.source import source_templates as st + + nx, ny = naxis + wcs = WCS(naxis=2) + wcs.wcs.ctype = ["LINEAR", "LINEAR"] + wcs.wcs.cunit = ["deg", "deg"] + wcs.wcs.cdelt = 2 * [pixel_scale / 3600] + wcs.wcs.crval = [0, 0] + wcs.wcs.crpix = crpix or [(nx + 1) / 2, (ny + 1) / 2] + + hdr = wcs.to_header() + hdr["BUNIT"] = "" + board = (np.indices((ny, nx)).sum(axis=0) % 2).astype(float) + hdu = fits.ImageHDU(header=hdr, data=board) + hdu.header["SPEC_REF"] = 0 + hdu.header["SPECMAG"] = 0 + hdu.header["SPECUNIT"] = "mag" + return Source(image_hdu=hdu, spectra=[st.vega_spectrum()]), board + + +def _chunked_fov_headers(x_mm, y_mm, pixel_size, pixel_scale, chunk): + """Reproduce FOVManager's chunking of a detector footprint.""" + plate_scale = pixel_scale / pixel_size + lims = [v / pixel_size * pixel_scale + for v in (x_mm[0], x_mm[1], y_mm[0], y_mm[1])] + + def edges(vmin, vmax): + out = [vmin] + for val in np.arange(vmin, vmax, chunk * pixel_scale): + if vmin < val < vmax and val > out[-1]: + out.append(float(val)) + return out + [vmax] + + for x0, x1 in zip(*(lambda e: (e[:-1], e[1:]))(edges(lims[0], lims[1]))): + for y0, y1 in zip(*(lambda e: (e[:-1], e[1:]))(edges(lims[2], lims[3]))): + skyhdr = imp_utils.header_from_list_of_xy( + [x0 / 3600, x1 / 3600], [y0 / 3600, y1 / 3600], + pixel_scale=pixel_scale / 3600) + dethdr, _ = imp_utils.det_wcs_from_sky_wcs( + WCS(skyhdr), pixel_scale, plate_scale) + skyhdr.update(dethdr.to_header()) + yield skyhdr + + +class TestChunkedFOVsTileTheImagePlane: + """A checkerboard must survive being cut into chunked FOVs and reassembled. + + ``chunk_size`` is 8 here so the whole thing runs in well under a second, + but the geometry is MICADO's: the detector footprint is not a whole number + of pixels across (MICADO's y axis is 189.32 mm / 0.015 mm = 12621.33 px), + so the last chunk in each axis is a fraction of a chunk wide and lands on + a pixel lattice offset from its neighbours'. The projection used to snap + those chunks inconsistently, which put a duplicated or a dropped + row/column at the seam and flipped the checkerboard phase across it. + """ + + PIXEL_SIZE = 0.01 # mm / pix + PIXEL_SCALE = 0.2 # arcsec / pix + CHUNK = 8 + + @pytest.mark.parametrize("half_mm, exact_extent", [ + (0.12, 24.0), # whole pixels, whole chunks + (0.105, 21.0), # whole pixels, fractional last chunk + (0.0866, 17.32), # fractional pixels -- MICADO's y axis + (0.0855, 17.10), + ]) + def test_checkerboard_survives_chunking(self, half_mm, exact_extent): + x_mm = y_mm = (-half_mm, half_mm) + assert (x_mm[1] - x_mm[0]) / self.PIXEL_SIZE == approx(exact_extent) + + imphdr = imp_utils.header_from_list_of_xy( + list(x_mm), list(y_mm), pixel_scale=self.PIXEL_SIZE, wcs_suffix="D") + imp = ImagePlane(imphdr) + + source, board = _checkerboard_source( + (imphdr["NAXIS1"], imphdr["NAXIS2"]), self.PIXEL_SCALE, + crpix=[imphdr["CRPIX1D"], imphdr["CRPIX2D"]]) + + fov_hdrs = list(_chunked_fov_headers( + x_mm, y_mm, self.PIXEL_SIZE, self.PIXEL_SCALE, self.CHUNK)) + assert len(fov_hdrs) == 9 + + for hdr in fov_hdrs: + fov = FieldOfView2D(hdr, waverange=[0.5, 2.5] * u.um, + area=1 * u.m**2) + fov.extract_from(source) + fov.view() + imp.add(fov.hdu, wcs_suffix="D") + + # no gaps and no double-counting at the chunk seams + got = imp.data / imp.data.max() + np.testing.assert_allclose(got, board, atol=1e-6) diff --git a/scopesim/tests/tests_optics/test_fov_utls.py b/scopesim/tests/tests_optics/test_fov_utls.py index ba08540dd..5a782a9b6 100644 --- a/scopesim/tests/tests_optics/test_fov_utls.py +++ b/scopesim/tests/tests_optics/test_fov_utls.py @@ -6,6 +6,8 @@ from synphot import Empirical1D, SourceSpectrum from synphot.units import PHOTLAM from astropy import units as u +from astropy.io import fits +from astropy.wcs import WCS from scopesim.optics.fov import FieldOfView, extract_range_from_spectrum from scopesim.optics import image_plane_utils as imp_utils @@ -128,3 +130,72 @@ def test_logs_msg_for_waverang_overlap_mismatch( extract_range_from_spectrum(spec, waverange) assert msg in caplog.text + +class TestExtractAreaKeepsTheInputPixelGrid: + """The cutout is made of input pixels, so its WCS must say so. + + The cutout WCS used to be re-derived from the world corners *before* the + slice was grown outwards to whole input pixels. That gave a WCS for a + differently sized image (``NAXISn`` disagreed with ``data.shape``, silently + corrected by ``fits.ImageHDU``, which left ``CRPIXn`` describing the wrong + array) sitting on the FOV's pixel lattice rather than the input's. The + cutout was then mis-registered by up to a whole pixel when projected, and + by differing amounts for neighbouring FOVs, so their shared edge gained a + duplicated or a dropped row/column. + """ + + @staticmethod + def _image_hdu(naxis=32, pixel_scale=0.1, crval=(0.0, 0.0)): + """Image with a distinct value in every pixel, in deg.""" + skywcs = WCS(naxis=2) + skywcs.wcs.ctype = ["LINEAR", "LINEAR"] + skywcs.wcs.cunit = ["deg", "deg"] + skywcs.wcs.cdelt = 2 * [pixel_scale / 3600] + skywcs.wcs.crval = list(crval) + skywcs.wcs.crpix = 2 * [(naxis + 1) / 2] + hdr = skywcs.to_header() + hdr["BUNIT"] = "" + data = np.arange(naxis * naxis, dtype=float).reshape(naxis, naxis) + return fits.ImageHDU(header=hdr, data=data) + + @staticmethod + def _fov(x_min, x_max, y_min, y_max, pixel_scale=0.1): + skyhdr = imp_utils.header_from_list_of_xy( + [x_min / 3600, x_max / 3600], [y_min / 3600, y_max / 3600], + pixel_scale=pixel_scale / 3600) + dethdr, _ = imp_utils.det_wcs_from_sky_wcs( + WCS(skyhdr), pixel_scale, pixel_scale / 0.01) + skyhdr.update(dethdr.to_header()) + return FieldOfView(skyhdr, [1.0, 2.0]) + + # FOVs deliberately offset from the image's pixel edges by a fraction of a + # pixel, which is what chunking produces whenever the detector extent is + # not a whole number of pixels. + @pytest.mark.parametrize("offset", [0.0, 0.16, 0.34, 0.5, 0.68, 0.84]) + def test_naxis_matches_data_shape(self, offset): + hdu = self._image_hdu() + fov = self._fov(-0.8 + offset * 0.1, 0.0 + offset * 0.1, + -0.8 + offset * 0.1, 0.0 + offset * 0.1) + cut = fov.extract_area_from_imagehdu(hdu, fov.get_corners("deg")[0]) + assert (cut.header["NAXIS2"], cut.header["NAXIS1"]) == cut.data.shape + + @pytest.mark.parametrize("offset", [0.0, 0.16, 0.34, 0.5, 0.68, 0.84]) + def test_cutout_pixels_keep_their_world_coordinates(self, offset): + hdu = self._image_hdu() + fov = self._fov(-0.8 + offset * 0.1, 0.0 + offset * 0.1, + -0.8 + offset * 0.1, 0.0 + offset * 0.1) + cut = fov.extract_area_from_imagehdu(hdu, fov.get_corners("deg")[0]) + + # every cutout value must sit at the world coordinate its value had in + # the input image + in_wcs = WCS(hdu.header, naxis=2) + cut_wcs = WCS(cut.header, naxis=2) + ny, nx = cut.data.shape + yy, xx = np.indices((ny, nx)) + world = cut_wcs.wcs_pix2world( + np.column_stack([xx.ravel(), yy.ravel()]), 0) + back = in_wcs.wcs_world2pix(world, 0).round(6) + expected = hdu.data[back[:, 1].astype(int), back[:, 0].astype(int)] + np.testing.assert_array_equal(cut.data.ravel(), expected) + # ... and land on whole input pixels, not between them + np.testing.assert_allclose(back, back.round(), atol=1e-6) From bb100a8b43a8fa437ad8003961ceb5b63d5e3995 Mon Sep 17 00:00:00 2001 From: Kieran Leschinski Date: Fri, 4 Sep 2026 20:08:07 +0200 Subject: [PATCH 18/87] Share one pixel lattice between an image plane and its chunks This is the last of the three off-by-one causes and the most invasive of them: it changes CRVALn for any region that is not a whole number of pixels across. It is separable from the two commits before it - drop it and the dropped row and the phase flips stay fixed, but MICADO keeps one duplicated row at the last chunk boundary in y. create_wcs_from_points rounds the requested extent to a whole NAXISn and then centres the grid on the region, folding that region's own rounding remainder into CRVAL. Each region therefore gets its own sub-pixel lattice. Chunked FOVs are all carved from one origin in whole-pixel steps - the image plane, the full-size chunks and the smaller last chunk all measure from the same detector edge - so they ought to share a lattice, and instead ended up offset from one another by fractions of a pixel. The projection then has to snap them to whole pixels, and the last chunk could land a whole pixel off its neighbours. MICADO's detector plane is 189.32 mm / 0.015 mm = 12621.33 px tall, so it hits this on every run: with chunk_size 2048 the last chunk in y is 333.33 px and its lattice is offset from the six full chunks'. When NAXISn truncates the region, anchor the lower pixel edge at points.min and let the leftover fraction fall off the far edge; regions carved from the same origin then coincide. When NAXISn covers the region there is nothing to truncate, so keep centring - that is what a region smaller than one pixel needs, where NAXISn was clamped up to 1. Verified on the full MICADO array (12621 x 12888 image plane, 4 mas checkerboard, chunk_size 2048), checking that the board's row and column phase alternates strictly across the whole image plane: origin/main rows: 10407 bad, cols: 10408 bad, plus a dropped row at y = 12288 first two commits rows: 166 bad, cols: 0 bad with this commit rows: 0 bad, cols: 0 bad --- scopesim/optics/image_plane_utils.py | 36 ++++++++++++++- .../tests_optics/test_ImagePlaneUtils.py | 46 +++++++++++++++++++ 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/scopesim/optics/image_plane_utils.py b/scopesim/optics/image_plane_utils.py index 940bcfb44..b6feb9230 100644 --- a/scopesim/optics/image_plane_utils.py +++ b/scopesim/optics/image_plane_utils.py @@ -214,7 +214,41 @@ def create_wcs_from_points( offset[zeroaxis] = 0 crpix = (naxis + 1) / 2 - crval = (points.min(axis=0) + points.max(axis=0) + offset) / 2 + + # Where to put the pixel lattice when NAXISn cannot cover the requested + # region exactly. + # + # If the grid truncates the region (naxis < extent, i.e. the region is not + # a whole number of pixels across), anchor the lower pixel edge at + # points.min and let the leftover fraction fall off the far edge. Every + # region carved out of the same origin in whole-pixel steps then shares one + # lattice - which is what chunked FOVs are: the image plane, the full-size + # chunks and the smaller last chunk all measure from the same detector + # edge. Centring each of them on its own midpoint instead, and absorbing + # its own rounding remainder into CRVAL, put them on lattices offset from + # one another by fractions of a pixel, so the projection had to snap them + # and the last chunk could land a whole pixel off its neighbours. + # + # MICADO's detector plane is 189.32 mm / 0.015 mm = 12621.33 px tall, so it + # hits this on every run. + # + # If the grid covers the region (naxis >= extent) there is nothing to + # truncate, so keep centring it on the region. That is the right thing for + # a region smaller than a single pixel, where naxis was clamped up to 1. + _pxs = float(pixel_scale.value if isinstance(pixel_scale, u.Quantity) + else pixel_scale) + _pts = points.value if isinstance(points, u.Quantity) else np.asarray(points) + _naxis = np.asarray(naxis, dtype=float) + _extent = np.atleast_1d(np.asarray(extent, dtype=float)) + + crval = np.where( + _naxis < _extent, + _pts.min(axis=0) + (_naxis / 2) * _pxs, # anchored + (_pts.min(axis=0) + _pts.max(axis=0) + + np.asarray(offset, dtype=float)) / 2, # centred + ) + if isinstance(points, u.Quantity): + crval = crval * points.unit # Cannot do `in "DX"` here because that would also match the empty string. linsuff = {"D", "X"} diff --git a/scopesim/tests/tests_optics/test_ImagePlaneUtils.py b/scopesim/tests/tests_optics/test_ImagePlaneUtils.py index dd95ae12f..d5c65c3e4 100644 --- a/scopesim/tests/tests_optics/test_ImagePlaneUtils.py +++ b/scopesim/tests/tests_optics/test_ImagePlaneUtils.py @@ -229,6 +229,52 @@ def test_overlay_images_works_for_3d_cubes_at_border(self): plt.show() +class TestLatticeIsSharedBetweenChunks: + """Regions carved from one origin in whole-pixel steps share a lattice. + + This is what chunked FOVs are: the image plane, the full-size chunks and + the smaller last chunk are all measured from the same detector edge. When + the region is not a whole number of pixels across, CRVAL used to absorb + each region's own rounding remainder, putting them on lattices offset from + one another by fractions of a pixel. + """ + + @pytest.mark.parametrize("extent_px", [24.0, 21.0, 17.32, 17.1, 12621 + 1/3]) + @pytest.mark.parametrize("chunk", [4, 8]) + def test_chunks_land_on_the_parent_lattice(self, extent_px, chunk): + pixel_scale = 0.1 + lo = -extent_px / 2 * pixel_scale + hi = lo + extent_px * pixel_scale + + parent, parent_naxis = imp_utils.create_wcs_from_points( + np.array([[lo, lo], [hi, hi]]), pixel_scale) + + edges = [lo] + for val in np.arange(lo, hi, chunk * pixel_scale): + if lo < val < hi and val > edges[-1]: + edges.append(float(val)) + edges.append(hi) + + for x0, x1 in zip(edges[:-1], edges[1:]): + child, child_naxis = imp_utils.create_wcs_from_points( + np.array([[x0, x0], [x1, x1]]), pixel_scale) + # the child's pixel 0 must sit on a whole-pixel offset of the + # parent's grid, otherwise the projection has to snap it + world = child.wcs_pix2world([[0, 0]], 0) + pix = parent.wcs_world2pix(world, 0)[0] + np.testing.assert_allclose(pix, np.round(pix), atol=1e-6) + + @pytest.mark.parametrize("pnts, pixel_scale", [ + (np.array([[0, 0]]), 5), # single point + (np.array([[-1, -1], [-1, 1], [1, -1], [1, 1]]), 5), # sub-pixel + ]) + def test_regions_smaller_than_a_pixel_stay_centred(self, pnts, pixel_scale): + wcs, naxis = imp_utils.create_wcs_from_points(pnts, pixel_scale) + np.testing.assert_array_equal(naxis, [1, 1]) + np.testing.assert_array_equal(wcs.wcs.crval, + pnts.mean(axis=0)) + + class TestOverlayImageSnapping: """The origin, not the centre, must be snapped to the pixel grid. From 3fd301db000d990d75219b9c378ce3607def6fb4 Mon Sep 17 00:00:00 2001 From: Kieran Leschinski Date: Fri, 4 Sep 2026 21:57:06 +0200 Subject: [PATCH 19/87] Generate chunk edges without np.arange's platform-dependent endpoint FOVManager._get_splits used np.arange(v_min, v_max, step) for the chunk boundaries. That returns ceil((v_max - v_min) / step) elements, which is one too many whenever the division rounds just above an integer - as it does for a span that is an exact multiple of step: (1.2000000000000002 - -1.2000000000000002) / 0.8 == 3.0000000000000004 Whether the surplus element lands exactly on v_max or one ulp below it depends on how the platform rounds v_min + i * step. It lands on v_max on x86, where FovVolumeList.split then discards it (v_max <= value). On arm64 macOS it lands below v_max, which is a perfectly legal split point, so split accepts it and carves off a volume ~1e-15 px wide. create_wcs_from_points rounds that up to a whole pixel, so it becomes a spurious one-pixel FOV, off the pixel lattice, adding flux at the edge of the array. A detector footprint that is an exact multiple of chunk_size therefore got 4 extra degenerate FOVs on macOS and none on Linux or Windows. Emulating the arm64 rounding locally, a 24 px footprint at chunk_size 8 gives: np.arange, x86 rounding 9 volumes, none degenerate np.arange, arm64 rounding 16 volumes, 4 of width 0 chunk_edges 9 volumes, none degenerate Compute the edges from the chunk count instead and return only interior points, so no edge can coincide with either bound on any platform. step still need not divide the span or be an integer, so this is not a range(). The two test helpers added in the previous commits duplicated the arange expression and so inherited the same fragility - that is what failed CI on macos-latest, on exactly the exact-multiple parametrisations. They now call chunk_edges rather than reimplementing it. Behaviour on x86 is unchanged: the full MICADO image plane (12621 x 12888) comes out bit-identical to the previous commit. --- scopesim/optics/fov_manager.py | 33 +++++++++-- .../test_source_fov_imageplane_play_nicely.py | 15 +++-- .../tests/tests_optics/test_FOVManager.py | 59 ++++++++++++++++++- .../tests_optics/test_ImagePlaneUtils.py | 8 +-- 4 files changed, 96 insertions(+), 19 deletions(-) diff --git a/scopesim/optics/fov_manager.py b/scopesim/optics/fov_manager.py index 14157fb3b..5dc3d64f4 100644 --- a/scopesim/optics/fov_manager.py +++ b/scopesim/optics/fov_manager.py @@ -61,6 +61,32 @@ logger = get_logger(__name__) +def chunk_edges(vmin, vmax, step): + """Return the interior points that cut ``[vmin, vmax]`` into `step` chunks. + + Only interior points are returned, so no edge can coincide with `vmin` or + `vmax`. `step` need not divide the span, and need not be an integer (see + HAWKI/test_hawki/test_full_package_hawki.py), so this cannot be a plain + ``range``. + + It cannot be ``np.arange(vmin, vmax, step)`` either. That returns + ``ceil((vmax - vmin) / step)`` elements, which is one too many when the + division rounds just above an integer - as it does for a span that is an + exact multiple of `step`, e.g. ``(1.2000000000000002 - -1.2000000000000002) + / 0.8 == 3.0000000000000004``. Whether the surplus element then lands + exactly on `vmax` or one ulp below it depends on how the platform rounds + ``vmin + i * step``: it lands on `vmax` on x86, but below it on arm64 + macOS. One ulp below `vmax` is a perfectly legal split point, so + ``FovVolumeList.split`` accepts it and carves off a volume ~1e-15 px wide, + which ``create_wcs_from_points`` then rounds up to a whole pixel - a + spurious one-pixel FOV, off the pixel lattice, double-counting flux at the + edge of the array. + + """ + n_chunks = max(1, int(np.ceil((vmax - vmin) / step - 1e-9))) + return vmin + step * np.arange(1, n_chunks) + + class FOVManager: """ A class to manage the (monochromatic) image windows covering the target. @@ -122,11 +148,8 @@ def _get_splits(self, pixel_scale): if vol_pix_area > max_seg_size: step = chunk_size * pixel_scale - # These are not always integers, unlike in the tests. - # See for example HAWKI/test_hawki/test_full_package_hawki.py. - # The np.arange can therefore not be changed to just a range. - yield (np.arange(vol["x_min"], vol["x_max"], step), - np.arange(vol["y_min"], vol["y_max"], step)) + yield (chunk_edges(vol["x_min"], vol["x_max"], step), + chunk_edges(vol["y_min"], vol["y_max"], step)) def generate_fovs_list(self) -> Iterator[FieldOfView]: """ diff --git a/scopesim/tests/tests_integrations/test_source_fov_imageplane_play_nicely.py b/scopesim/tests/tests_integrations/test_source_fov_imageplane_play_nicely.py index f87ea5ec2..8f7bd3899 100644 --- a/scopesim/tests/tests_integrations/test_source_fov_imageplane_play_nicely.py +++ b/scopesim/tests/tests_integrations/test_source_fov_imageplane_play_nicely.py @@ -12,6 +12,7 @@ from scopesim.optics import image_plane_utils as imp_utils from scopesim.optics.fov import FieldOfView2D +from scopesim.optics.fov_manager import chunk_edges from scopesim.optics.image_plane import ImagePlane from scopesim.tests.mocks.py_objects import source_objects as src @@ -99,14 +100,12 @@ def _chunked_fov_headers(x_mm, y_mm, pixel_size, pixel_scale, chunk): for v in (x_mm[0], x_mm[1], y_mm[0], y_mm[1])] def edges(vmin, vmax): - out = [vmin] - for val in np.arange(vmin, vmax, chunk * pixel_scale): - if vmin < val < vmax and val > out[-1]: - out.append(float(val)) - return out + [vmax] - - for x0, x1 in zip(*(lambda e: (e[:-1], e[1:]))(edges(lims[0], lims[1]))): - for y0, y1 in zip(*(lambda e: (e[:-1], e[1:]))(edges(lims[2], lims[3]))): + # the real chunk-edge helper, not a copy of it + return [vmin, *chunk_edges(vmin, vmax, chunk * pixel_scale), vmax] + + xe, ye = edges(lims[0], lims[1]), edges(lims[2], lims[3]) + for x0, x1 in zip(xe[:-1], xe[1:]): + for y0, y1 in zip(ye[:-1], ye[1:]): skyhdr = imp_utils.header_from_list_of_xy( [x0 / 3600, x1 / 3600], [y0 / 3600, y1 / 3600], pixel_scale=pixel_scale / 3600) diff --git a/scopesim/tests/tests_optics/test_FOVManager.py b/scopesim/tests/tests_optics/test_FOVManager.py index e1a055e71..f5f094ddf 100644 --- a/scopesim/tests/tests_optics/test_FOVManager.py +++ b/scopesim/tests/tests_optics/test_FOVManager.py @@ -3,7 +3,7 @@ import numpy as np from astropy import units as u -from scopesim.optics.fov_manager import FOVManager +from scopesim.optics.fov_manager import FOVManager, chunk_edges from scopesim.tests.mocks.py_objects import effects_objects as eo from scopesim.utils import from_currsys @@ -56,3 +56,60 @@ def test_returns_n_fovs_for_smaller_chunk_size(self, chunk_size, n_fovs): assert len(fovs) == 4 assert fov_skycorners.min(axis=0)[0] == approx(-1024 / 3600) # [deg] 2k detector / pixel_scale assert fovs[0].waverange[0] == 0.6 * u.um # filter blue edge + + +class TestChunkEdges: + """Chunk edges must be interior and platform-independent. + + np.arange(vmin, vmax, step) returns one element too many when + (vmax - vmin) / step rounds just above an integer, and whether that + element lands on vmax or one ulp below it depends on how the platform + rounds vmin + i * step. One ulp below vmax is a legal split point, so + FovVolumeList.split would carve off a ~1e-15 px volume that + create_wcs_from_points then rounds up into a spurious one-pixel FOV. + """ + + @pytest.mark.parametrize("span_px, chunk, n_chunks", [ + (24.0, 8, 3), # exact multiple -- the fragile case + (24.0, 4, 6), # exact multiple + (16.0, 8, 2), + (21.0, 8, 3), # fractional last chunk + (17.32, 8, 3), # fractional pixels, fractional last chunk + (12621 + 1 / 3, 2048, 7), # MICADO's y axis + (5.0, 8, 1), # smaller than one chunk + ]) + def test_edges_are_interior_and_count_is_right(self, span_px, chunk, + n_chunks): + pixel_scale = 0.1 + vmin = -span_px / 2 * pixel_scale + vmax = vmin + span_px * pixel_scale + edges = chunk_edges(vmin, vmax, chunk * pixel_scale) + + assert len(edges) == n_chunks - 1 + assert (edges > vmin).all() + assert (edges < vmax).all() + + @pytest.mark.parametrize("span_px, chunk", [(24.0, 8), (24.0, 4), + (16.0, 8), (32.0, 16)]) + def test_exact_multiples_give_no_sliver_chunk(self, span_px, chunk): + # A span that is an exact multiple of the chunk must come out as + # equal chunks, with no degenerate remainder however the float + # arithmetic rounds. + pixel_scale = 0.1 + vmin = -span_px / 2 * pixel_scale + vmax = vmin + span_px * pixel_scale + edges = chunk_edges(vmin, vmax, chunk * pixel_scale) + + bounds = np.array([vmin, *edges, vmax]) + widths = np.diff(bounds) / pixel_scale + assert len(widths) == span_px / chunk + np.testing.assert_allclose(widths, chunk, atol=1e-9) + + def test_a_split_one_ulp_inside_the_upper_bound_is_not_emitted(self): + # np.arange does emit such a value on arm64 macOS; assert directly + # that chunk_edges never does, on any platform. + vmin, vmax, step = -1.2000000000000002, 1.2000000000000002, 0.8 + edges = chunk_edges(vmin, vmax, step) + assert (edges < np.nextafter(vmax, -np.inf)).all() + # ... which np.arange does not guarantee + assert len(np.arange(vmin, vmax, step)) == len(edges) + 2 diff --git a/scopesim/tests/tests_optics/test_ImagePlaneUtils.py b/scopesim/tests/tests_optics/test_ImagePlaneUtils.py index d5c65c3e4..1832ec8e1 100644 --- a/scopesim/tests/tests_optics/test_ImagePlaneUtils.py +++ b/scopesim/tests/tests_optics/test_ImagePlaneUtils.py @@ -7,6 +7,7 @@ import matplotlib.pyplot as plt from scopesim.optics import image_plane_utils as imp_utils +from scopesim.optics.fov_manager import chunk_edges from scopesim.tests.mocks.py_objects import imagehdu_objects as imo @@ -249,11 +250,8 @@ def test_chunks_land_on_the_parent_lattice(self, extent_px, chunk): parent, parent_naxis = imp_utils.create_wcs_from_points( np.array([[lo, lo], [hi, hi]]), pixel_scale) - edges = [lo] - for val in np.arange(lo, hi, chunk * pixel_scale): - if lo < val < hi and val > edges[-1]: - edges.append(float(val)) - edges.append(hi) + # the real chunk-edge helper, not a copy of it + edges = [lo, *chunk_edges(lo, hi, chunk * pixel_scale), hi] for x0, x1 in zip(edges[:-1], edges[1:]): child, child_naxis = imp_utils.create_wcs_from_points( From 0070eafe3648c4f790cd8ab6e20f3b0f40a4b46a Mon Sep 17 00:00:00 2001 From: teutoburg Date: Sat, 5 Sep 2026 19:24:42 +0200 Subject: [PATCH 20/87] Comments and cleanup --- scopesim/effects/detector_list.py | 6 ++++-- scopesim/optics/fov_manager.py | 5 ++--- scopesim/optics/image_plane_utils.py | 4 ---- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/scopesim/effects/detector_list.py b/scopesim/effects/detector_list.py index 61fd4fcba..821229394 100644 --- a/scopesim/effects/detector_list.py +++ b/scopesim/effects/detector_list.py @@ -497,9 +497,11 @@ def _get_fov_limits(self, pixel_scale=None): def image_plane_header(self): """Create and return the Image Plane Header.""" # FIXME: Heavy property..... - points = self._get_corner_points().value new_wcs, naxis = create_wcs_from_points( - points, self.pixel_size.to_value(u.mm), "D") + self._get_corner_points(), + self.pixel_size.to(u.mm), + wcs_suffix="D", + ) hdr = fits.Header() hdr["NAXIS"] = 3 diff --git a/scopesim/optics/fov_manager.py b/scopesim/optics/fov_manager.py index 5dc3d64f4..af062eac8 100644 --- a/scopesim/optics/fov_manager.py +++ b/scopesim/optics/fov_manager.py @@ -170,7 +170,7 @@ def generate_fovs_list(self) -> Iterator[FieldOfView]: for effect in self.effects: self.volumes_list = effect.apply_to(self.volumes_list, **params) - # ..todo: add catch to split volumes larger than chunk_size + # TODO: add catch to split volumes larger than chunk_size pixel_scale = from_currsys(self.meta["pixel_scale"], self.cmds) plate_scale = from_currsys(self.meta["plate_scale"], self.cmds) @@ -182,7 +182,6 @@ def generate_fovs_list(self) -> Iterator[FieldOfView]: for vol in self.volumes_list: xs_min, xs_max = vol["x_min"] / 3600., vol["x_max"] / 3600. ys_min, ys_max = vol["y_min"] / 3600., vol["y_max"] / 3600. - waverange = (vol["wave_min"], vol["wave_max"]) skyhdr = ipu.header_from_list_of_xy([xs_min, xs_max], [ys_min, ys_max], pixel_scale=pixel_scale / 3600.) @@ -210,7 +209,7 @@ def generate_fovs_list(self) -> Iterator[FieldOfView]: new_fov = fovcls( skyhdr, - waverange, + (vol["wave_min"], vol["wave_max"]), detector_header=dethdr, cmds=self.cmds, **vol["meta"], diff --git a/scopesim/optics/image_plane_utils.py b/scopesim/optics/image_plane_utils.py index b6feb9230..7cb70d2bc 100644 --- a/scopesim/optics/image_plane_utils.py +++ b/scopesim/optics/image_plane_utils.py @@ -1221,8 +1221,6 @@ def det_wcs_from_sky_wcs( Shape of the image (``NAXIS1``, ``NAXIS2``). """ - # TODO: Using astropy units for now to avoid deg vs. arcsec confusion. - # Once Scopesim is consistent there, remove astropy units. pixel_scale <<= u.arcsec / u.pixel plate_scale <<= u.arcsec / u.mm logger.debug("Pixel scale: %s", pixel_scale) @@ -1268,8 +1266,6 @@ def sky_wcs_from_det_wcs(det_wcs: WCS, Shape of the image (``NAXIS1``, ``NAXIS2``). """ - # TODO: Using astropy units for now to avoid deg vs. arcsec confusion. - # Once Scopesim is consistent there, remove astropy units. pixel_scale <<= u.arcsec / u.pixel plate_scale <<= u.arcsec / u.mm logger.debug("Pixel scale: %s", pixel_scale) From ca8b8eb1efff6f15de81869d7b2315b8b6a3a04d Mon Sep 17 00:00:00 2001 From: teutoburg Date: Sun, 6 Sep 2026 02:23:49 +0200 Subject: [PATCH 21/87] Careful when cherry-picking... --- scopesim/effects/detector_list.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scopesim/effects/detector_list.py b/scopesim/effects/detector_list.py index 821229394..f3a1a5f09 100644 --- a/scopesim/effects/detector_list.py +++ b/scopesim/effects/detector_list.py @@ -499,7 +499,7 @@ def image_plane_header(self): # FIXME: Heavy property..... new_wcs, naxis = create_wcs_from_points( self._get_corner_points(), - self.pixel_size.to(u.mm), + self.pixel_size.to(u.mm) / u.pixel, wcs_suffix="D", ) From f37016765dec4af64b7a5c571c2f19da0823e86d Mon Sep 17 00:00:00 2001 From: teutoburg Date: Sun, 30 Aug 2026 23:05:14 +0200 Subject: [PATCH 22/87] Remove `apply_throughput_to_cube` This was only used in two places, and one of them has now been refactored to use something else, so no point in keeping the separate function, which in the end was just a one-liner anyway. --- scopesim/effects/spectral_efficiency.py | 13 ++++++----- scopesim/effects/ter_curves_utils.py | 30 ------------------------- 2 files changed, 8 insertions(+), 35 deletions(-) diff --git a/scopesim/effects/spectral_efficiency.py b/scopesim/effects/spectral_efficiency.py index 35fe76fc3..fde4c8297 100644 --- a/scopesim/effects/spectral_efficiency.py +++ b/scopesim/effects/spectral_efficiency.py @@ -11,7 +11,6 @@ from .effects import Effect from .ter_curves import TERCurve -from .ter_curves_utils import apply_throughput_to_cube from ..utils import figure_factory, get_logger @@ -91,9 +90,12 @@ def get_efficiencies(self): wavelength = tbl['wavelength'].quantity efficiency = tbl['efficiency'].value params.pop("filename", None) # don't pass filename to TERCurve! - effic_curve = TERCurve(array_dict={"wavelength":wavelength, - "transmission":efficiency}, - **params) + effic_curve = TERCurve( + array_dict={ + "wavelength": wavelength, + "transmission": efficiency, + }, + **params) efficiencies[name] = effic_curve hdul.close() @@ -111,7 +113,8 @@ def apply_to(self, obj, **kwargs): swcs = WCS(obj.hdu.header).spectral with u.set_enabled_equivalencies(u.spectral()): wave = swcs.pixel_to_world(np.arange(swcs.pixel_shape[0])) << u.um - obj.hdu = apply_throughput_to_cube(obj.hdu, effic.throughput, wave) + thru = effic.throughput(wave) + obj.hdu.data = obj.hdu.data * thru.value[:, None, None] return obj def plot(self): diff --git a/scopesim/effects/ter_curves_utils.py b/scopesim/effects/ter_curves_utils.py index 827bf742a..8dcf62320 100644 --- a/scopesim/effects/ter_curves_utils.py +++ b/scopesim/effects/ter_curves_utils.py @@ -359,36 +359,6 @@ def scale_spectrum(spectrum, filter_name, amplitude): return spectrum -def apply_throughput_to_cube( - cube: fits.ImageHDU, - thru: SpectralElement | SourceSpectrum, - wave_cube: u.Quantity, -) -> fits.ImageHDU: - """ - Apply throughput curve to a spectroscopic cube. - - Parameters - ---------- - cube : fits.ImageHDU - Three-dimensional image, dimension 0 (in python convention) is the - spectral dimension. - thru : SpectralElement | SourceSpectrum - Throughput curve, spectrum or filter. - wave_cube : u.Quantity["length"] - Wavelength axis of the cube. - - Returns - ------- - cube : fits.ImageHDU - Header unchanged, data multiplied with wavelength-dependent throughput. - - """ - # Note: wave_cube used to be converted to AA, but synphot understands um - # or whatever just as well... - cube.data *= thru(wave_cube).value[:, None, None] - return cube - - def add_edge_zeros(tbl, wave_colname): if isinstance(tbl, Table): vals = np.zeros(len(tbl.colnames)) From 070f4b9781ddcb6d2ca45c7ecb1c6c27883396bb Mon Sep 17 00:00:00 2001 From: teutoburg Date: Sun, 30 Aug 2026 23:46:28 +0200 Subject: [PATCH 23/87] mv `scale_spectrum` to `surface.py`, rm unused functions It was only used there, and the module is small enough that keeping it elsewhere doesn't make much sense. Also removes one cross-import between optics and effects. There is still `get_filter`, but maybe we can remove that as well, and the whole scale function actually, if we used the existing functionality in SpeXtra / synphot directly... The functions `zero_mag_flux` and `get_zero_mag_spectrum` were not used anywhere except in their own unit tests, and are definetely redundant with SpeXtra / synphot, so were removed entirely. --- scopesim/effects/ter_curves_utils.py | 147 +----------------- scopesim/optics/surface.py | 121 ++++++++++++-- scopesim/tests/tests_effects/test_Surface.py | 27 +++- .../tests/tests_effects/test_SurfaceList.py | 3 +- .../tests_effects/test_ter_curves_utils.py | 51 +----- 5 files changed, 138 insertions(+), 211 deletions(-) diff --git a/scopesim/effects/ter_curves_utils.py b/scopesim/effects/ter_curves_utils.py index 8dcf62320..b7454e2c2 100644 --- a/scopesim/effects/ter_curves_utils.py +++ b/scopesim/effects/ter_curves_utils.py @@ -8,12 +8,8 @@ from astropy.table import Table from astropy.io.votable import parse_single_table from astropy.io import ascii as ioascii -from astropy.io import fits -from synphot import SpectralElement, SourceSpectrum, Empirical1D, Observation -from synphot.units import PHOTLAM +from synphot import SpectralElement, Empirical1D -from scopesim.source.source_templates import vega_spectrum, st_spectrum, \ - ab_spectrum from ..utils import find_file, quantity_from_table, get_logger from ..server.download_utils import create_client, handle_download @@ -218,147 +214,6 @@ def get_filter(filter_name): return filt -def get_zero_mag_spectrum(system_name: str = "AB") -> SourceSpectrum: - """ - Return a synphot spectrum of 0 mag in given photometric system. - - Parameters - ---------- - system_name : {"AB", "Vega", "ST", "HST"} - Name of the photometric system. - - Raises - ------ - ValueError - Raised if `system_name` is invalid. - - Returns - ------- - spec : synphot.SourceSpectrum - Output spectrum. - - """ - if system_name.lower() == "vega": - return vega_spectrum() - if system_name.lower() == "ab": - return ab_spectrum() - if system_name.lower() in ("st", "hst"): - return st_spectrum() - raise ValueError(f"system_name {system_name} is unknown") - - -def zero_mag_flux(filter_name, photometric_system, return_filter=False): - """ - Return the zero magnitude photon flux for a filter. - - Acceptable filter names are those given in - ``scopesim.effects.ter_curves_utils.FILTER_DEFAULTS`` or a string with an - appropriate name for a filter in the Spanish-VO filter-service. Such - strings must use the naming convention: observatory/instrument.filter. E.g: - "paranal/HAWKI.Ks", or "Gemini/GMOS-N.CaT". - - Parameters - ---------- - filter_name : str - Name of the filter - see above. - - photometric_system : {"vega", "AB", "ST"} - Name of the photometric system - - return_filter : bool, optional - If True, also returns the filter curve object - - Returns - ------- - flux : float - [PHOTLAM] - filt : ``synphot.SpectralElement`` - If `return_filter` is True - - """ - filt = get_filter(filter_name) - spec = get_zero_mag_spectrum(photometric_system) - - obs = Observation(spec, filt) - flux = obs.effstim(flux_unit=PHOTLAM) - - if return_filter: - return flux, filt - return flux - - -def scale_spectrum(spectrum, filter_name, amplitude): - """ - Scale a SourceSpectrum to a value in a filter. - - Parameters - ---------- - spectrum : synphot.SourceSpectrum - - filter_name : str - Name of a filter from - - a local instrument package (available in ``rc.__search_path__``) - - a generic filter name (see ``ter_curves_utils.FILTER_DEFAULTS``) - - a spanish-vo filter service reference (e.g. "Paranal/HAWKI.Ks") - - amplitude : astropy.Quantity, float - The value that the spectrum should have in the given filter. Acceptable - astropy quantities are: - - u.mag : Vega magnitudes - - u.ABmag : AB magnitudes - - u.STmag : HST magnitudes - - u.Jy : Jansky per filter bandpass - Additionally the "FLAM" and "FNU" units from ``synphot.units`` can - be used when passing the quantity for `amplitude`. - - Returns - ------- - spectrum : synphot.SourceSpectrum - Input spectrum scaled to the given amplitude in the given filter. - - Examples - -------- - :: - - >>> from scopesim.source.source_templates import vega_spectrum - >>> from scopesim.effects.ter_curves_utils as ter_utils - >>> - >>> spec = vega_spectrum() - >>> vega_185 = ter_utils.scale_spectrum(spec, "Ks", -1.85 * u.mag) - >>> ab_0 = ter_utils.scale_spectrum(spec, "Ks", 0 * u.ABmag) - >>> jy_3630 = ter_utils.scale_spectrum(spec, "Ks", 3630 * u.Jy) - - """ - if isinstance(amplitude, u.Quantity): - if amplitude.unit.physical_type == "spectral flux density": - if amplitude.unit != u.ABmag: - amplitude = amplitude.to(u.ABmag) - ref_spec = ab_spectrum(amplitude.value) - - elif amplitude.unit.physical_type == "spectral flux density wav": - if amplitude.unit != u.STmag: - amplitude = amplitude.to(u.STmag) - ref_spec = st_spectrum(amplitude.value) - - elif amplitude.unit == u.mag: - ref_spec = vega_spectrum(amplitude.value) - - else: - raise ValueError(f"Units of amplitude must be one of " - f"[u.mag, u.ABmag, u.STmag, u.Jy]: {amplitude}") - else: - ref_spec = vega_spectrum(amplitude) - - filt = get_filter(filter_name) - ref_flux = Observation(ref_spec, filt).effstim(flux_unit=PHOTLAM) - - real_flux = Observation(spectrum, filt).effstim(flux_unit=PHOTLAM) - scale_factor = ref_flux / real_flux - spectrum *= scale_factor.value - - return spectrum - - def add_edge_zeros(tbl, wave_colname): if isinstance(tbl, Table): vals = np.zeros(len(tbl.colnames)) diff --git a/scopesim/optics/surface.py b/scopesim/optics/surface.py index fb7f70963..f60c9f0e4 100644 --- a/scopesim/optics/surface.py +++ b/scopesim/optics/surface.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + from pathlib import Path from dataclasses import dataclass from typing import Any @@ -8,14 +10,29 @@ from astropy.io import ascii as ioascii from astropy.table import Table -from synphot import SpectralElement -from synphot.models import Empirical1D +from synphot import SpectralElement, SourceSpectrum, Empirical1D, Observation +from synphot.units import PHOTLAM + +from ..utils import ( + get_meta_quantity, + quantify, + from_currsys, + convert_table_comments_to_dict, + find_file, + get_logger, +) +from ..source.source_templates import ( + vega_spectrum, + st_spectrum, + ab_spectrum, +) +from ..effects.ter_curves_utils import get_filter +from .surface_utils import ( + make_emission_from_emissivity, + make_emission_from_array, + extract_type_from_unit, +) -from ..effects import ter_curves_utils as ter_utils -from .surface_utils import (make_emission_from_emissivity, - make_emission_from_array, extract_type_from_unit) -from ..utils import (get_meta_quantity, quantify, from_currsys, - convert_table_comments_to_dict, find_file, get_logger) logger = get_logger(__name__) @@ -88,15 +105,15 @@ def wavelength(self): return self._get_array("wavelength") @property - def throughput(self): + def throughput(self) -> SpectralElement: return self._get_ter_property(self.meta.get("action", "transmission")) @property - def transmission(self): + def transmission(self) -> SpectralElement: return self._get_ter_property("transmission") @property - def emissivity(self): + def emissivity(self) -> SourceSpectrum: return self._get_ter_property("emissivity") @property @@ -144,7 +161,7 @@ def emission(self): filter_name = dic["filter_name"] if "filename_format" in dic: filter_name = dic["filename_format"].format(filter_name) - flux = ter_utils.scale_spectrum(flux, filter_name, amplitude) + flux = scale_spectrum(flux, filter_name, amplitude) return flux @@ -197,9 +214,12 @@ def _get_ter_property(self, ter_property, fmt="synphot"): fill_value = 0. if ter_property == "transmission" else np.nan if value_arr is not None and wave is not None and fmt == "synphot": - response_curve = SpectralElement(Empirical1D, points=wave, - lookup_table=value_arr, - fill_value=fill_value) + response_curve = SpectralElement( + Empirical1D, + points=wave, + lookup_table=value_arr, + fill_value=fill_value, + ) elif fmt == "array": response_curve = value_arr else: @@ -299,3 +319,76 @@ def __str__(self): msg = f"SpectralSurface [{cols}] \"{name}\"" return msg + + +# TODO: This feels WET with some SpeXtra / synphot functionality... +def scale_spectrum(spectrum, filter_name, amplitude): + """ + Scale a SourceSpectrum to a value in a filter. + + Parameters + ---------- + spectrum : synphot.SourceSpectrum + + filter_name : str + Name of a filter from + - a local instrument package (available in ``rc.__search_path__``) + - a generic filter name (see ``ter_curves_utils.FILTER_DEFAULTS``) + - a spanish-vo filter service reference (e.g. "Paranal/HAWKI.Ks") + + amplitude : astropy.Quantity, float + The value that the spectrum should have in the given filter. Acceptable + astropy quantities are: + - u.mag : Vega magnitudes + - u.ABmag : AB magnitudes + - u.STmag : HST magnitudes + - u.Jy : Jansky per filter bandpass + Additionally the "FLAM" and "FNU" units from ``synphot.units`` can + be used when passing the quantity for `amplitude`. + + Returns + ------- + spectrum : synphot.SourceSpectrum + Input spectrum scaled to the given amplitude in the given filter. + + Examples + -------- + :: + + >>> from scopesim.source.source_templates import vega_spectrum + >>> from scopesim.optics.surface import scale_spectrum + >>> + >>> spec = vega_spectrum() + >>> vega_185 = scale_spectrum(spec, "Ks", -1.85 * u.mag) + >>> ab_0 = scale_spectrum(spec, "Ks", 0 * u.ABmag) + >>> jy_3630 = scale_spectrum(spec, "Ks", 3630 * u.Jy) + + """ + if isinstance(amplitude, u.Quantity): + if amplitude.unit.physical_type == "spectral flux density": + if amplitude.unit != u.ABmag: + amplitude = amplitude.to(u.ABmag) + ref_spec = ab_spectrum(amplitude.value) + + elif amplitude.unit.physical_type == "spectral flux density wav": + if amplitude.unit != u.STmag: + amplitude = amplitude.to(u.STmag) + ref_spec = st_spectrum(amplitude.value) + + elif amplitude.unit == u.mag: + ref_spec = vega_spectrum(amplitude.value) + + else: + raise ValueError(f"Units of amplitude must be one of " + f"[u.mag, u.ABmag, u.STmag, u.Jy]: {amplitude}") + else: + ref_spec = vega_spectrum(amplitude) + + filt = get_filter(filter_name) + ref_flux = Observation(ref_spec, filt).effstim(flux_unit=PHOTLAM) + + real_flux = Observation(spectrum, filt).effstim(flux_unit=PHOTLAM) + scale_factor = ref_flux / real_flux + spectrum *= scale_factor.value + + return spectrum diff --git a/scopesim/tests/tests_effects/test_Surface.py b/scopesim/tests/tests_effects/test_Surface.py index d1e4823b8..141ffe28e 100644 --- a/scopesim/tests/tests_effects/test_Surface.py +++ b/scopesim/tests/tests_effects/test_Surface.py @@ -1,4 +1,6 @@ +# -*- coding: utf-8 -*- import pytest +from pytest import approx import numpy as np from astropy import units as u @@ -6,7 +8,8 @@ from synphot import SpectralElement, SourceSpectrum from scopesim.effects import TERCurve -from scopesim.optics.surface import SpectralSurface +from scopesim.optics.surface import SpectralSurface, scale_spectrum +from scopesim.source.source_templates import vega_spectrum PLOTS = False @@ -51,3 +54,25 @@ def test_returns_surface_object_for_arrays(self): wave = surf.surface.wavelength plt.plot(wave, surf.surface.emission(wave)) plt.show() + + +@pytest.mark.webtest +@pytest.mark.usefixtures("no_file_error") +class TestScaleSpectrum: + def test_scales_vega_spectrum_to_vega_ab_or_jansky(self): + spec = vega_spectrum() + vega_185 = scale_spectrum(spec, "Ks", -1.85 * u.mag) + ab_0 = scale_spectrum(spec, "Ks", 0 * u.ABmag) + jy_3630 = scale_spectrum(spec, "Ks", 3630 * u.Jy) + + wave = np.linspace(1.8, 2.5, 1000) * u.um + assert vega_185(wave).value == approx(ab_0(wave).value, rel=1e-2) + assert vega_185(wave).value == approx(jy_3630(wave).value, rel=1e-2) + + if PLOTS: + plt.plot(wave, spec(wave), "b") + plt.plot(wave, vega_185(wave), "r:") + plt.plot(wave, ab_0(wave), "g--") + plt.plot(wave, jy_3630(wave), "y-.") + plt.semilogy() + plt.show() diff --git a/scopesim/tests/tests_effects/test_SurfaceList.py b/scopesim/tests/tests_effects/test_SurfaceList.py index 8128dd161..465cffdc7 100644 --- a/scopesim/tests/tests_effects/test_SurfaceList.py +++ b/scopesim/tests/tests_effects/test_SurfaceList.py @@ -1,4 +1,5 @@ -""" Tests for the the new SurfaceList object """ +# -*- coding: utf-8 -*- +"""Tests for the the SurfaceList object.""" import pytest from pytest import approx diff --git a/scopesim/tests/tests_effects/test_ter_curves_utils.py b/scopesim/tests/tests_effects/test_ter_curves_utils.py index 555b0c41f..13206ee15 100644 --- a/scopesim/tests/tests_effects/test_ter_curves_utils.py +++ b/scopesim/tests/tests_effects/test_ter_curves_utils.py @@ -1,9 +1,9 @@ +# -*- coding: utf-8 -*- + import pytest -import scopesim.source.source_templates from pytest import approx import numpy as np -from matplotlib import pyplot as plt from astropy import units as u @@ -28,53 +28,6 @@ def test_returns_specific_from_svo(self): assert np.max(ks(wave)) > 0.75 -@pytest.mark.webtest -@pytest.mark.usefixtures("no_file_error") -@pytest.mark.parametrize("filter_name, phot_system, flux", - [("V", "vega", 995), - ("z", "AB", 602), - ("Ks", "vega", 43)]) -def test_zero_mag_flux(filter_name, phot_system, flux): - """ - Photon fluxes are in line with: - http://www.astronomy.ohio-state.edu/~martini/usefuldata.html - """ - new_flux = ter_utils.zero_mag_flux(filter_name, phot_system) - assert flux == approx(new_flux.value, rel=0.08) - - -@pytest.mark.webtest -@pytest.mark.usefixtures("no_file_error") -def test_compare_br_gamma(): - filt = "Ks" - flux_vega = ter_utils.zero_mag_flux(filt, "vega").value - flux_ab = ter_utils.zero_mag_flux(filt, "AB").value - delta_mag = 2.5 * np.log10(flux_ab/flux_vega) - assert delta_mag == approx(1.85, rel=0.01) - - -@pytest.mark.webtest -@pytest.mark.usefixtures("no_file_error") -class TestScaleSpectrum: - def test_scales_vega_spectrum_to_vega_ab_or_jansky(self): - spec = scopesim.source.source_templates.vega_spectrum() - vega_185 = ter_utils.scale_spectrum(spec, "Ks", -1.85 * u.mag) - ab_0 = ter_utils.scale_spectrum(spec, "Ks", 0 * u.ABmag) - jy_3630 = ter_utils.scale_spectrum(spec, "Ks", 3630 * u.Jy) - - wave = np.linspace(1.8, 2.5, 1000) * u.um - assert vega_185(wave).value == approx(ab_0(wave).value, rel=1e-2) - assert vega_185(wave).value == approx(jy_3630(wave).value, rel=1e-2) - - if PLOTS: - plt.plot(wave, spec(wave), "b") - plt.plot(wave, vega_185(wave), "r:") - plt.plot(wave, ab_0(wave), "g--") - plt.plot(wave, jy_3630(wave), "y-.") - plt.semilogy() - plt.show() - - @pytest.mark.webtest @pytest.mark.usefixtures("no_file_error") class TestGetFilterEffectiveWavelength: From 146a466b92da3aee2ced7fe10eac45434f8116cb Mon Sep 17 00:00:00 2001 From: teutoburg Date: Sun, 6 Sep 2026 17:26:30 +0200 Subject: [PATCH 24/87] Tiny cleanup in `OpticalTrain.readout()` - Remove two local vars only needed once. - Consistent naming of detector manager instance and list. --- scopesim/optics/optical_train.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/scopesim/optics/optical_train.py b/scopesim/optics/optical_train.py index 00823c7d8..45cfe947e 100644 --- a/scopesim/optics/optical_train.py +++ b/scopesim/optics/optical_train.py @@ -473,11 +473,12 @@ def readout(self, filename=None, reset=True, **kwargs): self.cmds[f"!OBS.{key}"] = value hduls = [] - for i, detector_array in enumerate(self.detector_managers): - array_effects = self.optics_manager.detector_array_effects - dtcr_effects = self.optics_manager.detector_effects - hdul = detector_array.readout( - self.image_planes, array_effects, dtcr_effects) + for i, detector_manager in enumerate(self.detector_managers): + hdul = detector_manager.readout( + self.image_planes, + self.optics_manager.detector_array_effects, + self.optics_manager.detector_effects, + ) fits_effects = self.optics_manager.get_all(ExtraFitsKeywords) if len(fits_effects) > 0: From 056b1af841133dd95d0ea76e5755c559eb68c18b Mon Sep 17 00:00:00 2001 From: teutoburg Date: Mon, 7 Sep 2026 01:04:20 +0200 Subject: [PATCH 25/87] More cleanup --- scopesim/effects/electronic/noise.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scopesim/effects/electronic/noise.py b/scopesim/effects/electronic/noise.py index 717fec584..00b6049ff 100644 --- a/scopesim/effects/electronic/noise.py +++ b/scopesim/effects/electronic/noise.py @@ -391,10 +391,6 @@ def __call__( ) -> np.ndarray: return data + dark_level * dit * ndit - @property - def random_seed(self) -> int: - return from_currsys(self.meta["random_seed"], self.cmds) - def _get_dark_level(self, det_meta) -> float: dark_level = float(from_currsys(self.meta["value"], self.cmds)) if isinstance(dark_level, Real): From bb32c7d44a24d4f37c6c13f8df0e9b2f6c2bfa69 Mon Sep 17 00:00:00 2001 From: "astarvienna-bot[bot]" <217837468+myciapp[bot]@users.noreply.github.com> Date: Sun, 6 Sep 2026 23:19:13 +0000 Subject: [PATCH 26/87] Bumping version from 0.12.0b2.dev5 to 0.12.0b3.dev0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index da766c6da..75e53403b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ScopeSim" -version = "0.12.0b2.dev5" +version = "0.12.0b3.dev0" description = "Generalised telescope observation simulator" license = {text = "GPL-3.0-or-later"} authors = [ From 81b83624803b64614c5888db7f599f5644da0b66 Mon Sep 17 00:00:00 2001 From: teutoburg Date: Mon, 7 Sep 2026 01:04:56 +0200 Subject: [PATCH 27/87] Add `Detector.det_id` convenience property We needed the line `det_meta[real_colname("id", det_meta)]` a few times now, so it's worth making that simpler. Also means not the whole meta, but rather just a simple int can be passed to the resolve methods. --- scopesim/detector/detector.py | 26 +++++++++++++++++++++--- scopesim/detector/detector_manager.py | 16 ++++++++------- scopesim/effects/electronic/electrons.py | 11 +++++----- scopesim/effects/electronic/noise.py | 24 ++++++++++------------ 4 files changed, 48 insertions(+), 29 deletions(-) diff --git a/scopesim/detector/detector.py b/scopesim/detector/detector.py index b6708d365..eb1ff3a98 100644 --- a/scopesim/detector/detector.py +++ b/scopesim/detector/detector.py @@ -7,9 +7,17 @@ from astropy.wcs import WCS from ..optics import ImagePlane -from ..optics.image_plane_utils import (add_imagehdu_to_imagehdu, - sky_wcs_from_det_wcs) -from ..utils import get_logger, from_currsys, stringify_dict, zeros_from_header +from ..optics.image_plane_utils import ( + add_imagehdu_to_imagehdu, + sky_wcs_from_det_wcs, +) +from ..utils import ( + get_logger, + from_currsys, + stringify_dict, + zeros_from_header, + real_colname, +) logger = get_logger(__name__) @@ -65,6 +73,18 @@ def hdu(self): return self._hdu + @property + def det_id(self) -> int | None: + """Return detector ID. + + Avoid calling it just "id", because ``id(self)`` is something else. + + If not found in `self.meta`, return None. + """ + if key := real_colname("id", self.meta): + return self.meta[key] + return None + @property def header(self): """Return header from internal HDU.""" diff --git a/scopesim/detector/detector_manager.py b/scopesim/detector/detector_manager.py index de6fd24a7..9c0f80b86 100644 --- a/scopesim/detector/detector_manager.py +++ b/scopesim/detector/detector_manager.py @@ -40,8 +40,13 @@ def __init__(self, detector_list=None, cmds=None, **kwargs): self._latest_exposure: HDUList | None = None - def readout(self, image_planes, array_effects=None, dtcr_effects=None, - **kwargs) -> HDUList: + def readout( + self, + image_planes, + array_effects: list[Effect] | None = None, + dtcr_effects: list[Effect] | None = None, + **kwargs + ) -> HDUList: """ Read out the detector array into a FITS HDU List. @@ -74,9 +79,6 @@ def readout(self, image_planes, array_effects=None, dtcr_effects=None, Output FITS HDU List. """ - # .. note:: Detector is what used to be called Chip - # DetectorManager is the old Detector - self._array_effects = array_effects or [] self._dtcr_effects = dtcr_effects or [] self.meta.update(kwargs) @@ -100,7 +102,7 @@ def readout(self, image_planes, array_effects=None, dtcr_effects=None, detector = effect.apply_to(detector) # 6. add necessary header keywords - # .. todo: add keywords + # TODO: add keywords # FIXME: Why is this applied twice ??? for effect in self._array_effects: @@ -148,7 +150,7 @@ def _make_primary_hdu(self): return prihdu def _make_effects_hdu(self): - # .. todo:: decide what goes into the effects table of meta data + # TODO: decide what goes into the effects table of meta data # effects = self._array_effects + self._dtcr_effects return TableHDU() diff --git a/scopesim/effects/electronic/electrons.py b/scopesim/effects/electronic/electrons.py index b28f55b05..b279df63e 100644 --- a/scopesim/effects/electronic/electrons.py +++ b/scopesim/effects/electronic/electrons.py @@ -18,7 +18,7 @@ from .. import Effect from ...detector import Detector from ...utils import figure_factory, check_keys -from ...utils import from_currsys, real_colname +from ...utils import from_currsys from . import logger @@ -309,14 +309,13 @@ def _should_apply(self) -> bool: def __call__(self, data: ArrayLike, gain: Real) -> NDArray: return data / gain - def _get_gain(self, det_meta) -> Real: + def _get_gain(self, det_id: int) -> Real: # Apply the gain value (copy from DarkCurrent) # Note that this does not cater for the case where the gain is given # as a plain dictionary. Should we implement that? if hasattr(self.cmds["!DET.gain"], "dic"): - dtcr_id = det_meta[real_colname("id", det_meta)] - gain = self.cmds["!DET.gain"].dic[dtcr_id] - logger.info(f"Detector {dtcr_id}: applying gain {gain}") + gain = self.cmds["!DET.gain"].dic[det_id] + logger.info(f"Detector {det_id}: applying gain {gain}") return gain if isinstance(self.cmds["!DET.gain"], Real): gain = self.cmds["!DET.gain"] @@ -333,7 +332,7 @@ def apply_to(self, obj, **kwargs): new_dtype = self.meta["dtype"] # Apply gain - obj.data = self(obj.data, self._get_gain(obj.meta)) + obj.data = self(obj.data, self._get_gain(obj.det_id)) # Type-conversion wraps around input values that are higher or lower than # the respective maximum and minimum values of the new data type. Before diff --git a/scopesim/effects/electronic/noise.py b/scopesim/effects/electronic/noise.py index 00b6049ff..c149ed5f7 100644 --- a/scopesim/effects/electronic/noise.py +++ b/scopesim/effects/electronic/noise.py @@ -11,7 +11,7 @@ from .. import Effect from ...detector import Detector -from ...utils import from_currsys, figure_factory, check_keys, real_colname +from ...utils import from_currsys, figure_factory, check_keys from . import logger @@ -247,19 +247,18 @@ class PixelResponseNonUniformity(Effect): def __init__(self, **kwargs): super().__init__(**kwargs) self.meta.update(kwargs) - self._gain_maps = {} # keyed by dtcr_id + self._gain_maps = {} # keyed by det_id def apply_to(self, obj, **kwargs): if not isinstance(obj, Detector): return obj random_seed = from_currsys(self.meta.get("prnu_seed"), self.cmds) - id_key = real_colname("id", obj.meta) - dtcr_id = obj.meta[id_key] if id_key is not None else None + det_id = obj.det_id prnu_std_meta = from_currsys(self.meta["prnu_std"], self.cmds) if isinstance(prnu_std_meta, Mapping): - prnu_std = float(from_currsys(prnu_std_meta[dtcr_id], self.cmds)) + prnu_std = float(from_currsys(prnu_std_meta[det_id], self.cmds)) elif isinstance(prnu_std_meta, Real): prnu_std = float(prnu_std_meta) else: @@ -269,16 +268,16 @@ def apply_to(self, obj, **kwargs): ) shape = obj.data.shape - if dtcr_id not in self._gain_maps: + if det_id not in self._gain_maps: rng = np.random.default_rng(random_seed) - self._gain_maps[dtcr_id] = rng.normal( + self._gain_maps[det_id] = rng.normal( loc=1.0, scale=prnu_std, size=shape, ) - if self._gain_maps[dtcr_id].shape != shape: + if self._gain_maps[det_id].shape != shape: raise ValueError("gain map shape mismatch") - obj.data = obj.data * self._gain_maps[dtcr_id] + obj.data = obj.data * self._gain_maps[det_id] return obj def plot(self, det_id=None): @@ -391,13 +390,12 @@ def __call__( ) -> np.ndarray: return data + dark_level * dit * ndit - def _get_dark_level(self, det_meta) -> float: + def _get_dark_level(self, det_id: int) -> float: dark_level = float(from_currsys(self.meta["value"], self.cmds)) if isinstance(dark_level, Real): return dark_level if isinstance(dark_level, Mapping): - dtcr_id = det_meta[real_colname("id", det_meta)] - return from_currsys(dark_level[dtcr_id], self.cmds) + return from_currsys(dark_level[det_id], self.cmds) raise ValueError( f"<{self.__class__.__name__}>.meta['value'] must be either " f"dict-like or scalar number, but is {dark_level}." @@ -408,7 +406,7 @@ def apply_to(self, obj, **kwargs): return obj # Dark level needs detector meta so can't go into __call__() - dark_level = self._get_dark_level(obj.meta) + dark_level = self._get_dark_level(obj.det_id) dit = from_currsys(self.meta["dit"], self.cmds) ndit = from_currsys(self.meta["ndit"], self.cmds) From c9954a731d877e40bf2dad39796e882f6d2220ce Mon Sep 17 00:00:00 2001 From: teutoburg Date: Mon, 7 Sep 2026 13:27:03 +0200 Subject: [PATCH 28/87] Hotfix after #1006 --- scopesim/effects/electronic/pixels.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scopesim/effects/electronic/pixels.py b/scopesim/effects/electronic/pixels.py index 93f746700..f97d1eb2c 100644 --- a/scopesim/effects/electronic/pixels.py +++ b/scopesim/effects/electronic/pixels.py @@ -140,9 +140,9 @@ class UnequalBinnedImage(BinnedImageBase): def __call__(self, data: ArrayLike) -> NDArray: height, width = data.shape binned_data = data.reshape(( - height//self.binx, - self.binx, - width//self.biny, + height//self.biny, + self.biny, + width//self.binx, self.binx )).sum(axis=3).sum(axis=1) return binned_data From 71428519e2c78f2a9f332d4b21d0baac47eb2bf5 Mon Sep 17 00:00:00 2001 From: jbrink Date: Tue, 1 Sep 2026 14:27:03 +0200 Subject: [PATCH 29/87] Fixes to FOV and implane-utils for correct flat_lamp scaling --- scopesim/optics/fov.py | 28 +++++++++++++++++++++------- scopesim/optics/image_plane_utils.py | 6 ++++-- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/scopesim/optics/fov.py b/scopesim/optics/fov.py index 503214f53..f229c2f10 100644 --- a/scopesim/optics/fov.py +++ b/scopesim/optics/fov.py @@ -605,15 +605,17 @@ def _make_scaled_hdu(self, field: ImageSourceField) -> fits.ImageHDU: """Deal with BUNIT and pixel area.""" # Note: Do not scale source data - make a copy first. field_hdu = field.field.copy() # .field is the HDU (yeah...) - field_hdu.data /= self.pixel_area.value + field_hdu.data /= field.pixel_area.value # TODO: Check if this scaling is actually correct. How does this # work with the add_imagehdu_to_imagehdu below? Isn't that # supposed to conserve flux? Test carefully!! if field.is_bunit_spatially_differential: + logger.debug("differential bunit...") # Field is in (PHOTLAM) arcsec-2, need to scale by pixarea - field_hdu.data *= field.pixel_area.value + field_hdu.data *= self.pixel_area.value else: + logger.debug("binned bunit...") # Pixel area doesn't cancel out, need to convert new_bunit = field.bunit / u.arcsec**2 field_hdu.header["BUNIT"] = new_bunit.to_string("fits") @@ -751,6 +753,7 @@ def _make_cubefields(self): image = np.sum(field.data, axis=0) * PHOTLAM/u.arcsec**2 image = (image * self.area * spectral_bin_width).to(u.ph/u.s/u.arcsec**2) + logger.debug("2D FOV make_cubefields: image.mean() = %f", image.mean().value) # FIXME: This might create a 2D ImageHDU with a 3D header, not ideal... yield fits.ImageHDU(data=image, header=field.header) @@ -763,6 +766,7 @@ def _make_imagefields(self, fov_waveset, bin_widths): * yield image to be added to canvas image """ for field in self._get_image_fields(): + logger.debug("2D FOV make_imagefields: field.data.sum() = %f", field.data.sum()) field_hdu = self._make_scaled_hdu(field) # Reevaluate spectrum onto FOV waveset @@ -773,6 +777,7 @@ def _make_imagefields(self, fov_waveset, bin_widths): # Rescale 2D flux weight map by integrated flux from spectrum field_hdu.data *= flux # ph s-1 arcsec-2 + logger.debug("2D FOV make_imagefields: field_hdu.data.mean() = %f ph s-1 arcsec-2", field_hdu.data.mean()) yield field_hdu def _make_tablefields(self, fov_waveset, bin_widths, use_photlam=False): @@ -851,11 +856,13 @@ def make_hdu(self): canvas_image_hdu = imp_utils.add_imagehdu_to_imagehdu( tmp_hdu, canvas_image_hdu, - conserve_flux=True, + conserve_flux=False, spline_order=self.spline_order) + logger.debug("2D FOV make_hdu: canvas_image_hdu.data.mean() = %f ph s-1 arcsec-2", canvas_image_hdu.data.mean()) # TODO: Move this further along at some point... canvas_image_hdu.data *= self.pixel_area.value # eliminate arcsec-2 + logger.debug("After scaling by pixel area: canvas_image_hdu.data.mean() = %f ph s-1 pix-1", canvas_image_hdu.data.mean()) for flux, weight, x, y in self._make_tablefields( fov_waveset, bin_widths): @@ -932,7 +939,8 @@ def _make_cubefields(self, fov_waveset): field_data = field_interp(fov_waveset.value) # Pixel scale conversion - field_data *= field.pixel_area / self.pixel_area + # field_data *= field.pixel_area / self.pixel_area ## not needed in flux density cubes?? + logger.debug("3D FOV make_cubefields: field_data.mean() = %f PHOTLAM arcsec-2", field_data.mean()) field_hdu = fits.ImageHDU(data=field_data, header=field.header) yield field_hdu @@ -957,12 +965,14 @@ def _make_imagefields(self, fov_waveset, spline_order): canvas_image_hdu = imp_utils.add_imagehdu_to_imagehdu( field_hdu, canvas_image_hdu, - spline_order=spline_order) + spline_order=spline_order, + conserve_flux=False, + ) spec = field.spectrum(fov_waveset) - # 2D * 1D -> 3D field_cube = canvas_image_hdu.data[None, :, :] * spec[:, None, None] + logger.debug("3D FOV make_imagefields: field_cube.mean() = %f", field_cube.mean().value) yield field_cube.value def _make_tablefields(self, fov_waveset): @@ -1113,7 +1123,9 @@ def make_hdu(self): canvas_cube_hdu = imp_utils.add_imagehdu_to_imagehdu( field_hdu, canvas_cube_hdu, - spline_order=self.spline_order) + spline_order=self.spline_order, + conserve_flux=False, + ) canvas_cube_hdu.data = sum(self._make_imagefields( fov_waveset, self.spline_order), @@ -1132,6 +1144,8 @@ def make_hdu(self): canvas_cube_hdu.data *= 1e4 # ph/s/AA/arcsec2 --> ph/s/um/arcsec2 canvas_cube_hdu.header["BUNIT"] = "ph s-1 um-1 arcsec-2" + logger.debug("3D FOV make_hdu: canvas_cube_hdu.data.mean() = %f", canvas_cube_hdu.data.mean()) + return canvas_cube_hdu # [ph s-1 um-1 (arcsec-2)] def plot_data(self): diff --git a/scopesim/optics/image_plane_utils.py b/scopesim/optics/image_plane_utils.py index 62f717315..e59fadc90 100644 --- a/scopesim/optics/image_plane_utils.py +++ b/scopesim/optics/image_plane_utils.py @@ -593,7 +593,9 @@ def rescale_imagehdu(imagehdu: fits.ImageHDU, pixel_scale: float | u.Quantity, new_im = np.nan_to_num(new_im, copy=False) sum_new = np.sum(new_im) if sum_new != 0: - new_im *= sum_orig / sum_new + flux_factor = sum_orig / sum_new + logger.debug("flux factor = %f", flux_factor) + new_im *= flux_factor elif sum_orig != 0: logger.warning( "rescale_imagehdu: all input flux (%g) was lost in resampling; " @@ -838,7 +840,7 @@ def add_imagehdu_to_imagehdu(image_hdu: fits.ImageHDU, spline_order=spline_order, conserve_flux=conserve_flux) # TODO: Perhaps add separately formatted WCS logger? - # logger.debug("fromrescale %s", WCS(new_hdu.header, key=canvas_wcs.wcs.alt)) + logger.debug("fromrescale %s", WCS(new_hdu.header, key=canvas_wcs.wcs.alt)) new_hdu = reorient_imagehdu(new_hdu, wcs_suffix=canvas_wcs.wcs.alt, spline_order=spline_order, From d5fd5295fcd7fd7dab9a46b91ca6a6d4469ce5ca Mon Sep 17 00:00:00 2001 From: teutoburg Date: Fri, 4 Sep 2026 12:34:23 +0200 Subject: [PATCH 30/87] Stick with binned flux for 2D images --- scopesim/optics/fov.py | 66 ++++++++++++++++++++++++++++-------------- 1 file changed, 44 insertions(+), 22 deletions(-) diff --git a/scopesim/optics/fov.py b/scopesim/optics/fov.py index f229c2f10..e995fab0e 100644 --- a/scopesim/optics/fov.py +++ b/scopesim/optics/fov.py @@ -605,6 +605,7 @@ def _make_scaled_hdu(self, field: ImageSourceField) -> fits.ImageHDU: """Deal with BUNIT and pixel area.""" # Note: Do not scale source data - make a copy first. field_hdu = field.field.copy() # .field is the HDU (yeah...) + logger.debug("scaling by %f", field.pixel_area.value) field_hdu.data /= field.pixel_area.value # TODO: Check if this scaling is actually correct. How does this @@ -613,12 +614,14 @@ def _make_scaled_hdu(self, field: ImageSourceField) -> fits.ImageHDU: if field.is_bunit_spatially_differential: logger.debug("differential bunit...") # Field is in (PHOTLAM) arcsec-2, need to scale by pixarea + logger.debug("scaling by %f", self.pixel_area.value) field_hdu.data *= self.pixel_area.value else: logger.debug("binned bunit...") # Pixel area doesn't cancel out, need to convert new_bunit = field.bunit / u.arcsec**2 field_hdu.header["BUNIT"] = new_bunit.to_string("fits") + logger.debug("BUNIT after scaling: %s", field_hdu.header["BUNIT"]) return field_hdu @@ -746,14 +749,16 @@ def _make_cubefields(self): # cube_fields come in with units of photlam/arcsec2, # need to convert to ph/s # We need to the voxel volume (spectral and solid angle) for that. - spectral_bin_width = (field.header["CDELT3"] * - u.Unit(field.header["CUNIT3"]) - ).to(u.Angstrom) + spectral_bin_width = ( + field.header["CDELT3"] * + u.Unit(field.header["CUNIT3"]) + ).to(u.Angstrom) # First collapse to image, then convert units image = np.sum(field.data, axis=0) * PHOTLAM/u.arcsec**2 - image = (image * self.area * - spectral_bin_width).to(u.ph/u.s/u.arcsec**2) - logger.debug("2D FOV make_cubefields: image.mean() = %f", image.mean().value) + image = (image * self.area * field.pixel_area * + spectral_bin_width).to(u.ph/u.s) + logger.debug("2D FOV make_cubefields: image.mean() = %f %s", + image.mean().value, image.unit) # FIXME: This might create a 2D ImageHDU with a 3D header, not ideal... yield fits.ImageHDU(data=image, header=field.header) @@ -766,8 +771,18 @@ def _make_imagefields(self, fov_waveset, bin_widths): * yield image to be added to canvas image """ for field in self._get_image_fields(): - logger.debug("2D FOV make_imagefields: field.data.sum() = %f", field.data.sum()) - field_hdu = self._make_scaled_hdu(field) + logger.debug("2D FOV make_imagefields: field.data.sum() = %f %s", + field.data.sum(), field.bunit) + + field_hdu = field.field.copy() + + if field.is_bunit_spatially_differential: + logger.warning("Differential BUNIT in 2D image is discouraged.") + logger.debug("scaling by %f", field.pixel_area.value) + field_hdu.data *= field.pixel_area.value + new_bunit = field.bunit / field.pixel_area.unit + field_hdu.header["BUNIT"] = new_bunit.to_string("fits") + logger.debug("BUNIT after scaling: %s", field_hdu.header["BUNIT"]) # Reevaluate spectrum onto FOV waveset spec = field.spectrum(fov_waveset) # PHOTLAM @@ -776,8 +791,10 @@ def _make_imagefields(self, fov_waveset, bin_widths): flux = (spec * bin_widths * self.area).to_value(u.ph / u.s).sum() # Rescale 2D flux weight map by integrated flux from spectrum - field_hdu.data *= flux # ph s-1 arcsec-2 - logger.debug("2D FOV make_imagefields: field_hdu.data.mean() = %f ph s-1 arcsec-2", field_hdu.data.mean()) + field_hdu.data *= flux # ph s-1 + logger.info( + "2D FOV make_imagefields: field_hdu.data.mean() = %f ph/s", + field_hdu.data.mean()) yield field_hdu def _make_tablefields(self, fov_waveset, bin_widths, use_photlam=False): @@ -856,13 +873,14 @@ def make_hdu(self): canvas_image_hdu = imp_utils.add_imagehdu_to_imagehdu( tmp_hdu, canvas_image_hdu, - conserve_flux=False, + conserve_flux=True, # binned flux needs this spline_order=self.spline_order) - logger.debug("2D FOV make_hdu: canvas_image_hdu.data.mean() = %f ph s-1 arcsec-2", canvas_image_hdu.data.mean()) - - # TODO: Move this further along at some point... - canvas_image_hdu.data *= self.pixel_area.value # eliminate arcsec-2 - logger.debug("After scaling by pixel area: canvas_image_hdu.data.mean() = %f ph s-1 pix-1", canvas_image_hdu.data.mean()) + logger.info( + "2D FOV make_hdu: canvas_image_hdu.data.mean() = %f ph/s/pix", + canvas_image_hdu.data.mean()) + logger.info( + "2D FOV make_hdu: canvas_image_hdu.data.max() = %f ph/s/pix", + canvas_image_hdu.data.max()) for flux, weight, x, y in self._make_tablefields( fov_waveset, bin_widths): @@ -939,8 +957,10 @@ def _make_cubefields(self, fov_waveset): field_data = field_interp(fov_waveset.value) # Pixel scale conversion - # field_data *= field.pixel_area / self.pixel_area ## not needed in flux density cubes?? - logger.debug("3D FOV make_cubefields: field_data.mean() = %f PHOTLAM arcsec-2", field_data.mean()) + field_data *= field.pixel_area / self.pixel_area + logger.debug( + "3D FOV make_cubefields: field_data.mean() = %f " + "PHOTLAM arcsec-2", field_data.mean()) field_hdu = fits.ImageHDU(data=field_data, header=field.header) yield field_hdu @@ -966,13 +986,14 @@ def _make_imagefields(self, fov_waveset, spline_order): field_hdu, canvas_image_hdu, spline_order=spline_order, - conserve_flux=False, + conserve_flux=True, ) spec = field.spectrum(fov_waveset) # 2D * 1D -> 3D field_cube = canvas_image_hdu.data[None, :, :] * spec[:, None, None] - logger.debug("3D FOV make_imagefields: field_cube.mean() = %f", field_cube.mean().value) + logger.debug("3D FOV make_imagefields: field_cube.mean() = %f", + field_cube.mean().value) yield field_cube.value def _make_tablefields(self, fov_waveset): @@ -1124,7 +1145,7 @@ def make_hdu(self): field_hdu, canvas_cube_hdu, spline_order=self.spline_order, - conserve_flux=False, + conserve_flux=True, ) canvas_cube_hdu.data = sum(self._make_imagefields( @@ -1144,7 +1165,8 @@ def make_hdu(self): canvas_cube_hdu.data *= 1e4 # ph/s/AA/arcsec2 --> ph/s/um/arcsec2 canvas_cube_hdu.header["BUNIT"] = "ph s-1 um-1 arcsec-2" - logger.debug("3D FOV make_hdu: canvas_cube_hdu.data.mean() = %f", canvas_cube_hdu.data.mean()) + logger.debug("3D FOV make_hdu: canvas_cube_hdu.data.mean() = %f", + canvas_cube_hdu.data.mean()) return canvas_cube_hdu # [ph s-1 um-1 (arcsec-2)] From ad3592f47927919392d945d4d645f59751fe197a Mon Sep 17 00:00:00 2001 From: oczoske Date: Wed, 9 Sep 2026 16:22:25 +0200 Subject: [PATCH 31/87] Apply throughput in-place for speed --- scopesim/effects/ter_curves.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scopesim/effects/ter_curves.py b/scopesim/effects/ter_curves.py index 1b32b6674..53641a043 100644 --- a/scopesim/effects/ter_curves.py +++ b/scopesim/effects/ter_curves.py @@ -147,7 +147,9 @@ def _apply_to_src(self, src: Source) -> None: # apply transmission to source spectra for fld in src.fields: if isinstance(fld, CubeSourceField): - fld.field.data = self(fld.field.data, fld.waveset) + ## in-place is much faster than self() + #fld.field.data = self(fld.field.data, fld.waveset) + fld.field.data *= thru(fld.waveset).value[:, None, None] elif isinstance(fld, SpectrumSourceField): fld.spectra = { isp: spec * thru # This works because synphot! From 549b04d1219b51810dc1069007249b6141daa77e Mon Sep 17 00:00:00 2001 From: jbrink Date: Wed, 9 Sep 2026 16:43:33 +0200 Subject: [PATCH 32/87] SPEC scaling now also fixed - moved all scaling to conserve-flux functionality in image_plane_utils and added a parameter to identify differential flux units (as used in cube sources). --- scopesim/optics/fov.py | 20 +++++++++++--------- scopesim/optics/image_plane_utils.py | 16 ++++++++++++---- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/scopesim/optics/fov.py b/scopesim/optics/fov.py index e995fab0e..141d72731 100644 --- a/scopesim/optics/fov.py +++ b/scopesim/optics/fov.py @@ -606,7 +606,6 @@ def _make_scaled_hdu(self, field: ImageSourceField) -> fits.ImageHDU: # Note: Do not scale source data - make a copy first. field_hdu = field.field.copy() # .field is the HDU (yeah...) logger.debug("scaling by %f", field.pixel_area.value) - field_hdu.data /= field.pixel_area.value # TODO: Check if this scaling is actually correct. How does this # work with the add_imagehdu_to_imagehdu below? Isn't that @@ -614,10 +613,11 @@ def _make_scaled_hdu(self, field: ImageSourceField) -> fits.ImageHDU: if field.is_bunit_spatially_differential: logger.debug("differential bunit...") # Field is in (PHOTLAM) arcsec-2, need to scale by pixarea - logger.debug("scaling by %f", self.pixel_area.value) - field_hdu.data *= self.pixel_area.value + # logger.debug("scaling by %f", self.pixel_area.value) + # field_hdu.data *= self.pixel_area.value else: logger.debug("binned bunit...") + field_hdu.data /= field.pixel_area.value # Pixel area doesn't cancel out, need to convert new_bunit = field.bunit / u.arcsec**2 field_hdu.header["BUNIT"] = new_bunit.to_string("fits") @@ -957,10 +957,9 @@ def _make_cubefields(self, fov_waveset): field_data = field_interp(fov_waveset.value) # Pixel scale conversion - field_data *= field.pixel_area / self.pixel_area + # field_data *= field.pixel_area / self.pixel_area logger.debug( - "3D FOV make_cubefields: field_data.mean() = %f " - "PHOTLAM arcsec-2", field_data.mean()) + "3D FOV make_cubefields: field_data.mean() = %f [%s]", field_data.mean(), field.bunit) field_hdu = fits.ImageHDU(data=field_data, header=field.header) yield field_hdu @@ -987,13 +986,15 @@ def _make_imagefields(self, fov_waveset, spline_order): canvas_image_hdu, spline_order=spline_order, conserve_flux=True, + differential=True, ) spec = field.spectrum(fov_waveset) # 2D * 1D -> 3D field_cube = canvas_image_hdu.data[None, :, :] * spec[:, None, None] - logger.debug("3D FOV make_imagefields: field_cube.mean() = %f", - field_cube.mean().value) + # logger.debug("3D FOV make_imagefields: field_cube.mean() = %f", + # field_cube.mean().value) + logger.debug(f"3D FOV make_imagefields: {field_cube.mean() = }") yield field_cube.value def _make_tablefields(self, fov_waveset): @@ -1145,7 +1146,8 @@ def make_hdu(self): field_hdu, canvas_cube_hdu, spline_order=self.spline_order, - conserve_flux=True, + conserve_flux=False, + differential=True, ) canvas_cube_hdu.data = sum(self._make_imagefields( diff --git a/scopesim/optics/image_plane_utils.py b/scopesim/optics/image_plane_utils.py index e59fadc90..307b0e0fd 100644 --- a/scopesim/optics/image_plane_utils.py +++ b/scopesim/optics/image_plane_utils.py @@ -485,7 +485,8 @@ def overlay_image(small_im, big_im, coords, mask=None, sub_pixel=False): def rescale_imagehdu(imagehdu: fits.ImageHDU, pixel_scale: float | u.Quantity, wcs_suffix: str = "", conserve_flux: bool = True, - spline_order: int = 1) -> fits.ImageHDU: + spline_order: int = 1, + differential: bool = False) -> fits.ImageHDU: """ Scale the .data array by the ratio of pixel_scale [deg] and CDELTn. @@ -540,6 +541,8 @@ def rescale_imagehdu(imagehdu: fits.ImageHDU, pixel_scale: float | u.Quantity, return imagehdu sum_orig = np.sum(imagehdu.data) + if differential: + sum_orig *= primary_wcs.wcs.cdelt[0]*primary_wcs.wcs.cdelt[1] # Perform the rescaling. Axes need to be inverted because python. zoom_np = zoom[::-1] @@ -592,9 +595,12 @@ def rescale_imagehdu(imagehdu: fits.ImageHDU, pixel_scale: float | u.Quantity, if conserve_flux: new_im = np.nan_to_num(new_im, copy=False) sum_new = np.sum(new_im) + if differential: + sum_new *= (pixel_scale.value**2) + if sum_new != 0: flux_factor = sum_orig / sum_new - logger.debug("flux factor = %f", flux_factor) + logger.info("flux factor = %f", flux_factor) new_im *= flux_factor elif sum_orig != 0: logger.warning( @@ -785,7 +791,8 @@ def add_imagehdu_to_imagehdu(image_hdu: fits.ImageHDU, canvas_hdu: fits.ImageHDU, spline_order: int = 1, wcs_suffix: str = "", - conserve_flux: bool = True) -> fits.ImageHDU: + conserve_flux: bool = True, + differential: bool = False) -> fits.ImageHDU: """ Re-project one ``fits.ImageHDU`` onto another ``fits.ImageHDU``. @@ -838,7 +845,8 @@ def add_imagehdu_to_imagehdu(image_hdu: fits.ImageHDU, new_hdu = rescale_imagehdu(image_hdu, pixel_scale=canvas_pixel_scale / conv_fac, wcs_suffix=canvas_wcs.wcs.alt, spline_order=spline_order, - conserve_flux=conserve_flux) + conserve_flux=conserve_flux, + differential=differential) # TODO: Perhaps add separately formatted WCS logger? logger.debug("fromrescale %s", WCS(new_hdu.header, key=canvas_wcs.wcs.alt)) new_hdu = reorient_imagehdu(new_hdu, From 4c120070ba3e88f6723df3ca66d7f22c3462404b Mon Sep 17 00:00:00 2001 From: jbrink Date: Wed, 9 Sep 2026 16:46:05 +0200 Subject: [PATCH 33/87] Removed stray f-string from debug logging. --- scopesim/optics/fov.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scopesim/optics/fov.py b/scopesim/optics/fov.py index 141d72731..da8489256 100644 --- a/scopesim/optics/fov.py +++ b/scopesim/optics/fov.py @@ -992,9 +992,8 @@ def _make_imagefields(self, fov_waveset, spline_order): spec = field.spectrum(fov_waveset) # 2D * 1D -> 3D field_cube = canvas_image_hdu.data[None, :, :] * spec[:, None, None] - # logger.debug("3D FOV make_imagefields: field_cube.mean() = %f", - # field_cube.mean().value) - logger.debug(f"3D FOV make_imagefields: {field_cube.mean() = }") + logger.debug("3D FOV make_imagefields: field_cube.mean() = %f", + field_cube.mean().value) yield field_cube.value def _make_tablefields(self, fov_waveset): From 14aeaae23d5b3fee65d8d2a561e479b9d6baa36c Mon Sep 17 00:00:00 2001 From: oczoske Date: Wed, 9 Sep 2026 18:01:47 +0200 Subject: [PATCH 34/87] Test for cube source --- scopesim/tests/tests_effects/test_TERCurve.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/scopesim/tests/tests_effects/test_TERCurve.py b/scopesim/tests/tests_effects/test_TERCurve.py index aadff5cc3..67c7ed5a5 100644 --- a/scopesim/tests/tests_effects/test_TERCurve.py +++ b/scopesim/tests/tests_effects/test_TERCurve.py @@ -1,8 +1,12 @@ +"""Tests for TERCurve class""" + import pytest import numpy as np +import numpy.testing as npt from matplotlib import pyplot as plt from astropy import units as u +from astropy.wcs import WCS from scopesim.effects import ter_curves as tc from scopesim.tests.mocks.py_objects import source_objects as so @@ -20,6 +24,8 @@ def _filter_wheel(mock_path_micado): "filename_format": fname, "current_filter": "Br-gamma"}) +# pylint: disable=missing-class-docstring, +# pylint: disable=missing-function-docstring class TestTERCurveApplyTo: def test_adds_bg_to_source_if_source_has_no_bg(self): @@ -47,6 +53,26 @@ def test_adds_bg_to_source_if_source_has_no_bg(self): plt.semilogy(wave, flux, "r") plt.show() + def test_applies_to_cube_source(self): + src = so._cube_source() + wcs = WCS(src.cube_fields[0].header).spectral + nlam = src.cube_fields[0].data.shape[0] + lam = wcs.all_pix2world(np.arange(nlam), 0)[0] * u.m + + eff = eo._filter_surface() + thru = eff.throughput(lam) + + # Save spectrum before application + orig = src.cube_fields[0].data[:, 25, 25] * 1. + + # Apply effect + src = eff.apply_to(src) + + # Spectrum after application + new = src.cube_fields[0].data[:, 25, 25] + + npt.assert_allclose(new, orig * thru) + class TestTERCurvePlot: def test_plots_only_transmission(self): From 828a28438c3cf96500f03379c088c9ab69bc6ffb Mon Sep 17 00:00:00 2001 From: teutoburg Date: Wed, 9 Sep 2026 19:54:48 +0200 Subject: [PATCH 35/87] Add `Detector.__str__()` method --- scopesim/detector/detector.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scopesim/detector/detector.py b/scopesim/detector/detector.py index eb1ff3a98..0bceceeb4 100644 --- a/scopesim/detector/detector.py +++ b/scopesim/detector/detector.py @@ -103,3 +103,8 @@ def data(self, data: ArrayLike): def image(self): """Return data from internal HDU.""" return self.data + + def __str__(self) -> str: + if self.det_id is not None: + return f"Detector {self.det_id:>2d}" + return "Detector" From 38d69ac9c43070c744497d9794c7bc72352e72c1 Mon Sep 17 00:00:00 2001 From: teutoburg Date: Wed, 9 Sep 2026 19:55:22 +0200 Subject: [PATCH 36/87] Use integer detector ID in tests Like in the real case. --- scopesim/tests/tests_effects/test_prnu.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scopesim/tests/tests_effects/test_prnu.py b/scopesim/tests/tests_effects/test_prnu.py index 4d74810f5..021155fe3 100644 --- a/scopesim/tests/tests_effects/test_prnu.py +++ b/scopesim/tests/tests_effects/test_prnu.py @@ -37,10 +37,10 @@ def test_dict_prnu_std(self): """Dict mode: different amplitude per detector ID.""" hdr = header_from_list_of_xy([-5, 5], [-5, 5], 1, "D") dtcr = Detector(hdr) - dtcr.meta["id"] = "H2RG" + dtcr.meta["id"] = 0 dtcr.data[:] = 1000 prnu = PixelResponseNonUniformity( - prnu_std={"H2RG": 0.005, "GeoSnap": 0.020}, prnu_seed=42) + prnu_std={0: 0.005, 1: 0.020}, prnu_seed=42) prnu.apply_to(dtcr) assert dtcr.data.std() > 0 From b729dae6f575de491528098a5e3e4ac66431e3e6 Mon Sep 17 00:00:00 2001 From: jbrink Date: Thu, 10 Sep 2026 10:16:38 +0200 Subject: [PATCH 37/87] Fixed conserve_flux=True parameter in 3D Fov cube case - cancels out, so does not change the results. Debug logging changes. --- scopesim/optics/fov.py | 4 ++-- scopesim/optics/image_plane_utils.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/scopesim/optics/fov.py b/scopesim/optics/fov.py index da8489256..58b1b433f 100644 --- a/scopesim/optics/fov.py +++ b/scopesim/optics/fov.py @@ -605,7 +605,6 @@ def _make_scaled_hdu(self, field: ImageSourceField) -> fits.ImageHDU: """Deal with BUNIT and pixel area.""" # Note: Do not scale source data - make a copy first. field_hdu = field.field.copy() # .field is the HDU (yeah...) - logger.debug("scaling by %f", field.pixel_area.value) # TODO: Check if this scaling is actually correct. How does this # work with the add_imagehdu_to_imagehdu below? Isn't that @@ -617,6 +616,7 @@ def _make_scaled_hdu(self, field: ImageSourceField) -> fits.ImageHDU: # field_hdu.data *= self.pixel_area.value else: logger.debug("binned bunit...") + logger.debug("scaling by %f", field.pixel_area.value) field_hdu.data /= field.pixel_area.value # Pixel area doesn't cancel out, need to convert new_bunit = field.bunit / u.arcsec**2 @@ -1145,7 +1145,7 @@ def make_hdu(self): field_hdu, canvas_cube_hdu, spline_order=self.spline_order, - conserve_flux=False, + conserve_flux=True, differential=True, ) diff --git a/scopesim/optics/image_plane_utils.py b/scopesim/optics/image_plane_utils.py index 307b0e0fd..439358fbd 100644 --- a/scopesim/optics/image_plane_utils.py +++ b/scopesim/optics/image_plane_utils.py @@ -541,6 +541,8 @@ def rescale_imagehdu(imagehdu: fits.ImageHDU, pixel_scale: float | u.Quantity, return imagehdu sum_orig = np.sum(imagehdu.data) + + # scale by pixel area if the flux is differential (i.e. per unit area) if differential: sum_orig *= primary_wcs.wcs.cdelt[0]*primary_wcs.wcs.cdelt[1] @@ -600,7 +602,7 @@ def rescale_imagehdu(imagehdu: fits.ImageHDU, pixel_scale: float | u.Quantity, if sum_new != 0: flux_factor = sum_orig / sum_new - logger.info("flux factor = %f", flux_factor) + logger.debug("flux factor = %f", flux_factor) new_im *= flux_factor elif sum_orig != 0: logger.warning( From fd9d080eb491dc71d8840c279ebb75af472f547c Mon Sep 17 00:00:00 2001 From: teutoburg Date: Wed, 9 Sep 2026 19:37:18 +0200 Subject: [PATCH 38/87] Formatting --- scopesim/effects/electronic/dmps.py | 15 ++++++++------ scopesim/effects/electronic/exposure.py | 14 +++++++------ scopesim/optics/optical_train.py | 27 ++++++++++++++++--------- 3 files changed, 35 insertions(+), 21 deletions(-) diff --git a/scopesim/effects/electronic/dmps.py b/scopesim/effects/electronic/dmps.py index 1f1bcf3fd..296703a88 100644 --- a/scopesim/effects/electronic/dmps.py +++ b/scopesim/effects/electronic/dmps.py @@ -73,9 +73,11 @@ def __init__(self, **kwargs): def apply_to(self, obj, **kwargs): logger.debug("Entering DetectorModePropertiesSetter.apply_to") - mode_name = kwargs.get("detector_readout_mode", - from_currsys("!OBS.detector_readout_mode", - self.cmds)) + mode_name = kwargs.get( + "detector_readout_mode", + from_currsys("!OBS.detector_readout_mode", self.cmds), + ) + if isinstance(obj, ImagePlane) and mode_name == "auto": mode_name = self.select_mode(obj, **kwargs) logger.info("Detector mode set to %s", mode_name) @@ -101,9 +103,10 @@ def select_mode(self, obj, **kwargs): saturation level (peak to full_well). """ immax = np.max(obj.data) - fillfrac = kwargs.get("fill_frac", - from_currsys("!OBS.auto_exposure.fill_frac", - self.cmds)) + fillfrac = kwargs.get( + "fill_frac", + from_currsys("!OBS.auto_exposure.fill_frac", self.cmds), + ) goodmodes = [] goodron = [] diff --git a/scopesim/effects/electronic/exposure.py b/scopesim/effects/electronic/exposure.py index f131387dd..0b20b698e 100644 --- a/scopesim/effects/electronic/exposure.py +++ b/scopesim/effects/electronic/exposure.py @@ -70,10 +70,10 @@ def _log_dit_ndit(dit: float, ndit: int) -> None: logger.info("Total exposure time: %.3f s", dit * ndit) def estimate_dit_ndit( - self, - exptime: float, - image_plane_max: float, - **kwargs + self, + exptime: float, + image_plane_max: float, + **kwargs ) -> tuple[float, int]: """ Automatically determine DIT and NDIT from exposure time. @@ -136,8 +136,10 @@ def apply_to(self, obj, **kwargs): # in both cases, `obj` is an ImagePlane. return obj - exptime = kwargs.pop("exptime", - from_currsys("!OBS.exptime", self.cmds)) + exptime = kwargs.pop( + "exptime", + from_currsys("!OBS.exptime", self.cmds), + ) mindit = from_currsys(self.meta["mindit"], self.cmds) # TODO: Remove this silly try-except once currsys works properly... diff --git a/scopesim/optics/optical_train.py b/scopesim/optics/optical_train.py index 45cfe947e..996aa9ecf 100644 --- a/scopesim/optics/optical_train.py +++ b/scopesim/optics/optical_train.py @@ -184,12 +184,19 @@ def update(self, **kwargs): self.optics_manager.update(**kwargs) opt_man = self.optics_manager - self.fov_manager = FOVManager(opt_man.fov_setup_effects, cmds=self.cmds, - **kwargs) - self.image_planes = [ImagePlane(hdr, self.cmds, **kwargs) - for hdr in opt_man.image_plane_headers] - self.detector_managers = [DetectorManager(det_list, cmds=self.cmds, **kwargs) - for det_list in opt_man.detector_setup_effects] + self.fov_manager = FOVManager( + opt_man.fov_setup_effects, + cmds=self.cmds, + **kwargs + ) + self.image_planes = [ + ImagePlane(hdr, self.cmds, **kwargs) + for hdr in opt_man.image_plane_headers + ] + self.detector_managers = [ + DetectorManager(det_list, cmds=self.cmds, **kwargs) + for det_list in opt_man.detector_setup_effects + ] # Move everything from CurrObs to CurrSys, so CurrObs is clean for # .observe and .readout. This is necessary because the setup and @@ -488,9 +495,11 @@ def readout(self, filename=None, reset=True, **kwargs): try: hdul = self.write_header(hdul) except Exception: - logger.exception("Header update failed, data will be " - "saved with incomplete header. See stack " - "trace for details.") + logger.exception( + "Header update failed, data will be saved with " + "incomplete header. See stack trace for details." + ) + if filename is not None and isinstance(filename, str): fname = filename From d81fe40d22a33a2e6302bde000a6343e6fb93a9d Mon Sep 17 00:00:00 2001 From: teutoburg Date: Thu, 10 Sep 2026 15:20:09 +0200 Subject: [PATCH 39/87] Formatting --- scopesim/optics/fov.py | 49 ++++++----------- scopesim/optics/image_plane_utils.py | 82 ++++++++++++++++++---------- 2 files changed, 70 insertions(+), 61 deletions(-) diff --git a/scopesim/optics/fov.py b/scopesim/optics/fov.py index 1c314c007..514ef7280 100644 --- a/scopesim/optics/fov.py +++ b/scopesim/optics/fov.py @@ -2,7 +2,6 @@ """Defines FieldOfView class.""" from warnings import warn -from copy import deepcopy from itertools import chain from collections.abc import Iterable, Generator @@ -32,7 +31,6 @@ get_logger, array_minmax, close_loop, - unit_includes_per_physical_type, figure_factory, image_plotter, cube_plotter, @@ -603,14 +601,8 @@ def _make_scaled_hdu(self, field: ImageSourceField) -> fits.ImageHDU: # Note: Do not scale source data - make a copy first. field_hdu = field.field.copy() # .field is the HDU (yeah...) - # TODO: Check if this scaling is actually correct. How does this - # work with the add_imagehdu_to_imagehdu below? Isn't that - # supposed to conserve flux? Test carefully!! if field.is_bunit_spatially_differential: logger.debug("differential bunit...") - # Field is in (PHOTLAM) arcsec-2, need to scale by pixarea - # logger.debug("scaling by %f", self.pixel_area.value) - # field_hdu.data *= self.pixel_area.value else: logger.debug("binned bunit...") logger.debug("scaling by %f", field.pixel_area.value) @@ -864,20 +856,16 @@ def make_hdu(self): data=np.zeros((self.header["NAXIS2"], self.header["NAXIS1"])), header=self.header) - for tmp_hdu in chain(self._make_cubefields(), - self._make_imagefields( - fov_waveset, bin_widths)): + for tmp_hdu in chain( + self._make_cubefields(), + self._make_imagefields(fov_waveset, bin_widths) + ): canvas_image_hdu = imp_utils.add_imagehdu_to_imagehdu( tmp_hdu, canvas_image_hdu, conserve_flux=True, # binned flux needs this - spline_order=self.spline_order) - logger.info( - "2D FOV make_hdu: canvas_image_hdu.data.mean() = %f ph/s/pix", - canvas_image_hdu.data.mean()) - logger.info( - "2D FOV make_hdu: canvas_image_hdu.data.max() = %f ph/s/pix", - canvas_image_hdu.data.max()) + spline_order=self.spline_order, + ) for flux, weight, x, y in self._make_tablefields( fov_waveset, bin_widths): @@ -947,16 +935,16 @@ def _make_cubefields(self, fov_waveset): # Assumption is that ImageHDUs have units of PHOTLAM arcsec-2 # TODO: Deal with this bounds_error in a more elegant way - field_interp = interp1d(field.waveset.to(u.um).value, - field.data, axis=0, kind="linear", - bounds_error=False, fill_value=0) + field_interp = interp1d( + field.waveset.to(u.um).value, + field.data, + axis=0, + kind="linear", + bounds_error=False, + fill_value=0, + ) field_data = field_interp(fov_waveset.value) - - # Pixel scale conversion - # field_data *= field.pixel_area / self.pixel_area - logger.debug( - "3D FOV make_cubefields: field_data.mean() = %f [%s]", field_data.mean(), field.bunit) field_hdu = fits.ImageHDU(data=field_data, header=field.header) yield field_hdu @@ -989,8 +977,6 @@ def _make_imagefields(self, fov_waveset, spline_order): spec = field.spectrum(fov_waveset) # 2D * 1D -> 3D field_cube = canvas_image_hdu.data[None, :, :] * spec[:, None, None] - logger.debug("3D FOV make_imagefields: field_cube.mean() = %f", - field_cube.mean().value) yield field_cube.value def _make_tablefields(self, fov_waveset): @@ -1146,9 +1132,10 @@ def make_hdu(self): differential=True, ) - canvas_cube_hdu.data = sum(self._make_imagefields( - fov_waveset, self.spline_order), - start=canvas_cube_hdu.data) + canvas_cube_hdu.data = sum( + self._make_imagefields(fov_waveset, self.spline_order), + start=canvas_cube_hdu.data, + ) for flux, x, y in self._make_tablefields(fov_waveset): # To prevent adding array values in this manner. diff --git a/scopesim/optics/image_plane_utils.py b/scopesim/optics/image_plane_utils.py index 2372b8191..7304d41d7 100644 --- a/scopesim/optics/image_plane_utils.py +++ b/scopesim/optics/image_plane_utils.py @@ -536,10 +536,14 @@ def overlay_image(small_im, big_im, coords, mask=None, sub_pixel=False): return big_im -def rescale_imagehdu(imagehdu: fits.ImageHDU, pixel_scale: float | u.Quantity, - wcs_suffix: str = "", conserve_flux: bool = True, - spline_order: int = 1, - differential: bool = False) -> fits.ImageHDU: +def rescale_imagehdu( + imagehdu: fits.ImageHDU, + pixel_scale: float | u.Quantity, + wcs_suffix: str = "", + conserve_flux: bool = True, + spline_order: int = 1, + differential: bool = False, +) -> fits.ImageHDU: """ Scale the .data array by the ratio of pixel_scale [deg] and CDELTn. @@ -597,7 +601,7 @@ def rescale_imagehdu(imagehdu: fits.ImageHDU, pixel_scale: float | u.Quantity, # scale by pixel area if the flux is differential (i.e. per unit area) if differential: - sum_orig *= primary_wcs.wcs.cdelt[0]*primary_wcs.wcs.cdelt[1] + sum_orig *= primary_wcs.wcs.cdelt[0] * primary_wcs.wcs.cdelt[1] # Perform the rescaling. Axes need to be inverted because python. zoom_np = zoom[::-1] @@ -709,9 +713,12 @@ def rescale_imagehdu(imagehdu: fits.ImageHDU, pixel_scale: float | u.Quantity, return imagehdu -def reorient_imagehdu(imagehdu: fits.ImageHDU, wcs_suffix: str = "", - conserve_flux: bool = True, - spline_order: int = 1) -> fits.ImageHDU: +def reorient_imagehdu( + imagehdu: fits.ImageHDU, + wcs_suffix: str = "", + conserve_flux: bool = True, + spline_order: int = 1, +) -> fits.ImageHDU: """ Apply an affine transformation to the image, as given in its header. @@ -769,9 +776,15 @@ def reorient_imagehdu(imagehdu: fits.ImageHDU, wcs_suffix: str = "", return imagehdu -def affine_map(input, matrix=None, rotation_angle: float = 0., - shear_angle: float = 0., scale_factor=None, - reshape: bool = True, spline_order: int = 3): +def affine_map( + input, + matrix=None, + rotation_angle: float = 0., + shear_angle: float = 0., + scale_factor=None, + reshape: bool = True, + spline_order: int = 3, +): """ Apply an affine transformation matrix to an image around its centre. @@ -842,12 +855,14 @@ def affine_map(input, matrix=None, rotation_angle: float = 0., return output -def add_imagehdu_to_imagehdu(image_hdu: fits.ImageHDU, - canvas_hdu: fits.ImageHDU, - spline_order: int = 1, - wcs_suffix: str = "", - conserve_flux: bool = True, - differential: bool = False) -> fits.ImageHDU: +def add_imagehdu_to_imagehdu( + image_hdu: fits.ImageHDU, + canvas_hdu: fits.ImageHDU, + spline_order: int = 1, + wcs_suffix: str = "", + conserve_flux: bool = True, + differential: bool = False, +) -> fits.ImageHDU: """ Re-project one ``fits.ImageHDU`` onto another ``fits.ImageHDU``. @@ -897,17 +912,22 @@ def add_imagehdu_to_imagehdu(image_hdu: fits.ImageHDU, canvas_pixel_scale = float(canvas_wcs.wcs.cdelt[0]) conv_fac = u.Unit(image_hdu.header[f"CUNIT1{wcs_suffix}"].lower()).to(canvas_wcs.wcs.cunit[0]) - new_hdu = rescale_imagehdu(image_hdu, pixel_scale=canvas_pixel_scale / conv_fac, - wcs_suffix=canvas_wcs.wcs.alt, - spline_order=spline_order, - conserve_flux=conserve_flux, - differential=differential) + new_hdu = rescale_imagehdu( + image_hdu, + pixel_scale=canvas_pixel_scale / conv_fac, + wcs_suffix=canvas_wcs.wcs.alt, + spline_order=spline_order, + conserve_flux=conserve_flux, + differential=differential, + ) # TODO: Perhaps add separately formatted WCS logger? logger.debug("fromrescale %s", WCS(new_hdu.header, key=canvas_wcs.wcs.alt)) - new_hdu = reorient_imagehdu(new_hdu, - wcs_suffix=canvas_wcs.wcs.alt, - spline_order=spline_order, - conserve_flux=conserve_flux) + new_hdu = reorient_imagehdu( + new_hdu, + wcs_suffix=canvas_wcs.wcs.alt, + spline_order=spline_order, + conserve_flux=conserve_flux, + ) img_center = np.array([[new_hdu.header[f"NAXIS{i+1}"] for i in range(new_hdu.header["NAXIS"])]]) @@ -1250,10 +1270,12 @@ def det_wcs_from_sky_wcs( return create_wcs_from_points(corners, pixel_size, "D") -def sky_wcs_from_det_wcs(det_wcs: WCS, - pixel_scale: float, - plate_scale: float, - naxis=None) -> tuple[WCS, np.ndarray]: +def sky_wcs_from_det_wcs( + det_wcs: WCS, + pixel_scale: float, + plate_scale: float, + naxis=None, +) -> tuple[WCS, np.ndarray]: """ Create celestial WCS from detector WCS using pixel and plate scales. From 16f2b019a9c70a2045b95fbcc36e662ff5180835 Mon Sep 17 00:00:00 2001 From: teutoburg Date: Wed, 9 Sep 2026 19:56:04 +0200 Subject: [PATCH 40/87] Allow readout ID, fix Path support --- scopesim/optics/optical_train.py | 48 +++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/scopesim/optics/optical_train.py b/scopesim/optics/optical_train.py index 996aa9ecf..19245c9e2 100644 --- a/scopesim/optics/optical_train.py +++ b/scopesim/optics/optical_train.py @@ -3,11 +3,13 @@ import copy from datetime import datetime +from pathlib import Path import numpy as np from scipy.interpolate import interp1d from astropy import units as u from astropy.wcs import WCS +from astropy.io.fits import HDUList from tqdm.auto import tqdm @@ -426,23 +428,39 @@ def prepare_source(self, source): return source @top_level_catch - def readout(self, filename=None, reset=True, **kwargs): + def readout( + self, + filename: Path | str | None = None, + reset: bool = True, + roid: int = 0, + **kwargs + ) -> list[HDUList]: """ Produce detector readouts for the observed image. Parameters ---------- - filename : str, optional - Where to save the FITS file - kwargs + filename : Path | str | None, optional + Full path or local name for to store the resulting FITS file. + If None (the default), the function still returns the result, but + will not automatically save it to disk. + reset : bool, optional + If true (the default), do not keep any parameters set during any + previous readout. This is usually what you want. + roid : int, optional + Readout ID, used for seeding randomness. If you run multiple + readouts on the same observation and want the random effects to + produce different results, set this parameter with e.g. the loop + counter. Simple integers are fine (0, 1, 2, ...), because this gets + wrangled into a proper random seed downstream. If you need to + reproduce a specific readout in a sequence, set e.g. `roid=4`. + **kwargs : + Any other kwargs passed to the detector effects. Returns ------- - hdu : fits.HDUList - - Notes - ----- - - Apply detector plane (0D, 2D) effects - z_order = 500..599 + list[HDUList] + List of HDUList objects. """ if reset: @@ -500,11 +518,15 @@ def readout(self, filename=None, reset=True, **kwargs): "incomplete header. See stack trace for details." ) + if filename is not None: + if len(self.detector_managers) == 1: + fname = filename + else: + if isinstance(filename, str): + fname = f"{i}_{filename}" + if isinstance(filename, Path): + fname = filename.parent / f"{i}_{filename.name}" - if filename is not None and isinstance(filename, str): - fname = filename - if len(self.detector_managers) > 1: - fname = f"{i}_{filename}" hdul.writeto(fname, overwrite=True) hduls.append(hdul) From 120e7dc7d8e601ce013ddb10c474a14e9abafaee Mon Sep 17 00:00:00 2001 From: teutoburg Date: Thu, 10 Sep 2026 14:38:31 +0200 Subject: [PATCH 41/87] Resolve random seed in `.observe()` Concretize seed from system entropy to keep constant during the observe run. For multiple observations, this is cleared above. --- scopesim/optics/optical_train.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/scopesim/optics/optical_train.py b/scopesim/optics/optical_train.py index 996aa9ecf..385c20e5e 100644 --- a/scopesim/optics/optical_train.py +++ b/scopesim/optics/optical_train.py @@ -245,6 +245,17 @@ def observe(self, orig_source=None, update=True, **kwargs): if update: self.update(**kwargs) + if self.cmds["!SIM.random.seed"] is None: + # Concretize seed from system entropy to keep constant during the + # observe run. For multiple observations, this is cleared above. + self.cmds["!SIM.random.seed"] = np.random.SeedSequence().entropy + elif self.cmds["!SIM.random.seed"] in {"None", "none", "NONE"}: + logger.warning( + "Got random seed '%s' (a string, not None!), use NULL in yaml", + self.cmds["!SIM.random.seed"] + ) + self.cmds["!SIM.random.seed"] = np.random.SeedSequence().entropy + # self.set_focus(**kwargs) # put focus back on current instrument package # Make a copy of the Source and prepare for observation (convert to From 9e7985536a863c6c7fbc9c26b3f7880657daa6d9 Mon Sep 17 00:00:00 2001 From: teutoburg Date: Thu, 10 Sep 2026 16:39:33 +0200 Subject: [PATCH 42/87] Make sure cmds is cleared in between observations --- scopesim/optics/optical_train.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scopesim/optics/optical_train.py b/scopesim/optics/optical_train.py index 385c20e5e..8393a2e91 100644 --- a/scopesim/optics/optical_train.py +++ b/scopesim/optics/optical_train.py @@ -169,9 +169,9 @@ def load(self, user_commands): self.yaml_dicts = self.cmds.yaml_dicts self.optics_manager = OpticsManager(self.yaml_dicts, self.cmds) - self.update() + self.update(migrate_cmds=True) - def update(self, **kwargs): + def update(self, migrate_cmds=False, **kwargs): """ Update the user-defined parameters and remake main internal classes. @@ -209,7 +209,8 @@ def update(self, **kwargs): # self.cmds.maps[0].clear() # HACK: recursive_update is needed to avoid overwriting emtpy !ABCs # TODO: or is it?? - self.cmds.maps[1].dic = recursive_update(self.cmds.maps[1].dic, self.cmds.maps[0].dic) + if migrate_cmds: + self.cmds.maps[1].dic = recursive_update(self.cmds.maps[1].dic, self.cmds.maps[0].dic) self.cmds.maps[0].dic.clear() @top_level_catch From 30b8d36401300bd3479e40c1a6c2260d5d9bde6a Mon Sep 17 00:00:00 2001 From: oczoske Date: Thu, 10 Sep 2026 22:33:53 +0200 Subject: [PATCH 43/87] Set INHERIT keyword and fix unit to FITS standard --- scopesim/effects/mosaic_trace_list.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scopesim/effects/mosaic_trace_list.py b/scopesim/effects/mosaic_trace_list.py index 542755ebf..6f3083304 100644 --- a/scopesim/effects/mosaic_trace_list.py +++ b/scopesim/effects/mosaic_trace_list.py @@ -271,8 +271,9 @@ def apply_to(self, det, **kwargs): det._hdu = fits.BinTableHDU.from_columns([ fits.Column(name="wavelength", format="D", array=lam, unit="um"), - fits.Column(name="spectrum", format="D", array=spec, unit="ADU"), + fits.Column(name="spectrum", format="D", array=spec, unit="adu"), ]) + det._hdu.header["INHERIT"] = True elif output_format == "table": ntrace = len(self.spectral_traces) @@ -298,8 +299,9 @@ def apply_to(self, det, **kwargs): tab = Table(data=[idarr, xarr, yarr, lamarr, specarr], names=["id", "x", "y", "wavelength", "spectrum"], - units=["", "arcsec", "arcsec", "um", "ADU"]) + units=["", "arcsec", "arcsec", "um", "adu"]) det._hdu = fits.BinTableHDU(data=tab) + det._hdu.header["INHERIT"] = True elif output_format == "image": pass From c29df4e454e0a4eadd435449d944e8bc632e0bc8 Mon Sep 17 00:00:00 2001 From: oczoske Date: Fri, 11 Sep 2026 18:15:42 +0200 Subject: [PATCH 44/87] Fix indexing in FoV extraction --- scopesim/optics/fov.py | 15 +++++---- .../tests/tests_optics/test_FieldOfView.py | 31 +++++-------------- 2 files changed, 17 insertions(+), 29 deletions(-) diff --git a/scopesim/optics/fov.py b/scopesim/optics/fov.py index 514ef7280..960baa107 100644 --- a/scopesim/optics/fov.py +++ b/scopesim/optics/fov.py @@ -330,12 +330,12 @@ def extract_area_from_imagehdu(self, imagehdu, corners): xyp.sort(axis=0) logger.debug("xyp:\n%s", xyp) - xy0p = np.max(((0, 0), np.floor(xyp[0]).astype(int)), axis=0) - xy1p = np.min(((naxis1, naxis2), np.ceil(xyp[1]).astype(int)), axis=0) - logger.debug("xy0p: %s; xy1p: %s", xy0p, xy1p) - - # Add 1 if the same - xy1p += (xy0p == xy1p) + xy0p = np.max(((0, 0), np.floor(xyp[0]).astype(np.intp)), axis=0) + # xyp are inclusive bounds; python (in)famously excludes upper bound, + # hence xy1p needs to be increased by one + xy1p = np.min(((naxis1, naxis2), + 1 + np.ceil(xyp[1]).astype(np.intp)), + axis=0) logger.debug("xy0p: %s; xy1p: %s", xy0p, xy1p) # Describe the cutout with the input WCS, shifted by the slice origin. @@ -356,6 +356,9 @@ def extract_area_from_imagehdu(self, imagehdu, corners): new_wcs = image_wcs.deepcopy() new_wcs.wcs.crpix = image_wcs.wcs.crpix - xy0p new_naxis = xy1p - xy0p + logger.debug("orig image wcs: %s", image_wcs) + logger.debug("new cutout wcs: %s", new_wcs) + logger.debug("new cutout naxis: %s", new_naxis) new_hdr = new_wcs.to_header() new_hdr.update({"NAXIS1": new_naxis[0], "NAXIS2": new_naxis[1]}) diff --git a/scopesim/tests/tests_optics/test_FieldOfView.py b/scopesim/tests/tests_optics/test_FieldOfView.py index 49577b070..de961eb0f 100644 --- a/scopesim/tests/tests_optics/test_FieldOfView.py +++ b/scopesim/tests/tests_optics/test_FieldOfView.py @@ -1,3 +1,6 @@ +# -*- coding: utf-8 -*- +"""Test for FieldOfView classes""" + import pytest from pytest import approx import numpy as np @@ -5,7 +8,6 @@ from astropy.io import fits from astropy.table import Table from matplotlib import pyplot as plt -from matplotlib.colors import LogNorm from scopesim.tests.mocks.py_objects import header_objects as ho from scopesim.tests.mocks.py_objects import source_objects as so @@ -69,9 +71,6 @@ def test_initialises_with_header_and_waverange(self): class TestExtractFrom: - # @pytest.mark.xfail(reason=("is_field_in_fov drops table if anything is " - # "outside fov volume, therefore no point source " - # "is extracted...")) def test_extract_point_sources_from_table(self): src = so._table_source() src.fields[0].field["x"] = [-15, -5, 0, 0] * u.arcsec @@ -88,7 +87,7 @@ def test_extract_2d_image_from_hduimage(self): fov = _fov_190_210_um() fov.extract_from(src) - assert fov.fields[0].data.shape == (51, 25) + assert fov.fields[0].data.shape == (51, 26) assert len(fov.fields[0].spectra[0].waveset) == 11 assert fov.fields[0].spectra[0].waveset[0].value == approx(19000) @@ -107,11 +106,8 @@ def test_extract_3d_cube_that_is_offset_relative_to_fov(self): fov = _fov_197_202_um() fov.extract_from(src) - assert fov.fields[0].field.shape == (3, 51, 25) + assert fov.fields[0].field.shape == (3, 51, 26) - # @pytest.mark.xfail(reason=("is_field_in_fov drops table if anything is " - # "outside fov volume, therefore no point source " - # "is extracted...")) def test_extract_one_of_each_type_from_source_object(self): src_table = so._table_source() # 4 sources, put two outside of FOV src_table.fields[0].field["x"] = [-15, -5, 0, 0] * u.arcsec @@ -124,7 +120,7 @@ def test_extract_one_of_each_type_from_source_object(self): fov.extract_from(src) assert fov.fields[0].field.shape == (3, 51, 51) - assert fov.fields[1].field.shape == (51, 25) + assert fov.fields[1].field.shape == (51, 26) assert len(fov.fields[2].field) == 2 # assert len(fov.spectra) == 3 @@ -145,17 +141,6 @@ def test_ignores_fields_outside_fov_boundary(self): assert len(fov.fields) == 0 - @pytest.mark.skip(reason="SPEC_REF is obsolete, just rm this test?") - def test_all_spectra_are_referenced_correctly(self): - src = so._image_source() + so._cube_source() + so._table_source() - fov = _fov_190_210_um() - fov.extract_from(src) - # check the same spectrum object is referenced by both lists - assert fov.fields[0].header["SPEC_REF"] == \ - src.fields[0].header["SPEC_REF"] - assert all(fov.fields[2][i]["ref"] == src.fields[2][i]["ref"] - for i in range(4)) - def test_contains_all_fields_inside_fov(self): src = so._image_source() + so._cube_source() + so._table_source() the_fov = FieldOfView(ho._basic_fov_header(), (1, 2) * u.um, @@ -294,7 +279,7 @@ def test_makes_cube_from_all_types_of_source_object(self): if PLOTS: im = cube.data[0, :, :] - plt.imshow(im, origin="lower", norm=LogNorm(), vmin=1e-8) + plt.imshow(im, origin="lower", norm="log", vmin=1e-8) plt.show() @pytest.mark.parametrize("src", [so._table_source(), @@ -434,7 +419,7 @@ def test_makes_image_from_all_types_of_source_object(self): if PLOTS: im = image.data - plt.imshow(im, origin="lower", norm=LogNorm(), vmin=1e-8) + plt.imshow(im, origin="lower", norm="log", vmin=1e-8) plt.show() From a64d9b0f884e90e2c54801c955ace57c1653ce09 Mon Sep 17 00:00:00 2001 From: jbrink Date: Fri, 11 Sep 2026 10:06:39 +0200 Subject: [PATCH 45/87] Added tests to catch the through-slit imaging issue --- .../test_source_fov_imageplane_play_nicely.py | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/scopesim/tests/tests_integrations/test_source_fov_imageplane_play_nicely.py b/scopesim/tests/tests_integrations/test_source_fov_imageplane_play_nicely.py index 8f7bd3899..3816a62a1 100644 --- a/scopesim/tests/tests_integrations/test_source_fov_imageplane_play_nicely.py +++ b/scopesim/tests/tests_integrations/test_source_fov_imageplane_play_nicely.py @@ -163,3 +163,110 @@ def test_checkerboard_survives_chunking(self, half_mm, exact_extent): # no gaps and no double-counting at the chunk seams got = imp.data / imp.data.max() np.testing.assert_allclose(got, board, atol=1e-6) + + +def _flat_source(naxis=64, pixel_scale=0.064): + """Uniformly illuminated image source, one everywhere, flat spectrum.""" + from astropy.io import fits + from synphot import Empirical1D, SourceSpectrum + from scopesim.source.source import Source + + wcs = WCS(naxis=2) + wcs.wcs.ctype = ["LINEAR", "LINEAR"] + wcs.wcs.cunit = ["arcsec", "arcsec"] + wcs.wcs.cdelt = 2 * [pixel_scale] + wcs.wcs.crval = [0, 0] + wcs.wcs.crpix = 2 * [(naxis + 1) / 2] + + hdu = fits.ImageHDU(data=np.ones((naxis, naxis)), header=wcs.to_header()) + spec = SourceSpectrum(Empirical1D, points=[0.5, 2.5] * u.um, + lookup_table=[1, 1] * u.Unit("ph s-1 m-2 um-1")) + return Source(image_hdu=hdu, spectra=[spec]) + + +def _slit_fov(x, y, pixel_scale, pixel_size): + """FOV covering a slit of x, y [arcsec], built like FOVManager does.""" + skyhdr = imp_utils.header_from_list_of_xy( + [v / 3600 for v in x], [v / 3600 for v in y], + pixel_scale=pixel_scale / 3600) + dethdr, _ = imp_utils.det_wcs_from_sky_wcs( + WCS(skyhdr), pixel_scale, pixel_scale / pixel_size) + skyhdr.update(dethdr.to_header()) + return FieldOfView2D(skyhdr, waverange=[1.0, 2.0] * u.um, area=1 * u.m**2) + + +class TestFlatSourceThroughASlitFOV: + """A flat lamp seen through a slit must fill the slit, uniformly. + + A slit FOV is typically *thinner than one source pixel*: MICADO's Short + slit is 0.016" tall while a flat-lamp source is sampled at 0.064"/pix. The + cutout is grown outwards to whole source pixels, so it covers more sky than + the slit does; if its WCS is then written on the *source's* pixel lattice + rather than on the FOV's, ``overlay_image`` crops the surplus away and the + flux through the slit comes out wrong -- for the MICADO case, exactly half + of the slit was left dark. This is what commit 9740ebd6 reverted in + ``FieldOfView.extract_area_from_imagehdu``. + + Note that these tests and ``TestChunkedFOVsTileTheImagePlane`` above are the + two sides of the same trade-off: the cutout WCS can describe either the + source's pixel lattice or the FOV's, and ``extract_area_from_imagehdu`` + currently has to pick one. Resolving it properly needs sub-pixel-accurate + cropping, i.e. weighting the partial edge pixels of the cutout. + """ + + SRC_PIXEL_SCALE = 0.064 # arcsec / pix, the flat lamp + FOV_PIXEL_SCALE = 0.004 # arcsec / pix, MICADO SPEC + PIXEL_SIZE = 0.015 # mm / pix + + # slit extents [arcsec], all thinner than or comparable to a source pixel + SLITS = [ + ((-1.5, 1.5), (-0.008, 0.008)), # MICADO Short slit, centred + ((-1.5, 1.5), (0.010, 0.026)), # ... offset within one source pixel + ((-1.5, 1.5), (-0.032, 0.032)), # exactly one source pixel tall + ((-1.5, 1.5), (-0.08, 0.08)), # 2.5 source pixels tall + ((-1.5, 1.5), (0.020, 0.060)), # straddling a source pixel edge + ((-2.0, 2.0), (-0.008, 0.008)), # longer slit + ] + + def _observe(self, x, y): + fov = _slit_fov(x, y, self.FOV_PIXEL_SCALE, self.PIXEL_SIZE) + fov.extract_from(_flat_source(pixel_scale=self.SRC_PIXEL_SCALE)) + fov.view() + return fov + + @pytest.mark.parametrize(("x", "y"), SLITS) + def test_flux_through_slit_matches_slit_area(self, x, y): + fov = self._observe(x, y) + + # each source pixel carries 1 ph/s here (flat 1 ph/s/m2/um over the + # 1 um wide FOV waverange, collecting area 1 m2), so the flux through + # the slit is just the slit area in source pixels + slit_area = (x[1] - x[0]) * (y[1] - y[0]) + expected = slit_area / self.SRC_PIXEL_SCALE**2 + + assert fov.hdu.data.sum() == approx(expected, rel=1e-6) + + # ... and it survives the projection onto the image plane + x_mm = [v * self.PIXEL_SIZE / self.FOV_PIXEL_SCALE for v in x] + y_mm = [v * self.PIXEL_SIZE / self.FOV_PIXEL_SCALE for v in y] + imp = ImagePlane(imp_utils.header_from_list_of_xy( + x_mm, y_mm, pixel_scale=self.PIXEL_SIZE, wcs_suffix="D")) + imp.add(fov.hdu, wcs_suffix="D") + + assert imp.data.sum() == approx(expected, rel=1e-6) + + @pytest.mark.parametrize(("x", "y"), SLITS) + def test_illumination_is_uniform_across_the_slit(self, x, y): + fov = self._observe(x, y) + data = fov.hdu.data + + if PLOTS: + plt.plot(data.mean(axis=1)) + plt.xlabel("pixel across slit") + plt.ylabel("flux [ph s-1 pix-1]") + plt.show() + + # a flat lamp illuminates the whole slit evenly: no dark rows at the + # edges, no step in the profile across the slit + assert data.min() > 0 + np.testing.assert_allclose(data, data.max(), rtol=1e-6) From 9c7b0dd2d93791bcdfe6a86c4c384d81cfd47cac Mon Sep 17 00:00:00 2001 From: "Fabian H." <73600109+teutoburg@users.noreply.github.com> Date: Sat, 12 Sep 2026 12:50:23 +0200 Subject: [PATCH 46/87] Skip all tests in test_rst_utils.py This functionality hasn't been used in years, and the tests sometimes cause re-runs because of some Tcl/Tkinter weirdness. We can always re-enable the tests if we use it again... --- scopesim/tests/tests_reports/test_rst_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scopesim/tests/tests_reports/test_rst_utils.py b/scopesim/tests/tests_reports/test_rst_utils.py index aca00adef..fee66a7ef 100644 --- a/scopesim/tests/tests_reports/test_rst_utils.py +++ b/scopesim/tests/tests_reports/test_rst_utils.py @@ -10,6 +10,7 @@ from scopesim.tests.mocks.py_objects import report_objects as ro from scopesim.tests.mocks.py_objects import effects_objects as eo +pytestmark = pytest.mark.skip # whole file CLEAN_UP = True PLOTS = False From 719360d7baac9c146a5bf7c8051b31497f9567db Mon Sep 17 00:00:00 2001 From: "Fabian H." <73600109+teutoburg@users.noreply.github.com> Date: Sat, 12 Sep 2026 13:27:45 +0200 Subject: [PATCH 47/87] Set mpl backend for tests Attempt to reduce the annoying Tcl/Tkinter fails seen on Windows. --- scopesim/tests/conftest.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scopesim/tests/conftest.py b/scopesim/tests/conftest.py index 0be2a072f..705fd5aa3 100644 --- a/scopesim/tests/conftest.py +++ b/scopesim/tests/conftest.py @@ -6,6 +6,7 @@ import pytest from unittest.mock import patch +import matplotlib import scopesim as sim from astar_utils import UniqueList @@ -15,6 +16,8 @@ sim.rc.__currsys__["!SIM.file.error_on_missing_file"] = True +matplotlib.use("agg") # Don't render plots in normal testing + @pytest.fixture(scope="function", autouse=True) def configure_logging(): From 5752b8f885d3083d73e6ed0cdd2c473262aad333 Mon Sep 17 00:00:00 2001 From: Kieran Leschinski Date: Fri, 4 Sep 2026 22:07:33 +0200 Subject: [PATCH 48/87] Remove unused `fov_grid()` methods Some were never called, others even raised an error. In `TERCurve`, the check was ported to the `apply_to()`. Originally clauded by @astronomyk, now rebased and reworded by @teutoburg. --- pyproject.toml | 5 +++- scopesim/effects/apertures.py | 19 ------------ scopesim/effects/detector_list.py | 25 ---------------- scopesim/effects/effects.py | 24 ++++++++------- scopesim/effects/shifts.py | 29 ++----------------- scopesim/effects/surface_list.py | 26 ----------------- scopesim/effects/ter_curves.py | 8 +++++ .../test_AtmosphericDispersion.py | 24 +++++++-------- .../test_AtmosphericDispersionCorrection.py | 8 +++-- scopesim/tests/tests_effects/test_ConstPSF.py | 1 - scopesim/tests/tests_effects/test_TERCurve.py | 15 ++++++++++ .../tests/tests_effects/test_slitwheel.py | 3 -- 12 files changed, 58 insertions(+), 129 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 75e53403b..4597b8c6e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -107,7 +107,10 @@ filterwarnings = [ "ignore:invalid value encountered in divide:RuntimeWarning", "ignore:Cannot merge meta key.*:astropy.utils.metadata.MergeConflictWarning", "ignore::astropy.wcs.wcs.FITSFixedWarning", - "default:The fov_grid*:DeprecationWarning", + # Only SpectralTraceList still calls the deprecated fov_grid internally. + # Delete this line once that call moves to an internal method; every other + # fov_grid DeprecationWarning is now an error. + "default:The fov_grid*:DeprecationWarning:scopesim.effects.spectral_trace_list", # Pytest Notebook stuff "ignore:Proactor*:RuntimeWarning", ] diff --git a/scopesim/effects/apertures.py b/scopesim/effects/apertures.py index fd6984a5b..47137a145 100644 --- a/scopesim/effects/apertures.py +++ b/scopesim/effects/apertures.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- """Effects related to field masks, including spectroscopic slits.""" -import warnings from typing import ClassVar import yaml @@ -131,18 +130,6 @@ def apply_to(self, obj, **kwargs): return obj - # Outdated. Remove when removing all old FOVManager code from effects - def fov_grid(self, which="edges", **kwargs): - """Return a header with the sky coordinates.""" - warnings.warn("The fov_grid method is deprecated and will be removed " - "in a future release.", DeprecationWarning, stacklevel=2) - if which == "edges": - self.meta.update(kwargs) - return self.header - elif which == "masks": - self.meta.update(kwargs) - return self.mask - @property def hdu(self): return fits.ImageHDU(data=self.mask, header=self.header) @@ -458,12 +445,6 @@ def apply_to(self, obj, **kwargs): """Use apply_to of current_slit.""" return self.current_slit.apply_to(obj, **kwargs) - def fov_grid(self, which="edges", **kwargs): - """See parent docstring.""" - warnings.warn("The fov_grid method is deprecated and will be removed " - "in a future release.", DeprecationWarning, stacklevel=2) - return self.current_slit.fov_grid(which=which, **kwargs) - def change_slit(self, slitname=None): """Change the current slit.""" if not slitname or slitname in self.slits.keys(): diff --git a/scopesim/effects/detector_list.py b/scopesim/effects/detector_list.py index f3a1a5f09..83979273e 100644 --- a/scopesim/effects/detector_list.py +++ b/scopesim/effects/detector_list.py @@ -242,31 +242,6 @@ def _get_corner_points(self): return points * u.mm - def fov_grid(self, which="edges", **kwargs): - """Return an ApertureMask object. kwargs are "pixel_scale" [arcsec].""" - # This has really been taken care of by apply_to now - # TODO v1.0: finally rm this completely - raise AttributeError("The DetectorList.fov_grid() method has been " - "removed in v0.10.0.") - # aperture_mask = None - # if which == "edges": - # self.meta.update(kwargs) - # self.meta = from_currsys(self.meta, self.cmds) - - # hdr = self.image_plane_header - # xy_mm = calc_footprint(hdr, "D") - # pixel_size = hdr["CDELT1D"] # mm - # pixel_scale = self.meta["pixel_scale"] # ["] - - # # x["] = x[mm] * ["] / [mm] - # xy_sky = xy_mm * pixel_scale / pixel_size - - # aperture_mask = ApertureMask(array_dict={"x": xy_sky[:, 0], - # "y": xy_sky[:, 1]}, - # pixel_scale=pixel_scale) - - # return aperture_mask - @property def image_plane_header(self): """Create and return the Image Plane Header.""" diff --git a/scopesim/effects/effects.py b/scopesim/effects/effects.py index 30b4bd38f..da47304af 100644 --- a/scopesim/effects/effects.py +++ b/scopesim/effects/effects.py @@ -11,20 +11,19 @@ from ..reports.rst_utils import table_to_rst -# FIXME: This docstring is out-of-date for several reasons: -# - Effects can act on objects other than Source (eg FOV, IMP, DET) -# - fov_grid is outdated - - # @dataclass(kw_only=True, eq=False) class Effect: """ Base class for representing the effects (artifacts) in an optical system. The ``Effect`` class is conceived to independently apply the changes that - an optical component (or series thereof) has on an incoming 3D description - of an on-sky object. In other words, **an Effect object should receive a - derivative of a ``Source`` object, alter it somehow, and return it**. + an optical component (or series thereof) has on an incoming description of + an on-sky object. **An Effect object should receive an object, alter it + somehow, and return an instance of the same class.** Besides ``Source``, + that object may be a ``FovVolumeList``, a ``FieldOfView``, an + ``ImagePlane`` or a ``Detector``; ``apply_to`` is expected to dispatch on + the type it is given and to pass anything it does not handle straight + through. The interface for the Effect base-class has been kept very general so that it can easily be sub-classed as data for new effects becomes available. @@ -32,9 +31,12 @@ class Effect: attributes: * ``self.meta`` - a dictionary to contain metadata. - * ``self.apply_to(obj, **kwargs)`` - a method which accepts a - Source-derivative and returns an instance of the same class as ``obj`` - * ``self.fov_grid(which="", **kwargs)`` + * ``self.apply_to(obj, **kwargs)`` - a method which accepts one of the + objects above and returns an instance of the same class as ``obj`` + + An Effect that needs to constrain the source-space volume the simulation + covers does so in the ``FovVolumeList`` branch of ``apply_to``, by calling + ``shrink``, ``extract`` or ``split`` on the list it is passed. Parameters diff --git a/scopesim/effects/shifts.py b/scopesim/effects/shifts.py index 1fc9d79bf..492550b71 100644 --- a/scopesim/effects/shifts.py +++ b/scopesim/effects/shifts.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- """TBA.""" -import warnings from typing import ClassVar import numpy as np @@ -27,17 +26,6 @@ def apply_to(self, obj, **kwargs): """See parent docstring.""" return obj - def fov_grid(self, which="shifts", **kwargs): - """See parent docstring.""" - warnings.warn("The fov_grid method is deprecated and will be removed " - "in a future release.", DeprecationWarning, stacklevel=2) - if which == "shifts": - col_names = ["wavelength", "dx", "dy"] - waves, dx, dy = [self.get_table(**kwargs)[col] - for col in col_names] - return waves, dx, dy - return None - def get_table(self, **kwargs): if self.table is None: names = ["wavelength", "dx", "dy"] @@ -70,7 +58,7 @@ class AtmosphericDispersion(Shift3D): Used to generate the wavelength bins based on shifts due to the atmosphere. Doesn't contain an ``apply_to`` function, but provides information through - the ``fov_grid`` function. + the ``get_table`` function. Required Parameters ------------------- @@ -133,7 +121,7 @@ def __init__(self, **kwargs): def get_table(self, **kwargs): """ - Called by the fov_grid method of Shift3D. + Compute the wavelength-dependent refraction shifts. Returns ------- @@ -248,19 +236,6 @@ def apply_to(self, fov, **kwargs): return fov - def fov_grid(self, which="shifts", **kwargs): - """See parent docstring.""" - warnings.warn("The fov_grid method is deprecated and will be removed " - "in a future release.", DeprecationWarning, stacklevel=2) - kwargs.update(self.meta) - if "quick_adc" in self.meta: - ad = AtmosphericDispersion(**self.meta) - waves, dx, dy = ad.fov_grid() - dx *= -(1 - self.meta["efficiency"]) - dy *= -(1 - self.meta["efficiency"]) - return waves, dx, dy - return None - def plot(self): return None diff --git a/scopesim/effects/surface_list.py b/scopesim/effects/surface_list.py index 70d3e0006..1db0cf948 100644 --- a/scopesim/effects/surface_list.py +++ b/scopesim/effects/surface_list.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- """TBA.""" -import warnings from collections import OrderedDict from copy import deepcopy from typing import ClassVar @@ -39,31 +38,6 @@ def __init__(self, **kwargs): surf_kwargs["filename"] = from_currsys(surf_kwargs["filename"], self.cmds) self.surfaces[surf_kwargs["name"]] = SpectralSurface(**surf_kwargs) - def fov_grid(self, which="waveset", **kwargs): - warnings.warn("The fov_grid method is deprecated and will be removed " - "in a future release.", DeprecationWarning, stacklevel=2) - wave_edges = [] - if which == "waveset": - self.meta.update(kwargs) - self.meta = from_currsys(self.meta, self.cmds) - wave_min = quantify(self.meta["wave_min"], u.um) - wave_max = quantify(self.meta["wave_max"], u.um) - # ..todo:: add 1001 to default.yaml somewhere - wave = np.linspace(wave_min, wave_max, 1001) - throughput = self.throughput(wave) - threshold = self.meta["minimum_throughput"] - valid_waves = np.where(throughput >= threshold)[0] - - if not len(valid_waves): - msg = ("No transmission found above the threshold " - f"{self.meta['minimum_throughput']} in this wavelength " - f"range {[self.meta['wave_min'], self.meta['wave_max']]}." - " Did you open the shutter?") - raise ValueError(msg) - - wave_edges = [min(wave[valid_waves]), max(wave[valid_waves])] - return wave_edges - @property def throughput(self): if self._throughput is None: diff --git a/scopesim/effects/ter_curves.py b/scopesim/effects/ter_curves.py index 53641a043..591301832 100644 --- a/scopesim/effects/ter_curves.py +++ b/scopesim/effects/ter_curves.py @@ -130,6 +130,14 @@ def _apply_to_fvl(self, fvl: FovVolumeList) -> None: wave = self.surface.throughput.waveset thru = self.surface.throughput(wave) valid_waves = np.argwhere(thru > 0) + + if not len(valid_waves): + raise ValueError( + f"No transmission found in {self.display_name} over the " + f"wavelength range [{wave.min().to_value(u.um):.3f}, " + f"{wave.max().to_value(u.um):.3f}] um. " + "Did you open the shutter?") + wave_min = wave[max(0, valid_waves[0][0] - 1)] wave_max = wave[min(len(wave) - 1, valid_waves[-1][0] + 1)] diff --git a/scopesim/tests/tests_effects/test_AtmosphericDispersion.py b/scopesim/tests/tests_effects/test_AtmosphericDispersion.py index 392d9e7a4..a578674a1 100644 --- a/scopesim/tests/tests_effects/test_AtmosphericDispersion.py +++ b/scopesim/tests/tests_effects/test_AtmosphericDispersion.py @@ -44,21 +44,17 @@ def test_throws_error_if_one_keyword_missing(self, atmo_yaml_dict): isinstance(AtmosphericDispersion(), AtmosphericDispersion) -class TestFovGrid: - def test_returns_list_of_3_arrays_with_correct_which(self, atmo_yaml_dict): +class TestGetTable: + def test_returns_wavelength_dx_dy_columns(self, atmo_yaml_dict): atmo_disp = AtmosphericDispersion(**atmo_yaml_dict["properties"]) - response = atmo_disp.fov_grid() - assert len(response) == 3 - assert all([isinstance(x, np.ndarray) for x in response]) - - def test_returns_non_with_wrong_which_keyword(self, atmo_yaml_dict): - atmo_disp = AtmosphericDispersion(**atmo_yaml_dict["properties"]) - response = atmo_disp.fov_grid(which="bogus") - assert response is None + tbl = atmo_disp.get_table() + assert tbl.colnames == ["wavelength", "dx", "dy"] + assert all(isinstance(tbl[col], np.ndarray) for col in tbl.colnames) def test_returns_similar_values_to_lasilla_website(self, atmo_yaml_dict): atmo_disp = AtmosphericDispersion(**atmo_yaml_dict["properties"]) - waves, dx, dy = atmo_disp.fov_grid() + tbl = atmo_disp.get_table() + waves, dx, dy = tbl["wavelength"], tbl["dx"], tbl["dy"] assert dy[0] - dy[-1] == approx(0.53, rel=1e-2) assert all(dx == 0) assert waves[0] == 0.5 and waves[-1] == 2.5 @@ -66,14 +62,16 @@ def test_returns_similar_values_to_lasilla_website(self, atmo_yaml_dict): def test_returns_same_results_when_turned_90_degrees(self, atmo_yaml_dict): atmo_yaml_dict["properties"]["pupil_angle"] = 90 atmo_disp = AtmosphericDispersion(**atmo_yaml_dict["properties"]) - waves, dx, dy = atmo_disp.fov_grid() + tbl = atmo_disp.get_table() + dx, dy = tbl["dx"], tbl["dy"] assert dx[0] - dx[-1] == approx(0.53, rel=1e-2) assert all([y == approx(0) for y in dy]) def test_returns_same_results_when_turned_30_degrees(self, atmo_yaml_dict): atmo_yaml_dict["properties"]["pupil_angle"] = 30 atmo_disp = AtmosphericDispersion(**atmo_yaml_dict["properties"]) - waves, dx, dy = atmo_disp.fov_grid() + tbl = atmo_disp.get_table() + dx, dy = tbl["dx"], tbl["dy"] dr = ((dx[0] - dx[-1])**2 + (dy[0] - dy[-1])**2)**0.5 assert dr == approx(0.53, rel=1e-2) diff --git a/scopesim/tests/tests_effects/test_AtmosphericDispersionCorrection.py b/scopesim/tests/tests_effects/test_AtmosphericDispersionCorrection.py index 4ff560b9f..55b83bb70 100644 --- a/scopesim/tests/tests_effects/test_AtmosphericDispersionCorrection.py +++ b/scopesim/tests/tests_effects/test_AtmosphericDispersionCorrection.py @@ -101,9 +101,11 @@ def test_shifts_between_adc_and_ad_are_opposite(self, atmo_params, waves, ad = AD(**atmo_params) adc = ADC(**atmo_params) - ad_shifts = ad.fov_grid() - ad_x_shift = np.interp(fov_wave_mid, ad_shifts[0], ad_shifts[1]) - ad_y_shift = np.interp(fov_wave_mid, ad_shifts[0], ad_shifts[2]) + ad_shifts = ad.get_table() + ad_x_shift = np.interp(fov_wave_mid, ad_shifts["wavelength"], + ad_shifts["dx"]) + ad_y_shift = np.interp(fov_wave_mid, ad_shifts["wavelength"], + ad_shifts["dy"]) adc.apply_to(fov) new_crpix_d = np.array([fov.header["CRPIX1D"], fov.header["CRPIX2D"]]) diff --git a/scopesim/tests/tests_effects/test_ConstPSF.py b/scopesim/tests/tests_effects/test_ConstPSF.py index d6270dd1c..27706b5d9 100644 --- a/scopesim/tests/tests_effects/test_ConstPSF.py +++ b/scopesim/tests/tests_effects/test_ConstPSF.py @@ -12,7 +12,6 @@ # - psf : returns self, as array returns a layer based on defaults # 4. should have the following methods: # - (?) apply_to(obj) -# - fov_grid(which="waveset") # - get_kernel(obj) """ diff --git a/scopesim/tests/tests_effects/test_TERCurve.py b/scopesim/tests/tests_effects/test_TERCurve.py index 67c7ed5a5..cf508a3c3 100644 --- a/scopesim/tests/tests_effects/test_TERCurve.py +++ b/scopesim/tests/tests_effects/test_TERCurve.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """Tests for TERCurve class""" import pytest @@ -7,8 +8,10 @@ from matplotlib import pyplot as plt from astropy import units as u from astropy.wcs import WCS +from astropy.table import Table from scopesim.effects import ter_curves as tc +from scopesim.optics.fov_volume_list import FovVolumeList from scopesim.tests.mocks.py_objects import source_objects as so from scopesim.tests.mocks.py_objects import effects_objects as eo @@ -73,6 +76,18 @@ def test_applies_to_cube_source(self): npt.assert_allclose(new, orig * thru) + def test_raises_helpful_error_when_nothing_transmits(self): + """A fully opaque surface used to die on a bare IndexError inside the + volume setup. The error must name the effect and the waverange.""" + tbl = Table(data=[[0.5, 1.0, 2.0, 3.0], [0.0, 0.0, 0.0, 0.0]], + names=["wavelength", "transmission"]) + tbl["wavelength"].unit = "um" + eff = tc.TERCurve(table=tbl, wave_min=0.5, wave_max=3.0, + name="closed shutter") + + with pytest.raises(ValueError, match="Did you open the shutter"): + eff.apply_to(FovVolumeList()) + class TestTERCurvePlot: def test_plots_only_transmission(self): diff --git a/scopesim/tests/tests_effects/test_slitwheel.py b/scopesim/tests/tests_effects/test_slitwheel.py index b46f10f19..707e3dac8 100644 --- a/scopesim/tests/tests_effects/test_slitwheel.py +++ b/scopesim/tests/tests_effects/test_slitwheel.py @@ -27,9 +27,6 @@ def test_has_aperture_masks(self, swheel): def test_current_slit_is_aperture_mask(self, swheel): assert isinstance(swheel.current_slit, ApertureMask) - def test_current_slit_has_fov_grid_method(self, swheel): - assert hasattr(swheel.current_slit, "fov_grid") - def test_change_to_known_slit(self, swheel): swheel.change_slit('A') assert swheel.current_slit.meta['name'] == 'A' From eecba478c37c838544e48f5351e6056259adf369 Mon Sep 17 00:00:00 2001 From: teutoburg Date: Sat, 12 Sep 2026 13:03:37 +0200 Subject: [PATCH 49/87] Remove old comments --- scopesim/source/source.py | 67 +-------------------------------------- 1 file changed, 1 insertion(+), 66 deletions(-) diff --git a/scopesim/source/source.py b/scopesim/source/source.py index b738cb92d..181aa63fb 100644 --- a/scopesim/source/source.py +++ b/scopesim/source/source.py @@ -1,24 +1,5 @@ +# -*- coding: utf-8 -*- """ -# old functionality to implement: -# - provide x, y, lam, spectra, weight, ref -# - overridden + : number, Source, SourceSpectrum -# - overridden * : number, SpectralElement -# - write to and read from file -# - shift all fields -# - rotate around the centre -# - photons_in_range returns the photons per spectrum in a wavelength range -# - image_in_range returns an image of the source for a wavelength range -# -# old functionality which will be removed: -# - project_onto_chip -# - apply_optical_train -# -# old structure --> new structure: -# - all data held in 6 arrays -# --> new dicts for fields, spectrum -# field can be a Table or an ImageHDU -# spectrum is a SourceSpectrum -# # Use cases: # image + spectrum # images + spectra @@ -242,27 +223,6 @@ def _from_imagehdu_and_spectra(self, image_hdu, **kwargs): assert not self.fields, "Constructor method must act on empty instance!" - if not image_hdu.header.get("BG_SRC"): - pass - # FIXME: This caused more problems than it solved! - # Find out if there's a good reason to mess with this, - # otherwise just remove... - - # image_hdu.header["CRVAL1"] = 0 - # image_hdu.header["CRVAL2"] = 0 - # image_hdu.header["CRPIX1"] = image_hdu.header["NAXIS1"] / 2 - # image_hdu.header["CRPIX2"] = image_hdu.header["NAXIS2"] / 2 - # #image_hdu.header["CRPIX1"] = (image_hdu.header["NAXIS1"] + 1) / 2 - # #image_hdu.header["CRPIX2"] = (image_hdu.header["NAXIS2"] + 1) / 2 - # # .. todo:: find where the actual problem is with negative CDELTs - # # .. todo:: --> abs(pixel_scale) in header_from_list_of_xy - # if image_hdu.header["CDELT1"] < 0: - # image_hdu.header["CDELT1"] *= -1 - # image_hdu.data = image_hdu.data[:, ::-1] - # if image_hdu.header["CDELT2"] < 0: - # image_hdu.header["CDELT2"] *= -1 - # image_hdu.data = image_hdu.data[::-1, :] - if isinstance(image_hdu, fits.PrimaryHDU): image_hdu = fits.ImageHDU(data=image_hdu.data, header=image_hdu.header) @@ -270,7 +230,6 @@ def _from_imagehdu_and_spectra(self, image_hdu, if not spectra: raise ValueError("No spectrum was provided.") - # image_hdu.header["SPEC_REF"] = len(self.spectra) assert len(spectra) == 1, f"_from_imagehdu_and_spectra needs single spectrum, ref was {image_hdu.header.get('SPEC_REF')}" image_hdu.header["SPEC_REF"] = 0 @@ -455,7 +414,6 @@ def bandpass(self, bandpass): self._bandpass = bandpass - # ..todo: rewrite this method def image_in_range(self, wave_min, wave_max, pixel_scale=1*u.arcsec, layers=None, area=None, spline_order=1, sub_pixel=False): if layers is None: @@ -481,17 +439,6 @@ def image_in_range(self, wave_min, wave_max, pixel_scale=1*u.arcsec, flux = self.photons_in_range(wave_min, wave_max, area, ref) # [ph s-1] or [ph s-1 m-2] come out of photons_in_range - # ## ..todo: CATCH UNITS HERE. DEAL WITH THEM PROPERLY - # Currently assuming that all images are scaled appropriately - # and that they have SPEC_REF - - # else: - # field = scale_imagehdu(field, area=area, - # solid_angle=pixel_scale**2, - # waverange=(wave_min, wave_max)) - # # [ph s-1] or [ph s-1 m-2] come out of photons_in_range - # flux = 1 - image = field.data * flux hdu = fits.ImageHDU(header=field.header, data=image) hdu_or_table = hdu @@ -537,18 +484,6 @@ def fluxes(self, wave_min, wave_max, **kwargs): def image(self, wave_min, wave_max, **kwargs): return self.image_in_range(wave_min, wave_max, **kwargs) - # @classmethod - # def load(cls, filename): - # """Load :class:'.Source' object from filename""" - # with open(filename, "rb") as fp1: - # src = pickle.load(fp1) - # return src - - # def dump(self, filename): - # """Save to filename as a pickle""" - # with open(filename, "wb") as fp1: - # pickle.dump(self, fp1) - def shift(self, dx: float = 0, dy: float = 0, layers=None) -> None: """ Shift the position of one or more fields w.r.t. the optical axis. From 1519e9c45080d5095ddec8323dcf7edb713687c3 Mon Sep 17 00:00:00 2001 From: teutoburg Date: Sun, 13 Sep 2026 18:23:57 +0200 Subject: [PATCH 50/87] SimCADO corpse removal All of this was only used in tests, not in any live part of ScopeSim. Some things were still used to setup or assert tests on live code, so I moved those to `/tests`. --- scopesim/optics/image_plane.py | 7 +- scopesim/optics/image_plane_utils.py | 340 +-------- scopesim/source/source.py | 74 -- scopesim/source/source_templates.py | 5 +- scopesim/source/source_utils.py | 90 +-- ...t_flux_is_conserved_through_full_system.py | 21 +- .../tests/tests_optics/test_ImagePlane.py | 698 ++++-------------- .../tests_optics/test_ImagePlaneUtils.py | 21 - scopesim/tests/tests_source/source_helpers.py | 95 +++ .../tests/tests_source/test_source_Source.py | 176 +---- 10 files changed, 267 insertions(+), 1260 deletions(-) create mode 100644 scopesim/tests/tests_source/source_helpers.py diff --git a/scopesim/optics/image_plane.py b/scopesim/optics/image_plane.py index e2b101f48..7b94adaf0 100644 --- a/scopesim/optics/image_plane.py +++ b/scopesim/optics/image_plane.py @@ -1,15 +1,10 @@ # -*- coding: utf-8 -*- """Contains ``ImagePlane`` class.""" -from warnings import warn - -import numpy as np - from astropy.io import fits -from astropy.table import Table from astropy.wcs import WCS -from .image_plane_utils import add_table_to_imagehdu, add_imagehdu_to_imagehdu +from .image_plane_utils import add_imagehdu_to_imagehdu from ..utils import ( from_currsys, diff --git a/scopesim/optics/image_plane_utils.py b/scopesim/optics/image_plane_utils.py index 7304d41d7..15f91daf5 100644 --- a/scopesim/optics/image_plane_utils.py +++ b/scopesim/optics/image_plane_utils.py @@ -1,160 +1,19 @@ # -*- coding: utf-8 -*- -from itertools import product -from collections.abc import Iterable - import numpy as np from numpy.typing import ArrayLike, NDArray from astropy import units as u from astropy.wcs import WCS, find_all_wcs from astropy.io import fits -from astropy.table import Table from astropy.utils.exceptions import AstropyWarning from scipy import ndimage as ndi -from ..utils import (unit_from_table, quantity_from_table, has_needed_keywords, - get_logger) +from ..utils import get_logger logger = get_logger(__name__) -def get_canvas_header(hdu_or_table_list, pixel_scale=1 * u.arcsec): - """ - Generate a fits.Header with a WCS that covers everything in the FOV. - - Parameters - ---------- - hdu_or_table_list : list - A list of Tables and/or ImageHDU py_objects - - pixel_scale : astropy.Quantity - [arcsec] The pixel scale of the projection. Default in 1 arcsec - - Returns - ------- - header : fits.Header - A Header containing a WCS and NAXISn values to build an ImageHDU - - """ - size_warning = ("Header dimension are {adverb} large: {num_pix}. " - "Any image made from this header will use more that " - ">{size} in memory") - - def _get_headers(hdus_or_tables): - tables = [] - for hdu_or_table in hdus_or_tables: - if isinstance(hdu_or_table, fits.ImageHDU): - yield hdu_or_table.header - elif isinstance(hdu_or_table, fits.Header): - yield hdu_or_table - elif isinstance(hdu_or_table, Table): - tables.append(hdu_or_table) - else: - raise TypeError( - "hdu_or_table_list may only contain fits.ImageHDU, Table " - f"or fits.Header, found {type(hdu_or_table)}.") - if tables: - yield _make_bounding_header_for_tables(*tables, - pixel_scale=pixel_scale) - - headers = list(_get_headers(hdu_or_table_list)) - - if not headers: - logger.warning("No tables or ImageHDUs were passed") - return None - - hdr = _make_bounding_header_from_headers(*headers, pixel_scale=pixel_scale) - - num_pix = hdr["NAXIS1"] * hdr["NAXIS2"] - if num_pix > 2 ** 28: - raise MemoryError(size_warning.format(adverb="too", num_pix=num_pix, - size="8 GB")) - if num_pix > 2 ** 25: # 2 * 4096**2 - logger.warning(size_warning.format(adverb="", num_pix=num_pix, - size="256 MB")) - return hdr - - -def _make_bounding_header_from_headers(*headers, pixel_scale=1*u.arcsec): - """ - Return a Header with WCS and NAXISn keywords bounding all input ImageHDUs. - - Parameters - ---------- - headers : list of fits.ImageHDU - pixel_scale : u.Quantity - [arcsec] - - Returns - ------- - hdr : fits.Header - - """ - wcs_suffix = "D" if pixel_scale.unit.physical_type == "length" else "" - unit = u.Unit(_get_unit_from_headers(*headers, wcs_suffix=wcs_suffix)) - - if unit.physical_type == "angle": - unit = "deg" - pixel_scale = pixel_scale.to_value(u.deg) - else: - pixel_scale = pixel_scale.to_value(unit) - - extents = [calc_footprint(header, wcs_suffix, unit) for header in headers] - pnts = np.vstack(extents) - - hdr = header_from_list_of_xy(pnts[:, 0], pnts[:, 1], - pixel_scale, wcs_suffix) - hdr["NAXIS1"] += 1 - hdr["NAXIS2"] += 1 - hdr[f"CRVAL1{wcs_suffix}"] -= 0.5 * pixel_scale - hdr[f"CRVAL2{wcs_suffix}"] -= 0.5 * pixel_scale - - return hdr - - -def _make_bounding_header_for_tables(*tables, pixel_scale=1*u.arcsec): - """ - Return a Header with WCS and NAXISn keywords bounding all input Tables. - - Parameters - ---------- - tables : list of astropy.Tables - [arcsec] Must contain columns: "x", "y" - - pixel_scale : u.Quantity - [arcsec] - - Returns - ------- - hdr : fits.Header - - """ - wcs_suffix = "D" if pixel_scale.unit.physical_type == "length" else "" - # FIXME: Convert to deg here? If yes, remove the arcsec=True below... - # Note: this could all be a lot simpler if we have consistent units, i.e. - # don't need to convert mm -> mm and arcsec -> arcsec (or deg) - new_unit = u.mm if wcs_suffix == "D" else u.arcsec # u.deg - tbl_unit = u.mm if wcs_suffix == "D" else u.arcsec - x_name = "x_mm" if wcs_suffix == "D" else "x" - y_name = "y_mm" if wcs_suffix == "D" else "y" - - pixel_scale = pixel_scale.to(new_unit) - - extents = [] - for table in tables: - extent = calc_table_footprint(table, x_name, y_name, - tbl_unit, new_unit, - padding=pixel_scale) - extents.append(extent) - pnts = np.vstack(extents) - - # TODO: check if this could just use create_wcs_from_points - hdr = header_from_list_of_xy(pnts[:, 0], pnts[:, 1], pixel_scale.value, - wcs_suffix, arcsec=wcs_suffix != "D") - return hdr - - def create_wcs_from_points( points: ArrayLike, pixel_scale: float, @@ -313,95 +172,6 @@ def header_from_list_of_xy(x, y, pixel_scale, wcs_suffix="", arcsec=False): return hdr -def add_table_to_imagehdu(table: Table, canvas_hdu: fits.ImageHDU, - sub_pixel: bool = True, - wcs_suffix: str = "") -> fits.ImageHDU: - """ - Add files from an astropy.Table to the image of an fits.ImageHDU. - - Parameters - ---------- - table : astropy.Table - Must contain the columns "x_mm", "y_mm", "flux" with the units in the - column attribute .unit, or in the table.meta dictionary as - "_unit". Default units are ``mm`` and ``ph / s / pix`` - - canvas_hdu : fits.ImageHDU - The ImageHDU onto which the table files should be projected. - This must include a valid WCS - - sub_pixel : bool, optional - Default is True. If True, sub-pixel shifts of files will be taken into - account when projecting onto the canvas pixel grid. This takes about 5x - longer than ignoring the sub-pixel shifts - - wcs_suffix : str, optional - - Returns - ------- - canvas_hdu : fits.ImageHDU - - """ - s = wcs_suffix - if not has_needed_keywords(canvas_hdu.header, s): - raise ValueError(f"canvas_hdu must include an appropriate WCS: {s}") - - f = quantity_from_table("flux", table, default_unit=u.Unit("ph s-1")) - if s == "D": - x = quantity_from_table("x_mm", table, default_unit=u.mm).to(u.mm) - y = quantity_from_table("y_mm", table, default_unit=u.mm).to(u.mm) - else: - arcsec = u.arcsec - canvas_unit = u.Unit(canvas_hdu.header[f"CUNIT1{s}"]) - x = quantity_from_table("x", table, default_unit=arcsec.to(canvas_unit)) - y = quantity_from_table("y", table, default_unit=arcsec.to(canvas_unit)) - x *= arcsec.to(canvas_unit) - y *= arcsec.to(canvas_unit) - - xpix, ypix = val2pix(canvas_hdu.header, x.value, y.value, s) - - naxis1 = canvas_hdu.header["NAXIS1"] - naxis2 = canvas_hdu.header["NAXIS2"] - # Occasionally 0 is returned as ~ -1e-11 - eps = -1e-7 - mask = (xpix >= eps) * (xpix < naxis1) * (ypix >= eps) * (ypix < naxis2) - - if sub_pixel: - canvas_hdu = _add_subpixel_sources_to_canvas( - canvas_hdu, xpix, ypix, f, mask) - else: - canvas_hdu = _add_intpixel_sources_to_canvas( - canvas_hdu, xpix, ypix, f, mask) - - return canvas_hdu - - -def _add_intpixel_sources_to_canvas(canvas_hdu, xpix, ypix, flux, mask): - canvas_hdu.header["comment"] = f"Adding {len(flux)} int-pixel files" - for xpx, ypx, flx, msk in zip(xpix.astype(int), ypix.astype(int), - flux, mask): - # To prevent adding array values in this manner. - assert not isinstance(xpx, Iterable), "xpx should be integer" - canvas_hdu.data[ypx, xpx] += flx.value * msk - - return canvas_hdu - - -def _add_subpixel_sources_to_canvas(canvas_hdu, xpix, ypix, flux, mask): - canvas_hdu.header["comment"] = f"Adding {len(flux)} sub-pixel files" - canvas_shape = canvas_hdu.data.shape - for xpx, ypx, flx, msk in zip(xpix, ypix, flux, mask): - if msk: - xx, yy, fracs = sub_pixel_fractions(xpx, ypx) - for x, y, frac in zip(xx, yy, fracs): - if y < canvas_shape[0] and x < canvas_shape[1]: - # To prevent adding array values in this manner. - assert not isinstance(x, Iterable), "x should be integer" - canvas_hdu.data[y, x] += frac * flx.value - - return canvas_hdu - - def sub_pixel_fractions(x, y): """ Make a list of pixel coordinates and weights to reflect sub-pixel shifts. @@ -955,6 +725,7 @@ def add_imagehdu_to_imagehdu( return canvas_hdu +# TODO: Use proper WCS functions instead and rm this on-foot implementation! def pix2val(header, x, y, wcs_suffix=""): """ Return the real coordinates [deg, mm] for coordinates from a Header WCS. @@ -995,6 +766,7 @@ def pix2val(header, x, y, wcs_suffix=""): return a, b +# TODO: Use proper WCS functions instead and rm this on-foot implementation! def val2pix(header, a, b, wcs_suffix=""): """ Return the pixel coordinates for real coordinates [deg, mm] from a WCS. @@ -1105,103 +877,6 @@ def calc_footprint(header, wcs_suffix="", new_unit: str = None): return xy1 -def calc_table_footprint( - table: Table, - x_name: str, - y_name: str, - tbl_unit: str, - new_unit: str, - padding: u.Quantity | None = None, -) -> NDArray: - """ - Equivalent to ``calc_footprint()``, but for tables instead of images. - - Parameters - ---------- - table : astropy.table.Table - Table containing data. - x_name : str - Name of the column in `table` to use as x-coordinates. - y_name : str - Name of the column in `table` to use as y-coordinates. - tbl_unit : str - Default unit to use for x and y if no units are found in `table`. - new_unit : str - Unit to convert x and y to, can be identical to `tbl_unit`. - padding : astropy.units.Quantity, optional - Constant value to subtract from minima and add to maxima. If used, must - be Quantity with same physical type as x and y. If None (default), no - padding is added. - - Returns - ------- - extent : (4, 2) array - Array containing corner points (clockwise from bottom left). Format and - order are equivalent to the output of - ``astropy.wcs.WCS.calc_footprint()``. - - """ - if padding is not None: - padding = padding.to_value(new_unit) - else: - padding = 0. - - x_convf = unit_from_table(x_name, table, tbl_unit).to(new_unit) - y_convf = unit_from_table(y_name, table, tbl_unit).to(new_unit) - - x_col = table[x_name] * x_convf - y_col = table[y_name] * y_convf - - x_min = x_col.min() - padding - x_max = x_col.max() + padding - y_min = y_col.min() - padding - y_max = y_col.max() + padding - - extent = np.array([[x_min, y_min], - [x_min, y_max], - [x_max, y_max], - [x_max, y_min]]) - - return extent - - -def split_header(hdr, chunk_size, wcs_suffix=""): - """ - Split a header into many smaller parts of the chunk_size. - - Parameters - ---------- - hdr - chunk_size - wcs_suffix - - Returns - ------- - hdr_list - """ - # TODO: test that this works - s = wcs_suffix - naxis1, naxis2 = hdr["NAXIS1"+s], hdr["NAXIS2"+s] - x0_pix, y0_pix = hdr["CRPIX1"+s], hdr["CRPIX2"+s] # pix - x0_sky, y0_sky = hdr["CRVAL1"+s], hdr["CRVAL2"+s] # deg - x_delt, y_delt = hdr["CDELT1"+s], hdr["CDELT2"+s] # deg / pix - - hdr_list = [] - for x1_pix in range(0, naxis1, chunk_size): - for y1_pix in range(0, naxis2, chunk_size): - x1_sky = x0_sky + (x1_pix - x0_pix) * x_delt - y1_sky = y0_sky + (y1_pix - y0_pix) * y_delt - x2_sky = x1_sky + x_delt * min(chunk_size, naxis1 - x1_pix) - y2_sky = y1_sky + y_delt * min(chunk_size, naxis2 - y1_pix) - - hdr_sky = header_from_list_of_xy([x1_sky, x2_sky], - [y1_sky, y2_sky], - pixel_scale=x_delt, wcs_suffix=s) - hdr_list.append(hdr_sky) - - return hdr_list - - def _fix_360(arr): """Fix the "full circle overflow" that occurs with deg.""" if isinstance(arr, u.Quantity): @@ -1214,15 +889,6 @@ def _fix_360(arr): return arr -def _get_unit_from_headers(*headers, wcs_suffix: str = "") -> str: - unit = headers[0][f"CUNIT1{wcs_suffix}"].lower() - assert all(header[f"CUNIT{i}{wcs_suffix}"].lower() == unit - for header, i in product(headers, range(1, 3))), \ - [(i, header[f"CUNIT{i}{wcs_suffix}"]) - for header, i in product(headers, range(1, 3))] - return unit - - def det_wcs_from_sky_wcs( sky_wcs: WCS, pixel_scale: float, diff --git a/scopesim/source/source.py b/scopesim/source/source.py index 181aa63fb..8d264b061 100644 --- a/scopesim/source/source.py +++ b/scopesim/source/source.py @@ -25,18 +25,14 @@ from synphot import SpectralElement, SourceSpectrum -from ..optics.image_plane import ImagePlane -from ..optics import image_plane_utils as imp_utils from .source_utils import ( validate_source_input, convert_to_list_of_spectra, - photons_in_range, ) from .source_fields import ( SourceField, TableSourceField, SpectrumSourceField, - HDUSourceField, ImageSourceField, CubeSourceField, ) @@ -44,7 +40,6 @@ find_file, is_fits, get_fits_type, - quantity_from_table, figure_factory, get_logger, ) @@ -414,75 +409,6 @@ def bandpass(self, bandpass): self._bandpass = bandpass - def image_in_range(self, wave_min, wave_max, pixel_scale=1*u.arcsec, - layers=None, area=None, spline_order=1, sub_pixel=False): - if layers is None: - layers = range(len(self.fields)) - fields = [self.fields[ii].field for ii in layers] - - hdr = imp_utils.get_canvas_header(fields, pixel_scale=pixel_scale) - im_plane = ImagePlane(hdr) - - for field in fields: - if isinstance(field, Table): - fluxes = self.photons_in_range(wave_min, wave_max, area, - field["ref"]) * field["weight"] - x = quantity_from_table("x", field, u.arcsec) - y = quantity_from_table("y", field, u.arcsec) - tbl = Table(names=["x", "y", "flux"], data=[x, y, fluxes]) - tbl.meta.update(field.meta) - hdu_or_table = tbl - - elif isinstance(field, fits.ImageHDU): - if field.header["SPEC_REF"] != "": - ref = [field.header["SPEC_REF"]] - flux = self.photons_in_range(wave_min, wave_max, area, ref) - # [ph s-1] or [ph s-1 m-2] come out of photons_in_range - - image = field.data * flux - hdu = fits.ImageHDU(header=field.header, data=image) - hdu_or_table = hdu - else: - continue - - im_plane.add(hdu_or_table, sub_pixel=sub_pixel, - spline_order=spline_order) - - return im_plane - - def photons_in_range(self, wave_min, wave_max, area=None, indices=None): - """ - - Parameters - ---------- - wave_min : float, u.Quantity - [um] - wave_max : float, u.Quantity - [um] - area : float, u.Quantity, optional - [m2] - indices : list of integers, optional - - Returns - ------- - counts : u.Quantity list - [ph / s / m2] if area is None - [ph / s] if area is passed - - """ - if indices is None: - indices = self.spectra.keys() - - spectra = [self.spectra[ii] for ii in indices] - counts = photons_in_range(spectra, wave_min, wave_max, area=area, - bandpass=self.bandpass) - return counts - - def fluxes(self, wave_min, wave_max, **kwargs): - return self.photons_in_range(wave_min, wave_max, **kwargs) - - def image(self, wave_min, wave_max, **kwargs): - return self.image_in_range(wave_min, wave_max, **kwargs) def shift(self, dx: float = 0, dy: float = 0, layers=None) -> None: """ diff --git a/scopesim/source/source_templates.py b/scopesim/source/source_templates.py index 0a88afbc6..7b4b04757 100644 --- a/scopesim/source/source_templates.py +++ b/scopesim/source/source_templates.py @@ -1,11 +1,11 @@ +# -*- coding: utf-8 -*- + from pathlib import Path import numpy as np - from astropy import units as u from astropy.table import Table from astropy.io import fits -from astropy.utils.decorators import deprecated_renamed_argument from synphot import SourceSpectrum, ConstFlux1D, Empirical1D from synphot.units import PHOTLAM @@ -30,7 +30,6 @@ def empty_sky(flux=0): return sky -@deprecated_renamed_argument("mag", "flux", "0.1.5") def star(x=0, y=0, flux=0): """ Source object for a single star in either vega, AB magnitudes, or Jansky diff --git a/scopesim/source/source_utils.py b/scopesim/source/source_utils.py index caf43a218..d969149be 100644 --- a/scopesim/source/source_utils.py +++ b/scopesim/source/source_utils.py @@ -4,11 +4,10 @@ from pathlib import Path import numpy as np -from scipy import integrate from astropy import wcs, units as u from astropy.io import fits from astropy.table import Table -from synphot import SourceSpectrum, Empirical1D, SpectralElement +from synphot import SourceSpectrum, Empirical1D from ..utils import find_file, get_logger, convert_table_comments_to_dict @@ -111,93 +110,6 @@ def _get_list(): return list(_get_list()) -# FIXME: typing: This should work with the more general (and true, since -# conversion is done anyway) Quantity["length"] (or "area" resp.), but -# doing so currently causes a NameError. Not sure what's going on. -def photons_in_range( - spectra: SourceSpectrum, - wave_min: u.Quantity[u.um] | float, - wave_max: u.Quantity[u.um] | float, - area: u.Quantity[u.m**2] | float | None = None, - bandpass: SpectralElement | None = None, -) -> u.Quantity[u.ph * u.s**-1 * u.m**-2] | u.Quantity[u.ph * u.s**-1]: - """ - Integrate photons from spectrum in given wavelength range. - - Parameters - ---------- - spectra : SourceSpectrum - Input spectrum. - wave_min : u.Quantity["length"] or float - Minimum wavelength. If float, assumes um. - wave_max : u.Quantity["length"] or float - Maximum wavelength. If float, assumes um. - area : u.Quantity["area"] or float, optional - Area to multiply with. If float, assumes m**2. The default is None. - bandpass : SpectralElement, optional - Filter to take into account, if any. The default is None. - - Returns - ------- - counts : astropy.units.Quantity - Either in ph/s/m**2 or just ph/s (if area was given). - - """ - # Note: Assuming um if given as float. - wave_min = (wave_min << u.um << u.Angstrom).value - wave_max = (wave_max << u.um << u.Angstrom).value - # Note: There appear to be some float shenanigans going on here, but - # rounding produces an error in the spectrum evaluation. Not sure what's - # going on, maybe it's fine as-is. - - counts = [] - for spec in spectra: - waveset = spec.waveset.value - mask = (waveset > wave_min) * (waveset < wave_max) - wave = np.array([wave_min, *waveset[mask], wave_max]) - flux = spec(wave).value - - # flux [ph s-1 cm-2] == flux [ph s-1 cm-2 AA-1] * wave [AA] - if isinstance(bandpass, SpectralElement): - bandpass.model.bounds_error = True - counts.append(integrate.trapezoid(bandpass(wave).value * flux, wave)) - else: - counts.append(integrate.trapezoid(flux, wave)) - - # counts = flux [ph s-1 cm-2] - counts = (counts * u.ph * u.s**-1 * u.cm**-2).to(u.ph * u.s**-1 * u.m**-2) - if area is not None: - counts *= (area << u.m**2) - - return counts - - -def scale_imagehdu(imagehdu, waverange, area=None): - """ - ..todo: implement this - - For the moment, all imagehdu must be accompanied by a spectrum in PHOTLAM - - Future functionality will include scaling here of: - ph s-1 - ph s-1 m-2 - ph s-1 m-2 - ph s-1 m-2 um-1 - ph s-1 m-2 um-1 arcsec-2 - J s-1 m-2 Hz-1 - J s-1 m-2 Hz-1 arcsec-2 - ABMAG - ABMAG arcsec-2 - VEGAMAG - VEGAMAG arcsec-2 - """ - if "SPEC_REF" not in imagehdu.header: - raise ValueError("For this version, an ImageHDU must be associated " - "with a spectrum. This will change in the future.") - - return imagehdu - - def make_img_wcs_header( pixel_scale: float, image_size: tuple[int, int], diff --git a/scopesim/tests/tests_integrations/test_flux_is_conserved_through_full_system.py b/scopesim/tests/tests_integrations/test_flux_is_conserved_through_full_system.py index 65690fe0d..6c7542b5c 100644 --- a/scopesim/tests/tests_integrations/test_flux_is_conserved_through_full_system.py +++ b/scopesim/tests/tests_integrations/test_flux_is_conserved_through_full_system.py @@ -1,18 +1,21 @@ +# -*- coding: utf-8 -*- import pytest from pytest import approx from unittest.mock import patch import numpy as np +import matplotlib.pyplot as plt from scopesim.optics.optical_train import OpticalTrain from scopesim.commands import UserCommands from scopesim.tests.mocks.py_objects.source_objects import ( - _image_source, _single_table_source, _table_source_overlapping) - -import matplotlib.pyplot as plt -from matplotlib.colors import LogNorm + _image_source, + _single_table_source, + _table_source_overlapping, +) +from scopesim.tests.tests_source.source_helpers import src_photons_in_range PLOTS = False @@ -48,11 +51,11 @@ def test_flux_is_conserved_for_no_bg_emission(self, cmds, tbl_src): opt.observe(tbl_src) im = opt.image_planes[0].image bg_flux = np.pi / 4 * np.prod(im.shape) - src_flux = tbl_src.photons_in_range(1, 2, 1)[0].value + src_flux = src_photons_in_range(tbl_src, 1, 2, 1)[0].value if PLOTS: implane = opt.image_planes[0] - plt.imshow(implane.image.T, origin="lower", norm=LogNorm()) + plt.imshow(implane.image.T, origin="lower", norm="log") plt.colorbar() plt.show() @@ -70,11 +73,11 @@ def test_flux_is_conserved_for_yes_bg_emission(self, non_unity_cmds, tbl_src): opt = OpticalTrain(non_unity_cmds) opt.observe(tbl_src) im = opt.image_planes[0].image - src_flux = tbl_src.photons_in_range(1, 2, 1)[0].value + src_flux = src_photons_in_range(tbl_src, 1, 2, 1)[0].value if PLOTS: implane = opt.image_planes[0] - plt.imshow(implane.image.T, origin="lower", norm=LogNorm()) + plt.imshow(implane.image.T, origin="lower", norm="log") plt.colorbar() plt.show() @@ -133,7 +136,7 @@ def test_stacked_stars(self): if PLOTS: fig, axes = plt.subplots(nrows=1, ncols=1, figsize=(12, 12)) - axes.imshow(np.sqrt(im_h), norm=LogNorm(), cmap="inferno") + axes.imshow(np.sqrt(im_h), norm="log", cmap="inferno") axes.set_title('H-band MORFEO "PSF Generic" ') plt.show() diff --git a/scopesim/tests/tests_optics/test_ImagePlane.py b/scopesim/tests/tests_optics/test_ImagePlane.py index 1a8a81190..c941e903e 100644 --- a/scopesim/tests/tests_optics/test_ImagePlane.py +++ b/scopesim/tests/tests_optics/test_ImagePlane.py @@ -1,9 +1,11 @@ +# -*- coding: utf-8 -*- """Tests for ImagePlane and some ImagePlaneUtils""" # pylint: disable=missing-class-docstring # pylint: disable=missing-function-docstring from copy import deepcopy +from itertools import product import pytest from pytest import approx @@ -11,22 +13,108 @@ import numpy as np from astropy.io import fits from astropy import units as u -from astropy.table import Table from astropy import wcs - import matplotlib.pyplot as plt -from matplotlib.colors import LogNorm import scopesim.optics.image_plane as opt_imp import scopesim.optics.image_plane_utils as imp_utils -from scopesim.tests.mocks.py_objects.imagehdu_objects import \ - _image_hdu_square, _image_hdu_rect, _image_hdu_three_wcs,\ - _image_hdu_3d_data +from scopesim.tests.mocks.py_objects.imagehdu_objects import ( + _image_hdu_square, + _image_hdu_rect, + _image_hdu_three_wcs, + _image_hdu_3d_data, +) PLOTS = False +# The three functions below were oringinally stored in image_plane_utils, +# but ultimately only used here, so I moved them here as helpers. +# TODO: Check if the same functionality can be achieved in another way. +# ScopeSim internally uses something else (imageplane headers from detectors), +# so tests should ideally mock that instead... +def get_canvas_header(hdu_or_table_list, pixel_scale=1*u.arcsec): + """ + Generate a fits.Header with a WCS that covers everything in the FOV. + + Parameters + ---------- + hdu_or_table_list : list + A list of Tables and/or ImageHDU py_objects + + pixel_scale : astropy.Quantity + [arcsec] The pixel scale of the projection. Default in 1 arcsec + + Returns + ------- + header : fits.Header + A Header containing a WCS and NAXISn values to build an ImageHDU + + """ + def _get_headers(hdus_or_tables): + for hdu_or_table in hdus_or_tables: + if isinstance(hdu_or_table, fits.ImageHDU): + yield hdu_or_table.header + elif isinstance(hdu_or_table, fits.Header): + yield hdu_or_table + else: + raise TypeError( + "hdu_or_table_list may only contain fits.ImageHDU " + f"or fits.Header, found {type(hdu_or_table)}.") + + headers = list(_get_headers(hdu_or_table_list)) + + hdr = _make_bounding_header_from_headers(*headers, pixel_scale=pixel_scale) + return hdr + + +def _make_bounding_header_from_headers(*headers, pixel_scale): + """ + Return a Header with WCS and NAXISn keywords bounding all input ImageHDUs. + + Parameters + ---------- + headers : list of fits.ImageHDU + pixel_scale : u.Quantity + [arcsec] + + Returns + ------- + hdr : fits.Header + + """ + def _get_unit_from_headers(*headers, wcs_suffix: str) -> str: + unit = headers[0][f"CUNIT1{wcs_suffix}"].lower() + assert all(header[f"CUNIT{i}{wcs_suffix}"].lower() == unit + for header, i in product(headers, range(1, 3))), \ + [(i, header[f"CUNIT{i}{wcs_suffix}"]) + for header, i in product(headers, range(1, 3))] + return unit + + wcs_suffix = "D" if pixel_scale.unit.physical_type == "length" else "" + unit = u.Unit(_get_unit_from_headers(*headers, wcs_suffix=wcs_suffix)) + + if unit.physical_type == "angle": + unit = "deg" + pixel_scale = pixel_scale.to_value(u.deg) + else: + pixel_scale = pixel_scale.to_value(unit) + + extents = [imp_utils.calc_footprint(header, wcs_suffix, unit) + for header in headers] + pnts = np.vstack(extents) + + hdr = imp_utils.header_from_list_of_xy( + pnts[:, 0], pnts[:, 1], pixel_scale, wcs_suffix) + hdr["NAXIS1"] += 1 + hdr["NAXIS2"] += 1 + hdr[f"CRVAL1{wcs_suffix}"] -= 0.5 * pixel_scale + hdr[f"CRVAL2{wcs_suffix}"] -= 0.5 * pixel_scale + + return hdr + + @pytest.fixture(scope="function", name="image_hdu_rect") def fixture_image_hdu_rect(): return _image_hdu_rect() @@ -57,361 +145,14 @@ def fixture_image_hdu_3d_data(): return _image_hdu_3d_data() -@pytest.fixture(scope="function", name="input_table") -def fixture_input_table(): - x = [-10, -10, 0, 10, 10] * u.arcsec - y = [-10, 10, 0, -10, 10] * u.arcsec - f = [1, 3, 1, 1, 5] - tbl = Table(names=["x", "y", "flux"], data=[x, y, f]) - - return tbl - - -@pytest.fixture(scope="function", name="input_table_mm") -def fixture_input_table_mm(): - x = [-10, -10, 0, 10, 10] * u.mm - y = [-10, 10, 0, -10, 10] * u.mm - f = [1, 3, 1, 1, 5] - tbl = Table(names=["x_mm", "y_mm", "flux"], data=[x, y, f]) - - return tbl - - -class TestCombineTableBoundaries: - def test_all_three_tables_are_inside_header_wcs(self, input_table): - tbl1 = deepcopy(input_table) - tbl2 = deepcopy(input_table) - tbl3 = deepcopy(input_table) - - tbl2["x"] -= 25 - tbl3["y"] -= 60 - - hdr = imp_utils._make_bounding_header_for_tables(tbl1, tbl2, tbl3) - as2deg = u.arcsec.to(u.deg) - for tbl in [tbl1, tbl2, tbl3]: - x, y = imp_utils.val2pix(hdr, tbl["x"]*as2deg, tbl["y"]*as2deg) - for xi, yi in zip(x, y): - assert 0 <= xi < hdr["NAXIS1"] - assert 0 <= yi < hdr["NAXIS2"] - - if PLOTS: - xy = imp_utils.calc_footprint(hdr) - x, y = xy[:, 0], xy[:, 1] - x, y = imp_utils.val2pix(hdr, x, y) - x0, y0 = imp_utils.val2pix(hdr, 0, 0) - - plt.plot(x, y, "b") - plt.plot(x0, y0, "ro") - for tbl in [tbl1, tbl2, tbl3]: - x, y = imp_utils.val2pix(hdr, tbl["x"] / 3600., - tbl["y"] / 3600.) - plt.plot(x, y, "k.") - - plt.show() - - def test_all_three_mm_tables_are_inside_header_wcs(self, input_table_mm): - tbl1 = deepcopy(input_table_mm) - tbl2 = deepcopy(input_table_mm) - tbl3 = deepcopy(input_table_mm) - - tbl2["x_mm"] += 50 - tbl3["y_mm"] += 25 - - hdr = imp_utils._make_bounding_header_for_tables(tbl1, tbl2, tbl3, - pixel_scale=100*u.um) - for tbl in [tbl1, tbl2, tbl3]: - x, y = imp_utils.val2pix(hdr, tbl["x_mm"], tbl["y_mm"], "D") - for xi, yi in zip(x, y): - assert 0 <= xi < hdr["NAXIS1"] - assert 0 <= yi < hdr["NAXIS2"] - - if PLOTS: - xy = imp_utils.calc_footprint(hdr, "D") - x, y = xy[:, 0], xy[:, 1] - x, y = imp_utils.val2pix(hdr, x, y, "D") - x0, y0 = imp_utils.val2pix(hdr, 0, 0, "D") - - plt.plot(x, y, "b") - plt.plot(x0, y0, "ro") - for tbl in [tbl1, tbl2, tbl3]: - x, y = imp_utils.val2pix(hdr, tbl["x_mm"], tbl["y_mm"], "D") - plt.plot(x, y, "k.") - - plt.show() - - -class TestCombineImageHDUBoundaries: - def test_all_two_imagehdus_are_inside_header_wcs(self, image_hdu_square, - image_hdu_rect): - - image_hdu_rect.header["CRVAL1"] -= 70 # * u.arcsec.to(u.deg) - image_hdu_square.header["CRVAL2"] += 70 # * u.arcsec.to(u.deg) - - hdr = imp_utils._make_bounding_header_from_headers( - image_hdu_square.header, image_hdu_rect.header) - for imhdr in [image_hdu_square.header, image_hdu_rect.header]: - xy = imp_utils.calc_footprint(imhdr) - x, y = xy[:, 0], xy[:, 1] - x *= u.arcsec.to(u.deg) - y *= u.arcsec.to(u.deg) - x, y = imp_utils.val2pix(hdr, x, y) - for xi, yi in zip(x, y): - assert 0 <= xi < hdr["NAXIS1"] - assert 0 <= yi < hdr["NAXIS2"] - - if PLOTS: - for imhdr in [image_hdu_square.header, image_hdu_rect.header, hdr]: - xy = imp_utils.calc_footprint(imhdr) - x, y = xy[:, 0], xy[:, 1] - xp, yp = imp_utils.val2pix(imhdr, x, y) - plt.plot(x, y, "r-") - - plt.plot(0, 0, "ro") - plt.gca().set_aspect(1) - plt.show() - - def test_all_two_mm_imagehdus_are_inside_header_wcs(self, - image_hdu_square_mm, - image_hdu_rect_mm): - - image_hdu_rect_mm.header["CRVAL1D"] -= 40 - image_hdu_square_mm.header["CRVAL2D"] += 80 - - hdr = imp_utils._make_bounding_header_from_headers( - image_hdu_square_mm.header, image_hdu_rect_mm.header, - pixel_scale=1*u.mm) - for imhdr in [image_hdu_square_mm.header, image_hdu_rect_mm.header]: - xy = imp_utils.calc_footprint(imhdr, "D") - x, y = xy[:, 0], xy[:, 1] - x, y = imp_utils.val2pix(hdr, x, y, "D") - for xi, yi in zip(x, y): - assert 0 <= xi < hdr["NAXIS1"] - assert 0 <= yi < hdr["NAXIS2"] - - if PLOTS: - for imhdr in [image_hdu_square_mm.header, - image_hdu_rect_mm.header, hdr]: - xy = imp_utils.calc_footprint(imhdr, "D") - x, y = xy[:, 0], xy[:, 1] - xp, yp = imp_utils.val2pix(imhdr, x, y, "D") - plt.plot(x, y, "r-") - - plt.plot(0, 0, "ro") - plt.gca().set_aspect(1) - plt.show() - - -class TestGetCanvasHeader: - def test_all_5_objects_are_inside_header_wcs(self, image_hdu_square, - image_hdu_rect, input_table): - - tbl1 = deepcopy(input_table) - tbl2 = deepcopy(input_table) - tbl3 = deepcopy(input_table) - - tbl2["x"] -= 150 - tbl3["y"] -= 100 - - image_hdu_rect.header["CRVAL1"] += 100 * u.arcsec.to(u.deg) - image_hdu_square.header["CRVAL1"] += 0 * u.arcsec.to(u.deg) - image_hdu_square.header["CRVAL2"] += 100 * u.arcsec.to(u.deg) - - hdr = imp_utils.get_canvas_header([image_hdu_square, tbl1, tbl2, tbl3, - image_hdu_rect], pixel_scale=1*u.arcsec) - - as2deg = u.arcsec.to(u.deg) - - for im in [image_hdu_square.header, image_hdu_rect.header]: - xy = imp_utils.calc_footprint(im) - x, y = xy[:, 0], xy[:, 1] - x, y = imp_utils.val2pix(hdr, x*as2deg, y*as2deg) - for xi, yi in zip(x, y): - assert 0 <= xi < hdr["NAXIS1"] - assert 0 <= yi < hdr["NAXIS2"] - - for tbl in [tbl1, tbl2, tbl3]: - x, y = imp_utils.val2pix(hdr, tbl["x"]*as2deg, tbl["y"]*as2deg) - for xi, yi in zip(x, y): - assert 0 <= xi < hdr["NAXIS1"] - assert 0 <= yi < hdr["NAXIS2"] - - if PLOTS: - - xy = imp_utils.calc_footprint(hdr) - x, y = xy[:, 0], xy[:, 1] - x, y = imp_utils.val2pix(hdr, x, y) - plt.plot(x, y, "b") - x0, y0 = imp_utils.val2pix(hdr, 0, 0) - plt.plot(x0, y0, "ro") - - for tbl in [tbl1, tbl2, tbl3]: - x, y = imp_utils.val2pix(hdr, tbl["x"]*as2deg, tbl["y"]*as2deg) - plt.plot(x, y, "k.") - - for im in [image_hdu_square.header, image_hdu_rect.header]: - xy = imp_utils.calc_footprint(im) - x, y = xy[:, 0], xy[:, 1] - x, y = imp_utils.val2pix(hdr, x, y) - plt.plot(x, y, "r-") - - plt.gca().set_aspect(1) - plt.show() - - def test_all_5_objects_are_inside_mm_header_wcs(self, image_hdu_square_mm, - image_hdu_rect_mm, - input_table_mm): - - tbl1 = deepcopy(input_table_mm) - tbl2 = deepcopy(input_table_mm) - tbl3 = deepcopy(input_table_mm) - - tbl2["x_mm"] -= 150 - tbl3["y_mm"] -= 100 - - image_hdu_rect_mm.header["CRVAL1D"] += 100 - image_hdu_square_mm.header["CRVAL1D"] += 0 - image_hdu_square_mm.header["CRVAL2D"] += 100 - - hdr = imp_utils.get_canvas_header([image_hdu_square_mm, tbl1, tbl2, - tbl3, image_hdu_rect_mm], - pixel_scale=1*u.mm) - - for im in [image_hdu_square_mm.header, image_hdu_rect_mm.header]: - xy = imp_utils.calc_footprint(im, "D") - x, y = xy[:, 0], xy[:, 1] - x, y = imp_utils.val2pix(hdr, x, y, "D") - for xi, yi in zip(x, y): - assert 0 <= xi < hdr["NAXIS1"] - assert 0 <= yi < hdr["NAXIS2"] - - for tbl in [tbl1, tbl2, tbl3]: - x, y = imp_utils.val2pix(hdr, tbl["x_mm"], tbl["y_mm"], "D") - for xi, yi in zip(x, y): - assert 0 <= xi < hdr["NAXIS1"] - assert 0 <= yi < hdr["NAXIS2"] - - if PLOTS: - - xy = imp_utils.calc_footprint(hdr, "D") - x, y = xy[:, 0], xy[:, 1] - x, y = imp_utils.val2pix(hdr, x, y, "D") - plt.plot(x, y, "b") - x0, y0 = imp_utils.val2pix(hdr, 0, 0, "D") - plt.plot(x0, y0, "ro") - - for tbl in [tbl1, tbl2, tbl3]: - x, y = imp_utils.val2pix(hdr, tbl["x_mm"], tbl["y_mm"], "D") - plt.plot(x, y, "k.") - - for im in [image_hdu_square_mm.header, image_hdu_rect_mm.header]: - xy = imp_utils.calc_footprint(im, "D") - x, y = xy[:, 0], xy[:, 1] - x, y = imp_utils.val2pix(hdr, x, y, "D") - plt.plot(x, y, "r-") - - plt.gca().set_aspect(1) - plt.show() - - -class TestAddTableToImageHDU: - def test_points_are_added_to_small_canvas(self, input_table): - tbl1 = deepcopy(input_table) - hdr = imp_utils.get_canvas_header([tbl1]) - - im = np.zeros((hdr["NAXIS2"], hdr["NAXIS1"])) - canvas_hdu = fits.ImageHDU(header=hdr, data=im) - canvas_hdu = imp_utils.add_table_to_imagehdu(tbl1, canvas_hdu) - - assert np.sum(canvas_hdu.data) == np.sum(tbl1["flux"]) - - if PLOTS: - # "top left is green, top right is yellow" - plt.imshow(canvas_hdu.data, origin="lower") - plt.show() - - def test_mm_points_are_added_to_small_canvas(self, input_table_mm): - tbl1 = deepcopy(input_table_mm) - hdr = imp_utils.get_canvas_header([tbl1], pixel_scale=2*u.mm) - - im = np.zeros((hdr["NAXIS2"], hdr["NAXIS1"])) - canvas_hdu = fits.ImageHDU(header=hdr, data=im) - canvas_hdu = imp_utils.add_table_to_imagehdu(tbl1, canvas_hdu, - wcs_suffix="D") - - assert np.sum(canvas_hdu.data) == np.sum(tbl1["flux"]) - - if PLOTS: - # "top left is green, top right is yellow" - plt.imshow(canvas_hdu.data, origin="lower") - plt.show() - - # add a paramaterised test here to check border cases - def test_points_are_added_to_massive_canvas(self, input_table): - tbl1 = deepcopy(input_table) - tbl2 = deepcopy(input_table) - tbl3 = deepcopy(input_table) - - tbl1["y"] += 50 - tbl2["x"] += 20 - tbl3["x"] -= 25 - tbl3["y"] -= 25 - - hdr = imp_utils.get_canvas_header([tbl1, tbl2, tbl3], - pixel_scale=1*u.arcsec) - - im = np.zeros((hdr["NAXIS2"], hdr["NAXIS1"])) - canvas_hdu = fits.ImageHDU(header=hdr, data=im) - - for tbl in [tbl1, tbl2, tbl3]: - canvas_hdu = imp_utils.add_table_to_imagehdu(tbl, canvas_hdu, True) - - total_flux = np.sum([tbl1["flux"], tbl2["flux"], tbl3["flux"]]) - assert np.sum(canvas_hdu.data) == total_flux - - if PLOTS: - x, y = imp_utils.val2pix(hdr, 0, 0) - plt.plot(x, y, "ro") - "top left is green, top right is yellow" - plt.imshow(canvas_hdu.data, origin="lower") - plt.show() - - def test_mm_points_are_added_to_massive_canvas(self, input_table_mm): - tbl1 = deepcopy(input_table_mm) - tbl2 = deepcopy(input_table_mm) - tbl3 = deepcopy(input_table_mm) - - tbl1["y_mm"] += 50 - tbl2["x_mm"] += 20 - tbl3["x_mm"] -= 25 - tbl3["y_mm"] -= 25 - - hdr = imp_utils.get_canvas_header([tbl1, tbl2, tbl3], - pixel_scale=1*u.mm) - im = np.zeros((hdr["NAXIS2"], hdr["NAXIS1"])) - canvas_hdu = fits.ImageHDU(header=hdr, data=im) - - for tbl in [tbl1, tbl2, tbl3]: - canvas_hdu = imp_utils.add_table_to_imagehdu(tbl, canvas_hdu, - True, wcs_suffix="D") - - total_flux = np.sum([tbl1["flux"], tbl1["flux"], tbl1["flux"]]) - assert np.sum(canvas_hdu.data) == total_flux - - if PLOTS: - x, y = imp_utils.val2pix(hdr, 0, 0, "D") - plt.plot(x, y, "ro") - # "top left is green, top right is yellow" - plt.imshow(canvas_hdu.data, origin="lower") - plt.show() - - class TestAddImageHDUToImageHDU: - def test_image_is_added_to_small_canvas(self, image_hdu_rect, - image_hdu_square): + def test_image_is_added_to_small_canvas( + self, image_hdu_rect, image_hdu_square, + ): im_hdu = image_hdu_rect im_hdu.header["CRVAL1"] -= 150*u.arcsec.to(u.deg) im_hdu.header["CRVAL2"] += 40*u.arcsec.to(u.deg) - hdr = imp_utils.get_canvas_header([im_hdu, image_hdu_square]) + hdr = get_canvas_header([im_hdu, image_hdu_square]) im = np.zeros((hdr["NAXIS2"], hdr["NAXIS1"])) canvas_hdu = fits.ImageHDU(header=hdr, data=im) @@ -436,18 +177,19 @@ def test_image_is_added_to_small_canvas(self, image_hdu_rect, plt.show() - def test_mm_image_is_added_to_small_canvas(self, image_hdu_rect_mm, - image_hdu_square_mm): + def test_mm_image_is_added_to_small_canvas( + self, image_hdu_rect_mm, image_hdu_square_mm, + ): im_hdu = image_hdu_rect_mm im_hdu.header["CRVAL1D"] -= 150 im_hdu.header["CRVAL2D"] += 40 - hdr = imp_utils.get_canvas_header([im_hdu, image_hdu_square_mm], - pixel_scale=1*u.mm) + hdr = get_canvas_header([im_hdu, image_hdu_square_mm], 1*u.mm) im = np.zeros((hdr["NAXIS2"], hdr["NAXIS1"])) canvas_hdu = fits.ImageHDU(header=hdr, data=im) - canvas_hdu = imp_utils.add_imagehdu_to_imagehdu(im_hdu, canvas_hdu, - wcs_suffix="D") + canvas_hdu = imp_utils.add_imagehdu_to_imagehdu( + im_hdu, canvas_hdu, wcs_suffix="D", + ) assert np.sum(canvas_hdu.data) == approx(np.sum(im_hdu.data)) @@ -466,32 +208,21 @@ def test_mm_image_is_added_to_small_canvas(self, image_hdu_rect_mm, plt.show() - def test_image_and_tables_on_large_canvas(self, input_table, image_hdu_rect, - image_hdu_square): - tbl1 = deepcopy(input_table) - tbl2 = deepcopy(input_table) - - tbl1["y"] -= 100 - tbl2["x"] += 100 - tbl2["y"] += 100 - + def test_images_on_large_canvas(self, image_hdu_rect, image_hdu_square): im_hdu = image_hdu_rect im_hdu.header["CRVAL1"] -= 150 # *u.arcsec.to(u.deg) im_hdu.header["CRVAL2"] += 20 # *u.arcsec.to(u.deg) - total_flux = (np.sum(tbl1["flux"]) + np.sum(tbl2["flux"]) + - im_hdu.data.sum() + image_hdu_square.data.sum()) + total_flux = im_hdu.data.sum() + image_hdu_square.data.sum() - hdr = imp_utils.get_canvas_header([im_hdu, image_hdu_square, - tbl1, tbl2], pixel_scale=3*u.arcsec) + hdr = get_canvas_header([im_hdu, image_hdu_square], 3*u.arcsec) im = np.zeros((hdr["NAXIS2"], hdr["NAXIS1"])) canvas_hdu = fits.ImageHDU(header=hdr, data=im) canvas_hdu = imp_utils.add_imagehdu_to_imagehdu(im_hdu, canvas_hdu) - canvas_hdu = imp_utils.add_imagehdu_to_imagehdu(image_hdu_square, - canvas_hdu) - canvas_hdu = imp_utils.add_table_to_imagehdu(tbl1, canvas_hdu) - canvas_hdu = imp_utils.add_table_to_imagehdu(tbl2, canvas_hdu) + canvas_hdu = imp_utils.add_imagehdu_to_imagehdu( + image_hdu_square, canvas_hdu, + ) assert np.sum(canvas_hdu.data) == approx(total_flux, rel=1e-2) @@ -511,39 +242,28 @@ def test_image_and_tables_on_large_canvas(self, input_table, image_hdu_rect, plt.show() - def test_mm_image_and_tables_on_large_canvas(self, input_table_mm, - image_hdu_rect_mm, - image_hdu_square_mm): + def test_mm_images_on_large_canvas( + self, image_hdu_rect_mm, image_hdu_square_mm, + ): image_hdu_rect = image_hdu_rect_mm image_hdu_square = image_hdu_square_mm - tbl1 = deepcopy(input_table_mm) - tbl2 = deepcopy(input_table_mm) - - tbl1["y_mm"] -= 100 - tbl2["x_mm"] += 100 - tbl2["y_mm"] += 100 im_hdu = image_hdu_rect im_hdu.header["CRVAL1D"] -= 150 im_hdu.header["CRVAL2D"] += 20 - hdr = imp_utils.get_canvas_header([im_hdu, image_hdu_square, - tbl1, tbl2], pixel_scale=3*u.mm) + hdr = get_canvas_header([im_hdu, image_hdu_square], 3*u.mm) im = np.zeros((hdr["NAXIS2"], hdr["NAXIS1"])) canvas_hdu = fits.ImageHDU(header=hdr, data=im) - canvas_hdu = imp_utils.add_table_to_imagehdu(tbl1, canvas_hdu, - wcs_suffix="D") - canvas_hdu = imp_utils.add_table_to_imagehdu(tbl2, canvas_hdu, - wcs_suffix="D") - canvas_hdu = imp_utils.add_imagehdu_to_imagehdu(im_hdu, canvas_hdu, - wcs_suffix="D") - canvas_hdu = imp_utils.add_imagehdu_to_imagehdu(image_hdu_square, - canvas_hdu, - wcs_suffix="D") - - total_flux = np.sum(tbl1["flux"]) + np.sum(tbl2["flux"]) + \ - np.sum(im_hdu.data) + np.sum(image_hdu_square.data) + canvas_hdu = imp_utils.add_imagehdu_to_imagehdu( + im_hdu, canvas_hdu, wcs_suffix="D", + ) + canvas_hdu = imp_utils.add_imagehdu_to_imagehdu( + image_hdu_square, canvas_hdu, wcs_suffix="D", + ) + + total_flux = np.sum(im_hdu.data) + np.sum(image_hdu_square.data) assert np.sum(canvas_hdu.data) == approx(total_flux, rel=5e-3) if PLOTS: @@ -564,11 +284,10 @@ def test_mm_image_and_tables_on_large_canvas(self, input_table_mm, class TestImagePlaneAdd: - def test_simple_add_imagehdu_conserves_flux(self, image_hdu_square, - image_hdu_rect): - fields = [image_hdu_rect, image_hdu_square] - hdr = imp_utils.get_canvas_header(pixel_scale=1 * u.arcsec, - hdu_or_table_list=fields) + def test_simple_add_imagehdu_conserves_flux( + self, image_hdu_square, image_hdu_rect, + ): + hdr = get_canvas_header([image_hdu_rect, image_hdu_square]) orig_sum = image_hdu_rect.data.sum() @@ -598,11 +317,10 @@ def test_add_many_tables_and_imagehdus(self, image_hdu_rect, image_hdu_square): im_hdu.header["CRVAL1"] -= 150*u.arcsec.to(u.deg) im_hdu.header["CRVAL2"] += 20*u.arcsec.to(u.deg) - fields = [im_hdu, image_hdu_square] - hdr = imp_utils.get_canvas_header(fields, pixel_scale=1 * u.arcsec) + hdr = get_canvas_header([im_hdu, image_hdu_square]) implane = opt_imp.ImagePlane(hdr) - implane.add(fields) + implane.add([im_hdu, image_hdu_square]) total_flux = np.sum(im_hdu.data) + np.sum(image_hdu_square.data) assert np.sum(implane.data) == approx(total_flux) @@ -618,11 +336,12 @@ def test_add_many_tables_and_imagehdus(self, image_hdu_rect, image_hdu_square): plt.plot(x0, y0, "ro") plt.gca().set_aspect(1) - plt.imshow(implane.data, origin="lower", norm=LogNorm()) + plt.imshow(implane.data, origin="lower", norm="log") plt.show() - def test_add_many_mm_tables_and_imagehdus( - self, image_hdu_rect_mm, image_hdu_square_mm): + def test_add_many_mm_imagehdus( + self, image_hdu_rect_mm, image_hdu_square_mm, + ): image_hdu_rect = image_hdu_rect_mm image_hdu_square = image_hdu_square_mm @@ -630,10 +349,9 @@ def test_add_many_mm_tables_and_imagehdus( im_hdu.header["CRVAL1D"] -= 150 # mm im_hdu.header["CRVAL2D"] += 20 - fields = [im_hdu, image_hdu_square] - hdr = imp_utils.get_canvas_header(fields, pixel_scale=1*u.mm) + hdr = get_canvas_header([im_hdu, image_hdu_square], 1*u.mm) implane = opt_imp.ImagePlane(hdr) - implane.add(fields, wcs_suffix="D") + implane.add([im_hdu, image_hdu_square], wcs_suffix="D") total_flux = np.sum(im_hdu.data) + np.sum(image_hdu_square.data) assert np.sum(implane.data) == approx(total_flux) @@ -649,7 +367,7 @@ def test_add_many_mm_tables_and_imagehdus( plt.plot(x0, y0, "ro") plt.gca().set_aspect(1) - plt.imshow(implane.data, origin="lower", norm=LogNorm()) + plt.imshow(implane.data, origin="lower", norm="log") plt.show() @@ -725,7 +443,7 @@ def test_rescale_works_on_3d_imageplane(self, image_hdu_3d_data): ############################################################################### -# ..todo: When you have time, reintegrate these tests, There are some good ones +# TODO: When you have time, reintegrate these tests, There are some good ones class TestGetSpatialExtentOfHeader: def test_returns_right_sky_coords_from_known_coords(self, image_hdu_square): @@ -742,141 +460,24 @@ def test_returns_right_sky_coords_from_known_coords(self, image_hdu_square): class TestMakeImagePlaneHeader: - def test_header_contains_future_naxis_pixel_sizes(self, image_hdu_square, - image_hdu_rect): - hdr = imp_utils.get_canvas_header([image_hdu_square, image_hdu_rect]) + def test_header_contains_future_naxis_pixel_sizes( + self, image_hdu_square, image_hdu_rect, + ): + hdr = get_canvas_header([image_hdu_square, image_hdu_rect]) assert hdr["NAXIS1"] == 100 + 1 assert hdr["NAXIS2"] == 200 + 1 @pytest.mark.parametrize("offset", -np.random.randint(200, 1001, 10)) - def test_header_contains_spread_out_regions(self, offset, image_hdu_square, - image_hdu_rect): + def test_header_contains_spread_out_regions( + self, offset, image_hdu_square, image_hdu_rect, + ): image_hdu_rect.header["CRVAL1"] += offset # *u.arcsec.to(u.deg) - hdr = imp_utils.get_canvas_header([image_hdu_square, image_hdu_rect]) + hdr = get_canvas_header([image_hdu_square, image_hdu_rect]) image_width = image_hdu_square.header["NAXIS1"] // 2 + \ image_hdu_rect.header["NAXIS1"] // 2 + abs(offset) + 1 assert hdr["NAXIS1"] == image_width - @pytest.mark.parametrize("offset", [0, 10, 25, 50]) - def test_header_has_correct_size_based_on_table_extremes(self, offset, - input_table): - tbl1 = input_table - tbl2 = deepcopy(input_table) - tbl3 = deepcopy(input_table) - tbl2["x"] += offset - tbl3["y"] += offset - hdr = imp_utils.get_canvas_header([tbl1, tbl2, tbl3], - pixel_scale=0.1*u.arcsec) - - assert hdr["NAXIS1"] == np.max(tbl1["x"] + tbl2["x"]) * 10 + 3 # 4 - assert hdr["NAXIS2"] == np.max(tbl1["y"] + tbl3["y"]) * 10 + 3 # 4 - - # @pytest.mark.parametrize("pix_scl", [5, 1, 0.5, 0.1]) - # def test_header_has_correct_size_with_tbl_and_image_input(self, input_table, - # image_hdu_square, - # pix_scl): - # input_table["x"] += 100 - # hdr = imp_utils.get_canvas_header([image_hdu_square, input_table], - # pixel_scale=pix_scl * u.arcsec) - # assert hdr["NAXIS1"] == approx(hdr["CRPIX1"] + - # np.max(input_table["x"]) / pix_scl + 1, - # abs=0.5) - - def test_header_does_not_contain_negative_naxis_keywords(self): - pass - -# -# class TestGetCornerSkyCoordsFromTable: -# def test_table_with_column_units_returns_right_values(self): -# x, y = [1] * u.arcmin, [1] * u.arcmin -# tbl = Table(names=["x", "y"], data=[x, y]) -# xsky, ysky = imp_utils.get_corner_sky_coords_from_table(tbl) -# -# assert xsky[0]*u.deg == x[0].to(u.deg) -# assert ysky[0]*u.deg == y[0].to(u.deg) -# -# def test_table_with_meta_units_returns_right_values(self): -# x, y = [1], [1] -# tbl = Table(names=["x", "y"], data=[x, y]) -# tbl.meta.update({"x_unit": u.arcmin, "y_unit": u.arcmin}) -# xsky, ysky = impl_utils.get_corner_sky_coords_from_table(tbl) -# -# assert xsky[0] == x[0]*u.arcmin.to(u.deg) -# assert ysky[0] == y[0]*u.arcmin.to(u.deg) -# -# def test_table_with_default_units_returns_right_values(self): -# x, y = [60], [60] # because default unit is arcsec -# tbl = Table(names=["x", "y"], data=[x, y]) -# xsky, ysky = impl_utils.get_corner_sky_coords_from_table(tbl) -# -# assert pytest.approx(xsky[0] == x[0] * u.arcmin.to(u.deg)) -# assert pytest.approx(ysky[0] == y[0] * u.arcmin.to(u.deg)) -# -# -# class TestGetCornerSkyCoords: -# def test_returns_coords_for_combination_of_table_and_header(self, -# image_hdu_square): -# x, y = [-100, 70] * u.arcsec, [0, 0] * u.arcsec -# tbl = Table(names=["x", "y"], data=[x, y]) -# xsky, ysky = impl_utils.get_corner_sky_coords([tbl, image_hdu_square]) -# -# assert np.all((xsky == x.to_value(u.deg))) -# - - -class TestAddTableToImageHDU2: - @pytest.mark.parametrize("xpix, ypix, value", - [(51, 51, 2), - (48, 51, 3), - (48, 48, 4), - (51, 48, 5)]) - def test_integer_pixel_fluxes_are_added_correctly(self, xpix, ypix, value, - image_hdu_square): - # Given the weird behaviour on pixel boundaries - x, y = [1.5, -1.5, -1.5, 1.5]*u.arcsec, [1.5, 1.5, -1.5, -1.5]*u.arcsec - flux = [1, 2, 3, 4] * u.Unit("ph s-1") - tbl = Table(names=["x", "y", "flux"], data=[x, y, flux]) - - hdu = imp_utils.add_table_to_imagehdu(tbl, image_hdu_square, - sub_pixel=False) - - assert hdu.data[ypix, xpix] == value - - @pytest.mark.parametrize("x, y, flux, xpix, ypix, value", - [([0], [0], [1], 50, 50, .25), - ([0.7], [0.7], [1], 50, 50, 0.64), - ([-0.7], [-0.7], [1], 48, 48, 0.04), - ([5.5], [-4.7], [1], 55, 45, 0.8), - ([5.5], [-4.7], [1], 55, 44, 0.2)]) - def test_sub_pixel_fluxes_are_added_correctly(self, x, y, flux, xpix, ypix, - value, image_hdu_square): - # Given the weird behaviour on pixel boundaries - tbl = Table(names=["x", "y", "flux"], - data=[x*u.arcsec, y*u.arcsec, flux*u.Unit("ph s-1")]) - hdu = imp_utils.add_table_to_imagehdu(tbl, image_hdu_square, - sub_pixel=True) - - if PLOTS: - plt.imshow(hdu.data[45:55, 45:55], origin="lower") - plt.colorbar() - plt.show() - - assert hdu.data[ypix, xpix] == approx(value + 1) - - @pytest.mark.parametrize("x, y, flux", - [([100, -100], [0, 0], [10, 10])]) - def test_source_outside_canvas_are_ignored(self, x, y, flux, - image_hdu_square): - # Given the weird behaviour on pixel boundaries - tbl = Table(names=["x", "y", "flux"], - data=[x * u.arcsec, y * u.arcsec, - flux * u.Unit("ph s-1")]) - hdu = imp_utils.add_table_to_imagehdu(tbl, image_hdu_square, - sub_pixel=True) - - assert np.sum(hdu.data) == np.prod(hdu.data.shape) - class TestAddImagehduToImageHDU: @pytest.mark.parametrize("angle", [0, 30, 45, 89]) @@ -915,11 +516,10 @@ def test_throws_error_when_initialised_with_nothing(self): with pytest.raises(TypeError): opt_imp.ImagePlane() - def test_initialises_with_header_with_hdu(self, image_hdu_square, - image_hdu_rect): - hdr = imp_utils.get_canvas_header(pixel_scale=0.1 * u.arcsec, - hdu_or_table_list=[image_hdu_rect, - image_hdu_square]) + def test_initialises_with_header_with_hdu( + self, image_hdu_square, image_hdu_rect, + ): + hdr = get_canvas_header([image_hdu_rect, image_hdu_square]) implane = opt_imp.ImagePlane(hdr) assert isinstance(implane, opt_imp.ImagePlane) assert isinstance(implane.hdu, fits.ImageHDU) diff --git a/scopesim/tests/tests_optics/test_ImagePlaneUtils.py b/scopesim/tests/tests_optics/test_ImagePlaneUtils.py index 1832ec8e1..c0116087a 100644 --- a/scopesim/tests/tests_optics/test_ImagePlaneUtils.py +++ b/scopesim/tests/tests_optics/test_ImagePlaneUtils.py @@ -14,27 +14,6 @@ PLOTS = False -class TestSplitHeader: - def test_is_the_header_split_into_the_right_amount_of_chunks(self): - hdr = imp_utils.header_from_list_of_xy([-1.024, 1.024], - [-1.024, 1.024], 0.002) - hdrs = imp_utils.split_header(hdr, 128) - area_sum = np.sum([hdr["NAXIS1"] * hdr["NAXIS2"] for hdr in hdrs]) - assert len(hdrs) == 64 - assert area_sum == hdr["NAXIS1"] * hdr["NAXIS2"] - - @pytest.mark.parametrize("x, y, pix", [(0.19, 0.2, 0.01), - (2.19, 1.55, 0.01), - (1.29, 1.2, 0.02), - (2.55, 3.75, 1)]) - def test_final_header_is_smaller_for_odd_size(self, x, y, pix): - hdr = imp_utils.header_from_list_of_xy([-1., x], - [-1., y], pix) - hdrs = imp_utils.split_header(hdr, 100) - area_sum = np.sum([hdr["NAXIS1"] * hdr["NAXIS2"] for hdr in hdrs]) - assert area_sum == hdr["NAXIS1"] * hdr["NAXIS2"] - - class TestAddImageHDUtoImageHDU: def big_small_hdus(self, big_wh=(20, 10), big_offsets=(0, 0), small_wh=(6, 3), small_offsets=(0, 0), pixel_scale=0.1): diff --git a/scopesim/tests/tests_source/source_helpers.py b/scopesim/tests/tests_source/source_helpers.py new file mode 100644 index 000000000..1a92fb99a --- /dev/null +++ b/scopesim/tests/tests_source/source_helpers.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- +"""Aux function for source testing.""" + +import numpy as np +from scipy import integrate +from astropy import units as u +from synphot import SourceSpectrum, SpectralElement + + +# This used to be in source_utils, but was ultimately only used in a few tests, +# so moved here as a simple helper function. +def photons_in_range( + spectra: SourceSpectrum, + wave_min: u.Quantity[u.um] | float, + wave_max: u.Quantity[u.um] | float, + area: u.Quantity[u.m**2] | float | None = None, + bandpass: SpectralElement | None = None, +) -> u.Quantity[u.ph * u.s**-1 * u.m**-2] | u.Quantity[u.ph * u.s**-1]: + """ + Integrate photons from spectrum in given wavelength range. + + Parameters + ---------- + spectra : SourceSpectrum + Input spectrum. + wave_min : u.Quantity["length"] or float + Minimum wavelength. If float, assumes um. + wave_max : u.Quantity["length"] or float + Maximum wavelength. If float, assumes um. + area : u.Quantity["area"] or float, optional + Area to multiply with. If float, assumes m**2. The default is None. + bandpass : SpectralElement, optional + Filter to take into account, if any. The default is None. + + Returns + ------- + counts : astropy.units.Quantity + Either in ph/s/m**2 or just ph/s (if area was given). + + """ + # Note: Assuming um if given as float. + wave_min = (wave_min << u.um << u.Angstrom).value + wave_max = (wave_max << u.um << u.Angstrom).value + # Note: There appear to be some float shenanigans going on here, but + # rounding produces an error in the spectrum evaluation. Not sure what's + # going on, maybe it's fine as-is. + + counts = [] + for spec in spectra: + waveset = spec.waveset.value + mask = (waveset > wave_min) * (waveset < wave_max) + wave = np.array([wave_min, *waveset[mask], wave_max]) + flux = spec(wave).value + + # flux [ph s-1 cm-2] == flux [ph s-1 cm-2 AA-1] * wave [AA] + if isinstance(bandpass, SpectralElement): + bandpass.model.bounds_error = True + counts.append(integrate.trapezoid(bandpass(wave).value * flux, wave)) + else: + counts.append(integrate.trapezoid(flux, wave)) + + # counts = flux [ph s-1 cm-2] + counts = (counts * u.ph * u.s**-1 * u.cm**-2).to(u.ph * u.s**-1 * u.m**-2) + if area is not None: + counts *= (area << u.m**2) + + return counts + + +# This used to be a method of Source, but was ultimately only used in two tests, +# so moved here as a simple helper function. +def src_photons_in_range(src, wave_min, wave_max, area=None): + """ + + Parameters + ---------- + wave_min : float, u.Quantity + [um] + wave_max : float, u.Quantity + [um] + area : float, u.Quantity, optional + [m2] + + Returns + ------- + counts : u.Quantity list + [ph / s / m2] if area is None + [ph / s] if area is passed + + """ + indices = src.spectra.keys() + spectra = [src.spectra[ii] for ii in indices] + counts = photons_in_range(spectra, wave_min, wave_max, area=area, + bandpass=src.bandpass) + return counts diff --git a/scopesim/tests/tests_source/test_source_Source.py b/scopesim/tests/tests_source/test_source_Source.py index feea28e44..2f5b06fbe 100644 --- a/scopesim/tests/tests_source/test_source_Source.py +++ b/scopesim/tests/tests_source/test_source_Source.py @@ -1,8 +1,10 @@ -# actually for Source +# -*- coding: utf-8 -*- + import pytest from pytest import approx import numpy as np +import matplotlib.pyplot as plt from astropy.io import fits from astropy.io import ascii as ioascii @@ -10,7 +12,7 @@ from astropy import units as u from astropy import wcs -from synphot import SourceSpectrum, SpectralElement +from synphot import SourceSpectrum from synphot.models import Empirical1D from synphot.units import PHOTLAM @@ -18,14 +20,10 @@ from scopesim.source.source import Source from scopesim.source.source_fields import CubeSourceField, TableSourceField -from scopesim.optics.image_plane import ImagePlane from scopesim.utils import convert_table_comments_to_dict from scopesim.tests.mocks.py_objects import source_objects as so -import matplotlib.pyplot as plt -from matplotlib.colors import LogNorm - PLOTS = False @@ -261,83 +259,6 @@ def test_empty_source_is_the_additive_identity(self, image_source): assert len(new_source_2.fields) == len(new_source_2._meta_dicts) -class TestSourceImageInRange: - @pytest.mark.skip(reason=( - "This was the only place where the 'add table to ImagePlane' branch " - "was still used. Figure out what actually gets tested here and then " - "try to do it in another way." - )) - def test_returns_an_image_plane_object(self, table_source): - im = table_source.image_in_range(1*u.um, 2*u.um) - assert isinstance(im, ImagePlane) - - @pytest.mark.skip(reason=( - "This was the only place where the 'add table to ImagePlane' branch " - "was still used. Figure out what actually gets tested here and then " - "try to do it in another way." - )) - def test_flux_from_table_on_image_is_as_expected(self, table_source): - ph = table_source.photons_in_range(1*u.um, 2*u.um) - ref = table_source.fields[0].field["ref"] - weight = table_source.fields[0].field["weight"] - counts = np.sum([ph.value[r] * w for r, w in zip(ref, weight)]) - - im = table_source.image_in_range(1*u.um, 2*u.um) - assert np.sum(im.image) == approx(counts) - - @pytest.mark.parametrize("pix_scl", [0.1, 0.2, 0.4, 1.0]) - def test_flux_from_imagehdu_is_as_expected(self, image_source, pix_scl): - ph = image_source.photons_in_range(1*u.um, 2*u.um)[0].value - im_sum = ph * np.sum(image_source.fields[0].data) - im = image_source.image_in_range(1*u.um, 2*u.um, pix_scl*u.arcsec) - assert np.sum(im.image) == approx(im_sum) - - def test_combines_more_that_one_field_into_image(self, image_source, - table_source): - ph = table_source.photons_in_range(1 * u.um, 2 * u.um) - fld = table_source.fields[0] - tbl_sum = u.Quantity([ph[fld.field["ref"][ii]] * fld.field["weight"][ii] - for ii in range(len(fld.field))]) - tbl_sum = np.sum(tbl_sum.value) - - ph = image_source.photons_in_range(1 * u.um, 2 * u.um)[0] - im_sum = np.sum(image_source.fields[0].data) * ph.value - - comb_src = image_source - im_plane = comb_src.image_in_range(1*u.um, 2*u.um, 0.3*u.arcsec) - - assert np.sum(im_plane.image) == approx(im_sum) - - if PLOTS: - plt.imshow(im_plane.image.T, origin="lower", norm=LogNorm()) - plt.show() - - -class TestSourcePhotonsInRange: - def test_correct_photons_are_returned_for_table_source(self, table_source): - ph = table_source.photons_in_range(1, 2) - assert np.allclose(ph.value, [4., 2., 2.]) - - def test_correct_photons_are_returned_for_image_source(self, image_source): - ph = image_source.photons_in_range(1, 2) - assert np.allclose(ph.value, [2.]) - - def test_correct_photons_are_returned_for_no_spectra(self, image_source): - image_source.fields[0].spectra = {} - ph = image_source.photons_in_range(1, 2) - assert len(ph) == 0 - - @pytest.mark.parametrize("area, expected", [(None, 2), (1, 2), (10, 20)]) - def test_photons_increase_with_area(self, area, expected, image_source): - ph = image_source.photons_in_range(1, 2, area=area) - assert ph[0].value == approx(expected) - - def test_photons_returned_only_for_indices(self, table_source): - ph = table_source.photons_in_range(1, 2, indices=[0, 2]) - assert len(ph) == 2 - assert np.allclose(ph.value, [4, 2]) - - @pytest.mark.xfail class TestSourceShift: def test_that_it_does_what_it_should(self): @@ -350,65 +271,6 @@ def test_that_it_does_what_it_should(self): assert False -class TestPhotonsInRange: - @pytest.mark.parametrize("ii, n_ph", - [(0, 50), - (1, 8e12)]) - def test_returns_correct_number_of_photons_for_one_spectrum(self, ii, n_ph, - input_spectra): - spec = input_spectra[ii] - counts = source_utils.photons_in_range([spec], 1, 2) - assert np.isclose(counts.value, n_ph, rtol=2e-3) - - def test_returns_ones_for_unity_spectrum(self): - flux = np.ones(11) * u.Unit("ph s-1 m-2 um-1") - wave = np.linspace(1, 2, 11) * u.um - spec = SourceSpectrum(Empirical1D, points=wave, lookup_table=flux) - counts = source_utils.photons_in_range([spec], 1 * u.um, 2 * u.um) - assert counts.value == approx(1) - - @pytest.mark.parametrize("area, expected_units", - [(1*u.m**2, u.ph / u.s), - (None, u.ph / u.s / u.m**2)]) - def test_returns_correct_units_with_without_area_argument(self, area, - expected_units): - flux = np.ones(11) * u.Unit("ph s-1 m-2 um-1") - wave = np.linspace(1, 2, 11) * u.um - spec = SourceSpectrum(Empirical1D, points=wave, lookup_table=flux) - counts = source_utils.photons_in_range([spec], 1 * u.um, 2 * u.um, - area=area) - assert counts.unit == expected_units - - def test_returns_correct_half_flux_with_bandpass(self): - flux = np.ones(11) * u.Unit("ph s-1 m-2 um-1") - wave = np.linspace(0.5, 2.5, 11) * u.um - spec = SourceSpectrum(Empirical1D, points=wave, lookup_table=flux) - bandpass = SpectralElement(Empirical1D, - points=np.linspace(1, 2, 13)*u.um, - lookup_table=0.5 * np.ones(13)) - counts = source_utils.photons_in_range([spec], 1 * u.um, 2 * u.um, - bandpass=bandpass) - assert counts.value == approx(0.5) - - @pytest.mark.parametrize("flux, area, expected", - [(np.linspace(0, 1, 11), 1E4*u.cm**2, 0.25), - (np.linspace(0, 1, 11)**2, None, 0.13625), - (np.linspace(0, 1, 11)**0.5, 100, 34.931988)]) - def test_with_bandpass_and_area_returns_correct_value(self, flux, area, - expected): - flux *= u.Unit("ph s-1 m-2 um-1") - spec = SourceSpectrum(Empirical1D, - points=np.linspace(0.5, 2.5, 11) * u.um, - lookup_table=flux) - bandpass = SpectralElement(Empirical1D, - points=np.linspace(1, 2, 13)*u.um, - lookup_table=0.5 * np.ones(13)) - counts = source_utils.photons_in_range([spec], 1 * u.um, 2 * u.um, - bandpass=bandpass, - area=area) - assert counts.value == approx(expected) - - class TestSpectraListConverter: def test_works_for_arrays(self): spec = source_utils.convert_to_list_of_spectra( @@ -465,33 +327,3 @@ def test_throws_for_invalid_ref(): # Minimal Table tbl = Table(data=[[0, 1]], names=["ref"]) TableSourceField(tbl, {0: None}) - - -# -# class TestScaleImageHDU: -# def test_scaling_properly_for_si_photlam_in_header(self): -# hdu = fits.ImageHDU(data=np.ones((10,10))) -# hdu.header["CDELT1"] = 0.1 * u.arcsec.to(u.deg) -# hdu.header["CDELT2"] = 0.1 * u.arcsec.to(u.deg) -# hdu.header["BUNIT"] = "ph s-1 m-2 um-1" -# waverange = (1, 2)*u.um -# scaled_hdu = source2_utils.scale_imagehdu(hdu, waverange) -# assert scaled_hdu -# -# def test_scaling_properly_for_cgs_photlam_per_arcsec2_in_header(self): -# pass -# -# def test_scaling_properly_for_photlam_per_arcsec2_no_area_in_header(self): -# pass -# -# def test_scaling_properly_for_Jansky_in_header(self): -# pass -# -# def test_scaling_properly_for_Jansky_per_arcsec_in_header(self): -# pass -# -# def test_scaling_properly_for_photlam_and_bscale_in_header(self): -# pass -# -# def test_scaling_properly_for_photlam_and_bscale_bzero_in_header(self): -# pass From 019abf13822219f9c620c1595ac11de82b9ca48c Mon Sep 17 00:00:00 2001 From: teutoburg Date: Sun, 13 Sep 2026 22:16:51 +0200 Subject: [PATCH 51/87] Some more cleanup and removals --- docs/source/_ext/scopesim_sphinx_ext.py | 13 ++++++++--- scopesim/effects/effects_utils.py | 13 ++--------- scopesim/effects/ter_curves_utils.py | 1 + .../tests/tests_effects/test_effect_utils.py | 23 ++++--------------- 4 files changed, 17 insertions(+), 33 deletions(-) diff --git a/docs/source/_ext/scopesim_sphinx_ext.py b/docs/source/_ext/scopesim_sphinx_ext.py index 4ab146d1e..ea7957f8f 100644 --- a/docs/source/_ext/scopesim_sphinx_ext.py +++ b/docs/source/_ext/scopesim_sphinx_ext.py @@ -1,6 +1,13 @@ +# -*- coding: utf-8 -*- + +# TODO: What does this (rather clunky) Sphinx extension do that isn't covered +# by autodoc/autosummary?? If it's redundant, just get rid of it tbh. + import os import numpy as np -from scopesim.effects import effects_utils as eu + +# This function is only used here. If this extension is removed, rm the func. +from scopesim.effects.effects_utils import scopesim_effect_classes def setup(app): @@ -8,7 +15,7 @@ def setup(app): "../effects_docstrings")) os.makedirs(output_dir, exist_ok=True) - efs_dict = eu.scopesim_effect_classes() + efs_dict = scopesim_effect_classes() eff_types = np.unique([eff.split(".")[0] for eff in efs_dict]) eff_type_strs = {eff_type: f"{eff_type}\n{'='*len(eff_type)}\n\n" for eff_type in eff_types} @@ -37,7 +44,7 @@ def setup(app): :glob: * - + """ with open(os.path.join(output_dir, "index.rst"), "w") as f: diff --git a/scopesim/effects/effects_utils.py b/scopesim/effects/effects_utils.py index f9522ded1..f96dceeb9 100644 --- a/scopesim/effects/effects_utils.py +++ b/scopesim/effects/effects_utils.py @@ -1,10 +1,9 @@ +# -*- coding: utf-8 -*- """TBA.""" import inspect from collections.abc import Iterable -from astropy.table import Table - from .. import effects as efs from ..utils import get_logger @@ -51,15 +50,7 @@ def is_spectroscope(effects): return any(isinstance(eff, spec_classes) for eff in effects) -def empty_surface_list(**kwargs): - tbl = Table(names=["name", "outer", "inner", "angle", - "temperature", "action", "filename"], - data=[["test"], [0.], [0.], [0.], [0.], ["none"], ["none"]], - meta={"outer_unit": "m", "inner_unit": "m", - "angle_unit": "deg", "temperature_unit": "deg_C"}) - return efs.SurfaceList(table=tbl[:0], **kwargs) - - +# This is only used for the questionable sphinx extension. def scopesim_effect_classes(base_effect=efs.Effect): members = inspect.getmembers(efs) efs_dict = {".".join([cls.__module__, cls.__name__]).replace("scopesim.effects.", ""): cls diff --git a/scopesim/effects/ter_curves_utils.py b/scopesim/effects/ter_curves_utils.py index b7454e2c2..dd2f2f1c5 100644 --- a/scopesim/effects/ter_curves_utils.py +++ b/scopesim/effects/ter_curves_utils.py @@ -49,6 +49,7 @@ PATH_SVO_DATA = PATH_HERE.parent / "data" / "svo" +# This function is only used in the AnisoCADO PSF... def get_filter_effective_wavelength(filter_name): # TODO: This is technically stored in the SVO XML file as WavelengthEff... # (actually WavelengthMean, by definition of formula ...) diff --git a/scopesim/tests/tests_effects/test_effect_utils.py b/scopesim/tests/tests_effects/test_effect_utils.py index 6a2779116..8e2611cea 100644 --- a/scopesim/tests/tests_effects/test_effect_utils.py +++ b/scopesim/tests/tests_effects/test_effect_utils.py @@ -1,10 +1,10 @@ +# -*- coding: utf-8 -*- + import pytest from scopesim import effects as efs -from scopesim.effects import effects_utils as eu, GaussianDiffractionPSF, \ - SurfaceList -from scopesim.tests.mocks.py_objects.effects_objects import _surf_list, \ - _surf_list_empty, _filter_surface +from scopesim.effects import effects_utils as eu, GaussianDiffractionPSF + from scopesim.tests.mocks.py_objects.yaml_objects import _atmo_yaml_dict @@ -13,21 +13,6 @@ def atmo_yaml_dict(): return _atmo_yaml_dict() -@pytest.fixture(scope="function") -def surf_list(): - return _surf_list() - - -@pytest.fixture(scope="function") -def surf_list_empty(): - return _surf_list_empty() - - -@pytest.fixture(scope="function") -def filter_surface(): - return _filter_surface() - - class TestMakeEffect: def test_it_creates_an_effects_object(self, atmo_yaml_dict): effdic = atmo_yaml_dict["effects"][0] From 5d69ce98e52430460d3efe25e7639ee1e091049a Mon Sep 17 00:00:00 2001 From: oczoske Date: Mon, 14 Sep 2026 17:08:00 +0200 Subject: [PATCH 52/87] Ensure positive CDELT and pixel area --- scopesim/optics/fov.py | 1 + scopesim/utils.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/scopesim/optics/fov.py b/scopesim/optics/fov.py index 960baa107..4f37d75c1 100644 --- a/scopesim/optics/fov.py +++ b/scopesim/optics/fov.py @@ -355,6 +355,7 @@ def extract_area_from_imagehdu(self, imagehdu, corners): # up a duplicated or a dropped row/column. new_wcs = image_wcs.deepcopy() new_wcs.wcs.crpix = image_wcs.wcs.crpix - xy0p + new_wcs.wcs.cdelt = np.abs(new_wcs.wcs.cdelt) new_naxis = xy1p - xy0p logger.debug("orig image wcs: %s", image_wcs) logger.debug("new cutout wcs: %s", new_wcs) diff --git a/scopesim/utils.py b/scopesim/utils.py index a22c19ae7..62d506a52 100644 --- a/scopesim/utils.py +++ b/scopesim/utils.py @@ -556,6 +556,8 @@ def unit_includes_per_physical_type(unit, physical_type): def pixel_area(header: fits.Header) -> u.Quantity[u.arcsec**2]: """Calculate area covered by one pixel in arcsec**2 from header. + The pixel area is always positive. + .. versionadded:: 0.11.1 """ @@ -571,7 +573,7 @@ def pixel_area(header: fits.Header) -> u.Quantity[u.arcsec**2]: area = (header["CDELT1"] * u.Unit(header["CUNIT1"]) * header["CDELT2"] * u.Unit(header["CUNIT2"])) - return area.to(u.arcsec**2) + return np.abs(area.to(u.arcsec**2)) def has_needed_keywords(header, suffix=""): From cf8915d2d2f20b766e58bb0355bf5c3c268aced0 Mon Sep 17 00:00:00 2001 From: teutoburg Date: Mon, 7 Sep 2026 01:30:56 +0200 Subject: [PATCH 53/87] Improve randomness, refactor electronic effects - Add `ElectronicEffect` base class - Use composite random seed - Use `Detector.det_id` property everywhere --- scopesim/effects/electronic/__init__.py | 30 +- scopesim/effects/electronic/electrons.py | 62 +- scopesim/effects/electronic/noise.py | 550 +++++++++--------- scopesim/effects/electronic/pixels.py | 44 +- scopesim/optics/optical_train.py | 2 +- .../tests_effects/test_BasicReadoutNoise.py | 6 +- 6 files changed, 345 insertions(+), 349 deletions(-) diff --git a/scopesim/effects/electronic/__init__.py b/scopesim/effects/electronic/__init__.py index 5d10e0d42..abc4912e7 100644 --- a/scopesim/effects/electronic/__init__.py +++ b/scopesim/effects/electronic/__init__.py @@ -19,9 +19,37 @@ - InterPixelCapacitance - apply IPC kernel to detector readout """ -from ...utils import get_logger +from ...utils import get_logger, check_keys +from ...detector import Detector +from .. import Effect + logger = get_logger(__name__) + +class ElectronicEffect(Effect): + """Base class for electronic effects. + + This will eventually replace the 800-range zorder (or parts of it). + """ + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.meta.update(kwargs) + check_keys(self.meta, self.required_keys, action="error") + + def _apply_to_det(self, det: Detector) -> None: + """Subclasses can override if more params needed in call.""" + logger.debug("Apply %s to %s", self.display_name, det) + det.data = self(det.data) + + def apply_to(self, obj, **kwargs): + """See parent docstring.""" + if isinstance(obj, Detector): + self._apply_to_det(obj) + + return obj + + from .electrons import LinearityCurve, ADConversion, InterPixelCapacitance from .noise import (Bias, PoorMansHxRGReadoutNoise, BasicReadoutNoise, ShotNoise, DarkCurrent, PixelResponseNonUniformity) diff --git a/scopesim/effects/electronic/electrons.py b/scopesim/effects/electronic/electrons.py index b279df63e..c2bee25b2 100644 --- a/scopesim/effects/electronic/electrons.py +++ b/scopesim/effects/electronic/electrons.py @@ -15,14 +15,11 @@ from numpy.typing import ArrayLike, NDArray from scipy.signal import oaconvolve -from .. import Effect -from ...detector import Detector -from ...utils import figure_factory, check_keys -from ...utils import from_currsys -from . import logger +from ...utils import from_currsys, figure_factory +from . import ElectronicEffect, Detector, logger -class LinearityCurve(Effect): +class LinearityCurve(ElectronicEffect): """ Detector linearity effect. @@ -57,12 +54,6 @@ class LinearityCurve(Effect): report_plot_include: ClassVar[bool] = True report_table_include: ClassVar[bool] = False - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.meta.update(kwargs) - - check_keys(self.meta, self.required_keys, action="error") - def __call__(self, data: ArrayLike, ndit: int = 1) -> NDArray: incident = self.incident * ndit measured = self.measured * ndit @@ -80,13 +71,12 @@ def measured(self) -> NDArray: return self.table["measured"] return np.asarray(from_currsys(self.meta["measured"], self.cmds)) - def apply_to(self, obj, **kwargs): - if not isinstance(obj, Detector): - return obj + def _apply_to_det(self, det: Detector) -> None: + """Subclasses can override if more params needed in call.""" + logger.debug("Apply %s to %s", self.display_name, det) ndit = from_currsys(self.meta["ndit"], self.cmds) - obj.data = self(obj.data, ndit) - return obj + det.data = self(det.data, ndit) def plot(self, **kwargs): fig, ax = figure_factory() @@ -102,7 +92,7 @@ def plot(self, **kwargs): return fig -class InterPixelCapacitance(Effect): +class InterPixelCapacitance(ElectronicEffect): r"""Inter-pixel capacitance effect. The effect models cross-talk due to inter-pixel capacitance with @@ -156,8 +146,6 @@ class InterPixelCapacitance(Effect): def __init__(self, **kwargs): super().__init__(**kwargs) - - self.meta.update(kwargs) self.kernel = self._build_kernel(kwargs) def _build_kernel(self, params): @@ -183,14 +171,10 @@ def _build_kernel(self, params): ]) return kernel - def apply_to(self, det, **kwargs): - if not isinstance(det, Detector): - logger.debug("%s applied to %s", self.display_name, - det.__class__.__name__) - return det - + def _apply_to_det(self, det: Detector) -> None: + """Subclasses can override if more params needed in call.""" + logger.debug("Apply %s to %s", self.display_name, det) det.data = oaconvolve(det.data, self.kernel, mode="same") - return det def update(self, **kwargs): """Update the IPC kernel. @@ -228,7 +212,7 @@ def __str__(self): return msg -class ADConversion(Effect): +class ADConversion(ElectronicEffect): """Analogue-Digital Conversion effect. The effect applies the gain factor (electrons/ADU) to the detector readouts @@ -293,7 +277,7 @@ def _should_apply(self) -> bool: logger.warning("Cannot access cmds for ADConversion effect.") return True - # ..todo: need to deal with this case more realistically + # TODO: need to deal with this case more realistically # Is this still necessary? #if self.cmds.get("!OBS.autoexpset", False): # logger.info("DIT, NDIT determined by AutoExposure -> " @@ -325,14 +309,14 @@ def _get_gain(self, det_id: int) -> Real: f"{self.__class__.__name__}.meta['gain'] must be either " f"dict or float, but is {self.cmds['!DET.gain']}") - def apply_to(self, obj, **kwargs): - if not isinstance(obj, Detector): - return obj + def _apply_to_det(self, det: Detector) -> None: + """Subclasses can override if more params needed in call.""" + logger.debug("Apply %s to %s", self.display_name, det) new_dtype = self.meta["dtype"] # Apply gain - obj.data = self(obj.data, self._get_gain(obj.det_id)) + det.data = self(det.data, self._get_gain(det.det_id)) # Type-conversion wraps around input values that are higher or lower than # the respective maximum and minimum values of the new data type. Before @@ -341,14 +325,14 @@ def apply_to(self, obj, **kwargs): if np.issubdtype(new_dtype, np.integer): minval = np.iinfo(new_dtype).min maxval = np.iinfo(new_dtype).max - minvals_mask = obj.data < minval - maxvals_mask = obj.data > maxval + minvals_mask = det.data < minval + maxvals_mask = det.data > maxval if minvals_mask.any(): - obj.data[minvals_mask] = minval + det.data[minvals_mask] = minval logger.warning( f"Effect ADConversion: {minvals_mask.sum()} negative pixels") if maxvals_mask.any(): - obj.data[maxvals_mask] = maxval + det.data[maxvals_mask] = maxval logger.warning( f"Effect ADConversion: {maxvals_mask.sum()} saturated pixels") @@ -356,6 +340,4 @@ def apply_to(self, obj, **kwargs): # set to the modified data. It should be fine to simply re-assign the # data attribute, but just in case it's not... logger.debug("Applying digitization to dtype %s.", new_dtype) - obj.data = obj.data.astype(new_dtype) - - return obj + det.data = det.data.astype(new_dtype) diff --git a/scopesim/effects/electronic/noise.py b/scopesim/effects/electronic/noise.py index c149ed5f7..69d22a118 100644 --- a/scopesim/effects/electronic/noise.py +++ b/scopesim/effects/electronic/noise.py @@ -1,31 +1,33 @@ # -*- coding: utf-8 -*- """Any kinds of electronic or photonic noise.""" +from hashlib import sha256 from typing import ClassVar from collections.abc import Mapping from numbers import Real # matches int, float and all the numpy scalars import numpy as np from numpy.typing import ArrayLike, NDArray -from astropy.io import fits -from .. import Effect -from ...detector import Detector -from ...utils import from_currsys, figure_factory, check_keys -from . import logger +from ...utils import from_currsys, figure_factory +from . import ElectronicEffect, Detector, logger -class Bias(Effect): +# TODO: Potential refactoring in multiple effects here: +# The various __call__ now get a det_id, which is what the _get_... need +# for their lookup. Either pass the rng to __call__ instead of det_id, +# or call the _get_.. in there. Needs some consideration of outside +# callers, would they rather pass which one? And how about det_id = None? +# Also could refactor the _get_... (also elsewhere) to a abstract value +# plus the concrete key. + + +class Bias(ElectronicEffect): """Adds a constant bias level to readout.""" required_keys = {"bias"} z_order: ClassVar[tuple[int, ...]] = (855,) - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.meta.update(kwargs) - check_keys(self.meta, self.required_keys, action="error") - def __call__(self, data: ArrayLike) -> NDArray: return data + self.bias_level @@ -33,15 +35,169 @@ def __call__(self, data: ArrayLike) -> NDArray: def bias_level(self) -> float: return from_currsys(self.meta["bias"], self.cmds) - def apply_to(self, obj, **kwargs): - if not isinstance(obj, Detector): - return obj - obj.data = self(obj.data) - return obj +class DarkCurrent(ElectronicEffect): + """ + required: dit, ndit, value + """ + + required_keys = {"value", "dit", "ndit"} + z_order: ClassVar[tuple[int, ...]] = (830,) + + def __call__( + self, + data: ArrayLike, + dark_level: float, + dit: float, + ndit: int, + ) -> NDArray: + return data + dark_level * dit * ndit + + def _get_dark_level(self, det_id: int) -> float: + dark_level = float(from_currsys(self.meta["value"], self.cmds)) + if isinstance(dark_level, Real): + return dark_level + if isinstance(dark_level, Mapping): + return from_currsys(dark_level[det_id], self.cmds) + raise TypeError( + f"<{self.__class__.__name__}>.meta['value'] must be either " + f"dict-like or scalar number, but is {dark_level}." + ) + + def _apply_to_det(self, det: Detector) -> None: + logger.debug("Apply %s to %s", self.display_name, det) + + # Dark level needs detector meta so can't go into __call__() + dark_level = self._get_dark_level(det.det_id) + dit = from_currsys(self.meta["dit"], self.cmds) + ndit = from_currsys(self.meta["ndit"], self.cmds) + + det.data = self(det.data, dark_level, dit, ndit) + + def plot(self, det, **kwargs): + """Plot effect.""" + dit = from_currsys(self.meta["dit"], self.cmds) + ndit = from_currsys(self.meta["ndit"], self.cmds) + total_time = dit * ndit + times = np.linspace(0, 2*total_time, 10) + dtcr = self.apply_to(det) + dark_level = dtcr.data[0, 0] / total_time # just read one pixel + levels = dark_level * times + fig, ax = figure_factory() + ax.plot(times, levels, **kwargs) + ax.set_xlabel("time") + ax.set_ylabel("dark level") + return ax + +# TODO: Add tests for seed resolving, start with code from #975, but for +# basic_instrument, and work from there. +class RandomEffect(ElectronicEffect): + """Mixin class for Effects that need random seeds.""" -class PoorMansHxRGReadoutNoise(Effect): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.meta["random_seed"] = "!SIM.random.seed" # Default + + @property + def root_seed(self) -> int: + """Resolve root seed from cmds, or use value from effect kwargs.""" + return from_currsys(self.meta["random_seed"], self.cmds) + + @classmethod + def cls_seed(cls) -> int: + """Generate reproducible seed from hash of class name.""" + digest = sha256(cls.__name__.encode("utf-8")).digest() + return int.from_bytes(digest[:16], "little") + + @property + def readout_seed(self) -> int: + """Readout ID, or 0 if not found in cmds.""" + return self.cmds.get("!OBS.roid", 0) if self.cmds is not None else 0 + + @property + def random_seed(self) -> int | None: + """Composite random seed, or None.""" + seed = [self.readout_seed, self.cls_seed(), self.root_seed] + # TODO: Consider removing this if root seed is resolved upstream! + if None in seed: + return None + return seed + + def create_rng(self, det_id: int | None = None) -> np.random.Generator: + """ + Instantiate np.random.Generator using composite seed. + + Parameters + ---------- + det_id : int | None, optional + Detector ID. If None (the default), detector ID is not included in + the composite seed, meaning results will look identical on all + detectors passed to the effect in the same readout. + + Returns + ------- + np.random.Generator + New Generator instance. + + """ + if det_id is not None: + seed = [det_id, *self.random_seed] + else: + seed = self.random_seed + return np.random.default_rng(seed) + + def plot(self, det): + """Plot effect image.""" + detector = self.apply_to(det) + fig, ax = figure_factory() + ax.imshow(detector.data) + return ax + + def plot_hist(self, det, **kwargs): + """Plot effect histogram.""" + detector = self.apply_to(det) + fig, ax = figure_factory() + ax.hist(detector.data.flatten()) + return ax + + +class BasicReadoutNoise(RandomEffect): + """Readout noise computed as: ron * sqrt(NDIT).""" + + required_keys = {"noise_std", "ndit"} + z_order: ClassVar[tuple[int, ...]] = (811,) + + def __call__( + self, + data: ArrayLike, + ndit: int = 1, + det_id: int | None = None, + ) -> NDArray: + rng = self.create_rng(det_id) + return data + self._create_noise_frame(data.shape, rng, ndit) + + @property + def noise_std(self) -> float: + return from_currsys(self.meta["noise_std"], self.cmds) + + def _create_noise_frame( + self, + shape: tuple[int, ...], + rng: np.random.Generator, + ndit: int, + ) -> NDArray: + scale = self.noise_std * np.sqrt(float(ndit)) + return rng.normal(loc=0, scale=scale, size=shape) + + def _apply_to_det(self, det: Detector) -> None: + logger.debug("Apply %s to %s", self.display_name, det) + + ndit = from_currsys(self.meta["ndit"], self.cmds) + det.data = self(det.data, ndit) + + +class PoorMansHxRGReadoutNoise(BasicReadoutNoise): required_keys = {"noise_std", "n_channels", "ndit"} z_order: ClassVar[tuple[int, ...]] = (811,) report_plot_include: ClassVar[bool] = False @@ -54,60 +210,34 @@ def __init__(self, **kwargs): "read_fraction": 0.4, "line_fraction": 0.25, "channel_fraction": 0.05, - "random_seed": "!SIM.random.seed", } self.meta.update(params) self.meta.update(kwargs) - check_keys(self.meta, self.required_keys, action="error") - - def __call__(self, data: ArrayLike, ndit: int = 1) -> NDArray: - self.meta = from_currsys(self.meta, self.cmds) - - ron_frame = self._make_ron_frame(data.shape) - - stacked_ron_frame = np.zeros_like(ron_frame) - for i in range(ndit): - dx = np.random.randint(0, ron_frame.shape[1]) - dy = np.random.randint(0, ron_frame.shape[0]) - stacked_ron_frame += np.roll(ron_frame, (dy, dx), axis=(0, 1)) - - # TODO: this .T is ugly. Work out where things are getting switched and remove it! - return data + stacked_ron_frame.T - - @property - def noise_std(self) -> int: - return from_currsys(self.meta["noise_std"], self.cmds) - @property def n_channels(self) -> int: return from_currsys(self.meta["n_channels"], self.cmds) - @property - def random_seed(self) -> int: - return from_currsys(self.meta["random_seed"], self.cmds) - - @staticmethod - def _pseudo_random_field( - rng, - scale: float = 1., - size: tuple[int, ...] = (1024, 1024), + def _create_noise_frame( + self, + shape: tuple[int, ...], + rng: np.random.Generator, + ndit: int, ) -> NDArray: - n = 256 - image = np.zeros(size) - batch = rng.normal(loc=0, scale=scale, size=(2*n, 2*n)) - for y in range(0, size[1], n): - for x in range(0, size[0], n): - i, j = rng.integers(n, size=2) - dx, dy = min(size[0]-x, n), min(size[1]-y, n) - image[x:x+dx, y:y+dy] = batch[i:i+dx, j:j+dy] + ron_frame = self._make_ron_frame(rng, shape) + stacked_ron_frame = np.zeros_like(ron_frame) - return image + for i in range(ndit): + stacked_ron_frame += np.roll( + ron_frame, + rng.integers((0, 0), ron_frame.shape), + axis=(0, 1), + ) - def _make_ron_frame(self, shape: tuple[int, ...]) -> NDArray: - # TODO: Add yaml-settable seed here - rng = np.random.default_rng(self.random_seed) + return stacked_ron_frame + def _make_ron_frame(self, rng, shape: tuple[int, ...]) -> NDArray: + self.meta = from_currsys(self.meta, self.cmds) # TODO: Is this needed? channel_fraction = self.meta["channel_fraction"] line_fraction = self.meta["line_fraction"] pedestal_fraction = self.meta["pedestal_fraction"] @@ -136,78 +266,27 @@ def _make_ron_frame(self, shape: tuple[int, ...]) -> NDArray: axis=0, ) - return (pixel + line).T + channel[:shape[0]] - - def apply_to(self, det, **kwargs): - if not isinstance(det, Detector): - return det - - ndit = from_currsys(self.meta["ndit"], self.cmds) - det.data = self(det.data, ndit) - return det - - def plot(self, det, **kwargs): - """Plot effect image.""" - dtcr = self.apply_to(det) - fig, ax = figure_factory() - ax.imshow(dtcr.data, origin="lower") - - def plot_hist(self, det, **kwargs): - """Plot effect histogram.""" - dtcr = self.apply_to(det) - fig, ax = figure_factory() - ax.hist(dtcr.data.flatten()) - - -class BasicReadoutNoise(Effect): - """Readout noise computed as: ron * sqrt(NDIT).""" - - required_keys = {"noise_std", "ndit"} - z_order: ClassVar[tuple[int, ...]] = (811,) - - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.meta["random_seed"] = "!SIM.random.seed" - self.meta.update(kwargs) + return (pixel + line) + channel[:shape[0], None] - check_keys(self.meta, self.required_keys, action="error") - - def __call__(self, data: ArrayLike, ndit: int = 1) -> NDArray: - rng = np.random.default_rng(self.random_seed) - scale = self.noise_std * np.sqrt(float(ndit)) - return data + rng.normal(loc=0, scale=scale, size=data.shape) - - @property - def noise_std(self) -> float: - return from_currsys(self.meta["noise_std"], self.cmds) - - @property - def random_seed(self) -> int: - return from_currsys(self.meta["random_seed"], self.cmds) - - def apply_to(self, det, **kwargs): - if not isinstance(det, Detector): - return det - - ndit = from_currsys(self.meta["ndit"], self.cmds) - det.data = self(det.data, ndit) - return det - - def plot(self, det): - """Plot effect image.""" - dtcr = self.apply_to(det) - fig, ax = figure_factory() - ax.imshow(dtcr.data) + @staticmethod + def _pseudo_random_field( + rng, + scale: float = 1., + size: tuple[int, ...] = (1024, 1024), + ) -> NDArray: + n = 256 + image = np.zeros(size) + batch = rng.normal(loc=0, scale=scale, size=(2*n, 2*n)) + for y in range(0, size[1], n): + for x in range(0, size[0], n): + i, j = rng.integers(n, size=2) + dx, dy = min(size[0]-x, n), min(size[1]-y, n) + image[x:x+dx, y:y+dy] = batch[i:i+dx, j:j+dy] - def plot_hist(self, det, **kwargs): - """Plot effect histogram.""" - dtcr = self.apply_to(det) - fig, ax = figure_factory() - ax.hist(dtcr.data.flatten()) + return image -# TODO: Is this really a "noise" effect? Sounds more like "electrons" tbh. -class PixelResponseNonUniformity(Effect): +class PixelResponseNonUniformity(RandomEffect): """Pixel Response Non-Uniformity (PRNU). Models the fixed pattern of per-pixel gain variations across the detector @@ -223,8 +302,6 @@ class PixelResponseNonUniformity(Effect): prnu_std : float or dict Standard deviation of the per-pixel gain distribution. - prnu_seed : int, fixed - include: "!DET.include_prnu" Example @@ -236,49 +313,61 @@ class PixelResponseNonUniformity(Effect): class: PixelResponseNonUniformity kwargs: prnu_std: 0.001 - prnu_seed: 42 include: "!DET.include_prnu" """ - required_keys: ClassVar[set] = set() + required_keys: ClassVar[set] = {"prnu_std"} z_order: ClassVar[tuple[int, ...]] = (805,) def __init__(self, **kwargs): super().__init__(**kwargs) - self.meta.update(kwargs) self._gain_maps = {} # keyed by det_id - def apply_to(self, obj, **kwargs): - if not isinstance(obj, Detector): - return obj - - random_seed = from_currsys(self.meta.get("prnu_seed"), self.cmds) - det_id = obj.det_id - - prnu_std_meta = from_currsys(self.meta["prnu_std"], self.cmds) - if isinstance(prnu_std_meta, Mapping): - prnu_std = float(from_currsys(prnu_std_meta[det_id], self.cmds)) - elif isinstance(prnu_std_meta, Real): - prnu_std = float(prnu_std_meta) - else: - raise TypeError( - ".meta['prnu_std'] must be a float " - f"or a dict keyed by detector ID, got {type(prnu_std_meta)}" - ) - - shape = obj.data.shape + def __call__( + self, + data: ArrayLike, + prnu_std: float, + det_id: int | None = None, + ) -> NDArray: if det_id not in self._gain_maps: - rng = np.random.default_rng(random_seed) + # TODO: Should not use roid here??? + rng = self.create_rng(det_id) self._gain_maps[det_id] = rng.normal( - loc=1.0, scale=prnu_std, size=shape, + loc=1.0, scale=prnu_std, size=data.shape, ) - if self._gain_maps[det_id].shape != shape: - raise ValueError("gain map shape mismatch") + return data * self._gain_maps[det_id] - obj.data = obj.data * self._gain_maps[det_id] - return obj + @property + def random_seed(self) -> int | None: + """Composite random seed, or None. + + Override from base class to remove ROID, because PRNU should be + identical in one observation. + """ + seed = [self.cls_seed(), self.root_seed] + # TODO: Consider removing this if root seed is resolved upstream! + if None in seed: + return None + return seed + + def _get_prnu_std(self, det_id: int) -> float: + prnu_std = from_currsys(self.meta["prnu_std"], self.cmds) + if isinstance(prnu_std, Real): + return prnu_std + if isinstance(prnu_std, Mapping): + return from_currsys(prnu_std[det_id], self.cmds) + raise TypeError( + f"<{self.__class__.__name__}>.meta['value'] must be either " + f"dict-like or scalar number, but is {prnu_std}." + ) + + def _apply_to_det(self, det: Detector) -> None: + logger.debug("Apply %s to %s", self.display_name, det) + + prnu_std = self._get_prnu_std(det.det_id) + det.data = self(det.data, prnu_std) def plot(self, det_id=None): """Plot effect.""" @@ -291,19 +380,38 @@ def plot(self, det_id=None): im = ax.imshow(gain_map, origin="lower", aspect="auto", vmin=1 - dev, vmax=1 + dev) fig.colorbar(im, ax=ax, label="per-pixel gain") - return fig + return fig, ax -class ShotNoise(Effect): - z_order: ClassVar[tuple[int, ...]] = (820,) +class ShotNoise(RandomEffect): + """Poissonian photon noise. - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.meta["random_seed"] = "!SIM.random.seed" - self.meta.update(kwargs) + Notes + ----- - def __call__(self, data: ArrayLike) -> NDArray: - rng = np.random.default_rng(self.random_seed) + Numpy has a problem with generating Poisson distributions above certain + values. E.g. on linux, numpy.random.poisson(1e20) raises ValueError: lam + value too large. The value might be smaller on other (operating) systems. + + The poisson and normal distribution are basically the same + above ~100 counts: + poisson(x) ~= normal(mu=x, sigma=x**0.5) + + Therefore a limit of 1e7 is used, above which the Poisson distribution is + approximated with a normal distribution. + + Also, the normal distribution takes only 60% as long as the Poisson + distribution for large arrays. + + Special values should be handled with care: + - Negative values are mapped to 0; there cannot be negative flux. + - numpy.nan are implicitly passed through the normal distribution; + because the Poisson distribution cannot handle them. + """ + z_order: ClassVar[tuple[int, ...]] = (820,) + + def __call__(self, data: ArrayLike, det_id: int | None = None) -> NDArray: + rng = self.create_rng(det_id) # Check if there are negative values in the data. values_negative = data < 0 @@ -318,111 +426,9 @@ def __call__(self, data: ArrayLike) -> NDArray: # Apply a normal distribution to the high values. values_high = ~values_low - data[values_high] = rng.normal(data[values_high], np.sqrt(data[values_high])) - - return data - - @property - def random_seed(self) -> int: - return from_currsys(self.meta["random_seed"], self.cmds) - - def apply_to(self, det, **kwargs): - if not isinstance(det, Detector): - return det - - self.meta["random_seed"] = from_currsys( - self.meta["random_seed"], self.cmds) - - # numpy has a problem with generating Poisson distributions above - # certain values. E.g. on linux, numpy.random.poisson(1e20) raises - # ValueError: lam value too large - # The value might be smaller on other (operating) systems. - # - # The poisson and normal distribution are basically the same - # above ~100 counts: - # poisson(x) ~= normal(mu=x, sigma=x**0.5) - # - # Therefore a limit of 1e7 is used, above which the Poisson - # distribution is approximated with a normal distribution. - # - # Also, the normal distribution takes only 60% as long as the - # poisson distribution for large arrays. - # - # Special values should be handled with care: - # - Negative values are mapped to 0; there cannot be negative flux. - # - numpy.nan are implicitly passed through the normal distribution; - # because the Poisson distribution cannot handle them. - - det._hdu = fits.ImageHDU(header=det.header, data=self(det.data)) - return det - - def plot(self, det): - """Plot effect image.""" - dtcr = self.apply_to(det) - fig, ax = figure_factory() - ax.imshow(dtcr.data) - - def plot_hist(self, det, **kwargs): - """Plot effect histogram.""" - dtcr = self.apply_to(det) - fig, ax = figure_factory() - ax.hist(dtcr.data.flatten()) - - -class DarkCurrent(Effect): - """ - required: dit, ndit, value - """ - - required_keys = {"value", "dit", "ndit"} - z_order: ClassVar[tuple[int, ...]] = (830,) - - def __init__(self, **kwargs): - super().__init__(**kwargs) - check_keys(self.meta, self.required_keys, action="error") - - def __call__( - self, - data: np.ndarray, - dark_level: float, - dit: float, - ndit: int, - ) -> np.ndarray: - return data + dark_level * dit * ndit - - def _get_dark_level(self, det_id: int) -> float: - dark_level = float(from_currsys(self.meta["value"], self.cmds)) - if isinstance(dark_level, Real): - return dark_level - if isinstance(dark_level, Mapping): - return from_currsys(dark_level[det_id], self.cmds) - raise ValueError( - f"<{self.__class__.__name__}>.meta['value'] must be either " - f"dict-like or scalar number, but is {dark_level}." + data[values_high] = rng.normal( + loc=data[values_high], + scale=np.sqrt(data[values_high]), ) - def apply_to(self, obj, **kwargs): - if not isinstance(obj, Detector): - return obj - - # Dark level needs detector meta so can't go into __call__() - dark_level = self._get_dark_level(obj.det_id) - dit = from_currsys(self.meta["dit"], self.cmds) - ndit = from_currsys(self.meta["ndit"], self.cmds) - - obj.data = self(obj.data, dark_level, dit, ndit) - return obj - - def plot(self, det, **kwargs): - """Plot effect.""" - dit = from_currsys(self.meta["dit"], self.cmds) - ndit = from_currsys(self.meta["ndit"], self.cmds) - total_time = dit * ndit - times = np.linspace(0, 2*total_time, 10) - dtcr = self.apply_to(det) - dark_level = dtcr.data[0, 0] / total_time # just read one pixel - levels = dark_level * times - fig, ax = figure_factory() - ax.plot(times, levels, **kwargs) - ax.set_xlabel("time") - ax.set_ylabel("dark level") + return data diff --git a/scopesim/effects/electronic/pixels.py b/scopesim/effects/electronic/pixels.py index f97d1eb2c..e26b26e96 100644 --- a/scopesim/effects/electronic/pixels.py +++ b/scopesim/effects/electronic/pixels.py @@ -5,13 +5,11 @@ from numpy.typing import ArrayLike, NDArray -from .. import Effect -from ...detector import Detector -from ...utils import from_currsys, figure_factory, check_keys, real_colname -from .. import logger +from ...utils import from_currsys, figure_factory +from . import ElectronicEffect, Detector, logger -class ReferencePixelBorder(Effect): +class ReferencePixelBorder(ElectronicEffect): """Remove signal from reference pixels. Detectors often have a number of rows and columns around the edges masked. @@ -59,26 +57,20 @@ def __call__(self, data: ArrayLike, border: dict[int, int]) -> NDArray: data[:, -border[3]:] = 0 return data - def _get_border(self, det_meta) -> dict[int, int]: + def _get_border(self, det_id: int) -> dict[int, int]: if hasattr(self.meta["border"], "dic"): - dtcr_id = det_meta[real_colname("id", det_meta)] - return self.meta["border"].dic[dtcr_id] + return self.meta["border"].dic[det_id] if isinstance(self.meta["border"], list): return self.meta["border"] - raise ValueError( + raise TypeError( f"{self.__class__.__name__}.meta['border'] must be either " f"dict or list, but is {self.meta['border']}") - def apply_to(self, obj, **kwargs): - """Mask border pixels.""" - if not isinstance(obj, Detector): - logger.warning( - "ReferencePixelBorder: got non-detector object: %s", type(obj)) - return obj - + def _apply_to_det(self, det: Detector) -> None: + """Subclasses can override if more params needed in call.""" + logger.debug("Apply %s to %s", self.display_name, det) logger.info(f"Applying border {from_currsys(self.meta['border'])}") - obj.data = self(obj.data, self._get_border(obj.meta)) - return obj + det.data = self(det.data, self._get_border(det.det_id)) def plot(self, det, **kwargs): """Show the masked detector image.""" @@ -96,25 +88,11 @@ def __str__(self) -> str: return msg -class BinnedImageBase(Effect): +class BinnedImageBase(ElectronicEffect): """Base class for binning effects.""" z_order: ClassVar[tuple[int, ...]] = (870,) - def __init__(self, **kwargs): - super().__init__(**kwargs) - check_keys(self.meta, self.required_keys, action="error") - - def __call__(self, data: ArrayLike) -> NDArray: - raise NotImplementedError("Subclasses must implement this.") - - def apply_to(self, det, **kwargs): - if not isinstance(det, Detector): - return det - - det.data = self(det.data) - return det - class BinnedImage(BinnedImageBase): required_keys = {"bin_size"} diff --git a/scopesim/optics/optical_train.py b/scopesim/optics/optical_train.py index 8121670de..3046de244 100644 --- a/scopesim/optics/optical_train.py +++ b/scopesim/optics/optical_train.py @@ -486,7 +486,7 @@ def readout( # self.cmds["!OBS.dit"] = None # self.cmds["!OBS.ndit"] = None # TODO: This is still hacky but seems to work for now... - params = {"exptime": None} + params = {"exptime": None, "roid": roid} params.update(kwargs) if params["exptime"] is not None and params.get("dit") is None: params.update( diff --git a/scopesim/tests/tests_effects/test_BasicReadoutNoise.py b/scopesim/tests/tests_effects/test_BasicReadoutNoise.py index e119dfa29..1b0ac3e30 100644 --- a/scopesim/tests/tests_effects/test_BasicReadoutNoise.py +++ b/scopesim/tests/tests_effects/test_BasicReadoutNoise.py @@ -46,7 +46,8 @@ def test_stdev_increase_with_square_of_n(self, n): "pedestal_fraction": 0.3, "read_fraction": 0.4, }) - frames = np.array([ron._make_ron_frame((256, 256)) + rng = np.random.default_rng(42) + frames = np.array([ron._make_ron_frame(rng, (256, 256)) for _ in range(n)]) assert np.std(np.sum(frames, axis=0)) == approx(10*n**0.5, rel=0.3) @@ -59,5 +60,6 @@ def test_makes_frame_sizes_for_non_integer_n_channels(self, shape): "pedestal_fraction": 0.25, "read_fraction": 0.25, }) - frame = ron._make_ron_frame(shape) + rng = np.random.default_rng(42) + frame = ron._make_ron_frame(rng, shape) assert frame.shape > (0, 0) From 5a6f25bd6ac5759d0fbf913c4e48bd7cf87591f3 Mon Sep 17 00:00:00 2001 From: oczoske Date: Thu, 17 Sep 2026 12:39:38 +0200 Subject: [PATCH 54/87] Include dark current in AutoExposure --- scopesim/effects/electronic/exposure.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/scopesim/effects/electronic/exposure.py b/scopesim/effects/electronic/exposure.py index 0b20b698e..342e9ac67 100644 --- a/scopesim/effects/electronic/exposure.py +++ b/scopesim/effects/electronic/exposure.py @@ -102,8 +102,17 @@ def estimate_dit_ndit( "fill_frac", from_currsys(self.meta["fill_frac"], self.cmds) ) + try: + dark_current = kwargs.get( + "dark_current", + from_currsys(self.meta["dark_current"], self.cmds) + ) + except KeyError: + logger.warning("No dark current found for %s", + self.display_name) + dark_current = 0 - dit_nosat = fill_frac * full_well / image_plane_max + dit_nosat = fill_frac * full_well / (image_plane_max + dark_current) logger.info("Required DIT without saturation: %.3f s", dit_nosat) # np.ceil so that dit is at most what is required for fill_frac From bb15056aa6158008cf3dabf801e351367faa0a94 Mon Sep 17 00:00:00 2001 From: oczoske Date: Thu, 17 Sep 2026 13:42:41 +0200 Subject: [PATCH 55/87] A few more tests --- .../tests/tests_effects/test_AutoExposure.py | 45 ++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/scopesim/tests/tests_effects/test_AutoExposure.py b/scopesim/tests/tests_effects/test_AutoExposure.py index 2876de7bf..d836d78dc 100644 --- a/scopesim/tests/tests_effects/test_AutoExposure.py +++ b/scopesim/tests/tests_effects/test_AutoExposure.py @@ -32,6 +32,7 @@ def fixture_autoexposure(): """Instantiate an AutoExposure object""" return AutoExposure(fill_frac=0.75, full_well=1e5, + dark_current=0, mindit=0.011, exptime=None) @@ -40,6 +41,7 @@ class TestAutoExposure: def test_initialises_correctly(self): autoexposure = AutoExposure(fill_frac=0.75, full_well=1e5, + dark_current=0, mindit=0.011) assert isinstance(autoexposure, AutoExposure) @@ -102,7 +104,7 @@ def test_fill_frac_acts_correctly(self, imageplane): assert out_ndit_1 == fill_2 / fill_1 * out_ndit_2 @pytest.mark.xfail(reason=("This now fails because a user-specified DIT " - "and NDIT will now just go thourgh unmodified. " + "and NDIT will now just go through unmodified. " "Decide what to do with this test later...")) def test_exptime_specified_by_dit_ndit(self, autoexposure, imageplane): """ @@ -139,3 +141,44 @@ def test_exptime_at_least_mindit(self, imageplane): assert dit == mindit assert ndit == 1 + + def test_takes_dark_current_into_account(self, imageplane): + exptime = 3600 + autoexp_1 = AutoExposure(fill_frac=0.75, + full_well=1e6, + mindit=1.3, + dark_current=0) + autoexp_1.cmds["!OBS.exptime"] = exptime + autoexp_1.apply_to(imageplane) + dit1 = autoexp_1.cmds["!OBS.dit"] + + autoexp_2 = AutoExposure(fill_frac=0.75, + full_well=1e6, + mindit=1.3, + dark_current=1e4) + autoexp_2.cmds["!OBS.exptime"] = exptime + autoexp_2.apply_to(imageplane) + dit2 = autoexp_2.cmds["!OBS.dit"] + + assert dit2 < dit1 + + def test_no_dark_gives_zero_dark(self, imageplane): + exptime = 3600 + # no dark provided + autoexp_1 = AutoExposure(fill_frac=0.75, + full_well=1e6, + mindit=1.3) + autoexp_1.cmds["!OBS.exptime"] = exptime + autoexp_1.apply_to(imageplane) + dit1 = autoexp_1.cmds["!OBS.dit"] + + # zero dark explicitly provided + autoexp_2 = AutoExposure(fill_frac=0.75, + full_well=1e6, + mindit=1.3, + dark_current=0.) + autoexp_2.cmds["!OBS.exptime"] = exptime + autoexp_2.apply_to(imageplane) + dit2 = autoexp_2.cmds["!OBS.dit"] + + assert dit2 == dit1 From 3aabed0be597f05ac1ef980ee01821805d9346bf Mon Sep 17 00:00:00 2001 From: oczoske Date: Thu, 17 Sep 2026 14:00:43 +0200 Subject: [PATCH 56/87] test for warning issued --- scopesim/tests/tests_effects/test_AutoExposure.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/scopesim/tests/tests_effects/test_AutoExposure.py b/scopesim/tests/tests_effects/test_AutoExposure.py index d836d78dc..179d1419d 100644 --- a/scopesim/tests/tests_effects/test_AutoExposure.py +++ b/scopesim/tests/tests_effects/test_AutoExposure.py @@ -1,7 +1,8 @@ """Tests for Effect AutoExposure.""" +import logging + import pytest -from unittest.mock import patch from scopesim import UserCommands from scopesim.optics.image_plane import ImagePlane @@ -9,9 +10,10 @@ from scopesim.tests.mocks.py_objects.imagehdu_objects import _image_hdu_square -# pylint: disable=no-self-use, missing-class-docstring +# pylint: disable=missing-class-docstring # pylint: disable=missing-function-docstring +LOGGER = logging.getLogger(__name__) def _patched_cmds(exptime=1, dit=None, ndit=None): return UserCommands(properties={"!OBS.exptime": exptime, @@ -162,14 +164,16 @@ def test_takes_dark_current_into_account(self, imageplane): assert dit2 < dit1 - def test_no_dark_gives_zero_dark(self, imageplane): + def test_no_dark_gives_zero_dark(self, imageplane, caplog): exptime = 3600 # no dark provided autoexp_1 = AutoExposure(fill_frac=0.75, full_well=1e6, mindit=1.3) autoexp_1.cmds["!OBS.exptime"] = exptime + caplog.set_level(logging.WARNING) autoexp_1.apply_to(imageplane) + assert "No dark current found for" in caplog.text dit1 = autoexp_1.cmds["!OBS.dit"] # zero dark explicitly provided From 1239de591a717691aca3c5eb5f08df5b6000e619 Mon Sep 17 00:00:00 2001 From: oczoske Date: Thu, 17 Sep 2026 17:21:48 +0200 Subject: [PATCH 57/87] Cosmetics --- scopesim/effects/micado_ifu_trace_list.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/scopesim/effects/micado_ifu_trace_list.py b/scopesim/effects/micado_ifu_trace_list.py index cd2595b44..390407a06 100644 --- a/scopesim/effects/micado_ifu_trace_list.py +++ b/scopesim/effects/micado_ifu_trace_list.py @@ -94,12 +94,12 @@ def apply_to_fov(self, obj): ymin = spt.meta["fov"]["y_min"] ymax = spt.meta["fov"]["y_max"] - slicewcs = fovwcs.deepcopy() slicewcs.wcs.ctype = ["LINEAR", "LINEAR", slicewcs.wcs.ctype[2]] slicewcs.wcs.crpix[1] = (ny_slice + 1) / 2 + # todo: here we silently assume that ymin, ymax are in degrees slicewcs.wcs.crval[1] = (ymin + ymax) / 2 / 3600 slicewcs.wcs.cdelt[1] = (ymax - ymin) / ny_slice / 3600 slicewcs_spat = slicewcs.sub(2) @@ -112,6 +112,7 @@ def apply_to_fov(self, obj): slicecube = np.zeros((n_z, ny_slice, n_x)) for islice in range(n_z): + # todo: replace by explicit bilinear interpolation ifov = RectBivariateSpline(np.arange(n_y), np.arange(n_x), fovcube[islice], kx=1, ky=1) @@ -139,8 +140,6 @@ def apply_to_fov(self, obj): return obj - - def make_spectral_traces(self): """Make a spectral trace for each combination of order and aperture""" self.ext_data = self._file[0].header["EDATA"] @@ -157,15 +156,17 @@ def make_spectral_traces(self): for spslice, apid in enumerate(self.slicelist["id"]): specid = f"{row['description']}_{apid:02d}" - thistrace = MicadoIFUSpectralTrace(trace_hdu, self.slicelist, spslice, - **params) + thistrace = MicadoIFUSpectralTrace( + trace_hdu, + self.slicelist, + spslice, + **params) thistrace.meta["trace_id"] = specid spec_traces[specid] = thistrace self.spectral_traces = spec_traces - class MicadoIFUSpectralTrace(SpectralTrace): """SpectralTrace for the MICADO IFU""" @@ -187,8 +188,8 @@ def __init__(self, trace_tbl, aplist, spslice, cmds=None, **kwargs): self.meta["fov"] = self.fov_grid() lam = np.linspace(self.wave_min, self.wave_max, 1001) * u.um - off_y = aplist['offset'][spslice] - dlam = np.median(self.dlam_per_pix(lam)) / self.meta["pixsize"] * off_y + off_y = aplist['offset'][spslice] # offset on detector in mm + dlam = np.median(self.dlam_per_pix(lam)) * off_y / self.meta["pixsize"] self.xy2lam.posttransform = (det_offset, {"offset": -dlam}) self.xilam2x.pretransform_y = (det_offset, {"offset": -dlam}) self.xilam2y.pretransform_y = (det_offset, {"offset": -dlam}) From 4af9ce564d8bc2da15bae6d3f03416883b527524 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 18 Sep 2026 01:31:22 +0000 Subject: [PATCH 58/87] Bump soupsieve from 2.8.4 to 2.9 Bumps [soupsieve](https://github.com/facelessuser/soupsieve) from 2.8.4 to 2.9. - [Release notes](https://github.com/facelessuser/soupsieve/releases) - [Commits](https://github.com/facelessuser/soupsieve/compare/2.8.4...2.9) --- updated-dependencies: - dependency-name: soupsieve dependency-version: '2.9' dependency-type: indirect ... Signed-off-by: dependabot[bot] --- poetry.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index ebb77fc36..b7aff9557 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3797,14 +3797,14 @@ files = [ [[package]] name = "soupsieve" -version = "2.8.4" +version = "2.9.2" description = "A modern CSS selector implementation for Beautiful Soup." optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main", "dev", "docs"] files = [ - {file = "soupsieve-2.8.4-py3-none-any.whl", hash = "sha256:e7e6b0769c8f51ed59acab6e994b00621096cfb1c640a7509295987388fbaf65"}, - {file = "soupsieve-2.8.4.tar.gz", hash = "sha256:e121fd02e975c695e4e9e8774a5ee35d74714b59307868dcc5319ad2d9e3328e"}, + {file = "soupsieve-2.9.2-py3-none-any.whl", hash = "sha256:8089a26fd974ca7a1f30276d3d8492ab266ab15af581642dfe8aa162e0c1c823"}, + {file = "soupsieve-2.9.2.tar.gz", hash = "sha256:4a55d8cf158a9c2e587fa4922f1bbb91d68ac829e2d6f25403a85747c71daf74"}, ] [[package]] From d828994f5c53ea54f833625b4b0f75cdf5ab5f62 Mon Sep 17 00:00:00 2001 From: "astarvienna-bot[bot]" <217837468+myciapp[bot]@users.noreply.github.com> Date: Fri, 18 Sep 2026 09:16:45 +0000 Subject: [PATCH 59/87] Bumping version from 0.12.0b3.dev0 to 0.12.0b3.dev1 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4597b8c6e..3b247f257 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ScopeSim" -version = "0.12.0b3.dev0" +version = "0.12.0b3.dev1" description = "Generalised telescope observation simulator" license = {text = "GPL-3.0-or-later"} authors = [ From 0c30b870888a42c7fbe52df0a652419363a352f0 Mon Sep 17 00:00:00 2001 From: oczoske Date: Fri, 18 Sep 2026 12:43:21 +0200 Subject: [PATCH 60/87] import numpy --- scopesim/optics/image_plane.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scopesim/optics/image_plane.py b/scopesim/optics/image_plane.py index 9cda002b4..3ecba4d4f 100644 --- a/scopesim/optics/image_plane.py +++ b/scopesim/optics/image_plane.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- """Contains ``ImagePlane`` class.""" +import numpy as np from astropy.io import fits from astropy.wcs import WCS From 26c780f0811eac56b25101205a4fee6330a89eee Mon Sep 17 00:00:00 2001 From: oczoske Date: Fri, 18 Sep 2026 17:47:39 +0200 Subject: [PATCH 61/87] Defuse trace outside fov --- scopesim/effects/micado_ifu_trace_list.py | 6 +++++- scopesim/effects/spectral_trace_list_utils.py | 9 +++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/scopesim/effects/micado_ifu_trace_list.py b/scopesim/effects/micado_ifu_trace_list.py index 390407a06..f10342501 100644 --- a/scopesim/effects/micado_ifu_trace_list.py +++ b/scopesim/effects/micado_ifu_trace_list.py @@ -128,7 +128,11 @@ def apply_to_fov(self, obj): slicefov.cube = fits.ImageHDU(header=slicewcs.to_header(), data=slicecube) # slicefov.cube.writeto(f"slicefov_{sptid}.fits", overwrite=True) - slicefov.hdu = spt.map_spectra_to_focal_plane(slicefov) + try: + slicefov.hdu = spt.map_spectra_to_focal_plane(slicefov) + except ValueError as err: + logger.debug(err) + if slicefov.hdu is not None: sxmin = slicefov.hdu.header["XMIN"] sxmax = slicefov.hdu.header["XMAX"] diff --git a/scopesim/effects/spectral_trace_list_utils.py b/scopesim/effects/spectral_trace_list_utils.py index 2c00c98ad..b3a4d7576 100644 --- a/scopesim/effects/spectral_trace_list_utils.py +++ b/scopesim/effects/spectral_trace_list_utils.py @@ -175,6 +175,7 @@ def map_spectra_to_focal_plane(self, fov) -> fits.ImageHDU: where this image lies in the focal plane. """ logger.debug("Mapping %s", fov.trace_id) + # Initialise the image based on the footprint of the spectral # trace and the focal plane WCS wave_min = fov.meta["wave_min"].value # [um] @@ -188,8 +189,10 @@ def map_spectra_to_focal_plane(self, fov) -> fits.ImageHDU: xi_max=xi_max, ) - if xlim_mm is None: - raise ValueError("xlim_mm is None") + if xlim_mm is None or ylim_mm is None: + # Cases where there is no overlap betwen slit/trace and + # fov are acceptable. + return None fov_header = fov.header det_header = fov.detector_header @@ -531,6 +534,7 @@ def footprint(self, wave_min=None, wave_max=None, xi_min=None, xi_max=None): # Requested wavelenth range is entirely outside definition range: # no footprint if wave_min > np.max(wave_val) or wave_max < np.min(wave_val): + logger.info("Trace not in wavelength range") return None, None # Restrict to overlap of requested range and definition range @@ -559,6 +563,7 @@ def footprint(self, wave_min=None, wave_max=None, xi_min=None, xi_max=None): # Requested slit range is entirely outside definition range: # no footprint if xi_min > np.max(xi_val) or xi_max < np.min(xi_val): + logger.info("Slit outside range (spatial)") return None, None # Restrict to overlap of requested range and definition range From 00b4a93fc73bd96b0d98d6b4f750cdc74a48d014 Mon Sep 17 00:00:00 2001 From: oczoske Date: Fri, 18 Sep 2026 18:03:13 +0200 Subject: [PATCH 62/87] comments forgotten in #922 --- scopesim/effects/micado_ifu_trace_list.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scopesim/effects/micado_ifu_trace_list.py b/scopesim/effects/micado_ifu_trace_list.py index f10342501..42eaea14e 100644 --- a/scopesim/effects/micado_ifu_trace_list.py +++ b/scopesim/effects/micado_ifu_trace_list.py @@ -99,7 +99,8 @@ def apply_to_fov(self, obj): slicewcs.wcs.ctype = ["LINEAR", "LINEAR", slicewcs.wcs.ctype[2]] slicewcs.wcs.crpix[1] = (ny_slice + 1) / 2 - # todo: here we silently assume that ymin, ymax are in degrees + # TODO: here we silently assume that ymin, ymax are in arcsec + # and wcs in degrees slicewcs.wcs.crval[1] = (ymin + ymax) / 2 / 3600 slicewcs.wcs.cdelt[1] = (ymax - ymin) / ny_slice / 3600 slicewcs_spat = slicewcs.sub(2) @@ -112,7 +113,7 @@ def apply_to_fov(self, obj): slicecube = np.zeros((n_z, ny_slice, n_x)) for islice in range(n_z): - # todo: replace by explicit bilinear interpolation + # TODO: replace by explicit bilinear interpolation ifov = RectBivariateSpline(np.arange(n_y), np.arange(n_x), fovcube[islice], kx=1, ky=1) @@ -127,7 +128,7 @@ def apply_to_fov(self, obj): slicefov.meta["trace_id"] = sptid slicefov.cube = fits.ImageHDU(header=slicewcs.to_header(), data=slicecube) - # slicefov.cube.writeto(f"slicefov_{sptid}.fits", overwrite=True) + try: slicefov.hdu = spt.map_spectra_to_focal_plane(slicefov) except ValueError as err: @@ -143,7 +144,6 @@ def apply_to_fov(self, obj): obj.hdu = fits.ImageHDU(data=fovimage, header=obj.detector_header) return obj - def make_spectral_traces(self): """Make a spectral trace for each combination of order and aperture""" self.ext_data = self._file[0].header["EDATA"] From 8c8bdc3998d02002b52a52a2b3b56220d949fea4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 18 Sep 2026 20:55:58 +0000 Subject: [PATCH 63/87] Bump anyio from 3.7.1 to 4.14.2 Bumps [anyio](https://github.com/agronholm/anyio) from 3.7.1 to 4.14.2. - [Release notes](https://github.com/agronholm/anyio/releases) - [Commits](https://github.com/agronholm/anyio/compare/3.7.1...4.14.2) --- updated-dependencies: - dependency-name: anyio dependency-version: 4.14.2 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- poetry.lock | 28 +++++++--------------------- 1 file changed, 7 insertions(+), 21 deletions(-) diff --git a/poetry.lock b/poetry.lock index b7aff9557..eaa19d9ef 100644 --- a/poetry.lock +++ b/poetry.lock @@ -50,24 +50,22 @@ numpy = ">=2.2.6,<3.0.0" [[package]] name = "anyio" -version = "3.7.1" -description = "High level compatibility layer for multiple asynchronous event loop implementations" +version = "4.15.1" +description = "High-level concurrency and networking framework on top of asyncio or Trio" optional = false -python-versions = ">=3.7" +python-versions = ">=3.10" groups = ["main", "dev"] files = [ - {file = "anyio-3.7.1-py3-none-any.whl", hash = "sha256:91dee416e570e92c64041bd18b900d1d6fa78dff7048769ce5ac5ddad004fbb5"}, - {file = "anyio-3.7.1.tar.gz", hash = "sha256:44a3c9aba0f5defa43261a8b3efb97891f2bd7d804e0e1f56419befa1adfc780"}, + {file = "anyio-4.15.1-py3-none-any.whl", hash = "sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101"}, + {file = "anyio-4.15.1.tar.gz", hash = "sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94"}, ] [package.dependencies] idna = ">=2.8" -sniffio = ">=1.1" +typing_extensions = {version = ">=4.16.0", markers = "python_version < \"3.15\""} [package.extras] -doc = ["Sphinx", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme (>=1.2.2)", "sphinxcontrib-jquery"] -test = ["anyio[trio]", "coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "mock (>=4) ; python_version < \"3.8\"", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17) ; python_version < \"3.12\" and platform_python_implementation == \"CPython\" and platform_system != \"Windows\""] -trio = ["trio (<0.22)"] +trio = ["trio (>=0.32.0)"] [[package]] name = "appnope" @@ -3771,18 +3769,6 @@ pyyaml = ">=6.0.3,<7.0.0" [package.extras] syn = ["synphot (>=1.7.0,<2.0.0)"] -[[package]] -name = "sniffio" -version = "1.3.0" -description = "Sniff out which async library your code is running under" -optional = false -python-versions = ">=3.7" -groups = ["main", "dev"] -files = [ - {file = "sniffio-1.3.0-py3-none-any.whl", hash = "sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384"}, - {file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"}, -] - [[package]] name = "snowballstemmer" version = "2.2.0" From 00bc44329f472902005470ea038f7a1909f0fadb Mon Sep 17 00:00:00 2001 From: "astarvienna-bot[bot]" <217837468+myciapp[bot]@users.noreply.github.com> Date: Fri, 18 Sep 2026 21:23:18 +0000 Subject: [PATCH 64/87] Bumping version from 0.12.0b3.dev1 to 0.12.0b3.dev2 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3b247f257..81028dc13 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ScopeSim" -version = "0.12.0b3.dev1" +version = "0.12.0b3.dev2" description = "Generalised telescope observation simulator" license = {text = "GPL-3.0-or-later"} authors = [ From 19450cfa85877584fb713e6a16908da625cf0d28 Mon Sep 17 00:00:00 2001 From: teutoburg Date: Sat, 19 Sep 2026 22:01:32 +0200 Subject: [PATCH 65/87] Bumping version from 0.12.0b3.dev2 to 0.12.0rc0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 81028dc13..4651c5530 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ScopeSim" -version = "0.12.0b3.dev2" +version = "0.12.0rc0" description = "Generalised telescope observation simulator" license = {text = "GPL-3.0-or-later"} authors = [ From 5cafd646f17439ada8dd887dbf763c4f6be27de5 Mon Sep 17 00:00:00 2001 From: "astarvienna-bot[bot]" <217837468+myciapp[bot]@users.noreply.github.com> Date: Sat, 19 Sep 2026 20:02:10 +0000 Subject: [PATCH 66/87] Replaced next version placeholders with 0.12.0 --- scopesim/effects/mosaic_trace_list.py | 2 +- scopesim/effects/psfs/discrete.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scopesim/effects/mosaic_trace_list.py b/scopesim/effects/mosaic_trace_list.py index 6f3083304..34e6bb66e 100644 --- a/scopesim/effects/mosaic_trace_list.py +++ b/scopesim/effects/mosaic_trace_list.py @@ -239,7 +239,7 @@ class MosaicOutputFormat(MosaicSpectralTraceList): "image": The output is a (pseudo-)detector image where the spectrum for each fibre occupies a row. - .. versionadded:: PLACEHOLDER_NEXT_RELEASE_VERSION + .. versionadded:: 0.12.0 """ required_keys = {"filename", "format"} diff --git a/scopesim/effects/psfs/discrete.py b/scopesim/effects/psfs/discrete.py index 3e85adf9c..64d852efc 100644 --- a/scopesim/effects/psfs/discrete.py +++ b/scopesim/effects/psfs/discrete.py @@ -172,7 +172,7 @@ class FieldConstantPSF(DiscretePSF): Fixed handling of background level and rounded edges to avoid visible "squares" in the image. - .. versionchanged:: PLACEHOLDER_NEXT_RELEASE_VERSION + .. versionchanged:: 0.12.0 Interpolation spline order can now be set via "!SIM.psf.interp_order". From 3ace6b3d7473a804048c85ddd623b2e0dbb6ba68 Mon Sep 17 00:00:00 2001 From: teutoburg Date: Sat, 19 Sep 2026 22:12:56 +0200 Subject: [PATCH 67/87] Bumping version from 0.12.0rc0 to 0.12.0rc1 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4651c5530..74e457216 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ScopeSim" -version = "0.12.0rc0" +version = "0.12.0rc1" description = "Generalised telescope observation simulator" license = {text = "GPL-3.0-or-later"} authors = [ From e0e8357be9a7a3573f117917bb28d664dc5daf93 Mon Sep 17 00:00:00 2001 From: "Fabian H." <73600109+teutoburg@users.noreply.github.com> Date: Mon, 21 Sep 2026 12:56:00 +0200 Subject: [PATCH 68/87] Remove no longer needed FOV logging --- scopesim/optics/fov.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/scopesim/optics/fov.py b/scopesim/optics/fov.py index 4f37d75c1..6ed2d2e74 100644 --- a/scopesim/optics/fov.py +++ b/scopesim/optics/fov.py @@ -764,9 +764,6 @@ def _make_imagefields(self, fov_waveset, bin_widths): * yield image to be added to canvas image """ for field in self._get_image_fields(): - logger.debug("2D FOV make_imagefields: field.data.sum() = %f %s", - field.data.sum(), field.bunit) - field_hdu = field.field.copy() if field.is_bunit_spatially_differential: @@ -785,9 +782,6 @@ def _make_imagefields(self, fov_waveset, bin_widths): # Rescale 2D flux weight map by integrated flux from spectrum field_hdu.data *= flux # ph s-1 - logger.info( - "2D FOV make_imagefields: field_hdu.data.mean() = %f ph/s", - field_hdu.data.mean()) yield field_hdu def _make_tablefields(self, fov_waveset, bin_widths, use_photlam=False): From ebd042b11db8b6a1767f364c8705d602c52a9c82 Mon Sep 17 00:00:00 2001 From: "Fabian H." <73600109+teutoburg@users.noreply.github.com> Date: Mon, 21 Sep 2026 13:09:00 +0200 Subject: [PATCH 69/87] Add 'prerelease' option to versioning in workflow --- .github/workflows/prepare_release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/prepare_release.yml b/.github/workflows/prepare_release.yml index 705405e4f..ba82cfc6f 100644 --- a/.github/workflows/prepare_release.yml +++ b/.github/workflows/prepare_release.yml @@ -10,6 +10,7 @@ on: - patch - minor - major + - prerelease - prepatch - preminor - premajor From b535b1384440b2664766dd525fd49d80e9868334 Mon Sep 17 00:00:00 2001 From: "astarvienna-bot[bot]" <217837468+myciapp[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:10:04 +0000 Subject: [PATCH 70/87] Bumping version from 0.12.0rc1 to 0.12.0rc2 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 74e457216..7942931bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ScopeSim" -version = "0.12.0rc1" +version = "0.12.0rc2" description = "Generalised telescope observation simulator" license = {text = "GPL-3.0-or-later"} authors = [ From 338356ce0fb5e4ed5002cef5dd5df3d74cbd6a9e Mon Sep 17 00:00:00 2001 From: "astarvienna-bot[bot]" <217837468+myciapp[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:12:51 +0000 Subject: [PATCH 71/87] Bumping version from 0.12.0rc2 to 0.12.0rc3.dev0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7942931bc..ff545c329 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ScopeSim" -version = "0.12.0rc2" +version = "0.12.0rc3.dev0" description = "Generalised telescope observation simulator" license = {text = "GPL-3.0-or-later"} authors = [ From 7f5d53ec0908a79d7a3a70084335864f88479643 Mon Sep 17 00:00:00 2001 From: "Fabian H." <73600109+teutoburg@users.noreply.github.com> Date: Wed, 23 Sep 2026 20:04:59 +0200 Subject: [PATCH 72/87] Add release notes for v0.12.0 to CHANGELOG --- CHANGELOG.md | 97 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 013c8fa8c..e2df097a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,100 @@ +# Version 0.12.0 +**2026-09-24** + +Large release with many important bug fixes centered around flux conservation and pixel shifts. + +> [!IMPORTANT] +> The minimum required Python version for this package is now **3.12** (see Dependency Changes). + +## Feature highlights +### General +- Improved handling of random seeds. + Nosie can now vary per readout (needs different `roid`) and is fully reproducible. + For instruments (and modes) with multiple detectors, the random seed varies per detector. +- PSF interpolation was made more efficient in some cases, and the interpolation order can now be specified. + +### MICADO +- Calibration mode added (see IRDB). +- IFU and offset slits implemented. + +### MOSAIC +- Tabular output format for the mIFU mode added. + +> [!WARNING] +> We expect backwards incompatibility, so we strongly recommend updating your IRDB packages as well. + +## What's Changed +### API Changes +* Reduce `OpticalTrain.write_header()` to the bare minimum by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/935 +* PSF interpolation order by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/950 +* Fully remove deprecated functionality by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/963 +* AtmoLibraryTERCurve works with parameters other than pwv by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/972 +* Make laser parameters configurable; utility function for tunable laser by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/970 +### Bugs fixed +* Catch wrong inputs to `UserCommands` by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/934 +* Dark value can be given as number without floating point by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/941 +* Remove reflectance factor from integrating sphere background by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/962 +* Fix scalar Quantity pattern matching by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/964 +* Fix flux loss, aliasing, and placement errors in image-plane resampling and overlay by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/958 +* Fix NameError in `SpectralTrace.plot` when `plot_trace_id` is used without `plot_outline` by @astronomyk in https://github.com/AstarVienna/ScopeSim/pull/981 +* Fix `XiLamImage` overwriting the cunit of its primary WCS by @astronomyk in https://github.com/AstarVienna/ScopeSim/pull/982 +* Skip empty FOVs by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1001 +* Fix transposed axes in `make_image_interpolations` by @astronomyk in https://github.com/AstarVienna/ScopeSim/pull/984 +* Fix 2D image rescaling by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1016 +* Fix yet another off-by-one bug by @astronomyk in https://github.com/AstarVienna/ScopeSim/pull/1013 +* SPEC scaling now also fixed by @janusbrink in https://github.com/AstarVienna/ScopeSim/pull/1018 +* Set INHERIT keyword and fix unit to FITS standard by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/1022 +* Fix indexing in FoV extraction by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1024 +### Changes to or addition of Effects +* MOSAIC mIFU: tabular output by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/968 +* Refactor `PSF.apply_to()` by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1002 +* Refactor `TERCurve.apply_to()` by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1004 +* Refactor `.apply_to()` in electronic effects by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1006 +* Apply throughput in-place for speed by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/1017 +* `AutoExposure` to take dark current into account by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/1031 +* Improve randomness, refactor electronic effects by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1030 +* Define slit transforms for image slicer IFUs and offset slits by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/922 +### New Features or Improvements +* Use optimal inverse transform for rectification by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/961 +* Skip PSF convolution for exactly-uniform fields by @astronomyk in https://github.com/AstarVienna/ScopeSim/pull/973 +* More useful error when download fails by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/990 +* Allow readout ID, fix Path support by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1020 +* Resolve random seed in `.observe()` by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1021 +### Dependency Changes +* Bump the dev-dependencies group with 3 updates by @dependabot[bot] in https://github.com/AstarVienna/ScopeSim/pull/940 +* Switch from httpx to httpxyz by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/924 +* Bump various dependencies by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/946 +* Bump astar-utils from 0.5.3 to 0.6.0 by @dependabot[bot] in https://github.com/AstarVienna/ScopeSim/pull/951 +* Bump actions/checkout from 6 to 7 by @dependabot[bot] in https://github.com/AstarVienna/ScopeSim/pull/947 +* Bump various dependencies by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/954 +* Bump actions/setup-python from 6 to 7 by @dependabot[bot] in https://github.com/AstarVienna/ScopeSim/pull/960 +* Drop support for Python 3.10 and 3.11 by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/969 +* Bump matplotlib and skycalc-ipy by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/977 +* Bump tornado from 6.5.7 to 6.5.8 by @dependabot[bot] in https://github.com/AstarVienna/ScopeSim/pull/1011 +* Bump tqdm from 4.68.4 to 4.70.0 by @dependabot[bot] in https://github.com/AstarVienna/ScopeSim/pull/1009 +* Bump the dev-dependencies group across 1 directory with 4 updates by @dependabot[bot] in https://github.com/AstarVienna/ScopeSim/pull/1008 +* Bump soupsieve from 2.8.4 to 2.9 by @dependabot[bot] in https://github.com/AstarVienna/ScopeSim/pull/1032 +* Bump anyio from 3.7.1 to 4.14.2 by @dependabot[bot] in https://github.com/AstarVienna/ScopeSim/pull/1033 +### Other Changes +* Add prepare-release workflow from elsewhere by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/971 +* Remove obsolete logic to deal with unversioned versions by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/979 +* Make `Transform2D` per-call transform overrides non-persistent by @astronomyk in https://github.com/AstarVienna/ScopeSim/pull/983 +* Cache the detector layout file in `MetisLMSSpectralTraceList` to avoid reading the file N*28 times for every `OpticalTrain` usage by @astronomyk in https://github.com/AstarVienna/ScopeSim/pull/987 +* Improve handling of spectral trace mapping and rectification by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/999 +* Cleanup PSF by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1003 +* Use correct NumPy type hints by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1005 +* Mark test_python_image_coords as webtest, because face is downloaded. by @hugobuddel in https://github.com/AstarVienna/ScopeSim/pull/1012 +* Cleanup in `TER_curve_utils` etc. by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1014 +* Add `Detector.det_id` convenience property by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1015 +* Formatting and cleanup by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1019 +* Set mpl backend for tests by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1026 +* Skip all tests in test_rst_utils.py by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1025 +* Remove unused `fov_grid()` methods by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1027 +* SimCADO corpse removal by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1029 + +**Full Changelog**: https://github.com/AstarVienna/ScopeSim/compare/v0.11.4...v0.12.0 + + # Version 0.11.4 **2026-05-20** From bd9ddcf7bb8ccc197132409e12d2df8d88f98049 Mon Sep 17 00:00:00 2001 From: oczoske Date: Thu, 24 Sep 2026 09:01:38 +0200 Subject: [PATCH 73/87] Update CHANGELOG.md --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2df097a6..d6b92f39f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ Large release with many important bug fixes centered around flux conservation an ## Feature highlights ### General - Improved handling of random seeds. - Nosie can now vary per readout (needs different `roid`) and is fully reproducible. + Noise can now vary per readout (needs different `roid`) and is fully reproducible. For instruments (and modes) with multiple detectors, the random seed varies per detector. - PSF interpolation was made more efficient in some cases, and the interpolation order can now be specified. @@ -29,7 +29,6 @@ Large release with many important bug fixes centered around flux conservation an * PSF interpolation order by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/950 * Fully remove deprecated functionality by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/963 * AtmoLibraryTERCurve works with parameters other than pwv by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/972 -* Make laser parameters configurable; utility function for tunable laser by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/970 ### Bugs fixed * Catch wrong inputs to `UserCommands` by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/934 * Dark value can be given as number without floating point by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/941 @@ -47,6 +46,7 @@ Large release with many important bug fixes centered around flux conservation an * Fix indexing in FoV extraction by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1024 ### Changes to or addition of Effects * MOSAIC mIFU: tabular output by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/968 +* METIS WCU: Make laser parameters configurable; utility function for tunable laser by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/970 * Refactor `PSF.apply_to()` by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1002 * Refactor `TERCurve.apply_to()` by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1004 * Refactor `.apply_to()` in electronic effects by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1006 From 80c93cbfb7054efbfb6ab2ef6d2ed18cb5574a3f Mon Sep 17 00:00:00 2001 From: "astarvienna-bot[bot]" <217837468+myciapp[bot]@users.noreply.github.com> Date: Thu, 24 Sep 2026 12:34:47 +0000 Subject: [PATCH 74/87] Bumping version from 0.12.0rc3.dev0 to 0.12.0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ff545c329..4b22f7aa7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ScopeSim" -version = "0.12.0rc3.dev0" +version = "0.12.0" description = "Generalised telescope observation simulator" license = {text = "GPL-3.0-or-later"} authors = [ From 4ed185bd98606122aa5d1bdc4552c76ca1f0ee84 Mon Sep 17 00:00:00 2001 From: "astarvienna-bot[bot]" <217837468+myciapp[bot]@users.noreply.github.com> Date: Thu, 24 Sep 2026 12:35:36 +0000 Subject: [PATCH 75/87] Bumping version from 0.12.0 to 0.12.1.dev0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4b22f7aa7..61752913c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ScopeSim" -version = "0.12.0" +version = "0.12.1.dev0" description = "Generalised telescope observation simulator" license = {text = "GPL-3.0-or-later"} authors = [ From a8f524fb33a147d8373223db1bce754e980e92fd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 15:44:47 +0000 Subject: [PATCH 76/87] Bump tqdm from 4.70.0 to 4.70.1 in the main-dependencies group Bumps the main-dependencies group with 1 update: [tqdm](https://github.com/tqdm/tqdm). Updates `tqdm` from 4.70.0 to 4.70.1 - [Release notes](https://github.com/tqdm/tqdm/releases) - [Commits](https://github.com/tqdm/tqdm/compare/v4.70.0...v4.70.1) --- updated-dependencies: - dependency-name: tqdm dependency-version: 4.70.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: main-dependencies ... Signed-off-by: dependabot[bot] --- poetry.lock | 26 +++++++++++++------------- pyproject.toml | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/poetry.lock b/poetry.lock index eaa19d9ef..284ea6bed 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.5.1 and should not be changed by hand. [[package]] name = "accessible-pygments" @@ -1247,7 +1247,7 @@ comm = ">=0.1.1" debugpy = ">=1.6.5" ipython = ">=7.23.1" jupyter-client = ">=8.9.0" -jupyter-core = ">=5.1,<6.0.dev0 || >=6.1.dev0" +jupyter-core = ">=5.1,<6.0 || >=6.1.dev0" matplotlib-inline = ">=0.1" nest-asyncio2 = ">=1.7.0" packaging = ">=22" @@ -1579,7 +1579,7 @@ files = [ ipykernel = ">=6.14" ipython = "*" jupyter-client = ">=7.0.0" -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +jupyter-core = ">=4.12,<5.0 || >=5.1.dev0" prompt-toolkit = ">=3.0.30" pygments = "*" pyzmq = ">=17" @@ -1668,14 +1668,14 @@ anyio = ">=3.1.0" argon2-cffi = ">=21.1" jinja2 = ">=3.0.3" jupyter-client = ">=7.4.4" -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +jupyter-core = ">=4.12,<5.0 || >=5.1.dev0" jupyter-events = ">=0.11.0" jupyter-server-terminals = ">=0.4.4" nbconvert = ">=6.4.4" nbformat = ">=5.3.0" packaging = ">=22.0" prometheus-client = ">=0.9" -pywinpty = {version = ">=2.0.1,<3.0.4 || >3.0.4", markers = "os_name == \"nt\""} +pywinpty = {version = ">=2.0.1,!=3.0.4", markers = "os_name == \"nt\""} pyzmq = ">=24" send2trash = ">=1.8.2" terminado = ">=0.8.3" @@ -1746,7 +1746,7 @@ files = [ [package.dependencies] async-lru = ">=1.0.0" httpx = ">=0.25.0,<1" -ipykernel = ">=6.5.0,<6.30.0 || >6.30.0" +ipykernel = ">=6.5.0,!=6.30.0" jinja2 = ">=3.0.3" jupyter-builder = ">=1.0.2" jupyter-core = "*" @@ -2482,7 +2482,7 @@ files = [ [package.dependencies] fastjsonschema = ">=2.15" jsonschema = ">=2.6" -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +jupyter-core = ">=4.12,<5.0 || >=5.1.dev0" traitlets = ">=5.1" [package.extras] @@ -2504,9 +2504,9 @@ files = [ [package.dependencies] docutils = ">=0.18.1" jinja2 = "*" -nbconvert = ">=5.3,<5.4 || >5.4" +nbconvert = ">=5.3,!=5.4" nbformat = "*" -sphinx = ">=1.8,<8.2.0 || >8.2.0,<8.2.1 || >8.2.1" +sphinx = ">=1.8,!=8.2.0,!=8.2.1" traitlets = ">=5" [[package]] @@ -4232,14 +4232,14 @@ files = [ [[package]] name = "tqdm" -version = "4.70.0" +version = "4.70.1" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "tqdm-4.70.0-py3-none-any.whl", hash = "sha256:7f585706bfddbdebf89daac705b2dfcc16890130727d3197ca62c732b4310953"}, - {file = "tqdm-4.70.0.tar.gz", hash = "sha256:55b0b0dbd97462d06ebee91e4dac24ed4d4702be82b24f07e6c1d27e08cea220"}, + {file = "tqdm-4.70.1-py3-none-any.whl", hash = "sha256:c293e525e6fef9c20e8728fd4612df02a0aa31bb5fe91ecd93e123b1b7bffa73"}, + {file = "tqdm-4.70.1.tar.gz", hash = "sha256:cefd0eca11b2a37a3aee776544d4f4ae913f02688135b5556b8788dfa474afc4"}, ] [package.dependencies] @@ -4412,4 +4412,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">=3.12,<3.15" -content-hash = "ed6ef2a6cb8814ee6ab96c983711908f3102cbfdeb28136410bf675ad2da745c" +content-hash = "7c06b18379d2badb1c8603f9d29d48487af53f925335fbc9a95aa031614ebc6f" diff --git a/pyproject.toml b/pyproject.toml index 61752913c..c3865c647 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ dependencies = [ "lxml (>=6.1.1,<7.0.0)", "pyyaml (>=6.0.3,<7.0.0)", "more-itertools (>=10.2.0,<11.0.0)", - "tqdm (>=4.70.0,<5.0.0)", + "tqdm (>=4.70.1,<5.0.0)", "packaging (>=26.0,<28.0)", "synphot (>=1.7.0,<2.0.0)", From 720022edef580f14797a93d1250f162d75416c55 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 15:46:20 +0000 Subject: [PATCH 77/87] Bump sphinxcontrib-mermaid Bumps the dev-dependencies group with 1 update: [sphinxcontrib-mermaid](https://github.com/mgaitan/sphinxcontrib-mermaid). Updates `sphinxcontrib-mermaid` from 2.1.0 to 2.1.1 - [Changelog](https://github.com/mgaitan/sphinxcontrib-mermaid/blob/master/CHANGELOG.md) - [Commits](https://github.com/mgaitan/sphinxcontrib-mermaid/compare/2.1.0...2.1.1) --- updated-dependencies: - dependency-name: sphinxcontrib-mermaid dependency-version: 2.1.1 dependency-type: direct:development update-type: version-update:semver-patch dependency-group: dev-dependencies ... Signed-off-by: dependabot[bot] --- poetry.lock | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/poetry.lock b/poetry.lock index eaa19d9ef..babc24283 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.5.1 and should not be changed by hand. [[package]] name = "accessible-pygments" @@ -1247,7 +1247,7 @@ comm = ">=0.1.1" debugpy = ">=1.6.5" ipython = ">=7.23.1" jupyter-client = ">=8.9.0" -jupyter-core = ">=5.1,<6.0.dev0 || >=6.1.dev0" +jupyter-core = ">=5.1,<6.0 || >=6.1.dev0" matplotlib-inline = ">=0.1" nest-asyncio2 = ">=1.7.0" packaging = ">=22" @@ -1579,7 +1579,7 @@ files = [ ipykernel = ">=6.14" ipython = "*" jupyter-client = ">=7.0.0" -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +jupyter-core = ">=4.12,<5.0 || >=5.1.dev0" prompt-toolkit = ">=3.0.30" pygments = "*" pyzmq = ">=17" @@ -1668,14 +1668,14 @@ anyio = ">=3.1.0" argon2-cffi = ">=21.1" jinja2 = ">=3.0.3" jupyter-client = ">=7.4.4" -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +jupyter-core = ">=4.12,<5.0 || >=5.1.dev0" jupyter-events = ">=0.11.0" jupyter-server-terminals = ">=0.4.4" nbconvert = ">=6.4.4" nbformat = ">=5.3.0" packaging = ">=22.0" prometheus-client = ">=0.9" -pywinpty = {version = ">=2.0.1,<3.0.4 || >3.0.4", markers = "os_name == \"nt\""} +pywinpty = {version = ">=2.0.1,!=3.0.4", markers = "os_name == \"nt\""} pyzmq = ">=24" send2trash = ">=1.8.2" terminado = ">=0.8.3" @@ -1746,7 +1746,7 @@ files = [ [package.dependencies] async-lru = ">=1.0.0" httpx = ">=0.25.0,<1" -ipykernel = ">=6.5.0,<6.30.0 || >6.30.0" +ipykernel = ">=6.5.0,!=6.30.0" jinja2 = ">=3.0.3" jupyter-builder = ">=1.0.2" jupyter-core = "*" @@ -2482,7 +2482,7 @@ files = [ [package.dependencies] fastjsonschema = ">=2.15" jsonschema = ">=2.6" -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +jupyter-core = ">=4.12,<5.0 || >=5.1.dev0" traitlets = ">=5.1" [package.extras] @@ -2504,9 +2504,9 @@ files = [ [package.dependencies] docutils = ">=0.18.1" jinja2 = "*" -nbconvert = ">=5.3,<5.4 || >5.4" +nbconvert = ">=5.3,!=5.4" nbformat = "*" -sphinx = ">=1.8,<8.2.0 || >8.2.0,<8.2.1 || >8.2.1" +sphinx = ">=1.8,!=8.2.0,!=8.2.1" traitlets = ">=5" [[package]] @@ -3948,14 +3948,14 @@ test = ["flake8", "mypy", "pytest"] [[package]] name = "sphinxcontrib-mermaid" -version = "2.1.0" +version = "2.1.1" description = "Mermaid diagrams in your Sphinx-powered docs" optional = false python-versions = ">=3.10" groups = ["docs"] files = [ - {file = "sphinxcontrib_mermaid-2.1.0-py3-none-any.whl", hash = "sha256:417cd144ec4b28852f46ba653f02ce8e538881c812111671a4c30344e87f2112"}, - {file = "sphinxcontrib_mermaid-2.1.0.tar.gz", hash = "sha256:13c5f9ac395cb6abf403eca34e228dc9fb3a30c9d960dbf3e40e9a8cef969549"}, + {file = "sphinxcontrib_mermaid-2.1.1-py3-none-any.whl", hash = "sha256:3b9ac1a8e0b5154dec3c0acf9b5c54dd9ba58540e416f20858c300814c25648f"}, + {file = "sphinxcontrib_mermaid-2.1.1.tar.gz", hash = "sha256:f8d7f1b2f0ad69b7ebdd5cbf532fb4ee107893b5373bc2d0e9634089c0b2cfee"}, ] [package.dependencies] From 99c06335070c99201528b75b52b8a39b64b86a0d Mon Sep 17 00:00:00 2001 From: "astarvienna-bot[bot]" <217837468+myciapp[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 19:25:45 +0000 Subject: [PATCH 78/87] Bumping version from 0.12.1.dev0 to 0.12.1.dev1 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 61752913c..141acea6d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ScopeSim" -version = "0.12.1.dev0" +version = "0.12.1.dev1" description = "Generalised telescope observation simulator" license = {text = "GPL-3.0-or-later"} authors = [ From a256ed3d7709e6aafa1bca305a00c96e5b175cf5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 19:35:25 +0000 Subject: [PATCH 79/87] Bump urllib3 from 2.7.0 to 2.8.0 Bumps [urllib3](https://github.com/urllib3/urllib3) from 2.7.0 to 2.8.0. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/2.7.0...2.8.0) --- updated-dependencies: - dependency-name: urllib3 dependency-version: 2.8.0 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index babc24283..5eab9023c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -4308,14 +4308,14 @@ dev = ["flake8", "flake8-annotations", "flake8-bandit", "flake8-bugbear", "flake [[package]] name = "urllib3" -version = "2.7.0" +version = "2.8.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.10" groups = ["main", "dev", "docs"] files = [ - {file = "urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897"}, - {file = "urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c"}, + {file = "urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3"}, + {file = "urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63"}, ] [package.extras] From 4cda19bd343a15b0c19d34f1a905233cb55bd7ea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 19:39:34 +0000 Subject: [PATCH 80/87] Bump tornado from 6.5.8 to 6.5.9 Bumps [tornado](https://github.com/tornadoweb/tornado) from 6.5.8 to 6.5.9. - [Changelog](https://github.com/tornadoweb/tornado/blob/master/docs/releases.rst) - [Commits](https://github.com/tornadoweb/tornado/compare/v6.5.8...v6.5.9) --- updated-dependencies: - dependency-name: tornado dependency-version: 6.5.9 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- poetry.lock | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/poetry.lock b/poetry.lock index babc24283..1ad9fe772 100644 --- a/poetry.lock +++ b/poetry.lock @@ -4212,22 +4212,22 @@ test = ["flake8", "isort", "pytest"] [[package]] name = "tornado" -version = "6.5.8" +version = "6.5.9" description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." optional = false python-versions = ">=3.9" groups = ["dev", "docs", "test"] files = [ - {file = "tornado-6.5.8-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:cc6aa787d7cfab7c3d35189dc7a56fbd2399a569624c730c6b55b3d6531d0403"}, - {file = "tornado-6.5.8-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9715b5eb79735b2bcd454ce216a9275b7c0470e64ea1bf5742f78b2f72b26eeb"}, - {file = "tornado-6.5.8-cp39-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:547d63f450d570c14fe0e8db2cfb14c9bbd1c2503b4a6612586267955aa47b58"}, - {file = "tornado-6.5.8-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7e2360a0ffbe145eca8af0b19cb7203d79b1a98dd4cccdd6b368f6f49c2e3808"}, - {file = "tornado-6.5.8-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:5d242290bdf7ab3151bc1065fdd75c0dcc21cbc7b49f22a4c56329c2d6566d22"}, - {file = "tornado-6.5.8-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:7b94ff0e128fe0542f3bd331fb44d06260fc4ac16881545159f34ef08aad4195"}, - {file = "tornado-6.5.8-cp39-abi3-win32.whl", hash = "sha256:67832909c4779c64942380cb5f044a5c6163d00831472d80e25e115de9917836"}, - {file = "tornado-6.5.8-cp39-abi3-win_amd64.whl", hash = "sha256:11881db6b7c168494be2c2d12e65931451bdf7ee718535418ae1d8855dd5a0ee"}, - {file = "tornado-6.5.8-cp39-abi3-win_arm64.whl", hash = "sha256:68a7468c7e289f8514d7d664101753903217eff1bb6822c6b5994a0b5f5bcb26"}, - {file = "tornado-6.5.8.tar.gz", hash = "sha256:9452e1b208a8bd771e2cb1f2ff564985b9b214bdebbe622793e1799e0a6bd23f"}, + {file = "tornado-6.5.9-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:dea3bee433b73d6312d993ce89ad5f2b8d3ed9fde56e79ba5342b9edb78d342a"}, + {file = "tornado-6.5.9-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:22bed49cdb55292d1d58f46008ba2cc3020e5b84995ef6ae749676c8b5309514"}, + {file = "tornado-6.5.9-cp39-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0cc0876a58eee9143476f24901b1b8eb4d52443283ade2b6bca650dd1da04f84"}, + {file = "tornado-6.5.9-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc0d703cc7ec92b47ebfe236c97cd765c1f2e45744a2e156e6677e54c948274d"}, + {file = "tornado-6.5.9-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:93f9ab0226ef625b94cacb7981a5af104f1735ea02f92282abe5c389109fb31d"}, + {file = "tornado-6.5.9-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5801bbf94dfde8d21087b6aa0b17f3a5dab6110dfe44e7e4492df5de393ac15b"}, + {file = "tornado-6.5.9-cp39-abi3-win32.whl", hash = "sha256:dca9f377e80efe5dd4d52a2776b67492a81a39b2d931de169b6b36d0c1da444c"}, + {file = "tornado-6.5.9-cp39-abi3-win_amd64.whl", hash = "sha256:f810730ac71eadf009e6e5cb6d534d096887310f5649cff19d7e0d19c0bf4ee7"}, + {file = "tornado-6.5.9-cp39-abi3-win_arm64.whl", hash = "sha256:b5f4d92798337260c6d9f9f0e400c7539dd5e55cf93479a3f8508c9aa09c2b3b"}, + {file = "tornado-6.5.9.tar.gz", hash = "sha256:4d868544ebdf2fc155239a74397f65ebfea9b4d3635296c96b5dfb0701c354f5"}, ] [[package]] From 3fe382130f1a65f14e4d5e43e7fe6d6bdecb9497 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 19:42:24 +0000 Subject: [PATCH 81/87] Bump notebook from 7.6.1 to 7.6.3 Bumps [notebook](https://github.com/jupyter/notebook) from 7.6.1 to 7.6.3. - [Release notes](https://github.com/jupyter/notebook/releases) - [Changelog](https://github.com/jupyter/notebook/blob/@jupyter-notebook/tree@7.6.3/CHANGELOG.md) - [Commits](https://github.com/jupyter/notebook/compare/@jupyter-notebook/tree@7.6.1...@jupyter-notebook/tree@7.6.3) --- updated-dependencies: - dependency-name: notebook dependency-version: 7.6.3 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- poetry.lock | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/poetry.lock b/poetry.lock index babc24283..3baf85d80 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1733,14 +1733,14 @@ test = ["bash-kernel", "pytest"] [[package]] name = "jupyterlab" -version = "4.6.3" +version = "4.6.4" description = "JupyterLab computational environment" optional = false python-versions = ">=3.10" groups = ["dev"] files = [ - {file = "jupyterlab-4.6.3-py3-none-any.whl", hash = "sha256:0a1ebc6567186f1eabd99536e94df7ed9e96d1e7c5ddf3e4406ae16e88abacb7"}, - {file = "jupyterlab-4.6.3.tar.gz", hash = "sha256:2e3db6e3a12495ebd188276e985bf5ac502fbde3d1e8628819920210008de498"}, + {file = "jupyterlab-4.6.4-py3-none-any.whl", hash = "sha256:15b13f991d3985129c797eb84d9949eeb8b6615e14b444868e642411f2c418b2"}, + {file = "jupyterlab-4.6.4.tar.gz", hash = "sha256:404f49b081819378524886c9db66dba57a5565981eff885830df1baba3a17df5"}, ] [package.dependencies] @@ -2535,20 +2535,20 @@ files = [ [[package]] name = "notebook" -version = "7.6.1" +version = "7.6.3" description = "Jupyter Notebook - A web-based notebook environment for interactive computing" optional = false python-versions = ">=3.10" groups = ["dev"] files = [ - {file = "notebook-7.6.1-py3-none-any.whl", hash = "sha256:6ea1e4c926f0dc490444ddcc335797a3dacda470a805d01031872fb119988ba2"}, - {file = "notebook-7.6.1.tar.gz", hash = "sha256:0b45fd1010668dd4808c40914d957706dbf044a677d28764dc881b74dfcede82"}, + {file = "notebook-7.6.3-py3-none-any.whl", hash = "sha256:ad7e0eb765fba836cd4a2ab0c7a3a26cde1d91665fbf6f533b6ae7b2de6d88d2"}, + {file = "notebook-7.6.3.tar.gz", hash = "sha256:e2c08e469c0ae20bb0b3214f0ab77e79653317a2f8e5b34c10361c66874a5b50"}, ] [package.dependencies] jupyter-builder = ">=1.0.2,<2" jupyter-server = ">=2.19.0,<3" -jupyterlab = ">=4.6.2,<4.7" +jupyterlab = ">=4.6.4,<4.7" jupyterlab-server = ">=2.28.0,<3" notebook-shim = ">=0.2,<0.3" tornado = ">=6.2.0" From dea4b10d5e78d15841b7f3c54740935cb5908d1a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 19:42:54 +0000 Subject: [PATCH 82/87] Bump jupyterlab from 4.6.3 to 4.6.4 Bumps [jupyterlab](https://github.com/jupyterlab/jupyterlab) from 4.6.3 to 4.6.4. - [Release notes](https://github.com/jupyterlab/jupyterlab/releases) - [Changelog](https://github.com/jupyterlab/jupyterlab/blob/main/RELEASE.md) - [Commits](https://github.com/jupyterlab/jupyterlab/compare/@jupyterlab/lsp@4.6.3...@jupyterlab/lsp@4.6.4) --- updated-dependencies: - dependency-name: jupyterlab dependency-version: 4.6.4 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index babc24283..e7db5b54e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1733,14 +1733,14 @@ test = ["bash-kernel", "pytest"] [[package]] name = "jupyterlab" -version = "4.6.3" +version = "4.6.4" description = "JupyterLab computational environment" optional = false python-versions = ">=3.10" groups = ["dev"] files = [ - {file = "jupyterlab-4.6.3-py3-none-any.whl", hash = "sha256:0a1ebc6567186f1eabd99536e94df7ed9e96d1e7c5ddf3e4406ae16e88abacb7"}, - {file = "jupyterlab-4.6.3.tar.gz", hash = "sha256:2e3db6e3a12495ebd188276e985bf5ac502fbde3d1e8628819920210008de498"}, + {file = "jupyterlab-4.6.4-py3-none-any.whl", hash = "sha256:15b13f991d3985129c797eb84d9949eeb8b6615e14b444868e642411f2c418b2"}, + {file = "jupyterlab-4.6.4.tar.gz", hash = "sha256:404f49b081819378524886c9db66dba57a5565981eff885830df1baba3a17df5"}, ] [package.dependencies] From 43228458cb1ce971d8434fde11ea6f9b88f49f2f Mon Sep 17 00:00:00 2001 From: "astarvienna-bot[bot]" <217837468+myciapp[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 19:56:17 +0000 Subject: [PATCH 83/87] Bumping version from 0.12.1.dev1 to 0.12.1.dev2 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 141acea6d..f31788528 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ScopeSim" -version = "0.12.1.dev1" +version = "0.12.1.dev2" description = "Generalised telescope observation simulator" license = {text = "GPL-3.0-or-later"} authors = [ From 8200eaa3e2de285469f12c60c4db36bfa24cbbf9 Mon Sep 17 00:00:00 2001 From: "astarvienna-bot[bot]" <217837468+myciapp[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 22:37:18 +0000 Subject: [PATCH 84/87] Bumping version from 0.12.1.dev2 to 0.12.1.dev3 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f31788528..9189bd408 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ScopeSim" -version = "0.12.1.dev2" +version = "0.12.1.dev3" description = "Generalised telescope observation simulator" license = {text = "GPL-3.0-or-later"} authors = [ From 91d452fb55494b865db6fd018a09eae33c68e251 Mon Sep 17 00:00:00 2001 From: "astarvienna-bot[bot]" <217837468+myciapp[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 22:42:58 +0000 Subject: [PATCH 85/87] Bumping version from 0.12.1.dev3 to 0.12.1.dev4 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b51c3ba4e..efa2773ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ScopeSim" -version = "0.12.1.dev3" +version = "0.12.1.dev4" description = "Generalised telescope observation simulator" license = {text = "GPL-3.0-or-later"} authors = [ From d8f2c096beb84e47e5d12295678f2c703f1736e2 Mon Sep 17 00:00:00 2001 From: "astarvienna-bot[bot]" <217837468+myciapp[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 22:50:44 +0000 Subject: [PATCH 86/87] Bumping version from 0.12.1.dev4 to 0.12.1.dev5 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index efa2773ca..ff0d93783 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ScopeSim" -version = "0.12.1.dev4" +version = "0.12.1.dev5" description = "Generalised telescope observation simulator" license = {text = "GPL-3.0-or-later"} authors = [ From 2d24516855e35bdffa5acd0485dbacd41855c727 Mon Sep 17 00:00:00 2001 From: "astarvienna-bot[bot]" <217837468+myciapp[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 23:51:20 +0000 Subject: [PATCH 87/87] Bumping version from 0.12.1.dev5 to 0.12.1.dev6 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ff0d93783..f6ecd946f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ScopeSim" -version = "0.12.1.dev5" +version = "0.12.1.dev6" description = "Generalised telescope observation simulator" license = {text = "GPL-3.0-or-later"} authors = [