diff --git a/.github/workflows/prepare_release.yml b/.github/workflows/prepare_release.yml index 705405e4f..ba82cfc6f 100644 --- a/.github/workflows/prepare_release.yml +++ b/.github/workflows/prepare_release.yml @@ -10,6 +10,7 @@ on: - patch - minor - major + - prerelease - prepatch - preminor - premajor diff --git a/CHANGELOG.md b/CHANGELOG.md index 013c8fa8c..d6b92f39f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,100 @@ +# Version 0.12.0 +**2026-09-24** + +Large release with many important bug fixes centered around flux conservation and pixel shifts. + +> [!IMPORTANT] +> The minimum required Python version for this package is now **3.12** (see Dependency Changes). + +## Feature highlights +### General +- Improved handling of random seeds. + Noise can now vary per readout (needs different `roid`) and is fully reproducible. + For instruments (and modes) with multiple detectors, the random seed varies per detector. +- PSF interpolation was made more efficient in some cases, and the interpolation order can now be specified. + +### MICADO +- Calibration mode added (see IRDB). +- IFU and offset slits implemented. + +### MOSAIC +- Tabular output format for the mIFU mode added. + +> [!WARNING] +> We expect backwards incompatibility, so we strongly recommend updating your IRDB packages as well. + +## What's Changed +### API Changes +* Reduce `OpticalTrain.write_header()` to the bare minimum by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/935 +* PSF interpolation order by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/950 +* Fully remove deprecated functionality by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/963 +* AtmoLibraryTERCurve works with parameters other than pwv by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/972 +### Bugs fixed +* Catch wrong inputs to `UserCommands` by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/934 +* Dark value can be given as number without floating point by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/941 +* Remove reflectance factor from integrating sphere background by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/962 +* Fix scalar Quantity pattern matching by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/964 +* Fix flux loss, aliasing, and placement errors in image-plane resampling and overlay by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/958 +* Fix NameError in `SpectralTrace.plot` when `plot_trace_id` is used without `plot_outline` by @astronomyk in https://github.com/AstarVienna/ScopeSim/pull/981 +* Fix `XiLamImage` overwriting the cunit of its primary WCS by @astronomyk in https://github.com/AstarVienna/ScopeSim/pull/982 +* Skip empty FOVs by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1001 +* Fix transposed axes in `make_image_interpolations` by @astronomyk in https://github.com/AstarVienna/ScopeSim/pull/984 +* Fix 2D image rescaling by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1016 +* Fix yet another off-by-one bug by @astronomyk in https://github.com/AstarVienna/ScopeSim/pull/1013 +* SPEC scaling now also fixed by @janusbrink in https://github.com/AstarVienna/ScopeSim/pull/1018 +* Set INHERIT keyword and fix unit to FITS standard by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/1022 +* Fix indexing in FoV extraction by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1024 +### Changes to or addition of Effects +* MOSAIC mIFU: tabular output by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/968 +* METIS WCU: Make laser parameters configurable; utility function for tunable laser by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/970 +* Refactor `PSF.apply_to()` by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1002 +* Refactor `TERCurve.apply_to()` by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1004 +* Refactor `.apply_to()` in electronic effects by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1006 +* Apply throughput in-place for speed by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/1017 +* `AutoExposure` to take dark current into account by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/1031 +* Improve randomness, refactor electronic effects by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1030 +* Define slit transforms for image slicer IFUs and offset slits by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/922 +### New Features or Improvements +* Use optimal inverse transform for rectification by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/961 +* Skip PSF convolution for exactly-uniform fields by @astronomyk in https://github.com/AstarVienna/ScopeSim/pull/973 +* More useful error when download fails by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/990 +* Allow readout ID, fix Path support by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1020 +* Resolve random seed in `.observe()` by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1021 +### Dependency Changes +* Bump the dev-dependencies group with 3 updates by @dependabot[bot] in https://github.com/AstarVienna/ScopeSim/pull/940 +* Switch from httpx to httpxyz by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/924 +* Bump various dependencies by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/946 +* Bump astar-utils from 0.5.3 to 0.6.0 by @dependabot[bot] in https://github.com/AstarVienna/ScopeSim/pull/951 +* Bump actions/checkout from 6 to 7 by @dependabot[bot] in https://github.com/AstarVienna/ScopeSim/pull/947 +* Bump various dependencies by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/954 +* Bump actions/setup-python from 6 to 7 by @dependabot[bot] in https://github.com/AstarVienna/ScopeSim/pull/960 +* Drop support for Python 3.10 and 3.11 by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/969 +* Bump matplotlib and skycalc-ipy by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/977 +* Bump tornado from 6.5.7 to 6.5.8 by @dependabot[bot] in https://github.com/AstarVienna/ScopeSim/pull/1011 +* Bump tqdm from 4.68.4 to 4.70.0 by @dependabot[bot] in https://github.com/AstarVienna/ScopeSim/pull/1009 +* Bump the dev-dependencies group across 1 directory with 4 updates by @dependabot[bot] in https://github.com/AstarVienna/ScopeSim/pull/1008 +* Bump soupsieve from 2.8.4 to 2.9 by @dependabot[bot] in https://github.com/AstarVienna/ScopeSim/pull/1032 +* Bump anyio from 3.7.1 to 4.14.2 by @dependabot[bot] in https://github.com/AstarVienna/ScopeSim/pull/1033 +### Other Changes +* Add prepare-release workflow from elsewhere by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/971 +* Remove obsolete logic to deal with unversioned versions by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/979 +* Make `Transform2D` per-call transform overrides non-persistent by @astronomyk in https://github.com/AstarVienna/ScopeSim/pull/983 +* Cache the detector layout file in `MetisLMSSpectralTraceList` to avoid reading the file N*28 times for every `OpticalTrain` usage by @astronomyk in https://github.com/AstarVienna/ScopeSim/pull/987 +* Improve handling of spectral trace mapping and rectification by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/999 +* Cleanup PSF by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1003 +* Use correct NumPy type hints by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1005 +* Mark test_python_image_coords as webtest, because face is downloaded. by @hugobuddel in https://github.com/AstarVienna/ScopeSim/pull/1012 +* Cleanup in `TER_curve_utils` etc. by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1014 +* Add `Detector.det_id` convenience property by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1015 +* Formatting and cleanup by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1019 +* Set mpl backend for tests by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1026 +* Skip all tests in test_rst_utils.py by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1025 +* Remove unused `fov_grid()` methods by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1027 +* SimCADO corpse removal by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1029 + +**Full Changelog**: https://github.com/AstarVienna/ScopeSim/compare/v0.11.4...v0.12.0 + + # Version 0.11.4 **2026-05-20** diff --git a/docs/source/_ext/scopesim_sphinx_ext.py b/docs/source/_ext/scopesim_sphinx_ext.py index 4ab146d1e..ea7957f8f 100644 --- a/docs/source/_ext/scopesim_sphinx_ext.py +++ b/docs/source/_ext/scopesim_sphinx_ext.py @@ -1,6 +1,13 @@ +# -*- coding: utf-8 -*- + +# TODO: What does this (rather clunky) Sphinx extension do that isn't covered +# by autodoc/autosummary?? If it's redundant, just get rid of it tbh. + import os import numpy as np -from scopesim.effects import effects_utils as eu + +# This function is only used here. If this extension is removed, rm the func. +from scopesim.effects.effects_utils import scopesim_effect_classes def setup(app): @@ -8,7 +15,7 @@ def setup(app): "../effects_docstrings")) os.makedirs(output_dir, exist_ok=True) - efs_dict = eu.scopesim_effect_classes() + efs_dict = scopesim_effect_classes() eff_types = np.unique([eff.split(".")[0] for eff in efs_dict]) eff_type_strs = {eff_type: f"{eff_type}\n{'='*len(eff_type)}\n\n" for eff_type in eff_types} @@ -37,7 +44,7 @@ def setup(app): :glob: * - + """ with open(os.path.join(output_dir, "index.rst"), "w") as f: diff --git a/poetry.lock b/poetry.lock index ebb77fc36..a6af398fc 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.5.1 and should not be changed by hand. [[package]] name = "accessible-pygments" @@ -50,24 +50,22 @@ numpy = ">=2.2.6,<3.0.0" [[package]] name = "anyio" -version = "3.7.1" -description = "High level compatibility layer for multiple asynchronous event loop implementations" +version = "4.15.1" +description = "High-level concurrency and networking framework on top of asyncio or Trio" optional = false -python-versions = ">=3.7" +python-versions = ">=3.10" groups = ["main", "dev"] files = [ - {file = "anyio-3.7.1-py3-none-any.whl", hash = "sha256:91dee416e570e92c64041bd18b900d1d6fa78dff7048769ce5ac5ddad004fbb5"}, - {file = "anyio-3.7.1.tar.gz", hash = "sha256:44a3c9aba0f5defa43261a8b3efb97891f2bd7d804e0e1f56419befa1adfc780"}, + {file = "anyio-4.15.1-py3-none-any.whl", hash = "sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101"}, + {file = "anyio-4.15.1.tar.gz", hash = "sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94"}, ] [package.dependencies] idna = ">=2.8" -sniffio = ">=1.1" +typing_extensions = {version = ">=4.16.0", markers = "python_version < \"3.15\""} [package.extras] -doc = ["Sphinx", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme (>=1.2.2)", "sphinxcontrib-jquery"] -test = ["anyio[trio]", "coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "mock (>=4) ; python_version < \"3.8\"", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17) ; python_version < \"3.12\" and platform_python_implementation == \"CPython\" and platform_system != \"Windows\""] -trio = ["trio (<0.22)"] +trio = ["trio (>=0.32.0)"] [[package]] name = "appnope" @@ -1249,7 +1247,7 @@ comm = ">=0.1.1" debugpy = ">=1.6.5" ipython = ">=7.23.1" jupyter-client = ">=8.9.0" -jupyter-core = ">=5.1,<6.0.dev0 || >=6.1.dev0" +jupyter-core = ">=5.1,<6.0 || >=6.1.dev0" matplotlib-inline = ">=0.1" nest-asyncio2 = ">=1.7.0" packaging = ">=22" @@ -1581,7 +1579,7 @@ files = [ ipykernel = ">=6.14" ipython = "*" jupyter-client = ">=7.0.0" -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +jupyter-core = ">=4.12,<5.0 || >=5.1.dev0" prompt-toolkit = ">=3.0.30" pygments = "*" pyzmq = ">=17" @@ -1670,14 +1668,14 @@ anyio = ">=3.1.0" argon2-cffi = ">=21.1" jinja2 = ">=3.0.3" jupyter-client = ">=7.4.4" -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +jupyter-core = ">=4.12,<5.0 || >=5.1.dev0" jupyter-events = ">=0.11.0" jupyter-server-terminals = ">=0.4.4" nbconvert = ">=6.4.4" nbformat = ">=5.3.0" packaging = ">=22.0" prometheus-client = ">=0.9" -pywinpty = {version = ">=2.0.1,<3.0.4 || >3.0.4", markers = "os_name == \"nt\""} +pywinpty = {version = ">=2.0.1,!=3.0.4", markers = "os_name == \"nt\""} pyzmq = ">=24" send2trash = ">=1.8.2" terminado = ">=0.8.3" @@ -1735,20 +1733,20 @@ test = ["bash-kernel", "pytest"] [[package]] name = "jupyterlab" -version = "4.6.3" +version = "4.6.4" description = "JupyterLab computational environment" optional = false python-versions = ">=3.10" groups = ["dev"] files = [ - {file = "jupyterlab-4.6.3-py3-none-any.whl", hash = "sha256:0a1ebc6567186f1eabd99536e94df7ed9e96d1e7c5ddf3e4406ae16e88abacb7"}, - {file = "jupyterlab-4.6.3.tar.gz", hash = "sha256:2e3db6e3a12495ebd188276e985bf5ac502fbde3d1e8628819920210008de498"}, + {file = "jupyterlab-4.6.4-py3-none-any.whl", hash = "sha256:15b13f991d3985129c797eb84d9949eeb8b6615e14b444868e642411f2c418b2"}, + {file = "jupyterlab-4.6.4.tar.gz", hash = "sha256:404f49b081819378524886c9db66dba57a5565981eff885830df1baba3a17df5"}, ] [package.dependencies] async-lru = ">=1.0.0" httpx = ">=0.25.0,<1" -ipykernel = ">=6.5.0,<6.30.0 || >6.30.0" +ipykernel = ">=6.5.0,!=6.30.0" jinja2 = ">=3.0.3" jupyter-builder = ">=1.0.2" jupyter-core = "*" @@ -2484,7 +2482,7 @@ files = [ [package.dependencies] fastjsonschema = ">=2.15" jsonschema = ">=2.6" -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +jupyter-core = ">=4.12,<5.0 || >=5.1.dev0" traitlets = ">=5.1" [package.extras] @@ -2506,9 +2504,9 @@ files = [ [package.dependencies] docutils = ">=0.18.1" jinja2 = "*" -nbconvert = ">=5.3,<5.4 || >5.4" +nbconvert = ">=5.3,!=5.4" nbformat = "*" -sphinx = ">=1.8,<8.2.0 || >8.2.0,<8.2.1 || >8.2.1" +sphinx = ">=1.8,!=8.2.0,!=8.2.1" traitlets = ">=5" [[package]] @@ -2537,20 +2535,20 @@ files = [ [[package]] name = "notebook" -version = "7.6.1" +version = "7.6.3" description = "Jupyter Notebook - A web-based notebook environment for interactive computing" optional = false python-versions = ">=3.10" groups = ["dev"] files = [ - {file = "notebook-7.6.1-py3-none-any.whl", hash = "sha256:6ea1e4c926f0dc490444ddcc335797a3dacda470a805d01031872fb119988ba2"}, - {file = "notebook-7.6.1.tar.gz", hash = "sha256:0b45fd1010668dd4808c40914d957706dbf044a677d28764dc881b74dfcede82"}, + {file = "notebook-7.6.3-py3-none-any.whl", hash = "sha256:ad7e0eb765fba836cd4a2ab0c7a3a26cde1d91665fbf6f533b6ae7b2de6d88d2"}, + {file = "notebook-7.6.3.tar.gz", hash = "sha256:e2c08e469c0ae20bb0b3214f0ab77e79653317a2f8e5b34c10361c66874a5b50"}, ] [package.dependencies] jupyter-builder = ">=1.0.2,<2" jupyter-server = ">=2.19.0,<3" -jupyterlab = ">=4.6.2,<4.7" +jupyterlab = ">=4.6.4,<4.7" jupyterlab-server = ">=2.28.0,<3" notebook-shim = ">=0.2,<0.3" tornado = ">=6.2.0" @@ -3771,18 +3769,6 @@ pyyaml = ">=6.0.3,<7.0.0" [package.extras] syn = ["synphot (>=1.7.0,<2.0.0)"] -[[package]] -name = "sniffio" -version = "1.3.0" -description = "Sniff out which async library your code is running under" -optional = false -python-versions = ">=3.7" -groups = ["main", "dev"] -files = [ - {file = "sniffio-1.3.0-py3-none-any.whl", hash = "sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384"}, - {file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"}, -] - [[package]] name = "snowballstemmer" version = "2.2.0" @@ -3797,14 +3783,14 @@ files = [ [[package]] name = "soupsieve" -version = "2.8.4" +version = "2.9.2" description = "A modern CSS selector implementation for Beautiful Soup." optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main", "dev", "docs"] files = [ - {file = "soupsieve-2.8.4-py3-none-any.whl", hash = "sha256:e7e6b0769c8f51ed59acab6e994b00621096cfb1c640a7509295987388fbaf65"}, - {file = "soupsieve-2.8.4.tar.gz", hash = "sha256:e121fd02e975c695e4e9e8774a5ee35d74714b59307868dcc5319ad2d9e3328e"}, + {file = "soupsieve-2.9.2-py3-none-any.whl", hash = "sha256:8089a26fd974ca7a1f30276d3d8492ab266ab15af581642dfe8aa162e0c1c823"}, + {file = "soupsieve-2.9.2.tar.gz", hash = "sha256:4a55d8cf158a9c2e587fa4922f1bbb91d68ac829e2d6f25403a85747c71daf74"}, ] [[package]] @@ -3962,14 +3948,14 @@ test = ["flake8", "mypy", "pytest"] [[package]] name = "sphinxcontrib-mermaid" -version = "2.1.0" +version = "2.1.1" description = "Mermaid diagrams in your Sphinx-powered docs" optional = false python-versions = ">=3.10" groups = ["docs"] files = [ - {file = "sphinxcontrib_mermaid-2.1.0-py3-none-any.whl", hash = "sha256:417cd144ec4b28852f46ba653f02ce8e538881c812111671a4c30344e87f2112"}, - {file = "sphinxcontrib_mermaid-2.1.0.tar.gz", hash = "sha256:13c5f9ac395cb6abf403eca34e228dc9fb3a30c9d960dbf3e40e9a8cef969549"}, + {file = "sphinxcontrib_mermaid-2.1.1-py3-none-any.whl", hash = "sha256:3b9ac1a8e0b5154dec3c0acf9b5c54dd9ba58540e416f20858c300814c25648f"}, + {file = "sphinxcontrib_mermaid-2.1.1.tar.gz", hash = "sha256:f8d7f1b2f0ad69b7ebdd5cbf532fb4ee107893b5373bc2d0e9634089c0b2cfee"}, ] [package.dependencies] @@ -4226,34 +4212,34 @@ test = ["flake8", "isort", "pytest"] [[package]] name = "tornado" -version = "6.5.8" +version = "6.5.9" description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." optional = false python-versions = ">=3.9" groups = ["dev", "docs", "test"] files = [ - {file = "tornado-6.5.8-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:cc6aa787d7cfab7c3d35189dc7a56fbd2399a569624c730c6b55b3d6531d0403"}, - {file = "tornado-6.5.8-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9715b5eb79735b2bcd454ce216a9275b7c0470e64ea1bf5742f78b2f72b26eeb"}, - {file = "tornado-6.5.8-cp39-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:547d63f450d570c14fe0e8db2cfb14c9bbd1c2503b4a6612586267955aa47b58"}, - {file = "tornado-6.5.8-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7e2360a0ffbe145eca8af0b19cb7203d79b1a98dd4cccdd6b368f6f49c2e3808"}, - {file = "tornado-6.5.8-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:5d242290bdf7ab3151bc1065fdd75c0dcc21cbc7b49f22a4c56329c2d6566d22"}, - {file = "tornado-6.5.8-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:7b94ff0e128fe0542f3bd331fb44d06260fc4ac16881545159f34ef08aad4195"}, - {file = "tornado-6.5.8-cp39-abi3-win32.whl", hash = "sha256:67832909c4779c64942380cb5f044a5c6163d00831472d80e25e115de9917836"}, - {file = "tornado-6.5.8-cp39-abi3-win_amd64.whl", hash = "sha256:11881db6b7c168494be2c2d12e65931451bdf7ee718535418ae1d8855dd5a0ee"}, - {file = "tornado-6.5.8-cp39-abi3-win_arm64.whl", hash = "sha256:68a7468c7e289f8514d7d664101753903217eff1bb6822c6b5994a0b5f5bcb26"}, - {file = "tornado-6.5.8.tar.gz", hash = "sha256:9452e1b208a8bd771e2cb1f2ff564985b9b214bdebbe622793e1799e0a6bd23f"}, + {file = "tornado-6.5.9-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:dea3bee433b73d6312d993ce89ad5f2b8d3ed9fde56e79ba5342b9edb78d342a"}, + {file = "tornado-6.5.9-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:22bed49cdb55292d1d58f46008ba2cc3020e5b84995ef6ae749676c8b5309514"}, + {file = "tornado-6.5.9-cp39-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0cc0876a58eee9143476f24901b1b8eb4d52443283ade2b6bca650dd1da04f84"}, + {file = "tornado-6.5.9-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc0d703cc7ec92b47ebfe236c97cd765c1f2e45744a2e156e6677e54c948274d"}, + {file = "tornado-6.5.9-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:93f9ab0226ef625b94cacb7981a5af104f1735ea02f92282abe5c389109fb31d"}, + {file = "tornado-6.5.9-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5801bbf94dfde8d21087b6aa0b17f3a5dab6110dfe44e7e4492df5de393ac15b"}, + {file = "tornado-6.5.9-cp39-abi3-win32.whl", hash = "sha256:dca9f377e80efe5dd4d52a2776b67492a81a39b2d931de169b6b36d0c1da444c"}, + {file = "tornado-6.5.9-cp39-abi3-win_amd64.whl", hash = "sha256:f810730ac71eadf009e6e5cb6d534d096887310f5649cff19d7e0d19c0bf4ee7"}, + {file = "tornado-6.5.9-cp39-abi3-win_arm64.whl", hash = "sha256:b5f4d92798337260c6d9f9f0e400c7539dd5e55cf93479a3f8508c9aa09c2b3b"}, + {file = "tornado-6.5.9.tar.gz", hash = "sha256:4d868544ebdf2fc155239a74397f65ebfea9b4d3635296c96b5dfb0701c354f5"}, ] [[package]] name = "tqdm" -version = "4.70.0" +version = "4.70.1" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "tqdm-4.70.0-py3-none-any.whl", hash = "sha256:7f585706bfddbdebf89daac705b2dfcc16890130727d3197ca62c732b4310953"}, - {file = "tqdm-4.70.0.tar.gz", hash = "sha256:55b0b0dbd97462d06ebee91e4dac24ed4d4702be82b24f07e6c1d27e08cea220"}, + {file = "tqdm-4.70.1-py3-none-any.whl", hash = "sha256:c293e525e6fef9c20e8728fd4612df02a0aa31bb5fe91ecd93e123b1b7bffa73"}, + {file = "tqdm-4.70.1.tar.gz", hash = "sha256:cefd0eca11b2a37a3aee776544d4f4ae913f02688135b5556b8788dfa474afc4"}, ] [package.dependencies] @@ -4322,14 +4308,14 @@ dev = ["flake8", "flake8-annotations", "flake8-bandit", "flake8-bugbear", "flake [[package]] name = "urllib3" -version = "2.7.0" +version = "2.8.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.10" groups = ["main", "dev", "docs"] files = [ - {file = "urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897"}, - {file = "urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c"}, + {file = "urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3"}, + {file = "urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63"}, ] [package.extras] @@ -4426,4 +4412,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">=3.12,<3.15" -content-hash = "ed6ef2a6cb8814ee6ab96c983711908f3102cbfdeb28136410bf675ad2da745c" +content-hash = "7c06b18379d2badb1c8603f9d29d48487af53f925335fbc9a95aa031614ebc6f" diff --git a/pyproject.toml b/pyproject.toml index da766c6da..f6ecd946f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ScopeSim" -version = "0.12.0b2.dev5" +version = "0.12.1.dev6" description = "Generalised telescope observation simulator" license = {text = "GPL-3.0-or-later"} authors = [ @@ -29,7 +29,7 @@ dependencies = [ "lxml (>=6.1.1,<7.0.0)", "pyyaml (>=6.0.3,<7.0.0)", "more-itertools (>=10.2.0,<11.0.0)", - "tqdm (>=4.70.0,<5.0.0)", + "tqdm (>=4.70.1,<5.0.0)", "packaging (>=26.0,<28.0)", "synphot (>=1.7.0,<2.0.0)", @@ -107,7 +107,10 @@ filterwarnings = [ "ignore:invalid value encountered in divide:RuntimeWarning", "ignore:Cannot merge meta key.*:astropy.utils.metadata.MergeConflictWarning", "ignore::astropy.wcs.wcs.FITSFixedWarning", - "default:The fov_grid*:DeprecationWarning", + # Only SpectralTraceList still calls the deprecated fov_grid internally. + # Delete this line once that call moves to an internal method; every other + # fov_grid DeprecationWarning is now an error. + "default:The fov_grid*:DeprecationWarning:scopesim.effects.spectral_trace_list", # Pytest Notebook stuff "ignore:Proactor*:RuntimeWarning", ] diff --git a/scopesim/detector/detector.py b/scopesim/detector/detector.py index 2b0e43ed8..9e1336217 100644 --- a/scopesim/detector/detector.py +++ b/scopesim/detector/detector.py @@ -1,12 +1,23 @@ +# -*- coding: utf-8 -*- + import numpy as np +from numpy.typing import ArrayLike from astropy.io.fits import ImageHDU from astropy.wcs import WCS from ..optics import ImagePlane -from ..optics.image_plane_utils import (add_imagehdu_to_imagehdu, - sky_wcs_from_det_wcs) -from ..utils import get_logger, from_currsys, stringify_dict, zeros_from_header +from ..optics.image_plane_utils import ( + add_imagehdu_to_imagehdu, + sky_wcs_from_det_wcs, +) +from ..utils import ( + get_logger, + from_currsys, + stringify_dict, + zeros_from_header, + real_colname, +) logger = get_logger(__name__) @@ -14,6 +25,7 @@ class Detector: def __init__(self, header, cmds=None, **kwargs): + self._native_header = header.copy() self._hdu = ImageHDU(header=header, data=zeros_from_header(header)) self.meta = {} self.meta.update(header) @@ -40,10 +52,9 @@ def extract_from(self, image_plane, spline_order=1, reset=True): self.header.update(sky_wcs.to_header()) def reset(self): - """Reset internal HDU data to all-zeros-array.""" - # The detector might have been converted to integers by the - # Quantization effect, so it is not possible to use zeros_like. - self._hdu.data = np.zeros(self._hdu.data.shape) + """Reset charge and restore the configured, unbinned detector geometry.""" + self._hdu = ImageHDU(header=self._native_header, data=zeros_from_header(self._native_header)) + self.meta.update(self._native_header) @property def hdu(self): @@ -62,6 +73,18 @@ def hdu(self): return self._hdu + @property + def det_id(self) -> int | None: + """Return detector ID. + + Avoid calling it just "id", because ``id(self)`` is something else. + + If not found in `self.meta`, return None. + """ + if key := real_colname("id", self.meta): + return self.meta[key] + return None + @property def header(self): """Return header from internal HDU.""" @@ -72,7 +95,16 @@ def data(self): """Return data from internal HDU.""" return self._hdu.data + @data.setter + def data(self, data: ArrayLike): + self._hdu.data = data + @property def image(self): """Return data from internal HDU.""" return self.data + + def __str__(self) -> str: + if self.det_id is not None: + return f"Detector {self.det_id!s:>2}" + return "Detector" diff --git a/scopesim/detector/detector_manager.py b/scopesim/detector/detector_manager.py index de6fd24a7..9c0f80b86 100644 --- a/scopesim/detector/detector_manager.py +++ b/scopesim/detector/detector_manager.py @@ -40,8 +40,13 @@ def __init__(self, detector_list=None, cmds=None, **kwargs): self._latest_exposure: HDUList | None = None - def readout(self, image_planes, array_effects=None, dtcr_effects=None, - **kwargs) -> HDUList: + def readout( + self, + image_planes, + array_effects: list[Effect] | None = None, + dtcr_effects: list[Effect] | None = None, + **kwargs + ) -> HDUList: """ Read out the detector array into a FITS HDU List. @@ -74,9 +79,6 @@ def readout(self, image_planes, array_effects=None, dtcr_effects=None, Output FITS HDU List. """ - # .. note:: Detector is what used to be called Chip - # DetectorManager is the old Detector - self._array_effects = array_effects or [] self._dtcr_effects = dtcr_effects or [] self.meta.update(kwargs) @@ -100,7 +102,7 @@ def readout(self, image_planes, array_effects=None, dtcr_effects=None, detector = effect.apply_to(detector) # 6. add necessary header keywords - # .. todo: add keywords + # TODO: add keywords # FIXME: Why is this applied twice ??? for effect in self._array_effects: @@ -148,7 +150,7 @@ def _make_primary_hdu(self): return prihdu def _make_effects_hdu(self): - # .. todo:: decide what goes into the effects table of meta data + # TODO: decide what goes into the effects table of meta data # effects = self._array_effects + self._dtcr_effects return TableHDU() diff --git a/scopesim/effects/__init__.py b/scopesim/effects/__init__.py index 3c5d75a84..2191fa0ba 100644 --- a/scopesim/effects/__init__.py +++ b/scopesim/effects/__init__.py @@ -8,6 +8,7 @@ from .spectral_trace_list import * from .spectral_efficiency import * from .metis_lms_trace_list import * +from .micado_ifu_trace_list import * from .mosaic_trace_list import * from .surface_list import * from .ter_curves import * diff --git a/scopesim/effects/apertures.py b/scopesim/effects/apertures.py index 756e78602..199914541 100644 --- a/scopesim/effects/apertures.py +++ b/scopesim/effects/apertures.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- """Effects related to field masks, including spectroscopic slits.""" -import warnings from typing import ClassVar import yaml @@ -118,6 +117,7 @@ def apply_to(self, obj, **kwargs): """See parent docstring.""" if isinstance(obj, FovVolumeList): logger.debug("Executing %s, FoV setup", self.meta['name']) + x = quantity_from_table("x", self.table, u.arcsec).to_value(u.arcsec) y = quantity_from_table("y", self.table, @@ -128,22 +128,18 @@ def apply_to(self, obj, **kwargs): for vol in obj.volumes: vol["meta"]["xi_min"] = min(x) * u.arcsec vol["meta"]["xi_max"] = max(x) * u.arcsec + vol["meta"]["eta_min"] = min(y) * u.arcsec + vol["meta"]["eta_max"] = max(y) * u.arcsec vol["meta"]["slit_name"] = self.meta["name"] + if "offset_type" in self.meta: + vol["meta"]["offset_type"] = self.meta["offset_type"] + vol["meta"]["offset_x"] = self.meta["offset_x"] + vol["meta"]["offset_y"] = self.meta["offset_y"] + else: + vol["meta"]["offset_type"] = "none" return obj - # Outdated. Remove when removing all old FOVManager code from effects - def fov_grid(self, which="edges", **kwargs): - """Return a header with the sky coordinates.""" - warnings.warn("The fov_grid method is deprecated and will be removed " - "in a future release.", DeprecationWarning, stacklevel=2) - if which == "edges": - self.meta.update(kwargs) - return self.header - elif which == "masks": - self.meta.update(kwargs) - return self.mask - @property def hdu(self): return fits.ImageHDU(data=self.mask, header=self.header) @@ -311,6 +307,8 @@ def apply_to(self, obj, **kwargs): # ..todo: HUGE HACK - Get rid of this! vol["meta"]["xi_min"] = row["left"] * u.arcsec vol["meta"]["xi_max"] = row["right"] * u.arcsec + vol["meta"]["eta_min"] = row["bottom"] * u.arcsec + vol["meta"]["eta_max"] = row["top"] * u.arcsec vol["conserve_image"] = row["conserve_image"] vol["shape"] = row["shape"] vol["angle"] = row["angle"] @@ -459,12 +457,6 @@ def apply_to(self, obj, **kwargs): """Use apply_to of current_slit.""" return self.current_slit.apply_to(obj, **kwargs) - def fov_grid(self, which="edges", **kwargs): - """See parent docstring.""" - warnings.warn("The fov_grid method is deprecated and will be removed " - "in a future release.", DeprecationWarning, stacklevel=2) - return self.current_slit.fov_grid(which=which, **kwargs) - def change_slit(self, slitname=None): """Change the current slit.""" if not slitname or slitname in self.slits.keys(): diff --git a/scopesim/effects/atmo_dispersion.py b/scopesim/effects/atmo_dispersion.py index 268cd7fee..3fd85fd18 100644 --- a/scopesim/effects/atmo_dispersion.py +++ b/scopesim/effects/atmo_dispersion.py @@ -1,5 +1,5 @@ from typing import ClassVar, Any -from functools import partial +from functools import partial, lru_cache import numpy as np from astropy import units as u from astropy.coordinates import SkyCoord, EarthLocation @@ -9,6 +9,7 @@ from astropy.coordinates import SkyCoord, HADec from scipy.interpolate import make_interp_spline, RegularGridInterpolator from scipy.ndimage import map_coordinates +from scipy.sparse import csr_array from .effects import Effect from ..optics.fov import FieldOfView3D @@ -61,15 +62,17 @@ class ShiftFoV3D(Effect): def __init__(self, **kwargs): super().__init__(**kwargs) + self.meta.setdefault("method", None) - if self.meta.get("method", None) not in [None, "roll", "map_coords"]: + if self.meta["method"] not in [None, "roll", "map_coords"]: raise ValueError("Method must be 'roll' or 'map_coords' if provided.") def get_shifts(self, obj): """ Shifts calculated from obj (FoV3D) properties or from init data. Overwrite this function when sub-classing. - Should return interp spline values for dx and dy at data wavelength bins in arcsec + Return dx and dy at cube wavelength bins in arcsec. The FOV can be + unrendered when its incoming illumination needs spatial padding. """ if self.data is not None: if isinstance(self.data, Table): @@ -81,14 +84,28 @@ def get_shifts(self, obj): logger.warning("No shifts as a function of wavelength provided in init.") raise NotImplementedError - swcs = WCS(obj.hdu.header).spectral - with u.set_enabled_equivalencies(u.spectral()): - wave = swcs.pixel_to_world(np.arange(swcs.pixel_shape[0])) << u.um - return dx(wave), dy(wave) + wave = _cube_wavelengths(obj) + return dx(wave), dy(wave) + + def _padded_header(self, header, shifts, rounded=False): + """Spatial support needed before applying this displacement.""" + header = header.copy() + # Cubic interpolation needs neighboring samples as well as the shift. + guard = 0 if self.meta['method'] == 'roll' else 4 + for axis, displacement in enumerate(shifts, start=1): + cells = (displacement/(abs(header[f'CDELT{axis}'])*u.deg)).decompose().value + if rounded: + cells = np.round(cells, 1) + if self.meta['method'] == 'roll': + cells = np.round(np.asarray(cells, dtype=np.float32)) + padding = int(np.ceil(np.max(np.abs(cells)))) + guard if np.any(cells) else 0 + header[f'NAXIS{axis}'] += 2*padding + header[f'CRPIX{axis}'] += padding + return header def apply_to(self, obj, **kwargs): if isinstance(obj, FieldOfView3D) and obj.hdu is not None and obj.hdu.header['NAXIS'] == 3: - dxs, dys = self.get_shifts(obj) + dxs, dys = kwargs['_shifts'] if '_shifts' in kwargs else self.get_shifts(obj) # get bin widths in spatial axis and divide to get dxs and dys in steps dxs = np.round((dxs / (obj.hdu.header["CDELT1"]*u.deg)).decompose().value, 1) @@ -119,19 +136,29 @@ def roll_shift(data, dx, dy): raise ValueError("dx and dy must have shape (nwave,)") out = np.empty_like(data) - for w in range(nw): - out[w] = np.roll(data[w], (dy[w], dx[w]), axis=(0, 1)) - # zero out the rolled in part, select the correct slice based on sign of dxint, dyint - xslice = slice(0, dx[w]) if dx[w] > 0 else slice(nx-dx[w], nx) - yslice = slice(0, dy[w]) if dy[w] > 0 else slice(ny-dy[w], ny) - out[w][yslice, xslice] = 0 + boundaries = np.r_[0, np.flatnonzero((np.diff(dx) != 0) | (np.diff(dy) != 0))+1, nw] + for start, stop in zip(boundaries[:-1], boundaries[1:]): + xshift, yshift = dx[start], dy[start] + if abs(xshift) >= nx or abs(yshift) >= ny: + out[start:stop] = 0 + continue + if xshift == 0 and yshift == 0: + out[start:stop] = data[start:stop] + continue + out[start:stop] = np.roll(data[start:stop], (yshift, xshift), axis=(1, 2)) + if xshift != 0: + xslice = slice(0, xshift) if xshift > 0 else slice(nx + xshift, nx) + out[start:stop, :, xslice] = 0 + if yshift != 0: + yslice = slice(0, yshift) if yshift > 0 else slice(ny + yshift, ny) + out[start:stop, yslice, :] = 0 return out @staticmethod - def map_coords_shift(data, dx, dy, chunk_w: int=8, order: int=3): + def map_coords_shift(data, dx, dy, chunk_w: int=64, order: int=3): """ Shift each wavelength plane data[w, :, :] by (dx[w], dy[w]) using - scipy.ndimage.map_coordinates with chunked processing. + scipy.ndimage.map_coordinates independently on each spatial plane. Parameters ---------- @@ -142,7 +169,7 @@ def map_coords_shift(data, dx, dy, chunk_w: int=8, order: int=3): dy : np.ndarray, shape (nwave,) Shift along y (axis=0) in pixels. Positive moves content toward +y. chunk_w : int - Number of wavelength planes to process per chunk. Default 8. + Number of independent wavelength planes processed together. order : int Spline interpolation order. Default 3. @@ -156,36 +183,34 @@ def map_coords_shift(data, dx, dy, chunk_w: int=8, order: int=3): if dx.shape != (nw,) or dy.shape != (nw,): raise ValueError("dx and dy must have shape (nwave,)") - # Base grids for a single chunk (will be broadcast) - yi = np.arange(ny, dtype=np.float32)[None, :, None] # (1, ny, 1) - xi = np.arange(nx, dtype=np.float32)[None, None, :] # (1, 1, nx) - - out = np.empty_like(data) - prefilter = (order > 1) - - for w0 in range(0, nw, chunk_w): - w1 = min(w0 + chunk_w, nw) - k = w1 - w0 - - # wave index grid for this chunk: (k, 1, 1) - wi = np.arange(w0, w1, dtype=np.float32)[:, None, None] - - # To shift content by +dx/+dy, sample input at (x - dx, y - dy) - xcoords = xi - dx[w0:w1][:, None, None] # (k, 1, nx) -> broadcast to (k, ny, nx) - ycoords = yi - dy[w0:w1][:, None, None] # (k, ny, 1) -> broadcast to (k, ny, nx) - - # Broadcast to full chunk volume - xcoords = np.broadcast_to(xcoords, (k, ny, nx)) - ycoords = np.broadcast_to(ycoords, (k, ny, nx)) - wcoords = np.broadcast_to(wi, (k, ny, nx)) - - # Axis order in coords must match axis order of data: (wave, y, x) - coords = np.array([wcoords, ycoords, xcoords], dtype=np.float32) # (3, k, ny, nx) - - out[w0:w1, :, :] = map_coordinates( - data, coords, order=order, mode='nearest', prefilter=prefilter - ) - return out + # The same rounded displacement recurs in many wavelength planes. + # Reuse its spatial interpolation operator, without ever interpolating + # along wavelength or filtering the same profile thousands of times. + shifted = data + for axis, displacement, length in ((2, dx, nx), (1, dy, ny)): + if not np.any(displacement): + continue + out = np.empty(data.shape, dtype=float) + for offset in np.unique(displacement): + operator = _spatial_shift_operator(length, float(offset), order) + planes = np.flatnonzero(displacement == offset) + for start in range(0, len(planes), chunk_w): + selected = planes[start:start + chunk_w] + block = np.moveaxis(shifted[selected], axis, 0) + result = operator @ block.reshape(length, -1) + out[selected] = np.moveaxis(result.reshape(block.shape), 0, axis) + shifted = out + return shifted.astype(data.dtype, copy=False) + + +@lru_cache(maxsize=128) +def _spatial_shift_operator(length, shift, order): + # Apply scipy's interpolation to unit basis profiles once. Cubic spline + # coefficients decay exponentially; omit only weights below 1e-13. + row, column = np.indices((length, length), dtype=float) + response = map_coordinates(np.eye(length), [row, column - shift], order=order, mode='nearest', prefilter=order > 1) + response[abs(response) < 1e-13] = 0 + return csr_array(response.T) class ADShift(ShiftFoV3D): @@ -236,7 +261,7 @@ def apply_to(self, obj, **kwargs): def get_shifts(self, obj: FieldOfView3D): shift_arcsec = self._get_shifts_arcsec(obj) - pos_angle_y = field_rotation_pa_y(obj.hdu.header) + pos_angle_y = field_rotation_pa_y(obj.header if obj.hdu is None else obj.hdu.header) par_angle = get_parallactic_angle(self.target, self.location, self.time) theta = np.deg2rad(par_angle - pos_angle_y) # angle of zenith direction from +y-axis counter-clockwise logger.info(f'Direction of zenith, measured counter-clockwise from +y axis is {int(par_angle - pos_angle_y)}.') @@ -249,9 +274,7 @@ def _get_shifts_arcsec(self, obj: FieldOfView3D): wave_ref = from_currsys(self.meta["wave_ref_um"], self.cmds) * u.um n_ref = self.n(wave_ref) - swcs = WCS(obj.hdu.header).spectral - with u.set_enabled_equivalencies(u.spectral()): - wave = swcs.pixel_to_world(np.arange(swcs.pixel_shape[0])) << u.um + wave = _cube_wavelengths(obj) n_wave = self.n(wave) delta_n = n_wave - n_ref return 206265 * delta_n * np.tan(self.zenith_angle.to(u.rad).value) * u.arcsec @@ -309,9 +332,7 @@ def get_shifts(self, obj: FieldOfView3D): """ Computes an AD correction residual for a given zenith angle. """ - swcs = WCS(obj.hdu.header).spectral - with u.set_enabled_equivalencies(u.spectral()): - wave = swcs.pixel_to_world(np.arange(swcs.pixel_shape[0])) << u.um + wave = _cube_wavelengths(obj) shifts = np.zeros_like(wave.value) * u.arcsec if self.data is not None and isinstance(self.data, Table): @@ -354,7 +375,7 @@ def get_shifts(self, obj: FieldOfView3D): shifts += ad_shift ad.zenith_angle = self.zenith_angle - pos_angle_y = field_rotation_pa_y(obj.hdu.header) + pos_angle_y = field_rotation_pa_y(obj.header if obj.hdu is None else obj.hdu.header) par_angle = get_parallactic_angle(self.target, self.location, self.time) theta = np.deg2rad(par_angle - pos_angle_y) # angle of zenith direction from +y-axis counter-clockwise logger.info(f'Direction of zenith, measured counter-clockwise from +y axis is {int(par_angle - pos_angle_y)}.') @@ -401,12 +422,20 @@ def _get_ad_shift(self, ad_kwargs): ########################### AD utils ############################### +def _cube_wavelengths(obj): + if obj.hdu is None: + return obj.cube_waveset.to(u.um) + swcs = WCS(obj.hdu.header).spectral + with u.set_enabled_equivalencies(u.spectral()): + return swcs.pixel_to_world(np.arange(swcs.pixel_shape[0])).to(u.um) + + def field_rotation_pa_y(header) -> float: """ Returns PA of detector +Y axis on sky, in deg (N->E convention). 0 deg = Y points north, 90 deg = Y points east """ - w = WCS(header).dropaxis(-1) + w = WCS(header, naxis=2) # local anchor (image center) x0, y0 = w.wcs.crpix lon0, lat0 = w.pixel_to_world(x0, y0) diff --git a/scopesim/effects/detector_list.py b/scopesim/effects/detector_list.py index 6e197f75f..4a3c5ea55 100644 --- a/scopesim/effects/detector_list.py +++ b/scopesim/effects/detector_list.py @@ -243,31 +243,6 @@ def _get_corner_points(self): return points * u.mm - def fov_grid(self, which="edges", **kwargs): - """Return an ApertureMask object. kwargs are "pixel_scale" [arcsec].""" - # This has really been taken care of by apply_to now - # TODO v1.0: finally rm this completely - raise AttributeError("The DetectorList.fov_grid() method has been " - "removed in v0.10.0.") - # aperture_mask = None - # if which == "edges": - # self.meta.update(kwargs) - # self.meta = from_currsys(self.meta, self.cmds) - - # hdr = self.image_plane_header - # xy_mm = calc_footprint(hdr, "D") - # pixel_size = hdr["CDELT1D"] # mm - # pixel_scale = self.meta["pixel_scale"] # ["] - - # # x["] = x[mm] * ["] / [mm] - # xy_sky = xy_mm * pixel_scale / pixel_size - - # aperture_mask = ApertureMask(array_dict={"x": xy_sky[:, 0], - # "y": xy_sky[:, 1]}, - # pixel_scale=pixel_scale) - - # return aperture_mask - @property def image_plane_header(self): """Create and return the Image Plane Header.""" @@ -355,16 +330,17 @@ def detector_headers(self, ids=None): hdr["PC2_1"] = -sang hdr["PC2_2"] = cang - # hdr["GAIN"] = row["gain"] - if "id" in row: - hdr["DET_ID"] = row["id"] - hdr["EXTNAME"] = f"DET_{row['id']}" - row_dict = { f"HIERARCH {col.upper()}" if len(col) > 8 else col: row[col] for col in row.colnames } hdr.update(row_dict) + if "id" in row.colnames: + # Keep routing identifiers numeric when detector metadata is + # serialized to FITS (NumPy integers otherwise become strings). + hdr["ID"] = int(row["id"]) + hdr["DET_ID"] = int(row["id"]) + hdr["EXTNAME"] = f"DET_{row['id']}" hdrs.append(hdr) return hdrs @@ -498,9 +474,11 @@ def _get_fov_limits(self, pixel_scale=None): def image_plane_header(self): """Create and return the Image Plane Header.""" # FIXME: Heavy property..... - points = self._get_corner_points().value new_wcs, naxis = create_wcs_from_points( - points, self.pixel_size.to_value(u.mm), "D") + self._get_corner_points(), + self.pixel_size.to(u.mm) / u.pixel, + wcs_suffix="D", + ) hdr = fits.Header() hdr["NAXIS"] = 3 diff --git a/scopesim/effects/effects.py b/scopesim/effects/effects.py index 30b4bd38f..da47304af 100644 --- a/scopesim/effects/effects.py +++ b/scopesim/effects/effects.py @@ -11,20 +11,19 @@ from ..reports.rst_utils import table_to_rst -# FIXME: This docstring is out-of-date for several reasons: -# - Effects can act on objects other than Source (eg FOV, IMP, DET) -# - fov_grid is outdated - - # @dataclass(kw_only=True, eq=False) class Effect: """ Base class for representing the effects (artifacts) in an optical system. The ``Effect`` class is conceived to independently apply the changes that - an optical component (or series thereof) has on an incoming 3D description - of an on-sky object. In other words, **an Effect object should receive a - derivative of a ``Source`` object, alter it somehow, and return it**. + an optical component (or series thereof) has on an incoming description of + an on-sky object. **An Effect object should receive an object, alter it + somehow, and return an instance of the same class.** Besides ``Source``, + that object may be a ``FovVolumeList``, a ``FieldOfView``, an + ``ImagePlane`` or a ``Detector``; ``apply_to`` is expected to dispatch on + the type it is given and to pass anything it does not handle straight + through. The interface for the Effect base-class has been kept very general so that it can easily be sub-classed as data for new effects becomes available. @@ -32,9 +31,12 @@ class Effect: attributes: * ``self.meta`` - a dictionary to contain metadata. - * ``self.apply_to(obj, **kwargs)`` - a method which accepts a - Source-derivative and returns an instance of the same class as ``obj`` - * ``self.fov_grid(which="", **kwargs)`` + * ``self.apply_to(obj, **kwargs)`` - a method which accepts one of the + objects above and returns an instance of the same class as ``obj`` + + An Effect that needs to constrain the source-space volume the simulation + covers does so in the ``FovVolumeList`` branch of ``apply_to``, by calling + ``shrink``, ``extract`` or ``split`` on the list it is passed. Parameters diff --git a/scopesim/effects/effects_utils.py b/scopesim/effects/effects_utils.py index f9522ded1..f96dceeb9 100644 --- a/scopesim/effects/effects_utils.py +++ b/scopesim/effects/effects_utils.py @@ -1,10 +1,9 @@ +# -*- coding: utf-8 -*- """TBA.""" import inspect from collections.abc import Iterable -from astropy.table import Table - from .. import effects as efs from ..utils import get_logger @@ -51,15 +50,7 @@ def is_spectroscope(effects): return any(isinstance(eff, spec_classes) for eff in effects) -def empty_surface_list(**kwargs): - tbl = Table(names=["name", "outer", "inner", "angle", - "temperature", "action", "filename"], - data=[["test"], [0.], [0.], [0.], [0.], ["none"], ["none"]], - meta={"outer_unit": "m", "inner_unit": "m", - "angle_unit": "deg", "temperature_unit": "deg_C"}) - return efs.SurfaceList(table=tbl[:0], **kwargs) - - +# This is only used for the questionable sphinx extension. def scopesim_effect_classes(base_effect=efs.Effect): members = inspect.getmembers(efs) efs_dict = {".".join([cls.__module__, cls.__name__]).replace("scopesim.effects.", ""): cls diff --git a/scopesim/effects/electronic/__init__.py b/scopesim/effects/electronic/__init__.py index 80a5ea6db..991d1aa33 100644 --- a/scopesim/effects/electronic/__init__.py +++ b/scopesim/effects/electronic/__init__.py @@ -19,14 +19,42 @@ - InterPixelCapacitance - apply IPC kernel to detector readout """ -from ...utils import get_logger +from ...utils import get_logger, check_keys +from ...detector import Detector +from .. import Effect + logger = get_logger(__name__) + +class ElectronicEffect(Effect): + """Base class for electronic effects. + + This will eventually replace the 800-range zorder (or parts of it). + """ + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.meta.update(kwargs) + check_keys(self.meta, self.required_keys, action="error") + + def _apply_to_det(self, det: Detector) -> None: + """Subclasses can override if more params needed in call.""" + logger.debug("Apply %s to %s", self.display_name, det) + det.data = self(det.data) + + def apply_to(self, obj, **kwargs): + """See parent docstring.""" + if isinstance(obj, Detector): + self._apply_to_det(obj) + + return obj + + from .electrons import LinearityCurve, ADConversion, InterPixelCapacitance from .noise import (Bias, PoorMansHxRGReadoutNoise, BasicReadoutNoise, - PixelResponseNonUniformity, ShotNoise, DarkCurrent) -from .exposure import AutoExposure, ExposureIntegration, ExposureOutput -from .pixels import ReferencePixelBorder, BinnedImage, UnequalBinnedImage + PixelResponseNonUniformity, ShotNoise, AvalancheShotNoise, DarkCurrent, ReadoutNoiseFromTable) +from .exposure import AutoExposure, ExposureIntegration, ExposureOutput, ExposureLimit +from .pixels import ReferencePixelBorder, BinnedImage, UnequalBinnedImage, OnChipBinning from .dmps import DetectorModePropertiesSetter diff --git a/scopesim/effects/electronic/dmps.py b/scopesim/effects/electronic/dmps.py index 1f1bcf3fd..2cf3e2130 100644 --- a/scopesim/effects/electronic/dmps.py +++ b/scopesim/effects/electronic/dmps.py @@ -23,6 +23,9 @@ class DetectorModePropertiesSetter(Effect): mode_properties : dict A dictionary containing the DET parameters to be changed for each mode. See below for an example yaml entry. + mode_key : str + Command selecting the mode, by default ``!OBS.detector_readout_mode``. + Separate keys allow independent detector arms to select their modes. Examples -------- @@ -70,11 +73,12 @@ def __init__(self, **kwargs): check_keys(self.meta, self.required_keys, action="error") self.mode_properties = kwargs["mode_properties"] + self.mode_key = kwargs.get("mode_key", "!OBS.detector_readout_mode") def apply_to(self, obj, **kwargs): logger.debug("Entering DetectorModePropertiesSetter.apply_to") mode_name = kwargs.get("detector_readout_mode", - from_currsys("!OBS.detector_readout_mode", + from_currsys(self.mode_key, self.cmds)) if isinstance(obj, ImagePlane) and mode_name == "auto": mode_name = self.select_mode(obj, **kwargs) @@ -82,7 +86,7 @@ def apply_to(self, obj, **kwargs): self.meta["detector_readout_mode"] = mode_name props_dict = self.mode_properties[mode_name] - self.cmds["!OBS.detector_readout_mode"] = mode_name + self.cmds[self.mode_key] = mode_name for key, value in props_dict.items(): self.cmds[key] = value logger.debug("DMPS setting %s: %s", key, value) @@ -101,9 +105,10 @@ def select_mode(self, obj, **kwargs): saturation level (peak to full_well). """ immax = np.max(obj.data) - fillfrac = kwargs.get("fill_frac", - from_currsys("!OBS.auto_exposure.fill_frac", - self.cmds)) + fillfrac = kwargs.get( + "fill_frac", + from_currsys("!OBS.auto_exposure.fill_frac", self.cmds), + ) goodmodes = [] goodron = [] diff --git a/scopesim/effects/electronic/electrons.py b/scopesim/effects/electronic/electrons.py index c860362bd..ac3e13236 100644 --- a/scopesim/effects/electronic/electrons.py +++ b/scopesim/effects/electronic/electrons.py @@ -9,19 +9,17 @@ """ from typing import ClassVar +from numbers import Real # matches int, float and all the numpy scalars import numpy as np +from numpy.typing import ArrayLike, NDArray from scipy.signal import oaconvolve -from .. import Effect -from ...detector import Detector -from ...utils import figure_factory, check_keys -from ...utils import from_currsys, real_colname, get_logger -from . import logger +from ...utils import from_currsys, figure_factory +from . import ElectronicEffect, Detector, logger -logger = get_logger(__name__) -class LinearityCurve(Effect): +class LinearityCurve(ElectronicEffect): """ Detector linearity effect. @@ -56,28 +54,29 @@ class LinearityCurve(Effect): report_plot_include: ClassVar[bool] = True report_table_include: ClassVar[bool] = False - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.meta.update(kwargs) + def __call__(self, data: ArrayLike, ndit: int = 1) -> NDArray: + incident = self.incident * ndit + measured = self.measured * ndit + return np.interp(data, incident, measured) - check_keys(self.meta, self.required_keys, action="error") - - def apply_to(self, obj, **kwargs): - if not isinstance(obj, Detector): - return obj + @property + def incident(self) -> NDArray: + if self.table is not None: + return self.table["incident"] + return np.asarray(from_currsys(self.meta["incident"], self.cmds)) - ndit = from_currsys(self.meta["ndit"], self.cmds) + @property + def measured(self) -> NDArray: if self.table is not None: - incident = self.table["incident"] * ndit - measured = self.table["measured"] * ndit - else: - incident = np.asarray(from_currsys(self.meta["incident"], - self.cmds)) * ndit - measured = np.asarray(from_currsys(self.meta["measured"], - self.cmds)) * ndit - obj._hdu.data = np.interp(obj._hdu.data, incident, measured) + return self.table["measured"] + return np.asarray(from_currsys(self.meta["measured"], self.cmds)) + + def _apply_to_det(self, det: Detector) -> None: + """Subclasses can override if more params needed in call.""" + logger.debug("Apply %s to %s", self.display_name, det) - return obj + ndit = from_currsys(self.meta["ndit"], self.cmds) + det.data = self(det.data, ndit) def plot(self, **kwargs): fig, ax = figure_factory() @@ -93,7 +92,7 @@ def plot(self, **kwargs): return fig -class InterPixelCapacitance(Effect): +class InterPixelCapacitance(ElectronicEffect): r"""Inter-pixel capacitance effect. The effect models cross-talk due to inter-pixel capacitance with @@ -147,8 +146,6 @@ class InterPixelCapacitance(Effect): def __init__(self, **kwargs): super().__init__(**kwargs) - - self.meta.update(kwargs) self.kernel = self._build_kernel(kwargs) def _build_kernel(self, params): @@ -174,15 +171,10 @@ def _build_kernel(self, params): ]) return kernel - def apply_to(self, det, **kwargs): - if not isinstance(det, Detector): - logger.debug("%s applied to %s", self.display_name, - det.__class__.__name__) - return det - - newdata = oaconvolve(det._hdu.data, self.kernel, mode="same") - det._hdu.data = newdata - return det + def _apply_to_det(self, det: Detector) -> None: + """Subclasses can override if more params needed in call.""" + logger.debug("Apply %s to %s", self.display_name, det) + det.data = oaconvolve(det.data, self.kernel, mode="same") def update(self, **kwargs): """Update the IPC kernel. @@ -220,7 +212,7 @@ def __str__(self): return msg -class ADConversion(Effect): +class ADConversion(ElectronicEffect): """Analogue-Digital Conversion effect. The effect applies the gain factor (electrons/ADU) to the detector readouts @@ -271,7 +263,7 @@ def __init__(self, **kwargs): super().__init__(**kwargs) params = { "dtype": "uint16", - "gain": 1. # default, usually overridden from yaml + "gain": "!DET.gain" } self.meta.update(params) self.meta.update(kwargs) @@ -285,7 +277,7 @@ def _should_apply(self) -> bool: logger.warning("Cannot access cmds for ADConversion effect.") return True - # ..todo: need to deal with this case more realistically + # TODO: need to deal with this case more realistically # Is this still necessary? #if self.cmds.get("!OBS.autoexpset", False): # logger.info("DIT, NDIT determined by AutoExposure -> " @@ -298,28 +290,34 @@ def _should_apply(self) -> bool: return True - def apply_to(self, obj, **kwargs): - if not isinstance(obj, Detector): - return obj - - new_dtype = self.meta["dtype"] + def __call__(self, data: ArrayLike, gain: Real) -> NDArray: + return data / gain + def _get_gain(self, det_id: int) -> Real: # Apply the gain value (copy from DarkCurrent) # Note that this does not cater for the case where the gain is given # as a plain dictionary. Should we implement that? - if hasattr(self.cmds["!DET.gain"], "dic"): - dtcr_id = obj.meta[real_colname("id", obj.meta)] - gain = self.cmds["!DET.gain"].dic[dtcr_id] - logger.info(f"Detector {dtcr_id}: applying gain {gain}") - elif isinstance(self.cmds["!DET.gain"], (float, int)): - gain = self.cmds["!DET.gain"] + configured_gain = from_currsys(self.meta["gain"], self.cmds) + if hasattr(configured_gain, "dic"): + gain = configured_gain.dic[det_id] + logger.info(f"Detector {det_id}: applying gain {gain}") + return gain + if isinstance(configured_gain, Real): + gain = configured_gain logger.info(f"Applying gain {gain}") - else: - raise ValueError(".meta['gain'] must be either " - f"dict or float, but is {self.cmds['!DET.gain']}") + return gain + raise ValueError( + f"{self.__class__.__name__}.meta['gain'] must be either " + f"dict or float, but is {configured_gain}") + + def _apply_to_det(self, det: Detector) -> None: + """Subclasses can override if more params needed in call.""" + logger.debug("Apply %s to %s", self.display_name, det) + + new_dtype = self.meta["dtype"] # Apply gain - obj._hdu.data /= gain + det.data = self(det.data, self._get_gain(det.det_id)) # Type-conversion wraps around input values that are higher or lower than # the respective maximum and minimum values of the new data type. Before @@ -328,14 +326,14 @@ def apply_to(self, obj, **kwargs): if np.issubdtype(new_dtype, np.integer): minval = np.iinfo(new_dtype).min maxval = np.iinfo(new_dtype).max - minvals_mask = obj._hdu.data < minval - maxvals_mask = obj._hdu.data > maxval + minvals_mask = det.data < minval + maxvals_mask = det.data > maxval if minvals_mask.any(): - obj._hdu.data[minvals_mask] = minval + det.data[minvals_mask] = minval logger.warning( f"Effect ADConversion: {minvals_mask.sum()} negative pixels") if maxvals_mask.any(): - obj._hdu.data[maxvals_mask] = maxval + det.data[maxvals_mask] = maxval logger.warning( f"Effect ADConversion: {maxvals_mask.sum()} saturated pixels") @@ -343,6 +341,4 @@ def apply_to(self, obj, **kwargs): # set to the modified data. It should be fine to simply re-assign the # data attribute, but just in case it's not... logger.debug("Applying digitization to dtype %s.", new_dtype) - obj._hdu.data = obj._hdu.data.astype(new_dtype) - - return obj + det.data = det.data.astype(new_dtype) diff --git a/scopesim/effects/electronic/exposure.py b/scopesim/effects/electronic/exposure.py index 8ff88e1e7..328eefb90 100644 --- a/scopesim/effects/electronic/exposure.py +++ b/scopesim/effects/electronic/exposure.py @@ -2,6 +2,7 @@ """Exposure actions.""" from typing import ClassVar +from warnings import warn import numpy as np @@ -70,10 +71,10 @@ def _log_dit_ndit(dit: float, ndit: int) -> None: logger.info("Total exposure time: %.3f s", dit * ndit) def estimate_dit_ndit( - self, - exptime: float, - image_plane_max: float, - **kwargs + self, + exptime: float, + image_plane_max: float, + **kwargs ) -> tuple[float, int]: """ Automatically determine DIT and NDIT from exposure time. @@ -102,8 +103,17 @@ def estimate_dit_ndit( "fill_frac", from_currsys(self.meta["fill_frac"], self.cmds) ) + try: + dark_current = kwargs.get( + "dark_current", + from_currsys(self.meta["dark_current"], self.cmds) + ) + except KeyError: + logger.warning("No dark current found for %s", + self.display_name) + dark_current = 0 - dit_nosat = fill_frac * full_well / image_plane_max + dit_nosat = fill_frac * full_well / (image_plane_max + dark_current) logger.info("Required DIT without saturation: %.3f s", dit_nosat) # np.ceil so that dit is at most what is required for fill_frac @@ -136,8 +146,10 @@ def apply_to(self, obj, **kwargs): # in both cases, `obj` is an ImagePlane. return obj - exptime = kwargs.pop("exptime", - from_currsys("!OBS.exptime", self.cmds)) + exptime = kwargs.pop( + "exptime", + from_currsys("!OBS.exptime", self.cmds), + ) mindit = from_currsys(self.meta["mindit"], self.cmds) # TODO: Remove this silly try-except once currsys works properly... @@ -217,7 +229,7 @@ def apply_to(self, obj, **kwargs): logger.debug("Exposure: DIT = %s s, NDIT = %s", dit, ndit) if self.current_mode == "average": - obj._hdu.data /= ndit + obj.data = obj.data / ndit return obj @@ -276,6 +288,38 @@ def apply_to(self, obj, **kwargs): "readout call." ) - obj._hdu.data *= dit * ndit + obj.header["EXPTIME"] = (dit * ndit, "Total accumulated integration time [s]") + obj.header['DIT'] = (dit, 'Integration time per exposure [s]') + obj.header['NDIT'] = ndit + obj.data = obj.data * dit * ndit + + return obj + + + +class ExposureLimit(Effect): + """Warn about expected charge per integration without changing the exposure. + Apply after integration, dark current and on-chip binning, before noise. + The configured usable well refers to a charge packet at this readout stage. + Software binning is deliberately later and does not change this limit. + """ + + required_keys = {"usable_well", "ndit"} + z_order = (819,) + + def __init__(self, **kwargs): + super().__init__(**kwargs) + check_keys(self.meta, self.required_keys, action="error") + + def apply_to(self, obj, **kwargs): + if isinstance(obj, Detector): + well = from_currsys(self.meta["usable_well"], self.cmds) + ndit = from_currsys(self.meta["ndit"], self.cmds) + peak = obj.data.max() / ndit + obj.header["PEAKDIT"] = (peak, "Expected peak charge per integration [electron]") + obj.meta["WELLDEP"] = well + obj.header["WELLDEP"] = (well, "Usable charge per integration [input electron]") + if peak > well: + warn(f"Detector {obj.det_id}: peak {peak:.1f} e per integration exceeds usable well {well:.1f} e; DIT/NDIT are unchanged.", UserWarning, stacklevel=2) return obj diff --git a/scopesim/effects/electronic/noise.py b/scopesim/effects/electronic/noise.py index 635b5f238..e9d9198a0 100644 --- a/scopesim/effects/electronic/noise.py +++ b/scopesim/effects/electronic/noise.py @@ -1,146 +1,308 @@ # -*- coding: utf-8 -*- """Any kinds of electronic or photonic noise.""" +from hashlib import sha256 from typing import ClassVar -from numbers import Real +from collections.abc import Mapping +from numbers import Real # matches int, float and all the numpy scalars import numpy as np -from astropy.io import fits +from numpy.typing import ArrayLike, NDArray -from .. import Effect -from ...detector import Detector -from ...utils import from_currsys, figure_factory, check_keys, real_colname -from . import logger +from ...utils import from_currsys, figure_factory +from . import ElectronicEffect, Detector, logger -class Bias(Effect): +# TODO: Potential refactoring in multiple effects here: +# The various __call__ now get a det_id, which is what the _get_... need +# for their lookup. Either pass the rng to __call__ instead of det_id, +# or call the _get_.. in there. Needs some consideration of outside +# callers, would they rather pass which one? And how about det_id = None? +# Also could refactor the _get_... (also elsewhere) to a abstract value +# plus the concrete key. + + +class Bias(ElectronicEffect): """Adds a constant bias level to readout.""" required_keys = {"bias"} z_order: ClassVar[tuple[int, ...]] = (855,) - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.meta.update(kwargs) - check_keys(self.meta, self.required_keys, action="error") - - def apply_to(self, obj, **kwargs): - if not isinstance(obj, Detector): - return obj + def __call__(self, data: ArrayLike) -> NDArray: + return data + self.bias_level - biaslevel = from_currsys(self.meta["bias"], self.cmds) - # Can't do in-place because of Quantization data type conflicts. - obj._hdu.data = obj._hdu.data + biaslevel + @property + def bias_level(self) -> float: + return from_currsys(self.meta["bias"], self.cmds) - return obj +class DarkCurrent(ElectronicEffect): + """ + required: dit, ndit, value + """ -class PoorMansHxRGReadoutNoise(Effect): - required_keys = {"noise_std", "n_channels", "ndit"} - z_order: ClassVar[tuple[int, ...]] = (811,) - report_plot_include: ClassVar[bool] = False - report_table_include: ClassVar[bool] = False + required_keys = {"value", "dit", "ndit"} + z_order: ClassVar[tuple[int, ...]] = (830,) - def __init__(self, **kwargs): - super().__init__(**kwargs) - params = { - "pedestal_fraction": 0.3, - "read_fraction": 0.4, - "line_fraction": 0.25, - "channel_fraction": 0.05, - "random_seed": "!SIM.random.seed", - } - self.meta.update(params) - self.meta.update(kwargs) + def __call__( + self, + data: ArrayLike, + dark_level: float, + dit: float, + ndit: int, + ) -> NDArray: + return data + dark_level * dit * ndit + + def _get_dark_level(self, det_id: int) -> float: + dark_level = from_currsys(self.meta["value"], self.cmds) + if isinstance(dark_level, Real): + return dark_level + if isinstance(dark_level, Mapping): + return from_currsys(dark_level[det_id], self.cmds) + raise TypeError( + f"<{self.__class__.__name__}>.meta['value'] must be either " + f"dict-like or scalar number, but is {dark_level}." + ) - check_keys(self.meta, self.required_keys, action="error") + def _apply_to_det(self, det: Detector) -> None: + logger.debug("Apply %s to %s", self.display_name, det) - def apply_to(self, det, **kwargs): - if not isinstance(det, Detector): - return det + # Dark level needs detector meta so can't go into __call__() + dark_level = self._get_dark_level(det.det_id) + dit = from_currsys(self.meta["dit"], self.cmds) + ndit = from_currsys(self.meta["ndit"], self.cmds) - self.meta["random_seed"] = from_currsys(self.meta["random_seed"], - self.cmds) - if self.meta["random_seed"] is not None: - np.random.seed(self.meta["random_seed"]) + det.data = self(det.data, dark_level, dit, ndit) - self.meta = from_currsys(self.meta, self.cmds) - ron_keys = ["noise_std", "n_channels", "channel_fraction", - "line_fraction", "pedestal_fraction", "read_fraction"] - ron_kwargs = {key: self.meta[key] for key in ron_keys} - ron_kwargs["image_shape"] = det._hdu.data.shape + def plot(self, det, **kwargs): + """Plot effect.""" + dit = from_currsys(self.meta["dit"], self.cmds) + ndit = from_currsys(self.meta["ndit"], self.cmds) + total_time = dit * ndit + times = np.linspace(0, 2*total_time, 10) + dtcr = self.apply_to(det) + dark_level = dtcr.data[0, 0] / total_time # just read one pixel + levels = dark_level * times + fig, ax = figure_factory() + ax.plot(times, levels, **kwargs) + ax.set_xlabel("time") + ax.set_ylabel("dark level") + return ax - ron_frame = _make_ron_frame(**ron_kwargs) - stacked_ron_frame = np.zeros_like(ron_frame) - for i in range(self.meta["ndit"]): - dx = np.random.randint(0, ron_frame.shape[1]) - dy = np.random.randint(0, ron_frame.shape[0]) - stacked_ron_frame += np.roll(ron_frame, (dy, dx), axis=(0, 1)) - # TODO: this .T is ugly. Work out where things are getting switched and remove it! - # Can't do in-place because of Quantization data type conflicts. - det._hdu.data = det._hdu.data + stacked_ron_frame.T +# TODO: Add tests for seed resolving, start with code from #975, but for +# basic_instrument, and work from there. +class RandomEffect(ElectronicEffect): + """Mixin class for Effects that need random seeds.""" - return det + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.meta["random_seed"] = "!SIM.random.seed" # Default + + @property + def root_seed(self) -> int: + """Resolve root seed from cmds, or use value from effect kwargs.""" + return from_currsys(self.meta["random_seed"], self.cmds) + + @classmethod + def cls_seed(cls) -> int: + """Generate reproducible seed from hash of class name.""" + digest = sha256(cls.__name__.encode("utf-8")).digest() + return int.from_bytes(digest[:16], "little") + + @property + def readout_seed(self) -> int: + """Readout ID, or 0 if not found in cmds.""" + return self.cmds.get("!OBS.roid", 0) if self.cmds is not None else 0 + + @property + def random_seed(self) -> int | None: + """Composite random seed, or None.""" + seed = [self.readout_seed, self.cls_seed(), self.root_seed] + # TODO: Consider removing this if root seed is resolved upstream! + if None in seed: + return None + return seed + + def create_rng(self, det_id: int | None = None) -> np.random.Generator: + """ + Instantiate np.random.Generator using composite seed. + + Parameters + ---------- + det_id : int | None, optional + Detector ID. If None (the default), detector ID is not included in + the composite seed, meaning results will look identical on all + detectors passed to the effect in the same readout. + + Returns + ------- + np.random.Generator + New Generator instance. + + """ + seed = self.random_seed + if det_id is not None and seed is not None: + seed = [det_id, *seed] + return np.random.default_rng(seed) - def plot(self, det, **kwargs): + def plot(self, det): """Plot effect image.""" - dtcr = self.apply_to(det) + detector = self.apply_to(det) fig, ax = figure_factory() - ax.imshow(dtcr.data, origin="lower") + ax.imshow(detector.data) + return ax def plot_hist(self, det, **kwargs): """Plot effect histogram.""" - dtcr = self.apply_to(det) + detector = self.apply_to(det) fig, ax = figure_factory() - ax.hist(dtcr.data.flatten()) + ax.hist(detector.data.flatten()) + return ax -class BasicReadoutNoise(Effect): +class BasicReadoutNoise(RandomEffect): """Readout noise computed as: ron * sqrt(NDIT).""" required_keys = {"noise_std", "ndit"} z_order: ClassVar[tuple[int, ...]] = (811,) + def __call__( + self, + data: ArrayLike, + ndit: int = 1, + det_id: int | None = None, + ) -> NDArray: + rng = self.create_rng(det_id) + return data + self._create_noise_frame(data.shape, rng, ndit) + + @property + def noise_std(self) -> float: + return from_currsys(self.meta["noise_std"], self.cmds) + + def _create_noise_frame( + self, + shape: tuple[int, ...], + rng: np.random.Generator, + ndit: int, + ) -> NDArray: + scale = self.noise_std * np.sqrt(float(ndit)) + return rng.normal(loc=0, scale=scale, size=shape) + + def _apply_to_det(self, det: Detector) -> None: + logger.debug("Apply %s to %s", self.display_name, det) + + ndit = from_currsys(self.meta["ndit"], self.cmds) + det.data = self(det.data, ndit, det_id=det.det_id) + + +class PoorMansHxRGReadoutNoise(BasicReadoutNoise): + required_keys = {"noise_std", "n_channels", "ndit"} + z_order: ClassVar[tuple[int, ...]] = (811,) + report_plot_include: ClassVar[bool] = False + report_table_include: ClassVar[bool] = False + def __init__(self, **kwargs): super().__init__(**kwargs) - self.meta["random_seed"] = "!SIM.random.seed" + params = { + "pedestal_fraction": 0.3, + "read_fraction": 0.4, + "line_fraction": 0.25, + "channel_fraction": 0.05, + } + self.meta.update(params) self.meta.update(kwargs) - check_keys(self.meta, self.required_keys, action="error") + @property + def n_channels(self) -> int: + return from_currsys(self.meta["n_channels"], self.cmds) + + def _create_noise_frame( + self, + shape: tuple[int, ...], + rng: np.random.Generator, + ndit: int, + ) -> NDArray: + ron_frame = self._make_ron_frame(rng, shape) + stacked_ron_frame = np.zeros_like(ron_frame) - def apply_to(self, det, **kwargs): - if not isinstance(det, Detector): - return det + for i in range(ndit): + stacked_ron_frame += np.roll( + ron_frame, + rng.integers((0, 0), ron_frame.shape), + axis=(0, 1), + ) - ndit = from_currsys(self.meta["ndit"], self.cmds) - ron = from_currsys(self.meta["noise_std"], self.cmds) - noise_std = ron * np.sqrt(float(ndit)) + return stacked_ron_frame + + def _make_ron_frame(self, rng, shape: tuple[int, ...]) -> NDArray: + self.meta = from_currsys(self.meta, self.cmds) # TODO: Is this needed? + channel_fraction = self.meta["channel_fraction"] + line_fraction = self.meta["line_fraction"] + pedestal_fraction = self.meta["pedestal_fraction"] + read_fraction = self.meta["read_fraction"] + + pixel_std = self.noise_std * (pedestal_fraction + read_fraction)**0.5 + if shape < (1024, 1024): + pixel = rng.normal(loc=0, scale=pixel_std, size=shape) + line = rng.normal( + loc=0, + scale=self.noise_std * line_fraction**0.5, + size=shape[1], + ) + else: + # TODO: Why bother with this pseudo random function? + pixel = self._pseudo_random_field(rng, scale=pixel_std, size=shape) + line = pixel[0] + + channel = np.repeat( + rng.normal( + loc=0, + scale=self.noise_std * channel_fraction**0.5, + size=self.n_channels, + ), + max(1, shape[0] // self.n_channels) + 1, + axis=0, + ) - random_seed = from_currsys(self.meta["random_seed"], self.cmds) - if random_seed is not None: - np.random.seed(random_seed) - # Can't do in-place because of Quantization data type conflicts. - det._hdu.data = det._hdu.data + np.random.normal( - loc=0, scale=noise_std, size=det._hdu.data.shape) + return (pixel + line) + channel[:shape[0], None] - return det + @staticmethod + def _pseudo_random_field( + rng, + scale: float = 1., + size: tuple[int, ...] = (1024, 1024), + ) -> NDArray: + n = 256 + image = np.zeros(size) + batch = rng.normal(loc=0, scale=scale, size=(2*n, 2*n)) + for y in range(0, size[1], n): + for x in range(0, size[0], n): + i, j = rng.integers(n, size=2) + dx, dy = min(size[0]-x, n), min(size[1]-y, n) + image[x:x+dx, y:y+dy] = batch[i:i+dx, j:j+dy] - def plot(self, det): - """Plot effect image.""" - dtcr = self.apply_to(det) - fig, ax = figure_factory() - ax.imshow(dtcr.data) + return image - def plot_hist(self, det, **kwargs): - """Plot effect histogram.""" - dtcr = self.apply_to(det) - fig, ax = figure_factory() - ax.hist(dtcr.data.flatten()) + +class ReadoutNoiseFromTable(BasicReadoutNoise): + """Interpolate measured read noise in electrons as a function of DIT [s].""" + + required_keys = {"filename", "dit", "ndit"} + + def apply_to(self, det, **kwargs): + if not isinstance(det, Detector): + return det + dit = from_currsys(self.meta["dit"], self.cmds) + times = self.table["dit"] + if not times[0] <= dit <= times[-1]: + raise ValueError(f"DIT {dit} s is outside the measured read-noise table") + self.meta["noise_std"] = np.interp(dit, times, self.table["read_noise"]) + det.header["RDNOISE"] = (self.meta["noise_std"], "Read noise per integration [electron]") + return super().apply_to(det, **kwargs) -# TODO: Is this really a "noise" effect? Sounds more like "electrons" tbh. -class PixelResponseNonUniformity(Effect): +class PixelResponseNonUniformity(RandomEffect): """Pixel Response Non-Uniformity (PRNU). Models the fixed pattern of per-pixel gain variations across the detector @@ -156,8 +318,6 @@ class PixelResponseNonUniformity(Effect): prnu_std : float or dict Standard deviation of the per-pixel gain distribution. - prnu_seed : int, fixed - include: "!DET.include_prnu" Example @@ -169,50 +329,61 @@ class PixelResponseNonUniformity(Effect): class: PixelResponseNonUniformity kwargs: prnu_std: 0.001 - prnu_seed: 42 include: "!DET.include_prnu" """ - required_keys: ClassVar[set] = set() + required_keys: ClassVar[set] = {"prnu_std"} z_order: ClassVar[tuple[int, ...]] = (805,) def __init__(self, **kwargs): super().__init__(**kwargs) - self.meta.update(kwargs) - self._gain_maps = {} # keyed by dtcr_id - - def apply_to(self, obj, **kwargs): - if not isinstance(obj, Detector): - return obj - - random_seed = from_currsys(self.meta.get("prnu_seed"), self.cmds) - id_key = real_colname("id", obj.meta) - dtcr_id = obj.meta[id_key] if id_key is not None else None - - prnu_std_meta = from_currsys(self.meta["prnu_std"], self.cmds) - if isinstance(prnu_std_meta, dict): - prnu_std = float(from_currsys(prnu_std_meta[dtcr_id], self.cmds)) - elif isinstance(prnu_std_meta, Real): - prnu_std = float(prnu_std_meta) - else: - raise TypeError( - ".meta['prnu_std'] must be a float " - f"or a dict keyed by detector ID, got {type(prnu_std_meta)}" + self._gain_maps = {} # keyed by det_id + + def __call__( + self, + data: ArrayLike, + prnu_std: float, + det_id: int | None = None, + ) -> NDArray: + if det_id not in self._gain_maps: + # TODO: Should not use roid here??? + rng = self.create_rng(det_id) + self._gain_maps[det_id] = rng.normal( + loc=1.0, scale=prnu_std, size=data.shape, ) - shape = obj._hdu.data.shape - if dtcr_id not in self._gain_maps: - rng = np.random.default_rng(random_seed) - self._gain_maps[dtcr_id] = rng.normal( - loc=1.0, scale=prnu_std, size=shape, - ) + return data * self._gain_maps[det_id] + + @property + def random_seed(self) -> int | None: + """Composite random seed, or None. + + Override from base class to remove ROID, because PRNU should be + identical in one observation. + """ + seed = [self.cls_seed(), self.root_seed] + # TODO: Consider removing this if root seed is resolved upstream! + if None in seed: + return None + return seed + + def _get_prnu_std(self, det_id: int) -> float: + prnu_std = from_currsys(self.meta["prnu_std"], self.cmds) + if isinstance(prnu_std, Real): + return prnu_std + if isinstance(prnu_std, Mapping): + return from_currsys(prnu_std[det_id], self.cmds) + raise TypeError( + f"<{self.__class__.__name__}>.meta['value'] must be either " + f"dict-like or scalar number, but is {prnu_std}." + ) - if self._gain_maps[dtcr_id].shape != shape: - raise ValueError("gain map shape mismatch") + def _apply_to_det(self, det: Detector) -> None: + logger.debug("Apply %s to %s", self.display_name, det) - obj._hdu.data = obj._hdu.data * self._gain_maps[dtcr_id] - return obj + prnu_std = self._get_prnu_std(det.det_id) + det.data = self(det.data, prnu_std) def plot(self, det_id=None): """Plot the cached gain map.""" @@ -225,46 +396,41 @@ def plot(self, det_id=None): im = ax.imshow(gain_map, origin="lower", aspect="auto", vmin=1 - dev, vmax=1 + dev) fig.colorbar(im, ax=ax, label="per-pixel gain") - return fig + return fig, ax -class ShotNoise(Effect): - z_order: ClassVar[tuple[int, ...]] = (820,) +class ShotNoise(RandomEffect): + """Poissonian photon noise. - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.meta["random_seed"] = "!SIM.random.seed" - self.meta.update(kwargs) + Notes + ----- - def apply_to(self, det, **kwargs): - if not isinstance(det, Detector): - return det + Numpy has a problem with generating Poisson distributions above certain + values. E.g. on linux, numpy.random.poisson(1e20) raises ValueError: lam + value too large. The value might be smaller on other (operating) systems. + + The poisson and normal distribution are basically the same + above ~100 counts: + poisson(x) ~= normal(mu=x, sigma=x**0.5) + + Therefore a limit of 1e7 is used, above which the Poisson distribution is + approximated with a normal distribution. + + Also, the normal distribution takes only 60% as long as the Poisson + distribution for large arrays. - self.meta["random_seed"] = from_currsys(self.meta["random_seed"], - self.cmds) - rng = np.random.default_rng(self.meta["random_seed"]) - - # numpy has a problem with generating Poisson distributions above - # certain values. E.g. on linux, numpy.random.poisson(1e20) raises - # ValueError: lam value too large - # The value might be smaller on other (operating) systems. - # - # The poisson and normal distribution are basically the same - # above ~100 counts: - # poisson(x) ~= normal(mu=x, sigma=x**0.5) - # - # Therefore a limit of 1e7 is used, above which the Poisson - # distribution is approximated with a normal distribution. - # - # Also, the normal distribution takes only 60% as long as the - # poisson distribution for large arrays. - # - # Special values should be handled with care: - # - Negative values are mapped to 0; there cannot be negative flux. - # - numpy.nan are implicitly passed through the normal distribution; - # because the Poisson distribution cannot handle them. - - data = det._hdu.data + Special values should be handled with care: + - Negative values are mapped to 0; there cannot be negative flux. + - numpy.nan are implicitly passed through the normal distribution; + because the Poisson distribution cannot handle them. + """ + z_order: ClassVar[tuple[int, ...]] = (820,) + + def _apply_to_det(self, det): + det.data = self(det.data, det_id=det.det_id) + + def __call__(self, data: ArrayLike, det_id: int | None = None) -> NDArray: + rng = self.create_rng(det_id) # Check if there are negative values in the data. values_negative = data < 0 @@ -279,108 +445,43 @@ def apply_to(self, det, **kwargs): # Apply a normal distribution to the high values. values_high = ~values_low - data[values_high] = rng.normal(data[values_high], np.sqrt(data[values_high])) - - new_imagehdu = fits.ImageHDU( - data=data, - header=det._hdu.header, + data[values_high] = rng.normal( + loc=data[values_high], + scale=np.sqrt(data[values_high]), ) - det._hdu = new_imagehdu - return det + return data - def plot(self, det): - """Plot effect image.""" - dtcr = self.apply_to(det) - fig, ax = figure_factory() - ax.imshow(dtcr.data) - def plot_hist(self, det, **kwargs): - """Plot effect histogram.""" - dtcr = self.apply_to(det) - fig, ax = figure_factory() - ax.hist(dtcr.data.flatten()) +class AvalancheShotNoise(ShotNoise): + """Input-referred photon multiplication noise, with independent dark charge. -class DarkCurrent(Effect): - """ - required: dit, ndit, value + Conditional on N detected photons, a gamma variate of mean N and variance + N(F-1) gives total photon variance F*mu. F is a variance multiplier, not + an RMS factor. The adopted input-referred dark term remains Poisson. + Apply after ExposureIntegration and DarkCurrent, before read noise. """ - required_keys = {"value", "dit", "ndit"} - z_order: ClassVar[tuple[int, ...]] = (830,) - - def __init__(self, **kwargs): - super().__init__(**kwargs) - check_keys(self.meta, self.required_keys, action="error") - - def apply_to(self, obj, **kwargs): - if not isinstance(obj, Detector): - return obj - - if isinstance(from_currsys(self.meta["value"], self.cmds), dict): - dtcr_id = obj.meta[real_colname("id", obj.meta)] - dark = from_currsys(self.meta["value"][dtcr_id], self.cmds) - elif isinstance(from_currsys(self.meta["value"], self.cmds), Real): - dark = float(from_currsys(self.meta["value"], self.cmds)) - else: - raise ValueError(".meta['value'] must be either " - f"dict or float, but is {self.meta['value']}") - + required_keys = {"excess_noise_factor", "dark_current", "dit", "ndit"} + + @property + def excess_noise_factor(self): + return from_currsys(self.meta["excess_noise_factor"], self.cmds) + + def __call__(self, data, dark_charge=0., det_id=None): + factor = self.excess_noise_factor + if factor < 1: + raise ValueError("The avalanche variance multiplier must be at least one.") + if factor == 1: + return super().__call__(data, det_id=det_id) + rng = self.create_rng(det_id) + photons = rng.poisson(data - dark_charge) + signal = rng.gamma(photons / (factor - 1), factor - 1) + return signal + rng.poisson(dark_charge, size=data.shape) + + def _apply_to_det(self, det): + dark = from_currsys(self.meta["dark_current"], self.cmds) dit = from_currsys(self.meta["dit"], self.cmds) ndit = from_currsys(self.meta["ndit"], self.cmds) - - # Can't do in-place because of Quantization data type conflicts. - obj._hdu.data = obj._hdu.data + dark * dit * ndit - - return obj - - def plot(self, det, **kwargs): - """Plot effect.""" - dit = from_currsys(self.meta["dit"], self.cmds) - ndit = from_currsys(self.meta["ndit"], self.cmds) - total_time = dit * ndit - times = np.linspace(0, 2*total_time, 10) - dtcr = self.apply_to(det) - dark_level = dtcr.data[0, 0] / total_time # just read one pixel - levels = dark_level * times - fig, ax = figure_factory() - ax.plot(times, levels, **kwargs) - ax.set_xlabel("time") - ax.set_ylabel("dark level") - - -def _pseudo_random_field(scale=1, size=(1024, 1024)): - n = 256 - image = np.zeros(size) - batch = np.random.normal(loc=0, scale=scale, size=(2*n, 2*n)) - for y in range(0, size[1], n): - for x in range(0, size[0], n): - i, j = np.random.randint(n, size=2) - dx, dy = min(size[0]-x, n), min(size[1]-y, n) - image[x:x+dx, y:y+dy] = batch[i:i+dx, j:j+dy] - - return image - - -def _make_ron_frame(image_shape, noise_std, n_channels, channel_fraction, - line_fraction, pedestal_fraction, read_fraction): - shape = image_shape - w_chan = max(1, shape[0] // n_channels) - - pixel_std = noise_std * (pedestal_fraction + read_fraction)**0.5 - line_std = noise_std * line_fraction**0.5 - if shape < (1024, 1024): - pixel = np.random.normal(loc=0, scale=pixel_std, size=shape) - line = np.random.normal(loc=0, scale=line_std, size=shape[1]) - else: - pixel = _pseudo_random_field(scale=pixel_std, size=shape) - line = pixel[0] - - channel_std = noise_std * channel_fraction**0.5 - channel = np.repeat(np.random.normal(loc=0, scale=channel_std, - size=n_channels), w_chan + 1, axis=0) - - ron_frame = (pixel + line).T + channel[:shape[0]] - - return ron_frame + det.data = self(det.data, dark * dit * ndit, det_id=det.det_id) diff --git a/scopesim/effects/electronic/pixels.py b/scopesim/effects/electronic/pixels.py index 5c8c8c8cc..efb966cf2 100644 --- a/scopesim/effects/electronic/pixels.py +++ b/scopesim/effects/electronic/pixels.py @@ -3,13 +3,13 @@ from typing import ClassVar -from .. import Effect -from ...detector import Detector -from ...utils import from_currsys, figure_factory, check_keys, real_colname -from .. import logger +from numpy.typing import ArrayLike, NDArray +from ...utils import from_currsys, figure_factory +from . import ElectronicEffect, Detector, logger -class ReferencePixelBorder(Effect): + +class ReferencePixelBorder(ElectronicEffect): """Remove signal from reference pixels. Detectors often have a number of rows and columns around the edges masked. @@ -46,33 +46,31 @@ def __init__(self, **kwargs): raise ValueError( "Parameter 'border' must have exactly four values.") - def apply_to(self, obj, **kwargs): - """Mask border pixels.""" - if not isinstance(obj, Detector): - logger.warning( - "ReferencePixelBorder: got non-detector object: %s", type(obj)) - return obj - - logger.info(f"Applying border {from_currsys(self.meta['border'])}") - if hasattr(self.meta["border"], "dic"): - dtcr_id = obj.meta[real_colname("id", obj.meta)] - border = self.meta["border"].dic[dtcr_id] - elif isinstance(self.meta["border"], list): - border = self.meta["border"] - else: - raise ValueError( - ".meta['border'] must be either " - f"dict or list, but is {self.meta['border']}") - + def __call__(self, data: ArrayLike, border: dict[int, int]) -> NDArray: if border[0] > 0: - obj.data[:border[0], :] = 0 + data[:border[0], :] = 0 if border[1] > 0: - obj.data[:, :border[1]] = 0 + data[:, :border[1]] = 0 if border[2] > 0: - obj.data[-border[2]:, :] = 0 + data[-border[2]:, :] = 0 if border[3] > 0: - obj.data[:, -border[3]:] = 0 - return obj + data[:, -border[3]:] = 0 + return data + + def _get_border(self, det_id: int) -> dict[int, int]: + if hasattr(self.meta["border"], "dic"): + return self.meta["border"].dic[det_id] + if isinstance(self.meta["border"], list): + return self.meta["border"] + raise TypeError( + f"{self.__class__.__name__}.meta['border'] must be either " + f"dict or list, but is {self.meta['border']}") + + def _apply_to_det(self, det: Detector) -> None: + """Subclasses can override if more params needed in call.""" + logger.debug("Apply %s to %s", self.display_name, det) + logger.info(f"Applying border {from_currsys(self.meta['border'])}") + det.data = self(det.data, self._get_border(det.det_id)) def plot(self, det, **kwargs): """Show the masked detector image.""" @@ -90,44 +88,88 @@ def __str__(self) -> str: return msg -class BinnedImage(Effect): - required_keys = {"bin_size"} - z_order: ClassVar[tuple[int, ...]] = (870,) +class BinnedImageBase(ElectronicEffect): + """Base class for binning effects.""" - def __init__(self, **kwargs): - super().__init__(**kwargs) - check_keys(self.meta, self.required_keys, action="error") + z_order: ClassVar[tuple[int, ...]] = (870,) - def apply_to(self, det, **kwargs): - if not isinstance(det, Detector): - return det + def __call__(self, data: ArrayLike) -> NDArray: + bx, by = self.bin_factors + ny, nx = data.shape + return data.reshape(ny//by, by, nx//bx, bx).sum(axis=(1, 3)) + + def bin_header(self, native_header): + """WCS of the summed pixel blocks, also used by trace coordinates.""" + bx, by = self.bin_factors + if not isinstance(bx, int) or not isinstance(by, int) or min(bx, by) < 1: + raise ValueError('Bin factors must be positive integers') + if 'allowed_bins' in self.meta and [bx, by] not in self.meta['allowed_bins']: + raise ValueError(f'Unsupported binning: {bx} x {by}') + header = native_header.copy() + for axis, factor in ((1, bx), (2, by)): + if header[f'NAXIS{axis}'] % factor: + raise ValueError('Binning must divide the full detector dimensions') + header[f'NAXIS{axis}'] //= factor + for suffix in [''] + list('ABCDEFGHIJKLMNOPQRSTUVWXYZ'): + if f'CRPIX1{suffix}' not in header: + continue + for axis, factor in ((1, bx), (2, by)): + header[f'CRPIX{axis}{suffix}'] = (header[f'CRPIX{axis}{suffix}'] - .5)/factor + .5 + if f'CD1_1{suffix}' in header: + for i in (1, 2): + for j, factor in ((1, bx), (2, by)): + header[f'CD{i}_{j}{suffix}'] *= factor + else: + header[f'CDELT1{suffix}'] *= bx + header[f'CDELT2{suffix}'] *= by + if f'PC1_2{suffix}' in header: + header[f'PC1_2{suffix}'] *= by/bx + if f'PC2_1{suffix}' in header: + header[f'PC2_1{suffix}'] *= bx/by + header['BINX'], header['BINY'] = bx, by + return header + + def _apply_to_det(self, det): + header = self.bin_header(det.header) + det.data = self(det.data) + det.header.update(header) + det.meta.update(header) + + +class BinnedImage(BinnedImageBase): + required_keys = {"bin_size"} - bs = from_currsys(self.meta["bin_size"], self.cmds) - image = det._hdu.data - h, w = image.shape - new_image = image.reshape((h//bs, bs, w//bs, bs)) - det._hdu.data = new_image.sum(axis=3).sum(axis=1) + @property + def bin_factors(self): + return self.bin_size, self.bin_size - return det + @property + def bin_size(self) -> int: + return from_currsys(self.meta["bin_size"], self.cmds) -class UnequalBinnedImage(Effect): +class UnequalBinnedImage(BinnedImageBase): required_keys = {"binx","biny"} - z_order: ClassVar[tuple[int, ...]] = (870,) - def __init__(self, **kwargs): - super().__init__(**kwargs) - check_keys(self.meta, self.required_keys, action="error") + @property + def bin_factors(self): + return self.binx, self.biny + + @property + def binx(self) -> int: + return from_currsys(self.meta["binx"], self.cmds) + + @property + def biny(self) -> int: + return from_currsys(self.meta["biny"], self.cmds) - def apply_to(self, det, **kwargs): - if not isinstance(det, Detector): - return det - bx = from_currsys(self.meta["binx"], self.cmds) - by = from_currsys(self.meta["biny"], self.cmds) - image = det._hdu.data - h, w = image.shape - new_image = image.reshape((h//bx, bx, w//by, by)) - det._hdu.data = new_image.sum(axis=3).sum(axis=1) +class OnChipBinning(UnequalBinnedImage): + """Sum charge in FITS x/y pixel blocks before adding read noise. + + Unlike a software sum after readout, each output pixel receives one read + noise draw. Configure this effect between charge integration and read noise. + WCS reference pixels refer to the centers of the new pixel blocks. + """ - return det + z_order = (815,) diff --git a/scopesim/effects/fits_headers.py b/scopesim/effects/fits_headers.py index 9110f3331..987117edf 100644 --- a/scopesim/effects/fits_headers.py +++ b/scopesim/effects/fits_headers.py @@ -153,6 +153,9 @@ class ExtraFitsKeywords(Effect): header['HIERARCH ESO DET DIT'] = (5, '[s] exposure length') header['EXTNAME'] = "DET2.DATA" + A group may additionally specify ``detector_id``. It then applies only to + HDULists containing that ``DET_ID``, including their primary headers. + Resolved and un-resolved keywords --------------------------------- ScopeSim uses bang-strings to resolve global parameters. @@ -285,6 +288,12 @@ def apply_to(self, hdul, **kwargs): return hdul for dic in self.dict_list: + # Each detector array produces its own HDUList. A detector selector + # also permits arm-specific metadata in that array's primary HDU. + if "detector_id" in dic: + detector_ids = [hdu.header["DET_ID"] for hdu in hdul if "DET_ID" in hdu.header] + if dic["detector_id"] not in detector_ids: + continue resolved = flatten_dict(dic.get("keywords", {}), resolve=True, optics_manager=opt_train) unresolved = flatten_dict(dic.get("unresolved_keywords", {})) diff --git a/scopesim/effects/illumination.py b/scopesim/effects/illumination.py index d4ff12bfe..5179aa7db 100644 --- a/scopesim/effects/illumination.py +++ b/scopesim/effects/illumination.py @@ -212,6 +212,7 @@ def _effect_from_spec(spec, cmds=None): def _detector_qe_from_spec(spec, filename=None, cmds=None): + spec = from_currsys(spec, cmds) if spec is not None: if isinstance(spec, str): return SpectralQuantumEfficiency(filename=spec, cmds=cmds) @@ -538,6 +539,7 @@ class ImagePlaneBackground(Effect): trace list. The supplied value or model should already include downstream throughput and detector QE. For tapered QE maps, use an average or otherwise representative positional QE when deriving this non-dispersed background. + A FITS ``filename`` and ``extname`` can supply an already registered map. """ z_order: ClassVar[tuple[int, ...]] = (760,) @@ -562,7 +564,7 @@ def apply_to(self, obj, **kwargs): return obj shape = obj.hdu.data.shape - if self._map is None or shape != self._map_shape: + if self._file is not None or self._map is None or shape != self._map_shape: self._map = self._make_map(shape) self._map_shape = shape @@ -570,7 +572,13 @@ def apply_to(self, obj, **kwargs): return obj def _make_map(self, shape): - if self._model is None: + if self._file is not None: + # The calibration image must be registered to this image plane. + extname = str(from_currsys(self.meta["extname"], self.cmds)) + background = self._file[extname].data + if background.shape != shape: + raise ValueError("Background calibration shape differs from the image plane") + elif self._model is None: value = from_currsys(self.meta["value"], self.cmds) background = np.full(shape, value, dtype=np.float32) else: diff --git a/scopesim/effects/metis_lms_trace_list.py b/scopesim/effects/metis_lms_trace_list.py index 877360da0..64f6f460d 100644 --- a/scopesim/effects/metis_lms_trace_list.py +++ b/scopesim/effects/metis_lms_trace_list.py @@ -68,109 +68,108 @@ def __init__(self, **kwargs): self.slicelist["top"].max() - self.slicelist["bottom"].min()]) - def apply_to(self, obj, **kwargs): - """See parent docstring.""" - if isinstance(obj, FovVolumeList): - # Create a single volume that covers the aperture and - # the maximum wavelength range of LMS - volumes = [spectral_trace.fov_grid() - for spectral_trace in self.spectral_traces.values()] - - wave_min = min(vol["wave_min"] for vol in volumes) - wave_max = max(vol["wave_max"] for vol in volumes) - extracted_vols = obj.extract(axes=["wave"], - edges=([[wave_min, wave_max]])) - obj.volumes = extracted_vols - - if isinstance(obj, FieldOfView): - # Application to field of view - logger.debug("Executing %s, FoV", self.meta['name']) - if obj.hdu is not None and obj.hdu.header["NAXIS"] == 3: - obj.cube = obj.hdu - elif obj.hdu is None and obj.cube is None: - obj.cube = obj.make_hdu() - - fovcube = obj.cube.data - n_z, n_y, n_x = fovcube.shape - fovwcs = WCS(obj.cube.header) - # Make this linear to avoid jump at RA 0 deg - fovwcs.wcs.ctype = ["LINEAR", "LINEAR", fovwcs.wcs.ctype[2]] - fovwcs_spat = fovwcs.sub(2) - ny_slice = self.meta["slice_samples"] - - # Spatial pixel coordinates - xslice, yslice = np.meshgrid(np.arange(n_x), - np.arange(ny_slice)) - - fovimage = np.zeros((obj.detector_header["NAXIS2"], - obj.detector_header["NAXIS1"]), - dtype=np.float32) - - for sptid, spt in tqdm(self.spectral_traces.items(), - desc=" Spectral Traces", position=2): - ymin = spt.meta["fov"]["y_min"] - ymax = spt.meta["fov"]["y_max"] - - slicewcs = fovwcs.deepcopy() - - slicewcs.wcs.ctype = ["LINEAR", "LINEAR", - slicewcs.wcs.ctype[2]] - slicewcs.wcs.crpix[1] = (ny_slice + 1) / 2 - slicewcs.wcs.crval[1] = (ymin + ymax) / 2 / 3600 - slicewcs.wcs.cdelt[1] = (ymax - ymin) / ny_slice / 3600 - slicewcs_spat = slicewcs.sub(2) - - # World coordinates for the slice - xworld, yworld = slicewcs_spat.all_pix2world(xslice, - yslice, 0) - # FOV pixel coordinates for the slice - xfov, yfov = fovwcs_spat.all_world2pix(xworld, yworld, 0) - - slicecube = np.zeros((n_z, ny_slice, n_x)) - for islice in range(n_z): - ifov = RectBivariateSpline( - np.arange(n_y), - np.arange(n_x), - fovcube[islice], - kx=1, - ky=1, - ) - slicecube[islice] = ifov(yfov, xfov, grid=False) - - slicefov = FieldOfView3D( - obj.header, - [obj.meta["wave_min"], obj.meta["wave_max"]], - ) - slicefov.detector_header = obj.detector_header - slicefov.meta["xi_min"] = obj.meta["xi_min"] - slicefov.meta["xi_max"] = obj.meta["xi_max"] - slicefov.meta["trace_id"] = sptid - slicefov.cube = fits.ImageHDU( - header=slicewcs.to_header(), - data=slicecube, - ) - # slicefov.cube.writeto(f"slicefov_{sptid}.fits", overwrite=True) + # Overrides for the two apply_to() methods + def apply_to_fovvolumelist(self, obj): + """Setup - try: - # If footprint is outside FOV, this will assign None to the hdu - slicefov.hdu = spt.map_spectra_to_focal_plane(slicefov) - except ValueError as err: # xlim_mm is None - logger.error(err) - continue + Create a single volume that covers the aperture and + the maximum wavelength range of LMS. + """ + logger.debug("Executing %s: Setup", self.meta['name']) + volumes = [spectral_trace.fov_grid() + for spectral_trace in self.spectral_traces.values()] + + wave_min = min(vol["wave_min"] for vol in volumes) + wave_max = max(vol["wave_max"] for vol in volumes) + extracted_vols = obj.extract(axes=["wave"], + edges=([[wave_min, wave_max]])) + obj.volumes = extracted_vols + return obj - if slicefov.hdu is None: # footprint outside FOV - continue + def apply_to_fov(self, obj): + """Application to field of view""" + + logger.debug("Executing %s, FoV", self.meta['name']) + if obj.hdu is not None and obj.hdu.header["NAXIS"] == 3: + obj.cube = obj.hdu + elif obj.hdu is None and obj.cube is None: + obj.cube = obj.make_hdu() + + fovcube = obj.cube.data + n_z, n_y, n_x = fovcube.shape + fovwcs = WCS(obj.cube.header) + # Make this linear to avoid jump at RA 0 deg + fovwcs.wcs.ctype = ["LINEAR", "LINEAR", fovwcs.wcs.ctype[2]] + fovwcs_spat = fovwcs.sub(2) + ny_slice = self.meta["slice_samples"] + + # Spatial pixel coordinates + xslice, yslice = np.meshgrid(np.arange(n_x), + np.arange(ny_slice)) + + fovimage = np.zeros((obj.detector_header["NAXIS2"], + obj.detector_header["NAXIS1"]), + dtype=np.float32) + + for sptid, spt in tqdm(self.spectral_traces.items(), + desc=" Spectral Traces", position=2): + ymin = spt.meta["fov"]["y_min"] + ymax = spt.meta["fov"]["y_max"] + + slicewcs = fovwcs.deepcopy() + + slicewcs.wcs.ctype = ["LINEAR", "LINEAR", + slicewcs.wcs.ctype[2]] + slicewcs.wcs.crpix[1] = (ny_slice + 1) / 2 + slicewcs.wcs.crval[1] = (ymin + ymax) / 2 / 3600 + slicewcs.wcs.cdelt[1] = (ymax - ymin) / ny_slice / 3600 + slicewcs_spat = slicewcs.sub(2) + + # World coordinates for the slice + xworld, yworld = slicewcs_spat.all_pix2world(xslice, + yslice, 0) + # FOV pixel coordinates for the slice + xfov, yfov = fovwcs_spat.all_world2pix(xworld, yworld, 0) + + slicecube = np.zeros((n_z, ny_slice, n_x)) + for islice in range(n_z): + ifov = RectBivariateSpline(np.arange(n_y), + np.arange(n_x), + fovcube[islice], kx=1, ky=1) + slicecube[islice] = ifov(yfov, xfov, grid=False) + + slicefov = FieldOfView3D(obj.header, + [obj.meta["wave_min"], + obj.meta["wave_max"]]) + slicefov.detector_header = obj.detector_header + slicefov.meta["xi_min"] = obj.meta["xi_min"] + slicefov.meta["xi_max"] = obj.meta["xi_max"] + slicefov.meta["eta_min"] = ymin * u.arcsec + slicefov.meta["eta_max"] = ymax * u.arcsec + slicefov.meta["trace_id"] = sptid + slicefov.cube = fits.ImageHDU(header=slicewcs.to_header(), + data=slicecube) + # slicefov.cube.writeto(f"slicefov_{sptid}.fits", overwrite=True) + try: + # If footprint is outside FOV, this will assign None to the hdu + slicefov.hdu = spt.map_spectra_to_focal_plane(slicefov) + except ValueError as err: # xlim_mm is None + logger.error(err) + continue - sxmin = slicefov.hdu.header["XMIN"] - sxmax = slicefov.hdu.header["XMAX"] - symin = slicefov.hdu.header["YMIN"] - symax = slicefov.hdu.header["YMAX"] - fovimage[symin:symax, sxmin:sxmax] += slicefov.hdu.data + if slicefov.hdu is None: # footprint outside FOV + continue - obj.hdu = fits.ImageHDU(data=fovimage, header=obj.detector_header) + sxmin = slicefov.hdu.header["XMIN"] + sxmax = slicefov.hdu.header["XMAX"] + symin = slicefov.hdu.header["YMIN"] + symax = slicefov.hdu.header["YMAX"] + fovimage[symin:symax, sxmin:sxmax] += slicefov.hdu.data + obj.hdu = fits.ImageHDU(data=fovimage, header=obj.detector_header) return obj + def make_spectral_traces(self): """Compute the transformations by interpolation.""" # nslice = len(self._file["Aperture List"].data) @@ -325,6 +324,7 @@ class MetisLMSSpectralTrace(SpectralTrace): """SpectralTrace for the METIS LM spectrograph.""" _class_params = { + **SpectralTrace._class_params, "naxis1": 122, "nslice": 28, "slicewidth": 0.0207, # arcsec @@ -396,7 +396,7 @@ def get_waverange(self, det_mm_lims): lam0 = from_currsys(self.meta["wavelen"], self.cmds) xi0 = 0. - ymid = self.xilam2y(xi0, lam0)[0] # estimate y level of trace + ymid = self.xilam2y(xi0, lam0) # scalar input gives a scalar trace coordinate waverange = self.xy2lam(np.array([xmin, xmax]), np.array([ymid, ymid]), grid=False) diff --git a/scopesim/effects/micado_ifu_trace_list.py b/scopesim/effects/micado_ifu_trace_list.py new file mode 100644 index 000000000..42eaea14e --- /dev/null +++ b/scopesim/effects/micado_ifu_trace_list.py @@ -0,0 +1,222 @@ +# -*- coding: utf-8 -*- +"""SpectralTraceList and SpectralTrace for the MICADO IFU.""" + +import copy +import warnings + +from tqdm.auto import tqdm +import numpy as np +from scipy.interpolate import RectBivariateSpline + +from astropy.io import fits +from astropy.io import ascii as ioascii +from astropy.table import Table +from astropy.wcs import WCS +from astropy import units as u + +from ..utils import from_currsys, find_file, quantify, get_logger +from .spectral_trace_list import SpectralTraceList +from .spectral_trace_list_utils import SpectralTrace +from .spectral_trace_list_utils import Transform2D +from .spectral_trace_list_utils import make_image_interpolations, det_offset +from .apertures import ApertureMask +from .ter_curves import TERCurve +from ..optics.fov import FieldOfView, FieldOfView3D +from ..optics.fov_volume_list import FovVolumeList + +logger = get_logger(__name__) + + +class MicadoIFUSpectralTraceList(SpectralTraceList): + """SpectralTraceList for the MICADO IFU.""" + + _class_params = { + "naxis1": 112, + "nslice": 32, + "slicewidth": 0.012, # arcsec + "wave_min": 1.5, + "wave_max": 2.5, # todo: do not hardcode + } + + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.wave_min = self.meta["wave_min"] + self.wave_max = self.meta["wave_max"] + + def apply_to_fovvolumelist(self, obj): + """ + Setup + + Create a single volume that covers the aperture and the maximum + wavelength range of the MICADO IFU. + """ + logger.debug("Executing %s: Setup", self.meta['name']) + volumes = [spectral_trace.fov_grid() + for spectral_trace in self.spectral_traces.values()] + + wave_min = min(vol["wave_min"] for vol in volumes) + wave_max = max(vol["wave_max"] for vol in volumes) + extracted_vols = obj.extract(axes=["wave"], + edges=([[wave_min, wave_max]])) + obj.volumes = extracted_vols + self.wave_min = wave_min + self.wave_max = wave_max + return obj + + def apply_to_fov(self, obj): + """Apply to FieldOfView""" + + logger.debug("Executing %s, FoV", self.meta['name']) + if obj.hdu is not None and obj.hdu.header["NAXIS"] == 3: + obj.cube = obj.hdu + elif obj.hdu is None and obj.cube is None: + obj.cube = obj.make_hdu() + + fovcube = obj.cube.data + n_z, n_y, n_x = fovcube.shape + fovwcs = WCS(obj.cube.header) + # Make this linear to avoid jump at RA 0 deg + fovwcs.wcs.ctype = ["LINEAR", "LINEAR", fovwcs.wcs.ctype[2]] + fovwcs_spat = fovwcs.sub(2) + ny_slice = self.meta["slice_samples"] + + # Spatial pixel coordinates + xslice, yslice = np.meshgrid(np.arange(n_x), + np.arange(ny_slice)) + + fovimage = np.zeros((obj.detector_header["NAXIS2"], + obj.detector_header["NAXIS1"]), + dtype=np.float32) + + for sptid, spt in tqdm(self.spectral_traces.items(), + desc=" Spectral Traces", position=2): + ymin = spt.meta["fov"]["y_min"] + ymax = spt.meta["fov"]["y_max"] + + slicewcs = fovwcs.deepcopy() + + slicewcs.wcs.ctype = ["LINEAR", "LINEAR", + slicewcs.wcs.ctype[2]] + slicewcs.wcs.crpix[1] = (ny_slice + 1) / 2 + # TODO: here we silently assume that ymin, ymax are in arcsec + # and wcs in degrees + slicewcs.wcs.crval[1] = (ymin + ymax) / 2 / 3600 + slicewcs.wcs.cdelt[1] = (ymax - ymin) / ny_slice / 3600 + slicewcs_spat = slicewcs.sub(2) + + # World coordinates for the slice + xworld, yworld = slicewcs_spat.all_pix2world(xslice, + yslice, 0) + # FOV pixel coordinates for the slice + xfov, yfov = fovwcs_spat.all_world2pix(xworld, yworld, 0) + + slicecube = np.zeros((n_z, ny_slice, n_x)) + for islice in range(n_z): + # TODO: replace by explicit bilinear interpolation + ifov = RectBivariateSpline(np.arange(n_y), + np.arange(n_x), + fovcube[islice], kx=1, ky=1) + slicecube[islice] = ifov(yfov, xfov, grid=False) + + slicefov = FieldOfView3D(obj.header, + [obj.meta["wave_min"], + obj.meta["wave_max"]]) + slicefov.detector_header = obj.detector_header + slicefov.meta["xi_min"] = spt.meta["fov"]["xi_min"] + slicefov.meta["xi_max"] = spt.meta["fov"]["xi_max"] + slicefov.meta["trace_id"] = sptid + slicefov.cube = fits.ImageHDU(header=slicewcs.to_header(), + data=slicecube) + + try: + slicefov.hdu = spt.map_spectra_to_focal_plane(slicefov) + except ValueError as err: + logger.debug(err) + + if slicefov.hdu is not None: + sxmin = slicefov.hdu.header["XMIN"] + sxmax = slicefov.hdu.header["XMAX"] + symin = slicefov.hdu.header["YMIN"] + symax = slicefov.hdu.header["YMAX"] + fovimage[symin:symax, sxmin:sxmax] += slicefov.hdu.data + + obj.hdu = fits.ImageHDU(data=fovimage, header=obj.detector_header) + return obj + + def make_spectral_traces(self): + """Make a spectral trace for each combination of order and aperture""" + self.ext_data = self._file[0].header["EDATA"] + self.ext_cat = self._file[0].header["ECAT"] + self.catalog = Table(self._file[self.ext_cat].data) + self.slicelist = self._file["Aperture List"].data + spec_traces = {} + for row in self.catalog: + if row["image_plane_id"] == -99: + continue + params = {col: row[col] for col in row.colnames} + params.update(self.meta) + trace_hdu = self._file[row["extension_id"]] + + for spslice, apid in enumerate(self.slicelist["id"]): + specid = f"{row['description']}_{apid:02d}" + thistrace = MicadoIFUSpectralTrace( + trace_hdu, + self.slicelist, + spslice, + **params) + thistrace.meta["trace_id"] = specid + spec_traces[specid] = thistrace + + self.spectral_traces = spec_traces + + +class MicadoIFUSpectralTrace(SpectralTrace): + """SpectralTrace for the MICADO IFU""" + + _class_params = { + "naxis1": 112, + "nslice": 32, + "slicewidth": 0.012, # arcsec + "pixsize": 0.015, # mm + } + + + def __init__(self, trace_tbl, aplist, spslice, cmds=None, **kwargs): + super().__init__(trace_tbl, **kwargs) + + self._set_dispersion(self.wave_min, self.wave_max) + # Provisional + self.aplist = aplist + self.meta["slice"] = spslice + self.meta["fov"] = self.fov_grid() + + lam = np.linspace(self.wave_min, self.wave_max, 1001) * u.um + off_y = aplist['offset'][spslice] # offset on detector in mm + dlam = np.median(self.dlam_per_pix(lam)) * off_y / self.meta["pixsize"] + self.xy2lam.posttransform = (det_offset, {"offset": -dlam}) + self.xilam2x.pretransform_y = (det_offset, {"offset": -dlam}) + self.xilam2y.pretransform_y = (det_offset, {"offset": -dlam}) + + def fov_grid(self): + """ + Provide information on the source space volume required by the effect. + """ + aperture = self.aplist[self.meta["slice"]] + x_min = aperture["left"] + x_max = aperture["right"] + y_min = aperture["bottom"] + y_max = aperture["top"] + xi_min = aperture["xi1"] + xi_max = aperture["xi2"] + + # ..todo: just a hack - xi and x are the same except xi is a quantity + xi_min = quantify(xi_min, u.arcsec) + xi_max = quantify(xi_max, u.arcsec) + + return {"x_min": x_min, "x_max": x_max, + "y_min": y_min, "y_max": y_max, + "xi_min": xi_min, "xi_max": xi_max, + "wave_min": self.meta["wave_min"], + "wave_max": self.meta["wave_max"], + "trace_id": self.trace_id} diff --git a/scopesim/effects/mosaic_trace_list.py b/scopesim/effects/mosaic_trace_list.py index 8b23e3ad3..c74c9d214 100644 --- a/scopesim/effects/mosaic_trace_list.py +++ b/scopesim/effects/mosaic_trace_list.py @@ -239,7 +239,7 @@ class MosaicOutputFormat(MosaicSpectralTraceList): "image": The output is a (pseudo-)detector image where the spectrum for each fibre occupies a row. - .. versionadded:: PLACEHOLDER_NEXT_RELEASE_VERSION + .. versionadded:: 0.12.0 """ required_keys = {"filename", "format"} @@ -253,8 +253,11 @@ def apply_to(self, det, **kwargs): type(det)) return det - image = det._hdu.data - detwcs = WCS(det._hdu.header, key="D") + image = det.data + detwcs = WCS(det.header, key="D") + # ExposureIntegration has already recorded the actual accumulated time. + # New table HDUs need those cards, but not the detector-image WCS. + exposure_header = fits.Header(card for card in det.header.cards if card.keyword in ("DIT", "NDIT", "EXPTIME")) output_format = from_currsys(self.meta["format"], self.cmds) @@ -271,8 +274,9 @@ def apply_to(self, det, **kwargs): det._hdu = fits.BinTableHDU.from_columns([ fits.Column(name="wavelength", format="D", array=lam, unit="um"), - fits.Column(name="spectrum", format="D", array=spec, unit="ADU"), + fits.Column(name="spectrum", format="D", array=spec, unit="adu"), ]) + det._hdu.header["INHERIT"] = True elif output_format == "table": ntrace = len(self.spectral_traces) @@ -298,8 +302,9 @@ def apply_to(self, det, **kwargs): tab = Table(data=[idarr, xarr, yarr, lamarr, specarr], names=["id", "x", "y", "wavelength", "spectrum"], - units=["", "arcsec", "arcsec", "um", "ADU"]) + units=["", "arcsec", "arcsec", "um", "adu"]) det._hdu = fits.BinTableHDU(data=tab) + det._hdu.header["INHERIT"] = True elif output_format == "image": pass @@ -307,6 +312,7 @@ def apply_to(self, det, **kwargs): else: logger.error("Unknown output format: %s", output_format) + det.header.update(exposure_header) return det diff --git a/scopesim/effects/psfs/analytical.py b/scopesim/effects/psfs/analytical.py index dc39f5207..1e8bb0ff4 100644 --- a/scopesim/effects/psfs/analytical.py +++ b/scopesim/effects/psfs/analytical.py @@ -2,23 +2,31 @@ """Contains simple Vibration, NCPA, Seeing and Diffraction PSFs.""" from typing import ClassVar, Any -from functools import partial +from functools import partial, lru_cache +from pathlib import Path import numpy as np from astropy import units as u from astropy.convolution import Gaussian2DKernel -from astropy.modeling.models import Moffat2D from scipy.interpolate import make_interp_spline +from scipy.special import gammaln, stdtr +from astropy.wcs import WCS from .. import DataContainer from ...optics import ImagePlane -from ...optics.fov import FieldOfView +from ...optics.fov import FieldOfView, FieldOfView3D from ...utils import (from_currsys, quantify, quantity_from_table, figure_factory, check_keys, get_logger, find_file, get_zenith_angle, get_observation_info_from_cmds) from . import PSF, PoorMansFOV logger = get_logger(__name__) + +def _psf_table_key(filename, cmds): + path = Path(find_file(from_currsys(filename, cmds))) + return str(path), path.read_bytes() + + class AnalyticalPSF(PSF): """Base class for analytical PSFs.""" @@ -261,9 +269,13 @@ def fwhm(self, wavelengths): fwhm = self.meta["fwhm"] if isinstance(fwhm, dict) and {"pivot_wave", "pivot_wave_unit"}.issubset(fwhm): target, location, time, _ = get_observation_info_from_cmds(self.cmds) - key = tuple(fwhm.items()), target, location, time + # The seeing law depends on zenith angle, not the coordinate frame + # instance (SkyCoord equality raises for different observation times). + key = tuple(fwhm.items()), get_zenith_angle(target, location, time) elif isinstance(fwhm, dict): key = tuple(fwhm.items()) + elif isinstance(fwhm, str): + key = _psf_table_key(fwhm, self.cmds) else: key = fwhm @@ -311,8 +323,9 @@ def get_fwhm_interp(self): raise TypeError("fwhm kwarg must be of type dict or float or str") def get_kernel(self, fov): - pixel_scale = fov.header["CDELT1"] * u.deg.to(u.arcsec) + pixel_scale = abs(fov.header["CDELT1"]) * u.deg.to(u.arcsec) pixel_scale = quantify(pixel_scale, u.arcsec) + ystep = abs(fov.header["CDELT2"] / fov.header["CDELT1"]) # Sample the wavelength-dependent seeing law with a bounded number of # representative points. Kernel sizing is driven by the broadest PSF @@ -324,14 +337,14 @@ def get_kernel(self, fov): target = self._target_enclosed_energy() max_ksize = self._max_kernel_size() - ksize = self._minimum_kernel_size(np.max(fwhms)) + ksize = self._minimum_kernel_size(np.max(fwhms) / min(1., ystep)) if max_ksize is not None: ksize = min(ksize, max_ksize) - kernel, norm = self._make_moffat_kernel(gammas, alpha, ksize) + kernel, norm = self._make_moffat_kernel(gammas, alpha, ksize, ystep) while norm < target and (max_ksize is None or ksize < max_ksize): ksize = self._next_kernel_size(ksize, max_ksize) - kernel, norm = self._make_moffat_kernel(gammas, alpha, ksize) + kernel, norm = self._make_moffat_kernel(gammas, alpha, ksize, ystep) if norm < target: logger.warning( @@ -354,6 +367,65 @@ def get_kernel(self, fov): kernel /= norm return kernel + def point_source_image(self, fov, x, y): + """Pixel-integrated effective PSF at an actual sky position [arcsec].""" + dx, dy = abs(fov.header['CDELT1'])*3600, abs(fov.header['CDELT2'])*3600 + xp, yp = WCS(fov.header, naxis=2).all_world2pix(x/3600, y/3600, 0) + xx = (np.arange(fov.header['NAXIS1']) - xp)[None, :]*dx + yy = (np.arange(fov.header['NAXIS2']) - yp)[:, None]*dy + wave = self._sample_kernel_wavelengths(fov.waveset) + gammas = self.fwhm2gamma(self.fwhm(wave), self.alpha).to_value(u.arcsec) + return moffat_cell_integrals(xx, yy, dx, dy, gammas, self.alpha) + + def _apply_convolution(self, obj, spatial_shifts=()): + if isinstance(obj, FieldOfView3D) and obj.hdu is None: + if obj.trace_id is not None: + # Finite cross-slit cells carry a uniform illumination density. + # Resolve the effective PSF before shearing those cells in + # wavelength. Refine only as needed to resolve this effective PSF. + wave = self._sample_kernel_wavelengths(obj.waveset) + width = (obj.meta['eta_max'] - obj.meta['eta_min']).to_value(u.arcsec) + fwhm = self.fwhm(wave).to_value(u.arcsec).min() + obj.meta['effective_psf_fwhm'] = fwhm*u.arcsec + step = fwhm / 10 + rows = max(obj.header['NAXIS2'], int(np.ceil(width / step))) + obj.header['NAXIS2'] = rows + obj.header['CDELT2'] = np.copysign(width / rows / 3600, obj.header['CDELT2']) + obj.header['CRPIX2'] = (rows + 1) / 2 + obj.header['CRVAL2'] = ((obj.meta['eta_min'] + obj.meta['eta_max']) / 2).to_value(u.deg) + length = (obj.meta['xi_max'] - obj.meta['xi_min']).to_value(u.arcsec) + columns = max(obj.header['NAXIS1'], int(np.ceil(6*length/fwhm))) + obj.header['NAXIS1'] = columns + obj.header['CDELT1'] = np.copysign(length/columns/3600, obj.header['CDELT1']) + obj.header['CRPIX1'] = (columns+1)/2 + obj.header['CRVAL1'] = ((obj.meta['xi_min'] + obj.meta['xi_max'])/2).to_value(u.deg) + obj.meta['pixel_area'] = None + obj._volume = None + if 'slit_spectral_width' in obj.meta: + obj.meta['slit_spectral_width'] *= min(1., fwhm / width) + obj.hdu = obj.make_hdu(spatial_psf=self, spatial_shifts=spatial_shifts) + else: + super()._apply_convolution(obj) + + def padded_header(self, fov, point_sources=False): + """Spatial support needed to illuminate this FOV through the PSF.""" + if point_sources: + # Points use analytic cell integrals, so constructing a convolution + # kernel merely to determine source-selection bounds is redundant. + wave = self._sample_kernel_wavelengths(fov.waveset) + gamma = self.fwhm2gamma(self.fwhm(wave), self.alpha).to_value(u.arcsec).max() + radius = gamma*np.sqrt((1-self._target_enclosed_energy())**(1/(1-self.alpha))-1) + padding_x = int(np.ceil(radius/(abs(fov.header['CDELT1'])*3600))) + padding_y = int(np.ceil(radius/(abs(fov.header['CDELT2'])*3600))) + else: + kernel = self.get_kernel(fov) + padding_y, padding_x = kernel.shape[0]//2, kernel.shape[1]//2 + header = fov.header.copy() + for axis, padding in ((1, padding_x), (2, padding_y)): + header[f'NAXIS{axis}'] += 2*padding + header[f'CRPIX{axis}'] += padding + return header + def _sample_kernel_wavelengths(self, waveset: u.Quantity) -> u.Quantity: max_samples = max(2, int(from_currsys( self.meta.get("max_kernel_wavelength_samples", 16), self.cmds))) @@ -387,22 +459,9 @@ def _minimum_kernel_size(self, max_fwhm_pix: float) -> int: kx = float(from_currsys(self.meta.get("kernel_size", 4.0), self.cmds)) return self._ensure_odd_int(kx * max_fwhm_pix) - def _make_moffat_kernel(self, gammas: np.ndarray, alpha:float, ksize: int) -> tuple[np.ndarray, float]: - amplitude = (alpha - 1) / (np.pi * gammas**2) - x, y = np.meshgrid( - np.arange(ksize) - ksize // 2, - np.arange(ksize) - ksize // 2, - ) - cube = Moffat2D.evaluate( - x=x[None, ...], - y=y[None, ...], - amplitude=amplitude[:, None, None], - x_0=0, - y_0=0, - gamma=gammas[:, None, None], - alpha=alpha, - ) - kernel = np.mean(cube, axis=0) + def _make_moffat_kernel(self, gammas, alpha, ksize, ystep=1.): + positions = np.arange(ksize) - ksize//2 + kernel = moffat_cell_integrals(positions[None, :], positions[:, None]*ystep, 1., ystep, gammas, alpha) if from_currsys(self.meta.get("rounded_edges", False), self.cmds): kernel = self._round_kernel_edges(kernel) return kernel, float(np.sum(kernel)) @@ -451,6 +510,33 @@ def fwhm_from_table(table): return lambda wavelengths: interp(wavelengths.to_value(u.um)) +@lru_cache(maxsize=16) +def _legendre_rule(order): + nodes, weights = np.polynomial.legendre.leggauss(order) + nodes.flags.writeable = weights.flags.writeable = False + return nodes, weights + + +def moffat_cell_integrals(x, y, dx, dy, gammas, alpha): + """Integrate normalized Moffat profiles over rectangles, averaging wavelengths. + + Integrate the conditional Student-t profile analytically along y and use + Gauss-Legendre quadrature along x. This resolves narrow profiles without + oversampling a spectral cube. All coordinates share one angular unit. + """ + nodes, weights = _legendre_rule(max(8, int(np.ceil(8*dx/np.min(gammas))))) + xx = x[..., None] + dx*nodes/2 + lower, upper = y[..., None] - dy/2, y[..., None] + dy/2 + integral = np.zeros(np.broadcast_shapes(x.shape, y.shape)) + normalization = np.exp(gammaln(alpha - .5) - gammaln(alpha - 1)) / np.sqrt(np.pi) + for gamma in gammas: + marginal = normalization/gamma * (1 + (xx/gamma)**2)**(.5 - alpha) + scale = np.sqrt((2*alpha - 1)/(gamma**2 + xx**2)) + probability = stdtr(2*alpha - 1, upper*scale) - stdtr(2*alpha - 1, lower*scale) + integral += np.sum(marginal*probability*weights, axis=-1)*dx/2 + return integral/len(gammas) + + class AOEnhanceablePSF(MoffatPSF): """ Wavelength dependent Moffat PSF but with added AO scaling. @@ -505,9 +591,10 @@ def natural_alpha(self): @property def ao_alpha(self): - if self._ao_alpha is None or self._ao_alpha[1] != self.meta['ao_table']: + key = _psf_table_key(self.meta['ao_table'], self.cmds), self.meta['alpha'] + if self._ao_alpha is None or self._ao_alpha[1] != key: _, alpha = self.ao_table_data() - self._ao_alpha = alpha, self.meta['ao_table'] + self._ao_alpha = alpha, key return self._ao_alpha[0] @property @@ -530,9 +617,10 @@ def ao_table_data(self): @property def ao_scale(self): - if self._ao_interp is None or self._ao_interp[1] != self.meta['ao_table']: + key = _psf_table_key(self.meta['ao_table'], self.cmds) + if self._ao_interp is None or self._ao_interp[1] != key: ao_table, _ = self.ao_table_data() - self._ao_interp = self.fwhm_from_table(ao_table), self.meta['ao_table'] + self._ao_interp = self.fwhm_from_table(ao_table), key return self._ao_interp[0] def ao_fwhm(self, wavelengths): diff --git a/scopesim/effects/psfs/discrete.py b/scopesim/effects/psfs/discrete.py index 3e85adf9c..8b5c33355 100644 --- a/scopesim/effects/psfs/discrete.py +++ b/scopesim/effects/psfs/discrete.py @@ -172,7 +172,7 @@ class FieldConstantPSF(DiscretePSF): Fixed handling of background level and rounded edges to avoid visible "squares" in the image. - .. versionchanged:: PLACEHOLDER_NEXT_RELEASE_VERSION + .. versionchanged:: 0.12.0 Interpolation spline order can now be set via "!SIM.psf.interp_order". @@ -197,8 +197,6 @@ def get_kernel(self, fov): """Find nearest wavelength and build PSF kernel from file.""" idx = _nearest_index(fov.wavelength, self._waveset) ext = self.kernel_indices[idx] - if ext == self.current_layer_id: - return self.kernel try: spline_order = from_currsys(self.meta["interp_order"], cmds=self.cmds) @@ -207,13 +205,13 @@ def get_kernel(self, fov): logger.warning("PSF parameter interp_order not found, using 1 (linear interpolation).") spline_order = 1 - if fov.hdu.header["NAXIS"] == 3: self.current_layer_id = ext self.make_psf_cube(fov, spline_order) return self.kernel - self.kernel = self._file[ext].data + # Keep the loaded reference independent of callers modifying a kernel. + self.kernel = self._file[ext].data.copy() self.current_layer_id = ext hdr = self._file[ext].header @@ -265,6 +263,7 @@ def make_psf_cube(self, fov, spline_order=1): - fov.hdu.header["CRPIX3"]) + fov.hdu.header["CRVAL3"] ) + lam *= u.Unit(fov.hdu.header['CUNIT3']).to(u.um) if not self.meta.get("interp_psf", True): lam = np.array([lam[len(lam)//2]]) diff --git a/scopesim/effects/rotation.py b/scopesim/effects/rotation.py index 7b208bfcc..d1d48b578 100644 --- a/scopesim/effects/rotation.py +++ b/scopesim/effects/rotation.py @@ -40,5 +40,5 @@ def apply_to(self, obj, **kwargs): return obj rotations = from_currsys(self.meta["rotations"], self.cmds) - obj._hdu.data = np.rot90(obj._hdu.data, rotations) + obj.data = np.rot90(obj.data, rotations) return obj diff --git a/scopesim/effects/selector_wheel.py b/scopesim/effects/selector_wheel.py index 7ee03f242..c58692572 100644 --- a/scopesim/effects/selector_wheel.py +++ b/scopesim/effects/selector_wheel.py @@ -13,12 +13,13 @@ import importlib from numbers import Integral -from ..utils import (check_keys, get_logger, real_colname) +from ..utils import from_currsys, check_keys, get_logger, real_colname from .effects import Effect from ..optics.fov_volume_list import FovVolumeList from ..optics.fov import FieldOfView from ..optics.image_plane import ImagePlane from ..detector.detector import Detector +from ..source.source import Source logger = get_logger(__name__) @@ -62,8 +63,10 @@ def __init__(self, **kwargs): self.wheel_effects = {} for wheel_entry in self.meta["wheel"]: selector_value = wheel_entry["selector_value"] # can be single value or list of values - effect_class_name = wheel_entry["effect_class"] + effect_class_name = from_currsys(wheel_entry["effect_class"], self.cmds) effect_kwargs = wheel_entry.get("effect_kwargs", {}) + if isinstance(effect_kwargs, str): + effect_kwargs = from_currsys(effect_kwargs, self.cmds) # Dynamically import the effect class effect_module = importlib.import_module("scopesim.effects") @@ -78,8 +81,14 @@ def __init__(self, **kwargs): self.z_order = self._resolve_z_order() - def apply_to(self, obj, **kwargs): - """Based on the selector_key's value in obj.meta, apply the corresponding effect from the wheel.""" + def apply_to(self, obj, fov=None, **kwargs): + """Select from obj.meta, or fov.meta when applying branch effects to a Source.""" + if isinstance(obj, Source) and fov is not None: + selector_value = fov.meta[self.meta["selector_key"]] + effect_to_apply = self.get_effect(selector_value) + if effect_to_apply is not None: + obj = effect_to_apply.apply_to(obj, fov=fov, **kwargs) + if isinstance(obj, FieldOfView): if self.meta['selector_key'] not in obj.meta.keys(): raise ValueError(f"Selector key {self.meta['selector_key']} not found in FieldOfView meta.") diff --git a/scopesim/effects/shifts.py b/scopesim/effects/shifts.py index 1fc9d79bf..492550b71 100644 --- a/scopesim/effects/shifts.py +++ b/scopesim/effects/shifts.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- """TBA.""" -import warnings from typing import ClassVar import numpy as np @@ -27,17 +26,6 @@ def apply_to(self, obj, **kwargs): """See parent docstring.""" return obj - def fov_grid(self, which="shifts", **kwargs): - """See parent docstring.""" - warnings.warn("The fov_grid method is deprecated and will be removed " - "in a future release.", DeprecationWarning, stacklevel=2) - if which == "shifts": - col_names = ["wavelength", "dx", "dy"] - waves, dx, dy = [self.get_table(**kwargs)[col] - for col in col_names] - return waves, dx, dy - return None - def get_table(self, **kwargs): if self.table is None: names = ["wavelength", "dx", "dy"] @@ -70,7 +58,7 @@ class AtmosphericDispersion(Shift3D): Used to generate the wavelength bins based on shifts due to the atmosphere. Doesn't contain an ``apply_to`` function, but provides information through - the ``fov_grid`` function. + the ``get_table`` function. Required Parameters ------------------- @@ -133,7 +121,7 @@ def __init__(self, **kwargs): def get_table(self, **kwargs): """ - Called by the fov_grid method of Shift3D. + Compute the wavelength-dependent refraction shifts. Returns ------- @@ -248,19 +236,6 @@ def apply_to(self, fov, **kwargs): return fov - def fov_grid(self, which="shifts", **kwargs): - """See parent docstring.""" - warnings.warn("The fov_grid method is deprecated and will be removed " - "in a future release.", DeprecationWarning, stacklevel=2) - kwargs.update(self.meta) - if "quick_adc" in self.meta: - ad = AtmosphericDispersion(**self.meta) - waves, dx, dy = ad.fov_grid() - dx *= -(1 - self.meta["efficiency"]) - dy *= -(1 - self.meta["efficiency"]) - return waves, dx, dy - return None - def plot(self): return None diff --git a/scopesim/effects/spectral_efficiency.py b/scopesim/effects/spectral_efficiency.py index e84a35587..b551b72fc 100644 --- a/scopesim/effects/spectral_efficiency.py +++ b/scopesim/effects/spectral_efficiency.py @@ -11,8 +11,7 @@ from .effects import Effect from .ter_curves import TERCurve -from .ter_curves_utils import apply_throughput_to_cube -from ..utils import figure_factory, get_logger, check_keys +from ..utils import figure_factory, get_logger, check_keys, from_currsys from .data_container import DataContainer from ..optics import echelle @@ -93,9 +92,12 @@ def get_efficiencies(self): wavelength = tbl['wavelength'].quantity efficiency = tbl['efficiency'].value params.pop("filename", None) # don't pass filename to TERCurve! - effic_curve = TERCurve(array_dict={"wavelength":wavelength, - "transmission":efficiency}, - **params) + effic_curve = TERCurve( + array_dict={ + "wavelength": wavelength, + "transmission": efficiency, + }, + **params) efficiencies[name] = effic_curve hdul.close() @@ -110,11 +112,7 @@ def apply_to(self, obj, **kwargs): logger.warning("No grating efficiency for trace %s", trace_id) return obj - swcs = WCS(obj.hdu.header).spectral - with u.set_enabled_equivalencies(u.spectral()): - wave = swcs.pixel_to_world(np.arange(swcs.pixel_shape[0])) << u.um - obj.hdu = apply_throughput_to_cube(obj.hdu, effic.throughput, wave) - return obj + return effic.apply_to(obj) def plot(self): """Plot the grating efficiencies.""" @@ -146,7 +144,7 @@ def __init__(self, **kwargs): self.efficiency_generator = self._generate_efficiency_curve_func() def _generate_efficiency_curve_func(self) -> Callable: - trace_params = self.table + trace_params = from_currsys(self.table, self.cmds) spectrographs = {} for row in trace_params: prefix = row["prefix"] # note trance ids are assumed to be prefix_{order} @@ -170,13 +168,17 @@ def _generate_efficiency_curve_func(self) -> Callable: xdisp_groove_length = u.Unit(trace_params.meta["xdisp_freq_unit"]) / row['xdisp_freq'] echelle_groove_length = u.Unit(trace_params.meta["disp_freq_unit"]) / row['disp_freq'] pix_per_res_elem = row['fwhm'] + prescription = {} + if 'collimator1_focal_length' in trace_params.colnames: + for key in ('collimator1_focal_length', 'collimator2_focal_length', 'entrance_slit_scale'): + prescription[key] = row[key] * u.Unit(trace_params.meta[key + '_unit']) spectrographs[prefix] = echelle.spectrograph_factory(min_wave, max_wave, focal_len, design_res, echelle_angle, min_order, max_order, echelle_groove_length, pix_per_res_elem, disp_npix, xdisp_npix, pix_size, xdisp_groove_length=xdisp_groove_length, xdisp_beta_center=xdisp_beta_center, - dispersion_focal_len=dispersion_focal_len) + dispersion_focal_len=dispersion_focal_len, **prescription) self._spectrographs = spectrographs def efficiency_curve(trace_id, wavelength): @@ -194,9 +196,12 @@ def apply_to(self, obj, **kwargs): """Interface between FieldOfView and SpectralEfficiency.""" trace_id = obj.trace_id - swcs = WCS(obj.hdu.header).spectral - with u.set_enabled_equivalencies(u.spectral()): - wave = swcs.pixel_to_world(np.arange(swcs.pixel_shape[0])) << u.um + if obj.hdu is None: + wave = obj.cube_waveset + else: + swcs = WCS(obj.hdu.header).spectral + with u.set_enabled_equivalencies(u.spectral()): + wave = swcs.pixel_to_world(np.arange(swcs.pixel_shape[0])) << u.um efficiency = self.efficiency_generator(trace_id, wave) params = {"description": trace_id} @@ -204,8 +209,7 @@ def apply_to(self, obj, **kwargs): effic_curve = TERCurve(array_dict={"wavelength": wave, "transmission": efficiency}, **params) self.efficiencies[trace_id] = effic_curve - obj.hdu = apply_throughput_to_cube(obj.hdu, effic_curve.throughput, wave) - return obj + return effic_curve.apply_to(obj) def plot(self): """Plot the grating efficiencies.""" diff --git a/scopesim/effects/spectral_trace_list.py b/scopesim/effects/spectral_trace_list.py index 7d4c6a7ce..b10cbb49d 100644 --- a/scopesim/effects/spectral_trace_list.py +++ b/scopesim/effects/spectral_trace_list.py @@ -22,7 +22,7 @@ from ..optics.fov_volume_list import FovVolumeList from .effects import Effect from .ter_curves import FilterCurve -from .spectral_trace_list_utils import SpectralTrace, make_image_interpolations +from .spectral_trace_list_utils import SpectralTrace, make_image_interpolations, det_offset from .data_container import DataContainer from ..optics import echelle @@ -128,8 +128,6 @@ def __init__(self, cmds=None, **kwargs): "x_colname": "x", "y_colname": "y", "s_colname": "s", - "offset_x": 0, # [mm] in detector plane - "offset_y": 0, # [mm] in detector plane "wave_colname": "wavelength", "center_on_wave_mid": False, "dwave": 0.002, # [um] for finding the best fit dispersion @@ -151,6 +149,8 @@ def make_spectral_traces(self): self.catalog = Table(self._file[self.ext_cat].data) spec_traces = {} for row in self.catalog: + if row["image_plane_id"] == -99: + continue params = {col: row[col] for col in row.colnames} params.update(self.meta) hdu = self._file[row["extension_id"]] @@ -185,6 +185,7 @@ def update_meta(self): self.meta["y_min"] = min(ylim) self.meta["y_max"] = max(ylim) + def apply_to(self, obj, **kwargs): """ Interface between ``FieldOfView`` and ``SpectralTraceList``. @@ -199,76 +200,119 @@ def apply_to(self, obj, **kwargs): list, identified by meta["trace_id"]. """ if isinstance(obj, FovVolumeList): - logger.debug("%s applied to %s", self.display_name, - obj.__class__.__name__) - # Setup of FieldOfView object - # volumes = [spectral_trace.fov_grid() - # for spectral_trace in self.spectral_traces.values()] - - new_vols_list = [] - - # for vol in volumes: - for spt in self.spectral_traces.values(): - vol = spt.fov_grid() - wave_edges = [vol["wave_min"], vol["wave_max"]] - if "x_min" in vol: - x_edges = [vol["x_min"], vol["x_max"]] - y_edges = [vol["y_min"], vol["y_max"]] - extracted_vols = obj.extract( - axes=["wave", "x", "y"], - edges=(wave_edges, x_edges, y_edges), - aperture_id=vol["aperture_id"]) - else: - extracted_vols = obj.extract( - axes=["wave"], - edges=(wave_edges, ), - aperture_id=vol["aperture_id"]) - - for ex_vol in extracted_vols: - ex_vol["meta"].update(vol) - ex_vol["meta"].pop("wave_min") - ex_vol["meta"].pop("wave_max") - new_vols_list.extend(extracted_vols) - - obj.volumes = new_vols_list + obj = self.apply_to_fovvolumelist(obj) if isinstance(obj, FieldOfView): - logger.debug("%s applied to %s", self.display_name, - obj.__class__.__name__) - # Application to field of view - if obj.hdu is not None and obj.hdu.header["NAXIS"] == 3: - obj.cube = obj.hdu - elif obj.hdu is not None and obj.hdu.header["NAXIS"] == 2: - # todo: catch the case of obj.hdu.header["NAXIS"] == 2 - # for MAAT - pass - elif obj.hdu is None and obj.cube is None: - logger.info("Making cube") - obj.cube = obj.make_hdu() - - # # Check whether an offset slit is used. If so, recompute spectral traces. - # offset_x = obj.cube.header["CRVAL1D"] - # offset_y = obj.cube.header["CRVAL2D"] - # if (offset_x != self.meta["offset_x"] or - # offset_y != self.meta["offset_y"]): - # logger.debug("Recomputing spectral traces for offset (%.1g, %.1g)", - # offset_x, offset_y) - # self.meta["offset_x"] = offset_x - # self.meta["offset_y"] = offset_y - # self.make_spectral_traces() - # self.update_meta() - - spt = self.spectral_traces[obj.trace_id] - try: - # If footprint is outside FOV, this will assign None to the hdu - obj.hdu = spt.map_spectra_to_focal_plane(obj) - except ValueError as err: # xlim_mm is None - logger.error(err) - obj.image_plane_id = spt.meta["image_plane_id"] + obj = self.apply_to_fov(obj) logger.debug("%s done", self.display_name) return obj + def apply_to_fovvolumelist(self, obj): + """ + Setup of SpectralTraceList + + During setup of the required FieldOfView objects, the + SpectralTraceList is asked for the source space volumes that + it requires (spatial limits and wavelength limits). + """ + logger.debug("%s applied to %s", self.display_name, + obj.__class__.__name__) + + new_vols_list = [] + + for spt in self.spectral_traces.values(): + vol = spt.fov_grid() + wave_edges = [vol["wave_min"], vol["wave_max"]] + if "x_min" in vol: + x_edges = [vol["x_min"], vol["x_max"]] + y_edges = [vol["y_min"], vol["y_max"]] + extracted_vols = obj.extract( + axes=["wave", "x", "y"], + edges=(wave_edges, x_edges, y_edges), + aperture_id=vol["aperture_id"]) + else: + extracted_vols = obj.extract( + axes=["wave"], + edges=(wave_edges, ), + aperture_id=vol["aperture_id"]) + + for ex_vol in extracted_vols: + ex_vol["meta"].update(vol) + ex_vol["meta"].pop("wave_min") + ex_vol["meta"].pop("wave_max") + new_vols_list.extend(extracted_vols) + + obj.volumes = new_vols_list + return obj + + + def apply_to_fov(self, obj): + """Apply to FieldOfView + + During "observation" the method is passed a single FieldOfView + object and applies the mapping to the image plane to it. + The FieldOfView object is associated to one SpectralTrace from the + list, identified by meta["trace_id"]. + """ + logger.debug("%s applied to %s %s", self.display_name, + obj.__class__.__name__, obj.trace_id) + + if obj.hdu is not None and obj.hdu.header["NAXIS"] == 3: + obj.cube = obj.hdu + elif obj.hdu is not None and obj.hdu.header["NAXIS"] == 2: + # todo: catch the case of obj.hdu.header["NAXIS"] == 2 + # for MAAT + pass + elif obj.hdu is None and obj.cube is None: + logger.info("Making cube") + obj.cube = obj.make_hdu() + + # Check whether an offset slit is used. If so, recompute spectral traces. + # TODO: this needs to be removed. If transforms are available that + # can be applied to the actual slit position, they ought be used + # without modification. If they are not available but need to be + # hacked, then that hack needs to be defined via the yaml and only + # be used in a particular case, not in general. + #offset_x = obj.cube.header["CRVAL1D"] + #offset_y = obj.cube.header["CRVAL2D"] + #if (offset_x != self.meta["offset_x"] or + # offset_y != self.meta["offset_y"]): + # logger.debug("Recomputing spectral traces for offset (%.1g, %.1g)", + # offset_x, offset_y) + # self.meta["offset_x"] = offset_x + # self.meta["offset_y"] = offset_y + # self.make_spectral_traces() + # self.update_meta() + + spt = self.spectral_traces[obj.trace_id] + remove_pre = False + if ("offset_type" in obj.meta and + obj.meta["offset_type"] == "detector"): + self.meta["offset_type"] = "detector" + self.meta["offset_x"] = obj.meta["offset_x"] + self.meta["offset_y"] = obj.meta["offset_y"] + spt.xy2lam.pretransform_x = (det_offset, + {"offset": obj.meta["offset_x"]}) + spt.xy2lam.pretransform_y = (det_offset, + {"offset": obj.meta["offset_y"]}) + spt.xilam2x.posttransform = (det_offset, + {"offset": -obj.meta["offset_x"]}) + spt.xilam2y.posttransform = (det_offset, + {"offset": -obj.meta["offset_y"]}) + remove_pre = True + + obj.hdu = spt.map_spectra_to_focal_plane(obj) + obj.image_plane_id = spt.meta["image_plane_id"] + + if remove_pre: + spt.xy2lam.pretransform_y = None + spt.xy2lam.pretransform_x = None + spt.xilam2x.posttransform = None + spt.xilam2y.posttransform = None + + return obj + @property def footprint(self): """Return the footprint of the entire SpectralTraceList.""" @@ -365,6 +409,20 @@ def rectify_traces(self, hdulist, xi_min=None, xi_max=None, interps=None, for i, trace_id in tqdm(enumerate(self.spectral_traces, start=1), desc=" Traces", total=len(self.spectral_traces)): + spt = self[trace_id] + remove_pre = False + if ("offset_type" in self.meta and + self.meta["offset_type"] == "detector"): + spt.xy2lam.pretransform_x = (det_offset, + {"offset": self.meta["offset_x"]}) + spt.xy2lam.pretransform_y = (det_offset, + {"offset": self.meta["offset_y"]}) + spt.xilam2x.posttransform = (det_offset, + {"offset": -self.meta["offset_x"]}) + spt.xilam2y.posttransform = (det_offset, + {"offset": -self.meta["offset_y"]}) + remove_pre = True + try: hdu = self[trace_id].rectify( hdulist, @@ -385,6 +443,12 @@ def rectify_traces(self, hdulist, xi_min=None, xi_max=None, interps=None, outhdul.append(hdu) outhdul[0].header[f"EXTNAME{i}"] = trace_id + if remove_pre: + spt.xy2lam.pretransform_y = None + spt.xy2lam.pretransform_x = None + spt.xilam2x.posttransform = None + spt.xilam2y.posttransform = None + outhdul[0].header.update(inhdul[0].header) return outhdul @@ -614,6 +678,19 @@ class EchelleSpectralTraceList(SpectralTraceList): nIR 2 0 40 24 970 2500 20000 64.2 225 225 4.7 0.7 10.0 10 0.015 4096 4096 45 175 10 x ---------------------------------------------------------------- + Optional ``detector_offset_x`` and ``detector_offset_y`` columns specify + the rotated analytical camera origin relative to detector center, positive + right/up. Both columns and their ``*_unit`` metadata are required together; + values must have length units. Explicit zeros place that origin at detector + center. Offsets are applied after rotation, before detector clipping, with + no automatic recentering. Omitting both columns retains automatic placement. + + With ``physical_slit_dispersion=True``, supply ``collimator1_focal_length``, + ``collimator2_focal_length`` and ``entrance_slit_scale`` columns with their + ``*_unit`` metadata. The entrance scale is length per on-sky arcsecond. + These determine the illumination angle and asymmetric pupil relay; + the along-slit plate scale is not used for across-slit dispersion. + The calculated traces are stored in the same HDUList format as required by SpectralTraceList, and supplied to the parent class through hdulist kwarg. @@ -626,10 +703,13 @@ def __init__(self, cmds=None, **kwargs): self.cmds = cmds trace_param_filename = kwargs.pop("filename") + self.physical_slit_dispersion = kwargs.pop("physical_slit_dispersion", False) + self.slit_samples = kwargs.pop("slit_samples", 3) save_generated_hdulist = kwargs.pop("save_generated_hdulist", False) generated_hdulist_filename = kwargs.pop( "generated_hdulist_filename", "analytical_echelle_traces.fits") trace_params = DataContainer(filename=trace_param_filename) + trace_params.table = from_currsys(trace_params.table, self.cmds) hdulist = self._generate_trace_hdulist(trace_params) if save_generated_hdulist: output_path = Path(generated_hdulist_filename).expanduser() @@ -648,6 +728,7 @@ def _generate_trace_hdulist(self, trace_params): hdul[0].header["EDATA"] = 2 trace_ids, ap_ids, im_ids = [], [], [] + self._spectrographs = {} for row in trace_params.table: prefix = row["prefix"] min_order = row['m0'] - row['n'] + 1 @@ -676,13 +757,19 @@ def _generate_trace_hdulist(self, trace_params): xdisp_groove_length = u.Unit(trace_params.meta["xdisp_freq_unit"]) / row['xdisp_freq'] echelle_groove_length = u.Unit(trace_params.meta["disp_freq_unit"]) / row['disp_freq'] pix_per_res_elem = row['fwhm'] + prescription = {} + if 'collimator1_focal_length' in trace_params.table.colnames: + for key in ('collimator1_focal_length', 'collimator2_focal_length', 'entrance_slit_scale'): + prescription[key] = row[key] * u.Unit(trace_params.meta[key + '_unit']) + nominal_width = row['nominal_slit_width'] * u.Unit(trace_params.meta['nominal_slit_width_unit']) + pix_per_res_elem = (nominal_width * prescription['entrance_slit_scale'] * focal_len / prescription['collimator2_focal_length'] / pix_size).to_value(u.dimensionless_unscaled) ss = echelle.spectrograph_factory(min_wave, max_wave, focal_len, design_res, echelle_angle, min_order, max_order, echelle_groove_length, pix_per_res_elem, disp_npix, xdisp_npix, pix_size, xdisp_groove_length=xdisp_groove_length, xdisp_beta_center=xdisp_beta_center, - dispersion_focal_len=dispersion_focal_len) + dispersion_focal_len=dispersion_focal_len, **prescription) _warn_if_echelle_design_res_inconsistent( prefix, design_res, @@ -693,7 +780,7 @@ def _generate_trace_hdulist(self, trace_params): ) slit_edge = (row['slitlength'] / 2) * u.Unit(trace_params.meta["slitlength_unit"]) - slit_pos = np.linspace(-slit_edge, slit_edge, num=3) + slit_pos = np.linspace(-slit_edge, slit_edge, num=self.slit_samples) if "plate_scale" in trace_params.table.colnames: plate_scale = row["plate_scale"] * u.Unit( trace_params.meta["plate_scale_unit"]) @@ -712,74 +799,52 @@ def _generate_trace_hdulist(self, trace_params): ).to_value(u.deg) cang = np.cos(np.deg2rad(detector_angle)) sang = np.sin(np.deg2rad(detector_angle)) - detector_x_padding = max( - 0.0, - ( - disp_npix * abs(cang) - + xdisp_npix * abs(sang) - - disp_npix - ) / 2, - ) - slit_x_padding = ( - abs(np.tan(np.deg2rad(detector_angle))) - * float(np.nanmax(np.abs(slit_offset_pix))) - ) - raw_x_min = 0.5 - detector_x_padding - slit_x_padding - raw_x_max = disp_npix - 0.5 + detector_x_padding + slit_x_padding - - def raw_detector_pixels(wave, order): - x = ss.wavelength_to_x_pixel(wave, order) - y = ss.wavelength_to_y_pixel(wave) - x = u.Quantity(x, copy=False).to_value( - u.dimensionless_unscaled) - y = u.Quantity(y, copy=False).to_value( - u.dimensionless_unscaled) - xpix = np.broadcast_to(x, (slit_offset_pix.size, wave.size)) - ypix = y[None, :] + slit_offset_pix[:, None] - return xpix, ypix - - rotated_x_min = rotated_y_min = np.inf - rotated_x_max = rotated_y_max = -np.inf - # The analytical echelle layout is relative, not yet detector - # placed. Rotate the footprint first, then place that rotated - # footprint on the detector. Detector padding is not part of the - # optical trace geometry; downstream detector/FOV code is - # responsible for clipping padded display or extraction regions. - for i, order in enumerate(ss.orders): - raw_x = np.linspace( - raw_x_min, raw_x_max, num=max(disp_npix, 2)) - wave = ss.x_pixel_to_wavelength(raw_x, order) - xpix, ypix = raw_detector_pixels(wave, order) - xrot = cang * xpix - sang * ypix - yrot = sang * xpix + cang * ypix - rotated_x_min = min(rotated_x_min, float(np.nanmin(xrot))) - rotated_x_max = max(rotated_x_max, float(np.nanmax(xrot))) - rotated_y_min = min(rotated_y_min, float(np.nanmin(yrot))) - rotated_y_max = max(rotated_y_max, float(np.nanmax(yrot))) - rotated_x_center = rotated_x_min + (rotated_x_max - rotated_x_min) / 2 - rotated_y_center = rotated_y_min + (rotated_y_max - rotated_y_min) / 2 - - def rotated_detector_pixels(wave, order): - xpix, ypix = raw_detector_pixels(wave, order) - xrot = cang * xpix - sang * ypix - yrot = sang * xpix + cang * ypix - return ( - xrot - rotated_x_center + disp_npix / 2, - yrot - rotated_y_center + xdisp_npix / 2, - ) + ss._detector_cos, ss._detector_sin = cang, sang + ss._slit_offset_pix = slit_offset_pix + self._spectrographs[prefix] = ss + if 'detector_offset_x' in trace_params.table.colnames or 'detector_offset_y' in trace_params.table.colnames: + # Multiplying an ASCII table's masked cell by a Unit yields + # zero, so reject missing values before attaching units. + if np.ma.is_masked(row['detector_offset_x']) or np.ma.is_masked(row['detector_offset_y']): + raise ValueError('Both detector offsets must have values.') + offset_x = row['detector_offset_x'] * u.Unit(trace_params.meta['detector_offset_x_unit']) + offset_y = row['detector_offset_y'] * u.Unit(trace_params.meta['detector_offset_y_unit']) + # Offsets locate the rotated camera origin relative to detector + # center. The raw mapping includes the unrotated pixel origin. + rotated_x_center = cang*disp_npix/2 - sang*xdisp_npix/2 - (offset_x/pix_size).to_value(u.dimensionless_unscaled) + rotated_y_center = sang*disp_npix/2 + cang*xdisp_npix/2 - (offset_y/pix_size).to_value(u.dimensionless_unscaled) + + # Project the translated detector rectangle back onto model x; + # slit offsets are in model y and cancel in this projection. + raw_x_center = cang*rotated_x_center + sang*rotated_y_center + raw_x_halfspan = (abs(cang)*disp_npix + abs(sang)*xdisp_npix)/2 + candidate_raw_x = np.linspace(raw_x_center - raw_x_halfspan, raw_x_center + raw_x_halfspan, disp_npix + 1) + else: + # Preserve the original automatic placement and sampling for + # parameter files that do not specify a detector position. + detector_x_padding = max(0., (disp_npix*abs(cang) + xdisp_npix*abs(sang) - disp_npix)/2) + slit_x_padding = abs(np.tan(np.deg2rad(detector_angle))) * np.max(np.abs(slit_offset_pix)) + raw_x_min = .5 - detector_x_padding - slit_x_padding + raw_x_max = disp_npix - .5 + detector_x_padding + slit_x_padding + candidate_raw_x = np.linspace(raw_x_min, raw_x_max, num=max(disp_npix, 2)) + rotated_x_min = rotated_y_min = np.inf + rotated_x_max = rotated_y_max = -np.inf + for order in ss.orders: + wave = ss.x_pixel_to_wavelength(candidate_raw_x, order) + xrot, yrot = ss._rotated_detector_pixels(wave, order) + rotated_x_min = min(rotated_x_min, xrot.min()) + rotated_x_max = max(rotated_x_max, xrot.max()) + rotated_y_min = min(rotated_y_min, yrot.min()) + rotated_y_max = max(rotated_y_max, yrot.max()) + rotated_x_center = rotated_x_min + (rotated_x_max - rotated_x_min)/2 + rotated_y_center = rotated_y_min + (rotated_y_max - rotated_y_min)/2 + ss._detector_center = (rotated_x_center, rotated_y_center) def on_detector(wave, order): - xpix, ypix = rotated_detector_pixels(wave, order) - return ( - (xpix >= 0) - & (xpix <= disp_npix) - & (ypix >= 0) - & (ypix <= xdisp_npix) - ) + xpix, ypix = ss._detector_pixels(wave, order) + return (xpix >= 0) & (xpix <= disp_npix) & (ypix >= 0) & (ypix <= xdisp_npix) - for i, order in enumerate(ss.orders): - candidate_raw_x = np.linspace( - raw_x_min, raw_x_max, num=max(disp_npix, 2)) + for order in ss.orders: candidate_wave = ss.x_pixel_to_wavelength(candidate_raw_x, order) valid_wave = np.any( on_detector(candidate_wave, order), axis=0) @@ -808,7 +873,7 @@ def on_detector(wave, order): continue s = np.tile(slit_pos, wave.size).reshape(wave.size, slit_pos.size).T.ravel() w = np.tile(wave, slit_offset_pix.size) - xpix, ypix = rotated_detector_pixels(wave, order) + xpix, ypix = ss._detector_pixels(wave, order) xval = (xpix.ravel() - disp_npix / 2) * pix_size.to(u.mm) yval = (ypix.ravel() - xdisp_npix / 2) * pix_size.to(u.mm) @@ -817,7 +882,12 @@ def on_detector(wave, order): 'x': xval, 'y': yval}) + if self.physical_slit_dispersion: + order_table['dlam_deta'] = ss.slit_dispersion(w, order) + trace_hdu = fits.BinTableHDU(order_table) + if self.physical_slit_dispersion: + trace_hdu.header['SLITCOL'] = 'dlam_deta' trace_hdu.header['DISPDIR'] = row['dispdir'] trace_hdu.header["EXTNAME"] = f'{prefix}_{order:d}' trace_hdu.header["DESIGNR"] = ( diff --git a/scopesim/effects/spectral_trace_list_utils.py b/scopesim/effects/spectral_trace_list_utils.py index ad1dd802e..09e0af973 100644 --- a/scopesim/effects/spectral_trace_list_utils.py +++ b/scopesim/effects/spectral_trace_list_utils.py @@ -14,8 +14,9 @@ import numpy as np -from scipy.interpolate import RectBivariateSpline -from scipy.interpolate import interp1d +from scipy.interpolate import interp1d, RectBivariateSpline +from scipy.ndimage import gaussian_filter1d, map_coordinates +from scipy.sparse import csr_matrix from astropy.table import Table, vstack from astropy.modeling import fitting @@ -90,12 +91,11 @@ class SpectralTrace: - x, y : [mm] """ + _pixel_quadrature_order = 4 _class_params = { "x_colname": "x", "y_colname": "y", "s_colname": "s", - "offset_x": 0, - "offset_y": 0, "wave_colname": "wavelength", "dwave": 0.002, "aperture_id": 0, @@ -105,6 +105,7 @@ class SpectralTrace: "pixel_size": None, "description": "", "trace_flux_jacobian": "!SIM.spectral.trace_flux_jacobian", + "fit_degree": 4, } def __init__(self, trace_tbl, cmds=None, **kwargs): @@ -130,6 +131,9 @@ def __init__(self, trace_tbl, cmds=None, **kwargs): ("PLTSCALE", "plate_scale"), ("DETPAD", "detector_pad"), ("DETANG", "detector_angle"), + ("FITDEG", "fit_degree"), + ("LSFRES", "optical_resolving_power"), + ("SLITCOL", "slit_dispersion_colname"), ): if header_key in trace_tbl.header: self.meta[meta_key] = trace_tbl.header[header_key] @@ -144,6 +148,7 @@ def __init__(self, trace_tbl, cmds=None, **kwargs): # Declaration of other attributes self._xilamimg = None + self._slit_cache = {} self.dlam_per_pix = None self.fit_inverse = True @@ -175,6 +180,7 @@ def fov_grid(self): "trace_id": self.trace_id, "aperture_id": aperture_id, "image_plane_id": image_plane_id} + def compute_interpolation_functions(self): """ Compute various interpolation functions between slit and focal plane. @@ -183,23 +189,31 @@ def compute_interpolation_functions(self): `xi` (spatial coordinate along the slit, in arcsec) and `lam` (wavelength, in um). - The interpolation functions include a shift in the focal-plane - coordinates, determined from the CRVAL of the source FOV. + #The interpolation functions include a shift in the focal-plane + #coordinates, determined from the CRVAL of the source FOV. """ - x_arr = self.table[self.meta["x_colname"]] + self.meta["offset_x"] - y_arr = self.table[self.meta["y_colname"]] + self.meta["offset_y"] + x_arr = self.table[self.meta["x_colname"]] + y_arr = self.table[self.meta["y_colname"]] + xi_arr = self.table[self.meta["s_colname"]] lam_arr = self.table[self.meta["wave_colname"]] self.wave_min = quantify(np.min(lam_arr), u.um).value self.wave_max = quantify(np.max(lam_arr), u.um).value - self.xy2xi = Transform2D.fit(x_arr, y_arr, xi_arr) - self.xy2lam = Transform2D.fit(x_arr, y_arr, lam_arr) - self.xilam2x = Transform2D.fit(xi_arr, lam_arr, x_arr) - self.xilam2y = Transform2D.fit(xi_arr, lam_arr, y_arr) - self._xiy2x = Transform2D.fit(xi_arr, y_arr, x_arr) - self._xiy2lam = Transform2D.fit(xi_arr, y_arr, lam_arr) + degree = self.meta["fit_degree"] + self.xy2xi = Transform2D.fit(x_arr, y_arr, xi_arr, degree=degree) + self.xy2lam = Transform2D.fit(x_arr, y_arr, lam_arr, degree=degree) + self.xilam2x = Transform2D.fit(xi_arr, lam_arr, x_arr, degree=degree) + self.xilam2y = Transform2D.fit(xi_arr, lam_arr, y_arr, degree=degree) + self._xiy2x = Transform2D.fit(xi_arr, y_arr, x_arr, degree=degree) + self._xiy2lam = Transform2D.fit(xi_arr, y_arr, lam_arr, degree=degree) + if "slit_dispersion_colname" in self.meta: + center = xi_arr == 0 + derivative = self.table[self.meta["slit_dispersion_colname"]].quantity.to_value(u.um / u.arcsec) + # FOVs are restricted to this trace; allow unit-conversion roundoff + # at the wavelength endpoints when interpolating the derivative. + self.slit_dispersion = interp1d(lam_arr[center], derivative[center], bounds_error=False, fill_value="extrapolate") if self.dispersion_axis == "unknown": dlam_dx, dlam_dy = self.xy2lam.gradient() @@ -223,8 +237,9 @@ def _focal_plane_grid(self, fov): xlim_mm, ylim_mm = self.footprint(wave_min=wave_min, wave_max=wave_max, xi_min=xi_min, xi_max=xi_max) - if xlim_mm is None: - logger.warning("xlim_mm is None") + if xlim_mm is None or ylim_mm is None: + # Cases where there is no overlap betwen slit/trace and + # fov are acceptable. return None det_header = fov.detector_header @@ -256,10 +271,31 @@ def _focal_plane_grid(self, fov): xmin_mm, ymin_mm = fpa_wcsd.all_pix2world(xmin, ymin, 0) xmax_mm, ymax_mm = fpa_wcsd.all_pix2world(xmax, ymax, 0) - x_mm = np.linspace(xmin_mm, xmax_mm, sub_naxis1, endpoint=False, dtype=np.float32) - y_mm = np.linspace(ymin_mm, ymax_mm, sub_naxis2, endpoint=False, dtype=np.float32) + x_mm = np.linspace(xmin_mm, xmax_mm, sub_naxis1, endpoint=False) + y_mm = np.linspace(ymin_mm, ymax_mm, sub_naxis2, endpoint=False) return xmin, ymin, x_mm, y_mm, det_wcs + def spectral_padding(self, fov): + """Input wavelength support for slit displacement and optical blur.""" + lower, upper = fov.meta['wave_min'].to_value(u.um), fov.meta['wave_max'].to_value(u.um) + wave = np.unique(u.Quantity(self.table[self.meta['wave_colname']]).to_value(u.um)) + wave = np.r_[lower, wave[(wave > lower) & (wave < upper)], upper] + width = (fov.meta['eta_max'] - fov.meta['eta_min']).to_value(u.arcsec) + if 'slit_dispersion_colname' in self.meta: + derivative = self.slit_dispersion(wave) + else: + pixsize = (fov.detector_header['CDELT1D']*u.Unit(fov.detector_header['CUNIT1D'])).to_value(u.mm) + self._set_dispersion(lower, upper, pixsize=pixsize) + step = abs(fov.header['CDELT2'])*3600 + derivative = self.dlam_per_pix(wave)/step + padding = np.max(abs(derivative))*width/2 + # Resolve the physical slit profile rather than using one fixed + # wavelength step for all orders. The PSF may request a finer step. + fov.meta['slit_spectral_width'] = np.min(abs(derivative))*width*u.um + if 'optical_resolving_power' in self.meta: + padding += 5*upper/(self.meta['optical_resolving_power']*np.sqrt(8*np.log(2))) + return padding*u.um + def map_spectra_to_focal_plane(self, fov): """ Apply the spectral trace mapping to a spectral cube. @@ -290,62 +326,57 @@ def map_spectra_to_focal_plane(self, fov): sub_naxis2 = len(y_mm) xmax = xmin + sub_naxis1 ymax = ymin + sub_naxis2 - image = np.zeros((sub_naxis2, sub_naxis1), dtype=np.float32) + image = np.zeros((sub_naxis2, sub_naxis1), dtype=np.float64) self._set_dispersion(wave_min, wave_max, pixsize=pixsize) - try: - xilam = XiLamImage(fov, self.dlam_per_pix) - self._xilamimg = xilam # ..todo: remove or make available with a debug flag? - except ValueError: - logger.warning(" ---> %s gave ValueError", self.trace_id) - - npix_xi, npix_lam = xilam.npix_xi, xilam.npix_lam - xilam_wcs = xilam.wcs - - # focal-plane coordinate images - ximg_fpa, yimg_fpa = np.meshgrid(x_mm, y_mm) - - # Image mapping (xi, lambda) on the focal plane - xi_fpa = self.xy2xi(ximg_fpa, yimg_fpa).astype(np.float32) - lam_fpa = self.xy2lam(ximg_fpa, yimg_fpa).astype(np.float32) - - # mask everything outside the wavelength range - mask = (xi_fpa >= xi_min) & (xi_fpa <= xi_max) - xi_fpa *= mask - lam_fpa *= mask - - # mask everything outside the wavelength range - # mask = (lam_fpa >= wave_min) & (lam_fpa <= wave_max) - # xi_fpa *= mask - # lam_fpa *= mask - - # Convert to pixel images - # These are the pixel coordinates in the image corresponding to - # xi, lambda - # It is much quicker to do the linear transformation by hand - # than to use the astropy.wcs functions for conversion. - i_img = ((lam_fpa - xilam_wcs.wcs.crval[0]) - / xilam_wcs.wcs.cdelt[0]).astype(int) - j_img = ((xi_fpa - xilam_wcs.wcs.crval[1]) - / xilam_wcs.wcs.cdelt[1]).astype(int) - - # truncate images to remove pixel coordinates outside the image - ijmask = ((i_img >= 0) * (i_img < npix_lam) - * (j_img >= 0) * (j_img < npix_xi)) - - # do the actual interpolation - # image is in [ph/s/um/arcsec] - image = xilam.interp(xi_fpa, lam_fpa, grid=False) * ijmask - - # Scale to ph / s / pixel - image *= self._trace_flux_scale( - ximg_fpa, yimg_fpa, pixsize, pixscale, det_header) + slit_dispersion = None + if "slit_dispersion_colname" in self.meta: + slit_dispersion = self.slit_dispersion + xilam = XiLamImage(fov, self.dlam_per_pix, slit_dispersion=slit_dispersion, _cache=self._slit_cache) + if "optical_resolving_power" in self.meta: + xilam.convolve_lsf(self.meta["optical_resolving_power"]) + self._xilamimg = xilam + + # Integrate the density over each detector pixel. The inverse optical + # map is locally affine within a pixel. Recompute centers and gradients: + # configured transforms can be edited in place between observations. + xx, yy = np.meshgrid(np.arange(sub_naxis1), np.arange(sub_naxis2)) + ximg_fpa, yimg_fpa = det_wcs.all_pix2world(xx, yy, 0) + lam_fpa = self.xy2lam(ximg_fpa, yimg_fpa) + matrix = det_wcs.pixel_scale_matrix + dxidx, dxidy = self.xy2xi.gradient() + dlamdx, dlamdy = self.xy2lam.gradient() + ax, ay = dxidx(ximg_fpa, yimg_fpa), dxidy(ximg_fpa, yimg_fpa) + bx, by = dlamdx(ximg_fpa, yimg_fpa), dlamdy(ximg_fpa, yimg_fpa) + xi_fpa = self.xy2xi(ximg_fpa, yimg_fpa) + xi_x, xi_y = ax*matrix[0, 0] + ay*matrix[1, 0], ax*matrix[0, 1] + ay*matrix[1, 1] + lam_x, lam_y = bx*matrix[0, 0] + by*matrix[1, 0], bx*matrix[0, 1] + by*matrix[1, 1] + scale = self._trace_flux_scale(ximg_fpa, yimg_fpa, pixsize, pixscale, det_header) + quadrature = np.full(2, self._pixel_quadrature_order) + if 'effective_psf_fwhm' in fov.meta: + middle = (wave_min + wave_max)/2 + xm, ym = self.xilam2x(0., middle), self.xilam2y(0., middle) + dxidx, dxidy = self.xy2xi.gradient() + dlamdx, dlamdy = self.xy2lam.gradient() + angular_dispersion = self.slit_dispersion(middle) if slit_dispersion is not None else -self.dlam_per_pix(middle)/pixscale + angular_map = np.array([[dxidx(xm, ym), dxidy(xm, ym)], [dlamdx(xm, ym)/angular_dispersion, dlamdy(xm, ym)/angular_dispersion]]) + angular_steps = np.linalg.norm(angular_map @ det_wcs.pixel_scale_matrix, axis=0) + psf_width = fov.meta['effective_psf_fwhm'].to_value(u.arcsec) + quadrature = np.maximum(quadrature, np.ceil(8*angular_steps/psf_width).astype(int)) + xnodes, xweights = np.polynomial.legendre.leggauss(quadrature[0]) + ynodes, yweights = np.polynomial.legendre.leggauss(quadrature[1]) + for nx, wx in zip(xnodes/2, xweights/2): + for ny, wy in zip(ynodes/2, yweights/2): + xi = xi_fpa + nx*xi_x + ny*xi_y + lam = lam_fpa + nx*lam_x + ny*lam_y + inside = (xi >= xi_min) & (xi < xi_max) & (lam >= wave_min) & (lam < wave_max) + image[inside] += xilam.interp(xi[inside], lam[inside], grid=False) * scale[inside] * wx * wy detector_qe = fov.meta.get("detector_qe") if detector_qe is not None: xpix_img, ypix_img = np.meshgrid( - np.arange(xmin, xmax, dtype=np.float32), - np.arange(ymin, ymax, dtype=np.float32), + np.arange(xmin, xmax, dtype=np.float64), + np.arange(ymin, ymax, dtype=np.float64), ) image = apply_detector_qe_to_trace_image( image, @@ -379,11 +410,7 @@ def map_spectra_to_focal_plane(self, fov): def _trace_flux_scale(self, x_mm, y_mm, pixsize, pixscale, det_header): """Return the local [arcsec um] per detector pixel flux scale.""" - try: - use_jacobian = bool(from_currsys( - self.meta["trace_flux_jacobian"], self.cmds)) - except ValueError: - use_jacobian = False + use_jacobian = from_currsys(self.meta["trace_flux_jacobian"], self.cmds) if not use_jacobian: dlam_by_dx, dlam_by_dy = self.xy2lam.gradient() @@ -400,13 +427,10 @@ def _trace_flux_scale(self, x_mm, y_mm, pixsize, pixscale, det_header): @staticmethod def _detector_pixel_area(det_header): """Return detector WCS pixel area in mm2.""" - try: - pixel_scale_matrix = WCS(det_header, key="D").pixel_scale_matrix - return np.abs(np.linalg.det(pixel_scale_matrix)) - except Exception: # pragma: no cover - FITS fallback for odd headers - cdelt1 = det_header["CDELT1D"] * u.Unit(det_header["CUNIT1D"]) - cdelt2 = det_header["CDELT2D"] * u.Unit(det_header["CUNIT2D"]) - return np.abs((cdelt1 * cdelt2).to_value(u.mm**2)) + detector_wcs = WCS(det_header, key="D") + pixel_area = np.abs(np.linalg.det(detector_wcs.pixel_scale_matrix)) + pixel_area *= u.Unit(detector_wcs.wcs.cunit[0]) * u.Unit(detector_wcs.wcs.cunit[1]) + return pixel_area.to_value(u.mm**2) def rectify( self, @@ -631,6 +655,7 @@ def footprint(self, wave_min=None, wave_max=None, xi_min=None, xi_max=None): # Requested wavelenth range is entirely outside definition range: # no footprint if wave_min > np.max(wave_val) or wave_max < np.min(wave_val): + logger.info("Trace not in wavelength range") return None, None # Restrict to overlap of requested range and definition range @@ -659,6 +684,7 @@ def footprint(self, wave_min=None, wave_max=None, xi_min=None, xi_max=None): # Requested slit range is entirely outside definition range: # no footprint if xi_min > np.max(xi_val) or xi_max < np.min(xi_val): + logger.info("Slit outside range (spatial)") return None, None # Restrict to overlap of requested range and definition range @@ -835,7 +861,41 @@ def __str__(self): return msg -class XiLamImage(): +def _uniform_sum_cdf(position, width_a, width_b): + """CDF of two centered uniform intervals, including a zero-width interval.""" + wide, narrow = np.maximum(width_a, width_b), np.minimum(width_a, width_b) + probability = np.clip(.5 + position / wide, 0, 1) + finite = narrow > 0 + a, b, x = wide[finite], narrow[finite], position[:, finite] + tail = np.clip((a + b)/2 - abs(x), 0, b)**2 / (2*a*b) + probability[:, finite] = np.where(abs(x) <= (a - b)/2, .5 + x/a, np.where(x < 0, tail, 1 - tail)) + return probability + + +def _slit_cell_operator(wavelength, derivative, eta, deta): + """Conservative overlap of wavelength bins sheared across one finite slit cell. + + Wavelength and its derivative use um and um/arcsec; eta uses arcsec. + Locally affine dispersion maps each rectangular (lambda, eta) cell to a + trapezoidal wavelength profile. Integrating its CDF at output edges keeps + the input photon integral, including lines narrower than a wavelength bin. + """ + step = wavelength[1] - wavelength[0] + center = np.arange(len(wavelength)) + derivative * eta / step + spectral_width = 1 + eta * np.gradient(derivative, wavelength) + slit_width = abs(derivative * deta / step) + assert np.all(spectral_width > 0), 'Slit displacement must preserve wavelength ordering.' + span = spectral_width + slit_width + first = np.floor(center - span/2 + .5).astype(int) + output = first[None, :] + np.arange(int(np.ceil(span.max())) + 1)[:, None] + lower = output - .5 - center + probability = _uniform_sum_cdf(lower + 1, spectral_width, slit_width) - _uniform_sum_cdf(lower, spectral_width, slit_width) + source = np.broadcast_to(np.arange(len(wavelength)), output.shape) + valid = (output >= 0) & (output < len(wavelength)) & (probability > 0) + return csr_matrix((probability[valid], (output[valid], source[valid])), shape=(len(wavelength), len(wavelength))) + + +class XiLamImage: """ Class to compute a rectified 2D spectrum. @@ -848,9 +908,13 @@ class XiLamImage(): dlam_per_pix : a 1-D interpolation function from wavelength (in um) to dispersion (in um/pixel); alternatively a number giving an average dispersion + slit_dispersion : callable, optional + Physical cross-slit derivative d(lambda)/d(eta) in um/arcsec as a + function of wavelength in um. When supplied, this replaces the + legacy coupling between sky-grid sampling and detector dispersion. """ - def __init__(self, fov, dlam_per_pix): + def __init__(self, fov, dlam_per_pix, slit_dispersion=None, _cache=None): # ..todo: we assume that we always have a cube. We use SpecCADO's # add_cube_layer method cube_wcs = WCS(fov.cube.header, key=" ") @@ -868,13 +932,20 @@ def __init__(self, fov, dlam_per_pix): (n_lam, n_eta, n_xi) = fov.cube.data.shape # arrays of cube coordinates - cube_xi = d_xi * np.arange(n_xi) + fov.meta["xi_min"].value - cube_eta = d_eta * (np.arange(n_eta) - (n_eta - 1) / 2) + cube_xi = cube_wcs.all_pix2world(np.arange(n_xi), np.zeros(n_xi), np.zeros(n_xi), 0)[0] + cube_eta = cube_wcs.all_pix2world(np.zeros(n_eta), np.arange(n_eta), np.zeros(n_eta), 0)[1] + if cube_wcs.has_celestial: + # Slit coordinates are offsets about zero; RA wraps at 360 degrees. + cube_xi = (cube_xi + 180) % 360 - 180 + cube_xi *= u.Unit(cube_wcs.wcs.cunit[0]).to(u.arcsec) + cube_eta *= u.Unit(cube_wcs.wcs.cunit[1]).to(u.arcsec) + # Each IFU slice has its own optical axis, independent of sky position. + eta_center = (fov.meta["eta_min"] + fov.meta["eta_max"]) / 2 + cube_eta -= eta_center.to_value(u.arcsec) cube_lam = wcs_lam.all_pix2world(np.arange(n_lam), 0)[0] cube_lam *= u.Unit(wcs_lam.wcs.cunit[0]).to(u.um) # Initialise the array to hold the xi-lambda image - self.image = np.zeros((n_xi, n_lam), dtype=np.float32) self.lam = cube_lam try: dlam_per_pix_val = dlam_per_pix(np.asarray(self.lam)) @@ -883,24 +954,31 @@ def __init__(self, fov, dlam_per_pix): logger.warning("Using scalar dlam_per_pix = %.2g", dlam_per_pix_val) - for i, eta in enumerate(cube_eta): - lam0 = self.lam + dlam_per_pix_val * eta / d_eta - - # lam0 is the target wavelength. We need to check that this - # overlaps with the wavelength range covered by the cube - if lam0.min() < cube_lam.max() and lam0.max() > cube_lam.min(): - plane = fov.cube.data[:, i, :].T - plane_interp = RectBivariateSpline(cube_xi, cube_lam, plane, - kx=1, ky=1) - self.image += plane_interp(cube_xi, lam0) - - self.image *= d_eta # ph/s/um/arcsec2 --> ph/s/um/arcsec + derivative = -dlam_per_pix_val / d_eta if slit_dispersion is None else slit_dispersion(self.lam) + derivative = np.broadcast_to(derivative, self.lam.shape) + # The operator depends only on these evaluated physical arrays. Include + # every element, so direct WCS/table/callable changes cannot alias it. + key = (cube_lam.tobytes(), cube_eta.tobytes(), d_eta, derivative.tobytes()) + if _cache is None: + _cache = {} + if _cache.get('key') != key: + _cache.clear() + operators = [_slit_cell_operator(cube_lam, derivative, eta, d_eta).tocoo() for eta in cube_eta] + rows = np.concatenate([operator.row for operator in operators]) + columns = np.concatenate([operator.col*n_eta+i for i, operator in enumerate(operators)]) + weights = np.concatenate([operator.data for operator in operators])*abs(d_eta) + # Interleaved wavelength/slit cells match the contiguous cube layout, + # avoiding a separate strided dense copy for every across-slit row. + _cache['operator'] = csr_matrix((weights, (rows, columns)), shape=(n_lam, n_lam*n_eta)) + _cache['key'] = key + cells = fov.cube.data.reshape(n_lam*n_eta, n_xi) + self.image = (_cache['operator'] @ cells).T # WCS for the xi-lambda image, i.e. the rectified 2D spectrum # Default WCS with xi in arcsec self.wcs = WCS(naxis=2) self.wcs.wcs.crpix = [1, 1] - self.wcs.wcs.crval = [self.lam[0], fov.meta["xi_min"].value] + self.wcs.wcs.crval = [self.lam[0], cube_xi[0]] self.wcs.wcs.pc = [[1, 0], [0, 1]] self.wcs.wcs.cdelt = [d_lam, d_xi] self.wcs.wcs.ctype = ["LINEAR", "LINEAR"] @@ -920,14 +998,30 @@ def __init__(self, fov, dlam_per_pix): self.xi = self.wcs.all_pix2world(self.lam[0], np.arange(n_xi), 0)[1] self.npix_xi = n_xi self.npix_lam = n_lam - # ..todo: cubic spline introduces negative values, linear does not. - # Alternative might be to cubic-spline interpolate on sqrt(image), - # with subsequent squaring of the result. This would require - # wrapping RectBivariateSpline in a new (sub)class. - spline_order = (1, 1) - self.interp = RectBivariateSpline(self.xi, self.lam, self.image, - kx=spline_order[0], - ky=spline_order[1]) + + def interp(self, xi, wavelength, grid=True): + """Bilinear density interpolation on the existing uniform WCS grid.""" + xi, wavelength = np.meshgrid(xi, wavelength, indexing='ij') if grid else np.broadcast_arrays(xi, wavelength) + row = (xi - self.xi[0])/self.wcs.wcs.cdelt[1] + column = (wavelength - self.lam[0])/self.wcs.wcs.cdelt[0] + return map_coordinates(self.image, [row, column], order=1, mode='nearest', prefilter=False) + + def convolve_lsf(self, resolving_power): + """Apply an optical Gaussian LSF at constant R, preserving photon density. + + This is the optical blur alone; slit illumination is integrated above. + Convolving lambda * I_lambda on a uniform log(lambda) grid preserves + integrated flux rather than treating I_lambda as a log-bin count. + """ + if resolving_power <= 0: + raise ValueError("Optical resolving power must be positive") + log_wave = np.log(self.lam) + n = int(np.ceil((log_wave[-1] - log_wave[0]) / np.min(np.diff(log_wave)))) + 1 + log_grid = np.linspace(log_wave[0], log_wave[-1], n) + sigma = 1 / (resolving_power * np.sqrt(8 * np.log(2)) * (log_grid[1] - log_grid[0])) + density = interp1d(log_wave, self.image * self.lam, axis=1)(log_grid) + blurred = gaussian_filter1d(density, sigma, axis=1, mode="nearest") + self.image = interp1d(log_grid, blurred, axis=1)(log_wave) / self.lam class Transform2D(): @@ -1233,13 +1327,18 @@ def _xiy2xlam_fit(layout, params): def make_image_interpolations(hdulist, **kwargs): - """Create 2D interpolation functions for images.""" + """Create 2D interpolation functions for images. + + The interpolation functions are called with (j, i) pixel coordinates, + i.e. the first argument corresponds to the row (NAXIS2) axis of the + image, the second to the column (NAXIS1) axis. + """ interps = [] for hdu in hdulist: if isinstance(hdu, fits.ImageHDU): interps.append( - RectBivariateSpline(np.arange(hdu.header["NAXIS1"]), - np.arange(hdu.header["NAXIS2"]), + RectBivariateSpline(np.arange(hdu.header["NAXIS2"]), + np.arange(hdu.header["NAXIS1"]), hdu.data, **kwargs) ) return interps @@ -1300,3 +1399,7 @@ def get_affine_parameters(coords): shears = (np.average(shears, axis=0) * rad2deg) - (90 + rotations) return rotations, shears + +def det_offset(x, offset): + """Apply offset to x""" + return x + offset diff --git a/scopesim/effects/surface_list.py b/scopesim/effects/surface_list.py index 12e37a1cf..811b367ec 100644 --- a/scopesim/effects/surface_list.py +++ b/scopesim/effects/surface_list.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- """TBA.""" -import warnings from collections import OrderedDict from copy import deepcopy from typing import ClassVar @@ -40,31 +39,6 @@ def __init__(self, **kwargs): surf_kwargs["filename"] = from_currsys(surf_kwargs["filename"], self.cmds) self.surfaces[surf_kwargs["name"]] = SpectralSurface(**surf_kwargs) - def fov_grid(self, which="waveset", **kwargs): - warnings.warn("The fov_grid method is deprecated and will be removed " - "in a future release.", DeprecationWarning, stacklevel=2) - wave_edges = [] - if which == "waveset": - self.meta.update(kwargs) - self.meta = from_currsys(self.meta, self.cmds) - wave_min = quantify(self.meta["wave_min"], u.um) - wave_max = quantify(self.meta["wave_max"], u.um) - # ..todo:: add 1001 to default.yaml somewhere - wave = np.linspace(wave_min, wave_max, 1001) - throughput = self.throughput(wave) - threshold = self.meta["minimum_throughput"] - valid_waves = np.where(throughput >= threshold)[0] - - if not len(valid_waves): - msg = ("No transmission found above the threshold " - f"{self.meta['minimum_throughput']} in this wavelength " - f"range {[self.meta['wave_min'], self.meta['wave_max']]}." - " Did you open the shutter?") - raise ValueError(msg) - - wave_edges = [min(wave[valid_waves]), max(wave[valid_waves])] - return wave_edges - @property def throughput(self): if self._throughput is None: @@ -255,6 +229,8 @@ def plot(self, which="x", wavelength=None, *, axes=None, **kwargs): def _row_quantity_or_value(table, colname, index): col = table[colname] value = col[index] + if isinstance(value, str): + return value if col.unit is not None or f"{colname}_unit" in table.meta: return quantity_from_table(colname, table)[index] return value.item() if hasattr(value, "item") else value diff --git a/scopesim/effects/ter_curves.py b/scopesim/effects/ter_curves.py index 340af7eab..f4a6e4219 100644 --- a/scopesim/effects/ter_curves.py +++ b/scopesim/effects/ter_curves.py @@ -4,29 +4,43 @@ import warnings from typing import ClassVar from collections.abc import Collection, Iterable +from numbers import Integral import numpy as np +from numpy.typing import ArrayLike, NDArray from scipy import integrate from astropy import units as u from astropy.io import fits from astropy.table import Table from astropy.wcs import WCS from copy import deepcopy +from synphot import SpectralElement, SourceSpectrum import skycalc_ipy from . import DataContainer from .effects import Effect -from .ter_curves_utils import (add_edge_zeros, combine_two_spectra, - apply_throughput_to_cube, download_svo_filter, - download_svo_filter_list) +from .ter_curves_utils import ( + add_edge_zeros, + download_svo_filter, + download_svo_filter_list, +) from ..optics.fov_volume_list import FovVolumeList from ..optics.fov import FieldOfView from ..optics.surface import SpectralSurface from ..source.source import Source -from ..source.source_fields import (CubeSourceField, SpectrumSourceField, - BackgroundSourceField) -from ..utils import (from_currsys, quantify, check_keys, find_file, - figure_factory, get_logger) +from ..source.source_fields import ( + CubeSourceField, + SpectrumSourceField, + BackgroundSourceField, +) +from ..utils import ( + from_currsys, + quantify, + check_keys, + find_file, + figure_factory, + get_logger, +) from ..server.download_utils import create_retriever logger = get_logger(__name__) @@ -156,62 +170,91 @@ def __init__(self, filename=None, **kwargs): self.surface.table = data self.surface.table.meta.update(self.meta) - def apply_to(self, obj, **kwargs): - if isinstance(obj, Source): - self.meta = from_currsys(self.meta, self.cmds) - wave_min = quantify(self.meta["wave_min"], u.um).to(u.AA) - wave_max = quantify(self.meta["wave_max"], u.um).to(u.AA) + def __call__(self, data: ArrayLike, waveset: u.Quantity[u.um]) -> NDArray: + return data * self.throughput(waveset).value[:, None, None] - thru = self.throughput + def _apply_to_fvl(self, fvl: FovVolumeList) -> None: + logger.debug("Apply %s, FoV setup", self.display_name) - # apply transmission to source spectra - for fld in obj.fields: - if isinstance(fld, CubeSourceField): - fld.field = apply_throughput_to_cube( - fld.field, thru, fld.waveset) - elif isinstance(fld, SpectrumSourceField): - fld.spectra = { - isp: combine_two_spectra(spec, thru, "multiply", - wave_min, wave_max) - for isp, spec in fld.spectra.items() - } - else: - # Rather log than raise here, can still move on - logger.error("Source field is neither Cube nor has " - "spectra, this shouldn't occur...") + wave = self.surface.throughput.waveset + thru = self.surface.throughput(wave) + valid_waves = np.argwhere(thru > 0) - # add the effect background to the source background field - if self.background_source is not None: - for bgs in self.background_source: - obj.append(bgs) + if not len(valid_waves): + raise ValueError( + f"No transmission found in {self.display_name} over the " + f"wavelength range [{wave.min().to_value(u.um):.3f}, " + f"{wave.max().to_value(u.um):.3f}] um. " + "Did you open the shutter?") - if isinstance(obj, FovVolumeList): - wave = self.surface.throughput.waveset - thru = self.surface.throughput(wave) - valid_waves = np.argwhere(thru > 0) - wave_min = wave[max(0, valid_waves[0][0] - 1)] - wave_max = wave[min(len(wave) - 1, valid_waves[-1][0] + 1)] + wave_min = wave[max(0, valid_waves[0][0] - 1)] + wave_max = wave[min(len(wave) - 1, valid_waves[-1][0] + 1)] + + fvl.shrink("wave", [wave_min.to_value(u.um), wave_max.to_value(u.um)]) + + def _apply_to_src(self, src: Source) -> None: + logger.debug("Apply %s, Source spectra", self.display_name) + + self.meta = from_currsys(self.meta, self.cmds) + wave_min = quantify(self.meta["wave_min"], u.um).to(u.AA) + wave_max = quantify(self.meta["wave_max"], u.um).to(u.AA) + + thru = self.throughput + + # apply transmission to source spectra + for fld in src.fields: + if isinstance(fld, CubeSourceField): + ## in-place is much faster than self() + #fld.field.data = self(fld.field.data, fld.waveset) + fld.field.data *= thru(fld.waveset).value[:, None, None] + elif isinstance(fld, SpectrumSourceField): + fld.spectra = { + isp: spec * thru # This works because synphot! + for isp, spec in fld.spectra.items() + } + else: + # Rather log than raise here, can still move on + logger.error( + "Source field is neither Cube nor has spectra, this " + "should not happen, but moving on.") + + # add the effect background to the source background field + if self.background_source is not None: + for bgs in self.background_source: + src.append(bgs) - obj.shrink("wave", [wave_min.to_value(u.um), - wave_max.to_value(u.um)]) + def apply_to(self, obj, **kwargs): + if isinstance(obj, Source): + self._apply_to_src(obj) + + if isinstance(obj, FovVolumeList): + self._apply_to_fvl(obj) ## If obj is a FieldOfView, apply the throughput if isinstance(obj, FieldOfView): thru = self.throughput - if obj.hdu is not None and obj.hdu.header['NAXIS'] == 3: + if obj.hdu is None: + # Retain native spectral knots until spatial rendering, so + # narrow lines survive multiplication by the throughput. + for field in obj.fields: + if isinstance(field, CubeSourceField): + field.data *= thru(field.waveset).value[:, None, None] + elif isinstance(field, SpectrumSourceField): + field.spectra = {ref: spec * thru for ref, spec in field.spectra.items()} + elif obj.hdu.header['NAXIS'] == 3: swcs = WCS(obj.hdu.header).spectral with u.set_enabled_equivalencies(u.spectral()): wave = swcs.pixel_to_world(np.arange(swcs.pixel_shape[0])) << u.um - obj.hdu = apply_throughput_to_cube(obj.hdu, thru, wave) + obj.hdu.data *= thru(wave).value[:, None, None] return obj @property - def emission(self): + def emission(self) -> SourceSpectrum: return self.surface.emission @property - def throughput(self): + def throughput(self) -> SpectralElement: return self.surface.throughput @property @@ -222,23 +265,16 @@ def background_source(self): bkg_hdr = fits.Header() bkg_hdr.update({ + "ROLE": "background", # not used yet "BG_SRC": True, "BG_SURF": self.display_name, "SPEC_REF": 0, - # Seem those are not needed... - # "CUNIT1": "ARCSEC", - # "CUNIT2": "ARCSEC", - # "CDELT1": 0, - # "CDELT2": 0, "BUNIT": "photlam arcsec-2", "SOLIDANG": "arcsec-2", }) bkg_fld = BackgroundSourceField(field=None, spectra=bkg_spec, header=bkg_hdr) bkg_src = Source(field=bkg_fld) - # Before BackgroundSourceField: - # bkg_hdu = fits.ImageHDU(header=bkg_hdr) - # bkg_src = Source(image_hdu=bkg_hdu, spectra=flux) self._background_source = bkg_src return [self._background_source] @@ -520,9 +556,14 @@ def load_skycalc_table(self): else: path = find_file(use_local_file) - fits_tbl = fits.getdata(path, ext=1) + with fits.open(path) as sky_file: + fits_tbl = sky_file[1].data + # Older cached files require an explicit unit if TUNIT is absent. + unit = self.meta["local_skycalc_wavelength_unit"] if "local_skycalc_wavelength_unit" in self.meta else sky_file[1].columns["lam"].unit + wave_unit = u.Unit(unit) fits_hdr = fits.getheader(path, ext=0) tbl = Table(fits_tbl) + tbl["lam"] = tbl["lam"] * wave_unit.to(u.um) tbl["lam"].unit = u.um for colname in tbl.colnames: if "flux" in colname: @@ -888,34 +929,6 @@ def __init__(self, cmds=None, **kwargs): # TODO: maybe use actually masked table here? self.table["transmission"][mask] = 0 - def fov_grid(self, which="waveset", **kwargs): - warnings.warn("The fov_grid method is deprecated and will be removed " - "in a future release.", DeprecationWarning, stacklevel=2) - if which == "waveset": - self.meta.update(kwargs) - self.meta = from_currsys(self.meta, self.cmds) - # ..todo:: replace the 101 with a variable in !SIM - wave = np.linspace(self.meta["wave_min"], - self.meta["wave_max"], 101) - wave = quantify(wave, u.um) - throughput = self.surface.transmission(wave) - min_thru = self.meta["minimum_throughput"] - valid_waves = np.where(throughput.value > min_thru)[0] - if len(valid_waves) > 0: - wave_edges = [min(wave[valid_waves].value), - max(wave[valid_waves].value)] * u.um - else: - raise ValueError("No transmission found above the threshold {}" - " in this wavelength range {}. Did you open " - "the shutter?" - "".format(self.meta["minimum_throughput"], - [self.meta["wave_min"], - self.meta["wave_max"]])) - else: - wave_edges = [] - - return wave_edges - @property def fwhm(self): wave = self.surface.wavelength @@ -1073,11 +1086,6 @@ def surface(self): def throughput(self): return self.current_filter.throughput - def fov_grid(self, which="waveset", **kwargs): - warnings.warn("The fov_grid method is deprecated and will be removed " - "in a future release.", DeprecationWarning, stacklevel=2) - return self.current_filter.fov_grid(which=which, **kwargs) - def change_filter(self, filtername=None): """Change the current filter.""" if filtername in self.filters.keys(): @@ -1557,6 +1565,7 @@ class DichroicTree(Effect): ``` This effect creates a volume of FoVs, one for each row in the dichroic tree table. + For imaging, pass a Source with its fov context to apply the branch throughput before rendering. """ required_keys = {"tree_filename", "dichroic_names", "filename_format"} z_order: ClassVar[tuple[int, ...]] = (117, 217, 617) @@ -1564,6 +1573,9 @@ class DichroicTree(Effect): def __init__(self, cmds=None, **kwargs): check_keys(kwargs, self.required_keys, action="error") super().__init__(cmds=cmds, **kwargs) + if "z_order" in self.meta: + z_order = self.meta["z_order"] + self.z_order = (int(z_order),) if isinstance(z_order, Integral) else tuple(z_order) # Load all the dichroic TER curves self.dichroics = {} @@ -1580,7 +1592,7 @@ def __init__(self, cmds=None, **kwargs): tree = DataContainer(filename=tree_path) self.table = tree.table - def apply_to(self, obj, **kwargs): + def apply_to(self, obj, fov=None, **kwargs): """ To the object, loop through each row of the table and apply the dichroics (defined by columns) with the corresponding action defined in the table cell. @@ -1618,9 +1630,11 @@ def apply_to(self, obj, **kwargs): obj.volumes = new_vols # 2. During observe - if isinstance(obj, FieldOfView): + if isinstance(obj, FieldOfView) or (isinstance(obj, Source) and fov is not None): logger.debug("Executing %s, observe", self.meta['name']) - obj_id = obj.meta.get(id_colname, None) + branch_fov = obj if isinstance(obj, FieldOfView) else fov + obj_id = branch_fov.meta[id_colname] + setattr(branch_fov, id_colname, obj_id) row = self.table[self.table[id_colname] == obj_id] if len(row) != 1: diff --git a/scopesim/effects/ter_curves_utils.py b/scopesim/effects/ter_curves_utils.py index cabc6d796..dd2f2f1c5 100644 --- a/scopesim/effects/ter_curves_utils.py +++ b/scopesim/effects/ter_curves_utils.py @@ -8,12 +8,8 @@ from astropy.table import Table from astropy.io.votable import parse_single_table from astropy.io import ascii as ioascii -from astropy.io import fits -from synphot import SpectralElement, SourceSpectrum, Empirical1D, Observation -from synphot.units import PHOTLAM +from synphot import SpectralElement, Empirical1D -from scopesim.source.source_templates import vega_spectrum, st_spectrum, \ - ab_spectrum from ..utils import find_file, quantity_from_table, get_logger from ..server.download_utils import create_client, handle_download @@ -53,6 +49,7 @@ PATH_SVO_DATA = PATH_HERE.parent / "data" / "svo" +# This function is only used in the AnisoCADO PSF... def get_filter_effective_wavelength(filter_name): # TODO: This is technically stored in the SVO XML file as WavelengthEff... # (actually WavelengthMean, by definition of formula ...) @@ -218,226 +215,6 @@ def get_filter(filter_name): return filt -def get_zero_mag_spectrum(system_name: str = "AB") -> SourceSpectrum: - """ - Return a synphot spectrum of 0 mag in given photometric system. - - Parameters - ---------- - system_name : {"AB", "Vega", "ST", "HST"} - Name of the photometric system. - - Raises - ------ - ValueError - Raised if `system_name` is invalid. - - Returns - ------- - spec : synphot.SourceSpectrum - Output spectrum. - - """ - if system_name.lower() == "vega": - return vega_spectrum() - if system_name.lower() == "ab": - return ab_spectrum() - if system_name.lower() in ("st", "hst"): - return st_spectrum() - raise ValueError(f"system_name {system_name} is unknown") - - -def zero_mag_flux(filter_name, photometric_system, return_filter=False): - """ - Return the zero magnitude photon flux for a filter. - - Acceptable filter names are those given in - ``scopesim.effects.ter_curves_utils.FILTER_DEFAULTS`` or a string with an - appropriate name for a filter in the Spanish-VO filter-service. Such - strings must use the naming convention: observatory/instrument.filter. E.g: - "paranal/HAWKI.Ks", or "Gemini/GMOS-N.CaT". - - Parameters - ---------- - filter_name : str - Name of the filter - see above. - - photometric_system : {"vega", "AB", "ST"} - Name of the photometric system - - return_filter : bool, optional - If True, also returns the filter curve object - - Returns - ------- - flux : float - [PHOTLAM] - filt : ``synphot.SpectralElement`` - If `return_filter` is True - - """ - filt = get_filter(filter_name) - spec = get_zero_mag_spectrum(photometric_system) - - obs = Observation(spec, filt) - flux = obs.effstim(flux_unit=PHOTLAM) - - if return_filter: - return flux, filt - return flux - - -def scale_spectrum(spectrum, filter_name, amplitude): - """ - Scale a SourceSpectrum to a value in a filter. - - Parameters - ---------- - spectrum : synphot.SourceSpectrum - - filter_name : str - Name of a filter from - - a local instrument package (available in ``rc.__search_path__``) - - a generic filter name (see ``ter_curves_utils.FILTER_DEFAULTS``) - - a spanish-vo filter service reference (e.g. "Paranal/HAWKI.Ks") - - amplitude : astropy.Quantity, float - The value that the spectrum should have in the given filter. Acceptable - astropy quantities are: - - u.mag : Vega magnitudes - - u.ABmag : AB magnitudes - - u.STmag : HST magnitudes - - u.Jy : Jansky per filter bandpass - Additionally the "FLAM" and "FNU" units from ``synphot.units`` can - be used when passing the quantity for `amplitude`. - - Returns - ------- - spectrum : synphot.SourceSpectrum - Input spectrum scaled to the given amplitude in the given filter. - - Examples - -------- - :: - - >>> from scopesim.source.source_templates import vega_spectrum - >>> from scopesim.effects.ter_curves_utils as ter_utils - >>> - >>> spec = vega_spectrum() - >>> vega_185 = ter_utils.scale_spectrum(spec, "Ks", -1.85 * u.mag) - >>> ab_0 = ter_utils.scale_spectrum(spec, "Ks", 0 * u.ABmag) - >>> jy_3630 = ter_utils.scale_spectrum(spec, "Ks", 3630 * u.Jy) - - """ - if isinstance(amplitude, u.Quantity): - if amplitude.unit.physical_type == "spectral flux density": - if amplitude.unit != u.ABmag: - amplitude = amplitude.to(u.ABmag) - ref_spec = ab_spectrum(amplitude.value) - - elif amplitude.unit.physical_type == "spectral flux density wav": - if amplitude.unit != u.STmag: - amplitude = amplitude.to(u.STmag) - ref_spec = st_spectrum(amplitude.value) - - elif amplitude.unit == u.mag: - ref_spec = vega_spectrum(amplitude.value) - - else: - raise ValueError(f"Units of amplitude must be one of " - f"[u.mag, u.ABmag, u.STmag, u.Jy]: {amplitude}") - else: - ref_spec = vega_spectrum(amplitude) - - filt = get_filter(filter_name) - ref_flux = Observation(ref_spec, filt).effstim(flux_unit=PHOTLAM) - - real_flux = Observation(spectrum, filt).effstim(flux_unit=PHOTLAM) - scale_factor = ref_flux / real_flux - spectrum *= scale_factor.value - - return spectrum - - -def apply_throughput_to_cube( - cube: fits.ImageHDU, - thru: SpectralElement | SourceSpectrum, - wave_cube: u.Quantity, -) -> fits.ImageHDU: - """ - Apply throughput curve to a spectroscopic cube. - - Parameters - ---------- - cube : fits.ImageHDU - Three-dimensional image, dimension 0 (in python convention) is the - spectral dimension. - thru : SpectralElement | SourceSpectrum - Throughput curve, spectrum or filter. - wave_cube : u.Quantity["length"] - Wavelength axis of the cube. - - Returns - ------- - cube : fits.ImageHDU - Header unchanged, data multiplied with wavelength-dependent throughput. - - """ - # Note: wave_cube used to be converted to AA, but synphot understands um - # or whatever just as well... - cube.data *= thru(wave_cube).value[:, None, None] - return cube - - -def combine_two_spectra(spec_a, spec_b, action, wave_min, wave_max): - """ - Combine transmission and/or emission spectrum with a common waverange. - - Spec_A is the source spectrum - Spec_B is either the transmission or emission that should be applied - - Parameters - ---------- - spec_a : synphot.SourceSpectrum - spec_b : synphot.SpectralElement, synphot.SourceSpectrum - action: {"multiply", "add"} - wave_min, wave_max : quantity - [Angstrom] - - Returns - ------- - new_source : synphot.SourceSpectrum - - """ - if spec_a.waveset is None: - wave_val = spec_b.waveset.value - else: - wave_val = spec_a.waveset.value - mask = (wave_val > wave_min.value) * (wave_val < wave_max.value) - - wave = ([wave_min.value] + list(wave_val[mask]) + [wave_max.value]) * u.AA - if "mult" in action.lower(): - spec_c = spec_a(wave) * spec_b(wave) - # Diagnostic plots - not for general use - # from matplotlib import pyplot as plt - # plt.plot(wave, spec_a(wave), label="spec_a") - # plt.plot(wave, spec_b(wave), label="spec_b") - # plt.plot(wave, spec_c, label="spec_c") - # plt.xlim(2.9e4, 4.2e4) - # plt.legend() - # plt.show() - elif "add" in action.lower(): - spec_c = spec_a(wave) + spec_b(wave) - else: - raise ValueError(f"action {action} unknown") - - new_source = SourceSpectrum(Empirical1D, points=wave, lookup_table=spec_c) - new_source.meta.update(spec_b.meta) - new_source.meta.update(spec_a.meta) - - return new_source - - def add_edge_zeros(tbl, wave_colname): if isinstance(tbl, Table): vals = np.zeros(len(tbl.colnames)) diff --git a/scopesim/optics/echelle.py b/scopesim/optics/echelle.py index 4c7614e63..709b2fd2e 100644 --- a/scopesim/optics/echelle.py +++ b/scopesim/optics/echelle.py @@ -11,7 +11,10 @@ def spectrograph_factory(min_wave: float|u.Quantity, max_wave: float|u.Quantity, echelle_groove_length: float|u.Quantity, pix_per_res_elem: float, disp_npix: int, xdisp_npix: int, pix_size: float|u.Quantity, xdisp_groove_length: float|u.Quantity = 0.0, xdisp_beta_center: float|u.Quantity = 0.0, - dispersion_focal_len: float|u.Quantity = None): + dispersion_focal_len: float|u.Quantity = None, + collimator1_focal_length: u.Quantity = None, + collimator2_focal_length: u.Quantity = None, + entrance_slit_scale: u.Quantity = None): """ Parameters @@ -47,6 +50,11 @@ def spectrograph_factory(min_wave: float|u.Quantity, max_wave: float|u.Quantity, dispersion_focal_len: float|u.Quantity Effective focal length for the echelle/main-dispersion detector coordinate. If float, assumed to be in mm. Defaults to focal_len. + collimator1_focal_length, collimator2_focal_length: u.Quantity + Physical collimator lengths. When supplied together, derive the + dispersion focal length from focal_len times their ratio. + entrance_slit_scale: u.Quantity + Entrance-slit length per on-sky angle, required with the collimators. Returns ------- @@ -98,6 +106,9 @@ def spectrograph_factory(min_wave: float|u.Quantity, max_wave: float|u.Quantity, pixels_per_res_elem=pix_per_res_elem, focal_length=focal_len, dispersion_focal_length=dispersion_focal_len, + collimator1_focal_length=collimator1_focal_length, + collimator2_focal_length=collimator2_focal_length, + entrance_slit_scale=entrance_slit_scale, grating=echelle_grating, detector=Detector(disp_npix, xdisp_npix, pix_size), cross_disperser=cross_disperser) @@ -191,16 +202,21 @@ def estimate_xdisp_angle_with_groove_len( if not isinstance(d, u.Quantity): d = d * u.mm - # assume we want a dispersion at the midpoint that matches the range to the detector length - t = detector_length / focal_length - k = (l_max - l_min) / d - x = k * (1 + np.sqrt(1 + t ** 2)) / 2 / t - beta_center_guess = np.arccos(np.sqrt(x)) - beta_half_range = np.arcsin(k / 2 / np.sqrt(x)) - beta_max = beta_center_guess + beta_half_range - beta_min = beta_center_guess - beta_half_range - alpha = np.arcsin(l_max / d - np.sin(beta_max)) - return alpha.to(u.rad), ((beta_min + beta_max) / 2).to(u.rad) + # The camera maps beta_center +/- half_angle to +/- detector_length/2. + # Subtracting the two grating equations eliminates the incidence angle: + # (l_max - l_min)/d = 2*cos(beta_center)*sin(half_angle). + if l_max <= l_min or detector_length <= 0 * u.mm or focal_length <= 0 * u.mm or d <= 0 * u.mm: + raise ValueError("Wavelength range, detector length, focal length and groove spacing must be positive.") + half_angle = np.arctan(detector_length / (2 * focal_length)) + cos_center = (l_max - l_min) / (2 * d * np.sin(half_angle)) + if not 0 < cos_center <= 1: + raise ValueError("The wavelength range cannot fit this detector with the specified groove spacing.") + beta_center = np.arccos(cos_center) + sin_alpha = l_min / d - np.sin(beta_center - half_angle) + if beta_center + half_angle >= 90 * u.deg or not -1 <= sin_alpha <= 1: + raise ValueError("The requested detector span requires a non-propagating grating angle.") + alpha = np.arcsin(sin_alpha) + return alpha.to(u.rad), beta_center.to(u.rad) @staticmethod def estimate_xdisp_freq_and_angle(l_min: float | u.Quantity, @@ -364,6 +380,9 @@ def __init__( detector: Detector, cross_disperser: GratingSetup = None, dispersion_focal_length: u.Quantity = None, + collimator1_focal_length: u.Quantity = None, + collimator2_focal_length: u.Quantity = None, + entrance_slit_scale: u.Quantity = None, ): """ @@ -398,6 +417,12 @@ def __init__( if dispersion_focal_length is None else dispersion_focal_length.to(u.mm) ) + if collimator1_focal_length is not None: + self.collimator1_focal_length = collimator1_focal_length.to(u.mm) + self.collimator2_focal_length = collimator2_focal_length.to(u.mm) + self.entrance_slit_scale = entrance_slit_scale.to(u.mm / u.arcsec) + # Angular dispersion is amplified by the asymmetric pupil relay. + self.dispersion_focal_length = self.focal_length * self.collimator1_focal_length / self.collimator2_focal_length self.pixel_scale = np.arctan( self.detector.pixel_size / self.dispersion_focal_length) self.beta_central_pixel = self.grating.beta_center @@ -432,6 +457,22 @@ def set_beta_center(self, beta, littrow: bool = False): if littrow: self.grating.alpha = beta + def slit_dispersion(self, wavelength, order): + """Apparent wavelength shift per across-slit arcsecond at fixed incident wavelength. + + m*lambda = d*(sin(alpha) + sin(beta)); increasing entrance-slit eta + increases alpha and reduces the wavelength inferred at nominal alpha. + The first collimator sets this angle; the pupil relay is already in + the camera's dispersion focal length and must not enter a second time. + """ + derivative = -self.grating.d * np.cos(self.grating.alpha) * self.entrance_slit_scale / (order * self.collimator1_focal_length) + return np.ones(np.shape(wavelength)) * derivative.to(u.um / u.arcsec) + + def projected_slit_width(self, wavelength, order, slit_width): + """Unrotated echelle-coordinate slit width, including local anamorphism.""" + half_shift = self.slit_dispersion(wavelength, order) * slit_width / 2 + return abs(self.wavelength_to_x_pixel(wavelength + half_shift, order) - self.wavelength_to_x_pixel(wavelength - half_shift, order)) * u.pixel + @property def orders(self): try: @@ -558,17 +599,15 @@ def order_mask(self, wave, fsr_edge: bool = False): def edge_wave(self, fsr=True): """ :param fsr: True to return the FSR edge wavelengths, False to return the detector edges - :return: wavelengths at the edges of the detector (or FSR) for each order, always include central pixel wavelength + :return: the two endpoint wavelengths for each order """ - pix = np.array([0, self.detector.n_pix_x // 2, self.detector.n_pix_x-1]) + .5 - fiducial_waves = self.x_pixel_to_wavelength(pix, self.orders[:, None]) + pixels = np.array([0, self.detector.n_pix_x//2, self.detector.n_pix_x - 1]) + .5 + fiducial_waves = self.x_pixel_to_wavelength(pixels, self.orders[:, None]) if not fsr: return fiducial_waves[:, [0, -1]] - + # Retain the original central-pixel wavelength convention for the FSR. central_fsr = fiducial_waves[:, 1] / self.orders - fsr_edges = (u.Quantity([-central_fsr / 2, central_fsr / 2]) + fiducial_waves[:, 1]).T - - return fsr_edges + return (u.Quantity([-central_fsr/2, central_fsr/2]) + fiducial_waves[:, 1]).T def central_wave(self, order): """ @@ -616,6 +655,22 @@ def x_pixel_wavelengths(self, pixel_phase=.5): def wavelength_to_y_pixel(self, wave): return self.y_pixel_for_beta(self.cross_disperser.beta(wave, 1)) + def _rotated_detector_pixels(self, wave, order): + x = self.wavelength_to_x_pixel(wave, order).to_value(u.dimensionless_unscaled) + y = self.wavelength_to_y_pixel(wave).to_value(u.dimensionless_unscaled) + xpix = np.broadcast_to(x, (self._slit_offset_pix.size, wave.size)) + ypix = y[None, :] + self._slit_offset_pix[:, None] + return self._detector_cos*xpix - self._detector_sin*ypix, self._detector_sin*xpix + self._detector_cos*ypix + + def _detector_pixels(self, wave, order): + """The trace generator's placed coordinates for all configured slit nodes. + + Unlike the clipped trace table, this mapping can also inspect the full + FSR outside the detector. It uses the same rotation and placement. + """ + x, y = self._rotated_detector_pixels(wave, order) + return x - self._detector_center[0] + self.detector.n_pix_x/2, y - self._detector_center[1] + self.detector.n_pix_y/2 + def y_pixel_to_wavelength(self, pixel, m): """ :param pixel: pixel diff --git a/scopesim/optics/fov.py b/scopesim/optics/fov.py index ef251938d..3f56c4e28 100644 --- a/scopesim/optics/fov.py +++ b/scopesim/optics/fov.py @@ -1,14 +1,14 @@ # -*- coding: utf-8 -*- """Defines FieldOfView class.""" +from copy import copy from warnings import warn -from copy import deepcopy from itertools import chain from collections.abc import Iterable, Generator -from weakref import WeakKeyDictionary import numpy as np -from scipy.interpolate import interp1d +from scipy.integrate import cumulative_trapezoid +from scipy.signal import fftconvolve from astropy import units as u from astropy.io import fits @@ -33,7 +33,6 @@ get_logger, array_minmax, close_loop, - unit_includes_per_physical_type, figure_factory, image_plotter, cube_plotter, @@ -45,7 +44,6 @@ logger = get_logger(__name__) -_SPECTRUM_WAVESET_AA_CACHE = WeakKeyDictionary() class FieldOfView: @@ -77,6 +75,7 @@ def __init__(self, header, waverange, detector_header=None, cmds=None, **kwargs) "wave_max": quantify(waverange[1], u.um), "wave_bin_n": 1, "wave_bin_type": "linear", + 'spectral_padding': 0*u.um, "area": 0 * u.m**2, "pixel_area": None, # [arcsec] @@ -136,7 +135,7 @@ def pixel_area(self): self.meta["pixel_area"] = pixel_area(self.header) return self.meta["pixel_area"] - def extract_from(self, src) -> None: + def extract_from(self, src, spatial_header=None, _spectral_wavesets=None) -> None: """ Extract relevent fields from source object. @@ -158,17 +157,22 @@ def extract_from(self, src) -> None: """ assert isinstance(src, Source), f"expected Source: {type(src)}" + self.fields = [] + self.hdu = None - fields_in_fov = list(self.get_fields_in_fov(src.fields)) + corners_arcsec, _ = self.get_corners("arcsec") + if spatial_header is not None: + corners_arcsec = imp_utils.calc_footprint(spatial_header, new_unit='arcsec') + corners_deg = corners_arcsec / 3600 + fields_in_fov = list(self.get_fields_in_fov(src.fields, corners_arcsec)) if not fields_in_fov: logger.warning("No fields in FOV.") else: logger.debug("%d fields in FOV", len(fields_in_fov)) - corners_arcsec, _ = self.get_corners("arcsec") - corners_deg, _ = self.get_corners("deg") minmax = array_minmax(corners_arcsec) * u.arcsec + source_range = self.waverange + np.array([-1, 1])*self.meta['spectral_padding'] for field in fields_in_fov: if isinstance(field, TableSourceField): @@ -180,7 +184,7 @@ def extract_from(self, src) -> None: new_fld = TableSourceField( field=extracted, spectra={ - ref: extract_range_from_spectrum(spec, self.waverange) + ref: extract_range_from_spectrum(spec, source_range, _waveset=None if _spectral_wavesets is None else _spectral_wavesets[spec]) for ref, spec in field.spectra.items() if ref in extracted["ref"] }, @@ -194,7 +198,7 @@ def extract_from(self, src) -> None: new_fld = ImageSourceField( field=extracted, spectra={ - ref: extract_range_from_spectrum(spec, self.waverange) + ref: extract_range_from_spectrum(spec, source_range, _waveset=None if _spectral_wavesets is None else _spectral_wavesets[spec]) for ref, spec in field.spectra.items() }, ) @@ -212,7 +216,7 @@ def extract_from(self, src) -> None: field=None, header=field.header, spectra={ - ref: extract_range_from_spectrum(spec, self.waverange) + ref: extract_range_from_spectrum(spec, source_range, _waveset=None if _spectral_wavesets is None else _spectral_wavesets[spec]) for ref, spec in field.spectra.items() }, ) @@ -250,12 +254,13 @@ def view(self, sub_pixel: bool | None = None): def flatten(self): """If cube, collapse along first axis.""" if self.hdu and self.hdu.header["NAXIS"] == 3: - image = np.sum(self.hdu.data, axis=0) + image = np.sum(self.hdu.data, axis=0, dtype=np.float64) self.hdu.data = image - def get_fields_in_fov(self, fields: Iterable[SourceField]) -> Generator: + def get_fields_in_fov(self, fields: Iterable[SourceField], fov_corners=None) -> Generator: """Return True if Source.field footprint is inside FOV footprint.""" - fov_corners, _ = self.get_corners("arcsec") + if fov_corners is None: + fov_corners, _ = self.get_corners("arcsec") for field in fields: field_corners = field.get_corners("arcsec") @@ -335,35 +340,35 @@ def extract_area_from_imagehdu(self, imagehdu, corners): xyp.sort(axis=0) logger.debug("xyp:\n%s", xyp) - xy0p = np.max(((0, 0), np.floor(xyp[0]).astype(int)), axis=0) - xy1p = np.min(((naxis1, naxis2), np.ceil(xyp[1]).astype(int)), axis=0) - logger.debug("xy0p: %s; xy1p: %s", xy0p, xy1p) - - # Add 1 if the same - xy1p += (xy0p == xy1p) + xy0p = np.max(((0, 0), np.floor(xyp[0]).astype(np.intp)), axis=0) + # xyp are inclusive bounds; python (in)famously excludes upper bound, + # hence xy1p needs to be increased by one + xy1p = np.min(((naxis1, naxis2), + 1 + np.ceil(xyp[1]).astype(np.intp)), + axis=0) logger.debug("xy0p: %s; xy1p: %s", xy0p, xy1p) - new_wcs, new_naxis = imp_utils.create_wcs_from_points( - np.array([xy0s, xy1s]).round(11), pixel_scale=hdr["CDELT1"]) - - # TODO: Come back at some point and figure out if the failing tests - # here are relevant or can be ignored... - # FIXME: Commented out for now because it appears too often IRL... - # try: - # roundtrip = new_wcs.wcs_world2pix( - # np.array([xy0s, xy1s - .5*image_wcs.wcs.cdelt]), 0).round(5) - # np.testing.assert_array_equal(roundtrip[0], [0, 0]) - # np.testing.assert_array_equal(roundtrip[1], new_naxis - [1, 1]) - # except AssertionError: - # logger.exception("WCS roundtrip assertion failed.") - # FIXME: Related to the above, this sometimes fails: - # np.testing.assert_equal(xy1p - xy0p, new_naxis) - # This occurs when the floor and ceil in xy0s and xy1s produce an - # off-by-one error. Using .round instead for both would solve things - # in those cases, but breaks in other cases. Find a proper solution! - # Note: This is not super fatal, because the resulting projections - # will trim off that extra pixel later on, but this should still - # be addressed. + # Describe the cutout with the input WCS, shifted by the slice origin. + # The cutout's pixels ARE input pixels, so this is exact: the cutout + # stays on the input's pixel lattice and NAXISn always agrees with + # data.shape. + # + # Deriving the WCS from [xy0s, xy1s] instead (via + # create_wcs_from_points) got both wrong. Those are the world corners + # *before* the floor/ceil above grew the slice outwards to whole input + # pixels, so the resulting WCS described a differently sized image + # (NAXISn disagreed with data.shape by a pixel, silently corrected by + # fits.ImageHDU, which left CRPIXn describing the wrong array) placed + # on the FOV's lattice rather than the input's. Projecting that cutout + # then mis-registered it by up to a whole pixel, and by differing + # amounts for neighbouring FOVs, so the seam between two FOVs picked + # up a duplicated or a dropped row/column. + new_wcs = image_wcs.deepcopy() + new_wcs.wcs.crpix = image_wcs.wcs.crpix - xy0p + new_naxis = xy1p - xy0p + logger.debug("orig image wcs: %s", image_wcs) + logger.debug("new cutout wcs: %s", new_wcs) + logger.debug("new cutout naxis: %s", new_naxis) new_hdr = new_wcs.to_header() new_hdr.update({"NAXIS1": new_naxis[0], "NAXIS2": new_naxis[1]}) @@ -403,10 +408,14 @@ def _extract_volume_from_cube(self, cubehdu, new_hdr, xy0p, xy1p): hdu_waves = hdu_waves.quantity.to(u.um) wdel = hdr["CDELT3"] * u.Unit(hdr.get("CUNIT3", "AA")) - fov_wmin, fov_wmax = self.waverange + fov_wmin, fov_wmax = self.waverange + np.array([-1, 1])*self.meta['spectral_padding'] # need to go [+/-] half a bin - mask = ((hdu_waves > fov_wmin - 0.5 * wdel) * - (hdu_waves <= fov_wmax + 0.5 * wdel)) + margin = abs(wdel)/2 + if isinstance(self, FieldOfView3D): + # Spectroscopic inputs remain densities until bin integration; + # retain neighbours on both sides of the output bin edges. + margin = abs(wdel) + np.diff(self.cube_waveset[:2])[0]/2 + mask = (hdu_waves > fov_wmin - margin) & (hdu_waves <= fov_wmax + margin) # OC [2021-12-14] if fov range is not covered by the source return nothing if not np.any(mask): @@ -421,12 +430,11 @@ def _extract_volume_from_cube(self, cubehdu, new_hdr, xy0p, xy1p): f0 = (abs(hdu_waves[i0p] - fov_wmin + 0.5 * wdel) % wdel) / wdel # blue edge f1 = (abs(hdu_waves[i1p] - fov_wmax - 0.5 * wdel) % wdel) / wdel # red edge - data = cubehdu.data[i0p:i1p+1, - xy0p[1]:xy1p[1], - xy0p[0]:xy1p[0]] - data[0, :, :] *= f0.to(1).round(11).value # decompose dimensionless - if i1p > i0p: - data[-1, :, :] *= f1.to(1).round(11).value + data = cubehdu.data[i0p:i1p+1, xy0p[1]:xy1p[1], xy0p[0]:xy1p[0]].copy() + if not isinstance(self, FieldOfView3D): + data[0] *= f0.to_value(1).round(11) + if i1p > i0p: + data[-1] *= f1.to_value(1).round(11) # w0, w1 : the closest cube wavelengths outside the fov edge wavelengths # fov_waves : the fov edge wavelengths @@ -491,10 +499,11 @@ def waveset(self): for cube in cube_fields]) _waveset = cube_fields[naxis3_max].waveset elif specfields := self._get_fields(SpectrumSourceField): - _waveset = np.concatenate([ + _waveset = np.concatenate([self.waverange << u.um] + [ spec.waveset.to(u.um) for field in specfields for spec in field.spectra.values() + if spec.waveset is not None ]) else: _waveset = self.waverange << u.um @@ -608,18 +617,17 @@ def _make_scaled_hdu(self, field: ImageSourceField) -> fits.ImageHDU: """Deal with BUNIT and pixel area.""" # Note: Do not scale source data - make a copy first. field_hdu = field.field.copy() # .field is the HDU (yeah...) - field_hdu.data /= self.pixel_area.value - # TODO: Check if this scaling is actually correct. How does this - # work with the add_imagehdu_to_imagehdu below? Isn't that - # supposed to conserve flux? Test carefully!! if field.is_bunit_spatially_differential: - # Field is in (PHOTLAM) arcsec-2, need to scale by pixarea - field_hdu.data *= field.pixel_area.value + logger.debug("differential bunit...") else: + logger.debug("binned bunit...") + logger.debug("scaling by %f", field.pixel_area.value) + field_hdu.data /= field.pixel_area.value # Pixel area doesn't cancel out, need to convert new_bunit = field.bunit / u.arcsec**2 field_hdu.header["BUNIT"] = new_bunit.to_string("fits") + logger.debug("BUNIT after scaling: %s", field_hdu.header["BUNIT"]) return field_hdu @@ -643,14 +651,10 @@ def _make_cubefields(self): * yield scaled flux to be added to canvas flux """ for field in self._get_cube_fields(): - fluxes = field.data.sum(axis=(1, 2)) - fov_waveset_fluxes = np.interp(self.waveset, field.waveset, fluxes) - - # .lower() is needed because astropy doesn't recognize PHOTLAM - field_unit = field.header.get("BUNIT", PHOTLAM).lower() - flux_scale_factor = u.Unit(field_unit).to(PHOTLAM) - - yield fov_waveset_fluxes * flux_scale_factor + fluxes = field.data.sum(axis=(1, 2), dtype=np.float64)*u.Unit(field.header['BUNIT'].lower()) + if field.is_bunit_spatially_differential: + fluxes *= field.pixel_area + yield np.interp(self.waveset, field.waveset, fluxes.to_value(PHOTLAM)) def _make_imagefields(self): """ @@ -661,7 +665,7 @@ def _make_imagefields(self): * yield spectrum multiply by sum to be added to canvas flux """ for field in self._get_image_fields(): - weight = np.sum(field.data) # Shouldn't that be 1 by convention? + weight = np.sum(field.data, dtype=np.float64) # Shouldn't that be 1 by convention? yield field.spectrum(self.waveset).value * weight def _make_tablefields(self): @@ -747,13 +751,16 @@ def _make_cubefields(self): # cube_fields come in with units of photlam/arcsec2, # need to convert to ph/s # We need to the voxel volume (spectral and solid angle) for that. - spectral_bin_width = (field.header["CDELT3"] * - u.Unit(field.header["CUNIT3"]) - ).to(u.Angstrom) + spectral_bin_width = ( + field.header["CDELT3"] * + u.Unit(field.header["CUNIT3"]) + ).to(u.Angstrom) # First collapse to image, then convert units - image = np.sum(field.data, axis=0) * PHOTLAM/u.arcsec**2 - image = (image * self.area * - spectral_bin_width).to(u.ph/u.s/u.arcsec**2) + image = np.sum(field.data, axis=0, dtype=np.float64) * PHOTLAM/u.arcsec**2 + image = (image * self.area * field.pixel_area * + spectral_bin_width).to(u.ph/u.s) + logger.debug("2D FOV make_cubefields: image.mean() = %f %s", + image.mean().value, image.unit) # FIXME: This might create a 2D ImageHDU with a 3D header, not ideal... yield fits.ImageHDU(data=image, header=field.header) @@ -766,7 +773,15 @@ def _make_imagefields(self, fov_waveset, bin_widths): * yield image to be added to canvas image """ for field in self._get_image_fields(): - field_hdu = self._make_scaled_hdu(field) + field_hdu = field.field.copy() + + if field.is_bunit_spatially_differential: + logger.warning("Differential BUNIT in 2D image is discouraged.") + logger.debug("scaling by %f", field.pixel_area.value) + field_hdu.data *= field.pixel_area.value + new_bunit = field.bunit / field.pixel_area.unit + field_hdu.header["BUNIT"] = new_bunit.to_string("fits") + logger.debug("BUNIT after scaling: %s", field_hdu.header["BUNIT"]) # Reevaluate spectrum onto FOV waveset spec = field.spectrum(fov_waveset) # PHOTLAM @@ -775,7 +790,7 @@ def _make_imagefields(self, fov_waveset, bin_widths): flux = (spec * bin_widths * self.area).to_value(u.ph / u.s).sum() # Rescale 2D flux weight map by integrated flux from spectrum - field_hdu.data *= flux # ph s-1 arcsec-2 + field_hdu.data *= flux # ph s-1 yield field_hdu def _make_tablefields(self, fov_waveset, bin_widths, use_photlam=False): @@ -807,8 +822,8 @@ def _make_tablefields(self, fov_waveset, bin_widths, use_photlam=False): else: # Note: these had x/ypix+0.5 until a06ab75 # TODO: could these be something more numpythonic grid-ish? - x = np.array(xpix).astype(int) - y = np.array(ypix).astype(int) # quickest way to round + x = np.floor(xpix + .5).astype(int) + y = np.floor(ypix + .5).astype(int) flux = np.array([fluxes[int(ref)] for ref in field.field["ref"]]) yield flux, np.array(field.field["weight"]), x, y @@ -848,17 +863,16 @@ def make_hdu(self): data=np.zeros((self.header["NAXIS2"], self.header["NAXIS1"])), header=self.header) - for tmp_hdu in chain(self._make_cubefields(), - self._make_imagefields( - fov_waveset, bin_widths)): + for tmp_hdu in chain( + self._make_cubefields(), + self._make_imagefields(fov_waveset, bin_widths) + ): canvas_image_hdu = imp_utils.add_imagehdu_to_imagehdu( tmp_hdu, canvas_image_hdu, - conserve_flux=True, - spline_order=self.spline_order) - - # TODO: Move this further along at some point... - canvas_image_hdu.data *= self.pixel_area.value # eliminate arcsec-2 + conserve_flux=True, # binned flux needs this + spline_order=self.spline_order, + ) for flux, weight, x, y in self._make_tablefields( fov_waveset, bin_widths): @@ -867,14 +881,11 @@ def make_hdu(self): # enabled, it is therefore not necessary to deploy the fix # below in the else-branch. assert not isinstance(x, Iterable), "x must be an integer" - try: + if 0 <= x < canvas_image_hdu.data.shape[1] and 0 <= y < canvas_image_hdu.data.shape[0]: canvas_image_hdu.data[y, x] += flux * weight - except IndexError: - continue else: # Mask out any stars that were pushed out of the fov by rounding - mask = ((x < canvas_image_hdu.data.shape[1]) * - (y < canvas_image_hdu.data.shape[0])) + mask = (x >= 0) & (y >= 0) & (x < canvas_image_hdu.data.shape[1]) & (y < canvas_image_hdu.data.shape[0]) # This used to contain this line: # canvas_image_hdu.data[y[mask], x[mask]] += flux[mask] * weight[mask] @@ -915,12 +926,22 @@ class FieldOfView3D(FieldOfView): """ + @property + def cube_waveset(self): + wave_unit = u.Unit(from_currsys('!SIM.spectral.wave_unit', self.cmds)) + step = from_currsys('!SIM.spectral.spectral_bin_width', self.cmds) + if 'slit_spectral_width' in self.meta: + step = min(step, self.meta['slit_spectral_width'].to_value(wave_unit)/20) + start, stop = self.meta['wave_min'].value, self.meta['wave_max'].value + padding = int(np.ceil(self.meta['spectral_padding'].to_value(wave_unit)/step)) + return (start + np.arange(-padding, int((stop - start)/step) + padding)*step)*wave_unit + def _make_cubefields(self, fov_waveset): """ Find Cube fields. * rescale and reorient cubes - * interp1d smaller cubes with waveset + * integrate native spectral samples into the FOV wavelength bins * yield cubes to be added to cavas cube """ for field in self._get_cube_fields(): @@ -928,18 +949,12 @@ def _make_cubefields(self, fov_waveset): # Assumption is that ImageHDUs have units of PHOTLAM arcsec-2 # TODO: Deal with this bounds_error in a more elegant way - field_interp = interp1d(field.waveset.to(u.um).value, - field.data, axis=0, kind="linear", - bounds_error=False, fill_value=0) - - field_data = field_interp(fov_waveset.value) - - # Pixel scale conversion - field_data *= field.pixel_area / self.pixel_area + field_data = rebin_spectral_density(field.waveset.to_value(u.um), field.data, spectral_bin_edges(fov_waveset.to_value(u.um))) field_hdu = fits.ImageHDU(data=field_data, header=field.header) + field_hdu.header.update(CRPIX3=1., CRVAL3=fov_waveset[0].to_value(u.um), CDELT3=np.diff(fov_waveset[:2])[0].to_value(u.um), CUNIT3='um', CTYPE3='WAVE') yield field_hdu - def _make_imagefields(self, fov_waveset, spline_order): + def _make_imagefields(self, fov_waveset, spline_order, spatial_psf=None): """ Find Image fields. @@ -954,18 +969,19 @@ def _make_imagefields(self, fov_waveset, spline_order): # ImageHDUs have photons/second/pixel. field_hdu = self._make_scaled_hdu(field) - canvas_image_hdu = fits.ImageHDU( - data=zeros_from_header(self.header, ndims=2), - header=self.header) - canvas_image_hdu = imp_utils.add_imagehdu_to_imagehdu( - field_hdu, - canvas_image_hdu, - spline_order=spline_order) - - spec = field.spectrum(fov_waveset) - + if spatial_psf is None: + image = imp_utils.project_spatial_density(field_hdu, self.header) + else: + padded = spatial_psf.padded_header(self) + image = imp_utils.project_spatial_density(field_hdu, padded) + image = fftconvolve(image, spatial_psf.get_kernel(self), mode='same') + px = (padded['NAXIS1'] - self.header['NAXIS1'])//2 + py = (padded['NAXIS2'] - self.header['NAXIS2'])//2 + image = image[py:py + self.header['NAXIS2'], px:px + self.header['NAXIS1']] + + spec = average_spectrum(field.spectrum, fov_waveset) # 2D * 1D -> 3D - field_cube = canvas_image_hdu.data[None, :, :] * spec[:, None, None] + field_cube = image[None, :, :] * spec[:, None, None] yield field_cube.value def _make_tablefields(self, fov_waveset): @@ -986,7 +1002,7 @@ def _make_tablefields(self, fov_waveset): # if this is the case and if yes, remove the condition here!! # Fluxes (yes that's the correct plural) in PHOTLAM fluxes = { - ref: spec(fov_waveset) + ref: average_spectrum(spec, fov_waveset) for ref, spec in field.spectra.items() if ref in field.field["ref"] } @@ -1004,7 +1020,7 @@ def _make_tablefields(self, fov_waveset): for i, j, k in zip(xs, ys, fracs): yield flux_vector * k, i, j else: - yield flux_vector, int(xpix), int(ypix) + yield flux_vector, int(np.floor(xpix + .5)), int(np.floor(ypix + .5)) def _make_backfields(self, fov_waveset): for field in self._get_background_fields(): @@ -1013,84 +1029,50 @@ def _make_backfields(self, fov_waveset): # Remember, cube output needs PHOTLAM arcsec-2 ! # So if SOLIDANG or BUNIT or whatever is not in arcsec-2, # the spectrum shoule be scaled accordingly! - spec = field.spectrum(fov_waveset) + spec = average_spectrum(field.spectrum, fov_waveset) yield spec[:, None, None].value - def make_hdu(self): - """ - TBA. - - Used for IFUs, slit spectrographs, and coherent MOSs (e.g.KMOS) - - Returned cube units are ``ph s-1 voxel-1`` - - .. note:: ``self.make_cube()`` does NOT store anything in ``self.cube`` - - self.cube and self.make_cube() are deliberately kept seperately - so that self.cube will not be accidently overwritten by a rogue - call from an Effect object. - - All Effect objects should specifically test whether - ``self.cube is None`` before assigning a new cube it - - The cube is made with these steps: - - 1. Make waveset and canvas cube:: - - if at least one cube: - set waveset to equal largest cube waveset - else: - make waveset from self.meta values - make canvas cube based on waveset of largest cube and NAXIS1,2 from fov.header - - 2. Find Cube fields (see ``FieldOfView._make_cube_cubefields()``). - 3. Find Image fields (see ``FieldOfView._make_cube_imagefields()``). - 4. Find Table fields (see ``FieldOfView._make_cube_tablefields()``). - - ``PHOTLAM = ph / (cm2 * s * AA)``. - Original source fields are in units of: - - - tables: (PHOTLAM in spectrum) - - images: arcsec-2 (PHOTLAM in spectrum) - - cubes: PHOTLAM arcsec-2 - - .. warning:: Input Images and Cubes should have units of PHOTLAM arcsec-2 - - Returns - ------- - canvas_cube_hdu : fits.ImageHDU - [ph s-1 um-1 arcsec-2] # as needed by SpectralTrace + def make_hdu(self, spatial_psf=None, spatial_shifts=()): + """Render spectral surface-brightness density in ph/s/um/arcsec2. + Native spectra are integrated into the cube wavelength cells. Point + sources use cell-integrated PSF illumination; resolved fields are + convolved before the physical aperture is clipped. Collecting area is + applied here. Slit and detector cell integrals are applied by the trace. """ + if spatial_shifts: + # Keep incoming illumination outside the aperture until every + # atmospheric displacement is applied. The physical slit metadata + # and cell sizes stay unchanged; only temporary spatial support grows. + shifts = [(effect, effect.get_shifts(self)) for effect in spatial_shifts] + padded = copy(self) + padded.header = self.header.copy() + for effect, displacement in shifts: + # This is the final PSF grid. Earlier source selection keeps + # conservative support because its coarser grid can still change. + padded.header = effect._padded_header(padded.header, displacement, rounded=True) + padded.hdu = padded.make_hdu(spatial_psf=spatial_psf) + for effect, displacement in shifts: + effect.apply_to(padded, _shifts=displacement) + px = (padded.header['NAXIS1'] - self.header['NAXIS1'])//2 + py = (padded.header['NAXIS2'] - self.header['NAXIS2'])//2 + header = padded.hdu.header.copy() + header.update(CRPIX1=self.header['CRPIX1'], CRPIX2=self.header['CRPIX2']) + data = padded.hdu.data[:, py:py + self.header['NAXIS2'], px:px + self.header['NAXIS1']].copy() + return fits.ImageHDU(data=data, header=header) + # 1. Make waveset and canvas cube (area, bin_width are applied at end) # TODO: Why is this not self.waveset? What's different? # -> For non-cube input but cube output (e.g. 2D image in spec mode), # waverange needs to be resampled (I guess) to spectral_bin_width, # but self.waverange can only access the fields. # -> Perhaps change that?? - wave_unit = u.Unit(from_currsys("!SIM.spectral.wave_unit", self.cmds)) - fov_waveset = np.arange( - self.meta["wave_min"].value, self.meta["wave_max"].value, - from_currsys("!SIM.spectral.spectral_bin_width", self.cmds)) * wave_unit - - # Note: There is an edge case where some floating point madness - # results in the arange ticking over by another bin by just - # .000000000005 (yeah...), which creates and off-by-one error - # further down the line. - # TODO: Find a better way to solve this, perhaps with linspace... - n_wave = int( - (self.meta["wave_max"].value - self.meta["wave_min"].value) / - from_currsys("!SIM.spectral.spectral_bin_width", self.cmds) - ) - fov_waveset = fov_waveset[:n_wave].to(u.um) - - # make canvas cube based on waveset of largest cube and NAXIS1,2 - # from fov.header - canvas_cube_hdu = fits.ImageHDU( - data=np.zeros((len(fov_waveset), - self.header["NAXIS2"], - self.header["NAXIS1"])), - header=self.header) + fov_waveset = self.cube_waveset.to(u.um) + + # Retain double precision when combining faint sources with sky and + # thermal backgrounds; subtraction can expose storage-rounding error. + shape = (len(fov_waveset), self.header['NAXIS2'], self.header['NAXIS1']) + canvas_cube_hdu = fits.ImageHDU(data=np.zeros(shape, dtype=np.float64), header=self.header) # set BUNIT initially to PHOTLAM / arcsec**2 canvas_cube_hdu.header["BUNIT"] = "ph cm-2 s-1 AA-1 arcsec-2" @@ -1113,28 +1095,47 @@ def make_hdu(self): assert canvas_cube_hdu.header["NAXIS3"] == self.detector_header["NAXIS3"] for field_hdu in self._make_cubefields(fov_waveset): - canvas_cube_hdu = imp_utils.add_imagehdu_to_imagehdu( - field_hdu, - canvas_cube_hdu, - spline_order=self.spline_order) - - canvas_cube_hdu.data = sum(self._make_imagefields( - fov_waveset, self.spline_order), - start=canvas_cube_hdu.data) - - for flux, x, y in self._make_tablefields(fov_waveset): - # To prevent adding array values in this manner. - assert not isinstance(x, Iterable), "x should be integer" - canvas_cube_hdu.data[:, y, x] += flux + if spatial_psf is None: + canvas_cube_hdu.data += imp_utils.project_spatial_density(field_hdu, self.header) + else: + padded = spatial_psf.padded_header(self) + kernel = spatial_psf.get_kernel(self)[None, :, :] + px = (padded['NAXIS1'] - self.header['NAXIS1'])//2 + py = (padded['NAXIS2'] - self.header['NAXIS2'])//2 + # Bound the temporary padded cube, even for long input spectra. + for start in range(0, len(fov_waveset), 64): + chunk = fits.ImageHDU(data=field_hdu.data[start:start + 64], header=field_hdu.header) + data = imp_utils.project_spatial_density(chunk, padded) + data = fftconvolve(data, kernel, mode='same', axes=(-2, -1)) + canvas_cube_hdu.data[start:start + 64] += data[:, py:py + self.header['NAXIS2'], px:px + self.header['NAXIS1']] + + for image in self._make_imagefields(fov_waveset, self.spline_order, spatial_psf): + canvas_cube_hdu.data += image + + if spatial_psf is None: + for flux, x, y in self._make_tablefields(fov_waveset): + if 0 <= x < self.header['NAXIS1'] and 0 <= y < self.header['NAXIS2']: + canvas_cube_hdu.data[:, y, x] += flux + else: + for field in self._get_table_fields(): + spectra = {ref: average_spectrum(spec, fov_waveset).value for ref, spec in field.spectra.items()} + for row in field.field: + image = spatial_psf.point_source_image(self, row['x'], row['y'])/self.pixel_area.value + for start in range(0, len(fov_waveset), 64): + spectrum = spectra[row['ref']][start:start + 64] + canvas_cube_hdu.data[start:start + 64] += spectrum[:, None, None]*image[None, :, :]*row['weight'] - canvas_cube_hdu.data = sum(self._make_backfields(fov_waveset), - start=canvas_cube_hdu.data) + for background in self._make_backfields(fov_waveset): + canvas_cube_hdu.data += background # SpectralTrace wants ph/s/um/arcsec2 --> get rid of m2, leave um canvas_cube_hdu.data *= self.area.to(u.cm ** 2).value canvas_cube_hdu.data *= 1e4 # ph/s/AA/arcsec2 --> ph/s/um/arcsec2 canvas_cube_hdu.header["BUNIT"] = "ph s-1 um-1 arcsec-2" + logger.debug("3D FOV make_hdu: canvas_cube_hdu.data.mean() = %f", + canvas_cube_hdu.data.mean()) + return canvas_cube_hdu # [ph s-1 um-1 (arcsec-2)] def plot_data(self): @@ -1164,12 +1165,14 @@ def replace_nans(field, cmds) -> None: "set in '!SIM.computing.nan_fill_value'.") -def extract_range_from_spectrum(spectrum, waverange): +def extract_range_from_spectrum(spectrum, waverange, _waveset=None): assert isinstance(spectrum, SourceSpectrum), ( f"spectrum must be of type synphot.SourceSpectrum: {type(spectrum)}") wave_min, wave_max = quantify(waverange, u.um).to(u.AA).value - spec_waveset = _spectrum_waveset_aa_value(spectrum) + spec_waveset = _spectrum_waveset_aa_value(spectrum) if _waveset is None else _waveset + if spec_waveset is None: + return spectrum mask = (spec_waveset > wave_min) & (spec_waveset < wave_max) # FIXME: Why did I comment this out in 2023? Seems useful to have... @@ -1192,9 +1195,40 @@ def extract_range_from_spectrum(spectrum, waverange): def _spectrum_waveset_aa_value(spectrum): - """Return cached spectrum waveset values in Angstrom.""" - spec_waveset = _SPECTRUM_WAVESET_AA_CACHE.get(spectrum) - if spec_waveset is None: - spec_waveset = spectrum.waveset.to(u.AA).value - _SPECTRUM_WAVESET_AA_CACHE[spectrum] = spec_waveset - return spec_waveset + # A spectrum can change its sampling in place, including through redshift. + waveset = spectrum.waveset + return None if waveset is None else waveset.to_value(u.AA) + + +def spectral_bin_edges(centers): + """Edges of bins whose centers are the supplied wavelength samples.""" + half_steps = np.diff(centers) / 2 + return np.r_[centers[0] - half_steps[0], centers[:-1] + half_steps, centers[-1] + half_steps[-1]] + + +def rebin_spectral_density(wave, density, edges): + """Integrate a piecewise-linear density, with zero flux beyond its support. + + Wavelength is the first axis; density may also carry spatial dimensions. + The result remains a density, averaged over each requested bin. + """ + if wave[1] < wave[0]: + wave, density = wave[::-1], density[::-1] + integral = cumulative_trapezoid(density, wave, axis=0, initial=0) + clipped = np.clip(edges, wave[0], wave[-1]) + index = np.clip(np.searchsorted(wave, clipped, side='right') - 1, 0, len(wave) - 2) + dimensions = (len(edges),) + (1,)*(density.ndim - 1) + offset = (clipped - wave[index]).reshape(dimensions) + step = (wave[index + 1] - wave[index]).reshape(dimensions) + at_edges = integral[index] + density[index]*offset + (density[index + 1] - density[index])*offset**2/(2*step) + widths = np.diff(edges).reshape((len(edges) - 1,) + (1,)*(density.ndim - 1)) + return np.diff(at_edges, axis=0) / widths + + +def average_spectrum(spectrum, wavelengths): + """Average PHOTLAM over numerical bins without discarding native line samples.""" + edges = spectral_bin_edges(wavelengths.to_value(u.AA)) + native = _spectrum_waveset_aa_value(spectrum) + wave = edges if native is None else np.union1d(edges, native[(native > edges[0]) & (native < edges[-1])]) + flux = spectrum(wave*u.AA).to_value(PHOTLAM) + return rebin_spectral_density(wave, flux, edges) * PHOTLAM diff --git a/scopesim/optics/fov_manager.py b/scopesim/optics/fov_manager.py index 29f7ef888..261f48f86 100644 --- a/scopesim/optics/fov_manager.py +++ b/scopesim/optics/fov_manager.py @@ -61,6 +61,32 @@ logger = get_logger(__name__) +def chunk_edges(vmin, vmax, step): + """Return the interior points that cut ``[vmin, vmax]`` into `step` chunks. + + Only interior points are returned, so no edge can coincide with `vmin` or + `vmax`. `step` need not divide the span, and need not be an integer (see + HAWKI/test_hawki/test_full_package_hawki.py), so this cannot be a plain + ``range``. + + It cannot be ``np.arange(vmin, vmax, step)`` either. That returns + ``ceil((vmax - vmin) / step)`` elements, which is one too many when the + division rounds just above an integer - as it does for a span that is an + exact multiple of `step`, e.g. ``(1.2000000000000002 - -1.2000000000000002) + / 0.8 == 3.0000000000000004``. Whether the surplus element then lands + exactly on `vmax` or one ulp below it depends on how the platform rounds + ``vmin + i * step``: it lands on `vmax` on x86, but below it on arm64 + macOS. One ulp below `vmax` is a perfectly legal split point, so + ``FovVolumeList.split`` accepts it and carves off a volume ~1e-15 px wide, + which ``create_wcs_from_points`` then rounds up to a whole pixel - a + spurious one-pixel FOV, off the pixel lattice, double-counting flux at the + edge of the array. + + """ + n_chunks = max(1, int(np.ceil((vmax - vmin) / step - 1e-9))) + return vmin + step * np.arange(1, n_chunks) + + class FOVManager: """ A class to manage the (monochromatic) image windows covering the target. @@ -122,11 +148,8 @@ def _get_splits(self, pixel_scale): if vol_pix_area > max_seg_size: step = chunk_size * pixel_scale - # These are not always integers, unlike in the tests. - # See for example HAWKI/test_hawki/test_full_package_hawki.py. - # The np.arange can therefore not be changed to just a range. - yield (np.arange(vol["x_min"], vol["x_max"], step), - np.arange(vol["y_min"], vol["y_max"], step)) + yield (chunk_edges(vol["x_min"], vol["x_max"], step), + chunk_edges(vol["y_min"], vol["y_max"], step)) def generate_fovs_list(self) -> Iterator[FieldOfView]: """ @@ -152,7 +175,7 @@ def generate_fovs_list(self) -> Iterator[FieldOfView]: self.volumes_list = effect.apply_to( self.volumes_list, **effect_params) - # ..todo: add catch to split volumes larger than chunk_size + # TODO: add catch to split volumes larger than chunk_size pixel_scale = from_currsys(self.meta["pixel_scale"], self.cmds) splits = (chain.from_iterable(split) @@ -166,7 +189,10 @@ def generate_fovs_list(self) -> Iterator[FieldOfView]: for vol in self.volumes_list: image_plane_id = vol["meta"].get("image_plane_id") det_eff = None - if image_plane_id is not None: + if image_plane_id is None and len(detector_effects) == 1: + # Single-plane fibre/imaging packages need no arm-routing key. + det_eff, = detector_effects + elif image_plane_id is not None: for candidate in detector_effects: if (from_currsys(candidate.meta["image_plane_id"], self.cmds) == image_plane_id): @@ -178,29 +204,40 @@ def generate_fovs_list(self) -> Iterator[FieldOfView]: "plate_scale": self.meta["plate_scale"], } if det_eff is not None: - scale_meta["pixel_scale"] = det_eff.meta.get( - "pixel_scale", scale_meta["pixel_scale"]) - scale_meta["plate_scale"] = det_eff.meta.get( - "plate_scale", scale_meta["plate_scale"]) + for key in ('pixel_scale', 'plate_scale'): + if det_eff.meta[key] != f'!INST.{key}': + scale_meta[key] = det_eff.meta[key] scales = from_currsys(scale_meta, self.cmds) pixel_scale = scales["pixel_scale"] plate_scale = scales["plate_scale"] xs_min, xs_max = vol["x_min"] / 3600., vol["x_max"] / 3600. ys_min, ys_max = vol["y_min"] / 3600., vol["y_max"] / 3600. - waverange = (vol["wave_min"], vol["wave_max"]) skyhdr = ipu.header_from_list_of_xy([xs_min, xs_max], [ys_min, ys_max], pixel_scale=pixel_scale / 3600.) + if self.is_spectroscope and "eta_min" in vol["meta"]: + # Tile the physical slit exactly. Detector sampling sets the + # approximate integration step, never the admitted slit area. + for axis, lower, upper in ((1, xs_min, xs_max), (2, ys_min, ys_max)): + count = max(1, int(np.ceil((upper - lower)*3600/pixel_scale))) + skyhdr[f'NAXIS{axis}'] = count + skyhdr[f'CDELT{axis}'] = (upper - lower)/count + skyhdr[f'CRPIX{axis}'] = (count + 1)/2 + skyhdr[f'CRVAL{axis}'] = (lower + upper)/2 + fov_dethdr, _ = ipu.det_wcs_from_sky_wcs( WCS(skyhdr), pixel_scale, plate_scale) skyhdr.update(fov_dethdr.to_header()) - dethdr = fov_dethdr + dethdr = fov_dethdr.to_header() + dethdr.update(NAXIS=2, NAXIS1=skyhdr['NAXIS1'], NAXIS2=skyhdr['NAXIS2']) # useful for spectroscopy mode where slit dimensions is not the same # as detector dimensions - if decouple and det_eff is not None: + if decouple: + if det_eff is None: + raise ValueError('A decoupled FOV needs an image-plane ID or one unambiguous detector layout.') dethdr = det_eff.image_plane_header # TODO: Why is this .image_plane_header and not # .detector_headers()[0] or something? @@ -219,7 +256,7 @@ def generate_fovs_list(self) -> Iterator[FieldOfView]: new_fov = fovcls( skyhdr, - waverange, + (vol["wave_min"], vol["wave_max"]), detector_header=dethdr, cmds=self.cmds, **fov_meta, diff --git a/scopesim/optics/image_plane.py b/scopesim/optics/image_plane.py index e2b101f48..f65c86f73 100644 --- a/scopesim/optics/image_plane.py +++ b/scopesim/optics/image_plane.py @@ -1,15 +1,11 @@ # -*- coding: utf-8 -*- """Contains ``ImagePlane`` class.""" - -from warnings import warn - import numpy as np from astropy.io import fits -from astropy.table import Table from astropy.wcs import WCS -from .image_plane_utils import add_table_to_imagehdu, add_imagehdu_to_imagehdu +from .image_plane_utils import add_imagehdu_to_imagehdu from ..utils import ( from_currsys, @@ -68,7 +64,7 @@ def __init__(self, header, cmds=None, **kwargs): raise ValueError(f"header must have a valid image-plane WCS: " f"{dict(header)}") - image = zeros_from_header(header) + image = zeros_from_header(header, dtype=np.float64) self.hdu = fits.ImageHDU(data=image, header=header) self.hdu.header["BUNIT"] = "ph s-1" # photons per second (per pixel) diff --git a/scopesim/optics/image_plane_utils.py b/scopesim/optics/image_plane_utils.py index 5781366da..d64c995a7 100644 --- a/scopesim/optics/image_plane_utils.py +++ b/scopesim/optics/image_plane_utils.py @@ -1,159 +1,20 @@ # -*- coding: utf-8 -*- -from itertools import product -from collections.abc import Iterable - import numpy as np from numpy.typing import ArrayLike, NDArray from astropy import units as u from astropy.wcs import WCS, find_all_wcs from astropy.io import fits -from astropy.table import Table from astropy.utils.exceptions import AstropyWarning from scipy import ndimage as ndi +from scipy.sparse import csr_array -from ..utils import (unit_from_table, quantity_from_table, has_needed_keywords, - get_logger) +from ..utils import get_logger logger = get_logger(__name__) -def get_canvas_header(hdu_or_table_list, pixel_scale=1 * u.arcsec): - """ - Generate a fits.Header with a WCS that covers everything in the FOV. - - Parameters - ---------- - hdu_or_table_list : list - A list of Tables and/or ImageHDU py_objects - - pixel_scale : astropy.Quantity - [arcsec] The pixel scale of the projection. Default in 1 arcsec - - Returns - ------- - header : fits.Header - A Header containing a WCS and NAXISn values to build an ImageHDU - - """ - size_warning = ("Header dimension are {adverb} large: {num_pix}. " - "Any image made from this header will use more that " - ">{size} in memory") - - def _get_headers(hdus_or_tables): - tables = [] - for hdu_or_table in hdus_or_tables: - if isinstance(hdu_or_table, fits.ImageHDU): - yield hdu_or_table.header - elif isinstance(hdu_or_table, fits.Header): - yield hdu_or_table - elif isinstance(hdu_or_table, Table): - tables.append(hdu_or_table) - else: - raise TypeError( - "hdu_or_table_list may only contain fits.ImageHDU, Table " - f"or fits.Header, found {type(hdu_or_table)}.") - if tables: - yield _make_bounding_header_for_tables(*tables, - pixel_scale=pixel_scale) - - headers = list(_get_headers(hdu_or_table_list)) - - if not headers: - logger.warning("No tables or ImageHDUs were passed") - return None - - hdr = _make_bounding_header_from_headers(*headers, pixel_scale=pixel_scale) - - num_pix = hdr["NAXIS1"] * hdr["NAXIS2"] - if num_pix > 2 ** 28: - raise MemoryError(size_warning.format(adverb="too", num_pix=num_pix, - size="8 GB")) - if num_pix > 2 ** 25: # 2 * 4096**2 - logger.warning(size_warning.format(adverb="", num_pix=num_pix, - size="256 MB")) - return hdr - - -def _make_bounding_header_from_headers(*headers, pixel_scale=1*u.arcsec): - """ - Return a Header with WCS and NAXISn keywords bounding all input ImageHDUs. - - Parameters - ---------- - headers : list of fits.ImageHDU - pixel_scale : u.Quantity - [arcsec] - - Returns - ------- - hdr : fits.Header - - """ - wcs_suffix = "D" if pixel_scale.unit.physical_type == "length" else "" - unit = u.Unit(_get_unit_from_headers(*headers, wcs_suffix=wcs_suffix)) - - if unit.physical_type == "angle": - unit = "deg" - pixel_scale = pixel_scale.to_value(u.deg) - else: - pixel_scale = pixel_scale.to_value(unit) - - extents = [calc_footprint(header, wcs_suffix, unit) for header in headers] - pnts = np.vstack(extents) - - hdr = header_from_list_of_xy(pnts[:, 0], pnts[:, 1], - pixel_scale, wcs_suffix) - hdr["NAXIS1"] += 1 - hdr["NAXIS2"] += 1 - hdr[f"CRVAL1{wcs_suffix}"] -= 0.5 * pixel_scale - hdr[f"CRVAL2{wcs_suffix}"] -= 0.5 * pixel_scale - - return hdr - - -def _make_bounding_header_for_tables(*tables, pixel_scale=1*u.arcsec): - """ - Return a Header with WCS and NAXISn keywords bounding all input Tables. - - Parameters - ---------- - tables : list of astropy.Tables - [arcsec] Must contain columns: "x", "y" - - pixel_scale : u.Quantity - [arcsec] - - Returns - ------- - hdr : fits.Header - - """ - wcs_suffix = "D" if pixel_scale.unit.physical_type == "length" else "" - # Note: this could all be a lot simpler if we have consistent units, i.e. - # don't need to convert mm -> mm and arcsec -> arcsec (or deg) - new_unit = u.mm if wcs_suffix == "D" else u.deg # arcsec here causes problems later when all header units are expected to be same - tbl_unit = u.mm if wcs_suffix == "D" else u.arcsec - x_name = "x_mm" if wcs_suffix == "D" else "x" - y_name = "y_mm" if wcs_suffix == "D" else "y" - - pixel_scale = pixel_scale.to(new_unit) - - extents = [] - for table in tables: - extent = calc_table_footprint(table, x_name, y_name, - tbl_unit, new_unit, - padding=pixel_scale) - extents.append(extent) - pnts = np.vstack(extents) - - # TODO: check if this could just use create_wcs_from_points - hdr = header_from_list_of_xy(pnts[:, 0], pnts[:, 1], pixel_scale.value, - wcs_suffix)#, arcsec=wcs_suffix != "D") - return hdr - - def create_wcs_from_points( points: ArrayLike, pixel_scale: float, @@ -213,7 +74,41 @@ def create_wcs_from_points( offset[zeroaxis] = 0 crpix = (naxis + 1) / 2 - crval = (points.min(axis=0) + points.max(axis=0) + offset) / 2 + + # Where to put the pixel lattice when NAXISn cannot cover the requested + # region exactly. + # + # If the grid truncates the region (naxis < extent, i.e. the region is not + # a whole number of pixels across), anchor the lower pixel edge at + # points.min and let the leftover fraction fall off the far edge. Every + # region carved out of the same origin in whole-pixel steps then shares one + # lattice - which is what chunked FOVs are: the image plane, the full-size + # chunks and the smaller last chunk all measure from the same detector + # edge. Centring each of them on its own midpoint instead, and absorbing + # its own rounding remainder into CRVAL, put them on lattices offset from + # one another by fractions of a pixel, so the projection had to snap them + # and the last chunk could land a whole pixel off its neighbours. + # + # MICADO's detector plane is 189.32 mm / 0.015 mm = 12621.33 px tall, so it + # hits this on every run. + # + # If the grid covers the region (naxis >= extent) there is nothing to + # truncate, so keep centring it on the region. That is the right thing for + # a region smaller than a single pixel, where naxis was clamped up to 1. + _pxs = float(pixel_scale.value if isinstance(pixel_scale, u.Quantity) + else pixel_scale) + _pts = points.value if isinstance(points, u.Quantity) else np.asarray(points) + _naxis = np.asarray(naxis, dtype=float) + _extent = np.atleast_1d(np.asarray(extent, dtype=float)) + + crval = np.where( + _naxis < _extent, + _pts.min(axis=0) + (_naxis / 2) * _pxs, # anchored + (_pts.min(axis=0) + _pts.max(axis=0) + + np.asarray(offset, dtype=float)) / 2, # centred + ) + if isinstance(points, u.Quantity): + crval = crval * points.unit # Cannot do `in "DX"` here because that would also match the empty string. linsuff = {"D", "X"} @@ -278,95 +173,6 @@ def header_from_list_of_xy(x, y, pixel_scale, wcs_suffix="", arcsec=False): return hdr -def add_table_to_imagehdu(table: Table, canvas_hdu: fits.ImageHDU, - sub_pixel: bool = True, - wcs_suffix: str = "") -> fits.ImageHDU: - """ - Add files from an astropy.Table to the image of an fits.ImageHDU. - - Parameters - ---------- - table : astropy.Table - Must contain the columns "x_mm", "y_mm", "flux" with the units in the - column attribute .unit, or in the table.meta dictionary as - "_unit". Default units are ``mm`` and ``ph / s / pix`` - - canvas_hdu : fits.ImageHDU - The ImageHDU onto which the table files should be projected. - This must include a valid WCS - - sub_pixel : bool, optional - Default is True. If True, sub-pixel shifts of files will be taken into - account when projecting onto the canvas pixel grid. This takes about 5x - longer than ignoring the sub-pixel shifts - - wcs_suffix : str, optional - - Returns - ------- - canvas_hdu : fits.ImageHDU - - """ - s = wcs_suffix - if not has_needed_keywords(canvas_hdu.header, s): - raise ValueError(f"canvas_hdu must include an appropriate WCS: {s}") - - f = quantity_from_table("flux", table, default_unit=u.Unit("ph s-1")) - if s == "D": - x = quantity_from_table("x_mm", table, default_unit=u.mm).to(u.mm) - y = quantity_from_table("y_mm", table, default_unit=u.mm).to(u.mm) - else: - arcsec = u.arcsec - canvas_unit = u.Unit(canvas_hdu.header[f"CUNIT1{s}"]) - x = quantity_from_table("x", table, default_unit=arcsec.to(canvas_unit)) - y = quantity_from_table("y", table, default_unit=arcsec.to(canvas_unit)) - x *= arcsec.to(canvas_unit) - y *= arcsec.to(canvas_unit) - - xpix, ypix = val2pix(canvas_hdu.header, x.value, y.value, s) - - naxis1 = canvas_hdu.header["NAXIS1"] - naxis2 = canvas_hdu.header["NAXIS2"] - # Occasionally 0 is returned as ~ -1e-11 - eps = -1e-7 - mask = (xpix >= eps) * (xpix < naxis1) * (ypix >= eps) * (ypix < naxis2) - - if sub_pixel: - canvas_hdu = _add_subpixel_sources_to_canvas( - canvas_hdu, xpix, ypix, f, mask) - else: - canvas_hdu = _add_intpixel_sources_to_canvas( - canvas_hdu, xpix, ypix, f, mask) - - return canvas_hdu - - -def _add_intpixel_sources_to_canvas(canvas_hdu, xpix, ypix, flux, mask): - canvas_hdu.header["comment"] = f"Adding {len(flux)} int-pixel files" - for xpx, ypx, flx, msk in zip(xpix.astype(int), ypix.astype(int), - flux, mask): - # To prevent adding array values in this manner. - assert not isinstance(xpx, Iterable), "xpx should be integer" - canvas_hdu.data[ypx, xpx] += flx.value * msk - - return canvas_hdu - - -def _add_subpixel_sources_to_canvas(canvas_hdu, xpix, ypix, flux, mask): - canvas_hdu.header["comment"] = f"Adding {len(flux)} sub-pixel files" - canvas_shape = canvas_hdu.data.shape - for xpx, ypx, flx, msk in zip(xpix, ypix, flux, mask): - if msk: - xx, yy, fracs = sub_pixel_fractions(xpx, ypx) - for x, y, frac in zip(xx, yy, fracs): - if y < canvas_shape[0] and x < canvas_shape[1]: - # To prevent adding array values in this manner. - assert not isinstance(x, Iterable), "x should be integer" - canvas_hdu.data[y, x] += frac * flx.value - - return canvas_hdu - - def sub_pixel_fractions(x, y): """ Make a list of pixel coordinates and weights to reflect sub-pixel shifts. @@ -436,13 +242,32 @@ def overlay_image(small_im, big_im, coords, mask=None, sub_pixel=False): coords = np.array([*coords, (big_im.shape[0] - 1) / 2]) # FIXME: this would not be necessary if we used WCS instead of manual 2pix - # Round to 1e-4 pix before ceil: WCS deg-space round-trips leave - # float dust of order 1e-8..1e-6 pix on integer-valued coords, which - # ceil would amplify to a full-pixel shift; genuine sub-pixel intent - # cannot be finer than 0.5 pix here (sub_pixel is not implemented), - # and the half-integer even-shape convention is preserved exactly. - coords = np.ceil(np.asarray(coords).round(4)).astype(np.intp) - idx = coords.astype(int)[::-1] - np.array(small_im.shape) // 2 + # Round to 1e-4 pix first: WCS deg-space round-trips leave float dust of + # order 1e-8..1e-6 pix on integer-valued coords, which the snap below + # would otherwise amplify to a full-pixel shift; genuine sub-pixel intent + # cannot be finer than 0.5 pix here (sub_pixel is not implemented). + coords = np.asarray(coords, dtype=float).round(4) + + # `idx` is the origin, in array order, at which small_im must be placed to + # centre it on `coords` (both are 0-based pixel-centre coordinates, as + # returned by WCS.wcs_world2pix(..., 0)). + # + # Snap that origin, NOT `coords` itself. The old expression + # np.ceil(coords)[::-1] - np.array(small_im.shape) // 2 + # is off by up to a whole pixel whenever `coords` is not on the lattice + # small_im implies (half-integer for even shapes, integer for odd ones), + # and the direction of that error flips with the parity of the shape. Two + # adjacent FOVs of different sizes therefore get displaced opposite ways + # and their shared edge picks up a duplicated or a dropped row/column. + # Snapping the origin instead keeps every placement consistent, whatever + # the shape. + # + # Half-integer origins (a real sub-pixel shift, which sub_pixel=True would + # have to handle properly) are rounded towards the lower index, matching + # what the old expression did for even shapes. + idx = np.ceil( + coords[::-1] - (np.array(small_im.shape) - 1) / 2 - 0.5 + ).astype(np.intp) # Image ranges idx1 = np.maximum(0, idx) @@ -482,9 +307,14 @@ def overlay_image(small_im, big_im, coords, mask=None, sub_pixel=False): return big_im -def rescale_imagehdu(imagehdu: fits.ImageHDU, pixel_scale: float | u.Quantity, - wcs_suffix: str = "", conserve_flux: bool = True, - spline_order: int = 1) -> fits.ImageHDU: +def rescale_imagehdu( + imagehdu: fits.ImageHDU, + pixel_scale: float | u.Quantity, + wcs_suffix: str = "", + conserve_flux: bool = True, + spline_order: int = 1, + differential: bool = False, +) -> fits.ImageHDU: """ Scale the .data array by the ratio of pixel_scale [deg] and CDELTn. @@ -540,6 +370,10 @@ def rescale_imagehdu(imagehdu: fits.ImageHDU, pixel_scale: float | u.Quantity, sum_orig = np.sum(imagehdu.data) + # scale by pixel area if the flux is differential (i.e. per unit area) + if differential: + sum_orig *= primary_wcs.wcs.cdelt[0] * primary_wcs.wcs.cdelt[1] + # Perform the rescaling. Axes need to be inverted because python. zoom_np = zoom[::-1] data = imagehdu.data @@ -591,8 +425,13 @@ def rescale_imagehdu(imagehdu: fits.ImageHDU, pixel_scale: float | u.Quantity, if conserve_flux: new_im = np.nan_to_num(new_im, copy=False) sum_new = np.sum(new_im) + if differential: + sum_new *= (pixel_scale.value**2) + if sum_new != 0: - new_im *= sum_orig / sum_new + flux_factor = sum_orig / sum_new + logger.debug("flux factor = %f", flux_factor) + new_im *= flux_factor elif sum_orig != 0: logger.warning( "rescale_imagehdu: all input flux (%g) was lost in resampling; " @@ -645,9 +484,12 @@ def rescale_imagehdu(imagehdu: fits.ImageHDU, pixel_scale: float | u.Quantity, return imagehdu -def reorient_imagehdu(imagehdu: fits.ImageHDU, wcs_suffix: str = "", - conserve_flux: bool = True, - spline_order: int = 1) -> fits.ImageHDU: +def reorient_imagehdu( + imagehdu: fits.ImageHDU, + wcs_suffix: str = "", + conserve_flux: bool = True, + spline_order: int = 1, +) -> fits.ImageHDU: """ Apply an affine transformation to the image, as given in its header. @@ -705,9 +547,15 @@ def reorient_imagehdu(imagehdu: fits.ImageHDU, wcs_suffix: str = "", return imagehdu -def affine_map(input, matrix=None, rotation_angle: float = 0., - shear_angle: float = 0., scale_factor=None, - reshape: bool = True, spline_order: int = 3): +def affine_map( + input, + matrix=None, + rotation_angle: float = 0., + shear_angle: float = 0., + scale_factor=None, + reshape: bool = True, + spline_order: int = 3, +): """ Apply an affine transformation matrix to an image around its centre. @@ -778,11 +626,46 @@ def affine_map(input, matrix=None, rotation_angle: float = 0., return output -def add_imagehdu_to_imagehdu(image_hdu: fits.ImageHDU, - canvas_hdu: fits.ImageHDU, - spline_order: int = 1, - wcs_suffix: str = "", - conserve_flux: bool = True) -> fits.ImageHDU: +def project_spatial_density(image_hdu, canvas_header): + """Average a surface brightness image/cube over destination pixel areas. + + Overlaps are exact for aligned rectangular pixels, including signed and + unequal scales. Rotated inputs retain the existing affine interpolation + before this area integration. No global flux renormalization is applied + after clipping to the destination footprint. + """ + source = reorient_imagehdu(image_hdu.copy()) + source_wcs = WCS(source.header, naxis=2) + canvas_wcs = WCS(canvas_header, naxis=2) + nx, ny = source.data.shape[-1], source.data.shape[-2] + x = np.arange(canvas_header['NAXIS1'] + 1) - .5 + y = np.arange(canvas_header['NAXIS2'] + 1) - .5 + xedges, _ = source_wcs.all_world2pix(*canvas_wcs.all_pix2world(x, np.zeros_like(x), 0), 0) + _, yedges = source_wcs.all_world2pix(*canvas_wcs.all_pix2world(np.zeros_like(y), y, 0), 0) + + weights = [] + for count, edges in ((nx, xedges), (ny, yedges)): + lower = np.minimum(edges[:-1], edges[1:])[:, None] + upper = np.maximum(edges[:-1], edges[1:])[:, None] + input_lower = np.arange(count)[None, :] - .5 + overlap = np.maximum(0, np.minimum(upper, input_lower + 1) - np.maximum(lower, input_lower)) + weights.append(csr_array(overlap/(upper - lower))) + wx, wy = weights + planes = source.data.reshape(-1, ny, nx) + rows = wy @ planes.transpose(1, 0, 2).reshape(ny, -1) + rows = rows.reshape(wy.shape[0], len(planes), nx).transpose(1, 0, 2) + mapped = (wx @ rows.reshape(-1, nx).T).T + return mapped.reshape(source.data.shape[:-2] + (wy.shape[0], wx.shape[0])) + + +def add_imagehdu_to_imagehdu( + image_hdu: fits.ImageHDU, + canvas_hdu: fits.ImageHDU, + spline_order: int = 1, + wcs_suffix: str = "", + conserve_flux: bool = True, + differential: bool = False, +) -> fits.ImageHDU: """ Re-project one ``fits.ImageHDU`` onto another ``fits.ImageHDU``. @@ -832,16 +715,22 @@ def add_imagehdu_to_imagehdu(image_hdu: fits.ImageHDU, canvas_pixel_scale = float(canvas_wcs.wcs.cdelt[0]) conv_fac = u.Unit(image_hdu.header[f"CUNIT1{wcs_suffix}"].lower()).to(canvas_wcs.wcs.cunit[0]) - new_hdu = rescale_imagehdu(image_hdu, pixel_scale=canvas_pixel_scale / conv_fac, - wcs_suffix=canvas_wcs.wcs.alt, - spline_order=spline_order, - conserve_flux=conserve_flux) + new_hdu = rescale_imagehdu( + image_hdu, + pixel_scale=canvas_pixel_scale / conv_fac, + wcs_suffix=canvas_wcs.wcs.alt, + spline_order=spline_order, + conserve_flux=conserve_flux, + differential=differential, + ) # TODO: Perhaps add separately formatted WCS logger? - # logger.debug("fromrescale %s", WCS(new_hdu.header, key=canvas_wcs.wcs.alt)) - new_hdu = reorient_imagehdu(new_hdu, - wcs_suffix=canvas_wcs.wcs.alt, - spline_order=spline_order, - conserve_flux=conserve_flux) + logger.debug("fromrescale %s", WCS(new_hdu.header, key=canvas_wcs.wcs.alt)) + new_hdu = reorient_imagehdu( + new_hdu, + wcs_suffix=canvas_wcs.wcs.alt, + spline_order=spline_order, + conserve_flux=conserve_flux, + ) img_center = np.array([[new_hdu.header[f"NAXIS{i+1}"] for i in range(new_hdu.header["NAXIS"])]]) @@ -869,6 +758,7 @@ def add_imagehdu_to_imagehdu(image_hdu: fits.ImageHDU, return canvas_hdu +# TODO: Use proper WCS functions instead and rm this on-foot implementation! def pix2val(header, x, y, wcs_suffix=""): """ Return the real coordinates [deg, mm] for coordinates from a Header WCS. @@ -909,6 +799,7 @@ def pix2val(header, x, y, wcs_suffix=""): return a, b +# TODO: Use proper WCS functions instead and rm this on-foot implementation! def val2pix(header, a, b, wcs_suffix=""): """ Return the pixel coordinates for real coordinates [deg, mm] from a WCS. @@ -1019,103 +910,6 @@ def calc_footprint(header, wcs_suffix="", new_unit: str = None): return xy1 -def calc_table_footprint( - table: Table, - x_name: str, - y_name: str, - tbl_unit: str, - new_unit: str, - padding: u.Quantity | None = None, -) -> NDArray: - """ - Equivalent to ``calc_footprint()``, but for tables instead of images. - - Parameters - ---------- - table : astropy.table.Table - Table containing data. - x_name : str - Name of the column in `table` to use as x-coordinates. - y_name : str - Name of the column in `table` to use as y-coordinates. - tbl_unit : str - Default unit to use for x and y if no units are found in `table`. - new_unit : str - Unit to convert x and y to, can be identical to `tbl_unit`. - padding : astropy.units.Quantity, optional - Constant value to subtract from minima and add to maxima. If used, must - be Quantity with same physical type as x and y. If None (default), no - padding is added. - - Returns - ------- - extent : (4, 2) array - Array containing corner points (clockwise from bottom left). Format and - order are equivalent to the output of - ``astropy.wcs.WCS.calc_footprint()``. - - """ - if padding is not None: - padding = padding.to_value(new_unit) - else: - padding = 0. - - x_convf = unit_from_table(x_name, table, tbl_unit).to(new_unit) - y_convf = unit_from_table(y_name, table, tbl_unit).to(new_unit) - - x_col = table[x_name] * x_convf - y_col = table[y_name] * y_convf - - x_min = x_col.min() - padding - x_max = x_col.max() + padding - y_min = y_col.min() - padding - y_max = y_col.max() + padding - - extent = np.array([[x_min, y_min], - [x_min, y_max], - [x_max, y_max], - [x_max, y_min]]) - - return extent - - -def split_header(hdr, chunk_size, wcs_suffix=""): - """ - Split a header into many smaller parts of the chunk_size. - - Parameters - ---------- - hdr - chunk_size - wcs_suffix - - Returns - ------- - hdr_list - """ - # TODO: test that this works - s = wcs_suffix - naxis1, naxis2 = hdr["NAXIS1"+s], hdr["NAXIS2"+s] - x0_pix, y0_pix = hdr["CRPIX1"+s], hdr["CRPIX2"+s] # pix - x0_sky, y0_sky = hdr["CRVAL1"+s], hdr["CRVAL2"+s] # deg - x_delt, y_delt = hdr["CDELT1"+s], hdr["CDELT2"+s] # deg / pix - - hdr_list = [] - for x1_pix in range(0, naxis1, chunk_size): - for y1_pix in range(0, naxis2, chunk_size): - x1_sky = x0_sky + (x1_pix - x0_pix) * x_delt - y1_sky = y0_sky + (y1_pix - y0_pix) * y_delt - x2_sky = x1_sky + x_delt * min(chunk_size, naxis1 - x1_pix) - y2_sky = y1_sky + y_delt * min(chunk_size, naxis2 - y1_pix) - - hdr_sky = header_from_list_of_xy([x1_sky, x2_sky], - [y1_sky, y2_sky], - pixel_scale=x_delt, wcs_suffix=s) - hdr_list.append(hdr_sky) - - return hdr_list - - def _fix_360(arr): """Fix the "full circle overflow" that occurs with deg.""" if isinstance(arr, u.Quantity): @@ -1128,15 +922,6 @@ def _fix_360(arr): return arr -def _get_unit_from_headers(*headers, wcs_suffix: str = "") -> str: - unit = headers[0][f"CUNIT1{wcs_suffix}"].lower() - assert all(header[f"CUNIT{i}{wcs_suffix}"].lower() == unit - for header, i in product(headers, range(1, 3))), \ - [(i, header[f"CUNIT{i}{wcs_suffix}"]) - for header, i in product(headers, range(1, 3))] - return unit - - def det_wcs_from_sky_wcs( sky_wcs: WCS, pixel_scale: float, @@ -1167,8 +952,6 @@ def det_wcs_from_sky_wcs( Shape of the image (``NAXIS1``, ``NAXIS2``). """ - # TODO: Using astropy units for now to avoid deg vs. arcsec confusion. - # Once Scopesim is consistent there, remove astropy units. pixel_scale <<= u.arcsec / u.pixel plate_scale <<= u.arcsec / u.mm logger.debug("Pixel scale: %s", pixel_scale) @@ -1186,10 +969,12 @@ def det_wcs_from_sky_wcs( return create_wcs_from_points(corners, pixel_size, "D") -def sky_wcs_from_det_wcs(det_wcs: WCS, - pixel_scale: float, - plate_scale: float, - naxis=None) -> tuple[WCS, np.ndarray]: +def sky_wcs_from_det_wcs( + det_wcs: WCS, + pixel_scale: float, + plate_scale: float, + naxis=None, +) -> tuple[WCS, np.ndarray]: """ Create celestial WCS from detector WCS using pixel and plate scales. @@ -1214,8 +999,6 @@ def sky_wcs_from_det_wcs(det_wcs: WCS, Shape of the image (``NAXIS1``, ``NAXIS2``). """ - # TODO: Using astropy units for now to avoid deg vs. arcsec confusion. - # Once Scopesim is consistent there, remove astropy units. pixel_scale <<= u.arcsec / u.pixel plate_scale <<= u.arcsec / u.mm logger.debug("Pixel scale: %s", pixel_scale) diff --git a/scopesim/optics/optical_train.py b/scopesim/optics/optical_train.py index 8ef81cd9c..ebad53f79 100644 --- a/scopesim/optics/optical_train.py +++ b/scopesim/optics/optical_train.py @@ -1,19 +1,18 @@ # -*- coding: utf-8 -*- import copy -import os from collections.abc import Mapping from datetime import datetime +from pathlib import Path import numpy as np -from scipy.interpolate import interp1d from astropy import units as u from astropy.table import QTable from astropy.wcs import WCS +from astropy.io.fits import HDUList from tqdm.auto import tqdm -from synphot import SourceSpectrum, Empirical1D from synphot.units import PHOTLAM from astar_utils.nested_mapping import RecursiveNestedMapping, is_bangkey, recursive_update @@ -21,12 +20,14 @@ from .optics_manager import OpticsManager from .fov_manager import FOVManager from .image_plane import ImagePlane +from .fov import FieldOfView3D from ..commands.user_commands import UserCommands from ..detector import DetectorManager -from ..effects import ExtraFitsKeywords +from ..effects import ExtraFitsKeywords, SelectorWheel, MoffatPSF, TERCurve, SpectralEfficiency, EchelleSpectralEfficiency, SpectralTraceList +from ..effects.ter_curves import DichroicTree, FilterWheelBase, TaperedQuantumEfficiency +from ..effects.atmo_dispersion import ShiftFoV3D from ..utils import from_currsys, top_level_catch, get_logger from ..source.source_templates import empty_sky -from ..source.source_fields import SpectrumSourceField from .. import __version__ @@ -37,8 +38,14 @@ N_PROCESSES = mp.cpu_count() - 1 USE_MULTIPROCESSING = False -def extract_source(fov, source): - fov.extract_from(source) +def extract_source(fov, source, source_effects=(), spatial_header=None, _spectral_wavesets=None): + if source_effects: + source = source.make_copy() + for effect in source_effects: + source = effect.apply_to(source, fov=fov) + # Branch filtering changes the spectrum and its native wavelength grid. + _spectral_wavesets = None + fov.extract_from(source, spatial_header=spatial_header, _spectral_wavesets=_spectral_wavesets) return fov def view_fov(fov, hdu_type): @@ -121,6 +128,7 @@ def __init__(self, cmds=None): self.detector_managers = [] self.yaml_dicts = None self._last_source = None + self._readout_id = 0 if cmds is not None: self.load(cmds) @@ -171,9 +179,9 @@ def load(self, user_commands): self.yaml_dicts = self.cmds.yaml_dicts self.optics_manager = OpticsManager(self.yaml_dicts, self.cmds) - self.update() + self.update(migrate_cmds=True) - def update(self, **kwargs): + def update(self, migrate_cmds=False, **kwargs): """ Update the user-defined parameters and remake main internal classes. @@ -186,12 +194,19 @@ def update(self, **kwargs): self.optics_manager.update(**kwargs) opt_man = self.optics_manager - self.fov_manager = FOVManager(opt_man.fov_setup_effects, cmds=self.cmds, - **kwargs) - self.image_planes = [ImagePlane(hdr, self.cmds, **kwargs) - for hdr in opt_man.image_plane_headers] - self.detector_managers = [DetectorManager(det_list, cmds=self.cmds, **kwargs) - for det_list in opt_man.detector_setup_effects] + self.fov_manager = FOVManager( + opt_man.fov_setup_effects, + cmds=self.cmds, + **kwargs + ) + self.image_planes = [ + ImagePlane(hdr, self.cmds, **kwargs) + for hdr in opt_man.image_plane_headers + ] + self.detector_managers = [ + DetectorManager(det_list, cmds=self.cmds, **kwargs) + for det_list in opt_man.detector_setup_effects + ] # Move everything from CurrObs to CurrSys, so CurrObs is clean for # .observe and .readout. This is necessary because the setup and @@ -204,7 +219,8 @@ def update(self, **kwargs): # self.cmds.maps[0].clear() # HACK: recursive_update is needed to avoid overwriting emtpy !ABCs # TODO: or is it?? - self.cmds.maps[1].dic = recursive_update(self.cmds.maps[1].dic, self.cmds.maps[0].dic) + if migrate_cmds: + self.cmds.maps[1].dic = recursive_update(self.cmds.maps[1].dic, self.cmds.maps[0].dic) self.cmds.maps[0].dic.clear() def trace_detector_coordinates(self, *, xi=0 * u.arcsec, @@ -225,12 +241,13 @@ def trace_detector_coordinates(self, *, xi=0 * u.arcsec, explicit definition of which output image owns each trace coordinate. """ from ..effects import ( - BinnedImage, + DetectorModePropertiesSetter, + ImagePlaneBackground, Rotate90CCD, SelectorWheel, SpectralTraceList, - UnequalBinnedImage, ) + from ..effects.electronic.pixels import BinnedImageBase trace_lists = self.optics_manager.get_all(SpectralTraceList) if len(trace_lists) != 1: @@ -239,14 +256,18 @@ def trace_detector_coordinates(self, *, xi=0 * u.arcsec, "SpectralTraceList.") trace_list = trace_lists[0] - if self.optics_manager.image_plane_effects: - raise NotImplementedError( - "Trace detector coordinates do not yet support image-plane " - "effects.") - if self.optics_manager.detector_array_effects: - raise NotImplementedError( - "Trace detector coordinates do not yet support detector-array " - "effects.") + for effect in self.optics_manager.image_plane_effects: + selected = effect.wheel_effects.values() if isinstance(effect, SelectorWheel) else [effect] + if any(not isinstance(item, ImagePlaneBackground) for item in selected): + raise NotImplementedError('Trace detector coordinates only support additive image-plane backgrounds.') + for effect in self.optics_manager.detector_array_effects: + if not isinstance(effect, DetectorModePropertiesSetter): + raise NotImplementedError('Trace detector coordinates do not yet support this detector-array effect.') + # Mode setters update commands, not image coordinates. Resolve an + # explicit mode before reading any binning command it can set. + if from_currsys(effect.mode_key, self.cmds) == 'auto': + raise NotImplementedError('Select an explicit detector readout mode before inspecting coordinates.') + effect.apply_to(None) fovs_by_trace = {} for fov in self.fov_manager.fovs: @@ -317,31 +338,14 @@ def image_origin_on_canvas(shape_xy, image_wcs, canvas_wcs): "detectors on one image plane.") detector = detector_manager[0] detector_id = detector.header["ID"] + readout_header = detector._native_header.copy() for effect in self.optics_manager.detector_effects: - selected_effect = ( - effect.wheel_effects.get(detector_id) - if isinstance(effect, SelectorWheel) - else effect - ) + selected_effect = effect.get_effect(detector_id) if isinstance(effect, SelectorWheel) else effect if selected_effect is None: continue - if isinstance(selected_effect, BinnedImage): - bin_size = from_currsys( - selected_effect.meta["bin_size"], self.cmds) - if bin_size != 1: - raise NotImplementedError( - "Trace detector coordinates do not yet support " - "post-extraction detector binning.") - if isinstance(selected_effect, UnequalBinnedImage): - binx = from_currsys( - selected_effect.meta["binx"], self.cmds) - biny = from_currsys( - selected_effect.meta["biny"], self.cmds) - if binx != 1 or biny != 1: - raise NotImplementedError( - "Trace detector coordinates do not yet support " - "post-extraction detector binning.") + if isinstance(selected_effect, BinnedImageBase): + readout_header = selected_effect.bin_header(readout_header) if isinstance(selected_effect, Rotate90CCD): rotations = from_currsys( selected_effect.meta["rotations"], self.cmds) @@ -350,7 +354,7 @@ def image_origin_on_canvas(shape_xy, image_wcs, canvas_wcs): "Trace detector coordinates do not yet support " "post-extraction detector rotation.") - detector_wcs = WCS(detector.header, key="D") + detector_wcs = WCS(detector._native_header, key="D") image_plane_wcs = image_plane._det_wcs if ( not np.allclose(image_plane_wcs.wcs.pc, np.eye(2)) @@ -420,13 +424,7 @@ def image_origin_on_canvas(shape_xy, image_wcs, canvas_wcs): f"Trace {trace_name!r} has overlapping FOV wavelength " "ranges.") - ( - _, - _, - x_grid_mm, - y_grid_mm, - subimage_wcs, - ) = trace._focal_plane_grid(fov) + _, _, x_grid_mm, y_grid_mm, subimage_wcs = trace._focal_plane_grid(fov) if ( not np.allclose(subimage_wcs.wcs.pc, np.eye(2)) or not np.allclose( @@ -441,16 +439,7 @@ def image_origin_on_canvas(shape_xy, image_wcs, canvas_wcs): x_mm = trace.xilam2x(xi_flat[in_fov], wave_flat[in_fov]) y_mm = trace.xilam2y(xi_flat[in_fov], wave_flat[in_fov]) - subimage_x = ( - (x_mm - x_grid_mm[0]) - / (x_grid_mm[-1] - x_grid_mm[0]) - * (len(x_grid_mm) - 1) - ) - subimage_y = ( - (y_mm - y_grid_mm[0]) - / (y_grid_mm[-1] - y_grid_mm[0]) - * (len(y_grid_mm) - 1) - ) + subimage_x, subimage_y = subimage_wcs.all_world2pix(x_mm, y_mm, 0) subimage_shape = np.array( [len(x_grid_mm), len(y_grid_mm)]) @@ -468,6 +457,8 @@ def image_origin_on_canvas(shape_xy, image_wcs, canvas_wcs): f"Requested wavelengths for trace {trace_name!r} " "do not map through a configured FOV.") + focal_x, focal_y = detector_wcs.all_pix2world(x_pix, y_pix, 0) + x_pix, y_pix = WCS(readout_header, key='D').all_world2pix(focal_x, focal_y, 0) point_count = wave_grid.size columns["readout_index"].extend( np.full(point_count, readout_index)) @@ -525,6 +516,18 @@ def observe(self, orig_source=None, update=True, **kwargs): """ if update: self.update(**kwargs) + self._readout_id = 0 + + if self.cmds["!SIM.random.seed"] is None: + # Concretize seed from system entropy to keep constant during the + # observe run. For multiple observations, this is cleared above. + self.cmds["!SIM.random.seed"] = np.random.SeedSequence().entropy + elif self.cmds["!SIM.random.seed"] in {"None", "none", "NONE"}: + logger.warning( + "Got random seed '%s' (a string, not None!), use NULL in yaml", + self.cmds["!SIM.random.seed"] + ) + self.cmds["!SIM.random.seed"] = np.random.SeedSequence().entropy # self.set_focus(**kwargs) # put focus back on current instrument package @@ -538,9 +541,26 @@ def observe(self, orig_source=None, update=True, **kwargs): source = self.prepare_source(source) # [1D - transmission curves] - for effect in self.optics_manager.source_effects: + source_effects = self.optics_manager.source_effects + branch_effects = [] + if not self.fov_manager.is_spectroscope: + for ii, effect in enumerate(source_effects): + if isinstance(effect, (DichroicTree, SelectorWheel)): + # Attenuation and emission downstream of a split belong to each branch. + source_effects, branch_effects = source_effects[:ii], source_effects[ii:] + break + for effect in source_effects: source = effect.apply_to(source) + # This source is a private copy, after all source effects. FOV extraction + # copies native-sampled spectra before FOV effects can modify them. Evaluate + # native sampling once for this observation; retain no cache on the + # source, spectrum, FOV or train, and never reuse it on the next call. + spectral_wavesets = {} + for spectrum in source.spectra.values(): + waveset = spectrum.waveset + spectral_wavesets[spectrum] = None if waveset is None else waveset.to_value(u.AA) + # [3D - Atmospheric shifts, PSF, NCPAs, Grating shift/distortion] # START OF MULTIPROCESSING @@ -552,7 +572,7 @@ def observe(self, orig_source=None, update=True, **kwargs): with mp.Pool(processes=N_PROCESSES) as pool: fovs = pool.starmap(extract_source, - zip(fovs, [source] * len(fovs))) + zip(fovs, [source] * len(fovs), [branch_effects] * len(fovs))) with mp.Pool(processes=N_PROCESSES) as pool: fovs = pool.starmap(view_fov, @@ -568,14 +588,59 @@ def observe(self, orig_source=None, update=True, **kwargs): fovs = self.fov_manager.fovs nobar = len(fovs) <= 1 for fov in tqdm(fovs, desc=" FOVs", position=0, disable=nobar): - fov.extract_from(source) - - fov.view() - foveffs = self.optics_manager.fov_effects + # FOVManager can reuse a FOV whose previous observation is + # already mapped to detector coordinates. Prepare new sky + # illumination from its physical header, not that old HDU. + fov.hdu = None + foveffs = [] + for effect in self.optics_manager.fov_effects: + if isinstance(effect, SelectorWheel): + effect = effect.get_effect(fov.meta[effect.meta['selector_key']]) + if isinstance(effect, FilterWheelBase): + effect = effect.current_filter + if effect is not None: + foveffs.append(effect) + + for effect in foveffs: + # Slit shear needs spectral support before extraction. + # Fibre bundles have no single FOV trace and map separately. + if isinstance(effect, SpectralTraceList) and isinstance(fov, FieldOfView3D) and fov.trace_id is not None: + fov.meta['spectral_padding'] = effect.spectral_traces[fov.trace_id].spectral_padding(fov) + + # Render at the PSF stage when preceding effects only multiply + # spectra. Point positions then stay continuous, and resolved + # images are convolved before clipping to the aperture. + psf_indices = [i for i, effect in enumerate(foveffs) if isinstance(effect, MoffatPSF)] + defer_render = isinstance(fov, FieldOfView3D) and len(psf_indices) == 1 + if defer_render: + psf_index = psf_indices[0] + defer_render = all(isinstance(effect, (TERCurve, SpectralEfficiency, EchelleSpectralEfficiency, TaperedQuantumEfficiency)) for effect in foveffs[:psf_index]) + spatial_shifts = [] + if defer_render: + for effect in foveffs[psf_index + 1:]: + if isinstance(effect, ShiftFoV3D): + spatial_shifts.append(effect) + elif not isinstance(effect, (TERCurve, DichroicTree, SpectralEfficiency, EchelleSpectralEfficiency, TaperedQuantumEfficiency)): + break + # Wavelength-only multipliers commute with a displacement + # within each spectral plane; stop before other spatial effects. + point_sources = not (source.image_fields or source.cube_fields) + spatial_header = foveffs[psf_index].padded_header(fov, point_sources=point_sources) + for effect in spatial_shifts: + spatial_header = effect._padded_header(spatial_header, effect.get_shifts(fov)) + extract_source(fov, source, branch_effects, spatial_header=spatial_header, _spectral_wavesets=spectral_wavesets) + else: + extract_source(fov, source, branch_effects, _spectral_wavesets=spectral_wavesets) + fov.view() nobar = len(foveffs) <= 1 for effect in tqdm(foveffs, disable=nobar, desc=" FOV effects", position=1):#, leave=False): - fov = effect.apply_to(fov) + if effect in spatial_shifts: + continue # Applied to incoming illumination at the PSF stage. + if defer_render and effect is foveffs[psf_index] and spatial_shifts: + effect._apply_convolution(fov, spatial_shifts=spatial_shifts) + else: + fov = effect.apply_to(fov) if fov.hdu is None: logger.info(" Skipping empty FOV") @@ -590,6 +655,10 @@ def observe(self, orig_source=None, update=True, **kwargs): self.image_planes[fov.image_plane_id].header.update( WCS(fov.hdu).to_header()) + # Mapping is complete. Keep the rendered FOV and trace's + # rectified spectrum, without retaining every input cube. + fov.cube = None + # ..todo: finish off the multiple image plane stuff # END OF MULTIPROCESSING @@ -623,28 +692,12 @@ def prepare_source(self, source): # TODO: this is not sufficiently general # TODO: Maybe move this to source_fields?? - for field in source.fields: - if not isinstance(field, SpectrumSourceField): - continue - for ispec, spec in field.spectra.items(): - # Put on fov wavegrid - wave_min = min(fov.meta["wave_min"] for fov in self.fov_manager.fovs) - wave_max = max(fov.meta["wave_max"] for fov in self.fov_manager.fovs) - wave_unit = u.Unit(from_currsys("!SIM.spectral.wave_unit", self.cmds)) - dwave = from_currsys("!SIM.spectral.spectral_bin_width", self.cmds) # Not a quantity - fov_waveset = np.arange(wave_min.value, wave_max.value, dwave) * wave_unit - fov_waveset = fov_waveset.to(u.um) - - field.spectra[ispec] = SourceSpectrum(Empirical1D, - points=fov_waveset, - lookup_table=spec(fov_waveset)) - for cube in source.cube_fields: header, data, wave = cube.header, cube.data, cube.waveset # Need to check whether BUNIT is per arcsec2 or per pixel inunit = u.Unit(header["BUNIT"]) - data = data.astype(np.float32) * inunit + data = data.astype(np.float64, copy=False) * inunit factor = 1 for base, power in zip(inunit.bases, inunit.powers): if (base**power).is_equivalent(u.arcsec**(-2)): @@ -673,61 +726,52 @@ def prepare_source(self, source): cube.header["CUNIT1"] = "deg" cube.header["CUNIT2"] = "deg" - # Put on fov wavegrid - wave_min = min(fov.meta["wave_min"] for fov in self.fov_manager.fovs) - wave_max = max(fov.meta["wave_max"] for fov in self.fov_manager.fovs) - wave_unit = u.Unit(from_currsys("!SIM.spectral.wave_unit", self.cmds)) - dwave = from_currsys("!SIM.spectral.spectral_bin_width", self.cmds) # Not a quantity - fov_waveset = np.arange(wave_min.value, wave_max.value, dwave) * wave_unit - fov_waveset = fov_waveset.to(u.um) - - if (wave.to(u.um).min() > fov_waveset.max() or - wave.to(u.um).max() < fov_waveset.min()): - logger.warning("Source and FOV wavesets disjoint.") - - # Interpolate into new data cube. - # This is done layer by layer for memory reasons. - new_data = np.zeros((fov_waveset.shape[0], data.shape[1], data.shape[2]), - dtype=np.float32) - for j in range(data.shape[1]): - cube_interp = interp1d(wave.to_value(u.um), data[:, j, :], - axis=0, kind="linear", - bounds_error=False, fill_value=0) - new_data[:, j, :] = cube_interp(fov_waveset.value) - - cube.data = new_data - cube.header["CTYPE3"] = "WAVE" - cube.header["CRPIX3"] = 1 - cube.header["CRVAL3"] = wave_min.value - cube.header["CDELT3"] = dwave - cube.header["CUNIT3"] = wave_unit.name - + # Keep the native wavelength samples until the spatial FOV is cut. + # Rebinning the entire source here loses narrow lines and expands + # resolved cubes over the full instrument band unnecessarily. + cube.data = data cube.wcs = WCS(cube.field) return source @top_level_catch - def readout(self, filename=None, reset=True, **kwargs): + def readout( + self, + filename: Path | str | None = None, + reset: bool = True, + roid: int | None = None, + **kwargs + ) -> list[HDUList]: """ Produce detector readouts for the observed image. Parameters ---------- - filename : str, optional - Where to save the FITS file - kwargs + filename : Path | str | None, optional + Full path or local name for to store the resulting FITS file. + If None (the default), the function still returns the result, but + will not automatically save it to disk. + reset : bool, optional + If true (the default), do not keep any parameters set during any + previous readout. This is usually what you want. + roid : int or None, optional + Readout ID for reproducible noise. By default, successive calls + use 0, 1, 2, ... within each observation. Supply an explicit ID to + replay that noise realization with the same root seed. + **kwargs : + Any other kwargs passed to the detector effects. Returns ------- - hdu : fits.HDUList - - Notes - ----- - - Apply detector plane (0D, 2D) effects - z_order = 500..599 + list[HDUList] + List of HDUList objects. """ if reset: self.cmds.clear() + if roid is None: + roid = self._readout_id + self._readout_id += 1 # Hack to make sure AutoExposure and Quantization work properly. # Should probably be removed once #428 is fixed properly. @@ -737,8 +781,8 @@ def readout(self, filename=None, reset=True, **kwargs): # self.cmds["!OBS.dit"] = None # self.cmds["!OBS.ndit"] = None # TODO: This is still hacky but seems to work for now... - params = {"exptime": None} - params.update(kwargs) + params = {"exptime": None, "roid": roid} + params.update({key.removeprefix('!OBS.'): value for key, value in kwargs.items()}) if params["exptime"] is not None and params.get("dit") is None: params.update( dit=None, @@ -753,19 +797,20 @@ def readout(self, filename=None, reset=True, **kwargs): for key, value in params.items(): if is_bangkey(key): - # skip if already explicitly !ABC.xyz logger.debug("%s: %s given in kwargs", key, value) + self.cmds[key] = value continue # otherwise fallback to !OBS logger.debug("%s: %s given in kwargs, put in !OBS", key, value) self.cmds[f"!OBS.{key}"] = value hduls = [] - for i, detector_array in enumerate(self.detector_managers): - array_effects = self.optics_manager.detector_array_effects - dtcr_effects = self.optics_manager.detector_effects - hdul = detector_array.readout( - self.image_planes, array_effects, dtcr_effects) + for i, detector_manager in enumerate(self.detector_managers): + hdul = detector_manager.readout( + self.image_planes, + self.optics_manager.detector_array_effects, + self.optics_manager.detector_effects, + ) fits_effects = self.optics_manager.get_all(ExtraFitsKeywords) if len(fits_effects) > 0: @@ -773,16 +818,22 @@ def readout(self, filename=None, reset=True, **kwargs): hdul = effect.apply_to(hdul, optical_train=self) else: try: - hdul = self.write_header(hdul, detector_array) + hdul = self.write_header(hdul, detector_manager) except Exception: - logger.exception("Header update failed, data will be " - "saved with incomplete header. See stack " - "trace for details.") - - if filename is not None and isinstance(filename, str): - fname = filename - if len(self.detector_managers) > 1: - fname = f"{os.path.dirname(filename)}/{i}_{os.path.basename(filename)}" + logger.exception( + "Header update failed, data will be saved with " + "incomplete header. See stack trace for details." + ) + + if filename is not None: + if len(self.detector_managers) == 1: + fname = filename + else: + if isinstance(filename, str): + fname = f"{i}_{filename}" + if isinstance(filename, Path): + fname = filename.parent / f"{i}_{filename.name}" + hdul.writeto(fname, overwrite=True) hduls.append(hdul) @@ -817,10 +868,12 @@ def write_header(self, hdulist, detector_array): # Image HDUs for hdu in hdulist[1:]: - hdu.header["EXPTIME"] = from_currsys("!OBS.exptime", self.cmds), "[s]" - hdu.header["DIT"] = from_currsys("!OBS.dit", self.cmds), "[s]" - hdu.header["NDIT"] = from_currsys("!OBS.ndit", self.cmds) - hdu.header["DETECTOR"] = detector_array._detector_list.meta["detector"] + # Arm-specific integration effects already recorded their actual + # exposure. Global command values are only for minimal setups. + for key in ('EXPTIME', 'DIT', 'NDIT'): + if key not in hdu.header: + hdu.header[key] = from_currsys(f'!OBS.{key.lower()}', self.cmds) + hdu.header['DETECTOR'] = detector_array._detector_list.display_name return hdulist def shutdown(self): diff --git a/scopesim/optics/surface.py b/scopesim/optics/surface.py index fb7f70963..f60c9f0e4 100644 --- a/scopesim/optics/surface.py +++ b/scopesim/optics/surface.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + from pathlib import Path from dataclasses import dataclass from typing import Any @@ -8,14 +10,29 @@ from astropy.io import ascii as ioascii from astropy.table import Table -from synphot import SpectralElement -from synphot.models import Empirical1D +from synphot import SpectralElement, SourceSpectrum, Empirical1D, Observation +from synphot.units import PHOTLAM + +from ..utils import ( + get_meta_quantity, + quantify, + from_currsys, + convert_table_comments_to_dict, + find_file, + get_logger, +) +from ..source.source_templates import ( + vega_spectrum, + st_spectrum, + ab_spectrum, +) +from ..effects.ter_curves_utils import get_filter +from .surface_utils import ( + make_emission_from_emissivity, + make_emission_from_array, + extract_type_from_unit, +) -from ..effects import ter_curves_utils as ter_utils -from .surface_utils import (make_emission_from_emissivity, - make_emission_from_array, extract_type_from_unit) -from ..utils import (get_meta_quantity, quantify, from_currsys, - convert_table_comments_to_dict, find_file, get_logger) logger = get_logger(__name__) @@ -88,15 +105,15 @@ def wavelength(self): return self._get_array("wavelength") @property - def throughput(self): + def throughput(self) -> SpectralElement: return self._get_ter_property(self.meta.get("action", "transmission")) @property - def transmission(self): + def transmission(self) -> SpectralElement: return self._get_ter_property("transmission") @property - def emissivity(self): + def emissivity(self) -> SourceSpectrum: return self._get_ter_property("emissivity") @property @@ -144,7 +161,7 @@ def emission(self): filter_name = dic["filter_name"] if "filename_format" in dic: filter_name = dic["filename_format"].format(filter_name) - flux = ter_utils.scale_spectrum(flux, filter_name, amplitude) + flux = scale_spectrum(flux, filter_name, amplitude) return flux @@ -197,9 +214,12 @@ def _get_ter_property(self, ter_property, fmt="synphot"): fill_value = 0. if ter_property == "transmission" else np.nan if value_arr is not None and wave is not None and fmt == "synphot": - response_curve = SpectralElement(Empirical1D, points=wave, - lookup_table=value_arr, - fill_value=fill_value) + response_curve = SpectralElement( + Empirical1D, + points=wave, + lookup_table=value_arr, + fill_value=fill_value, + ) elif fmt == "array": response_curve = value_arr else: @@ -299,3 +319,76 @@ def __str__(self): msg = f"SpectralSurface [{cols}] \"{name}\"" return msg + + +# TODO: This feels WET with some SpeXtra / synphot functionality... +def scale_spectrum(spectrum, filter_name, amplitude): + """ + Scale a SourceSpectrum to a value in a filter. + + Parameters + ---------- + spectrum : synphot.SourceSpectrum + + filter_name : str + Name of a filter from + - a local instrument package (available in ``rc.__search_path__``) + - a generic filter name (see ``ter_curves_utils.FILTER_DEFAULTS``) + - a spanish-vo filter service reference (e.g. "Paranal/HAWKI.Ks") + + amplitude : astropy.Quantity, float + The value that the spectrum should have in the given filter. Acceptable + astropy quantities are: + - u.mag : Vega magnitudes + - u.ABmag : AB magnitudes + - u.STmag : HST magnitudes + - u.Jy : Jansky per filter bandpass + Additionally the "FLAM" and "FNU" units from ``synphot.units`` can + be used when passing the quantity for `amplitude`. + + Returns + ------- + spectrum : synphot.SourceSpectrum + Input spectrum scaled to the given amplitude in the given filter. + + Examples + -------- + :: + + >>> from scopesim.source.source_templates import vega_spectrum + >>> from scopesim.optics.surface import scale_spectrum + >>> + >>> spec = vega_spectrum() + >>> vega_185 = scale_spectrum(spec, "Ks", -1.85 * u.mag) + >>> ab_0 = scale_spectrum(spec, "Ks", 0 * u.ABmag) + >>> jy_3630 = scale_spectrum(spec, "Ks", 3630 * u.Jy) + + """ + if isinstance(amplitude, u.Quantity): + if amplitude.unit.physical_type == "spectral flux density": + if amplitude.unit != u.ABmag: + amplitude = amplitude.to(u.ABmag) + ref_spec = ab_spectrum(amplitude.value) + + elif amplitude.unit.physical_type == "spectral flux density wav": + if amplitude.unit != u.STmag: + amplitude = amplitude.to(u.STmag) + ref_spec = st_spectrum(amplitude.value) + + elif amplitude.unit == u.mag: + ref_spec = vega_spectrum(amplitude.value) + + else: + raise ValueError(f"Units of amplitude must be one of " + f"[u.mag, u.ABmag, u.STmag, u.Jy]: {amplitude}") + else: + ref_spec = vega_spectrum(amplitude) + + filt = get_filter(filter_name) + ref_flux = Observation(ref_spec, filt).effstim(flux_unit=PHOTLAM) + + real_flux = Observation(spectrum, filt).effstim(flux_unit=PHOTLAM) + scale_factor = ref_flux / real_flux + spectrum *= scale_factor.value + + return spectrum diff --git a/scopesim/source/source.py b/scopesim/source/source.py index b738cb92d..8d264b061 100644 --- a/scopesim/source/source.py +++ b/scopesim/source/source.py @@ -1,24 +1,5 @@ +# -*- coding: utf-8 -*- """ -# old functionality to implement: -# - provide x, y, lam, spectra, weight, ref -# - overridden + : number, Source, SourceSpectrum -# - overridden * : number, SpectralElement -# - write to and read from file -# - shift all fields -# - rotate around the centre -# - photons_in_range returns the photons per spectrum in a wavelength range -# - image_in_range returns an image of the source for a wavelength range -# -# old functionality which will be removed: -# - project_onto_chip -# - apply_optical_train -# -# old structure --> new structure: -# - all data held in 6 arrays -# --> new dicts for fields, spectrum -# field can be a Table or an ImageHDU -# spectrum is a SourceSpectrum -# # Use cases: # image + spectrum # images + spectra @@ -44,18 +25,14 @@ from synphot import SpectralElement, SourceSpectrum -from ..optics.image_plane import ImagePlane -from ..optics import image_plane_utils as imp_utils from .source_utils import ( validate_source_input, convert_to_list_of_spectra, - photons_in_range, ) from .source_fields import ( SourceField, TableSourceField, SpectrumSourceField, - HDUSourceField, ImageSourceField, CubeSourceField, ) @@ -63,7 +40,6 @@ find_file, is_fits, get_fits_type, - quantity_from_table, figure_factory, get_logger, ) @@ -242,27 +218,6 @@ def _from_imagehdu_and_spectra(self, image_hdu, **kwargs): assert not self.fields, "Constructor method must act on empty instance!" - if not image_hdu.header.get("BG_SRC"): - pass - # FIXME: This caused more problems than it solved! - # Find out if there's a good reason to mess with this, - # otherwise just remove... - - # image_hdu.header["CRVAL1"] = 0 - # image_hdu.header["CRVAL2"] = 0 - # image_hdu.header["CRPIX1"] = image_hdu.header["NAXIS1"] / 2 - # image_hdu.header["CRPIX2"] = image_hdu.header["NAXIS2"] / 2 - # #image_hdu.header["CRPIX1"] = (image_hdu.header["NAXIS1"] + 1) / 2 - # #image_hdu.header["CRPIX2"] = (image_hdu.header["NAXIS2"] + 1) / 2 - # # .. todo:: find where the actual problem is with negative CDELTs - # # .. todo:: --> abs(pixel_scale) in header_from_list_of_xy - # if image_hdu.header["CDELT1"] < 0: - # image_hdu.header["CDELT1"] *= -1 - # image_hdu.data = image_hdu.data[:, ::-1] - # if image_hdu.header["CDELT2"] < 0: - # image_hdu.header["CDELT2"] *= -1 - # image_hdu.data = image_hdu.data[::-1, :] - if isinstance(image_hdu, fits.PrimaryHDU): image_hdu = fits.ImageHDU(data=image_hdu.data, header=image_hdu.header) @@ -270,7 +225,6 @@ def _from_imagehdu_and_spectra(self, image_hdu, if not spectra: raise ValueError("No spectrum was provided.") - # image_hdu.header["SPEC_REF"] = len(self.spectra) assert len(spectra) == 1, f"_from_imagehdu_and_spectra needs single spectrum, ref was {image_hdu.header.get('SPEC_REF')}" image_hdu.header["SPEC_REF"] = 0 @@ -455,99 +409,6 @@ def bandpass(self, bandpass): self._bandpass = bandpass - # ..todo: rewrite this method - def image_in_range(self, wave_min, wave_max, pixel_scale=1*u.arcsec, - layers=None, area=None, spline_order=1, sub_pixel=False): - if layers is None: - layers = range(len(self.fields)) - fields = [self.fields[ii].field for ii in layers] - - hdr = imp_utils.get_canvas_header(fields, pixel_scale=pixel_scale) - im_plane = ImagePlane(hdr) - - for field in fields: - if isinstance(field, Table): - fluxes = self.photons_in_range(wave_min, wave_max, area, - field["ref"]) * field["weight"] - x = quantity_from_table("x", field, u.arcsec) - y = quantity_from_table("y", field, u.arcsec) - tbl = Table(names=["x", "y", "flux"], data=[x, y, fluxes]) - tbl.meta.update(field.meta) - hdu_or_table = tbl - - elif isinstance(field, fits.ImageHDU): - if field.header["SPEC_REF"] != "": - ref = [field.header["SPEC_REF"]] - flux = self.photons_in_range(wave_min, wave_max, area, ref) - # [ph s-1] or [ph s-1 m-2] come out of photons_in_range - - # ## ..todo: CATCH UNITS HERE. DEAL WITH THEM PROPERLY - # Currently assuming that all images are scaled appropriately - # and that they have SPEC_REF - - # else: - # field = scale_imagehdu(field, area=area, - # solid_angle=pixel_scale**2, - # waverange=(wave_min, wave_max)) - # # [ph s-1] or [ph s-1 m-2] come out of photons_in_range - # flux = 1 - - image = field.data * flux - hdu = fits.ImageHDU(header=field.header, data=image) - hdu_or_table = hdu - else: - continue - - im_plane.add(hdu_or_table, sub_pixel=sub_pixel, - spline_order=spline_order) - - return im_plane - - def photons_in_range(self, wave_min, wave_max, area=None, indices=None): - """ - - Parameters - ---------- - wave_min : float, u.Quantity - [um] - wave_max : float, u.Quantity - [um] - area : float, u.Quantity, optional - [m2] - indices : list of integers, optional - - Returns - ------- - counts : u.Quantity list - [ph / s / m2] if area is None - [ph / s] if area is passed - - """ - if indices is None: - indices = self.spectra.keys() - - spectra = [self.spectra[ii] for ii in indices] - counts = photons_in_range(spectra, wave_min, wave_max, area=area, - bandpass=self.bandpass) - return counts - - def fluxes(self, wave_min, wave_max, **kwargs): - return self.photons_in_range(wave_min, wave_max, **kwargs) - - def image(self, wave_min, wave_max, **kwargs): - return self.image_in_range(wave_min, wave_max, **kwargs) - - # @classmethod - # def load(cls, filename): - # """Load :class:'.Source' object from filename""" - # with open(filename, "rb") as fp1: - # src = pickle.load(fp1) - # return src - - # def dump(self, filename): - # """Save to filename as a pickle""" - # with open(filename, "wb") as fp1: - # pickle.dump(self, fp1) def shift(self, dx: float = 0, dy: float = 0, layers=None) -> None: """ diff --git a/scopesim/source/source_templates.py b/scopesim/source/source_templates.py index 0a88afbc6..7b4b04757 100644 --- a/scopesim/source/source_templates.py +++ b/scopesim/source/source_templates.py @@ -1,11 +1,11 @@ +# -*- coding: utf-8 -*- + from pathlib import Path import numpy as np - from astropy import units as u from astropy.table import Table from astropy.io import fits -from astropy.utils.decorators import deprecated_renamed_argument from synphot import SourceSpectrum, ConstFlux1D, Empirical1D from synphot.units import PHOTLAM @@ -30,7 +30,6 @@ def empty_sky(flux=0): return sky -@deprecated_renamed_argument("mag", "flux", "0.1.5") def star(x=0, y=0, flux=0): """ Source object for a single star in either vega, AB magnitudes, or Jansky diff --git a/scopesim/source/source_utils.py b/scopesim/source/source_utils.py index caf43a218..d969149be 100644 --- a/scopesim/source/source_utils.py +++ b/scopesim/source/source_utils.py @@ -4,11 +4,10 @@ from pathlib import Path import numpy as np -from scipy import integrate from astropy import wcs, units as u from astropy.io import fits from astropy.table import Table -from synphot import SourceSpectrum, Empirical1D, SpectralElement +from synphot import SourceSpectrum, Empirical1D from ..utils import find_file, get_logger, convert_table_comments_to_dict @@ -111,93 +110,6 @@ def _get_list(): return list(_get_list()) -# FIXME: typing: This should work with the more general (and true, since -# conversion is done anyway) Quantity["length"] (or "area" resp.), but -# doing so currently causes a NameError. Not sure what's going on. -def photons_in_range( - spectra: SourceSpectrum, - wave_min: u.Quantity[u.um] | float, - wave_max: u.Quantity[u.um] | float, - area: u.Quantity[u.m**2] | float | None = None, - bandpass: SpectralElement | None = None, -) -> u.Quantity[u.ph * u.s**-1 * u.m**-2] | u.Quantity[u.ph * u.s**-1]: - """ - Integrate photons from spectrum in given wavelength range. - - Parameters - ---------- - spectra : SourceSpectrum - Input spectrum. - wave_min : u.Quantity["length"] or float - Minimum wavelength. If float, assumes um. - wave_max : u.Quantity["length"] or float - Maximum wavelength. If float, assumes um. - area : u.Quantity["area"] or float, optional - Area to multiply with. If float, assumes m**2. The default is None. - bandpass : SpectralElement, optional - Filter to take into account, if any. The default is None. - - Returns - ------- - counts : astropy.units.Quantity - Either in ph/s/m**2 or just ph/s (if area was given). - - """ - # Note: Assuming um if given as float. - wave_min = (wave_min << u.um << u.Angstrom).value - wave_max = (wave_max << u.um << u.Angstrom).value - # Note: There appear to be some float shenanigans going on here, but - # rounding produces an error in the spectrum evaluation. Not sure what's - # going on, maybe it's fine as-is. - - counts = [] - for spec in spectra: - waveset = spec.waveset.value - mask = (waveset > wave_min) * (waveset < wave_max) - wave = np.array([wave_min, *waveset[mask], wave_max]) - flux = spec(wave).value - - # flux [ph s-1 cm-2] == flux [ph s-1 cm-2 AA-1] * wave [AA] - if isinstance(bandpass, SpectralElement): - bandpass.model.bounds_error = True - counts.append(integrate.trapezoid(bandpass(wave).value * flux, wave)) - else: - counts.append(integrate.trapezoid(flux, wave)) - - # counts = flux [ph s-1 cm-2] - counts = (counts * u.ph * u.s**-1 * u.cm**-2).to(u.ph * u.s**-1 * u.m**-2) - if area is not None: - counts *= (area << u.m**2) - - return counts - - -def scale_imagehdu(imagehdu, waverange, area=None): - """ - ..todo: implement this - - For the moment, all imagehdu must be accompanied by a spectrum in PHOTLAM - - Future functionality will include scaling here of: - ph s-1 - ph s-1 m-2 - ph s-1 m-2 - ph s-1 m-2 um-1 - ph s-1 m-2 um-1 arcsec-2 - J s-1 m-2 Hz-1 - J s-1 m-2 Hz-1 arcsec-2 - ABMAG - ABMAG arcsec-2 - VEGAMAG - VEGAMAG arcsec-2 - """ - if "SPEC_REF" not in imagehdu.header: - raise ValueError("For this version, an ImageHDU must be associated " - "with a spectrum. This will change in the future.") - - return imagehdu - - def make_img_wcs_header( pixel_scale: float, image_size: tuple[int, int], diff --git a/scopesim/tests/conftest.py b/scopesim/tests/conftest.py index 0be2a072f..705fd5aa3 100644 --- a/scopesim/tests/conftest.py +++ b/scopesim/tests/conftest.py @@ -6,6 +6,7 @@ import pytest from unittest.mock import patch +import matplotlib import scopesim as sim from astar_utils import UniqueList @@ -15,6 +16,8 @@ sim.rc.__currsys__["!SIM.file.error_on_missing_file"] = True +matplotlib.use("agg") # Don't render plots in normal testing + @pytest.fixture(scope="function", autouse=True) def configure_logging(): diff --git a/scopesim/tests/test_utils_functions.py b/scopesim/tests/test_utils_functions.py index a1df0f1b3..1469b32d9 100644 --- a/scopesim/tests/test_utils_functions.py +++ b/scopesim/tests/test_utils_functions.py @@ -199,6 +199,15 @@ def test_converts_list(self): def test_converts_numpy_array(self): assert utils.from_currsys(np.array(["!SIM.random.seed"]*2))[1] is None + @pytest.mark.parametrize("dtype", [np.float32, np.float64, np.int32]) + def test_numeric_calibration_array_keeps_values_and_precision(self, dtype): + values = np.arange(12, dtype=dtype).reshape(3, 4) + resolved = utils.from_currsys(values) + np.testing.assert_array_equal(resolved, values) + assert resolved.dtype == dtype + resolved[0, 0] = 20 + assert values[0, 0] == 0 + def test_converts_dict(self): assert utils.from_currsys({"seed": "!SIM.random.seed"})["seed"] is None diff --git a/scopesim/tests/tests_effects/test_ADConversion.py b/scopesim/tests/tests_effects/test_ADConversion.py index daece3751..eba0ad615 100644 --- a/scopesim/tests/tests_effects/test_ADConversion.py +++ b/scopesim/tests/tests_effects/test_ADConversion.py @@ -32,9 +32,9 @@ def fixture_mock_detector(): def fixture_detector_with_data(mock_detector): """Instantiate a Detector with some data""" det = mock_detector - width = det._hdu.data.shape[1] - det._hdu.data[:] = 1.2 - det._hdu.data[:, width//2] = 1.99 + width = det.data.shape[1] + det.data[:] = 1.2 + det.data[:, width//2] = 1.99 return det @@ -73,11 +73,11 @@ def test_converts_pixels_to_integer_with_custom(self, detector_with_data): def test_applies_gain(self, mock_detector, gain): det = mock_detector data = np.random.randn(100, 100) * 100. + 1000. - det._hdu.data = 1. * data + det.data = 1. * data adconverter = ADConversion(gain=gain) adconverter.cmds = {"!DET.gain": gain, "!OBS.ndit": 1} adconverter.apply_to(det) - assert np.all((data/gain).astype(int) == det._hdu.data) + assert np.all((data/gain).astype(int) == det.data) @pytest.mark.usefixtures("patch_mock_path_micado") def test_applies_gain_list_to_detector_list(self): @@ -104,10 +104,10 @@ def test_caps_data_overflow(self, mock_detector, newtype, maxval, minval): data = medval * np.ones((100, 100), dtype=np.float64) data[50, 50] = 1.e12 data[60, 60] = -100000. - det._hdu.data = data + det.data = data adconverter = ADConversion(gain=gain, dtype=newtype) adconverter.cmds = {"!DET.gain": gain, "!OBS.ndit": 1} adconverter.apply_to(det) - assert(np.median(det._hdu.data) == newtype(medval/gain)) - assert(np.max(det._hdu.data) == maxval) - assert(np.min(det._hdu.data) == minval) + assert(np.median(det.data) == newtype(medval/gain)) + assert(np.max(det.data) == maxval) + assert(np.min(det.data) == minval) diff --git a/scopesim/tests/tests_effects/test_AtmosphericDispersion.py b/scopesim/tests/tests_effects/test_AtmosphericDispersion.py index 392d9e7a4..a72cc531f 100644 --- a/scopesim/tests/tests_effects/test_AtmosphericDispersion.py +++ b/scopesim/tests/tests_effects/test_AtmosphericDispersion.py @@ -3,12 +3,27 @@ import numpy as np +from scopesim.effects.atmo_dispersion import ADCShift + from scopesim.effects.shifts import AtmosphericDispersion, \ get_pixel_border_waves_from_atmo_disp from ..mocks.py_objects.yaml_objects import _atmo_yaml_dict +def test_repeated_integer_adc_shifts_preserve_positions_and_clip_edges(): + cube = np.zeros((10, 5, 7)) + cube[:, 2, 3] = 1 + dx = np.array([-1.2, -1.2, 0., 0., .9, .9, 8., -8., 0., 0.]) + dy = np.array([1., 1., 0., 0., -1., -1., 0., 0., 6., -6.]) + shifted = ADCShift.roll_shift(cube, dx, dy) + expected = np.zeros_like(cube) + expected[:2, 3, 2] = 1 + expected[2:4, 2, 3] = 1 + expected[4:6, 1, 4] = 1 + np.testing.assert_array_equal(shifted, expected) + + @pytest.fixture(scope="function") def atmo_yaml_dict(): atmo_dict = _atmo_yaml_dict() @@ -44,21 +59,17 @@ def test_throws_error_if_one_keyword_missing(self, atmo_yaml_dict): isinstance(AtmosphericDispersion(), AtmosphericDispersion) -class TestFovGrid: - def test_returns_list_of_3_arrays_with_correct_which(self, atmo_yaml_dict): - atmo_disp = AtmosphericDispersion(**atmo_yaml_dict["properties"]) - response = atmo_disp.fov_grid() - assert len(response) == 3 - assert all([isinstance(x, np.ndarray) for x in response]) - - def test_returns_non_with_wrong_which_keyword(self, atmo_yaml_dict): +class TestGetTable: + def test_returns_wavelength_dx_dy_columns(self, atmo_yaml_dict): atmo_disp = AtmosphericDispersion(**atmo_yaml_dict["properties"]) - response = atmo_disp.fov_grid(which="bogus") - assert response is None + tbl = atmo_disp.get_table() + assert tbl.colnames == ["wavelength", "dx", "dy"] + assert all(isinstance(tbl[col], np.ndarray) for col in tbl.colnames) def test_returns_similar_values_to_lasilla_website(self, atmo_yaml_dict): atmo_disp = AtmosphericDispersion(**atmo_yaml_dict["properties"]) - waves, dx, dy = atmo_disp.fov_grid() + tbl = atmo_disp.get_table() + waves, dx, dy = tbl["wavelength"], tbl["dx"], tbl["dy"] assert dy[0] - dy[-1] == approx(0.53, rel=1e-2) assert all(dx == 0) assert waves[0] == 0.5 and waves[-1] == 2.5 @@ -66,14 +77,16 @@ def test_returns_similar_values_to_lasilla_website(self, atmo_yaml_dict): def test_returns_same_results_when_turned_90_degrees(self, atmo_yaml_dict): atmo_yaml_dict["properties"]["pupil_angle"] = 90 atmo_disp = AtmosphericDispersion(**atmo_yaml_dict["properties"]) - waves, dx, dy = atmo_disp.fov_grid() + tbl = atmo_disp.get_table() + dx, dy = tbl["dx"], tbl["dy"] assert dx[0] - dx[-1] == approx(0.53, rel=1e-2) assert all([y == approx(0) for y in dy]) def test_returns_same_results_when_turned_30_degrees(self, atmo_yaml_dict): atmo_yaml_dict["properties"]["pupil_angle"] = 30 atmo_disp = AtmosphericDispersion(**atmo_yaml_dict["properties"]) - waves, dx, dy = atmo_disp.fov_grid() + tbl = atmo_disp.get_table() + dx, dy = tbl["dx"], tbl["dy"] dr = ((dx[0] - dx[-1])**2 + (dy[0] - dy[-1])**2)**0.5 assert dr == approx(0.53, rel=1e-2) diff --git a/scopesim/tests/tests_effects/test_AtmosphericDispersionCorrection.py b/scopesim/tests/tests_effects/test_AtmosphericDispersionCorrection.py index 4ff560b9f..55b83bb70 100644 --- a/scopesim/tests/tests_effects/test_AtmosphericDispersionCorrection.py +++ b/scopesim/tests/tests_effects/test_AtmosphericDispersionCorrection.py @@ -101,9 +101,11 @@ def test_shifts_between_adc_and_ad_are_opposite(self, atmo_params, waves, ad = AD(**atmo_params) adc = ADC(**atmo_params) - ad_shifts = ad.fov_grid() - ad_x_shift = np.interp(fov_wave_mid, ad_shifts[0], ad_shifts[1]) - ad_y_shift = np.interp(fov_wave_mid, ad_shifts[0], ad_shifts[2]) + ad_shifts = ad.get_table() + ad_x_shift = np.interp(fov_wave_mid, ad_shifts["wavelength"], + ad_shifts["dx"]) + ad_y_shift = np.interp(fov_wave_mid, ad_shifts["wavelength"], + ad_shifts["dy"]) adc.apply_to(fov) new_crpix_d = np.array([fov.header["CRPIX1D"], fov.header["CRPIX2D"]]) diff --git a/scopesim/tests/tests_effects/test_AutoExposure.py b/scopesim/tests/tests_effects/test_AutoExposure.py index 2876de7bf..179d1419d 100644 --- a/scopesim/tests/tests_effects/test_AutoExposure.py +++ b/scopesim/tests/tests_effects/test_AutoExposure.py @@ -1,7 +1,8 @@ """Tests for Effect AutoExposure.""" +import logging + import pytest -from unittest.mock import patch from scopesim import UserCommands from scopesim.optics.image_plane import ImagePlane @@ -9,9 +10,10 @@ from scopesim.tests.mocks.py_objects.imagehdu_objects import _image_hdu_square -# pylint: disable=no-self-use, missing-class-docstring +# pylint: disable=missing-class-docstring # pylint: disable=missing-function-docstring +LOGGER = logging.getLogger(__name__) def _patched_cmds(exptime=1, dit=None, ndit=None): return UserCommands(properties={"!OBS.exptime": exptime, @@ -32,6 +34,7 @@ def fixture_autoexposure(): """Instantiate an AutoExposure object""" return AutoExposure(fill_frac=0.75, full_well=1e5, + dark_current=0, mindit=0.011, exptime=None) @@ -40,6 +43,7 @@ class TestAutoExposure: def test_initialises_correctly(self): autoexposure = AutoExposure(fill_frac=0.75, full_well=1e5, + dark_current=0, mindit=0.011) assert isinstance(autoexposure, AutoExposure) @@ -102,7 +106,7 @@ def test_fill_frac_acts_correctly(self, imageplane): assert out_ndit_1 == fill_2 / fill_1 * out_ndit_2 @pytest.mark.xfail(reason=("This now fails because a user-specified DIT " - "and NDIT will now just go thourgh unmodified. " + "and NDIT will now just go through unmodified. " "Decide what to do with this test later...")) def test_exptime_specified_by_dit_ndit(self, autoexposure, imageplane): """ @@ -139,3 +143,46 @@ def test_exptime_at_least_mindit(self, imageplane): assert dit == mindit assert ndit == 1 + + def test_takes_dark_current_into_account(self, imageplane): + exptime = 3600 + autoexp_1 = AutoExposure(fill_frac=0.75, + full_well=1e6, + mindit=1.3, + dark_current=0) + autoexp_1.cmds["!OBS.exptime"] = exptime + autoexp_1.apply_to(imageplane) + dit1 = autoexp_1.cmds["!OBS.dit"] + + autoexp_2 = AutoExposure(fill_frac=0.75, + full_well=1e6, + mindit=1.3, + dark_current=1e4) + autoexp_2.cmds["!OBS.exptime"] = exptime + autoexp_2.apply_to(imageplane) + dit2 = autoexp_2.cmds["!OBS.dit"] + + assert dit2 < dit1 + + def test_no_dark_gives_zero_dark(self, imageplane, caplog): + exptime = 3600 + # no dark provided + autoexp_1 = AutoExposure(fill_frac=0.75, + full_well=1e6, + mindit=1.3) + autoexp_1.cmds["!OBS.exptime"] = exptime + caplog.set_level(logging.WARNING) + autoexp_1.apply_to(imageplane) + assert "No dark current found for" in caplog.text + dit1 = autoexp_1.cmds["!OBS.dit"] + + # zero dark explicitly provided + autoexp_2 = AutoExposure(fill_frac=0.75, + full_well=1e6, + mindit=1.3, + dark_current=0.) + autoexp_2.cmds["!OBS.exptime"] = exptime + autoexp_2.apply_to(imageplane) + dit2 = autoexp_2.cmds["!OBS.dit"] + + assert dit2 == dit1 diff --git a/scopesim/tests/tests_effects/test_BasicReadoutNoise.py b/scopesim/tests/tests_effects/test_BasicReadoutNoise.py index 89bc4029c..1b0ac3e30 100644 --- a/scopesim/tests/tests_effects/test_BasicReadoutNoise.py +++ b/scopesim/tests/tests_effects/test_BasicReadoutNoise.py @@ -2,7 +2,7 @@ from pytest import approx import numpy as np -from scopesim.effects.electronic.noise import PoorMansHxRGReadoutNoise, _make_ron_frame +from scopesim.effects.electronic.noise import PoorMansHxRGReadoutNoise from scopesim.tests.mocks.py_objects.detector_objects import _basic_detector @@ -23,7 +23,7 @@ def test_creates_noise_std_as_needed(self, noise_std): ron = PoorMansHxRGReadoutNoise(noise_std=noise_std, n_channels=64, ndit=1) dtcr = ron.apply_to(dtcr) - assert np.std(dtcr._hdu.data) == approx(noise_std, rel=0.05) + assert np.std(dtcr.data) == approx(noise_std, rel=0.05) @pytest.mark.parametrize("noise_std", [1, 9, 100]) @pytest.mark.parametrize("ndit", [1, 9, 100]) @@ -31,7 +31,7 @@ def test_noise_reduces_with_square_root_of_ndit(self, ndit, noise_std): dtcr = _basic_detector(width=256) ron = PoorMansHxRGReadoutNoise(noise_std=noise_std, n_channels=64, ndit=ndit) dtcr = ron.apply_to(dtcr) - noise_real = np.std(dtcr._hdu.data) + noise_real = np.std(dtcr.data) assert noise_real == approx(noise_std * ndit**0.5, rel=0.1) @@ -39,12 +39,27 @@ def test_noise_reduces_with_square_root_of_ndit(self, ndit, noise_std): class TestMakeRonFrame: @pytest.mark.parametrize("n", (1, 4, 9, 25)) def test_stdev_increase_with_square_of_n(self, n): - frames = np.array([_make_ron_frame((256, 256), 10, 2, 0.1, 0.2, 0.3, 0.4) + ron = PoorMansHxRGReadoutNoise(noise_std=10, n_channels=2, ndit=1) + ron.meta.update({ + "channel_fraction": 0.1, + "line_fraction": 0.2, + "pedestal_fraction": 0.3, + "read_fraction": 0.4, + }) + rng = np.random.default_rng(42) + frames = np.array([ron._make_ron_frame(rng, (256, 256)) for _ in range(n)]) assert np.std(np.sum(frames, axis=0)) == approx(10*n**0.5, rel=0.3) @pytest.mark.parametrize("shape", [(3, 7), (7, 3)]) def test_makes_frame_sizes_for_non_integer_n_channels(self, shape): - n_channels = 2 - frame = _make_ron_frame(shape, 5, n_channels, 0.25, 0.25, 0.25, 0.25) + ron = PoorMansHxRGReadoutNoise(noise_std=5, n_channels=2, ndit=1) + ron.meta.update({ + "channel_fraction": 0.25, + "line_fraction": 0.25, + "pedestal_fraction": 0.25, + "read_fraction": 0.25, + }) + rng = np.random.default_rng(42) + frame = ron._make_ron_frame(rng, shape) assert frame.shape > (0, 0) diff --git a/scopesim/tests/tests_effects/test_ConstPSF.py b/scopesim/tests/tests_effects/test_ConstPSF.py index d6270dd1c..cfbbd337f 100644 --- a/scopesim/tests/tests_effects/test_ConstPSF.py +++ b/scopesim/tests/tests_effects/test_ConstPSF.py @@ -12,14 +12,16 @@ # - psf : returns self, as array returns a layer based on defaults # 4. should have the following methods: # - (?) apply_to(obj) -# - fov_grid(which="waveset") # - get_kernel(obj) """ import pytest +from types import SimpleNamespace import numpy as np from numpy import testing as npt +from astropy import units as u +from astropy.io import fits import matplotlib.pyplot as plt @@ -77,6 +79,37 @@ def test_initialised_with_psf_name_and_file_format(self, mock_path): class TestGetKernel: + def test_returned_kernel_does_not_alias_reference_or_next_call(self, const_psf, centre_fov): + centre_fov.view() + initial = const_psf.get_kernel(centre_fov) + expected = initial.copy() + initial.fill(0.) + npt.assert_array_equal(const_psf.get_kernel(centre_fov), expected) + + def test_reference_image_edits_reach_next_kernel(self, const_psf, centre_fov): + centre_fov.view() + first = const_psf.get_kernel(centre_fov).copy() + ext = const_psf.current_layer_id + const_psf._file[ext].data[:] = np.roll(const_psf._file[ext].data, 3, axis=0) + second = const_psf.get_kernel(centre_fov) + assert not np.allclose(first, second) + + def test_spectral_kernel_follows_current_grid_and_units(self, const_psf): + const_psf.cmds = {'!SIM.computing.psf_maxsize': 21} + header = fits.Header({'CDELT1': .004/3600, 'CDELT2': .004/3600, 'CUNIT1': 'deg', 'CUNIT2': 'deg', 'CDELT3': .01, 'CRPIX3': 1., 'CRVAL3': 1.55, 'CUNIT3': 'um'}) + first = fits.ImageHDU(np.zeros((3, 15, 15)), header) + fov = SimpleNamespace(header=first.header, hdu=first, wavelength=1.6*u.um) + initial = const_psf.get_kernel(fov).copy() + second = fits.ImageHDU(np.zeros((5, 15, 15)), header) + fov.header, fov.hdu = second.header, second + expanded = const_psf.get_kernel(fov).copy() + assert expanded.shape[0] == 5 + npt.assert_allclose(expanded[:3], initial) + second.header['CUNIT3'] = 'nm' + second.header['CDELT3'] *= 1000 + second.header['CRVAL3'] *= 1000 + npt.assert_allclose(const_psf.get_kernel(fov), expanded, rtol=1e-6) + @pytest.mark.parametrize("waves, max_pixel", [([1.1, 1.3], 1 / 3.), ([1.5, 1.7], 1 / 3.), diff --git a/scopesim/tests/tests_effects/test_InterPixelCapacitance.py b/scopesim/tests/tests_effects/test_InterPixelCapacitance.py index fb6adb5ee..bed34660b 100644 --- a/scopesim/tests/tests_effects/test_InterPixelCapacitance.py +++ b/scopesim/tests/tests_effects/test_InterPixelCapacitance.py @@ -87,7 +87,7 @@ def fixture_detector(): """Instantiate a random detector object""" hdu = fits.ImageHDU(data=np.random.randn(214, 333)) det = Detector(hdu.header) - det._hdu.data = hdu.data + det.data = hdu.data return det class TestApply: @@ -102,7 +102,7 @@ def test_convolve_unit_source(self): [0., 1., 0.], [0., 0., 0.]])) det = Detector(hdu.header) - det._hdu.data = hdu.data + det.data = hdu.data ipc = IPC(kernel=np.random.rand(3, 3)) newdet = ipc.apply_to(det) np.testing.assert_allclose(newdet.hdu.data, ipc.kernel) diff --git a/scopesim/tests/tests_effects/test_LinearityCurve.py b/scopesim/tests/tests_effects/test_LinearityCurve.py index a0612000d..e29a39830 100644 --- a/scopesim/tests/tests_effects/test_LinearityCurve.py +++ b/scopesim/tests/tests_effects/test_LinearityCurve.py @@ -46,11 +46,11 @@ class TestApplyTo: (60, 60), (1e5, 60)]) def test_only_applied_to_detector(self, in_flux, out_flux, basic_lincurve): dtcr = _basic_detector() - dtcr._hdu.data[0, 0] = in_flux + dtcr.data[0, 0] = in_flux new_dtcr = basic_lincurve.apply_to(dtcr) - assert new_dtcr._hdu.data[0, 0] == out_flux - assert np.min(new_dtcr._hdu.data) == 0 + assert new_dtcr.data[0, 0] == out_flux + assert np.min(new_dtcr.data) == 0 def test_bypasses_non_detector_objects(self, basic_lincurve): new_dict = basic_lincurve.apply_to({"gigawatts": 1.21}) diff --git a/scopesim/tests/tests_effects/test_MoffatPSF.py b/scopesim/tests/tests_effects/test_MoffatPSF.py index 1119df8dd..03b7bbccf 100644 --- a/scopesim/tests/tests_effects/test_MoffatPSF.py +++ b/scopesim/tests/tests_effects/test_MoffatPSF.py @@ -1,14 +1,43 @@ import logging +from unittest.mock import Mock import numpy as np import pytest from astropy import units as u +from astropy.coordinates import AltAz, EarthLocation, SkyCoord +from astropy.table import Table +from astropy.time import Time +from astropy.utils import iers -from scopesim.effects.psfs.analytical import MoffatPSF +from scopesim.commands import UserCommands +from scopesim.effects.psfs.analytical import AOEnhanceablePSF, MoffatPSF + + +def test_seeing_cache_uses_physical_zenith_angle(monkeypatch): + from astropy.coordinates import SkyCoord, AltAz, EarthLocation + from astropy.time import Time + from scopesim.effects.psfs import analytical + + location = EarthLocation.from_geodetic(0*u.deg, 30*u.deg) + altitude = 60*u.deg + times = iter(Time(60310 + np.arange(10)/86400, format='mjd')) + + def observing_info(cmds): + time = next(times) + target = SkyCoord(alt=altitude, az=0*u.deg, frame=AltAz(obstime=time, location=location)) + return target, location, time, 'dark' + + monkeypatch.setattr(analytical, 'get_observation_info_from_cmds', observing_info) + psf = MoffatPSF(alpha=4.765, fwhm={'seeing': .7, 'seeing_unit': 'arcsec', 'pivot_wave': 500, 'pivot_wave_unit': 'nm'}) + first = psf.fwhm(np.array([.5])*u.um) + np.testing.assert_allclose(psf.fwhm(np.array([.5])*u.um), first) + altitude = 30*u.deg + expected = .7 / np.sin(altitude.to_value(u.rad))**.6 + assert psf.fwhm(np.array([.5])*u.um)[0].to_value(u.arcsec) == pytest.approx(expected) class SimpleFov: - header = {"CDELT1": 0.1 / 3600} + header = {"CDELT1": 0.1 / 3600, "CDELT2": 0.1 / 3600} meta = {"wave_min": 1.0 * u.um, "wave_max": 1.1 * u.um} waveset = np.linspace(1.0, 1.1, 5) * u.um @@ -16,7 +45,11 @@ class SimpleFov: def test_constant_fwhm_does_not_require_observation_commands(): psf = MoffatPSF(alpha=4.765, fwhm=0.7) - assert isinstance(psf, MoffatPSF) + wave = np.array([500., 1000.])*u.nm + np.testing.assert_allclose(psf.fwhm(wave).to_value(u.arcsec), [.7, .7]) + cached = psf._fwhm_interp[0] + np.testing.assert_allclose(psf.fwhm(wave.to(u.um)).to_value(u.arcsec), [.7, .7]) + assert psf._fwhm_interp[0] is cached def test_kernel_grows_to_flux_accuracy_without_renormalizing(): @@ -87,3 +120,165 @@ def test_legacy_renormalization_can_be_requested(): kernel = psf.get_kernel(SimpleFov()) assert np.sum(kernel) == pytest.approx(1.0) + + +@pytest.fixture +def seeing_cmds(monkeypatch): + monkeypatch.setattr(iers.conf, 'auto_download', False) + # The regression uses historical timestamps, independent of table freshness. + monkeypatch.setattr(iers.conf, 'auto_max_age', None) + monkeypatch.setattr(Time, 'now', lambda: Time('2026-10-18T10:50:25.709309')) + return UserCommands(properties={ + '!ATMO.longitude': -155.47833, '!ATMO.latitude': 19.82833, '!ATMO.altitude': 4160., + '!OBS.alt': 65.38002267134289, '!OBS.az': 0., '!OBS.airmass': None, + '!OBS.ra': None, '!OBS.dec': None, '!OBS.brightness': 'dark', + }) + + +@pytest.fixture +def natural_psf(seeing_cmds): + return MoffatPSF(cmds=seeing_cmds, alpha=4.765, + fwhm={'seeing': .65, 'seeing_unit': 'arcsec', 'pivot_wave': 500., 'pivot_wave_unit': 'nm'}) + + +def test_nighttime_timestamp_change_reuses_the_same_seeing(natural_psf, monkeypatch): + wave = np.array([500., 1000.])*u.nm + first = natural_psf.fwhm(wave) + cached = natural_psf._fwhm_interp[0] + # These are the two times in the performance notebook's original traceback. + monkeypatch.setattr(Time, 'now', lambda: Time('2026-10-18T10:50:25.735796')) + second = natural_psf.fwhm(wave.to(u.um)) + np.testing.assert_allclose(first.to_value(u.arcsec), [.68825435, .59916022], atol=1e-8, rtol=0) + np.testing.assert_array_equal(second, first) + assert natural_psf._fwhm_interp[0] is cached + + +def test_seeing_and_pivot_changes_refresh_the_cache(natural_psf): + wave = np.array([500., 1000.])*u.nm + first = natural_psf.fwhm(wave) + natural_psf.meta['fwhm']['seeing'] = 1.3 + np.testing.assert_allclose(natural_psf.fwhm(wave).to_value(u.arcsec), 2*first.to_value(u.arcsec)) + natural_psf.meta['fwhm']['pivot_wave'] = 1000. + np.testing.assert_allclose(natural_psf.fwhm(wave).to_value(u.arcsec), 2*2**.2*first.to_value(u.arcsec)) + + +def test_altitude_change_refreshes_the_seeing(natural_psf): + wave = np.array([500.])*u.nm + natural_psf.cmds['!OBS.alt'] = 90. + np.testing.assert_allclose(natural_psf.fwhm(wave).to_value(u.arcsec), [.65]) + natural_psf.cmds['!OBS.alt'] = 30. + np.testing.assert_allclose(natural_psf.fwhm(wave).to_value(u.arcsec), [.65*2**.6]) + + +def test_radec_observation_time_changes_the_seeing(natural_psf): + natural_psf.cmds['!OBS.ra'], natural_psf.cmds['!OBS.dec'] = 30., 20. + target = SkyCoord(ra=30*u.deg, dec=20*u.deg, distance=1*u.AU) + location = EarthLocation.from_geodetic(-155.47833*u.deg, 19.82833*u.deg, 4160*u.m) + widths = [] + for time in ['2026-10-18T10:50:25', '2026-10-18T12:50:25']: + natural_psf.cmds['!OBS.mjdobs'] = time + alt = target.transform_to(AltAz(obstime=Time(time), location=location)).alt + assert alt > 0*u.deg + width = natural_psf.fwhm(np.array([500.])*u.nm).to_value(u.arcsec)[0] + np.testing.assert_allclose(width, .65/np.sin(alt)**.6) + widths.append(width) + assert not np.isclose(widths[0], widths[1]) + + +def test_relative_ao_uses_the_same_natural_seeing_cache(seeing_cmds, tmp_path, monkeypatch): + path = tmp_path / 'ao.dat' + Table({'wavelength': [.4, .5, 1., 2.5], 'fwhm': [.5, .5, .5, .5]}, meta={'comments': ['wavelength_unit : um']}).write(path, format='ascii') + psf = AOEnhanceablePSF(cmds=seeing_cmds, ao_table=str(path), is_absolute=False, enable_ao=True, alpha=4.765, + fwhm={'seeing': .65, 'seeing_unit': 'arcsec', 'pivot_wave': 500., 'pivot_wave_unit': 'nm'}) + wave = np.array([500., 1000.])*u.nm + first = psf.fwhm(wave) + cached = psf._fwhm_interp[0] + monkeypatch.setattr(Time, 'now', lambda: Time('2026-10-18T10:50:25.735796')) + np.testing.assert_array_equal(psf.fwhm(wave), first) + assert psf._fwhm_interp[0] is cached + psf.meta['enable_ao'] = False + np.testing.assert_allclose(psf.fwhm(wave).to_value(u.arcsec), 2*first.to_value(u.arcsec)) + + +def test_table_fwhm_reuses_interpolator_without_observation_commands(tmp_path): + path = tmp_path / 'fwhm.dat' + Table({'wavelength': [.4, .5, 1., 2.5], 'fwhm': [.4, .5, 1., 2.5]}, meta={'comments': ['wavelength_unit : um']}).write(path, format='ascii') + psf = MoffatPSF(alpha=4.765, fwhm=str(path)) + wave = np.array([500., 1000.])*u.nm + np.testing.assert_allclose(psf.fwhm(wave).to_value(u.arcsec), [.5, 1.]) + cached = psf._fwhm_interp[0] + np.testing.assert_allclose(psf.fwhm(wave.to(u.um)).to_value(u.arcsec), [.5, 1.]) + assert psf._fwhm_interp[0] is cached + + +def test_returned_kernel_mutation_does_not_change_later_psf(): + psf = MoffatPSF(alpha=4.765, fwhm=.7) + first = psf.get_kernel(SimpleFov()) + expected = first.copy() + first.fill(0.) + np.testing.assert_array_equal(psf.get_kernel(SimpleFov()), expected) + + +def test_overwriting_fwhm_table_refreshes_interpolator(tmp_path): + path = tmp_path/'fwhm.dat' + table = Table({'wavelength': [.4, .5, 1., 2.5], 'fwhm': [.4, .5, 1., 2.5]}, meta={'comments': ['wavelength_unit : um']}) + table.write(path, format='ascii') + psf = MoffatPSF(alpha=4.765, fwhm=str(path)) + wave = np.array([.5, 1.])*u.um + first = psf.fwhm(wave) + table['fwhm'] *= 2 + table.write(path, format='ascii', overwrite=True) + np.testing.assert_allclose(psf.fwhm(wave), 2*first) + + +def test_ao_cache_tracks_resolved_path_contents_and_alpha(tmp_path): + first, second = tmp_path/'first.dat', tmp_path/'second.dat' + table = Table({'wavelength': [.4, .5, 1., 2.5], 'fwhm': [.3, .3, .3, .3]}, meta={'comments': ['wavelength_unit : um']}) + table.write(first, format='ascii') + table['fwhm'] *= 2 + table.write(second, format='ascii') + cmds = UserCommands(properties={'!INST.ao_file': str(first)}) + psf = AOEnhanceablePSF(cmds=cmds, ao_table='!INST.ao_file', is_absolute=True, enable_ao=True, alpha=4.765) + # PSF construction resolves command references. Also exercise a reference + # assigned directly to the configured metadata after construction. + psf.meta['ao_table'] = '!INST.ao_file' + wave = np.array([.5, 1.])*u.um + np.testing.assert_allclose(psf.fwhm(wave).value, .3) + assert psf.alpha == 4.765 + cmds['!INST.ao_file'] = str(second) + np.testing.assert_allclose(psf.fwhm(wave).value, .6) + table['fwhm'] /= 3 + table.write(second, format='ascii', overwrite=True) + np.testing.assert_allclose(psf.fwhm(wave).value, .2) + psf.meta['alpha'] = 3. + assert psf.alpha == 3. + + +@pytest.mark.parametrize("psf_class", [MoffatPSF, AOEnhanceablePSF]) +def test_observation_configuration_is_only_required_when_evaluating(psf_class, monkeypatch): + observation_info = Mock(side_effect=KeyError("!OBS.airmass")) + monkeypatch.setattr("scopesim.effects.psfs.analytical.get_observation_info_from_cmds", observation_info) + kwargs = {"alpha": 4.765, "fwhm": {"seeing": 0.7, "seeing_unit": "arcsec", "pivot_wave": 500, "pivot_wave_unit": "nm"}} + if psf_class is AOEnhanceablePSF: + kwargs.update(ao_table="unused.dat", enable_ao=False, is_absolute=False) + psf = psf_class(**kwargs) + observation_info.assert_not_called() + with pytest.raises(KeyError, match="airmass"): + psf.fwhm(np.array([0.5]) * u.um) + + + +def test_ao_toggle_preserves_natural_and_absolute_widths(tmp_path): + filename = tmp_path / "ao.ecsv" + table = Table({"wavelength": np.array([0.5, 1.0, 1.5, 2.0]) * u.um, "fwhm": [0.5, 0.5, 0.5, 0.5]}) + table.write(filename, format="ascii.ecsv") + psf = AOEnhanceablePSF(alpha=4.765, fwhm=0.7, ao_table=str(filename), enable_ao=False, is_absolute=False) + wavelengths = np.array([1.0]) * u.um + np.testing.assert_allclose(psf.fwhm(wavelengths).to_value(u.arcsec), [0.7]) + psf.meta["enable_ao"] = True + np.testing.assert_allclose(psf.fwhm(wavelengths).to_value(u.arcsec), [0.35]) + psf.meta["is_absolute"] = True + np.testing.assert_allclose(psf.fwhm(wavelengths).to_value(u.arcsec), [0.5]) + psf.meta["fwhm"] = 0.9 + psf.meta["enable_ao"] = False + np.testing.assert_allclose(psf.fwhm(wavelengths).to_value(u.arcsec), [0.9]) diff --git a/scopesim/tests/tests_effects/test_OnChipBinning.py b/scopesim/tests/tests_effects/test_OnChipBinning.py new file mode 100644 index 000000000..ec3347f70 --- /dev/null +++ b/scopesim/tests/tests_effects/test_OnChipBinning.py @@ -0,0 +1,75 @@ +import numpy as np +import pytest +from scopesim.detector import Detector +from scopesim.optics.image_plane_utils import header_from_list_of_xy +from astropy.wcs import WCS +from scopesim.effects.electronic import OnChipBinning, UnequalBinnedImage, BasicReadoutNoise + + +def detector(nx=120, ny=80): + header = header_from_list_of_xy([-.015*nx/2, .015*nx/2], [-.015*ny/2, .015*ny/2], .015, 'D') + return Detector(header) + + + +@pytest.mark.parametrize('bx,by', [(1, 1), (1, 2), (2, 2)]) +def test_binning_charge_and_focal_coordinates(bx, by): + det = detector() + det.data[:] = 7 + original = WCS(det.header, key='D') + OnChipBinning(binx=bx, biny=by).apply_to(det) + assert det.data.shape == (80//by, 120//bx) + assert np.all(det.data == 7*bx*by) + assert det.data.sum() == 7*120*80 + binned = WCS(det.hdu.header, key='D') + expected = original.all_pix2world((np.array([10, 20])+.5)*bx-.5, (np.array([12, 23])+.5)*by-.5, 0) + np.testing.assert_allclose(binned.all_pix2world([10, 20], [12, 23], 0), expected, atol=1e-12) + det.reset() + assert det.data.shape == (80, 120) + assert det.header['CDELT2D'] == .015 + OnChipBinning(binx=bx, biny=by).apply_to(det) + assert det.data.shape == (80//by, 120//bx) + + + +def test_binning_rotated_cd_matrix(): + det = detector() + det.header['CD1_1D'], det.header['CD1_2D'] = .012, -.009 + det.header['CD2_1D'], det.header['CD2_2D'] = .009, .012 + wcs = WCS(det.header, key='D') + OnChipBinning(binx=1, biny=2).apply_to(det) + actual = WCS(det.header, key='D').all_pix2world(9, 15, 0) + np.testing.assert_allclose(actual, wcs.all_pix2world(9, 30.5, 0), atol=1e-12) + + + +def test_binning_rejects_unsupported_modes(): + with pytest.raises(ValueError, match='Unsupported'): + OnChipBinning(binx=2, biny=1, allowed_bins=[[1, 1], [1, 2], [2, 2]]).apply_to(detector()) + with pytest.raises(ValueError): + OnChipBinning(binx=1, biny=3).apply_to(detector()) + + + +def test_ccd_bin_does_not_multiply_amplifier_noise(): + det = detector(600, 400) + OnChipBinning(binx=2, biny=2).apply_to(det) + BasicReadoutNoise(noise_std=3, ndit=1, random_seed=17).apply_to(det) + assert det.data.std() == pytest.approx(3, rel=.02) + + +def test_software_sum_accumulates_independent_read_noise(): + det = detector(600, 400) + BasicReadoutNoise(noise_std=3, ndit=1, random_seed=17).apply_to(det) + UnequalBinnedImage(binx=2, biny=2).apply_to(det) + assert det.data.std() == pytest.approx(6, rel=.02) + assert det.header['CDELT1D'] == pytest.approx(.03) + assert det.header['CDELT2D'] == pytest.approx(.03) + + +def test_header_projection_matches_applied_binning(): + det = detector() + binning = OnChipBinning(binx=1, biny=2) + expected = binning.bin_header(det.header) + binning.apply_to(det) + assert det.header == expected diff --git a/scopesim/tests/tests_effects/test_ReadoutNoiseFromTable.py b/scopesim/tests/tests_effects/test_ReadoutNoiseFromTable.py new file mode 100644 index 000000000..ef372b486 --- /dev/null +++ b/scopesim/tests/tests_effects/test_ReadoutNoiseFromTable.py @@ -0,0 +1,25 @@ +import numpy as np +import pytest +from scopesim.detector import Detector +from scopesim.optics.image_plane_utils import header_from_list_of_xy +from scopesim.effects.electronic import ReadoutNoiseFromTable, ExposureOutput + + +def detector(nx=120, ny=80): + header = header_from_list_of_xy([-.015*nx/2, .015*nx/2], [-.015*ny/2, .015*ny/2], .015, 'D') + return Detector(header) + + + +@pytest.mark.parametrize('ndit', [1, 4, 9]) +def test_nir_averaging_preserves_signal_and_reduces_noise(tmp_path, ndit): + noise_file = tmp_path / 'noise.dat' + noise_file.write_text('dit read_noise\n10 12\n100 6\n') + det = detector(500, 400) + det.data[:] = 30*ndit + ReadoutNoiseFromTable(filename=str(noise_file), dit=55, ndit=ndit, random_seed=21).apply_to(det) + ExposureOutput(dit=55, ndit=ndit, mode='average').apply_to(det) + assert det.data.mean() == pytest.approx(30, abs=.1) + assert det.data.std() == pytest.approx(9/np.sqrt(ndit), rel=.02) + with pytest.raises(ValueError, match='outside'): + ReadoutNoiseFromTable(filename=str(noise_file), dit=101, ndit=ndit).apply_to(det) diff --git a/scopesim/tests/tests_effects/test_SkycalcTERCurve.py b/scopesim/tests/tests_effects/test_SkycalcTERCurve.py index 689621b10..b87995c81 100644 --- a/scopesim/tests/tests_effects/test_SkycalcTERCurve.py +++ b/scopesim/tests/tests_effects/test_SkycalcTERCurve.py @@ -59,7 +59,9 @@ def test_initialises_with_non_skycalc_keys(self): def test_initialise_with_local_skycalc_file(self, mock_path): sky_ter = SkycalcTERCurve( - use_local_skycalc_file=str(mock_path / "skycalc_override.fits")) + use_local_skycalc_file=str(mock_path / "skycalc_override.fits"), + local_skycalc_wavelength_unit=u.um) + # This legacy fixture predates wavelength units in its FITS columns. assert sky_ter.skycalc_table is not None diff --git a/scopesim/tests/tests_effects/test_SpectralTraceList.py b/scopesim/tests/tests_effects/test_SpectralTraceList.py index 8461857a2..5ff0633aa 100644 --- a/scopesim/tests/tests_effects/test_SpectralTraceList.py +++ b/scopesim/tests/tests_effects/test_SpectralTraceList.py @@ -6,6 +6,7 @@ import numpy as np from astropy import units as u from astropy.io import fits +from astropy.table import Table from scopesim.effects.spectral_trace_list import SpectralTraceList, \ @@ -137,6 +138,93 @@ def _echelle_cmds(): class TestEchelleSpectralTraceList: + @pytest.mark.parametrize('angle', [-25, 0, 25]) + @pytest.mark.parametrize('offset', [(0., 0.), (1.5, -1.2)]) + def test_fixed_placement_and_detector_intersections(self, tmp_path, angle, offset): + params_file = tmp_path / 'echelle_trace_parameters.dat' + _write_echelle_trace_params(params_file, detector_angle=angle) + params = Table.read(params_file, format='ascii') + for key, value in {'m0': 75, 'n': 6, 'min_wave': 960, 'max_wave': 1340, 'n_disp': 2048, 'n_xdisp': 1024, 'disp_freq': 24., 'xdisp_freq': 350.}.items(): + params[key][0] = value + params['xbeta_center'] = [10.9] + params['detector_offset_x'], params['detector_offset_y'] = [offset[0]], [offset[1]] + params.meta['comments'].extend(['detector_offset_x_unit : mm', 'detector_offset_y_unit : mm']) + params.write(params_file, format='ascii', overwrite=True) + effect = EchelleSpectralTraceList(cmds=_echelle_cmds(), filename=str(params_file), wave_colname='wavelength', s_colname='s') + model = effect._spectrographs['b'] + nx, ny = model.detector.n_pix_x, model.detector.n_pix_y + raw_grid = np.linspace(-2*nx, 3*nx, 50001) + for order, edges in zip(model.orders, model.edge_wave(fsr=True)): + wave = np.linspace(edges[0], edges[1], 4*nx + 1) + x, y = model._detector_pixels(wave, order) + # Rotate about the camera origin, then translate in detector axes. + xraw = model.wavelength_to_x_pixel(wave, order).value - nx/2 + yraw = model.wavelength_to_y_pixel(wave).value[None, :] - ny/2 + model._slit_offset_pix[:, None] + c, s = np.cos(np.deg2rad(angle)), np.sin(np.deg2rad(angle)) + expected_x = c*xraw - s*yraw + nx/2 + offset[0]/.015 + expected_y = s*xraw + c*yraw + ny/2 + offset[1]/.015 + np.testing.assert_allclose(x, expected_x, atol=1e-9, rtol=0) + np.testing.assert_allclose(y, expected_y, atol=1e-9, rtol=0) + # Translation changes placement, not dispersion or slit length. + xrot, yrot = model._rotated_detector_pixels(wave, order) + np.testing.assert_allclose(np.diff(x), np.diff(xrot), atol=1e-9, rtol=0) + np.testing.assert_allclose(np.diff(y), np.diff(yrot), atol=1e-9, rtol=0) + slit_length = np.hypot(x[2] - x[0], y[2] - y[0]) + np.testing.assert_allclose(slit_length, model._slit_offset_pix[2] - model._slit_offset_pix[0], atol=1e-9, rtol=0) + + # A deliberately oversized wavelength search independently checks + # that the translated clipping search has not lost either edge. + wave = model.x_pixel_to_wavelength(raw_grid, order) + x, y = model._detector_pixels(wave, order) + inside = np.any((x >= 0) & (x <= nx) & (y >= 0) & (y <= ny), axis=0) + trace_id = f'b_{order}' + if not inside.any(): + assert trace_id not in effect.spectral_traces + continue + trace = effect.spectral_traces[trace_id] + actual = model.wavelength_to_x_pixel(u.Quantity(trace.table['wavelength']), order).value + expected = raw_grid[inside][[0, -1]] + # The generator locates each edge on a roughly one-pixel grid. + np.testing.assert_allclose([actual.min(), actual.max()], expected, atol=3, rtol=0) + + @pytest.mark.parametrize('missing', ['detector_offset_x', 'detector_offset_y', 'detector_offset_x_unit', 'detector_offset_y_unit']) + def test_incomplete_placement_fails(self, tmp_path, missing): + path = tmp_path / 'echelle_trace_parameters.dat' + _write_echelle_trace_params(path) + params = Table.read(path, format='ascii') + for axis in ['x', 'y']: + key = f'detector_offset_{axis}' + if missing != key: + params[key] = [0.] + if missing != f'{key}_unit': + params.meta['comments'].append(f'{key}_unit : mm') + params.write(path, format='ascii', overwrite=True) + with pytest.raises(KeyError, match=missing): + EchelleSpectralTraceList(cmds=_echelle_cmds(), filename=str(path)) + + @pytest.mark.parametrize('missing_value', ['detector_offset_x', 'detector_offset_y']) + def test_blank_placement_value_does_not_become_zero(self, tmp_path, missing_value): + path = tmp_path / 'echelle_trace_parameters.dat' + _write_echelle_trace_params(path) + params = Table.read(path, format='ascii') + for axis in ['x', 'y']: + key = f'detector_offset_{axis}' + params[key] = np.ma.array([0.], mask=[key == missing_value]) + params.meta['comments'].append(f'{key}_unit : mm') + params.write(path, format='ascii', overwrite=True) + with pytest.raises(ValueError, match='Both detector offsets must have values'): + EchelleSpectralTraceList(cmds=_echelle_cmds(), filename=str(path)) + + def test_placement_requires_length_units(self, tmp_path): + path = tmp_path / 'echelle_trace_parameters.dat' + _write_echelle_trace_params(path) + params = Table.read(path, format='ascii') + params['detector_offset_x'], params['detector_offset_y'] = [0.], [0.] + params.meta['comments'].extend(['detector_offset_x_unit : arcsec', 'detector_offset_y_unit : mm']) + params.write(path, format='ascii', overwrite=True) + with pytest.raises(u.UnitConversionError): + EchelleSpectralTraceList(cmds=_echelle_cmds(), filename=str(path)) + def test_does_not_write_generated_hdulist_by_default(self, tmp_path): params_file = tmp_path / "echelle_trace_parameters.dat" _write_echelle_trace_params(params_file) @@ -197,6 +285,13 @@ def test_detector_angle_clips_to_detector_not_padding(self, tmp_path): assert trace.meta["detector_angle"] == 25 assert trace.meta["detector_pad"] == 10 + model = spt._spectrographs['b'] + order = int(trace.meta['trace_id'].split('_')[1]) + wave = u.Quantity(trace.table['wavelength'][:n_wave]) + model_x, model_y = model._detector_pixels(wave, order) + np.testing.assert_allclose(model_x, xpix, atol=1e-8, rtol=0) + np.testing.assert_allclose(model_y, ypix, atol=1e-8, rtol=0) + padded_file = tmp_path / "echelle_trace_parameters_more_padding.dat" _write_echelle_trace_params( padded_file, @@ -224,7 +319,9 @@ def test_detector_angle_clips_to_detector_not_padding(self, tmp_path): ) unrotated_trace = next(iter(unrotated.spectral_traces.values())) - assert trace.wave_min == pytest.approx(unrotated_trace.wave_min) - assert trace.wave_max == pytest.approx(unrotated_trace.wave_max) + # Rotation changes which wavelengths reach the rectangular detector; + # clipping must occur after rotation, not at the old unrotated edges. + assert trace.wave_min < trace.wave_max + assert not np.allclose([trace.wave_min, trace.wave_max], [unrotated_trace.wave_min, unrotated_trace.wave_max]) assert not np.allclose(trace.table["x"], unrotated_trace.table["x"]) assert not np.allclose(trace.table["y"], unrotated_trace.table["y"]) diff --git a/scopesim/tests/tests_effects/test_SpectralTraceListUtils.py b/scopesim/tests/tests_effects/test_SpectralTraceListUtils.py index 7b8b7a97c..04be6beef 100644 --- a/scopesim/tests/tests_effects/test_SpectralTraceListUtils.py +++ b/scopesim/tests/tests_effects/test_SpectralTraceListUtils.py @@ -196,7 +196,7 @@ def test_call_gives_correct_result(self, quadratic): # matrix and explicit function td2d = Transform2D(quadratic['matrix']) - assert td2d(x, y) == quadratic['function'](x, y) + assert td2d(x, y) == pytest.approx(quadratic['function'](x, y)) def test_gradient_gives_correct_result(self, quadratic): x = np.random.randn() @@ -205,8 +205,8 @@ def test_gradient_gives_correct_result(self, quadratic): tf2d = Transform2D(quadratic['matrix']) tf2d_grad = tf2d.gradient() - assert tf2d_grad[0](x, y) == quadratic['gradient'][0](x, y) - assert tf2d_grad[1](x, y) == quadratic['gradient'][1](x, y) + assert tf2d_grad[0](x, y) == pytest.approx(quadratic['gradient'][0](x, y)) + assert tf2d_grad[1](x, y) == pytest.approx(quadratic['gradient'][1](x, y)) def test_grid_true_gives_correct_shape(self, tf2d): n_x, n_y = 12, 3 @@ -303,7 +303,7 @@ def __init__(self, n_lam=20, n_eta=3, n_xi=11): hdr["CUNIT1"], hdr["CUNIT2"], hdr["CUNIT3"] = "arcsec", "arcsec", "um" self.cube = fits.ImageHDU( data=np.ones((n_lam, n_eta, n_xi)), header=hdr) - self.meta = {"xi_min": -0.5 * u.arcsec, "xi_max": 0.5 * u.arcsec} + self.meta = {"xi_min": -0.5 * u.arcsec, "xi_max": 0.5 * u.arcsec, "eta_min": -0.5 * u.arcsec, "eta_max": 0.5 * u.arcsec} class TestXiLamImage: @@ -333,3 +333,13 @@ def test_interpolation_is_accurate(self): interps = make_image_interpolations(hdul, kx=1, ky=1) imginterp = interps[0](yy, xx, grid=False) assert np.allclose(imginterp, img) + + def test_works_with_non_square_image(self): + # The axes passed to RectBivariateSpline were transposed, which + # raised ValueError for any image with NAXIS1 != NAXIS2 + xx, yy = np.meshgrid(np.arange(80), np.arange(50)) + img = np.sin(xx/6) * np.cos(yy/7) # shape (50, 80) + hdul = fits.HDUList(fits.ImageHDU(data=img)) + interps = make_image_interpolations(hdul, kx=1, ky=1) + imginterp = interps[0](yy, xx, grid=False) + assert np.allclose(imginterp, img) diff --git a/scopesim/tests/tests_effects/test_Surface.py b/scopesim/tests/tests_effects/test_Surface.py index d1e4823b8..141ffe28e 100644 --- a/scopesim/tests/tests_effects/test_Surface.py +++ b/scopesim/tests/tests_effects/test_Surface.py @@ -1,4 +1,6 @@ +# -*- coding: utf-8 -*- import pytest +from pytest import approx import numpy as np from astropy import units as u @@ -6,7 +8,8 @@ from synphot import SpectralElement, SourceSpectrum from scopesim.effects import TERCurve -from scopesim.optics.surface import SpectralSurface +from scopesim.optics.surface import SpectralSurface, scale_spectrum +from scopesim.source.source_templates import vega_spectrum PLOTS = False @@ -51,3 +54,25 @@ def test_returns_surface_object_for_arrays(self): wave = surf.surface.wavelength plt.plot(wave, surf.surface.emission(wave)) plt.show() + + +@pytest.mark.webtest +@pytest.mark.usefixtures("no_file_error") +class TestScaleSpectrum: + def test_scales_vega_spectrum_to_vega_ab_or_jansky(self): + spec = vega_spectrum() + vega_185 = scale_spectrum(spec, "Ks", -1.85 * u.mag) + ab_0 = scale_spectrum(spec, "Ks", 0 * u.ABmag) + jy_3630 = scale_spectrum(spec, "Ks", 3630 * u.Jy) + + wave = np.linspace(1.8, 2.5, 1000) * u.um + assert vega_185(wave).value == approx(ab_0(wave).value, rel=1e-2) + assert vega_185(wave).value == approx(jy_3630(wave).value, rel=1e-2) + + if PLOTS: + plt.plot(wave, spec(wave), "b") + plt.plot(wave, vega_185(wave), "r:") + plt.plot(wave, ab_0(wave), "g--") + plt.plot(wave, jy_3630(wave), "y-.") + plt.semilogy() + plt.show() diff --git a/scopesim/tests/tests_effects/test_SurfaceList.py b/scopesim/tests/tests_effects/test_SurfaceList.py index 8128dd161..465cffdc7 100644 --- a/scopesim/tests/tests_effects/test_SurfaceList.py +++ b/scopesim/tests/tests_effects/test_SurfaceList.py @@ -1,4 +1,5 @@ -""" Tests for the the new SurfaceList object """ +# -*- coding: utf-8 -*- +"""Tests for the the SurfaceList object.""" import pytest from pytest import approx diff --git a/scopesim/tests/tests_effects/test_TERCurve.py b/scopesim/tests/tests_effects/test_TERCurve.py index 68160654c..331ebe03b 100644 --- a/scopesim/tests/tests_effects/test_TERCurve.py +++ b/scopesim/tests/tests_effects/test_TERCurve.py @@ -1,12 +1,19 @@ +# -*- coding: utf-8 -*- +"""Tests for TERCurve class""" + import pytest import numpy as np +import numpy.testing as npt from matplotlib import pyplot as plt from astropy import units as u +from astropy.wcs import WCS +from astropy.table import Table from scopesim.effects import ter_curves as tc from scopesim.optics.fov import FieldOfView from scopesim.tests.mocks.py_objects.header_objects import _fov_header +from scopesim.optics.fov_volume_list import FovVolumeList from scopesim.tests.mocks.py_objects import source_objects as so from scopesim.tests.mocks.py_objects import effects_objects as eo @@ -22,6 +29,8 @@ def _filter_wheel(mock_path_micado): "filename_format": fname, "current_filter": "Br-gamma"}) +# pylint: disable=missing-class-docstring, +# pylint: disable=missing-function-docstring class TestTERCurveApplyTo: def test_adds_bg_to_source_if_source_has_no_bg(self): @@ -49,6 +58,38 @@ def test_adds_bg_to_source_if_source_has_no_bg(self): plt.semilogy(wave, flux, "r") plt.show() + def test_applies_to_cube_source(self): + src = so._cube_source() + wcs = WCS(src.cube_fields[0].header).spectral + nlam = src.cube_fields[0].data.shape[0] + lam = wcs.all_pix2world(np.arange(nlam), 0)[0] * u.m + + eff = eo._filter_surface() + thru = eff.throughput(lam) + + # Save spectrum before application + orig = src.cube_fields[0].data[:, 25, 25] * 1. + + # Apply effect + src = eff.apply_to(src) + + # Spectrum after application + new = src.cube_fields[0].data[:, 25, 25] + + npt.assert_allclose(new, orig * thru) + + def test_raises_helpful_error_when_nothing_transmits(self): + """A fully opaque surface used to die on a bare IndexError inside the + volume setup. The error must name the effect and the waverange.""" + tbl = Table(data=[[0.5, 1.0, 2.0, 3.0], [0.0, 0.0, 0.0, 0.0]], + names=["wavelength", "transmission"]) + tbl["wavelength"].unit = "um" + eff = tc.TERCurve(table=tbl, wave_min=0.5, wave_max=3.0, + name="closed shutter") + + with pytest.raises(ValueError, match="Did you open the shutter"): + eff.apply_to(FovVolumeList()) + class TestSpectralQuantumEfficiency: def test_is_fov_phase_only(self): @@ -302,9 +343,6 @@ def test_loads_all_filters(self, fwheel): def test_current_filter_is_filter(self, fwheel): assert isinstance(fwheel.current_filter, tc.FilterCurve) - def test_current_filter_has_fov_grid_method(self, fwheel): - assert hasattr(fwheel.current_filter, "fov_grid") - def test_change_to_known_filter(self, fwheel): fwheel.change_filter('Ks') assert fwheel.current_filter.meta["name"] == "Ks" diff --git a/scopesim/tests/tests_effects/test_calibrated_spectral_trace.py b/scopesim/tests/tests_effects/test_calibrated_spectral_trace.py new file mode 100644 index 000000000..e263f9ca6 --- /dev/null +++ b/scopesim/tests/tests_effects/test_calibrated_spectral_trace.py @@ -0,0 +1,147 @@ +from types import SimpleNamespace +import numpy as np +import pytest +from astropy import units as u +from astropy.io import fits +from scopesim.effects.spectral_trace_list_utils import XiLamImage, SpectralTrace +from astropy.table import Table +from scopesim.optics.image_plane_utils import header_from_list_of_xy + + +def test_single_precision_cube_is_integrated_in_double_precision(): + fov = line_cube(.05) + fov.cube.data = np.zeros(fov.cube.data.shape, dtype=np.float32) + fov.cube.data[1000, :, :] = 1. + fov.cube.data[1000, 0, :] = 1e8 + image = XiLamImage(fov, lambda wave: .00001, slit_dispersion=lambda wave: .002) + # Integration across eta must retain the 19 faint cells beside the bright + # cell. The remapped line lies well inside the spectral boundaries. + expected = (1e8 + 19)*.05 + np.testing.assert_allclose(image.image.sum(axis=1), expected, rtol=1e-12) + + +def line_cube(step, offset=0): + wave = np.linspace(.995, 1.005, 2001) + eta = np.arange(-.5+step/2, .5, step) + spatial = np.exp(-.5*((eta-offset)/.12)**2) + profile = np.exp(-.5*((wave-1)/.00002)**2) + cube = profile[:, None, None]*spatial[None, :, None]*np.ones((1, 1, 3)) + header = fits.Header({'CTYPE1': 'LINEAR', 'CTYPE2': 'LINEAR', 'CTYPE3': 'WAVE', 'CUNIT1': 'arcsec', 'CUNIT2': 'arcsec', 'CUNIT3': 'um', 'CDELT1': .1, 'CDELT2': step, 'CDELT3': wave[1]-wave[0], 'CRPIX1': 1., 'CRPIX2': 1., 'CRPIX3': 1., 'CRVAL1': 0., 'CRVAL2': eta[0], 'CRVAL3': wave[0]}) + return SimpleNamespace(cube=fits.ImageHDU(cube, header), meta={'xi_min': 0*u.arcsec, 'eta_min': -.5*u.arcsec, 'eta_max': .5*u.arcsec}) + + + +def test_slit_dispersion_is_independent_of_numerical_sky_grid(): + spectra = [XiLamImage(line_cube(step), lambda w: .00001, slit_dispersion=lambda w: .0002) for step in [.1, .05, .025]] + # Finite spatial quadrature changes, while the physical slit dispersion stays fixed. + for image in spectra[:-1]: + np.testing.assert_allclose(image.image, spectra[-1].image, atol=.001, rtol=.02) + + + +def test_calibrated_slit_shift_has_forward_model_sign(): + image = XiLamImage(line_cube(.025, offset=.1), lambda w: .00001, slit_dispersion=lambda w: .002) + centroid = np.sum(image.image[1]*image.lam)/image.image[1].sum() + assert centroid == pytest.approx(1.0002, abs=2e-6) + + + +def test_optical_lsf_preserves_photons_and_adds_variance(): + image = XiLamImage(line_cube(.05), lambda w: 0., slit_dispersion=lambda w: 0.) + before = np.trapezoid(image.image[1], image.lam) + image.convolve_lsf(10000) + after = np.trapezoid(image.image[1], image.lam) + assert after == pytest.approx(before, rel=.001) + center = np.sum(image.lam*image.image[1])/image.image[1].sum() + variance = np.sum((image.lam-center)**2*image.image[1])/image.image[1].sum() + assert variance == pytest.approx(.00002**2 + (1/10000/np.sqrt(8*np.log(2)))**2, rel=.02) + + +def test_slit_coordinates_follow_cube_wcs(): + fov = line_cube(.025) + fov.cube.header['CRVAL1'] = 2. + fov.cube.header['CRPIX1'] = 2. + fov.cube.header['CRVAL2'] += .1 + image = XiLamImage(fov, lambda wave: .00001, slit_dispersion=lambda wave: .002) + np.testing.assert_allclose(image.xi, [1.9, 2., 2.1]) + centroid = np.sum(image.image[1] * image.lam) / image.image[1].sum() + assert centroid == pytest.approx(1.0002, abs=2e-6) + + +def test_translating_an_ifu_slice_preserves_its_local_spectrum(): + fov = line_cube(.025) + centered = XiLamImage(fov, lambda wave: .00001, slit_dispersion=lambda wave: .002) + fov.cube.header['CRVAL2'] += 2. + fov.meta['eta_min'] += 2*u.arcsec + fov.meta['eta_max'] += 2*u.arcsec + translated = XiLamImage(fov, lambda wave: .00001, slit_dispersion=lambda wave: .002) + np.testing.assert_allclose(translated.image, centered.image, atol=1e-7) + + +def test_missing_aperture_geometry_fails(): + fov = line_cube(.025) + del fov.meta['eta_min'] + with pytest.raises(KeyError, match='eta_min'): + XiLamImage(fov, lambda wave: .00001, slit_dispersion=lambda wave: .002) + + +def test_detector_jacobian_uses_physical_pixel_area(): + header = fits.Header({'CTYPE1D': 'LINEAR', 'CTYPE2D': 'LINEAR', 'CUNIT1D': 'm', 'CUNIT2D': 'm', 'CDELT1D': .000015, 'CDELT2D': .000030, 'CRPIX1D': 1., 'CRPIX2D': 1., 'CRVAL1D': 0., 'CRVAL2D': 0.}) + assert SpectralTrace._detector_pixel_area(header) == pytest.approx(.015*.030) + + +def test_rectified_interpolation_reproduces_a_linear_density_field(): + fov = line_cube(.1) + wave = .995 + np.arange(2001)*fov.cube.header['CDELT3'] + xi = np.arange(3)*.1 + fov.cube.data[:] = 2 + 3*xi[None, None, :] + 5*wave[:, None, None] + image = XiLamImage(fov, lambda w: 0., slit_dispersion=lambda w: 0.) + positions, wavelengths = np.array([.03, .17]), np.array([.998, 1.003]) + expected = 2 + 3*positions + 5*wavelengths + np.testing.assert_allclose(image.interp(positions, wavelengths, grid=False), expected, rtol=2e-6) + + +@pytest.mark.parametrize('change', ['wavelength_origin', 'wavelength_step', 'wavelength_pc', 'slit_origin', 'slit_step', 'slit_pc', 'slice_center', 'dispersion', 'source', 'units']) +def test_reused_slit_operator_matches_fresh_after_direct_changes(change): + fov = line_cube(.1) + cache = {} + slope = [.002] + dispersion = lambda wave: slope[0]*wave + XiLamImage(fov, lambda wave: .00001, slit_dispersion=dispersion, _cache=cache) + if change == 'wavelength_origin': fov.cube.header['CRVAL3'] += .001 + if change == 'wavelength_step': fov.cube.header['CDELT3'] *= 1.1 + if change == 'wavelength_pc': fov.cube.header['PC3_3'] = 1.1 + if change == 'slit_origin': fov.cube.header['CRVAL2'] += .04 + if change == 'slit_step': fov.cube.header['CDELT2'] *= 1.1 + if change == 'slit_pc': fov.cube.header['PC2_2'] = 1.1 + if change == 'slice_center': fov.meta['eta_min'] += .04*u.arcsec + if change == 'dispersion': slope[0] *= 1.3 + if change == 'source': fov.cube.data *= 2. + if change == 'units': + fov.cube.header['CUNIT2'], fov.cube.header['CUNIT3'] = 'deg', 'nm' + for keyword in ('CDELT2', 'CRVAL2'): fov.cube.header[keyword] /= 3600 + for keyword in ('CDELT3', 'CRVAL3'): fov.cube.header[keyword] *= 1000 + reused = XiLamImage(fov, lambda wave: .00001, slit_dispersion=dispersion, _cache=cache) + fresh = XiLamImage(fov, lambda wave: .00001, slit_dispersion=dispersion) + np.testing.assert_array_equal(reused.image, fresh.image) + + +@pytest.mark.parametrize('coordinate', ['xy2lam', 'xy2xi']) +def test_direct_optical_transform_edits_reach_next_projection(coordinate): + xi, wave = np.meshgrid(np.linspace(-1., 1., 5), np.linspace(.99, 1.01, 9)) + table = Table({'wavelength': wave.ravel()*u.um, 's': xi.ravel()*u.arcsec, 'x': (wave.ravel()-1)*1000*u.mm, 'y': xi.ravel()*.15*u.mm}) + reused, fresh = [SpectralTrace(table.copy(), trace_flux_jacobian=True) for _ in range(2)] + fov = line_cube(.1) + fov.cube.data *= np.array([1., 2., 4.])[None, None, :] + fov.header = fov.cube.header + fov.detector_header = header_from_list_of_xy([-10., 10.], [-2., 2.], .015, 'D') + fov.trace_id = 'linear' + fov.meta.update(wave_min=.995*u.um, wave_max=1.005*u.um, xi_min=-.05*u.arcsec, xi_max=.25*u.arcsec) + before = reused.map_spectra_to_focal_plane(fov).data.copy() + for trace in (reused, fresh): + if coordinate == 'xy2lam': trace.xy2lam.matrix[0, 0] += .0002 + if coordinate == 'xy2xi': trace.xy2xi.matrix[0, 0] += .04 + actual = reused.map_spectra_to_focal_plane(fov).data + expected = fresh.map_spectra_to_focal_plane(fov).data + np.testing.assert_array_equal(actual, expected) + assert np.max(abs(actual-before)) > .001*before.max() diff --git a/scopesim/tests/tests_effects/test_detector_arm_configuration.py b/scopesim/tests/tests_effects/test_detector_arm_configuration.py new file mode 100644 index 000000000..2b81134c4 --- /dev/null +++ b/scopesim/tests/tests_effects/test_detector_arm_configuration.py @@ -0,0 +1,53 @@ +import numpy as np +import pytest +from scopesim.detector import Detector +from scopesim.optics.image_plane_utils import header_from_list_of_xy +from astropy.io import fits +from scopesim.commands import UserCommands +from scopesim.effects import DetectorList, ExtraFitsKeywords, DetectorModePropertiesSetter +from scopesim.effects.electronic import ADConversion + + +def detector(nx=120, ny=80): + header = header_from_list_of_xy([-.015*nx/2, .015*nx/2], [-.015*ny/2, .015*ny/2], .015, 'D') + return Detector(header) + + + +def test_gain_uses_selected_arm_and_missing_gain_fails(): + det = detector() + det.data[:] = 12 + ADConversion(gain='!DET.gain_uvb', dtype='float32', cmds={'!DET.gain_uvb': .6}).apply_to(det) + np.testing.assert_allclose(det.data, 20) + with pytest.raises(ValueError, match='gain_uvb'): + ADConversion(gain='!DET.gain_uvb', dtype='float32', cmds={}).apply_to(det) + + + +def test_detector_specific_primary_headers(): + eff = ExtraFitsKeywords(header_dict=[{'detector_id': 1, 'ext_number': [0, 1], 'keywords': {'ARM': 'VIS'}}]) + for detector_id in [0, 1, 2]: + hdul = fits.HDUList([fits.PrimaryHDU(), fits.ImageHDU(header=fits.Header({'DET_ID': detector_id}))]) + eff.apply_to(hdul) + assert ('ARM' in hdul[0].header) == (detector_id == 1) + + + +def test_detector_layout_sets_real_id_header(): + effect = DetectorList(array_dict={'id': [2], 'x_cen': [0.], 'y_cen': [0.], 'x_size': [16], 'y_size': [8], 'pixel_size': [.018], 'angle': [0.], 'gain': [2.29]}, x_cen_unit='mm', y_cen_unit='mm', x_size_unit='pixel', y_size_unit='pixel', pixel_size_unit='mm', angle_unit='deg', gain_unit='electron/adu') + header = effect.detector_headers()[0] + assert header['DET_ID'] == 2 + assert header['EXTNAME'] == 'DET_2' + det = Detector(header) + assert det.meta['ID'] == 2 + assert det.hdu.header['DET_ID'] == 2 + + + +def test_independent_detector_readout_modes(): + cmds = UserCommands(properties={'!DET.mode_blue': 'slow', '!DET.mode_red': 'fast'}) + for arm in ['blue', 'red']: + modes = {'slow': {f'!DET.gain_{arm}': .6}, 'fast': {f'!DET.gain_{arm}': 1.8}} + DetectorModePropertiesSetter(mode_properties=modes, mode_key=f'!DET.mode_{arm}', cmds=cmds).apply_to(None) + assert cmds['!DET.gain_blue'] == .6 + assert cmds['!DET.gain_red'] == 1.8 diff --git a/scopesim/tests/tests_effects/test_detector_operating_points.py b/scopesim/tests/tests_effects/test_detector_operating_points.py new file mode 100644 index 000000000..1763f1d26 --- /dev/null +++ b/scopesim/tests/tests_effects/test_detector_operating_points.py @@ -0,0 +1,85 @@ +import warnings + +import numpy as np +import pytest + +from scopesim import UserCommands +from scopesim.detector import Detector +from scopesim.effects.electronic import AvalancheShotNoise, DarkCurrent, ExposureIntegration, ExposureLimit, OnChipBinning, BasicReadoutNoise +from scopesim.optics.image_plane_utils import header_from_list_of_xy + + +@pytest.mark.parametrize('factor', [1., 1.2]) +@pytest.mark.parametrize('photons,dark', [(0., 4.), (.2, .01), (50., 3.)]) +def test_avalanche_preserves_physical_mean_and_only_amplifies_photon_variance(factor, photons, dark): + cmds = UserCommands(properties={'!SIM.random.seed': 7143}) + effect = AvalancheShotNoise(excess_noise_factor=factor, dark_current=dark, dit=1., ndit=1, cmds=cmds) + data = effect(np.full(400000, photons + dark), dark_charge=dark, det_id=2) + mean, variance = photons + dark, factor * photons + dark + assert data.mean() == pytest.approx(mean, abs=5*np.sqrt(variance/data.size)) + assert data.var() == pytest.approx(variance, rel=.025) + + +def test_independent_integrations_and_read_noise_after_avalanche(): + cmds = UserCommands(properties={'!SIM.random.seed': 736}) + count, photons, dark, ron, factor = 12, 7., .4, .5, 1.2 + effect = AvalancheShotNoise(excess_noise_factor=factor, dark_current=dark, dit=1., ndit=count, cmds=cmds) + data = effect(np.full(400000, count*(photons+dark)), dark_charge=count*dark, det_id=3) + data = BasicReadoutNoise(noise_std=ron, ndit=count, cmds=cmds)(data, ndit=count, det_id=3) + assert data.mean() == pytest.approx(count*(photons+dark), abs=.1) + assert data.var() == pytest.approx(count*(factor*photons + dark + ron**2), rel=.02) + cmds['!OBS.roid'] = 2 + other = effect(np.full(400000, count*(photons+dark)), dark_charge=count*dark, det_id=3) + assert not np.array_equal(data, other) + + +def test_exposure_limit_warns_after_charge_binning_without_changing_counts_or_commands(): + cmds = UserCommands(properties={'!OBS.dit': 10., '!OBS.ndit': 4}) + header = header_from_list_of_xy([-2, 2], [-2, 2], 1., 'D') + header['ID'] = 0 + detector = Detector(header, cmds=cmds) + detector.data[:] = 400. + limit = ExposureLimit(usable_well=15000., ndit='!OBS.ndit', cmds=cmds) + ExposureIntegration(dit='!OBS.dit', ndit='!OBS.ndit', cmds=cmds).apply_to(detector) + DarkCurrent(value=0., dit='!OBS.dit', ndit='!OBS.ndit', cmds=cmds).apply_to(detector) + with warnings.catch_warnings(record=True) as caught: + limit.apply_to(detector) + assert not caught # Summed 16000 e is only 4000 e per integration. + OnChipBinning(binx=2, biny=2).apply_to(detector) + before = detector.data.copy() + with pytest.warns(UserWarning, match='peak 16000.0 e.*DIT/NDIT are unchanged'): + limit.apply_to(detector) + np.testing.assert_array_equal(detector.data, before) + assert detector.header['PEAKDIT'] == 16000 + assert cmds['!OBS.dit'] == 10 and cmds['!OBS.ndit'] == 4 + + + +def test_avalanche_supports_unseeded_readouts(): + cmds = UserCommands(properties={'!SIM.random.seed': None}) + effect = AvalancheShotNoise(excess_noise_factor=1.2, dark_current=0., dit=1., ndit=1, cmds=cmds) + first = effect(np.full(1000, 10.), det_id=3) + second = effect(np.full(1000, 10.), det_id=3) + assert not np.array_equal(first, second) + + +def test_detector_ids_produce_independent_read_noise(): + cmds = UserCommands(properties={'!SIM.random.seed': 7143}) + effect = BasicReadoutNoise(noise_std=2., ndit=1, cmds=cmds) + header = header_from_list_of_xy([-50, 50], [-50, 50], 1., 'D') + header['ID'] = 1 + first = Detector(header.copy(), cmds=cmds) + header['ID'] = 2 + second = Detector(header.copy(), cmds=cmds) + effect.apply_to(first) + effect.apply_to(second) + assert abs(np.corrcoef(first.data.ravel(), second.data.ravel())[0, 1]) < .05 + + +def test_selector_detector_properties_keep_exposure_references_live(): + from scopesim.effects import SelectorWheel + from scopesim.utils import from_currsys + cmds = UserCommands(properties={'!DET.noise_class': 'BasicReadoutNoise', '!DET.noise_kwargs': {'noise_std': 2., 'ndit': '!OBS.ndit'}, '!OBS.ndit': 1}) + effect = SelectorWheel(selector_key='detector_id', wheel=[{'selector_value': 0, 'effect_class': '!DET.noise_class', 'effect_kwargs': '!DET.noise_kwargs'}], cmds=cmds) + cmds['!OBS.ndit'] = 7 + assert from_currsys(effect.get_effect(0).meta['ndit'], cmds) == 7 diff --git a/scopesim/tests/tests_effects/test_effect_utils.py b/scopesim/tests/tests_effects/test_effect_utils.py index 6a2779116..8e2611cea 100644 --- a/scopesim/tests/tests_effects/test_effect_utils.py +++ b/scopesim/tests/tests_effects/test_effect_utils.py @@ -1,10 +1,10 @@ +# -*- coding: utf-8 -*- + import pytest from scopesim import effects as efs -from scopesim.effects import effects_utils as eu, GaussianDiffractionPSF, \ - SurfaceList -from scopesim.tests.mocks.py_objects.effects_objects import _surf_list, \ - _surf_list_empty, _filter_surface +from scopesim.effects import effects_utils as eu, GaussianDiffractionPSF + from scopesim.tests.mocks.py_objects.yaml_objects import _atmo_yaml_dict @@ -13,21 +13,6 @@ def atmo_yaml_dict(): return _atmo_yaml_dict() -@pytest.fixture(scope="function") -def surf_list(): - return _surf_list() - - -@pytest.fixture(scope="function") -def surf_list_empty(): - return _surf_list_empty() - - -@pytest.fixture(scope="function") -def filter_surface(): - return _filter_surface() - - class TestMakeEffect: def test_it_creates_an_effects_object(self, atmo_yaml_dict): effdic = atmo_yaml_dict["effects"][0] diff --git a/scopesim/tests/tests_effects/test_exposure.py b/scopesim/tests/tests_effects/test_exposure.py index 18a1f091d..e3203c353 100644 --- a/scopesim/tests/tests_effects/test_exposure.py +++ b/scopesim/tests/tests_effects/test_exposure.py @@ -39,7 +39,7 @@ def fixture_exposureoutput(): @pytest.fixture(name="detector", scope="function") def fixture_detector(): det = Detector(_image_hdu_square().header) - det._hdu.data[:] = 1.e5 + det.data[:] = 1.e5 return det class TestExposureIntegration: @@ -47,9 +47,9 @@ def test_initialises_correctly(self, exposureintegration): assert isinstance(exposureintegration, ExposureIntegration) def test_integrates_correctly(self, exposureintegration, detector): - orig = 1. * detector._hdu.data + orig = 1. * detector.data assert isinstance(exposureintegration.apply_to(detector), Detector) - assert np.allclose(detector._hdu.data, 4 * orig) + assert np.allclose(detector.data, 4 * orig) def test_fails_without_dit(self): with pytest.raises(ValueError): @@ -91,17 +91,17 @@ def test_cannot_set_to_unknown_mode(self, exposureoutput): (2., 5), (3, 36)]) def test_applies_average(self, dit, ndit, detector): - det_mean = detector._hdu.data.mean() + det_mean = detector.data.mean() exposureoutput = ExposureOutput("average", dit=dit, ndit=ndit) result = exposureoutput.apply_to(detector) - assert np.isclose(result._hdu.data.mean(), det_mean / ndit) + assert np.isclose(result.data.mean(), det_mean / ndit) @pytest.mark.parametrize("dit, ndit", [(1., 1), (2., 5), (3, 36)]) def test_applies_sum(self, dit, ndit, detector): - det_mean = detector._hdu.data.mean() + det_mean = detector.data.mean() exposureoutput = ExposureOutput("sum", dit=dit, ndit=ndit) result = exposureoutput.apply_to(detector) - assert np.isclose(result._hdu.data.mean(), det_mean) + assert np.isclose(result.data.mean(), det_mean) diff --git a/scopesim/tests/tests_effects/test_finite_spectral_cells.py b/scopesim/tests/tests_effects/test_finite_spectral_cells.py new file mode 100644 index 000000000..4f74407bc --- /dev/null +++ b/scopesim/tests/tests_effects/test_finite_spectral_cells.py @@ -0,0 +1,36 @@ +"""Photon integrals and line moments across finite slit and wavelength cells.""" +import numpy as np +import pytest +from scopesim.effects.spectral_trace_list_utils import _slit_cell_operator + + +@pytest.mark.parametrize('phase', [0., .25, .5, .75]) +@pytest.mark.parametrize('slope', [0., .02]) +def test_slit_remap_preserves_photons_and_illumination_centroid(phase, slope): + wave = np.linspace(.99, 1.01, 2001) + derivative = -.001 + slope * (wave - 1.) + step = wave[1] - wave[0] + eta, deta = .0825 + phase*step/abs(derivative[1000]), .03 + photons = np.zeros(len(wave)) + photons[1000] = 17. + output = _slit_cell_operator(wave, derivative, eta, deta) @ photons + assert output.sum() == pytest.approx(17., rel=1e-12) + center = np.sum(output * wave) / output.sum() + assert abs(center - (1. + eta*derivative[1000])) < .01*step + + +def test_zero_slit_dispersion_is_identity(): + wave = np.linspace(1., 1.1, 101) + photons = np.arange(len(wave)) + 1. + actual = _slit_cell_operator(wave, np.zeros_like(wave), .1, .1) @ photons + np.testing.assert_allclose(actual, photons, rtol=1e-12) + + +def test_finite_slit_cell_width_has_uniform_variance(): + wave = np.linspace(.99, 1.01, 10001) + derivative = np.full_like(wave, -.001) + photons = np.zeros(len(wave)) + photons[5000] = 1. + output = _slit_cell_operator(wave, derivative, 0., .1) @ photons + variance = np.sum(output * (wave - 1.)**2) + assert variance == pytest.approx((.001*.1)**2 / 12, rel=.002) diff --git a/scopesim/tests/tests_effects/test_imaging_branches.py b/scopesim/tests/tests_effects/test_imaging_branches.py new file mode 100644 index 000000000..ce308f392 --- /dev/null +++ b/scopesim/tests/tests_effects/test_imaging_branches.py @@ -0,0 +1,115 @@ +"""Branch throughput must be applied before integrating an imaging spectrum.""" + +from types import SimpleNamespace + +import numpy as np +import pytest + +from scopesim.commands import UserCommands +from scopesim.effects import DichroicTree, SelectorWheel, TERCurve +from scopesim.optics.image_plane import ImagePlane +from scopesim.optics.optical_train import OpticalTrain, extract_source +from scopesim.source.source import Source +from scopesim.tests.mocks.py_objects.fov_objects import _centre_fov +from scopesim.tests.mocks.py_objects.source_objects import _table_source + + +@pytest.fixture +def tree(tmp_path): + (tmp_path / 'tree.dat').write_text('image_plane_id first second\n0 R X\n1 T R\n2 T T\n') + for name, transmission in [('first', 0.75), ('second', 0.6)]: + curve = '# wavelength_unit: um\nwavelength transmission reflection emissivity\n' + curve += f'0.5 {transmission} {1 - transmission} 0\n2.5 {transmission} {1 - transmission} 0\n' + (tmp_path / f'{name}.dat').write_text(curve) + return DichroicTree(name='tree', tree_filename=str(tmp_path / 'tree.dat'), dichroic_names=['first', 'second'], filename_format=str(tmp_path / '{}.dat')) + + +@pytest.fixture +def selector(): + wheel = [] + for arm, transmission in enumerate([0.25, 0.75]): + curve = {'wavelength': [0.5, 2.5], 'transmission': [transmission, transmission], 'emissivity': [0, 0]} + wheel.append({'selector_value': arm, 'effect_class': 'QuantumEfficiencyCurve', 'effect_kwargs': {'array_dict': curve, 'wavelength_unit': 'um'}}) + return SelectorWheel(selector_key='image_plane_id', wheel=wheel, allowed_missing_selector_values=[2]) + + +@pytest.fixture +def source(): + source = _table_source() + background = TERCurve(array_dict={'wavelength': [0.5, 2.5], 'transmission': [1, 1], 'emission': [1e-8, 1e-8]}, wavelength_unit='um', emission_unit='photlam') + source.append(background.background_source[0]) + return source + + +@pytest.mark.parametrize('effect_name, transmissions', [('tree', [0.25, 0.3, 0.45]), ('selector', [0.25, 0.75, 1.0])]) +def test_branch_attenuates_coloured_sources_and_background_independently(request, effect_name, transmissions, source): + effect = request.getfixturevalue(effect_name) + reference = extract_source(_centre_fov(n=10), source).make_hdu().data + for arm, transmission in enumerate(transmissions): + fov = _centre_fov(n=10) + fov.meta['image_plane_id'] = arm + extract_source(fov, source, [effect]) + np.testing.assert_allclose(fov.make_hdu().data, transmission * reference) + # Rendering each branch must leave the shared source, including its sky, intact. + np.testing.assert_array_equal(extract_source(_centre_fov(n=10), source).make_hdu().data, reference) + + +@pytest.mark.parametrize('effect_name', ['tree', 'selector']) +def test_source_without_fov_context_is_unchanged(request, effect_name, source): + effect = request.getfixturevalue(effect_name) + reference = extract_source(_centre_fov(n=10), source).make_hdu().data + effect.apply_to(source) + np.testing.assert_array_equal(extract_source(_centre_fov(n=10), source).make_hdu().data, reference) + + +@pytest.mark.parametrize('effect_name', ['tree', 'selector']) +def test_missing_branch_id_fails(request, effect_name, source): + effect = request.getfixturevalue(effect_name) + fov = _centre_fov(n=10) + fov.meta.clear() + with pytest.raises(KeyError, match='image_plane_id'): + effect.apply_to(source, fov=fov) + + +def test_observe_keeps_downstream_emission_after_branch_split(tree, source): + common = TERCurve(array_dict={'wavelength': [0.5, 2.5], 'transmission': [0.8, 0.8], 'emissivity': [0, 0]}, wavelength_unit='um') + downstream = TERCurve(array_dict={'wavelength': [0.5, 2.5], 'transmission': [0.5, 0.5], 'emission': [2e-8, 2e-8]}, wavelength_unit='um', emission_unit='photlam') + train = OpticalTrain() + train.cmds = UserCommands() + fovs = [_centre_fov(n=10) for arm in range(3)] + for arm, fov in enumerate(fovs): + fov.meta['image_plane_id'] = arm + train.fov_manager = SimpleNamespace(fovs=fovs, is_spectroscope=False) + train.optics_manager = SimpleNamespace(source_effects=[common, tree, downstream], fov_effects=[], image_plane_effects=[]) + train.image_planes = [ImagePlane(fov.header) for fov in fovs] + + prepared = train.prepare_source(source.make_copy()) + reference = extract_source(_centre_fov(n=10), prepared).make_hdu().data + emission = extract_source(_centre_fov(n=10), downstream.background_source[0]).make_hdu().data + train.observe(source, update=False) + for arm, transmission in enumerate([0.25, 0.3, 0.45]): + expected = reference * 0.8 * transmission * 0.5 + emission + np.testing.assert_allclose(train._last_fovs[arm].hdu.data, expected) + assert train._last_fovs[arm].image_plane_id == arm + + +@pytest.mark.parametrize('spectrum_id, expected', [(0, 0.5), (1, 0.5 + 1 / 24), (2, 0.5 - 1 / 24)]) +def test_wavelength_dependent_branch_preserves_source_colour(spectrum_id, expected): + spectrum = _table_source().fields[0].spectra[spectrum_id] + source = Source(x=[0], y=[0], ref=[0], weight=[1], spectra=[spectrum]) + curve = {'wavelength': [0.5, 2.5], 'transmission': [0, 1], 'emissivity': [0, 0]} + selector = SelectorWheel(selector_key='image_plane_id', wheel=[{'selector_value': 0, 'effect_class': 'QuantumEfficiencyCurve', 'effect_kwargs': {'array_dict': curve, 'wavelength_unit': 'um'}}]) + reference = extract_source(_centre_fov(n=10), source).make_hdu().data.sum() + fov = _centre_fov(n=10) + fov.meta['image_plane_id'] = 0 + measured = extract_source(fov, source, [selector]).make_hdu().data.sum() / reference + # Linear spectra and throughput give these analytic integrals over 1--2 um. + # The input spectra are sampled every 0.02 um. + np.testing.assert_allclose(measured, expected, rtol=0, atol=1e-4) + + +@pytest.mark.parametrize('z_order, expected', [([117, 217, 517], (117, 217, 517)), (517, (517,))]) +def test_dichroic_source_stage_is_opt_in(tree, z_order, expected): + assert tree.z_order == (117, 217, 617) + configured = DichroicTree(z_order=z_order, **{key: tree.meta[key] for key in tree.required_keys}) + assert configured.z_order == expected diff --git a/scopesim/tests/tests_effects/test_local_skycalc.py b/scopesim/tests/tests_effects/test_local_skycalc.py new file mode 100644 index 000000000..8baab5ba1 --- /dev/null +++ b/scopesim/tests/tests_effects/test_local_skycalc.py @@ -0,0 +1,24 @@ +import numpy as np +import pytest +from astropy import units as u +from astropy.table import Table +from scopesim.effects import SkycalcTERCurve + + +@pytest.mark.parametrize('wave_unit', [u.nm, u.um]) +def test_local_skycalc_respects_wavelength_unit(tmp_path, wave_unit): + wave = np.array([.5, 1., 2.]) * u.um + table = Table([wave.to(wave_unit), [.5, .8, .9], [100., 200., 300.]], names=['lam', 'trans', 'flux']) + path = tmp_path / 'sky.fits' + table.write(path) + sky = SkycalcTERCurve(use_local_skycalc_file=str(path)) + np.testing.assert_allclose(sky.skycalc_table['wavelength'], wave.value) + assert sky.surface.transmission(1*u.um).value == pytest.approx(.8) + + + +def test_skycalc_missing_wavelength_unit_fails(tmp_path): + path = tmp_path / 'sky.fits' + Table([[500., 1000., 2000.], [.5, .8, .9], [100., 200., 300.]], names=['lam', 'trans', 'flux']).write(path) + with pytest.raises(TypeError, match='Unit'): + SkycalcTERCurve(use_local_skycalc_file=str(path)) diff --git a/scopesim/tests/tests_effects/test_mosaic_trace_list.py b/scopesim/tests/tests_effects/test_mosaic_trace_list.py index da3207e76..f5bef8e8d 100644 --- a/scopesim/tests/tests_effects/test_mosaic_trace_list.py +++ b/scopesim/tests/tests_effects/test_mosaic_trace_list.py @@ -12,6 +12,7 @@ MosaicSpectralTraceList, MosaicSpectralTrace, MosaicOutputFormat) +from scopesim.effects.electronic.exposure import ExposureIntegration from scopesim.optics.image_plane_utils import header_from_list_of_xy from scopesim.detector import Detector @@ -104,7 +105,7 @@ def fixture_detector(): xsize, ysize = 50, 4096 hdr = header_from_list_of_xy([-xsize/2, xsize/2], [-ysize/2, ysize/2], 1, "D") dtcr = Detector(hdr) - dtcr._hdu.data[rows, :] = 1 + dtcr.data[rows, :] = 1 return dtcr @pytest.fixture(name="apply_collapse1d", scope="class") @@ -159,7 +160,7 @@ def test_collapse1d_detector_has_bintable(self, apply_collapse1d): assert isinstance(apply_collapse1d._hdu, fits.BinTableHDU) def test_collapse1d_gives_correct_result(self, apply_collapse1d): - assert np.all(apply_collapse1d._hdu.data['spectrum'] == 7) + assert np.all(apply_collapse1d.data['spectrum'] == 7) @pytest.mark.usefixtures("patch_mock_path_mosaic") @@ -181,7 +182,7 @@ def test_converttotable_detector_has_bintable(self, apply_converttotable): assert isinstance(apply_converttotable._hdu, fits.BinTableHDU) def test_converttotable_gives_correct_result(self, apply_converttotable): - for row in apply_converttotable._hdu.data: + for row in apply_converttotable.data: assert np.all(row['spectrum'] == 1) @pytest.mark.usefixtures("patch_mock_path_mosaic") @@ -201,3 +202,25 @@ def test_image_returns_detector(self, apply_image): def test_image_detector_has_image(self, apply_image): assert isinstance(apply_image._hdu, fits.ImageHDU) + + +@pytest.mark.usefixtures("patch_mock_path_mosaic") +@pytest.mark.parametrize("output_format", ["collapse1d", "table", "image"]) +def test_output_format_preserves_accumulated_counts_and_exposure(output_format): + header = header_from_list_of_xy([-25, 25], [-2048, 2048], 1, "D") + detector = Detector(header) + rows = [2023, 2031, 2039, 2047, 2055, 2063, 2071] + detector.data[rows, :] = 1 + rate_sum = detector.data.sum() + ExposureIntegration(dit=17., ndit=3).apply_to(detector) + output = MosaicOutputFormat(filename="TRACE_MOS-LR-J.fits", format=output_format).apply_to(detector) + assert output.header["DIT"] == 17. + assert output.header["NDIT"] == 3 + assert output.header["EXPTIME"] == 51. + if output_format == "image": + counts = output.data.sum() + assert "CTYPE1D" in output.header + else: + counts = output.data["spectrum"].sum() + assert "CTYPE1D" not in output.header + assert counts == pytest.approx(rate_sum * 51.) diff --git a/scopesim/tests/tests_effects/test_prnu.py b/scopesim/tests/tests_effects/test_prnu.py index 7d9b4dfb1..fb671ca96 100644 --- a/scopesim/tests/tests_effects/test_prnu.py +++ b/scopesim/tests/tests_effects/test_prnu.py @@ -12,26 +12,44 @@ def make_detector(value=1000, size=10): hdr = header_from_list_of_xy([-size / 2, size / 2], [-size / 2, size / 2], 1, "D") dtcr = Detector(hdr) - dtcr._hdu.data[:] = value + dtcr.data[:] = value return dtcr -def test_output_std_matches_prnu_std(): - prnu_std = 0.05 - dtcr = make_detector(value=1000, size=100) - PixelResponseNonUniformity(prnu_std=prnu_std, prnu_seed=42).apply_to(dtcr) - rel_std = dtcr._hdu.data.std() / dtcr._hdu.data.mean() - assert abs(rel_std - prnu_std) < 0.01 - - -def test_gain_map_is_reused(): - dtcr1 = make_detector() - dtcr2 = make_detector() - prnu = PixelResponseNonUniformity(prnu_std=0.01, prnu_seed=42) - prnu.apply_to(dtcr1) - prnu.apply_to(dtcr2) - np.testing.assert_array_equal(dtcr1._hdu.data, dtcr2._hdu.data) - +class TestApplyTo: + def test_output_std_matches_prnu_std(self): + """Relative std of output should be close to prnu_std.""" + prnu_std = 0.05 + dtcr = make_detector(value=1000, size=100) + PixelResponseNonUniformity(prnu_std=prnu_std, prnu_seed=42).apply_to(dtcr) + rel_std = dtcr.data.std() / dtcr.data.mean() + assert abs(rel_std - prnu_std) < 0.01 + + def test_gain_map_is_reused(self): + """Same map should be applied on repeated calls (fixed pattern).""" + dtcr1 = make_detector() + dtcr2 = make_detector() + prnu = PixelResponseNonUniformity(prnu_std=0.01, prnu_seed=42) + prnu.apply_to(dtcr1) + prnu.apply_to(dtcr2) + np.testing.assert_array_equal(dtcr1.data, dtcr2.data) + + def test_dict_prnu_std(self): + """Dict mode: different amplitude per detector ID.""" + hdr = header_from_list_of_xy([-5, 5], [-5, 5], 1, "D") + dtcr = Detector(hdr) + dtcr.meta["id"] = 0 + dtcr.data[:] = 1000 + prnu = PixelResponseNonUniformity( + prnu_std={0: 0.005, 1: 0.020}, prnu_seed=42) + prnu.apply_to(dtcr) + assert dtcr.data.std() > 0 + + def test_multiplicative_zero_signal(self): + """Zero signal should remain zero — confirms multiplicative (not additive).""" + dtcr = make_detector(value=0) + PixelResponseNonUniformity(prnu_std=0.01, prnu_seed=42).apply_to(dtcr) + assert dtcr.data.sum() == 0 def test_dict_prnu_std(): hdr = header_from_list_of_xy([-5, 5], [-5, 5], 1, "D") diff --git a/scopesim/tests/tests_effects/test_registered_background.py b/scopesim/tests/tests_effects/test_registered_background.py new file mode 100644 index 000000000..54606b9a5 --- /dev/null +++ b/scopesim/tests/tests_effects/test_registered_background.py @@ -0,0 +1,32 @@ +import numpy as np +import pytest +from scopesim.detector import Detector +from scopesim.optics.image_plane_utils import header_from_list_of_xy +from astropy.io import fits +from scopesim.effects import ImagePlaneBackground +from scopesim.optics.image_plane import ImagePlane + + +def detector(nx=120, ny=80): + header = header_from_list_of_xy([-.015*nx/2, .015*nx/2], [-.015*ny/2, .015*ny/2], .015, 'D') + return Detector(header) + + + +def test_background_uses_current_named_calibration(tmp_path): + path = tmp_path / 'background.fits' + fits.HDUList([fits.PrimaryHDU(), fits.ImageHDU(np.full((80, 120), .15), name='0.9'), fits.ImageHDU(np.full((80, 120), .04), name='0.9JH')]).writeto(path) + cmds = {'!INST.slit': '0.9'} + effect = ImagePlaneBackground(filename=str(path), extname='!INST.slit', cmds=cmds) + plane = ImagePlane(detector().header) + effect.apply_to(plane) + np.testing.assert_allclose(plane.data, .15) + cmds['!INST.slit'] = '0.9JH' + plane.data[:] = 0 + effect.apply_to(plane) + np.testing.assert_allclose(plane.data, .04) + with pytest.raises(ValueError, match='shape'): + effect.apply_to(ImagePlane(detector(60, 40).header)) + cmds['!INST.slit'] = 'missing' + with pytest.raises(KeyError): + effect.apply_to(plane) diff --git a/scopesim/tests/tests_effects/test_shift_planes.py b/scopesim/tests/tests_effects/test_shift_planes.py new file mode 100644 index 000000000..5f4bcabe1 --- /dev/null +++ b/scopesim/tests/tests_effects/test_shift_planes.py @@ -0,0 +1,14 @@ +import numpy as np +from scipy.ndimage import map_coordinates +from scopesim.effects.atmo_dispersion import ShiftFoV3D + + +def test_dispersion_does_not_mix_wavelengths(): + data = np.zeros((19, 15, 17)) + data[:, 7, 8] = np.arange(19) + dx, dy = np.linspace(-.2, .2, 19), np.linspace(.4, .6, 19) + shifted = ShiftFoV3D.map_coords_shift(data, dx, dy) + yy, xx = np.indices(data.shape[1:]) + for wave in range(19): + expected = map_coordinates(data[wave], [yy-dy[wave], xx-dx[wave]], order=3, mode='nearest') + np.testing.assert_allclose(shifted[wave], expected, atol=2e-5) diff --git a/scopesim/tests/tests_effects/test_shot_noise.py b/scopesim/tests/tests_effects/test_shot_noise.py index ccdca344f..c7cde5b55 100644 --- a/scopesim/tests/tests_effects/test_shot_noise.py +++ b/scopesim/tests/tests_effects/test_shot_noise.py @@ -21,10 +21,10 @@ def test_applies_dark_current_with_level_of_dit(self): value_too_high = 1e20 hdr = header_from_list_of_xy([-hw, hw], [-hw, hw], 1, "D") dtcr = Detector(hdr) - dtcr._hdu.data[0][0] = value_normal - dtcr._hdu.data[0][1] = -1 - dtcr._hdu.data[1][0] = value_too_high - dtcr._hdu.data[1][1] = np.nan + dtcr.data[0][0] = value_normal + dtcr.data[0][1] = -1 + dtcr.data[1][0] = value_too_high + dtcr.data[1][1] = np.nan sn = ShotNoise() # To prevent that data[0][0] does not change by chance. @@ -33,10 +33,10 @@ def test_applies_dark_current_with_level_of_dit(self): dtcr = sn.apply_to(dtcr) # Ensure that the values have changed. - assert dtcr._hdu.data[0][0] != value_normal - assert dtcr._hdu.data[1][0] != value_too_high + assert dtcr.data[0][0] != value_normal + assert dtcr.data[1][0] != value_too_high # Sensibility checks on the values. - assert 1 < dtcr._hdu.data[0][0] < value_normal * 2 - assert np.isnan(dtcr._hdu.data[0][1]) or dtcr._hdu.data[0][1] == 0 - assert np.isnan(dtcr._hdu.data[1][1]) + assert 1 < dtcr.data[0][0] < value_normal * 2 + assert np.isnan(dtcr.data[0][1]) or dtcr.data[0][1] == 0 + assert np.isnan(dtcr.data[1][1]) diff --git a/scopesim/tests/tests_effects/test_slitwheel.py b/scopesim/tests/tests_effects/test_slitwheel.py index b46f10f19..707e3dac8 100644 --- a/scopesim/tests/tests_effects/test_slitwheel.py +++ b/scopesim/tests/tests_effects/test_slitwheel.py @@ -27,9 +27,6 @@ def test_has_aperture_masks(self, swheel): def test_current_slit_is_aperture_mask(self, swheel): assert isinstance(swheel.current_slit, ApertureMask) - def test_current_slit_has_fov_grid_method(self, swheel): - assert hasattr(swheel.current_slit, "fov_grid") - def test_change_to_known_slit(self, swheel): swheel.change_slit('A') assert swheel.current_slit.meta['name'] == 'A' diff --git a/scopesim/tests/tests_effects/test_spatial_cube_shifts.py b/scopesim/tests/tests_effects/test_spatial_cube_shifts.py new file mode 100644 index 000000000..e8ef5a18b --- /dev/null +++ b/scopesim/tests/tests_effects/test_spatial_cube_shifts.py @@ -0,0 +1,30 @@ +import numpy as np +import pytest +from scipy.ndimage import map_coordinates +from scopesim.effects.atmo_dispersion import ShiftFoV3D + + +@pytest.mark.parametrize('shape', [(13, 5, 7), (9, 1, 4), (9, 8, 1)]) +def test_chunked_cubic_shift_matches_independent_spatial_planes(shape): + data = np.random.default_rng(127).uniform(size=shape) + dx = np.linspace(-2.3, 3.7, shape[0], dtype=np.float32) + dy = np.linspace(1.8, -.4, shape[0], dtype=np.float32) + yy, xx = np.indices(shape[1:], dtype=np.float32) + expected = np.stack([map_coordinates(plane, [yy - sy, xx - sx], order=3, mode='nearest') for plane, sx, sy in zip(data, dx, dy)]) + actual = ShiftFoV3D.map_coords_shift(data, dx, dy, chunk_w=4) + np.testing.assert_allclose(actual, expected, rtol=2e-6, atol=1e-6) + + +def test_spatial_shift_does_not_broaden_a_spectral_line(): + cube = np.zeros((11, 5, 7)) + cube[5, 2, 3] = 1. + shifted = ShiftFoV3D.map_coords_shift(cube, np.full(11, .3), np.full(11, -.2)) + np.testing.assert_array_equal(shifted[:5], 0.) + np.testing.assert_array_equal(shifted[6:], 0.) + + +@pytest.mark.parametrize('dx,dy', [(1, 0), (-1, 0), (0, 1), (0, -1), (2, -2)]) +def test_integer_roll_does_not_wrap_light_at_edges(dx, dy): + cube = np.ones((1, 5, 7)) + shifted = ShiftFoV3D.roll_shift(cube, [dx], [dy]) + assert shifted.sum() == (7 - abs(dx))*(5 - abs(dy)) diff --git a/scopesim/tests/tests_effects/test_ter_curves_utils.py b/scopesim/tests/tests_effects/test_ter_curves_utils.py index 555b0c41f..13206ee15 100644 --- a/scopesim/tests/tests_effects/test_ter_curves_utils.py +++ b/scopesim/tests/tests_effects/test_ter_curves_utils.py @@ -1,9 +1,9 @@ +# -*- coding: utf-8 -*- + import pytest -import scopesim.source.source_templates from pytest import approx import numpy as np -from matplotlib import pyplot as plt from astropy import units as u @@ -28,53 +28,6 @@ def test_returns_specific_from_svo(self): assert np.max(ks(wave)) > 0.75 -@pytest.mark.webtest -@pytest.mark.usefixtures("no_file_error") -@pytest.mark.parametrize("filter_name, phot_system, flux", - [("V", "vega", 995), - ("z", "AB", 602), - ("Ks", "vega", 43)]) -def test_zero_mag_flux(filter_name, phot_system, flux): - """ - Photon fluxes are in line with: - http://www.astronomy.ohio-state.edu/~martini/usefuldata.html - """ - new_flux = ter_utils.zero_mag_flux(filter_name, phot_system) - assert flux == approx(new_flux.value, rel=0.08) - - -@pytest.mark.webtest -@pytest.mark.usefixtures("no_file_error") -def test_compare_br_gamma(): - filt = "Ks" - flux_vega = ter_utils.zero_mag_flux(filt, "vega").value - flux_ab = ter_utils.zero_mag_flux(filt, "AB").value - delta_mag = 2.5 * np.log10(flux_ab/flux_vega) - assert delta_mag == approx(1.85, rel=0.01) - - -@pytest.mark.webtest -@pytest.mark.usefixtures("no_file_error") -class TestScaleSpectrum: - def test_scales_vega_spectrum_to_vega_ab_or_jansky(self): - spec = scopesim.source.source_templates.vega_spectrum() - vega_185 = ter_utils.scale_spectrum(spec, "Ks", -1.85 * u.mag) - ab_0 = ter_utils.scale_spectrum(spec, "Ks", 0 * u.ABmag) - jy_3630 = ter_utils.scale_spectrum(spec, "Ks", 3630 * u.Jy) - - wave = np.linspace(1.8, 2.5, 1000) * u.um - assert vega_185(wave).value == approx(ab_0(wave).value, rel=1e-2) - assert vega_185(wave).value == approx(jy_3630(wave).value, rel=1e-2) - - if PLOTS: - plt.plot(wave, spec(wave), "b") - plt.plot(wave, vega_185(wave), "r:") - plt.plot(wave, ab_0(wave), "g--") - plt.plot(wave, jy_3630(wave), "y-.") - plt.semilogy() - plt.show() - - @pytest.mark.webtest @pytest.mark.usefixtures("no_file_error") class TestGetFilterEffectiveWavelength: diff --git a/scopesim/tests/tests_integrations/test_flux_is_conserved_through_full_system.py b/scopesim/tests/tests_integrations/test_flux_is_conserved_through_full_system.py index 65690fe0d..6c7542b5c 100644 --- a/scopesim/tests/tests_integrations/test_flux_is_conserved_through_full_system.py +++ b/scopesim/tests/tests_integrations/test_flux_is_conserved_through_full_system.py @@ -1,18 +1,21 @@ +# -*- coding: utf-8 -*- import pytest from pytest import approx from unittest.mock import patch import numpy as np +import matplotlib.pyplot as plt from scopesim.optics.optical_train import OpticalTrain from scopesim.commands import UserCommands from scopesim.tests.mocks.py_objects.source_objects import ( - _image_source, _single_table_source, _table_source_overlapping) - -import matplotlib.pyplot as plt -from matplotlib.colors import LogNorm + _image_source, + _single_table_source, + _table_source_overlapping, +) +from scopesim.tests.tests_source.source_helpers import src_photons_in_range PLOTS = False @@ -48,11 +51,11 @@ def test_flux_is_conserved_for_no_bg_emission(self, cmds, tbl_src): opt.observe(tbl_src) im = opt.image_planes[0].image bg_flux = np.pi / 4 * np.prod(im.shape) - src_flux = tbl_src.photons_in_range(1, 2, 1)[0].value + src_flux = src_photons_in_range(tbl_src, 1, 2, 1)[0].value if PLOTS: implane = opt.image_planes[0] - plt.imshow(implane.image.T, origin="lower", norm=LogNorm()) + plt.imshow(implane.image.T, origin="lower", norm="log") plt.colorbar() plt.show() @@ -70,11 +73,11 @@ def test_flux_is_conserved_for_yes_bg_emission(self, non_unity_cmds, tbl_src): opt = OpticalTrain(non_unity_cmds) opt.observe(tbl_src) im = opt.image_planes[0].image - src_flux = tbl_src.photons_in_range(1, 2, 1)[0].value + src_flux = src_photons_in_range(tbl_src, 1, 2, 1)[0].value if PLOTS: implane = opt.image_planes[0] - plt.imshow(implane.image.T, origin="lower", norm=LogNorm()) + plt.imshow(implane.image.T, origin="lower", norm="log") plt.colorbar() plt.show() @@ -133,7 +136,7 @@ def test_stacked_stars(self): if PLOTS: fig, axes = plt.subplots(nrows=1, ncols=1, figsize=(12, 12)) - axes.imshow(np.sqrt(im_h), norm=LogNorm(), cmap="inferno") + axes.imshow(np.sqrt(im_h), norm="log", cmap="inferno") axes.set_title('H-band MORFEO "PSF Generic" ') plt.show() diff --git a/scopesim/tests/tests_integrations/test_source_fov_imageplane_play_nicely.py b/scopesim/tests/tests_integrations/test_source_fov_imageplane_play_nicely.py index 82209d5ab..3816a62a1 100644 --- a/scopesim/tests/tests_integrations/test_source_fov_imageplane_play_nicely.py +++ b/scopesim/tests/tests_integrations/test_source_fov_imageplane_play_nicely.py @@ -8,7 +8,11 @@ from astropy import units as u +from astropy.wcs import WCS + +from scopesim.optics import image_plane_utils as imp_utils from scopesim.optics.fov import FieldOfView2D +from scopesim.optics.fov_manager import chunk_edges from scopesim.optics.image_plane import ImagePlane from scopesim.tests.mocks.py_objects import source_objects as src @@ -64,3 +68,205 @@ def test_can_extract_the_source_in_a_fov(self, fov_hdr, comb_src, plt.subplot(133) plt.imshow(imp.hdu.data, origin="lower", norm=LogNorm()) plt.show() + +def _checkerboard_source(naxis, pixel_scale, crpix=None): + """0/1 checkerboard image source, one board square per pixel.""" + from astropy.io import fits + from scopesim.source.source import Source + from scopesim.source import source_templates as st + + nx, ny = naxis + wcs = WCS(naxis=2) + wcs.wcs.ctype = ["LINEAR", "LINEAR"] + wcs.wcs.cunit = ["deg", "deg"] + wcs.wcs.cdelt = 2 * [pixel_scale / 3600] + wcs.wcs.crval = [0, 0] + wcs.wcs.crpix = crpix or [(nx + 1) / 2, (ny + 1) / 2] + + hdr = wcs.to_header() + hdr["BUNIT"] = "" + board = (np.indices((ny, nx)).sum(axis=0) % 2).astype(float) + hdu = fits.ImageHDU(header=hdr, data=board) + hdu.header["SPEC_REF"] = 0 + hdu.header["SPECMAG"] = 0 + hdu.header["SPECUNIT"] = "mag" + return Source(image_hdu=hdu, spectra=[st.vega_spectrum()]), board + + +def _chunked_fov_headers(x_mm, y_mm, pixel_size, pixel_scale, chunk): + """Reproduce FOVManager's chunking of a detector footprint.""" + plate_scale = pixel_scale / pixel_size + lims = [v / pixel_size * pixel_scale + for v in (x_mm[0], x_mm[1], y_mm[0], y_mm[1])] + + def edges(vmin, vmax): + # the real chunk-edge helper, not a copy of it + return [vmin, *chunk_edges(vmin, vmax, chunk * pixel_scale), vmax] + + xe, ye = edges(lims[0], lims[1]), edges(lims[2], lims[3]) + for x0, x1 in zip(xe[:-1], xe[1:]): + for y0, y1 in zip(ye[:-1], ye[1:]): + skyhdr = imp_utils.header_from_list_of_xy( + [x0 / 3600, x1 / 3600], [y0 / 3600, y1 / 3600], + pixel_scale=pixel_scale / 3600) + dethdr, _ = imp_utils.det_wcs_from_sky_wcs( + WCS(skyhdr), pixel_scale, plate_scale) + skyhdr.update(dethdr.to_header()) + yield skyhdr + + +class TestChunkedFOVsTileTheImagePlane: + """A checkerboard must survive being cut into chunked FOVs and reassembled. + + ``chunk_size`` is 8 here so the whole thing runs in well under a second, + but the geometry is MICADO's: the detector footprint is not a whole number + of pixels across (MICADO's y axis is 189.32 mm / 0.015 mm = 12621.33 px), + so the last chunk in each axis is a fraction of a chunk wide and lands on + a pixel lattice offset from its neighbours'. The projection used to snap + those chunks inconsistently, which put a duplicated or a dropped + row/column at the seam and flipped the checkerboard phase across it. + """ + + PIXEL_SIZE = 0.01 # mm / pix + PIXEL_SCALE = 0.2 # arcsec / pix + CHUNK = 8 + + @pytest.mark.parametrize("half_mm, exact_extent", [ + (0.12, 24.0), # whole pixels, whole chunks + (0.105, 21.0), # whole pixels, fractional last chunk + (0.0866, 17.32), # fractional pixels -- MICADO's y axis + (0.0855, 17.10), + ]) + def test_checkerboard_survives_chunking(self, half_mm, exact_extent): + x_mm = y_mm = (-half_mm, half_mm) + assert (x_mm[1] - x_mm[0]) / self.PIXEL_SIZE == approx(exact_extent) + + imphdr = imp_utils.header_from_list_of_xy( + list(x_mm), list(y_mm), pixel_scale=self.PIXEL_SIZE, wcs_suffix="D") + imp = ImagePlane(imphdr) + + source, board = _checkerboard_source( + (imphdr["NAXIS1"], imphdr["NAXIS2"]), self.PIXEL_SCALE, + crpix=[imphdr["CRPIX1D"], imphdr["CRPIX2D"]]) + + fov_hdrs = list(_chunked_fov_headers( + x_mm, y_mm, self.PIXEL_SIZE, self.PIXEL_SCALE, self.CHUNK)) + assert len(fov_hdrs) == 9 + + for hdr in fov_hdrs: + fov = FieldOfView2D(hdr, waverange=[0.5, 2.5] * u.um, + area=1 * u.m**2) + fov.extract_from(source) + fov.view() + imp.add(fov.hdu, wcs_suffix="D") + + # no gaps and no double-counting at the chunk seams + got = imp.data / imp.data.max() + np.testing.assert_allclose(got, board, atol=1e-6) + + +def _flat_source(naxis=64, pixel_scale=0.064): + """Uniformly illuminated image source, one everywhere, flat spectrum.""" + from astropy.io import fits + from synphot import Empirical1D, SourceSpectrum + from scopesim.source.source import Source + + wcs = WCS(naxis=2) + wcs.wcs.ctype = ["LINEAR", "LINEAR"] + wcs.wcs.cunit = ["arcsec", "arcsec"] + wcs.wcs.cdelt = 2 * [pixel_scale] + wcs.wcs.crval = [0, 0] + wcs.wcs.crpix = 2 * [(naxis + 1) / 2] + + hdu = fits.ImageHDU(data=np.ones((naxis, naxis)), header=wcs.to_header()) + spec = SourceSpectrum(Empirical1D, points=[0.5, 2.5] * u.um, + lookup_table=[1, 1] * u.Unit("ph s-1 m-2 um-1")) + return Source(image_hdu=hdu, spectra=[spec]) + + +def _slit_fov(x, y, pixel_scale, pixel_size): + """FOV covering a slit of x, y [arcsec], built like FOVManager does.""" + skyhdr = imp_utils.header_from_list_of_xy( + [v / 3600 for v in x], [v / 3600 for v in y], + pixel_scale=pixel_scale / 3600) + dethdr, _ = imp_utils.det_wcs_from_sky_wcs( + WCS(skyhdr), pixel_scale, pixel_scale / pixel_size) + skyhdr.update(dethdr.to_header()) + return FieldOfView2D(skyhdr, waverange=[1.0, 2.0] * u.um, area=1 * u.m**2) + + +class TestFlatSourceThroughASlitFOV: + """A flat lamp seen through a slit must fill the slit, uniformly. + + A slit FOV is typically *thinner than one source pixel*: MICADO's Short + slit is 0.016" tall while a flat-lamp source is sampled at 0.064"/pix. The + cutout is grown outwards to whole source pixels, so it covers more sky than + the slit does; if its WCS is then written on the *source's* pixel lattice + rather than on the FOV's, ``overlay_image`` crops the surplus away and the + flux through the slit comes out wrong -- for the MICADO case, exactly half + of the slit was left dark. This is what commit 9740ebd6 reverted in + ``FieldOfView.extract_area_from_imagehdu``. + + Note that these tests and ``TestChunkedFOVsTileTheImagePlane`` above are the + two sides of the same trade-off: the cutout WCS can describe either the + source's pixel lattice or the FOV's, and ``extract_area_from_imagehdu`` + currently has to pick one. Resolving it properly needs sub-pixel-accurate + cropping, i.e. weighting the partial edge pixels of the cutout. + """ + + SRC_PIXEL_SCALE = 0.064 # arcsec / pix, the flat lamp + FOV_PIXEL_SCALE = 0.004 # arcsec / pix, MICADO SPEC + PIXEL_SIZE = 0.015 # mm / pix + + # slit extents [arcsec], all thinner than or comparable to a source pixel + SLITS = [ + ((-1.5, 1.5), (-0.008, 0.008)), # MICADO Short slit, centred + ((-1.5, 1.5), (0.010, 0.026)), # ... offset within one source pixel + ((-1.5, 1.5), (-0.032, 0.032)), # exactly one source pixel tall + ((-1.5, 1.5), (-0.08, 0.08)), # 2.5 source pixels tall + ((-1.5, 1.5), (0.020, 0.060)), # straddling a source pixel edge + ((-2.0, 2.0), (-0.008, 0.008)), # longer slit + ] + + def _observe(self, x, y): + fov = _slit_fov(x, y, self.FOV_PIXEL_SCALE, self.PIXEL_SIZE) + fov.extract_from(_flat_source(pixel_scale=self.SRC_PIXEL_SCALE)) + fov.view() + return fov + + @pytest.mark.parametrize(("x", "y"), SLITS) + def test_flux_through_slit_matches_slit_area(self, x, y): + fov = self._observe(x, y) + + # each source pixel carries 1 ph/s here (flat 1 ph/s/m2/um over the + # 1 um wide FOV waverange, collecting area 1 m2), so the flux through + # the slit is just the slit area in source pixels + slit_area = (x[1] - x[0]) * (y[1] - y[0]) + expected = slit_area / self.SRC_PIXEL_SCALE**2 + + assert fov.hdu.data.sum() == approx(expected, rel=1e-6) + + # ... and it survives the projection onto the image plane + x_mm = [v * self.PIXEL_SIZE / self.FOV_PIXEL_SCALE for v in x] + y_mm = [v * self.PIXEL_SIZE / self.FOV_PIXEL_SCALE for v in y] + imp = ImagePlane(imp_utils.header_from_list_of_xy( + x_mm, y_mm, pixel_scale=self.PIXEL_SIZE, wcs_suffix="D")) + imp.add(fov.hdu, wcs_suffix="D") + + assert imp.data.sum() == approx(expected, rel=1e-6) + + @pytest.mark.parametrize(("x", "y"), SLITS) + def test_illumination_is_uniform_across_the_slit(self, x, y): + fov = self._observe(x, y) + data = fov.hdu.data + + if PLOTS: + plt.plot(data.mean(axis=1)) + plt.xlabel("pixel across slit") + plt.ylabel("flux [ph s-1 pix-1]") + plt.show() + + # a flat lamp illuminates the whole slit evenly: no dark rows at the + # edges, no step in the profile across the slit + assert data.min() > 0 + np.testing.assert_allclose(data, data.max(), rtol=1e-6) diff --git a/scopesim/tests/tests_optics/test_FOVManager.py b/scopesim/tests/tests_optics/test_FOVManager.py index a99477156..19ee5a6fb 100644 --- a/scopesim/tests/tests_optics/test_FOVManager.py +++ b/scopesim/tests/tests_optics/test_FOVManager.py @@ -4,7 +4,7 @@ from astropy import units as u from scopesim.effects import DetectorWindow -from scopesim.optics.fov_manager import FOVManager +from scopesim.optics.fov_manager import FOVManager, chunk_edges from scopesim.tests.mocks.py_objects import effects_objects as eo from scopesim.utils import from_currsys @@ -97,3 +97,59 @@ def apply_to(self, obj, **kwargs): assert fov.detector_header["NAXIS1"] == 100 assert fov.detector_header["NAXIS2"] == 100 assert fov.detector_header["CDELT1D"] == approx(0.01) + +class TestChunkEdges: + """Chunk edges must be interior and platform-independent. + + np.arange(vmin, vmax, step) returns one element too many when + (vmax - vmin) / step rounds just above an integer, and whether that + element lands on vmax or one ulp below it depends on how the platform + rounds vmin + i * step. One ulp below vmax is a legal split point, so + FovVolumeList.split would carve off a ~1e-15 px volume that + create_wcs_from_points then rounds up into a spurious one-pixel FOV. + """ + + @pytest.mark.parametrize("span_px, chunk, n_chunks", [ + (24.0, 8, 3), # exact multiple -- the fragile case + (24.0, 4, 6), # exact multiple + (16.0, 8, 2), + (21.0, 8, 3), # fractional last chunk + (17.32, 8, 3), # fractional pixels, fractional last chunk + (12621 + 1 / 3, 2048, 7), # MICADO's y axis + (5.0, 8, 1), # smaller than one chunk + ]) + def test_edges_are_interior_and_count_is_right(self, span_px, chunk, + n_chunks): + pixel_scale = 0.1 + vmin = -span_px / 2 * pixel_scale + vmax = vmin + span_px * pixel_scale + edges = chunk_edges(vmin, vmax, chunk * pixel_scale) + + assert len(edges) == n_chunks - 1 + assert (edges > vmin).all() + assert (edges < vmax).all() + + @pytest.mark.parametrize("span_px, chunk", [(24.0, 8), (24.0, 4), + (16.0, 8), (32.0, 16)]) + def test_exact_multiples_give_no_sliver_chunk(self, span_px, chunk): + # A span that is an exact multiple of the chunk must come out as + # equal chunks, with no degenerate remainder however the float + # arithmetic rounds. + pixel_scale = 0.1 + vmin = -span_px / 2 * pixel_scale + vmax = vmin + span_px * pixel_scale + edges = chunk_edges(vmin, vmax, chunk * pixel_scale) + + bounds = np.array([vmin, *edges, vmax]) + widths = np.diff(bounds) / pixel_scale + assert len(widths) == span_px / chunk + np.testing.assert_allclose(widths, chunk, atol=1e-9) + + def test_a_split_one_ulp_inside_the_upper_bound_is_not_emitted(self): + # np.arange does emit such a value on arm64 macOS; assert directly + # that chunk_edges never does, on any platform. + vmin, vmax, step = -1.2000000000000002, 1.2000000000000002, 0.8 + edges = chunk_edges(vmin, vmax, step) + assert (edges < np.nextafter(vmax, -np.inf)).all() + # ... which np.arange does not guarantee + assert len(np.arange(vmin, vmax, step)) == len(edges) + 2 diff --git a/scopesim/tests/tests_optics/test_FieldOfView.py b/scopesim/tests/tests_optics/test_FieldOfView.py index 49577b070..bab799ad9 100644 --- a/scopesim/tests/tests_optics/test_FieldOfView.py +++ b/scopesim/tests/tests_optics/test_FieldOfView.py @@ -1,3 +1,6 @@ +# -*- coding: utf-8 -*- +"""Test for FieldOfView classes""" + import pytest from pytest import approx import numpy as np @@ -5,7 +8,6 @@ from astropy.io import fits from astropy.table import Table from matplotlib import pyplot as plt -from matplotlib.colors import LogNorm from scopesim.tests.mocks.py_objects import header_objects as ho from scopesim.tests.mocks.py_objects import source_objects as so @@ -69,9 +71,6 @@ def test_initialises_with_header_and_waverange(self): class TestExtractFrom: - # @pytest.mark.xfail(reason=("is_field_in_fov drops table if anything is " - # "outside fov volume, therefore no point source " - # "is extracted...")) def test_extract_point_sources_from_table(self): src = so._table_source() src.fields[0].field["x"] = [-15, -5, 0, 0] * u.arcsec @@ -88,7 +87,7 @@ def test_extract_2d_image_from_hduimage(self): fov = _fov_190_210_um() fov.extract_from(src) - assert fov.fields[0].data.shape == (51, 25) + assert fov.fields[0].data.shape == (51, 26) assert len(fov.fields[0].spectra[0].waveset) == 11 assert fov.fields[0].spectra[0].waveset[0].value == approx(19000) @@ -107,11 +106,8 @@ def test_extract_3d_cube_that_is_offset_relative_to_fov(self): fov = _fov_197_202_um() fov.extract_from(src) - assert fov.fields[0].field.shape == (3, 51, 25) + assert fov.fields[0].field.shape == (3, 51, 26) - # @pytest.mark.xfail(reason=("is_field_in_fov drops table if anything is " - # "outside fov volume, therefore no point source " - # "is extracted...")) def test_extract_one_of_each_type_from_source_object(self): src_table = so._table_source() # 4 sources, put two outside of FOV src_table.fields[0].field["x"] = [-15, -5, 0, 0] * u.arcsec @@ -124,7 +120,7 @@ def test_extract_one_of_each_type_from_source_object(self): fov.extract_from(src) assert fov.fields[0].field.shape == (3, 51, 51) - assert fov.fields[1].field.shape == (51, 25) + assert fov.fields[1].field.shape == (51, 26) assert len(fov.fields[2].field) == 2 # assert len(fov.spectra) == 3 @@ -145,17 +141,6 @@ def test_ignores_fields_outside_fov_boundary(self): assert len(fov.fields) == 0 - @pytest.mark.skip(reason="SPEC_REF is obsolete, just rm this test?") - def test_all_spectra_are_referenced_correctly(self): - src = so._image_source() + so._cube_source() + so._table_source() - fov = _fov_190_210_um() - fov.extract_from(src) - # check the same spectrum object is referenced by both lists - assert fov.fields[0].header["SPEC_REF"] == \ - src.fields[0].header["SPEC_REF"] - assert all(fov.fields[2][i]["ref"] == src.fields[2][i]["ref"] - for i in range(4)) - def test_contains_all_fields_inside_fov(self): src = so._image_source() + so._cube_source() + so._table_source() the_fov = FieldOfView(ho._basic_fov_header(), (1, 2) * u.um, @@ -179,123 +164,6 @@ def test_handles_nans(self): class TestMakeCube: - @pytest.mark.xfail(reason="cube flux is broken") - def test_makes_cube_from_table(self): - src_table = so._table_source() # 10x10" @ 0.2"/pix, [0.5, 2.5]m @ 0.02µm - fov = _fov_190_210_um(ndim=3) - fov.extract_from(src_table) - - cube = fov.make_hdu() - - in_sum = 0 - waveset = fov.spectra[0].waveset - for x, y, ref, weight in src_table.fields[0]: - flux = src_table.spectra[ref](waveset).to(u.ph/u.s/u.m**2/u.um).value - in_sum += np.sum(flux) * weight - - out_sum = np.sum(cube.data) - - assert out_sum == approx(in_sum, rel=0.01) - - if PLOTS: - plt.imshow(cube.data[0, :, :], origin="lower") - plt.show() - - @pytest.mark.xfail(reason="cube flux is broken") - def test_makes_cube_from_imagehdu(self): - src_image = so._image_source() # 10x10" @ 0.2"/pix, [0.5, 2.5]m @ 0.02µm - fov = _fov_190_210_um(ndim=3) - fov.extract_from(src_image) - - cube = fov.make_hdu() - - waveset = np.linspace(1.9, 2.1, np.shape(cube)[0]) * u.um - spec = fov.spectra[0](waveset).to(u.ph/u.s/u.m**2/u.um).value - in_sum = np.sum(src_image.fields[0].data) * np.sum(spec) - out_sum = np.sum(cube.data) - - assert out_sum == approx(in_sum, rel=0.01) - - if PLOTS: - plt.imshow(cube.data[0, :, :], origin="lower") - plt.show() - - @pytest.mark.xfail(reason="cube flux is broken") - def test_makes_cube_from_other_cube_imagehdu(self): - import scopesim as sim - sim.rc.__currsys__["!SIM.spectral.spectral_bin_width"] = 0.01 - src_cube = so._cube_source() # 10x10" @ 0.2"/pix, [0.5, 2.5]m @ 0.02µm - fov = _fov_197_202_um(ndim=3) - fov.extract_from(src_cube) - - cube = fov.make_hdu() - - # layer 74 to 77 are extracted by FOV - in_sum = np.sum(src_cube.fields[0].data[74:77, :, :]) - - out_sum = np.sum(cube.data) - - assert out_sum == approx(in_sum) - - if PLOTS: - plt.imshow(cube.data[0, :, :], origin="lower") - plt.show() - - @pytest.mark.xfail(reason="cube flux is broken") - def test_makes_cube_from_two_similar_cube_imagehdus(self): - src_cube = so._cube_source() + so._cube_source(dx=1) # 2 cubes 10x10" @ 0.2"/pix, [0.5, 2.5]m @ 0.02µm - fov = _fov_197_202_um(ndim=3) - fov.extract_from(src_cube) - - cube = fov.make_hdu() - - # layer 74 to 77 are extracted by FOV - bin_widths = np.array([0.01, 0.02, 0.01])[:, None, None] * 1e4 # um -> AA - in_sum = 2 * np.sum(src_cube.fields[0].data[74:77, :, :] * bin_widths) - - out_sum = np.sum(cube.data) - - assert out_sum == approx(in_sum, rel=1e-3) - - if PLOTS: - plt.imshow(cube.data[0, :, :], origin="lower") - plt.show() - - @pytest.mark.xfail(reason="cube flux is broken") - def test_makes_cube_from_all_types_of_source_object(self): - src_all = so._table_source() + \ - so._image_source(dx=-4, dy=-4) + \ - so._cube_source(weight=1e-8, dx=4) - - fov = _fov_190_210_um(ndim=3) - fov.extract_from(src_all) - - cube = fov.make_hdu() - - # sum up the expected flux in the output cube - # bin_width * half width edge bin * PHOTLAM -> SI - waveset = fov.spectra[0].waveset - - table_sum = 0 - for x, y, ref, weight in src_all.fields[0]: - flux = src_all.spectra[ref](waveset).value - table_sum += np.sum(flux) * weight - - ref = src_all.fields[1].header["SPEC_REF"] - spec = fov.spectra[ref](waveset).value - image_sum = np.sum(src_all.fields[1].data) * np.sum(spec) - - cube_sum = np.sum(src_all.fields[2].data[70:81, :, :]) * 0.02 * 0.95 - - in_sum = table_sum + image_sum + cube_sum - out_sum = np.sum(cube.data) - - assert out_sum == approx(in_sum, rel=0.02) - - if PLOTS: - im = cube.data[0, :, :] - plt.imshow(im, origin="lower", norm=LogNorm(), vmin=1e-8) - plt.show() @pytest.mark.parametrize("src", [so._table_source(), so._image_source(), @@ -434,7 +302,7 @@ def test_makes_image_from_all_types_of_source_object(self): if PLOTS: im = image.data - plt.imshow(im, origin="lower", norm=LogNorm(), vmin=1e-8) + plt.imshow(im, origin="lower", norm="log", vmin=1e-8) plt.show() @@ -468,14 +336,18 @@ def test_make_spectrum_from_image(self): def test_make_spectrum_from_cube(self): src_cube = so._cube_source() # 10x10" @ 0.2"/pix, [0.5, 2.5]m @ 0.02µm fov = _fov_197_202_um(ndim=1) + original = src_cube.fields[0].data.copy() fov.extract_from(src_cube) spec = fov.make_hdu() - in_sum = np.sum(src_cube.fields[0].data[74:77, :, :]) * 1e-8 + # 1.98 and 2.00 um cells are complete; the 2.02 um cell is half + # inside this FOV. Cropping must not change the input cube in place. + in_sum = (original[74:76].sum() + original[76].sum()/2)*1e-8 out_sum = np.sum(spec(fov.waveset).value) assert in_sum == approx(out_sum) + np.testing.assert_array_equal(src_cube.fields[0].data, original) def test_makes_spectrum_from_all_types_of_source_object(self): src_table = so._table_source() @@ -509,49 +381,50 @@ def test_makes_spectrum_from_all_types_of_source_object(self): plt.show() -@pytest.mark.skip(reason="needs split FOVs after subclass refactor") class TestMakeSpectrumImageCubeAllPlayNicely: - def test_make_cube_and_make_spectrum_return_the_same_fluxes(self): - src_all = so._table_source() + \ - so._image_source(dx=-4, dy=-4) + \ - so._cube_source(weight=1e-8, dx=4) - - fov = _fov_190_210_um(ndim=3) - fov.extract_from(src_all) - - # if photlam, units of ph / s / cm2 / AA, else units of ph / s / voxel - cube = fov.make_hdu() - # cube_waves = get_cube_waveset(cube.header) - cube_spectrum = cube.data.sum(axis=2).sum(axis=1) - - # always units of ph / s / cm-2 / AA-1 - waves = fov.waveset - spectrum = fov.make_hdu()(waves).value - - bin_width = 0.02 # um - photlam_to_si = 1e8 # cm-2 AA-1 --> m-2 um-1 - edge_compensation = 0.91 - - spectrum *= bin_width * photlam_to_si * edge_compensation - - if PLOTS: - plt.plot(waves, spectrum, "k") - # plt.plot(cube_waves, cube_spectrum, "r") - plt.show() - - assert np.sum(cube.data) == approx(np.sum(spectrum), rel=0.001) - - def test_make_cube_and_make_image_return_the_same_fluxes(self): - src_all = so._cube_source(weight=1e-8, dx=4) - # so._image_source(dx=-4, dy=-4) # - # so._table_source()# + \ - - fov = _fov_190_210_um() - fov.extract_from(src_all) - - # if photlam, units of ph / s / cm2 / AA, else units of ph / s / voxel - cube_sum = np.sum(fov.make_hdu().data) - # if photlam, units of ph / s / cm2 / AA, else units of ph / s / pixel - image_sum = np.sum(fov.make_hdu().data) - - assert cube_sum == approx(image_sum, rel=0.05) + @pytest.mark.parametrize("kind", ["table", "image", "cube", "two_cubes", "mixed"]) + @pytest.mark.parametrize("spectral_step", [.01, .005]) + def test_photon_integrals_agree_between_representations(self, kind, spectral_step): + from synphot import SourceSpectrum, Empirical1D + from scopesim.commands import UserCommands + from scopesim.optics.optical_train import OpticalTrain + + # A line fully inside the band isolates representation and grid errors + # from partial end bins. Its integral is the native trapezoidal integral. + wave = np.linspace(1.5, 2.5, 101)*u.um + profile = np.exp(-.5*((wave.to_value(u.um)-2.)/.02)**2) + spectrum = SourceSpectrum(Empirical1D, points=wave, lookup_table=profile*u.ph/u.s/u.m**2/u.um) + table = so._table_source() + table.fields[0].spectra = {ref: spectrum for ref in table.fields[0].spectra} + image = so._image_source() + image.fields[0].spectra = {0: spectrum} + cube = so._cube_source() + cube.fields[0].data = cube.fields[0].data[-1][None, :, :]/4 * profile[:, None, None] + cube.fields[0].header['CRVAL3'] = 2. + cube.fields[0].header['CDELT3'] = .01 + cube.fields[0].header['CUNIT3'] = 'um' + sources = {'table': table, 'image': image, 'cube': cube, 'two_cubes': cube+cube, 'mixed': table+image+cube} + source = sources[kind] + spatial_weight = {'table': table.fields[0].field['weight'].sum(), 'image': image.fields[0].data.sum(), 'cube': cube.fields[0].data[50].sum()} + spatial_weight['two_cubes'] = 2*spatial_weight['cube'] + spatial_weight['mixed'] = spatial_weight['table']+spatial_weight['image']+spatial_weight['cube'] + expected = spatial_weight[kind]*np.trapezoid(profile, wave.to_value(u.um)) + source = OpticalTrain().prepare_source(source) + + commands = UserCommands(properties={'!SIM.spectral.spectral_bin_width': spectral_step}) + rates = [] + for cls in (FieldOfView1D, FieldOfView2D, FieldOfView3D): + fov = cls(ho._fov_header(), [1.9, 2.1]*u.um, area=1*u.m**2, cmds=commands) + fov.extract_from(source) + product = fov.make_hdu() + if cls is FieldOfView1D: + samples = product(fov.waveset).to(u.ph/u.s/u.m**2/u.um) + rate = np.trapezoid(samples, fov.waveset.to(u.um))*fov.area + rates.append(rate.to_value(u.ph/u.s)) + elif cls is FieldOfView2D: + rates.append(product.data.sum()) + else: + rate = product.data.sum()*u.Unit(product.header['BUNIT'])*fov.pixel_area*product.header['CDELT3']*u.um + rates.append(rate.to_value(u.ph/u.s)) + # The line tails outside the band contain less than 10^-6 of its flux. + assert rates == approx([expected]*3, rel=2e-5) diff --git a/scopesim/tests/tests_optics/test_ImagePlane.py b/scopesim/tests/tests_optics/test_ImagePlane.py index 1a8a81190..2866f3451 100644 --- a/scopesim/tests/tests_optics/test_ImagePlane.py +++ b/scopesim/tests/tests_optics/test_ImagePlane.py @@ -1,9 +1,11 @@ +# -*- coding: utf-8 -*- """Tests for ImagePlane and some ImagePlaneUtils""" # pylint: disable=missing-class-docstring # pylint: disable=missing-function-docstring from copy import deepcopy +from itertools import product import pytest from pytest import approx @@ -11,22 +13,108 @@ import numpy as np from astropy.io import fits from astropy import units as u -from astropy.table import Table from astropy import wcs - import matplotlib.pyplot as plt -from matplotlib.colors import LogNorm import scopesim.optics.image_plane as opt_imp import scopesim.optics.image_plane_utils as imp_utils -from scopesim.tests.mocks.py_objects.imagehdu_objects import \ - _image_hdu_square, _image_hdu_rect, _image_hdu_three_wcs,\ - _image_hdu_3d_data +from scopesim.tests.mocks.py_objects.imagehdu_objects import ( + _image_hdu_square, + _image_hdu_rect, + _image_hdu_three_wcs, + _image_hdu_3d_data, +) PLOTS = False +# The three functions below were oringinally stored in image_plane_utils, +# but ultimately only used here, so I moved them here as helpers. +# TODO: Check if the same functionality can be achieved in another way. +# ScopeSim internally uses something else (imageplane headers from detectors), +# so tests should ideally mock that instead... +def get_canvas_header(hdu_or_table_list, pixel_scale=1*u.arcsec): + """ + Generate a fits.Header with a WCS that covers everything in the FOV. + + Parameters + ---------- + hdu_or_table_list : list + A list of Tables and/or ImageHDU py_objects + + pixel_scale : astropy.Quantity + [arcsec] The pixel scale of the projection. Default in 1 arcsec + + Returns + ------- + header : fits.Header + A Header containing a WCS and NAXISn values to build an ImageHDU + + """ + def _get_headers(hdus_or_tables): + for hdu_or_table in hdus_or_tables: + if isinstance(hdu_or_table, fits.ImageHDU): + yield hdu_or_table.header + elif isinstance(hdu_or_table, fits.Header): + yield hdu_or_table + else: + raise TypeError( + "hdu_or_table_list may only contain fits.ImageHDU " + f"or fits.Header, found {type(hdu_or_table)}.") + + headers = list(_get_headers(hdu_or_table_list)) + + hdr = _make_bounding_header_from_headers(*headers, pixel_scale=pixel_scale) + return hdr + + +def _make_bounding_header_from_headers(*headers, pixel_scale): + """ + Return a Header with WCS and NAXISn keywords bounding all input ImageHDUs. + + Parameters + ---------- + headers : list of fits.ImageHDU + pixel_scale : u.Quantity + [arcsec] + + Returns + ------- + hdr : fits.Header + + """ + def _get_unit_from_headers(*headers, wcs_suffix: str) -> str: + unit = headers[0][f"CUNIT1{wcs_suffix}"].lower() + assert all(header[f"CUNIT{i}{wcs_suffix}"].lower() == unit + for header, i in product(headers, range(1, 3))), \ + [(i, header[f"CUNIT{i}{wcs_suffix}"]) + for header, i in product(headers, range(1, 3))] + return unit + + wcs_suffix = "D" if pixel_scale.unit.physical_type == "length" else "" + unit = u.Unit(_get_unit_from_headers(*headers, wcs_suffix=wcs_suffix)) + + if unit.physical_type == "angle": + unit = "deg" + pixel_scale = pixel_scale.to_value(u.deg) + else: + pixel_scale = pixel_scale.to_value(unit) + + extents = [imp_utils.calc_footprint(header, wcs_suffix, unit) + for header in headers] + pnts = np.vstack(extents) + + hdr = imp_utils.header_from_list_of_xy( + pnts[:, 0], pnts[:, 1], pixel_scale, wcs_suffix) + hdr["NAXIS1"] += 1 + hdr["NAXIS2"] += 1 + hdr[f"CRVAL1{wcs_suffix}"] -= 0.5 * pixel_scale + hdr[f"CRVAL2{wcs_suffix}"] -= 0.5 * pixel_scale + + return hdr + + @pytest.fixture(scope="function", name="image_hdu_rect") def fixture_image_hdu_rect(): return _image_hdu_rect() @@ -57,361 +145,14 @@ def fixture_image_hdu_3d_data(): return _image_hdu_3d_data() -@pytest.fixture(scope="function", name="input_table") -def fixture_input_table(): - x = [-10, -10, 0, 10, 10] * u.arcsec - y = [-10, 10, 0, -10, 10] * u.arcsec - f = [1, 3, 1, 1, 5] - tbl = Table(names=["x", "y", "flux"], data=[x, y, f]) - - return tbl - - -@pytest.fixture(scope="function", name="input_table_mm") -def fixture_input_table_mm(): - x = [-10, -10, 0, 10, 10] * u.mm - y = [-10, 10, 0, -10, 10] * u.mm - f = [1, 3, 1, 1, 5] - tbl = Table(names=["x_mm", "y_mm", "flux"], data=[x, y, f]) - - return tbl - - -class TestCombineTableBoundaries: - def test_all_three_tables_are_inside_header_wcs(self, input_table): - tbl1 = deepcopy(input_table) - tbl2 = deepcopy(input_table) - tbl3 = deepcopy(input_table) - - tbl2["x"] -= 25 - tbl3["y"] -= 60 - - hdr = imp_utils._make_bounding_header_for_tables(tbl1, tbl2, tbl3) - as2deg = u.arcsec.to(u.deg) - for tbl in [tbl1, tbl2, tbl3]: - x, y = imp_utils.val2pix(hdr, tbl["x"]*as2deg, tbl["y"]*as2deg) - for xi, yi in zip(x, y): - assert 0 <= xi < hdr["NAXIS1"] - assert 0 <= yi < hdr["NAXIS2"] - - if PLOTS: - xy = imp_utils.calc_footprint(hdr) - x, y = xy[:, 0], xy[:, 1] - x, y = imp_utils.val2pix(hdr, x, y) - x0, y0 = imp_utils.val2pix(hdr, 0, 0) - - plt.plot(x, y, "b") - plt.plot(x0, y0, "ro") - for tbl in [tbl1, tbl2, tbl3]: - x, y = imp_utils.val2pix(hdr, tbl["x"] / 3600., - tbl["y"] / 3600.) - plt.plot(x, y, "k.") - - plt.show() - - def test_all_three_mm_tables_are_inside_header_wcs(self, input_table_mm): - tbl1 = deepcopy(input_table_mm) - tbl2 = deepcopy(input_table_mm) - tbl3 = deepcopy(input_table_mm) - - tbl2["x_mm"] += 50 - tbl3["y_mm"] += 25 - - hdr = imp_utils._make_bounding_header_for_tables(tbl1, tbl2, tbl3, - pixel_scale=100*u.um) - for tbl in [tbl1, tbl2, tbl3]: - x, y = imp_utils.val2pix(hdr, tbl["x_mm"], tbl["y_mm"], "D") - for xi, yi in zip(x, y): - assert 0 <= xi < hdr["NAXIS1"] - assert 0 <= yi < hdr["NAXIS2"] - - if PLOTS: - xy = imp_utils.calc_footprint(hdr, "D") - x, y = xy[:, 0], xy[:, 1] - x, y = imp_utils.val2pix(hdr, x, y, "D") - x0, y0 = imp_utils.val2pix(hdr, 0, 0, "D") - - plt.plot(x, y, "b") - plt.plot(x0, y0, "ro") - for tbl in [tbl1, tbl2, tbl3]: - x, y = imp_utils.val2pix(hdr, tbl["x_mm"], tbl["y_mm"], "D") - plt.plot(x, y, "k.") - - plt.show() - - -class TestCombineImageHDUBoundaries: - def test_all_two_imagehdus_are_inside_header_wcs(self, image_hdu_square, - image_hdu_rect): - - image_hdu_rect.header["CRVAL1"] -= 70 # * u.arcsec.to(u.deg) - image_hdu_square.header["CRVAL2"] += 70 # * u.arcsec.to(u.deg) - - hdr = imp_utils._make_bounding_header_from_headers( - image_hdu_square.header, image_hdu_rect.header) - for imhdr in [image_hdu_square.header, image_hdu_rect.header]: - xy = imp_utils.calc_footprint(imhdr) - x, y = xy[:, 0], xy[:, 1] - x *= u.arcsec.to(u.deg) - y *= u.arcsec.to(u.deg) - x, y = imp_utils.val2pix(hdr, x, y) - for xi, yi in zip(x, y): - assert 0 <= xi < hdr["NAXIS1"] - assert 0 <= yi < hdr["NAXIS2"] - - if PLOTS: - for imhdr in [image_hdu_square.header, image_hdu_rect.header, hdr]: - xy = imp_utils.calc_footprint(imhdr) - x, y = xy[:, 0], xy[:, 1] - xp, yp = imp_utils.val2pix(imhdr, x, y) - plt.plot(x, y, "r-") - - plt.plot(0, 0, "ro") - plt.gca().set_aspect(1) - plt.show() - - def test_all_two_mm_imagehdus_are_inside_header_wcs(self, - image_hdu_square_mm, - image_hdu_rect_mm): - - image_hdu_rect_mm.header["CRVAL1D"] -= 40 - image_hdu_square_mm.header["CRVAL2D"] += 80 - - hdr = imp_utils._make_bounding_header_from_headers( - image_hdu_square_mm.header, image_hdu_rect_mm.header, - pixel_scale=1*u.mm) - for imhdr in [image_hdu_square_mm.header, image_hdu_rect_mm.header]: - xy = imp_utils.calc_footprint(imhdr, "D") - x, y = xy[:, 0], xy[:, 1] - x, y = imp_utils.val2pix(hdr, x, y, "D") - for xi, yi in zip(x, y): - assert 0 <= xi < hdr["NAXIS1"] - assert 0 <= yi < hdr["NAXIS2"] - - if PLOTS: - for imhdr in [image_hdu_square_mm.header, - image_hdu_rect_mm.header, hdr]: - xy = imp_utils.calc_footprint(imhdr, "D") - x, y = xy[:, 0], xy[:, 1] - xp, yp = imp_utils.val2pix(imhdr, x, y, "D") - plt.plot(x, y, "r-") - - plt.plot(0, 0, "ro") - plt.gca().set_aspect(1) - plt.show() - - -class TestGetCanvasHeader: - def test_all_5_objects_are_inside_header_wcs(self, image_hdu_square, - image_hdu_rect, input_table): - - tbl1 = deepcopy(input_table) - tbl2 = deepcopy(input_table) - tbl3 = deepcopy(input_table) - - tbl2["x"] -= 150 - tbl3["y"] -= 100 - - image_hdu_rect.header["CRVAL1"] += 100 * u.arcsec.to(u.deg) - image_hdu_square.header["CRVAL1"] += 0 * u.arcsec.to(u.deg) - image_hdu_square.header["CRVAL2"] += 100 * u.arcsec.to(u.deg) - - hdr = imp_utils.get_canvas_header([image_hdu_square, tbl1, tbl2, tbl3, - image_hdu_rect], pixel_scale=1*u.arcsec) - - as2deg = u.arcsec.to(u.deg) - - for im in [image_hdu_square.header, image_hdu_rect.header]: - xy = imp_utils.calc_footprint(im) - x, y = xy[:, 0], xy[:, 1] - x, y = imp_utils.val2pix(hdr, x*as2deg, y*as2deg) - for xi, yi in zip(x, y): - assert 0 <= xi < hdr["NAXIS1"] - assert 0 <= yi < hdr["NAXIS2"] - - for tbl in [tbl1, tbl2, tbl3]: - x, y = imp_utils.val2pix(hdr, tbl["x"]*as2deg, tbl["y"]*as2deg) - for xi, yi in zip(x, y): - assert 0 <= xi < hdr["NAXIS1"] - assert 0 <= yi < hdr["NAXIS2"] - - if PLOTS: - - xy = imp_utils.calc_footprint(hdr) - x, y = xy[:, 0], xy[:, 1] - x, y = imp_utils.val2pix(hdr, x, y) - plt.plot(x, y, "b") - x0, y0 = imp_utils.val2pix(hdr, 0, 0) - plt.plot(x0, y0, "ro") - - for tbl in [tbl1, tbl2, tbl3]: - x, y = imp_utils.val2pix(hdr, tbl["x"]*as2deg, tbl["y"]*as2deg) - plt.plot(x, y, "k.") - - for im in [image_hdu_square.header, image_hdu_rect.header]: - xy = imp_utils.calc_footprint(im) - x, y = xy[:, 0], xy[:, 1] - x, y = imp_utils.val2pix(hdr, x, y) - plt.plot(x, y, "r-") - - plt.gca().set_aspect(1) - plt.show() - - def test_all_5_objects_are_inside_mm_header_wcs(self, image_hdu_square_mm, - image_hdu_rect_mm, - input_table_mm): - - tbl1 = deepcopy(input_table_mm) - tbl2 = deepcopy(input_table_mm) - tbl3 = deepcopy(input_table_mm) - - tbl2["x_mm"] -= 150 - tbl3["y_mm"] -= 100 - - image_hdu_rect_mm.header["CRVAL1D"] += 100 - image_hdu_square_mm.header["CRVAL1D"] += 0 - image_hdu_square_mm.header["CRVAL2D"] += 100 - - hdr = imp_utils.get_canvas_header([image_hdu_square_mm, tbl1, tbl2, - tbl3, image_hdu_rect_mm], - pixel_scale=1*u.mm) - - for im in [image_hdu_square_mm.header, image_hdu_rect_mm.header]: - xy = imp_utils.calc_footprint(im, "D") - x, y = xy[:, 0], xy[:, 1] - x, y = imp_utils.val2pix(hdr, x, y, "D") - for xi, yi in zip(x, y): - assert 0 <= xi < hdr["NAXIS1"] - assert 0 <= yi < hdr["NAXIS2"] - - for tbl in [tbl1, tbl2, tbl3]: - x, y = imp_utils.val2pix(hdr, tbl["x_mm"], tbl["y_mm"], "D") - for xi, yi in zip(x, y): - assert 0 <= xi < hdr["NAXIS1"] - assert 0 <= yi < hdr["NAXIS2"] - - if PLOTS: - - xy = imp_utils.calc_footprint(hdr, "D") - x, y = xy[:, 0], xy[:, 1] - x, y = imp_utils.val2pix(hdr, x, y, "D") - plt.plot(x, y, "b") - x0, y0 = imp_utils.val2pix(hdr, 0, 0, "D") - plt.plot(x0, y0, "ro") - - for tbl in [tbl1, tbl2, tbl3]: - x, y = imp_utils.val2pix(hdr, tbl["x_mm"], tbl["y_mm"], "D") - plt.plot(x, y, "k.") - - for im in [image_hdu_square_mm.header, image_hdu_rect_mm.header]: - xy = imp_utils.calc_footprint(im, "D") - x, y = xy[:, 0], xy[:, 1] - x, y = imp_utils.val2pix(hdr, x, y, "D") - plt.plot(x, y, "r-") - - plt.gca().set_aspect(1) - plt.show() - - -class TestAddTableToImageHDU: - def test_points_are_added_to_small_canvas(self, input_table): - tbl1 = deepcopy(input_table) - hdr = imp_utils.get_canvas_header([tbl1]) - - im = np.zeros((hdr["NAXIS2"], hdr["NAXIS1"])) - canvas_hdu = fits.ImageHDU(header=hdr, data=im) - canvas_hdu = imp_utils.add_table_to_imagehdu(tbl1, canvas_hdu) - - assert np.sum(canvas_hdu.data) == np.sum(tbl1["flux"]) - - if PLOTS: - # "top left is green, top right is yellow" - plt.imshow(canvas_hdu.data, origin="lower") - plt.show() - - def test_mm_points_are_added_to_small_canvas(self, input_table_mm): - tbl1 = deepcopy(input_table_mm) - hdr = imp_utils.get_canvas_header([tbl1], pixel_scale=2*u.mm) - - im = np.zeros((hdr["NAXIS2"], hdr["NAXIS1"])) - canvas_hdu = fits.ImageHDU(header=hdr, data=im) - canvas_hdu = imp_utils.add_table_to_imagehdu(tbl1, canvas_hdu, - wcs_suffix="D") - - assert np.sum(canvas_hdu.data) == np.sum(tbl1["flux"]) - - if PLOTS: - # "top left is green, top right is yellow" - plt.imshow(canvas_hdu.data, origin="lower") - plt.show() - - # add a paramaterised test here to check border cases - def test_points_are_added_to_massive_canvas(self, input_table): - tbl1 = deepcopy(input_table) - tbl2 = deepcopy(input_table) - tbl3 = deepcopy(input_table) - - tbl1["y"] += 50 - tbl2["x"] += 20 - tbl3["x"] -= 25 - tbl3["y"] -= 25 - - hdr = imp_utils.get_canvas_header([tbl1, tbl2, tbl3], - pixel_scale=1*u.arcsec) - - im = np.zeros((hdr["NAXIS2"], hdr["NAXIS1"])) - canvas_hdu = fits.ImageHDU(header=hdr, data=im) - - for tbl in [tbl1, tbl2, tbl3]: - canvas_hdu = imp_utils.add_table_to_imagehdu(tbl, canvas_hdu, True) - - total_flux = np.sum([tbl1["flux"], tbl2["flux"], tbl3["flux"]]) - assert np.sum(canvas_hdu.data) == total_flux - - if PLOTS: - x, y = imp_utils.val2pix(hdr, 0, 0) - plt.plot(x, y, "ro") - "top left is green, top right is yellow" - plt.imshow(canvas_hdu.data, origin="lower") - plt.show() - - def test_mm_points_are_added_to_massive_canvas(self, input_table_mm): - tbl1 = deepcopy(input_table_mm) - tbl2 = deepcopy(input_table_mm) - tbl3 = deepcopy(input_table_mm) - - tbl1["y_mm"] += 50 - tbl2["x_mm"] += 20 - tbl3["x_mm"] -= 25 - tbl3["y_mm"] -= 25 - - hdr = imp_utils.get_canvas_header([tbl1, tbl2, tbl3], - pixel_scale=1*u.mm) - im = np.zeros((hdr["NAXIS2"], hdr["NAXIS1"])) - canvas_hdu = fits.ImageHDU(header=hdr, data=im) - - for tbl in [tbl1, tbl2, tbl3]: - canvas_hdu = imp_utils.add_table_to_imagehdu(tbl, canvas_hdu, - True, wcs_suffix="D") - - total_flux = np.sum([tbl1["flux"], tbl1["flux"], tbl1["flux"]]) - assert np.sum(canvas_hdu.data) == total_flux - - if PLOTS: - x, y = imp_utils.val2pix(hdr, 0, 0, "D") - plt.plot(x, y, "ro") - # "top left is green, top right is yellow" - plt.imshow(canvas_hdu.data, origin="lower") - plt.show() - - class TestAddImageHDUToImageHDU: - def test_image_is_added_to_small_canvas(self, image_hdu_rect, - image_hdu_square): + def test_image_is_added_to_small_canvas( + self, image_hdu_rect, image_hdu_square, + ): im_hdu = image_hdu_rect im_hdu.header["CRVAL1"] -= 150*u.arcsec.to(u.deg) im_hdu.header["CRVAL2"] += 40*u.arcsec.to(u.deg) - hdr = imp_utils.get_canvas_header([im_hdu, image_hdu_square]) + hdr = get_canvas_header([im_hdu, image_hdu_square]) im = np.zeros((hdr["NAXIS2"], hdr["NAXIS1"])) canvas_hdu = fits.ImageHDU(header=hdr, data=im) @@ -436,18 +177,19 @@ def test_image_is_added_to_small_canvas(self, image_hdu_rect, plt.show() - def test_mm_image_is_added_to_small_canvas(self, image_hdu_rect_mm, - image_hdu_square_mm): + def test_mm_image_is_added_to_small_canvas( + self, image_hdu_rect_mm, image_hdu_square_mm, + ): im_hdu = image_hdu_rect_mm im_hdu.header["CRVAL1D"] -= 150 im_hdu.header["CRVAL2D"] += 40 - hdr = imp_utils.get_canvas_header([im_hdu, image_hdu_square_mm], - pixel_scale=1*u.mm) + hdr = get_canvas_header([im_hdu, image_hdu_square_mm], 1*u.mm) im = np.zeros((hdr["NAXIS2"], hdr["NAXIS1"])) canvas_hdu = fits.ImageHDU(header=hdr, data=im) - canvas_hdu = imp_utils.add_imagehdu_to_imagehdu(im_hdu, canvas_hdu, - wcs_suffix="D") + canvas_hdu = imp_utils.add_imagehdu_to_imagehdu( + im_hdu, canvas_hdu, wcs_suffix="D", + ) assert np.sum(canvas_hdu.data) == approx(np.sum(im_hdu.data)) @@ -466,32 +208,21 @@ def test_mm_image_is_added_to_small_canvas(self, image_hdu_rect_mm, plt.show() - def test_image_and_tables_on_large_canvas(self, input_table, image_hdu_rect, - image_hdu_square): - tbl1 = deepcopy(input_table) - tbl2 = deepcopy(input_table) - - tbl1["y"] -= 100 - tbl2["x"] += 100 - tbl2["y"] += 100 - + def test_images_on_large_canvas(self, image_hdu_rect, image_hdu_square): im_hdu = image_hdu_rect im_hdu.header["CRVAL1"] -= 150 # *u.arcsec.to(u.deg) im_hdu.header["CRVAL2"] += 20 # *u.arcsec.to(u.deg) - total_flux = (np.sum(tbl1["flux"]) + np.sum(tbl2["flux"]) + - im_hdu.data.sum() + image_hdu_square.data.sum()) + total_flux = im_hdu.data.sum() + image_hdu_square.data.sum() - hdr = imp_utils.get_canvas_header([im_hdu, image_hdu_square, - tbl1, tbl2], pixel_scale=3*u.arcsec) + hdr = get_canvas_header([im_hdu, image_hdu_square], 3*u.arcsec) im = np.zeros((hdr["NAXIS2"], hdr["NAXIS1"])) canvas_hdu = fits.ImageHDU(header=hdr, data=im) canvas_hdu = imp_utils.add_imagehdu_to_imagehdu(im_hdu, canvas_hdu) - canvas_hdu = imp_utils.add_imagehdu_to_imagehdu(image_hdu_square, - canvas_hdu) - canvas_hdu = imp_utils.add_table_to_imagehdu(tbl1, canvas_hdu) - canvas_hdu = imp_utils.add_table_to_imagehdu(tbl2, canvas_hdu) + canvas_hdu = imp_utils.add_imagehdu_to_imagehdu( + image_hdu_square, canvas_hdu, + ) assert np.sum(canvas_hdu.data) == approx(total_flux, rel=1e-2) @@ -511,39 +242,28 @@ def test_image_and_tables_on_large_canvas(self, input_table, image_hdu_rect, plt.show() - def test_mm_image_and_tables_on_large_canvas(self, input_table_mm, - image_hdu_rect_mm, - image_hdu_square_mm): + def test_mm_images_on_large_canvas( + self, image_hdu_rect_mm, image_hdu_square_mm, + ): image_hdu_rect = image_hdu_rect_mm image_hdu_square = image_hdu_square_mm - tbl1 = deepcopy(input_table_mm) - tbl2 = deepcopy(input_table_mm) - - tbl1["y_mm"] -= 100 - tbl2["x_mm"] += 100 - tbl2["y_mm"] += 100 im_hdu = image_hdu_rect im_hdu.header["CRVAL1D"] -= 150 im_hdu.header["CRVAL2D"] += 20 - hdr = imp_utils.get_canvas_header([im_hdu, image_hdu_square, - tbl1, tbl2], pixel_scale=3*u.mm) + hdr = get_canvas_header([im_hdu, image_hdu_square], 3*u.mm) im = np.zeros((hdr["NAXIS2"], hdr["NAXIS1"])) canvas_hdu = fits.ImageHDU(header=hdr, data=im) - canvas_hdu = imp_utils.add_table_to_imagehdu(tbl1, canvas_hdu, - wcs_suffix="D") - canvas_hdu = imp_utils.add_table_to_imagehdu(tbl2, canvas_hdu, - wcs_suffix="D") - canvas_hdu = imp_utils.add_imagehdu_to_imagehdu(im_hdu, canvas_hdu, - wcs_suffix="D") - canvas_hdu = imp_utils.add_imagehdu_to_imagehdu(image_hdu_square, - canvas_hdu, - wcs_suffix="D") - - total_flux = np.sum(tbl1["flux"]) + np.sum(tbl2["flux"]) + \ - np.sum(im_hdu.data) + np.sum(image_hdu_square.data) + canvas_hdu = imp_utils.add_imagehdu_to_imagehdu( + im_hdu, canvas_hdu, wcs_suffix="D", + ) + canvas_hdu = imp_utils.add_imagehdu_to_imagehdu( + image_hdu_square, canvas_hdu, wcs_suffix="D", + ) + + total_flux = np.sum(im_hdu.data) + np.sum(image_hdu_square.data) assert np.sum(canvas_hdu.data) == approx(total_flux, rel=5e-3) if PLOTS: @@ -564,11 +284,19 @@ def test_mm_image_and_tables_on_large_canvas(self, input_table_mm, class TestImagePlaneAdd: - def test_simple_add_imagehdu_conserves_flux(self, image_hdu_square, - image_hdu_rect): - fields = [image_hdu_rect, image_hdu_square] - hdr = imp_utils.get_canvas_header(pixel_scale=1 * u.arcsec, - hdu_or_table_list=fields) + def test_faint_signal_is_retained_on_bright_background(self, image_hdu_square): + plane = opt_imp.ImagePlane(image_hdu_square.header) + background = fits.ImageHDU(np.full(image_hdu_square.data.shape, 1e8), image_hdu_square.header) + signal = fits.ImageHDU(np.ones(image_hdu_square.data.shape), image_hdu_square.header) + plane.add(background) + before = plane.data.copy() + plane.add(signal) + np.testing.assert_allclose(plane.data - before, 1., atol=1e-7) + + def test_simple_add_imagehdu_conserves_flux( + self, image_hdu_square, image_hdu_rect, + ): + hdr = get_canvas_header([image_hdu_rect, image_hdu_square]) orig_sum = image_hdu_rect.data.sum() @@ -598,11 +326,10 @@ def test_add_many_tables_and_imagehdus(self, image_hdu_rect, image_hdu_square): im_hdu.header["CRVAL1"] -= 150*u.arcsec.to(u.deg) im_hdu.header["CRVAL2"] += 20*u.arcsec.to(u.deg) - fields = [im_hdu, image_hdu_square] - hdr = imp_utils.get_canvas_header(fields, pixel_scale=1 * u.arcsec) + hdr = get_canvas_header([im_hdu, image_hdu_square]) implane = opt_imp.ImagePlane(hdr) - implane.add(fields) + implane.add([im_hdu, image_hdu_square]) total_flux = np.sum(im_hdu.data) + np.sum(image_hdu_square.data) assert np.sum(implane.data) == approx(total_flux) @@ -618,11 +345,12 @@ def test_add_many_tables_and_imagehdus(self, image_hdu_rect, image_hdu_square): plt.plot(x0, y0, "ro") plt.gca().set_aspect(1) - plt.imshow(implane.data, origin="lower", norm=LogNorm()) + plt.imshow(implane.data, origin="lower", norm="log") plt.show() - def test_add_many_mm_tables_and_imagehdus( - self, image_hdu_rect_mm, image_hdu_square_mm): + def test_add_many_mm_imagehdus( + self, image_hdu_rect_mm, image_hdu_square_mm, + ): image_hdu_rect = image_hdu_rect_mm image_hdu_square = image_hdu_square_mm @@ -630,10 +358,9 @@ def test_add_many_mm_tables_and_imagehdus( im_hdu.header["CRVAL1D"] -= 150 # mm im_hdu.header["CRVAL2D"] += 20 - fields = [im_hdu, image_hdu_square] - hdr = imp_utils.get_canvas_header(fields, pixel_scale=1*u.mm) + hdr = get_canvas_header([im_hdu, image_hdu_square], 1*u.mm) implane = opt_imp.ImagePlane(hdr) - implane.add(fields, wcs_suffix="D") + implane.add([im_hdu, image_hdu_square], wcs_suffix="D") total_flux = np.sum(im_hdu.data) + np.sum(image_hdu_square.data) assert np.sum(implane.data) == approx(total_flux) @@ -649,7 +376,7 @@ def test_add_many_mm_tables_and_imagehdus( plt.plot(x0, y0, "ro") plt.gca().set_aspect(1) - plt.imshow(implane.data, origin="lower", norm=LogNorm()) + plt.imshow(implane.data, origin="lower", norm="log") plt.show() @@ -725,7 +452,7 @@ def test_rescale_works_on_3d_imageplane(self, image_hdu_3d_data): ############################################################################### -# ..todo: When you have time, reintegrate these tests, There are some good ones +# TODO: When you have time, reintegrate these tests, There are some good ones class TestGetSpatialExtentOfHeader: def test_returns_right_sky_coords_from_known_coords(self, image_hdu_square): @@ -742,141 +469,24 @@ def test_returns_right_sky_coords_from_known_coords(self, image_hdu_square): class TestMakeImagePlaneHeader: - def test_header_contains_future_naxis_pixel_sizes(self, image_hdu_square, - image_hdu_rect): - hdr = imp_utils.get_canvas_header([image_hdu_square, image_hdu_rect]) + def test_header_contains_future_naxis_pixel_sizes( + self, image_hdu_square, image_hdu_rect, + ): + hdr = get_canvas_header([image_hdu_square, image_hdu_rect]) assert hdr["NAXIS1"] == 100 + 1 assert hdr["NAXIS2"] == 200 + 1 @pytest.mark.parametrize("offset", -np.random.randint(200, 1001, 10)) - def test_header_contains_spread_out_regions(self, offset, image_hdu_square, - image_hdu_rect): + def test_header_contains_spread_out_regions( + self, offset, image_hdu_square, image_hdu_rect, + ): image_hdu_rect.header["CRVAL1"] += offset # *u.arcsec.to(u.deg) - hdr = imp_utils.get_canvas_header([image_hdu_square, image_hdu_rect]) + hdr = get_canvas_header([image_hdu_square, image_hdu_rect]) image_width = image_hdu_square.header["NAXIS1"] // 2 + \ image_hdu_rect.header["NAXIS1"] // 2 + abs(offset) + 1 assert hdr["NAXIS1"] == image_width - @pytest.mark.parametrize("offset", [0, 10, 25, 50]) - def test_header_has_correct_size_based_on_table_extremes(self, offset, - input_table): - tbl1 = input_table - tbl2 = deepcopy(input_table) - tbl3 = deepcopy(input_table) - tbl2["x"] += offset - tbl3["y"] += offset - hdr = imp_utils.get_canvas_header([tbl1, tbl2, tbl3], - pixel_scale=0.1*u.arcsec) - - assert hdr["NAXIS1"] == np.max(tbl1["x"] + tbl2["x"]) * 10 + 3 # 4 - assert hdr["NAXIS2"] == np.max(tbl1["y"] + tbl3["y"]) * 10 + 3 # 4 - - # @pytest.mark.parametrize("pix_scl", [5, 1, 0.5, 0.1]) - # def test_header_has_correct_size_with_tbl_and_image_input(self, input_table, - # image_hdu_square, - # pix_scl): - # input_table["x"] += 100 - # hdr = imp_utils.get_canvas_header([image_hdu_square, input_table], - # pixel_scale=pix_scl * u.arcsec) - # assert hdr["NAXIS1"] == approx(hdr["CRPIX1"] + - # np.max(input_table["x"]) / pix_scl + 1, - # abs=0.5) - - def test_header_does_not_contain_negative_naxis_keywords(self): - pass - -# -# class TestGetCornerSkyCoordsFromTable: -# def test_table_with_column_units_returns_right_values(self): -# x, y = [1] * u.arcmin, [1] * u.arcmin -# tbl = Table(names=["x", "y"], data=[x, y]) -# xsky, ysky = imp_utils.get_corner_sky_coords_from_table(tbl) -# -# assert xsky[0]*u.deg == x[0].to(u.deg) -# assert ysky[0]*u.deg == y[0].to(u.deg) -# -# def test_table_with_meta_units_returns_right_values(self): -# x, y = [1], [1] -# tbl = Table(names=["x", "y"], data=[x, y]) -# tbl.meta.update({"x_unit": u.arcmin, "y_unit": u.arcmin}) -# xsky, ysky = impl_utils.get_corner_sky_coords_from_table(tbl) -# -# assert xsky[0] == x[0]*u.arcmin.to(u.deg) -# assert ysky[0] == y[0]*u.arcmin.to(u.deg) -# -# def test_table_with_default_units_returns_right_values(self): -# x, y = [60], [60] # because default unit is arcsec -# tbl = Table(names=["x", "y"], data=[x, y]) -# xsky, ysky = impl_utils.get_corner_sky_coords_from_table(tbl) -# -# assert pytest.approx(xsky[0] == x[0] * u.arcmin.to(u.deg)) -# assert pytest.approx(ysky[0] == y[0] * u.arcmin.to(u.deg)) -# -# -# class TestGetCornerSkyCoords: -# def test_returns_coords_for_combination_of_table_and_header(self, -# image_hdu_square): -# x, y = [-100, 70] * u.arcsec, [0, 0] * u.arcsec -# tbl = Table(names=["x", "y"], data=[x, y]) -# xsky, ysky = impl_utils.get_corner_sky_coords([tbl, image_hdu_square]) -# -# assert np.all((xsky == x.to_value(u.deg))) -# - - -class TestAddTableToImageHDU2: - @pytest.mark.parametrize("xpix, ypix, value", - [(51, 51, 2), - (48, 51, 3), - (48, 48, 4), - (51, 48, 5)]) - def test_integer_pixel_fluxes_are_added_correctly(self, xpix, ypix, value, - image_hdu_square): - # Given the weird behaviour on pixel boundaries - x, y = [1.5, -1.5, -1.5, 1.5]*u.arcsec, [1.5, 1.5, -1.5, -1.5]*u.arcsec - flux = [1, 2, 3, 4] * u.Unit("ph s-1") - tbl = Table(names=["x", "y", "flux"], data=[x, y, flux]) - - hdu = imp_utils.add_table_to_imagehdu(tbl, image_hdu_square, - sub_pixel=False) - - assert hdu.data[ypix, xpix] == value - - @pytest.mark.parametrize("x, y, flux, xpix, ypix, value", - [([0], [0], [1], 50, 50, .25), - ([0.7], [0.7], [1], 50, 50, 0.64), - ([-0.7], [-0.7], [1], 48, 48, 0.04), - ([5.5], [-4.7], [1], 55, 45, 0.8), - ([5.5], [-4.7], [1], 55, 44, 0.2)]) - def test_sub_pixel_fluxes_are_added_correctly(self, x, y, flux, xpix, ypix, - value, image_hdu_square): - # Given the weird behaviour on pixel boundaries - tbl = Table(names=["x", "y", "flux"], - data=[x*u.arcsec, y*u.arcsec, flux*u.Unit("ph s-1")]) - hdu = imp_utils.add_table_to_imagehdu(tbl, image_hdu_square, - sub_pixel=True) - - if PLOTS: - plt.imshow(hdu.data[45:55, 45:55], origin="lower") - plt.colorbar() - plt.show() - - assert hdu.data[ypix, xpix] == approx(value + 1) - - @pytest.mark.parametrize("x, y, flux", - [([100, -100], [0, 0], [10, 10])]) - def test_source_outside_canvas_are_ignored(self, x, y, flux, - image_hdu_square): - # Given the weird behaviour on pixel boundaries - tbl = Table(names=["x", "y", "flux"], - data=[x * u.arcsec, y * u.arcsec, - flux * u.Unit("ph s-1")]) - hdu = imp_utils.add_table_to_imagehdu(tbl, image_hdu_square, - sub_pixel=True) - - assert np.sum(hdu.data) == np.prod(hdu.data.shape) - class TestAddImagehduToImageHDU: @pytest.mark.parametrize("angle", [0, 30, 45, 89]) @@ -915,11 +525,10 @@ def test_throws_error_when_initialised_with_nothing(self): with pytest.raises(TypeError): opt_imp.ImagePlane() - def test_initialises_with_header_with_hdu(self, image_hdu_square, - image_hdu_rect): - hdr = imp_utils.get_canvas_header(pixel_scale=0.1 * u.arcsec, - hdu_or_table_list=[image_hdu_rect, - image_hdu_square]) + def test_initialises_with_header_with_hdu( + self, image_hdu_square, image_hdu_rect, + ): + hdr = get_canvas_header([image_hdu_rect, image_hdu_square]) implane = opt_imp.ImagePlane(hdr) assert isinstance(implane, opt_imp.ImagePlane) assert isinstance(implane.hdu, fits.ImageHDU) diff --git a/scopesim/tests/tests_optics/test_ImagePlaneUtils.py b/scopesim/tests/tests_optics/test_ImagePlaneUtils.py index 8658dd423..c0116087a 100644 --- a/scopesim/tests/tests_optics/test_ImagePlaneUtils.py +++ b/scopesim/tests/tests_optics/test_ImagePlaneUtils.py @@ -7,33 +7,13 @@ import matplotlib.pyplot as plt from scopesim.optics import image_plane_utils as imp_utils +from scopesim.optics.fov_manager import chunk_edges from scopesim.tests.mocks.py_objects import imagehdu_objects as imo PLOTS = False -class TestSplitHeader: - def test_is_the_header_split_into_the_right_amount_of_chunks(self): - hdr = imp_utils.header_from_list_of_xy([-1.024, 1.024], - [-1.024, 1.024], 0.002) - hdrs = imp_utils.split_header(hdr, 128) - area_sum = np.sum([hdr["NAXIS1"] * hdr["NAXIS2"] for hdr in hdrs]) - assert len(hdrs) == 64 - assert area_sum == hdr["NAXIS1"] * hdr["NAXIS2"] - - @pytest.mark.parametrize("x, y, pix", [(0.19, 0.2, 0.01), - (2.19, 1.55, 0.01), - (1.29, 1.2, 0.02), - (2.55, 3.75, 1)]) - def test_final_header_is_smaller_for_odd_size(self, x, y, pix): - hdr = imp_utils.header_from_list_of_xy([-1., x], - [-1., y], pix) - hdrs = imp_utils.split_header(hdr, 100) - area_sum = np.sum([hdr["NAXIS1"] * hdr["NAXIS2"] for hdr in hdrs]) - assert area_sum == hdr["NAXIS1"] * hdr["NAXIS2"] - - class TestAddImageHDUtoImageHDU: def big_small_hdus(self, big_wh=(20, 10), big_offsets=(0, 0), small_wh=(6, 3), small_offsets=(0, 0), pixel_scale=0.1): @@ -229,6 +209,93 @@ def test_overlay_images_works_for_3d_cubes_at_border(self): plt.show() +class TestLatticeIsSharedBetweenChunks: + """Regions carved from one origin in whole-pixel steps share a lattice. + + This is what chunked FOVs are: the image plane, the full-size chunks and + the smaller last chunk are all measured from the same detector edge. When + the region is not a whole number of pixels across, CRVAL used to absorb + each region's own rounding remainder, putting them on lattices offset from + one another by fractions of a pixel. + """ + + @pytest.mark.parametrize("extent_px", [24.0, 21.0, 17.32, 17.1, 12621 + 1/3]) + @pytest.mark.parametrize("chunk", [4, 8]) + def test_chunks_land_on_the_parent_lattice(self, extent_px, chunk): + pixel_scale = 0.1 + lo = -extent_px / 2 * pixel_scale + hi = lo + extent_px * pixel_scale + + parent, parent_naxis = imp_utils.create_wcs_from_points( + np.array([[lo, lo], [hi, hi]]), pixel_scale) + + # the real chunk-edge helper, not a copy of it + edges = [lo, *chunk_edges(lo, hi, chunk * pixel_scale), hi] + + for x0, x1 in zip(edges[:-1], edges[1:]): + child, child_naxis = imp_utils.create_wcs_from_points( + np.array([[x0, x0], [x1, x1]]), pixel_scale) + # the child's pixel 0 must sit on a whole-pixel offset of the + # parent's grid, otherwise the projection has to snap it + world = child.wcs_pix2world([[0, 0]], 0) + pix = parent.wcs_world2pix(world, 0)[0] + np.testing.assert_allclose(pix, np.round(pix), atol=1e-6) + + @pytest.mark.parametrize("pnts, pixel_scale", [ + (np.array([[0, 0]]), 5), # single point + (np.array([[-1, -1], [-1, 1], [1, -1], [1, 1]]), 5), # sub-pixel + ]) + def test_regions_smaller_than_a_pixel_stay_centred(self, pnts, pixel_scale): + wcs, naxis = imp_utils.create_wcs_from_points(pnts, pixel_scale) + np.testing.assert_array_equal(naxis, [1, 1]) + np.testing.assert_array_equal(wcs.wcs.crval, + pnts.mean(axis=0)) + + +class TestOverlayImageSnapping: + """The origin, not the centre, must be snapped to the pixel grid. + + ``ceil(coords) - shape // 2`` snapped the centre instead, which is off by + up to a whole pixel whenever ``coords`` is not on the lattice implied by + ``small_im.shape``, in a direction that flips with the parity of that + shape. Adjacent FOVs of different sizes were therefore displaced opposite + ways, and their shared edge gained a duplicated or a dropped row/column. + """ + + @pytest.mark.parametrize("size", [1, 2, 3, 4, 7, 8]) + def test_places_image_on_its_own_lattice_without_shift(self, size): + big = np.zeros((32, 32)) + small = np.ones((size, size)) + # centre coordinate that the shape's own lattice implies + centre = 10 + (size - 1) / 2 + imp_utils.overlay_image(small, big, (centre, centre)) + rows = np.where(big.any(axis=1))[0] + assert rows[0] == 10 + assert rows[-1] == 10 + size - 1 + + @pytest.mark.parametrize("size", [1, 2, 3, 4, 7, 8]) + def test_snaps_sub_pixel_offset_the_same_way_for_any_shape(self, size): + # A 0.3 pix offset must never move the image by a whole pixel, whatever + # the parity of the shape. + big = np.zeros((32, 32)) + small = np.ones((size, size)) + centre = 10 + (size - 1) / 2 + 0.3 + imp_utils.overlay_image(small, big, (centre, centre)) + rows = np.where(big.any(axis=1))[0] + assert rows[0] == 10 + assert rows[-1] == 10 + size - 1 + + def test_adjacent_differently_sized_images_tile_exactly(self): + # Mimics chunked FOVs, where the last chunk is smaller than the rest. + big = np.zeros((1, 20)) + origin = 0 + for size in (8, 8, 4): + small = np.ones((1, size)) + imp_utils.overlay_image(small, big, (origin + (size - 1) / 2, 0)) + origin += size + assert (big == 1).all() + + class TestRescaleImageHDU: @pytest.mark.parametrize("pixel_scale", [0.3, 0.5, 1, 2, 3]) def test_rescales_a_2D_imagehdu(self, pixel_scale): diff --git a/scopesim/tests/tests_optics/test_OpticalTrain.py b/scopesim/tests/tests_optics/test_OpticalTrain.py index 581b5e29f..1585e350d 100644 --- a/scopesim/tests/tests_optics/test_OpticalTrain.py +++ b/scopesim/tests/tests_optics/test_OpticalTrain.py @@ -23,6 +23,11 @@ Effect, SpectralTraceList, UnequalBinnedImage, + OnChipBinning, + BasicReadoutNoise, + ImagePlaneBackground, + DetectorModePropertiesSetter, + SelectorWheel, ) from scopesim.utils import find_file @@ -197,6 +202,33 @@ def test_ignore_effects_works(self, cmds_with_ignore): class TestTraceDetectorCoordinates: + def test_explicit_detector_mode_resolves_its_binning_before_mapping(self): + train, _ = spectral_geometry_train() + before = train.trace_detector_coordinates() + train.cmds = UserCommands(properties={'!OBS.detector_readout_mode': 'slow'}) + setter = DetectorModePropertiesSetter(mode_properties={'slow': {'!DET.binx': 2}}, cmds=train.cmds) + train.optics_manager.detector_array_effects = [setter] + train.optics_manager.detector_effects = [OnChipBinning(binx='!DET.binx', biny=1, cmds=train.cmds)] + after = train.trace_detector_coordinates() + np.testing.assert_allclose(after['detector_x'], (before['detector_x'] + .5*u.pixel)/2 - .5*u.pixel) + np.testing.assert_array_equal(after['detector_y'], before['detector_y']) + + @pytest.mark.parametrize('selected', [False, True]) + def test_additive_background_preserves_trace_coordinates(self, selected): + train, _ = spectral_geometry_train() + before = train.trace_detector_coordinates() + background = SelectorWheel(selector_key='image_plane_id', wheel=[{'selector_value': [0], 'effect_class': 'ImagePlaneBackground', 'effect_kwargs': {'value': 1.}}]) if selected else ImagePlaneBackground(value=1.) + train.optics_manager.image_plane_effects = [background] + after = train.trace_detector_coordinates() + np.testing.assert_array_equal(before['detector_x'], after['detector_x']) + np.testing.assert_array_equal(before['detector_y'], after['detector_y']) + + def test_unsupported_image_plane_transform_is_not_silently_ignored(self): + train, _ = spectral_geometry_train() + train.optics_manager.image_plane_effects = [Effect()] + with pytest.raises(NotImplementedError, match='additive'): + train.trace_detector_coordinates() + def test_maps_live_trace_to_zero_origin_detector_pixels(self): train, _ = spectral_geometry_train() @@ -205,8 +237,8 @@ def test_maps_live_trace_to_zero_origin_detector_pixels(self): assert coordinates["trace_id"][0] == "linear" assert coordinates["readout_index"][0] == 0 - assert coordinates["detector_x"][0].to_value(u.pixel) == approx(49) - assert coordinates["detector_y"][0].to_value(u.pixel) == approx(39) + assert coordinates['detector_x'][0].to_value(u.pixel) == approx(49.5) + assert coordinates['detector_y'][0].to_value(u.pixel) == approx(39.5) def test_includes_renderer_endpoint_sampling(self): train, _ = spectral_geometry_train() @@ -214,12 +246,9 @@ def test_includes_renderer_endpoint_sampling(self): coordinates = train.trace_detector_coordinates( wavelengths={"linear": [1.25] * u.um}) - # The rendered subimage samples -0.65..0.65 mm at 13 inclusive - # positions. The trace's -0.25 mm point therefore lands here, rather - # than at the ideal detector-WCS coordinate x=47. - expected_x = 43 + (-0.25 + 0.65) / (0.65 + 0.65) * 12 - assert coordinates["detector_x"][0].to_value(u.pixel) == approx( - expected_x) + # The upstream renderer now uses native detector centers, excluding + # the next pixel beyond the subimage's last column. + assert coordinates['detector_x'][0].to_value(u.pixel) == approx(47.) def test_returns_all_trace_wavelengths_and_requested_slit_positions(self): train, _ = spectral_geometry_train() @@ -247,8 +276,8 @@ def test_uses_changed_live_trace_transform(self): wavelengths={"linear": [1.5] * u.um}) trace = trace_list.spectral_traces["linear"] - trace.meta["offset_y"] = 0.1 - trace.compute_interpolation_functions() + trace.xilam2y.posttransform = (lambda y: y + .1, {}) + trace.xy2lam.pretransform_y = (lambda y: y - .1, {}) after = train.trace_detector_coordinates( wavelengths={"linear": [1.5] * u.um}) @@ -261,14 +290,19 @@ def test_rejects_multiple_detectors_on_one_image_plane(self): with pytest.raises(NotImplementedError, match="multiple detectors"): train.trace_detector_coordinates() - def test_rejects_post_extraction_binning(self): + @pytest.mark.parametrize('binning_class', [UnequalBinnedImage, OnChipBinning]) + def test_binned_coordinates_match_before_and_after_readout_geometry(self, binning_class): train, _ = spectral_geometry_train() - train.optics_manager.detector_effects = [ - UnequalBinnedImage(binx=2, biny=1), - ] - - with pytest.raises(NotImplementedError, match="binning"): - train.trace_detector_coordinates() + native = train.trace_detector_coordinates() + binning = binning_class(binx=2, biny=1) + train.optics_manager.detector_effects = [binning] + before = train.trace_detector_coordinates() + np.testing.assert_allclose(before['detector_x'], (native['detector_x'] + .5*u.pixel)/2 - .5*u.pixel) + np.testing.assert_allclose(before['detector_y'], native['detector_y']) + binning.apply_to(train.detector_managers[0][0]) + after = train.trace_detector_coordinates() + np.testing.assert_array_equal(before['detector_x'], after['detector_x']) + np.testing.assert_array_equal(before['detector_y'], after['detector_y']) @pytest.mark.slow @@ -381,6 +415,28 @@ def test_works_with_a_pointer_to_fits_imagehdu(self, cmds): @pytest.mark.usefixtures("patch_mock_path") class TestReadout: + def test_explicit_readout_keys_and_reset(self, unity_cmds): + opt = OpticalTrain(deepcopy(unity_cmds)) + for seconds in (3600., 300., 3600.): + opt.readout(**{'!OBS.dit': seconds, '!OBS.ndit': 1, '!OBS.dit_blue': seconds / 2}) + assert opt.cmds['!OBS.dit'] == seconds + assert opt.cmds['!OBS.dit_blue'] == seconds / 2 + opt.readout(dit=17., ndit=2) + assert opt.cmds['!OBS.dit'] == 17. + assert opt.cmds['!OBS.ndit'] == 2 + + def test_successive_noise_realizations_can_be_replayed(self, unity_cmds, unity_src): + commands = deepcopy(unity_cmds) + commands['!SIM.random.seed'] = 17 + opt = OpticalTrain(commands) + opt.optics_manager.add_effect(BasicReadoutNoise(noise_std=3., ndit=1, cmds=opt.cmds)) + opt.observe(unity_src) + first = opt.readout()[0][1].data + second = opt.readout()[0][1].data + replay = opt.readout(roid=0)[0][1].data + assert not np.array_equal(first, second) + np.testing.assert_array_equal(first, replay) + def test_readout_works_when_source_observed(self, unity_cmds, unity_src): opt = OpticalTrain(unity_cmds) diff --git a/scopesim/tests/tests_optics/test_OpticsManager.py b/scopesim/tests/tests_optics/test_OpticsManager.py index 1c6b77cac..cd055f5d2 100644 --- a/scopesim/tests/tests_optics/test_OpticsManager.py +++ b/scopesim/tests/tests_optics/test_OpticsManager.py @@ -59,7 +59,7 @@ def test_effect_order_table_uses_runtime_phase_order(self, detector_setup_rows = tbl[tbl["phase"] == "detector_setup"] assert source_rows["effect"].tolist() == ["detector_qe_curve"] - assert fov_rows["effect"].tolist() == ["detector_qe_curve"] + assert len(fov_rows) == 0 # Ordinary QE is applied once, at the source. assert detector_setup_rows["effect"].tolist() == [ "micado_detector_geometry"] @@ -76,7 +76,7 @@ def test_effect_order_table_summarises_selector_wheels(self): "selector_key": "aperture_id", "wheel": [{ "selector_value": [1, 2], - "effect_class": "TERCurve", + "effect_class": "SpectralQuantumEfficiency", "effect_kwargs": {"filename": "TER_blank.dat"}, }], }, @@ -86,14 +86,14 @@ def test_effect_order_table_summarises_selector_wheels(self): opt_man = opt_mgr.OpticsManager([yaml_dict]) tbl = opt_man.effect_order_table() - source_row = tbl[tbl["phase"] == "source"][0] fov_row = tbl[tbl["phase"] == "fov"][0] - assert source_row["effect"] == "selected_qe" - assert source_row["class"] == "SelectorWheel" - assert source_row["selector_key"] == "aperture_id" - assert source_row["selector_values"] == "1, 2" - assert source_row["selected_classes"] == "TERCurve" + assert len(tbl[tbl['phase'] == 'source']) == 0 + assert fov_row['effect'] == 'selected_qe' + assert fov_row['class'] == 'SelectorWheel' + assert fov_row['selector_key'] == 'aperture_id' + assert fov_row['selector_values'] == '1, 2' + assert fov_row['selected_classes'] == 'SpectralQuantumEfficiency' assert fov_row["phase_z_orders"] == "610" diff --git a/scopesim/tests/tests_optics/test_adc_slit_illumination.py b/scopesim/tests/tests_optics/test_adc_slit_illumination.py new file mode 100644 index 000000000..3df011292 --- /dev/null +++ b/scopesim/tests/tests_optics/test_adc_slit_illumination.py @@ -0,0 +1,190 @@ +import numpy as np +import pytest +from astropy import units as u +from astropy.io import fits +from synphot import SourceSpectrum, ConstFlux1D +from synphot.units import PHOTLAM + +from scopesim import Source +from scopesim.effects.atmo_dispersion import ShiftFoV3D, _cube_wavelengths +from scopesim.effects.psfs.analytical import MoffatPSF +from scopesim.source.source_fields import BackgroundSourceField +from scopesim.optics.image_plane_utils import header_from_list_of_xy +from scopesim.tests.tests_optics.test_slit_spatial_integration import physical_fov + + +class Displacement(ShiftFoV3D): + def __init__(self, dx, dy, **kwargs): + super().__init__(**kwargs) + self.dx, self.dy = dx*u.arcsec, dy*u.arcsec + + def get_shifts(self, obj): + wave = _cube_wavelengths(obj) + return np.broadcast_to(self.dx, wave.shape, subok=True), np.broadcast_to(self.dy, wave.shape, subok=True) + + +def render(fov, source, psf, shifts): + support = psf.padded_header(fov, point_sources=not (source.image_fields or source.cube_fields)) + for effect in shifts: + support = effect._padded_header(support, effect.get_shifts(fov)) + fov.extract_from(source, spatial_header=support) + psf._apply_convolution(fov, spatial_shifts=shifts) + return fov.hdu.data*fov.pixel_area.value/1e4 + + +@pytest.mark.parametrize('width', [.33, .7, .9]) +@pytest.mark.parametrize('position,steps', [((0., 0.), (0, 1)), ((.037, .4), (0, -2)), ((5.6, .1), (-2, 1)), ((7., 0.), (-15, 0))]) +@pytest.mark.parametrize('sign', [-1, 1]) +def test_shifted_point_uses_incoming_psf_before_slit_and_ends(width, position, steps, sign): + fov = physical_fov(width) + fov.header['CDELT2'] *= sign + original = fov.header.copy() + cell = np.array([fov.header['CDELT1'], fov.header['CDELT2']])*3600 + displacement = np.array(steps)*cell + shift = Displacement(*displacement, method='roll') + psf = MoffatPSF(alpha=4.765, fwhm=.7) + source = Source(x=[position[0]], y=[position[1]], ref=[0], weight=[1.], spectra=[SourceSpectrum(ConstFlux1D, amplitude=1*PHOTLAM)]) + actual = render(fov, source, psf, [shift]) + expected = psf.point_source_image(fov, *(position + displacement)) + np.testing.assert_allclose(actual, np.broadcast_to(expected, actual.shape), rtol=2e-6, atol=1e-12) + for key in ('NAXIS1', 'NAXIS2', 'CRPIX1', 'CRPIX2', 'CDELT1', 'CDELT2', 'CRPIX1D', 'CRPIX2D'): + assert fov.header[key] == original[key] + assert fov.hdu.header[key] == original[key] + + +@pytest.mark.parametrize('method', ['roll', 'map_coords']) +def test_uniform_sky_is_not_lost_at_shifted_slit_edges(method): + fov = physical_fov(.7) + source = Source() + source.fields.append(BackgroundSourceField(field=None, header=fits.Header(), spectra={0: SourceSpectrum(ConstFlux1D, amplitude=1*PHOTLAM)})) + psf = MoffatPSF(alpha=4.765, fwhm=.7) + shift = Displacement(.23, -.17, method=method) + actual = render(fov, source, psf, [shift]) + np.testing.assert_allclose(actual, fov.pixel_area.value, rtol=1e-12) + + +def test_wavelength_dependent_shift_keeps_spectral_planes_independent(): + fov = physical_fov(.7) + count = len(fov.cube_waveset) + cell_y = fov.header['CDELT2']*3600 + displacement = np.resize([-1., 0., 1.], count)*cell_y + shift = Displacement(0., displacement, method='roll') + psf = MoffatPSF(alpha=4.765, fwhm=.7) + source = Source(x=[0], y=[.2], ref=[0], weight=[1.], spectra=[SourceSpectrum(ConstFlux1D, amplitude=1*PHOTLAM)]) + actual = render(fov, source, psf, [shift]) + expected = np.stack([psf.point_source_image(fov, 0., .2 + dy) for dy in displacement]) + np.testing.assert_allclose(actual, expected, rtol=1e-6) + + +def test_two_displacements_keep_light_that_reenters_slit(): + fov = physical_fov(.7) + step = fov.header['CDELT2']*3600 + shifts = [Displacement(0., 4*step, method='roll'), Displacement(0., -4*step, method='roll')] + psf = MoffatPSF(alpha=4.765, fwhm=.7) + source = Source(x=[0], y=[0], ref=[0], weight=[1.], spectra=[SourceSpectrum(ConstFlux1D, amplitude=1*PHOTLAM)]) + actual = render(fov, source, psf, shifts) + expected = psf.point_source_image(fov, 0., 0.) + np.testing.assert_allclose(actual, np.broadcast_to(expected, actual.shape), rtol=1e-6) + + +def test_shifted_resolved_image_and_cube_agree(): + header = header_from_list_of_xy([-1/3600, 1/3600], [-1/3600, 1/3600], .05/3600) + yy, xx = np.indices((header['NAXIS2'], header['NAXIS1'])) + density = np.exp(-((xx-20)**2 + (yy-28)**2)/8) + image_source = Source(image_hdu=fits.ImageHDU(data=density*.05**2, header=header), spectra=[SourceSpectrum(ConstFlux1D, amplitude=1*PHOTLAM)]) + cube_header = header.copy() + cube_header.update(CRPIX3=1., CRVAL3=.49, CDELT3=.01, CUNIT3='um', CTYPE3='WAVE', BUNIT='photlam arcsec-2') + cube_source = Source(cube=fits.ImageHDU(data=np.broadcast_to(density, (13, *density.shape)).copy(), header=cube_header)) + psf = MoffatPSF(alpha=4.765, fwhm=.7) + shift = Displacement(.15, -.2, method='roll') + images = [render(physical_fov(.7), source, psf, [shift]) for source in (image_source, cube_source)] + np.testing.assert_allclose(images[0], images[1], rtol=2e-6, atol=1e-10) + fov = physical_fov(.7) + cell = np.array([fov.header['CDELT1'], fov.header['CDELT2']])*3600 + displacement = np.rint(np.round(np.array([.15, -.2])/cell, 1))*cell + moved = image_source.make_copy() + moved.shift(*displacement) + expected = render(fov, moved, psf, []) + np.testing.assert_allclose(images[0], expected, rtol=2e-6, atol=1e-10) + + +@pytest.mark.parametrize('method', ['roll', 'map_coords']) +def test_zero_shift_reproduces_unshifted_render(method): + psf = MoffatPSF(alpha=4.765, fwhm=.7) + source = Source(x=[.1], y=[.2], ref=[0], weight=[1.], spectra=[SourceSpectrum(ConstFlux1D, amplitude=1*PHOTLAM)]) + unshifted = render(physical_fov(.7), source, psf, []) + shift = Displacement(0., 0., method=method) + fov = physical_fov(.7) + assert shift._padded_header(fov.header, shift.get_shifts(fov)) == fov.header + shifted = render(fov, source, psf, [shift]) + np.testing.assert_allclose(unshifted, shifted, rtol=1e-14, atol=1e-14) + + +@pytest.mark.parametrize('position', [(0., 0.), (.1, .4), (5.6, 0.)]) +def test_fractional_shift_agrees_with_pre_slit_psf_flux_and_centroid(position): + fov = physical_fov(.7) + cell = np.array([fov.header['CDELT1'], fov.header['CDELT2']])*3600 + shift = Displacement(*(np.array([.3, -.4])*cell), method='map_coords') + psf = MoffatPSF(alpha=4.765, fwhm=.7) + source = Source(x=[position[0]], y=[position[1]], ref=[0], weight=[1.], spectra=[SourceSpectrum(ConstFlux1D, amplitude=1*PHOTLAM)]) + actual = render(fov, source, psf, [shift])[0] + expected = psf.point_source_image(fov, *(position + np.array([.3, -.4])*cell)) + assert actual.sum() == pytest.approx(expected.sum(), rel=1e-3) + yy, xx = np.indices(actual.shape) + for coordinate in (xx, yy): + measured = np.sum(actual*coordinate)/actual.sum() + reference = np.sum(expected*coordinate)/expected.sum() + assert measured == pytest.approx(reference, abs=.01) + + +@pytest.mark.parametrize('intervening_throughput', [False, True]) +def test_observe_applies_shift_before_slit_through_wavelength_throughput(intervening_throughput): + from types import SimpleNamespace + from scopesim import OpticalTrain + from scopesim.effects import TERCurve + + psf = MoffatPSF(alpha=4.765, fwhm=.7) + images = [] + for move_source in (False, True): + fov = physical_fov(.7) + displacement = -2*fov.header['CDELT2']*3600 + source_y = .4 + displacement if move_source else .4 + source = Source(x=[0], y=[source_y], ref=[0], weight=[1.], spectra=[SourceSpectrum(ConstFlux1D, amplitude=1*PHOTLAM)]) + effects = [psf] + if intervening_throughput: + effects.append(TERCurve(array_dict={'wavelength': [.4, .7], 'transmission': [.2, .8]}, wavelength_unit='um')) + if not move_source: + effects.append(Displacement(0., displacement, method='roll')) + train = OpticalTrain() + train.cmds = fov.cmds + train.fov_manager = SimpleNamespace(fovs=[fov], is_spectroscope=True) + train.optics_manager = SimpleNamespace(source_effects=[], fov_effects=effects, image_plane_effects=[]) + train.image_planes = [SimpleNamespace(add=lambda hdu, **kwargs: images.append(hdu.data.copy()))] + train.observe(source, update=False) + train.observe(source, update=False) + for image in images[1:]: + np.testing.assert_allclose(images[0], image, rtol=2e-6, atol=1e-12) + + +@pytest.mark.parametrize('method', ['roll', 'map_coords']) +@pytest.mark.parametrize('fraction', [1e-12, .04, .54, .56]) +def test_final_padding_uses_applied_displacement_but_source_support_is_conservative(method, fraction): + fov = physical_fov(.7) + cell_y = fov.header['CDELT2']*3600 + shift = Displacement(1e-14, fraction*cell_y, method=method) + shifts = shift.get_shifts(fov) + source_support = shift._padded_header(fov.header, shifts) + final_support = shift._padded_header(fov.header, shifts, rounded=True) + assert source_support['NAXIS2'] > fov.header['NAXIS2'] + assert final_support['NAXIS1'] == fov.header['NAXIS1'] + applied = np.round(fraction, 1) + if method == 'roll': + applied = np.round(np.float32(applied)) + padding = int(np.ceil(abs(applied))) + (4 if method == 'map_coords' else 0) if applied else 0 + assert final_support['NAXIS2'] == fov.header['NAXIS2'] + 2*padding + if applied == 0: + psf = MoffatPSF(alpha=4.765, fwhm=.7) + source = Source(x=[.1], y=[.2], ref=[0], weight=[1.], spectra=[SourceSpectrum(ConstFlux1D, amplitude=1*PHOTLAM)]) + actual = render(fov, source, psf, [shift]) + expected = render(physical_fov(.7), source, psf, []) + np.testing.assert_array_equal(actual, expected) diff --git a/scopesim/tests/tests_optics/test_echelle.py b/scopesim/tests/tests_optics/test_echelle.py new file mode 100644 index 000000000..2370f03dd --- /dev/null +++ b/scopesim/tests/tests_optics/test_echelle.py @@ -0,0 +1,102 @@ +"""Planar grating geometry checked through the camera's forward projection.""" + +import numpy as np +import pytest +from astropy import units as u + +from scopesim.optics.echelle import GratingSetup, spectrograph_factory + + +# Wavelength endpoints [nm], cross-disperser density [mm^-1], detector height [pix]. +CHANNELS = [(310, 430, 2000, 4096), (430, 650, 1100, 4096), (650, 980, 625, 4096), + (960, 1340, 350, 2048), (1470, 1830, 300, 2048), (1990, 2450, 270, 2048)] + + +def test_asymmetric_pupil_slit_dispersion_and_anamorphism(): + instrument = spectrograph_factory(960, 1340, 225, 18000, 65.6, 58, 80, 1/24, 4.46, + 2048, 2048, .015, xdisp_groove_length=1/350, xdisp_beta_center=10.9, + collimator1_focal_length=1500*u.mm, collimator2_focal_length=1250*u.mm, entrance_slit_scale=.5306*u.mm/u.arcsec) + assert instrument.dispersion_focal_length == 270*u.mm + center = instrument.central_wave(65) + derivative = instrument.slit_dispersion(center, 65) + assert derivative < 0*u.um/u.arcsec + for slit, expected in ((.7, 17800), (.33, 37800)): + resolution = (center / abs(derivative * slit*u.arcsec)).to_value(u.dimensionless_unscaled) + assert resolution == pytest.approx(expected, rel=.01) + width = instrument.projected_slit_width(center, 65, .7*u.arcsec) + assert width.to_value(u.pixel) == pytest.approx(4.45704, rel=1e-5) + wavelengths = center * np.array([.99, 1., 1.01]) + widths = instrument.projected_slit_width(wavelengths, 65, .7*u.arcsec) + assert np.all(np.diff(widths) > 0*u.pixel) + half_angle = (.35*u.arcsec * instrument.entrance_slit_scale / instrument.collimator1_focal_length).value * u.rad + beta_low = np.arcsin(65*wavelengths/instrument.grating.d - np.sin(instrument.grating.alpha + half_angle)) + beta_high = np.arcsin(65*wavelengths/instrument.grating.d - np.sin(instrument.grating.alpha - half_angle)) + edge_width = abs(instrument.x_pixel_for_beta(beta_high) - instrument.x_pixel_for_beta(beta_low)) + # Independent finite slit edges retain the same anamorphism as dispersion. + np.testing.assert_allclose(widths.to_value(u.pixel), edge_width.value, rtol=2e-7) + step = center*1e-7 + dispersion = (instrument.wavelength_to_x_pixel(wavelengths+step, 65)-instrument.wavelength_to_x_pixel(wavelengths-step, 65))/(2*step) + measured_delta = widths.to_value(u.pixel)/dispersion + np.testing.assert_allclose(measured_delta.to_value(u.nm), abs(derivative*.7*u.arcsec).to_value(u.nm), rtol=2e-7) + eta = .0825*u.arcsec + dalpha = (eta * instrument.entrance_slit_scale / instrument.collimator1_focal_length).to_value(u.dimensionless_unscaled) * u.rad + exact = instrument.grating.d / 65 * (np.sin(instrument.grating.alpha) - np.sin(instrument.grating.alpha + dalpha)) + assert (derivative * eta).to_value(u.nm) == pytest.approx(exact.to_value(u.nm), rel=1e-4) + + +def test_incomplete_physical_prescription_fails(): + with pytest.raises(AttributeError): + spectrograph_factory(960, 1340, 225, 18000, 65.6, 58, 80, 1/24, 4.46, 2048, 2048, .015, + xdisp_groove_length=1/350, collimator1_focal_length=1500*u.mm) + + +@pytest.mark.parametrize("low,high,density,height", CHANNELS) +def test_fixed_groove_endpoints_fill_camera(low, high, density, height): + instrument = spectrograph_factory(low, high, 225, 18000, 65.6, 40, 50, 1/24, 4.46, + 2048, height, .015, xdisp_groove_length=1/density, dispersion_focal_len=270) + wavelength = np.array([low, high]) * u.nm + pixels = instrument.wavelength_to_y_pixel(wavelength) + # Algebraic inverse/forward evaluation; 1e-7 pixel allows floating-point cancellation. + np.testing.assert_allclose(pixels.value, [0, height], atol=1e-7, rtol=0) + recovered = instrument.y_pixel_to_wavelength(pixels, 1) + np.testing.assert_allclose(recovered.to_value(u.nm), wavelength.value, atol=1e-9, rtol=0) + + +def test_units_and_signed_incidence(): + numeric = GratingSetup.estimate_xdisp_angle_with_groove_len(1470, 1830, 30.72, 225, 1/300) + physical = GratingSetup.estimate_xdisp_angle_with_groove_len(1.47*u.um, 1.83*u.um, 3.072*u.cm, .225*u.m, (1/300)*u.mm) + assert physical[0] < 0 * u.rad + for plain, quantity in zip(numeric, physical): + assert plain.unit == u.rad + assert quantity.unit == u.rad + assert plain.to_value(u.rad) == pytest.approx(quantity.to_value(u.rad), abs=1e-12) + + +@pytest.mark.parametrize("inputs", [(310, 430, 1, 225, .0005), (3000, 3010, 61.44, 225, .0001), + (430, 310, 61.44, 225, .0005), (310, 430, 0, 225, .0005)]) +def test_impossible_geometry_fails(inputs): + with pytest.raises(ValueError): + GratingSetup.estimate_xdisp_angle_with_groove_len(*inputs) + + +def test_explicit_angle_keeps_configured_geometry(): + instrument = spectrograph_factory(960, 1340, 225, 18000, 65.6, 57, 80, 1/24, 4.46, + 2048, 2048, .015, xdisp_groove_length=1/350, xdisp_beta_center=12) + assert instrument.cross_disperser.alpha.to_value(u.deg) == pytest.approx(12) + assert instrument.cross_disperser.beta_center.to_value(u.deg) == pytest.approx(12) + + +def test_free_frequency_endpoints_fill_camera(): + instrument = spectrograph_factory(960, 1340, 225, 18000, 65.6, 57, 80, 1/24, 4.46, + 2048, 2048, .015, xdisp_groove_length=np.nan) + pixels = instrument.wavelength_to_y_pixel(np.array([960, 1340]) * u.nm) + np.testing.assert_allclose(pixels.value, [0, 2048], atol=1e-7, rtol=0) + + +def test_fsr_retains_the_central_pixel_wavelength_convention(): + instrument = spectrograph_factory(1990, 2450, 225, 18000, 65.6, 31, 38, 1/24, 4.46, + 2048, 2048, .015, xdisp_groove_length=1/270, dispersion_focal_len=270) + edges = instrument.edge_wave(fsr=True) + center = instrument.x_pixel_to_wavelength(instrument.detector.n_pix_x//2 + .5, instrument.orders) + np.testing.assert_allclose(edges.mean(axis=1).to_value(u.nm), center.to_value(u.nm), rtol=1e-12) + np.testing.assert_allclose(np.diff(edges, axis=1).ravel().to_value(u.nm), (center/instrument.orders).to_value(u.nm), rtol=1e-12) diff --git a/scopesim/tests/tests_optics/test_fov_utls.py b/scopesim/tests/tests_optics/test_fov_utls.py index a6acb844b..dd0821dd1 100644 --- a/scopesim/tests/tests_optics/test_fov_utls.py +++ b/scopesim/tests/tests_optics/test_fov_utls.py @@ -6,6 +6,8 @@ from synphot import Empirical1D, SourceSpectrum from synphot.units import PHOTLAM from astropy import units as u +from astropy.io import fits +from astropy.wcs import WCS from scopesim.optics.fov import FieldOfView, extract_range_from_spectrum from scopesim.optics import image_plane_utils as imp_utils @@ -151,3 +153,72 @@ def test_logs_msg_for_waverang_overlap_mismatch( extract_range_from_spectrum(spec, waverange) assert msg in caplog.text + +class TestExtractAreaKeepsTheInputPixelGrid: + """The cutout is made of input pixels, so its WCS must say so. + + The cutout WCS used to be re-derived from the world corners *before* the + slice was grown outwards to whole input pixels. That gave a WCS for a + differently sized image (``NAXISn`` disagreed with ``data.shape``, silently + corrected by ``fits.ImageHDU``, which left ``CRPIXn`` describing the wrong + array) sitting on the FOV's pixel lattice rather than the input's. The + cutout was then mis-registered by up to a whole pixel when projected, and + by differing amounts for neighbouring FOVs, so their shared edge gained a + duplicated or a dropped row/column. + """ + + @staticmethod + def _image_hdu(naxis=32, pixel_scale=0.1, crval=(0.0, 0.0)): + """Image with a distinct value in every pixel, in deg.""" + skywcs = WCS(naxis=2) + skywcs.wcs.ctype = ["LINEAR", "LINEAR"] + skywcs.wcs.cunit = ["deg", "deg"] + skywcs.wcs.cdelt = 2 * [pixel_scale / 3600] + skywcs.wcs.crval = list(crval) + skywcs.wcs.crpix = 2 * [(naxis + 1) / 2] + hdr = skywcs.to_header() + hdr["BUNIT"] = "" + data = np.arange(naxis * naxis, dtype=float).reshape(naxis, naxis) + return fits.ImageHDU(header=hdr, data=data) + + @staticmethod + def _fov(x_min, x_max, y_min, y_max, pixel_scale=0.1): + skyhdr = imp_utils.header_from_list_of_xy( + [x_min / 3600, x_max / 3600], [y_min / 3600, y_max / 3600], + pixel_scale=pixel_scale / 3600) + dethdr, _ = imp_utils.det_wcs_from_sky_wcs( + WCS(skyhdr), pixel_scale, pixel_scale / 0.01) + skyhdr.update(dethdr.to_header()) + return FieldOfView(skyhdr, [1.0, 2.0]) + + # FOVs deliberately offset from the image's pixel edges by a fraction of a + # pixel, which is what chunking produces whenever the detector extent is + # not a whole number of pixels. + @pytest.mark.parametrize("offset", [0.0, 0.16, 0.34, 0.5, 0.68, 0.84]) + def test_naxis_matches_data_shape(self, offset): + hdu = self._image_hdu() + fov = self._fov(-0.8 + offset * 0.1, 0.0 + offset * 0.1, + -0.8 + offset * 0.1, 0.0 + offset * 0.1) + cut = fov.extract_area_from_imagehdu(hdu, fov.get_corners("deg")[0]) + assert (cut.header["NAXIS2"], cut.header["NAXIS1"]) == cut.data.shape + + @pytest.mark.parametrize("offset", [0.0, 0.16, 0.34, 0.5, 0.68, 0.84]) + def test_cutout_pixels_keep_their_world_coordinates(self, offset): + hdu = self._image_hdu() + fov = self._fov(-0.8 + offset * 0.1, 0.0 + offset * 0.1, + -0.8 + offset * 0.1, 0.0 + offset * 0.1) + cut = fov.extract_area_from_imagehdu(hdu, fov.get_corners("deg")[0]) + + # every cutout value must sit at the world coordinate its value had in + # the input image + in_wcs = WCS(hdu.header, naxis=2) + cut_wcs = WCS(cut.header, naxis=2) + ny, nx = cut.data.shape + yy, xx = np.indices((ny, nx)) + world = cut_wcs.wcs_pix2world( + np.column_stack([xx.ravel(), yy.ravel()]), 0) + back = in_wcs.wcs_world2pix(world, 0).round(6) + expected = hdu.data[back[:, 1].astype(int), back[:, 0].astype(int)] + np.testing.assert_array_equal(cut.data.ravel(), expected) + # ... and land on whole input pixels, not between them + np.testing.assert_allclose(back, back.round(), atol=1e-6) diff --git a/scopesim/tests/tests_optics/test_slit_spatial_integration.py b/scopesim/tests/tests_optics/test_slit_spatial_integration.py new file mode 100644 index 000000000..2a12e4ba5 --- /dev/null +++ b/scopesim/tests/tests_optics/test_slit_spatial_integration.py @@ -0,0 +1,146 @@ +from types import SimpleNamespace +import numpy as np +import pytest +from astropy import units as u +from astropy.wcs import WCS +from astropy.io import fits +from synphot import SourceSpectrum, ConstFlux1D +from synphot.units import PHOTLAM +from scipy.special import stdtr +from scopesim import Source, UserCommands +from scopesim.effects import TERCurve, DetectorWindow +from scopesim.effects.psfs.analytical import MoffatPSF +from scopesim.optics.fov import FieldOfView3D +from scopesim.optics.fov_manager import FOVManager +from scopesim.optics.image_plane_utils import header_from_list_of_xy, det_wcs_from_sky_wcs, project_spatial_density + + +def slit_fov(width, step=.1570545): + nx, ny = int(np.ceil(11/step)), int(np.ceil(width/step)) + header = header_from_list_of_xy([-5.5/3600, 5.5/3600], [-width/7200, width/7200], step/3600) + header.update(NAXIS1=nx, NAXIS2=ny, CRPIX1=(nx+1)/2, CRPIX2=(ny+1)/2, CRVAL1=0., CRVAL2=0., CDELT1=11/nx/3600, CDELT2=width/ny/3600) + return SimpleNamespace(header=header, waveset=np.linspace(.5, .6, 16)*u.um) + + +@pytest.mark.parametrize('width', [.30, .33, .70]) +@pytest.mark.parametrize('fwhm', [.10, .14, .7]) +@pytest.mark.parametrize('offset', [0, .13, .4]) +def test_integrated_point_psf_matches_analytic_slit_loss(width, fwhm, offset): + alpha = 4.765 + psf = MoffatPSF(alpha=alpha, fwhm=fwhm, rounded_edges=False) + image = psf.point_source_image(slit_fov(width), .037, offset) + gamma = fwhm/(2*np.sqrt(2**(1/alpha) - 1)) + scale = np.sqrt(2*alpha - 2)/gamma + expected = stdtr(2*alpha - 2, (width/2 - offset)*scale) - stdtr(2*alpha - 2, (-width/2 - offset)*scale) + # The analytic comparison has an infinite slit length; the model has 11 arcsec. + assert image.sum() == pytest.approx(expected, abs=2e-5) + + +def test_off_slit_source_remains_in_psf_extraction_support(): + fov = slit_fov(.33) + psf = MoffatPSF(alpha=4.765, fwhm=.7) + padded = psf.padded_header(fov) + x, y = WCS(padded).all_world2pix(0, .4/3600, 0) + assert -.5 < x < padded['NAXIS1'] - .5 + assert -.5 < y < padded['NAXIS2'] - .5 + assert psf.point_source_image(fov, 0, .4).sum() > .1 + + +def test_kernel_cache_responds_to_seeing_and_pixel_scale(): + fov = slit_fov(.33) + psf = MoffatPSF(alpha=4.765, fwhm=.7, rounded_edges=False) + first = psf.get_kernel(fov) + np.testing.assert_array_equal(psf.get_kernel(fov), first) + psf.meta['fwhm'] = .2 + narrower = psf.get_kernel(fov) + assert narrower.max() > first.max() + fov.header['CDELT2'] /= 2 + finer = psf.get_kernel(fov) + assert finer.max() < narrower.max() + + +def physical_fov(width=.33): + header = slit_fov(width).header + detector_wcs, _ = det_wcs_from_sky_wcs(WCS(header), .1570545, 10.) + header.update(detector_wcs.to_header()) + cmds = UserCommands(properties={'!SIM.spectral.spectral_bin_width': .01}) + return FieldOfView3D(header, [.5, .6], cmds=cmds, area=1*u.cm**2) + + +def test_throughput_then_psf_admits_off_slit_point_light(): + fov = physical_fov() + psf = MoffatPSF(alpha=4.765, fwhm=.7, rounded_edges=False) + source = Source(x=[.037], y=[.4], ref=[0], weight=[1.], spectra=[SourceSpectrum(ConstFlux1D, amplitude=1*PHOTLAM)]) + fov.extract_from(source, spatial_header=psf.padded_header(fov)) + transmission = TERCurve(array_dict={'wavelength': [.4, .7], 'transmission': [.5, .5]}, wavelength_unit='um') + transmission.apply_to(fov) + psf.apply_to(fov) + expected = psf.point_source_image(fov, .037, .4).sum()*.5 + actual = fov.hdu.data.sum(axis=(1, 2))*fov.pixel_area.value/1e4 + np.testing.assert_allclose(actual, expected, rtol=1e-6) + + +@pytest.mark.parametrize('sign', [-1, 1]) +def test_surface_brightness_uses_signed_wcs_and_pixel_overlaps(sign): + header = header_from_list_of_xy([-.8/3600, .8/3600], [-.6/3600, .6/3600], .1/3600) + header['CDELT1'] *= sign + field = fits.ImageHDU(data=np.full((header['NAXIS2'], header['NAXIS1']), 7.), header=header) + target = slit_fov(.33).header + projected = project_spatial_density(field, target) + # Only the small input's length illuminates this 11-arcsec slit. + expected_area = header['NAXIS1']*.1*.33 + assert projected.sum()*abs(target['CDELT1']*target['CDELT2'])*3600**2 == pytest.approx(7*expected_area, rel=1e-6) + + +def test_resolved_image_and_cube_have_same_psf_and_spectral_normalization(): + header = header_from_list_of_xy([-1/3600, 1/3600], [-1/3600, 1/3600], .05/3600) + yy, xx = np.indices((header['NAXIS2'], header['NAXIS1'])) + density = np.exp(-((xx - 20)**2 + (yy - 26)**2)/8) + image = fits.ImageHDU(data=density*.05**2, header=header) + spectrum = SourceSpectrum(ConstFlux1D, amplitude=1*PHOTLAM) + image_source = Source(image_hdu=image, spectra=[spectrum]) + cube_header = header.copy() + cube_header.update(CRPIX3=1., CRVAL3=.49, CDELT3=.01, CUNIT3='um', CTYPE3='WAVE', BUNIT='photlam arcsec-2') + cube = fits.ImageHDU(data=np.broadcast_to(density, (13, *density.shape)).copy(), header=cube_header) + cube_source = Source(cube=cube) + psf = MoffatPSF(alpha=4.765, fwhm=.7, rounded_edges=False) + rendered = [] + for source in (image_source, cube_source): + fov = physical_fov() + fov.extract_from(source, spatial_header=psf.padded_header(fov)) + psf.apply_to(fov) + rendered.append(fov.hdu.data) + np.testing.assert_allclose(rendered[0], rendered[1], rtol=2e-6, atol=1e-10) + + +@pytest.mark.parametrize('width', [.30, .33, .70]) +def test_fov_manager_tiles_physical_slit_width(width): + class SlitVolume: + def apply_to(self, volumes, **kwargs): + for volume in volumes: + volume.update(x_min=-5.5, x_max=5.5, y_min=-width/2, y_max=width/2) + volume['meta'].update(eta_min=-width/2*u.arcsec, eta_max=width/2*u.arcsec, image_plane_id=0) + return volumes + + detector = DetectorWindow(pixel_size=.015, x=0, y=0, width=100, height=100, units='pixel', image_plane_id=0, pixel_scale=.1570545, plate_scale=10.) + manager = FOVManager([SlitVolume(), detector], pixel_scale=.1570545, plate_scale=10., preload_fovs=False) + manager.is_spectroscope = True + fov = next(manager.generate_fovs_list()) + assert fov.header['NAXIS2']*fov.header['CDELT2']*3600 == pytest.approx(width) + assert fov.header['NAXIS1']*fov.header['CDELT1']*3600 == pytest.approx(11.) + + +@pytest.mark.parametrize('edge', ['left', 'right']) +def test_subpixel_deposition_does_not_wrap_or_index_beyond_slit(edge): + fov = physical_fov() + fov.meta['sub_pixel'] = True + xpixel = -.25 if edge == 'left' else fov.header['NAXIS1'] - .75 + ypixel = (fov.header['NAXIS2'] - 1)/2 + x, y = WCS(fov.header, naxis=2).all_pix2world(xpixel, ypixel, 0) + source = Source(x=[x*3600], y=[y*3600], ref=[0], weight=[1.], spectra=[SourceSpectrum(ConstFlux1D, amplitude=1*PHOTLAM)]) + fov.extract_from(source) + cube = fov.make_hdu() + admitted = cube.data.sum(axis=(1, 2))*fov.pixel_area.value/1e4 + np.testing.assert_allclose(admitted, .75, rtol=1e-6) + opposite = -1 if edge == 'left' else 0 + np.testing.assert_array_equal(cube.data[:, :, opposite], 0.) diff --git a/scopesim/tests/tests_optics/test_spectral_bin_integration.py b/scopesim/tests/tests_optics/test_spectral_bin_integration.py new file mode 100644 index 000000000..a037bb5b5 --- /dev/null +++ b/scopesim/tests/tests_optics/test_spectral_bin_integration.py @@ -0,0 +1,78 @@ +import numpy as np +import pytest +from astropy import units as u +from synphot import SourceSpectrum, Empirical1D +from synphot.units import PHOTLAM +from scopesim.optics.fov import average_spectrum, spectral_bin_edges, rebin_spectral_density +from scopesim.optics.optical_train import OpticalTrain +from scopesim.source.source import Source +from scopesim.source.source_templates import ab_spectrum + + +@pytest.mark.parametrize('resolving_power', [20000, 200000]) +@pytest.mark.parametrize('phase', [0, .2, .5, .8]) +def test_narrow_line_photons_survive_coarse_grid_phase(resolving_power, phase): + center = 5000 + phase*.1 + sigma = center / resolving_power / np.sqrt(8*np.log(2)) + native = center + np.linspace(-8, 8, 193)*sigma + flux = np.exp(-.5*((native - center)/sigma)**2) / (sigma*np.sqrt(2*np.pi)) + spectrum = SourceSpectrum(Empirical1D, points=native*u.AA, lookup_table=flux*PHOTLAM) + wave = np.arange(4998, 5002, .1)*u.AA + density = average_spectrum(spectrum, wave) + photons = (density * np.diff(spectral_bin_edges(wave.to_value(u.AA)))*u.AA).sum() + assert photons.to_value(u.ph/u.s/u.cm**2) == pytest.approx(1, rel=.001) + + +def test_flat_ab_continuum_bin_integral(): + spectrum = ab_spectrum(mag=20) + wave = np.linspace(5000, 5100, 101)*u.AA + edges = spectral_bin_edges(wave.to_value(u.AA)) + # Constant f_nu gives photon density proportional to 1/lambda. + scale = spectrum(5000*u.AA).to_value(PHOTLAM)*5000 + expected = scale*np.log(edges[1:]/edges[:-1])/np.diff(edges) + np.testing.assert_allclose(average_spectrum(spectrum, wave).to_value(PHOTLAM), expected, rtol=1e-6) + + +def test_mutating_spectrum_redshift_updates_native_sampling(): + points = np.array([4999., 4999.99, 5000., 5000.01, 5001.])*u.AA + spectrum = SourceSpectrum(Empirical1D, points=points, lookup_table=[0., 0., 1., 0., 0.]) + average_spectrum(spectrum, np.arange(4998., 5002., .1)*u.AA) + spectrum.z = .01 + fresh = SourceSpectrum(Empirical1D, points=points, lookup_table=[0., 0., 1., 0., 0.], z=.01) + wave = np.arange(5048., 5052., .1)*u.AA + actual = average_spectrum(spectrum, wave) + np.testing.assert_allclose(actual, average_spectrum(fresh, wave), rtol=1e-12) + assert actual.sum() > 0 + + +@pytest.mark.parametrize('descending', [False, True]) +def test_cube_rebinning_preserves_flux_spatial_weights_and_support(descending): + wave = np.array([1., 2., 3.]) + weights = np.array([[1., 2.], [3., 4.]]) + density = wave[:, None, None]*weights + if descending: + wave, density = wave[::-1], density[::-1] + rebinned = rebin_spectral_density(wave, density, np.array([.5, 1.5, 2.5, 3.5])) + np.testing.assert_allclose(rebinned, np.array([.625, 2., 1.375])[:, None, None]*weights) + np.testing.assert_allclose(rebinned.sum(axis=0), 4*weights) + + +def test_prepare_source_keeps_native_line_sampling(): + wave = np.linspace(4999.9, 5000.1, 101)*u.AA + spectrum = SourceSpectrum(Empirical1D, points=wave, lookup_table=np.exp(-((wave.value - 5000)/.01)**2)*PHOTLAM) + source = Source(x=[0.], y=[0.], ref=[0], weight=[1.], spectra=[spectrum]) + train = OpticalTrain() + prepared = train.prepare_source(source) + np.testing.assert_array_equal(prepared.spectra[0].waveset, spectrum.waveset) + + +def test_prepare_cube_retains_faint_contrast_in_double_precision(): + from scopesim.tests.mocks.py_objects.source_objects import _cube_source + source = _cube_source() + field = source.cube_fields[0] + field.data.fill(1e8) + field.data[:, 0, 0] += 1. + prepared = OpticalTrain().prepare_source(source) + data = prepared.cube_fields[0].data + contrast = (data[:, 0, 0] - data[:, 0, 1])/data[:, 0, 1] + np.testing.assert_allclose(contrast, 1e-8, rtol=1e-6) diff --git a/scopesim/tests/tests_reports/test_rst_utils.py b/scopesim/tests/tests_reports/test_rst_utils.py index aca00adef..fee66a7ef 100644 --- a/scopesim/tests/tests_reports/test_rst_utils.py +++ b/scopesim/tests/tests_reports/test_rst_utils.py @@ -10,6 +10,7 @@ from scopesim.tests.mocks.py_objects import report_objects as ro from scopesim.tests.mocks.py_objects import effects_objects as eo +pytestmark = pytest.mark.skip # whole file CLEAN_UP = True PLOTS = False diff --git a/scopesim/tests/tests_source/source_helpers.py b/scopesim/tests/tests_source/source_helpers.py new file mode 100644 index 000000000..1a92fb99a --- /dev/null +++ b/scopesim/tests/tests_source/source_helpers.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- +"""Aux function for source testing.""" + +import numpy as np +from scipy import integrate +from astropy import units as u +from synphot import SourceSpectrum, SpectralElement + + +# This used to be in source_utils, but was ultimately only used in a few tests, +# so moved here as a simple helper function. +def photons_in_range( + spectra: SourceSpectrum, + wave_min: u.Quantity[u.um] | float, + wave_max: u.Quantity[u.um] | float, + area: u.Quantity[u.m**2] | float | None = None, + bandpass: SpectralElement | None = None, +) -> u.Quantity[u.ph * u.s**-1 * u.m**-2] | u.Quantity[u.ph * u.s**-1]: + """ + Integrate photons from spectrum in given wavelength range. + + Parameters + ---------- + spectra : SourceSpectrum + Input spectrum. + wave_min : u.Quantity["length"] or float + Minimum wavelength. If float, assumes um. + wave_max : u.Quantity["length"] or float + Maximum wavelength. If float, assumes um. + area : u.Quantity["area"] or float, optional + Area to multiply with. If float, assumes m**2. The default is None. + bandpass : SpectralElement, optional + Filter to take into account, if any. The default is None. + + Returns + ------- + counts : astropy.units.Quantity + Either in ph/s/m**2 or just ph/s (if area was given). + + """ + # Note: Assuming um if given as float. + wave_min = (wave_min << u.um << u.Angstrom).value + wave_max = (wave_max << u.um << u.Angstrom).value + # Note: There appear to be some float shenanigans going on here, but + # rounding produces an error in the spectrum evaluation. Not sure what's + # going on, maybe it's fine as-is. + + counts = [] + for spec in spectra: + waveset = spec.waveset.value + mask = (waveset > wave_min) * (waveset < wave_max) + wave = np.array([wave_min, *waveset[mask], wave_max]) + flux = spec(wave).value + + # flux [ph s-1 cm-2] == flux [ph s-1 cm-2 AA-1] * wave [AA] + if isinstance(bandpass, SpectralElement): + bandpass.model.bounds_error = True + counts.append(integrate.trapezoid(bandpass(wave).value * flux, wave)) + else: + counts.append(integrate.trapezoid(flux, wave)) + + # counts = flux [ph s-1 cm-2] + counts = (counts * u.ph * u.s**-1 * u.cm**-2).to(u.ph * u.s**-1 * u.m**-2) + if area is not None: + counts *= (area << u.m**2) + + return counts + + +# This used to be a method of Source, but was ultimately only used in two tests, +# so moved here as a simple helper function. +def src_photons_in_range(src, wave_min, wave_max, area=None): + """ + + Parameters + ---------- + wave_min : float, u.Quantity + [um] + wave_max : float, u.Quantity + [um] + area : float, u.Quantity, optional + [m2] + + Returns + ------- + counts : u.Quantity list + [ph / s / m2] if area is None + [ph / s] if area is passed + + """ + indices = src.spectra.keys() + spectra = [src.spectra[ii] for ii in indices] + counts = photons_in_range(spectra, wave_min, wave_max, area=area, + bandpass=src.bandpass) + return counts diff --git a/scopesim/tests/tests_source/test_source_Source.py b/scopesim/tests/tests_source/test_source_Source.py index feea28e44..2f5b06fbe 100644 --- a/scopesim/tests/tests_source/test_source_Source.py +++ b/scopesim/tests/tests_source/test_source_Source.py @@ -1,8 +1,10 @@ -# actually for Source +# -*- coding: utf-8 -*- + import pytest from pytest import approx import numpy as np +import matplotlib.pyplot as plt from astropy.io import fits from astropy.io import ascii as ioascii @@ -10,7 +12,7 @@ from astropy import units as u from astropy import wcs -from synphot import SourceSpectrum, SpectralElement +from synphot import SourceSpectrum from synphot.models import Empirical1D from synphot.units import PHOTLAM @@ -18,14 +20,10 @@ from scopesim.source.source import Source from scopesim.source.source_fields import CubeSourceField, TableSourceField -from scopesim.optics.image_plane import ImagePlane from scopesim.utils import convert_table_comments_to_dict from scopesim.tests.mocks.py_objects import source_objects as so -import matplotlib.pyplot as plt -from matplotlib.colors import LogNorm - PLOTS = False @@ -261,83 +259,6 @@ def test_empty_source_is_the_additive_identity(self, image_source): assert len(new_source_2.fields) == len(new_source_2._meta_dicts) -class TestSourceImageInRange: - @pytest.mark.skip(reason=( - "This was the only place where the 'add table to ImagePlane' branch " - "was still used. Figure out what actually gets tested here and then " - "try to do it in another way." - )) - def test_returns_an_image_plane_object(self, table_source): - im = table_source.image_in_range(1*u.um, 2*u.um) - assert isinstance(im, ImagePlane) - - @pytest.mark.skip(reason=( - "This was the only place where the 'add table to ImagePlane' branch " - "was still used. Figure out what actually gets tested here and then " - "try to do it in another way." - )) - def test_flux_from_table_on_image_is_as_expected(self, table_source): - ph = table_source.photons_in_range(1*u.um, 2*u.um) - ref = table_source.fields[0].field["ref"] - weight = table_source.fields[0].field["weight"] - counts = np.sum([ph.value[r] * w for r, w in zip(ref, weight)]) - - im = table_source.image_in_range(1*u.um, 2*u.um) - assert np.sum(im.image) == approx(counts) - - @pytest.mark.parametrize("pix_scl", [0.1, 0.2, 0.4, 1.0]) - def test_flux_from_imagehdu_is_as_expected(self, image_source, pix_scl): - ph = image_source.photons_in_range(1*u.um, 2*u.um)[0].value - im_sum = ph * np.sum(image_source.fields[0].data) - im = image_source.image_in_range(1*u.um, 2*u.um, pix_scl*u.arcsec) - assert np.sum(im.image) == approx(im_sum) - - def test_combines_more_that_one_field_into_image(self, image_source, - table_source): - ph = table_source.photons_in_range(1 * u.um, 2 * u.um) - fld = table_source.fields[0] - tbl_sum = u.Quantity([ph[fld.field["ref"][ii]] * fld.field["weight"][ii] - for ii in range(len(fld.field))]) - tbl_sum = np.sum(tbl_sum.value) - - ph = image_source.photons_in_range(1 * u.um, 2 * u.um)[0] - im_sum = np.sum(image_source.fields[0].data) * ph.value - - comb_src = image_source - im_plane = comb_src.image_in_range(1*u.um, 2*u.um, 0.3*u.arcsec) - - assert np.sum(im_plane.image) == approx(im_sum) - - if PLOTS: - plt.imshow(im_plane.image.T, origin="lower", norm=LogNorm()) - plt.show() - - -class TestSourcePhotonsInRange: - def test_correct_photons_are_returned_for_table_source(self, table_source): - ph = table_source.photons_in_range(1, 2) - assert np.allclose(ph.value, [4., 2., 2.]) - - def test_correct_photons_are_returned_for_image_source(self, image_source): - ph = image_source.photons_in_range(1, 2) - assert np.allclose(ph.value, [2.]) - - def test_correct_photons_are_returned_for_no_spectra(self, image_source): - image_source.fields[0].spectra = {} - ph = image_source.photons_in_range(1, 2) - assert len(ph) == 0 - - @pytest.mark.parametrize("area, expected", [(None, 2), (1, 2), (10, 20)]) - def test_photons_increase_with_area(self, area, expected, image_source): - ph = image_source.photons_in_range(1, 2, area=area) - assert ph[0].value == approx(expected) - - def test_photons_returned_only_for_indices(self, table_source): - ph = table_source.photons_in_range(1, 2, indices=[0, 2]) - assert len(ph) == 2 - assert np.allclose(ph.value, [4, 2]) - - @pytest.mark.xfail class TestSourceShift: def test_that_it_does_what_it_should(self): @@ -350,65 +271,6 @@ def test_that_it_does_what_it_should(self): assert False -class TestPhotonsInRange: - @pytest.mark.parametrize("ii, n_ph", - [(0, 50), - (1, 8e12)]) - def test_returns_correct_number_of_photons_for_one_spectrum(self, ii, n_ph, - input_spectra): - spec = input_spectra[ii] - counts = source_utils.photons_in_range([spec], 1, 2) - assert np.isclose(counts.value, n_ph, rtol=2e-3) - - def test_returns_ones_for_unity_spectrum(self): - flux = np.ones(11) * u.Unit("ph s-1 m-2 um-1") - wave = np.linspace(1, 2, 11) * u.um - spec = SourceSpectrum(Empirical1D, points=wave, lookup_table=flux) - counts = source_utils.photons_in_range([spec], 1 * u.um, 2 * u.um) - assert counts.value == approx(1) - - @pytest.mark.parametrize("area, expected_units", - [(1*u.m**2, u.ph / u.s), - (None, u.ph / u.s / u.m**2)]) - def test_returns_correct_units_with_without_area_argument(self, area, - expected_units): - flux = np.ones(11) * u.Unit("ph s-1 m-2 um-1") - wave = np.linspace(1, 2, 11) * u.um - spec = SourceSpectrum(Empirical1D, points=wave, lookup_table=flux) - counts = source_utils.photons_in_range([spec], 1 * u.um, 2 * u.um, - area=area) - assert counts.unit == expected_units - - def test_returns_correct_half_flux_with_bandpass(self): - flux = np.ones(11) * u.Unit("ph s-1 m-2 um-1") - wave = np.linspace(0.5, 2.5, 11) * u.um - spec = SourceSpectrum(Empirical1D, points=wave, lookup_table=flux) - bandpass = SpectralElement(Empirical1D, - points=np.linspace(1, 2, 13)*u.um, - lookup_table=0.5 * np.ones(13)) - counts = source_utils.photons_in_range([spec], 1 * u.um, 2 * u.um, - bandpass=bandpass) - assert counts.value == approx(0.5) - - @pytest.mark.parametrize("flux, area, expected", - [(np.linspace(0, 1, 11), 1E4*u.cm**2, 0.25), - (np.linspace(0, 1, 11)**2, None, 0.13625), - (np.linspace(0, 1, 11)**0.5, 100, 34.931988)]) - def test_with_bandpass_and_area_returns_correct_value(self, flux, area, - expected): - flux *= u.Unit("ph s-1 m-2 um-1") - spec = SourceSpectrum(Empirical1D, - points=np.linspace(0.5, 2.5, 11) * u.um, - lookup_table=flux) - bandpass = SpectralElement(Empirical1D, - points=np.linspace(1, 2, 13)*u.um, - lookup_table=0.5 * np.ones(13)) - counts = source_utils.photons_in_range([spec], 1 * u.um, 2 * u.um, - bandpass=bandpass, - area=area) - assert counts.value == approx(expected) - - class TestSpectraListConverter: def test_works_for_arrays(self): spec = source_utils.convert_to_list_of_spectra( @@ -465,33 +327,3 @@ def test_throws_for_invalid_ref(): # Minimal Table tbl = Table(data=[[0, 1]], names=["ref"]) TableSourceField(tbl, {0: None}) - - -# -# class TestScaleImageHDU: -# def test_scaling_properly_for_si_photlam_in_header(self): -# hdu = fits.ImageHDU(data=np.ones((10,10))) -# hdu.header["CDELT1"] = 0.1 * u.arcsec.to(u.deg) -# hdu.header["CDELT2"] = 0.1 * u.arcsec.to(u.deg) -# hdu.header["BUNIT"] = "ph s-1 m-2 um-1" -# waverange = (1, 2)*u.um -# scaled_hdu = source2_utils.scale_imagehdu(hdu, waverange) -# assert scaled_hdu -# -# def test_scaling_properly_for_cgs_photlam_per_arcsec2_in_header(self): -# pass -# -# def test_scaling_properly_for_photlam_per_arcsec2_no_area_in_header(self): -# pass -# -# def test_scaling_properly_for_Jansky_in_header(self): -# pass -# -# def test_scaling_properly_for_Jansky_per_arcsec_in_header(self): -# pass -# -# def test_scaling_properly_for_photlam_and_bscale_in_header(self): -# pass -# -# def test_scaling_properly_for_photlam_and_bscale_bzero_in_header(self): -# pass diff --git a/scopesim/utils.py b/scopesim/utils.py index a5d0e42b8..a6c60801d 100644 --- a/scopesim/utils.py +++ b/scopesim/utils.py @@ -560,6 +560,8 @@ def unit_includes_per_physical_type(unit, physical_type): def pixel_area(header: fits.Header) -> u.Quantity[u.arcsec**2]: """Calculate area covered by one pixel in arcsec**2 from header. + The pixel area is always positive. + .. versionadded:: 0.11.1 """ @@ -575,7 +577,7 @@ def pixel_area(header: fits.Header) -> u.Quantity[u.arcsec**2]: area = (header["CDELT1"] * u.Unit(header["CUNIT1"]) * header["CDELT2"] * u.Unit(header["CUNIT2"])) - return area.to(u.arcsec**2) + return np.abs(area.to(u.arcsec**2)) def has_needed_keywords(header, suffix=""): @@ -608,6 +610,9 @@ def from_currsys(item, cmds=None): item.meta = item_meta if isinstance(item, np.ndarray) and not isinstance(item, u.Quantity): + # Numeric calibration arrays cannot contain command references. + if item.dtype.kind in "biufc": + return item.copy() item = np.array([from_currsys(x, cmds) for x in item]) if isinstance(item, list): @@ -1259,4 +1264,4 @@ def _get_next_moon_cached(moontype="full", today=None, location_key=None): nextm = times[mask][np.argmax(moonalts[mask])] else: raise ValueError(f"Invalid moon type: {moontype}, should be 'full', 'new' or 'half'.") - return nextm \ No newline at end of file + return nextm