diff --git a/.github/workflows/prepare_release.yml b/.github/workflows/prepare_release.yml index 705405e4f..ba82cfc6f 100644 --- a/.github/workflows/prepare_release.yml +++ b/.github/workflows/prepare_release.yml @@ -10,6 +10,7 @@ on: - patch - minor - major + - prerelease - prepatch - preminor - premajor diff --git a/CHANGELOG.md b/CHANGELOG.md index 013c8fa8c..d6b92f39f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,100 @@ +# Version 0.12.0 +**2026-09-24** + +Large release with many important bug fixes centered around flux conservation and pixel shifts. + +> [!IMPORTANT] +> The minimum required Python version for this package is now **3.12** (see Dependency Changes). + +## Feature highlights +### General +- Improved handling of random seeds. + Noise can now vary per readout (needs different `roid`) and is fully reproducible. + For instruments (and modes) with multiple detectors, the random seed varies per detector. +- PSF interpolation was made more efficient in some cases, and the interpolation order can now be specified. + +### MICADO +- Calibration mode added (see IRDB). +- IFU and offset slits implemented. + +### MOSAIC +- Tabular output format for the mIFU mode added. + +> [!WARNING] +> We expect backwards incompatibility, so we strongly recommend updating your IRDB packages as well. + +## What's Changed +### API Changes +* Reduce `OpticalTrain.write_header()` to the bare minimum by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/935 +* PSF interpolation order by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/950 +* Fully remove deprecated functionality by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/963 +* AtmoLibraryTERCurve works with parameters other than pwv by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/972 +### Bugs fixed +* Catch wrong inputs to `UserCommands` by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/934 +* Dark value can be given as number without floating point by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/941 +* Remove reflectance factor from integrating sphere background by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/962 +* Fix scalar Quantity pattern matching by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/964 +* Fix flux loss, aliasing, and placement errors in image-plane resampling and overlay by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/958 +* Fix NameError in `SpectralTrace.plot` when `plot_trace_id` is used without `plot_outline` by @astronomyk in https://github.com/AstarVienna/ScopeSim/pull/981 +* Fix `XiLamImage` overwriting the cunit of its primary WCS by @astronomyk in https://github.com/AstarVienna/ScopeSim/pull/982 +* Skip empty FOVs by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1001 +* Fix transposed axes in `make_image_interpolations` by @astronomyk in https://github.com/AstarVienna/ScopeSim/pull/984 +* Fix 2D image rescaling by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1016 +* Fix yet another off-by-one bug by @astronomyk in https://github.com/AstarVienna/ScopeSim/pull/1013 +* SPEC scaling now also fixed by @janusbrink in https://github.com/AstarVienna/ScopeSim/pull/1018 +* Set INHERIT keyword and fix unit to FITS standard by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/1022 +* Fix indexing in FoV extraction by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1024 +### Changes to or addition of Effects +* MOSAIC mIFU: tabular output by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/968 +* METIS WCU: Make laser parameters configurable; utility function for tunable laser by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/970 +* Refactor `PSF.apply_to()` by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1002 +* Refactor `TERCurve.apply_to()` by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1004 +* Refactor `.apply_to()` in electronic effects by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1006 +* Apply throughput in-place for speed by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/1017 +* `AutoExposure` to take dark current into account by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/1031 +* Improve randomness, refactor electronic effects by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1030 +* Define slit transforms for image slicer IFUs and offset slits by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/922 +### New Features or Improvements +* Use optimal inverse transform for rectification by @oczoske in https://github.com/AstarVienna/ScopeSim/pull/961 +* Skip PSF convolution for exactly-uniform fields by @astronomyk in https://github.com/AstarVienna/ScopeSim/pull/973 +* More useful error when download fails by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/990 +* Allow readout ID, fix Path support by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1020 +* Resolve random seed in `.observe()` by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1021 +### Dependency Changes +* Bump the dev-dependencies group with 3 updates by @dependabot[bot] in https://github.com/AstarVienna/ScopeSim/pull/940 +* Switch from httpx to httpxyz by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/924 +* Bump various dependencies by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/946 +* Bump astar-utils from 0.5.3 to 0.6.0 by @dependabot[bot] in https://github.com/AstarVienna/ScopeSim/pull/951 +* Bump actions/checkout from 6 to 7 by @dependabot[bot] in https://github.com/AstarVienna/ScopeSim/pull/947 +* Bump various dependencies by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/954 +* Bump actions/setup-python from 6 to 7 by @dependabot[bot] in https://github.com/AstarVienna/ScopeSim/pull/960 +* Drop support for Python 3.10 and 3.11 by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/969 +* Bump matplotlib and skycalc-ipy by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/977 +* Bump tornado from 6.5.7 to 6.5.8 by @dependabot[bot] in https://github.com/AstarVienna/ScopeSim/pull/1011 +* Bump tqdm from 4.68.4 to 4.70.0 by @dependabot[bot] in https://github.com/AstarVienna/ScopeSim/pull/1009 +* Bump the dev-dependencies group across 1 directory with 4 updates by @dependabot[bot] in https://github.com/AstarVienna/ScopeSim/pull/1008 +* Bump soupsieve from 2.8.4 to 2.9 by @dependabot[bot] in https://github.com/AstarVienna/ScopeSim/pull/1032 +* Bump anyio from 3.7.1 to 4.14.2 by @dependabot[bot] in https://github.com/AstarVienna/ScopeSim/pull/1033 +### Other Changes +* Add prepare-release workflow from elsewhere by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/971 +* Remove obsolete logic to deal with unversioned versions by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/979 +* Make `Transform2D` per-call transform overrides non-persistent by @astronomyk in https://github.com/AstarVienna/ScopeSim/pull/983 +* Cache the detector layout file in `MetisLMSSpectralTraceList` to avoid reading the file N*28 times for every `OpticalTrain` usage by @astronomyk in https://github.com/AstarVienna/ScopeSim/pull/987 +* Improve handling of spectral trace mapping and rectification by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/999 +* Cleanup PSF by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1003 +* Use correct NumPy type hints by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1005 +* Mark test_python_image_coords as webtest, because face is downloaded. by @hugobuddel in https://github.com/AstarVienna/ScopeSim/pull/1012 +* Cleanup in `TER_curve_utils` etc. by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1014 +* Add `Detector.det_id` convenience property by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1015 +* Formatting and cleanup by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1019 +* Set mpl backend for tests by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1026 +* Skip all tests in test_rst_utils.py by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1025 +* Remove unused `fov_grid()` methods by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1027 +* SimCADO corpse removal by @teutoburg in https://github.com/AstarVienna/ScopeSim/pull/1029 + +**Full Changelog**: https://github.com/AstarVienna/ScopeSim/compare/v0.11.4...v0.12.0 + + # Version 0.11.4 **2026-05-20** diff --git a/docs/source/_ext/scopesim_sphinx_ext.py b/docs/source/_ext/scopesim_sphinx_ext.py index 4ab146d1e..ea7957f8f 100644 --- a/docs/source/_ext/scopesim_sphinx_ext.py +++ b/docs/source/_ext/scopesim_sphinx_ext.py @@ -1,6 +1,13 @@ +# -*- coding: utf-8 -*- + +# TODO: What does this (rather clunky) Sphinx extension do that isn't covered +# by autodoc/autosummary?? If it's redundant, just get rid of it tbh. + import os import numpy as np -from scopesim.effects import effects_utils as eu + +# This function is only used here. If this extension is removed, rm the func. +from scopesim.effects.effects_utils import scopesim_effect_classes def setup(app): @@ -8,7 +15,7 @@ def setup(app): "../effects_docstrings")) os.makedirs(output_dir, exist_ok=True) - efs_dict = eu.scopesim_effect_classes() + efs_dict = scopesim_effect_classes() eff_types = np.unique([eff.split(".")[0] for eff in efs_dict]) eff_type_strs = {eff_type: f"{eff_type}\n{'='*len(eff_type)}\n\n" for eff_type in eff_types} @@ -37,7 +44,7 @@ def setup(app): :glob: * - + """ with open(os.path.join(output_dir, "index.rst"), "w") as f: diff --git a/poetry.lock b/poetry.lock index ebb77fc36..a6af398fc 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.5.1 and should not be changed by hand. [[package]] name = "accessible-pygments" @@ -50,24 +50,22 @@ numpy = ">=2.2.6,<3.0.0" [[package]] name = "anyio" -version = "3.7.1" -description = "High level compatibility layer for multiple asynchronous event loop implementations" +version = "4.15.1" +description = "High-level concurrency and networking framework on top of asyncio or Trio" optional = false -python-versions = ">=3.7" +python-versions = ">=3.10" groups = ["main", "dev"] files = [ - {file = "anyio-3.7.1-py3-none-any.whl", hash = "sha256:91dee416e570e92c64041bd18b900d1d6fa78dff7048769ce5ac5ddad004fbb5"}, - {file = "anyio-3.7.1.tar.gz", hash = "sha256:44a3c9aba0f5defa43261a8b3efb97891f2bd7d804e0e1f56419befa1adfc780"}, + {file = "anyio-4.15.1-py3-none-any.whl", hash = "sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101"}, + {file = "anyio-4.15.1.tar.gz", hash = "sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94"}, ] [package.dependencies] idna = ">=2.8" -sniffio = ">=1.1" +typing_extensions = {version = ">=4.16.0", markers = "python_version < \"3.15\""} [package.extras] -doc = ["Sphinx", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme (>=1.2.2)", "sphinxcontrib-jquery"] -test = ["anyio[trio]", "coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "mock (>=4) ; python_version < \"3.8\"", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17) ; python_version < \"3.12\" and platform_python_implementation == \"CPython\" and platform_system != \"Windows\""] -trio = ["trio (<0.22)"] +trio = ["trio (>=0.32.0)"] [[package]] name = "appnope" @@ -1249,7 +1247,7 @@ comm = ">=0.1.1" debugpy = ">=1.6.5" ipython = ">=7.23.1" jupyter-client = ">=8.9.0" -jupyter-core = ">=5.1,<6.0.dev0 || >=6.1.dev0" +jupyter-core = ">=5.1,<6.0 || >=6.1.dev0" matplotlib-inline = ">=0.1" nest-asyncio2 = ">=1.7.0" packaging = ">=22" @@ -1581,7 +1579,7 @@ files = [ ipykernel = ">=6.14" ipython = "*" jupyter-client = ">=7.0.0" -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +jupyter-core = ">=4.12,<5.0 || >=5.1.dev0" prompt-toolkit = ">=3.0.30" pygments = "*" pyzmq = ">=17" @@ -1670,14 +1668,14 @@ anyio = ">=3.1.0" argon2-cffi = ">=21.1" jinja2 = ">=3.0.3" jupyter-client = ">=7.4.4" -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +jupyter-core = ">=4.12,<5.0 || >=5.1.dev0" jupyter-events = ">=0.11.0" jupyter-server-terminals = ">=0.4.4" nbconvert = ">=6.4.4" nbformat = ">=5.3.0" packaging = ">=22.0" prometheus-client = ">=0.9" -pywinpty = {version = ">=2.0.1,<3.0.4 || >3.0.4", markers = "os_name == \"nt\""} +pywinpty = {version = ">=2.0.1,!=3.0.4", markers = "os_name == \"nt\""} pyzmq = ">=24" send2trash = ">=1.8.2" terminado = ">=0.8.3" @@ -1735,20 +1733,20 @@ test = ["bash-kernel", "pytest"] [[package]] name = "jupyterlab" -version = "4.6.3" +version = "4.6.4" description = "JupyterLab computational environment" optional = false python-versions = ">=3.10" groups = ["dev"] files = [ - {file = "jupyterlab-4.6.3-py3-none-any.whl", hash = "sha256:0a1ebc6567186f1eabd99536e94df7ed9e96d1e7c5ddf3e4406ae16e88abacb7"}, - {file = "jupyterlab-4.6.3.tar.gz", hash = "sha256:2e3db6e3a12495ebd188276e985bf5ac502fbde3d1e8628819920210008de498"}, + {file = "jupyterlab-4.6.4-py3-none-any.whl", hash = "sha256:15b13f991d3985129c797eb84d9949eeb8b6615e14b444868e642411f2c418b2"}, + {file = "jupyterlab-4.6.4.tar.gz", hash = "sha256:404f49b081819378524886c9db66dba57a5565981eff885830df1baba3a17df5"}, ] [package.dependencies] async-lru = ">=1.0.0" httpx = ">=0.25.0,<1" -ipykernel = ">=6.5.0,<6.30.0 || >6.30.0" +ipykernel = ">=6.5.0,!=6.30.0" jinja2 = ">=3.0.3" jupyter-builder = ">=1.0.2" jupyter-core = "*" @@ -2484,7 +2482,7 @@ files = [ [package.dependencies] fastjsonschema = ">=2.15" jsonschema = ">=2.6" -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +jupyter-core = ">=4.12,<5.0 || >=5.1.dev0" traitlets = ">=5.1" [package.extras] @@ -2506,9 +2504,9 @@ files = [ [package.dependencies] docutils = ">=0.18.1" jinja2 = "*" -nbconvert = ">=5.3,<5.4 || >5.4" +nbconvert = ">=5.3,!=5.4" nbformat = "*" -sphinx = ">=1.8,<8.2.0 || >8.2.0,<8.2.1 || >8.2.1" +sphinx = ">=1.8,!=8.2.0,!=8.2.1" traitlets = ">=5" [[package]] @@ -2537,20 +2535,20 @@ files = [ [[package]] name = "notebook" -version = "7.6.1" +version = "7.6.3" description = "Jupyter Notebook - A web-based notebook environment for interactive computing" optional = false python-versions = ">=3.10" groups = ["dev"] files = [ - {file = "notebook-7.6.1-py3-none-any.whl", hash = "sha256:6ea1e4c926f0dc490444ddcc335797a3dacda470a805d01031872fb119988ba2"}, - {file = "notebook-7.6.1.tar.gz", hash = "sha256:0b45fd1010668dd4808c40914d957706dbf044a677d28764dc881b74dfcede82"}, + {file = "notebook-7.6.3-py3-none-any.whl", hash = "sha256:ad7e0eb765fba836cd4a2ab0c7a3a26cde1d91665fbf6f533b6ae7b2de6d88d2"}, + {file = "notebook-7.6.3.tar.gz", hash = "sha256:e2c08e469c0ae20bb0b3214f0ab77e79653317a2f8e5b34c10361c66874a5b50"}, ] [package.dependencies] jupyter-builder = ">=1.0.2,<2" jupyter-server = ">=2.19.0,<3" -jupyterlab = ">=4.6.2,<4.7" +jupyterlab = ">=4.6.4,<4.7" jupyterlab-server = ">=2.28.0,<3" notebook-shim = ">=0.2,<0.3" tornado = ">=6.2.0" @@ -3771,18 +3769,6 @@ pyyaml = ">=6.0.3,<7.0.0" [package.extras] syn = ["synphot (>=1.7.0,<2.0.0)"] -[[package]] -name = "sniffio" -version = "1.3.0" -description = "Sniff out which async library your code is running under" -optional = false -python-versions = ">=3.7" -groups = ["main", "dev"] -files = [ - {file = "sniffio-1.3.0-py3-none-any.whl", hash = "sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384"}, - {file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"}, -] - [[package]] name = "snowballstemmer" version = "2.2.0" @@ -3797,14 +3783,14 @@ files = [ [[package]] name = "soupsieve" -version = "2.8.4" +version = "2.9.2" description = "A modern CSS selector implementation for Beautiful Soup." optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main", "dev", "docs"] files = [ - {file = "soupsieve-2.8.4-py3-none-any.whl", hash = "sha256:e7e6b0769c8f51ed59acab6e994b00621096cfb1c640a7509295987388fbaf65"}, - {file = "soupsieve-2.8.4.tar.gz", hash = "sha256:e121fd02e975c695e4e9e8774a5ee35d74714b59307868dcc5319ad2d9e3328e"}, + {file = "soupsieve-2.9.2-py3-none-any.whl", hash = "sha256:8089a26fd974ca7a1f30276d3d8492ab266ab15af581642dfe8aa162e0c1c823"}, + {file = "soupsieve-2.9.2.tar.gz", hash = "sha256:4a55d8cf158a9c2e587fa4922f1bbb91d68ac829e2d6f25403a85747c71daf74"}, ] [[package]] @@ -3962,14 +3948,14 @@ test = ["flake8", "mypy", "pytest"] [[package]] name = "sphinxcontrib-mermaid" -version = "2.1.0" +version = "2.1.1" description = "Mermaid diagrams in your Sphinx-powered docs" optional = false python-versions = ">=3.10" groups = ["docs"] files = [ - {file = "sphinxcontrib_mermaid-2.1.0-py3-none-any.whl", hash = "sha256:417cd144ec4b28852f46ba653f02ce8e538881c812111671a4c30344e87f2112"}, - {file = "sphinxcontrib_mermaid-2.1.0.tar.gz", hash = "sha256:13c5f9ac395cb6abf403eca34e228dc9fb3a30c9d960dbf3e40e9a8cef969549"}, + {file = "sphinxcontrib_mermaid-2.1.1-py3-none-any.whl", hash = "sha256:3b9ac1a8e0b5154dec3c0acf9b5c54dd9ba58540e416f20858c300814c25648f"}, + {file = "sphinxcontrib_mermaid-2.1.1.tar.gz", hash = "sha256:f8d7f1b2f0ad69b7ebdd5cbf532fb4ee107893b5373bc2d0e9634089c0b2cfee"}, ] [package.dependencies] @@ -4226,34 +4212,34 @@ test = ["flake8", "isort", "pytest"] [[package]] name = "tornado" -version = "6.5.8" +version = "6.5.9" description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." optional = false python-versions = ">=3.9" groups = ["dev", "docs", "test"] files = [ - {file = "tornado-6.5.8-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:cc6aa787d7cfab7c3d35189dc7a56fbd2399a569624c730c6b55b3d6531d0403"}, - {file = "tornado-6.5.8-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9715b5eb79735b2bcd454ce216a9275b7c0470e64ea1bf5742f78b2f72b26eeb"}, - {file = "tornado-6.5.8-cp39-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:547d63f450d570c14fe0e8db2cfb14c9bbd1c2503b4a6612586267955aa47b58"}, - {file = "tornado-6.5.8-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7e2360a0ffbe145eca8af0b19cb7203d79b1a98dd4cccdd6b368f6f49c2e3808"}, - {file = "tornado-6.5.8-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:5d242290bdf7ab3151bc1065fdd75c0dcc21cbc7b49f22a4c56329c2d6566d22"}, - {file = "tornado-6.5.8-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:7b94ff0e128fe0542f3bd331fb44d06260fc4ac16881545159f34ef08aad4195"}, - {file = "tornado-6.5.8-cp39-abi3-win32.whl", hash = "sha256:67832909c4779c64942380cb5f044a5c6163d00831472d80e25e115de9917836"}, - {file = "tornado-6.5.8-cp39-abi3-win_amd64.whl", hash = "sha256:11881db6b7c168494be2c2d12e65931451bdf7ee718535418ae1d8855dd5a0ee"}, - {file = "tornado-6.5.8-cp39-abi3-win_arm64.whl", hash = "sha256:68a7468c7e289f8514d7d664101753903217eff1bb6822c6b5994a0b5f5bcb26"}, - {file = "tornado-6.5.8.tar.gz", hash = "sha256:9452e1b208a8bd771e2cb1f2ff564985b9b214bdebbe622793e1799e0a6bd23f"}, + {file = "tornado-6.5.9-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:dea3bee433b73d6312d993ce89ad5f2b8d3ed9fde56e79ba5342b9edb78d342a"}, + {file = "tornado-6.5.9-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:22bed49cdb55292d1d58f46008ba2cc3020e5b84995ef6ae749676c8b5309514"}, + {file = "tornado-6.5.9-cp39-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0cc0876a58eee9143476f24901b1b8eb4d52443283ade2b6bca650dd1da04f84"}, + {file = "tornado-6.5.9-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc0d703cc7ec92b47ebfe236c97cd765c1f2e45744a2e156e6677e54c948274d"}, + {file = "tornado-6.5.9-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:93f9ab0226ef625b94cacb7981a5af104f1735ea02f92282abe5c389109fb31d"}, + {file = "tornado-6.5.9-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5801bbf94dfde8d21087b6aa0b17f3a5dab6110dfe44e7e4492df5de393ac15b"}, + {file = "tornado-6.5.9-cp39-abi3-win32.whl", hash = "sha256:dca9f377e80efe5dd4d52a2776b67492a81a39b2d931de169b6b36d0c1da444c"}, + {file = "tornado-6.5.9-cp39-abi3-win_amd64.whl", hash = "sha256:f810730ac71eadf009e6e5cb6d534d096887310f5649cff19d7e0d19c0bf4ee7"}, + {file = "tornado-6.5.9-cp39-abi3-win_arm64.whl", hash = "sha256:b5f4d92798337260c6d9f9f0e400c7539dd5e55cf93479a3f8508c9aa09c2b3b"}, + {file = "tornado-6.5.9.tar.gz", hash = "sha256:4d868544ebdf2fc155239a74397f65ebfea9b4d3635296c96b5dfb0701c354f5"}, ] [[package]] name = "tqdm" -version = "4.70.0" +version = "4.70.1" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "tqdm-4.70.0-py3-none-any.whl", hash = "sha256:7f585706bfddbdebf89daac705b2dfcc16890130727d3197ca62c732b4310953"}, - {file = "tqdm-4.70.0.tar.gz", hash = "sha256:55b0b0dbd97462d06ebee91e4dac24ed4d4702be82b24f07e6c1d27e08cea220"}, + {file = "tqdm-4.70.1-py3-none-any.whl", hash = "sha256:c293e525e6fef9c20e8728fd4612df02a0aa31bb5fe91ecd93e123b1b7bffa73"}, + {file = "tqdm-4.70.1.tar.gz", hash = "sha256:cefd0eca11b2a37a3aee776544d4f4ae913f02688135b5556b8788dfa474afc4"}, ] [package.dependencies] @@ -4322,14 +4308,14 @@ dev = ["flake8", "flake8-annotations", "flake8-bandit", "flake8-bugbear", "flake [[package]] name = "urllib3" -version = "2.7.0" +version = "2.8.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.10" groups = ["main", "dev", "docs"] files = [ - {file = "urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897"}, - {file = "urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c"}, + {file = "urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3"}, + {file = "urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63"}, ] [package.extras] @@ -4426,4 +4412,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">=3.12,<3.15" -content-hash = "ed6ef2a6cb8814ee6ab96c983711908f3102cbfdeb28136410bf675ad2da745c" +content-hash = "7c06b18379d2badb1c8603f9d29d48487af53f925335fbc9a95aa031614ebc6f" diff --git a/pyproject.toml b/pyproject.toml index da766c6da..f6ecd946f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ScopeSim" -version = "0.12.0b2.dev5" +version = "0.12.1.dev6" description = "Generalised telescope observation simulator" license = {text = "GPL-3.0-or-later"} authors = [ @@ -29,7 +29,7 @@ dependencies = [ "lxml (>=6.1.1,<7.0.0)", "pyyaml (>=6.0.3,<7.0.0)", "more-itertools (>=10.2.0,<11.0.0)", - "tqdm (>=4.70.0,<5.0.0)", + "tqdm (>=4.70.1,<5.0.0)", "packaging (>=26.0,<28.0)", "synphot (>=1.7.0,<2.0.0)", @@ -107,7 +107,10 @@ filterwarnings = [ "ignore:invalid value encountered in divide:RuntimeWarning", "ignore:Cannot merge meta key.*:astropy.utils.metadata.MergeConflictWarning", "ignore::astropy.wcs.wcs.FITSFixedWarning", - "default:The fov_grid*:DeprecationWarning", + # Only SpectralTraceList still calls the deprecated fov_grid internally. + # Delete this line once that call moves to an internal method; every other + # fov_grid DeprecationWarning is now an error. + "default:The fov_grid*:DeprecationWarning:scopesim.effects.spectral_trace_list", # Pytest Notebook stuff "ignore:Proactor*:RuntimeWarning", ] diff --git a/scopesim/detector/detector.py b/scopesim/detector/detector.py index 2b0e43ed8..b99c5dba1 100644 --- a/scopesim/detector/detector.py +++ b/scopesim/detector/detector.py @@ -1,12 +1,23 @@ +# -*- coding: utf-8 -*- + import numpy as np +from numpy.typing import ArrayLike from astropy.io.fits import ImageHDU from astropy.wcs import WCS from ..optics import ImagePlane -from ..optics.image_plane_utils import (add_imagehdu_to_imagehdu, - sky_wcs_from_det_wcs) -from ..utils import get_logger, from_currsys, stringify_dict, zeros_from_header +from ..optics.image_plane_utils import ( + add_imagehdu_to_imagehdu, + sky_wcs_from_det_wcs, +) +from ..utils import ( + get_logger, + from_currsys, + stringify_dict, + zeros_from_header, + real_colname, +) logger = get_logger(__name__) @@ -62,6 +73,18 @@ def hdu(self): return self._hdu + @property + def det_id(self) -> int | None: + """Return detector ID. + + Avoid calling it just "id", because ``id(self)`` is something else. + + If not found in `self.meta`, return None. + """ + if key := real_colname("id", self.meta): + return self.meta[key] + return None + @property def header(self): """Return header from internal HDU.""" @@ -72,7 +95,16 @@ def data(self): """Return data from internal HDU.""" return self._hdu.data + @data.setter + def data(self, data: ArrayLike): + self._hdu.data = data + @property def image(self): """Return data from internal HDU.""" return self.data + + def __str__(self) -> str: + if self.det_id is not None: + return f"Detector {self.det_id!s:>2}" + return "Detector" diff --git a/scopesim/detector/detector_manager.py b/scopesim/detector/detector_manager.py index de6fd24a7..9c0f80b86 100644 --- a/scopesim/detector/detector_manager.py +++ b/scopesim/detector/detector_manager.py @@ -40,8 +40,13 @@ def __init__(self, detector_list=None, cmds=None, **kwargs): self._latest_exposure: HDUList | None = None - def readout(self, image_planes, array_effects=None, dtcr_effects=None, - **kwargs) -> HDUList: + def readout( + self, + image_planes, + array_effects: list[Effect] | None = None, + dtcr_effects: list[Effect] | None = None, + **kwargs + ) -> HDUList: """ Read out the detector array into a FITS HDU List. @@ -74,9 +79,6 @@ def readout(self, image_planes, array_effects=None, dtcr_effects=None, Output FITS HDU List. """ - # .. note:: Detector is what used to be called Chip - # DetectorManager is the old Detector - self._array_effects = array_effects or [] self._dtcr_effects = dtcr_effects or [] self.meta.update(kwargs) @@ -100,7 +102,7 @@ def readout(self, image_planes, array_effects=None, dtcr_effects=None, detector = effect.apply_to(detector) # 6. add necessary header keywords - # .. todo: add keywords + # TODO: add keywords # FIXME: Why is this applied twice ??? for effect in self._array_effects: @@ -148,7 +150,7 @@ def _make_primary_hdu(self): return prihdu def _make_effects_hdu(self): - # .. todo:: decide what goes into the effects table of meta data + # TODO: decide what goes into the effects table of meta data # effects = self._array_effects + self._dtcr_effects return TableHDU() diff --git a/scopesim/effects/__init__.py b/scopesim/effects/__init__.py index 3c5d75a84..2191fa0ba 100644 --- a/scopesim/effects/__init__.py +++ b/scopesim/effects/__init__.py @@ -8,6 +8,7 @@ from .spectral_trace_list import * from .spectral_efficiency import * from .metis_lms_trace_list import * +from .micado_ifu_trace_list import * from .mosaic_trace_list import * from .surface_list import * from .ter_curves import * diff --git a/scopesim/effects/apertures.py b/scopesim/effects/apertures.py index 756e78602..aca858141 100644 --- a/scopesim/effects/apertures.py +++ b/scopesim/effects/apertures.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- """Effects related to field masks, including spectroscopic slits.""" -import warnings from typing import ClassVar import yaml @@ -118,6 +117,7 @@ def apply_to(self, obj, **kwargs): """See parent docstring.""" if isinstance(obj, FovVolumeList): logger.debug("Executing %s, FoV setup", self.meta['name']) + x = quantity_from_table("x", self.table, u.arcsec).to_value(u.arcsec) y = quantity_from_table("y", self.table, @@ -130,20 +130,14 @@ def apply_to(self, obj, **kwargs): vol["meta"]["xi_max"] = max(x) * u.arcsec vol["meta"]["slit_name"] = self.meta["name"] + if "offset_type" in self.meta: + vol["meta"]["offset_type"] = self.meta["offset_type"] + vol["meta"]["offset_x"] = self.meta["offset_x"] + vol["meta"]["offset_y"] = self.meta["offset_y"] + else: + vol["meta"]["offset_type"] = "none" return obj - # Outdated. Remove when removing all old FOVManager code from effects - def fov_grid(self, which="edges", **kwargs): - """Return a header with the sky coordinates.""" - warnings.warn("The fov_grid method is deprecated and will be removed " - "in a future release.", DeprecationWarning, stacklevel=2) - if which == "edges": - self.meta.update(kwargs) - return self.header - elif which == "masks": - self.meta.update(kwargs) - return self.mask - @property def hdu(self): return fits.ImageHDU(data=self.mask, header=self.header) @@ -459,12 +453,6 @@ def apply_to(self, obj, **kwargs): """Use apply_to of current_slit.""" return self.current_slit.apply_to(obj, **kwargs) - def fov_grid(self, which="edges", **kwargs): - """See parent docstring.""" - warnings.warn("The fov_grid method is deprecated and will be removed " - "in a future release.", DeprecationWarning, stacklevel=2) - return self.current_slit.fov_grid(which=which, **kwargs) - def change_slit(self, slitname=None): """Change the current slit.""" if not slitname or slitname in self.slits.keys(): diff --git a/scopesim/effects/detector_list.py b/scopesim/effects/detector_list.py index 6e197f75f..7cb075146 100644 --- a/scopesim/effects/detector_list.py +++ b/scopesim/effects/detector_list.py @@ -243,31 +243,6 @@ def _get_corner_points(self): return points * u.mm - def fov_grid(self, which="edges", **kwargs): - """Return an ApertureMask object. kwargs are "pixel_scale" [arcsec].""" - # This has really been taken care of by apply_to now - # TODO v1.0: finally rm this completely - raise AttributeError("The DetectorList.fov_grid() method has been " - "removed in v0.10.0.") - # aperture_mask = None - # if which == "edges": - # self.meta.update(kwargs) - # self.meta = from_currsys(self.meta, self.cmds) - - # hdr = self.image_plane_header - # xy_mm = calc_footprint(hdr, "D") - # pixel_size = hdr["CDELT1D"] # mm - # pixel_scale = self.meta["pixel_scale"] # ["] - - # # x["] = x[mm] * ["] / [mm] - # xy_sky = xy_mm * pixel_scale / pixel_size - - # aperture_mask = ApertureMask(array_dict={"x": xy_sky[:, 0], - # "y": xy_sky[:, 1]}, - # pixel_scale=pixel_scale) - - # return aperture_mask - @property def image_plane_header(self): """Create and return the Image Plane Header.""" @@ -498,9 +473,11 @@ def _get_fov_limits(self, pixel_scale=None): def image_plane_header(self): """Create and return the Image Plane Header.""" # FIXME: Heavy property..... - points = self._get_corner_points().value new_wcs, naxis = create_wcs_from_points( - points, self.pixel_size.to_value(u.mm), "D") + self._get_corner_points(), + self.pixel_size.to(u.mm) / u.pixel, + wcs_suffix="D", + ) hdr = fits.Header() hdr["NAXIS"] = 3 diff --git a/scopesim/effects/effects.py b/scopesim/effects/effects.py index 30b4bd38f..da47304af 100644 --- a/scopesim/effects/effects.py +++ b/scopesim/effects/effects.py @@ -11,20 +11,19 @@ from ..reports.rst_utils import table_to_rst -# FIXME: This docstring is out-of-date for several reasons: -# - Effects can act on objects other than Source (eg FOV, IMP, DET) -# - fov_grid is outdated - - # @dataclass(kw_only=True, eq=False) class Effect: """ Base class for representing the effects (artifacts) in an optical system. The ``Effect`` class is conceived to independently apply the changes that - an optical component (or series thereof) has on an incoming 3D description - of an on-sky object. In other words, **an Effect object should receive a - derivative of a ``Source`` object, alter it somehow, and return it**. + an optical component (or series thereof) has on an incoming description of + an on-sky object. **An Effect object should receive an object, alter it + somehow, and return an instance of the same class.** Besides ``Source``, + that object may be a ``FovVolumeList``, a ``FieldOfView``, an + ``ImagePlane`` or a ``Detector``; ``apply_to`` is expected to dispatch on + the type it is given and to pass anything it does not handle straight + through. The interface for the Effect base-class has been kept very general so that it can easily be sub-classed as data for new effects becomes available. @@ -32,9 +31,12 @@ class Effect: attributes: * ``self.meta`` - a dictionary to contain metadata. - * ``self.apply_to(obj, **kwargs)`` - a method which accepts a - Source-derivative and returns an instance of the same class as ``obj`` - * ``self.fov_grid(which="", **kwargs)`` + * ``self.apply_to(obj, **kwargs)`` - a method which accepts one of the + objects above and returns an instance of the same class as ``obj`` + + An Effect that needs to constrain the source-space volume the simulation + covers does so in the ``FovVolumeList`` branch of ``apply_to``, by calling + ``shrink``, ``extract`` or ``split`` on the list it is passed. Parameters diff --git a/scopesim/effects/effects_utils.py b/scopesim/effects/effects_utils.py index f9522ded1..f96dceeb9 100644 --- a/scopesim/effects/effects_utils.py +++ b/scopesim/effects/effects_utils.py @@ -1,10 +1,9 @@ +# -*- coding: utf-8 -*- """TBA.""" import inspect from collections.abc import Iterable -from astropy.table import Table - from .. import effects as efs from ..utils import get_logger @@ -51,15 +50,7 @@ def is_spectroscope(effects): return any(isinstance(eff, spec_classes) for eff in effects) -def empty_surface_list(**kwargs): - tbl = Table(names=["name", "outer", "inner", "angle", - "temperature", "action", "filename"], - data=[["test"], [0.], [0.], [0.], [0.], ["none"], ["none"]], - meta={"outer_unit": "m", "inner_unit": "m", - "angle_unit": "deg", "temperature_unit": "deg_C"}) - return efs.SurfaceList(table=tbl[:0], **kwargs) - - +# This is only used for the questionable sphinx extension. def scopesim_effect_classes(base_effect=efs.Effect): members = inspect.getmembers(efs) efs_dict = {".".join([cls.__module__, cls.__name__]).replace("scopesim.effects.", ""): cls diff --git a/scopesim/effects/electronic/__init__.py b/scopesim/effects/electronic/__init__.py index 80a5ea6db..c35dfe06d 100644 --- a/scopesim/effects/electronic/__init__.py +++ b/scopesim/effects/electronic/__init__.py @@ -19,9 +19,37 @@ - InterPixelCapacitance - apply IPC kernel to detector readout """ -from ...utils import get_logger +from ...utils import get_logger, check_keys +from ...detector import Detector +from .. import Effect + logger = get_logger(__name__) + +class ElectronicEffect(Effect): + """Base class for electronic effects. + + This will eventually replace the 800-range zorder (or parts of it). + """ + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.meta.update(kwargs) + check_keys(self.meta, self.required_keys, action="error") + + def _apply_to_det(self, det: Detector) -> None: + """Subclasses can override if more params needed in call.""" + logger.debug("Apply %s to %s", self.display_name, det) + det.data = self(det.data) + + def apply_to(self, obj, **kwargs): + """See parent docstring.""" + if isinstance(obj, Detector): + self._apply_to_det(obj) + + return obj + + from .electrons import LinearityCurve, ADConversion, InterPixelCapacitance from .noise import (Bias, PoorMansHxRGReadoutNoise, BasicReadoutNoise, PixelResponseNonUniformity, ShotNoise, DarkCurrent) diff --git a/scopesim/effects/electronic/dmps.py b/scopesim/effects/electronic/dmps.py index 1f1bcf3fd..296703a88 100644 --- a/scopesim/effects/electronic/dmps.py +++ b/scopesim/effects/electronic/dmps.py @@ -73,9 +73,11 @@ def __init__(self, **kwargs): def apply_to(self, obj, **kwargs): logger.debug("Entering DetectorModePropertiesSetter.apply_to") - mode_name = kwargs.get("detector_readout_mode", - from_currsys("!OBS.detector_readout_mode", - self.cmds)) + mode_name = kwargs.get( + "detector_readout_mode", + from_currsys("!OBS.detector_readout_mode", self.cmds), + ) + if isinstance(obj, ImagePlane) and mode_name == "auto": mode_name = self.select_mode(obj, **kwargs) logger.info("Detector mode set to %s", mode_name) @@ -101,9 +103,10 @@ def select_mode(self, obj, **kwargs): saturation level (peak to full_well). """ immax = np.max(obj.data) - fillfrac = kwargs.get("fill_frac", - from_currsys("!OBS.auto_exposure.fill_frac", - self.cmds)) + fillfrac = kwargs.get( + "fill_frac", + from_currsys("!OBS.auto_exposure.fill_frac", self.cmds), + ) goodmodes = [] goodron = [] diff --git a/scopesim/effects/electronic/electrons.py b/scopesim/effects/electronic/electrons.py index c860362bd..c2bee25b2 100644 --- a/scopesim/effects/electronic/electrons.py +++ b/scopesim/effects/electronic/electrons.py @@ -9,19 +9,17 @@ """ from typing import ClassVar +from numbers import Real # matches int, float and all the numpy scalars import numpy as np +from numpy.typing import ArrayLike, NDArray from scipy.signal import oaconvolve -from .. import Effect -from ...detector import Detector -from ...utils import figure_factory, check_keys -from ...utils import from_currsys, real_colname, get_logger -from . import logger +from ...utils import from_currsys, figure_factory +from . import ElectronicEffect, Detector, logger -logger = get_logger(__name__) -class LinearityCurve(Effect): +class LinearityCurve(ElectronicEffect): """ Detector linearity effect. @@ -56,28 +54,29 @@ class LinearityCurve(Effect): report_plot_include: ClassVar[bool] = True report_table_include: ClassVar[bool] = False - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.meta.update(kwargs) + def __call__(self, data: ArrayLike, ndit: int = 1) -> NDArray: + incident = self.incident * ndit + measured = self.measured * ndit + return np.interp(data, incident, measured) - check_keys(self.meta, self.required_keys, action="error") - - def apply_to(self, obj, **kwargs): - if not isinstance(obj, Detector): - return obj + @property + def incident(self) -> NDArray: + if self.table is not None: + return self.table["incident"] + return np.asarray(from_currsys(self.meta["incident"], self.cmds)) - ndit = from_currsys(self.meta["ndit"], self.cmds) + @property + def measured(self) -> NDArray: if self.table is not None: - incident = self.table["incident"] * ndit - measured = self.table["measured"] * ndit - else: - incident = np.asarray(from_currsys(self.meta["incident"], - self.cmds)) * ndit - measured = np.asarray(from_currsys(self.meta["measured"], - self.cmds)) * ndit - obj._hdu.data = np.interp(obj._hdu.data, incident, measured) + return self.table["measured"] + return np.asarray(from_currsys(self.meta["measured"], self.cmds)) + + def _apply_to_det(self, det: Detector) -> None: + """Subclasses can override if more params needed in call.""" + logger.debug("Apply %s to %s", self.display_name, det) - return obj + ndit = from_currsys(self.meta["ndit"], self.cmds) + det.data = self(det.data, ndit) def plot(self, **kwargs): fig, ax = figure_factory() @@ -93,7 +92,7 @@ def plot(self, **kwargs): return fig -class InterPixelCapacitance(Effect): +class InterPixelCapacitance(ElectronicEffect): r"""Inter-pixel capacitance effect. The effect models cross-talk due to inter-pixel capacitance with @@ -147,8 +146,6 @@ class InterPixelCapacitance(Effect): def __init__(self, **kwargs): super().__init__(**kwargs) - - self.meta.update(kwargs) self.kernel = self._build_kernel(kwargs) def _build_kernel(self, params): @@ -174,15 +171,10 @@ def _build_kernel(self, params): ]) return kernel - def apply_to(self, det, **kwargs): - if not isinstance(det, Detector): - logger.debug("%s applied to %s", self.display_name, - det.__class__.__name__) - return det - - newdata = oaconvolve(det._hdu.data, self.kernel, mode="same") - det._hdu.data = newdata - return det + def _apply_to_det(self, det: Detector) -> None: + """Subclasses can override if more params needed in call.""" + logger.debug("Apply %s to %s", self.display_name, det) + det.data = oaconvolve(det.data, self.kernel, mode="same") def update(self, **kwargs): """Update the IPC kernel. @@ -220,7 +212,7 @@ def __str__(self): return msg -class ADConversion(Effect): +class ADConversion(ElectronicEffect): """Analogue-Digital Conversion effect. The effect applies the gain factor (electrons/ADU) to the detector readouts @@ -285,7 +277,7 @@ def _should_apply(self) -> bool: logger.warning("Cannot access cmds for ADConversion effect.") return True - # ..todo: need to deal with this case more realistically + # TODO: need to deal with this case more realistically # Is this still necessary? #if self.cmds.get("!OBS.autoexpset", False): # logger.info("DIT, NDIT determined by AutoExposure -> " @@ -298,28 +290,33 @@ def _should_apply(self) -> bool: return True - def apply_to(self, obj, **kwargs): - if not isinstance(obj, Detector): - return obj - - new_dtype = self.meta["dtype"] + def __call__(self, data: ArrayLike, gain: Real) -> NDArray: + return data / gain + def _get_gain(self, det_id: int) -> Real: # Apply the gain value (copy from DarkCurrent) # Note that this does not cater for the case where the gain is given # as a plain dictionary. Should we implement that? if hasattr(self.cmds["!DET.gain"], "dic"): - dtcr_id = obj.meta[real_colname("id", obj.meta)] - gain = self.cmds["!DET.gain"].dic[dtcr_id] - logger.info(f"Detector {dtcr_id}: applying gain {gain}") - elif isinstance(self.cmds["!DET.gain"], (float, int)): + gain = self.cmds["!DET.gain"].dic[det_id] + logger.info(f"Detector {det_id}: applying gain {gain}") + return gain + if isinstance(self.cmds["!DET.gain"], Real): gain = self.cmds["!DET.gain"] logger.info(f"Applying gain {gain}") - else: - raise ValueError(".meta['gain'] must be either " - f"dict or float, but is {self.cmds['!DET.gain']}") + return gain + raise ValueError( + f"{self.__class__.__name__}.meta['gain'] must be either " + f"dict or float, but is {self.cmds['!DET.gain']}") + + def _apply_to_det(self, det: Detector) -> None: + """Subclasses can override if more params needed in call.""" + logger.debug("Apply %s to %s", self.display_name, det) + + new_dtype = self.meta["dtype"] # Apply gain - obj._hdu.data /= gain + det.data = self(det.data, self._get_gain(det.det_id)) # Type-conversion wraps around input values that are higher or lower than # the respective maximum and minimum values of the new data type. Before @@ -328,14 +325,14 @@ def apply_to(self, obj, **kwargs): if np.issubdtype(new_dtype, np.integer): minval = np.iinfo(new_dtype).min maxval = np.iinfo(new_dtype).max - minvals_mask = obj._hdu.data < minval - maxvals_mask = obj._hdu.data > maxval + minvals_mask = det.data < minval + maxvals_mask = det.data > maxval if minvals_mask.any(): - obj._hdu.data[minvals_mask] = minval + det.data[minvals_mask] = minval logger.warning( f"Effect ADConversion: {minvals_mask.sum()} negative pixels") if maxvals_mask.any(): - obj._hdu.data[maxvals_mask] = maxval + det.data[maxvals_mask] = maxval logger.warning( f"Effect ADConversion: {maxvals_mask.sum()} saturated pixels") @@ -343,6 +340,4 @@ def apply_to(self, obj, **kwargs): # set to the modified data. It should be fine to simply re-assign the # data attribute, but just in case it's not... logger.debug("Applying digitization to dtype %s.", new_dtype) - obj._hdu.data = obj._hdu.data.astype(new_dtype) - - return obj + det.data = det.data.astype(new_dtype) diff --git a/scopesim/effects/electronic/exposure.py b/scopesim/effects/electronic/exposure.py index 8ff88e1e7..342e9ac67 100644 --- a/scopesim/effects/electronic/exposure.py +++ b/scopesim/effects/electronic/exposure.py @@ -70,10 +70,10 @@ def _log_dit_ndit(dit: float, ndit: int) -> None: logger.info("Total exposure time: %.3f s", dit * ndit) def estimate_dit_ndit( - self, - exptime: float, - image_plane_max: float, - **kwargs + self, + exptime: float, + image_plane_max: float, + **kwargs ) -> tuple[float, int]: """ Automatically determine DIT and NDIT from exposure time. @@ -102,8 +102,17 @@ def estimate_dit_ndit( "fill_frac", from_currsys(self.meta["fill_frac"], self.cmds) ) + try: + dark_current = kwargs.get( + "dark_current", + from_currsys(self.meta["dark_current"], self.cmds) + ) + except KeyError: + logger.warning("No dark current found for %s", + self.display_name) + dark_current = 0 - dit_nosat = fill_frac * full_well / image_plane_max + dit_nosat = fill_frac * full_well / (image_plane_max + dark_current) logger.info("Required DIT without saturation: %.3f s", dit_nosat) # np.ceil so that dit is at most what is required for fill_frac @@ -136,8 +145,10 @@ def apply_to(self, obj, **kwargs): # in both cases, `obj` is an ImagePlane. return obj - exptime = kwargs.pop("exptime", - from_currsys("!OBS.exptime", self.cmds)) + exptime = kwargs.pop( + "exptime", + from_currsys("!OBS.exptime", self.cmds), + ) mindit = from_currsys(self.meta["mindit"], self.cmds) # TODO: Remove this silly try-except once currsys works properly... @@ -217,7 +228,7 @@ def apply_to(self, obj, **kwargs): logger.debug("Exposure: DIT = %s s, NDIT = %s", dit, ndit) if self.current_mode == "average": - obj._hdu.data /= ndit + obj.data = obj.data / ndit return obj @@ -276,6 +287,6 @@ def apply_to(self, obj, **kwargs): "readout call." ) - obj._hdu.data *= dit * ndit + obj.data = obj.data * dit * ndit return obj diff --git a/scopesim/effects/electronic/noise.py b/scopesim/effects/electronic/noise.py index 635b5f238..555218549 100644 --- a/scopesim/effects/electronic/noise.py +++ b/scopesim/effects/electronic/noise.py @@ -1,146 +1,292 @@ # -*- coding: utf-8 -*- """Any kinds of electronic or photonic noise.""" +from hashlib import sha256 from typing import ClassVar -from numbers import Real +from collections.abc import Mapping +from numbers import Real # matches int, float and all the numpy scalars import numpy as np -from astropy.io import fits +from numpy.typing import ArrayLike, NDArray -from .. import Effect -from ...detector import Detector -from ...utils import from_currsys, figure_factory, check_keys, real_colname -from . import logger +from ...utils import from_currsys, figure_factory +from . import ElectronicEffect, Detector, logger -class Bias(Effect): +# TODO: Potential refactoring in multiple effects here: +# The various __call__ now get a det_id, which is what the _get_... need +# for their lookup. Either pass the rng to __call__ instead of det_id, +# or call the _get_.. in there. Needs some consideration of outside +# callers, would they rather pass which one? And how about det_id = None? +# Also could refactor the _get_... (also elsewhere) to a abstract value +# plus the concrete key. + + +class Bias(ElectronicEffect): """Adds a constant bias level to readout.""" required_keys = {"bias"} z_order: ClassVar[tuple[int, ...]] = (855,) - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.meta.update(kwargs) - check_keys(self.meta, self.required_keys, action="error") - - def apply_to(self, obj, **kwargs): - if not isinstance(obj, Detector): - return obj + def __call__(self, data: ArrayLike) -> NDArray: + return data + self.bias_level - biaslevel = from_currsys(self.meta["bias"], self.cmds) - # Can't do in-place because of Quantization data type conflicts. - obj._hdu.data = obj._hdu.data + biaslevel + @property + def bias_level(self) -> float: + return from_currsys(self.meta["bias"], self.cmds) - return obj +class DarkCurrent(ElectronicEffect): + """ + required: dit, ndit, value + """ -class PoorMansHxRGReadoutNoise(Effect): - required_keys = {"noise_std", "n_channels", "ndit"} - z_order: ClassVar[tuple[int, ...]] = (811,) - report_plot_include: ClassVar[bool] = False - report_table_include: ClassVar[bool] = False + required_keys = {"value", "dit", "ndit"} + z_order: ClassVar[tuple[int, ...]] = (830,) - def __init__(self, **kwargs): - super().__init__(**kwargs) - params = { - "pedestal_fraction": 0.3, - "read_fraction": 0.4, - "line_fraction": 0.25, - "channel_fraction": 0.05, - "random_seed": "!SIM.random.seed", - } - self.meta.update(params) - self.meta.update(kwargs) + def __call__( + self, + data: ArrayLike, + dark_level: float, + dit: float, + ndit: int, + ) -> NDArray: + return data + dark_level * dit * ndit + + def _get_dark_level(self, det_id: int) -> float: + dark_level = float(from_currsys(self.meta["value"], self.cmds)) + if isinstance(dark_level, Real): + return dark_level + if isinstance(dark_level, Mapping): + return from_currsys(dark_level[det_id], self.cmds) + raise TypeError( + f"<{self.__class__.__name__}>.meta['value'] must be either " + f"dict-like or scalar number, but is {dark_level}." + ) - check_keys(self.meta, self.required_keys, action="error") + def _apply_to_det(self, det: Detector) -> None: + logger.debug("Apply %s to %s", self.display_name, det) - def apply_to(self, det, **kwargs): - if not isinstance(det, Detector): - return det + # Dark level needs detector meta so can't go into __call__() + dark_level = self._get_dark_level(det.det_id) + dit = from_currsys(self.meta["dit"], self.cmds) + ndit = from_currsys(self.meta["ndit"], self.cmds) - self.meta["random_seed"] = from_currsys(self.meta["random_seed"], - self.cmds) - if self.meta["random_seed"] is not None: - np.random.seed(self.meta["random_seed"]) + det.data = self(det.data, dark_level, dit, ndit) - self.meta = from_currsys(self.meta, self.cmds) - ron_keys = ["noise_std", "n_channels", "channel_fraction", - "line_fraction", "pedestal_fraction", "read_fraction"] - ron_kwargs = {key: self.meta[key] for key in ron_keys} - ron_kwargs["image_shape"] = det._hdu.data.shape + def plot(self, det, **kwargs): + """Plot effect.""" + dit = from_currsys(self.meta["dit"], self.cmds) + ndit = from_currsys(self.meta["ndit"], self.cmds) + total_time = dit * ndit + times = np.linspace(0, 2*total_time, 10) + dtcr = self.apply_to(det) + dark_level = dtcr.data[0, 0] / total_time # just read one pixel + levels = dark_level * times + fig, ax = figure_factory() + ax.plot(times, levels, **kwargs) + ax.set_xlabel("time") + ax.set_ylabel("dark level") + return ax - ron_frame = _make_ron_frame(**ron_kwargs) - stacked_ron_frame = np.zeros_like(ron_frame) - for i in range(self.meta["ndit"]): - dx = np.random.randint(0, ron_frame.shape[1]) - dy = np.random.randint(0, ron_frame.shape[0]) - stacked_ron_frame += np.roll(ron_frame, (dy, dx), axis=(0, 1)) - # TODO: this .T is ugly. Work out where things are getting switched and remove it! - # Can't do in-place because of Quantization data type conflicts. - det._hdu.data = det._hdu.data + stacked_ron_frame.T +# TODO: Add tests for seed resolving, start with code from #975, but for +# basic_instrument, and work from there. +class RandomEffect(ElectronicEffect): + """Mixin class for Effects that need random seeds.""" - return det + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.meta["random_seed"] = "!SIM.random.seed" # Default + + @property + def root_seed(self) -> int: + """Resolve root seed from cmds, or use value from effect kwargs.""" + return from_currsys(self.meta["random_seed"], self.cmds) + + @classmethod + def cls_seed(cls) -> int: + """Generate reproducible seed from hash of class name.""" + digest = sha256(cls.__name__.encode("utf-8")).digest() + return int.from_bytes(digest[:16], "little") + + @property + def readout_seed(self) -> int: + """Readout ID, or 0 if not found in cmds.""" + return self.cmds.get("!OBS.roid", 0) if self.cmds is not None else 0 + + @property + def random_seed(self) -> int | None: + """Composite random seed, or None.""" + seed = [self.readout_seed, self.cls_seed(), self.root_seed] + # TODO: Consider removing this if root seed is resolved upstream! + if None in seed: + return None + return seed + + def create_rng(self, det_id: int | None = None) -> np.random.Generator: + """ + Instantiate np.random.Generator using composite seed. + + Parameters + ---------- + det_id : int | None, optional + Detector ID. If None (the default), detector ID is not included in + the composite seed, meaning results will look identical on all + detectors passed to the effect in the same readout. + + Returns + ------- + np.random.Generator + New Generator instance. + + """ + if det_id is not None: + seed = [det_id, *self.random_seed] + else: + seed = self.random_seed + return np.random.default_rng(seed) - def plot(self, det, **kwargs): + def plot(self, det): """Plot effect image.""" - dtcr = self.apply_to(det) + detector = self.apply_to(det) fig, ax = figure_factory() - ax.imshow(dtcr.data, origin="lower") + ax.imshow(detector.data) + return ax def plot_hist(self, det, **kwargs): """Plot effect histogram.""" - dtcr = self.apply_to(det) + detector = self.apply_to(det) fig, ax = figure_factory() - ax.hist(dtcr.data.flatten()) + ax.hist(detector.data.flatten()) + return ax -class BasicReadoutNoise(Effect): +class BasicReadoutNoise(RandomEffect): """Readout noise computed as: ron * sqrt(NDIT).""" required_keys = {"noise_std", "ndit"} z_order: ClassVar[tuple[int, ...]] = (811,) + def __call__( + self, + data: ArrayLike, + ndit: int = 1, + det_id: int | None = None, + ) -> NDArray: + rng = self.create_rng(det_id) + return data + self._create_noise_frame(data.shape, rng, ndit) + + @property + def noise_std(self) -> float: + return from_currsys(self.meta["noise_std"], self.cmds) + + def _create_noise_frame( + self, + shape: tuple[int, ...], + rng: np.random.Generator, + ndit: int, + ) -> NDArray: + scale = self.noise_std * np.sqrt(float(ndit)) + return rng.normal(loc=0, scale=scale, size=shape) + + def _apply_to_det(self, det: Detector) -> None: + logger.debug("Apply %s to %s", self.display_name, det) + + ndit = from_currsys(self.meta["ndit"], self.cmds) + det.data = self(det.data, ndit) + + +class PoorMansHxRGReadoutNoise(BasicReadoutNoise): + required_keys = {"noise_std", "n_channels", "ndit"} + z_order: ClassVar[tuple[int, ...]] = (811,) + report_plot_include: ClassVar[bool] = False + report_table_include: ClassVar[bool] = False + def __init__(self, **kwargs): super().__init__(**kwargs) - self.meta["random_seed"] = "!SIM.random.seed" + params = { + "pedestal_fraction": 0.3, + "read_fraction": 0.4, + "line_fraction": 0.25, + "channel_fraction": 0.05, + } + self.meta.update(params) self.meta.update(kwargs) - check_keys(self.meta, self.required_keys, action="error") - - def apply_to(self, det, **kwargs): - if not isinstance(det, Detector): - return det + @property + def n_channels(self) -> int: + return from_currsys(self.meta["n_channels"], self.cmds) + + def _create_noise_frame( + self, + shape: tuple[int, ...], + rng: np.random.Generator, + ndit: int, + ) -> NDArray: + ron_frame = self._make_ron_frame(rng, shape) + stacked_ron_frame = np.zeros_like(ron_frame) - ndit = from_currsys(self.meta["ndit"], self.cmds) - ron = from_currsys(self.meta["noise_std"], self.cmds) - noise_std = ron * np.sqrt(float(ndit)) + for i in range(ndit): + stacked_ron_frame += np.roll( + ron_frame, + rng.integers((0, 0), ron_frame.shape), + axis=(0, 1), + ) - random_seed = from_currsys(self.meta["random_seed"], self.cmds) - if random_seed is not None: - np.random.seed(random_seed) - # Can't do in-place because of Quantization data type conflicts. - det._hdu.data = det._hdu.data + np.random.normal( - loc=0, scale=noise_std, size=det._hdu.data.shape) + return stacked_ron_frame + + def _make_ron_frame(self, rng, shape: tuple[int, ...]) -> NDArray: + self.meta = from_currsys(self.meta, self.cmds) # TODO: Is this needed? + channel_fraction = self.meta["channel_fraction"] + line_fraction = self.meta["line_fraction"] + pedestal_fraction = self.meta["pedestal_fraction"] + read_fraction = self.meta["read_fraction"] + + pixel_std = self.noise_std * (pedestal_fraction + read_fraction)**0.5 + if shape < (1024, 1024): + pixel = rng.normal(loc=0, scale=pixel_std, size=shape) + line = rng.normal( + loc=0, + scale=self.noise_std * line_fraction**0.5, + size=shape[1], + ) + else: + # TODO: Why bother with this pseudo random function? + pixel = self._pseudo_random_field(rng, scale=pixel_std, size=shape) + line = pixel[0] + + channel = np.repeat( + rng.normal( + loc=0, + scale=self.noise_std * channel_fraction**0.5, + size=self.n_channels, + ), + max(1, shape[0] // self.n_channels) + 1, + axis=0, + ) - return det + return (pixel + line) + channel[:shape[0], None] - def plot(self, det): - """Plot effect image.""" - dtcr = self.apply_to(det) - fig, ax = figure_factory() - ax.imshow(dtcr.data) + @staticmethod + def _pseudo_random_field( + rng, + scale: float = 1., + size: tuple[int, ...] = (1024, 1024), + ) -> NDArray: + n = 256 + image = np.zeros(size) + batch = rng.normal(loc=0, scale=scale, size=(2*n, 2*n)) + for y in range(0, size[1], n): + for x in range(0, size[0], n): + i, j = rng.integers(n, size=2) + dx, dy = min(size[0]-x, n), min(size[1]-y, n) + image[x:x+dx, y:y+dy] = batch[i:i+dx, j:j+dy] - def plot_hist(self, det, **kwargs): - """Plot effect histogram.""" - dtcr = self.apply_to(det) - fig, ax = figure_factory() - ax.hist(dtcr.data.flatten()) + return image -# TODO: Is this really a "noise" effect? Sounds more like "electrons" tbh. -class PixelResponseNonUniformity(Effect): +class PixelResponseNonUniformity(RandomEffect): """Pixel Response Non-Uniformity (PRNU). Models the fixed pattern of per-pixel gain variations across the detector @@ -156,8 +302,6 @@ class PixelResponseNonUniformity(Effect): prnu_std : float or dict Standard deviation of the per-pixel gain distribution. - prnu_seed : int, fixed - include: "!DET.include_prnu" Example @@ -169,50 +313,61 @@ class PixelResponseNonUniformity(Effect): class: PixelResponseNonUniformity kwargs: prnu_std: 0.001 - prnu_seed: 42 include: "!DET.include_prnu" """ - required_keys: ClassVar[set] = set() + required_keys: ClassVar[set] = {"prnu_std"} z_order: ClassVar[tuple[int, ...]] = (805,) def __init__(self, **kwargs): super().__init__(**kwargs) - self.meta.update(kwargs) - self._gain_maps = {} # keyed by dtcr_id - - def apply_to(self, obj, **kwargs): - if not isinstance(obj, Detector): - return obj - - random_seed = from_currsys(self.meta.get("prnu_seed"), self.cmds) - id_key = real_colname("id", obj.meta) - dtcr_id = obj.meta[id_key] if id_key is not None else None - - prnu_std_meta = from_currsys(self.meta["prnu_std"], self.cmds) - if isinstance(prnu_std_meta, dict): - prnu_std = float(from_currsys(prnu_std_meta[dtcr_id], self.cmds)) - elif isinstance(prnu_std_meta, Real): - prnu_std = float(prnu_std_meta) - else: - raise TypeError( - ".meta['prnu_std'] must be a float " - f"or a dict keyed by detector ID, got {type(prnu_std_meta)}" + self._gain_maps = {} # keyed by det_id + + def __call__( + self, + data: ArrayLike, + prnu_std: float, + det_id: int | None = None, + ) -> NDArray: + if det_id not in self._gain_maps: + # TODO: Should not use roid here??? + rng = self.create_rng(det_id) + self._gain_maps[det_id] = rng.normal( + loc=1.0, scale=prnu_std, size=data.shape, ) - shape = obj._hdu.data.shape - if dtcr_id not in self._gain_maps: - rng = np.random.default_rng(random_seed) - self._gain_maps[dtcr_id] = rng.normal( - loc=1.0, scale=prnu_std, size=shape, - ) + return data * self._gain_maps[det_id] + + @property + def random_seed(self) -> int | None: + """Composite random seed, or None. + + Override from base class to remove ROID, because PRNU should be + identical in one observation. + """ + seed = [self.cls_seed(), self.root_seed] + # TODO: Consider removing this if root seed is resolved upstream! + if None in seed: + return None + return seed + + def _get_prnu_std(self, det_id: int) -> float: + prnu_std = from_currsys(self.meta["prnu_std"], self.cmds) + if isinstance(prnu_std, Real): + return prnu_std + if isinstance(prnu_std, Mapping): + return from_currsys(prnu_std[det_id], self.cmds) + raise TypeError( + f"<{self.__class__.__name__}>.meta['value'] must be either " + f"dict-like or scalar number, but is {prnu_std}." + ) - if self._gain_maps[dtcr_id].shape != shape: - raise ValueError("gain map shape mismatch") + def _apply_to_det(self, det: Detector) -> None: + logger.debug("Apply %s to %s", self.display_name, det) - obj._hdu.data = obj._hdu.data * self._gain_maps[dtcr_id] - return obj + prnu_std = self._get_prnu_std(det.det_id) + det.data = self(det.data, prnu_std) def plot(self, det_id=None): """Plot the cached gain map.""" @@ -225,46 +380,38 @@ def plot(self, det_id=None): im = ax.imshow(gain_map, origin="lower", aspect="auto", vmin=1 - dev, vmax=1 + dev) fig.colorbar(im, ax=ax, label="per-pixel gain") - return fig + return fig, ax -class ShotNoise(Effect): - z_order: ClassVar[tuple[int, ...]] = (820,) +class ShotNoise(RandomEffect): + """Poissonian photon noise. - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.meta["random_seed"] = "!SIM.random.seed" - self.meta.update(kwargs) + Notes + ----- + + Numpy has a problem with generating Poisson distributions above certain + values. E.g. on linux, numpy.random.poisson(1e20) raises ValueError: lam + value too large. The value might be smaller on other (operating) systems. + + The poisson and normal distribution are basically the same + above ~100 counts: + poisson(x) ~= normal(mu=x, sigma=x**0.5) + + Therefore a limit of 1e7 is used, above which the Poisson distribution is + approximated with a normal distribution. - def apply_to(self, det, **kwargs): - if not isinstance(det, Detector): - return det - - self.meta["random_seed"] = from_currsys(self.meta["random_seed"], - self.cmds) - rng = np.random.default_rng(self.meta["random_seed"]) - - # numpy has a problem with generating Poisson distributions above - # certain values. E.g. on linux, numpy.random.poisson(1e20) raises - # ValueError: lam value too large - # The value might be smaller on other (operating) systems. - # - # The poisson and normal distribution are basically the same - # above ~100 counts: - # poisson(x) ~= normal(mu=x, sigma=x**0.5) - # - # Therefore a limit of 1e7 is used, above which the Poisson - # distribution is approximated with a normal distribution. - # - # Also, the normal distribution takes only 60% as long as the - # poisson distribution for large arrays. - # - # Special values should be handled with care: - # - Negative values are mapped to 0; there cannot be negative flux. - # - numpy.nan are implicitly passed through the normal distribution; - # because the Poisson distribution cannot handle them. - - data = det._hdu.data + Also, the normal distribution takes only 60% as long as the Poisson + distribution for large arrays. + + Special values should be handled with care: + - Negative values are mapped to 0; there cannot be negative flux. + - numpy.nan are implicitly passed through the normal distribution; + because the Poisson distribution cannot handle them. + """ + z_order: ClassVar[tuple[int, ...]] = (820,) + + def __call__(self, data: ArrayLike, det_id: int | None = None) -> NDArray: + rng = self.create_rng(det_id) # Check if there are negative values in the data. values_negative = data < 0 @@ -279,108 +426,9 @@ def apply_to(self, det, **kwargs): # Apply a normal distribution to the high values. values_high = ~values_low - data[values_high] = rng.normal(data[values_high], np.sqrt(data[values_high])) - - new_imagehdu = fits.ImageHDU( - data=data, - header=det._hdu.header, + data[values_high] = rng.normal( + loc=data[values_high], + scale=np.sqrt(data[values_high]), ) - det._hdu = new_imagehdu - return det - - def plot(self, det): - """Plot effect image.""" - dtcr = self.apply_to(det) - fig, ax = figure_factory() - ax.imshow(dtcr.data) - - def plot_hist(self, det, **kwargs): - """Plot effect histogram.""" - dtcr = self.apply_to(det) - fig, ax = figure_factory() - ax.hist(dtcr.data.flatten()) - - -class DarkCurrent(Effect): - """ - required: dit, ndit, value - """ - - required_keys = {"value", "dit", "ndit"} - z_order: ClassVar[tuple[int, ...]] = (830,) - - def __init__(self, **kwargs): - super().__init__(**kwargs) - check_keys(self.meta, self.required_keys, action="error") - - def apply_to(self, obj, **kwargs): - if not isinstance(obj, Detector): - return obj - - if isinstance(from_currsys(self.meta["value"], self.cmds), dict): - dtcr_id = obj.meta[real_colname("id", obj.meta)] - dark = from_currsys(self.meta["value"][dtcr_id], self.cmds) - elif isinstance(from_currsys(self.meta["value"], self.cmds), Real): - dark = float(from_currsys(self.meta["value"], self.cmds)) - else: - raise ValueError(".meta['value'] must be either " - f"dict or float, but is {self.meta['value']}") - - dit = from_currsys(self.meta["dit"], self.cmds) - ndit = from_currsys(self.meta["ndit"], self.cmds) - - # Can't do in-place because of Quantization data type conflicts. - obj._hdu.data = obj._hdu.data + dark * dit * ndit - - return obj - - def plot(self, det, **kwargs): - """Plot effect.""" - dit = from_currsys(self.meta["dit"], self.cmds) - ndit = from_currsys(self.meta["ndit"], self.cmds) - total_time = dit * ndit - times = np.linspace(0, 2*total_time, 10) - dtcr = self.apply_to(det) - dark_level = dtcr.data[0, 0] / total_time # just read one pixel - levels = dark_level * times - fig, ax = figure_factory() - ax.plot(times, levels, **kwargs) - ax.set_xlabel("time") - ax.set_ylabel("dark level") - - -def _pseudo_random_field(scale=1, size=(1024, 1024)): - n = 256 - image = np.zeros(size) - batch = np.random.normal(loc=0, scale=scale, size=(2*n, 2*n)) - for y in range(0, size[1], n): - for x in range(0, size[0], n): - i, j = np.random.randint(n, size=2) - dx, dy = min(size[0]-x, n), min(size[1]-y, n) - image[x:x+dx, y:y+dy] = batch[i:i+dx, j:j+dy] - - return image - - -def _make_ron_frame(image_shape, noise_std, n_channels, channel_fraction, - line_fraction, pedestal_fraction, read_fraction): - shape = image_shape - w_chan = max(1, shape[0] // n_channels) - - pixel_std = noise_std * (pedestal_fraction + read_fraction)**0.5 - line_std = noise_std * line_fraction**0.5 - if shape < (1024, 1024): - pixel = np.random.normal(loc=0, scale=pixel_std, size=shape) - line = np.random.normal(loc=0, scale=line_std, size=shape[1]) - else: - pixel = _pseudo_random_field(scale=pixel_std, size=shape) - line = pixel[0] - - channel_std = noise_std * channel_fraction**0.5 - channel = np.repeat(np.random.normal(loc=0, scale=channel_std, - size=n_channels), w_chan + 1, axis=0) - - ron_frame = (pixel + line).T + channel[:shape[0]] - - return ron_frame + return data diff --git a/scopesim/effects/electronic/pixels.py b/scopesim/effects/electronic/pixels.py index 5c8c8c8cc..e26b26e96 100644 --- a/scopesim/effects/electronic/pixels.py +++ b/scopesim/effects/electronic/pixels.py @@ -3,13 +3,13 @@ from typing import ClassVar -from .. import Effect -from ...detector import Detector -from ...utils import from_currsys, figure_factory, check_keys, real_colname -from .. import logger +from numpy.typing import ArrayLike, NDArray +from ...utils import from_currsys, figure_factory +from . import ElectronicEffect, Detector, logger -class ReferencePixelBorder(Effect): + +class ReferencePixelBorder(ElectronicEffect): """Remove signal from reference pixels. Detectors often have a number of rows and columns around the edges masked. @@ -46,33 +46,31 @@ def __init__(self, **kwargs): raise ValueError( "Parameter 'border' must have exactly four values.") - def apply_to(self, obj, **kwargs): - """Mask border pixels.""" - if not isinstance(obj, Detector): - logger.warning( - "ReferencePixelBorder: got non-detector object: %s", type(obj)) - return obj - - logger.info(f"Applying border {from_currsys(self.meta['border'])}") - if hasattr(self.meta["border"], "dic"): - dtcr_id = obj.meta[real_colname("id", obj.meta)] - border = self.meta["border"].dic[dtcr_id] - elif isinstance(self.meta["border"], list): - border = self.meta["border"] - else: - raise ValueError( - ".meta['border'] must be either " - f"dict or list, but is {self.meta['border']}") - + def __call__(self, data: ArrayLike, border: dict[int, int]) -> NDArray: if border[0] > 0: - obj.data[:border[0], :] = 0 + data[:border[0], :] = 0 if border[1] > 0: - obj.data[:, :border[1]] = 0 + data[:, :border[1]] = 0 if border[2] > 0: - obj.data[-border[2]:, :] = 0 + data[-border[2]:, :] = 0 if border[3] > 0: - obj.data[:, -border[3]:] = 0 - return obj + data[:, -border[3]:] = 0 + return data + + def _get_border(self, det_id: int) -> dict[int, int]: + if hasattr(self.meta["border"], "dic"): + return self.meta["border"].dic[det_id] + if isinstance(self.meta["border"], list): + return self.meta["border"] + raise TypeError( + f"{self.__class__.__name__}.meta['border'] must be either " + f"dict or list, but is {self.meta['border']}") + + def _apply_to_det(self, det: Detector) -> None: + """Subclasses can override if more params needed in call.""" + logger.debug("Apply %s to %s", self.display_name, det) + logger.info(f"Applying border {from_currsys(self.meta['border'])}") + det.data = self(det.data, self._get_border(det.det_id)) def plot(self, det, **kwargs): """Show the masked detector image.""" @@ -90,44 +88,47 @@ def __str__(self) -> str: return msg -class BinnedImage(Effect): - required_keys = {"bin_size"} +class BinnedImageBase(ElectronicEffect): + """Base class for binning effects.""" + z_order: ClassVar[tuple[int, ...]] = (870,) - def __init__(self, **kwargs): - super().__init__(**kwargs) - check_keys(self.meta, self.required_keys, action="error") - def apply_to(self, det, **kwargs): - if not isinstance(det, Detector): - return det +class BinnedImage(BinnedImageBase): + required_keys = {"bin_size"} - bs = from_currsys(self.meta["bin_size"], self.cmds) - image = det._hdu.data - h, w = image.shape - new_image = image.reshape((h//bs, bs, w//bs, bs)) - det._hdu.data = new_image.sum(axis=3).sum(axis=1) + def __call__(self, data: ArrayLike) -> NDArray: + height, width = data.shape + binned_data = data.reshape(( + height//self.bin_size, + self.bin_size, + width//self.bin_size, + self.bin_size + )).sum(axis=3).sum(axis=1) + return binned_data - return det + @property + def bin_size(self) -> int: + return from_currsys(self.meta["bin_size"], self.cmds) -class UnequalBinnedImage(Effect): +class UnequalBinnedImage(BinnedImageBase): required_keys = {"binx","biny"} - z_order: ClassVar[tuple[int, ...]] = (870,) - - def __init__(self, **kwargs): - super().__init__(**kwargs) - check_keys(self.meta, self.required_keys, action="error") - - def apply_to(self, det, **kwargs): - if not isinstance(det, Detector): - return det - - bx = from_currsys(self.meta["binx"], self.cmds) - by = from_currsys(self.meta["biny"], self.cmds) - image = det._hdu.data - h, w = image.shape - new_image = image.reshape((h//bx, bx, w//by, by)) - det._hdu.data = new_image.sum(axis=3).sum(axis=1) - return det + def __call__(self, data: ArrayLike) -> NDArray: + height, width = data.shape + binned_data = data.reshape(( + height//self.biny, + self.biny, + width//self.binx, + self.binx + )).sum(axis=3).sum(axis=1) + return binned_data + + @property + def binx(self) -> int: + return from_currsys(self.meta["binx"], self.cmds) + + @property + def biny(self) -> int: + return from_currsys(self.meta["biny"], self.cmds) diff --git a/scopesim/effects/metis_lms_trace_list.py b/scopesim/effects/metis_lms_trace_list.py index 877360da0..92780daca 100644 --- a/scopesim/effects/metis_lms_trace_list.py +++ b/scopesim/effects/metis_lms_trace_list.py @@ -68,109 +68,106 @@ def __init__(self, **kwargs): self.slicelist["top"].max() - self.slicelist["bottom"].min()]) - def apply_to(self, obj, **kwargs): - """See parent docstring.""" - if isinstance(obj, FovVolumeList): - # Create a single volume that covers the aperture and - # the maximum wavelength range of LMS - volumes = [spectral_trace.fov_grid() - for spectral_trace in self.spectral_traces.values()] - - wave_min = min(vol["wave_min"] for vol in volumes) - wave_max = max(vol["wave_max"] for vol in volumes) - extracted_vols = obj.extract(axes=["wave"], - edges=([[wave_min, wave_max]])) - obj.volumes = extracted_vols - - if isinstance(obj, FieldOfView): - # Application to field of view - logger.debug("Executing %s, FoV", self.meta['name']) - if obj.hdu is not None and obj.hdu.header["NAXIS"] == 3: - obj.cube = obj.hdu - elif obj.hdu is None and obj.cube is None: - obj.cube = obj.make_hdu() - - fovcube = obj.cube.data - n_z, n_y, n_x = fovcube.shape - fovwcs = WCS(obj.cube.header) - # Make this linear to avoid jump at RA 0 deg - fovwcs.wcs.ctype = ["LINEAR", "LINEAR", fovwcs.wcs.ctype[2]] - fovwcs_spat = fovwcs.sub(2) - ny_slice = self.meta["slice_samples"] - - # Spatial pixel coordinates - xslice, yslice = np.meshgrid(np.arange(n_x), - np.arange(ny_slice)) - - fovimage = np.zeros((obj.detector_header["NAXIS2"], - obj.detector_header["NAXIS1"]), - dtype=np.float32) - - for sptid, spt in tqdm(self.spectral_traces.items(), - desc=" Spectral Traces", position=2): - ymin = spt.meta["fov"]["y_min"] - ymax = spt.meta["fov"]["y_max"] - - slicewcs = fovwcs.deepcopy() - - slicewcs.wcs.ctype = ["LINEAR", "LINEAR", - slicewcs.wcs.ctype[2]] - slicewcs.wcs.crpix[1] = (ny_slice + 1) / 2 - slicewcs.wcs.crval[1] = (ymin + ymax) / 2 / 3600 - slicewcs.wcs.cdelt[1] = (ymax - ymin) / ny_slice / 3600 - slicewcs_spat = slicewcs.sub(2) - - # World coordinates for the slice - xworld, yworld = slicewcs_spat.all_pix2world(xslice, - yslice, 0) - # FOV pixel coordinates for the slice - xfov, yfov = fovwcs_spat.all_world2pix(xworld, yworld, 0) - - slicecube = np.zeros((n_z, ny_slice, n_x)) - for islice in range(n_z): - ifov = RectBivariateSpline( - np.arange(n_y), - np.arange(n_x), - fovcube[islice], - kx=1, - ky=1, - ) - slicecube[islice] = ifov(yfov, xfov, grid=False) - - slicefov = FieldOfView3D( - obj.header, - [obj.meta["wave_min"], obj.meta["wave_max"]], - ) - slicefov.detector_header = obj.detector_header - slicefov.meta["xi_min"] = obj.meta["xi_min"] - slicefov.meta["xi_max"] = obj.meta["xi_max"] - slicefov.meta["trace_id"] = sptid - slicefov.cube = fits.ImageHDU( - header=slicewcs.to_header(), - data=slicecube, - ) - # slicefov.cube.writeto(f"slicefov_{sptid}.fits", overwrite=True) + # Overrides for the two apply_to() methods + def apply_to_fovvolumelist(self, obj): + """Setup - try: - # If footprint is outside FOV, this will assign None to the hdu - slicefov.hdu = spt.map_spectra_to_focal_plane(slicefov) - except ValueError as err: # xlim_mm is None - logger.error(err) - continue + Create a single volume that covers the aperture and + the maximum wavelength range of LMS. + """ + logger.debug("Executing %s: Setup", self.meta['name']) + volumes = [spectral_trace.fov_grid() + for spectral_trace in self.spectral_traces.values()] + + wave_min = min(vol["wave_min"] for vol in volumes) + wave_max = max(vol["wave_max"] for vol in volumes) + extracted_vols = obj.extract(axes=["wave"], + edges=([[wave_min, wave_max]])) + obj.volumes = extracted_vols + return obj - if slicefov.hdu is None: # footprint outside FOV - continue + def apply_to_fov(self, obj): + """Application to field of view""" + + logger.debug("Executing %s, FoV", self.meta['name']) + if obj.hdu is not None and obj.hdu.header["NAXIS"] == 3: + obj.cube = obj.hdu + elif obj.hdu is None and obj.cube is None: + obj.cube = obj.make_hdu() + + fovcube = obj.cube.data + n_z, n_y, n_x = fovcube.shape + fovwcs = WCS(obj.cube.header) + # Make this linear to avoid jump at RA 0 deg + fovwcs.wcs.ctype = ["LINEAR", "LINEAR", fovwcs.wcs.ctype[2]] + fovwcs_spat = fovwcs.sub(2) + ny_slice = self.meta["slice_samples"] + + # Spatial pixel coordinates + xslice, yslice = np.meshgrid(np.arange(n_x), + np.arange(ny_slice)) + + fovimage = np.zeros((obj.detector_header["NAXIS2"], + obj.detector_header["NAXIS1"]), + dtype=np.float32) + + for sptid, spt in tqdm(self.spectral_traces.items(), + desc=" Spectral Traces", position=2): + ymin = spt.meta["fov"]["y_min"] + ymax = spt.meta["fov"]["y_max"] + + slicewcs = fovwcs.deepcopy() + + slicewcs.wcs.ctype = ["LINEAR", "LINEAR", + slicewcs.wcs.ctype[2]] + slicewcs.wcs.crpix[1] = (ny_slice + 1) / 2 + slicewcs.wcs.crval[1] = (ymin + ymax) / 2 / 3600 + slicewcs.wcs.cdelt[1] = (ymax - ymin) / ny_slice / 3600 + slicewcs_spat = slicewcs.sub(2) + + # World coordinates for the slice + xworld, yworld = slicewcs_spat.all_pix2world(xslice, + yslice, 0) + # FOV pixel coordinates for the slice + xfov, yfov = fovwcs_spat.all_world2pix(xworld, yworld, 0) + + slicecube = np.zeros((n_z, ny_slice, n_x)) + for islice in range(n_z): + ifov = RectBivariateSpline(np.arange(n_y), + np.arange(n_x), + fovcube[islice], kx=1, ky=1) + slicecube[islice] = ifov(yfov, xfov, grid=False) + + slicefov = FieldOfView3D(obj.header, + [obj.meta["wave_min"], + obj.meta["wave_max"]]) + slicefov.detector_header = obj.detector_header + slicefov.meta["xi_min"] = obj.meta["xi_min"] + slicefov.meta["xi_max"] = obj.meta["xi_max"] + slicefov.meta["trace_id"] = sptid + slicefov.cube = fits.ImageHDU(header=slicewcs.to_header(), + data=slicecube) + # slicefov.cube.writeto(f"slicefov_{sptid}.fits", overwrite=True) + try: + # If footprint is outside FOV, this will assign None to the hdu + slicefov.hdu = spt.map_spectra_to_focal_plane(slicefov) + except ValueError as err: # xlim_mm is None + logger.error(err) + continue - sxmin = slicefov.hdu.header["XMIN"] - sxmax = slicefov.hdu.header["XMAX"] - symin = slicefov.hdu.header["YMIN"] - symax = slicefov.hdu.header["YMAX"] - fovimage[symin:symax, sxmin:sxmax] += slicefov.hdu.data + if slicefov.hdu is None: # footprint outside FOV + continue - obj.hdu = fits.ImageHDU(data=fovimage, header=obj.detector_header) + sxmin = slicefov.hdu.header["XMIN"] + sxmax = slicefov.hdu.header["XMAX"] + symin = slicefov.hdu.header["YMIN"] + symax = slicefov.hdu.header["YMAX"] + fovimage[symin:symax, sxmin:sxmax] += slicefov.hdu.data + obj.hdu = fits.ImageHDU(data=fovimage, header=obj.detector_header) return obj + def make_spectral_traces(self): """Compute the transformations by interpolation.""" # nslice = len(self._file["Aperture List"].data) diff --git a/scopesim/effects/micado_ifu_trace_list.py b/scopesim/effects/micado_ifu_trace_list.py new file mode 100644 index 000000000..42eaea14e --- /dev/null +++ b/scopesim/effects/micado_ifu_trace_list.py @@ -0,0 +1,222 @@ +# -*- coding: utf-8 -*- +"""SpectralTraceList and SpectralTrace for the MICADO IFU.""" + +import copy +import warnings + +from tqdm.auto import tqdm +import numpy as np +from scipy.interpolate import RectBivariateSpline + +from astropy.io import fits +from astropy.io import ascii as ioascii +from astropy.table import Table +from astropy.wcs import WCS +from astropy import units as u + +from ..utils import from_currsys, find_file, quantify, get_logger +from .spectral_trace_list import SpectralTraceList +from .spectral_trace_list_utils import SpectralTrace +from .spectral_trace_list_utils import Transform2D +from .spectral_trace_list_utils import make_image_interpolations, det_offset +from .apertures import ApertureMask +from .ter_curves import TERCurve +from ..optics.fov import FieldOfView, FieldOfView3D +from ..optics.fov_volume_list import FovVolumeList + +logger = get_logger(__name__) + + +class MicadoIFUSpectralTraceList(SpectralTraceList): + """SpectralTraceList for the MICADO IFU.""" + + _class_params = { + "naxis1": 112, + "nslice": 32, + "slicewidth": 0.012, # arcsec + "wave_min": 1.5, + "wave_max": 2.5, # todo: do not hardcode + } + + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.wave_min = self.meta["wave_min"] + self.wave_max = self.meta["wave_max"] + + def apply_to_fovvolumelist(self, obj): + """ + Setup + + Create a single volume that covers the aperture and the maximum + wavelength range of the MICADO IFU. + """ + logger.debug("Executing %s: Setup", self.meta['name']) + volumes = [spectral_trace.fov_grid() + for spectral_trace in self.spectral_traces.values()] + + wave_min = min(vol["wave_min"] for vol in volumes) + wave_max = max(vol["wave_max"] for vol in volumes) + extracted_vols = obj.extract(axes=["wave"], + edges=([[wave_min, wave_max]])) + obj.volumes = extracted_vols + self.wave_min = wave_min + self.wave_max = wave_max + return obj + + def apply_to_fov(self, obj): + """Apply to FieldOfView""" + + logger.debug("Executing %s, FoV", self.meta['name']) + if obj.hdu is not None and obj.hdu.header["NAXIS"] == 3: + obj.cube = obj.hdu + elif obj.hdu is None and obj.cube is None: + obj.cube = obj.make_hdu() + + fovcube = obj.cube.data + n_z, n_y, n_x = fovcube.shape + fovwcs = WCS(obj.cube.header) + # Make this linear to avoid jump at RA 0 deg + fovwcs.wcs.ctype = ["LINEAR", "LINEAR", fovwcs.wcs.ctype[2]] + fovwcs_spat = fovwcs.sub(2) + ny_slice = self.meta["slice_samples"] + + # Spatial pixel coordinates + xslice, yslice = np.meshgrid(np.arange(n_x), + np.arange(ny_slice)) + + fovimage = np.zeros((obj.detector_header["NAXIS2"], + obj.detector_header["NAXIS1"]), + dtype=np.float32) + + for sptid, spt in tqdm(self.spectral_traces.items(), + desc=" Spectral Traces", position=2): + ymin = spt.meta["fov"]["y_min"] + ymax = spt.meta["fov"]["y_max"] + + slicewcs = fovwcs.deepcopy() + + slicewcs.wcs.ctype = ["LINEAR", "LINEAR", + slicewcs.wcs.ctype[2]] + slicewcs.wcs.crpix[1] = (ny_slice + 1) / 2 + # TODO: here we silently assume that ymin, ymax are in arcsec + # and wcs in degrees + slicewcs.wcs.crval[1] = (ymin + ymax) / 2 / 3600 + slicewcs.wcs.cdelt[1] = (ymax - ymin) / ny_slice / 3600 + slicewcs_spat = slicewcs.sub(2) + + # World coordinates for the slice + xworld, yworld = slicewcs_spat.all_pix2world(xslice, + yslice, 0) + # FOV pixel coordinates for the slice + xfov, yfov = fovwcs_spat.all_world2pix(xworld, yworld, 0) + + slicecube = np.zeros((n_z, ny_slice, n_x)) + for islice in range(n_z): + # TODO: replace by explicit bilinear interpolation + ifov = RectBivariateSpline(np.arange(n_y), + np.arange(n_x), + fovcube[islice], kx=1, ky=1) + slicecube[islice] = ifov(yfov, xfov, grid=False) + + slicefov = FieldOfView3D(obj.header, + [obj.meta["wave_min"], + obj.meta["wave_max"]]) + slicefov.detector_header = obj.detector_header + slicefov.meta["xi_min"] = spt.meta["fov"]["xi_min"] + slicefov.meta["xi_max"] = spt.meta["fov"]["xi_max"] + slicefov.meta["trace_id"] = sptid + slicefov.cube = fits.ImageHDU(header=slicewcs.to_header(), + data=slicecube) + + try: + slicefov.hdu = spt.map_spectra_to_focal_plane(slicefov) + except ValueError as err: + logger.debug(err) + + if slicefov.hdu is not None: + sxmin = slicefov.hdu.header["XMIN"] + sxmax = slicefov.hdu.header["XMAX"] + symin = slicefov.hdu.header["YMIN"] + symax = slicefov.hdu.header["YMAX"] + fovimage[symin:symax, sxmin:sxmax] += slicefov.hdu.data + + obj.hdu = fits.ImageHDU(data=fovimage, header=obj.detector_header) + return obj + + def make_spectral_traces(self): + """Make a spectral trace for each combination of order and aperture""" + self.ext_data = self._file[0].header["EDATA"] + self.ext_cat = self._file[0].header["ECAT"] + self.catalog = Table(self._file[self.ext_cat].data) + self.slicelist = self._file["Aperture List"].data + spec_traces = {} + for row in self.catalog: + if row["image_plane_id"] == -99: + continue + params = {col: row[col] for col in row.colnames} + params.update(self.meta) + trace_hdu = self._file[row["extension_id"]] + + for spslice, apid in enumerate(self.slicelist["id"]): + specid = f"{row['description']}_{apid:02d}" + thistrace = MicadoIFUSpectralTrace( + trace_hdu, + self.slicelist, + spslice, + **params) + thistrace.meta["trace_id"] = specid + spec_traces[specid] = thistrace + + self.spectral_traces = spec_traces + + +class MicadoIFUSpectralTrace(SpectralTrace): + """SpectralTrace for the MICADO IFU""" + + _class_params = { + "naxis1": 112, + "nslice": 32, + "slicewidth": 0.012, # arcsec + "pixsize": 0.015, # mm + } + + + def __init__(self, trace_tbl, aplist, spslice, cmds=None, **kwargs): + super().__init__(trace_tbl, **kwargs) + + self._set_dispersion(self.wave_min, self.wave_max) + # Provisional + self.aplist = aplist + self.meta["slice"] = spslice + self.meta["fov"] = self.fov_grid() + + lam = np.linspace(self.wave_min, self.wave_max, 1001) * u.um + off_y = aplist['offset'][spslice] # offset on detector in mm + dlam = np.median(self.dlam_per_pix(lam)) * off_y / self.meta["pixsize"] + self.xy2lam.posttransform = (det_offset, {"offset": -dlam}) + self.xilam2x.pretransform_y = (det_offset, {"offset": -dlam}) + self.xilam2y.pretransform_y = (det_offset, {"offset": -dlam}) + + def fov_grid(self): + """ + Provide information on the source space volume required by the effect. + """ + aperture = self.aplist[self.meta["slice"]] + x_min = aperture["left"] + x_max = aperture["right"] + y_min = aperture["bottom"] + y_max = aperture["top"] + xi_min = aperture["xi1"] + xi_max = aperture["xi2"] + + # ..todo: just a hack - xi and x are the same except xi is a quantity + xi_min = quantify(xi_min, u.arcsec) + xi_max = quantify(xi_max, u.arcsec) + + return {"x_min": x_min, "x_max": x_max, + "y_min": y_min, "y_max": y_max, + "xi_min": xi_min, "xi_max": xi_max, + "wave_min": self.meta["wave_min"], + "wave_max": self.meta["wave_max"], + "trace_id": self.trace_id} diff --git a/scopesim/effects/mosaic_trace_list.py b/scopesim/effects/mosaic_trace_list.py index 8b23e3ad3..34e6bb66e 100644 --- a/scopesim/effects/mosaic_trace_list.py +++ b/scopesim/effects/mosaic_trace_list.py @@ -239,7 +239,7 @@ class MosaicOutputFormat(MosaicSpectralTraceList): "image": The output is a (pseudo-)detector image where the spectrum for each fibre occupies a row. - .. versionadded:: PLACEHOLDER_NEXT_RELEASE_VERSION + .. versionadded:: 0.12.0 """ required_keys = {"filename", "format"} @@ -253,8 +253,8 @@ def apply_to(self, det, **kwargs): type(det)) return det - image = det._hdu.data - detwcs = WCS(det._hdu.header, key="D") + image = det.data + detwcs = WCS(det.header, key="D") output_format = from_currsys(self.meta["format"], self.cmds) @@ -271,8 +271,9 @@ def apply_to(self, det, **kwargs): det._hdu = fits.BinTableHDU.from_columns([ fits.Column(name="wavelength", format="D", array=lam, unit="um"), - fits.Column(name="spectrum", format="D", array=spec, unit="ADU"), + fits.Column(name="spectrum", format="D", array=spec, unit="adu"), ]) + det._hdu.header["INHERIT"] = True elif output_format == "table": ntrace = len(self.spectral_traces) @@ -298,8 +299,9 @@ def apply_to(self, det, **kwargs): tab = Table(data=[idarr, xarr, yarr, lamarr, specarr], names=["id", "x", "y", "wavelength", "spectrum"], - units=["", "arcsec", "arcsec", "um", "ADU"]) + units=["", "arcsec", "arcsec", "um", "adu"]) det._hdu = fits.BinTableHDU(data=tab) + det._hdu.header["INHERIT"] = True elif output_format == "image": pass diff --git a/scopesim/effects/psfs/discrete.py b/scopesim/effects/psfs/discrete.py index 3e85adf9c..64d852efc 100644 --- a/scopesim/effects/psfs/discrete.py +++ b/scopesim/effects/psfs/discrete.py @@ -172,7 +172,7 @@ class FieldConstantPSF(DiscretePSF): Fixed handling of background level and rounded edges to avoid visible "squares" in the image. - .. versionchanged:: PLACEHOLDER_NEXT_RELEASE_VERSION + .. versionchanged:: 0.12.0 Interpolation spline order can now be set via "!SIM.psf.interp_order". diff --git a/scopesim/effects/rotation.py b/scopesim/effects/rotation.py index 7b208bfcc..d1d48b578 100644 --- a/scopesim/effects/rotation.py +++ b/scopesim/effects/rotation.py @@ -40,5 +40,5 @@ def apply_to(self, obj, **kwargs): return obj rotations = from_currsys(self.meta["rotations"], self.cmds) - obj._hdu.data = np.rot90(obj._hdu.data, rotations) + obj.data = np.rot90(obj.data, rotations) return obj diff --git a/scopesim/effects/shifts.py b/scopesim/effects/shifts.py index 1fc9d79bf..492550b71 100644 --- a/scopesim/effects/shifts.py +++ b/scopesim/effects/shifts.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- """TBA.""" -import warnings from typing import ClassVar import numpy as np @@ -27,17 +26,6 @@ def apply_to(self, obj, **kwargs): """See parent docstring.""" return obj - def fov_grid(self, which="shifts", **kwargs): - """See parent docstring.""" - warnings.warn("The fov_grid method is deprecated and will be removed " - "in a future release.", DeprecationWarning, stacklevel=2) - if which == "shifts": - col_names = ["wavelength", "dx", "dy"] - waves, dx, dy = [self.get_table(**kwargs)[col] - for col in col_names] - return waves, dx, dy - return None - def get_table(self, **kwargs): if self.table is None: names = ["wavelength", "dx", "dy"] @@ -70,7 +58,7 @@ class AtmosphericDispersion(Shift3D): Used to generate the wavelength bins based on shifts due to the atmosphere. Doesn't contain an ``apply_to`` function, but provides information through - the ``fov_grid`` function. + the ``get_table`` function. Required Parameters ------------------- @@ -133,7 +121,7 @@ def __init__(self, **kwargs): def get_table(self, **kwargs): """ - Called by the fov_grid method of Shift3D. + Compute the wavelength-dependent refraction shifts. Returns ------- @@ -248,19 +236,6 @@ def apply_to(self, fov, **kwargs): return fov - def fov_grid(self, which="shifts", **kwargs): - """See parent docstring.""" - warnings.warn("The fov_grid method is deprecated and will be removed " - "in a future release.", DeprecationWarning, stacklevel=2) - kwargs.update(self.meta) - if "quick_adc" in self.meta: - ad = AtmosphericDispersion(**self.meta) - waves, dx, dy = ad.fov_grid() - dx *= -(1 - self.meta["efficiency"]) - dy *= -(1 - self.meta["efficiency"]) - return waves, dx, dy - return None - def plot(self): return None diff --git a/scopesim/effects/spectral_efficiency.py b/scopesim/effects/spectral_efficiency.py index e84a35587..472389889 100644 --- a/scopesim/effects/spectral_efficiency.py +++ b/scopesim/effects/spectral_efficiency.py @@ -11,7 +11,6 @@ from .effects import Effect from .ter_curves import TERCurve -from .ter_curves_utils import apply_throughput_to_cube from ..utils import figure_factory, get_logger, check_keys from .data_container import DataContainer from ..optics import echelle @@ -93,9 +92,12 @@ def get_efficiencies(self): wavelength = tbl['wavelength'].quantity efficiency = tbl['efficiency'].value params.pop("filename", None) # don't pass filename to TERCurve! - effic_curve = TERCurve(array_dict={"wavelength":wavelength, - "transmission":efficiency}, - **params) + effic_curve = TERCurve( + array_dict={ + "wavelength": wavelength, + "transmission": efficiency, + }, + **params) efficiencies[name] = effic_curve hdul.close() @@ -113,7 +115,8 @@ def apply_to(self, obj, **kwargs): swcs = WCS(obj.hdu.header).spectral with u.set_enabled_equivalencies(u.spectral()): wave = swcs.pixel_to_world(np.arange(swcs.pixel_shape[0])) << u.um - obj.hdu = apply_throughput_to_cube(obj.hdu, effic.throughput, wave) + thru = effic.throughput(wave) + obj.hdu.data = obj.hdu.data * thru.value[:, None, None] return obj def plot(self): @@ -204,7 +207,7 @@ def apply_to(self, obj, **kwargs): effic_curve = TERCurve(array_dict={"wavelength": wave, "transmission": efficiency}, **params) self.efficiencies[trace_id] = effic_curve - obj.hdu = apply_throughput_to_cube(obj.hdu, effic_curve.throughput, wave) + obj.hdu.data *= effic_curve.throughput(wave).value[:, None, None] return obj def plot(self): diff --git a/scopesim/effects/spectral_trace_list.py b/scopesim/effects/spectral_trace_list.py index 7d4c6a7ce..6597ebb20 100644 --- a/scopesim/effects/spectral_trace_list.py +++ b/scopesim/effects/spectral_trace_list.py @@ -22,7 +22,7 @@ from ..optics.fov_volume_list import FovVolumeList from .effects import Effect from .ter_curves import FilterCurve -from .spectral_trace_list_utils import SpectralTrace, make_image_interpolations +from .spectral_trace_list_utils import SpectralTrace, make_image_interpolations, det_offset from .data_container import DataContainer from ..optics import echelle @@ -128,8 +128,6 @@ def __init__(self, cmds=None, **kwargs): "x_colname": "x", "y_colname": "y", "s_colname": "s", - "offset_x": 0, # [mm] in detector plane - "offset_y": 0, # [mm] in detector plane "wave_colname": "wavelength", "center_on_wave_mid": False, "dwave": 0.002, # [um] for finding the best fit dispersion @@ -151,6 +149,8 @@ def make_spectral_traces(self): self.catalog = Table(self._file[self.ext_cat].data) spec_traces = {} for row in self.catalog: + if row["image_plane_id"] == -99: + continue params = {col: row[col] for col in row.colnames} params.update(self.meta) hdu = self._file[row["extension_id"]] @@ -185,6 +185,7 @@ def update_meta(self): self.meta["y_min"] = min(ylim) self.meta["y_max"] = max(ylim) + def apply_to(self, obj, **kwargs): """ Interface between ``FieldOfView`` and ``SpectralTraceList``. @@ -199,76 +200,119 @@ def apply_to(self, obj, **kwargs): list, identified by meta["trace_id"]. """ if isinstance(obj, FovVolumeList): - logger.debug("%s applied to %s", self.display_name, - obj.__class__.__name__) - # Setup of FieldOfView object - # volumes = [spectral_trace.fov_grid() - # for spectral_trace in self.spectral_traces.values()] - - new_vols_list = [] - - # for vol in volumes: - for spt in self.spectral_traces.values(): - vol = spt.fov_grid() - wave_edges = [vol["wave_min"], vol["wave_max"]] - if "x_min" in vol: - x_edges = [vol["x_min"], vol["x_max"]] - y_edges = [vol["y_min"], vol["y_max"]] - extracted_vols = obj.extract( - axes=["wave", "x", "y"], - edges=(wave_edges, x_edges, y_edges), - aperture_id=vol["aperture_id"]) - else: - extracted_vols = obj.extract( - axes=["wave"], - edges=(wave_edges, ), - aperture_id=vol["aperture_id"]) - - for ex_vol in extracted_vols: - ex_vol["meta"].update(vol) - ex_vol["meta"].pop("wave_min") - ex_vol["meta"].pop("wave_max") - new_vols_list.extend(extracted_vols) - - obj.volumes = new_vols_list + obj = self.apply_to_fovvolumelist(obj) if isinstance(obj, FieldOfView): - logger.debug("%s applied to %s", self.display_name, - obj.__class__.__name__) - # Application to field of view - if obj.hdu is not None and obj.hdu.header["NAXIS"] == 3: - obj.cube = obj.hdu - elif obj.hdu is not None and obj.hdu.header["NAXIS"] == 2: - # todo: catch the case of obj.hdu.header["NAXIS"] == 2 - # for MAAT - pass - elif obj.hdu is None and obj.cube is None: - logger.info("Making cube") - obj.cube = obj.make_hdu() - - # # Check whether an offset slit is used. If so, recompute spectral traces. - # offset_x = obj.cube.header["CRVAL1D"] - # offset_y = obj.cube.header["CRVAL2D"] - # if (offset_x != self.meta["offset_x"] or - # offset_y != self.meta["offset_y"]): - # logger.debug("Recomputing spectral traces for offset (%.1g, %.1g)", - # offset_x, offset_y) - # self.meta["offset_x"] = offset_x - # self.meta["offset_y"] = offset_y - # self.make_spectral_traces() - # self.update_meta() - - spt = self.spectral_traces[obj.trace_id] - try: - # If footprint is outside FOV, this will assign None to the hdu - obj.hdu = spt.map_spectra_to_focal_plane(obj) - except ValueError as err: # xlim_mm is None - logger.error(err) - obj.image_plane_id = spt.meta["image_plane_id"] + obj = self.apply_to_fov(obj) logger.debug("%s done", self.display_name) return obj + def apply_to_fovvolumelist(self, obj): + """ + Setup of SpectralTraceList + + During setup of the required FieldOfView objects, the + SpectralTraceList is asked for the source space volumes that + it requires (spatial limits and wavelength limits). + """ + logger.debug("%s applied to %s", self.display_name, + obj.__class__.__name__) + + new_vols_list = [] + + for spt in self.spectral_traces.values(): + vol = spt.fov_grid() + wave_edges = [vol["wave_min"], vol["wave_max"]] + if "x_min" in vol: + x_edges = [vol["x_min"], vol["x_max"]] + y_edges = [vol["y_min"], vol["y_max"]] + extracted_vols = obj.extract( + axes=["wave", "x", "y"], + edges=(wave_edges, x_edges, y_edges), + aperture_id=vol["aperture_id"]) + else: + extracted_vols = obj.extract( + axes=["wave"], + edges=(wave_edges, ), + aperture_id=vol["aperture_id"]) + + for ex_vol in extracted_vols: + ex_vol["meta"].update(vol) + ex_vol["meta"].pop("wave_min") + ex_vol["meta"].pop("wave_max") + new_vols_list.extend(extracted_vols) + + obj.volumes = new_vols_list + return obj + + + def apply_to_fov(self, obj): + """Apply to FieldOfView + + During "observation" the method is passed a single FieldOfView + object and applies the mapping to the image plane to it. + The FieldOfView object is associated to one SpectralTrace from the + list, identified by meta["trace_id"]. + """ + logger.debug("%s applied to %s %s", self.display_name, + obj.__class__.__name__, obj.trace_id) + + if obj.hdu is not None and obj.hdu.header["NAXIS"] == 3: + obj.cube = obj.hdu + elif obj.hdu is not None and obj.hdu.header["NAXIS"] == 2: + # todo: catch the case of obj.hdu.header["NAXIS"] == 2 + # for MAAT + pass + elif obj.hdu is None and obj.cube is None: + logger.info("Making cube") + obj.cube = obj.make_hdu() + + # Check whether an offset slit is used. If so, recompute spectral traces. + # TODO: this needs to be removed. If transforms are available that + # can be applied to the actual slit position, they ought be used + # without modification. If they are not available but need to be + # hacked, then that hack needs to be defined via the yaml and only + # be used in a particular case, not in general. + #offset_x = obj.cube.header["CRVAL1D"] + #offset_y = obj.cube.header["CRVAL2D"] + #if (offset_x != self.meta["offset_x"] or + # offset_y != self.meta["offset_y"]): + # logger.debug("Recomputing spectral traces for offset (%.1g, %.1g)", + # offset_x, offset_y) + # self.meta["offset_x"] = offset_x + # self.meta["offset_y"] = offset_y + # self.make_spectral_traces() + # self.update_meta() + + spt = self.spectral_traces[obj.trace_id] + remove_pre = False + if ("offset_type" in obj.meta and + obj.meta["offset_type"] == "detector"): + self.meta["offset_type"] = "detector" + self.meta["offset_x"] = obj.meta["offset_x"] + self.meta["offset_y"] = obj.meta["offset_y"] + spt.xy2lam.pretransform_x = (det_offset, + {"offset": obj.meta["offset_x"]}) + spt.xy2lam.pretransform_y = (det_offset, + {"offset": obj.meta["offset_y"]}) + spt.xilam2x.posttransform = (det_offset, + {"offset": -obj.meta["offset_x"]}) + spt.xilam2y.posttransform = (det_offset, + {"offset": -obj.meta["offset_y"]}) + remove_pre = True + + obj.hdu = spt.map_spectra_to_focal_plane(obj) + obj.image_plane_id = spt.meta["image_plane_id"] + + if remove_pre: + spt.xy2lam.pretransform_y = None + spt.xy2lam.pretransform_x = None + spt.xilam2x.posttransform = None + spt.xilam2y.posttransform = None + + return obj + @property def footprint(self): """Return the footprint of the entire SpectralTraceList.""" @@ -365,6 +409,20 @@ def rectify_traces(self, hdulist, xi_min=None, xi_max=None, interps=None, for i, trace_id in tqdm(enumerate(self.spectral_traces, start=1), desc=" Traces", total=len(self.spectral_traces)): + spt = self[trace_id] + remove_pre = False + if ("offset_type" in self.meta and + self.meta["offset_type"] == "detector"): + spt.xy2lam.pretransform_x = (det_offset, + {"offset": self.meta["offset_x"]}) + spt.xy2lam.pretransform_y = (det_offset, + {"offset": self.meta["offset_y"]}) + spt.xilam2x.posttransform = (det_offset, + {"offset": -self.meta["offset_x"]}) + spt.xilam2y.posttransform = (det_offset, + {"offset": -self.meta["offset_y"]}) + remove_pre = True + try: hdu = self[trace_id].rectify( hdulist, @@ -385,6 +443,12 @@ def rectify_traces(self, hdulist, xi_min=None, xi_max=None, interps=None, outhdul.append(hdu) outhdul[0].header[f"EXTNAME{i}"] = trace_id + if remove_pre: + spt.xy2lam.pretransform_y = None + spt.xy2lam.pretransform_x = None + spt.xilam2x.posttransform = None + spt.xilam2y.posttransform = None + outhdul[0].header.update(inhdul[0].header) return outhdul diff --git a/scopesim/effects/spectral_trace_list_utils.py b/scopesim/effects/spectral_trace_list_utils.py index ad1dd802e..d5635bc71 100644 --- a/scopesim/effects/spectral_trace_list_utils.py +++ b/scopesim/effects/spectral_trace_list_utils.py @@ -94,8 +94,6 @@ class SpectralTrace: "x_colname": "x", "y_colname": "y", "s_colname": "s", - "offset_x": 0, - "offset_y": 0, "wave_colname": "wavelength", "dwave": 0.002, "aperture_id": 0, @@ -175,6 +173,7 @@ def fov_grid(self): "trace_id": self.trace_id, "aperture_id": aperture_id, "image_plane_id": image_plane_id} + def compute_interpolation_functions(self): """ Compute various interpolation functions between slit and focal plane. @@ -183,11 +182,12 @@ def compute_interpolation_functions(self): `xi` (spatial coordinate along the slit, in arcsec) and `lam` (wavelength, in um). - The interpolation functions include a shift in the focal-plane - coordinates, determined from the CRVAL of the source FOV. + #The interpolation functions include a shift in the focal-plane + #coordinates, determined from the CRVAL of the source FOV. """ - x_arr = self.table[self.meta["x_colname"]] + self.meta["offset_x"] - y_arr = self.table[self.meta["y_colname"]] + self.meta["offset_y"] + x_arr = self.table[self.meta["x_colname"]] + y_arr = self.table[self.meta["y_colname"]] + xi_arr = self.table[self.meta["s_colname"]] lam_arr = self.table[self.meta["wave_colname"]] @@ -223,8 +223,9 @@ def _focal_plane_grid(self, fov): xlim_mm, ylim_mm = self.footprint(wave_min=wave_min, wave_max=wave_max, xi_min=xi_min, xi_max=xi_max) - if xlim_mm is None: - logger.warning("xlim_mm is None") + if xlim_mm is None or ylim_mm is None: + # Cases where there is no overlap betwen slit/trace and + # fov are acceptable. return None det_header = fov.detector_header @@ -631,6 +632,7 @@ def footprint(self, wave_min=None, wave_max=None, xi_min=None, xi_max=None): # Requested wavelenth range is entirely outside definition range: # no footprint if wave_min > np.max(wave_val) or wave_max < np.min(wave_val): + logger.info("Trace not in wavelength range") return None, None # Restrict to overlap of requested range and definition range @@ -659,6 +661,7 @@ def footprint(self, wave_min=None, wave_max=None, xi_min=None, xi_max=None): # Requested slit range is entirely outside definition range: # no footprint if xi_min > np.max(xi_val) or xi_max < np.min(xi_val): + logger.info("Slit outside range (spatial)") return None, None # Restrict to overlap of requested range and definition range @@ -1233,13 +1236,18 @@ def _xiy2xlam_fit(layout, params): def make_image_interpolations(hdulist, **kwargs): - """Create 2D interpolation functions for images.""" + """Create 2D interpolation functions for images. + + The interpolation functions are called with (j, i) pixel coordinates, + i.e. the first argument corresponds to the row (NAXIS2) axis of the + image, the second to the column (NAXIS1) axis. + """ interps = [] for hdu in hdulist: if isinstance(hdu, fits.ImageHDU): interps.append( - RectBivariateSpline(np.arange(hdu.header["NAXIS1"]), - np.arange(hdu.header["NAXIS2"]), + RectBivariateSpline(np.arange(hdu.header["NAXIS2"]), + np.arange(hdu.header["NAXIS1"]), hdu.data, **kwargs) ) return interps @@ -1300,3 +1308,7 @@ def get_affine_parameters(coords): shears = (np.average(shears, axis=0) * rad2deg) - (90 + rotations) return rotations, shears + +def det_offset(x, offset): + """Apply offset to x""" + return x + offset diff --git a/scopesim/effects/surface_list.py b/scopesim/effects/surface_list.py index 12e37a1cf..811b367ec 100644 --- a/scopesim/effects/surface_list.py +++ b/scopesim/effects/surface_list.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- """TBA.""" -import warnings from collections import OrderedDict from copy import deepcopy from typing import ClassVar @@ -40,31 +39,6 @@ def __init__(self, **kwargs): surf_kwargs["filename"] = from_currsys(surf_kwargs["filename"], self.cmds) self.surfaces[surf_kwargs["name"]] = SpectralSurface(**surf_kwargs) - def fov_grid(self, which="waveset", **kwargs): - warnings.warn("The fov_grid method is deprecated and will be removed " - "in a future release.", DeprecationWarning, stacklevel=2) - wave_edges = [] - if which == "waveset": - self.meta.update(kwargs) - self.meta = from_currsys(self.meta, self.cmds) - wave_min = quantify(self.meta["wave_min"], u.um) - wave_max = quantify(self.meta["wave_max"], u.um) - # ..todo:: add 1001 to default.yaml somewhere - wave = np.linspace(wave_min, wave_max, 1001) - throughput = self.throughput(wave) - threshold = self.meta["minimum_throughput"] - valid_waves = np.where(throughput >= threshold)[0] - - if not len(valid_waves): - msg = ("No transmission found above the threshold " - f"{self.meta['minimum_throughput']} in this wavelength " - f"range {[self.meta['wave_min'], self.meta['wave_max']]}." - " Did you open the shutter?") - raise ValueError(msg) - - wave_edges = [min(wave[valid_waves]), max(wave[valid_waves])] - return wave_edges - @property def throughput(self): if self._throughput is None: @@ -255,6 +229,8 @@ def plot(self, which="x", wavelength=None, *, axes=None, **kwargs): def _row_quantity_or_value(table, colname, index): col = table[colname] value = col[index] + if isinstance(value, str): + return value if col.unit is not None or f"{colname}_unit" in table.meta: return quantity_from_table(colname, table)[index] return value.item() if hasattr(value, "item") else value diff --git a/scopesim/effects/ter_curves.py b/scopesim/effects/ter_curves.py index 340af7eab..6efd90f7b 100644 --- a/scopesim/effects/ter_curves.py +++ b/scopesim/effects/ter_curves.py @@ -5,28 +5,41 @@ from typing import ClassVar from collections.abc import Collection, Iterable import numpy as np +from numpy.typing import ArrayLike, NDArray from scipy import integrate from astropy import units as u from astropy.io import fits from astropy.table import Table from astropy.wcs import WCS from copy import deepcopy +from synphot import SpectralElement, SourceSpectrum import skycalc_ipy from . import DataContainer from .effects import Effect -from .ter_curves_utils import (add_edge_zeros, combine_two_spectra, - apply_throughput_to_cube, download_svo_filter, - download_svo_filter_list) +from .ter_curves_utils import ( + add_edge_zeros, + download_svo_filter, + download_svo_filter_list, +) from ..optics.fov_volume_list import FovVolumeList from ..optics.fov import FieldOfView from ..optics.surface import SpectralSurface from ..source.source import Source -from ..source.source_fields import (CubeSourceField, SpectrumSourceField, - BackgroundSourceField) -from ..utils import (from_currsys, quantify, check_keys, find_file, - figure_factory, get_logger) +from ..source.source_fields import ( + CubeSourceField, + SpectrumSourceField, + BackgroundSourceField, +) +from ..utils import ( + from_currsys, + quantify, + check_keys, + find_file, + figure_factory, + get_logger, +) from ..server.download_utils import create_retriever logger = get_logger(__name__) @@ -156,44 +169,65 @@ def __init__(self, filename=None, **kwargs): self.surface.table = data self.surface.table.meta.update(self.meta) - def apply_to(self, obj, **kwargs): - if isinstance(obj, Source): - self.meta = from_currsys(self.meta, self.cmds) - wave_min = quantify(self.meta["wave_min"], u.um).to(u.AA) - wave_max = quantify(self.meta["wave_max"], u.um).to(u.AA) + def __call__(self, data: ArrayLike, waveset: u.Quantity[u.um]) -> NDArray: + return data * self.throughput(waveset).value[:, None, None] - thru = self.throughput + def _apply_to_fvl(self, fvl: FovVolumeList) -> None: + logger.debug("Apply %s, FoV setup", self.display_name) - # apply transmission to source spectra - for fld in obj.fields: - if isinstance(fld, CubeSourceField): - fld.field = apply_throughput_to_cube( - fld.field, thru, fld.waveset) - elif isinstance(fld, SpectrumSourceField): - fld.spectra = { - isp: combine_two_spectra(spec, thru, "multiply", - wave_min, wave_max) - for isp, spec in fld.spectra.items() - } - else: - # Rather log than raise here, can still move on - logger.error("Source field is neither Cube nor has " - "spectra, this shouldn't occur...") + wave = self.surface.throughput.waveset + thru = self.surface.throughput(wave) + valid_waves = np.argwhere(thru > 0) - # add the effect background to the source background field - if self.background_source is not None: - for bgs in self.background_source: - obj.append(bgs) + if not len(valid_waves): + raise ValueError( + f"No transmission found in {self.display_name} over the " + f"wavelength range [{wave.min().to_value(u.um):.3f}, " + f"{wave.max().to_value(u.um):.3f}] um. " + "Did you open the shutter?") - if isinstance(obj, FovVolumeList): - wave = self.surface.throughput.waveset - thru = self.surface.throughput(wave) - valid_waves = np.argwhere(thru > 0) - wave_min = wave[max(0, valid_waves[0][0] - 1)] - wave_max = wave[min(len(wave) - 1, valid_waves[-1][0] + 1)] + wave_min = wave[max(0, valid_waves[0][0] - 1)] + wave_max = wave[min(len(wave) - 1, valid_waves[-1][0] + 1)] + + fvl.shrink("wave", [wave_min.to_value(u.um), wave_max.to_value(u.um)]) + + def _apply_to_src(self, src: Source) -> None: + logger.debug("Apply %s, Source spectra", self.display_name) + + self.meta = from_currsys(self.meta, self.cmds) + wave_min = quantify(self.meta["wave_min"], u.um).to(u.AA) + wave_max = quantify(self.meta["wave_max"], u.um).to(u.AA) + + thru = self.throughput + + # apply transmission to source spectra + for fld in src.fields: + if isinstance(fld, CubeSourceField): + ## in-place is much faster than self() + #fld.field.data = self(fld.field.data, fld.waveset) + fld.field.data *= thru(fld.waveset).value[:, None, None] + elif isinstance(fld, SpectrumSourceField): + fld.spectra = { + isp: spec * thru # This works because synphot! + for isp, spec in fld.spectra.items() + } + else: + # Rather log than raise here, can still move on + logger.error( + "Source field is neither Cube nor has spectra, this " + "should not happen, but moving on.") - obj.shrink("wave", [wave_min.to_value(u.um), - wave_max.to_value(u.um)]) + # add the effect background to the source background field + if self.background_source is not None: + for bgs in self.background_source: + src.append(bgs) + + def apply_to(self, obj, **kwargs): + if isinstance(obj, Source): + self._apply_to_src(obj) + + if isinstance(obj, FovVolumeList): + self._apply_to_fvl(obj) ## If obj is a FieldOfView, apply the throughput if isinstance(obj, FieldOfView): @@ -202,16 +236,16 @@ def apply_to(self, obj, **kwargs): swcs = WCS(obj.hdu.header).spectral with u.set_enabled_equivalencies(u.spectral()): wave = swcs.pixel_to_world(np.arange(swcs.pixel_shape[0])) << u.um - obj.hdu = apply_throughput_to_cube(obj.hdu, thru, wave) + obj.hdu.data *= thru(wave).value[:, None, None] return obj @property - def emission(self): + def emission(self) -> SourceSpectrum: return self.surface.emission @property - def throughput(self): + def throughput(self) -> SpectralElement: return self.surface.throughput @property @@ -222,23 +256,16 @@ def background_source(self): bkg_hdr = fits.Header() bkg_hdr.update({ + "ROLE": "background", # not used yet "BG_SRC": True, "BG_SURF": self.display_name, "SPEC_REF": 0, - # Seem those are not needed... - # "CUNIT1": "ARCSEC", - # "CUNIT2": "ARCSEC", - # "CDELT1": 0, - # "CDELT2": 0, "BUNIT": "photlam arcsec-2", "SOLIDANG": "arcsec-2", }) bkg_fld = BackgroundSourceField(field=None, spectra=bkg_spec, header=bkg_hdr) bkg_src = Source(field=bkg_fld) - # Before BackgroundSourceField: - # bkg_hdu = fits.ImageHDU(header=bkg_hdr) - # bkg_src = Source(image_hdu=bkg_hdu, spectra=flux) self._background_source = bkg_src return [self._background_source] @@ -888,34 +915,6 @@ def __init__(self, cmds=None, **kwargs): # TODO: maybe use actually masked table here? self.table["transmission"][mask] = 0 - def fov_grid(self, which="waveset", **kwargs): - warnings.warn("The fov_grid method is deprecated and will be removed " - "in a future release.", DeprecationWarning, stacklevel=2) - if which == "waveset": - self.meta.update(kwargs) - self.meta = from_currsys(self.meta, self.cmds) - # ..todo:: replace the 101 with a variable in !SIM - wave = np.linspace(self.meta["wave_min"], - self.meta["wave_max"], 101) - wave = quantify(wave, u.um) - throughput = self.surface.transmission(wave) - min_thru = self.meta["minimum_throughput"] - valid_waves = np.where(throughput.value > min_thru)[0] - if len(valid_waves) > 0: - wave_edges = [min(wave[valid_waves].value), - max(wave[valid_waves].value)] * u.um - else: - raise ValueError("No transmission found above the threshold {}" - " in this wavelength range {}. Did you open " - "the shutter?" - "".format(self.meta["minimum_throughput"], - [self.meta["wave_min"], - self.meta["wave_max"]])) - else: - wave_edges = [] - - return wave_edges - @property def fwhm(self): wave = self.surface.wavelength @@ -1073,11 +1072,6 @@ def surface(self): def throughput(self): return self.current_filter.throughput - def fov_grid(self, which="waveset", **kwargs): - warnings.warn("The fov_grid method is deprecated and will be removed " - "in a future release.", DeprecationWarning, stacklevel=2) - return self.current_filter.fov_grid(which=which, **kwargs) - def change_filter(self, filtername=None): """Change the current filter.""" if filtername in self.filters.keys(): diff --git a/scopesim/effects/ter_curves_utils.py b/scopesim/effects/ter_curves_utils.py index cabc6d796..dd2f2f1c5 100644 --- a/scopesim/effects/ter_curves_utils.py +++ b/scopesim/effects/ter_curves_utils.py @@ -8,12 +8,8 @@ from astropy.table import Table from astropy.io.votable import parse_single_table from astropy.io import ascii as ioascii -from astropy.io import fits -from synphot import SpectralElement, SourceSpectrum, Empirical1D, Observation -from synphot.units import PHOTLAM +from synphot import SpectralElement, Empirical1D -from scopesim.source.source_templates import vega_spectrum, st_spectrum, \ - ab_spectrum from ..utils import find_file, quantity_from_table, get_logger from ..server.download_utils import create_client, handle_download @@ -53,6 +49,7 @@ PATH_SVO_DATA = PATH_HERE.parent / "data" / "svo" +# This function is only used in the AnisoCADO PSF... def get_filter_effective_wavelength(filter_name): # TODO: This is technically stored in the SVO XML file as WavelengthEff... # (actually WavelengthMean, by definition of formula ...) @@ -218,226 +215,6 @@ def get_filter(filter_name): return filt -def get_zero_mag_spectrum(system_name: str = "AB") -> SourceSpectrum: - """ - Return a synphot spectrum of 0 mag in given photometric system. - - Parameters - ---------- - system_name : {"AB", "Vega", "ST", "HST"} - Name of the photometric system. - - Raises - ------ - ValueError - Raised if `system_name` is invalid. - - Returns - ------- - spec : synphot.SourceSpectrum - Output spectrum. - - """ - if system_name.lower() == "vega": - return vega_spectrum() - if system_name.lower() == "ab": - return ab_spectrum() - if system_name.lower() in ("st", "hst"): - return st_spectrum() - raise ValueError(f"system_name {system_name} is unknown") - - -def zero_mag_flux(filter_name, photometric_system, return_filter=False): - """ - Return the zero magnitude photon flux for a filter. - - Acceptable filter names are those given in - ``scopesim.effects.ter_curves_utils.FILTER_DEFAULTS`` or a string with an - appropriate name for a filter in the Spanish-VO filter-service. Such - strings must use the naming convention: observatory/instrument.filter. E.g: - "paranal/HAWKI.Ks", or "Gemini/GMOS-N.CaT". - - Parameters - ---------- - filter_name : str - Name of the filter - see above. - - photometric_system : {"vega", "AB", "ST"} - Name of the photometric system - - return_filter : bool, optional - If True, also returns the filter curve object - - Returns - ------- - flux : float - [PHOTLAM] - filt : ``synphot.SpectralElement`` - If `return_filter` is True - - """ - filt = get_filter(filter_name) - spec = get_zero_mag_spectrum(photometric_system) - - obs = Observation(spec, filt) - flux = obs.effstim(flux_unit=PHOTLAM) - - if return_filter: - return flux, filt - return flux - - -def scale_spectrum(spectrum, filter_name, amplitude): - """ - Scale a SourceSpectrum to a value in a filter. - - Parameters - ---------- - spectrum : synphot.SourceSpectrum - - filter_name : str - Name of a filter from - - a local instrument package (available in ``rc.__search_path__``) - - a generic filter name (see ``ter_curves_utils.FILTER_DEFAULTS``) - - a spanish-vo filter service reference (e.g. "Paranal/HAWKI.Ks") - - amplitude : astropy.Quantity, float - The value that the spectrum should have in the given filter. Acceptable - astropy quantities are: - - u.mag : Vega magnitudes - - u.ABmag : AB magnitudes - - u.STmag : HST magnitudes - - u.Jy : Jansky per filter bandpass - Additionally the "FLAM" and "FNU" units from ``synphot.units`` can - be used when passing the quantity for `amplitude`. - - Returns - ------- - spectrum : synphot.SourceSpectrum - Input spectrum scaled to the given amplitude in the given filter. - - Examples - -------- - :: - - >>> from scopesim.source.source_templates import vega_spectrum - >>> from scopesim.effects.ter_curves_utils as ter_utils - >>> - >>> spec = vega_spectrum() - >>> vega_185 = ter_utils.scale_spectrum(spec, "Ks", -1.85 * u.mag) - >>> ab_0 = ter_utils.scale_spectrum(spec, "Ks", 0 * u.ABmag) - >>> jy_3630 = ter_utils.scale_spectrum(spec, "Ks", 3630 * u.Jy) - - """ - if isinstance(amplitude, u.Quantity): - if amplitude.unit.physical_type == "spectral flux density": - if amplitude.unit != u.ABmag: - amplitude = amplitude.to(u.ABmag) - ref_spec = ab_spectrum(amplitude.value) - - elif amplitude.unit.physical_type == "spectral flux density wav": - if amplitude.unit != u.STmag: - amplitude = amplitude.to(u.STmag) - ref_spec = st_spectrum(amplitude.value) - - elif amplitude.unit == u.mag: - ref_spec = vega_spectrum(amplitude.value) - - else: - raise ValueError(f"Units of amplitude must be one of " - f"[u.mag, u.ABmag, u.STmag, u.Jy]: {amplitude}") - else: - ref_spec = vega_spectrum(amplitude) - - filt = get_filter(filter_name) - ref_flux = Observation(ref_spec, filt).effstim(flux_unit=PHOTLAM) - - real_flux = Observation(spectrum, filt).effstim(flux_unit=PHOTLAM) - scale_factor = ref_flux / real_flux - spectrum *= scale_factor.value - - return spectrum - - -def apply_throughput_to_cube( - cube: fits.ImageHDU, - thru: SpectralElement | SourceSpectrum, - wave_cube: u.Quantity, -) -> fits.ImageHDU: - """ - Apply throughput curve to a spectroscopic cube. - - Parameters - ---------- - cube : fits.ImageHDU - Three-dimensional image, dimension 0 (in python convention) is the - spectral dimension. - thru : SpectralElement | SourceSpectrum - Throughput curve, spectrum or filter. - wave_cube : u.Quantity["length"] - Wavelength axis of the cube. - - Returns - ------- - cube : fits.ImageHDU - Header unchanged, data multiplied with wavelength-dependent throughput. - - """ - # Note: wave_cube used to be converted to AA, but synphot understands um - # or whatever just as well... - cube.data *= thru(wave_cube).value[:, None, None] - return cube - - -def combine_two_spectra(spec_a, spec_b, action, wave_min, wave_max): - """ - Combine transmission and/or emission spectrum with a common waverange. - - Spec_A is the source spectrum - Spec_B is either the transmission or emission that should be applied - - Parameters - ---------- - spec_a : synphot.SourceSpectrum - spec_b : synphot.SpectralElement, synphot.SourceSpectrum - action: {"multiply", "add"} - wave_min, wave_max : quantity - [Angstrom] - - Returns - ------- - new_source : synphot.SourceSpectrum - - """ - if spec_a.waveset is None: - wave_val = spec_b.waveset.value - else: - wave_val = spec_a.waveset.value - mask = (wave_val > wave_min.value) * (wave_val < wave_max.value) - - wave = ([wave_min.value] + list(wave_val[mask]) + [wave_max.value]) * u.AA - if "mult" in action.lower(): - spec_c = spec_a(wave) * spec_b(wave) - # Diagnostic plots - not for general use - # from matplotlib import pyplot as plt - # plt.plot(wave, spec_a(wave), label="spec_a") - # plt.plot(wave, spec_b(wave), label="spec_b") - # plt.plot(wave, spec_c, label="spec_c") - # plt.xlim(2.9e4, 4.2e4) - # plt.legend() - # plt.show() - elif "add" in action.lower(): - spec_c = spec_a(wave) + spec_b(wave) - else: - raise ValueError(f"action {action} unknown") - - new_source = SourceSpectrum(Empirical1D, points=wave, lookup_table=spec_c) - new_source.meta.update(spec_b.meta) - new_source.meta.update(spec_a.meta) - - return new_source - - def add_edge_zeros(tbl, wave_colname): if isinstance(tbl, Table): vals = np.zeros(len(tbl.colnames)) diff --git a/scopesim/optics/fov.py b/scopesim/optics/fov.py index ef251938d..d38f4abe5 100644 --- a/scopesim/optics/fov.py +++ b/scopesim/optics/fov.py @@ -2,7 +2,6 @@ """Defines FieldOfView class.""" from warnings import warn -from copy import deepcopy from itertools import chain from collections.abc import Iterable, Generator from weakref import WeakKeyDictionary @@ -33,7 +32,6 @@ get_logger, array_minmax, close_loop, - unit_includes_per_physical_type, figure_factory, image_plotter, cube_plotter, @@ -335,35 +333,36 @@ def extract_area_from_imagehdu(self, imagehdu, corners): xyp.sort(axis=0) logger.debug("xyp:\n%s", xyp) - xy0p = np.max(((0, 0), np.floor(xyp[0]).astype(int)), axis=0) - xy1p = np.min(((naxis1, naxis2), np.ceil(xyp[1]).astype(int)), axis=0) + xy0p = np.max(((0, 0), np.floor(xyp[0]).astype(np.intp)), axis=0) + # xyp are inclusive bounds; python (in)famously excludes upper bound, + # hence xy1p needs to be increased by one + xy1p = np.min(((naxis1, naxis2), + 1 + np.ceil(xyp[1]).astype(np.intp)), + axis=0) logger.debug("xy0p: %s; xy1p: %s", xy0p, xy1p) - # Add 1 if the same - xy1p += (xy0p == xy1p) - logger.debug("xy0p: %s; xy1p: %s", xy0p, xy1p) - - new_wcs, new_naxis = imp_utils.create_wcs_from_points( - np.array([xy0s, xy1s]).round(11), pixel_scale=hdr["CDELT1"]) - - # TODO: Come back at some point and figure out if the failing tests - # here are relevant or can be ignored... - # FIXME: Commented out for now because it appears too often IRL... - # try: - # roundtrip = new_wcs.wcs_world2pix( - # np.array([xy0s, xy1s - .5*image_wcs.wcs.cdelt]), 0).round(5) - # np.testing.assert_array_equal(roundtrip[0], [0, 0]) - # np.testing.assert_array_equal(roundtrip[1], new_naxis - [1, 1]) - # except AssertionError: - # logger.exception("WCS roundtrip assertion failed.") - # FIXME: Related to the above, this sometimes fails: - # np.testing.assert_equal(xy1p - xy0p, new_naxis) - # This occurs when the floor and ceil in xy0s and xy1s produce an - # off-by-one error. Using .round instead for both would solve things - # in those cases, but breaks in other cases. Find a proper solution! - # Note: This is not super fatal, because the resulting projections - # will trim off that extra pixel later on, but this should still - # be addressed. + # Describe the cutout with the input WCS, shifted by the slice origin. + # The cutout's pixels ARE input pixels, so this is exact: the cutout + # stays on the input's pixel lattice and NAXISn always agrees with + # data.shape. + # + # Deriving the WCS from [xy0s, xy1s] instead (via + # create_wcs_from_points) got both wrong. Those are the world corners + # *before* the floor/ceil above grew the slice outwards to whole input + # pixels, so the resulting WCS described a differently sized image + # (NAXISn disagreed with data.shape by a pixel, silently corrected by + # fits.ImageHDU, which left CRPIXn describing the wrong array) placed + # on the FOV's lattice rather than the input's. Projecting that cutout + # then mis-registered it by up to a whole pixel, and by differing + # amounts for neighbouring FOVs, so the seam between two FOVs picked + # up a duplicated or a dropped row/column. + new_wcs = image_wcs.deepcopy() + new_wcs.wcs.crpix = image_wcs.wcs.crpix - xy0p + new_wcs.wcs.cdelt = np.abs(new_wcs.wcs.cdelt) + new_naxis = xy1p - xy0p + logger.debug("orig image wcs: %s", image_wcs) + logger.debug("new cutout wcs: %s", new_wcs) + logger.debug("new cutout naxis: %s", new_naxis) new_hdr = new_wcs.to_header() new_hdr.update({"NAXIS1": new_naxis[0], "NAXIS2": new_naxis[1]}) @@ -608,18 +607,17 @@ def _make_scaled_hdu(self, field: ImageSourceField) -> fits.ImageHDU: """Deal with BUNIT and pixel area.""" # Note: Do not scale source data - make a copy first. field_hdu = field.field.copy() # .field is the HDU (yeah...) - field_hdu.data /= self.pixel_area.value - # TODO: Check if this scaling is actually correct. How does this - # work with the add_imagehdu_to_imagehdu below? Isn't that - # supposed to conserve flux? Test carefully!! if field.is_bunit_spatially_differential: - # Field is in (PHOTLAM) arcsec-2, need to scale by pixarea - field_hdu.data *= field.pixel_area.value + logger.debug("differential bunit...") else: + logger.debug("binned bunit...") + logger.debug("scaling by %f", field.pixel_area.value) + field_hdu.data /= field.pixel_area.value # Pixel area doesn't cancel out, need to convert new_bunit = field.bunit / u.arcsec**2 field_hdu.header["BUNIT"] = new_bunit.to_string("fits") + logger.debug("BUNIT after scaling: %s", field_hdu.header["BUNIT"]) return field_hdu @@ -747,13 +745,16 @@ def _make_cubefields(self): # cube_fields come in with units of photlam/arcsec2, # need to convert to ph/s # We need to the voxel volume (spectral and solid angle) for that. - spectral_bin_width = (field.header["CDELT3"] * - u.Unit(field.header["CUNIT3"]) - ).to(u.Angstrom) + spectral_bin_width = ( + field.header["CDELT3"] * + u.Unit(field.header["CUNIT3"]) + ).to(u.Angstrom) # First collapse to image, then convert units image = np.sum(field.data, axis=0) * PHOTLAM/u.arcsec**2 - image = (image * self.area * - spectral_bin_width).to(u.ph/u.s/u.arcsec**2) + image = (image * self.area * field.pixel_area * + spectral_bin_width).to(u.ph/u.s) + logger.debug("2D FOV make_cubefields: image.mean() = %f %s", + image.mean().value, image.unit) # FIXME: This might create a 2D ImageHDU with a 3D header, not ideal... yield fits.ImageHDU(data=image, header=field.header) @@ -766,7 +767,15 @@ def _make_imagefields(self, fov_waveset, bin_widths): * yield image to be added to canvas image """ for field in self._get_image_fields(): - field_hdu = self._make_scaled_hdu(field) + field_hdu = field.field.copy() + + if field.is_bunit_spatially_differential: + logger.warning("Differential BUNIT in 2D image is discouraged.") + logger.debug("scaling by %f", field.pixel_area.value) + field_hdu.data *= field.pixel_area.value + new_bunit = field.bunit / field.pixel_area.unit + field_hdu.header["BUNIT"] = new_bunit.to_string("fits") + logger.debug("BUNIT after scaling: %s", field_hdu.header["BUNIT"]) # Reevaluate spectrum onto FOV waveset spec = field.spectrum(fov_waveset) # PHOTLAM @@ -775,7 +784,7 @@ def _make_imagefields(self, fov_waveset, bin_widths): flux = (spec * bin_widths * self.area).to_value(u.ph / u.s).sum() # Rescale 2D flux weight map by integrated flux from spectrum - field_hdu.data *= flux # ph s-1 arcsec-2 + field_hdu.data *= flux # ph s-1 yield field_hdu def _make_tablefields(self, fov_waveset, bin_widths, use_photlam=False): @@ -848,17 +857,16 @@ def make_hdu(self): data=np.zeros((self.header["NAXIS2"], self.header["NAXIS1"])), header=self.header) - for tmp_hdu in chain(self._make_cubefields(), - self._make_imagefields( - fov_waveset, bin_widths)): + for tmp_hdu in chain( + self._make_cubefields(), + self._make_imagefields(fov_waveset, bin_widths) + ): canvas_image_hdu = imp_utils.add_imagehdu_to_imagehdu( tmp_hdu, canvas_image_hdu, - conserve_flux=True, - spline_order=self.spline_order) - - # TODO: Move this further along at some point... - canvas_image_hdu.data *= self.pixel_area.value # eliminate arcsec-2 + conserve_flux=True, # binned flux needs this + spline_order=self.spline_order, + ) for flux, weight, x, y in self._make_tablefields( fov_waveset, bin_widths): @@ -928,14 +936,16 @@ def _make_cubefields(self, fov_waveset): # Assumption is that ImageHDUs have units of PHOTLAM arcsec-2 # TODO: Deal with this bounds_error in a more elegant way - field_interp = interp1d(field.waveset.to(u.um).value, - field.data, axis=0, kind="linear", - bounds_error=False, fill_value=0) + field_interp = interp1d( + field.waveset.to(u.um).value, + field.data, + axis=0, + kind="linear", + bounds_error=False, + fill_value=0, + ) field_data = field_interp(fov_waveset.value) - - # Pixel scale conversion - field_data *= field.pixel_area / self.pixel_area field_hdu = fits.ImageHDU(data=field_data, header=field.header) yield field_hdu @@ -960,10 +970,12 @@ def _make_imagefields(self, fov_waveset, spline_order): canvas_image_hdu = imp_utils.add_imagehdu_to_imagehdu( field_hdu, canvas_image_hdu, - spline_order=spline_order) + spline_order=spline_order, + conserve_flux=True, + differential=True, + ) spec = field.spectrum(fov_waveset) - # 2D * 1D -> 3D field_cube = canvas_image_hdu.data[None, :, :] * spec[:, None, None] yield field_cube.value @@ -1116,11 +1128,15 @@ def make_hdu(self): canvas_cube_hdu = imp_utils.add_imagehdu_to_imagehdu( field_hdu, canvas_cube_hdu, - spline_order=self.spline_order) + spline_order=self.spline_order, + conserve_flux=True, + differential=True, + ) - canvas_cube_hdu.data = sum(self._make_imagefields( - fov_waveset, self.spline_order), - start=canvas_cube_hdu.data) + canvas_cube_hdu.data = sum( + self._make_imagefields(fov_waveset, self.spline_order), + start=canvas_cube_hdu.data, + ) for flux, x, y in self._make_tablefields(fov_waveset): # To prevent adding array values in this manner. @@ -1135,6 +1151,9 @@ def make_hdu(self): canvas_cube_hdu.data *= 1e4 # ph/s/AA/arcsec2 --> ph/s/um/arcsec2 canvas_cube_hdu.header["BUNIT"] = "ph s-1 um-1 arcsec-2" + logger.debug("3D FOV make_hdu: canvas_cube_hdu.data.mean() = %f", + canvas_cube_hdu.data.mean()) + return canvas_cube_hdu # [ph s-1 um-1 (arcsec-2)] def plot_data(self): diff --git a/scopesim/optics/fov_manager.py b/scopesim/optics/fov_manager.py index 29f7ef888..4e7a894fe 100644 --- a/scopesim/optics/fov_manager.py +++ b/scopesim/optics/fov_manager.py @@ -61,6 +61,32 @@ logger = get_logger(__name__) +def chunk_edges(vmin, vmax, step): + """Return the interior points that cut ``[vmin, vmax]`` into `step` chunks. + + Only interior points are returned, so no edge can coincide with `vmin` or + `vmax`. `step` need not divide the span, and need not be an integer (see + HAWKI/test_hawki/test_full_package_hawki.py), so this cannot be a plain + ``range``. + + It cannot be ``np.arange(vmin, vmax, step)`` either. That returns + ``ceil((vmax - vmin) / step)`` elements, which is one too many when the + division rounds just above an integer - as it does for a span that is an + exact multiple of `step`, e.g. ``(1.2000000000000002 - -1.2000000000000002) + / 0.8 == 3.0000000000000004``. Whether the surplus element then lands + exactly on `vmax` or one ulp below it depends on how the platform rounds + ``vmin + i * step``: it lands on `vmax` on x86, but below it on arm64 + macOS. One ulp below `vmax` is a perfectly legal split point, so + ``FovVolumeList.split`` accepts it and carves off a volume ~1e-15 px wide, + which ``create_wcs_from_points`` then rounds up to a whole pixel - a + spurious one-pixel FOV, off the pixel lattice, double-counting flux at the + edge of the array. + + """ + n_chunks = max(1, int(np.ceil((vmax - vmin) / step - 1e-9))) + return vmin + step * np.arange(1, n_chunks) + + class FOVManager: """ A class to manage the (monochromatic) image windows covering the target. @@ -122,11 +148,8 @@ def _get_splits(self, pixel_scale): if vol_pix_area > max_seg_size: step = chunk_size * pixel_scale - # These are not always integers, unlike in the tests. - # See for example HAWKI/test_hawki/test_full_package_hawki.py. - # The np.arange can therefore not be changed to just a range. - yield (np.arange(vol["x_min"], vol["x_max"], step), - np.arange(vol["y_min"], vol["y_max"], step)) + yield (chunk_edges(vol["x_min"], vol["x_max"], step), + chunk_edges(vol["y_min"], vol["y_max"], step)) def generate_fovs_list(self) -> Iterator[FieldOfView]: """ @@ -152,7 +175,7 @@ def generate_fovs_list(self) -> Iterator[FieldOfView]: self.volumes_list = effect.apply_to( self.volumes_list, **effect_params) - # ..todo: add catch to split volumes larger than chunk_size + # TODO: add catch to split volumes larger than chunk_size pixel_scale = from_currsys(self.meta["pixel_scale"], self.cmds) splits = (chain.from_iterable(split) @@ -188,7 +211,6 @@ def generate_fovs_list(self) -> Iterator[FieldOfView]: xs_min, xs_max = vol["x_min"] / 3600., vol["x_max"] / 3600. ys_min, ys_max = vol["y_min"] / 3600., vol["y_max"] / 3600. - waverange = (vol["wave_min"], vol["wave_max"]) skyhdr = ipu.header_from_list_of_xy([xs_min, xs_max], [ys_min, ys_max], pixel_scale=pixel_scale / 3600.) @@ -219,7 +241,7 @@ def generate_fovs_list(self) -> Iterator[FieldOfView]: new_fov = fovcls( skyhdr, - waverange, + (vol["wave_min"], vol["wave_max"]), detector_header=dethdr, cmds=self.cmds, **fov_meta, diff --git a/scopesim/optics/image_plane.py b/scopesim/optics/image_plane.py index e2b101f48..3ecba4d4f 100644 --- a/scopesim/optics/image_plane.py +++ b/scopesim/optics/image_plane.py @@ -1,15 +1,11 @@ # -*- coding: utf-8 -*- """Contains ``ImagePlane`` class.""" - -from warnings import warn - import numpy as np from astropy.io import fits -from astropy.table import Table from astropy.wcs import WCS -from .image_plane_utils import add_table_to_imagehdu, add_imagehdu_to_imagehdu +from .image_plane_utils import add_imagehdu_to_imagehdu from ..utils import ( from_currsys, @@ -68,7 +64,7 @@ def __init__(self, header, cmds=None, **kwargs): raise ValueError(f"header must have a valid image-plane WCS: " f"{dict(header)}") - image = zeros_from_header(header) + image = zeros_from_header(header, dtype=np.float32) self.hdu = fits.ImageHDU(data=image, header=header) self.hdu.header["BUNIT"] = "ph s-1" # photons per second (per pixel) diff --git a/scopesim/optics/image_plane_utils.py b/scopesim/optics/image_plane_utils.py index 5781366da..15f91daf5 100644 --- a/scopesim/optics/image_plane_utils.py +++ b/scopesim/optics/image_plane_utils.py @@ -1,159 +1,19 @@ # -*- coding: utf-8 -*- -from itertools import product -from collections.abc import Iterable - import numpy as np from numpy.typing import ArrayLike, NDArray from astropy import units as u from astropy.wcs import WCS, find_all_wcs from astropy.io import fits -from astropy.table import Table from astropy.utils.exceptions import AstropyWarning from scipy import ndimage as ndi -from ..utils import (unit_from_table, quantity_from_table, has_needed_keywords, - get_logger) +from ..utils import get_logger logger = get_logger(__name__) -def get_canvas_header(hdu_or_table_list, pixel_scale=1 * u.arcsec): - """ - Generate a fits.Header with a WCS that covers everything in the FOV. - - Parameters - ---------- - hdu_or_table_list : list - A list of Tables and/or ImageHDU py_objects - - pixel_scale : astropy.Quantity - [arcsec] The pixel scale of the projection. Default in 1 arcsec - - Returns - ------- - header : fits.Header - A Header containing a WCS and NAXISn values to build an ImageHDU - - """ - size_warning = ("Header dimension are {adverb} large: {num_pix}. " - "Any image made from this header will use more that " - ">{size} in memory") - - def _get_headers(hdus_or_tables): - tables = [] - for hdu_or_table in hdus_or_tables: - if isinstance(hdu_or_table, fits.ImageHDU): - yield hdu_or_table.header - elif isinstance(hdu_or_table, fits.Header): - yield hdu_or_table - elif isinstance(hdu_or_table, Table): - tables.append(hdu_or_table) - else: - raise TypeError( - "hdu_or_table_list may only contain fits.ImageHDU, Table " - f"or fits.Header, found {type(hdu_or_table)}.") - if tables: - yield _make_bounding_header_for_tables(*tables, - pixel_scale=pixel_scale) - - headers = list(_get_headers(hdu_or_table_list)) - - if not headers: - logger.warning("No tables or ImageHDUs were passed") - return None - - hdr = _make_bounding_header_from_headers(*headers, pixel_scale=pixel_scale) - - num_pix = hdr["NAXIS1"] * hdr["NAXIS2"] - if num_pix > 2 ** 28: - raise MemoryError(size_warning.format(adverb="too", num_pix=num_pix, - size="8 GB")) - if num_pix > 2 ** 25: # 2 * 4096**2 - logger.warning(size_warning.format(adverb="", num_pix=num_pix, - size="256 MB")) - return hdr - - -def _make_bounding_header_from_headers(*headers, pixel_scale=1*u.arcsec): - """ - Return a Header with WCS and NAXISn keywords bounding all input ImageHDUs. - - Parameters - ---------- - headers : list of fits.ImageHDU - pixel_scale : u.Quantity - [arcsec] - - Returns - ------- - hdr : fits.Header - - """ - wcs_suffix = "D" if pixel_scale.unit.physical_type == "length" else "" - unit = u.Unit(_get_unit_from_headers(*headers, wcs_suffix=wcs_suffix)) - - if unit.physical_type == "angle": - unit = "deg" - pixel_scale = pixel_scale.to_value(u.deg) - else: - pixel_scale = pixel_scale.to_value(unit) - - extents = [calc_footprint(header, wcs_suffix, unit) for header in headers] - pnts = np.vstack(extents) - - hdr = header_from_list_of_xy(pnts[:, 0], pnts[:, 1], - pixel_scale, wcs_suffix) - hdr["NAXIS1"] += 1 - hdr["NAXIS2"] += 1 - hdr[f"CRVAL1{wcs_suffix}"] -= 0.5 * pixel_scale - hdr[f"CRVAL2{wcs_suffix}"] -= 0.5 * pixel_scale - - return hdr - - -def _make_bounding_header_for_tables(*tables, pixel_scale=1*u.arcsec): - """ - Return a Header with WCS and NAXISn keywords bounding all input Tables. - - Parameters - ---------- - tables : list of astropy.Tables - [arcsec] Must contain columns: "x", "y" - - pixel_scale : u.Quantity - [arcsec] - - Returns - ------- - hdr : fits.Header - - """ - wcs_suffix = "D" if pixel_scale.unit.physical_type == "length" else "" - # Note: this could all be a lot simpler if we have consistent units, i.e. - # don't need to convert mm -> mm and arcsec -> arcsec (or deg) - new_unit = u.mm if wcs_suffix == "D" else u.deg # arcsec here causes problems later when all header units are expected to be same - tbl_unit = u.mm if wcs_suffix == "D" else u.arcsec - x_name = "x_mm" if wcs_suffix == "D" else "x" - y_name = "y_mm" if wcs_suffix == "D" else "y" - - pixel_scale = pixel_scale.to(new_unit) - - extents = [] - for table in tables: - extent = calc_table_footprint(table, x_name, y_name, - tbl_unit, new_unit, - padding=pixel_scale) - extents.append(extent) - pnts = np.vstack(extents) - - # TODO: check if this could just use create_wcs_from_points - hdr = header_from_list_of_xy(pnts[:, 0], pnts[:, 1], pixel_scale.value, - wcs_suffix)#, arcsec=wcs_suffix != "D") - return hdr - - def create_wcs_from_points( points: ArrayLike, pixel_scale: float, @@ -213,7 +73,41 @@ def create_wcs_from_points( offset[zeroaxis] = 0 crpix = (naxis + 1) / 2 - crval = (points.min(axis=0) + points.max(axis=0) + offset) / 2 + + # Where to put the pixel lattice when NAXISn cannot cover the requested + # region exactly. + # + # If the grid truncates the region (naxis < extent, i.e. the region is not + # a whole number of pixels across), anchor the lower pixel edge at + # points.min and let the leftover fraction fall off the far edge. Every + # region carved out of the same origin in whole-pixel steps then shares one + # lattice - which is what chunked FOVs are: the image plane, the full-size + # chunks and the smaller last chunk all measure from the same detector + # edge. Centring each of them on its own midpoint instead, and absorbing + # its own rounding remainder into CRVAL, put them on lattices offset from + # one another by fractions of a pixel, so the projection had to snap them + # and the last chunk could land a whole pixel off its neighbours. + # + # MICADO's detector plane is 189.32 mm / 0.015 mm = 12621.33 px tall, so it + # hits this on every run. + # + # If the grid covers the region (naxis >= extent) there is nothing to + # truncate, so keep centring it on the region. That is the right thing for + # a region smaller than a single pixel, where naxis was clamped up to 1. + _pxs = float(pixel_scale.value if isinstance(pixel_scale, u.Quantity) + else pixel_scale) + _pts = points.value if isinstance(points, u.Quantity) else np.asarray(points) + _naxis = np.asarray(naxis, dtype=float) + _extent = np.atleast_1d(np.asarray(extent, dtype=float)) + + crval = np.where( + _naxis < _extent, + _pts.min(axis=0) + (_naxis / 2) * _pxs, # anchored + (_pts.min(axis=0) + _pts.max(axis=0) + + np.asarray(offset, dtype=float)) / 2, # centred + ) + if isinstance(points, u.Quantity): + crval = crval * points.unit # Cannot do `in "DX"` here because that would also match the empty string. linsuff = {"D", "X"} @@ -278,95 +172,6 @@ def header_from_list_of_xy(x, y, pixel_scale, wcs_suffix="", arcsec=False): return hdr -def add_table_to_imagehdu(table: Table, canvas_hdu: fits.ImageHDU, - sub_pixel: bool = True, - wcs_suffix: str = "") -> fits.ImageHDU: - """ - Add files from an astropy.Table to the image of an fits.ImageHDU. - - Parameters - ---------- - table : astropy.Table - Must contain the columns "x_mm", "y_mm", "flux" with the units in the - column attribute .unit, or in the table.meta dictionary as - "_unit". Default units are ``mm`` and ``ph / s / pix`` - - canvas_hdu : fits.ImageHDU - The ImageHDU onto which the table files should be projected. - This must include a valid WCS - - sub_pixel : bool, optional - Default is True. If True, sub-pixel shifts of files will be taken into - account when projecting onto the canvas pixel grid. This takes about 5x - longer than ignoring the sub-pixel shifts - - wcs_suffix : str, optional - - Returns - ------- - canvas_hdu : fits.ImageHDU - - """ - s = wcs_suffix - if not has_needed_keywords(canvas_hdu.header, s): - raise ValueError(f"canvas_hdu must include an appropriate WCS: {s}") - - f = quantity_from_table("flux", table, default_unit=u.Unit("ph s-1")) - if s == "D": - x = quantity_from_table("x_mm", table, default_unit=u.mm).to(u.mm) - y = quantity_from_table("y_mm", table, default_unit=u.mm).to(u.mm) - else: - arcsec = u.arcsec - canvas_unit = u.Unit(canvas_hdu.header[f"CUNIT1{s}"]) - x = quantity_from_table("x", table, default_unit=arcsec.to(canvas_unit)) - y = quantity_from_table("y", table, default_unit=arcsec.to(canvas_unit)) - x *= arcsec.to(canvas_unit) - y *= arcsec.to(canvas_unit) - - xpix, ypix = val2pix(canvas_hdu.header, x.value, y.value, s) - - naxis1 = canvas_hdu.header["NAXIS1"] - naxis2 = canvas_hdu.header["NAXIS2"] - # Occasionally 0 is returned as ~ -1e-11 - eps = -1e-7 - mask = (xpix >= eps) * (xpix < naxis1) * (ypix >= eps) * (ypix < naxis2) - - if sub_pixel: - canvas_hdu = _add_subpixel_sources_to_canvas( - canvas_hdu, xpix, ypix, f, mask) - else: - canvas_hdu = _add_intpixel_sources_to_canvas( - canvas_hdu, xpix, ypix, f, mask) - - return canvas_hdu - - -def _add_intpixel_sources_to_canvas(canvas_hdu, xpix, ypix, flux, mask): - canvas_hdu.header["comment"] = f"Adding {len(flux)} int-pixel files" - for xpx, ypx, flx, msk in zip(xpix.astype(int), ypix.astype(int), - flux, mask): - # To prevent adding array values in this manner. - assert not isinstance(xpx, Iterable), "xpx should be integer" - canvas_hdu.data[ypx, xpx] += flx.value * msk - - return canvas_hdu - - -def _add_subpixel_sources_to_canvas(canvas_hdu, xpix, ypix, flux, mask): - canvas_hdu.header["comment"] = f"Adding {len(flux)} sub-pixel files" - canvas_shape = canvas_hdu.data.shape - for xpx, ypx, flx, msk in zip(xpix, ypix, flux, mask): - if msk: - xx, yy, fracs = sub_pixel_fractions(xpx, ypx) - for x, y, frac in zip(xx, yy, fracs): - if y < canvas_shape[0] and x < canvas_shape[1]: - # To prevent adding array values in this manner. - assert not isinstance(x, Iterable), "x should be integer" - canvas_hdu.data[y, x] += frac * flx.value - - return canvas_hdu - - def sub_pixel_fractions(x, y): """ Make a list of pixel coordinates and weights to reflect sub-pixel shifts. @@ -436,13 +241,32 @@ def overlay_image(small_im, big_im, coords, mask=None, sub_pixel=False): coords = np.array([*coords, (big_im.shape[0] - 1) / 2]) # FIXME: this would not be necessary if we used WCS instead of manual 2pix - # Round to 1e-4 pix before ceil: WCS deg-space round-trips leave - # float dust of order 1e-8..1e-6 pix on integer-valued coords, which - # ceil would amplify to a full-pixel shift; genuine sub-pixel intent - # cannot be finer than 0.5 pix here (sub_pixel is not implemented), - # and the half-integer even-shape convention is preserved exactly. - coords = np.ceil(np.asarray(coords).round(4)).astype(np.intp) - idx = coords.astype(int)[::-1] - np.array(small_im.shape) // 2 + # Round to 1e-4 pix first: WCS deg-space round-trips leave float dust of + # order 1e-8..1e-6 pix on integer-valued coords, which the snap below + # would otherwise amplify to a full-pixel shift; genuine sub-pixel intent + # cannot be finer than 0.5 pix here (sub_pixel is not implemented). + coords = np.asarray(coords, dtype=float).round(4) + + # `idx` is the origin, in array order, at which small_im must be placed to + # centre it on `coords` (both are 0-based pixel-centre coordinates, as + # returned by WCS.wcs_world2pix(..., 0)). + # + # Snap that origin, NOT `coords` itself. The old expression + # np.ceil(coords)[::-1] - np.array(small_im.shape) // 2 + # is off by up to a whole pixel whenever `coords` is not on the lattice + # small_im implies (half-integer for even shapes, integer for odd ones), + # and the direction of that error flips with the parity of the shape. Two + # adjacent FOVs of different sizes therefore get displaced opposite ways + # and their shared edge picks up a duplicated or a dropped row/column. + # Snapping the origin instead keeps every placement consistent, whatever + # the shape. + # + # Half-integer origins (a real sub-pixel shift, which sub_pixel=True would + # have to handle properly) are rounded towards the lower index, matching + # what the old expression did for even shapes. + idx = np.ceil( + coords[::-1] - (np.array(small_im.shape) - 1) / 2 - 0.5 + ).astype(np.intp) # Image ranges idx1 = np.maximum(0, idx) @@ -482,9 +306,14 @@ def overlay_image(small_im, big_im, coords, mask=None, sub_pixel=False): return big_im -def rescale_imagehdu(imagehdu: fits.ImageHDU, pixel_scale: float | u.Quantity, - wcs_suffix: str = "", conserve_flux: bool = True, - spline_order: int = 1) -> fits.ImageHDU: +def rescale_imagehdu( + imagehdu: fits.ImageHDU, + pixel_scale: float | u.Quantity, + wcs_suffix: str = "", + conserve_flux: bool = True, + spline_order: int = 1, + differential: bool = False, +) -> fits.ImageHDU: """ Scale the .data array by the ratio of pixel_scale [deg] and CDELTn. @@ -540,6 +369,10 @@ def rescale_imagehdu(imagehdu: fits.ImageHDU, pixel_scale: float | u.Quantity, sum_orig = np.sum(imagehdu.data) + # scale by pixel area if the flux is differential (i.e. per unit area) + if differential: + sum_orig *= primary_wcs.wcs.cdelt[0] * primary_wcs.wcs.cdelt[1] + # Perform the rescaling. Axes need to be inverted because python. zoom_np = zoom[::-1] data = imagehdu.data @@ -591,8 +424,13 @@ def rescale_imagehdu(imagehdu: fits.ImageHDU, pixel_scale: float | u.Quantity, if conserve_flux: new_im = np.nan_to_num(new_im, copy=False) sum_new = np.sum(new_im) + if differential: + sum_new *= (pixel_scale.value**2) + if sum_new != 0: - new_im *= sum_orig / sum_new + flux_factor = sum_orig / sum_new + logger.debug("flux factor = %f", flux_factor) + new_im *= flux_factor elif sum_orig != 0: logger.warning( "rescale_imagehdu: all input flux (%g) was lost in resampling; " @@ -645,9 +483,12 @@ def rescale_imagehdu(imagehdu: fits.ImageHDU, pixel_scale: float | u.Quantity, return imagehdu -def reorient_imagehdu(imagehdu: fits.ImageHDU, wcs_suffix: str = "", - conserve_flux: bool = True, - spline_order: int = 1) -> fits.ImageHDU: +def reorient_imagehdu( + imagehdu: fits.ImageHDU, + wcs_suffix: str = "", + conserve_flux: bool = True, + spline_order: int = 1, +) -> fits.ImageHDU: """ Apply an affine transformation to the image, as given in its header. @@ -705,9 +546,15 @@ def reorient_imagehdu(imagehdu: fits.ImageHDU, wcs_suffix: str = "", return imagehdu -def affine_map(input, matrix=None, rotation_angle: float = 0., - shear_angle: float = 0., scale_factor=None, - reshape: bool = True, spline_order: int = 3): +def affine_map( + input, + matrix=None, + rotation_angle: float = 0., + shear_angle: float = 0., + scale_factor=None, + reshape: bool = True, + spline_order: int = 3, +): """ Apply an affine transformation matrix to an image around its centre. @@ -778,11 +625,14 @@ def affine_map(input, matrix=None, rotation_angle: float = 0., return output -def add_imagehdu_to_imagehdu(image_hdu: fits.ImageHDU, - canvas_hdu: fits.ImageHDU, - spline_order: int = 1, - wcs_suffix: str = "", - conserve_flux: bool = True) -> fits.ImageHDU: +def add_imagehdu_to_imagehdu( + image_hdu: fits.ImageHDU, + canvas_hdu: fits.ImageHDU, + spline_order: int = 1, + wcs_suffix: str = "", + conserve_flux: bool = True, + differential: bool = False, +) -> fits.ImageHDU: """ Re-project one ``fits.ImageHDU`` onto another ``fits.ImageHDU``. @@ -832,16 +682,22 @@ def add_imagehdu_to_imagehdu(image_hdu: fits.ImageHDU, canvas_pixel_scale = float(canvas_wcs.wcs.cdelt[0]) conv_fac = u.Unit(image_hdu.header[f"CUNIT1{wcs_suffix}"].lower()).to(canvas_wcs.wcs.cunit[0]) - new_hdu = rescale_imagehdu(image_hdu, pixel_scale=canvas_pixel_scale / conv_fac, - wcs_suffix=canvas_wcs.wcs.alt, - spline_order=spline_order, - conserve_flux=conserve_flux) + new_hdu = rescale_imagehdu( + image_hdu, + pixel_scale=canvas_pixel_scale / conv_fac, + wcs_suffix=canvas_wcs.wcs.alt, + spline_order=spline_order, + conserve_flux=conserve_flux, + differential=differential, + ) # TODO: Perhaps add separately formatted WCS logger? - # logger.debug("fromrescale %s", WCS(new_hdu.header, key=canvas_wcs.wcs.alt)) - new_hdu = reorient_imagehdu(new_hdu, - wcs_suffix=canvas_wcs.wcs.alt, - spline_order=spline_order, - conserve_flux=conserve_flux) + logger.debug("fromrescale %s", WCS(new_hdu.header, key=canvas_wcs.wcs.alt)) + new_hdu = reorient_imagehdu( + new_hdu, + wcs_suffix=canvas_wcs.wcs.alt, + spline_order=spline_order, + conserve_flux=conserve_flux, + ) img_center = np.array([[new_hdu.header[f"NAXIS{i+1}"] for i in range(new_hdu.header["NAXIS"])]]) @@ -869,6 +725,7 @@ def add_imagehdu_to_imagehdu(image_hdu: fits.ImageHDU, return canvas_hdu +# TODO: Use proper WCS functions instead and rm this on-foot implementation! def pix2val(header, x, y, wcs_suffix=""): """ Return the real coordinates [deg, mm] for coordinates from a Header WCS. @@ -909,6 +766,7 @@ def pix2val(header, x, y, wcs_suffix=""): return a, b +# TODO: Use proper WCS functions instead and rm this on-foot implementation! def val2pix(header, a, b, wcs_suffix=""): """ Return the pixel coordinates for real coordinates [deg, mm] from a WCS. @@ -1019,103 +877,6 @@ def calc_footprint(header, wcs_suffix="", new_unit: str = None): return xy1 -def calc_table_footprint( - table: Table, - x_name: str, - y_name: str, - tbl_unit: str, - new_unit: str, - padding: u.Quantity | None = None, -) -> NDArray: - """ - Equivalent to ``calc_footprint()``, but for tables instead of images. - - Parameters - ---------- - table : astropy.table.Table - Table containing data. - x_name : str - Name of the column in `table` to use as x-coordinates. - y_name : str - Name of the column in `table` to use as y-coordinates. - tbl_unit : str - Default unit to use for x and y if no units are found in `table`. - new_unit : str - Unit to convert x and y to, can be identical to `tbl_unit`. - padding : astropy.units.Quantity, optional - Constant value to subtract from minima and add to maxima. If used, must - be Quantity with same physical type as x and y. If None (default), no - padding is added. - - Returns - ------- - extent : (4, 2) array - Array containing corner points (clockwise from bottom left). Format and - order are equivalent to the output of - ``astropy.wcs.WCS.calc_footprint()``. - - """ - if padding is not None: - padding = padding.to_value(new_unit) - else: - padding = 0. - - x_convf = unit_from_table(x_name, table, tbl_unit).to(new_unit) - y_convf = unit_from_table(y_name, table, tbl_unit).to(new_unit) - - x_col = table[x_name] * x_convf - y_col = table[y_name] * y_convf - - x_min = x_col.min() - padding - x_max = x_col.max() + padding - y_min = y_col.min() - padding - y_max = y_col.max() + padding - - extent = np.array([[x_min, y_min], - [x_min, y_max], - [x_max, y_max], - [x_max, y_min]]) - - return extent - - -def split_header(hdr, chunk_size, wcs_suffix=""): - """ - Split a header into many smaller parts of the chunk_size. - - Parameters - ---------- - hdr - chunk_size - wcs_suffix - - Returns - ------- - hdr_list - """ - # TODO: test that this works - s = wcs_suffix - naxis1, naxis2 = hdr["NAXIS1"+s], hdr["NAXIS2"+s] - x0_pix, y0_pix = hdr["CRPIX1"+s], hdr["CRPIX2"+s] # pix - x0_sky, y0_sky = hdr["CRVAL1"+s], hdr["CRVAL2"+s] # deg - x_delt, y_delt = hdr["CDELT1"+s], hdr["CDELT2"+s] # deg / pix - - hdr_list = [] - for x1_pix in range(0, naxis1, chunk_size): - for y1_pix in range(0, naxis2, chunk_size): - x1_sky = x0_sky + (x1_pix - x0_pix) * x_delt - y1_sky = y0_sky + (y1_pix - y0_pix) * y_delt - x2_sky = x1_sky + x_delt * min(chunk_size, naxis1 - x1_pix) - y2_sky = y1_sky + y_delt * min(chunk_size, naxis2 - y1_pix) - - hdr_sky = header_from_list_of_xy([x1_sky, x2_sky], - [y1_sky, y2_sky], - pixel_scale=x_delt, wcs_suffix=s) - hdr_list.append(hdr_sky) - - return hdr_list - - def _fix_360(arr): """Fix the "full circle overflow" that occurs with deg.""" if isinstance(arr, u.Quantity): @@ -1128,15 +889,6 @@ def _fix_360(arr): return arr -def _get_unit_from_headers(*headers, wcs_suffix: str = "") -> str: - unit = headers[0][f"CUNIT1{wcs_suffix}"].lower() - assert all(header[f"CUNIT{i}{wcs_suffix}"].lower() == unit - for header, i in product(headers, range(1, 3))), \ - [(i, header[f"CUNIT{i}{wcs_suffix}"]) - for header, i in product(headers, range(1, 3))] - return unit - - def det_wcs_from_sky_wcs( sky_wcs: WCS, pixel_scale: float, @@ -1167,8 +919,6 @@ def det_wcs_from_sky_wcs( Shape of the image (``NAXIS1``, ``NAXIS2``). """ - # TODO: Using astropy units for now to avoid deg vs. arcsec confusion. - # Once Scopesim is consistent there, remove astropy units. pixel_scale <<= u.arcsec / u.pixel plate_scale <<= u.arcsec / u.mm logger.debug("Pixel scale: %s", pixel_scale) @@ -1186,10 +936,12 @@ def det_wcs_from_sky_wcs( return create_wcs_from_points(corners, pixel_size, "D") -def sky_wcs_from_det_wcs(det_wcs: WCS, - pixel_scale: float, - plate_scale: float, - naxis=None) -> tuple[WCS, np.ndarray]: +def sky_wcs_from_det_wcs( + det_wcs: WCS, + pixel_scale: float, + plate_scale: float, + naxis=None, +) -> tuple[WCS, np.ndarray]: """ Create celestial WCS from detector WCS using pixel and plate scales. @@ -1214,8 +966,6 @@ def sky_wcs_from_det_wcs(det_wcs: WCS, Shape of the image (``NAXIS1``, ``NAXIS2``). """ - # TODO: Using astropy units for now to avoid deg vs. arcsec confusion. - # Once Scopesim is consistent there, remove astropy units. pixel_scale <<= u.arcsec / u.pixel plate_scale <<= u.arcsec / u.mm logger.debug("Pixel scale: %s", pixel_scale) diff --git a/scopesim/optics/optical_train.py b/scopesim/optics/optical_train.py index 8ef81cd9c..cd8ab65bd 100644 --- a/scopesim/optics/optical_train.py +++ b/scopesim/optics/optical_train.py @@ -1,15 +1,16 @@ # -*- coding: utf-8 -*- import copy -import os from collections.abc import Mapping from datetime import datetime +from pathlib import Path import numpy as np from scipy.interpolate import interp1d from astropy import units as u from astropy.table import QTable from astropy.wcs import WCS +from astropy.io.fits import HDUList from tqdm.auto import tqdm @@ -171,9 +172,9 @@ def load(self, user_commands): self.yaml_dicts = self.cmds.yaml_dicts self.optics_manager = OpticsManager(self.yaml_dicts, self.cmds) - self.update() + self.update(migrate_cmds=True) - def update(self, **kwargs): + def update(self, migrate_cmds=False, **kwargs): """ Update the user-defined parameters and remake main internal classes. @@ -186,12 +187,19 @@ def update(self, **kwargs): self.optics_manager.update(**kwargs) opt_man = self.optics_manager - self.fov_manager = FOVManager(opt_man.fov_setup_effects, cmds=self.cmds, - **kwargs) - self.image_planes = [ImagePlane(hdr, self.cmds, **kwargs) - for hdr in opt_man.image_plane_headers] - self.detector_managers = [DetectorManager(det_list, cmds=self.cmds, **kwargs) - for det_list in opt_man.detector_setup_effects] + self.fov_manager = FOVManager( + opt_man.fov_setup_effects, + cmds=self.cmds, + **kwargs + ) + self.image_planes = [ + ImagePlane(hdr, self.cmds, **kwargs) + for hdr in opt_man.image_plane_headers + ] + self.detector_managers = [ + DetectorManager(det_list, cmds=self.cmds, **kwargs) + for det_list in opt_man.detector_setup_effects + ] # Move everything from CurrObs to CurrSys, so CurrObs is clean for # .observe and .readout. This is necessary because the setup and @@ -204,7 +212,8 @@ def update(self, **kwargs): # self.cmds.maps[0].clear() # HACK: recursive_update is needed to avoid overwriting emtpy !ABCs # TODO: or is it?? - self.cmds.maps[1].dic = recursive_update(self.cmds.maps[1].dic, self.cmds.maps[0].dic) + if migrate_cmds: + self.cmds.maps[1].dic = recursive_update(self.cmds.maps[1].dic, self.cmds.maps[0].dic) self.cmds.maps[0].dic.clear() def trace_detector_coordinates(self, *, xi=0 * u.arcsec, @@ -526,6 +535,17 @@ def observe(self, orig_source=None, update=True, **kwargs): if update: self.update(**kwargs) + if self.cmds["!SIM.random.seed"] is None: + # Concretize seed from system entropy to keep constant during the + # observe run. For multiple observations, this is cleared above. + self.cmds["!SIM.random.seed"] = np.random.SeedSequence().entropy + elif self.cmds["!SIM.random.seed"] in {"None", "none", "NONE"}: + logger.warning( + "Got random seed '%s' (a string, not None!), use NULL in yaml", + self.cmds["!SIM.random.seed"] + ) + self.cmds["!SIM.random.seed"] = np.random.SeedSequence().entropy + # self.set_focus(**kwargs) # put focus back on current instrument package # Make a copy of the Source and prepare for observation (convert to @@ -707,23 +727,39 @@ def prepare_source(self, source): return source @top_level_catch - def readout(self, filename=None, reset=True, **kwargs): + def readout( + self, + filename: Path | str | None = None, + reset: bool = True, + roid: int = 0, + **kwargs + ) -> list[HDUList]: """ Produce detector readouts for the observed image. Parameters ---------- - filename : str, optional - Where to save the FITS file - kwargs + filename : Path | str | None, optional + Full path or local name for to store the resulting FITS file. + If None (the default), the function still returns the result, but + will not automatically save it to disk. + reset : bool, optional + If true (the default), do not keep any parameters set during any + previous readout. This is usually what you want. + roid : int, optional + Readout ID, used for seeding randomness. If you run multiple + readouts on the same observation and want the random effects to + produce different results, set this parameter with e.g. the loop + counter. Simple integers are fine (0, 1, 2, ...), because this gets + wrangled into a proper random seed downstream. If you need to + reproduce a specific readout in a sequence, set e.g. `roid=4`. + **kwargs : + Any other kwargs passed to the detector effects. Returns ------- - hdu : fits.HDUList - - Notes - ----- - - Apply detector plane (0D, 2D) effects - z_order = 500..599 + list[HDUList] + List of HDUList objects. """ if reset: @@ -737,7 +773,7 @@ def readout(self, filename=None, reset=True, **kwargs): # self.cmds["!OBS.dit"] = None # self.cmds["!OBS.ndit"] = None # TODO: This is still hacky but seems to work for now... - params = {"exptime": None} + params = {"exptime": None, "roid": roid} params.update(kwargs) if params["exptime"] is not None and params.get("dit") is None: params.update( @@ -761,11 +797,12 @@ def readout(self, filename=None, reset=True, **kwargs): self.cmds[f"!OBS.{key}"] = value hduls = [] - for i, detector_array in enumerate(self.detector_managers): - array_effects = self.optics_manager.detector_array_effects - dtcr_effects = self.optics_manager.detector_effects - hdul = detector_array.readout( - self.image_planes, array_effects, dtcr_effects) + for i, detector_manager in enumerate(self.detector_managers): + hdul = detector_manager.readout( + self.image_planes, + self.optics_manager.detector_array_effects, + self.optics_manager.detector_effects, + ) fits_effects = self.optics_manager.get_all(ExtraFitsKeywords) if len(fits_effects) > 0: @@ -773,16 +810,22 @@ def readout(self, filename=None, reset=True, **kwargs): hdul = effect.apply_to(hdul, optical_train=self) else: try: - hdul = self.write_header(hdul, detector_array) + hdul = self.write_header(hdul, detector_manager) except Exception: - logger.exception("Header update failed, data will be " - "saved with incomplete header. See stack " - "trace for details.") - - if filename is not None and isinstance(filename, str): - fname = filename - if len(self.detector_managers) > 1: - fname = f"{os.path.dirname(filename)}/{i}_{os.path.basename(filename)}" + logger.exception( + "Header update failed, data will be saved with " + "incomplete header. See stack trace for details." + ) + + if filename is not None: + if len(self.detector_managers) == 1: + fname = filename + else: + if isinstance(filename, str): + fname = f"{i}_{filename}" + if isinstance(filename, Path): + fname = filename.parent / f"{i}_{filename.name}" + hdul.writeto(fname, overwrite=True) hduls.append(hdul) diff --git a/scopesim/optics/surface.py b/scopesim/optics/surface.py index fb7f70963..f60c9f0e4 100644 --- a/scopesim/optics/surface.py +++ b/scopesim/optics/surface.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + from pathlib import Path from dataclasses import dataclass from typing import Any @@ -8,14 +10,29 @@ from astropy.io import ascii as ioascii from astropy.table import Table -from synphot import SpectralElement -from synphot.models import Empirical1D +from synphot import SpectralElement, SourceSpectrum, Empirical1D, Observation +from synphot.units import PHOTLAM + +from ..utils import ( + get_meta_quantity, + quantify, + from_currsys, + convert_table_comments_to_dict, + find_file, + get_logger, +) +from ..source.source_templates import ( + vega_spectrum, + st_spectrum, + ab_spectrum, +) +from ..effects.ter_curves_utils import get_filter +from .surface_utils import ( + make_emission_from_emissivity, + make_emission_from_array, + extract_type_from_unit, +) -from ..effects import ter_curves_utils as ter_utils -from .surface_utils import (make_emission_from_emissivity, - make_emission_from_array, extract_type_from_unit) -from ..utils import (get_meta_quantity, quantify, from_currsys, - convert_table_comments_to_dict, find_file, get_logger) logger = get_logger(__name__) @@ -88,15 +105,15 @@ def wavelength(self): return self._get_array("wavelength") @property - def throughput(self): + def throughput(self) -> SpectralElement: return self._get_ter_property(self.meta.get("action", "transmission")) @property - def transmission(self): + def transmission(self) -> SpectralElement: return self._get_ter_property("transmission") @property - def emissivity(self): + def emissivity(self) -> SourceSpectrum: return self._get_ter_property("emissivity") @property @@ -144,7 +161,7 @@ def emission(self): filter_name = dic["filter_name"] if "filename_format" in dic: filter_name = dic["filename_format"].format(filter_name) - flux = ter_utils.scale_spectrum(flux, filter_name, amplitude) + flux = scale_spectrum(flux, filter_name, amplitude) return flux @@ -197,9 +214,12 @@ def _get_ter_property(self, ter_property, fmt="synphot"): fill_value = 0. if ter_property == "transmission" else np.nan if value_arr is not None and wave is not None and fmt == "synphot": - response_curve = SpectralElement(Empirical1D, points=wave, - lookup_table=value_arr, - fill_value=fill_value) + response_curve = SpectralElement( + Empirical1D, + points=wave, + lookup_table=value_arr, + fill_value=fill_value, + ) elif fmt == "array": response_curve = value_arr else: @@ -299,3 +319,76 @@ def __str__(self): msg = f"SpectralSurface [{cols}] \"{name}\"" return msg + + +# TODO: This feels WET with some SpeXtra / synphot functionality... +def scale_spectrum(spectrum, filter_name, amplitude): + """ + Scale a SourceSpectrum to a value in a filter. + + Parameters + ---------- + spectrum : synphot.SourceSpectrum + + filter_name : str + Name of a filter from + - a local instrument package (available in ``rc.__search_path__``) + - a generic filter name (see ``ter_curves_utils.FILTER_DEFAULTS``) + - a spanish-vo filter service reference (e.g. "Paranal/HAWKI.Ks") + + amplitude : astropy.Quantity, float + The value that the spectrum should have in the given filter. Acceptable + astropy quantities are: + - u.mag : Vega magnitudes + - u.ABmag : AB magnitudes + - u.STmag : HST magnitudes + - u.Jy : Jansky per filter bandpass + Additionally the "FLAM" and "FNU" units from ``synphot.units`` can + be used when passing the quantity for `amplitude`. + + Returns + ------- + spectrum : synphot.SourceSpectrum + Input spectrum scaled to the given amplitude in the given filter. + + Examples + -------- + :: + + >>> from scopesim.source.source_templates import vega_spectrum + >>> from scopesim.optics.surface import scale_spectrum + >>> + >>> spec = vega_spectrum() + >>> vega_185 = scale_spectrum(spec, "Ks", -1.85 * u.mag) + >>> ab_0 = scale_spectrum(spec, "Ks", 0 * u.ABmag) + >>> jy_3630 = scale_spectrum(spec, "Ks", 3630 * u.Jy) + + """ + if isinstance(amplitude, u.Quantity): + if amplitude.unit.physical_type == "spectral flux density": + if amplitude.unit != u.ABmag: + amplitude = amplitude.to(u.ABmag) + ref_spec = ab_spectrum(amplitude.value) + + elif amplitude.unit.physical_type == "spectral flux density wav": + if amplitude.unit != u.STmag: + amplitude = amplitude.to(u.STmag) + ref_spec = st_spectrum(amplitude.value) + + elif amplitude.unit == u.mag: + ref_spec = vega_spectrum(amplitude.value) + + else: + raise ValueError(f"Units of amplitude must be one of " + f"[u.mag, u.ABmag, u.STmag, u.Jy]: {amplitude}") + else: + ref_spec = vega_spectrum(amplitude) + + filt = get_filter(filter_name) + ref_flux = Observation(ref_spec, filt).effstim(flux_unit=PHOTLAM) + + real_flux = Observation(spectrum, filt).effstim(flux_unit=PHOTLAM) + scale_factor = ref_flux / real_flux + spectrum *= scale_factor.value + + return spectrum diff --git a/scopesim/source/source.py b/scopesim/source/source.py index b738cb92d..8d264b061 100644 --- a/scopesim/source/source.py +++ b/scopesim/source/source.py @@ -1,24 +1,5 @@ +# -*- coding: utf-8 -*- """ -# old functionality to implement: -# - provide x, y, lam, spectra, weight, ref -# - overridden + : number, Source, SourceSpectrum -# - overridden * : number, SpectralElement -# - write to and read from file -# - shift all fields -# - rotate around the centre -# - photons_in_range returns the photons per spectrum in a wavelength range -# - image_in_range returns an image of the source for a wavelength range -# -# old functionality which will be removed: -# - project_onto_chip -# - apply_optical_train -# -# old structure --> new structure: -# - all data held in 6 arrays -# --> new dicts for fields, spectrum -# field can be a Table or an ImageHDU -# spectrum is a SourceSpectrum -# # Use cases: # image + spectrum # images + spectra @@ -44,18 +25,14 @@ from synphot import SpectralElement, SourceSpectrum -from ..optics.image_plane import ImagePlane -from ..optics import image_plane_utils as imp_utils from .source_utils import ( validate_source_input, convert_to_list_of_spectra, - photons_in_range, ) from .source_fields import ( SourceField, TableSourceField, SpectrumSourceField, - HDUSourceField, ImageSourceField, CubeSourceField, ) @@ -63,7 +40,6 @@ find_file, is_fits, get_fits_type, - quantity_from_table, figure_factory, get_logger, ) @@ -242,27 +218,6 @@ def _from_imagehdu_and_spectra(self, image_hdu, **kwargs): assert not self.fields, "Constructor method must act on empty instance!" - if not image_hdu.header.get("BG_SRC"): - pass - # FIXME: This caused more problems than it solved! - # Find out if there's a good reason to mess with this, - # otherwise just remove... - - # image_hdu.header["CRVAL1"] = 0 - # image_hdu.header["CRVAL2"] = 0 - # image_hdu.header["CRPIX1"] = image_hdu.header["NAXIS1"] / 2 - # image_hdu.header["CRPIX2"] = image_hdu.header["NAXIS2"] / 2 - # #image_hdu.header["CRPIX1"] = (image_hdu.header["NAXIS1"] + 1) / 2 - # #image_hdu.header["CRPIX2"] = (image_hdu.header["NAXIS2"] + 1) / 2 - # # .. todo:: find where the actual problem is with negative CDELTs - # # .. todo:: --> abs(pixel_scale) in header_from_list_of_xy - # if image_hdu.header["CDELT1"] < 0: - # image_hdu.header["CDELT1"] *= -1 - # image_hdu.data = image_hdu.data[:, ::-1] - # if image_hdu.header["CDELT2"] < 0: - # image_hdu.header["CDELT2"] *= -1 - # image_hdu.data = image_hdu.data[::-1, :] - if isinstance(image_hdu, fits.PrimaryHDU): image_hdu = fits.ImageHDU(data=image_hdu.data, header=image_hdu.header) @@ -270,7 +225,6 @@ def _from_imagehdu_and_spectra(self, image_hdu, if not spectra: raise ValueError("No spectrum was provided.") - # image_hdu.header["SPEC_REF"] = len(self.spectra) assert len(spectra) == 1, f"_from_imagehdu_and_spectra needs single spectrum, ref was {image_hdu.header.get('SPEC_REF')}" image_hdu.header["SPEC_REF"] = 0 @@ -455,99 +409,6 @@ def bandpass(self, bandpass): self._bandpass = bandpass - # ..todo: rewrite this method - def image_in_range(self, wave_min, wave_max, pixel_scale=1*u.arcsec, - layers=None, area=None, spline_order=1, sub_pixel=False): - if layers is None: - layers = range(len(self.fields)) - fields = [self.fields[ii].field for ii in layers] - - hdr = imp_utils.get_canvas_header(fields, pixel_scale=pixel_scale) - im_plane = ImagePlane(hdr) - - for field in fields: - if isinstance(field, Table): - fluxes = self.photons_in_range(wave_min, wave_max, area, - field["ref"]) * field["weight"] - x = quantity_from_table("x", field, u.arcsec) - y = quantity_from_table("y", field, u.arcsec) - tbl = Table(names=["x", "y", "flux"], data=[x, y, fluxes]) - tbl.meta.update(field.meta) - hdu_or_table = tbl - - elif isinstance(field, fits.ImageHDU): - if field.header["SPEC_REF"] != "": - ref = [field.header["SPEC_REF"]] - flux = self.photons_in_range(wave_min, wave_max, area, ref) - # [ph s-1] or [ph s-1 m-2] come out of photons_in_range - - # ## ..todo: CATCH UNITS HERE. DEAL WITH THEM PROPERLY - # Currently assuming that all images are scaled appropriately - # and that they have SPEC_REF - - # else: - # field = scale_imagehdu(field, area=area, - # solid_angle=pixel_scale**2, - # waverange=(wave_min, wave_max)) - # # [ph s-1] or [ph s-1 m-2] come out of photons_in_range - # flux = 1 - - image = field.data * flux - hdu = fits.ImageHDU(header=field.header, data=image) - hdu_or_table = hdu - else: - continue - - im_plane.add(hdu_or_table, sub_pixel=sub_pixel, - spline_order=spline_order) - - return im_plane - - def photons_in_range(self, wave_min, wave_max, area=None, indices=None): - """ - - Parameters - ---------- - wave_min : float, u.Quantity - [um] - wave_max : float, u.Quantity - [um] - area : float, u.Quantity, optional - [m2] - indices : list of integers, optional - - Returns - ------- - counts : u.Quantity list - [ph / s / m2] if area is None - [ph / s] if area is passed - - """ - if indices is None: - indices = self.spectra.keys() - - spectra = [self.spectra[ii] for ii in indices] - counts = photons_in_range(spectra, wave_min, wave_max, area=area, - bandpass=self.bandpass) - return counts - - def fluxes(self, wave_min, wave_max, **kwargs): - return self.photons_in_range(wave_min, wave_max, **kwargs) - - def image(self, wave_min, wave_max, **kwargs): - return self.image_in_range(wave_min, wave_max, **kwargs) - - # @classmethod - # def load(cls, filename): - # """Load :class:'.Source' object from filename""" - # with open(filename, "rb") as fp1: - # src = pickle.load(fp1) - # return src - - # def dump(self, filename): - # """Save to filename as a pickle""" - # with open(filename, "wb") as fp1: - # pickle.dump(self, fp1) def shift(self, dx: float = 0, dy: float = 0, layers=None) -> None: """ diff --git a/scopesim/source/source_templates.py b/scopesim/source/source_templates.py index 0a88afbc6..7b4b04757 100644 --- a/scopesim/source/source_templates.py +++ b/scopesim/source/source_templates.py @@ -1,11 +1,11 @@ +# -*- coding: utf-8 -*- + from pathlib import Path import numpy as np - from astropy import units as u from astropy.table import Table from astropy.io import fits -from astropy.utils.decorators import deprecated_renamed_argument from synphot import SourceSpectrum, ConstFlux1D, Empirical1D from synphot.units import PHOTLAM @@ -30,7 +30,6 @@ def empty_sky(flux=0): return sky -@deprecated_renamed_argument("mag", "flux", "0.1.5") def star(x=0, y=0, flux=0): """ Source object for a single star in either vega, AB magnitudes, or Jansky diff --git a/scopesim/source/source_utils.py b/scopesim/source/source_utils.py index caf43a218..d969149be 100644 --- a/scopesim/source/source_utils.py +++ b/scopesim/source/source_utils.py @@ -4,11 +4,10 @@ from pathlib import Path import numpy as np -from scipy import integrate from astropy import wcs, units as u from astropy.io import fits from astropy.table import Table -from synphot import SourceSpectrum, Empirical1D, SpectralElement +from synphot import SourceSpectrum, Empirical1D from ..utils import find_file, get_logger, convert_table_comments_to_dict @@ -111,93 +110,6 @@ def _get_list(): return list(_get_list()) -# FIXME: typing: This should work with the more general (and true, since -# conversion is done anyway) Quantity["length"] (or "area" resp.), but -# doing so currently causes a NameError. Not sure what's going on. -def photons_in_range( - spectra: SourceSpectrum, - wave_min: u.Quantity[u.um] | float, - wave_max: u.Quantity[u.um] | float, - area: u.Quantity[u.m**2] | float | None = None, - bandpass: SpectralElement | None = None, -) -> u.Quantity[u.ph * u.s**-1 * u.m**-2] | u.Quantity[u.ph * u.s**-1]: - """ - Integrate photons from spectrum in given wavelength range. - - Parameters - ---------- - spectra : SourceSpectrum - Input spectrum. - wave_min : u.Quantity["length"] or float - Minimum wavelength. If float, assumes um. - wave_max : u.Quantity["length"] or float - Maximum wavelength. If float, assumes um. - area : u.Quantity["area"] or float, optional - Area to multiply with. If float, assumes m**2. The default is None. - bandpass : SpectralElement, optional - Filter to take into account, if any. The default is None. - - Returns - ------- - counts : astropy.units.Quantity - Either in ph/s/m**2 or just ph/s (if area was given). - - """ - # Note: Assuming um if given as float. - wave_min = (wave_min << u.um << u.Angstrom).value - wave_max = (wave_max << u.um << u.Angstrom).value - # Note: There appear to be some float shenanigans going on here, but - # rounding produces an error in the spectrum evaluation. Not sure what's - # going on, maybe it's fine as-is. - - counts = [] - for spec in spectra: - waveset = spec.waveset.value - mask = (waveset > wave_min) * (waveset < wave_max) - wave = np.array([wave_min, *waveset[mask], wave_max]) - flux = spec(wave).value - - # flux [ph s-1 cm-2] == flux [ph s-1 cm-2 AA-1] * wave [AA] - if isinstance(bandpass, SpectralElement): - bandpass.model.bounds_error = True - counts.append(integrate.trapezoid(bandpass(wave).value * flux, wave)) - else: - counts.append(integrate.trapezoid(flux, wave)) - - # counts = flux [ph s-1 cm-2] - counts = (counts * u.ph * u.s**-1 * u.cm**-2).to(u.ph * u.s**-1 * u.m**-2) - if area is not None: - counts *= (area << u.m**2) - - return counts - - -def scale_imagehdu(imagehdu, waverange, area=None): - """ - ..todo: implement this - - For the moment, all imagehdu must be accompanied by a spectrum in PHOTLAM - - Future functionality will include scaling here of: - ph s-1 - ph s-1 m-2 - ph s-1 m-2 - ph s-1 m-2 um-1 - ph s-1 m-2 um-1 arcsec-2 - J s-1 m-2 Hz-1 - J s-1 m-2 Hz-1 arcsec-2 - ABMAG - ABMAG arcsec-2 - VEGAMAG - VEGAMAG arcsec-2 - """ - if "SPEC_REF" not in imagehdu.header: - raise ValueError("For this version, an ImageHDU must be associated " - "with a spectrum. This will change in the future.") - - return imagehdu - - def make_img_wcs_header( pixel_scale: float, image_size: tuple[int, int], diff --git a/scopesim/tests/conftest.py b/scopesim/tests/conftest.py index 0be2a072f..705fd5aa3 100644 --- a/scopesim/tests/conftest.py +++ b/scopesim/tests/conftest.py @@ -6,6 +6,7 @@ import pytest from unittest.mock import patch +import matplotlib import scopesim as sim from astar_utils import UniqueList @@ -15,6 +16,8 @@ sim.rc.__currsys__["!SIM.file.error_on_missing_file"] = True +matplotlib.use("agg") # Don't render plots in normal testing + @pytest.fixture(scope="function", autouse=True) def configure_logging(): diff --git a/scopesim/tests/tests_effects/test_ADConversion.py b/scopesim/tests/tests_effects/test_ADConversion.py index daece3751..eba0ad615 100644 --- a/scopesim/tests/tests_effects/test_ADConversion.py +++ b/scopesim/tests/tests_effects/test_ADConversion.py @@ -32,9 +32,9 @@ def fixture_mock_detector(): def fixture_detector_with_data(mock_detector): """Instantiate a Detector with some data""" det = mock_detector - width = det._hdu.data.shape[1] - det._hdu.data[:] = 1.2 - det._hdu.data[:, width//2] = 1.99 + width = det.data.shape[1] + det.data[:] = 1.2 + det.data[:, width//2] = 1.99 return det @@ -73,11 +73,11 @@ def test_converts_pixels_to_integer_with_custom(self, detector_with_data): def test_applies_gain(self, mock_detector, gain): det = mock_detector data = np.random.randn(100, 100) * 100. + 1000. - det._hdu.data = 1. * data + det.data = 1. * data adconverter = ADConversion(gain=gain) adconverter.cmds = {"!DET.gain": gain, "!OBS.ndit": 1} adconverter.apply_to(det) - assert np.all((data/gain).astype(int) == det._hdu.data) + assert np.all((data/gain).astype(int) == det.data) @pytest.mark.usefixtures("patch_mock_path_micado") def test_applies_gain_list_to_detector_list(self): @@ -104,10 +104,10 @@ def test_caps_data_overflow(self, mock_detector, newtype, maxval, minval): data = medval * np.ones((100, 100), dtype=np.float64) data[50, 50] = 1.e12 data[60, 60] = -100000. - det._hdu.data = data + det.data = data adconverter = ADConversion(gain=gain, dtype=newtype) adconverter.cmds = {"!DET.gain": gain, "!OBS.ndit": 1} adconverter.apply_to(det) - assert(np.median(det._hdu.data) == newtype(medval/gain)) - assert(np.max(det._hdu.data) == maxval) - assert(np.min(det._hdu.data) == minval) + assert(np.median(det.data) == newtype(medval/gain)) + assert(np.max(det.data) == maxval) + assert(np.min(det.data) == minval) diff --git a/scopesim/tests/tests_effects/test_AtmosphericDispersion.py b/scopesim/tests/tests_effects/test_AtmosphericDispersion.py index 392d9e7a4..a578674a1 100644 --- a/scopesim/tests/tests_effects/test_AtmosphericDispersion.py +++ b/scopesim/tests/tests_effects/test_AtmosphericDispersion.py @@ -44,21 +44,17 @@ def test_throws_error_if_one_keyword_missing(self, atmo_yaml_dict): isinstance(AtmosphericDispersion(), AtmosphericDispersion) -class TestFovGrid: - def test_returns_list_of_3_arrays_with_correct_which(self, atmo_yaml_dict): +class TestGetTable: + def test_returns_wavelength_dx_dy_columns(self, atmo_yaml_dict): atmo_disp = AtmosphericDispersion(**atmo_yaml_dict["properties"]) - response = atmo_disp.fov_grid() - assert len(response) == 3 - assert all([isinstance(x, np.ndarray) for x in response]) - - def test_returns_non_with_wrong_which_keyword(self, atmo_yaml_dict): - atmo_disp = AtmosphericDispersion(**atmo_yaml_dict["properties"]) - response = atmo_disp.fov_grid(which="bogus") - assert response is None + tbl = atmo_disp.get_table() + assert tbl.colnames == ["wavelength", "dx", "dy"] + assert all(isinstance(tbl[col], np.ndarray) for col in tbl.colnames) def test_returns_similar_values_to_lasilla_website(self, atmo_yaml_dict): atmo_disp = AtmosphericDispersion(**atmo_yaml_dict["properties"]) - waves, dx, dy = atmo_disp.fov_grid() + tbl = atmo_disp.get_table() + waves, dx, dy = tbl["wavelength"], tbl["dx"], tbl["dy"] assert dy[0] - dy[-1] == approx(0.53, rel=1e-2) assert all(dx == 0) assert waves[0] == 0.5 and waves[-1] == 2.5 @@ -66,14 +62,16 @@ def test_returns_similar_values_to_lasilla_website(self, atmo_yaml_dict): def test_returns_same_results_when_turned_90_degrees(self, atmo_yaml_dict): atmo_yaml_dict["properties"]["pupil_angle"] = 90 atmo_disp = AtmosphericDispersion(**atmo_yaml_dict["properties"]) - waves, dx, dy = atmo_disp.fov_grid() + tbl = atmo_disp.get_table() + dx, dy = tbl["dx"], tbl["dy"] assert dx[0] - dx[-1] == approx(0.53, rel=1e-2) assert all([y == approx(0) for y in dy]) def test_returns_same_results_when_turned_30_degrees(self, atmo_yaml_dict): atmo_yaml_dict["properties"]["pupil_angle"] = 30 atmo_disp = AtmosphericDispersion(**atmo_yaml_dict["properties"]) - waves, dx, dy = atmo_disp.fov_grid() + tbl = atmo_disp.get_table() + dx, dy = tbl["dx"], tbl["dy"] dr = ((dx[0] - dx[-1])**2 + (dy[0] - dy[-1])**2)**0.5 assert dr == approx(0.53, rel=1e-2) diff --git a/scopesim/tests/tests_effects/test_AtmosphericDispersionCorrection.py b/scopesim/tests/tests_effects/test_AtmosphericDispersionCorrection.py index 4ff560b9f..55b83bb70 100644 --- a/scopesim/tests/tests_effects/test_AtmosphericDispersionCorrection.py +++ b/scopesim/tests/tests_effects/test_AtmosphericDispersionCorrection.py @@ -101,9 +101,11 @@ def test_shifts_between_adc_and_ad_are_opposite(self, atmo_params, waves, ad = AD(**atmo_params) adc = ADC(**atmo_params) - ad_shifts = ad.fov_grid() - ad_x_shift = np.interp(fov_wave_mid, ad_shifts[0], ad_shifts[1]) - ad_y_shift = np.interp(fov_wave_mid, ad_shifts[0], ad_shifts[2]) + ad_shifts = ad.get_table() + ad_x_shift = np.interp(fov_wave_mid, ad_shifts["wavelength"], + ad_shifts["dx"]) + ad_y_shift = np.interp(fov_wave_mid, ad_shifts["wavelength"], + ad_shifts["dy"]) adc.apply_to(fov) new_crpix_d = np.array([fov.header["CRPIX1D"], fov.header["CRPIX2D"]]) diff --git a/scopesim/tests/tests_effects/test_AutoExposure.py b/scopesim/tests/tests_effects/test_AutoExposure.py index 2876de7bf..179d1419d 100644 --- a/scopesim/tests/tests_effects/test_AutoExposure.py +++ b/scopesim/tests/tests_effects/test_AutoExposure.py @@ -1,7 +1,8 @@ """Tests for Effect AutoExposure.""" +import logging + import pytest -from unittest.mock import patch from scopesim import UserCommands from scopesim.optics.image_plane import ImagePlane @@ -9,9 +10,10 @@ from scopesim.tests.mocks.py_objects.imagehdu_objects import _image_hdu_square -# pylint: disable=no-self-use, missing-class-docstring +# pylint: disable=missing-class-docstring # pylint: disable=missing-function-docstring +LOGGER = logging.getLogger(__name__) def _patched_cmds(exptime=1, dit=None, ndit=None): return UserCommands(properties={"!OBS.exptime": exptime, @@ -32,6 +34,7 @@ def fixture_autoexposure(): """Instantiate an AutoExposure object""" return AutoExposure(fill_frac=0.75, full_well=1e5, + dark_current=0, mindit=0.011, exptime=None) @@ -40,6 +43,7 @@ class TestAutoExposure: def test_initialises_correctly(self): autoexposure = AutoExposure(fill_frac=0.75, full_well=1e5, + dark_current=0, mindit=0.011) assert isinstance(autoexposure, AutoExposure) @@ -102,7 +106,7 @@ def test_fill_frac_acts_correctly(self, imageplane): assert out_ndit_1 == fill_2 / fill_1 * out_ndit_2 @pytest.mark.xfail(reason=("This now fails because a user-specified DIT " - "and NDIT will now just go thourgh unmodified. " + "and NDIT will now just go through unmodified. " "Decide what to do with this test later...")) def test_exptime_specified_by_dit_ndit(self, autoexposure, imageplane): """ @@ -139,3 +143,46 @@ def test_exptime_at_least_mindit(self, imageplane): assert dit == mindit assert ndit == 1 + + def test_takes_dark_current_into_account(self, imageplane): + exptime = 3600 + autoexp_1 = AutoExposure(fill_frac=0.75, + full_well=1e6, + mindit=1.3, + dark_current=0) + autoexp_1.cmds["!OBS.exptime"] = exptime + autoexp_1.apply_to(imageplane) + dit1 = autoexp_1.cmds["!OBS.dit"] + + autoexp_2 = AutoExposure(fill_frac=0.75, + full_well=1e6, + mindit=1.3, + dark_current=1e4) + autoexp_2.cmds["!OBS.exptime"] = exptime + autoexp_2.apply_to(imageplane) + dit2 = autoexp_2.cmds["!OBS.dit"] + + assert dit2 < dit1 + + def test_no_dark_gives_zero_dark(self, imageplane, caplog): + exptime = 3600 + # no dark provided + autoexp_1 = AutoExposure(fill_frac=0.75, + full_well=1e6, + mindit=1.3) + autoexp_1.cmds["!OBS.exptime"] = exptime + caplog.set_level(logging.WARNING) + autoexp_1.apply_to(imageplane) + assert "No dark current found for" in caplog.text + dit1 = autoexp_1.cmds["!OBS.dit"] + + # zero dark explicitly provided + autoexp_2 = AutoExposure(fill_frac=0.75, + full_well=1e6, + mindit=1.3, + dark_current=0.) + autoexp_2.cmds["!OBS.exptime"] = exptime + autoexp_2.apply_to(imageplane) + dit2 = autoexp_2.cmds["!OBS.dit"] + + assert dit2 == dit1 diff --git a/scopesim/tests/tests_effects/test_BasicReadoutNoise.py b/scopesim/tests/tests_effects/test_BasicReadoutNoise.py index 89bc4029c..1b0ac3e30 100644 --- a/scopesim/tests/tests_effects/test_BasicReadoutNoise.py +++ b/scopesim/tests/tests_effects/test_BasicReadoutNoise.py @@ -2,7 +2,7 @@ from pytest import approx import numpy as np -from scopesim.effects.electronic.noise import PoorMansHxRGReadoutNoise, _make_ron_frame +from scopesim.effects.electronic.noise import PoorMansHxRGReadoutNoise from scopesim.tests.mocks.py_objects.detector_objects import _basic_detector @@ -23,7 +23,7 @@ def test_creates_noise_std_as_needed(self, noise_std): ron = PoorMansHxRGReadoutNoise(noise_std=noise_std, n_channels=64, ndit=1) dtcr = ron.apply_to(dtcr) - assert np.std(dtcr._hdu.data) == approx(noise_std, rel=0.05) + assert np.std(dtcr.data) == approx(noise_std, rel=0.05) @pytest.mark.parametrize("noise_std", [1, 9, 100]) @pytest.mark.parametrize("ndit", [1, 9, 100]) @@ -31,7 +31,7 @@ def test_noise_reduces_with_square_root_of_ndit(self, ndit, noise_std): dtcr = _basic_detector(width=256) ron = PoorMansHxRGReadoutNoise(noise_std=noise_std, n_channels=64, ndit=ndit) dtcr = ron.apply_to(dtcr) - noise_real = np.std(dtcr._hdu.data) + noise_real = np.std(dtcr.data) assert noise_real == approx(noise_std * ndit**0.5, rel=0.1) @@ -39,12 +39,27 @@ def test_noise_reduces_with_square_root_of_ndit(self, ndit, noise_std): class TestMakeRonFrame: @pytest.mark.parametrize("n", (1, 4, 9, 25)) def test_stdev_increase_with_square_of_n(self, n): - frames = np.array([_make_ron_frame((256, 256), 10, 2, 0.1, 0.2, 0.3, 0.4) + ron = PoorMansHxRGReadoutNoise(noise_std=10, n_channels=2, ndit=1) + ron.meta.update({ + "channel_fraction": 0.1, + "line_fraction": 0.2, + "pedestal_fraction": 0.3, + "read_fraction": 0.4, + }) + rng = np.random.default_rng(42) + frames = np.array([ron._make_ron_frame(rng, (256, 256)) for _ in range(n)]) assert np.std(np.sum(frames, axis=0)) == approx(10*n**0.5, rel=0.3) @pytest.mark.parametrize("shape", [(3, 7), (7, 3)]) def test_makes_frame_sizes_for_non_integer_n_channels(self, shape): - n_channels = 2 - frame = _make_ron_frame(shape, 5, n_channels, 0.25, 0.25, 0.25, 0.25) + ron = PoorMansHxRGReadoutNoise(noise_std=5, n_channels=2, ndit=1) + ron.meta.update({ + "channel_fraction": 0.25, + "line_fraction": 0.25, + "pedestal_fraction": 0.25, + "read_fraction": 0.25, + }) + rng = np.random.default_rng(42) + frame = ron._make_ron_frame(rng, shape) assert frame.shape > (0, 0) diff --git a/scopesim/tests/tests_effects/test_ConstPSF.py b/scopesim/tests/tests_effects/test_ConstPSF.py index d6270dd1c..27706b5d9 100644 --- a/scopesim/tests/tests_effects/test_ConstPSF.py +++ b/scopesim/tests/tests_effects/test_ConstPSF.py @@ -12,7 +12,6 @@ # - psf : returns self, as array returns a layer based on defaults # 4. should have the following methods: # - (?) apply_to(obj) -# - fov_grid(which="waveset") # - get_kernel(obj) """ diff --git a/scopesim/tests/tests_effects/test_InterPixelCapacitance.py b/scopesim/tests/tests_effects/test_InterPixelCapacitance.py index fb6adb5ee..bed34660b 100644 --- a/scopesim/tests/tests_effects/test_InterPixelCapacitance.py +++ b/scopesim/tests/tests_effects/test_InterPixelCapacitance.py @@ -87,7 +87,7 @@ def fixture_detector(): """Instantiate a random detector object""" hdu = fits.ImageHDU(data=np.random.randn(214, 333)) det = Detector(hdu.header) - det._hdu.data = hdu.data + det.data = hdu.data return det class TestApply: @@ -102,7 +102,7 @@ def test_convolve_unit_source(self): [0., 1., 0.], [0., 0., 0.]])) det = Detector(hdu.header) - det._hdu.data = hdu.data + det.data = hdu.data ipc = IPC(kernel=np.random.rand(3, 3)) newdet = ipc.apply_to(det) np.testing.assert_allclose(newdet.hdu.data, ipc.kernel) diff --git a/scopesim/tests/tests_effects/test_LinearityCurve.py b/scopesim/tests/tests_effects/test_LinearityCurve.py index a0612000d..e29a39830 100644 --- a/scopesim/tests/tests_effects/test_LinearityCurve.py +++ b/scopesim/tests/tests_effects/test_LinearityCurve.py @@ -46,11 +46,11 @@ class TestApplyTo: (60, 60), (1e5, 60)]) def test_only_applied_to_detector(self, in_flux, out_flux, basic_lincurve): dtcr = _basic_detector() - dtcr._hdu.data[0, 0] = in_flux + dtcr.data[0, 0] = in_flux new_dtcr = basic_lincurve.apply_to(dtcr) - assert new_dtcr._hdu.data[0, 0] == out_flux - assert np.min(new_dtcr._hdu.data) == 0 + assert new_dtcr.data[0, 0] == out_flux + assert np.min(new_dtcr.data) == 0 def test_bypasses_non_detector_objects(self, basic_lincurve): new_dict = basic_lincurve.apply_to({"gigawatts": 1.21}) diff --git a/scopesim/tests/tests_effects/test_SpectralTraceListUtils.py b/scopesim/tests/tests_effects/test_SpectralTraceListUtils.py index 7b8b7a97c..dfaa2d7f6 100644 --- a/scopesim/tests/tests_effects/test_SpectralTraceListUtils.py +++ b/scopesim/tests/tests_effects/test_SpectralTraceListUtils.py @@ -333,3 +333,13 @@ def test_interpolation_is_accurate(self): interps = make_image_interpolations(hdul, kx=1, ky=1) imginterp = interps[0](yy, xx, grid=False) assert np.allclose(imginterp, img) + + def test_works_with_non_square_image(self): + # The axes passed to RectBivariateSpline were transposed, which + # raised ValueError for any image with NAXIS1 != NAXIS2 + xx, yy = np.meshgrid(np.arange(80), np.arange(50)) + img = np.sin(xx/6) * np.cos(yy/7) # shape (50, 80) + hdul = fits.HDUList(fits.ImageHDU(data=img)) + interps = make_image_interpolations(hdul, kx=1, ky=1) + imginterp = interps[0](yy, xx, grid=False) + assert np.allclose(imginterp, img) diff --git a/scopesim/tests/tests_effects/test_Surface.py b/scopesim/tests/tests_effects/test_Surface.py index d1e4823b8..141ffe28e 100644 --- a/scopesim/tests/tests_effects/test_Surface.py +++ b/scopesim/tests/tests_effects/test_Surface.py @@ -1,4 +1,6 @@ +# -*- coding: utf-8 -*- import pytest +from pytest import approx import numpy as np from astropy import units as u @@ -6,7 +8,8 @@ from synphot import SpectralElement, SourceSpectrum from scopesim.effects import TERCurve -from scopesim.optics.surface import SpectralSurface +from scopesim.optics.surface import SpectralSurface, scale_spectrum +from scopesim.source.source_templates import vega_spectrum PLOTS = False @@ -51,3 +54,25 @@ def test_returns_surface_object_for_arrays(self): wave = surf.surface.wavelength plt.plot(wave, surf.surface.emission(wave)) plt.show() + + +@pytest.mark.webtest +@pytest.mark.usefixtures("no_file_error") +class TestScaleSpectrum: + def test_scales_vega_spectrum_to_vega_ab_or_jansky(self): + spec = vega_spectrum() + vega_185 = scale_spectrum(spec, "Ks", -1.85 * u.mag) + ab_0 = scale_spectrum(spec, "Ks", 0 * u.ABmag) + jy_3630 = scale_spectrum(spec, "Ks", 3630 * u.Jy) + + wave = np.linspace(1.8, 2.5, 1000) * u.um + assert vega_185(wave).value == approx(ab_0(wave).value, rel=1e-2) + assert vega_185(wave).value == approx(jy_3630(wave).value, rel=1e-2) + + if PLOTS: + plt.plot(wave, spec(wave), "b") + plt.plot(wave, vega_185(wave), "r:") + plt.plot(wave, ab_0(wave), "g--") + plt.plot(wave, jy_3630(wave), "y-.") + plt.semilogy() + plt.show() diff --git a/scopesim/tests/tests_effects/test_SurfaceList.py b/scopesim/tests/tests_effects/test_SurfaceList.py index 8128dd161..465cffdc7 100644 --- a/scopesim/tests/tests_effects/test_SurfaceList.py +++ b/scopesim/tests/tests_effects/test_SurfaceList.py @@ -1,4 +1,5 @@ -""" Tests for the the new SurfaceList object """ +# -*- coding: utf-8 -*- +"""Tests for the the SurfaceList object.""" import pytest from pytest import approx diff --git a/scopesim/tests/tests_effects/test_TERCurve.py b/scopesim/tests/tests_effects/test_TERCurve.py index 68160654c..331ebe03b 100644 --- a/scopesim/tests/tests_effects/test_TERCurve.py +++ b/scopesim/tests/tests_effects/test_TERCurve.py @@ -1,12 +1,19 @@ +# -*- coding: utf-8 -*- +"""Tests for TERCurve class""" + import pytest import numpy as np +import numpy.testing as npt from matplotlib import pyplot as plt from astropy import units as u +from astropy.wcs import WCS +from astropy.table import Table from scopesim.effects import ter_curves as tc from scopesim.optics.fov import FieldOfView from scopesim.tests.mocks.py_objects.header_objects import _fov_header +from scopesim.optics.fov_volume_list import FovVolumeList from scopesim.tests.mocks.py_objects import source_objects as so from scopesim.tests.mocks.py_objects import effects_objects as eo @@ -22,6 +29,8 @@ def _filter_wheel(mock_path_micado): "filename_format": fname, "current_filter": "Br-gamma"}) +# pylint: disable=missing-class-docstring, +# pylint: disable=missing-function-docstring class TestTERCurveApplyTo: def test_adds_bg_to_source_if_source_has_no_bg(self): @@ -49,6 +58,38 @@ def test_adds_bg_to_source_if_source_has_no_bg(self): plt.semilogy(wave, flux, "r") plt.show() + def test_applies_to_cube_source(self): + src = so._cube_source() + wcs = WCS(src.cube_fields[0].header).spectral + nlam = src.cube_fields[0].data.shape[0] + lam = wcs.all_pix2world(np.arange(nlam), 0)[0] * u.m + + eff = eo._filter_surface() + thru = eff.throughput(lam) + + # Save spectrum before application + orig = src.cube_fields[0].data[:, 25, 25] * 1. + + # Apply effect + src = eff.apply_to(src) + + # Spectrum after application + new = src.cube_fields[0].data[:, 25, 25] + + npt.assert_allclose(new, orig * thru) + + def test_raises_helpful_error_when_nothing_transmits(self): + """A fully opaque surface used to die on a bare IndexError inside the + volume setup. The error must name the effect and the waverange.""" + tbl = Table(data=[[0.5, 1.0, 2.0, 3.0], [0.0, 0.0, 0.0, 0.0]], + names=["wavelength", "transmission"]) + tbl["wavelength"].unit = "um" + eff = tc.TERCurve(table=tbl, wave_min=0.5, wave_max=3.0, + name="closed shutter") + + with pytest.raises(ValueError, match="Did you open the shutter"): + eff.apply_to(FovVolumeList()) + class TestSpectralQuantumEfficiency: def test_is_fov_phase_only(self): @@ -302,9 +343,6 @@ def test_loads_all_filters(self, fwheel): def test_current_filter_is_filter(self, fwheel): assert isinstance(fwheel.current_filter, tc.FilterCurve) - def test_current_filter_has_fov_grid_method(self, fwheel): - assert hasattr(fwheel.current_filter, "fov_grid") - def test_change_to_known_filter(self, fwheel): fwheel.change_filter('Ks') assert fwheel.current_filter.meta["name"] == "Ks" diff --git a/scopesim/tests/tests_effects/test_effect_utils.py b/scopesim/tests/tests_effects/test_effect_utils.py index 6a2779116..8e2611cea 100644 --- a/scopesim/tests/tests_effects/test_effect_utils.py +++ b/scopesim/tests/tests_effects/test_effect_utils.py @@ -1,10 +1,10 @@ +# -*- coding: utf-8 -*- + import pytest from scopesim import effects as efs -from scopesim.effects import effects_utils as eu, GaussianDiffractionPSF, \ - SurfaceList -from scopesim.tests.mocks.py_objects.effects_objects import _surf_list, \ - _surf_list_empty, _filter_surface +from scopesim.effects import effects_utils as eu, GaussianDiffractionPSF + from scopesim.tests.mocks.py_objects.yaml_objects import _atmo_yaml_dict @@ -13,21 +13,6 @@ def atmo_yaml_dict(): return _atmo_yaml_dict() -@pytest.fixture(scope="function") -def surf_list(): - return _surf_list() - - -@pytest.fixture(scope="function") -def surf_list_empty(): - return _surf_list_empty() - - -@pytest.fixture(scope="function") -def filter_surface(): - return _filter_surface() - - class TestMakeEffect: def test_it_creates_an_effects_object(self, atmo_yaml_dict): effdic = atmo_yaml_dict["effects"][0] diff --git a/scopesim/tests/tests_effects/test_exposure.py b/scopesim/tests/tests_effects/test_exposure.py index 18a1f091d..e3203c353 100644 --- a/scopesim/tests/tests_effects/test_exposure.py +++ b/scopesim/tests/tests_effects/test_exposure.py @@ -39,7 +39,7 @@ def fixture_exposureoutput(): @pytest.fixture(name="detector", scope="function") def fixture_detector(): det = Detector(_image_hdu_square().header) - det._hdu.data[:] = 1.e5 + det.data[:] = 1.e5 return det class TestExposureIntegration: @@ -47,9 +47,9 @@ def test_initialises_correctly(self, exposureintegration): assert isinstance(exposureintegration, ExposureIntegration) def test_integrates_correctly(self, exposureintegration, detector): - orig = 1. * detector._hdu.data + orig = 1. * detector.data assert isinstance(exposureintegration.apply_to(detector), Detector) - assert np.allclose(detector._hdu.data, 4 * orig) + assert np.allclose(detector.data, 4 * orig) def test_fails_without_dit(self): with pytest.raises(ValueError): @@ -91,17 +91,17 @@ def test_cannot_set_to_unknown_mode(self, exposureoutput): (2., 5), (3, 36)]) def test_applies_average(self, dit, ndit, detector): - det_mean = detector._hdu.data.mean() + det_mean = detector.data.mean() exposureoutput = ExposureOutput("average", dit=dit, ndit=ndit) result = exposureoutput.apply_to(detector) - assert np.isclose(result._hdu.data.mean(), det_mean / ndit) + assert np.isclose(result.data.mean(), det_mean / ndit) @pytest.mark.parametrize("dit, ndit", [(1., 1), (2., 5), (3, 36)]) def test_applies_sum(self, dit, ndit, detector): - det_mean = detector._hdu.data.mean() + det_mean = detector.data.mean() exposureoutput = ExposureOutput("sum", dit=dit, ndit=ndit) result = exposureoutput.apply_to(detector) - assert np.isclose(result._hdu.data.mean(), det_mean) + assert np.isclose(result.data.mean(), det_mean) diff --git a/scopesim/tests/tests_effects/test_mosaic_trace_list.py b/scopesim/tests/tests_effects/test_mosaic_trace_list.py index da3207e76..39433af43 100644 --- a/scopesim/tests/tests_effects/test_mosaic_trace_list.py +++ b/scopesim/tests/tests_effects/test_mosaic_trace_list.py @@ -104,7 +104,7 @@ def fixture_detector(): xsize, ysize = 50, 4096 hdr = header_from_list_of_xy([-xsize/2, xsize/2], [-ysize/2, ysize/2], 1, "D") dtcr = Detector(hdr) - dtcr._hdu.data[rows, :] = 1 + dtcr.data[rows, :] = 1 return dtcr @pytest.fixture(name="apply_collapse1d", scope="class") @@ -159,7 +159,7 @@ def test_collapse1d_detector_has_bintable(self, apply_collapse1d): assert isinstance(apply_collapse1d._hdu, fits.BinTableHDU) def test_collapse1d_gives_correct_result(self, apply_collapse1d): - assert np.all(apply_collapse1d._hdu.data['spectrum'] == 7) + assert np.all(apply_collapse1d.data['spectrum'] == 7) @pytest.mark.usefixtures("patch_mock_path_mosaic") @@ -181,7 +181,7 @@ def test_converttotable_detector_has_bintable(self, apply_converttotable): assert isinstance(apply_converttotable._hdu, fits.BinTableHDU) def test_converttotable_gives_correct_result(self, apply_converttotable): - for row in apply_converttotable._hdu.data: + for row in apply_converttotable.data: assert np.all(row['spectrum'] == 1) @pytest.mark.usefixtures("patch_mock_path_mosaic") diff --git a/scopesim/tests/tests_effects/test_prnu.py b/scopesim/tests/tests_effects/test_prnu.py index 7d9b4dfb1..fb671ca96 100644 --- a/scopesim/tests/tests_effects/test_prnu.py +++ b/scopesim/tests/tests_effects/test_prnu.py @@ -12,26 +12,44 @@ def make_detector(value=1000, size=10): hdr = header_from_list_of_xy([-size / 2, size / 2], [-size / 2, size / 2], 1, "D") dtcr = Detector(hdr) - dtcr._hdu.data[:] = value + dtcr.data[:] = value return dtcr -def test_output_std_matches_prnu_std(): - prnu_std = 0.05 - dtcr = make_detector(value=1000, size=100) - PixelResponseNonUniformity(prnu_std=prnu_std, prnu_seed=42).apply_to(dtcr) - rel_std = dtcr._hdu.data.std() / dtcr._hdu.data.mean() - assert abs(rel_std - prnu_std) < 0.01 - - -def test_gain_map_is_reused(): - dtcr1 = make_detector() - dtcr2 = make_detector() - prnu = PixelResponseNonUniformity(prnu_std=0.01, prnu_seed=42) - prnu.apply_to(dtcr1) - prnu.apply_to(dtcr2) - np.testing.assert_array_equal(dtcr1._hdu.data, dtcr2._hdu.data) - +class TestApplyTo: + def test_output_std_matches_prnu_std(self): + """Relative std of output should be close to prnu_std.""" + prnu_std = 0.05 + dtcr = make_detector(value=1000, size=100) + PixelResponseNonUniformity(prnu_std=prnu_std, prnu_seed=42).apply_to(dtcr) + rel_std = dtcr.data.std() / dtcr.data.mean() + assert abs(rel_std - prnu_std) < 0.01 + + def test_gain_map_is_reused(self): + """Same map should be applied on repeated calls (fixed pattern).""" + dtcr1 = make_detector() + dtcr2 = make_detector() + prnu = PixelResponseNonUniformity(prnu_std=0.01, prnu_seed=42) + prnu.apply_to(dtcr1) + prnu.apply_to(dtcr2) + np.testing.assert_array_equal(dtcr1.data, dtcr2.data) + + def test_dict_prnu_std(self): + """Dict mode: different amplitude per detector ID.""" + hdr = header_from_list_of_xy([-5, 5], [-5, 5], 1, "D") + dtcr = Detector(hdr) + dtcr.meta["id"] = 0 + dtcr.data[:] = 1000 + prnu = PixelResponseNonUniformity( + prnu_std={0: 0.005, 1: 0.020}, prnu_seed=42) + prnu.apply_to(dtcr) + assert dtcr.data.std() > 0 + + def test_multiplicative_zero_signal(self): + """Zero signal should remain zero — confirms multiplicative (not additive).""" + dtcr = make_detector(value=0) + PixelResponseNonUniformity(prnu_std=0.01, prnu_seed=42).apply_to(dtcr) + assert dtcr.data.sum() == 0 def test_dict_prnu_std(): hdr = header_from_list_of_xy([-5, 5], [-5, 5], 1, "D") diff --git a/scopesim/tests/tests_effects/test_shot_noise.py b/scopesim/tests/tests_effects/test_shot_noise.py index ccdca344f..c7cde5b55 100644 --- a/scopesim/tests/tests_effects/test_shot_noise.py +++ b/scopesim/tests/tests_effects/test_shot_noise.py @@ -21,10 +21,10 @@ def test_applies_dark_current_with_level_of_dit(self): value_too_high = 1e20 hdr = header_from_list_of_xy([-hw, hw], [-hw, hw], 1, "D") dtcr = Detector(hdr) - dtcr._hdu.data[0][0] = value_normal - dtcr._hdu.data[0][1] = -1 - dtcr._hdu.data[1][0] = value_too_high - dtcr._hdu.data[1][1] = np.nan + dtcr.data[0][0] = value_normal + dtcr.data[0][1] = -1 + dtcr.data[1][0] = value_too_high + dtcr.data[1][1] = np.nan sn = ShotNoise() # To prevent that data[0][0] does not change by chance. @@ -33,10 +33,10 @@ def test_applies_dark_current_with_level_of_dit(self): dtcr = sn.apply_to(dtcr) # Ensure that the values have changed. - assert dtcr._hdu.data[0][0] != value_normal - assert dtcr._hdu.data[1][0] != value_too_high + assert dtcr.data[0][0] != value_normal + assert dtcr.data[1][0] != value_too_high # Sensibility checks on the values. - assert 1 < dtcr._hdu.data[0][0] < value_normal * 2 - assert np.isnan(dtcr._hdu.data[0][1]) or dtcr._hdu.data[0][1] == 0 - assert np.isnan(dtcr._hdu.data[1][1]) + assert 1 < dtcr.data[0][0] < value_normal * 2 + assert np.isnan(dtcr.data[0][1]) or dtcr.data[0][1] == 0 + assert np.isnan(dtcr.data[1][1]) diff --git a/scopesim/tests/tests_effects/test_slitwheel.py b/scopesim/tests/tests_effects/test_slitwheel.py index b46f10f19..707e3dac8 100644 --- a/scopesim/tests/tests_effects/test_slitwheel.py +++ b/scopesim/tests/tests_effects/test_slitwheel.py @@ -27,9 +27,6 @@ def test_has_aperture_masks(self, swheel): def test_current_slit_is_aperture_mask(self, swheel): assert isinstance(swheel.current_slit, ApertureMask) - def test_current_slit_has_fov_grid_method(self, swheel): - assert hasattr(swheel.current_slit, "fov_grid") - def test_change_to_known_slit(self, swheel): swheel.change_slit('A') assert swheel.current_slit.meta['name'] == 'A' diff --git a/scopesim/tests/tests_effects/test_ter_curves_utils.py b/scopesim/tests/tests_effects/test_ter_curves_utils.py index 555b0c41f..13206ee15 100644 --- a/scopesim/tests/tests_effects/test_ter_curves_utils.py +++ b/scopesim/tests/tests_effects/test_ter_curves_utils.py @@ -1,9 +1,9 @@ +# -*- coding: utf-8 -*- + import pytest -import scopesim.source.source_templates from pytest import approx import numpy as np -from matplotlib import pyplot as plt from astropy import units as u @@ -28,53 +28,6 @@ def test_returns_specific_from_svo(self): assert np.max(ks(wave)) > 0.75 -@pytest.mark.webtest -@pytest.mark.usefixtures("no_file_error") -@pytest.mark.parametrize("filter_name, phot_system, flux", - [("V", "vega", 995), - ("z", "AB", 602), - ("Ks", "vega", 43)]) -def test_zero_mag_flux(filter_name, phot_system, flux): - """ - Photon fluxes are in line with: - http://www.astronomy.ohio-state.edu/~martini/usefuldata.html - """ - new_flux = ter_utils.zero_mag_flux(filter_name, phot_system) - assert flux == approx(new_flux.value, rel=0.08) - - -@pytest.mark.webtest -@pytest.mark.usefixtures("no_file_error") -def test_compare_br_gamma(): - filt = "Ks" - flux_vega = ter_utils.zero_mag_flux(filt, "vega").value - flux_ab = ter_utils.zero_mag_flux(filt, "AB").value - delta_mag = 2.5 * np.log10(flux_ab/flux_vega) - assert delta_mag == approx(1.85, rel=0.01) - - -@pytest.mark.webtest -@pytest.mark.usefixtures("no_file_error") -class TestScaleSpectrum: - def test_scales_vega_spectrum_to_vega_ab_or_jansky(self): - spec = scopesim.source.source_templates.vega_spectrum() - vega_185 = ter_utils.scale_spectrum(spec, "Ks", -1.85 * u.mag) - ab_0 = ter_utils.scale_spectrum(spec, "Ks", 0 * u.ABmag) - jy_3630 = ter_utils.scale_spectrum(spec, "Ks", 3630 * u.Jy) - - wave = np.linspace(1.8, 2.5, 1000) * u.um - assert vega_185(wave).value == approx(ab_0(wave).value, rel=1e-2) - assert vega_185(wave).value == approx(jy_3630(wave).value, rel=1e-2) - - if PLOTS: - plt.plot(wave, spec(wave), "b") - plt.plot(wave, vega_185(wave), "r:") - plt.plot(wave, ab_0(wave), "g--") - plt.plot(wave, jy_3630(wave), "y-.") - plt.semilogy() - plt.show() - - @pytest.mark.webtest @pytest.mark.usefixtures("no_file_error") class TestGetFilterEffectiveWavelength: diff --git a/scopesim/tests/tests_integrations/test_flux_is_conserved_through_full_system.py b/scopesim/tests/tests_integrations/test_flux_is_conserved_through_full_system.py index 65690fe0d..6c7542b5c 100644 --- a/scopesim/tests/tests_integrations/test_flux_is_conserved_through_full_system.py +++ b/scopesim/tests/tests_integrations/test_flux_is_conserved_through_full_system.py @@ -1,18 +1,21 @@ +# -*- coding: utf-8 -*- import pytest from pytest import approx from unittest.mock import patch import numpy as np +import matplotlib.pyplot as plt from scopesim.optics.optical_train import OpticalTrain from scopesim.commands import UserCommands from scopesim.tests.mocks.py_objects.source_objects import ( - _image_source, _single_table_source, _table_source_overlapping) - -import matplotlib.pyplot as plt -from matplotlib.colors import LogNorm + _image_source, + _single_table_source, + _table_source_overlapping, +) +from scopesim.tests.tests_source.source_helpers import src_photons_in_range PLOTS = False @@ -48,11 +51,11 @@ def test_flux_is_conserved_for_no_bg_emission(self, cmds, tbl_src): opt.observe(tbl_src) im = opt.image_planes[0].image bg_flux = np.pi / 4 * np.prod(im.shape) - src_flux = tbl_src.photons_in_range(1, 2, 1)[0].value + src_flux = src_photons_in_range(tbl_src, 1, 2, 1)[0].value if PLOTS: implane = opt.image_planes[0] - plt.imshow(implane.image.T, origin="lower", norm=LogNorm()) + plt.imshow(implane.image.T, origin="lower", norm="log") plt.colorbar() plt.show() @@ -70,11 +73,11 @@ def test_flux_is_conserved_for_yes_bg_emission(self, non_unity_cmds, tbl_src): opt = OpticalTrain(non_unity_cmds) opt.observe(tbl_src) im = opt.image_planes[0].image - src_flux = tbl_src.photons_in_range(1, 2, 1)[0].value + src_flux = src_photons_in_range(tbl_src, 1, 2, 1)[0].value if PLOTS: implane = opt.image_planes[0] - plt.imshow(implane.image.T, origin="lower", norm=LogNorm()) + plt.imshow(implane.image.T, origin="lower", norm="log") plt.colorbar() plt.show() @@ -133,7 +136,7 @@ def test_stacked_stars(self): if PLOTS: fig, axes = plt.subplots(nrows=1, ncols=1, figsize=(12, 12)) - axes.imshow(np.sqrt(im_h), norm=LogNorm(), cmap="inferno") + axes.imshow(np.sqrt(im_h), norm="log", cmap="inferno") axes.set_title('H-band MORFEO "PSF Generic" ') plt.show() diff --git a/scopesim/tests/tests_integrations/test_source_fov_imageplane_play_nicely.py b/scopesim/tests/tests_integrations/test_source_fov_imageplane_play_nicely.py index 82209d5ab..3816a62a1 100644 --- a/scopesim/tests/tests_integrations/test_source_fov_imageplane_play_nicely.py +++ b/scopesim/tests/tests_integrations/test_source_fov_imageplane_play_nicely.py @@ -8,7 +8,11 @@ from astropy import units as u +from astropy.wcs import WCS + +from scopesim.optics import image_plane_utils as imp_utils from scopesim.optics.fov import FieldOfView2D +from scopesim.optics.fov_manager import chunk_edges from scopesim.optics.image_plane import ImagePlane from scopesim.tests.mocks.py_objects import source_objects as src @@ -64,3 +68,205 @@ def test_can_extract_the_source_in_a_fov(self, fov_hdr, comb_src, plt.subplot(133) plt.imshow(imp.hdu.data, origin="lower", norm=LogNorm()) plt.show() + +def _checkerboard_source(naxis, pixel_scale, crpix=None): + """0/1 checkerboard image source, one board square per pixel.""" + from astropy.io import fits + from scopesim.source.source import Source + from scopesim.source import source_templates as st + + nx, ny = naxis + wcs = WCS(naxis=2) + wcs.wcs.ctype = ["LINEAR", "LINEAR"] + wcs.wcs.cunit = ["deg", "deg"] + wcs.wcs.cdelt = 2 * [pixel_scale / 3600] + wcs.wcs.crval = [0, 0] + wcs.wcs.crpix = crpix or [(nx + 1) / 2, (ny + 1) / 2] + + hdr = wcs.to_header() + hdr["BUNIT"] = "" + board = (np.indices((ny, nx)).sum(axis=0) % 2).astype(float) + hdu = fits.ImageHDU(header=hdr, data=board) + hdu.header["SPEC_REF"] = 0 + hdu.header["SPECMAG"] = 0 + hdu.header["SPECUNIT"] = "mag" + return Source(image_hdu=hdu, spectra=[st.vega_spectrum()]), board + + +def _chunked_fov_headers(x_mm, y_mm, pixel_size, pixel_scale, chunk): + """Reproduce FOVManager's chunking of a detector footprint.""" + plate_scale = pixel_scale / pixel_size + lims = [v / pixel_size * pixel_scale + for v in (x_mm[0], x_mm[1], y_mm[0], y_mm[1])] + + def edges(vmin, vmax): + # the real chunk-edge helper, not a copy of it + return [vmin, *chunk_edges(vmin, vmax, chunk * pixel_scale), vmax] + + xe, ye = edges(lims[0], lims[1]), edges(lims[2], lims[3]) + for x0, x1 in zip(xe[:-1], xe[1:]): + for y0, y1 in zip(ye[:-1], ye[1:]): + skyhdr = imp_utils.header_from_list_of_xy( + [x0 / 3600, x1 / 3600], [y0 / 3600, y1 / 3600], + pixel_scale=pixel_scale / 3600) + dethdr, _ = imp_utils.det_wcs_from_sky_wcs( + WCS(skyhdr), pixel_scale, plate_scale) + skyhdr.update(dethdr.to_header()) + yield skyhdr + + +class TestChunkedFOVsTileTheImagePlane: + """A checkerboard must survive being cut into chunked FOVs and reassembled. + + ``chunk_size`` is 8 here so the whole thing runs in well under a second, + but the geometry is MICADO's: the detector footprint is not a whole number + of pixels across (MICADO's y axis is 189.32 mm / 0.015 mm = 12621.33 px), + so the last chunk in each axis is a fraction of a chunk wide and lands on + a pixel lattice offset from its neighbours'. The projection used to snap + those chunks inconsistently, which put a duplicated or a dropped + row/column at the seam and flipped the checkerboard phase across it. + """ + + PIXEL_SIZE = 0.01 # mm / pix + PIXEL_SCALE = 0.2 # arcsec / pix + CHUNK = 8 + + @pytest.mark.parametrize("half_mm, exact_extent", [ + (0.12, 24.0), # whole pixels, whole chunks + (0.105, 21.0), # whole pixels, fractional last chunk + (0.0866, 17.32), # fractional pixels -- MICADO's y axis + (0.0855, 17.10), + ]) + def test_checkerboard_survives_chunking(self, half_mm, exact_extent): + x_mm = y_mm = (-half_mm, half_mm) + assert (x_mm[1] - x_mm[0]) / self.PIXEL_SIZE == approx(exact_extent) + + imphdr = imp_utils.header_from_list_of_xy( + list(x_mm), list(y_mm), pixel_scale=self.PIXEL_SIZE, wcs_suffix="D") + imp = ImagePlane(imphdr) + + source, board = _checkerboard_source( + (imphdr["NAXIS1"], imphdr["NAXIS2"]), self.PIXEL_SCALE, + crpix=[imphdr["CRPIX1D"], imphdr["CRPIX2D"]]) + + fov_hdrs = list(_chunked_fov_headers( + x_mm, y_mm, self.PIXEL_SIZE, self.PIXEL_SCALE, self.CHUNK)) + assert len(fov_hdrs) == 9 + + for hdr in fov_hdrs: + fov = FieldOfView2D(hdr, waverange=[0.5, 2.5] * u.um, + area=1 * u.m**2) + fov.extract_from(source) + fov.view() + imp.add(fov.hdu, wcs_suffix="D") + + # no gaps and no double-counting at the chunk seams + got = imp.data / imp.data.max() + np.testing.assert_allclose(got, board, atol=1e-6) + + +def _flat_source(naxis=64, pixel_scale=0.064): + """Uniformly illuminated image source, one everywhere, flat spectrum.""" + from astropy.io import fits + from synphot import Empirical1D, SourceSpectrum + from scopesim.source.source import Source + + wcs = WCS(naxis=2) + wcs.wcs.ctype = ["LINEAR", "LINEAR"] + wcs.wcs.cunit = ["arcsec", "arcsec"] + wcs.wcs.cdelt = 2 * [pixel_scale] + wcs.wcs.crval = [0, 0] + wcs.wcs.crpix = 2 * [(naxis + 1) / 2] + + hdu = fits.ImageHDU(data=np.ones((naxis, naxis)), header=wcs.to_header()) + spec = SourceSpectrum(Empirical1D, points=[0.5, 2.5] * u.um, + lookup_table=[1, 1] * u.Unit("ph s-1 m-2 um-1")) + return Source(image_hdu=hdu, spectra=[spec]) + + +def _slit_fov(x, y, pixel_scale, pixel_size): + """FOV covering a slit of x, y [arcsec], built like FOVManager does.""" + skyhdr = imp_utils.header_from_list_of_xy( + [v / 3600 for v in x], [v / 3600 for v in y], + pixel_scale=pixel_scale / 3600) + dethdr, _ = imp_utils.det_wcs_from_sky_wcs( + WCS(skyhdr), pixel_scale, pixel_scale / pixel_size) + skyhdr.update(dethdr.to_header()) + return FieldOfView2D(skyhdr, waverange=[1.0, 2.0] * u.um, area=1 * u.m**2) + + +class TestFlatSourceThroughASlitFOV: + """A flat lamp seen through a slit must fill the slit, uniformly. + + A slit FOV is typically *thinner than one source pixel*: MICADO's Short + slit is 0.016" tall while a flat-lamp source is sampled at 0.064"/pix. The + cutout is grown outwards to whole source pixels, so it covers more sky than + the slit does; if its WCS is then written on the *source's* pixel lattice + rather than on the FOV's, ``overlay_image`` crops the surplus away and the + flux through the slit comes out wrong -- for the MICADO case, exactly half + of the slit was left dark. This is what commit 9740ebd6 reverted in + ``FieldOfView.extract_area_from_imagehdu``. + + Note that these tests and ``TestChunkedFOVsTileTheImagePlane`` above are the + two sides of the same trade-off: the cutout WCS can describe either the + source's pixel lattice or the FOV's, and ``extract_area_from_imagehdu`` + currently has to pick one. Resolving it properly needs sub-pixel-accurate + cropping, i.e. weighting the partial edge pixels of the cutout. + """ + + SRC_PIXEL_SCALE = 0.064 # arcsec / pix, the flat lamp + FOV_PIXEL_SCALE = 0.004 # arcsec / pix, MICADO SPEC + PIXEL_SIZE = 0.015 # mm / pix + + # slit extents [arcsec], all thinner than or comparable to a source pixel + SLITS = [ + ((-1.5, 1.5), (-0.008, 0.008)), # MICADO Short slit, centred + ((-1.5, 1.5), (0.010, 0.026)), # ... offset within one source pixel + ((-1.5, 1.5), (-0.032, 0.032)), # exactly one source pixel tall + ((-1.5, 1.5), (-0.08, 0.08)), # 2.5 source pixels tall + ((-1.5, 1.5), (0.020, 0.060)), # straddling a source pixel edge + ((-2.0, 2.0), (-0.008, 0.008)), # longer slit + ] + + def _observe(self, x, y): + fov = _slit_fov(x, y, self.FOV_PIXEL_SCALE, self.PIXEL_SIZE) + fov.extract_from(_flat_source(pixel_scale=self.SRC_PIXEL_SCALE)) + fov.view() + return fov + + @pytest.mark.parametrize(("x", "y"), SLITS) + def test_flux_through_slit_matches_slit_area(self, x, y): + fov = self._observe(x, y) + + # each source pixel carries 1 ph/s here (flat 1 ph/s/m2/um over the + # 1 um wide FOV waverange, collecting area 1 m2), so the flux through + # the slit is just the slit area in source pixels + slit_area = (x[1] - x[0]) * (y[1] - y[0]) + expected = slit_area / self.SRC_PIXEL_SCALE**2 + + assert fov.hdu.data.sum() == approx(expected, rel=1e-6) + + # ... and it survives the projection onto the image plane + x_mm = [v * self.PIXEL_SIZE / self.FOV_PIXEL_SCALE for v in x] + y_mm = [v * self.PIXEL_SIZE / self.FOV_PIXEL_SCALE for v in y] + imp = ImagePlane(imp_utils.header_from_list_of_xy( + x_mm, y_mm, pixel_scale=self.PIXEL_SIZE, wcs_suffix="D")) + imp.add(fov.hdu, wcs_suffix="D") + + assert imp.data.sum() == approx(expected, rel=1e-6) + + @pytest.mark.parametrize(("x", "y"), SLITS) + def test_illumination_is_uniform_across_the_slit(self, x, y): + fov = self._observe(x, y) + data = fov.hdu.data + + if PLOTS: + plt.plot(data.mean(axis=1)) + plt.xlabel("pixel across slit") + plt.ylabel("flux [ph s-1 pix-1]") + plt.show() + + # a flat lamp illuminates the whole slit evenly: no dark rows at the + # edges, no step in the profile across the slit + assert data.min() > 0 + np.testing.assert_allclose(data, data.max(), rtol=1e-6) diff --git a/scopesim/tests/tests_optics/test_FOVManager.py b/scopesim/tests/tests_optics/test_FOVManager.py index a99477156..19ee5a6fb 100644 --- a/scopesim/tests/tests_optics/test_FOVManager.py +++ b/scopesim/tests/tests_optics/test_FOVManager.py @@ -4,7 +4,7 @@ from astropy import units as u from scopesim.effects import DetectorWindow -from scopesim.optics.fov_manager import FOVManager +from scopesim.optics.fov_manager import FOVManager, chunk_edges from scopesim.tests.mocks.py_objects import effects_objects as eo from scopesim.utils import from_currsys @@ -97,3 +97,59 @@ def apply_to(self, obj, **kwargs): assert fov.detector_header["NAXIS1"] == 100 assert fov.detector_header["NAXIS2"] == 100 assert fov.detector_header["CDELT1D"] == approx(0.01) + +class TestChunkEdges: + """Chunk edges must be interior and platform-independent. + + np.arange(vmin, vmax, step) returns one element too many when + (vmax - vmin) / step rounds just above an integer, and whether that + element lands on vmax or one ulp below it depends on how the platform + rounds vmin + i * step. One ulp below vmax is a legal split point, so + FovVolumeList.split would carve off a ~1e-15 px volume that + create_wcs_from_points then rounds up into a spurious one-pixel FOV. + """ + + @pytest.mark.parametrize("span_px, chunk, n_chunks", [ + (24.0, 8, 3), # exact multiple -- the fragile case + (24.0, 4, 6), # exact multiple + (16.0, 8, 2), + (21.0, 8, 3), # fractional last chunk + (17.32, 8, 3), # fractional pixels, fractional last chunk + (12621 + 1 / 3, 2048, 7), # MICADO's y axis + (5.0, 8, 1), # smaller than one chunk + ]) + def test_edges_are_interior_and_count_is_right(self, span_px, chunk, + n_chunks): + pixel_scale = 0.1 + vmin = -span_px / 2 * pixel_scale + vmax = vmin + span_px * pixel_scale + edges = chunk_edges(vmin, vmax, chunk * pixel_scale) + + assert len(edges) == n_chunks - 1 + assert (edges > vmin).all() + assert (edges < vmax).all() + + @pytest.mark.parametrize("span_px, chunk", [(24.0, 8), (24.0, 4), + (16.0, 8), (32.0, 16)]) + def test_exact_multiples_give_no_sliver_chunk(self, span_px, chunk): + # A span that is an exact multiple of the chunk must come out as + # equal chunks, with no degenerate remainder however the float + # arithmetic rounds. + pixel_scale = 0.1 + vmin = -span_px / 2 * pixel_scale + vmax = vmin + span_px * pixel_scale + edges = chunk_edges(vmin, vmax, chunk * pixel_scale) + + bounds = np.array([vmin, *edges, vmax]) + widths = np.diff(bounds) / pixel_scale + assert len(widths) == span_px / chunk + np.testing.assert_allclose(widths, chunk, atol=1e-9) + + def test_a_split_one_ulp_inside_the_upper_bound_is_not_emitted(self): + # np.arange does emit such a value on arm64 macOS; assert directly + # that chunk_edges never does, on any platform. + vmin, vmax, step = -1.2000000000000002, 1.2000000000000002, 0.8 + edges = chunk_edges(vmin, vmax, step) + assert (edges < np.nextafter(vmax, -np.inf)).all() + # ... which np.arange does not guarantee + assert len(np.arange(vmin, vmax, step)) == len(edges) + 2 diff --git a/scopesim/tests/tests_optics/test_FieldOfView.py b/scopesim/tests/tests_optics/test_FieldOfView.py index 49577b070..de961eb0f 100644 --- a/scopesim/tests/tests_optics/test_FieldOfView.py +++ b/scopesim/tests/tests_optics/test_FieldOfView.py @@ -1,3 +1,6 @@ +# -*- coding: utf-8 -*- +"""Test for FieldOfView classes""" + import pytest from pytest import approx import numpy as np @@ -5,7 +8,6 @@ from astropy.io import fits from astropy.table import Table from matplotlib import pyplot as plt -from matplotlib.colors import LogNorm from scopesim.tests.mocks.py_objects import header_objects as ho from scopesim.tests.mocks.py_objects import source_objects as so @@ -69,9 +71,6 @@ def test_initialises_with_header_and_waverange(self): class TestExtractFrom: - # @pytest.mark.xfail(reason=("is_field_in_fov drops table if anything is " - # "outside fov volume, therefore no point source " - # "is extracted...")) def test_extract_point_sources_from_table(self): src = so._table_source() src.fields[0].field["x"] = [-15, -5, 0, 0] * u.arcsec @@ -88,7 +87,7 @@ def test_extract_2d_image_from_hduimage(self): fov = _fov_190_210_um() fov.extract_from(src) - assert fov.fields[0].data.shape == (51, 25) + assert fov.fields[0].data.shape == (51, 26) assert len(fov.fields[0].spectra[0].waveset) == 11 assert fov.fields[0].spectra[0].waveset[0].value == approx(19000) @@ -107,11 +106,8 @@ def test_extract_3d_cube_that_is_offset_relative_to_fov(self): fov = _fov_197_202_um() fov.extract_from(src) - assert fov.fields[0].field.shape == (3, 51, 25) + assert fov.fields[0].field.shape == (3, 51, 26) - # @pytest.mark.xfail(reason=("is_field_in_fov drops table if anything is " - # "outside fov volume, therefore no point source " - # "is extracted...")) def test_extract_one_of_each_type_from_source_object(self): src_table = so._table_source() # 4 sources, put two outside of FOV src_table.fields[0].field["x"] = [-15, -5, 0, 0] * u.arcsec @@ -124,7 +120,7 @@ def test_extract_one_of_each_type_from_source_object(self): fov.extract_from(src) assert fov.fields[0].field.shape == (3, 51, 51) - assert fov.fields[1].field.shape == (51, 25) + assert fov.fields[1].field.shape == (51, 26) assert len(fov.fields[2].field) == 2 # assert len(fov.spectra) == 3 @@ -145,17 +141,6 @@ def test_ignores_fields_outside_fov_boundary(self): assert len(fov.fields) == 0 - @pytest.mark.skip(reason="SPEC_REF is obsolete, just rm this test?") - def test_all_spectra_are_referenced_correctly(self): - src = so._image_source() + so._cube_source() + so._table_source() - fov = _fov_190_210_um() - fov.extract_from(src) - # check the same spectrum object is referenced by both lists - assert fov.fields[0].header["SPEC_REF"] == \ - src.fields[0].header["SPEC_REF"] - assert all(fov.fields[2][i]["ref"] == src.fields[2][i]["ref"] - for i in range(4)) - def test_contains_all_fields_inside_fov(self): src = so._image_source() + so._cube_source() + so._table_source() the_fov = FieldOfView(ho._basic_fov_header(), (1, 2) * u.um, @@ -294,7 +279,7 @@ def test_makes_cube_from_all_types_of_source_object(self): if PLOTS: im = cube.data[0, :, :] - plt.imshow(im, origin="lower", norm=LogNorm(), vmin=1e-8) + plt.imshow(im, origin="lower", norm="log", vmin=1e-8) plt.show() @pytest.mark.parametrize("src", [so._table_source(), @@ -434,7 +419,7 @@ def test_makes_image_from_all_types_of_source_object(self): if PLOTS: im = image.data - plt.imshow(im, origin="lower", norm=LogNorm(), vmin=1e-8) + plt.imshow(im, origin="lower", norm="log", vmin=1e-8) plt.show() diff --git a/scopesim/tests/tests_optics/test_ImagePlane.py b/scopesim/tests/tests_optics/test_ImagePlane.py index 1a8a81190..c941e903e 100644 --- a/scopesim/tests/tests_optics/test_ImagePlane.py +++ b/scopesim/tests/tests_optics/test_ImagePlane.py @@ -1,9 +1,11 @@ +# -*- coding: utf-8 -*- """Tests for ImagePlane and some ImagePlaneUtils""" # pylint: disable=missing-class-docstring # pylint: disable=missing-function-docstring from copy import deepcopy +from itertools import product import pytest from pytest import approx @@ -11,22 +13,108 @@ import numpy as np from astropy.io import fits from astropy import units as u -from astropy.table import Table from astropy import wcs - import matplotlib.pyplot as plt -from matplotlib.colors import LogNorm import scopesim.optics.image_plane as opt_imp import scopesim.optics.image_plane_utils as imp_utils -from scopesim.tests.mocks.py_objects.imagehdu_objects import \ - _image_hdu_square, _image_hdu_rect, _image_hdu_three_wcs,\ - _image_hdu_3d_data +from scopesim.tests.mocks.py_objects.imagehdu_objects import ( + _image_hdu_square, + _image_hdu_rect, + _image_hdu_three_wcs, + _image_hdu_3d_data, +) PLOTS = False +# The three functions below were oringinally stored in image_plane_utils, +# but ultimately only used here, so I moved them here as helpers. +# TODO: Check if the same functionality can be achieved in another way. +# ScopeSim internally uses something else (imageplane headers from detectors), +# so tests should ideally mock that instead... +def get_canvas_header(hdu_or_table_list, pixel_scale=1*u.arcsec): + """ + Generate a fits.Header with a WCS that covers everything in the FOV. + + Parameters + ---------- + hdu_or_table_list : list + A list of Tables and/or ImageHDU py_objects + + pixel_scale : astropy.Quantity + [arcsec] The pixel scale of the projection. Default in 1 arcsec + + Returns + ------- + header : fits.Header + A Header containing a WCS and NAXISn values to build an ImageHDU + + """ + def _get_headers(hdus_or_tables): + for hdu_or_table in hdus_or_tables: + if isinstance(hdu_or_table, fits.ImageHDU): + yield hdu_or_table.header + elif isinstance(hdu_or_table, fits.Header): + yield hdu_or_table + else: + raise TypeError( + "hdu_or_table_list may only contain fits.ImageHDU " + f"or fits.Header, found {type(hdu_or_table)}.") + + headers = list(_get_headers(hdu_or_table_list)) + + hdr = _make_bounding_header_from_headers(*headers, pixel_scale=pixel_scale) + return hdr + + +def _make_bounding_header_from_headers(*headers, pixel_scale): + """ + Return a Header with WCS and NAXISn keywords bounding all input ImageHDUs. + + Parameters + ---------- + headers : list of fits.ImageHDU + pixel_scale : u.Quantity + [arcsec] + + Returns + ------- + hdr : fits.Header + + """ + def _get_unit_from_headers(*headers, wcs_suffix: str) -> str: + unit = headers[0][f"CUNIT1{wcs_suffix}"].lower() + assert all(header[f"CUNIT{i}{wcs_suffix}"].lower() == unit + for header, i in product(headers, range(1, 3))), \ + [(i, header[f"CUNIT{i}{wcs_suffix}"]) + for header, i in product(headers, range(1, 3))] + return unit + + wcs_suffix = "D" if pixel_scale.unit.physical_type == "length" else "" + unit = u.Unit(_get_unit_from_headers(*headers, wcs_suffix=wcs_suffix)) + + if unit.physical_type == "angle": + unit = "deg" + pixel_scale = pixel_scale.to_value(u.deg) + else: + pixel_scale = pixel_scale.to_value(unit) + + extents = [imp_utils.calc_footprint(header, wcs_suffix, unit) + for header in headers] + pnts = np.vstack(extents) + + hdr = imp_utils.header_from_list_of_xy( + pnts[:, 0], pnts[:, 1], pixel_scale, wcs_suffix) + hdr["NAXIS1"] += 1 + hdr["NAXIS2"] += 1 + hdr[f"CRVAL1{wcs_suffix}"] -= 0.5 * pixel_scale + hdr[f"CRVAL2{wcs_suffix}"] -= 0.5 * pixel_scale + + return hdr + + @pytest.fixture(scope="function", name="image_hdu_rect") def fixture_image_hdu_rect(): return _image_hdu_rect() @@ -57,361 +145,14 @@ def fixture_image_hdu_3d_data(): return _image_hdu_3d_data() -@pytest.fixture(scope="function", name="input_table") -def fixture_input_table(): - x = [-10, -10, 0, 10, 10] * u.arcsec - y = [-10, 10, 0, -10, 10] * u.arcsec - f = [1, 3, 1, 1, 5] - tbl = Table(names=["x", "y", "flux"], data=[x, y, f]) - - return tbl - - -@pytest.fixture(scope="function", name="input_table_mm") -def fixture_input_table_mm(): - x = [-10, -10, 0, 10, 10] * u.mm - y = [-10, 10, 0, -10, 10] * u.mm - f = [1, 3, 1, 1, 5] - tbl = Table(names=["x_mm", "y_mm", "flux"], data=[x, y, f]) - - return tbl - - -class TestCombineTableBoundaries: - def test_all_three_tables_are_inside_header_wcs(self, input_table): - tbl1 = deepcopy(input_table) - tbl2 = deepcopy(input_table) - tbl3 = deepcopy(input_table) - - tbl2["x"] -= 25 - tbl3["y"] -= 60 - - hdr = imp_utils._make_bounding_header_for_tables(tbl1, tbl2, tbl3) - as2deg = u.arcsec.to(u.deg) - for tbl in [tbl1, tbl2, tbl3]: - x, y = imp_utils.val2pix(hdr, tbl["x"]*as2deg, tbl["y"]*as2deg) - for xi, yi in zip(x, y): - assert 0 <= xi < hdr["NAXIS1"] - assert 0 <= yi < hdr["NAXIS2"] - - if PLOTS: - xy = imp_utils.calc_footprint(hdr) - x, y = xy[:, 0], xy[:, 1] - x, y = imp_utils.val2pix(hdr, x, y) - x0, y0 = imp_utils.val2pix(hdr, 0, 0) - - plt.plot(x, y, "b") - plt.plot(x0, y0, "ro") - for tbl in [tbl1, tbl2, tbl3]: - x, y = imp_utils.val2pix(hdr, tbl["x"] / 3600., - tbl["y"] / 3600.) - plt.plot(x, y, "k.") - - plt.show() - - def test_all_three_mm_tables_are_inside_header_wcs(self, input_table_mm): - tbl1 = deepcopy(input_table_mm) - tbl2 = deepcopy(input_table_mm) - tbl3 = deepcopy(input_table_mm) - - tbl2["x_mm"] += 50 - tbl3["y_mm"] += 25 - - hdr = imp_utils._make_bounding_header_for_tables(tbl1, tbl2, tbl3, - pixel_scale=100*u.um) - for tbl in [tbl1, tbl2, tbl3]: - x, y = imp_utils.val2pix(hdr, tbl["x_mm"], tbl["y_mm"], "D") - for xi, yi in zip(x, y): - assert 0 <= xi < hdr["NAXIS1"] - assert 0 <= yi < hdr["NAXIS2"] - - if PLOTS: - xy = imp_utils.calc_footprint(hdr, "D") - x, y = xy[:, 0], xy[:, 1] - x, y = imp_utils.val2pix(hdr, x, y, "D") - x0, y0 = imp_utils.val2pix(hdr, 0, 0, "D") - - plt.plot(x, y, "b") - plt.plot(x0, y0, "ro") - for tbl in [tbl1, tbl2, tbl3]: - x, y = imp_utils.val2pix(hdr, tbl["x_mm"], tbl["y_mm"], "D") - plt.plot(x, y, "k.") - - plt.show() - - -class TestCombineImageHDUBoundaries: - def test_all_two_imagehdus_are_inside_header_wcs(self, image_hdu_square, - image_hdu_rect): - - image_hdu_rect.header["CRVAL1"] -= 70 # * u.arcsec.to(u.deg) - image_hdu_square.header["CRVAL2"] += 70 # * u.arcsec.to(u.deg) - - hdr = imp_utils._make_bounding_header_from_headers( - image_hdu_square.header, image_hdu_rect.header) - for imhdr in [image_hdu_square.header, image_hdu_rect.header]: - xy = imp_utils.calc_footprint(imhdr) - x, y = xy[:, 0], xy[:, 1] - x *= u.arcsec.to(u.deg) - y *= u.arcsec.to(u.deg) - x, y = imp_utils.val2pix(hdr, x, y) - for xi, yi in zip(x, y): - assert 0 <= xi < hdr["NAXIS1"] - assert 0 <= yi < hdr["NAXIS2"] - - if PLOTS: - for imhdr in [image_hdu_square.header, image_hdu_rect.header, hdr]: - xy = imp_utils.calc_footprint(imhdr) - x, y = xy[:, 0], xy[:, 1] - xp, yp = imp_utils.val2pix(imhdr, x, y) - plt.plot(x, y, "r-") - - plt.plot(0, 0, "ro") - plt.gca().set_aspect(1) - plt.show() - - def test_all_two_mm_imagehdus_are_inside_header_wcs(self, - image_hdu_square_mm, - image_hdu_rect_mm): - - image_hdu_rect_mm.header["CRVAL1D"] -= 40 - image_hdu_square_mm.header["CRVAL2D"] += 80 - - hdr = imp_utils._make_bounding_header_from_headers( - image_hdu_square_mm.header, image_hdu_rect_mm.header, - pixel_scale=1*u.mm) - for imhdr in [image_hdu_square_mm.header, image_hdu_rect_mm.header]: - xy = imp_utils.calc_footprint(imhdr, "D") - x, y = xy[:, 0], xy[:, 1] - x, y = imp_utils.val2pix(hdr, x, y, "D") - for xi, yi in zip(x, y): - assert 0 <= xi < hdr["NAXIS1"] - assert 0 <= yi < hdr["NAXIS2"] - - if PLOTS: - for imhdr in [image_hdu_square_mm.header, - image_hdu_rect_mm.header, hdr]: - xy = imp_utils.calc_footprint(imhdr, "D") - x, y = xy[:, 0], xy[:, 1] - xp, yp = imp_utils.val2pix(imhdr, x, y, "D") - plt.plot(x, y, "r-") - - plt.plot(0, 0, "ro") - plt.gca().set_aspect(1) - plt.show() - - -class TestGetCanvasHeader: - def test_all_5_objects_are_inside_header_wcs(self, image_hdu_square, - image_hdu_rect, input_table): - - tbl1 = deepcopy(input_table) - tbl2 = deepcopy(input_table) - tbl3 = deepcopy(input_table) - - tbl2["x"] -= 150 - tbl3["y"] -= 100 - - image_hdu_rect.header["CRVAL1"] += 100 * u.arcsec.to(u.deg) - image_hdu_square.header["CRVAL1"] += 0 * u.arcsec.to(u.deg) - image_hdu_square.header["CRVAL2"] += 100 * u.arcsec.to(u.deg) - - hdr = imp_utils.get_canvas_header([image_hdu_square, tbl1, tbl2, tbl3, - image_hdu_rect], pixel_scale=1*u.arcsec) - - as2deg = u.arcsec.to(u.deg) - - for im in [image_hdu_square.header, image_hdu_rect.header]: - xy = imp_utils.calc_footprint(im) - x, y = xy[:, 0], xy[:, 1] - x, y = imp_utils.val2pix(hdr, x*as2deg, y*as2deg) - for xi, yi in zip(x, y): - assert 0 <= xi < hdr["NAXIS1"] - assert 0 <= yi < hdr["NAXIS2"] - - for tbl in [tbl1, tbl2, tbl3]: - x, y = imp_utils.val2pix(hdr, tbl["x"]*as2deg, tbl["y"]*as2deg) - for xi, yi in zip(x, y): - assert 0 <= xi < hdr["NAXIS1"] - assert 0 <= yi < hdr["NAXIS2"] - - if PLOTS: - - xy = imp_utils.calc_footprint(hdr) - x, y = xy[:, 0], xy[:, 1] - x, y = imp_utils.val2pix(hdr, x, y) - plt.plot(x, y, "b") - x0, y0 = imp_utils.val2pix(hdr, 0, 0) - plt.plot(x0, y0, "ro") - - for tbl in [tbl1, tbl2, tbl3]: - x, y = imp_utils.val2pix(hdr, tbl["x"]*as2deg, tbl["y"]*as2deg) - plt.plot(x, y, "k.") - - for im in [image_hdu_square.header, image_hdu_rect.header]: - xy = imp_utils.calc_footprint(im) - x, y = xy[:, 0], xy[:, 1] - x, y = imp_utils.val2pix(hdr, x, y) - plt.plot(x, y, "r-") - - plt.gca().set_aspect(1) - plt.show() - - def test_all_5_objects_are_inside_mm_header_wcs(self, image_hdu_square_mm, - image_hdu_rect_mm, - input_table_mm): - - tbl1 = deepcopy(input_table_mm) - tbl2 = deepcopy(input_table_mm) - tbl3 = deepcopy(input_table_mm) - - tbl2["x_mm"] -= 150 - tbl3["y_mm"] -= 100 - - image_hdu_rect_mm.header["CRVAL1D"] += 100 - image_hdu_square_mm.header["CRVAL1D"] += 0 - image_hdu_square_mm.header["CRVAL2D"] += 100 - - hdr = imp_utils.get_canvas_header([image_hdu_square_mm, tbl1, tbl2, - tbl3, image_hdu_rect_mm], - pixel_scale=1*u.mm) - - for im in [image_hdu_square_mm.header, image_hdu_rect_mm.header]: - xy = imp_utils.calc_footprint(im, "D") - x, y = xy[:, 0], xy[:, 1] - x, y = imp_utils.val2pix(hdr, x, y, "D") - for xi, yi in zip(x, y): - assert 0 <= xi < hdr["NAXIS1"] - assert 0 <= yi < hdr["NAXIS2"] - - for tbl in [tbl1, tbl2, tbl3]: - x, y = imp_utils.val2pix(hdr, tbl["x_mm"], tbl["y_mm"], "D") - for xi, yi in zip(x, y): - assert 0 <= xi < hdr["NAXIS1"] - assert 0 <= yi < hdr["NAXIS2"] - - if PLOTS: - - xy = imp_utils.calc_footprint(hdr, "D") - x, y = xy[:, 0], xy[:, 1] - x, y = imp_utils.val2pix(hdr, x, y, "D") - plt.plot(x, y, "b") - x0, y0 = imp_utils.val2pix(hdr, 0, 0, "D") - plt.plot(x0, y0, "ro") - - for tbl in [tbl1, tbl2, tbl3]: - x, y = imp_utils.val2pix(hdr, tbl["x_mm"], tbl["y_mm"], "D") - plt.plot(x, y, "k.") - - for im in [image_hdu_square_mm.header, image_hdu_rect_mm.header]: - xy = imp_utils.calc_footprint(im, "D") - x, y = xy[:, 0], xy[:, 1] - x, y = imp_utils.val2pix(hdr, x, y, "D") - plt.plot(x, y, "r-") - - plt.gca().set_aspect(1) - plt.show() - - -class TestAddTableToImageHDU: - def test_points_are_added_to_small_canvas(self, input_table): - tbl1 = deepcopy(input_table) - hdr = imp_utils.get_canvas_header([tbl1]) - - im = np.zeros((hdr["NAXIS2"], hdr["NAXIS1"])) - canvas_hdu = fits.ImageHDU(header=hdr, data=im) - canvas_hdu = imp_utils.add_table_to_imagehdu(tbl1, canvas_hdu) - - assert np.sum(canvas_hdu.data) == np.sum(tbl1["flux"]) - - if PLOTS: - # "top left is green, top right is yellow" - plt.imshow(canvas_hdu.data, origin="lower") - plt.show() - - def test_mm_points_are_added_to_small_canvas(self, input_table_mm): - tbl1 = deepcopy(input_table_mm) - hdr = imp_utils.get_canvas_header([tbl1], pixel_scale=2*u.mm) - - im = np.zeros((hdr["NAXIS2"], hdr["NAXIS1"])) - canvas_hdu = fits.ImageHDU(header=hdr, data=im) - canvas_hdu = imp_utils.add_table_to_imagehdu(tbl1, canvas_hdu, - wcs_suffix="D") - - assert np.sum(canvas_hdu.data) == np.sum(tbl1["flux"]) - - if PLOTS: - # "top left is green, top right is yellow" - plt.imshow(canvas_hdu.data, origin="lower") - plt.show() - - # add a paramaterised test here to check border cases - def test_points_are_added_to_massive_canvas(self, input_table): - tbl1 = deepcopy(input_table) - tbl2 = deepcopy(input_table) - tbl3 = deepcopy(input_table) - - tbl1["y"] += 50 - tbl2["x"] += 20 - tbl3["x"] -= 25 - tbl3["y"] -= 25 - - hdr = imp_utils.get_canvas_header([tbl1, tbl2, tbl3], - pixel_scale=1*u.arcsec) - - im = np.zeros((hdr["NAXIS2"], hdr["NAXIS1"])) - canvas_hdu = fits.ImageHDU(header=hdr, data=im) - - for tbl in [tbl1, tbl2, tbl3]: - canvas_hdu = imp_utils.add_table_to_imagehdu(tbl, canvas_hdu, True) - - total_flux = np.sum([tbl1["flux"], tbl2["flux"], tbl3["flux"]]) - assert np.sum(canvas_hdu.data) == total_flux - - if PLOTS: - x, y = imp_utils.val2pix(hdr, 0, 0) - plt.plot(x, y, "ro") - "top left is green, top right is yellow" - plt.imshow(canvas_hdu.data, origin="lower") - plt.show() - - def test_mm_points_are_added_to_massive_canvas(self, input_table_mm): - tbl1 = deepcopy(input_table_mm) - tbl2 = deepcopy(input_table_mm) - tbl3 = deepcopy(input_table_mm) - - tbl1["y_mm"] += 50 - tbl2["x_mm"] += 20 - tbl3["x_mm"] -= 25 - tbl3["y_mm"] -= 25 - - hdr = imp_utils.get_canvas_header([tbl1, tbl2, tbl3], - pixel_scale=1*u.mm) - im = np.zeros((hdr["NAXIS2"], hdr["NAXIS1"])) - canvas_hdu = fits.ImageHDU(header=hdr, data=im) - - for tbl in [tbl1, tbl2, tbl3]: - canvas_hdu = imp_utils.add_table_to_imagehdu(tbl, canvas_hdu, - True, wcs_suffix="D") - - total_flux = np.sum([tbl1["flux"], tbl1["flux"], tbl1["flux"]]) - assert np.sum(canvas_hdu.data) == total_flux - - if PLOTS: - x, y = imp_utils.val2pix(hdr, 0, 0, "D") - plt.plot(x, y, "ro") - # "top left is green, top right is yellow" - plt.imshow(canvas_hdu.data, origin="lower") - plt.show() - - class TestAddImageHDUToImageHDU: - def test_image_is_added_to_small_canvas(self, image_hdu_rect, - image_hdu_square): + def test_image_is_added_to_small_canvas( + self, image_hdu_rect, image_hdu_square, + ): im_hdu = image_hdu_rect im_hdu.header["CRVAL1"] -= 150*u.arcsec.to(u.deg) im_hdu.header["CRVAL2"] += 40*u.arcsec.to(u.deg) - hdr = imp_utils.get_canvas_header([im_hdu, image_hdu_square]) + hdr = get_canvas_header([im_hdu, image_hdu_square]) im = np.zeros((hdr["NAXIS2"], hdr["NAXIS1"])) canvas_hdu = fits.ImageHDU(header=hdr, data=im) @@ -436,18 +177,19 @@ def test_image_is_added_to_small_canvas(self, image_hdu_rect, plt.show() - def test_mm_image_is_added_to_small_canvas(self, image_hdu_rect_mm, - image_hdu_square_mm): + def test_mm_image_is_added_to_small_canvas( + self, image_hdu_rect_mm, image_hdu_square_mm, + ): im_hdu = image_hdu_rect_mm im_hdu.header["CRVAL1D"] -= 150 im_hdu.header["CRVAL2D"] += 40 - hdr = imp_utils.get_canvas_header([im_hdu, image_hdu_square_mm], - pixel_scale=1*u.mm) + hdr = get_canvas_header([im_hdu, image_hdu_square_mm], 1*u.mm) im = np.zeros((hdr["NAXIS2"], hdr["NAXIS1"])) canvas_hdu = fits.ImageHDU(header=hdr, data=im) - canvas_hdu = imp_utils.add_imagehdu_to_imagehdu(im_hdu, canvas_hdu, - wcs_suffix="D") + canvas_hdu = imp_utils.add_imagehdu_to_imagehdu( + im_hdu, canvas_hdu, wcs_suffix="D", + ) assert np.sum(canvas_hdu.data) == approx(np.sum(im_hdu.data)) @@ -466,32 +208,21 @@ def test_mm_image_is_added_to_small_canvas(self, image_hdu_rect_mm, plt.show() - def test_image_and_tables_on_large_canvas(self, input_table, image_hdu_rect, - image_hdu_square): - tbl1 = deepcopy(input_table) - tbl2 = deepcopy(input_table) - - tbl1["y"] -= 100 - tbl2["x"] += 100 - tbl2["y"] += 100 - + def test_images_on_large_canvas(self, image_hdu_rect, image_hdu_square): im_hdu = image_hdu_rect im_hdu.header["CRVAL1"] -= 150 # *u.arcsec.to(u.deg) im_hdu.header["CRVAL2"] += 20 # *u.arcsec.to(u.deg) - total_flux = (np.sum(tbl1["flux"]) + np.sum(tbl2["flux"]) + - im_hdu.data.sum() + image_hdu_square.data.sum()) + total_flux = im_hdu.data.sum() + image_hdu_square.data.sum() - hdr = imp_utils.get_canvas_header([im_hdu, image_hdu_square, - tbl1, tbl2], pixel_scale=3*u.arcsec) + hdr = get_canvas_header([im_hdu, image_hdu_square], 3*u.arcsec) im = np.zeros((hdr["NAXIS2"], hdr["NAXIS1"])) canvas_hdu = fits.ImageHDU(header=hdr, data=im) canvas_hdu = imp_utils.add_imagehdu_to_imagehdu(im_hdu, canvas_hdu) - canvas_hdu = imp_utils.add_imagehdu_to_imagehdu(image_hdu_square, - canvas_hdu) - canvas_hdu = imp_utils.add_table_to_imagehdu(tbl1, canvas_hdu) - canvas_hdu = imp_utils.add_table_to_imagehdu(tbl2, canvas_hdu) + canvas_hdu = imp_utils.add_imagehdu_to_imagehdu( + image_hdu_square, canvas_hdu, + ) assert np.sum(canvas_hdu.data) == approx(total_flux, rel=1e-2) @@ -511,39 +242,28 @@ def test_image_and_tables_on_large_canvas(self, input_table, image_hdu_rect, plt.show() - def test_mm_image_and_tables_on_large_canvas(self, input_table_mm, - image_hdu_rect_mm, - image_hdu_square_mm): + def test_mm_images_on_large_canvas( + self, image_hdu_rect_mm, image_hdu_square_mm, + ): image_hdu_rect = image_hdu_rect_mm image_hdu_square = image_hdu_square_mm - tbl1 = deepcopy(input_table_mm) - tbl2 = deepcopy(input_table_mm) - - tbl1["y_mm"] -= 100 - tbl2["x_mm"] += 100 - tbl2["y_mm"] += 100 im_hdu = image_hdu_rect im_hdu.header["CRVAL1D"] -= 150 im_hdu.header["CRVAL2D"] += 20 - hdr = imp_utils.get_canvas_header([im_hdu, image_hdu_square, - tbl1, tbl2], pixel_scale=3*u.mm) + hdr = get_canvas_header([im_hdu, image_hdu_square], 3*u.mm) im = np.zeros((hdr["NAXIS2"], hdr["NAXIS1"])) canvas_hdu = fits.ImageHDU(header=hdr, data=im) - canvas_hdu = imp_utils.add_table_to_imagehdu(tbl1, canvas_hdu, - wcs_suffix="D") - canvas_hdu = imp_utils.add_table_to_imagehdu(tbl2, canvas_hdu, - wcs_suffix="D") - canvas_hdu = imp_utils.add_imagehdu_to_imagehdu(im_hdu, canvas_hdu, - wcs_suffix="D") - canvas_hdu = imp_utils.add_imagehdu_to_imagehdu(image_hdu_square, - canvas_hdu, - wcs_suffix="D") - - total_flux = np.sum(tbl1["flux"]) + np.sum(tbl2["flux"]) + \ - np.sum(im_hdu.data) + np.sum(image_hdu_square.data) + canvas_hdu = imp_utils.add_imagehdu_to_imagehdu( + im_hdu, canvas_hdu, wcs_suffix="D", + ) + canvas_hdu = imp_utils.add_imagehdu_to_imagehdu( + image_hdu_square, canvas_hdu, wcs_suffix="D", + ) + + total_flux = np.sum(im_hdu.data) + np.sum(image_hdu_square.data) assert np.sum(canvas_hdu.data) == approx(total_flux, rel=5e-3) if PLOTS: @@ -564,11 +284,10 @@ def test_mm_image_and_tables_on_large_canvas(self, input_table_mm, class TestImagePlaneAdd: - def test_simple_add_imagehdu_conserves_flux(self, image_hdu_square, - image_hdu_rect): - fields = [image_hdu_rect, image_hdu_square] - hdr = imp_utils.get_canvas_header(pixel_scale=1 * u.arcsec, - hdu_or_table_list=fields) + def test_simple_add_imagehdu_conserves_flux( + self, image_hdu_square, image_hdu_rect, + ): + hdr = get_canvas_header([image_hdu_rect, image_hdu_square]) orig_sum = image_hdu_rect.data.sum() @@ -598,11 +317,10 @@ def test_add_many_tables_and_imagehdus(self, image_hdu_rect, image_hdu_square): im_hdu.header["CRVAL1"] -= 150*u.arcsec.to(u.deg) im_hdu.header["CRVAL2"] += 20*u.arcsec.to(u.deg) - fields = [im_hdu, image_hdu_square] - hdr = imp_utils.get_canvas_header(fields, pixel_scale=1 * u.arcsec) + hdr = get_canvas_header([im_hdu, image_hdu_square]) implane = opt_imp.ImagePlane(hdr) - implane.add(fields) + implane.add([im_hdu, image_hdu_square]) total_flux = np.sum(im_hdu.data) + np.sum(image_hdu_square.data) assert np.sum(implane.data) == approx(total_flux) @@ -618,11 +336,12 @@ def test_add_many_tables_and_imagehdus(self, image_hdu_rect, image_hdu_square): plt.plot(x0, y0, "ro") plt.gca().set_aspect(1) - plt.imshow(implane.data, origin="lower", norm=LogNorm()) + plt.imshow(implane.data, origin="lower", norm="log") plt.show() - def test_add_many_mm_tables_and_imagehdus( - self, image_hdu_rect_mm, image_hdu_square_mm): + def test_add_many_mm_imagehdus( + self, image_hdu_rect_mm, image_hdu_square_mm, + ): image_hdu_rect = image_hdu_rect_mm image_hdu_square = image_hdu_square_mm @@ -630,10 +349,9 @@ def test_add_many_mm_tables_and_imagehdus( im_hdu.header["CRVAL1D"] -= 150 # mm im_hdu.header["CRVAL2D"] += 20 - fields = [im_hdu, image_hdu_square] - hdr = imp_utils.get_canvas_header(fields, pixel_scale=1*u.mm) + hdr = get_canvas_header([im_hdu, image_hdu_square], 1*u.mm) implane = opt_imp.ImagePlane(hdr) - implane.add(fields, wcs_suffix="D") + implane.add([im_hdu, image_hdu_square], wcs_suffix="D") total_flux = np.sum(im_hdu.data) + np.sum(image_hdu_square.data) assert np.sum(implane.data) == approx(total_flux) @@ -649,7 +367,7 @@ def test_add_many_mm_tables_and_imagehdus( plt.plot(x0, y0, "ro") plt.gca().set_aspect(1) - plt.imshow(implane.data, origin="lower", norm=LogNorm()) + plt.imshow(implane.data, origin="lower", norm="log") plt.show() @@ -725,7 +443,7 @@ def test_rescale_works_on_3d_imageplane(self, image_hdu_3d_data): ############################################################################### -# ..todo: When you have time, reintegrate these tests, There are some good ones +# TODO: When you have time, reintegrate these tests, There are some good ones class TestGetSpatialExtentOfHeader: def test_returns_right_sky_coords_from_known_coords(self, image_hdu_square): @@ -742,141 +460,24 @@ def test_returns_right_sky_coords_from_known_coords(self, image_hdu_square): class TestMakeImagePlaneHeader: - def test_header_contains_future_naxis_pixel_sizes(self, image_hdu_square, - image_hdu_rect): - hdr = imp_utils.get_canvas_header([image_hdu_square, image_hdu_rect]) + def test_header_contains_future_naxis_pixel_sizes( + self, image_hdu_square, image_hdu_rect, + ): + hdr = get_canvas_header([image_hdu_square, image_hdu_rect]) assert hdr["NAXIS1"] == 100 + 1 assert hdr["NAXIS2"] == 200 + 1 @pytest.mark.parametrize("offset", -np.random.randint(200, 1001, 10)) - def test_header_contains_spread_out_regions(self, offset, image_hdu_square, - image_hdu_rect): + def test_header_contains_spread_out_regions( + self, offset, image_hdu_square, image_hdu_rect, + ): image_hdu_rect.header["CRVAL1"] += offset # *u.arcsec.to(u.deg) - hdr = imp_utils.get_canvas_header([image_hdu_square, image_hdu_rect]) + hdr = get_canvas_header([image_hdu_square, image_hdu_rect]) image_width = image_hdu_square.header["NAXIS1"] // 2 + \ image_hdu_rect.header["NAXIS1"] // 2 + abs(offset) + 1 assert hdr["NAXIS1"] == image_width - @pytest.mark.parametrize("offset", [0, 10, 25, 50]) - def test_header_has_correct_size_based_on_table_extremes(self, offset, - input_table): - tbl1 = input_table - tbl2 = deepcopy(input_table) - tbl3 = deepcopy(input_table) - tbl2["x"] += offset - tbl3["y"] += offset - hdr = imp_utils.get_canvas_header([tbl1, tbl2, tbl3], - pixel_scale=0.1*u.arcsec) - - assert hdr["NAXIS1"] == np.max(tbl1["x"] + tbl2["x"]) * 10 + 3 # 4 - assert hdr["NAXIS2"] == np.max(tbl1["y"] + tbl3["y"]) * 10 + 3 # 4 - - # @pytest.mark.parametrize("pix_scl", [5, 1, 0.5, 0.1]) - # def test_header_has_correct_size_with_tbl_and_image_input(self, input_table, - # image_hdu_square, - # pix_scl): - # input_table["x"] += 100 - # hdr = imp_utils.get_canvas_header([image_hdu_square, input_table], - # pixel_scale=pix_scl * u.arcsec) - # assert hdr["NAXIS1"] == approx(hdr["CRPIX1"] + - # np.max(input_table["x"]) / pix_scl + 1, - # abs=0.5) - - def test_header_does_not_contain_negative_naxis_keywords(self): - pass - -# -# class TestGetCornerSkyCoordsFromTable: -# def test_table_with_column_units_returns_right_values(self): -# x, y = [1] * u.arcmin, [1] * u.arcmin -# tbl = Table(names=["x", "y"], data=[x, y]) -# xsky, ysky = imp_utils.get_corner_sky_coords_from_table(tbl) -# -# assert xsky[0]*u.deg == x[0].to(u.deg) -# assert ysky[0]*u.deg == y[0].to(u.deg) -# -# def test_table_with_meta_units_returns_right_values(self): -# x, y = [1], [1] -# tbl = Table(names=["x", "y"], data=[x, y]) -# tbl.meta.update({"x_unit": u.arcmin, "y_unit": u.arcmin}) -# xsky, ysky = impl_utils.get_corner_sky_coords_from_table(tbl) -# -# assert xsky[0] == x[0]*u.arcmin.to(u.deg) -# assert ysky[0] == y[0]*u.arcmin.to(u.deg) -# -# def test_table_with_default_units_returns_right_values(self): -# x, y = [60], [60] # because default unit is arcsec -# tbl = Table(names=["x", "y"], data=[x, y]) -# xsky, ysky = impl_utils.get_corner_sky_coords_from_table(tbl) -# -# assert pytest.approx(xsky[0] == x[0] * u.arcmin.to(u.deg)) -# assert pytest.approx(ysky[0] == y[0] * u.arcmin.to(u.deg)) -# -# -# class TestGetCornerSkyCoords: -# def test_returns_coords_for_combination_of_table_and_header(self, -# image_hdu_square): -# x, y = [-100, 70] * u.arcsec, [0, 0] * u.arcsec -# tbl = Table(names=["x", "y"], data=[x, y]) -# xsky, ysky = impl_utils.get_corner_sky_coords([tbl, image_hdu_square]) -# -# assert np.all((xsky == x.to_value(u.deg))) -# - - -class TestAddTableToImageHDU2: - @pytest.mark.parametrize("xpix, ypix, value", - [(51, 51, 2), - (48, 51, 3), - (48, 48, 4), - (51, 48, 5)]) - def test_integer_pixel_fluxes_are_added_correctly(self, xpix, ypix, value, - image_hdu_square): - # Given the weird behaviour on pixel boundaries - x, y = [1.5, -1.5, -1.5, 1.5]*u.arcsec, [1.5, 1.5, -1.5, -1.5]*u.arcsec - flux = [1, 2, 3, 4] * u.Unit("ph s-1") - tbl = Table(names=["x", "y", "flux"], data=[x, y, flux]) - - hdu = imp_utils.add_table_to_imagehdu(tbl, image_hdu_square, - sub_pixel=False) - - assert hdu.data[ypix, xpix] == value - - @pytest.mark.parametrize("x, y, flux, xpix, ypix, value", - [([0], [0], [1], 50, 50, .25), - ([0.7], [0.7], [1], 50, 50, 0.64), - ([-0.7], [-0.7], [1], 48, 48, 0.04), - ([5.5], [-4.7], [1], 55, 45, 0.8), - ([5.5], [-4.7], [1], 55, 44, 0.2)]) - def test_sub_pixel_fluxes_are_added_correctly(self, x, y, flux, xpix, ypix, - value, image_hdu_square): - # Given the weird behaviour on pixel boundaries - tbl = Table(names=["x", "y", "flux"], - data=[x*u.arcsec, y*u.arcsec, flux*u.Unit("ph s-1")]) - hdu = imp_utils.add_table_to_imagehdu(tbl, image_hdu_square, - sub_pixel=True) - - if PLOTS: - plt.imshow(hdu.data[45:55, 45:55], origin="lower") - plt.colorbar() - plt.show() - - assert hdu.data[ypix, xpix] == approx(value + 1) - - @pytest.mark.parametrize("x, y, flux", - [([100, -100], [0, 0], [10, 10])]) - def test_source_outside_canvas_are_ignored(self, x, y, flux, - image_hdu_square): - # Given the weird behaviour on pixel boundaries - tbl = Table(names=["x", "y", "flux"], - data=[x * u.arcsec, y * u.arcsec, - flux * u.Unit("ph s-1")]) - hdu = imp_utils.add_table_to_imagehdu(tbl, image_hdu_square, - sub_pixel=True) - - assert np.sum(hdu.data) == np.prod(hdu.data.shape) - class TestAddImagehduToImageHDU: @pytest.mark.parametrize("angle", [0, 30, 45, 89]) @@ -915,11 +516,10 @@ def test_throws_error_when_initialised_with_nothing(self): with pytest.raises(TypeError): opt_imp.ImagePlane() - def test_initialises_with_header_with_hdu(self, image_hdu_square, - image_hdu_rect): - hdr = imp_utils.get_canvas_header(pixel_scale=0.1 * u.arcsec, - hdu_or_table_list=[image_hdu_rect, - image_hdu_square]) + def test_initialises_with_header_with_hdu( + self, image_hdu_square, image_hdu_rect, + ): + hdr = get_canvas_header([image_hdu_rect, image_hdu_square]) implane = opt_imp.ImagePlane(hdr) assert isinstance(implane, opt_imp.ImagePlane) assert isinstance(implane.hdu, fits.ImageHDU) diff --git a/scopesim/tests/tests_optics/test_ImagePlaneUtils.py b/scopesim/tests/tests_optics/test_ImagePlaneUtils.py index 8658dd423..c0116087a 100644 --- a/scopesim/tests/tests_optics/test_ImagePlaneUtils.py +++ b/scopesim/tests/tests_optics/test_ImagePlaneUtils.py @@ -7,33 +7,13 @@ import matplotlib.pyplot as plt from scopesim.optics import image_plane_utils as imp_utils +from scopesim.optics.fov_manager import chunk_edges from scopesim.tests.mocks.py_objects import imagehdu_objects as imo PLOTS = False -class TestSplitHeader: - def test_is_the_header_split_into_the_right_amount_of_chunks(self): - hdr = imp_utils.header_from_list_of_xy([-1.024, 1.024], - [-1.024, 1.024], 0.002) - hdrs = imp_utils.split_header(hdr, 128) - area_sum = np.sum([hdr["NAXIS1"] * hdr["NAXIS2"] for hdr in hdrs]) - assert len(hdrs) == 64 - assert area_sum == hdr["NAXIS1"] * hdr["NAXIS2"] - - @pytest.mark.parametrize("x, y, pix", [(0.19, 0.2, 0.01), - (2.19, 1.55, 0.01), - (1.29, 1.2, 0.02), - (2.55, 3.75, 1)]) - def test_final_header_is_smaller_for_odd_size(self, x, y, pix): - hdr = imp_utils.header_from_list_of_xy([-1., x], - [-1., y], pix) - hdrs = imp_utils.split_header(hdr, 100) - area_sum = np.sum([hdr["NAXIS1"] * hdr["NAXIS2"] for hdr in hdrs]) - assert area_sum == hdr["NAXIS1"] * hdr["NAXIS2"] - - class TestAddImageHDUtoImageHDU: def big_small_hdus(self, big_wh=(20, 10), big_offsets=(0, 0), small_wh=(6, 3), small_offsets=(0, 0), pixel_scale=0.1): @@ -229,6 +209,93 @@ def test_overlay_images_works_for_3d_cubes_at_border(self): plt.show() +class TestLatticeIsSharedBetweenChunks: + """Regions carved from one origin in whole-pixel steps share a lattice. + + This is what chunked FOVs are: the image plane, the full-size chunks and + the smaller last chunk are all measured from the same detector edge. When + the region is not a whole number of pixels across, CRVAL used to absorb + each region's own rounding remainder, putting them on lattices offset from + one another by fractions of a pixel. + """ + + @pytest.mark.parametrize("extent_px", [24.0, 21.0, 17.32, 17.1, 12621 + 1/3]) + @pytest.mark.parametrize("chunk", [4, 8]) + def test_chunks_land_on_the_parent_lattice(self, extent_px, chunk): + pixel_scale = 0.1 + lo = -extent_px / 2 * pixel_scale + hi = lo + extent_px * pixel_scale + + parent, parent_naxis = imp_utils.create_wcs_from_points( + np.array([[lo, lo], [hi, hi]]), pixel_scale) + + # the real chunk-edge helper, not a copy of it + edges = [lo, *chunk_edges(lo, hi, chunk * pixel_scale), hi] + + for x0, x1 in zip(edges[:-1], edges[1:]): + child, child_naxis = imp_utils.create_wcs_from_points( + np.array([[x0, x0], [x1, x1]]), pixel_scale) + # the child's pixel 0 must sit on a whole-pixel offset of the + # parent's grid, otherwise the projection has to snap it + world = child.wcs_pix2world([[0, 0]], 0) + pix = parent.wcs_world2pix(world, 0)[0] + np.testing.assert_allclose(pix, np.round(pix), atol=1e-6) + + @pytest.mark.parametrize("pnts, pixel_scale", [ + (np.array([[0, 0]]), 5), # single point + (np.array([[-1, -1], [-1, 1], [1, -1], [1, 1]]), 5), # sub-pixel + ]) + def test_regions_smaller_than_a_pixel_stay_centred(self, pnts, pixel_scale): + wcs, naxis = imp_utils.create_wcs_from_points(pnts, pixel_scale) + np.testing.assert_array_equal(naxis, [1, 1]) + np.testing.assert_array_equal(wcs.wcs.crval, + pnts.mean(axis=0)) + + +class TestOverlayImageSnapping: + """The origin, not the centre, must be snapped to the pixel grid. + + ``ceil(coords) - shape // 2`` snapped the centre instead, which is off by + up to a whole pixel whenever ``coords`` is not on the lattice implied by + ``small_im.shape``, in a direction that flips with the parity of that + shape. Adjacent FOVs of different sizes were therefore displaced opposite + ways, and their shared edge gained a duplicated or a dropped row/column. + """ + + @pytest.mark.parametrize("size", [1, 2, 3, 4, 7, 8]) + def test_places_image_on_its_own_lattice_without_shift(self, size): + big = np.zeros((32, 32)) + small = np.ones((size, size)) + # centre coordinate that the shape's own lattice implies + centre = 10 + (size - 1) / 2 + imp_utils.overlay_image(small, big, (centre, centre)) + rows = np.where(big.any(axis=1))[0] + assert rows[0] == 10 + assert rows[-1] == 10 + size - 1 + + @pytest.mark.parametrize("size", [1, 2, 3, 4, 7, 8]) + def test_snaps_sub_pixel_offset_the_same_way_for_any_shape(self, size): + # A 0.3 pix offset must never move the image by a whole pixel, whatever + # the parity of the shape. + big = np.zeros((32, 32)) + small = np.ones((size, size)) + centre = 10 + (size - 1) / 2 + 0.3 + imp_utils.overlay_image(small, big, (centre, centre)) + rows = np.where(big.any(axis=1))[0] + assert rows[0] == 10 + assert rows[-1] == 10 + size - 1 + + def test_adjacent_differently_sized_images_tile_exactly(self): + # Mimics chunked FOVs, where the last chunk is smaller than the rest. + big = np.zeros((1, 20)) + origin = 0 + for size in (8, 8, 4): + small = np.ones((1, size)) + imp_utils.overlay_image(small, big, (origin + (size - 1) / 2, 0)) + origin += size + assert (big == 1).all() + + class TestRescaleImageHDU: @pytest.mark.parametrize("pixel_scale", [0.3, 0.5, 1, 2, 3]) def test_rescales_a_2D_imagehdu(self, pixel_scale): diff --git a/scopesim/tests/tests_optics/test_OpticalTrain.py b/scopesim/tests/tests_optics/test_OpticalTrain.py index 581b5e29f..c1931499a 100644 --- a/scopesim/tests/tests_optics/test_OpticalTrain.py +++ b/scopesim/tests/tests_optics/test_OpticalTrain.py @@ -205,8 +205,8 @@ def test_maps_live_trace_to_zero_origin_detector_pixels(self): assert coordinates["trace_id"][0] == "linear" assert coordinates["readout_index"][0] == 0 - assert coordinates["detector_x"][0].to_value(u.pixel) == approx(49) - assert coordinates["detector_y"][0].to_value(u.pixel) == approx(39) + assert coordinates['detector_x'][0].to_value(u.pixel) == approx(49.5) + assert coordinates['detector_y'][0].to_value(u.pixel) == approx(39.5) def test_includes_renderer_endpoint_sampling(self): train, _ = spectral_geometry_train() @@ -214,12 +214,9 @@ def test_includes_renderer_endpoint_sampling(self): coordinates = train.trace_detector_coordinates( wavelengths={"linear": [1.25] * u.um}) - # The rendered subimage samples -0.65..0.65 mm at 13 inclusive - # positions. The trace's -0.25 mm point therefore lands here, rather - # than at the ideal detector-WCS coordinate x=47. - expected_x = 43 + (-0.25 + 0.65) / (0.65 + 0.65) * 12 - assert coordinates["detector_x"][0].to_value(u.pixel) == approx( - expected_x) + # The upstream renderer now uses native detector centers, excluding + # the next pixel beyond the subimage's last column. + assert coordinates['detector_x'][0].to_value(u.pixel) == approx(47.) def test_returns_all_trace_wavelengths_and_requested_slit_positions(self): train, _ = spectral_geometry_train() @@ -247,8 +244,8 @@ def test_uses_changed_live_trace_transform(self): wavelengths={"linear": [1.5] * u.um}) trace = trace_list.spectral_traces["linear"] - trace.meta["offset_y"] = 0.1 - trace.compute_interpolation_functions() + trace.xilam2y.posttransform = (lambda y: y + .1, {}) + trace.xy2lam.pretransform_y = (lambda y: y - .1, {}) after = train.trace_detector_coordinates( wavelengths={"linear": [1.5] * u.um}) diff --git a/scopesim/tests/tests_optics/test_fov_utls.py b/scopesim/tests/tests_optics/test_fov_utls.py index a6acb844b..dd0821dd1 100644 --- a/scopesim/tests/tests_optics/test_fov_utls.py +++ b/scopesim/tests/tests_optics/test_fov_utls.py @@ -6,6 +6,8 @@ from synphot import Empirical1D, SourceSpectrum from synphot.units import PHOTLAM from astropy import units as u +from astropy.io import fits +from astropy.wcs import WCS from scopesim.optics.fov import FieldOfView, extract_range_from_spectrum from scopesim.optics import image_plane_utils as imp_utils @@ -151,3 +153,72 @@ def test_logs_msg_for_waverang_overlap_mismatch( extract_range_from_spectrum(spec, waverange) assert msg in caplog.text + +class TestExtractAreaKeepsTheInputPixelGrid: + """The cutout is made of input pixels, so its WCS must say so. + + The cutout WCS used to be re-derived from the world corners *before* the + slice was grown outwards to whole input pixels. That gave a WCS for a + differently sized image (``NAXISn`` disagreed with ``data.shape``, silently + corrected by ``fits.ImageHDU``, which left ``CRPIXn`` describing the wrong + array) sitting on the FOV's pixel lattice rather than the input's. The + cutout was then mis-registered by up to a whole pixel when projected, and + by differing amounts for neighbouring FOVs, so their shared edge gained a + duplicated or a dropped row/column. + """ + + @staticmethod + def _image_hdu(naxis=32, pixel_scale=0.1, crval=(0.0, 0.0)): + """Image with a distinct value in every pixel, in deg.""" + skywcs = WCS(naxis=2) + skywcs.wcs.ctype = ["LINEAR", "LINEAR"] + skywcs.wcs.cunit = ["deg", "deg"] + skywcs.wcs.cdelt = 2 * [pixel_scale / 3600] + skywcs.wcs.crval = list(crval) + skywcs.wcs.crpix = 2 * [(naxis + 1) / 2] + hdr = skywcs.to_header() + hdr["BUNIT"] = "" + data = np.arange(naxis * naxis, dtype=float).reshape(naxis, naxis) + return fits.ImageHDU(header=hdr, data=data) + + @staticmethod + def _fov(x_min, x_max, y_min, y_max, pixel_scale=0.1): + skyhdr = imp_utils.header_from_list_of_xy( + [x_min / 3600, x_max / 3600], [y_min / 3600, y_max / 3600], + pixel_scale=pixel_scale / 3600) + dethdr, _ = imp_utils.det_wcs_from_sky_wcs( + WCS(skyhdr), pixel_scale, pixel_scale / 0.01) + skyhdr.update(dethdr.to_header()) + return FieldOfView(skyhdr, [1.0, 2.0]) + + # FOVs deliberately offset from the image's pixel edges by a fraction of a + # pixel, which is what chunking produces whenever the detector extent is + # not a whole number of pixels. + @pytest.mark.parametrize("offset", [0.0, 0.16, 0.34, 0.5, 0.68, 0.84]) + def test_naxis_matches_data_shape(self, offset): + hdu = self._image_hdu() + fov = self._fov(-0.8 + offset * 0.1, 0.0 + offset * 0.1, + -0.8 + offset * 0.1, 0.0 + offset * 0.1) + cut = fov.extract_area_from_imagehdu(hdu, fov.get_corners("deg")[0]) + assert (cut.header["NAXIS2"], cut.header["NAXIS1"]) == cut.data.shape + + @pytest.mark.parametrize("offset", [0.0, 0.16, 0.34, 0.5, 0.68, 0.84]) + def test_cutout_pixels_keep_their_world_coordinates(self, offset): + hdu = self._image_hdu() + fov = self._fov(-0.8 + offset * 0.1, 0.0 + offset * 0.1, + -0.8 + offset * 0.1, 0.0 + offset * 0.1) + cut = fov.extract_area_from_imagehdu(hdu, fov.get_corners("deg")[0]) + + # every cutout value must sit at the world coordinate its value had in + # the input image + in_wcs = WCS(hdu.header, naxis=2) + cut_wcs = WCS(cut.header, naxis=2) + ny, nx = cut.data.shape + yy, xx = np.indices((ny, nx)) + world = cut_wcs.wcs_pix2world( + np.column_stack([xx.ravel(), yy.ravel()]), 0) + back = in_wcs.wcs_world2pix(world, 0).round(6) + expected = hdu.data[back[:, 1].astype(int), back[:, 0].astype(int)] + np.testing.assert_array_equal(cut.data.ravel(), expected) + # ... and land on whole input pixels, not between them + np.testing.assert_allclose(back, back.round(), atol=1e-6) diff --git a/scopesim/tests/tests_reports/test_rst_utils.py b/scopesim/tests/tests_reports/test_rst_utils.py index aca00adef..fee66a7ef 100644 --- a/scopesim/tests/tests_reports/test_rst_utils.py +++ b/scopesim/tests/tests_reports/test_rst_utils.py @@ -10,6 +10,7 @@ from scopesim.tests.mocks.py_objects import report_objects as ro from scopesim.tests.mocks.py_objects import effects_objects as eo +pytestmark = pytest.mark.skip # whole file CLEAN_UP = True PLOTS = False diff --git a/scopesim/tests/tests_source/source_helpers.py b/scopesim/tests/tests_source/source_helpers.py new file mode 100644 index 000000000..1a92fb99a --- /dev/null +++ b/scopesim/tests/tests_source/source_helpers.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- +"""Aux function for source testing.""" + +import numpy as np +from scipy import integrate +from astropy import units as u +from synphot import SourceSpectrum, SpectralElement + + +# This used to be in source_utils, but was ultimately only used in a few tests, +# so moved here as a simple helper function. +def photons_in_range( + spectra: SourceSpectrum, + wave_min: u.Quantity[u.um] | float, + wave_max: u.Quantity[u.um] | float, + area: u.Quantity[u.m**2] | float | None = None, + bandpass: SpectralElement | None = None, +) -> u.Quantity[u.ph * u.s**-1 * u.m**-2] | u.Quantity[u.ph * u.s**-1]: + """ + Integrate photons from spectrum in given wavelength range. + + Parameters + ---------- + spectra : SourceSpectrum + Input spectrum. + wave_min : u.Quantity["length"] or float + Minimum wavelength. If float, assumes um. + wave_max : u.Quantity["length"] or float + Maximum wavelength. If float, assumes um. + area : u.Quantity["area"] or float, optional + Area to multiply with. If float, assumes m**2. The default is None. + bandpass : SpectralElement, optional + Filter to take into account, if any. The default is None. + + Returns + ------- + counts : astropy.units.Quantity + Either in ph/s/m**2 or just ph/s (if area was given). + + """ + # Note: Assuming um if given as float. + wave_min = (wave_min << u.um << u.Angstrom).value + wave_max = (wave_max << u.um << u.Angstrom).value + # Note: There appear to be some float shenanigans going on here, but + # rounding produces an error in the spectrum evaluation. Not sure what's + # going on, maybe it's fine as-is. + + counts = [] + for spec in spectra: + waveset = spec.waveset.value + mask = (waveset > wave_min) * (waveset < wave_max) + wave = np.array([wave_min, *waveset[mask], wave_max]) + flux = spec(wave).value + + # flux [ph s-1 cm-2] == flux [ph s-1 cm-2 AA-1] * wave [AA] + if isinstance(bandpass, SpectralElement): + bandpass.model.bounds_error = True + counts.append(integrate.trapezoid(bandpass(wave).value * flux, wave)) + else: + counts.append(integrate.trapezoid(flux, wave)) + + # counts = flux [ph s-1 cm-2] + counts = (counts * u.ph * u.s**-1 * u.cm**-2).to(u.ph * u.s**-1 * u.m**-2) + if area is not None: + counts *= (area << u.m**2) + + return counts + + +# This used to be a method of Source, but was ultimately only used in two tests, +# so moved here as a simple helper function. +def src_photons_in_range(src, wave_min, wave_max, area=None): + """ + + Parameters + ---------- + wave_min : float, u.Quantity + [um] + wave_max : float, u.Quantity + [um] + area : float, u.Quantity, optional + [m2] + + Returns + ------- + counts : u.Quantity list + [ph / s / m2] if area is None + [ph / s] if area is passed + + """ + indices = src.spectra.keys() + spectra = [src.spectra[ii] for ii in indices] + counts = photons_in_range(spectra, wave_min, wave_max, area=area, + bandpass=src.bandpass) + return counts diff --git a/scopesim/tests/tests_source/test_source_Source.py b/scopesim/tests/tests_source/test_source_Source.py index feea28e44..2f5b06fbe 100644 --- a/scopesim/tests/tests_source/test_source_Source.py +++ b/scopesim/tests/tests_source/test_source_Source.py @@ -1,8 +1,10 @@ -# actually for Source +# -*- coding: utf-8 -*- + import pytest from pytest import approx import numpy as np +import matplotlib.pyplot as plt from astropy.io import fits from astropy.io import ascii as ioascii @@ -10,7 +12,7 @@ from astropy import units as u from astropy import wcs -from synphot import SourceSpectrum, SpectralElement +from synphot import SourceSpectrum from synphot.models import Empirical1D from synphot.units import PHOTLAM @@ -18,14 +20,10 @@ from scopesim.source.source import Source from scopesim.source.source_fields import CubeSourceField, TableSourceField -from scopesim.optics.image_plane import ImagePlane from scopesim.utils import convert_table_comments_to_dict from scopesim.tests.mocks.py_objects import source_objects as so -import matplotlib.pyplot as plt -from matplotlib.colors import LogNorm - PLOTS = False @@ -261,83 +259,6 @@ def test_empty_source_is_the_additive_identity(self, image_source): assert len(new_source_2.fields) == len(new_source_2._meta_dicts) -class TestSourceImageInRange: - @pytest.mark.skip(reason=( - "This was the only place where the 'add table to ImagePlane' branch " - "was still used. Figure out what actually gets tested here and then " - "try to do it in another way." - )) - def test_returns_an_image_plane_object(self, table_source): - im = table_source.image_in_range(1*u.um, 2*u.um) - assert isinstance(im, ImagePlane) - - @pytest.mark.skip(reason=( - "This was the only place where the 'add table to ImagePlane' branch " - "was still used. Figure out what actually gets tested here and then " - "try to do it in another way." - )) - def test_flux_from_table_on_image_is_as_expected(self, table_source): - ph = table_source.photons_in_range(1*u.um, 2*u.um) - ref = table_source.fields[0].field["ref"] - weight = table_source.fields[0].field["weight"] - counts = np.sum([ph.value[r] * w for r, w in zip(ref, weight)]) - - im = table_source.image_in_range(1*u.um, 2*u.um) - assert np.sum(im.image) == approx(counts) - - @pytest.mark.parametrize("pix_scl", [0.1, 0.2, 0.4, 1.0]) - def test_flux_from_imagehdu_is_as_expected(self, image_source, pix_scl): - ph = image_source.photons_in_range(1*u.um, 2*u.um)[0].value - im_sum = ph * np.sum(image_source.fields[0].data) - im = image_source.image_in_range(1*u.um, 2*u.um, pix_scl*u.arcsec) - assert np.sum(im.image) == approx(im_sum) - - def test_combines_more_that_one_field_into_image(self, image_source, - table_source): - ph = table_source.photons_in_range(1 * u.um, 2 * u.um) - fld = table_source.fields[0] - tbl_sum = u.Quantity([ph[fld.field["ref"][ii]] * fld.field["weight"][ii] - for ii in range(len(fld.field))]) - tbl_sum = np.sum(tbl_sum.value) - - ph = image_source.photons_in_range(1 * u.um, 2 * u.um)[0] - im_sum = np.sum(image_source.fields[0].data) * ph.value - - comb_src = image_source - im_plane = comb_src.image_in_range(1*u.um, 2*u.um, 0.3*u.arcsec) - - assert np.sum(im_plane.image) == approx(im_sum) - - if PLOTS: - plt.imshow(im_plane.image.T, origin="lower", norm=LogNorm()) - plt.show() - - -class TestSourcePhotonsInRange: - def test_correct_photons_are_returned_for_table_source(self, table_source): - ph = table_source.photons_in_range(1, 2) - assert np.allclose(ph.value, [4., 2., 2.]) - - def test_correct_photons_are_returned_for_image_source(self, image_source): - ph = image_source.photons_in_range(1, 2) - assert np.allclose(ph.value, [2.]) - - def test_correct_photons_are_returned_for_no_spectra(self, image_source): - image_source.fields[0].spectra = {} - ph = image_source.photons_in_range(1, 2) - assert len(ph) == 0 - - @pytest.mark.parametrize("area, expected", [(None, 2), (1, 2), (10, 20)]) - def test_photons_increase_with_area(self, area, expected, image_source): - ph = image_source.photons_in_range(1, 2, area=area) - assert ph[0].value == approx(expected) - - def test_photons_returned_only_for_indices(self, table_source): - ph = table_source.photons_in_range(1, 2, indices=[0, 2]) - assert len(ph) == 2 - assert np.allclose(ph.value, [4, 2]) - - @pytest.mark.xfail class TestSourceShift: def test_that_it_does_what_it_should(self): @@ -350,65 +271,6 @@ def test_that_it_does_what_it_should(self): assert False -class TestPhotonsInRange: - @pytest.mark.parametrize("ii, n_ph", - [(0, 50), - (1, 8e12)]) - def test_returns_correct_number_of_photons_for_one_spectrum(self, ii, n_ph, - input_spectra): - spec = input_spectra[ii] - counts = source_utils.photons_in_range([spec], 1, 2) - assert np.isclose(counts.value, n_ph, rtol=2e-3) - - def test_returns_ones_for_unity_spectrum(self): - flux = np.ones(11) * u.Unit("ph s-1 m-2 um-1") - wave = np.linspace(1, 2, 11) * u.um - spec = SourceSpectrum(Empirical1D, points=wave, lookup_table=flux) - counts = source_utils.photons_in_range([spec], 1 * u.um, 2 * u.um) - assert counts.value == approx(1) - - @pytest.mark.parametrize("area, expected_units", - [(1*u.m**2, u.ph / u.s), - (None, u.ph / u.s / u.m**2)]) - def test_returns_correct_units_with_without_area_argument(self, area, - expected_units): - flux = np.ones(11) * u.Unit("ph s-1 m-2 um-1") - wave = np.linspace(1, 2, 11) * u.um - spec = SourceSpectrum(Empirical1D, points=wave, lookup_table=flux) - counts = source_utils.photons_in_range([spec], 1 * u.um, 2 * u.um, - area=area) - assert counts.unit == expected_units - - def test_returns_correct_half_flux_with_bandpass(self): - flux = np.ones(11) * u.Unit("ph s-1 m-2 um-1") - wave = np.linspace(0.5, 2.5, 11) * u.um - spec = SourceSpectrum(Empirical1D, points=wave, lookup_table=flux) - bandpass = SpectralElement(Empirical1D, - points=np.linspace(1, 2, 13)*u.um, - lookup_table=0.5 * np.ones(13)) - counts = source_utils.photons_in_range([spec], 1 * u.um, 2 * u.um, - bandpass=bandpass) - assert counts.value == approx(0.5) - - @pytest.mark.parametrize("flux, area, expected", - [(np.linspace(0, 1, 11), 1E4*u.cm**2, 0.25), - (np.linspace(0, 1, 11)**2, None, 0.13625), - (np.linspace(0, 1, 11)**0.5, 100, 34.931988)]) - def test_with_bandpass_and_area_returns_correct_value(self, flux, area, - expected): - flux *= u.Unit("ph s-1 m-2 um-1") - spec = SourceSpectrum(Empirical1D, - points=np.linspace(0.5, 2.5, 11) * u.um, - lookup_table=flux) - bandpass = SpectralElement(Empirical1D, - points=np.linspace(1, 2, 13)*u.um, - lookup_table=0.5 * np.ones(13)) - counts = source_utils.photons_in_range([spec], 1 * u.um, 2 * u.um, - bandpass=bandpass, - area=area) - assert counts.value == approx(expected) - - class TestSpectraListConverter: def test_works_for_arrays(self): spec = source_utils.convert_to_list_of_spectra( @@ -465,33 +327,3 @@ def test_throws_for_invalid_ref(): # Minimal Table tbl = Table(data=[[0, 1]], names=["ref"]) TableSourceField(tbl, {0: None}) - - -# -# class TestScaleImageHDU: -# def test_scaling_properly_for_si_photlam_in_header(self): -# hdu = fits.ImageHDU(data=np.ones((10,10))) -# hdu.header["CDELT1"] = 0.1 * u.arcsec.to(u.deg) -# hdu.header["CDELT2"] = 0.1 * u.arcsec.to(u.deg) -# hdu.header["BUNIT"] = "ph s-1 m-2 um-1" -# waverange = (1, 2)*u.um -# scaled_hdu = source2_utils.scale_imagehdu(hdu, waverange) -# assert scaled_hdu -# -# def test_scaling_properly_for_cgs_photlam_per_arcsec2_in_header(self): -# pass -# -# def test_scaling_properly_for_photlam_per_arcsec2_no_area_in_header(self): -# pass -# -# def test_scaling_properly_for_Jansky_in_header(self): -# pass -# -# def test_scaling_properly_for_Jansky_per_arcsec_in_header(self): -# pass -# -# def test_scaling_properly_for_photlam_and_bscale_in_header(self): -# pass -# -# def test_scaling_properly_for_photlam_and_bscale_bzero_in_header(self): -# pass diff --git a/scopesim/utils.py b/scopesim/utils.py index a5d0e42b8..36b2e5080 100644 --- a/scopesim/utils.py +++ b/scopesim/utils.py @@ -560,6 +560,8 @@ def unit_includes_per_physical_type(unit, physical_type): def pixel_area(header: fits.Header) -> u.Quantity[u.arcsec**2]: """Calculate area covered by one pixel in arcsec**2 from header. + The pixel area is always positive. + .. versionadded:: 0.11.1 """ @@ -575,7 +577,7 @@ def pixel_area(header: fits.Header) -> u.Quantity[u.arcsec**2]: area = (header["CDELT1"] * u.Unit(header["CUNIT1"]) * header["CDELT2"] * u.Unit(header["CUNIT2"])) - return area.to(u.arcsec**2) + return np.abs(area.to(u.arcsec**2)) def has_needed_keywords(header, suffix=""):