merge version for LANDHYPERNET data release (v2.3.0)

.gitignore

@@ -24,3 +24,6 @@ docs/sphinx/content/API
 /hypernets_processor/etc/working_directory.txt
 /hypernets_processor/etc/processor.config
 /data/calibration_files_ascii
+.eggs/
+.vscode/
+__pycache__/

.readthedocs.yml

@@ -9,7 +9,7 @@ version: 2
 build:
   os: ubuntu-22.04
   tools:
-    python: "3.9"
+    python: "3.12"
 
 # Build documentation in the docs/ directory with Sphinx
 sphinx:

.vscode/settings.json

@@ -0,0 +1,9 @@
+{
+    "python-envs.pythonProjects": [
+        {
+            "path": ".",
+            "envManager": "ms-python.python:conda",
+            "packageManager": "ms-python.python:conda"
+        }
+    ]
+}

docs/sphinx/conf.py

@@ -18,7 +18,8 @@
 
 import os
 import sys
-sys.path.insert(0, os.path.abspath('sphinx'))
+
+sys.path.insert(0, os.path.abspath("sphinx"))
 
 
 def run_apidoc(_):
@@ -37,15 +38,19 @@ def run_apidoc(_):
     try:
         # Sphinx 1.7+
         from sphinx.ext import apidoc
+
         apidoc.main(argv)
     except ImportError:
         # Sphinx 1.6 (and earlier)
         from sphinx import apidoc
+
         argv.insert(0, apidoc.__file__)
         apidoc.main(argv)
 
+
 def setup(app):
-    app.connect('builder-inited', run_apidoc)
+    app.connect("builder-inited", run_apidoc)
+
 
 # -- General configuration ------------------------------------------------
 
@@ -57,41 +62,41 @@ def setup(app):
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
-    'sphinx.ext.autodoc',
-    'sphinx.ext.mathjax',
+    "sphinx.ext.autodoc",
+    "sphinx.ext.mathjax",
 ]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # The suffix(es) of source filenames.
 # You can specify multiple suffix as a list of string:
 #
 # source_suffix = ['.rst', '.md']
-source_suffix = '.rst'
+source_suffix = ".rst"
 
 # The encoding of source files.
 #
 # source_encoding = 'utf-8-sig'
 
 # The master toctree document.
-master_doc = 'index'
-atbd = 'atbd'
-userguide = 'userguide'
+master_doc = "index"
+atbd = "atbd"
+userguide = "userguide"
 
 # General information about the project.
-project = u'hypernets_processor'
-copyright = u'2020, Hypernets Project Team'
-author = u'Hypernets Project Team'
+project = "hypernets_processor"
+copyright = "2020, Hypernets Project Team"
+author = "Hypernets Project Team"
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = u'2.0'
+version = "2.0"
 # The full version, including alpha/beta/rc tags.
-release = u'2.0'
+release = "2.0"
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -134,7 +139,7 @@ def setup(app):
 # show_authors = False
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = "sphinx"
 
 # A list of ignored prefixes for module index sorting.
 # modindex_common_prefix = []
@@ -151,7 +156,7 @@ def setup(app):
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'sphinx_rtd_theme'
+html_theme = "sphinx_rtd_theme"
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
@@ -185,7 +190,7 @@ def setup(app):
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+html_static_path = ["_static"]
 
 # html_context = {
 #     'css_files': [
@@ -271,36 +276,43 @@ def setup(app):
 # html_search_scorer = 'scorer.js'
 
 # Output file base name for HTML help builder.
-htmlhelp_basename = 'hypernets_processordocs'
+htmlhelp_basename = "hypernets_processordocs"
 
 # -- Options for LaTeX output ---------------------------------------------
 
 latex_elements = {
-     # The paper size ('letterpaper' or 'a4paper').
-     #
-     # 'papersize': 'letterpaper',
-
-     # The font size ('10pt', '11pt' or '12pt').
-     #
-     # 'pointsize': '10pt',
-
-     # Additional stuff for the LaTeX preamble.
-     #
-     # 'preamble': '',
-
-     # Latex figure (float) alignment
-     #
-     # 'figure_align': 'htbp',
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
 }
 
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-    (atbd, 'hypernets_processor_atbd.tex', u'Hypernets Processor Algorithm Theoretical Basis Document',
-     u'Hypernets Project Team', 'manual'),
-    (userguide, 'hypernets_processor_userguide.tex', u'Hypernets Processor User Guide',
-     u'Hypernets Project Team', 'manual'),
+    (
+        atbd,
+        "hypernets_processor_atbd.tex",
+        "Hypernets Processor Algorithm Theoretical Basis Document",
+        "Hypernets Project Team",
+        "manual",
+    ),
+    (
+        userguide,
+        "hypernets_processor_userguide.tex",
+        "Hypernets Processor User Guide",
+        "Hypernets Project Team",
+        "manual",
+    ),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
@@ -341,8 +353,13 @@ def setup(app):
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
 man_pages = [
-    (master_doc, 'hypernets_processor', u'hypernets_processor Documentation',
-     [author], 1)
+    (
+        master_doc,
+        "hypernets_processor",
+        "hypernets_processor Documentation",
+        [author],
+        1,
+    )
 ]
 
 # If true, show URL addresses after external links.
@@ -356,9 +373,15 @@ def setup(app):
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-    (master_doc, 'hypernets_processor', u'Hypernets Processor User Guide',
-     author, 'hypernets_processor', 'One line description of project.',
-     'Miscellaneous'),
+    (
+        master_doc,
+        "hypernets_processor",
+        "Hypernets Processor User Guide",
+        author,
+        "hypernets_processor",
+        "One line description of project.",
+        "Miscellaneous",
+    ),
 ]
 
 # Documents to append as an appendix to all manuals.

docs/sphinx/content/atbd/processing/post_processing.rst

@@ -5,13 +5,136 @@
 
 .. _post_processing:
 
+Post Processing - Site-specific Quality Checks
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Post Processing - Site-specific Quality checks
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Once the L2A datasets have been produced, a final set of site-specific quality checks and masks are
-applied. The site-specific quality checks and masks are determined after inspection of the first months of
-data in consultation of the site owners. The resulting masks are applied to the L2A
-dataset and stored as L2B. The same masks are also applied to the L1B datasets and stored as L1D. Only
-the L1D and L2B data will be distributed to satellite validation users.
+The LANDHYPERNET L2A data still contains erroneous sequences which are not 
+suitable for Cal/Val, and thus not suitable for public distribution.
+The standard L2A processing as described in the previous sections applies general quality checks to the
+data. However, no site-specific QC has been applied in the processing to L2A.
 
-These post-processing quality checks are still under development and will be detailed here in the future.
+To ensure the highest quality of the L2A data over the Cal/Val sites, additional
+quality checks are applied to the data. This is done in two stages:
+
+1. An offline analysis is performed for each deployment period at each site, to identify
+   periods with known issues (e.g. due to site heterogeneity, bad deployment conditions,
+   unsuitable viewing or solar angles, etc). Based on this offline analysis, a range of config
+   values are determined for each site and deployment period, which are then stored in the job.config file (see :ref:`config`).
+
+2. During processing to L2B, the site-specific quality checks are applied to the data,
+   using the config values determined in the offline analysis. Only data suitable for Cal/Val
+   are retained in the L2B data.
+
+Flags Set in the L2A Data
+-------------------------
+
+Data flagged with a critical issue in L2A and L1B data are removed. Specifically, the following flags are removed (see also :ref:`using_hypernets`)::
+
+   bad_flags=["pt_ref_invalid", "half_of_scans_masked", "not_enough_dark_scans", "not_enough_rad_scans",
+           "not_enough_irr_scans", "no_clear_sky_irradiance", "variable_irradiance",
+           "half_of_uncertainties_too_big", "discontinuity_VNIR_SWIR", "single_irradiance_used"]
+
+See :ref:`flags` for more information on these flags.
+
+Sequences Outside Any Valid Deployment Date Ranges
+--------------------------------------------------
+
+The datetime of the sequence being processed is simply checked against the valid deployment ranges 
+(see https://www.landhypernet.org.uk/site_descriptions for these date ranges for each site).
+
+Periods with Bad Deployment Conditions
+--------------------------------------
+
+Such date/time/datetime periods can be revealed as part of the offline analysis, or can be known by the site owner 
+due to expert knowledge about the site, knowledge about the deployment, etc.
+
+Periods with Site Heterogeneity
+-------------------------------
+
+Homogeneous periods are defined using NDVI at vegetated sites. We first define a reference period through manual analysis 
+of images from sites and conversations with site owners and calculate the standard deviation of the NDVI in this period. 
+The homogeneous periods are then defined as periods where a rolling standard deviation of NDVI is below 1.5 times the reference standard deviation.
+Any measurements outside these homogeneous periods are removed.
+
+.. figure:: NDVI_LOBE.png
+   :width: 600px
+   :align: center
+   :alt: NDVI timeseries for LOBE April 2025 – August 2025
+
+   (Top) NDVI timeseries for LOBE April 2025 – August 2025. Green dots show NDVI of a measurement; blue line shows rolling average of 100 measurements with rolling standard deviation as shaded region. (Bottom) Rolling standard deviation of NDVI timeseries; red, orange, and yellow lines show 1, 2, and 3 times the reference standard deviation, respectively, where the reference is defined as the standard deviation of a known homogeneous period (here July 2025).
+
+Unsuitable Viewing Zenith & Azimuth Angle
+-----------------------------------------
+
+Such angles with tolerances can be revealed as part of the offline analysis (e.g. see Figure below), or can be known by the site owner due to expert site knowledge, knowledge about the deployment, etc. 
+To mask out shadows, there is also the option to specify the relative azimuth angle (shadows typically at 0 degrees) and to select all zenith angles smaller than the solar zenith angle.
+
+.. figure:: NDVI_JSIT.png
+   :width: 600px
+   :align: center
+   :alt: NDVI Timeseries for JSIT Jan – Jul 2025 coloured by viewing zenith angle
+
+   NDVI Timeseries for JSIT Jan – Jul 2025 coloured by viewing zenith angle, lower NDVI can be seen close to nadir (0 and 5 degrees). Close to nadir the sensor is looking at bare soil next to the mast and this data is removed in L2B.
+
+Unsuitable Sun Zenith Angle
+--------------------------
+
+Any measurements with a solar zenith angle above 60 degrees are removed. This value is changed for some high-latitude sites where very low solar zeniths occur; these are defined in offline analysis.
+
+Poorly Performing Wavelength Ranges
+-----------------------------------
+
+Wavelength ranges with known issues (such as calibration failures) are masked.
+
+.. figure:: CLEAR_SKY_JSIT.png
+   :width: 600px
+   :align: center
+   :alt: Clear sky check plot for JSIT irradiance measurements
+
+   Clear sky check plot for JSIT irradiance measurements on 2024-05-17 at 9:00 UTC. There is an erroneous peak in the measured irradiance around 585 nm.
+
+A Note on Clear-Sky Modelling
+-----------------------------
+
+Site-specific clear sky models are generated using the LibRadtran Radiative Transfer software. The surface was set to be Lambertian with reflectance equal to the mean L2A spectral reflectance for the specific site at a viewing zenith angle of 20 degrees. Atmospheric properties were taken from ERA5 reanalysis data averaged over 2023. One set of models was run with no aerosols, and one set of models was run with the median aerosol optical depth taken from CAMS reanalysis data over the site in 2023. BOA irradiance calculations (including direct and diffuse contributions) were then performed using RT for a range of different solar zenith angles (0 to 80 in steps of 10). The altitude for each simulation was also set specifically from the Copernicus DEM, extracted at each site. During analysis, models are interpolated to the solar zenith angle of the measurements.
+
+Misalignment of the HYPSTAR Sensor
+----------------------------------
+
+Sequences with erroneous irradiances were spotted with the likely cause being a slight misalignment of the HYPSTAR sensor to the vertical during the irradiance measurements, causing a trend in the ratio between a clear sky model and the irradiance measurements throughout the day. The observed ratio between the clear sky model with median aerosols (described above) and the irradiance measurements is used to fit a modelled ratio. The modelled ratio has the following free parameters: misalignment viewing zenith angle (the tilt from the vertical); misalignment viewing azimuth angle (the azimuthal direction of the tilt with 0 as North); offset (to account for errors in the clear sky model and drift in the calibration). These free parameters are fit using a Markov Chain Monte Carlo (MCMC) process. The data is then normalised using a correction factor so that the average irradiance is the same as prior to the misalignment correction.
+
+.. figure:: MISALIGNMENT_MODEL.png
+   :width: 600px
+   :align: center
+   :alt: Illustration of the misalignment modelling results
+
+   Illustration of the misalignment modelling results. Data points show data from JAES May-Sep 2023 in bins of time-of-day. Errorbars show standard deviation within the bin. Top: observed ratio of clear sky model and irradiance measurements, which shows some unexpected variability during the day. 2nd panel: modelled ratio, allowing for viewing zenith and azimuth angle to be pointed not exactly at nadir (also includes an offset to account for mismatches between model and observations). 3rd panel: residual between observations and models. 4th panel: normalised corrected ratios (by using the modelled misalignment, but not applying offset). Bottom: solar zenith angle in each bin.
+
+More Stringent Clear Sky Check
+------------------------------
+
+The improved clear sky consists of discarding any sequences where the measured irradiances (after correction for misalignment) are either more than 5% higher than the model without aerosols at 550 nm, or more than 10% smaller than the clear sky model with median aerosols. Note that various thresholds and aerosol loads were experimented with, and this combination was found to hold the best trade-off between masking erroneous sequences and wrongfully masking valid sequences.
+
+.. figure:: CLOUDY_SCENE_LOBE.png
+   :width: 600px
+   :align: center
+   :alt: Example of cloudy scene from LOBE
+
+   Example of cloudy scene from LOBE which passed L2A QC. More stringent L2B site-specific cloud checks correctly identify this as a cloudy scene.
+
+Unsuitable Reflectances for Specific Site
+-----------------------------------------
+
+Based on offline analysis, realistic upper and lower bounds on the reflectance values are provided in bins along multiple dimensions (typically solar zenith angle, viewing zenith angle, relative azimuth angle, and optionally temporal bins), to allow for angular variability due to BRDF effects (and optionally seasonal effects). The bounds themselves come from binning the data within these bins, and then iteratively removing outliers (as compared to other values in the bin, as well as fits to the overall angular and temporal variability).
+
+.. figure:: REFLECTANCE_BOUNDS_GHNA.png
+   :width: 600px
+   :align: center
+   :alt: Illustration of the reflectance bounds for GHNA May 2022 – Oct 2023
+
+   Illustration of the reflectance bounds for GHNA May 2022 – Oct 2023. The bounds are shown in blue, data which pass in green, and data which are removed in red.
+
+Other Unusual Conditions
+------------------------
+
+Any remaining issues, due to unusual conditions that occur occasionally, can be omitted by manually specifying the sequence ID.