Feature/earthdata downloader

bluemath_tk/downloaders/_transfer.py

@@ -8,6 +8,7 @@
 
 import logging
 import os
+import threading
 import time
 from typing import Callable, Optional, Tuple, Type, TypeVar
 
@@ -18,6 +19,10 @@
 PART_SUFFIX = ".part"
 CHUNK_SIZE = 8 * 1024 * 1024
 
+# netCDF-C/HDF5 are not thread safe: opening files from several download threads
+# at once gives HDF errors or segfaults, so validate_file opens one at a time
+_NETCDF_LOCK = threading.Lock()
+
 T = TypeVar("T")
 
 
@@ -217,12 +222,13 @@ def validate_file(path: str) -> None:
         magic = f.read(8)
 
     if magic.startswith(b"CDF") or magic.startswith(b"\x89HDF"):
-        import netCDF4
+        with _NETCDF_LOCK:
+            import netCDF4
 
-        try:
-            netCDF4.Dataset(path).close()
-        except Exception as e:
-            raise IOError(f"Corrupt NetCDF file {path}: {e}") from e
+            try:
+                netCDF4.Dataset(path).close()
+            except Exception as e:
+                raise IOError(f"Corrupt NetCDF file {path}: {e}") from e
 
     elif magic.startswith(b"GRIB"):
         with open(path, "rb") as f:

bluemath_tk/downloaders/earthdata/SWOT/SWOT_config.json

@@ -0,0 +1,13 @@
+{
+    "description": "Surface Water and Ocean Topography (SWOT) mission, CNES/NASA, distributed by PO.DAAC",
+    "cmr_url": "https://cmr.earthdata.nasa.gov/search/granules.umm_json",
+    "datasets": {
+        "L2_LR_SSH_WINDWAVE_D": {
+            "description": "KaRIn L2 low-rate wind and wave product (swh_karin, wind_speed_karin, sig0_karin), 2 km swath grid, one file per half-orbit pass, version D",
+            "short_name": "SWOT_L2_LR_SSH_WINDWAVE_D",
+            "filename_pattern": "^SWOT_L2_LR_SSH_WindWave_.*\\.nc$",
+            "local_dir": "SWOT/L2_LR_SSH_WINDWAVE_D/{year}-{month:02d}",
+            "start": "2023-03"
+        }
+    }
+}

bluemath_tk/downloaders/aviso/__init__.py → bluemath_tk/downloaders/earthdata/__init__.py

@@ -1,11 +1,11 @@
 """
 Project: BlueMath_tk
-Sub-Module: downloaders.aviso
+Sub-Module: downloaders.earthdata
 Author: GeoOcean Research Group, Universidad de Cantabria
 Repository: https://github.com/GeoOcean/BlueMath_tk.git
 Status: Under development (Working)
 """
 
-from .aviso_downloader import AvisoDownloader
+from .earthdata_downloader import EarthdataDownloader
 
-__all__ = ["AvisoDownloader"]
+__all__ = ["EarthdataDownloader"]