Feature/imos downloader

bluemath_tk/downloaders/_download_result.py

@@ -112,13 +112,11 @@ def add_error(
         if context:
             error_dict["context"] = context
         self.errors.append(error_dict)
-        if file_path not in self.error_files:
-            self.error_files.append(file_path)
+        self._append_unique("error_files", file_path)
 
     def add_downloaded(self, file_path: str):
         """Add a successfully downloaded file."""
-        if file_path not in self.downloaded_files:
-            self.downloaded_files.append(file_path)
+        self._append_unique("downloaded_files", file_path)
 
     def add_skipped(self, file_path: str, reason: str = ""):
         """
@@ -131,11 +129,26 @@ def add_skipped(self, file_path: str, reason: str = ""):
         reason : str, optional
             Reason why the file was skipped.
         """
-        if file_path not in self.skipped_files:
-            self.skipped_files.append(file_path)
+        self._append_unique("skipped_files", file_path)
         if reason:
             self.metadata.setdefault("skip_reasons", {})[file_path] = reason
 
+    def _append_unique(self, name: str, file_path: str) -> None:
+        """
+        Append to one of the file lists unless already there.
+
+        A set kept next to each list makes the check O(1), so results of
+        hundreds of thousands of files stay fast. It is rebuilt if the list was
+        changed directly.
+        """
+        files = getattr(self, name)
+        seen = self.__dict__.setdefault("_seen", {}).get(name)
+        if seen is None or len(seen) != len(files):
+            seen = self._seen[name] = set(files)
+        if file_path not in seen:
+            seen.add(file_path)
+            files.append(file_path)
+
     def merge(self, other: "DownloadResult") -> "DownloadResult":
         """
         Add the files, errors and metadata of another result to this one.
@@ -155,11 +168,9 @@ def merge(self, other: "DownloadResult") -> "DownloadResult":
         for file_path in other.downloaded_files:
             self.add_downloaded(file_path)
         for file_path in other.skipped_files:
-            if file_path not in self.skipped_files:
-                self.skipped_files.append(file_path)
+            self._append_unique("skipped_files", file_path)
         for file_path in other.error_files:
-            if file_path not in self.error_files:
-                self.error_files.append(file_path)
+            self._append_unique("error_files", file_path)
         self.errors.extend(other.errors)
         for key, value in other.metadata.items():
             if isinstance(value, dict):

bluemath_tk/downloaders/_transfer.py

@@ -6,6 +6,7 @@
 file validation.
 """
 
+import hashlib
 import logging
 import os
 import threading
@@ -202,6 +203,19 @@ def http_download(
             )
 
 
+def file_checksum(path: str, algorithm: str) -> str:
+    """
+    Hex digest of a file; algorithm as providers write it ("MD5", "SHA-256", ...).
+    """
+
+    digest = hashlib.new(algorithm.lower().replace("-", ""))
+    with open(path, "rb") as f:
+        for block in iter(lambda: f.read(8 * 1024 * 1024), b""):
+            digest.update(block)
+
+    return digest.hexdigest()
+
+
 def validate_file(path: str) -> None:
     """
     Check that a downloaded file is readable, based on its magic bytes.

bluemath_tk/downloaders/earthdata/earthdata_downloader.py

@@ -14,10 +14,8 @@
 Each dataset of a product is one CMR collection, given by its short name.
 """
 
-import hashlib
 import netrc
 import os
-import re
 import threading
 from concurrent.futures import ThreadPoolExecutor
 from datetime import datetime
@@ -28,7 +26,13 @@
 
 from .._base_downloaders import BaseDownloader, load_config
 from .._download_result import DownloadResult
-from .._transfer import http_download, http_get, make_session, retry_call
+from .._transfer import (
+    file_checksum,
+    http_download,
+    http_get,
+    make_session,
+    retry_call,
+)
 
 _HERE = os.path.dirname(__file__)
 URS_HOST = "urs.earthdata.nasa.gov"
@@ -133,19 +137,6 @@ def parse_granule(item: dict) -> List[dict]:
     return rows
 
 
-def file_checksum(path: str, algorithm: str) -> str:
-    """
-    Hex digest of a file ("MD5", "SHA-256", ... as written in CMR).
-    """
-
-    digest = hashlib.new(algorithm.lower().replace("-", ""))
-    with open(path, "rb") as f:
-        for block in iter(lambda: f.read(8 * 1024 * 1024), b""):
-            digest.update(block)
-
-    return digest.hexdigest()
-
-
 class EarthdataDownloader(BaseDownloader):
     """
     Downloader for NASA Earthdata collections, found through the CMR search API.

bluemath_tk/downloaders/imos/SRS/SRS_config.json

@@ -0,0 +1,12 @@
+{
+    "description": "IMOS Satellite Remote Sensing (SRS) facility, public AODN bucket on AWS S3",
+    "bucket_url": "https://imos-data.s3-ap-southeast-2.amazonaws.com/",
+    "datasets": {
+        "Wave-Wind-Altimetry-DM00": {
+            "description": "Multi-mission altimeter wave height and wind speed (Ribal and Young 2019, updated), 1 Hz along-track, one file per mission and 1x1 degree cell, 1985-present, calibrated and raw values with quality flags",
+            "prefix": "IMOS/SRS/Surface-Waves/Wave-Wind-Altimetry-DM00/",
+            "local_dir": "SRS/Surface-Waves/Wave-Wind-Altimetry-DM00",
+            "filename_pattern": "^IMOS_SRS-Surface-Waves_MW_.*-DM00\\.nc$"
+        }
+    }
+}

bluemath_tk/downloaders/imos/__init__.py

@@ -0,0 +1,11 @@
+"""
+Project: BlueMath_tk
+Sub-Module: downloaders.imos
+Author: GeoOcean Research Group, Universidad de Cantabria
+Repository: https://github.com/GeoOcean/BlueMath_tk.git
+Status: Under development (Working)
+"""
+
+from .imos_downloader import IMOSDownloader
+
+__all__ = ["IMOSDownloader"]