Retry URL imports as a browser when a site answers 403 (fixes Rumble)

backend/app/ingest/ytdlp.py

@@ -32,7 +32,8 @@
 
 import yt_dlp
 from fastapi import HTTPException, status
-from yt_dlp.utils import DownloadCancelled, DownloadError
+from yt_dlp.networking.impersonate import ImpersonateTarget
+from yt_dlp.utils import DownloadCancelled, DownloadError, YoutubeDLError
 
 from app import music
 from app.config import settings
@@ -132,11 +133,10 @@ def probe_url(url: str, workdir: Path) -> dict:
         "extract_flat": "in_playlist",
         "playlist_items": f"1:{max(1, settings.url_import_max_playlist_items)}",
     }
-    with yt_dlp.YoutubeDL(options) as ydl:
-        try:
-            info = ydl.extract_info(url, download=False)
-        except DownloadError as exc:
-            raise UrlImportError(readable_error(exc)) from exc
+    try:
+        info = _extract(options, url, download=False)
+    except DownloadError as exc:
+        raise UrlImportError(readable_error(exc)) from exc
     if not info:
         raise UrlImportError("Nothing importable was found at that link")
     return info
@@ -265,13 +265,12 @@ def download(
             {"key": "FFmpegSubtitlesConvertor", "format": "vtt", "when": "before_dl"}
         )
 
-    with yt_dlp.YoutubeDL(options) as ydl:
-        try:
-            result = ydl.extract_info(url, download=True)
-        except DownloadCancelled as exc:
-            raise JobCancelled(str(exc)) from exc
-        except DownloadError as exc:
-            raise UrlImportError(readable_error(exc)) from exc
+    try:
+        result = _extract(options, url, download=True)
+    except DownloadCancelled as exc:
+        raise JobCancelled(str(exc)) from exc
+    except DownloadError as exc:
+        raise UrlImportError(readable_error(exc)) from exc
 
     media = _media_path(result, workdir)
     return Downloaded(
@@ -451,6 +450,44 @@ def _base_options(workdir: Path) -> dict[str, Any]:
     return options
 
 
+def _extract(options: dict[str, Any], url: str, *, download: bool) -> Optional[dict]:
+    """`extract_info`, retried once as a browser when the site answers 403.
+
+    Sites behind Cloudflare's bot screen (Rumble, and others that come and go) refuse a
+    request on its TLS fingerprint before reading a single header, so no user agent or
+    cookie gets past it; only a request that handshakes like Chrome does. yt-dlp can
+    send one through curl_cffi (the `curl-cffi` extra in requirements.txt).
+
+    Only on a 403, and not for every request, because impersonation swaps the HTTP
+    stack under every site that already works — YouTube's extractor in particular is
+    tuned against yt-dlp's own client, and a change there would fail on YouTube's
+    schedule rather than ours. A 403 that impersonation does not cure (a geo-block, a
+    private video) costs one extra page load and reports the same error.
+    """
+    try:
+        with yt_dlp.YoutubeDL(options) as ydl:
+            return ydl.extract_info(url, download=download)
+    except DownloadError as exc:
+        if "impersonate" in options or not _is_forbidden(exc):
+            raise
+        first = exc
+
+    logger.info("url import: %s answered 403, retrying as a browser", url)
+    try:
+        ydl = yt_dlp.YoutubeDL({**options, "impersonate": ImpersonateTarget()})
+    except YoutubeDLError:
+        # curl_cffi is not installed (or is a version yt-dlp does not support). The
+        # original 403 is the truthful answer; readable_error says what to install.
+        raise first
+    with ydl:
+        return ydl.extract_info(url, download=download)
+
+
+def _is_forbidden(exc: Exception) -> bool:
+    message = str(getattr(exc, "msg", None) or exc).lower()
+    return "403" in message and "forbidden" in message
+
+
 _ERROR_PREFIX = re.compile(r"^(?:ERROR:\s*)?(?:\[[^\]]+\]\s*)?(?:[\w-]+:\s+)?")
 
 # Messages worth replacing with what to actually do about them.
@@ -487,6 +524,15 @@ def readable_error(exc: Exception) -> str:
             "docs/url-import.md."
         )
 
+    if "403" in lowered and "forbidden" in lowered:
+        return (
+            "The site refused this server (HTTP 403). If the backend image predates "
+            "curl-cffi in backend/requirements.txt, rebuild it so the import can retry "
+            "as a browser. Otherwise the site blocks this server's IP or wants a "
+            "signed-in session: export your browser's cookies for it to a file and set "
+            "URL_IMPORT_COOKIES_FILE — see docs/url-import.md."
+        )
+
     first_line = message.splitlines()[0] if message else ""
     cleaned = _ERROR_PREFIX.sub("", first_line, count=1).strip() or "The import failed"
     if any(phrase in lowered for phrase in _NEEDS_UPDATE):

backend/requirements.txt

@@ -44,7 +44,14 @@ openpyxl==3.1.5
 # Pinned like everything else here, with one difference worth knowing: YouTube changes
 # under yt-dlp every few weeks, so this pin *will* start failing on its own schedule. The
 # fix is always the same — bump it to the latest release and `docker compose up --build`.
-yt-dlp[default,deno]==2026.8.19
+#
+# `curl-cffi` lets yt-dlp handshake like Chrome. Rumble (and other sites behind
+# Cloudflare's bot screen) answer a plain Python client with 403 before reading a header;
+# ytdlp._extract retries those as a browser. Pinned to what this yt-dlp's own extra
+# accepts (<0.18): yt-dlp refuses a curl_cffi it was not built against. abi3 manylinux
+# wheels for x86_64 and aarch64, so nothing compiles.
+yt-dlp[default,deno,curl-cffi]==2026.10.26
+curl-cffi==0.17.0
 # How enrichment and search reach Gecko Suite (AI_BACKEND=suite), and the agent loop the
 # assistant shares with the other apps. A commit, not a branch: the client and the
 # suite's API change together, and a deployment must not pick up one without the other.

backend/tests/test_url_import.py

@@ -17,7 +17,7 @@
 
 import pytest
 from sqlmodel import col, select
-from yt_dlp.utils import DownloadError
+from yt_dlp.utils import DownloadError, YoutubeDLError
 
 from app.ingest import ytdlp
 from app.jobs import enrichment as enrichment_jobs
@@ -84,8 +84,15 @@ class FakeYoutubeDL:
     made: list = []
     # When set, raised from `extract_info` for the matching URL.
     errors: dict = {}
+    # URLs that answer 403 unless the client impersonates a browser (Cloudflare).
+    browser_only: set = set()
+    # Whether curl_cffi is "installed": without it yt-dlp refuses `impersonate` at
+    # construction, which is the real library's behaviour.
+    can_impersonate: bool = True
 
     def __init__(self, params):
+        if params.get("impersonate") and not FakeYoutubeDL.can_impersonate:
+            raise YoutubeDLError('Impersonate target "" is not available.')
         self.params = params
         FakeYoutubeDL.made.append(params)
 
@@ -98,6 +105,11 @@ def __exit__(self, *exc):
     def extract_info(self, url, download=False):
         if url in FakeYoutubeDL.errors:
             raise FakeYoutubeDL.errors[url]
+        if url in FakeYoutubeDL.browser_only and not self.params.get("impersonate"):
+            raise DownloadError(
+                f"ERROR: [Rumble] {url}: Unable to download webpage: HTTP Error 403: "
+                "Forbidden (caused by <HTTPError 403: Forbidden>)"
+            )
         info = copy.deepcopy(FakeYoutubeDL.infos[url])
         if not download:
             return info
@@ -134,6 +146,8 @@ def fake_ytdlp(monkeypatch):
     FakeYoutubeDL.infos = {VIDEO_URL: video_info()}
     FakeYoutubeDL.made = []
     FakeYoutubeDL.errors = {}
+    FakeYoutubeDL.browser_only = set()
+    FakeYoutubeDL.can_impersonate = True
     monkeypatch.setattr(ytdlp.yt_dlp, "YoutubeDL", FakeYoutubeDL)
     # Every name resolves somewhere public, so the SSRF guard never touches real DNS.
     monkeypatch.setattr(
@@ -659,6 +673,45 @@ def test_yt_dlp_failures_become_readable_messages(library, session, monkeypatch,
     assert not job["error_message"].startswith("ERROR")
 
 
+def test_a_site_that_refuses_python_is_retried_as_a_browser(library, session, monkeypatch):
+    """Rumble behind Cloudflare: 403 to yt-dlp's own client, fine to Chrome's."""
+    FakeYoutubeDL.browser_only.add(VIDEO_URL)
+    job = _activity(library, _import(library, session, monkeypatch))
+
+    assert job["status"] == "done", job["error_message"]
+    # Probe and download each tried plainly first, then as a browser.
+    assert [bool(p.get("impersonate")) for p in FakeYoutubeDL.made] == [False, True, False, True]
+
+
+def test_a_403_that_impersonation_does_not_cure_says_what_to_do(library, session, monkeypatch):
+    FakeYoutubeDL.errors[VIDEO_URL] = DownloadError(
+        "ERROR: [Rumble] v1: Unable to download webpage: HTTP Error 403: Forbidden"
+    )
+    job = _activity(library, _import(library, session, monkeypatch))
+
+    assert job["status"] == "error"
+    assert "HTTP 403" in job["error_message"]
+    assert "URL_IMPORT_COOKIES_FILE" in job["error_message"]
+    assert [bool(p.get("impersonate")) for p in FakeYoutubeDL.made] == [False, True]
+
+
+def test_without_curl_cffi_the_403_is_reported_not_crashed_on(library, session, monkeypatch):
+    FakeYoutubeDL.browser_only.add(VIDEO_URL)
+    FakeYoutubeDL.can_impersonate = False
+    job = _activity(library, _import(library, session, monkeypatch))
+
+    assert job["status"] == "error"
+    assert "HTTP 403" in job["error_message"]
+    assert "curl-cffi" in job["error_message"]
+
+
+def test_other_failures_are_not_retried_as_a_browser(library, session, monkeypatch):
+    FakeYoutubeDL.errors[VIDEO_URL] = DownloadError("ERROR: [youtube] abc123: Private video")
+    _import(library, session, monkeypatch)
+
+    assert not any(p.get("impersonate") for p in FakeYoutubeDL.made)
+
+
 def test_cancelling_mid_download_stops_the_job(library, session, monkeypatch):
     """The progress hook is the checkpoint: the reporter raises JobCancelled, which the
     hook turns into the exception yt-dlp is written to let through."""

docs/url-import.md

@@ -83,10 +83,21 @@ YouTube changes under yt-dlp every few weeks, and a pinned yt-dlp eventually sto
 working on its own schedule. The error message says so ("…yt-dlp needs updating"). The
 fix is always the same:
 
-1. Set `yt-dlp[default,deno]==<latest>` in `backend/requirements.txt`
-   ([releases](https://pypi.org/project/yt-dlp/#history)).
+1. Set `yt-dlp[default,deno,curl-cffi]==<latest>` in `backend/requirements.txt`
+   ([releases](https://pypi.org/project/yt-dlp/#history)), and keep the `curl-cffi` pin
+   beside it inside the range that release's extra accepts.
 2. `docker compose up --build -d`.
 
+### HTTP 403 and browser impersonation
+
+Rumble, and other sites behind Cloudflare's bot screen, refuse a plain Python client on
+its TLS handshake (`Unable to download webpage: HTTP Error 403: Forbidden`) before any
+header or cookie is read. The `curl-cffi` extra lets yt-dlp handshake like Chrome, and
+`ytdlp._extract` retries any 403 that way, once. Only on a 403: sites that already work,
+YouTube above all, keep yt-dlp's own client. A 403 that survives the retry is a block on
+the server's IP or a site that wants a signed-in session; the cookies file below is the
+remedy for the second.
+
 ### Deno
 
 Since late 2025 YouTube serves almost no downloadable formats to a client that cannot run