From 211a0391fa126d032eba68510d95cf6d3a5ebe0c Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 7 Nov 2026 02:50:22 +0000 Subject: [PATCH] Retry URL imports as a browser when a site answers 403 Rumble now sits behind Cloudflare's bot screen, which refuses yt-dlp's Python client on its TLS fingerprint: "Unable to download webpage: HTTP Error 403: Forbidden". yt-dlp can handshake like Chrome through curl_cffi, which the install did not include. Add the curl-cffi extra (pinned to 0.17.0, abi3 wheels for x86_64 and aarch64), bump yt-dlp to 2026.10.26, and retry a probe or download once with impersonation when it fails with 403. Other sites keep yt-dlp's own client. A 403 that survives the retry, or that cannot be retried because curl_cffi is missing, now says what to do instead of yt-dlp's raw line. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01V1geD4inqJxe1F6HuWV82R --- backend/app/ingest/ytdlp.py | 72 ++++++++++++++++++++++++++------ backend/requirements.txt | 9 +++- backend/tests/test_url_import.py | 55 +++++++++++++++++++++++- docs/url-import.md | 15 ++++++- 4 files changed, 134 insertions(+), 17 deletions(-) diff --git a/backend/app/ingest/ytdlp.py b/backend/app/ingest/ytdlp.py index e1f6921..8fd3eb1 100644 --- a/backend/app/ingest/ytdlp.py +++ b/backend/app/ingest/ytdlp.py @@ -32,7 +32,8 @@ import yt_dlp from fastapi import HTTPException, status -from yt_dlp.utils import DownloadCancelled, DownloadError +from yt_dlp.networking.impersonate import ImpersonateTarget +from yt_dlp.utils import DownloadCancelled, DownloadError, YoutubeDLError from app import music from app.config import settings @@ -132,11 +133,10 @@ def probe_url(url: str, workdir: Path) -> dict: "extract_flat": "in_playlist", "playlist_items": f"1:{max(1, settings.url_import_max_playlist_items)}", } - with yt_dlp.YoutubeDL(options) as ydl: - try: - info = ydl.extract_info(url, download=False) - except DownloadError as exc: - raise UrlImportError(readable_error(exc)) from exc + try: + info = _extract(options, url, download=False) + except DownloadError as exc: + raise UrlImportError(readable_error(exc)) from exc if not info: raise UrlImportError("Nothing importable was found at that link") return info @@ -265,13 +265,12 @@ def download( {"key": "FFmpegSubtitlesConvertor", "format": "vtt", "when": "before_dl"} ) - with yt_dlp.YoutubeDL(options) as ydl: - try: - result = ydl.extract_info(url, download=True) - except DownloadCancelled as exc: - raise JobCancelled(str(exc)) from exc - except DownloadError as exc: - raise UrlImportError(readable_error(exc)) from exc + try: + result = _extract(options, url, download=True) + except DownloadCancelled as exc: + raise JobCancelled(str(exc)) from exc + except DownloadError as exc: + raise UrlImportError(readable_error(exc)) from exc media = _media_path(result, workdir) return Downloaded( @@ -451,6 +450,44 @@ def _base_options(workdir: Path) -> dict[str, Any]: return options +def _extract(options: dict[str, Any], url: str, *, download: bool) -> Optional[dict]: + """`extract_info`, retried once as a browser when the site answers 403. + + Sites behind Cloudflare's bot screen (Rumble, and others that come and go) refuse a + request on its TLS fingerprint before reading a single header, so no user agent or + cookie gets past it; only a request that handshakes like Chrome does. yt-dlp can + send one through curl_cffi (the `curl-cffi` extra in requirements.txt). + + Only on a 403, and not for every request, because impersonation swaps the HTTP + stack under every site that already works — YouTube's extractor in particular is + tuned against yt-dlp's own client, and a change there would fail on YouTube's + schedule rather than ours. A 403 that impersonation does not cure (a geo-block, a + private video) costs one extra page load and reports the same error. + """ + try: + with yt_dlp.YoutubeDL(options) as ydl: + return ydl.extract_info(url, download=download) + except DownloadError as exc: + if "impersonate" in options or not _is_forbidden(exc): + raise + first = exc + + logger.info("url import: %s answered 403, retrying as a browser", url) + try: + ydl = yt_dlp.YoutubeDL({**options, "impersonate": ImpersonateTarget()}) + except YoutubeDLError: + # curl_cffi is not installed (or is a version yt-dlp does not support). The + # original 403 is the truthful answer; readable_error says what to install. + raise first + with ydl: + return ydl.extract_info(url, download=download) + + +def _is_forbidden(exc: Exception) -> bool: + message = str(getattr(exc, "msg", None) or exc).lower() + return "403" in message and "forbidden" in message + + _ERROR_PREFIX = re.compile(r"^(?:ERROR:\s*)?(?:\[[^\]]+\]\s*)?(?:[\w-]+:\s+)?") # Messages worth replacing with what to actually do about them. @@ -487,6 +524,15 @@ def readable_error(exc: Exception) -> str: "docs/url-import.md." ) + if "403" in lowered and "forbidden" in lowered: + return ( + "The site refused this server (HTTP 403). If the backend image predates " + "curl-cffi in backend/requirements.txt, rebuild it so the import can retry " + "as a browser. Otherwise the site blocks this server's IP or wants a " + "signed-in session: export your browser's cookies for it to a file and set " + "URL_IMPORT_COOKIES_FILE — see docs/url-import.md." + ) + first_line = message.splitlines()[0] if message else "" cleaned = _ERROR_PREFIX.sub("", first_line, count=1).strip() or "The import failed" if any(phrase in lowered for phrase in _NEEDS_UPDATE): diff --git a/backend/requirements.txt b/backend/requirements.txt index f867e16..e1eb803 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -44,7 +44,14 @@ openpyxl==3.1.5 # Pinned like everything else here, with one difference worth knowing: YouTube changes # under yt-dlp every few weeks, so this pin *will* start failing on its own schedule. The # fix is always the same — bump it to the latest release and `docker compose up --build`. -yt-dlp[default,deno]==2026.8.19 +# +# `curl-cffi` lets yt-dlp handshake like Chrome. Rumble (and other sites behind +# Cloudflare's bot screen) answer a plain Python client with 403 before reading a header; +# ytdlp._extract retries those as a browser. Pinned to what this yt-dlp's own extra +# accepts (<0.18): yt-dlp refuses a curl_cffi it was not built against. abi3 manylinux +# wheels for x86_64 and aarch64, so nothing compiles. +yt-dlp[default,deno,curl-cffi]==2026.10.26 +curl-cffi==0.17.0 # How enrichment and search reach Gecko Suite (AI_BACKEND=suite), and the agent loop the # assistant shares with the other apps. A commit, not a branch: the client and the # suite's API change together, and a deployment must not pick up one without the other. diff --git a/backend/tests/test_url_import.py b/backend/tests/test_url_import.py index d3ff13d..66df9d9 100644 --- a/backend/tests/test_url_import.py +++ b/backend/tests/test_url_import.py @@ -17,7 +17,7 @@ import pytest from sqlmodel import col, select -from yt_dlp.utils import DownloadError +from yt_dlp.utils import DownloadError, YoutubeDLError from app.ingest import ytdlp from app.jobs import enrichment as enrichment_jobs @@ -84,8 +84,15 @@ class FakeYoutubeDL: made: list = [] # When set, raised from `extract_info` for the matching URL. errors: dict = {} + # URLs that answer 403 unless the client impersonates a browser (Cloudflare). + browser_only: set = set() + # Whether curl_cffi is "installed": without it yt-dlp refuses `impersonate` at + # construction, which is the real library's behaviour. + can_impersonate: bool = True def __init__(self, params): + if params.get("impersonate") and not FakeYoutubeDL.can_impersonate: + raise YoutubeDLError('Impersonate target "" is not available.') self.params = params FakeYoutubeDL.made.append(params) @@ -98,6 +105,11 @@ def __exit__(self, *exc): def extract_info(self, url, download=False): if url in FakeYoutubeDL.errors: raise FakeYoutubeDL.errors[url] + if url in FakeYoutubeDL.browser_only and not self.params.get("impersonate"): + raise DownloadError( + f"ERROR: [Rumble] {url}: Unable to download webpage: HTTP Error 403: " + "Forbidden (caused by )" + ) info = copy.deepcopy(FakeYoutubeDL.infos[url]) if not download: return info @@ -134,6 +146,8 @@ def fake_ytdlp(monkeypatch): FakeYoutubeDL.infos = {VIDEO_URL: video_info()} FakeYoutubeDL.made = [] FakeYoutubeDL.errors = {} + FakeYoutubeDL.browser_only = set() + FakeYoutubeDL.can_impersonate = True monkeypatch.setattr(ytdlp.yt_dlp, "YoutubeDL", FakeYoutubeDL) # Every name resolves somewhere public, so the SSRF guard never touches real DNS. monkeypatch.setattr( @@ -659,6 +673,45 @@ def test_yt_dlp_failures_become_readable_messages(library, session, monkeypatch, assert not job["error_message"].startswith("ERROR") +def test_a_site_that_refuses_python_is_retried_as_a_browser(library, session, monkeypatch): + """Rumble behind Cloudflare: 403 to yt-dlp's own client, fine to Chrome's.""" + FakeYoutubeDL.browser_only.add(VIDEO_URL) + job = _activity(library, _import(library, session, monkeypatch)) + + assert job["status"] == "done", job["error_message"] + # Probe and download each tried plainly first, then as a browser. + assert [bool(p.get("impersonate")) for p in FakeYoutubeDL.made] == [False, True, False, True] + + +def test_a_403_that_impersonation_does_not_cure_says_what_to_do(library, session, monkeypatch): + FakeYoutubeDL.errors[VIDEO_URL] = DownloadError( + "ERROR: [Rumble] v1: Unable to download webpage: HTTP Error 403: Forbidden" + ) + job = _activity(library, _import(library, session, monkeypatch)) + + assert job["status"] == "error" + assert "HTTP 403" in job["error_message"] + assert "URL_IMPORT_COOKIES_FILE" in job["error_message"] + assert [bool(p.get("impersonate")) for p in FakeYoutubeDL.made] == [False, True] + + +def test_without_curl_cffi_the_403_is_reported_not_crashed_on(library, session, monkeypatch): + FakeYoutubeDL.browser_only.add(VIDEO_URL) + FakeYoutubeDL.can_impersonate = False + job = _activity(library, _import(library, session, monkeypatch)) + + assert job["status"] == "error" + assert "HTTP 403" in job["error_message"] + assert "curl-cffi" in job["error_message"] + + +def test_other_failures_are_not_retried_as_a_browser(library, session, monkeypatch): + FakeYoutubeDL.errors[VIDEO_URL] = DownloadError("ERROR: [youtube] abc123: Private video") + _import(library, session, monkeypatch) + + assert not any(p.get("impersonate") for p in FakeYoutubeDL.made) + + def test_cancelling_mid_download_stops_the_job(library, session, monkeypatch): """The progress hook is the checkpoint: the reporter raises JobCancelled, which the hook turns into the exception yt-dlp is written to let through.""" diff --git a/docs/url-import.md b/docs/url-import.md index 6b2071a..a894177 100644 --- a/docs/url-import.md +++ b/docs/url-import.md @@ -83,10 +83,21 @@ YouTube changes under yt-dlp every few weeks, and a pinned yt-dlp eventually sto working on its own schedule. The error message says so ("…yt-dlp needs updating"). The fix is always the same: -1. Set `yt-dlp[default,deno]==` in `backend/requirements.txt` - ([releases](https://pypi.org/project/yt-dlp/#history)). +1. Set `yt-dlp[default,deno,curl-cffi]==` in `backend/requirements.txt` + ([releases](https://pypi.org/project/yt-dlp/#history)), and keep the `curl-cffi` pin + beside it inside the range that release's extra accepts. 2. `docker compose up --build -d`. +### HTTP 403 and browser impersonation + +Rumble, and other sites behind Cloudflare's bot screen, refuse a plain Python client on +its TLS handshake (`Unable to download webpage: HTTP Error 403: Forbidden`) before any +header or cookie is read. The `curl-cffi` extra lets yt-dlp handshake like Chrome, and +`ytdlp._extract` retries any 403 that way, once. Only on a 403: sites that already work, +YouTube above all, keep yt-dlp's own client. A 403 that survives the retry is a block on +the server's IP or a site that wants a signed-in session; the cookies file below is the +remedy for the second. + ### Deno Since late 2025 YouTube serves almost no downloadable formats to a client that cannot run