From a6cb1e977ebad63c9164c9f3eb727139bf10972d Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 7 Nov 2026 03:28:23 +0000 Subject: [PATCH 1/3] Import a Cloudflare-challenged Rumble page through its embed Impersonating a browser was not enough on the production server: Rumble's watch page answers its IP with Cloudflare's JavaScript challenge (403, cf-mitigated: challenge), which nothing but a real browser can pass. The same server reaches Rumble's oEmbed API and embed player without a challenge. So when a rumble.com watch page still answers 403 as a browser, ask oEmbed for the video's embed id and import rumble.com/embed//, recording the watch page as the asset's source URL so the already-imported check still matches. The embed lacks the description and tags; the title, channel, date, duration and thumbnail come through. The 403 message now names the likely cause, a Cloudflare challenge bound to the server's IP, instead of suggesting a cookies file, which cannot carry that clearance from another machine. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01V1geD4inqJxe1F6HuWV82R --- backend/app/ingest/ytdlp.py | 100 ++++++++++++++++++++++++------- backend/tests/test_url_import.py | 72 ++++++++++++++++++++-- docs/url-import.md | 23 ++++++- 3 files changed, 166 insertions(+), 29 deletions(-) diff --git a/backend/app/ingest/ytdlp.py b/backend/app/ingest/ytdlp.py index 8fd3eb1..7b23569 100644 --- a/backend/app/ingest/ytdlp.py +++ b/backend/app/ingest/ytdlp.py @@ -21,6 +21,7 @@ from __future__ import annotations +import json import logging import re import shutil @@ -451,18 +452,21 @@ def _base_options(workdir: Path) -> dict[str, Any]: def _extract(options: dict[str, Any], url: str, *, download: bool) -> Optional[dict]: - """`extract_info`, retried once as a browser when the site answers 403. + """`extract_info`, with two fallbacks for a site that answers 403. - Sites behind Cloudflare's bot screen (Rumble, and others that come and go) refuse a + First, the same request as a browser. Sites behind Cloudflare's bot screen refuse a request on its TLS fingerprint before reading a single header, so no user agent or cookie gets past it; only a request that handshakes like Chrome does. yt-dlp can - send one through curl_cffi (the `curl-cffi` extra in requirements.txt). - - Only on a 403, and not for every request, because impersonation swaps the HTTP - stack under every site that already works — YouTube's extractor in particular is - tuned against yt-dlp's own client, and a change there would fail on YouTube's - schedule rather than ours. A 403 that impersonation does not cure (a geo-block, a - private video) costs one extra page load and reports the same error. + send one through curl_cffi (the `curl-cffi` extra in requirements.txt). Only on a + 403, and not for every request, because impersonation swaps the HTTP stack under + every site that already works — YouTube's extractor in particular is tuned against + yt-dlp's own client, and a change there would fail on YouTube's schedule. + + Second, for Rumble only, the video's embed instead of its watch page. Cloudflare + sends some server IPs a JavaScript challenge on the watch page that no importer can + answer, while the oEmbed API and the embed player behind it stay open (checked from + a blocked server, Nov 2026). The embed carries less — no description or tags — but + a video with its title beats an error. """ try: with yt_dlp.YoutubeDL(options) as ydl: @@ -470,17 +474,67 @@ def _extract(options: dict[str, Any], url: str, *, download: bool) -> Optional[d except DownloadError as exc: if "impersonate" in options or not _is_forbidden(exc): raise - first = exc + failure = exc + + browser = _as_browser(options) + if browser is not None: + logger.info("url import: %s answered 403, retrying as a browser", url) + try: + with yt_dlp.YoutubeDL(browser) as ydl: + return ydl.extract_info(url, download=download) + except DownloadError as exc: + if not _is_forbidden(exc): + raise + failure = exc + + embed = _rumble_embed_url(url, browser or options) + if embed is None: + raise failure + logger.info("url import: %s is challenged, importing its embed %s", url, embed) + with yt_dlp.YoutubeDL(browser or options) as ydl: + info = ydl.extract_info(embed, download=download) + if info: + # The page people know the video by, so the asset's source link and the + # already-imported check both name it rather than the player. + info["webpage_url"] = url + return info - logger.info("url import: %s answered 403, retrying as a browser", url) + +def _as_browser(options: dict[str, Any]) -> Optional[dict[str, Any]]: + """`options` impersonating a browser, or None when curl_cffi is missing (or is a + version yt-dlp does not support): yt-dlp refuses the option at construction.""" + browser = {**options, "impersonate": ImpersonateTarget()} try: - ydl = yt_dlp.YoutubeDL({**options, "impersonate": ImpersonateTarget()}) + yt_dlp.YoutubeDL(browser).close() except YoutubeDLError: - # curl_cffi is not installed (or is a version yt-dlp does not support). The - # original 403 is the truthful answer; readable_error says what to install. - raise first - with ydl: - return ydl.extract_info(url, download=download) + return None + return browser + + +_RUMBLE_HOSTS = ("rumble.com", "www.rumble.com") +_RUMBLE_EMBED = re.compile(r"https://rumble\.com/embed/(?:[0-9a-z]+\.)?([0-9a-z]+)") + + +def _rumble_embed_url(url: str, options: dict[str, Any]) -> Optional[str]: + """The embed player URL for a Rumble watch page, from Rumble's oEmbed API. + + The embed id is not the watch page's id, and the page that would map one to the + other is the one being challenged. Fetched + through yt-dlp so the cookies, timeout and impersonation of the import apply. + """ + parts = urllib.parse.urlsplit(url) + if (parts.hostname or "").lower() not in _RUMBLE_HOSTS or parts.path.startswith("/embed/"): + return None + api = "https://rumble.com/api/Media/oembed.json?" + urllib.parse.urlencode({"url": url}) + try: + with yt_dlp.YoutubeDL(options) as ydl: + with ydl.urlopen(api) as response: + data = json.loads(response.read()) + except (YoutubeDLError, OSError, ValueError) as exc: + logger.info("url import: Rumble oEmbed for %s failed: %s", url, exc) + return None + match = _RUMBLE_EMBED.search(str(data.get("html") or "") if isinstance(data, dict) else "") + return f"https://rumble.com/embed/{match.group(1)}/" if match else None def _is_forbidden(exc: Exception) -> bool: @@ -526,11 +580,13 @@ def readable_error(exc: Exception) -> str: if "403" in lowered and "forbidden" in lowered: return ( - "The site refused this server (HTTP 403). If the backend image predates " - "curl-cffi in backend/requirements.txt, rebuild it so the import can retry " - "as a browser. Otherwise the site blocks this server's IP or wants a " - "signed-in session: export your browser's cookies for it to a file and set " - "URL_IMPORT_COOKIES_FILE — see docs/url-import.md." + "The site refused this server (HTTP 403). " + "Usually that is Cloudflare challenging this server's IP address — a " + "JavaScript check no importer can answer, and one a cookies file from " + "another computer does not pass either, since the clearance is tied to the " + "IP that earned it. Download the video on your own computer and upload it. " + "If the backend image predates curl-cffi in backend/requirements.txt, " + "rebuild it first." ) first_line = message.splitlines()[0] if message else "" diff --git a/backend/tests/test_url_import.py b/backend/tests/test_url_import.py index 66df9d9..2dc2393 100644 --- a/backend/tests/test_url_import.py +++ b/backend/tests/test_url_import.py @@ -10,9 +10,11 @@ """ import copy +import io import json import shutil import socket +import urllib.parse from pathlib import Path import pytest @@ -82,10 +84,16 @@ class FakeYoutubeDL: infos: dict = {} made: list = [] + # (url, impersonating) for every `extract_info` call, in order. + extracted: list = [] # When set, raised from `extract_info` for the matching URL. errors: dict = {} # URLs that answer 403 unless the client impersonates a browser (Cloudflare). browser_only: set = set() + # URLs that answer 403 even to a browser (Cloudflare's JavaScript challenge). + challenged: set = set() + # What `urlopen` answers, by URL; anything else is a 404. + pages: dict = {} # Whether curl_cffi is "installed": without it yt-dlp refuses `impersonate` at # construction, which is the real library's behaviour. can_impersonate: bool = True @@ -102,10 +110,21 @@ def __enter__(self): def __exit__(self, *exc): return False + def close(self): + pass + + def urlopen(self, url): + if url not in FakeYoutubeDL.pages: + raise YoutubeDLError(f"HTTP Error 404: Not Found ({url})") + return io.BytesIO(json.dumps(FakeYoutubeDL.pages[url]).encode()) + def extract_info(self, url, download=False): + FakeYoutubeDL.extracted.append((url, bool(self.params.get("impersonate")))) if url in FakeYoutubeDL.errors: raise FakeYoutubeDL.errors[url] - if url in FakeYoutubeDL.browser_only and not self.params.get("impersonate"): + if url in FakeYoutubeDL.challenged or ( + url in FakeYoutubeDL.browser_only and not self.params.get("impersonate") + ): raise DownloadError( f"ERROR: [Rumble] {url}: Unable to download webpage: HTTP Error 403: " "Forbidden (caused by )" @@ -145,8 +164,11 @@ def extract_info(self, url, download=False): def fake_ytdlp(monkeypatch): FakeYoutubeDL.infos = {VIDEO_URL: video_info()} FakeYoutubeDL.made = [] + FakeYoutubeDL.extracted = [] FakeYoutubeDL.errors = {} FakeYoutubeDL.browser_only = set() + FakeYoutubeDL.challenged = set() + FakeYoutubeDL.pages = {} FakeYoutubeDL.can_impersonate = True monkeypatch.setattr(ytdlp.yt_dlp, "YoutubeDL", FakeYoutubeDL) # Every name resolves somewhere public, so the SSRF guard never touches real DNS. @@ -680,7 +702,7 @@ def test_a_site_that_refuses_python_is_retried_as_a_browser(library, session, mo assert job["status"] == "done", job["error_message"] # Probe and download each tried plainly first, then as a browser. - assert [bool(p.get("impersonate")) for p in FakeYoutubeDL.made] == [False, True, False, True] + assert [browser for _, browser in FakeYoutubeDL.extracted] == [False, True, False, True] def test_a_403_that_impersonation_does_not_cure_says_what_to_do(library, session, monkeypatch): @@ -691,8 +713,8 @@ def test_a_403_that_impersonation_does_not_cure_says_what_to_do(library, session assert job["status"] == "error" assert "HTTP 403" in job["error_message"] - assert "URL_IMPORT_COOKIES_FILE" in job["error_message"] - assert [bool(p.get("impersonate")) for p in FakeYoutubeDL.made] == [False, True] + assert "Cloudflare" in job["error_message"] + assert [browser for _, browser in FakeYoutubeDL.extracted] == [False, True] def test_without_curl_cffi_the_403_is_reported_not_crashed_on(library, session, monkeypatch): @@ -712,6 +734,48 @@ def test_other_failures_are_not_retried_as_a_browser(library, session, monkeypat assert not any(p.get("impersonate") for p in FakeYoutubeDL.made) +RUMBLE_PAGE = "https://rumble.com/v13047p-a-speech.html" +RUMBLE_OEMBED = "https://rumble.com/api/Media/oembed.json?url=" + urllib.parse.quote(RUMBLE_PAGE, safe="") +RUMBLE_EMBED = "https://rumble.com/embed/v10abcd/" + + +def test_a_challenged_rumble_page_is_imported_through_its_embed(library, session, monkeypatch): + """Cloudflare challenges the watch page from some server IPs; the oEmbed API and + the embed player stay open, so the video arrives anyway, filed under its page.""" + FakeYoutubeDL.challenged.add(RUMBLE_PAGE) + FakeYoutubeDL.pages[RUMBLE_OEMBED] = { + "type": "video", + "html": f'', + } + FakeYoutubeDL.infos[RUMBLE_EMBED] = video_info(webpage_url=RUMBLE_EMBED, extractor_key="RumbleEmbed") + job = _activity(library, _import(library, session, monkeypatch, url=RUMBLE_PAGE)) + + assert job["status"] == "done", job["error_message"] + originals = [a for a in session.exec(select(Asset)).all() if a.source_url] + assert [a.source_url for a in originals] == [RUMBLE_PAGE] + # Page plainly, page as a browser, then the embed: for the probe and the download. + assert FakeYoutubeDL.extracted == [ + (RUMBLE_PAGE, False), (RUMBLE_PAGE, True), (RUMBLE_EMBED, True), + ] * 2 + + +def test_a_challenged_rumble_page_without_an_embed_reports_the_challenge(library, session, monkeypatch): + FakeYoutubeDL.challenged.add(RUMBLE_PAGE) + FakeYoutubeDL.pages[RUMBLE_OEMBED] = {"type": "video", "html": ""} + job = _activity(library, _import(library, session, monkeypatch, url=RUMBLE_PAGE)) + + assert job["status"] == "error" + assert "Cloudflare" in job["error_message"] + + +def test_only_rumble_falls_back_to_an_embed(library, session, monkeypatch): + FakeYoutubeDL.challenged.add(VIDEO_URL) + job = _activity(library, _import(library, session, monkeypatch)) + + assert job["status"] == "error" + assert "HTTP 403" in job["error_message"] + + def test_cancelling_mid_download_stops_the_job(library, session, monkeypatch): """The progress hook is the checkpoint: the reporter raises JobCancelled, which the hook turns into the exception yt-dlp is written to let through.""" diff --git a/docs/url-import.md b/docs/url-import.md index a894177..78d08c4 100644 --- a/docs/url-import.md +++ b/docs/url-import.md @@ -94,9 +94,26 @@ Rumble, and other sites behind Cloudflare's bot screen, refuse a plain Python cl its TLS handshake (`Unable to download webpage: HTTP Error 403: Forbidden`) before any header or cookie is read. The `curl-cffi` extra lets yt-dlp handshake like Chrome, and `ytdlp._extract` retries any 403 that way, once. Only on a 403: sites that already work, -YouTube above all, keep yt-dlp's own client. A 403 that survives the retry is a block on -the server's IP or a site that wants a signed-in session; the cookies file below is the -remedy for the second. +YouTube above all, keep yt-dlp's own client. + +A 403 that survives that retry is usually Cloudflare's JavaScript challenge +(`cf-mitigated: challenge`, a "Just a moment…" page), which it sends to IPs it distrusts +— hosting providers' especially. Nothing that is not a real browser answers it, and a +cookies file exported elsewhere does not help: the clearance cookie is bound to the IP +and browser that earned it. For **Rumble** there is a way round: the challenge covers its +watch pages but not its oEmbed API or embed player, so the import asks oEmbed for the +video's embed id and imports `rumble.com/embed//` instead, filed under the watch +page's URL. The embed has the title, channel, date, duration and thumbnail, but no +description or tags. For any other site, download it elsewhere and upload the file. + +To see which case a server is in: + +```bash +docker compose exec backend python -c " +from curl_cffi import requests +r = requests.get('', impersonate='chrome') +print(r.status_code, r.headers.get('cf-mitigated'))" +``` ### Deno From 20ab449b1cefd133edca6b84e397aae96a5eea89 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 7 Nov 2026 03:35:18 +0000 Subject: [PATCH 2/3] Replace the Rumble embed fallback with an import proxy The production server proved the embed route useless: Cloudflare challenges its IP on Rumble's embed data endpoint (embedJS/u3) as well as the watch page, with or without a Referer, leaving only oEmbed open, which carries no video. No request from that IP can import from Rumble. So remove the fallback and add URL_IMPORT_PROXY, an http(s) or SOCKS proxy for imports, with URL_IMPORT_PROXY_HOSTS limiting it to the sites that need it (a name covers its subdomains; empty means every import). It is chosen by the pasted link and set as yt-dlp's proxy for the whole run, so the video files on the site's CDN go the same way. A value that is not a proxy URL stops startup. docs/url-import.md walks through a free proxy: an SSH reverse SOCKS tunnel from a computer at home, bound to Docker's bridge address. The 403 message now points there instead of at a cookies file, which cannot carry Cloudflare's clearance from another machine. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01V1geD4inqJxe1F6HuWV82R --- .env.example | 9 +++ backend/app/config.py | 22 ++++++ backend/app/ingest/ytdlp.py | 129 ++++++++++++------------------- backend/tests/test_url_import.py | 97 ++++++++++------------- docker-compose.yml | 2 + docs/url-import.md | 78 ++++++++++++++++--- 6 files changed, 188 insertions(+), 149 deletions(-) diff --git a/.env.example b/.env.example index c1a046b..5569ebe 100644 --- a/.env.example +++ b/.env.example @@ -95,6 +95,15 @@ URL_IMPORT_MAX_PLAYLIST_ITEMS=200 # is a path on your own machine. The file is a login: treat it like a password. URL_IMPORT_COOKIES_FILE= +# A proxy for imports from sites that refuse this server's IP outright — Cloudflare +# answers many VPS IPs with a JavaScript challenge nothing but a real browser passes +# (Rumble does this). http://, https://, socks4(a):// or socks5(h)://. An SSH tunnel to +# a computer at home makes a free one: docs/url-import.md#when-a-site-blocks-this-servers-ip +URL_IMPORT_PROXY= +# Comma-separated sites the proxy is for (a name covers its subdomains), e.g. rumble.com. +# Empty sends every import through it. +URL_IMPORT_PROXY_HOSTS= + # ─── Administrators ────────────────────────────────────────────────────────── # Who may edit the AI generation model catalogue, and see the server's disk space and diff --git a/backend/app/config.py b/backend/app/config.py index 047da4a..3448083 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -115,6 +115,15 @@ class Settings(BaseSettings): # For YouTube's "confirm you're not a bot" wall on datacenter IPs, and for # age-restricted or members-only videos; mount the file read-only and point here. url_import_cookies_file: str = "" + # A proxy for imports, e.g. socks5h://172.17.0.1:1080 — for sites that refuse this + # server's IP outright. Cloudflare answers some hosting providers' IPs with a + # JavaScript challenge that no importer can pass and no cookie from another machine + # clears (Rumble, Nov 2026); only a request from another IP gets through. + url_import_proxy: str = "" + # Comma-separated sites the proxy is for, e.g. "rumble.com"; a name covers its + # subdomains. Empty sends every import through it. Matched on the link that was + # pasted, and the whole import follows it, video files on a CDN included. + url_import_proxy_hosts: str = "" # ─── administration ────────────────────────────────────────────────────── # Comma-separated user ids or usernames. The Notes token carries no admin claim and @@ -201,6 +210,19 @@ def _reject_weak_secret(cls, value: str) -> str: ) return value + @field_validator("url_import_proxy") + @classmethod + def _known_proxy_scheme(cls, value: str) -> str: + value = value.strip() + scheme = value.split("://", 1)[0].lower() if "://" in value else "" + if value and scheme not in {"http", "https", "socks4", "socks4a", "socks5", "socks5h"}: + # yt-dlp would otherwise fail every import with its own, less obvious error. + raise ValueError( + "URL_IMPORT_PROXY must be a URL such as socks5h://host:1080 or " + "http://host:8080" + ) + return value + @field_validator("ai_backend") @classmethod def _known_ai_backend(cls, value: str) -> str: diff --git a/backend/app/ingest/ytdlp.py b/backend/app/ingest/ytdlp.py index 7b23569..16dd615 100644 --- a/backend/app/ingest/ytdlp.py +++ b/backend/app/ingest/ytdlp.py @@ -21,7 +21,6 @@ from __future__ import annotations -import json import logging import re import shutil @@ -130,7 +129,7 @@ def probe_url(url: str, workdir: Path) -> dict: their own jobs, which resolve them anyway. """ options = { - **_base_options(workdir), + **_base_options(workdir, url), "extract_flat": "in_playlist", "playlist_items": f"1:{max(1, settings.url_import_max_playlist_items)}", } @@ -220,7 +219,7 @@ def download( captions = choose_captions(info) if want_captions else None options: dict[str, Any] = { - **_base_options(workdir), + **_base_options(workdir, url), "paths": {"home": str(workdir), "temp": str(workdir)}, # Fixed names: the title is recorded as metadata, and a filename built from it # is a filename built from text the site controls. @@ -418,7 +417,7 @@ def error(self, msg: str) -> None: logger.info("yt-dlp: %s", msg) -def _base_options(workdir: Path) -> dict[str, Any]: +def _base_options(workdir: Path, url: str) -> dict[str, Any]: options: dict[str, Any] = { "quiet": True, "no_warnings": True, @@ -448,25 +447,45 @@ def _base_options(workdir: Path) -> dict[str, Any]: shutil.copyfile(source, copy) options["cookiefile"] = str(copy) + proxy = proxy_for(url) + if proxy: + options["proxy"] = proxy + return options +def proxy_for(url: str) -> str: + """URL_IMPORT_PROXY when it applies to `url`, else "". + + By the pasted link's host, and for the whole import: yt-dlp's `proxy` covers every + request of the run, so the video files on a site's CDN go the same way as its pages + — which is what a site blocking this server's IP needs, since the CDN usually blocks + it too. + """ + proxy = settings.url_import_proxy.strip() + if not proxy: + return "" + hosts = [h.strip().lower().lstrip(".") for h in settings.url_import_proxy_hosts.split(",")] + hosts = [h for h in hosts if h] + if not hosts: + return proxy + host = (urllib.parse.urlsplit(url).hostname or "").lower() + return proxy if any(host == h or host.endswith(f".{h}") for h in hosts) else "" + + def _extract(options: dict[str, Any], url: str, *, download: bool) -> Optional[dict]: - """`extract_info`, with two fallbacks for a site that answers 403. + """`extract_info`, retried once as a browser when the site answers 403. - First, the same request as a browser. Sites behind Cloudflare's bot screen refuse a + Sites behind Cloudflare's bot screen (Rumble, and others that come and go) refuse a request on its TLS fingerprint before reading a single header, so no user agent or cookie gets past it; only a request that handshakes like Chrome does. yt-dlp can - send one through curl_cffi (the `curl-cffi` extra in requirements.txt). Only on a - 403, and not for every request, because impersonation swaps the HTTP stack under - every site that already works — YouTube's extractor in particular is tuned against - yt-dlp's own client, and a change there would fail on YouTube's schedule. - - Second, for Rumble only, the video's embed instead of its watch page. Cloudflare - sends some server IPs a JavaScript challenge on the watch page that no importer can - answer, while the oEmbed API and the embed player behind it stay open (checked from - a blocked server, Nov 2026). The embed carries less — no description or tags — but - a video with its title beats an error. + send one through curl_cffi (the `curl-cffi` extra in requirements.txt). + + Only on a 403, and not for every request, because impersonation swaps the HTTP + stack under every site that already works — YouTube's extractor in particular is + tuned against yt-dlp's own client, and a change there would fail on YouTube's + schedule rather than ours. A 403 that impersonation does not cure (a geo-block, a + private video) costs one extra page load and reports the same error. """ try: with yt_dlp.YoutubeDL(options) as ydl: @@ -474,67 +493,17 @@ def _extract(options: dict[str, Any], url: str, *, download: bool) -> Optional[d except DownloadError as exc: if "impersonate" in options or not _is_forbidden(exc): raise - failure = exc - - browser = _as_browser(options) - if browser is not None: - logger.info("url import: %s answered 403, retrying as a browser", url) - try: - with yt_dlp.YoutubeDL(browser) as ydl: - return ydl.extract_info(url, download=download) - except DownloadError as exc: - if not _is_forbidden(exc): - raise - failure = exc - - embed = _rumble_embed_url(url, browser or options) - if embed is None: - raise failure - logger.info("url import: %s is challenged, importing its embed %s", url, embed) - with yt_dlp.YoutubeDL(browser or options) as ydl: - info = ydl.extract_info(embed, download=download) - if info: - # The page people know the video by, so the asset's source link and the - # already-imported check both name it rather than the player. - info["webpage_url"] = url - return info + first = exc - -def _as_browser(options: dict[str, Any]) -> Optional[dict[str, Any]]: - """`options` impersonating a browser, or None when curl_cffi is missing (or is a - version yt-dlp does not support): yt-dlp refuses the option at construction.""" - browser = {**options, "impersonate": ImpersonateTarget()} + logger.info("url import: %s answered 403, retrying as a browser", url) try: - yt_dlp.YoutubeDL(browser).close() + ydl = yt_dlp.YoutubeDL({**options, "impersonate": ImpersonateTarget()}) except YoutubeDLError: - return None - return browser - - -_RUMBLE_HOSTS = ("rumble.com", "www.rumble.com") -_RUMBLE_EMBED = re.compile(r"https://rumble\.com/embed/(?:[0-9a-z]+\.)?([0-9a-z]+)") - - -def _rumble_embed_url(url: str, options: dict[str, Any]) -> Optional[str]: - """The embed player URL for a Rumble watch page, from Rumble's oEmbed API. - - The embed id is not the watch page's id, and the page that would map one to the - other is the one being challenged. Fetched - through yt-dlp so the cookies, timeout and impersonation of the import apply. - """ - parts = urllib.parse.urlsplit(url) - if (parts.hostname or "").lower() not in _RUMBLE_HOSTS or parts.path.startswith("/embed/"): - return None - api = "https://rumble.com/api/Media/oembed.json?" + urllib.parse.urlencode({"url": url}) - try: - with yt_dlp.YoutubeDL(options) as ydl: - with ydl.urlopen(api) as response: - data = json.loads(response.read()) - except (YoutubeDLError, OSError, ValueError) as exc: - logger.info("url import: Rumble oEmbed for %s failed: %s", url, exc) - return None - match = _RUMBLE_EMBED.search(str(data.get("html") or "") if isinstance(data, dict) else "") - return f"https://rumble.com/embed/{match.group(1)}/" if match else None + # curl_cffi is not installed (or is a version yt-dlp does not support). The + # original 403 is the truthful answer; readable_error says what to install. + raise first + with ydl: + return ydl.extract_info(url, download=download) def _is_forbidden(exc: Exception) -> bool: @@ -580,13 +549,11 @@ def readable_error(exc: Exception) -> str: if "403" in lowered and "forbidden" in lowered: return ( - "The site refused this server (HTTP 403). " - "Usually that is Cloudflare challenging this server's IP address — a " - "JavaScript check no importer can answer, and one a cookies file from " - "another computer does not pass either, since the clearance is tied to the " - "IP that earned it. Download the video on your own computer and upload it. " - "If the backend image predates curl-cffi in backend/requirements.txt, " - "rebuild it first." + "The site refused this server (HTTP 403). Usually that is Cloudflare " + "challenging this server's IP address, which no importer can answer and a " + "cookies file from another computer does not clear. Send imports from that " + "site through another network with URL_IMPORT_PROXY, or download the video " + "on your own computer and upload it — see docs/url-import.md." ) first_line = message.splitlines()[0] if message else "" diff --git a/backend/tests/test_url_import.py b/backend/tests/test_url_import.py index 2dc2393..9d05431 100644 --- a/backend/tests/test_url_import.py +++ b/backend/tests/test_url_import.py @@ -10,11 +10,9 @@ """ import copy -import io import json import shutil import socket -import urllib.parse from pathlib import Path import pytest @@ -84,16 +82,10 @@ class FakeYoutubeDL: infos: dict = {} made: list = [] - # (url, impersonating) for every `extract_info` call, in order. - extracted: list = [] # When set, raised from `extract_info` for the matching URL. errors: dict = {} # URLs that answer 403 unless the client impersonates a browser (Cloudflare). browser_only: set = set() - # URLs that answer 403 even to a browser (Cloudflare's JavaScript challenge). - challenged: set = set() - # What `urlopen` answers, by URL; anything else is a 404. - pages: dict = {} # Whether curl_cffi is "installed": without it yt-dlp refuses `impersonate` at # construction, which is the real library's behaviour. can_impersonate: bool = True @@ -110,21 +102,10 @@ def __enter__(self): def __exit__(self, *exc): return False - def close(self): - pass - - def urlopen(self, url): - if url not in FakeYoutubeDL.pages: - raise YoutubeDLError(f"HTTP Error 404: Not Found ({url})") - return io.BytesIO(json.dumps(FakeYoutubeDL.pages[url]).encode()) - def extract_info(self, url, download=False): - FakeYoutubeDL.extracted.append((url, bool(self.params.get("impersonate")))) if url in FakeYoutubeDL.errors: raise FakeYoutubeDL.errors[url] - if url in FakeYoutubeDL.challenged or ( - url in FakeYoutubeDL.browser_only and not self.params.get("impersonate") - ): + if url in FakeYoutubeDL.browser_only and not self.params.get("impersonate"): raise DownloadError( f"ERROR: [Rumble] {url}: Unable to download webpage: HTTP Error 403: " "Forbidden (caused by )" @@ -164,11 +145,8 @@ def extract_info(self, url, download=False): def fake_ytdlp(monkeypatch): FakeYoutubeDL.infos = {VIDEO_URL: video_info()} FakeYoutubeDL.made = [] - FakeYoutubeDL.extracted = [] FakeYoutubeDL.errors = {} FakeYoutubeDL.browser_only = set() - FakeYoutubeDL.challenged = set() - FakeYoutubeDL.pages = {} FakeYoutubeDL.can_impersonate = True monkeypatch.setattr(ytdlp.yt_dlp, "YoutubeDL", FakeYoutubeDL) # Every name resolves somewhere public, so the SSRF guard never touches real DNS. @@ -702,7 +680,7 @@ def test_a_site_that_refuses_python_is_retried_as_a_browser(library, session, mo assert job["status"] == "done", job["error_message"] # Probe and download each tried plainly first, then as a browser. - assert [browser for _, browser in FakeYoutubeDL.extracted] == [False, True, False, True] + assert [bool(p.get("impersonate")) for p in FakeYoutubeDL.made] == [False, True, False, True] def test_a_403_that_impersonation_does_not_cure_says_what_to_do(library, session, monkeypatch): @@ -713,8 +691,8 @@ def test_a_403_that_impersonation_does_not_cure_says_what_to_do(library, session assert job["status"] == "error" assert "HTTP 403" in job["error_message"] - assert "Cloudflare" in job["error_message"] - assert [browser for _, browser in FakeYoutubeDL.extracted] == [False, True] + assert "URL_IMPORT_PROXY" in job["error_message"] + assert [bool(p.get("impersonate")) for p in FakeYoutubeDL.made] == [False, True] def test_without_curl_cffi_the_403_is_reported_not_crashed_on(library, session, monkeypatch): @@ -724,7 +702,7 @@ def test_without_curl_cffi_the_403_is_reported_not_crashed_on(library, session, assert job["status"] == "error" assert "HTTP 403" in job["error_message"] - assert "curl-cffi" in job["error_message"] + assert [bool(p.get("impersonate")) for p in FakeYoutubeDL.made] == [False] def test_other_failures_are_not_retried_as_a_browser(library, session, monkeypatch): @@ -734,46 +712,51 @@ def test_other_failures_are_not_retried_as_a_browser(library, session, monkeypat assert not any(p.get("impersonate") for p in FakeYoutubeDL.made) -RUMBLE_PAGE = "https://rumble.com/v13047p-a-speech.html" -RUMBLE_OEMBED = "https://rumble.com/api/Media/oembed.json?url=" + urllib.parse.quote(RUMBLE_PAGE, safe="") -RUMBLE_EMBED = "https://rumble.com/embed/v10abcd/" +RUMBLE_URL = "https://rumble.com/v13047p-a-speech.html" -def test_a_challenged_rumble_page_is_imported_through_its_embed(library, session, monkeypatch): - """Cloudflare challenges the watch page from some server IPs; the oEmbed API and - the embed player stay open, so the video arrives anyway, filed under its page.""" - FakeYoutubeDL.challenged.add(RUMBLE_PAGE) - FakeYoutubeDL.pages[RUMBLE_OEMBED] = { - "type": "video", - "html": f'', - } - FakeYoutubeDL.infos[RUMBLE_EMBED] = video_info(webpage_url=RUMBLE_EMBED, extractor_key="RumbleEmbed") - job = _activity(library, _import(library, session, monkeypatch, url=RUMBLE_PAGE)) +@pytest.mark.parametrize( + "hosts, url, proxied", + [ + ("", VIDEO_URL, True), # no list: every import + ("rumble.com", RUMBLE_URL, True), + ("rumble.com", "https://www.rumble.com/v1-x.html", True), # subdomains too + ("rumble.com", VIDEO_URL, False), + ("rumble.com", "https://notrumble.com/v1.html", False), # not a suffix match + (" Rumble.com , odysee.com ", RUMBLE_URL, True), + ], +) +def test_the_import_proxy_applies_to_the_sites_it_names( + library, session, monkeypatch, hosts, url, proxied +): + from app.config import settings + + monkeypatch.setattr(settings, "url_import_proxy", "socks5h://172.17.0.1:1080") + monkeypatch.setattr(settings, "url_import_proxy_hosts", hosts) + FakeYoutubeDL.infos[url] = video_info(webpage_url=url) + job = _activity(library, _import(library, session, monkeypatch, url=url)) assert job["status"] == "done", job["error_message"] - originals = [a for a in session.exec(select(Asset)).all() if a.source_url] - assert [a.source_url for a in originals] == [RUMBLE_PAGE] - # Page plainly, page as a browser, then the embed: for the probe and the download. - assert FakeYoutubeDL.extracted == [ - (RUMBLE_PAGE, False), (RUMBLE_PAGE, True), (RUMBLE_EMBED, True), - ] * 2 + expected = "socks5h://172.17.0.1:1080" if proxied else None + # Probe and download alike: a site that blocks the IP blocks its CDN too. + assert [p.get("proxy") for p in FakeYoutubeDL.made] == [expected, expected] -def test_a_challenged_rumble_page_without_an_embed_reports_the_challenge(library, session, monkeypatch): - FakeYoutubeDL.challenged.add(RUMBLE_PAGE) - FakeYoutubeDL.pages[RUMBLE_OEMBED] = {"type": "video", "html": ""} - job = _activity(library, _import(library, session, monkeypatch, url=RUMBLE_PAGE)) +def test_no_proxy_is_used_when_none_is_set(library, session, monkeypatch): + _import(library, session, monkeypatch) - assert job["status"] == "error" - assert "Cloudflare" in job["error_message"] + assert not any("proxy" in p for p in FakeYoutubeDL.made) -def test_only_rumble_falls_back_to_an_embed(library, session, monkeypatch): - FakeYoutubeDL.challenged.add(VIDEO_URL) - job = _activity(library, _import(library, session, monkeypatch)) +@pytest.mark.parametrize("value", ["172.17.0.1:1080", "ftp://proxy:21", "socks://x"]) +def test_a_proxy_that_is_not_a_proxy_url_stops_startup(monkeypatch, value): + from pydantic import ValidationError - assert job["status"] == "error" - assert "HTTP 403" in job["error_message"] + from app.config import Settings + + monkeypatch.setenv("URL_IMPORT_PROXY", value) + with pytest.raises(ValidationError, match="URL_IMPORT_PROXY"): + Settings() def test_cancelling_mid_download_stops_the_job(library, session, monkeypatch): diff --git a/docker-compose.yml b/docker-compose.yml index 39319e0..0d3ac82 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,6 +36,8 @@ services: - URL_IMPORT_MAX_HEIGHT=${URL_IMPORT_MAX_HEIGHT:-1080} - URL_IMPORT_MAX_PLAYLIST_ITEMS=${URL_IMPORT_MAX_PLAYLIST_ITEMS:-200} - URL_IMPORT_COOKIES_FILE=${URL_IMPORT_COOKIES_FILE:-} + - URL_IMPORT_PROXY=${URL_IMPORT_PROXY:-} + - URL_IMPORT_PROXY_HOSTS=${URL_IMPORT_PROXY_HOSTS:-} - ADMIN_USERS=${ADMIN_USERS:-} - GENERATION_CONCURRENCY=${GENERATION_CONCURRENCY:-2} - GENERATION_TIMEOUT_MINUTES=${GENERATION_TIMEOUT_MINUTES:-30} diff --git a/docs/url-import.md b/docs/url-import.md index 78d08c4..e040111 100644 --- a/docs/url-import.md +++ b/docs/url-import.md @@ -96,17 +96,12 @@ header or cookie is read. The `curl-cffi` extra lets yt-dlp handshake like Chrom `ytdlp._extract` retries any 403 that way, once. Only on a 403: sites that already work, YouTube above all, keep yt-dlp's own client. -A 403 that survives that retry is usually Cloudflare's JavaScript challenge -(`cf-mitigated: challenge`, a "Just a moment…" page), which it sends to IPs it distrusts -— hosting providers' especially. Nothing that is not a real browser answers it, and a -cookies file exported elsewhere does not help: the clearance cookie is bound to the IP -and browser that earned it. For **Rumble** there is a way round: the challenge covers its -watch pages but not its oEmbed API or embed player, so the import asks oEmbed for the -video's embed id and imports `rumble.com/embed//` instead, filed under the watch -page's URL. The embed has the title, channel, date, duration and thumbnail, but no -description or tags. For any other site, download it elsewhere and upload the file. - -To see which case a server is in: +A 403 that survives the retry is usually Cloudflare's JavaScript challenge, which it +sends to IPs it distrusts, hosting providers' especially. See the next section. + +### When a site blocks this server's IP + +Check from the server what the site answers: ```bash docker compose exec backend python -c " @@ -115,6 +110,65 @@ r = requests.get('', impersonate='chrome') print(r.status_code, r.headers.get('cf-mitigated'))" ``` +`403 challenge` is Cloudflare's JavaScript check ("Just a moment…"). Nothing but a real +browser answers it, and a cookies file does not carry the clearance across: the cookie is +bound to the IP and browser that earned it. Rumble does this to whole datacenter ranges, +and on every path of rumble.com that leads to a video — the watch page, the embed +player's data endpoint — leaving only oEmbed open, which carries no video (checked from a +blocked server, Nov 2026). + +What gets through is a request from another IP. `URL_IMPORT_PROXY` sends imports through +a proxy, and `URL_IMPORT_PROXY_HOSTS` limits it to the sites that need it. The whole +import goes through it, the video files on the site's CDN included. + +**A free proxy: an SSH tunnel to a computer at home.** OpenSSH (7.6 or newer) can turn +an SSH connection into a SOCKS proxy on the far end, so the server's requests leave +through your home connection, whose IP Cloudflare trusts. It works while that computer is +on and the tunnel is up; for always-on, use a residential proxy service or a VPN +container on the server instead, and set the same two variables. + +1. On the server, let a tunnel bind to Docker's bridge address rather than only to + loopback, which containers cannot reach. In `/etc/ssh/sshd_config`: + + ``` + GatewayPorts clientspecified + ``` + + then `systemctl reload ssh`. `ip -4 addr show docker0` shows the bridge address — + `172.17.0.1` unless Docker was configured otherwise. Nothing outside the machine can + reach that address. + +2. On the home computer, open the tunnel (`-N`: no shell; a `-R` with no destination is a + SOCKS proxy): + + ```bash + ssh -N -R 172.17.0.1:1080 root@ + ``` + + To keep it up across drops and reboots, run it under `autossh` or a systemd user unit. + +3. In the server's `.env`: + + ``` + URL_IMPORT_PROXY=socks5h://172.17.0.1:1080 + URL_IMPORT_PROXY_HOSTS=rumble.com + ``` + + then `docker compose up -d` (a restart, not a rebuild). `socks5h` resolves names at the + far end too. + +4. Check it from inside the container: + + ```bash + docker compose exec backend curl -s -o /dev/null -w '%{http_code}\n' \ + -x socks5h://172.17.0.1:1080 https://rumble.com/ + ``` + + `200` is the tunnel working. A timeout is usually the host firewall dropping traffic + from containers to the host (with ufw: `ufw allow in on docker0 to any port 1080`, + and the same for the compose network's `br-…` interface if that is where it comes + from). + ### Deno Since late 2025 YouTube serves almost no downloadable formats to a client that cannot run @@ -269,6 +323,8 @@ size free while that happens. | `URL_IMPORT_MAX_HEIGHT` | `1080` | Tallest video fetched. | | `URL_IMPORT_MAX_PLAYLIST_ITEMS` | `200` | Most videos one playlist or channel link may queue. | | `URL_IMPORT_COOKIES_FILE` | empty | Set for you by `docker-compose.cookies.yml` — see [YouTube cookies](#youtube-cookies). | +| `URL_IMPORT_PROXY` | empty | Proxy for imports (`socks5h://…`, `http://…`) — see [When a site blocks this server's IP](#when-a-site-blocks-this-servers-ip). | +| `URL_IMPORT_PROXY_HOSTS` | empty | Sites the proxy is for, comma-separated; empty means every import. | --- From f371242bf263c2d8d0edccf5cfb5f989c5009103 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 7 Nov 2026 04:12:40 +0000 Subject: [PATCH 3/3] Drop the import proxy; end a blocked import with a command to run at home Decided against the proxy: an SSH tunnel home exposes the home network to anything on the server, and a residential proxy is a subscription for an occasional video. Rumble videos will be downloaded locally and uploaded. So the 403 that survives the browser retry now says what it usually is, Cloudflare blocking this server's IP, and ends with the yt-dlp command that fetches the same link the way an import would: --embed-metadata so the upload reads back the channel, date and link, the title as filename, the same height cap and MP4 container. The link is percent-encoded for anything a shell would act on inside double quotes, so a pasted URL cannot run anything when the command is pasted into a terminal. The activity panel renders a line starting `yt-dlp ` as code with a copy button, since a command is only useful whole and exact. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01V1geD4inqJxe1F6HuWV82R --- .env.example | 9 --- backend/app/config.py | 22 ------ backend/app/ingest/ytdlp.py | 70 +++++++++-------- backend/tests/test_url_import.py | 62 ++++++--------- docker-compose.yml | 2 - docs/url-import.md | 78 ++++--------------- .../src/components/ActivityIndicator.test.tsx | 32 ++++++++ frontend/src/components/ActivityIndicator.tsx | 72 +++++++++++++++-- 8 files changed, 168 insertions(+), 179 deletions(-) diff --git a/.env.example b/.env.example index 5569ebe..c1a046b 100644 --- a/.env.example +++ b/.env.example @@ -95,15 +95,6 @@ URL_IMPORT_MAX_PLAYLIST_ITEMS=200 # is a path on your own machine. The file is a login: treat it like a password. URL_IMPORT_COOKIES_FILE= -# A proxy for imports from sites that refuse this server's IP outright — Cloudflare -# answers many VPS IPs with a JavaScript challenge nothing but a real browser passes -# (Rumble does this). http://, https://, socks4(a):// or socks5(h)://. An SSH tunnel to -# a computer at home makes a free one: docs/url-import.md#when-a-site-blocks-this-servers-ip -URL_IMPORT_PROXY= -# Comma-separated sites the proxy is for (a name covers its subdomains), e.g. rumble.com. -# Empty sends every import through it. -URL_IMPORT_PROXY_HOSTS= - # ─── Administrators ────────────────────────────────────────────────────────── # Who may edit the AI generation model catalogue, and see the server's disk space and diff --git a/backend/app/config.py b/backend/app/config.py index 3448083..047da4a 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -115,15 +115,6 @@ class Settings(BaseSettings): # For YouTube's "confirm you're not a bot" wall on datacenter IPs, and for # age-restricted or members-only videos; mount the file read-only and point here. url_import_cookies_file: str = "" - # A proxy for imports, e.g. socks5h://172.17.0.1:1080 — for sites that refuse this - # server's IP outright. Cloudflare answers some hosting providers' IPs with a - # JavaScript challenge that no importer can pass and no cookie from another machine - # clears (Rumble, Nov 2026); only a request from another IP gets through. - url_import_proxy: str = "" - # Comma-separated sites the proxy is for, e.g. "rumble.com"; a name covers its - # subdomains. Empty sends every import through it. Matched on the link that was - # pasted, and the whole import follows it, video files on a CDN included. - url_import_proxy_hosts: str = "" # ─── administration ────────────────────────────────────────────────────── # Comma-separated user ids or usernames. The Notes token carries no admin claim and @@ -210,19 +201,6 @@ def _reject_weak_secret(cls, value: str) -> str: ) return value - @field_validator("url_import_proxy") - @classmethod - def _known_proxy_scheme(cls, value: str) -> str: - value = value.strip() - scheme = value.split("://", 1)[0].lower() if "://" in value else "" - if value and scheme not in {"http", "https", "socks4", "socks4a", "socks5", "socks5h"}: - # yt-dlp would otherwise fail every import with its own, less obvious error. - raise ValueError( - "URL_IMPORT_PROXY must be a URL such as socks5h://host:1080 or " - "http://host:8080" - ) - return value - @field_validator("ai_backend") @classmethod def _known_ai_backend(cls, value: str) -> str: diff --git a/backend/app/ingest/ytdlp.py b/backend/app/ingest/ytdlp.py index 16dd615..2579766 100644 --- a/backend/app/ingest/ytdlp.py +++ b/backend/app/ingest/ytdlp.py @@ -129,14 +129,14 @@ def probe_url(url: str, workdir: Path) -> dict: their own jobs, which resolve them anyway. """ options = { - **_base_options(workdir, url), + **_base_options(workdir), "extract_flat": "in_playlist", "playlist_items": f"1:{max(1, settings.url_import_max_playlist_items)}", } try: info = _extract(options, url, download=False) except DownloadError as exc: - raise UrlImportError(readable_error(exc)) from exc + raise UrlImportError(readable_error(exc, url)) from exc if not info: raise UrlImportError("Nothing importable was found at that link") return info @@ -219,7 +219,7 @@ def download( captions = choose_captions(info) if want_captions else None options: dict[str, Any] = { - **_base_options(workdir, url), + **_base_options(workdir), "paths": {"home": str(workdir), "temp": str(workdir)}, # Fixed names: the title is recorded as metadata, and a filename built from it # is a filename built from text the site controls. @@ -270,7 +270,7 @@ def download( except DownloadCancelled as exc: raise JobCancelled(str(exc)) from exc except DownloadError as exc: - raise UrlImportError(readable_error(exc)) from exc + raise UrlImportError(readable_error(exc, url)) from exc media = _media_path(result, workdir) return Downloaded( @@ -417,7 +417,7 @@ def error(self, msg: str) -> None: logger.info("yt-dlp: %s", msg) -def _base_options(workdir: Path, url: str) -> dict[str, Any]: +def _base_options(workdir: Path) -> dict[str, Any]: options: dict[str, Any] = { "quiet": True, "no_warnings": True, @@ -447,32 +447,9 @@ def _base_options(workdir: Path, url: str) -> dict[str, Any]: shutil.copyfile(source, copy) options["cookiefile"] = str(copy) - proxy = proxy_for(url) - if proxy: - options["proxy"] = proxy - return options -def proxy_for(url: str) -> str: - """URL_IMPORT_PROXY when it applies to `url`, else "". - - By the pasted link's host, and for the whole import: yt-dlp's `proxy` covers every - request of the run, so the video files on a site's CDN go the same way as its pages - — which is what a site blocking this server's IP needs, since the CDN usually blocks - it too. - """ - proxy = settings.url_import_proxy.strip() - if not proxy: - return "" - hosts = [h.strip().lower().lstrip(".") for h in settings.url_import_proxy_hosts.split(",")] - hosts = [h for h in hosts if h] - if not hosts: - return proxy - host = (urllib.parse.urlsplit(url).hostname or "").lower() - return proxy if any(host == h or host.endswith(f".{h}") for h in hosts) else "" - - def _extract(options: dict[str, Any], url: str, *, download: bool) -> Optional[dict]: """`extract_info`, retried once as a browser when the site answers 403. @@ -526,7 +503,7 @@ def _is_forbidden(exc: Exception) -> bool: ) -def readable_error(exc: Exception) -> str: +def readable_error(exc: Exception, url: str = "") -> str: """A yt-dlp failure as something a person can act on. yt-dlp's messages are written for a terminal user — `ERROR: [youtube] abc123: …` — @@ -549,11 +526,11 @@ def readable_error(exc: Exception) -> str: if "403" in lowered and "forbidden" in lowered: return ( - "The site refused this server (HTTP 403). Usually that is Cloudflare " - "challenging this server's IP address, which no importer can answer and a " - "cookies file from another computer does not clear. Send imports from that " - "site through another network with URL_IMPORT_PROXY, or download the video " - "on your own computer and upload it — see docs/url-import.md." + "The site refused this server (HTTP 403). " + "Usually that is Cloudflare blocking this server's IP address, which no " + "importer can get past from here. Download it on your own computer and " + "upload the file:" + + (f"\n{local_download_command(url)}" if url else "") ) first_line = message.splitlines()[0] if message else "" @@ -566,6 +543,31 @@ def readable_error(exc: Exception) -> str: return cleaned +# Characters left as they are when a URL goes into a pasted command. Everything else — +# quotes, `$`, backticks, `!`, `\`, spaces — is percent-encoded, which leaves the link +# meaning the same thing and leaves nothing for a shell to expand inside double quotes. +# Double rather than single quotes because cmd.exe has no single-quoting at all; bash, +# zsh and PowerShell take either. +_COMMAND_SAFE = ":/?#[]@&=+,;%-._~*()'" + + +def local_download_command(url: str) -> str: + """The yt-dlp command that fetches `url` the way an import would, to run elsewhere. + + For a site that blocks this server but not a home connection. `--embed-metadata` + writes the channel, date and link into the file, which the upload path reads back + (`embedded_metadata.py`); the output name is the title, which an upload takes as the + asset's name. Same height cap and container as an import. Needs ffmpeg, as any + yt-dlp merge does. + """ + quoted = urllib.parse.quote(url, safe=_COMMAND_SAFE) + height = max(144, settings.url_import_max_height) + return ( + f'yt-dlp --embed-metadata -f "bv*[height<={height}]+ba/b" ' + f'--merge-output-format mp4 -o "%(title)s.%(ext)s" "{quoted}"' + ) + + # ─── mapping what the site said ────────────────────────────────────────────── diff --git a/backend/tests/test_url_import.py b/backend/tests/test_url_import.py index 9d05431..0f605b5 100644 --- a/backend/tests/test_url_import.py +++ b/backend/tests/test_url_import.py @@ -13,6 +13,7 @@ import json import shutil import socket +import urllib.parse from pathlib import Path import pytest @@ -691,7 +692,10 @@ def test_a_403_that_impersonation_does_not_cure_says_what_to_do(library, session assert job["status"] == "error" assert "HTTP 403" in job["error_message"] - assert "URL_IMPORT_PROXY" in job["error_message"] + # The last line is a command to run at home, for this link. + command = job["error_message"].splitlines()[-1] + assert command.startswith("yt-dlp --embed-metadata ") + assert command.endswith(f'"{VIDEO_URL}"') assert [bool(p.get("impersonate")) for p in FakeYoutubeDL.made] == [False, True] @@ -712,51 +716,29 @@ def test_other_failures_are_not_retried_as_a_browser(library, session, monkeypat assert not any(p.get("impersonate") for p in FakeYoutubeDL.made) -RUMBLE_URL = "https://rumble.com/v13047p-a-speech.html" - - -@pytest.mark.parametrize( - "hosts, url, proxied", - [ - ("", VIDEO_URL, True), # no list: every import - ("rumble.com", RUMBLE_URL, True), - ("rumble.com", "https://www.rumble.com/v1-x.html", True), # subdomains too - ("rumble.com", VIDEO_URL, False), - ("rumble.com", "https://notrumble.com/v1.html", False), # not a suffix match - (" Rumble.com , odysee.com ", RUMBLE_URL, True), - ], -) -def test_the_import_proxy_applies_to_the_sites_it_names( - library, session, monkeypatch, hosts, url, proxied -): +def test_the_local_command_fetches_what_an_import_would(monkeypatch): from app.config import settings - monkeypatch.setattr(settings, "url_import_proxy", "socks5h://172.17.0.1:1080") - monkeypatch.setattr(settings, "url_import_proxy_hosts", hosts) - FakeYoutubeDL.infos[url] = video_info(webpage_url=url) - job = _activity(library, _import(library, session, monkeypatch, url=url)) - - assert job["status"] == "done", job["error_message"] - expected = "socks5h://172.17.0.1:1080" if proxied else None - # Probe and download alike: a site that blocks the IP blocks its CDN too. - assert [p.get("proxy") for p in FakeYoutubeDL.made] == [expected, expected] - - -def test_no_proxy_is_used_when_none_is_set(library, session, monkeypatch): - _import(library, session, monkeypatch) - - assert not any("proxy" in p for p in FakeYoutubeDL.made) + monkeypatch.setattr(settings, "url_import_max_height", 720) + command = ytdlp.local_download_command("https://rumble.com/v13047p-a-speech.html") + assert command == ( + 'yt-dlp --embed-metadata -f "bv*[height<=720]+ba/b" --merge-output-format mp4 ' + '-o "%(title)s.%(ext)s" "https://rumble.com/v13047p-a-speech.html"' + ) -@pytest.mark.parametrize("value", ["172.17.0.1:1080", "ftp://proxy:21", "socks://x"]) -def test_a_proxy_that_is_not_a_proxy_url_stops_startup(monkeypatch, value): - from pydantic import ValidationError - from app.config import Settings +def test_the_local_command_leaves_a_shell_nothing_to_expand(): + """The link is whatever was pasted. Inside the command's double quotes, a `"`, `$`, + backtick, `!` or `\\` would end the quoting or run something; encoded, each is still + the same link to the site.""" + hostile = 'https://rumble.com/v1-x.html?a="b"&c=$(touch pwned)&d=`id`&e=!!&f=\\' + command = ytdlp.local_download_command(hostile) + quoted = command.rsplit(" ", 1)[-1] - monkeypatch.setenv("URL_IMPORT_PROXY", value) - with pytest.raises(ValidationError, match="URL_IMPORT_PROXY"): - Settings() + assert quoted.startswith('"') and quoted.endswith('"') + assert not any(ch in quoted[1:-1] for ch in '"$`!\\ ') + assert urllib.parse.unquote(quoted[1:-1]) == hostile def test_cancelling_mid_download_stops_the_job(library, session, monkeypatch): diff --git a/docker-compose.yml b/docker-compose.yml index 0d3ac82..39319e0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,8 +36,6 @@ services: - URL_IMPORT_MAX_HEIGHT=${URL_IMPORT_MAX_HEIGHT:-1080} - URL_IMPORT_MAX_PLAYLIST_ITEMS=${URL_IMPORT_MAX_PLAYLIST_ITEMS:-200} - URL_IMPORT_COOKIES_FILE=${URL_IMPORT_COOKIES_FILE:-} - - URL_IMPORT_PROXY=${URL_IMPORT_PROXY:-} - - URL_IMPORT_PROXY_HOSTS=${URL_IMPORT_PROXY_HOSTS:-} - ADMIN_USERS=${ADMIN_USERS:-} - GENERATION_CONCURRENCY=${GENERATION_CONCURRENCY:-2} - GENERATION_TIMEOUT_MINUTES=${GENERATION_TIMEOUT_MINUTES:-30} diff --git a/docs/url-import.md b/docs/url-import.md index e040111..278f6af 100644 --- a/docs/url-import.md +++ b/docs/url-import.md @@ -97,11 +97,11 @@ header or cookie is read. The `curl-cffi` extra lets yt-dlp handshake like Chrom YouTube above all, keep yt-dlp's own client. A 403 that survives the retry is usually Cloudflare's JavaScript challenge, which it -sends to IPs it distrusts, hosting providers' especially. See the next section. - -### When a site blocks this server's IP - -Check from the server what the site answers: +sends to IPs it distrusts — hosting providers' especially. Rumble does this to whole +datacenter ranges, on every path that leads to a video (checked from a blocked server, +Nov 2026). Nothing but a real browser answers it, and a cookies file exported elsewhere +does not help: the clearance cookie is bound to the IP and browser that earned it. To +check what a site answers this server: ```bash docker compose exec backend python -c " @@ -110,64 +110,16 @@ r = requests.get('', impersonate='chrome') print(r.status_code, r.headers.get('cf-mitigated'))" ``` -`403 challenge` is Cloudflare's JavaScript check ("Just a moment…"). Nothing but a real -browser answers it, and a cookies file does not carry the clearance across: the cookie is -bound to the IP and browser that earned it. Rumble does this to whole datacenter ranges, -and on every path of rumble.com that leads to a video — the watch page, the embed -player's data endpoint — leaving only oEmbed open, which carries no video (checked from a -blocked server, Nov 2026). - -What gets through is a request from another IP. `URL_IMPORT_PROXY` sends imports through -a proxy, and `URL_IMPORT_PROXY_HOSTS` limits it to the sites that need it. The whole -import goes through it, the video files on the site's CDN included. - -**A free proxy: an SSH tunnel to a computer at home.** OpenSSH (7.6 or newer) can turn -an SSH connection into a SOCKS proxy on the far end, so the server's requests leave -through your home connection, whose IP Cloudflare trusts. It works while that computer is -on and the tunnel is up; for always-on, use a residential proxy service or a VPN -container on the server instead, and set the same two variables. - -1. On the server, let a tunnel bind to Docker's bridge address rather than only to - loopback, which containers cannot reach. In `/etc/ssh/sshd_config`: - - ``` - GatewayPorts clientspecified - ``` - - then `systemctl reload ssh`. `ip -4 addr show docker0` shows the bridge address — - `172.17.0.1` unless Docker was configured otherwise. Nothing outside the machine can - reach that address. - -2. On the home computer, open the tunnel (`-N`: no shell; a `-R` with no destination is a - SOCKS proxy): - - ```bash - ssh -N -R 172.17.0.1:1080 root@ - ``` - - To keep it up across drops and reboots, run it under `autossh` or a systemd user unit. - -3. In the server's `.env`: - - ``` - URL_IMPORT_PROXY=socks5h://172.17.0.1:1080 - URL_IMPORT_PROXY_HOSTS=rumble.com - ``` - - then `docker compose up -d` (a restart, not a rebuild). `socks5h` resolves names at the - far end too. - -4. Check it from inside the container: - - ```bash - docker compose exec backend curl -s -o /dev/null -w '%{http_code}\n' \ - -x socks5h://172.17.0.1:1080 https://rumble.com/ - ``` +`403 challenge` is that. The import's error then ends with a yt-dlp command for the +link (`ytdlp.local_download_command`), shown in the activity panel with a copy button: run +it on a computer whose connection the site accepts (it needs yt-dlp and ffmpeg) and +upload the file. `--embed-metadata` writes the channel, date and link into the file, +which the upload reads back; the filename is the title, which becomes the asset's name. +The description, tags and chapters do not come along. - `200` is the tunnel working. A timeout is usually the host firewall dropping traffic - from containers to the host (with ufw: `ufw allow in on docker0 to any port 1080`, - and the same for the compose network's `br-…` interface if that is where it comes - from). +Routing imports through another network instead (a proxy, or an SSH tunnel home) was +written and set aside in davior/gam#75 (see its history): a tunnel exposes the home network to +anything on the server, and a residential proxy is a subscription for an occasional video. ### Deno @@ -323,8 +275,6 @@ size free while that happens. | `URL_IMPORT_MAX_HEIGHT` | `1080` | Tallest video fetched. | | `URL_IMPORT_MAX_PLAYLIST_ITEMS` | `200` | Most videos one playlist or channel link may queue. | | `URL_IMPORT_COOKIES_FILE` | empty | Set for you by `docker-compose.cookies.yml` — see [YouTube cookies](#youtube-cookies). | -| `URL_IMPORT_PROXY` | empty | Proxy for imports (`socks5h://…`, `http://…`) — see [When a site blocks this server's IP](#when-a-site-blocks-this-servers-ip). | -| `URL_IMPORT_PROXY_HOSTS` | empty | Sites the proxy is for, comma-separated; empty means every import. | --- diff --git a/frontend/src/components/ActivityIndicator.test.tsx b/frontend/src/components/ActivityIndicator.test.tsx index a541f22..24235e8 100644 --- a/frontend/src/components/ActivityIndicator.test.tsx +++ b/frontend/src/components/ActivityIndicator.test.tsx @@ -116,4 +116,36 @@ describe('ActivityIndicator', () => { expect(screen.queryByRole('button', { name: /cancel/i })).not.toBeInTheDocument() }) + + it('sets a command in an error apart, to be copied', async () => { + // Spied rather than replaced: userEvent.setup() installs its own clipboard stub. + const user = userEvent.setup() + const writeText = vi.spyOn(navigator.clipboard, 'writeText').mockResolvedValue() + const command = + 'yt-dlp --embed-metadata -f "bv*[height<=1080]+ba/b" --merge-output-format mp4 ' + + '-o "%(title)s.%(ext)s" "https://rumble.com/v1-x.html"' + renderWith([ + job({ + action: 'import_url', + status: 'error', + error_message: `The site refused this server (HTTP 403). Upload the file:\n${command}`, + }), + ]) + + await user.click(screen.getByRole('button', { name: 'Background activity' })) + expect(screen.getByText(command).tagName).toBe('CODE') + expect(screen.getByText(/refused this server/)).not.toHaveTextContent('yt-dlp') + + await user.click(screen.getByRole('button', { name: 'Copy this command' })) + expect(writeText).toHaveBeenCalledWith(command) + }) + + it('shows an error with no command as it is', async () => { + const user = userEvent.setup() + renderWith([job({ status: 'error', error_message: 'No Deepgram key' })]) + + await user.click(screen.getByRole('button', { name: 'Background activity' })) + expect(screen.getByText('No Deepgram key')).toBeInTheDocument() + expect(screen.queryByRole('button', { name: 'Copy this command' })).toBeNull() + }) }) diff --git a/frontend/src/components/ActivityIndicator.tsx b/frontend/src/components/ActivityIndicator.tsx index da36266..bc1b655 100644 --- a/frontend/src/components/ActivityIndicator.tsx +++ b/frontend/src/components/ActivityIndicator.tsx @@ -1,5 +1,5 @@ import { useEffect, useRef, useState } from 'react' -import { Activity, X } from 'lucide-react' +import { Activity, Check, Copy, X } from 'lucide-react' import type { ActivityJob } from '@/api/transcripts' import { isActive, useActivityStore } from '@/stores/activity' @@ -33,6 +33,60 @@ function subject(job: ActivityJob): string { return job.asset_name || 'Your whole library' } +/** + * A job's error, with any command in it set apart to be copied. + * + * An import from a site that blocks the server ends with the yt-dlp command that fetches + * the same video at home (ytdlp.local_download_command). It is only useful whole and + * exact, which a line of 12px prose in a narrow card is not, so a line that starts + * `yt-dlp ` renders as code with a copy button. Plain text in, so the message stays a + * sentence anywhere else it is shown. + */ +function JobError({ message }: { message: string }) { + const lines = message.split('\n') + const command = lines.find((line) => line.startsWith('yt-dlp ')) + const prose = lines.filter((line) => line !== command).join(' ') + const [copied, setCopied] = useState(false) + + const copy = async () => { + if (!command) return + try { + await navigator.clipboard.writeText(command) + setCopied(true) + setTimeout(() => setCopied(false), 1500) + } catch { + // Clipboard access needs a secure context. Prompting with the text beats a button + // that silently does nothing. + window.prompt('Copy this command', command) + } + } + + return ( + <> + {prose} + {command && ( + + + {command} + + + + )} + + ) +} + export default function ActivityIndicator() { const jobs = useActivityStore((s) => s.jobs) const start = useActivityStore((s) => s.start) @@ -125,14 +179,16 @@ export default function ActivityIndicator() { )} -

- {job.stalled - ? 'Not responding' - : job.status === 'error' - ? (job.error_message ?? 'Failed') - : job.detail || job.stage || job.status} +

+ {job.stalled ? ( + 'Not responding' + ) : job.status === 'error' ? ( + + ) : ( + job.detail || job.stage || job.status + )} {isActive(job) && job.progress > 0 && ` · ${job.progress}%`} -

+
))}