diff --git a/backend/app/ingest/ytdlp.py b/backend/app/ingest/ytdlp.py index 8fd3eb1..2579766 100644 --- a/backend/app/ingest/ytdlp.py +++ b/backend/app/ingest/ytdlp.py @@ -136,7 +136,7 @@ def probe_url(url: str, workdir: Path) -> dict: try: info = _extract(options, url, download=False) except DownloadError as exc: - raise UrlImportError(readable_error(exc)) from exc + raise UrlImportError(readable_error(exc, url)) from exc if not info: raise UrlImportError("Nothing importable was found at that link") return info @@ -270,7 +270,7 @@ def download( except DownloadCancelled as exc: raise JobCancelled(str(exc)) from exc except DownloadError as exc: - raise UrlImportError(readable_error(exc)) from exc + raise UrlImportError(readable_error(exc, url)) from exc media = _media_path(result, workdir) return Downloaded( @@ -503,7 +503,7 @@ def _is_forbidden(exc: Exception) -> bool: ) -def readable_error(exc: Exception) -> str: +def readable_error(exc: Exception, url: str = "") -> str: """A yt-dlp failure as something a person can act on. yt-dlp's messages are written for a terminal user — `ERROR: [youtube] abc123: …` — @@ -526,11 +526,11 @@ def readable_error(exc: Exception) -> str: if "403" in lowered and "forbidden" in lowered: return ( - "The site refused this server (HTTP 403). If the backend image predates " - "curl-cffi in backend/requirements.txt, rebuild it so the import can retry " - "as a browser. Otherwise the site blocks this server's IP or wants a " - "signed-in session: export your browser's cookies for it to a file and set " - "URL_IMPORT_COOKIES_FILE — see docs/url-import.md." + "The site refused this server (HTTP 403). " + "Usually that is Cloudflare blocking this server's IP address, which no " + "importer can get past from here. Download it on your own computer and " + "upload the file:" + + (f"\n{local_download_command(url)}" if url else "") ) first_line = message.splitlines()[0] if message else "" @@ -543,6 +543,31 @@ def readable_error(exc: Exception) -> str: return cleaned +# Characters left as they are when a URL goes into a pasted command. Everything else — +# quotes, `$`, backticks, `!`, `\`, spaces — is percent-encoded, which leaves the link +# meaning the same thing and leaves nothing for a shell to expand inside double quotes. +# Double rather than single quotes because cmd.exe has no single-quoting at all; bash, +# zsh and PowerShell take either. +_COMMAND_SAFE = ":/?#[]@&=+,;%-._~*()'" + + +def local_download_command(url: str) -> str: + """The yt-dlp command that fetches `url` the way an import would, to run elsewhere. + + For a site that blocks this server but not a home connection. `--embed-metadata` + writes the channel, date and link into the file, which the upload path reads back + (`embedded_metadata.py`); the output name is the title, which an upload takes as the + asset's name. Same height cap and container as an import. Needs ffmpeg, as any + yt-dlp merge does. + """ + quoted = urllib.parse.quote(url, safe=_COMMAND_SAFE) + height = max(144, settings.url_import_max_height) + return ( + f'yt-dlp --embed-metadata -f "bv*[height<={height}]+ba/b" ' + f'--merge-output-format mp4 -o "%(title)s.%(ext)s" "{quoted}"' + ) + + # ─── mapping what the site said ────────────────────────────────────────────── diff --git a/backend/tests/test_url_import.py b/backend/tests/test_url_import.py index 66df9d9..0f605b5 100644 --- a/backend/tests/test_url_import.py +++ b/backend/tests/test_url_import.py @@ -13,6 +13,7 @@ import json import shutil import socket +import urllib.parse from pathlib import Path import pytest @@ -691,7 +692,10 @@ def test_a_403_that_impersonation_does_not_cure_says_what_to_do(library, session assert job["status"] == "error" assert "HTTP 403" in job["error_message"] - assert "URL_IMPORT_COOKIES_FILE" in job["error_message"] + # The last line is a command to run at home, for this link. + command = job["error_message"].splitlines()[-1] + assert command.startswith("yt-dlp --embed-metadata ") + assert command.endswith(f'"{VIDEO_URL}"') assert [bool(p.get("impersonate")) for p in FakeYoutubeDL.made] == [False, True] @@ -702,7 +706,7 @@ def test_without_curl_cffi_the_403_is_reported_not_crashed_on(library, session, assert job["status"] == "error" assert "HTTP 403" in job["error_message"] - assert "curl-cffi" in job["error_message"] + assert [bool(p.get("impersonate")) for p in FakeYoutubeDL.made] == [False] def test_other_failures_are_not_retried_as_a_browser(library, session, monkeypatch): @@ -712,6 +716,31 @@ def test_other_failures_are_not_retried_as_a_browser(library, session, monkeypat assert not any(p.get("impersonate") for p in FakeYoutubeDL.made) +def test_the_local_command_fetches_what_an_import_would(monkeypatch): + from app.config import settings + + monkeypatch.setattr(settings, "url_import_max_height", 720) + command = ytdlp.local_download_command("https://rumble.com/v13047p-a-speech.html") + + assert command == ( + 'yt-dlp --embed-metadata -f "bv*[height<=720]+ba/b" --merge-output-format mp4 ' + '-o "%(title)s.%(ext)s" "https://rumble.com/v13047p-a-speech.html"' + ) + + +def test_the_local_command_leaves_a_shell_nothing_to_expand(): + """The link is whatever was pasted. Inside the command's double quotes, a `"`, `$`, + backtick, `!` or `\\` would end the quoting or run something; encoded, each is still + the same link to the site.""" + hostile = 'https://rumble.com/v1-x.html?a="b"&c=$(touch pwned)&d=`id`&e=!!&f=\\' + command = ytdlp.local_download_command(hostile) + quoted = command.rsplit(" ", 1)[-1] + + assert quoted.startswith('"') and quoted.endswith('"') + assert not any(ch in quoted[1:-1] for ch in '"$`!\\ ') + assert urllib.parse.unquote(quoted[1:-1]) == hostile + + def test_cancelling_mid_download_stops_the_job(library, session, monkeypatch): """The progress hook is the checkpoint: the reporter raises JobCancelled, which the hook turns into the exception yt-dlp is written to let through.""" diff --git a/docs/url-import.md b/docs/url-import.md index a894177..278f6af 100644 --- a/docs/url-import.md +++ b/docs/url-import.md @@ -94,9 +94,32 @@ Rumble, and other sites behind Cloudflare's bot screen, refuse a plain Python cl its TLS handshake (`Unable to download webpage: HTTP Error 403: Forbidden`) before any header or cookie is read. The `curl-cffi` extra lets yt-dlp handshake like Chrome, and `ytdlp._extract` retries any 403 that way, once. Only on a 403: sites that already work, -YouTube above all, keep yt-dlp's own client. A 403 that survives the retry is a block on -the server's IP or a site that wants a signed-in session; the cookies file below is the -remedy for the second. +YouTube above all, keep yt-dlp's own client. + +A 403 that survives the retry is usually Cloudflare's JavaScript challenge, which it +sends to IPs it distrusts — hosting providers' especially. Rumble does this to whole +datacenter ranges, on every path that leads to a video (checked from a blocked server, +Nov 2026). Nothing but a real browser answers it, and a cookies file exported elsewhere +does not help: the clearance cookie is bound to the IP and browser that earned it. To +check what a site answers this server: + +```bash +docker compose exec backend python -c " +from curl_cffi import requests +r = requests.get('', impersonate='chrome') +print(r.status_code, r.headers.get('cf-mitigated'))" +``` + +`403 challenge` is that. The import's error then ends with a yt-dlp command for the +link (`ytdlp.local_download_command`), shown in the activity panel with a copy button: run +it on a computer whose connection the site accepts (it needs yt-dlp and ffmpeg) and +upload the file. `--embed-metadata` writes the channel, date and link into the file, +which the upload reads back; the filename is the title, which becomes the asset's name. +The description, tags and chapters do not come along. + +Routing imports through another network instead (a proxy, or an SSH tunnel home) was +written and set aside in davior/gam#75 (see its history): a tunnel exposes the home network to +anything on the server, and a residential proxy is a subscription for an occasional video. ### Deno diff --git a/frontend/src/components/ActivityIndicator.test.tsx b/frontend/src/components/ActivityIndicator.test.tsx index a541f22..24235e8 100644 --- a/frontend/src/components/ActivityIndicator.test.tsx +++ b/frontend/src/components/ActivityIndicator.test.tsx @@ -116,4 +116,36 @@ describe('ActivityIndicator', () => { expect(screen.queryByRole('button', { name: /cancel/i })).not.toBeInTheDocument() }) + + it('sets a command in an error apart, to be copied', async () => { + // Spied rather than replaced: userEvent.setup() installs its own clipboard stub. + const user = userEvent.setup() + const writeText = vi.spyOn(navigator.clipboard, 'writeText').mockResolvedValue() + const command = + 'yt-dlp --embed-metadata -f "bv*[height<=1080]+ba/b" --merge-output-format mp4 ' + + '-o "%(title)s.%(ext)s" "https://rumble.com/v1-x.html"' + renderWith([ + job({ + action: 'import_url', + status: 'error', + error_message: `The site refused this server (HTTP 403). Upload the file:\n${command}`, + }), + ]) + + await user.click(screen.getByRole('button', { name: 'Background activity' })) + expect(screen.getByText(command).tagName).toBe('CODE') + expect(screen.getByText(/refused this server/)).not.toHaveTextContent('yt-dlp') + + await user.click(screen.getByRole('button', { name: 'Copy this command' })) + expect(writeText).toHaveBeenCalledWith(command) + }) + + it('shows an error with no command as it is', async () => { + const user = userEvent.setup() + renderWith([job({ status: 'error', error_message: 'No Deepgram key' })]) + + await user.click(screen.getByRole('button', { name: 'Background activity' })) + expect(screen.getByText('No Deepgram key')).toBeInTheDocument() + expect(screen.queryByRole('button', { name: 'Copy this command' })).toBeNull() + }) }) diff --git a/frontend/src/components/ActivityIndicator.tsx b/frontend/src/components/ActivityIndicator.tsx index da36266..bc1b655 100644 --- a/frontend/src/components/ActivityIndicator.tsx +++ b/frontend/src/components/ActivityIndicator.tsx @@ -1,5 +1,5 @@ import { useEffect, useRef, useState } from 'react' -import { Activity, X } from 'lucide-react' +import { Activity, Check, Copy, X } from 'lucide-react' import type { ActivityJob } from '@/api/transcripts' import { isActive, useActivityStore } from '@/stores/activity' @@ -33,6 +33,60 @@ function subject(job: ActivityJob): string { return job.asset_name || 'Your whole library' } +/** + * A job's error, with any command in it set apart to be copied. + * + * An import from a site that blocks the server ends with the yt-dlp command that fetches + * the same video at home (ytdlp.local_download_command). It is only useful whole and + * exact, which a line of 12px prose in a narrow card is not, so a line that starts + * `yt-dlp ` renders as code with a copy button. Plain text in, so the message stays a + * sentence anywhere else it is shown. + */ +function JobError({ message }: { message: string }) { + const lines = message.split('\n') + const command = lines.find((line) => line.startsWith('yt-dlp ')) + const prose = lines.filter((line) => line !== command).join(' ') + const [copied, setCopied] = useState(false) + + const copy = async () => { + if (!command) return + try { + await navigator.clipboard.writeText(command) + setCopied(true) + setTimeout(() => setCopied(false), 1500) + } catch { + // Clipboard access needs a secure context. Prompting with the text beats a button + // that silently does nothing. + window.prompt('Copy this command', command) + } + } + + return ( + <> + {prose} + {command && ( + + + {command} + + + + )} + + ) +} + export default function ActivityIndicator() { const jobs = useActivityStore((s) => s.jobs) const start = useActivityStore((s) => s.start) @@ -125,14 +179,16 @@ export default function ActivityIndicator() { )} -

- {job.stalled - ? 'Not responding' - : job.status === 'error' - ? (job.error_message ?? 'Failed') - : job.detail || job.stage || job.status} +

+ {job.stalled ? ( + 'Not responding' + ) : job.status === 'error' ? ( + + ) : ( + job.detail || job.stage || job.status + )} {isActive(job) && job.progress > 0 && ` · ${job.progress}%`} -

+
))}