When a site blocks the server, end the import error with a yt-dlp command to run at home

backend/app/ingest/ytdlp.py

@@ -136,7 +136,7 @@ def probe_url(url: str, workdir: Path) -> dict:
     try:
         info = _extract(options, url, download=False)
     except DownloadError as exc:
-        raise UrlImportError(readable_error(exc)) from exc
+        raise UrlImportError(readable_error(exc, url)) from exc
     if not info:
         raise UrlImportError("Nothing importable was found at that link")
     return info
@@ -270,7 +270,7 @@ def download(
     except DownloadCancelled as exc:
         raise JobCancelled(str(exc)) from exc
     except DownloadError as exc:
-        raise UrlImportError(readable_error(exc)) from exc
+        raise UrlImportError(readable_error(exc, url)) from exc
 
     media = _media_path(result, workdir)
     return Downloaded(
@@ -503,7 +503,7 @@ def _is_forbidden(exc: Exception) -> bool:
 )
 
 
-def readable_error(exc: Exception) -> str:
+def readable_error(exc: Exception, url: str = "") -> str:
     """A yt-dlp failure as something a person can act on.
 
     yt-dlp's messages are written for a terminal user — `ERROR: [youtube] abc123: …` —
@@ -526,11 +526,11 @@ def readable_error(exc: Exception) -> str:
 
     if "403" in lowered and "forbidden" in lowered:
         return (
-            "The site refused this server (HTTP 403). If the backend image predates "
-            "curl-cffi in backend/requirements.txt, rebuild it so the import can retry "
-            "as a browser. Otherwise the site blocks this server's IP or wants a "
-            "signed-in session: export your browser's cookies for it to a file and set "
-            "URL_IMPORT_COOKIES_FILE — see docs/url-import.md."
+            "The site refused this server (HTTP 403). "
+            "Usually that is Cloudflare blocking this server's IP address, which no "
+            "importer can get past from here. Download it on your own computer and "
+            "upload the file:"
+            + (f"\n{local_download_command(url)}" if url else "")
         )
 
     first_line = message.splitlines()[0] if message else ""
@@ -543,6 +543,31 @@ def readable_error(exc: Exception) -> str:
     return cleaned
 
 
+# Characters left as they are when a URL goes into a pasted command. Everything else —
+# quotes, `$`, backticks, `!`, `\`, spaces — is percent-encoded, which leaves the link
+# meaning the same thing and leaves nothing for a shell to expand inside double quotes.
+# Double rather than single quotes because cmd.exe has no single-quoting at all; bash,
+# zsh and PowerShell take either.
+_COMMAND_SAFE = ":/?#[]@&=+,;%-._~*()'"
+
+
+def local_download_command(url: str) -> str:
+    """The yt-dlp command that fetches `url` the way an import would, to run elsewhere.
+
+    For a site that blocks this server but not a home connection. `--embed-metadata`
+    writes the channel, date and link into the file, which the upload path reads back
+    (`embedded_metadata.py`); the output name is the title, which an upload takes as the
+    asset's name. Same height cap and container as an import. Needs ffmpeg, as any
+    yt-dlp merge does.
+    """
+    quoted = urllib.parse.quote(url, safe=_COMMAND_SAFE)
+    height = max(144, settings.url_import_max_height)
+    return (
+        f'yt-dlp --embed-metadata -f "bv*[height<={height}]+ba/b" '
+        f'--merge-output-format mp4 -o "%(title)s.%(ext)s" "{quoted}"'
+    )
+
+
 # ─── mapping what the site said ──────────────────────────────────────────────
 
 

backend/tests/test_url_import.py

@@ -13,6 +13,7 @@
 import json
 import shutil
 import socket
+import urllib.parse
 from pathlib import Path
 
 import pytest
@@ -691,7 +692,10 @@ def test_a_403_that_impersonation_does_not_cure_says_what_to_do(library, session
 
     assert job["status"] == "error"
     assert "HTTP 403" in job["error_message"]
-    assert "URL_IMPORT_COOKIES_FILE" in job["error_message"]
+    # The last line is a command to run at home, for this link.
+    command = job["error_message"].splitlines()[-1]
+    assert command.startswith("yt-dlp --embed-metadata ")
+    assert command.endswith(f'"{VIDEO_URL}"')
     assert [bool(p.get("impersonate")) for p in FakeYoutubeDL.made] == [False, True]
 
 
@@ -702,7 +706,7 @@ def test_without_curl_cffi_the_403_is_reported_not_crashed_on(library, session,
 
     assert job["status"] == "error"
     assert "HTTP 403" in job["error_message"]
-    assert "curl-cffi" in job["error_message"]
+    assert [bool(p.get("impersonate")) for p in FakeYoutubeDL.made] == [False]
 
 
 def test_other_failures_are_not_retried_as_a_browser(library, session, monkeypatch):
@@ -712,6 +716,31 @@ def test_other_failures_are_not_retried_as_a_browser(library, session, monkeypat
     assert not any(p.get("impersonate") for p in FakeYoutubeDL.made)
 
 
+def test_the_local_command_fetches_what_an_import_would(monkeypatch):
+    from app.config import settings
+
+    monkeypatch.setattr(settings, "url_import_max_height", 720)
+    command = ytdlp.local_download_command("https://rumble.com/v13047p-a-speech.html")
+
+    assert command == (
+        'yt-dlp --embed-metadata -f "bv*[height<=720]+ba/b" --merge-output-format mp4 '
+        '-o "%(title)s.%(ext)s" "https://rumble.com/v13047p-a-speech.html"'
+    )
+
+
+def test_the_local_command_leaves_a_shell_nothing_to_expand():
+    """The link is whatever was pasted. Inside the command's double quotes, a `"`, `$`,
+    backtick, `!` or `\\` would end the quoting or run something; encoded, each is still
+    the same link to the site."""
+    hostile = 'https://rumble.com/v1-x.html?a="b"&c=$(touch pwned)&d=`id`&e=!!&f=\\'
+    command = ytdlp.local_download_command(hostile)
+    quoted = command.rsplit(" ", 1)[-1]
+
+    assert quoted.startswith('"') and quoted.endswith('"')
+    assert not any(ch in quoted[1:-1] for ch in '"$`!\\ ')
+    assert urllib.parse.unquote(quoted[1:-1]) == hostile
+
+
 def test_cancelling_mid_download_stops_the_job(library, session, monkeypatch):
     """The progress hook is the checkpoint: the reporter raises JobCancelled, which the
     hook turns into the exception yt-dlp is written to let through."""

docs/url-import.md

@@ -94,9 +94,32 @@ Rumble, and other sites behind Cloudflare's bot screen, refuse a plain Python cl
 its TLS handshake (`Unable to download webpage: HTTP Error 403: Forbidden`) before any
 header or cookie is read. The `curl-cffi` extra lets yt-dlp handshake like Chrome, and
 `ytdlp._extract` retries any 403 that way, once. Only on a 403: sites that already work,
-YouTube above all, keep yt-dlp's own client. A 403 that survives the retry is a block on
-the server's IP or a site that wants a signed-in session; the cookies file below is the
-remedy for the second.
+YouTube above all, keep yt-dlp's own client.
+
+A 403 that survives the retry is usually Cloudflare's JavaScript challenge, which it
+sends to IPs it distrusts — hosting providers' especially. Rumble does this to whole
+datacenter ranges, on every path that leads to a video (checked from a blocked server,
+Nov 2026). Nothing but a real browser answers it, and a cookies file exported elsewhere
+does not help: the clearance cookie is bound to the IP and browser that earned it. To
+check what a site answers this server:
+
+```bash
+docker compose exec backend python -c "
+from curl_cffi import requests
+r = requests.get('<the page URL>', impersonate='chrome')
+print(r.status_code, r.headers.get('cf-mitigated'))"
+```
+
+`403 challenge` is that. The import's error then ends with a yt-dlp command for the
+link (`ytdlp.local_download_command`), shown in the activity panel with a copy button: run
+it on a computer whose connection the site accepts (it needs yt-dlp and ffmpeg) and
+upload the file. `--embed-metadata` writes the channel, date and link into the file,
+which the upload reads back; the filename is the title, which becomes the asset's name.
+The description, tags and chapters do not come along.
+
+Routing imports through another network instead (a proxy, or an SSH tunnel home) was
+written and set aside in davior/gam#75 (see its history): a tunnel exposes the home network to
+anything on the server, and a residential proxy is a subscription for an occasional video.
 
 ### Deno
 

frontend/src/components/ActivityIndicator.test.tsx

@@ -116,4 +116,36 @@ describe('ActivityIndicator', () => {
 
     expect(screen.queryByRole('button', { name: /cancel/i })).not.toBeInTheDocument()
   })
+
+  it('sets a command in an error apart, to be copied', async () => {
+    // Spied rather than replaced: userEvent.setup() installs its own clipboard stub.
+    const user = userEvent.setup()
+    const writeText = vi.spyOn(navigator.clipboard, 'writeText').mockResolvedValue()
+    const command =
+      'yt-dlp --embed-metadata -f "bv*[height<=1080]+ba/b" --merge-output-format mp4 ' +
+      '-o "%(title)s.%(ext)s" "https://rumble.com/v1-x.html"'
+    renderWith([
+      job({
+        action: 'import_url',
+        status: 'error',
+        error_message: `The site refused this server (HTTP 403). Upload the file:\n${command}`,
+      }),
+    ])
+
+    await user.click(screen.getByRole('button', { name: 'Background activity' }))
+    expect(screen.getByText(command).tagName).toBe('CODE')
+    expect(screen.getByText(/refused this server/)).not.toHaveTextContent('yt-dlp')
+
+    await user.click(screen.getByRole('button', { name: 'Copy this command' }))
+    expect(writeText).toHaveBeenCalledWith(command)
+  })
+
+  it('shows an error with no command as it is', async () => {
+    const user = userEvent.setup()
+    renderWith([job({ status: 'error', error_message: 'No Deepgram key' })])
+
+    await user.click(screen.getByRole('button', { name: 'Background activity' }))
+    expect(screen.getByText('No Deepgram key')).toBeInTheDocument()
+    expect(screen.queryByRole('button', { name: 'Copy this command' })).toBeNull()
+  })
 })

frontend/src/components/ActivityIndicator.tsx

@@ -1,5 +1,5 @@
 import { useEffect, useRef, useState } from 'react'
-import { Activity, X } from 'lucide-react'
+import { Activity, Check, Copy, X } from 'lucide-react'
 import type { ActivityJob } from '@/api/transcripts'
 import { isActive, useActivityStore } from '@/stores/activity'
 
@@ -33,6 +33,60 @@ function subject(job: ActivityJob): string {
   return job.asset_name || 'Your whole library'
 }
 
+/**
+ * A job's error, with any command in it set apart to be copied.
+ *
+ * An import from a site that blocks the server ends with the yt-dlp command that fetches
+ * the same video at home (ytdlp.local_download_command). It is only useful whole and
+ * exact, which a line of 12px prose in a narrow card is not, so a line that starts
+ * `yt-dlp ` renders as code with a copy button. Plain text in, so the message stays a
+ * sentence anywhere else it is shown.
+ */
+function JobError({ message }: { message: string }) {
+  const lines = message.split('\n')
+  const command = lines.find((line) => line.startsWith('yt-dlp '))
+  const prose = lines.filter((line) => line !== command).join(' ')
+  const [copied, setCopied] = useState(false)
+
+  const copy = async () => {
+    if (!command) return
+    try {
+      await navigator.clipboard.writeText(command)
+      setCopied(true)
+      setTimeout(() => setCopied(false), 1500)
+    } catch {
+      // Clipboard access needs a secure context. Prompting with the text beats a button
+      // that silently does nothing.
+      window.prompt('Copy this command', command)
+    }
+  }
+
+  return (
+    <>
+      <span>{prose}</span>
+      {command && (
+        <span className="mt-1 flex items-start gap-1">
+          <code className="block min-w-0 flex-1 select-all break-all rounded bg-gray-100 p-1.5 font-mono text-[11px] text-gray-800 dark:bg-gray-800 dark:text-gray-200">
+            {command}
+          </code>
+          <button
+            type="button"
+            className="btn btn-ghost shrink-0 p-1"
+            aria-label="Copy this command"
+            onClick={() => void copy()}
+          >
+            {copied ? (
+              <Check className="h-3.5 w-3.5" />
+            ) : (
+              <Copy className="h-3.5 w-3.5" />
+            )}
+          </button>
+        </span>
+      )}
+    </>
+  )
+}
+
 export default function ActivityIndicator() {
   const jobs = useActivityStore((s) => s.jobs)
   const start = useActivityStore((s) => s.start)
@@ -125,14 +179,16 @@ export default function ActivityIndicator() {
                 )}
               </div>
 
-              <p className="text-xs text-gray-500 dark:text-gray-400">
-                {job.stalled
-                  ? 'Not responding'
-                  : job.status === 'error'
-                    ? (job.error_message ?? 'Failed')
-                    : job.detail || job.stage || job.status}
+              <div className="text-xs text-gray-500 dark:text-gray-400">
+                {job.stalled ? (
+                  'Not responding'
+                ) : job.status === 'error' ? (
+                  <JobError message={job.error_message ?? 'Failed'} />
+                ) : (
+                  job.detail || job.stage || job.status
+                )}
                 {isActive(job) && job.progress > 0 && ` · ${job.progress}%`}
-              </p>
+              </div>
             </div>
           ))}
         </div>