From 3696662b58e1574ea5e93b757450dbdf85a88ec1 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sat, 29 Aug 2026 13:41:16 +0000 Subject: [PATCH] feat(social): Instagram at /ig/, Facebook at /fb/, and one failure rule MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two more namespaces, and they cost very different amounts — which is the useful part of this change rather than an aside. Instagram is X's shape: no feeds, collected through the RSSHub daemon already running in the poller, mirrored here. It was a canonical.js, a fetch.js and a route, because the namespace machinery already existed. Facebook is not. Three doors, measured rather than assumed: - facebook.com/feeds/page.php?format=rss20 answers 404 (removed) - mbasic.facebook.com/ answers 200 with a login wall - RSSHub carries a thousand namespaces and no Facebook one The only remaining route is Meta's Graph API, which returns a Page's posts only to somebody who ADMINISTERS that Page — reading a stranger's public Page needs a permission Meta grants rarely and only after review. So /fb/ takes no open submissions: a Page appears once its operator connects it via FB_PAGE_TOKENS, and an unconnected Page is told plainly that it is not collectable rather than shown a button that does nothing. Nothing here drives a logged-in session against that login wall. The rule from #157 is now shared rather than per-platform. src/failure.js owns "only a source that does not exist may count against the source", and X was refactored onto it rather than Instagram and Facebook being written against a copy. Its 60 tests passed unchanged, which is the evidence the refactor was behaviour-identical. Two bugs the tests caught, both mine: - identify.js said in a comment that X is tried before Instagram and had Instagram first, so a bare @handle resolved to Instagram - the Facebook collector took its display name from the ref, which is lowercased because it is an identity — so every item title read "somepage:" instead of "SomePage:". Display now comes from feed_url, which kept the original casing; identity still comes from the ref Also: an uncrawled social row now renders its canonical name instead of whatever the OPML import called it. 50,026 subreddits are imported and a few hundred crawled, so /r/programming was headed "reddit.com". A render decision, not a rewrite of stored titles — the crawler owns those. 1,345 tests across 12 packages, all green. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Q6QEgpuS4MLamogXtr2ZX6 --- .env.example | 18 + README.md | 57 ++- apps/web/next.config.mjs | 12 + apps/web/src/app/AddSocialSource.jsx | 115 ++++-- apps/web/src/app/SocialIndex.jsx | 95 +++-- .../app/api/fb/[page]/feed/[format]/route.js | 39 ++ .../api/ig/[username]/feed/[format]/route.js | 34 ++ .../api/ig/tag/[tag]/feed/[format]/route.js | 34 ++ apps/web/src/app/fb/[page]/page.jsx | 56 +++ apps/web/src/app/fb/page.jsx | 27 ++ apps/web/src/app/ig/[username]/page.jsx | 55 +++ apps/web/src/app/ig/page.jsx | 27 ++ apps/web/src/app/ig/tag/[tag]/page.jsx | 50 +++ apps/web/src/app/layout.jsx | 3 +- apps/web/src/lib/sitemap.js | 4 + apps/web/src/lib/socialPage.js | 11 +- apps/web/src/lib/socialRiver.js | 45 ++- packages/db/src/social.js | 16 +- packages/feed/src/slug.js | 6 + packages/ingest/src/crawl.js | 20 +- packages/ingest/test/social-crawl.test.js | 4 +- packages/social/index.js | 29 ++ packages/social/src/collect.js | 54 +++ packages/social/src/display.js | 68 ++++ packages/social/src/facebook/canonical.js | 201 ++++++++++ packages/social/src/facebook/fetch.js | 263 +++++++++++++ packages/social/src/failure.js | 69 ++++ packages/social/src/identify.js | 119 +++--- packages/social/src/instagram/canonical.js | 205 ++++++++++ packages/social/src/instagram/fetch.js | 125 ++++++ packages/social/src/x/fetch.js | 75 +--- packages/social/test/platforms.test.js | 369 ++++++++++++++++++ 32 files changed, 2108 insertions(+), 197 deletions(-) create mode 100644 apps/web/src/app/api/fb/[page]/feed/[format]/route.js create mode 100644 apps/web/src/app/api/ig/[username]/feed/[format]/route.js create mode 100644 apps/web/src/app/api/ig/tag/[tag]/feed/[format]/route.js create mode 100644 apps/web/src/app/fb/[page]/page.jsx create mode 100644 apps/web/src/app/fb/page.jsx create mode 100644 apps/web/src/app/ig/[username]/page.jsx create mode 100644 apps/web/src/app/ig/page.jsx create mode 100644 apps/web/src/app/ig/tag/[tag]/page.jsx create mode 100644 packages/social/src/collect.js create mode 100644 packages/social/src/display.js create mode 100644 packages/social/src/facebook/canonical.js create mode 100644 packages/social/src/facebook/fetch.js create mode 100644 packages/social/src/failure.js create mode 100644 packages/social/src/instagram/canonical.js create mode 100644 packages/social/src/instagram/fetch.js create mode 100644 packages/social/test/platforms.test.js diff --git a/.env.example b/.env.example index a72eade..21491ea 100644 --- a/.env.example +++ b/.env.example @@ -87,3 +87,21 @@ X_SESSIONS= X_FETCH_TIMEOUT_MS=15000 X_SESSION_COOLDOWN_SECONDS=900 + +# ------------------------------------------------------------------- Instagram +# Collected through the same embedded RSSHub daemon as X, so it needs no base +# URL of its own — only a logged-in Instagram cookie on the RSSHub side. +# Accounts and hashtags only; stories expire and are not collected. +IG_COOKIE= + +# -------------------------------------------------------------------- Facebook +# Facebook publishes no public feed, serves no page without a login, and has no +# bridge. The only route in is Meta's Graph API, which returns a Page's posts +# only to somebody who ADMINISTERS that Page - so /fb/ takes no open +# submissions, and a Page appears only once its operator connects it here. +# +# FB_PAGE_TOKENS=[{"page":"MyPage","token":"EAA..."}] +# +# A Page Access Token can post as the Page. Treat it exactly like the X session +# cookies above: vault, not a service env, and never a database column. +FB_PAGE_TOKENS= diff --git a/README.md b/README.md index 783c24f..a342f2e 100644 --- a/README.md +++ b/README.md @@ -65,8 +65,12 @@ pnpm --filter @rssamplifier/db migrate | `/x/list/` | One X list | | `/x/search?q=` | An X search as a feed — X's own operators pass through | | `/x/status` | Which provider is collecting X, and how it is doing | +| `/ig` | Every Instagram account and hashtag in the directory | +| `/ig/` | One account: `/ig/nasa`, and `/ig/tag/` for a hashtag | +| `/fb` | Facebook Pages whose operators have connected them | +| `/fb/` | One connected Page | -### The two social namespaces +### The four social namespaces Reddit and X both live under a prefix of their own, and for the same reason from opposite directions. Reddit publishes real RSS, so a subreddit resolves down the @@ -74,8 +78,12 @@ ordinary path and lands as an untyped row at a slug of its own — which is how 50,099 of them ended up filed among the blogs. X publishes nothing at all, so without a provider it is not submittable in the first place. -`packages/social` answers one question for both: **what is the canonical identity -of this thing?** `@OpenAI`, `x.com/OpenAI` and `https://twitter.com/openai/` are +Instagram is X's shape again — no feeds, collected through RSSHub — and cost +almost nothing to add, which is the point of having built the namespace once. +Facebook is its own thing entirely; see below. + +`packages/social` answers one question for all four: **what is the canonical +identity of this thing?** `@OpenAI`, `x.com/OpenAI` and `https://twitter.com/openai/` are one source (`x:user:openai`, at `/x/OpenAI`); `/r/programming`, `/r/Programming/` and `/r/programming/new/.rss` are one community (`r:sub:programming`, at `/r/programming`). One identity means one row, which means **one polling job no @@ -93,6 +101,49 @@ to tell them apart. of a row and links already point at it. The `/r/` and `/x/` address is the canonical one, which is what search engines are told. +## Facebook, and what `/fb/` can honestly be + +**There is no way to read an arbitrary public Facebook Page.** Three doors, all +measured on 2026-08-29 rather than assumed: + +- the old `facebook.com/feeds/page.php?format=rss20` endpoint answers **404** — + removed, not deprecated +- `mbasic.facebook.com/` answers 200 with a **login wall** +- RSSHub, which carries a thousand namespaces and maintains Twitter and + Instagram, has **no Facebook namespace at all** + +The one remaining door is Meta's Graph API, and it only opens for Pages the +caller **administers**. Reading somebody else's public Page needs the +`Page Public Content Access` feature, which requires App Review plus business +verification and is granted rarely. + +So `/fb/` is the one namespace here that does not take open submissions: a Page +appears when its operator connects it, by putting a Page Access Token in +`FB_PAGE_TOKENS`. A Page nobody has connected is not "not crawled yet", it is +not collectable, and the page says exactly that rather than offering a button +that would quietly do nothing. + +```bash +FB_PAGE_TOKENS='[{"page":"MyPage","token":"EAA..."}]' +``` + +What is deliberately absent: anything that drives a logged-in Facebook session +against that login wall. It breaks constantly, it is against Meta's terms, and +it would risk an account of ours to serve a directory nobody pays for. + +## Collecting Instagram + +Same shape as X, through the same RSSHub daemon — the `/instagram/2/…` web-api +routes, which authenticate with a cookie rather than the private-api routes, +which want a username and password. + +```bash +IG_COOKIE= # on the RSSHub side; see apps/poller/src/rsshub.js +``` + +Accounts and hashtags only. Stories expire, and a feed of things that have +already gone is worse than no feed. + ## Collecting X X has no feeds, so posts are collected through a provider and mirrored here. diff --git a/apps/web/next.config.mjs b/apps/web/next.config.mjs index a7f31c7..cc0276f 100644 --- a/apps/web/next.config.mjs +++ b/apps/web/next.config.mjs @@ -242,6 +242,18 @@ const nextConfig = { source: '/r/:subreddit.:format(rss|atom|json|xml|md)', destination: '/api/r/:subreddit/feed/:format', }, + { + source: '/ig/tag/:tag.:format(rss|atom|json|xml|md)', + destination: '/api/ig/tag/:tag/feed/:format', + }, + { + source: '/ig/:username.:format(rss|atom|json|xml|md)', + destination: '/api/ig/:username/feed/:format', + }, + { + source: '/fb/:page.:format(rss|atom|json|xml|md)', + destination: '/api/fb/:page/feed/:format', + }, // One category of it. The segments are the category pages' own paths, // duplicated from CATEGORIES in apps/web/src/lib/categories.js — this diff --git a/apps/web/src/app/AddSocialSource.jsx b/apps/web/src/app/AddSocialSource.jsx index e634e12..7a31016 100644 --- a/apps/web/src/app/AddSocialSource.jsx +++ b/apps/web/src/app/AddSocialSource.jsx @@ -1,33 +1,82 @@ import { siteUrl } from '../lib/db.js'; /** - * What `/r/somewhere` or `/x/somebody` shows when nobody has added it yet. + * What `/r/somewhere`, `/x/somebody`, `/ig/somebody` or `/fb/SomePage` shows + * when it is not in the directory yet. * - * A 404 would be the easy answer and the wrong one. The address is well formed, - * the thing at the other end almost certainly exists, and the visitor has - * already told us exactly what they want by typing it — so the page offers to - * add it rather than telling them they were wrong to ask. + * A 404 would be the easy answer and the wrong one for three of the four. The + * address is well formed, the thing at the other end almost certainly exists, + * and the visitor has already said exactly what they want by typing it — so the + * page offers to add it rather than telling them they were wrong to ask. * * A plain `
` to `/api/submit`, like every other control on * this site: it works with JavaScript off, and the endpoint answers an HTML - * caller with a 303 back to the source's own page. Nothing is fetched from X or - * Reddit while the visitor waits — the row is written, the poller collects on + * caller with a 303 back to the source's own page. Nothing is fetched from the + * platform while the visitor waits — the row is written, the poller collects on * its next tick, and this page is replaced by the real one within the minute * (§17, §37). * - * @param {{ network: 'x'|'reddit', label: string, input: string, canonical: string }} props + * **Facebook is the exception, and says so.** There is no public feed, no + * unauthenticated HTML and no provider; the only way in is a Page Access Token + * from whoever administers the Page. Offering an "add" button there would be a + * button that quietly does nothing, so it gets an explanation instead. + * + * @param {{ network: 'x'|'reddit'|'instagram'|'facebook', label: string, input: string, canonical: string }} props * `input` is what gets submitted — the canonical upstream URL, not what was * typed, so the source that gets created is the one this page is about. */ export default function AddSocialSource({ network, label, input, canonical }) { - const platform = network === 'x' ? 'X' : 'Reddit'; + const platform = PLATFORMS[network] ?? PLATFORMS.x; + const address = ( + <> + + {siteUrl()} + {canonical} + {' '} + in every format this site publishes: .rss, .atom,{' '} + .json and .md + + ); + + if (network === 'facebook') { + return ( +
+

{label}

+ +

+ This Facebook Page is not connected, and unlike the rest of the directory it cannot be + added by anyone who happens to want it. +

+ +

+ Facebook publishes no feed for a Page, serves no page without a login, and has no + third-party bridge we can use. The only remaining route is Meta’s own Graph API, + and it will only return a Page’s posts to somebody who administers that + Page — reading a stranger’s public Page needs a permission Meta grants + rarely and only after review. +

+ +

+ So if this is your Page, it can be connected: an administrator supplies a Page Access + Token and it appears here at {address}, collected on the same schedule as everything + else. If it is not your Page, there is nothing we can honestly offer — and we would + rather say that than mirror a scraper that breaks every few weeks. +

+ +

+ What is connected · X · Instagram ·{' '} + Reddit +

+
+ ); + } return (

{label}

- Nobody has added this {platform} source to the directory yet. Add it and RSS Amplifier + Nobody has added this {platform.name} source to the directory yet. Add it and RSS Amplifier will start collecting it — usually within a minute.

@@ -36,33 +85,33 @@ export default function AddSocialSource({ network, label, input, canonical }) { -

- Once it is here, it will be at{' '} - - {siteUrl()} - {canonical} - {' '} - in every format this site publishes:{' '} - .rss, .atom, .json and .md. That - address does not change, whatever we have to do behind it to keep collecting. -

+

Once it is here, it will be at {address}. That address does not change, whatever we have + to do behind it to keep collecting.

- {network === 'x' ? ( -

- X publishes no feeds of its own, so this is collected on your behalf and mirrored here. - Protected accounts are not collected, and posts arrive as fast as we can read them - rather than in real time. -

- ) : ( -

- Reddit publishes its own feed for this, and we read it on a schedule and keep a copy — - so the address above works whether or not Reddit is answering right now. -

- )} +

{platform.note}

- Browse what is already here + Browse what is already here

); } + +/** What to call each platform, and the one thing worth saying about it. */ +const PLATFORMS = { + x: { + name: 'X', + index: '/x', + note: 'X publishes no feeds of its own, so this is collected on your behalf and mirrored here. Protected accounts are not collected, and posts arrive as fast as we can read them rather than in real time.', + }, + reddit: { + name: 'Reddit', + index: '/r', + note: 'Reddit publishes its own feed for this, and we read it on a schedule and keep a copy — so the address above works whether or not Reddit is answering right now.', + }, + instagram: { + name: 'Instagram', + index: '/ig', + note: 'Instagram publishes no feeds, so this is collected on your behalf and mirrored here. Private accounts are not collected, and stories are not either — they expire, and a feed of things that have already gone is worse than no feed.', + }, +}; diff --git a/apps/web/src/app/SocialIndex.jsx b/apps/web/src/app/SocialIndex.jsx index bed0d1f..721ce87 100644 --- a/apps/web/src/app/SocialIndex.jsx +++ b/apps/web/src/app/SocialIndex.jsx @@ -1,5 +1,5 @@ import { social } from '@rssamplifier/db'; -import { socialPathFor } from '@rssamplifier/social'; +import { socialDisplayTitle, socialPathFor } from '@rssamplifier/social'; import { db, siteUrl } from '../lib/db.js'; import ListFilter from './ListFilter.jsx'; @@ -24,6 +24,51 @@ import { FILTER_FROM } from '../lib/listFilter.js'; /** How many sources a page of this listing holds. */ const PER_PAGE = 100; +/** + * What each namespace calls itself, and the one paragraph it owes a reader. + * + * A table rather than a chain of ternaries, which is what this was when there + * were two platforms and what stopped scaling at three. + */ +const LOOKS = { + reddit: { + platform: 'Reddit', + noun: 'communities and users', + base: '/r', + placeholder: 'r/programming', + addLabel: 'Add a subreddit or Reddit user', + blurb: + 'Reddit publishes a feed for every community. We read them on a schedule and keep a copy, so these addresses work whether or not Reddit is answering right now.', + }, + x: { + platform: 'X', + noun: 'accounts, searches and lists', + base: '/x', + placeholder: '@OpenAI', + addLabel: 'Add an X account, list or search', + blurb: + 'X publishes no feeds, so these are collected on your behalf and mirrored here — the posts you read come out of this directory, never out of X.', + }, + instagram: { + platform: 'Instagram', + noun: 'accounts and hashtags', + base: '/ig', + placeholder: 'ig/nasa', + addLabel: 'Add an Instagram account or hashtag', + blurb: + 'Instagram publishes no feeds either, so these are collected and mirrored the same way X is. Private accounts are not collected, and neither are stories — they expire, and a feed of things that have already gone is worse than no feed.', + }, + facebook: { + platform: 'Facebook', + noun: 'connected Pages', + base: '/fb', + placeholder: '', + addLabel: '', + blurb: + 'Facebook is the one platform here that cannot be added by whoever wants it. There is no public feed, no page without a login, and no bridge — only Meta’s Graph API, which returns a Page’s posts to somebody who administers that Page. So these are Pages whose operators connected them, and nothing else can be.', + }, +}; + /** * @param {{ network: 'x'|'reddit', page?: number }} props */ @@ -36,9 +81,7 @@ export default async function SocialIndex({ network, page = 1 }) { social.countSocialFeeds(client, network), ]); - const platform = network === 'x' ? 'X' : 'Reddit'; - const noun = network === 'x' ? 'accounts, searches and lists' : 'communities and users'; - const base = network === 'x' ? '/x' : '/r'; + const { platform, noun, base, blurb, placeholder, addLabel } = LOOKS[network]; return (
@@ -50,38 +93,23 @@ export default async function SocialIndex({ network, page = 1 }) { page here and a feed in four formats, at an address that does not change.

- {network === 'x' ? ( -

- X publishes no feeds, so these are collected on your behalf and mirrored here — the - posts you read come out of this directory, never out of X.{' '} - Turn a search into a feed, or{' '} - see how collection is going. -

- ) : ( -

- Reddit publishes a feed for every community. We read them on a schedule and keep a - copy, so these addresses work whether or not Reddit is answering right now. -

- )} +

{blurb}

-
- - - -
+ {/* Facebook has no add form, and that is not an oversight: a Page can + only be connected by somebody who administers it, so a box inviting + anyone to paste a Page would be a box that quietly does nothing. */} + {network === 'facebook' ? null : ( +
+ + + +
+ )} {rows.length >= FILTER_FROM ? ( {rows.map((row) => { const href = socialPathFor(row); + // The canonical name where the imported title says nothing — most + // of the catalogue is uncrawled and titled with the bare host. + const name = socialDisplayTitle(row, href.replace(/^\//, '')); return (
  • - {String(row.title ?? href)} + {name} {row.description ? — {String(row.description)} : null} {' '} diff --git a/apps/web/src/app/api/fb/[page]/feed/[format]/route.js b/apps/web/src/app/api/fb/[page]/feed/[format]/route.js new file mode 100644 index 0000000..3333f7c --- /dev/null +++ b/apps/web/src/app/api/fb/[page]/feed/[format]/route.js @@ -0,0 +1,39 @@ +import { riverFail } from '../../../../../../lib/river.js'; +import { facebookTarget, socialRiver } from '../../../../../../lib/socialRiver.js'; + +export const dynamic = 'force-dynamic'; + +/** + * One Facebook Page, as a feed of ours. `/fb/SomePage.rss` rewrites here. + * + * A Page that nobody has connected a token for will 404 here rather than + * serving an empty feed, and that is the honest answer: it is not "not crawled + * yet", it is not collectable at all. See @rssamplifier/social's + * facebook/canonical.js for why Facebook cannot work the way the other three do. + * + * @param {Request} req + * @param {{ params: Promise<{ page: string, format: string }> }} ctx + */ +export async function GET(req, { params }) { + const { page, format } = await params; + const url = new URL(req.url); + + const target = facebookTarget({ page }); + if (!target) { + return riverFail( + format, + 400, + `not a Facebook Page name: ${page}`, + 'Page names are 5-60 characters of A-Z, 0-9 and dot.', + ); + } + + return socialRiver({ + ref: target.ref, + canonical: target.canonical, + label: target.label, + format, + limit: url.searchParams.get('limit'), + req, + }); +} diff --git a/apps/web/src/app/api/ig/[username]/feed/[format]/route.js b/apps/web/src/app/api/ig/[username]/feed/[format]/route.js new file mode 100644 index 0000000..298e82a --- /dev/null +++ b/apps/web/src/app/api/ig/[username]/feed/[format]/route.js @@ -0,0 +1,34 @@ +import { riverFail } from '../../../../../../lib/river.js'; +import { instagramTarget, socialRiver } from '../../../../../../lib/socialRiver.js'; + +export const dynamic = 'force-dynamic'; + +/** + * One Instagram account, as a feed of ours. `/ig/nasa.rss` rewrites here. + * + * @param {Request} req + * @param {{ params: Promise<{ username: string, format: string }> }} ctx + */ +export async function GET(req, { params }) { + const { username, format } = await params; + const url = new URL(req.url); + + const target = instagramTarget({ username }); + if (!target) { + return riverFail( + format, + 400, + `not an Instagram handle: ${username}`, + 'Handles are up to 30 characters of A-Z, 0-9, dot and underscore.', + ); + } + + return socialRiver({ + ref: target.ref, + canonical: target.canonical, + label: target.label, + format, + limit: url.searchParams.get('limit'), + req, + }); +} diff --git a/apps/web/src/app/api/ig/tag/[tag]/feed/[format]/route.js b/apps/web/src/app/api/ig/tag/[tag]/feed/[format]/route.js new file mode 100644 index 0000000..0fda453 --- /dev/null +++ b/apps/web/src/app/api/ig/tag/[tag]/feed/[format]/route.js @@ -0,0 +1,34 @@ +import { riverFail } from '../../../../../../../lib/river.js'; +import { instagramTarget, socialRiver } from '../../../../../../../lib/socialRiver.js'; + +export const dynamic = 'force-dynamic'; + +/** + * One Instagram hashtag. `/ig/tag/coffee.rss` rewrites here. + * + * @param {Request} req + * @param {{ params: Promise<{ tag: string, format: string }> }} ctx + */ +export async function GET(req, { params }) { + const { tag, format } = await params; + const url = new URL(req.url); + + const target = instagramTarget({ tag }); + if (!target) { + return riverFail( + format, + 400, + `not an Instagram hashtag: ${tag}`, + 'Hashtags are letters, digits and underscore.', + ); + } + + return socialRiver({ + ref: target.ref, + canonical: target.canonical, + label: target.label, + format, + limit: url.searchParams.get('limit'), + req, + }); +} diff --git a/apps/web/src/app/fb/[page]/page.jsx b/apps/web/src/app/fb/[page]/page.jsx new file mode 100644 index 0000000..7817ee2 --- /dev/null +++ b/apps/web/src/app/fb/[page]/page.jsx @@ -0,0 +1,56 @@ +import { notFound } from 'next/navigation'; +import { facebookSource } from '@rssamplifier/social'; + +import { socialFeed, socialMetadata } from '../../../lib/socialPage.js'; +import AddSocialSource from '../../AddSocialSource.jsx'; +import FeedPage from '../../[slug]/page.jsx'; + +export const dynamic = 'force-dynamic'; + +/** + * One Facebook Page, at `/fb/SomePage`. + * + * The one namespace of the four where "not here yet" usually means "not + * possible" rather than "nobody has got round to it". Facebook has no public + * feed, no unauthenticated HTML and no provider — the only way in is a Page + * Access Token from whoever administers the Page — so the empty state explains + * that rather than offering a button that would quietly do nothing. + * + * @param {{ params: Promise<{ page: string }> }} props + */ +export async function generateMetadata({ params }) { + const { page } = await params; + const source = facebookSource(`fb/${page}`); + if (!source) return { title: 'Not found', robots: { index: false, follow: false } }; + + return socialMetadata({ + feed: await socialFeed(source.ref), + canonical: source.path, + label: source.title, + network: 'facebook', + }); +} + +/** + * @param {{ params: Promise<{ page: string }> }} props + */ +export default async function FacebookPagePage({ params }) { + const { page } = await params; + + const source = facebookSource(`fb/${page}`); + if (!source) notFound(); + + const feed = await socialFeed(source.ref); + if (!feed) { + return ( + + ); + } + + return FeedPage({ params: Promise.resolve({ slug: String(feed.slug) }) }); +} diff --git a/apps/web/src/app/fb/page.jsx b/apps/web/src/app/fb/page.jsx new file mode 100644 index 0000000..d0eba9a --- /dev/null +++ b/apps/web/src/app/fb/page.jsx @@ -0,0 +1,27 @@ +import { siteUrl } from '../../lib/db.js'; +import SocialIndex, { pageNumber } from '../SocialIndex.jsx'; + +export const dynamic = 'force-dynamic'; + +/** + * @param {{ searchParams: Promise> }} props + */ +export async function generateMetadata({ searchParams }) { + const page = pageNumber((await searchParams).page); + + return { + title: page === 1 ? 'Facebook' : `Facebook · page ${page}`, + description: + 'Facebook Pages whose operators have connected them, as feeds you can subscribe to. Facebook publishes no public feeds, so only connected Pages can appear here.', + alternates: { + canonical: page === 1 ? `${siteUrl()}/fb` : `${siteUrl()}/fb?page=${page}`, + }, + }; +} + +/** + * @param {{ searchParams: Promise> }} props + */ +export default async function FacebookIndexPage({ searchParams }) { + return ; +} diff --git a/apps/web/src/app/ig/[username]/page.jsx b/apps/web/src/app/ig/[username]/page.jsx new file mode 100644 index 0000000..77503b9 --- /dev/null +++ b/apps/web/src/app/ig/[username]/page.jsx @@ -0,0 +1,55 @@ +import { notFound } from 'next/navigation'; +import { instagramSource } from '@rssamplifier/social'; + +import { socialFeed, socialMetadata } from '../../../lib/socialPage.js'; +import AddSocialSource from '../../AddSocialSource.jsx'; +import FeedPage from '../../[slug]/page.jsx'; + +export const dynamic = 'force-dynamic'; + +/** + * One Instagram account, at `/ig/nasa`. + * + * Same arrangement as `/r/[subreddit]` and `/x/[username]`: the page is + * `/{slug}`'s, and what this route contributes is the name and the canonical + * tag. Three platforms sharing one page is the payoff for having built the + * namespace machinery once. + * + * @param {{ params: Promise<{ username: string }> }} props + */ +export async function generateMetadata({ params }) { + const { username } = await params; + const source = instagramSource(`ig/${username}`); + if (!source) return { title: 'Not found', robots: { index: false, follow: false } }; + + return socialMetadata({ + feed: await socialFeed(source.ref), + canonical: source.path, + label: source.title, + network: 'instagram', + }); +} + +/** + * @param {{ params: Promise<{ username: string }> }} props + */ +export default async function InstagramAccountPage({ params }) { + const { username } = await params; + + const source = instagramSource(`ig/${username}`); + if (!source) notFound(); + + const feed = await socialFeed(source.ref); + if (!feed) { + return ( + + ); + } + + return FeedPage({ params: Promise.resolve({ slug: String(feed.slug) }) }); +} diff --git a/apps/web/src/app/ig/page.jsx b/apps/web/src/app/ig/page.jsx new file mode 100644 index 0000000..7ba2ac7 --- /dev/null +++ b/apps/web/src/app/ig/page.jsx @@ -0,0 +1,27 @@ +import { siteUrl } from '../../lib/db.js'; +import SocialIndex, { pageNumber } from '../SocialIndex.jsx'; + +export const dynamic = 'force-dynamic'; + +/** + * @param {{ searchParams: Promise> }} props + */ +export async function generateMetadata({ searchParams }) { + const page = pageNumber((await searchParams).page); + + return { + title: page === 1 ? 'Instagram' : `Instagram · page ${page}`, + description: + 'Instagram accounts and hashtags as feeds you can subscribe to — collected by RSS Amplifier and served from here, never from Instagram.', + alternates: { + canonical: page === 1 ? `${siteUrl()}/ig` : `${siteUrl()}/ig?page=${page}`, + }, + }; +} + +/** + * @param {{ searchParams: Promise> }} props + */ +export default async function InstagramIndexPage({ searchParams }) { + return ; +} diff --git a/apps/web/src/app/ig/tag/[tag]/page.jsx b/apps/web/src/app/ig/tag/[tag]/page.jsx new file mode 100644 index 0000000..de593fa --- /dev/null +++ b/apps/web/src/app/ig/tag/[tag]/page.jsx @@ -0,0 +1,50 @@ +import { notFound } from 'next/navigation'; +import { instagramSource } from '@rssamplifier/social'; + +import { socialFeed, socialMetadata } from '../../../../lib/socialPage.js'; +import AddSocialSource from '../../../AddSocialSource.jsx'; +import FeedPage from '../../../[slug]/page.jsx'; + +export const dynamic = 'force-dynamic'; + +/** + * One Instagram hashtag, at `/ig/tag/coffee`. + * + * @param {{ params: Promise<{ tag: string }> }} props + */ +export async function generateMetadata({ params }) { + const { tag } = await params; + const source = instagramSource(`https://www.instagram.com/explore/tags/${tag}/`); + if (!source) return { title: 'Not found', robots: { index: false, follow: false } }; + + return socialMetadata({ + feed: await socialFeed(source.ref), + canonical: source.path, + label: source.title, + network: 'instagram', + }); +} + +/** + * @param {{ params: Promise<{ tag: string }> }} props + */ +export default async function InstagramTagPage({ params }) { + const { tag } = await params; + + const source = instagramSource(`https://www.instagram.com/explore/tags/${tag}/`); + if (!source) notFound(); + + const feed = await socialFeed(source.ref); + if (!feed) { + return ( + + ); + } + + return FeedPage({ params: Promise.resolve({ slug: String(feed.slug) }) }); +} diff --git a/apps/web/src/app/layout.jsx b/apps/web/src/app/layout.jsx index d33bb4a..b05d870 100644 --- a/apps/web/src/app/layout.jsx +++ b/apps/web/src/app/layout.jsx @@ -298,7 +298,8 @@ export default function RootLayout({ children }) { apart also stops the row growing a third arm every time a platform is added. */}

    - Platforms: Reddit · X + Platforms: Reddit · X ·{' '} + Instagram · Facebook

    Machine-readable: MCP server · CLI ·{' '} diff --git a/apps/web/src/lib/sitemap.js b/apps/web/src/lib/sitemap.js index 284cb68..8ae6082 100644 --- a/apps/web/src/lib/sitemap.js +++ b/apps/web/src/lib/sitemap.js @@ -48,6 +48,10 @@ export const STATIC_PAGES = [ // addresses to keep, and it does not need the sitemap's help to do it. { path: '/r', changefreq: 'daily', priority: '0.8' }, { path: '/x', changefreq: 'daily', priority: '0.8' }, + { path: '/ig', changefreq: 'daily', priority: '0.8' }, + // Weekly rather than daily: a namespace that only ever holds Pages somebody + // connected by hand changes on a very different clock from the other three. + { path: '/fb', changefreq: 'weekly', priority: '0.6' }, { path: '/search', changefreq: 'daily', priority: '0.8' }, { path: '/submit', changefreq: 'weekly', priority: '0.7' }, { path: '/signup', changefreq: 'monthly', priority: '0.6' }, diff --git a/apps/web/src/lib/socialPage.js b/apps/web/src/lib/socialPage.js index 8fc788c..23fdc67 100644 --- a/apps/web/src/lib/socialPage.js +++ b/apps/web/src/lib/socialPage.js @@ -1,4 +1,5 @@ import { social } from '@rssamplifier/db'; +import { socialDisplayTitle } from '@rssamplifier/social'; import { db, siteUrl } from './db.js'; import { feedAlternates } from './subscribe.js'; @@ -47,17 +48,17 @@ export function socialMetadata({ feed, canonical, label, network }) { }; } + const name = socialDisplayTitle(feed, label); + return { - title: String(feed.title ?? label), - description: String( - feed.description ?? `${label}, mirrored by the RSS Amplifier directory.`, - ), + title: name, + description: String(feed.description ?? `${name}, mirrored by the RSS Amplifier directory.`), alternates: { canonical: url, // The same four formats the rewrites serve. No playlists: a timeline and // a subreddit carry no enclosures, so announcing an `.m3u` would be // advertising an empty file. - types: feedAlternates(url, String(feed.title ?? label)), + types: feedAlternates(url, name), }, other: { 'x-social-network': network }, }; diff --git a/apps/web/src/lib/socialRiver.js b/apps/web/src/lib/socialRiver.js index 95b5057..323eaa3 100644 --- a/apps/web/src/lib/socialRiver.js +++ b/apps/web/src/lib/socialRiver.js @@ -1,5 +1,11 @@ import { q, social } from '@rssamplifier/db'; -import { redditSource, xSource } from '@rssamplifier/social'; +import { + facebookSource, + instagramSource, + redditSource, + socialDisplayTitle, + xSource, +} from '@rssamplifier/social'; import { db, siteUrl } from './db.js'; import { @@ -87,7 +93,10 @@ export async function socialRiver({ const rows = await q.itemsForFeed(client, String(feed.id), riverLimit(rawLimit)); const channel = { - title: String(feed.title ?? label ?? ref), + // The canonical name when the stored title says nothing — most of the + // imported subreddits have not been crawled yet and carry the bare host as + // their title. See socialDisplayTitle. + title: socialDisplayTitle(feed, label ?? ref), description: String( feed.description ?? `${label ?? ref}, mirrored by the RSS Amplifier directory.`, ), @@ -155,3 +164,35 @@ export function xTarget(params) { const [canonical, query = null] = source.path.split('?'); return { ref: source.ref, canonical, label: source.title, query }; } + +/** + * The same for `/ig/…`, across both modes. + * + * `ig/` rather than a bare handle, because the parser is being asked + * "is this Instagram?" and a bare handle is ambiguous with X — see the ordering + * note in @rssamplifier/social's identify.js. The route already knows which + * platform it is holding, so it says so. + * + * @param {{ username?: string, tag?: string }} params + * @returns {{ ref: string, canonical: string, label: string }|null} + */ +export function instagramTarget(params) { + const source = params.tag + ? instagramSource(`https://www.instagram.com/explore/tags/${params.tag}/`) + : instagramSource(`ig/${params.username}`); + + if (!source) return null; + return { ref: source.ref, canonical: source.path, label: source.title }; +} + +/** + * And for `/fb/`. + * + * @param {{ page?: string }} params + * @returns {{ ref: string, canonical: string, label: string }|null} + */ +export function facebookTarget(params) { + const source = facebookSource(`fb/${params.page}`); + if (!source) return null; + return { ref: source.ref, canonical: source.path, label: source.title }; +} diff --git a/packages/db/src/social.js b/packages/db/src/social.js index 2894662..34bb612 100644 --- a/packages/db/src/social.js +++ b/packages/db/src/social.js @@ -137,11 +137,17 @@ export async function upsertSocialSource(db, source) { source.feedUrl, source.siteUrl ?? null, source.title, - // X sources are polled more often than the 60-minute default, because a - // timeline is the one thing in this directory where an hour old is - // visibly stale. §17's "active source: 5 minutes" is the ceiling the - // crawler's own interval learning then works down from. - source.network === 'x' ? 5 : 60, + // Provider-collected sources start faster than the 60-minute default, + // because a timeline is the one thing in this directory where an hour old + // is visibly stale. §17's "active source: 5 minutes" is the starting + // point, not the resting one: the crawler's interval learning backs a + // quiet Page or account off on its own, which is why a rarely-posting + // Facebook Page costs nothing to start fast. + // + // Reddit is excluded deliberately — it is a real feed on somebody else's + // server, and 50,026 of them at five minutes is how you get rate-limited + // off a platform. See markHostThrottled in queries.js. + source.network === 'reddit' ? 60 : 5, now, now, now, diff --git a/packages/feed/src/slug.js b/packages/feed/src/slug.js index 41eedef..3976865 100644 --- a/packages/feed/src/slug.js +++ b/packages/feed/src/slug.js @@ -34,6 +34,12 @@ const RESERVED = new Set([ // above are listed for. 'r', 'x', + 'ig', + 'fb', + // Reserved as well as the short forms, so a feed cannot take the name we + // would use if either namespace is ever spelled out. + 'instagram', + 'facebook', // The people index, for the same reason as the categories above: a feed // slugged 'authors' would still be served, but only Next's static segment // would answer and the blog's own page would be unreachable. diff --git a/packages/ingest/src/crawl.js b/packages/ingest/src/crawl.js index eee2210..c1f7651 100644 --- a/packages/ingest/src/crawl.js +++ b/packages/ingest/src/crawl.js @@ -1,6 +1,6 @@ import { resolveFeed, scrapeFeed, feedTopics } from '@rssamplifier/feed'; import { q, authors } from '@rssamplifier/db'; -import { fetchXSource } from '@rssamplifier/social'; +import { fetchSocialSource, isCollected } from '@rssamplifier/social'; import { prepareCredits } from './enrich.js'; import { @@ -202,10 +202,10 @@ export function topicsFrom(feed = {}, storedItems = []) { async function collectSocial(feed, opts) { const runtime = opts.xRuntime ?? null; if (!runtime) { - return { ok: false, throttled: true, retryAfter: 3600, error: 'x-runtime-unavailable' }; + return { ok: false, throttled: true, retryAfter: 3600, error: 'social-runtime-unavailable' }; } - return (opts.x ?? fetchXSource)(feed, { runtime }); + return (opts.x ?? fetchSocialSource)(feed, { runtime }); } export async function crawlFeed(db, feed, opts = {}) { @@ -215,10 +215,16 @@ export async function crawlFeed(db, feed, opts = {}) { // The third way in. A feed is fetched, a scraped source is read off a page, // and a social source is collected through a provider — three methods, one // return shape, and everything past this point is identical for all three. - // That is what keeps X out of the rest of the pipeline entirely: dedupe, - // interval learning, keyword extraction, credits, FTS and syndication never - // learn that it exists (§30, AC-8). - const social = feed.social_network === 'x' ? 'x' : null; + // That is what keeps the platforms out of the rest of the pipeline entirely: + // dedupe, interval learning, keyword extraction, credits, FTS and syndication + // never learn that any of them exists (§30, AC-8). + // + // `isCollected` rather than a list of network names, because the two + // questions differ: Reddit *is* a social network and is *not* collected — it + // publishes real RSS and is fetched like any blog, and `/r/` is about naming + // it rather than about reading it. @rssamplifier/social owns that + // distinction, so adding a platform never edits this file. + const social = isCollected(feed); // A provider-backed source polls on a five-minute floor rather than an hour's // — see SOCIAL_MIN_INTERVAL. The floor is passed to every scheduling call diff --git a/packages/ingest/test/social-crawl.test.js b/packages/ingest/test/social-crawl.test.js index b9e0259..c242c10 100644 --- a/packages/ingest/test/social-crawl.test.js +++ b/packages/ingest/test/social-crawl.test.js @@ -178,14 +178,14 @@ test('a rate limit moves the schedule and touches no health column (§16)', asyn assert.equal(items.length, 2); }); -test('no X runtime is a reschedule, not a verdict on the source', async () => { +test('no social runtime is a reschedule, not a verdict on the source', async () => { const feed = await seedXSource(); const result = await crawlFeed(db, feed, {}); assert.equal(result.ok, false); assert.equal(result.throttled, true); - assert.equal(result.error, 'x-runtime-unavailable'); + assert.equal(result.error, 'social-runtime-unavailable'); const after = (await db.execute({ sql: 'select * from feeds where id = ?', args: [feed.id] })) .rows[0]; diff --git a/packages/social/index.js b/packages/social/index.js index d0fdc22..b0575b3 100644 --- a/packages/social/index.js +++ b/packages/social/index.js @@ -53,4 +53,33 @@ export { redditSpecFromRef, } from './src/reddit/canonical.js'; +export { + parseInstagramInput, + instagramRef, + instagramUrl, + instagramPath, + instagramSlug, + instagramTitle, + instagramSource, + instagramSpecFromRef, + INSTAGRAM_MODES, +} from './src/instagram/canonical.js'; +export { fetchInstagramSource } from './src/instagram/fetch.js'; + +export { + parseFacebookInput, + facebookRef, + facebookUrl, + facebookPath, + facebookSlug, + facebookTitle, + facebookSource, + facebookSpecFromRef, +} from './src/facebook/canonical.js'; +export { fetchFacebookSource, pageToken, connectedPages } from './src/facebook/fetch.js'; + +export { failureResult, retryAfterFor, ANOMALY_SECONDS, UNCONFIGURED_SECONDS } from './src/failure.js'; +export { fetchSocialSource, isCollected } from './src/collect.js'; + export { socialSourceFrom, socialPathFor, SOCIAL_NETWORKS } from './src/identify.js'; +export { socialDisplayTitle } from './src/display.js'; diff --git a/packages/social/src/collect.js b/packages/social/src/collect.js new file mode 100644 index 0000000..07baae6 --- /dev/null +++ b/packages/social/src/collect.js @@ -0,0 +1,54 @@ +/** + * One entry point for the crawler, whatever the platform is. + * + * `crawlFeed` should not grow a branch per network — it already holds three + * ingestion methods apart (fetch, scrape, collect) and that is the right number. + * So the choice of collector lives here, and the crawler asks one question: + * is this row collected rather than fetched? + * + * Reddit is deliberately absent. It publishes real RSS, so it is *fetched* like + * any blog and needs no collector at all — the `/r/` namespace is about naming + * it, not about reading it. That asymmetry is the whole reason `social_network` + * and "needs a collector" are two different questions. + */ + +import { fetchXSource } from './x/fetch.js'; +import { fetchInstagramSource } from './instagram/fetch.js'; +import { fetchFacebookSource } from './facebook/fetch.js'; + +/** + * The networks that cannot simply be fetched, and what collects them. + * + * Adding a platform is an entry here plus a `canonical.js` and a `fetch.js`. + * It is deliberately not a registry with lifecycle hooks: three collectors that + * share a return shape are easier to read than a framework that abstracts over + * two of them. + */ +const COLLECTORS = { + x: fetchXSource, + instagram: fetchInstagramSource, + facebook: fetchFacebookSource, +}; + +/** + * Is this row collected through a provider rather than fetched from a document? + * + * @param {{ social_network?: string|null }} feed + * @returns {boolean} + */ +export function isCollected(feed) { + return Boolean(COLLECTORS[String(feed?.social_network ?? '')]); +} + +/** + * Collect one social source, in the shape `crawlFeed` expects. + * + * @param {object} feed the row + * @param {{ runtime: object, limit?: number, signal?: AbortSignal }} opts + * @returns {Promise} + */ +export async function fetchSocialSource(feed, opts) { + const collect = COLLECTORS[String(feed?.social_network ?? '')]; + if (!collect) return { ok: false, error: `no collector for ${feed?.social_network}` }; + return collect(feed, opts); +} diff --git a/packages/social/src/display.js b/packages/social/src/display.js new file mode 100644 index 0000000..9b54828 --- /dev/null +++ b/packages/social/src/display.js @@ -0,0 +1,68 @@ +/** + * What to call a social source on a page, when its own title is no use. + * + * The 50,026 subreddits were bulk-imported from OPML, and the crawler has read + * a few hundred of them. Until it reaches one, the row's `title` is whatever + * the OPML said — and for that catalogue it is very often the bare host, so + * `/r/programming` renders a heading that says **"reddit.com"**. + * + * The fix is not to rewrite the stored titles. A title is the publisher's, and + * the crawler replaces it with the real one soon enough; overwriting the column + * would fight `markCrawlSuccess` on every subsequent crawl. This is a render + * decision instead: when the stored title carries no information, show the + * canonical name — `r/programming`, `@nasa on Instagram` — which we always know + * because the ref encodes it. + * + * Deliberately conservative. A real title always wins, including one that + * merely *contains* the platform's name, because "Reddit Blog" is a genuine + * feed title and not a placeholder. + */ + +/** + * Titles that say nothing about the source they are attached to. + * + * Matched whole, case-insensitively, after trimming. Anything longer or more + * specific is somebody's actual title and is left alone. + */ +const EMPTY_TITLES = new Set([ + 'reddit', + 'reddit.com', + 'www.reddit.com', + 'old.reddit.com', + 'instagram', + 'instagram.com', + 'www.instagram.com', + 'facebook', + 'facebook.com', + 'www.facebook.com', + 'x', + 'x.com', + 'twitter', + 'twitter.com', + '(untitled)', + 'untitled', + 'rss', + 'feed', + 'rss feed', +]); + +/** + * The name to render for a social row. + * + * @param {{ title?: unknown, feed_url?: unknown }} feed the stored row + * @param {string} label the canonical name, from the ref + * @returns {string} + */ +export function socialDisplayTitle(feed, label) { + const stored = String(feed?.title ?? '').trim(); + + if (!stored) return label; + if (EMPTY_TITLES.has(stored.toLowerCase())) return label; + + // A title that is just the URL it was imported from is the same non-answer in + // a different shape, and the OPML catalogue is full of them. + const url = String(feed?.feed_url ?? '').trim(); + if (url && (stored === url || url.includes(stored))) return label; + + return stored; +} diff --git a/packages/social/src/facebook/canonical.js b/packages/social/src/facebook/canonical.js new file mode 100644 index 0000000..dbdc47a --- /dev/null +++ b/packages/social/src/facebook/canonical.js @@ -0,0 +1,201 @@ +/** + * Facebook — and the honest limits on what `/fb/` can ever be. + * + * Read this before adding to it, because the shape of this file is decided by + * something outside the codebase. + * + * **There is no way to read an arbitrary public Facebook Page.** Three doors, + * all measured rather than assumed, on 2026-08-29: + * + * - the old `facebook.com/feeds/page.php?format=rss20` endpoint answers 404; + * it was removed, not deprecated + * - `mbasic.facebook.com/` answers 200 with a login wall + * - RSSHub, which carries a thousand namespaces and maintains Twitter and + * Instagram, has no Facebook namespace at all + * + * The remaining door is the Graph API, and it only opens for Pages the caller + * **administers**: reading somebody else's public Page needs the + * `Page Public Content Access` feature, which requires App Review and business + * verification and is granted rarely. So a Facebook source here is not + * something a stranger can submit — it is something the Page's own operator + * connects, by supplying a Page Access Token. + * + * That is a different bargain from the rest of this directory, where anyone may + * submit anything, and `/fb/` should not pretend otherwise: a page nobody has + * connected a token for is not "not crawled yet", it is "not collectable", and + * the page says so. + * + * What is deliberately *not* here: anything that drives a logged-in Facebook + * session against the login wall. It breaks constantly, it is against Meta's + * terms, and it would put an account of ours at risk to serve a directory + * nobody is paying for. + */ + +/** + * A Page's public name — the `/PageName` in a Facebook URL. + * + * Facebook calls it a "username" or "vanity URL" and permits letters, digits + * and dots, minimum five characters. + */ +const VANITY = /^[A-Za-z0-9.]{5,60}$/; + +/** A numeric Page id, which is what the Graph API actually addresses. */ +const PAGE_ID = /^[0-9]{6,25}$/; + +const HOSTS = new Set([ + 'facebook.com', + 'www.facebook.com', + 'm.facebook.com', + 'mbasic.facebook.com', + 'web.facebook.com', + 'fb.com', + 'www.fb.com', +]); + +/** + * Segments that are Facebook's own furniture rather than a Page name. + * + * `profile.php` is the important one: it is how every personal profile without + * a vanity URL is addressed, and a personal profile is not collectable by any + * means at all — there is no Graph API for somebody's own timeline. + */ +const NOT_A_PAGE = new Set([ + 'profile.php', + 'people', + 'groups', + 'events', + 'marketplace', + 'watch', + 'gaming', + 'story.php', + 'photo.php', + 'permalink.php', + 'sharer', + 'sharer.php', + 'dialog', + 'login', + 'login.php', + 'help', + 'policies', + 'privacy', + 'terms', + 'settings', + 'pages', + 'pg', +]); + +/** + * Read whatever a person pasted and say which Page they meant. + * + * @param {unknown} input + * @returns {{ mode: 'page', page: string }|null} + */ +export function parseFacebookInput(input) { + const raw = String(input ?? '').trim(); + if (!raw) return null; + + // `fb/SomePage` and `fb.com/SomePage` shorthands, plus a bare numeric id. + const short = /^\/?(?:fb|facebook)\/([A-Za-z0-9.]{5,60})\/?$/i.exec(raw); + if (short) return { mode: 'page', page: short[1] }; + + // A bare string of digits is deliberately NOT read as a Page id. It is also a + // plausible X list id, and a directory that guesses which platform a bare + // number belongs to will guess wrong in front of somebody eventually. + + const withScheme = /^https?:\/\//i.test(raw) ? raw : `https://${raw.replace(/^\/+/, '')}`; + + let url; + try { + url = new URL(withScheme); + } catch { + return null; + } + + if (!HOSTS.has(url.hostname.toLowerCase())) return null; + + const segments = url.pathname.split('/').filter(Boolean); + if (segments.length === 0) return null; + + const first = segments[0]; + const lower = first.toLowerCase(); + + // `/pages/Some-Name/123456` — the old form, whose numeric id is the reliable + // half and the only half the Graph API can use. + if (lower === 'pages') { + const id = segments.find((segment) => PAGE_ID.test(segment)); + return id ? { mode: 'page', page: id } : null; + } + + if (NOT_A_PAGE.has(lower)) return null; + + // A post, a photo or a video under a Page is a thing to read, not a source. + if (segments[1] && !['about', ''].includes(segments[1].toLowerCase())) return null; + + if (PAGE_ID.test(first)) return { mode: 'page', page: first }; + return VANITY.test(first) ? { mode: 'page', page: first } : null; +} + +/** + * @param {{ mode: string, page: string }} spec + * @returns {string|null} + */ +export function facebookRef(spec) { + if (!spec?.page) return null; + return spec.mode === 'page' ? `fb:page:${spec.page.toLowerCase()}` : null; +} + +/** The canonical address on Facebook's side. */ +export function facebookUrl(spec) { + return spec?.page ? `https://www.facebook.com/${spec.page}` : null; +} + +/** Where it lives on this site. */ +export function facebookPath(spec) { + return spec?.page ? `/fb/${spec.page}` : null; +} + +/** A title for a Page whose first collection has not happened yet. */ +export function facebookTitle(spec) { + return spec?.page ? `${spec.page} on Facebook` : 'Facebook'; +} + +/** The directory slug. */ +export function facebookSlug(spec) { + const ref = facebookRef(spec); + if (!ref) return null; + return ref + .replace(/^fb:page:/, 'fb-') + .replace(/[^a-z0-9-]+/g, '-') + .replace(/-{2,}/g, '-') + .replace(/^-+|-+$/g, ''); +} + +/** + * Everything a source row needs, from one pasted string. + * + * @param {unknown} input + * @returns {object|null} + */ +export function facebookSource(input) { + const spec = parseFacebookInput(input); + if (!spec) return null; + + const ref = facebookRef(spec); + const url = facebookUrl(spec); + const path = facebookPath(spec); + const slug = facebookSlug(spec); + if (!ref || !url || !path || !slug) return null; + + return { ...spec, ref, url, path, slug, title: facebookTitle(spec) }; +} + +/** + * Rebuild the spec from a stored ref. + * + * @param {unknown} ref + * @returns {{ mode: 'page', page: string }|null} + */ +export function facebookSpecFromRef(ref) { + const match = /^fb:page:([A-Za-z0-9.]{5,60}|[0-9]{6,25})$/.exec(String(ref ?? '')); + return match ? { mode: 'page', page: match[1] } : null; +} diff --git a/packages/social/src/facebook/fetch.js b/packages/social/src/facebook/fetch.js new file mode 100644 index 0000000..1e6fde9 --- /dev/null +++ b/packages/social/src/facebook/fetch.js @@ -0,0 +1,263 @@ +/** + * Collecting Facebook, for the Pages somebody has actually connected. + * + * See `./canonical.js` for why this is the only shape available: there is no + * way to read an arbitrary public Page, so a Facebook source is not something a + * stranger submits — it is something a Page's operator connects by supplying a + * Page Access Token. + * + * **Tokens live in the environment, keyed by Page**, never in a table, for + * exactly the reason X's session cookies do not: a Page Access Token can post + * as the Page. `FB_PAGE_TOKENS` is a JSON array, and a Page with no entry in it + * is not "not crawled yet" — it is not collectable, and both the crawler and + * the page say so rather than retrying for ever. + * + * This is the one collector in the package that talks to a real, supported, + * documented API rather than a bridge, which makes it the least likely of the + * three to break and the one with the smallest reach. That trade is Meta's, not + * ours. + */ + +import { providerGet } from '../x/providers/http.js'; +import { XUnavailable, XNoSuchSource } from '../x/errors.js'; +import { failureResult } from '../failure.js'; +import { facebookSpecFromRef } from './canonical.js'; + +/** + * Pinned rather than floating. Graph deprecates a version roughly every two + * years with a hard cutoff, and an unpinned call silently changes shape under + * you; a pinned one fails loudly on a date that can be looked up. + */ +const GRAPH = 'https://graph.facebook.com/v21.0'; + +/** + * What a post is, in the fewest fields that render. + * + * `message` is the caption and is absent on a share with no comment; + * `permalink_url` is the only stable public address; `full_picture` is the + * attached image where there is one. `story` carries the "X shared a link" + * sentence that stands in for a caption when there is none. + */ +const FIELDS = 'id,message,story,created_time,permalink_url,full_picture'; + +/** + * Collect one Facebook Page. + * + * @param {{ social_ref?: string, feed_url?: string, item_count?: number }} feed + * @param {{ runtime: { env?: Record, onEvent?: Function }, signal?: AbortSignal }} opts + * @returns {Promise} + */ +export async function fetchFacebookSource(feed, opts) { + const spec = facebookSpecFromRef(feed?.social_ref); + if (!spec) return { ok: false, error: 'invalid-facebook-ref' }; + + const env = opts.runtime?.env ?? process.env; + const onEvent = opts.runtime?.onEvent ?? (() => {}); + + // The Page's own spelling, for anything a human reads. + // + // `social_ref` is lowercased because it is an identity — `fb:page:somepage` + // has to match however the URL was typed — but lowercasing is not ours to do + // to somebody's name, and every item title carries it. `feed_url` kept the + // original casing at submission, so display comes from there and identity + // stays from the ref. + const display = displayName(feed?.feed_url) ?? spec.page; + + try { + const token = pageToken(env, spec.page); + if (!token) { + // Not an outage and not a broken Page: nobody has connected it. Phrased + // so `retryAfterFor` gives it the hour it deserves rather than retrying + // every twenty minutes for a token that is not coming. + throw new XUnavailable(`facebook: page ${spec.page} is not connected`); + } + + const url = new URL(`${GRAPH}/${encodeURIComponent(spec.page)}/posts`); + url.searchParams.set('fields', FIELDS); + url.searchParams.set('limit', '50'); + url.searchParams.set('access_token', token); + + onEvent('facebook.fetch.started', { ref: feed.social_ref }); + + const { body } = await providerGet(url, { + provider: 'facebook-graph', + headers: { accept: 'application/json' }, + timeoutMs: Number(env.X_FETCH_TIMEOUT_MS) || undefined, + // Injected by tests, so a suite never reaches a real upstream (§51). + fetch: opts.runtime?.fetch, + signal: opts.signal, + }); + + const payload = JSON.parse(body); + + // Graph answers 200 with an `error` object for several real failures, so + // the status code alone is not the answer — the same trap RSSHub sets. + if (payload?.error) { + const code = Number(payload.error.code); + // 100 (unknown path) and 803 (unresolvable alias) mean the Page is gone + // or renamed; that is the one failure genuinely about the source. + if (code === 100 || code === 803) { + throw new XNoSuchSource(`facebook: ${payload.error.message ?? 'no such page'}`); + } + throw new XUnavailable(`facebook: ${payload.error.message ?? 'graph error'}`); + } + + const items = (payload?.data ?? []).map((post) => toItem(post, display)).filter(Boolean); + + if (items.length === 0 && Number(feed?.item_count ?? 0) > 0) { + onEvent('facebook.fetch.failed', { ref: feed.social_ref, error: 'empty-result' }); + return { ok: false, throttled: true, retryAfter: 20 * 60, error: 'empty-result' }; + } + + onEvent('facebook.fetch.success', { ref: feed.social_ref, itemCount: items.length }); + + return { + ok: true, + feedUrl: feed.feed_url, + feed: { + title: `${display} on Facebook`, + description: `Posts from the ${display} Page on Facebook, mirrored by RSS Amplifier.`, + siteUrl: String(feed.feed_url), + language: null, + imageUrl: null, + categories: [], + kind: 'blog', + items, + }, + }; + } catch (error) { + onEvent('facebook.fetch.failed', { ref: feed.social_ref, error: String(error?.message ?? error) }); + return failureResult(error); + } +} + +/** + * One Graph post as one of our items. + * + * @param {object} post + * @param {string} display the Page's own spelling of its name + * @returns {object|null} + */ +function toItem(post, display) { + if (!post?.id) return null; + + // A caption, or the sentence Facebook writes when there is none. A post with + // neither is a bare photo, and gets a title that says so rather than an empty + // one — every format we render needs a title. + const text = String(post.message ?? post.story ?? '').trim(); + const first = text.split('\n').find(Boolean) ?? ''; + const title = first ? clip(first, 110) : '(photo)'; + + return { + // The Graph post id, never the permalink: a Page rename rewrites every + // permalink it has ever had, and a URL-keyed dedupe would re-ingest the + // whole Page the day that happens. + guid: `fb:${post.id}`, + url: post.permalink_url ?? `https://www.facebook.com/${display}`, + title: `${display}: ${title}`, + summary: clip(text, 400) || null, + contentHtml: html(text, post.full_picture, post.permalink_url), + author: display, + publishedAt: post.created_time ?? null, + imageUrl: post.full_picture ?? null, + categories: [], + audio: null, + }; +} + +/** + * The rendered body. Escaped rather than sanitised, because none of it is + * markup: Graph returns a plain-text caption, and the only tags in the output + * are ones this function wrote. + */ +function html(text, picture, permalink) { + const blocks = []; + if (text) blocks.push(`

    ${escapeHtml(text).replace(/\n/g, '
    ')}

    `); + if (picture) { + const img = ``; + blocks.push(permalink ? `

    ${img}

    ` : `

    ${img}

    `); + } + return blocks.join('\n') || '

    (no text)

    '; +} + +/** + * The token for one Page, from `FB_PAGE_TOKENS`. + * + * Structured JSON rather than parallel lists, for the reason spelled out in + * `../x/sessions.js`: positional pairs silently mispair when one entry is + * removed, and a mispaired credential authenticates as nobody. + * + * FB_PAGE_TOKENS=[{"page":"MyPage","token":"EAA..."}] + * + * Matched case-insensitively on either the vanity name or the numeric id, so a + * Page connected as `MyPage` is found by a source stored as `mypage`. + * + * @param {Record} env + * @param {string} page + * @returns {string|null} + */ +export function pageToken(env, page) { + const raw = String(env.FB_PAGE_TOKENS ?? '').trim(); + if (!raw) return null; + + try { + const parsed = JSON.parse(raw); + if (!Array.isArray(parsed)) return null; + + const wanted = String(page).toLowerCase(); + const match = parsed.find( + (entry) => + String(entry?.page ?? entry?.pageId ?? entry?.id ?? '').toLowerCase() === wanted, + ); + + const token = String(match?.token ?? match?.accessToken ?? '').trim(); + return token || null; + } catch { + // A malformed FB_PAGE_TOKENS must not stop the crawler booting, and must + // not be reported as "this Page is broken" either — it reads as no token, + // which is the honest answer. + return null; + } +} + +/** Which Pages have a token at all, for the status page and the add form. */ +export function connectedPages(env = process.env) { + const raw = String(env.FB_PAGE_TOKENS ?? '').trim(); + if (!raw) return []; + + try { + const parsed = JSON.parse(raw); + if (!Array.isArray(parsed)) return []; + return parsed + .map((entry) => String(entry?.page ?? entry?.pageId ?? entry?.id ?? '').trim()) + .filter(Boolean); + } catch { + return []; + } +} + +/** + * The Page name as it was written, read back off the stored URL. + * + * @param {unknown} feedUrl + * @returns {string|null} + */ +function displayName(feedUrl) { + const match = /facebook\.com\/([A-Za-z0-9.]{5,60})\/?$/.exec(String(feedUrl ?? '')); + return match ? match[1] : null; +} + +/** @param {string} value @param {number} max */ +function clip(value, max) { + const text = String(value ?? '').replace(/\s+/g, ' ').trim(); + return text.length > max ? `${text.slice(0, max).trimEnd()}…` : text; +} + +/** @param {string} value */ +function escapeHtml(value) { + return String(value ?? '') + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"'); +} diff --git a/packages/social/src/failure.js b/packages/social/src/failure.js new file mode 100644 index 0000000..73319eb --- /dev/null +++ b/packages/social/src/failure.js @@ -0,0 +1,69 @@ +/** + * What a collection failure means for the source, for every platform at once. + * + * This exists because the rule is easy to state, easy to get wrong, and + * expensive when it is: **only a source that does not exist may count against + * the source.** Everything else — a rate limit, a provider outage, a dead + * session, nothing configured to collect with — is a fact about us or about the + * upstream. + * + * `markCrawlFailure` retires a feed at ten consecutive failures, and a + * provider-collected source polls on a five-minute floor. Ten strikes is fifty + * minutes. PR #157 was written because that rule was implemented once, for X, + * and implemented wrongly; the fix belongs in one place rather than being + * rediscovered by each new platform. Instagram and Facebook were added against + * this function rather than against a copy of X's version of it. + */ + +/** + * How long to wait after an upstream anomaly, in seconds. + * + * Long enough that a hundred queued sources do not each rediscover the same + * outage inside one tick; short enough that a real recovery is picked up within + * the hour. + */ +export const ANOMALY_SECONDS = 20 * 60; + +/** + * And after "there is nothing configured to collect with", which is a + * deployment that has not happened rather than a service that is down. Asking + * again in twenty minutes answers no sooner than asking in an hour, and costs a + * thousand pointless wake-ups a day across the directory. + */ +export const UNCONFIGURED_SECONDS = 3600; + +/** + * Turn a thrown collection error into the result `crawlFeed` expects. + * + * @param {Error & { name?: string, retryAfter?: number|null }} error + * @returns {{ ok: false, error: string, throttled?: true, retryAfter?: number }} + */ +export function failureResult(error) { + const message = String(error?.message ?? 'collect-failed').slice(0, 200); + + // The one failure that is genuinely about the source: deleted, suspended, + // renamed, or protected. This is the only path to markCrawlFailure. + if (error?.name === 'XNoSuchSource') return { ok: false, error: message }; + + return { ok: false, throttled: true, retryAfter: retryAfterFor(error), error: message }; +} + +/** + * @param {Error & { name?: string, retryAfter?: number|null }} error + * @returns {number} seconds + */ +export function retryAfterFor(error) { + if (error?.name === 'XRateLimited') { + const named = Number(error.retryAfter); + return named > 0 ? named : ANOMALY_SECONDS; + } + + // Distinguished by message rather than by type, because `XUnavailable` covers + // both "nothing is configured" and "the thing that is configured is down", + // and those deserve very different patience. + if (/no .* provider is configured|no RSSHUB_BASE_URL|not connected/i.test(String(error?.message ?? ''))) { + return UNCONFIGURED_SECONDS; + } + + return ANOMALY_SECONDS; +} diff --git a/packages/social/src/identify.js b/packages/social/src/identify.js index e1bfd0e..03345cd 100644 --- a/packages/social/src/identify.js +++ b/packages/social/src/identify.js @@ -1,5 +1,5 @@ /** - * One question asked of both platforms: "is this URL one of ours?" + * One question asked of every platform: "is this URL one of ours?" * * The submit path, the importer and the crawler all need to recognise a social * URL before they know which platform it belongs to, and none of them should @@ -9,23 +9,66 @@ * It matters that this runs *before* the ordinary feed resolver. Reddit does * publish RSS, so `https://www.reddit.com/r/programming/` resolves perfectly * well as a plain feed — and lands as an untyped row at a slug of its own, - * which is exactly the 50,099-row outcome `reddit/canonical.js` describes. The + * which is exactly the 50,026-row outcome `reddit/canonical.js` describes. The * difference between a subreddit at `/r/programming` and a subreddit filed * among the blogs is entirely a matter of who looks at the URL first. */ import { xSource } from './x/canonical.js'; import { redditSource } from './reddit/canonical.js'; +import { instagramSource } from './instagram/canonical.js'; +import { facebookSource } from './facebook/canonical.js'; /** The networks that get a namespace of their own. */ -export const SOCIAL_NETWORKS = Object.freeze(['x', 'reddit']); +export const SOCIAL_NETWORKS = Object.freeze(['reddit', 'x', 'instagram', 'facebook']); + +/** + * The prefix each network's refs use, and the path each lives under. + * + * A table rather than four `if`s in `socialPathFor`, because that function is + * called from listings and would otherwise grow a branch per platform in a + * place nobody thinks to look when adding one. + */ +const BY_PREFIX = { + 'r:sub': (name) => `/r/${name}`, + 'r:user': (name) => `/r/u/${name}`, + 'x:user': (name) => `/x/${name}`, + 'x:replies': (name) => `/x/${name}/replies`, + 'x:media': (name) => `/x/${name}/media`, + 'x:list': (id) => `/x/list/${id}`, + 'x:search': (query) => `/x/search?q=${encodeURIComponent(query)}`, + 'ig:user': (name) => `/ig/${name}`, + 'ig:tag': (tag) => `/ig/tag/${tag}`, + 'fb:page': (page) => `/fb/${page}`, +}; + +/** + * The recognisers, in the order they get to claim a string. + * + * **The order is load-bearing in exactly one place.** A bare `@handle` is a + * valid input to both X and Instagram, and X is tried first because it had the + * spelling first and because `/submit` has accepted it since PR #156. Instagram + * is reachable explicitly, as `ig/handle` or a full URL — see the note in + * `instagram/canonical.js`. Everything else is disambiguated by hostname and + * the order is irrelevant. + */ +const RECOGNISERS = [ + ['reddit', redditSource], + ['facebook', facebookSource], + // X before Instagram, and only because of the bare-handle case. Both accept a + // bare `@handle`; whichever is asked first wins it. Reddit and Facebook are + // above them because neither claims a bare handle at all, so their position is + // arbitrary and their hostnames disambiguate them. + ['x', xSource], + ['instagram', instagramSource], +]; /** * Recognise a social source, or say it is not one. * * @param {unknown} input anything a person or an importer might supply * @returns {{ - * network: 'x'|'reddit', + * network: 'reddit'|'x'|'instagram'|'facebook', * ref: string, * slug: string, * title: string, @@ -35,33 +78,23 @@ export const SOCIAL_NETWORKS = Object.freeze(['x', 'reddit']); * }|null} */ export function socialSourceFrom(input) { - const reddit = redditSource(input); - if (reddit) { - return { - network: 'reddit', - ref: reddit.ref, - slug: reddit.slug, - title: reddit.title, - path: reddit.path, - feedUrl: reddit.feedUrl, - siteUrl: reddit.siteUrl, - }; - } + for (const [network, recognise] of RECOGNISERS) { + const found = recognise(input); + if (!found) continue; - const x = xSource(input); - if (x) { return { - network: 'x', - ref: x.ref, - slug: x.slug, - title: x.title, - path: x.path, - // For X there is no document at this address and nothing ever fetches it. - // It is here because `feeds.feed_url` is `not null unique` and is the - // column every other surface reads to show a human where a feed came - // from — see the header of x/canonical.js. - feedUrl: x.url, - siteUrl: x.url, + network, + ref: found.ref, + slug: found.slug, + title: found.title, + path: found.path, + // Reddit is the only one of the four with a document at the other end. + // For the rest this is the canonical public address of the thing on the + // platform's side: nothing fetches it, and it is stored because + // `feeds.feed_url` is `not null unique` and is what every surface reads + // to show a human where a feed came from. + feedUrl: found.feedUrl ?? found.url, + siteUrl: found.siteUrl ?? found.url ?? null, }; } @@ -75,28 +108,22 @@ export function socialSourceFrom(input) { * row whose ref predates this code — so a caller can use this everywhere * without checking whether a feed is social first. * - * @param {{ social_network?: string|null, social_ref?: string|null, slug?: string }} feed + * @param {{ social_ref?: string|null, slug?: string }} feed * @returns {string} */ export function socialPathFor(feed) { const ref = feed?.social_ref ? String(feed.social_ref) : null; - const slug = String(feed?.slug ?? ''); + if (!ref) return `/${String(feed?.slug ?? '')}`; - if (ref?.startsWith('r:')) { - const [, mode, name] = ref.split(':'); - if (name) return mode === 'user' ? `/r/u/${name}` : `/r/${name}`; - } + // Split on the *second* colon only: a search ref is `x:search:` and a + // query may contain colons of its own (`from:OpenAI`), so splitting on every + // colon would truncate it. + const separator = ref.indexOf(':', ref.indexOf(':') + 1); + if (separator === -1) return `/${String(feed?.slug ?? '')}`; - if (ref?.startsWith('x:')) { - const separator = ref.indexOf(':', 2); - const mode = ref.slice(2, separator); - const rest = ref.slice(separator + 1); - if (mode === 'user') return `/x/${rest}`; - if (mode === 'replies') return `/x/${rest}/replies`; - if (mode === 'media') return `/x/${rest}/media`; - if (mode === 'list') return `/x/list/${rest}`; - if (mode === 'search') return `/x/search?q=${encodeURIComponent(rest)}`; - } + const prefix = ref.slice(0, separator); + const rest = ref.slice(separator + 1); - return `/${slug}`; + const build = BY_PREFIX[prefix]; + return build && rest ? build(rest) : `/${String(feed?.slug ?? '')}`; } diff --git a/packages/social/src/instagram/canonical.js b/packages/social/src/instagram/canonical.js new file mode 100644 index 0000000..da72a23 --- /dev/null +++ b/packages/social/src/instagram/canonical.js @@ -0,0 +1,205 @@ +/** + * Instagram, named the way Instagram names itself. + * + * The third platform through this door and the first one that cost almost + * nothing, which is the point of having built the door. Instagram is X's shape + * rather than Reddit's — it publishes no feeds, so posts are collected through a + * provider and mirrored here — so this file is X's `canonical.js` with a + * different set of URL rules, and everything downstream is already written. + * + * Two modes, not five. An account and a hashtag are the two things Instagram + * exposes that behave like a feed; saved posts, stories and the explore tab are + * either private, expiring, or personalised, and none of the three is a thing a + * stranger can subscribe to. + */ + +/** The two things worth subscribing to. */ +export const INSTAGRAM_MODES = Object.freeze(['user', 'hashtag']); + +/** + * Instagram's rule for a handle: 1–30 of `[A-Za-z0-9._]`. + * + * Dots are legal here and are not legal on X, which is the one difference that + * matters when this file is read next to `../x/canonical.js` — a handle regex + * copied from there would silently reject a third of Instagram. + */ +const HANDLE = /^[A-Za-z0-9._]{1,30}$/; + +/** A hashtag: letters, digits and underscore, no dots and no leading digit-only. */ +const HASHTAG = /^[A-Za-z0-9_]{1,60}$/; + +const HOSTS = new Set([ + 'instagram.com', + 'www.instagram.com', + 'm.instagram.com', + 'l.instagram.com', + 'instagr.am', +]); + +/** + * Path segments that are Instagram's own furniture rather than a handle. + * + * `explore` is the one that matters: `/explore/tags/coffee` is how a hashtag is + * addressed, so reading the first segment as a username would create an account + * called @explore that can never return a post. + */ +const NOT_A_HANDLE = new Set([ + 'explore', + 'p', + 'reel', + 'reels', + 'tv', + 'stories', + 'direct', + 'accounts', + 'about', + 'developer', + 'legal', + 'privacy', + 'terms', + 'challenge', +]); + +/** + * Read whatever a person pasted and say which Instagram source they meant. + * + * @param {unknown} input + * @returns {{ mode: 'user'|'hashtag', username?: string, tag?: string }|null} + */ +export function parseInstagramInput(input) { + const raw = String(input ?? '').trim(); + if (!raw) return null; + + // `#coffee` — the shorthand for a hashtag, checked before anything else + // because a `#` in a URL is a fragment and would be thrown away by a parse. + const tag = /^#([A-Za-z0-9_]{1,60})$/.exec(raw); + if (tag) return { mode: 'hashtag', tag: tag[1] }; + + // `ig/somebody` — the explicit shorthand, and the one that matters for + // submission. A *bare* `@somebody` is deliberately not routed here by + // `socialSourceFrom`: it is ambiguous between X and Instagram, and X had it + // first. This parser still accepts a bare handle because the `/ig/` + // route calls it already knowing which platform it is holding. + const short = /^\/?(?:ig|instagram)\/(@?[A-Za-z0-9._]{1,30})\/?$/i.exec(raw); + if (short) { + const name = short[1].replace(/^@/, ''); + return HANDLE.test(name) ? { mode: 'user', username: name } : null; + } + + const bare = raw.replace(/^@/, ''); + if (HANDLE.test(bare) && !raw.includes('/') && !raw.includes(':')) { + return { mode: 'user', username: bare }; + } + + const withScheme = /^https?:\/\//i.test(raw) ? raw : `https://${raw.replace(/^\/+/, '')}`; + + let url; + try { + url = new URL(withScheme); + } catch { + return null; + } + + if (!HOSTS.has(url.hostname.toLowerCase())) return null; + + const segments = url.pathname.split('/').filter(Boolean); + if (segments.length === 0) return null; + + const first = segments[0].toLowerCase(); + + if (first === 'explore') { + const name = segments[1]?.toLowerCase() === 'tags' ? segments[2] : null; + return name && HASHTAG.test(name) ? { mode: 'hashtag', tag: name } : null; + } + + // A single post or reel is something to look at, not a source to follow — + // the same call `../x/canonical.js` makes about a status URL. + if (NOT_A_HANDLE.has(first)) return null; + if (!HANDLE.test(segments[0])) return null; + + // `/handle/reels`, `/handle/tagged` and friends are views of an account, and + // none of them is separately collectable through the provider we use. + if (segments[1]) return null; + + return { mode: 'user', username: segments[0] }; +} + +/** + * @param {{ mode: string, username?: string, tag?: string }} spec + * @returns {string|null} + */ +export function instagramRef(spec) { + if (!spec) return null; + if (spec.mode === 'user') return spec.username ? `ig:user:${spec.username.toLowerCase()}` : null; + if (spec.mode === 'hashtag') return spec.tag ? `ig:tag:${spec.tag.toLowerCase()}` : null; + return null; +} + +/** The canonical address on Instagram's side. */ +export function instagramUrl(spec) { + if (!spec) return null; + if (spec.mode === 'user') return `https://www.instagram.com/${spec.username}/`; + if (spec.mode === 'hashtag') return `https://www.instagram.com/explore/tags/${spec.tag}/`; + return null; +} + +/** Where it lives on this site. */ +export function instagramPath(spec) { + if (!spec) return null; + if (spec.mode === 'user') return `/ig/${spec.username}`; + if (spec.mode === 'hashtag') return `/ig/tag/${spec.tag}`; + return null; +} + +/** A title for a source whose first crawl has not landed yet. */ +export function instagramTitle(spec) { + if (spec?.mode === 'user') return `@${spec.username} on Instagram`; + if (spec?.mode === 'hashtag') return `#${spec.tag} on Instagram`; + return 'Instagram'; +} + +/** The directory slug. */ +export function instagramSlug(spec) { + const ref = instagramRef(spec); + if (!ref) return null; + return ref + .replace(/^ig:/, 'ig-') + .replace(/:/g, '-') + .replace(/[^a-z0-9-]+/g, '-') + .replace(/-{2,}/g, '-') + .replace(/^-+|-+$/g, ''); +} + +/** + * Everything a source row needs, from one pasted string. + * + * @param {unknown} input + * @returns {object|null} + */ +export function instagramSource(input) { + const spec = parseInstagramInput(input); + if (!spec) return null; + + const ref = instagramRef(spec); + const url = instagramUrl(spec); + const path = instagramPath(spec); + const slug = instagramSlug(spec); + if (!ref || !url || !path || !slug) return null; + + return { ...spec, ref, url, path, slug, title: instagramTitle(spec) }; +} + +/** + * Rebuild the spec from a stored ref, for the crawler. + * + * @param {unknown} ref + * @returns {{ mode: 'user'|'hashtag', username?: string, tag?: string }|null} + */ +export function instagramSpecFromRef(ref) { + const match = /^ig:(user|tag):(.+)$/.exec(String(ref ?? '')); + if (!match) return null; + + const [, kind, rest] = match; + if (kind === 'user') return HANDLE.test(rest) ? { mode: 'user', username: rest } : null; + return HASHTAG.test(rest) ? { mode: 'hashtag', tag: rest } : null; +} diff --git a/packages/social/src/instagram/fetch.js b/packages/social/src/instagram/fetch.js new file mode 100644 index 0000000..6f84545 --- /dev/null +++ b/packages/social/src/instagram/fetch.js @@ -0,0 +1,125 @@ +/** + * Collecting Instagram, which needs far less machinery than X did. + * + * X needed a normaliser because its posts have structure that survives into the + * rendering — a repost has no text of its own, a quote is two posts in one item, + * a reply is a fragment. Instagram has none of that: a post is a caption and + * some pictures, and RSSHub's own rendering of it is already the item we want. + * + * So this parses the bridge's RSS with the ordinary feed parser and changes + * exactly one thing: the identity. `parseFeed` keys an item on the bridge's + * ``, and a bridge's guid is the bridge's — swap RSSHub for anything else + * and every post in every Instagram feed changes identity, and every + * subscriber's reader marks the whole account unread. Re-keying on the post's + * own shortcode is what makes the collection method replaceable, which is the + * same promise `/x/` makes (AC-2). + */ + +import { parseFeed } from '@rssamplifier/feed'; + +import { providerGet } from '../x/providers/http.js'; +import { XUnavailable } from '../x/errors.js'; +import { failureResult } from '../failure.js'; +import { instagramSpecFromRef } from './canonical.js'; + +/** Instagram post shortcodes appear in `/p//` and `/reel//`. */ +const SHORTCODE = /\/(?:p|reel|tv)\/([A-Za-z0-9_-]{5,30})/; + +/** + * Collect one Instagram source, in the shape `crawlFeed` expects. + * + * @param {{ social_ref?: string, feed_url?: string, item_count?: number }} feed + * @param {{ runtime: { env?: Record, onEvent?: Function }, signal?: AbortSignal }} opts + * @returns {Promise} + */ +export async function fetchInstagramSource(feed, opts) { + const spec = instagramSpecFromRef(feed?.social_ref); + if (!spec) return { ok: false, error: 'invalid-instagram-ref' }; + + const env = opts.runtime?.env ?? process.env; + const onEvent = opts.runtime?.onEvent ?? (() => {}); + const base = String(env.RSSHUB_BASE_URL ?? '').replace(/\/+$/, ''); + + try { + if (!base) throw new XUnavailable('instagram: no RSSHUB_BASE_URL'); + + // The web-api route (`/instagram/2/...`) rather than the private-api one. + // Both exist upstream; this one authenticates with a cookie (`IG_COOKIE`) + // where the other wants a username and password, and storing somebody's + // Instagram password to read public posts is a trade nobody should make. + const url = new URL( + `${base}/instagram/2/${spec.mode === 'user' ? 'user' : 'hashtag'}/` + + encodeURIComponent(spec.mode === 'user' ? spec.username : spec.tag), + ); + if (env.RSSHUB_ACCESS_KEY) url.searchParams.set('key', String(env.RSSHUB_ACCESS_KEY)); + + onEvent('instagram.fetch.started', { ref: feed.social_ref }); + + const { body } = await providerGet(url, { + provider: 'rsshub', + timeoutMs: Number(env.X_FETCH_TIMEOUT_MS) || undefined, + // Injected by tests, so a suite never reaches a real upstream (§51). + fetch: opts.runtime?.fetch, + signal: opts.signal, + }); + + const parsed = parseFeed(body, String(url)); + if (!parsed) throw new XUnavailable('instagram: unparseable-response'); + + const items = (parsed.items ?? []).map(reKey).filter(Boolean); + + // An account that had posts yesterday and none today is almost always an + // upstream that answered 200 with a page it could not fill — the same + // reasoning, and the same treatment, as the X path (§16). + if (items.length === 0 && Number(feed?.item_count ?? 0) > 0) { + onEvent('instagram.fetch.failed', { ref: feed.social_ref, error: 'empty-result' }); + return { ok: false, throttled: true, retryAfter: 20 * 60, error: 'empty-result' }; + } + + onEvent('instagram.fetch.success', { ref: feed.social_ref, itemCount: items.length }); + + return { + ok: true, + feedUrl: feed.feed_url, + feed: { + title: parsed.title || titleFor(spec), + description: parsed.description || descriptionFor(spec), + siteUrl: String(feed.feed_url), + language: null, + imageUrl: parsed.imageUrl ?? null, + categories: [], + kind: 'blog', + items, + }, + }; + } catch (error) { + onEvent('instagram.fetch.failed', { ref: feed.social_ref, error: String(error?.message ?? error) }); + return failureResult(error); + } +} + +/** + * The item, keyed on the post rather than on the bridge. + * + * @param {object} item a `parseFeed` item + * @returns {object|null} + */ +function reKey(item) { + const code = SHORTCODE.exec(String(item?.url ?? ''))?.[1] ?? SHORTCODE.exec(String(item?.guid ?? ''))?.[1]; + + // No shortcode, no item. An item that cannot be deduplicated arrives again on + // every crawl for ever, and is invisible until the feed is all duplicates. + if (!code) return null; + + return { ...item, guid: `ig:${code}`, audio: null }; +} + +function titleFor(spec) { + return spec.mode === 'user' ? `@${spec.username} on Instagram` : `#${spec.tag} on Instagram`; +} + +function descriptionFor(spec) { + return spec.mode === 'user' + ? `Posts from @${spec.username} on Instagram, mirrored by RSS Amplifier.` + : `Instagram posts tagged #${spec.tag}, mirrored by RSS Amplifier.`; +} diff --git a/packages/social/src/x/fetch.js b/packages/social/src/x/fetch.js index 0b653ae..8dcea0e 100644 --- a/packages/social/src/x/fetch.js +++ b/packages/social/src/x/fetch.js @@ -20,18 +20,12 @@ * fall back to, because the database was always the thing being served. */ +import { failureResult, ANOMALY_SECONDS } from '../failure.js'; import { xSpecFromRef } from './canonical.js'; import { normalizeXFeed } from './normalize.js'; import { XRegistry } from './registry.js'; import { XSessionPool, sessionsFromEnv } from './sessions.js'; -/** - * How long to wait after an anomaly, in minutes. Long enough that a hundred - * queued sources do not all rediscover the same upstream problem inside a tick, - * short enough that a real recovery is picked up within the hour. - */ -const ANOMALY_MINUTES = 20; - /** * Build the runtime once, at boot. * @@ -58,7 +52,11 @@ export async function createXRuntime(opts = {}) { await Promise.all([registry.hydrate(), sessions.hydrate()]); - return { registry, sessions, onEvent: opts.onEvent ?? (() => {}) }; + // `env` travels with the runtime so the collectors that are not X — Instagram + // reads RSSHUB_BASE_URL, Facebook reads FB_PAGE_TOKENS — read the same + // environment this was built from rather than each reaching for process.env + // and becoming untestable. + return { registry, sessions, env, onEvent: opts.onEvent ?? (() => {}) }; } /** @@ -109,34 +107,13 @@ export async function fetchXSource(feed, opts) { { sessions, onEvent, signal: opts.signal }, ); } catch (error) { - // **Exactly one kind of failure is about the source.** A deleted, suspended - // or protected account is a fact about the account; everything else — a rate - // limit, a provider outage, a dead session, no provider configured at all — - // is a fact about us or about the upstream, and recording it against the - // account is how a directory deletes itself. - // - // The arithmetic, because it is what makes this urgent rather than tidy: - // `markCrawlFailure` retires a feed at ten consecutive failures, and an X - // source polls on a five-minute floor. Ten strikes is **fifty minutes**. So - // switching X on before a provider is reachable — which is the ordinary - // order of operations, since the flag is how you find out — would quietly - // mark every X source dead within the hour, and nothing in the logs would - // say "no provider configured" rather than "these accounts are broken". - // - // Returning `throttled` routes to `markThrottled`, which moves - // `next_fetch_at` and leaves `status`, `error_count`, `last_error` and - // `last_success_at` exactly as they were — the same treatment an ordinary - // publisher's 429 gets, and for the same reason (§16, §40). - if (error?.name === 'XNoSuchSource') { - return { ok: false, error: String(error?.message ?? 'no-such-source').slice(0, 200) }; - } - - return { - ok: false, - throttled: true, - retryAfter: retryAfterFor(error), - error: String(error?.message ?? 'x-fetch-failed').slice(0, 200), - }; + // **Exactly one kind of failure is about the source**, and the rule now + // lives in ../failure.js so that every platform shares it rather than each + // rediscovering it. See that file, and PR #157, for what it costs to get + // wrong: at a five-minute floor, ten strikes retires a source in fifty + // minutes, so "no provider configured" would delete the X directory within + // the hour of switching X on. + return failureResult(error); } const posts = result.posts ?? []; @@ -149,7 +126,7 @@ export async function fetchXSource(feed, opts) { // and worth crawling once a day (§16, "empty-result anomalies"). if (posts.length === 0 && Number(feed?.item_count ?? 0) > 0) { onEvent('x.fetch.failed', { provider: result.provider, error: 'empty-result' }); - return { ok: false, throttled: true, retryAfter: ANOMALY_MINUTES * 60, error: 'empty-result' }; + return { ok: false, throttled: true, retryAfter: ANOMALY_SECONDS, error: 'empty-result' }; } return { @@ -167,30 +144,6 @@ export async function fetchXSource(feed, opts) { }; } -/** - * How long to wait before trying this source again, by what went wrong. - * - * The three intervals are three different guesses about when the situation - * changes. A rate limit usually names its own; a provider outage is minutes; - * and "no provider is configured" is a deployment that has not happened yet, so - * asking again in ten minutes is a thousand pointless wake-ups a day across the - * directory and answers no sooner than asking in an hour. - * - * @param {Error & { retryAfter?: number|null }} error - * @returns {number} seconds - */ -function retryAfterFor(error) { - if (error?.name === 'XRateLimited') { - return Number(error.retryAfter) > 0 ? Number(error.retryAfter) : ANOMALY_MINUTES * 60; - } - // Nothing is set up to collect with. Distinguished by message rather than by - // type because `XUnavailable` covers both this and a provider that is merely - // down, and the two deserve very different patience. - if (/no X provider is configured/i.test(String(error?.message ?? ''))) return 3600; - - return ANOMALY_MINUTES * 60; -} - /** * The per-source toggles of §6.3, with the PRD's defaults. * diff --git a/packages/social/test/platforms.test.js b/packages/social/test/platforms.test.js new file mode 100644 index 0000000..31d08e7 --- /dev/null +++ b/packages/social/test/platforms.test.js @@ -0,0 +1,369 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; + +import { parseInstagramInput, instagramRef, instagramSource, instagramSpecFromRef } from '../src/instagram/canonical.js'; +import { parseFacebookInput, facebookRef, facebookSource, facebookSpecFromRef } from '../src/facebook/canonical.js'; +import { pageToken, connectedPages, fetchFacebookSource } from '../src/facebook/fetch.js'; +import { fetchInstagramSource } from '../src/instagram/fetch.js'; +import { socialSourceFrom, socialPathFor, SOCIAL_NETWORKS } from '../src/identify.js'; +import { isCollected, fetchSocialSource } from '../src/collect.js'; +import { failureResult, ANOMALY_SECONDS, UNCONFIGURED_SECONDS } from '../src/failure.js'; +import { socialDisplayTitle } from '../src/display.js'; +import { XNoSuchSource, XRateLimited, XUnavailable, XAuthFailed } from '../src/x/errors.js'; + +/* --------------------------------------------------------------- Instagram */ + +test('every spelling of an Instagram handle is one source', () => { + const forms = [ + 'ig/nasa', + '/ig/nasa', + 'ig/@nasa', + 'https://www.instagram.com/nasa/', + 'https://instagram.com/NASA', + 'https://m.instagram.com/nasa', + ]; + + assert.deepEqual([...new Set(forms.map((f) => instagramRef(parseInstagramInput(f))))], ['ig:user:nasa']); +}); + +test('dots are legal in an Instagram handle and are not on X', () => { + // A handle regex copied from x/canonical.js would silently reject these. + assert.equal(instagramRef(parseInstagramInput('ig/some.body')), 'ig:user:some.body'); + assert.equal(instagramRef(parseInstagramInput('https://www.instagram.com/a.b_c/')), 'ig:user:a.b_c'); +}); + +test('hashtags, in both spellings', () => { + assert.equal(instagramRef(parseInstagramInput('#coffee')), 'ig:tag:coffee'); + assert.equal( + instagramRef(parseInstagramInput('https://www.instagram.com/explore/tags/Coffee/')), + 'ig:tag:coffee', + ); +}); + +test('a post, a reel and a story are not sources', () => { + for (const path of ['/p/Cxyz123/', '/reel/Cxyz123/', '/stories/nasa/', '/explore/']) { + assert.equal(parseInstagramInput(`https://www.instagram.com${path}`), null, path); + } + // Nor is a view of an account. + assert.equal(parseInstagramInput('https://www.instagram.com/nasa/tagged/'), null); +}); + +test('an Instagram ref survives the round trip the crawler makes it do', () => { + for (const input of ['ig/nasa', '#coffee', 'https://www.instagram.com/some.body/']) { + const spec = parseInstagramInput(input); + assert.equal(instagramRef(instagramSpecFromRef(instagramRef(spec))), instagramRef(spec), input); + } +}); + +test('Instagram paths and slugs', () => { + assert.equal(instagramSource('ig/nasa').path, '/ig/nasa'); + assert.equal(instagramSource('#coffee').path, '/ig/tag/coffee'); + assert.equal(instagramSource('ig/nasa').slug, 'ig-user-nasa'); +}); + +/* ---------------------------------------------------------------- Facebook */ + +test('a Facebook Page is recognised in the spellings people paste', () => { + const forms = [ + 'fb/SomePage', + 'https://www.facebook.com/SomePage', + 'https://m.facebook.com/SomePage/', + 'https://fb.com/SomePage', + 'https://www.facebook.com/SomePage/about', + ]; + + assert.deepEqual([...new Set(forms.map((f) => facebookRef(parseFacebookInput(f))))], ['fb:page:somepage']); +}); + +test('a personal profile is not a Page, and cannot be', () => { + // There is no Graph API for somebody's own timeline at all. + assert.equal(parseFacebookInput('https://www.facebook.com/profile.php?id=100001'), null); + assert.equal(parseFacebookInput('https://www.facebook.com/groups/12345'), null); + assert.equal(parseFacebookInput('https://www.facebook.com/events/12345'), null); +}); + +test('a post under a Page is a thing to read, not a source', () => { + assert.equal(parseFacebookInput('https://www.facebook.com/SomePage/posts/12345'), null); + assert.equal(parseFacebookInput('https://www.facebook.com/SomePage/photos/12345'), null); +}); + +test('the old /pages/Name/id form keeps the id, which is the usable half', () => { + assert.equal( + facebookRef(parseFacebookInput('https://www.facebook.com/pages/Some-Name/123456789')), + 'fb:page:123456789', + ); +}); + +test('a bare number is not guessed to be a Facebook Page', () => { + // It is also a plausible X list id. Guessing would be wrong in front of + // somebody eventually. + assert.equal(parseFacebookInput('1234567890'), null); +}); + +test('a Facebook ref survives the round trip', () => { + for (const input of ['fb/SomePage', 'https://www.facebook.com/pages/N/123456789']) { + const ref = facebookRef(parseFacebookInput(input)); + assert.equal(facebookRef(facebookSpecFromRef(ref)), ref, input); + } +}); + +test('Page tokens come from structured config, keyed case-insensitively', () => { + const env = { FB_PAGE_TOKENS: '[{"page":"MyPage","token":"EAAsecret"},{"page":"123456789","token":"EAAother"}]' }; + + assert.equal(pageToken(env, 'mypage'), 'EAAsecret'); + assert.equal(pageToken(env, 'MyPage'), 'EAAsecret'); + assert.equal(pageToken(env, '123456789'), 'EAAother'); + assert.equal(pageToken(env, 'somebodyelse'), null); + assert.deepEqual(connectedPages(env), ['MyPage', '123456789']); + + // Malformed config reads as "no token", not as a crash and not as an error + // about the Page. + assert.equal(pageToken({ FB_PAGE_TOKENS: '{oops' }, 'mypage'), null); + assert.deepEqual(connectedPages({}), []); +}); + +test('an unconnected Page reschedules for an hour rather than being retired', async () => { + const result = await fetchFacebookSource( + { social_ref: 'fb:page:somepage', feed_url: 'https://www.facebook.com/SomePage' }, + { runtime: { env: {}, onEvent: () => {} } }, + ); + + assert.equal(result.ok, false); + assert.equal(result.throttled, true, 'not collectable is not the same as broken'); + assert.equal(result.retryAfter, UNCONFIGURED_SECONDS); +}); + +test('Graph answering 200 with an error object is still a failure', async () => { + const env = { FB_PAGE_TOKENS: '[{"page":"somepage","token":"t"}]' }; + const feed = { social_ref: 'fb:page:somepage', feed_url: 'https://www.facebook.com/SomePage' }; + + const graph = (payload) => async () => + new Response(JSON.stringify(payload), { status: 200, headers: { 'content-type': 'application/json' } }); + + // Code 100 means the Page is gone or renamed: the one failure genuinely about + // the source, and the only one allowed to reach markCrawlFailure. + const gone = await fetchFacebookSource(feed, { + runtime: { env, onEvent: () => {}, fetch: graph({ error: { code: 100, message: 'Unknown path' } }) }, + }); + assert.equal(gone.ok, false); + assert.equal(gone.throttled, undefined); + + // Anything else Graph complains about is ours or theirs, not the Page's. + const rate = await fetchFacebookSource(feed, { + runtime: { env, onEvent: () => {}, fetch: graph({ error: { code: 4, message: 'rate limited' } }) }, + }); + assert.equal(rate.throttled, true); +}); + +test('a Page that answers is turned into items keyed on the post id', async () => { + const env = { FB_PAGE_TOKENS: '[{"page":"somepage","token":"t"}]' }; + const payload = { + data: [ + { + id: '123_456', + message: 'Hello world\nsecond line', + created_time: '2026-08-29T10:00:00+0000', + permalink_url: 'https://www.facebook.com/SomePage/posts/456', + full_picture: 'https://scontent.example/1.jpg', + }, + { id: '123_789', story: 'SomePage shared a link.', created_time: '2026-08-29T09:00:00+0000' }, + ], + }; + + const result = await fetchFacebookSource( + { social_ref: 'fb:page:somepage', feed_url: 'https://www.facebook.com/SomePage' }, + { + runtime: { + env, + onEvent: () => {}, + fetch: async () => + new Response(JSON.stringify(payload), { + status: 200, + headers: { 'content-type': 'application/json' }, + }), + }, + }, + ); + + assert.equal(result.ok, true); + assert.deepEqual( + result.feed.items.map((i) => i.guid), + ['fb:123_456', 'fb:123_789'], + ); + + const [first, second] = result.feed.items; + assert.equal(first.title, 'SomePage: Hello world'); + // Graph returns plain text, so the only tags in the body are ones we wrote. + assert.match(first.contentHtml, /<b>world/); + assert.doesNotMatch(first.contentHtml, //); + assert.equal(first.imageUrl, 'https://scontent.example/1.jpg'); + + // A post with no caption still gets a title, because every format needs one. + assert.equal(second.title, 'SomePage: SomePage shared a link.'); +}); + +test('Instagram items are re-keyed on the post shortcode, not the bridge guid', async () => { + const rss = [ + '', + '', + 'NASAhttps://www.instagram.com/nasa/', + '', + 'A photo', + 'https://www.instagram.com/p/CxAbCdEfGhI/', + 'rsshub-internal-12345', + 'Sat, 29 Aug 2026 10:00:00 GMT', + 'a caption', + '', + '', + 'No shortcode', + 'https://www.instagram.com/nasa/', + 'nothing', + '', + '', + ].join('\n'); + + const result = await fetchInstagramSource( + { social_ref: 'ig:user:nasa', feed_url: 'https://www.instagram.com/nasa/' }, + { + runtime: { + env: { RSSHUB_BASE_URL: 'http://127.0.0.1:1200' }, + onEvent: () => {}, + fetch: async () => new Response(rss, { status: 200 }), + }, + }, + ); + + assert.equal(result.ok, true); + // The bridge's own guid is discarded: swapping bridges must not change the + // identity of every post in the feed (AC-2). An item with no shortcode is + // dropped rather than stored undeduplicatable. + assert.deepEqual( + result.feed.items.map((i) => i.guid), + ['ig:CxAbCdEfGhI'], + ); +}); + +/* -------------------------------------------------- the shared failure rule */ + +test('only a missing source may count against it, for every platform', () => { + assert.equal(failureResult(new XNoSuchSource('gone')).throttled, undefined); + + for (const error of [ + new XUnavailable('rsshub: upstream-502'), + new XAuthFailed('auth-failed-401'), + new XRateLimited('429', { retryAfter: 90 }), + ]) { + assert.equal(failureResult(error).throttled, true, error.name); + } + + assert.equal(failureResult(new XRateLimited('429', { retryAfter: 90 })).retryAfter, 90); + assert.equal(failureResult(new XUnavailable('boom')).retryAfter, ANOMALY_SECONDS); + assert.equal( + failureResult(new XUnavailable('instagram: no RSSHUB_BASE_URL')).retryAfter, + UNCONFIGURED_SECONDS, + ); +}); + +test('Instagram with no provider configured reschedules for an hour', async () => { + const result = await fetchInstagramSource( + { social_ref: 'ig:user:nasa', feed_url: 'https://www.instagram.com/nasa/' }, + { runtime: { env: {}, onEvent: () => {} } }, + ); + + assert.equal(result.ok, false); + assert.equal(result.throttled, true); + assert.equal(result.retryAfter, UNCONFIGURED_SECONDS); +}); + +/* ------------------------------------------------------ recognition + routing */ + +test('the four namespaces, and which of them needs a collector', () => { + assert.deepEqual([...SOCIAL_NETWORKS], ['reddit', 'x', 'instagram', 'facebook']); + + // Reddit is a social network and is NOT collected: it publishes real RSS and + // is fetched like any blog. That asymmetry is the whole reason the two + // questions are separate. + assert.equal(isCollected({ social_network: 'reddit' }), false); + assert.equal(isCollected({ social_network: 'x' }), true); + assert.equal(isCollected({ social_network: 'instagram' }), true); + assert.equal(isCollected({ social_network: 'facebook' }), true); + assert.equal(isCollected({}), false); +}); + +test('each platform is recognised, and nothing else is', () => { + assert.equal(socialSourceFrom('https://www.reddit.com/r/programming/').network, 'reddit'); + assert.equal(socialSourceFrom('https://x.com/OpenAI').network, 'x'); + assert.equal(socialSourceFrom('https://www.instagram.com/nasa/').network, 'instagram'); + assert.equal(socialSourceFrom('https://www.facebook.com/SomePage').network, 'facebook'); + assert.equal(socialSourceFrom('https://example.com/feed.xml'), null); +}); + +test('a bare handle stays X, which had the spelling first', () => { + // Ambiguous between X and Instagram; the ordering in identify.js decides it, + // and /submit has accepted @handle as X since #156. + assert.equal(socialSourceFrom('@OpenAI').network, 'x'); + assert.equal(socialSourceFrom('OpenAI').network, 'x'); + + // Instagram stays reachable explicitly. + assert.equal(socialSourceFrom('ig/OpenAI').network, 'instagram'); +}); + +test('a stored row of any platform knows its own address', () => { + assert.equal(socialPathFor({ social_ref: 'r:sub:programming' }), '/r/programming'); + assert.equal(socialPathFor({ social_ref: 'x:user:openai' }), '/x/openai'); + assert.equal(socialPathFor({ social_ref: 'ig:user:nasa' }), '/ig/nasa'); + assert.equal(socialPathFor({ social_ref: 'ig:tag:coffee' }), '/ig/tag/coffee'); + assert.equal(socialPathFor({ social_ref: 'fb:page:somepage' }), '/fb/somepage'); + assert.equal(socialPathFor({ slug: 'a-blog' }), '/a-blog'); +}); + +test('a search ref keeps colons in its query rather than being truncated', () => { + // `x:search:from:OpenAI lang:en` — splitting on every colon would cut the + // query in half and produce a path to a different search. + assert.equal( + socialPathFor({ social_ref: 'x:search:from:openai lang:en' }), + '/x/search?q=from%3Aopenai%20lang%3Aen', + ); +}); + +test('no public path ever names a provider', () => { + for (const input of ['r/programming', '@OpenAI', 'ig/nasa', 'fb/SomePage']) { + assert.doesNotMatch(socialSourceFrom(input).path, /rsshub|teapot|nitter|graph\.facebook/i, input); + } +}); + +test('the dispatcher refuses a network it has no collector for', async () => { + const result = await fetchSocialSource({ social_network: 'reddit' }, { runtime: {} }); + assert.equal(result.ok, false); + assert.match(result.error, /no collector/); +}); + +/* --------------------------------------------------------- what to call a row */ + +test('an uncrawled row gets its canonical name rather than the imported one', () => { + // The measured case: 50,026 subreddits came from an OPML catalogue and the + // crawler has read a few hundred, so most rows are titled with the bare host + // and /r/programming rendered a heading reading "reddit.com". + for (const title of ['reddit.com', 'Reddit', 'www.reddit.com', '(untitled)', '', ' ']) { + assert.equal(socialDisplayTitle({ title }, 'r/programming'), 'r/programming', JSON.stringify(title)); + } + + // A title that is just the URL it was imported from is the same non-answer. + assert.equal( + socialDisplayTitle( + { title: 'www.reddit.com/r/programming', feed_url: 'https://www.reddit.com/r/programming/.rss' }, + 'r/programming', + ), + 'r/programming', + ); +}); + +test('a real title always wins, including one that mentions the platform', () => { + // "Reddit Blog" is somebody's actual feed title, not a placeholder. + assert.equal(socialDisplayTitle({ title: 'Reddit Blog' }, 'r/blog'), 'Reddit Blog'); + assert.equal( + socialDisplayTitle({ title: 'programming' }, 'r/programming'), + 'programming', + ); + assert.equal(socialDisplayTitle({ title: 'NASA' }, '@nasa on Instagram'), 'NASA'); +});