diff --git a/apps/web/src/app/account/page.jsx b/apps/web/src/app/account/page.jsx index 4fbce17..f729b67 100644 --- a/apps/web/src/app/account/page.jsx +++ b/apps/web/src/app/account/page.jsx @@ -1,5 +1,5 @@ import { redirect } from 'next/navigation'; -import { accounts, apikeys } from '@rssamplifier/db'; +import { accounts, apikeys, dataset } from '@rssamplifier/db'; import Toolbar from '../Toolbar.jsx'; import { AddPasskey } from '../Passkey.jsx'; @@ -7,6 +7,7 @@ import { db } from '../../lib/db.js'; import { currentUser } from '../../lib/auth.js'; import { topicLabel } from '../../lib/following.js'; import { ANONYMOUS_HOURLY } from '../../lib/ratelimit.js'; +import { latestClosedWindow } from '../../lib/datasetWindow.js'; export const dynamic = 'force-dynamic'; @@ -29,13 +30,19 @@ export default async function AccountPage({ searchParams }) { const client = db(); const userId = String(user.id); - const [follows, credentials, topics, keys] = await Promise.all([ + const [follows, credentials, topics, keys, grant] = await Promise.all([ accounts.followedFeeds(client, userId), accounts.credentialsForUser(client, userId), accounts.followedTopics(client, userId), apikeys.keysForUser(client, userId), + dataset.activeGrant(client, userId), ]); + // Only for an account that has one, so the overwhelming majority of readers — + // who will never license the corpus — do not pay a query to be told they have + // no downloads. + const pulls = grant ? await dataset.recentDownloads(client, String(grant.id), 5) : []; + // Revoked keys are kept in the table so a stale key found in a log can still // be identified, but there is nothing for their owner to do about them. const liveKeys = keys.filter((k) => !k.revoked_at); @@ -138,6 +145,62 @@ export default async function AccountPage({ searchParams }) { + {/* Shown to every account, licensed or not, and that is deliberate. An + account with no licence is told so in one line with a link, which is + the answer to "did my access get set up?" — a question that otherwise + arrives by email. There is no button here because there is nothing to + press: a licence is written by hand after a conversation, and a page + that implied otherwise would be promising a checkout that does not + exist. */} +

Training data

+ {grant ? ( + <> +

+ This account holds a {String(grant.plan)} corpus licence, granted{' '} + {formatDate(grant.granted_at)} + {grant.expires_at ? ` and running until ${formatDate(grant.expires_at)}` : ''}. Pull{' '} + {Number(grant.per_window_downloads)} times per dataset per window, and{' '} + {Number(grant.full_dumps_per_day)} full-history dump + {Number(grant.full_dumps_per_day) === 1 ? '' : 's'} a day. Use a session or any of your + API keys as a bearer token — the licence belongs to the account, so rotating a key costs + you nothing. +

+

+ Newest complete window: {latestClosedWindow()}. The shape of every row is + at /api/dataset. +

+ {pulls.length === 0 ? ( +

Nothing pulled yet.

+ ) : ( +
+ {pulls.map((p, i) => ( +
+
{String(p.dataset)}
+
+ {p.full_dump ? 'full history' : String(p.window_start)} +
+ {/* A pull with no completed_at did not finish. Said plainly, + because it is the difference between "that window really + was small" and "your pipeline lost the connection", and + only one of those is worth investigating. */} + {p.completed_at + ? `${Number(p.rows_sent).toLocaleString('en-US')} rows · ${formatDate(p.created_at)}` + : `did not finish · started ${formatDate(p.created_at)}`} +
+
+ ))} +
+ )} + + ) : ( +

+ This account has no corpus licence, and needs none for anything the directory serves + openly — the API, the OPML export and the MCP server all answer without one. Bulk access + to the whole directory as a training corpus is licensed separately:{' '} + what is in it, and how to ask. +

+ )} + {/* What is followed, but not what it published: the river moved to /following, which is the one place the blogs and the topics are merged into a single list and the only one of the two that can be subscribed diff --git a/apps/web/src/app/api/dataset/[name]/route.js b/apps/web/src/app/api/dataset/[name]/route.js new file mode 100644 index 0000000..7639a0c --- /dev/null +++ b/apps/web/src/app/api/dataset/[name]/route.js @@ -0,0 +1,234 @@ +import { dataset } from '@rssamplifier/db'; + +import { db } from '../../../../lib/db.js'; +import { + datasetCaller, + resolveWindow, + latestClosedWindow, + startOfUtcDay, +} from '../../../../lib/dataset.js'; + +export const dynamic = 'force-dynamic'; + +/** + * One dataset, one slice, streamed as gzipped NDJSON. + * + * ## Why this streams instead of building a file + * + * There is nowhere to put a file. This deployment has no object storage, and the + * database it would be built from is the same one serving the site — so a + * "build a dump every four hours" job would be a large periodic read against a + * write-saturated database, producing an artifact most windows nobody collects. + * Streaming inverts it: the work happens when somebody actually wants the data, + * nothing larger than one page is ever resident, and the buyer starts receiving + * rows immediately rather than after the last one is read. + * + * The window boundaries are what make that safe to sell. A streamed response is + * not obviously reproducible — but a slice cut on a fixed clock is the same set + * of rows whoever asks, so "stream it twice and get the same corpus" holds + * without an artifact existing anywhere in between. See `lib/dataset.js`. + * + * ## Why NDJSON and not CSV or Parquet + * + * Because the rows are not rectangular: a post carries an HTML body and an + * author record carries a nested array of links, and both survive a line of JSON + * unchanged. It is also the format that can be produced incrementally — a line + * at a time, with no header to write first and no footer to get right — which is + * what lets a dump be a stream at all. Parquet would be a better fit for a + * buyer's storage and a worse fit for ours; converting on receipt is one line of + * their pipeline. + * + * ## Why the count in the audit log can be short + * + * `finishDownload` runs when the stream closes cleanly. If the buyer hangs up + * partway, it never runs, and the row keeps a null `completed_at` — which is the + * signal that the pull broke rather than that it was small. That is deliberate: + * a broken pull must still count against the window allowance, or hanging up + * becomes a free way to run the query in a loop. + */ + +/** + * @param {Request} req + * @param {{ params: Promise<{ name: string }> }} ctx + * @returns {Promise} + */ +export async function GET(req, { params }) { + const { name: raw } = await params; + // `.jsonl.gz` is stripped rather than required, so a caller may write the URL + // as a filename and get a file. Both spellings reach the same stream. + const name = String(raw ?? '') + .toLowerCase() + .replace(/\.(ndjson|jsonl)(\.gz)?$/, ''); + + const stream = dataset.streamFor(name); + if (!stream) { + return Response.json( + { + error: 'unknown-dataset', + detail: `No dataset called "${name}".`, + datasets: dataset.DATASETS, + manifest: '/api/dataset', + }, + { status: 404, headers: { 'access-control-allow-origin': '*' } }, + ); + } + + const caller = await datasetCaller(req); + if (!caller.ok) return caller.response; + + const { user, grant, apiKeyId } = caller; + const client = db(); + const url = new URL(req.url); + const full = url.searchParams.get('full') === '1'; + + // Two different allowances, because they bound two different costs. A window + // is a bounded index range; a full dump walks the table. + if (full) { + const taken = await dataset.fullDumpCount(client, String(grant.id), startOfUtcDay()); + const allowed = Number(grant.full_dumps_per_day) || 0; + if (taken >= allowed) { + return exhausted( + 'full-dump-limit', + `This licence allows ${allowed} full-history pull${allowed === 1 ? '' : 's'} a day and has used ${taken}. Incremental windows are not affected — take ${latestClosedWindow()} instead.`, + ); + } + } + + const slice = full + ? { ok: true, start: null, end: null } + : resolveWindow(url.searchParams.get('window')); + + if (!slice.ok) { + return Response.json( + { error: slice.error, detail: slice.detail, manifest: '/api/dataset' }, + { status: 400, headers: { 'access-control-allow-origin': '*' } }, + ); + } + + if (!full) { + const taken = await dataset.windowDownloadCount( + client, + String(grant.id), + name, + String(slice.start), + ); + const allowed = Number(grant.per_window_downloads) || 0; + if (taken >= allowed) { + return exhausted( + 'window-limit', + `This licence allows ${allowed} pulls of one dataset per ${dataset.WINDOW_HOURS}-hour window, and has taken ${taken} of ${name} for ${slice.start}.`, + ); + } + } + + // Opened before the first byte, closed when the last one lands. See the note + // at the top on why the order matters. + const downloadId = await dataset.startDownload(client, { + grantId: String(grant.id), + userId: String(user.id), + dataset: name, + windowStart: full ? null : String(slice.start), + fullDump: full, + apiKeyId, + }); + + const encoder = new TextEncoder(); + let rows = 0; + + const lines = new ReadableStream({ + async start(controller) { + try { + for await (const row of stream(client, { since: slice.start, until: slice.end })) { + controller.enqueue(encoder.encode(`${JSON.stringify(shape(name, row))}\n`)); + rows += 1; + } + await dataset.finishDownload(client, downloadId, rows); + } catch (err) { + // The response is already on the wire with a 200, so this cannot become + // an error status — the same bind the OPML export is in. The download + // row is left with a null `completed_at`, which is exactly the record of + // "this one broke", and the buyer sees a truncated file rather than a + // silently short one because the byte count will not match a re-pull of + // the same window. + console.error(`dataset stream failed partway: ${name}`, err); + } + controller.close(); + }, + }); + + const filename = full + ? `rssamplifier-${name}-full.jsonl.gz` + : `rssamplifier-${name}-${String(slice.start).replace(/[:.]/g, '')}.jsonl.gz`; + + return new Response(lines.pipeThrough(new CompressionStream('gzip')), { + headers: { + 'content-type': 'application/x-ndjson', + 'content-encoding': 'gzip', + 'content-disposition': `attachment; filename="${filename}"`, + // The buyer's pipeline reads these to decide when to ask again, so they + // are part of the interface rather than diagnostics. + 'x-dataset-name': name, + 'x-dataset-window': full ? 'full' : String(slice.start), + 'x-dataset-window-end': full ? '' : String(slice.end), + 'x-dataset-window-hours': String(dataset.WINDOW_HOURS), + 'x-dataset-next-window': latestClosedWindow(), + // No caching anywhere in between: a window is stable but a licence is not, + // and a proxy holding a corpus slice would serve it to the next caller. + 'cache-control': 'private, no-store', + 'access-control-allow-origin': '*', + }, + }); +} + +/** + * A row, as the buyer sees it. + * + * The database's column names are an implementation detail and several of them + * would be actively misleading in a corpus — `cursor_rowid` is paging state that + * means nothing outside this process, and `links` arrives from SQLite as a JSON + * *string* that would otherwise land in the file double-encoded and have to be + * parsed twice. + * + * Everything else keeps its snake_case name deliberately. The manifest describes + * the columns, `/crawlstats` and the open API use the same vocabulary, and + * renaming them here would mean the corpus and the documentation disagree. + * + * @param {string} name + * @param {Record} row + * @returns {Record} + */ +function shape(name, row) { + const { cursor_rowid: _cursor, ...rest } = row; + + if (name === 'authors' && typeof rest['links'] === 'string') { + try { + rest['links'] = JSON.parse(String(rest['links'])); + } catch { + // json_group_array cannot really produce invalid JSON, but a corpus line + // that fails to parse is worse than one whose links arrive as a string. + rest['links'] = []; + } + } + + if (name === 'feeds' && typeof rest['categories'] === 'string') { + try { + rest['categories'] = JSON.parse(String(rest['categories'])); + } catch { + rest['categories'] = []; + } + } + + return rest; +} + +/** + * @param {string} error + * @param {string} detail + * @returns {Response} + */ +function exhausted(error, detail) { + return Response.json( + { error, detail, manifest: '/api/dataset' }, + { status: 429, headers: { 'access-control-allow-origin': '*' } }, + ); +} diff --git a/apps/web/src/app/api/dataset/route.js b/apps/web/src/app/api/dataset/route.js new file mode 100644 index 0000000..183de1c --- /dev/null +++ b/apps/web/src/app/api/dataset/route.js @@ -0,0 +1,117 @@ +import { dataset } from '@rssamplifier/db'; + +import { latestClosedWindow, windowEnd } from '../../../lib/datasetWindow.js'; +import { siteUrl } from '../../../lib/db.js'; + +export const dynamic = 'force-dynamic'; + +/** + * The manifest: what the corpus contains, how it is cut, and how to take it. + * + * Open, and answers without a licence. That is the whole reason it is a separate + * route from the streams it describes. + * + * A buyer's first question is "what is actually in there", and the honest way to + * answer it is a machine-readable description they can read before paying rather + * than a paragraph of marketing they have to trust. It also means the pipeline + * that will pull this every four hours can discover the current boundary instead + * of computing it — so if the cadence ever changes, their code follows without + * an email. + * + * What it deliberately does not contain is a price. There is none in this + * codebase; licensing is negotiated, and the manifest points at /sales for it. + * + * It also does not contain row counts. A `count(*)` over `feed_items` is a + * multi-second scan of 4.7 million rows against a database whose write path is + * already saturated, and this is an unauthenticated endpoint — putting one here + * would be handing anybody a way to make the site slow. The sales page shows the + * cached directory figures instead, and a licensed caller learns the exact size + * of a window by taking it. + */ + +/** + * @returns {Response} + */ +export function GET() { + const newest = latestClosedWindow(); + const base = siteUrl(); + + return Response.json( + { + corpus: 'rssamplifier.com', + description: + 'The open small-web directory as a training corpus: independent blogs, podcasts and video feeds, crawled continuously and sliced on a fixed clock.', + + cadence: { + windowHours: dataset.WINDOW_HOURS, + // Named rather than implied. A pipeline reading this should walk + // boundaries in order; saying so here is cheaper than saying it in an + // email to every buyer. + semantics: + 'Each window is the half-open range [start, end) on the row timestamp named in `cutOn`. Windows are fixed against the Unix epoch, so a window is the same set of rows whoever asks and whenever they ask. Walk them in order to see every row exactly once.', + latestClosedWindow: newest, + latestWindowEnd: windowEnd(newest), + nextWindowOpensAt: windowEnd(newest), + }, + + datasets: { + feeds: { + description: + 'One row per feed in the directory: title, description, canonical feed and site URLs, language, category, item count and crawl state.', + cutOn: 'created_at — when the directory first saw the feed, so a window is the feeds that are new to it', + format: 'application/x-ndjson, gzipped', + url: `${base}/api/dataset/feeds`, + }, + items: { + description: + 'One row per post: title, summary, author, canonical URL, publication date, and the feed it belongs to. This is metadata at scale rather than prose — see `bodies` below.', + cutOn: 'created_at — when the crawler ingested the post', + format: 'application/x-ndjson, gzipped', + url: `${base}/api/dataset/items`, + // Said here, unprompted, because it is the one thing about this corpus + // a buyer could reasonably assume wrongly and only discover after + // paying. Posts ingested before the change still carry their body. + bodies: + 'Posts ingested since 2026-08 do not carry `content_html`: storing a body for every post was 10GB of a 14GB database and the crawler stopped writing it. Full article text lives in the `extracts` dataset, for the subset that has been fetched. Older rows still carry the body they were ingested with.', + }, + extracts: { + description: + 'The article itself, sanitized, for posts whose page has been fetched and parsed. Several thousand characters on average. This is the prose in the corpus.', + cutOn: 'fetched_at — when the article was read, not when it was published, so a window includes an old post that was fetched today', + format: 'application/x-ndjson, gzipped', + url: `${base}/api/dataset/extracts`, + coverage: + 'A fraction of `items`, not a parallel of it: an article is fetched when a reader opens the post, so this grows with attention rather than with the crawl.', + }, + authors: { + description: + 'The people behind the feeds, keyed on a URL they control, with their homepages and social profiles folded in as a JSON array.', + cutOn: 'created_at, filtered rather than seeked — `authors` is small enough to walk whole', + format: 'application/x-ndjson, gzipped', + url: `${base}/api/dataset/authors`, + }, + }, + + access: { + how: 'A licence, granted per buyer. Present a session cookie or an API key as a bearer token; the licence hangs off the account rather than the key, so keys can be rotated freely.', + enquiries: `${base}/sales`, + parameters: { + window: + 'ISO-8601 window boundary. Omit for the newest closed window. An unaligned or still-filling boundary is refused rather than rounded.', + full: 'full=1 takes the whole history instead of one window. Separately metered, and expensive — intended once, to seed a mirror, before switching to windows.', + }, + limits: + 'Per-window and per-day allowances are set on the licence and reported in the `x-dataset-*` response headers.', + }, + + provenance: { + source: + 'Public feeds their publishers chose to syndicate. The directory is open and anyone may submit to it.', + optOut: + 'A publisher may be excluded from the corpus while staying in the directory, on request to hello@rssamplifier.com. Excluded feeds and every post and article belonging to them are absent from every dataset above.', + openApi: `Nothing here is a restriction on the open API: ${base}/api/feeds, /api/search, /opml and the MCP server all answer without an account.`, + }, + }, + { headers: { 'access-control-allow-origin': '*', 'cache-control': 'public, max-age=60' } }, + ); +} diff --git a/apps/web/src/app/api/sales/contact/route.js b/apps/web/src/app/api/sales/contact/route.js new file mode 100644 index 0000000..29b99ef --- /dev/null +++ b/apps/web/src/app/api/sales/contact/route.js @@ -0,0 +1,193 @@ +import { dataset } from '@rssamplifier/db'; +import { sendEmail, emailEnabled } from '@rssamplifier/mail'; + +import { db } from '../../../../lib/db.js'; +import { requestMeta } from '../../../../lib/auth.js'; + +export const dynamic = 'force-dynamic'; + +/** + * The only way into a corpus conversation. + * + * ## Why a form here when /contact deliberately has none + * + * That page's comment says it plainly: "no form, because a form on a site with + * no accounts is one more thing to spam", and it routes to published addresses + * instead. The reasoning holds and this is not a reversal of it — it is a + * different trade. /contact's job is to get a stranger to the right mailbox, + * which a list of links does perfectly well. This page's job is to start a + * negotiation, and the fields below (what they want it for, at what scale) are + * exactly the ones that decide whether there is a deal and what it looks like. + * An email that omits them costs a round trip; a form that asks for them does + * not. So the spam is paid for rather than avoided, by the two guards below. + * + * ## Why it is written down before it is sent + * + * `sendEmail` reports a failure instead of throwing, and every caller in this + * codebase treats mail as optional infrastructure. That is right for a sign-in + * link, which the reader can simply request again — it is wrong for a sales + * enquiry, which the sender believes is delivered and will never send twice. So + * the row lands first, the mail is best-effort on top of it, and a Resend outage + * costs a notification rather than a customer. + */ + +/** How long the flood window is. */ +const WINDOW_MS = 60 * 60 * 1000; + +/** + * Enquiries one address may send in that window. + * + * Three, not one. A genuine sender who mistypes their own email address and + * resends is the most likely repeat here, and refusing them is worse than + * accepting a third message from a spammer we are also storing and rate + * limiting. + */ +const MAX_PER_WINDOW = 3; + +/** Where an enquiry is announced, when mail is configured. */ +const INBOX = process.env['SALES_EMAIL'] || 'hello@rssamplifier.com'; + +/** + * @param {Request} req + * @returns {Promise} + */ +export async function POST(req) { + const wantsHtml = (req.headers.get('accept') ?? '').includes('text/html'); + const body = await readBody(req); + if (!body) return respond(wantsHtml, 400, 'bad-request', 'That form could not be read.'); + + // The honeypot. A field no human sees, no browser fills and every naive bot + // completes. Answered with the same success the real path gives, because a + // bot told it failed is a bot that tries again differently. + if (body.website) { + return respond(wantsHtml, 200, null, null); + } + + const email = body.email.trim().toLowerCase(); + // Deliberately the weakest possible check. This address is going to be read by + // a person who will reply to it, not authenticated — a strict pattern here + // rejects the valid addresses nobody remembers are valid and gains nothing. + if (!email || !email.includes('@') || email.length > 200) { + return respond(wantsHtml, 400, 'bad-email', 'That does not look like an email address.'); + } + + const useCase = body.useCase.trim(); + if (useCase.length < 10) { + return respond( + wantsHtml, + 400, + 'no-use-case', + 'Please say what you want the corpus for — it is the field that decides what we can offer.', + ); + } + + const client = db(); + const meta = await requestMeta(); + + const recent = await dataset.enquiryCountFrom( + client, + meta.ipHash, + new Date(Date.now() - WINDOW_MS).toISOString(), + ); + if (recent >= MAX_PER_WINDOW) { + return respond( + wantsHtml, + 429, + 'too-many', + 'That is several enquiries in an hour. Email hello@rssamplifier.com directly and we will pick it up there.', + ); + } + + await dataset.insertEnquiry(client, { + name: body.name.trim().slice(0, 120) || null, + email: email.slice(0, 200), + org: body.org.trim().slice(0, 160) || null, + useCase: useCase.slice(0, 4000), + ipHash: meta.ipHash, + userAgent: meta.userAgent, + }); + + if (emailEnabled()) { + // Not awaited into the response. The enquiry is already stored, so the + // sender's answer does not depend on Resend being reachable this second, and + // making them wait on it would only ever make the form feel broken. + void sendEmail({ + to: INBOX, + subject: `Corpus enquiry: ${body.org.trim() || email}`, + text: [ + `From: ${body.name.trim() || '(no name)'} <${email}>`, + `Company: ${body.org.trim() || '(none given)'}`, + '', + 'What for:', + useCase.slice(0, 4000), + '', + // So a reply can be written without opening the database, and so the + // operator knows the row exists even if this mail is the only thing they + // ever see. + 'Stored in dataset_enquiries. Grant access with an insert into dataset_grants.', + ].join('\n'), + // Replying to the notification reaches the sender rather than ourselves, + // which is the difference between a queue and an inbox. + replyTo: email, + }).catch(() => {}); + } + + return respond(wantsHtml, 200, null, null); +} + +/** + * HTML callers get a 303 back to the page; everyone else gets JSON. + * + * @param {boolean} wantsHtml + * @param {number} status + * @param {string|null} error + * @param {string|null} detail + * @returns {Response} + */ +function respond(wantsHtml, status, error, detail) { + if (wantsHtml) { + const to = error ? `/sales?error=${encodeURIComponent(error)}#enquire` : '/sales?sent=1#enquire'; + return new Response(null, { status: 303, headers: { location: to } }); + } + + return Response.json(error ? { error, detail } : { ok: true }, { + status, + headers: { 'cache-control': 'no-store' }, + }); +} + +/** + * Read a form or a JSON body into the same shape. + * + * @param {Request} req + * @returns {Promise<{ name: string, email: string, org: string, useCase: string, website: string }|null>} + */ +async function readBody(req) { + try { + const source = (req.headers.get('content-type') ?? '').includes('application/json') + ? await req.json() + : Object.fromEntries(await req.formData()); + + return { + name: str(source?.name), + email: str(source?.email), + org: str(source?.org), + // `useCase` for a JSON caller, `use_case` for the form field — the form + // uses the name that reads correctly in HTML and the API uses the one that + // reads correctly in JavaScript, and neither should have to know about the + // other. + useCase: str(source?.useCase) || str(source?.use_case), + website: str(source?.website), + }; + } catch { + return null; + } +} + +/** + * @param {unknown} value + * @returns {string} + */ +function str(value) { + return value == null ? '' : String(value); +} diff --git a/apps/web/src/app/globals.css b/apps/web/src/app/globals.css index 6033d76..f5977ac 100644 --- a/apps/web/src/app/globals.css +++ b/apps/web/src/app/globals.css @@ -482,8 +482,21 @@ h2 { margin-bottom: 2rem; } +/* Stacked fields need air between them. /discover got away without this with + two, because a textarea and an input read as separate blocks on their own; + /sales has four in a row and they ran together into one grey slab. */ +.submit-box > input, +.submit-box > textarea { + margin-bottom: 0.75rem; +} + +/* `email` belongs here for the same reason `text` does, and its absence was a + real bug rather than a style preference: /discover and /sales both take an + address in a `.submit-box`, and without it the field rendered at the browser's + default size beside full-width siblings — a form that looks half broken. */ textarea, input[type='text'], +input[type='email'], input[type='search'] { width: 100%; font-family: var(--mono); diff --git a/apps/web/src/app/layout.jsx b/apps/web/src/app/layout.jsx index b05d870..a51031e 100644 --- a/apps/web/src/app/layout.jsx +++ b/apps/web/src/app/layout.jsx @@ -321,7 +321,11 @@ export default function RootLayout({ children }) {

About · Advertise ·{' '} - Contact · Privacy ·{' '} + {/* Beside Advertise rather than beside About: both are the + commercial half of the footer, and a reader looking for one is + the reader most likely to want the other. */} + Training data · Contact ·{' '} + Privacy ·{' '} Terms ·{' '} Source on GitHub diff --git a/apps/web/src/app/sales/page.jsx b/apps/web/src/app/sales/page.jsx new file mode 100644 index 0000000..2c5e2ad --- /dev/null +++ b/apps/web/src/app/sales/page.jsx @@ -0,0 +1,299 @@ +import { dataset } from '@rssamplifier/db'; + +import { categoryStats } from '../../lib/crawlstats.js'; +import { corpusFigures } from '../../lib/corpus.js'; +import { latestClosedWindow } from '../../lib/datasetWindow.js'; +import Toolbar from '../Toolbar.jsx'; + +export const metadata = { + title: 'Training data', + description: + 'License the RSS Amplifier directory as a training corpus: independent blogs, podcasts and video feeds, sliced on a four-hour clock and streamed as gzipped NDJSON.', +}; + +// Rendered per request, for exactly the reason /advertise is. Left alone, Next +// prerenders a page with no dynamic params at build time, and the build has no +// database — so the figures below would be whatever the build machine could see, +// frozen until the next deploy. On a page whose entire argument is the size and +// freshness of a dataset, stale numbers are worse than none. +export const dynamic = 'force-dynamic'; + +/** + * The corpus offer, without a rate card. + * + * ## Why there is no price on this page + * + * Because there is no price. Corpus licensing is negotiated per buyer — what + * they may keep after the term, whether they may redistribute it, whether + * attribution travels with the text — and those terms move the number by more + * than any volume tier would. A published figure would either be the wrong one + * for everybody, or a fiction with a "contact us for enterprise" underneath it, + * which is the same thing with extra steps. + * + * So this page does one job: say precisely what the data is, so that somebody + * can decide whether to start the conversation, and then start it. + * + * ## Why it is so specific about what the corpus is *not* + * + * Every number here is read live from the directory, and the paragraph about + * article bodies is the most important one on the page. "4.7 million posts" + * invites the reading "4.7 million articles of prose", and that is not what this + * database holds — bodies stopped being stored for every post in + * `0031_item_body_on_demand.sql`, and full text lives in a smaller table. A buyer + * who discovers that after signing is a refund and a reputation; a buyer who + * reads it here is a buyer who wanted the metadata anyway, or who asks us to + * change what the crawler stores, which is a conversation worth having. + * + * The same instinct runs through /advertise ("nothing here is estimated or + * rounded up") and /terms ("measured numbers are claims about the past that + * anybody can check"). This page inherits it because it is the page where the + * temptation to round up is strongest. + */ +export default async function SalesPage({ searchParams }) { + const params = await searchParams; + const sent = params?.sent === '1'; + const error = typeof params?.error === 'string' ? params.error : null; + + const [stats, corpus] = await Promise.all([categoryStats(), corpusFigures()]); + + const feeds = Number(stats?.total ?? 0); + const posts = (stats?.categories ?? []).reduce((n, c) => n + Number(c.items ?? 0), 0); + const crawledLastDay = (stats?.categories ?? []).reduce( + (n, c) => n + Number(c.crawledLastDay ?? 0), + 0, + ); + + return ( + <> +

Training data

+

+ RSS Amplifier crawls {fmt(feeds)} independent feeds — blogs, podcasts and video channels + that publish on their own sites rather than inside a platform. The whole directory is + available as a licensed corpus, sliced on a {dataset.WINDOW_HOURS}-hour clock and streamed + as gzipped NDJSON, so a training pipeline can pull the new rows every{' '} + {dataset.WINDOW_HOURS} hours and keep a mirror current indefinitely. +

+ +

What is in it

+
+
+
Feeds
+
{fmt(feeds)}
+
+
+
Post records
+
{fmt(posts)}
+
+ {corpus ? ( +
+
Full-text articles
+
{fmt(corpus.articles)}
+
+ ) : null} + {corpus ? ( +
+
Authors
+
{fmt(corpus.authors)}
+
+ ) : null} +
+
Feeds re-read in the last day
+
{fmt(crawledLastDay)}
+
+
+

+ Read live from the directory as this page loaded — the same numbers the{' '} + crawler status page reports, from the same cache. Nothing on this + page is estimated or rounded up. +

+ +

Post records and article text are different things

+

+ This is the one thing worth reading twice before you talk to us, because it is the thing + most likely to be assumed wrongly. +

+

+ The post records are the large dataset: title, summary, author, canonical + URL, publication date and the feed each post belongs to, {fmt(posts)} of them, growing by + hundreds of thousands a day. That is metadata at scale, and for a great many uses — link + graphs, recency signals, topic and language distribution, retrieval indexes — it is the part + that matters. +

+

+ The article text is the smaller one + {corpus ? `: ${fmt(corpus.articles)} articles` : ''}, sanitized, averaging{' '} + {corpus ? `${fmt(corpus.sampledAvgChars)} characters` : 'several thousand characters'}{' '} + {corpus ? ( + <> + (sampled over {fmt(corpus.sampleSize)} of them, not summed over all — the exact figure + is a quarter of a million row lookups this database should not be asked for on a page + load) + + ) : null} + . It exists because an article is fetched and cached when a reader opens the post, so it + grows with attention rather than with the crawl. Posts ingested before August 2026 also + carry the body their feed published; newer ones do not, because storing one for every post + was ten gigabytes of a fourteen gigabyte database. +

+

+ If you need prose at the scale of the metadata, say so — that is a change to what the + crawler stores rather than a parameter you can pass, and it is a conversation we are happy + to have. +

+ +

How it is delivered

+

+ Four streams — feeds, items, extracts and{' '} + authors — each one gzipped NDJSON, one JSON object per line, streamed rather + than downloaded from a prepared file. The full machine-readable description lives at{' '} + /api/dataset and needs no account, so you can read the exact + shape of every row before you talk to anybody. +

+

+ Every slice is a half-open range on a fixed {dataset.WINDOW_HOURS}-hour boundary, currently{' '} + {latestClosedWindow()}. That matters more than it sounds: a window is the same + set of rows whoever asks and whenever they ask, so a pipeline that walks boundaries in order + provably sees every row exactly once — no gaps from clock skew, no duplicates from a retry, + and a failed pull can simply be repeated. Only closed windows are served, because the one + containing the present is still filling. +

+

+ Authenticate with a session or an API key as a bearer token. The licence belongs to the + account rather than to the key, so keys can be rotated without telling us. A full-history + pull is available too, separately metered — intended once, to seed a mirror, before + switching to windows for good. +

+ +

Where it comes from

+

+ Public feeds their publishers chose to syndicate. Anyone may{' '} + submit a feed, every feed has{' '} + its own page here, and the directory has been open and free to read + since the day it launched — the JSON API,{' '} + the OPML export, llms.txt and{' '} + the MCP server all still answer without an account, and nothing on this + page changes that. What is licensed here is bulk access, which is a different artifact and a + real cost to serve. +

+

+ A publisher may be excluded from the corpus while staying in the directory, by writing to{' '} + hello@rssamplifier.com. It is a separate ask + from removal on purpose — “list my blog, but do not sell my writing to a model” is a + coherent position and nobody should have to leave the directory to hold it. Excluded feeds, + and every post and article belonging to them, are absent from every stream. + {corpus && corpus.optedOut > 0 ? ` ${fmt(corpus.optedOut)} have asked so far.` : ''} +

+ +

Talk to us

+ {sent ? ( +

+ Thank you — that is recorded and someone will reply to the address you gave. If you would + rather chase it, hello@rssamplifier.com{' '} + reaches the same people. +

+ ) : null} + {error ? ( +

+ {errorMessage(error)} +

+ ) : null} +

+ Licensing is per buyer, so there is no price list here: what you may keep, whether you may + redistribute it and whether attribution travels with the text all move the number more than + volume does. Tell us what you want it for and we will come back with terms. +

+ + {/* A plain form posting to /api/*, like every other control on this site: + it works with JavaScript off, and the route answers an HTML caller with + a 303 back to this anchor and a JSON caller with JSON. */} +
+

Only the email and the last box are required

+ + + +