From 609082f88e1861d252e394902c71f2c39d1c1545 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sat, 29 Aug 2026 18:55:36 +0000 Subject: [PATCH] Sell the directory as a corpus, on a four-hour clock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The directory stays open — every endpoint that answered a stranger yesterday still does, and llms.txt says so in as many words. What is new is a different artifact: the whole thing in bulk, which the open API was never shaped to serve and which this database cannot afford to give away. Four streams (feeds, items, extracts, authors) as gzipped NDJSON, cut on fixed four-hour UTC boundaries. The boundaries are the product, not the packaging: a window is the same set of rows whoever asks and whenever they ask, so a buyer walking them in order provably sees every row exactly once — no gaps from clock skew, no duplicates from a retry. Only closed windows are served, because the one containing the present is still filling. Streamed from a keyset cursor rather than built into a file: there is nowhere to put a file, and a periodic dump job would be a large read against a write-saturated database for an artifact most windows nobody collects. /sales carries no price and no rate card, only a contact form. Corpus licensing is negotiated per buyer, and a self-serve button would be selling terms nobody agreed to. Access is a hand-written row in dataset_grants after a conversation; there is deliberately no endpoint that mints one. Two things the page is careful about, because both would otherwise be assumed wrongly. Post records and article text are different datasets — content_html stopped being written in 0031, so the 4.7M figure is metadata and the prose is a smaller table — and publishers can now be excluded from the corpus while staying in the directory, which is a separate ask from removal on purpose. authors.email never leaves at all. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Pij2tFcRgqceMpheotFSoX --- apps/web/src/app/account/page.jsx | 67 +- apps/web/src/app/api/dataset/[name]/route.js | 234 ++++++ apps/web/src/app/api/dataset/route.js | 117 +++ apps/web/src/app/api/sales/contact/route.js | 193 +++++ apps/web/src/app/globals.css | 13 + apps/web/src/app/layout.jsx | 6 +- apps/web/src/app/sales/page.jsx | 299 ++++++++ apps/web/src/lib/corpus.js | 67 ++ apps/web/src/lib/dataset.js | 110 +++ apps/web/src/lib/datasetWindow.js | 133 ++++ apps/web/src/lib/llms.js | 9 +- apps/web/test/dataset-window.test.js | 121 +++ packages/db/index.js | 1 + .../20260829183852_dataset_access.sql | 150 ++++ packages/db/src/dataset.js | 703 ++++++++++++++++++ packages/db/test/dataset.test.js | 394 ++++++++++ packages/mail/src/send.js | 8 +- 17 files changed, 2619 insertions(+), 6 deletions(-) create mode 100644 apps/web/src/app/api/dataset/[name]/route.js create mode 100644 apps/web/src/app/api/dataset/route.js create mode 100644 apps/web/src/app/api/sales/contact/route.js create mode 100644 apps/web/src/app/sales/page.jsx create mode 100644 apps/web/src/lib/corpus.js create mode 100644 apps/web/src/lib/dataset.js create mode 100644 apps/web/src/lib/datasetWindow.js create mode 100644 apps/web/test/dataset-window.test.js create mode 100644 packages/db/migrations/20260829183852_dataset_access.sql create mode 100644 packages/db/src/dataset.js create mode 100644 packages/db/test/dataset.test.js diff --git a/apps/web/src/app/account/page.jsx b/apps/web/src/app/account/page.jsx index 4fbce17..f729b67 100644 --- a/apps/web/src/app/account/page.jsx +++ b/apps/web/src/app/account/page.jsx @@ -1,5 +1,5 @@ import { redirect } from 'next/navigation'; -import { accounts, apikeys } from '@rssamplifier/db'; +import { accounts, apikeys, dataset } from '@rssamplifier/db'; import Toolbar from '../Toolbar.jsx'; import { AddPasskey } from '../Passkey.jsx'; @@ -7,6 +7,7 @@ import { db } from '../../lib/db.js'; import { currentUser } from '../../lib/auth.js'; import { topicLabel } from '../../lib/following.js'; import { ANONYMOUS_HOURLY } from '../../lib/ratelimit.js'; +import { latestClosedWindow } from '../../lib/datasetWindow.js'; export const dynamic = 'force-dynamic'; @@ -29,13 +30,19 @@ export default async function AccountPage({ searchParams }) { const client = db(); const userId = String(user.id); - const [follows, credentials, topics, keys] = await Promise.all([ + const [follows, credentials, topics, keys, grant] = await Promise.all([ accounts.followedFeeds(client, userId), accounts.credentialsForUser(client, userId), accounts.followedTopics(client, userId), apikeys.keysForUser(client, userId), + dataset.activeGrant(client, userId), ]); + // Only for an account that has one, so the overwhelming majority of readers — + // who will never license the corpus — do not pay a query to be told they have + // no downloads. + const pulls = grant ? await dataset.recentDownloads(client, String(grant.id), 5) : []; + // Revoked keys are kept in the table so a stale key found in a log can still // be identified, but there is nothing for their owner to do about them. const liveKeys = keys.filter((k) => !k.revoked_at); @@ -138,6 +145,62 @@ export default async function AccountPage({ searchParams }) { + {/* Shown to every account, licensed or not, and that is deliberate. An + account with no licence is told so in one line with a link, which is + the answer to "did my access get set up?" — a question that otherwise + arrives by email. There is no button here because there is nothing to + press: a licence is written by hand after a conversation, and a page + that implied otherwise would be promising a checkout that does not + exist. */} +

Training data

+ {grant ? ( + <> +

+ This account holds a {String(grant.plan)} corpus licence, granted{' '} + {formatDate(grant.granted_at)} + {grant.expires_at ? ` and running until ${formatDate(grant.expires_at)}` : ''}. Pull{' '} + {Number(grant.per_window_downloads)} times per dataset per window, and{' '} + {Number(grant.full_dumps_per_day)} full-history dump + {Number(grant.full_dumps_per_day) === 1 ? '' : 's'} a day. Use a session or any of your + API keys as a bearer token — the licence belongs to the account, so rotating a key costs + you nothing. +

+

+ Newest complete window: {latestClosedWindow()}. The shape of every row is + at /api/dataset. +

+ {pulls.length === 0 ? ( +

Nothing pulled yet.

+ ) : ( +
+ {pulls.map((p, i) => ( +
+
{String(p.dataset)}
+
+ {p.full_dump ? 'full history' : String(p.window_start)} +
+ {/* A pull with no completed_at did not finish. Said plainly, + because it is the difference between "that window really + was small" and "your pipeline lost the connection", and + only one of those is worth investigating. */} + {p.completed_at + ? `${Number(p.rows_sent).toLocaleString('en-US')} rows · ${formatDate(p.created_at)}` + : `did not finish · started ${formatDate(p.created_at)}`} +
+
+ ))} +
+ )} + + ) : ( +

+ This account has no corpus licence, and needs none for anything the directory serves + openly — the API, the OPML export and the MCP server all answer without one. Bulk access + to the whole directory as a training corpus is licensed separately:{' '} + what is in it, and how to ask. +

+ )} + {/* What is followed, but not what it published: the river moved to /following, which is the one place the blogs and the topics are merged into a single list and the only one of the two that can be subscribed diff --git a/apps/web/src/app/api/dataset/[name]/route.js b/apps/web/src/app/api/dataset/[name]/route.js new file mode 100644 index 0000000..7639a0c --- /dev/null +++ b/apps/web/src/app/api/dataset/[name]/route.js @@ -0,0 +1,234 @@ +import { dataset } from '@rssamplifier/db'; + +import { db } from '../../../../lib/db.js'; +import { + datasetCaller, + resolveWindow, + latestClosedWindow, + startOfUtcDay, +} from '../../../../lib/dataset.js'; + +export const dynamic = 'force-dynamic'; + +/** + * One dataset, one slice, streamed as gzipped NDJSON. + * + * ## Why this streams instead of building a file + * + * There is nowhere to put a file. This deployment has no object storage, and the + * database it would be built from is the same one serving the site — so a + * "build a dump every four hours" job would be a large periodic read against a + * write-saturated database, producing an artifact most windows nobody collects. + * Streaming inverts it: the work happens when somebody actually wants the data, + * nothing larger than one page is ever resident, and the buyer starts receiving + * rows immediately rather than after the last one is read. + * + * The window boundaries are what make that safe to sell. A streamed response is + * not obviously reproducible — but a slice cut on a fixed clock is the same set + * of rows whoever asks, so "stream it twice and get the same corpus" holds + * without an artifact existing anywhere in between. See `lib/dataset.js`. + * + * ## Why NDJSON and not CSV or Parquet + * + * Because the rows are not rectangular: a post carries an HTML body and an + * author record carries a nested array of links, and both survive a line of JSON + * unchanged. It is also the format that can be produced incrementally — a line + * at a time, with no header to write first and no footer to get right — which is + * what lets a dump be a stream at all. Parquet would be a better fit for a + * buyer's storage and a worse fit for ours; converting on receipt is one line of + * their pipeline. + * + * ## Why the count in the audit log can be short + * + * `finishDownload` runs when the stream closes cleanly. If the buyer hangs up + * partway, it never runs, and the row keeps a null `completed_at` — which is the + * signal that the pull broke rather than that it was small. That is deliberate: + * a broken pull must still count against the window allowance, or hanging up + * becomes a free way to run the query in a loop. + */ + +/** + * @param {Request} req + * @param {{ params: Promise<{ name: string }> }} ctx + * @returns {Promise} + */ +export async function GET(req, { params }) { + const { name: raw } = await params; + // `.jsonl.gz` is stripped rather than required, so a caller may write the URL + // as a filename and get a file. Both spellings reach the same stream. + const name = String(raw ?? '') + .toLowerCase() + .replace(/\.(ndjson|jsonl)(\.gz)?$/, ''); + + const stream = dataset.streamFor(name); + if (!stream) { + return Response.json( + { + error: 'unknown-dataset', + detail: `No dataset called "${name}".`, + datasets: dataset.DATASETS, + manifest: '/api/dataset', + }, + { status: 404, headers: { 'access-control-allow-origin': '*' } }, + ); + } + + const caller = await datasetCaller(req); + if (!caller.ok) return caller.response; + + const { user, grant, apiKeyId } = caller; + const client = db(); + const url = new URL(req.url); + const full = url.searchParams.get('full') === '1'; + + // Two different allowances, because they bound two different costs. A window + // is a bounded index range; a full dump walks the table. + if (full) { + const taken = await dataset.fullDumpCount(client, String(grant.id), startOfUtcDay()); + const allowed = Number(grant.full_dumps_per_day) || 0; + if (taken >= allowed) { + return exhausted( + 'full-dump-limit', + `This licence allows ${allowed} full-history pull${allowed === 1 ? '' : 's'} a day and has used ${taken}. Incremental windows are not affected — take ${latestClosedWindow()} instead.`, + ); + } + } + + const slice = full + ? { ok: true, start: null, end: null } + : resolveWindow(url.searchParams.get('window')); + + if (!slice.ok) { + return Response.json( + { error: slice.error, detail: slice.detail, manifest: '/api/dataset' }, + { status: 400, headers: { 'access-control-allow-origin': '*' } }, + ); + } + + if (!full) { + const taken = await dataset.windowDownloadCount( + client, + String(grant.id), + name, + String(slice.start), + ); + const allowed = Number(grant.per_window_downloads) || 0; + if (taken >= allowed) { + return exhausted( + 'window-limit', + `This licence allows ${allowed} pulls of one dataset per ${dataset.WINDOW_HOURS}-hour window, and has taken ${taken} of ${name} for ${slice.start}.`, + ); + } + } + + // Opened before the first byte, closed when the last one lands. See the note + // at the top on why the order matters. + const downloadId = await dataset.startDownload(client, { + grantId: String(grant.id), + userId: String(user.id), + dataset: name, + windowStart: full ? null : String(slice.start), + fullDump: full, + apiKeyId, + }); + + const encoder = new TextEncoder(); + let rows = 0; + + const lines = new ReadableStream({ + async start(controller) { + try { + for await (const row of stream(client, { since: slice.start, until: slice.end })) { + controller.enqueue(encoder.encode(`${JSON.stringify(shape(name, row))}\n`)); + rows += 1; + } + await dataset.finishDownload(client, downloadId, rows); + } catch (err) { + // The response is already on the wire with a 200, so this cannot become + // an error status — the same bind the OPML export is in. The download + // row is left with a null `completed_at`, which is exactly the record of + // "this one broke", and the buyer sees a truncated file rather than a + // silently short one because the byte count will not match a re-pull of + // the same window. + console.error(`dataset stream failed partway: ${name}`, err); + } + controller.close(); + }, + }); + + const filename = full + ? `rssamplifier-${name}-full.jsonl.gz` + : `rssamplifier-${name}-${String(slice.start).replace(/[:.]/g, '')}.jsonl.gz`; + + return new Response(lines.pipeThrough(new CompressionStream('gzip')), { + headers: { + 'content-type': 'application/x-ndjson', + 'content-encoding': 'gzip', + 'content-disposition': `attachment; filename="${filename}"`, + // The buyer's pipeline reads these to decide when to ask again, so they + // are part of the interface rather than diagnostics. + 'x-dataset-name': name, + 'x-dataset-window': full ? 'full' : String(slice.start), + 'x-dataset-window-end': full ? '' : String(slice.end), + 'x-dataset-window-hours': String(dataset.WINDOW_HOURS), + 'x-dataset-next-window': latestClosedWindow(), + // No caching anywhere in between: a window is stable but a licence is not, + // and a proxy holding a corpus slice would serve it to the next caller. + 'cache-control': 'private, no-store', + 'access-control-allow-origin': '*', + }, + }); +} + +/** + * A row, as the buyer sees it. + * + * The database's column names are an implementation detail and several of them + * would be actively misleading in a corpus — `cursor_rowid` is paging state that + * means nothing outside this process, and `links` arrives from SQLite as a JSON + * *string* that would otherwise land in the file double-encoded and have to be + * parsed twice. + * + * Everything else keeps its snake_case name deliberately. The manifest describes + * the columns, `/crawlstats` and the open API use the same vocabulary, and + * renaming them here would mean the corpus and the documentation disagree. + * + * @param {string} name + * @param {Record} row + * @returns {Record} + */ +function shape(name, row) { + const { cursor_rowid: _cursor, ...rest } = row; + + if (name === 'authors' && typeof rest['links'] === 'string') { + try { + rest['links'] = JSON.parse(String(rest['links'])); + } catch { + // json_group_array cannot really produce invalid JSON, but a corpus line + // that fails to parse is worse than one whose links arrive as a string. + rest['links'] = []; + } + } + + if (name === 'feeds' && typeof rest['categories'] === 'string') { + try { + rest['categories'] = JSON.parse(String(rest['categories'])); + } catch { + rest['categories'] = []; + } + } + + return rest; +} + +/** + * @param {string} error + * @param {string} detail + * @returns {Response} + */ +function exhausted(error, detail) { + return Response.json( + { error, detail, manifest: '/api/dataset' }, + { status: 429, headers: { 'access-control-allow-origin': '*' } }, + ); +} diff --git a/apps/web/src/app/api/dataset/route.js b/apps/web/src/app/api/dataset/route.js new file mode 100644 index 0000000..183de1c --- /dev/null +++ b/apps/web/src/app/api/dataset/route.js @@ -0,0 +1,117 @@ +import { dataset } from '@rssamplifier/db'; + +import { latestClosedWindow, windowEnd } from '../../../lib/datasetWindow.js'; +import { siteUrl } from '../../../lib/db.js'; + +export const dynamic = 'force-dynamic'; + +/** + * The manifest: what the corpus contains, how it is cut, and how to take it. + * + * Open, and answers without a licence. That is the whole reason it is a separate + * route from the streams it describes. + * + * A buyer's first question is "what is actually in there", and the honest way to + * answer it is a machine-readable description they can read before paying rather + * than a paragraph of marketing they have to trust. It also means the pipeline + * that will pull this every four hours can discover the current boundary instead + * of computing it — so if the cadence ever changes, their code follows without + * an email. + * + * What it deliberately does not contain is a price. There is none in this + * codebase; licensing is negotiated, and the manifest points at /sales for it. + * + * It also does not contain row counts. A `count(*)` over `feed_items` is a + * multi-second scan of 4.7 million rows against a database whose write path is + * already saturated, and this is an unauthenticated endpoint — putting one here + * would be handing anybody a way to make the site slow. The sales page shows the + * cached directory figures instead, and a licensed caller learns the exact size + * of a window by taking it. + */ + +/** + * @returns {Response} + */ +export function GET() { + const newest = latestClosedWindow(); + const base = siteUrl(); + + return Response.json( + { + corpus: 'rssamplifier.com', + description: + 'The open small-web directory as a training corpus: independent blogs, podcasts and video feeds, crawled continuously and sliced on a fixed clock.', + + cadence: { + windowHours: dataset.WINDOW_HOURS, + // Named rather than implied. A pipeline reading this should walk + // boundaries in order; saying so here is cheaper than saying it in an + // email to every buyer. + semantics: + 'Each window is the half-open range [start, end) on the row timestamp named in `cutOn`. Windows are fixed against the Unix epoch, so a window is the same set of rows whoever asks and whenever they ask. Walk them in order to see every row exactly once.', + latestClosedWindow: newest, + latestWindowEnd: windowEnd(newest), + nextWindowOpensAt: windowEnd(newest), + }, + + datasets: { + feeds: { + description: + 'One row per feed in the directory: title, description, canonical feed and site URLs, language, category, item count and crawl state.', + cutOn: 'created_at — when the directory first saw the feed, so a window is the feeds that are new to it', + format: 'application/x-ndjson, gzipped', + url: `${base}/api/dataset/feeds`, + }, + items: { + description: + 'One row per post: title, summary, author, canonical URL, publication date, and the feed it belongs to. This is metadata at scale rather than prose — see `bodies` below.', + cutOn: 'created_at — when the crawler ingested the post', + format: 'application/x-ndjson, gzipped', + url: `${base}/api/dataset/items`, + // Said here, unprompted, because it is the one thing about this corpus + // a buyer could reasonably assume wrongly and only discover after + // paying. Posts ingested before the change still carry their body. + bodies: + 'Posts ingested since 2026-08 do not carry `content_html`: storing a body for every post was 10GB of a 14GB database and the crawler stopped writing it. Full article text lives in the `extracts` dataset, for the subset that has been fetched. Older rows still carry the body they were ingested with.', + }, + extracts: { + description: + 'The article itself, sanitized, for posts whose page has been fetched and parsed. Several thousand characters on average. This is the prose in the corpus.', + cutOn: 'fetched_at — when the article was read, not when it was published, so a window includes an old post that was fetched today', + format: 'application/x-ndjson, gzipped', + url: `${base}/api/dataset/extracts`, + coverage: + 'A fraction of `items`, not a parallel of it: an article is fetched when a reader opens the post, so this grows with attention rather than with the crawl.', + }, + authors: { + description: + 'The people behind the feeds, keyed on a URL they control, with their homepages and social profiles folded in as a JSON array.', + cutOn: 'created_at, filtered rather than seeked — `authors` is small enough to walk whole', + format: 'application/x-ndjson, gzipped', + url: `${base}/api/dataset/authors`, + }, + }, + + access: { + how: 'A licence, granted per buyer. Present a session cookie or an API key as a bearer token; the licence hangs off the account rather than the key, so keys can be rotated freely.', + enquiries: `${base}/sales`, + parameters: { + window: + 'ISO-8601 window boundary. Omit for the newest closed window. An unaligned or still-filling boundary is refused rather than rounded.', + full: 'full=1 takes the whole history instead of one window. Separately metered, and expensive — intended once, to seed a mirror, before switching to windows.', + }, + limits: + 'Per-window and per-day allowances are set on the licence and reported in the `x-dataset-*` response headers.', + }, + + provenance: { + source: + 'Public feeds their publishers chose to syndicate. The directory is open and anyone may submit to it.', + optOut: + 'A publisher may be excluded from the corpus while staying in the directory, on request to hello@rssamplifier.com. Excluded feeds and every post and article belonging to them are absent from every dataset above.', + openApi: `Nothing here is a restriction on the open API: ${base}/api/feeds, /api/search, /opml and the MCP server all answer without an account.`, + }, + }, + { headers: { 'access-control-allow-origin': '*', 'cache-control': 'public, max-age=60' } }, + ); +} diff --git a/apps/web/src/app/api/sales/contact/route.js b/apps/web/src/app/api/sales/contact/route.js new file mode 100644 index 0000000..29b99ef --- /dev/null +++ b/apps/web/src/app/api/sales/contact/route.js @@ -0,0 +1,193 @@ +import { dataset } from '@rssamplifier/db'; +import { sendEmail, emailEnabled } from '@rssamplifier/mail'; + +import { db } from '../../../../lib/db.js'; +import { requestMeta } from '../../../../lib/auth.js'; + +export const dynamic = 'force-dynamic'; + +/** + * The only way into a corpus conversation. + * + * ## Why a form here when /contact deliberately has none + * + * That page's comment says it plainly: "no form, because a form on a site with + * no accounts is one more thing to spam", and it routes to published addresses + * instead. The reasoning holds and this is not a reversal of it — it is a + * different trade. /contact's job is to get a stranger to the right mailbox, + * which a list of links does perfectly well. This page's job is to start a + * negotiation, and the fields below (what they want it for, at what scale) are + * exactly the ones that decide whether there is a deal and what it looks like. + * An email that omits them costs a round trip; a form that asks for them does + * not. So the spam is paid for rather than avoided, by the two guards below. + * + * ## Why it is written down before it is sent + * + * `sendEmail` reports a failure instead of throwing, and every caller in this + * codebase treats mail as optional infrastructure. That is right for a sign-in + * link, which the reader can simply request again — it is wrong for a sales + * enquiry, which the sender believes is delivered and will never send twice. So + * the row lands first, the mail is best-effort on top of it, and a Resend outage + * costs a notification rather than a customer. + */ + +/** How long the flood window is. */ +const WINDOW_MS = 60 * 60 * 1000; + +/** + * Enquiries one address may send in that window. + * + * Three, not one. A genuine sender who mistypes their own email address and + * resends is the most likely repeat here, and refusing them is worse than + * accepting a third message from a spammer we are also storing and rate + * limiting. + */ +const MAX_PER_WINDOW = 3; + +/** Where an enquiry is announced, when mail is configured. */ +const INBOX = process.env['SALES_EMAIL'] || 'hello@rssamplifier.com'; + +/** + * @param {Request} req + * @returns {Promise} + */ +export async function POST(req) { + const wantsHtml = (req.headers.get('accept') ?? '').includes('text/html'); + const body = await readBody(req); + if (!body) return respond(wantsHtml, 400, 'bad-request', 'That form could not be read.'); + + // The honeypot. A field no human sees, no browser fills and every naive bot + // completes. Answered with the same success the real path gives, because a + // bot told it failed is a bot that tries again differently. + if (body.website) { + return respond(wantsHtml, 200, null, null); + } + + const email = body.email.trim().toLowerCase(); + // Deliberately the weakest possible check. This address is going to be read by + // a person who will reply to it, not authenticated — a strict pattern here + // rejects the valid addresses nobody remembers are valid and gains nothing. + if (!email || !email.includes('@') || email.length > 200) { + return respond(wantsHtml, 400, 'bad-email', 'That does not look like an email address.'); + } + + const useCase = body.useCase.trim(); + if (useCase.length < 10) { + return respond( + wantsHtml, + 400, + 'no-use-case', + 'Please say what you want the corpus for — it is the field that decides what we can offer.', + ); + } + + const client = db(); + const meta = await requestMeta(); + + const recent = await dataset.enquiryCountFrom( + client, + meta.ipHash, + new Date(Date.now() - WINDOW_MS).toISOString(), + ); + if (recent >= MAX_PER_WINDOW) { + return respond( + wantsHtml, + 429, + 'too-many', + 'That is several enquiries in an hour. Email hello@rssamplifier.com directly and we will pick it up there.', + ); + } + + await dataset.insertEnquiry(client, { + name: body.name.trim().slice(0, 120) || null, + email: email.slice(0, 200), + org: body.org.trim().slice(0, 160) || null, + useCase: useCase.slice(0, 4000), + ipHash: meta.ipHash, + userAgent: meta.userAgent, + }); + + if (emailEnabled()) { + // Not awaited into the response. The enquiry is already stored, so the + // sender's answer does not depend on Resend being reachable this second, and + // making them wait on it would only ever make the form feel broken. + void sendEmail({ + to: INBOX, + subject: `Corpus enquiry: ${body.org.trim() || email}`, + text: [ + `From: ${body.name.trim() || '(no name)'} <${email}>`, + `Company: ${body.org.trim() || '(none given)'}`, + '', + 'What for:', + useCase.slice(0, 4000), + '', + // So a reply can be written without opening the database, and so the + // operator knows the row exists even if this mail is the only thing they + // ever see. + 'Stored in dataset_enquiries. Grant access with an insert into dataset_grants.', + ].join('\n'), + // Replying to the notification reaches the sender rather than ourselves, + // which is the difference between a queue and an inbox. + replyTo: email, + }).catch(() => {}); + } + + return respond(wantsHtml, 200, null, null); +} + +/** + * HTML callers get a 303 back to the page; everyone else gets JSON. + * + * @param {boolean} wantsHtml + * @param {number} status + * @param {string|null} error + * @param {string|null} detail + * @returns {Response} + */ +function respond(wantsHtml, status, error, detail) { + if (wantsHtml) { + const to = error ? `/sales?error=${encodeURIComponent(error)}#enquire` : '/sales?sent=1#enquire'; + return new Response(null, { status: 303, headers: { location: to } }); + } + + return Response.json(error ? { error, detail } : { ok: true }, { + status, + headers: { 'cache-control': 'no-store' }, + }); +} + +/** + * Read a form or a JSON body into the same shape. + * + * @param {Request} req + * @returns {Promise<{ name: string, email: string, org: string, useCase: string, website: string }|null>} + */ +async function readBody(req) { + try { + const source = (req.headers.get('content-type') ?? '').includes('application/json') + ? await req.json() + : Object.fromEntries(await req.formData()); + + return { + name: str(source?.name), + email: str(source?.email), + org: str(source?.org), + // `useCase` for a JSON caller, `use_case` for the form field — the form + // uses the name that reads correctly in HTML and the API uses the one that + // reads correctly in JavaScript, and neither should have to know about the + // other. + useCase: str(source?.useCase) || str(source?.use_case), + website: str(source?.website), + }; + } catch { + return null; + } +} + +/** + * @param {unknown} value + * @returns {string} + */ +function str(value) { + return value == null ? '' : String(value); +} diff --git a/apps/web/src/app/globals.css b/apps/web/src/app/globals.css index 6033d76..f5977ac 100644 --- a/apps/web/src/app/globals.css +++ b/apps/web/src/app/globals.css @@ -482,8 +482,21 @@ h2 { margin-bottom: 2rem; } +/* Stacked fields need air between them. /discover got away without this with + two, because a textarea and an input read as separate blocks on their own; + /sales has four in a row and they ran together into one grey slab. */ +.submit-box > input, +.submit-box > textarea { + margin-bottom: 0.75rem; +} + +/* `email` belongs here for the same reason `text` does, and its absence was a + real bug rather than a style preference: /discover and /sales both take an + address in a `.submit-box`, and without it the field rendered at the browser's + default size beside full-width siblings — a form that looks half broken. */ textarea, input[type='text'], +input[type='email'], input[type='search'] { width: 100%; font-family: var(--mono); diff --git a/apps/web/src/app/layout.jsx b/apps/web/src/app/layout.jsx index b05d870..a51031e 100644 --- a/apps/web/src/app/layout.jsx +++ b/apps/web/src/app/layout.jsx @@ -321,7 +321,11 @@ export default function RootLayout({ children }) {

About · Advertise ·{' '} - Contact · Privacy ·{' '} + {/* Beside Advertise rather than beside About: both are the + commercial half of the footer, and a reader looking for one is + the reader most likely to want the other. */} + Training data · Contact ·{' '} + Privacy ·{' '} Terms ·{' '} Source on GitHub diff --git a/apps/web/src/app/sales/page.jsx b/apps/web/src/app/sales/page.jsx new file mode 100644 index 0000000..2c5e2ad --- /dev/null +++ b/apps/web/src/app/sales/page.jsx @@ -0,0 +1,299 @@ +import { dataset } from '@rssamplifier/db'; + +import { categoryStats } from '../../lib/crawlstats.js'; +import { corpusFigures } from '../../lib/corpus.js'; +import { latestClosedWindow } from '../../lib/datasetWindow.js'; +import Toolbar from '../Toolbar.jsx'; + +export const metadata = { + title: 'Training data', + description: + 'License the RSS Amplifier directory as a training corpus: independent blogs, podcasts and video feeds, sliced on a four-hour clock and streamed as gzipped NDJSON.', +}; + +// Rendered per request, for exactly the reason /advertise is. Left alone, Next +// prerenders a page with no dynamic params at build time, and the build has no +// database — so the figures below would be whatever the build machine could see, +// frozen until the next deploy. On a page whose entire argument is the size and +// freshness of a dataset, stale numbers are worse than none. +export const dynamic = 'force-dynamic'; + +/** + * The corpus offer, without a rate card. + * + * ## Why there is no price on this page + * + * Because there is no price. Corpus licensing is negotiated per buyer — what + * they may keep after the term, whether they may redistribute it, whether + * attribution travels with the text — and those terms move the number by more + * than any volume tier would. A published figure would either be the wrong one + * for everybody, or a fiction with a "contact us for enterprise" underneath it, + * which is the same thing with extra steps. + * + * So this page does one job: say precisely what the data is, so that somebody + * can decide whether to start the conversation, and then start it. + * + * ## Why it is so specific about what the corpus is *not* + * + * Every number here is read live from the directory, and the paragraph about + * article bodies is the most important one on the page. "4.7 million posts" + * invites the reading "4.7 million articles of prose", and that is not what this + * database holds — bodies stopped being stored for every post in + * `0031_item_body_on_demand.sql`, and full text lives in a smaller table. A buyer + * who discovers that after signing is a refund and a reputation; a buyer who + * reads it here is a buyer who wanted the metadata anyway, or who asks us to + * change what the crawler stores, which is a conversation worth having. + * + * The same instinct runs through /advertise ("nothing here is estimated or + * rounded up") and /terms ("measured numbers are claims about the past that + * anybody can check"). This page inherits it because it is the page where the + * temptation to round up is strongest. + */ +export default async function SalesPage({ searchParams }) { + const params = await searchParams; + const sent = params?.sent === '1'; + const error = typeof params?.error === 'string' ? params.error : null; + + const [stats, corpus] = await Promise.all([categoryStats(), corpusFigures()]); + + const feeds = Number(stats?.total ?? 0); + const posts = (stats?.categories ?? []).reduce((n, c) => n + Number(c.items ?? 0), 0); + const crawledLastDay = (stats?.categories ?? []).reduce( + (n, c) => n + Number(c.crawledLastDay ?? 0), + 0, + ); + + return ( + <> +

Training data

+

+ RSS Amplifier crawls {fmt(feeds)} independent feeds — blogs, podcasts and video channels + that publish on their own sites rather than inside a platform. The whole directory is + available as a licensed corpus, sliced on a {dataset.WINDOW_HOURS}-hour clock and streamed + as gzipped NDJSON, so a training pipeline can pull the new rows every{' '} + {dataset.WINDOW_HOURS} hours and keep a mirror current indefinitely. +

+ +

What is in it

+
+
+
Feeds
+
{fmt(feeds)}
+
+
+
Post records
+
{fmt(posts)}
+
+ {corpus ? ( +
+
Full-text articles
+
{fmt(corpus.articles)}
+
+ ) : null} + {corpus ? ( +
+
Authors
+
{fmt(corpus.authors)}
+
+ ) : null} +
+
Feeds re-read in the last day
+
{fmt(crawledLastDay)}
+
+
+

+ Read live from the directory as this page loaded — the same numbers the{' '} + crawler status page reports, from the same cache. Nothing on this + page is estimated or rounded up. +

+ +

Post records and article text are different things

+

+ This is the one thing worth reading twice before you talk to us, because it is the thing + most likely to be assumed wrongly. +

+

+ The post records are the large dataset: title, summary, author, canonical + URL, publication date and the feed each post belongs to, {fmt(posts)} of them, growing by + hundreds of thousands a day. That is metadata at scale, and for a great many uses — link + graphs, recency signals, topic and language distribution, retrieval indexes — it is the part + that matters. +

+

+ The article text is the smaller one + {corpus ? `: ${fmt(corpus.articles)} articles` : ''}, sanitized, averaging{' '} + {corpus ? `${fmt(corpus.sampledAvgChars)} characters` : 'several thousand characters'}{' '} + {corpus ? ( + <> + (sampled over {fmt(corpus.sampleSize)} of them, not summed over all — the exact figure + is a quarter of a million row lookups this database should not be asked for on a page + load) + + ) : null} + . It exists because an article is fetched and cached when a reader opens the post, so it + grows with attention rather than with the crawl. Posts ingested before August 2026 also + carry the body their feed published; newer ones do not, because storing one for every post + was ten gigabytes of a fourteen gigabyte database. +

+

+ If you need prose at the scale of the metadata, say so — that is a change to what the + crawler stores rather than a parameter you can pass, and it is a conversation we are happy + to have. +

+ +

How it is delivered

+

+ Four streams — feeds, items, extracts and{' '} + authors — each one gzipped NDJSON, one JSON object per line, streamed rather + than downloaded from a prepared file. The full machine-readable description lives at{' '} + /api/dataset and needs no account, so you can read the exact + shape of every row before you talk to anybody. +

+

+ Every slice is a half-open range on a fixed {dataset.WINDOW_HOURS}-hour boundary, currently{' '} + {latestClosedWindow()}. That matters more than it sounds: a window is the same + set of rows whoever asks and whenever they ask, so a pipeline that walks boundaries in order + provably sees every row exactly once — no gaps from clock skew, no duplicates from a retry, + and a failed pull can simply be repeated. Only closed windows are served, because the one + containing the present is still filling. +

+

+ Authenticate with a session or an API key as a bearer token. The licence belongs to the + account rather than to the key, so keys can be rotated without telling us. A full-history + pull is available too, separately metered — intended once, to seed a mirror, before + switching to windows for good. +

+ +

Where it comes from

+

+ Public feeds their publishers chose to syndicate. Anyone may{' '} + submit a feed, every feed has{' '} + its own page here, and the directory has been open and free to read + since the day it launched — the JSON API,{' '} + the OPML export, llms.txt and{' '} + the MCP server all still answer without an account, and nothing on this + page changes that. What is licensed here is bulk access, which is a different artifact and a + real cost to serve. +

+

+ A publisher may be excluded from the corpus while staying in the directory, by writing to{' '} + hello@rssamplifier.com. It is a separate ask + from removal on purpose — “list my blog, but do not sell my writing to a model” is a + coherent position and nobody should have to leave the directory to hold it. Excluded feeds, + and every post and article belonging to them, are absent from every stream. + {corpus && corpus.optedOut > 0 ? ` ${fmt(corpus.optedOut)} have asked so far.` : ''} +

+ +

Talk to us

+ {sent ? ( +

+ Thank you — that is recorded and someone will reply to the address you gave. If you would + rather chase it, hello@rssamplifier.com{' '} + reaches the same people. +

+ ) : null} + {error ? ( +

+ {errorMessage(error)} +

+ ) : null} +

+ Licensing is per buyer, so there is no price list here: what you may keep, whether you may + redistribute it and whether attribution travels with the text all move the number more than + volume does. Tell us what you want it for and we will come back with terms. +

+ + {/* A plain form posting to /api/*, like every other control on this site: + it works with JavaScript off, and the route answers an HTML caller with + a 303 back to this anchor and a JSON caller with JSON. */} +
+

Only the email and the last box are required

+ + + +