Website: hosting headers, one file for the coverage figures

website/.gitignore

@@ -4,4 +4,5 @@ dist/
 public/playground/engine/
 wasm/target/
 public/playground/duckdb/
+public/playground/duckdb-version.js
 public/playground/sqlquack.toml

website/README.md

@@ -25,6 +25,17 @@ that bundles DuckDB 2.0. Swap it for a stable 2.x release once one ships: change
 `package.json`, run `npm install`, and check that stage 4 of the playground reports the expected
 DuckDB version.
 
+The headline coverage figures live in one file, `src/data/figures.json`: the syntax and
+translation percentages, their basis lines and the number of catalogued syntax forms. The homepage,
+`/docs/coverage/` and the `llms.txt` files all read it at build time. A value left as its own
+`{{NAME}}` token shows as a placeholder; `npm run build` warns about it and `npm run check:figures`
+fails until every figure is set, so run that before publishing.
+
+`public/_headers` sets caching for hosts that read it, such as Cloudflare Pages and Netlify:
+content-hashed Astro assets and the version-named DuckDB-WASM directory are immutable for a year;
+the engine revalidates after five minutes. Both hosts serve Brotli on their own. GitHub Pages
+ignores the file and serves gzip with a 10-minute cache.
+
 The logo uses the SQLQuack icon from `sqlstack-io/sqlstack` (`assets/icons/sqlquack.png`); the
 footer's SQLStack symbols are unchanged copies from the same repository.
 
@@ -45,4 +56,5 @@ npm run build    # static site in dist/, including the search index
 npm run preview  # serve dist/
 python3 scripts/check_links.py dist
 npm run test:engine   # the engine and its spans, after build_engine.sh
+npm run check:figures # fails while a coverage figure is still a placeholder
 ```

website/package.json

@@ -9,7 +9,8 @@
     "dev": "node scripts/prepare_playground.mjs && astro dev",
     "build": "node scripts/prepare_playground.mjs && astro build",
     "preview": "astro preview",
-    "test:engine": "node --test tests/playground.test.mjs"
+    "test:engine": "node --test tests/playground.test.mjs",
+    "check:figures": "node scripts/check_figures.mjs"
   },
   "dependencies": {
     "@astrojs/starlight": "^0.42.4",

website/public/_headers

@@ -0,0 +1,22 @@
+# Response headers for hosts that read a _headers file, such as Cloudflare Pages and Netlify. Both
+# compress with Brotli on their own, .wasm included, so no encoding is set here. GitHub Pages, which
+# the SQLClean and SQLBuild sites use, ignores this file: it serves gzip with a 10-minute cache.
+
+# Content-hashed by Astro: a changed file gets a new name.
+/_astro/*
+  Cache-Control: public, max-age=31536000, immutable
+
+# DuckDB-WASM, in a directory named for its version: a new version gets a new path.
+/playground/duckdb/*
+  Cache-Control: public, max-age=31536000, immutable
+
+# The playground engine and pages keep their names across builds, so they revalidate.
+/playground/engine/*
+  Cache-Control: public, max-age=300, must-revalidate
+
+/fonts/*
+  Cache-Control: public, max-age=2592000
+
+/*
+  X-Content-Type-Options: nosniff
+  Referrer-Policy: strict-origin-when-cross-origin

website/public/index.html

@@ -275,7 +275,7 @@ <h3>{{TRANSLATION_PCT}} of Snowflake statements translate</h3>
 						</div>
 						<div class="item">
 							<h3>{{SYNTAX_PCT}} of documented syntax verified</h3>
-							<p>Out of 3,348 documented Snowflake syntax forms. {{SYNTAX_BASIS}}</p>
+							<p>Out of {{SYNTAX_FORMS}} documented Snowflake syntax forms. {{SYNTAX_BASIS}}</p>
 						</div>
 						<div class="item">
 							<h3>Rules checked against Snowflake</h3>

website/public/playground/playground.js

@@ -22,8 +22,6 @@ function engine(op, fields) {
 	});
 }
 
-// DuckDB-WASM, served from this site (scripts/prepare_playground.mjs copies it in at build time).
-const DUCKDB_WASM = new URL('./duckdb/', import.meta.url);
 
 const SCHEMA = `{
   "ORDERS": [
@@ -177,6 +175,9 @@ let duckdb = null;
 let running = 0;
 
 async function loadDuckDB() {
+	// Served from this site, in a directory named for its version (scripts/prepare_playground.mjs).
+	const { DUCKDB_DIR } = await import('./duckdb-version.js');
+	const DUCKDB_WASM = new URL(DUCKDB_DIR, import.meta.url);
 	const module = await import(new URL('duckdb-browser.mjs', DUCKDB_WASM).href);
 	const worker = new Worker(new URL('duckdb-browser-base.worker.js', DUCKDB_WASM));
 	const db = new module.AsyncDuckDB(new module.VoidLogger(), worker);

website/scripts/check_figures.mjs

@@ -0,0 +1,23 @@
+// Fails while any headline figure in src/data/figures.json is still its {{NAME}} placeholder, or
+// while a built page in dist/ still shows one. Run before publishing: `npm run check:figures`.
+import { readFile, readdir } from 'node:fs/promises';
+import path from 'node:path';
+import { placeholders } from '../src/figures.mjs';
+
+const problems = placeholders().map((name) => `src/data/figures.json: ${name} is unset`);
+async function scan(dir) {
+	for (const entry of await readdir(dir, { withFileTypes: true }).catch(() => [])) {
+		const full = path.join(dir, entry.name);
+		if (entry.isDirectory()) await scan(full);
+		else if (/\.(html|txt)$/.test(entry.name)) {
+			const left = (await readFile(full, 'utf8')).match(/\{\{[A-Z\\_]+\}\}/g);
+			if (left) problems.push(`${full}: ${[...new Set(left)].join(', ')}`);
+		}
+	}
+}
+await scan('dist');
+if (problems.length) {
+	console.error(`placeholder figures remain:\n  ${problems.join('\n  ')}`);
+	process.exit(1);
+}
+console.log('every figure is set');

website/scripts/prepare_playground.mjs

@@ -1,16 +1,18 @@
 // Prepares the playground's served files, ignored by Git and rebuilt by `npm run build`:
-// DuckDB-WASM from node_modules into public/playground/duckdb/, so the playground serves it from
-// the site rather than a CDN (its browser module bundled with its Apache Arrow import), and the
-// repository's policy file, which the page offers to load. Run from website/.
+// DuckDB-WASM from node_modules into public/playground/duckdb/<version>/, so the playground serves
+// it from the site rather than a CDN (its browser module bundled with its Apache Arrow import), with
+// duckdb-version.js naming that directory; and the repository's policy file, which the page offers
+// to load. The version in the path lets hosts cache DuckDB as immutable (public/_headers).
+// Run from website/.
 import { build } from 'esbuild';
-import { copyFileSync, mkdirSync, readFileSync, rmSync } from 'node:fs';
+import { copyFileSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
 import { join } from 'node:path';
 
 const root = 'node_modules/@duckdb/duckdb-wasm';
 const { version } = JSON.parse(readFileSync(join(root, 'package.json'), 'utf8'));
-const out = 'public/playground/duckdb';
+const out = `public/playground/duckdb/${version}`;
 
-rmSync(out, { recursive: true, force: true });
+rmSync('public/playground/duckdb', { recursive: true, force: true });
 mkdirSync(out, { recursive: true });
 await build({
 	entryPoints: [join(root, 'dist/duckdb-browser.mjs')],
@@ -23,5 +25,6 @@ await build({
 for (const file of ['duckdb-base.wasm', 'duckdb-browser-base.worker.js']) {
 	copyFileSync(join(root, 'dist', file), join(out, file));
 }
+writeFileSync('public/playground/duckdb-version.js', `export const DUCKDB_DIR = './duckdb/${version}/';\n`);
 console.log(`duckdb-wasm ${version} vendored into ${out}`);
 copyFileSync('../sqlquack.toml', 'public/playground/sqlquack.toml');

website/src/content/docs/docs/coverage.mdx

@@ -11,7 +11,7 @@ None of them means every input is proven: a figure says what was checked, under
 | Measure | Figure | Basis |
 |---------|--------|-------|
 | Snowflake statements that translate | {figures.TRANSLATION_PCT} | {figures.TRANSLATION_BASIS} |
-| Documented Snowflake syntax forms verified | {figures.SYNTAX_PCT} of 3,348 | {figures.SYNTAX_BASIS} |
+| Documented Snowflake syntax forms verified | {figures.SYNTAX_PCT} of {figures.SYNTAX_FORMS} | {figures.SYNTAX_BASIS} |
 
 ## Translation
 
@@ -23,7 +23,7 @@ against results recorded on Snowflake, not every possible input.
 ## Syntax
 
 The syntax figure is the share of documented Snowflake syntax forms that meet every obligation
-declared for them. The catalogue holds 3,348 forms, one per documented syntax block or variant.
+declared for them. The catalogue holds {figures.SYNTAX_FORMS} forms, one per documented syntax block or variant.
 A form counts as verified only when it has at least one required obligation, every required
 obligation is satisfied by a case recorded on Snowflake, and an approved review of its family is
 current. Anything missing, ambiguous or stale counts as not verified.

website/src/data/figures.json

@@ -1,6 +1,8 @@
 {
-	"_comment": "Headline coverage figures, used by the homepage and docs/coverage. A value left as its own {{NAME}} token renders as a visible placeholder, and the build warns about it.",
+	"_comment": "Headline coverage figures: the only file to edit for them. The homepage, docs/coverage and llms.txt read it at build time. A value left as its own {{NAME}} token renders as a visible placeholder; `npm run build` warns, and `npm run check:figures` fails, while any is left.",
+	"_sources": "SYNTAX_FORMS is the `forms` count in coverage/syntax/baseline.json. SYNTAX_PCT is the verified share of those forms, as `make syntax-coverage` reports it. TRANSLATION_PCT is the share of actual Snowflake statements that convert in full.",
 	"SYNTAX_PCT": "{{SYNTAX_PCT}}",
+	"SYNTAX_FORMS": "3,475",
 	"SYNTAX_BASIS": "{{SYNTAX_BASIS}}",
 	"TRANSLATION_PCT": "{{TRANSLATION_PCT}}",
 	"TRANSLATION_BASIS": "{{TRANSLATION_BASIS}}"

website/src/figures.mjs

@@ -1,4 +1,4 @@
-// Fills {{NAME}} tokens from src/data/figures.json: in the static homepage at build time and
+// Fills {{NAME}} tokens from src/data/figures.json: in the built HTML and llms.txt files and
 // while `astro dev` serves it, and in MDX pages through the exported `figures`.
 import { readFile, readdir, writeFile } from 'node:fs/promises';
 import { fileURLToPath } from 'node:url';
@@ -7,18 +7,19 @@ import data from './data/figures.json' with { type: 'json' };
 
 export const figures = Object.fromEntries(Object.entries(data).filter(([name]) => !name.startsWith('_')));
 
+// Markdown output, such as llms.txt, escapes the underscores: {{SYNTAX\_PCT}}.
 export function fill(text) {
-	return text.replace(/\{\{([A-Z_]+)\}\}/g, (token, name) => figures[name] ?? token);
+	return text.replace(/\{\{([A-Z\\_]+)\}\}/g, (token, name) => figures[name.replaceAll('\\', '')] ?? token);
 }
 
-const placeholders = () => Object.entries(figures).filter(([name, value]) => value === `{{${name}}}`).map(([name]) => name);
+export const placeholders = () => Object.entries(figures).filter(([name, value]) => value === `{{${name}}}`).map(([name]) => name);
 
-async function htmlFiles(dir) {
+async function builtFiles(dir) {
 	const entries = await readdir(dir, { withFileTypes: true });
 	const nested = await Promise.all(
 		entries.map((entry) => {
 			const full = path.join(dir, entry.name);
-			return entry.isDirectory() ? htmlFiles(full) : entry.name.endsWith('.html') ? [full] : [];
+			return entry.isDirectory() ? builtFiles(full) : /\.(html|txt)$/.test(entry.name) ? [full] : [];
 		}),
 	);
 	return nested.flat();
@@ -37,7 +38,7 @@ export function figuresIntegration() {
 				});
 			},
 			'astro:build:done': async ({ dir, logger }) => {
-				for (const file of await htmlFiles(fileURLToPath(dir))) {
+				for (const file of await builtFiles(fileURLToPath(dir))) {
 					const html = await readFile(file, 'utf8');
 					const filled = fill(html);
 					if (filled !== html) await writeFile(file, filled);