From 8a1dab870d865d45d5871d57e5c03dc950df577b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 5 Nov 2026 15:53:49 +0000 Subject: [PATCH] Website: hosting headers, one file for the coverage figures - public/_headers caches content-hashed assets and DuckDB-WASM, now in a directory named for its version, as immutable; the engine revalidates. For hosts that read the file; GitHub Pages ignores it. - src/data/figures.json also holds the number of catalogued syntax forms (3,475, from coverage/syntax/baseline.json), which replaces the hard-coded 3,348. The llms.txt files are filled from it too. - npm run check:figures fails while any figure is still a placeholder. --- website/.gitignore | 1 + website/README.md | 12 +++++++++++ website/package.json | 3 ++- website/public/_headers | 22 +++++++++++++++++++++ website/public/index.html | 2 +- website/public/playground/playground.js | 5 +++-- website/scripts/check_figures.mjs | 23 ++++++++++++++++++++++ website/scripts/prepare_playground.mjs | 15 ++++++++------ website/src/content/docs/docs/coverage.mdx | 4 ++-- website/src/data/figures.json | 4 +++- website/src/figures.mjs | 13 ++++++------ 11 files changed, 85 insertions(+), 19 deletions(-) create mode 100644 website/public/_headers create mode 100644 website/scripts/check_figures.mjs diff --git a/website/.gitignore b/website/.gitignore index 0091ba03d..c45f47a34 100644 --- a/website/.gitignore +++ b/website/.gitignore @@ -4,4 +4,5 @@ dist/ public/playground/engine/ wasm/target/ public/playground/duckdb/ +public/playground/duckdb-version.js public/playground/sqlquack.toml diff --git a/website/README.md b/website/README.md index f55a81cac..8db1ef7f6 100644 --- a/website/README.md +++ b/website/README.md @@ -25,6 +25,17 @@ that bundles DuckDB 2.0. Swap it for a stable 2.x release once one ships: change `package.json`, run `npm install`, and check that stage 4 of the playground reports the expected DuckDB version. +The headline coverage figures live in one file, `src/data/figures.json`: the syntax and +translation percentages, their basis lines and the number of catalogued syntax forms. The homepage, +`/docs/coverage/` and the `llms.txt` files all read it at build time. A value left as its own +`{{NAME}}` token shows as a placeholder; `npm run build` warns about it and `npm run check:figures` +fails until every figure is set, so run that before publishing. + +`public/_headers` sets caching for hosts that read it, such as Cloudflare Pages and Netlify: +content-hashed Astro assets and the version-named DuckDB-WASM directory are immutable for a year; +the engine revalidates after five minutes. Both hosts serve Brotli on their own. GitHub Pages +ignores the file and serves gzip with a 10-minute cache. + The logo uses the SQLQuack icon from `sqlstack-io/sqlstack` (`assets/icons/sqlquack.png`); the footer's SQLStack symbols are unchanged copies from the same repository. @@ -45,4 +56,5 @@ npm run build # static site in dist/, including the search index npm run preview # serve dist/ python3 scripts/check_links.py dist npm run test:engine # the engine and its spans, after build_engine.sh +npm run check:figures # fails while a coverage figure is still a placeholder ``` diff --git a/website/package.json b/website/package.json index 6527de0a3..b5e977309 100644 --- a/website/package.json +++ b/website/package.json @@ -9,7 +9,8 @@ "dev": "node scripts/prepare_playground.mjs && astro dev", "build": "node scripts/prepare_playground.mjs && astro build", "preview": "astro preview", - "test:engine": "node --test tests/playground.test.mjs" + "test:engine": "node --test tests/playground.test.mjs", + "check:figures": "node scripts/check_figures.mjs" }, "dependencies": { "@astrojs/starlight": "^0.42.4", diff --git a/website/public/_headers b/website/public/_headers new file mode 100644 index 000000000..d28c6adf9 --- /dev/null +++ b/website/public/_headers @@ -0,0 +1,22 @@ +# Response headers for hosts that read a _headers file, such as Cloudflare Pages and Netlify. Both +# compress with Brotli on their own, .wasm included, so no encoding is set here. GitHub Pages, which +# the SQLClean and SQLBuild sites use, ignores this file: it serves gzip with a 10-minute cache. + +# Content-hashed by Astro: a changed file gets a new name. +/_astro/* + Cache-Control: public, max-age=31536000, immutable + +# DuckDB-WASM, in a directory named for its version: a new version gets a new path. +/playground/duckdb/* + Cache-Control: public, max-age=31536000, immutable + +# The playground engine and pages keep their names across builds, so they revalidate. +/playground/engine/* + Cache-Control: public, max-age=300, must-revalidate + +/fonts/* + Cache-Control: public, max-age=2592000 + +/* + X-Content-Type-Options: nosniff + Referrer-Policy: strict-origin-when-cross-origin diff --git a/website/public/index.html b/website/public/index.html index 514324743..97b684b27 100644 --- a/website/public/index.html +++ b/website/public/index.html @@ -275,7 +275,7 @@

{{TRANSLATION_PCT}} of Snowflake statements translate

{{SYNTAX_PCT}} of documented syntax verified

-

Out of 3,348 documented Snowflake syntax forms. {{SYNTAX_BASIS}}

+

Out of {{SYNTAX_FORMS}} documented Snowflake syntax forms. {{SYNTAX_BASIS}}

Rules checked against Snowflake

diff --git a/website/public/playground/playground.js b/website/public/playground/playground.js index 359f0e3ba..e2ffced55 100644 --- a/website/public/playground/playground.js +++ b/website/public/playground/playground.js @@ -22,8 +22,6 @@ function engine(op, fields) { }); } -// DuckDB-WASM, served from this site (scripts/prepare_playground.mjs copies it in at build time). -const DUCKDB_WASM = new URL('./duckdb/', import.meta.url); const SCHEMA = `{ "ORDERS": [ @@ -177,6 +175,9 @@ let duckdb = null; let running = 0; async function loadDuckDB() { + // Served from this site, in a directory named for its version (scripts/prepare_playground.mjs). + const { DUCKDB_DIR } = await import('./duckdb-version.js'); + const DUCKDB_WASM = new URL(DUCKDB_DIR, import.meta.url); const module = await import(new URL('duckdb-browser.mjs', DUCKDB_WASM).href); const worker = new Worker(new URL('duckdb-browser-base.worker.js', DUCKDB_WASM)); const db = new module.AsyncDuckDB(new module.VoidLogger(), worker); diff --git a/website/scripts/check_figures.mjs b/website/scripts/check_figures.mjs new file mode 100644 index 000000000..964770e3f --- /dev/null +++ b/website/scripts/check_figures.mjs @@ -0,0 +1,23 @@ +// Fails while any headline figure in src/data/figures.json is still its {{NAME}} placeholder, or +// while a built page in dist/ still shows one. Run before publishing: `npm run check:figures`. +import { readFile, readdir } from 'node:fs/promises'; +import path from 'node:path'; +import { placeholders } from '../src/figures.mjs'; + +const problems = placeholders().map((name) => `src/data/figures.json: ${name} is unset`); +async function scan(dir) { + for (const entry of await readdir(dir, { withFileTypes: true }).catch(() => [])) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) await scan(full); + else if (/\.(html|txt)$/.test(entry.name)) { + const left = (await readFile(full, 'utf8')).match(/\{\{[A-Z\\_]+\}\}/g); + if (left) problems.push(`${full}: ${[...new Set(left)].join(', ')}`); + } + } +} +await scan('dist'); +if (problems.length) { + console.error(`placeholder figures remain:\n ${problems.join('\n ')}`); + process.exit(1); +} +console.log('every figure is set'); diff --git a/website/scripts/prepare_playground.mjs b/website/scripts/prepare_playground.mjs index 884214453..14bd288a3 100644 --- a/website/scripts/prepare_playground.mjs +++ b/website/scripts/prepare_playground.mjs @@ -1,16 +1,18 @@ // Prepares the playground's served files, ignored by Git and rebuilt by `npm run build`: -// DuckDB-WASM from node_modules into public/playground/duckdb/, so the playground serves it from -// the site rather than a CDN (its browser module bundled with its Apache Arrow import), and the -// repository's policy file, which the page offers to load. Run from website/. +// DuckDB-WASM from node_modules into public/playground/duckdb//, so the playground serves +// it from the site rather than a CDN (its browser module bundled with its Apache Arrow import), with +// duckdb-version.js naming that directory; and the repository's policy file, which the page offers +// to load. The version in the path lets hosts cache DuckDB as immutable (public/_headers). +// Run from website/. import { build } from 'esbuild'; -import { copyFileSync, mkdirSync, readFileSync, rmSync } from 'node:fs'; +import { copyFileSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; const root = 'node_modules/@duckdb/duckdb-wasm'; const { version } = JSON.parse(readFileSync(join(root, 'package.json'), 'utf8')); -const out = 'public/playground/duckdb'; +const out = `public/playground/duckdb/${version}`; -rmSync(out, { recursive: true, force: true }); +rmSync('public/playground/duckdb', { recursive: true, force: true }); mkdirSync(out, { recursive: true }); await build({ entryPoints: [join(root, 'dist/duckdb-browser.mjs')], @@ -23,5 +25,6 @@ await build({ for (const file of ['duckdb-base.wasm', 'duckdb-browser-base.worker.js']) { copyFileSync(join(root, 'dist', file), join(out, file)); } +writeFileSync('public/playground/duckdb-version.js', `export const DUCKDB_DIR = './duckdb/${version}/';\n`); console.log(`duckdb-wasm ${version} vendored into ${out}`); copyFileSync('../sqlquack.toml', 'public/playground/sqlquack.toml'); diff --git a/website/src/content/docs/docs/coverage.mdx b/website/src/content/docs/docs/coverage.mdx index 717429f4d..cc860b1e9 100644 --- a/website/src/content/docs/docs/coverage.mdx +++ b/website/src/content/docs/docs/coverage.mdx @@ -11,7 +11,7 @@ None of them means every input is proven: a figure says what was checked, under | Measure | Figure | Basis | |---------|--------|-------| | Snowflake statements that translate | {figures.TRANSLATION_PCT} | {figures.TRANSLATION_BASIS} | -| Documented Snowflake syntax forms verified | {figures.SYNTAX_PCT} of 3,348 | {figures.SYNTAX_BASIS} | +| Documented Snowflake syntax forms verified | {figures.SYNTAX_PCT} of {figures.SYNTAX_FORMS} | {figures.SYNTAX_BASIS} | ## Translation @@ -23,7 +23,7 @@ against results recorded on Snowflake, not every possible input. ## Syntax The syntax figure is the share of documented Snowflake syntax forms that meet every obligation -declared for them. The catalogue holds 3,348 forms, one per documented syntax block or variant. +declared for them. The catalogue holds {figures.SYNTAX_FORMS} forms, one per documented syntax block or variant. A form counts as verified only when it has at least one required obligation, every required obligation is satisfied by a case recorded on Snowflake, and an approved review of its family is current. Anything missing, ambiguous or stale counts as not verified. diff --git a/website/src/data/figures.json b/website/src/data/figures.json index f3a57d7f1..b52039608 100644 --- a/website/src/data/figures.json +++ b/website/src/data/figures.json @@ -1,6 +1,8 @@ { - "_comment": "Headline coverage figures, used by the homepage and docs/coverage. A value left as its own {{NAME}} token renders as a visible placeholder, and the build warns about it.", + "_comment": "Headline coverage figures: the only file to edit for them. The homepage, docs/coverage and llms.txt read it at build time. A value left as its own {{NAME}} token renders as a visible placeholder; `npm run build` warns, and `npm run check:figures` fails, while any is left.", + "_sources": "SYNTAX_FORMS is the `forms` count in coverage/syntax/baseline.json. SYNTAX_PCT is the verified share of those forms, as `make syntax-coverage` reports it. TRANSLATION_PCT is the share of actual Snowflake statements that convert in full.", "SYNTAX_PCT": "{{SYNTAX_PCT}}", + "SYNTAX_FORMS": "3,475", "SYNTAX_BASIS": "{{SYNTAX_BASIS}}", "TRANSLATION_PCT": "{{TRANSLATION_PCT}}", "TRANSLATION_BASIS": "{{TRANSLATION_BASIS}}" diff --git a/website/src/figures.mjs b/website/src/figures.mjs index a9f81f848..20ff094b6 100644 --- a/website/src/figures.mjs +++ b/website/src/figures.mjs @@ -1,4 +1,4 @@ -// Fills {{NAME}} tokens from src/data/figures.json: in the static homepage at build time and +// Fills {{NAME}} tokens from src/data/figures.json: in the built HTML and llms.txt files and // while `astro dev` serves it, and in MDX pages through the exported `figures`. import { readFile, readdir, writeFile } from 'node:fs/promises'; import { fileURLToPath } from 'node:url'; @@ -7,18 +7,19 @@ import data from './data/figures.json' with { type: 'json' }; export const figures = Object.fromEntries(Object.entries(data).filter(([name]) => !name.startsWith('_'))); +// Markdown output, such as llms.txt, escapes the underscores: {{SYNTAX\_PCT}}. export function fill(text) { - return text.replace(/\{\{([A-Z_]+)\}\}/g, (token, name) => figures[name] ?? token); + return text.replace(/\{\{([A-Z\\_]+)\}\}/g, (token, name) => figures[name.replaceAll('\\', '')] ?? token); } -const placeholders = () => Object.entries(figures).filter(([name, value]) => value === `{{${name}}}`).map(([name]) => name); +export const placeholders = () => Object.entries(figures).filter(([name, value]) => value === `{{${name}}}`).map(([name]) => name); -async function htmlFiles(dir) { +async function builtFiles(dir) { const entries = await readdir(dir, { withFileTypes: true }); const nested = await Promise.all( entries.map((entry) => { const full = path.join(dir, entry.name); - return entry.isDirectory() ? htmlFiles(full) : entry.name.endsWith('.html') ? [full] : []; + return entry.isDirectory() ? builtFiles(full) : /\.(html|txt)$/.test(entry.name) ? [full] : []; }), ); return nested.flat(); @@ -37,7 +38,7 @@ export function figuresIntegration() { }); }, 'astro:build:done': async ({ dir, logger }) => { - for (const file of await htmlFiles(fileURLToPath(dir))) { + for (const file of await builtFiles(fileURLToPath(dir))) { const html = await readFile(file, 'utf8'); const filled = fill(html); if (filled !== html) await writeFile(file, filled);