playground: load ICU and JSON from the site before any SQL runs

website/package.json

@@ -9,7 +9,7 @@
     "dev": "node scripts/prepare_playground.mjs && astro dev",
     "build": "node scripts/prepare_playground.mjs && astro build",
     "preview": "astro preview",
-    "test:engine": "node --test tests/playground.test.mjs",
+    "test:engine": "node scripts/prepare_playground.mjs && node --test tests/playground.test.mjs",
     "check:figures": "node scripts/check_figures.mjs"
   },
   "dependencies": {

website/public/playground/playground.js

@@ -2,7 +2,7 @@
 // worker) and shows its stages; translated SQL runs in DuckDB-WASM, which downloads in the
 // background once the page is idle.
 
-import { DUCKDB_DIR, DUCKDB_WASM_BYTES } from './duckdb-version.js';
+import { DUCKDB_DIR, DUCKDB_EXTENSIONS, DUCKDB_WASM_BYTES } from './duckdb-version.js';
 import { ENGINE_DIR, ENGINE_WASM_BYTES } from './engine-version.js';
 import { CAPTIONS, EXAMPLE_GROUPS, EXAMPLES, ROWS, SCHEMA, stopLabel } from './examples.js';
 import { escape, previewLimit, rendered, resultTable, underlined } from './spans.js';
@@ -328,7 +328,17 @@ async function loadDuckDB() {
 	const module = await import(moduleUrl);
 	const db = new module.AsyncDuckDB(new module.VoidLogger(), new Worker(workerUrl));
 	await db.instantiate(wasmUrl);
-	const version = await (await db.connect()).query('SELECT version() AS v');
+	const conn = await db.connect();
+	// The extensions this build lacks, from the site, before any SQL: DuckDB autoloads ICU while
+	// binding the first statement that needs it, which corrupts that statement's function lookups.
+	await conn.query(`SET custom_extension_repository = '${here(`${DUCKDB_DIR}extensions`)}'`);
+	for (const extension of DUCKDB_EXTENSIONS) {
+		await conn.query(`LOAD ${extension}`).catch((error) => {
+			throw new Error(`the ${extension} extension did not load: ${error.message ?? error}`);
+		});
+	}
+	const version = await conn.query('SELECT version() AS v');
+	await conn.close();
 	return { db, version: String(version.toArray()[0].v) };
 }
 

website/scripts/prepare_playground.mjs

@@ -1,7 +1,8 @@
 // Prepares the playground's served files, ignored by Git and rebuilt by `npm run build`: the engine
 // from wasm/pkg into public/playground/engine/<hash>/, with engine-version.js naming that directory;
 // DuckDB-WASM from node_modules into public/playground/duckdb/<version>/, so the playground serves
-// it from the site rather than a CDN (its browser module bundled with its Apache Arrow import), with
+// it from the site rather than a CDN (its browser module bundled with its Apache Arrow import, and
+// the extensions the page loads), with
 // duckdb-version.js naming that directory; and the repository's policy file, which the page offers
 // to load. The version in the path lets hosts cache DuckDB as immutable (public/_headers).
 // Run from website/.
@@ -48,11 +49,48 @@ await build({
 for (const file of ['duckdb-base.wasm', 'duckdb-browser-base.worker.js']) {
 	copyFileSync(join(root, 'dist', file), join(out, file));
 }
+// The extensions the page loads before any SQL runs, served from the site in DuckDB's repository
+// layout so the page never fetches from extensions.duckdb.org. ICU must load before the first query:
+// this build lacks it, and autoloading it while a statement binds (CURRENT_DATE does) corrupts the
+// binder's function lookups. The sample schema's JSON column needs json. Pinned per DuckDB-WASM
+// version, since an extension only loads into the DuckDB build it was compiled for.
+const EXTENSIONS = {
+	'1.33.1-dev114.0': {
+		path: 'v2.0.0/wasm_eh',
+		sha256: {
+			icu: '0e379b9d688f0df82192f15f9405f60861f478956ad0282ce831572695458308',
+			json: '0d4e36ac68c53809c27a4f1e7801142409b2735ad2f7cb7f8b09b8db8d558984',
+		},
+	},
+};
+const pinned = EXTENSIONS[version];
+if (!pinned) {
+	console.error(`prepare_playground: no extensions pinned for duckdb-wasm ${version}; add them to EXTENSIONS`);
+	process.exit(1);
+}
+const downloads = 'node_modules/.cache/duckdb-extensions';
+mkdirSync(downloads, { recursive: true });
+mkdirSync(join(out, 'extensions', pinned.path), { recursive: true });
+for (const [name, sha256] of Object.entries(pinned.sha256)) {
+	const file = `${name}.duckdb_extension.wasm`;
+	const download = join(downloads, `${sha256}.wasm`);
+	if (!existsSync(download)) {
+		const url = `https://extensions.duckdb.org/${pinned.path}/${file}`;
+		const response = await fetch(url);
+		if (!response.ok) throw new Error(`${url}: HTTP ${response.status}`);
+		const bytes = Buffer.from(await response.arrayBuffer());
+		const actual = createHash('sha256').update(bytes).digest('hex');
+		if (actual !== sha256) throw new Error(`${url}: sha256 ${actual}, expected ${sha256}`);
+		writeFileSync(download, bytes);
+	}
+	copyFileSync(download, join(out, 'extensions', pinned.path, file));
+}
 // The wasm's decoded size lets the page show download progress whatever the transfer encoding.
 const duckdbBytes = statSync(join(out, 'duckdb-base.wasm')).size;
 writeFileSync(
 	'public/playground/duckdb-version.js',
-	`export const DUCKDB_DIR = './duckdb/${version}/';\nexport const DUCKDB_WASM_BYTES = ${duckdbBytes};\n`,
+	`export const DUCKDB_DIR = './duckdb/${version}/';\nexport const DUCKDB_WASM_BYTES = ${duckdbBytes};\n` +
+		`export const DUCKDB_EXTENSIONS = ${JSON.stringify(Object.keys(pinned.sha256))};\n`,
 );
-console.log(`duckdb-wasm ${version} vendored into ${out}`);
+console.log(`duckdb-wasm ${version} vendored into ${out}, with ${Object.keys(pinned.sha256).join(', ')}`);
 copyFileSync('../sqlquack.toml', 'public/playground/sqlquack.toml');

website/tests/playground.test.mjs

@@ -1,9 +1,10 @@
 // The playground engine and its span handling, under Node's test runner: `npm run test:engine`.
-// Needs the engine built into wasm/pkg (scripts/build_engine.sh).
+// Needs the engine built into wasm/pkg (scripts/build_engine.sh); the script prepares the playground first.
 import assert from 'node:assert/strict';
 import { readFileSync } from 'node:fs';
 import { createRequire } from 'node:module';
 import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
 import { test } from 'node:test';
 
 import { initSync, analyze, fixture_tables, run_failure } from '../wasm/pkg/sqlquack_wasm.js';
@@ -13,6 +14,27 @@ initSync({ module: readFileSync(new URL('../wasm/pkg/sqlquack_wasm_bg.wasm', imp
 
 const SCHEMA = JSON.stringify({ ORDERS: [['ORDER_ID', 'NUMBER(38,0)'], ['NOTE', 'VARCHAR']] });
 
+// A DuckDB as the page starts one: the DuckDB-WASM build it serves, the extensions it loads from the
+// site, then the sample schema's tables and rows. A fresh one per case, so no earlier case has
+// already loaded an extension a later case needs.
+async function pageDuckDB() {
+	const { SCHEMA: PAGE_SCHEMA, ROWS } = await import('../public/playground/examples.js');
+	const { DUCKDB_DIR, DUCKDB_EXTENSIONS } = await import('../public/playground/duckdb-version.js');
+	const require = createRequire(import.meta.url);
+	const duckdb = require('@duckdb/duckdb-wasm/dist/duckdb-node-blocking.cjs');
+	const dist = dirname(require.resolve('@duckdb/duckdb-wasm/dist/duckdb-node-blocking.cjs'));
+	const bundles = { base: { mainModule: join(dist, 'duckdb-base.wasm'), mainWorker: join(dist, 'duckdb-node-base.worker.cjs') } };
+	const db = await duckdb.createDuckDB(bundles, new duckdb.VoidLogger(), duckdb.NODE_RUNTIME);
+	await db.instantiate();
+	const conn = db.connect();
+	conn.query(`SET custom_extension_repository = '${fileURLToPath(new URL(`../public/playground/${DUCKDB_DIR}extensions`, import.meta.url))}'`);
+	for (const extension of DUCKDB_EXTENSIONS) conn.query(`LOAD ${extension}`);
+	for (const table of JSON.parse(fixture_tables(PAGE_SCHEMA)).tables) conn.query(table);
+	conn.query(ROWS);
+	// Every value as text, as the page reads them.
+	return (sql) => conn.query(`SELECT COLUMNS(*)::VARCHAR FROM (${sql}) AS result`).toArray().map((row) => Object.values(row.toJSON()));
+}
+
 const marked = (html) => [...html.matchAll(/<mark[^>]*>(.*?)<\/mark>/g)].map((match) => match[1]);
 
 test('spans after non-ASCII text underline the characters the diagnostic names', () => {
@@ -86,39 +108,52 @@ test('the conversion budget is timed in the browser and refuses with SQ0006 when
 });
 
 test('every example chip stops at the stage its label names, the Run stage included', async () => {
-	const { EXAMPLE_GROUPS, EXAMPLES, CAPTIONS, ROWS, SCHEMA: PAGE_SCHEMA } = await import('../public/playground/examples.js');
+	const { EXAMPLE_GROUPS, EXAMPLES, CAPTIONS, SCHEMA: PAGE_SCHEMA } = await import('../public/playground/examples.js');
 	const chips = EXAMPLE_GROUPS.flatMap((group) => group.examples);
 	assert.deepEqual(chips.map((chip) => chip.key).sort(), Object.keys(EXAMPLES).sort());
 	assert.equal(new Set(chips.map((chip) => chip.slug)).size, chips.length);
-	// Translated examples run as the page runs them: the page's fixture tables and rows, in the
-	// DuckDB-WASM build the page serves, with every value read as text.
-	const require = createRequire(import.meta.url);
-	const duckdb = require('@duckdb/duckdb-wasm/dist/duckdb-node-blocking.cjs');
-	const dist = dirname(require.resolve('@duckdb/duckdb-wasm/dist/duckdb-node-blocking.cjs'));
-	const bundles = { base: { mainModule: join(dist, 'duckdb-base.wasm'), mainWorker: join(dist, 'duckdb-node-base.worker.cjs') } };
-	const db = await duckdb.createDuckDB(bundles, new duckdb.VoidLogger(), duckdb.NODE_RUNTIME);
-	await db.instantiate();
-	const conn = db.connect();
-	for (const table of JSON.parse(fixture_tables(PAGE_SCHEMA)).tables) conn.query(table);
-	conn.query(ROWS);
-	const stops = (sql) => {
+	// Translated examples run as the page runs them, each in a fresh DuckDB.
+	const stops = async (sql) => {
 		const report = JSON.parse(analyze(sql, PAGE_SCHEMA));
 		const failed = ['parse', 'checks', 'translate'].find((stage) => report[stage].status !== 'pass');
 		if (failed) return failed;
+		const run = await pageDuckDB();
 		try {
-			conn.query(`SELECT COLUMNS(*)::VARCHAR FROM (${report.translate.statements[0].sql}) AS result`);
+			run(report.translate.statements[0].sql);
 			return null;
 		} catch {
 			return 'run';
 		}
 	};
-	for (const chip of chips) assert.equal(stops(EXAMPLES[chip.key]), chip.stops, chip.name);
+	for (const chip of chips) assert.equal(await stops(EXAMPLES[chip.key]), chip.stops, chip.name);
 	// The one deliberate refusal is a known divergence, explained by its caption.
 	const refused = JSON.parse(analyze(EXAMPLES.refused, PAGE_SCHEMA)).translate;
 	assert.deepEqual(refused.diagnostics.map((d) => d.code), ['SQ0008']);
 	assert.ok(CAPTIONS.refused);
 });
 
+test('date arithmetic against CURRENT_DATE runs in the in-browser DuckDB, on its first statement', async () => {
+	const { SCHEMA: PAGE_SCHEMA } = await import('../public/playground/examples.js');
+	const cases = [
+		// Last 30 days: the fixture's orders are older, and stay so.
+		['SELECT order_id FROM orders WHERE ordered_at >= DATEADD(day, -30, CURRENT_DATE)', []],
+		// Every sample customer signed up more than 30 days ago.
+		['SELECT customer_id FROM customers WHERE DATEDIFF(day, signup_date, CURRENT_DATE) > 30 ORDER BY customer_id', [['10'], ['11'], ['12']]],
+		// A month later than a month's last day is the next month's last day when it is shorter.
+		[
+			"SELECT DATEADD(month, 1, DATE '2024-01-31') AS leap, DATEADD(month, -1, DATE '2026-03-31') AS feb FROM orders " +
+				'WHERE ordered_at < DATEADD(year, 1, CURRENT_DATE) AND order_id = 1',
+			[['2024-02-29', '2026-02-28']],
+		],
+	];
+	for (const [sql, rows] of cases) {
+		const report = JSON.parse(analyze(sql, PAGE_SCHEMA));
+		assert.equal(report.translate.status, 'pass', sql);
+		const run = await pageDuckDB();
+		assert.deepEqual(run(report.translate.statements[0].sql), rows, sql);
+	}
+});
+
 test('a run error from the in-browser DuckDB is told apart from an error about the rows', () => {
 	// What the preview build of DuckDB-WASM reported for a translated DATEADD that native DuckDB 2.0 runs.
 	assert.ok(previewLimit("Binder Error: No function matches the given name and argument types '+(DATE, INTERVAL)'."));