From 0fcc4ce31ad6b60efdc07022a42a930accf20725 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 7 Nov 2026 11:17:14 +0000 Subject: [PATCH] playground: load ICU and JSON from the site before any SQL runs The DuckDB-WASM build the playground serves starts without ICU. The first statement that calls an ICU function, such as current_date(), autoloads it while that statement binds, and ICU's new overloads for +, - and date_diff corrupt the binder's lookups: garbled binder errors or "memory access out of bounds" for queries native DuckDB 2.0 runs, such as DATEADD(day, -30, CURRENT_DATE). The page now loads ICU, and json for the sample schema's JSON column, right after connecting, and says so when either fails to load. prepare_playground vendors both extension files for this DuckDB-WASM version, pinned by sha256, under the versioned duckdb directory, so the page never fetches from extensions.duckdb.org. The engine tests run each case in a fresh DuckDB loaded as the page loads it, so an earlier case can no longer load ICU for a later one, and cover last-30-days DATEADD, DATEDIFF against CURRENT_DATE and month-end DATEADD. --- website/package.json | 2 +- website/public/playground/playground.js | 14 +++++- website/scripts/prepare_playground.mjs | 44 ++++++++++++++-- website/tests/playground.test.mjs | 67 +++++++++++++++++++------ 4 files changed, 105 insertions(+), 22 deletions(-) diff --git a/website/package.json b/website/package.json index b5e977309..2ee5be474 100644 --- a/website/package.json +++ b/website/package.json @@ -9,7 +9,7 @@ "dev": "node scripts/prepare_playground.mjs && astro dev", "build": "node scripts/prepare_playground.mjs && astro build", "preview": "astro preview", - "test:engine": "node --test tests/playground.test.mjs", + "test:engine": "node scripts/prepare_playground.mjs && node --test tests/playground.test.mjs", "check:figures": "node scripts/check_figures.mjs" }, "dependencies": { diff --git a/website/public/playground/playground.js b/website/public/playground/playground.js index a6b9923e2..3f1029a56 100644 --- a/website/public/playground/playground.js +++ b/website/public/playground/playground.js @@ -2,7 +2,7 @@ // worker) and shows its stages; translated SQL runs in DuckDB-WASM, which downloads in the // background once the page is idle. -import { DUCKDB_DIR, DUCKDB_WASM_BYTES } from './duckdb-version.js'; +import { DUCKDB_DIR, DUCKDB_EXTENSIONS, DUCKDB_WASM_BYTES } from './duckdb-version.js'; import { ENGINE_DIR, ENGINE_WASM_BYTES } from './engine-version.js'; import { CAPTIONS, EXAMPLE_GROUPS, EXAMPLES, ROWS, SCHEMA, stopLabel } from './examples.js'; import { escape, previewLimit, rendered, resultTable, underlined } from './spans.js'; @@ -328,7 +328,17 @@ async function loadDuckDB() { const module = await import(moduleUrl); const db = new module.AsyncDuckDB(new module.VoidLogger(), new Worker(workerUrl)); await db.instantiate(wasmUrl); - const version = await (await db.connect()).query('SELECT version() AS v'); + const conn = await db.connect(); + // The extensions this build lacks, from the site, before any SQL: DuckDB autoloads ICU while + // binding the first statement that needs it, which corrupts that statement's function lookups. + await conn.query(`SET custom_extension_repository = '${here(`${DUCKDB_DIR}extensions`)}'`); + for (const extension of DUCKDB_EXTENSIONS) { + await conn.query(`LOAD ${extension}`).catch((error) => { + throw new Error(`the ${extension} extension did not load: ${error.message ?? error}`); + }); + } + const version = await conn.query('SELECT version() AS v'); + await conn.close(); return { db, version: String(version.toArray()[0].v) }; } diff --git a/website/scripts/prepare_playground.mjs b/website/scripts/prepare_playground.mjs index da006355b..b02e3e633 100644 --- a/website/scripts/prepare_playground.mjs +++ b/website/scripts/prepare_playground.mjs @@ -1,7 +1,8 @@ // Prepares the playground's served files, ignored by Git and rebuilt by `npm run build`: the engine // from wasm/pkg into public/playground/engine//, with engine-version.js naming that directory; // DuckDB-WASM from node_modules into public/playground/duckdb//, so the playground serves -// it from the site rather than a CDN (its browser module bundled with its Apache Arrow import), with +// it from the site rather than a CDN (its browser module bundled with its Apache Arrow import, and +// the extensions the page loads), with // duckdb-version.js naming that directory; and the repository's policy file, which the page offers // to load. The version in the path lets hosts cache DuckDB as immutable (public/_headers). // Run from website/. @@ -48,11 +49,48 @@ await build({ for (const file of ['duckdb-base.wasm', 'duckdb-browser-base.worker.js']) { copyFileSync(join(root, 'dist', file), join(out, file)); } +// The extensions the page loads before any SQL runs, served from the site in DuckDB's repository +// layout so the page never fetches from extensions.duckdb.org. ICU must load before the first query: +// this build lacks it, and autoloading it while a statement binds (CURRENT_DATE does) corrupts the +// binder's function lookups. The sample schema's JSON column needs json. Pinned per DuckDB-WASM +// version, since an extension only loads into the DuckDB build it was compiled for. +const EXTENSIONS = { + '1.33.1-dev114.0': { + path: 'v2.0.0/wasm_eh', + sha256: { + icu: '0e379b9d688f0df82192f15f9405f60861f478956ad0282ce831572695458308', + json: '0d4e36ac68c53809c27a4f1e7801142409b2735ad2f7cb7f8b09b8db8d558984', + }, + }, +}; +const pinned = EXTENSIONS[version]; +if (!pinned) { + console.error(`prepare_playground: no extensions pinned for duckdb-wasm ${version}; add them to EXTENSIONS`); + process.exit(1); +} +const downloads = 'node_modules/.cache/duckdb-extensions'; +mkdirSync(downloads, { recursive: true }); +mkdirSync(join(out, 'extensions', pinned.path), { recursive: true }); +for (const [name, sha256] of Object.entries(pinned.sha256)) { + const file = `${name}.duckdb_extension.wasm`; + const download = join(downloads, `${sha256}.wasm`); + if (!existsSync(download)) { + const url = `https://extensions.duckdb.org/${pinned.path}/${file}`; + const response = await fetch(url); + if (!response.ok) throw new Error(`${url}: HTTP ${response.status}`); + const bytes = Buffer.from(await response.arrayBuffer()); + const actual = createHash('sha256').update(bytes).digest('hex'); + if (actual !== sha256) throw new Error(`${url}: sha256 ${actual}, expected ${sha256}`); + writeFileSync(download, bytes); + } + copyFileSync(download, join(out, 'extensions', pinned.path, file)); +} // The wasm's decoded size lets the page show download progress whatever the transfer encoding. const duckdbBytes = statSync(join(out, 'duckdb-base.wasm')).size; writeFileSync( 'public/playground/duckdb-version.js', - `export const DUCKDB_DIR = './duckdb/${version}/';\nexport const DUCKDB_WASM_BYTES = ${duckdbBytes};\n`, + `export const DUCKDB_DIR = './duckdb/${version}/';\nexport const DUCKDB_WASM_BYTES = ${duckdbBytes};\n` + + `export const DUCKDB_EXTENSIONS = ${JSON.stringify(Object.keys(pinned.sha256))};\n`, ); -console.log(`duckdb-wasm ${version} vendored into ${out}`); +console.log(`duckdb-wasm ${version} vendored into ${out}, with ${Object.keys(pinned.sha256).join(', ')}`); copyFileSync('../sqlquack.toml', 'public/playground/sqlquack.toml'); diff --git a/website/tests/playground.test.mjs b/website/tests/playground.test.mjs index 27b0b0a61..715138772 100644 --- a/website/tests/playground.test.mjs +++ b/website/tests/playground.test.mjs @@ -1,9 +1,10 @@ // The playground engine and its span handling, under Node's test runner: `npm run test:engine`. -// Needs the engine built into wasm/pkg (scripts/build_engine.sh). +// Needs the engine built into wasm/pkg (scripts/build_engine.sh); the script prepares the playground first. import assert from 'node:assert/strict'; import { readFileSync } from 'node:fs'; import { createRequire } from 'node:module'; import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; import { test } from 'node:test'; import { initSync, analyze, fixture_tables, run_failure } from '../wasm/pkg/sqlquack_wasm.js'; @@ -13,6 +14,27 @@ initSync({ module: readFileSync(new URL('../wasm/pkg/sqlquack_wasm_bg.wasm', imp const SCHEMA = JSON.stringify({ ORDERS: [['ORDER_ID', 'NUMBER(38,0)'], ['NOTE', 'VARCHAR']] }); +// A DuckDB as the page starts one: the DuckDB-WASM build it serves, the extensions it loads from the +// site, then the sample schema's tables and rows. A fresh one per case, so no earlier case has +// already loaded an extension a later case needs. +async function pageDuckDB() { + const { SCHEMA: PAGE_SCHEMA, ROWS } = await import('../public/playground/examples.js'); + const { DUCKDB_DIR, DUCKDB_EXTENSIONS } = await import('../public/playground/duckdb-version.js'); + const require = createRequire(import.meta.url); + const duckdb = require('@duckdb/duckdb-wasm/dist/duckdb-node-blocking.cjs'); + const dist = dirname(require.resolve('@duckdb/duckdb-wasm/dist/duckdb-node-blocking.cjs')); + const bundles = { base: { mainModule: join(dist, 'duckdb-base.wasm'), mainWorker: join(dist, 'duckdb-node-base.worker.cjs') } }; + const db = await duckdb.createDuckDB(bundles, new duckdb.VoidLogger(), duckdb.NODE_RUNTIME); + await db.instantiate(); + const conn = db.connect(); + conn.query(`SET custom_extension_repository = '${fileURLToPath(new URL(`../public/playground/${DUCKDB_DIR}extensions`, import.meta.url))}'`); + for (const extension of DUCKDB_EXTENSIONS) conn.query(`LOAD ${extension}`); + for (const table of JSON.parse(fixture_tables(PAGE_SCHEMA)).tables) conn.query(table); + conn.query(ROWS); + // Every value as text, as the page reads them. + return (sql) => conn.query(`SELECT COLUMNS(*)::VARCHAR FROM (${sql}) AS result`).toArray().map((row) => Object.values(row.toJSON())); +} + const marked = (html) => [...html.matchAll(/]*>(.*?)<\/mark>/g)].map((match) => match[1]); test('spans after non-ASCII text underline the characters the diagnostic names', () => { @@ -86,39 +108,52 @@ test('the conversion budget is timed in the browser and refuses with SQ0006 when }); test('every example chip stops at the stage its label names, the Run stage included', async () => { - const { EXAMPLE_GROUPS, EXAMPLES, CAPTIONS, ROWS, SCHEMA: PAGE_SCHEMA } = await import('../public/playground/examples.js'); + const { EXAMPLE_GROUPS, EXAMPLES, CAPTIONS, SCHEMA: PAGE_SCHEMA } = await import('../public/playground/examples.js'); const chips = EXAMPLE_GROUPS.flatMap((group) => group.examples); assert.deepEqual(chips.map((chip) => chip.key).sort(), Object.keys(EXAMPLES).sort()); assert.equal(new Set(chips.map((chip) => chip.slug)).size, chips.length); - // Translated examples run as the page runs them: the page's fixture tables and rows, in the - // DuckDB-WASM build the page serves, with every value read as text. - const require = createRequire(import.meta.url); - const duckdb = require('@duckdb/duckdb-wasm/dist/duckdb-node-blocking.cjs'); - const dist = dirname(require.resolve('@duckdb/duckdb-wasm/dist/duckdb-node-blocking.cjs')); - const bundles = { base: { mainModule: join(dist, 'duckdb-base.wasm'), mainWorker: join(dist, 'duckdb-node-base.worker.cjs') } }; - const db = await duckdb.createDuckDB(bundles, new duckdb.VoidLogger(), duckdb.NODE_RUNTIME); - await db.instantiate(); - const conn = db.connect(); - for (const table of JSON.parse(fixture_tables(PAGE_SCHEMA)).tables) conn.query(table); - conn.query(ROWS); - const stops = (sql) => { + // Translated examples run as the page runs them, each in a fresh DuckDB. + const stops = async (sql) => { const report = JSON.parse(analyze(sql, PAGE_SCHEMA)); const failed = ['parse', 'checks', 'translate'].find((stage) => report[stage].status !== 'pass'); if (failed) return failed; + const run = await pageDuckDB(); try { - conn.query(`SELECT COLUMNS(*)::VARCHAR FROM (${report.translate.statements[0].sql}) AS result`); + run(report.translate.statements[0].sql); return null; } catch { return 'run'; } }; - for (const chip of chips) assert.equal(stops(EXAMPLES[chip.key]), chip.stops, chip.name); + for (const chip of chips) assert.equal(await stops(EXAMPLES[chip.key]), chip.stops, chip.name); // The one deliberate refusal is a known divergence, explained by its caption. const refused = JSON.parse(analyze(EXAMPLES.refused, PAGE_SCHEMA)).translate; assert.deepEqual(refused.diagnostics.map((d) => d.code), ['SQ0008']); assert.ok(CAPTIONS.refused); }); +test('date arithmetic against CURRENT_DATE runs in the in-browser DuckDB, on its first statement', async () => { + const { SCHEMA: PAGE_SCHEMA } = await import('../public/playground/examples.js'); + const cases = [ + // Last 30 days: the fixture's orders are older, and stay so. + ['SELECT order_id FROM orders WHERE ordered_at >= DATEADD(day, -30, CURRENT_DATE)', []], + // Every sample customer signed up more than 30 days ago. + ['SELECT customer_id FROM customers WHERE DATEDIFF(day, signup_date, CURRENT_DATE) > 30 ORDER BY customer_id', [['10'], ['11'], ['12']]], + // A month later than a month's last day is the next month's last day when it is shorter. + [ + "SELECT DATEADD(month, 1, DATE '2024-01-31') AS leap, DATEADD(month, -1, DATE '2026-03-31') AS feb FROM orders " + + 'WHERE ordered_at < DATEADD(year, 1, CURRENT_DATE) AND order_id = 1', + [['2024-02-29', '2026-02-28']], + ], + ]; + for (const [sql, rows] of cases) { + const report = JSON.parse(analyze(sql, PAGE_SCHEMA)); + assert.equal(report.translate.status, 'pass', sql); + const run = await pageDuckDB(); + assert.deepEqual(run(report.translate.statements[0].sql), rows, sql); + } +}); + test('a run error from the in-browser DuckDB is told apart from an error about the rows', () => { // What the preview build of DuckDB-WASM reported for a translated DATEADD that native DuckDB 2.0 runs. assert.ok(previewLimit("Binder Error: No function matches the given name and argument types '+(DATE, INTERVAL)'."));