Skip to content

Commit 0fcc4ce

Browse files
kvlongekvlonge
committed
playground: load ICU and JSON from the site before any SQL runs
The DuckDB-WASM build the playground serves starts without ICU. The first statement that calls an ICU function, such as current_date(), autoloads it while that statement binds, and ICU's new overloads for +, - and date_diff corrupt the binder's lookups: garbled binder errors or "memory access out of bounds" for queries native DuckDB 2.0 runs, such as DATEADD(day, -30, CURRENT_DATE). The page now loads ICU, and json for the sample schema's JSON column, right after connecting, and says so when either fails to load. prepare_playground vendors both extension files for this DuckDB-WASM version, pinned by sha256, under the versioned duckdb directory, so the page never fetches from extensions.duckdb.org. The engine tests run each case in a fresh DuckDB loaded as the page loads it, so an earlier case can no longer load ICU for a later one, and cover last-30-days DATEADD, DATEDIFF against CURRENT_DATE and month-end DATEADD.
1 parent 09d77da commit 0fcc4ce

4 files changed

Lines changed: 105 additions & 22 deletions

File tree

‎website/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
"dev": "node scripts/prepare_playground.mjs && astro dev",
1010
"build": "node scripts/prepare_playground.mjs && astro build",
1111
"preview": "astro preview",
12-
"test:engine": "node --test tests/playground.test.mjs",
12+
"test:engine": "node scripts/prepare_playground.mjs && node --test tests/playground.test.mjs",
1313
"check:figures": "node scripts/check_figures.mjs"
1414
},
1515
"dependencies": {

‎website/public/playground/playground.js‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
// worker) and shows its stages; translated SQL runs in DuckDB-WASM, which downloads in the
33
// background once the page is idle.
44

5-
import { DUCKDB_DIR, DUCKDB_WASM_BYTES } from './duckdb-version.js';
5+
import { DUCKDB_DIR, DUCKDB_EXTENSIONS, DUCKDB_WASM_BYTES } from './duckdb-version.js';
66
import { ENGINE_DIR, ENGINE_WASM_BYTES } from './engine-version.js';
77
import { CAPTIONS, EXAMPLE_GROUPS, EXAMPLES, ROWS, SCHEMA, stopLabel } from './examples.js';
88
import { escape, previewLimit, rendered, resultTable, underlined } from './spans.js';
@@ -328,7 +328,17 @@ async function loadDuckDB() {
328328
const module = await import(moduleUrl);
329329
const db = new module.AsyncDuckDB(new module.VoidLogger(), new Worker(workerUrl));
330330
await db.instantiate(wasmUrl);
331-
const version = await (await db.connect()).query('SELECT version() AS v');
331+
const conn = await db.connect();
332+
// The extensions this build lacks, from the site, before any SQL: DuckDB autoloads ICU while
333+
// binding the first statement that needs it, which corrupts that statement's function lookups.
334+
await conn.query(`SET custom_extension_repository = '${here(`${DUCKDB_DIR}extensions`)}'`);
335+
for (const extension of DUCKDB_EXTENSIONS) {
336+
await conn.query(`LOAD ${extension}`).catch((error) => {
337+
throw new Error(`the ${extension} extension did not load: ${error.message ?? error}`);
338+
});
339+
}
340+
const version = await conn.query('SELECT version() AS v');
341+
await conn.close();
332342
return { db, version: String(version.toArray()[0].v) };
333343
}
334344

‎website/scripts/prepare_playground.mjs‎

Lines changed: 41 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
// Prepares the playground's served files, ignored by Git and rebuilt by `npm run build`: the engine
22
// from wasm/pkg into public/playground/engine/<hash>/, with engine-version.js naming that directory;
33
// DuckDB-WASM from node_modules into public/playground/duckdb/<version>/, so the playground serves
4-
// it from the site rather than a CDN (its browser module bundled with its Apache Arrow import), with
4+
// it from the site rather than a CDN (its browser module bundled with its Apache Arrow import, and
5+
// the extensions the page loads), with
56
// duckdb-version.js naming that directory; and the repository's policy file, which the page offers
67
// to load. The version in the path lets hosts cache DuckDB as immutable (public/_headers).
78
// Run from website/.
@@ -48,11 +49,48 @@ await build({
4849
for (const file of ['duckdb-base.wasm', 'duckdb-browser-base.worker.js']) {
4950
copyFileSync(join(root, 'dist', file), join(out, file));
5051
}
52+
// The extensions the page loads before any SQL runs, served from the site in DuckDB's repository
53+
// layout so the page never fetches from extensions.duckdb.org. ICU must load before the first query:
54+
// this build lacks it, and autoloading it while a statement binds (CURRENT_DATE does) corrupts the
55+
// binder's function lookups. The sample schema's JSON column needs json. Pinned per DuckDB-WASM
56+
// version, since an extension only loads into the DuckDB build it was compiled for.
57+
const EXTENSIONS = {
58+
'1.33.1-dev114.0': {
59+
path: 'v2.0.0/wasm_eh',
60+
sha256: {
61+
icu: '0e379b9d688f0df82192f15f9405f60861f478956ad0282ce831572695458308',
62+
json: '0d4e36ac68c53809c27a4f1e7801142409b2735ad2f7cb7f8b09b8db8d558984',
63+
},
64+
},
65+
};
66+
const pinned = EXTENSIONS[version];
67+
if (!pinned) {
68+
console.error(`prepare_playground: no extensions pinned for duckdb-wasm ${version}; add them to EXTENSIONS`);
69+
process.exit(1);
70+
}
71+
const downloads = 'node_modules/.cache/duckdb-extensions';
72+
mkdirSync(downloads, { recursive: true });
73+
mkdirSync(join(out, 'extensions', pinned.path), { recursive: true });
74+
for (const [name, sha256] of Object.entries(pinned.sha256)) {
75+
const file = `${name}.duckdb_extension.wasm`;
76+
const download = join(downloads, `${sha256}.wasm`);
77+
if (!existsSync(download)) {
78+
const url = `https://extensions.duckdb.org/${pinned.path}/${file}`;
79+
const response = await fetch(url);
80+
if (!response.ok) throw new Error(`${url}: HTTP ${response.status}`);
81+
const bytes = Buffer.from(await response.arrayBuffer());
82+
const actual = createHash('sha256').update(bytes).digest('hex');
83+
if (actual !== sha256) throw new Error(`${url}: sha256 ${actual}, expected ${sha256}`);
84+
writeFileSync(download, bytes);
85+
}
86+
copyFileSync(download, join(out, 'extensions', pinned.path, file));
87+
}
5188
// The wasm's decoded size lets the page show download progress whatever the transfer encoding.
5289
const duckdbBytes = statSync(join(out, 'duckdb-base.wasm')).size;
5390
writeFileSync(
5491
'public/playground/duckdb-version.js',
55-
`export const DUCKDB_DIR = './duckdb/${version}/';\nexport const DUCKDB_WASM_BYTES = ${duckdbBytes};\n`,
92+
`export const DUCKDB_DIR = './duckdb/${version}/';\nexport const DUCKDB_WASM_BYTES = ${duckdbBytes};\n` +
93+
`export const DUCKDB_EXTENSIONS = ${JSON.stringify(Object.keys(pinned.sha256))};\n`,
5694
);
57-
console.log(`duckdb-wasm ${version} vendored into ${out}`);
95+
console.log(`duckdb-wasm ${version} vendored into ${out}, with ${Object.keys(pinned.sha256).join(', ')}`);
5896
copyFileSync('../sqlquack.toml', 'public/playground/sqlquack.toml');

‎website/tests/playground.test.mjs‎

Lines changed: 51 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
// The playground engine and its span handling, under Node's test runner: `npm run test:engine`.
2-
// Needs the engine built into wasm/pkg (scripts/build_engine.sh).
2+
// Needs the engine built into wasm/pkg (scripts/build_engine.sh); the script prepares the playground first.
33
import assert from 'node:assert/strict';
44
import { readFileSync } from 'node:fs';
55
import { createRequire } from 'node:module';
66
import { dirname, join } from 'node:path';
7+
import { fileURLToPath } from 'node:url';
78
import { test } from 'node:test';
89

910
import { initSync, analyze, fixture_tables, run_failure } from '../wasm/pkg/sqlquack_wasm.js';
@@ -13,6 +14,27 @@ initSync({ module: readFileSync(new URL('../wasm/pkg/sqlquack_wasm_bg.wasm', imp
1314

1415
const SCHEMA = JSON.stringify({ ORDERS: [['ORDER_ID', 'NUMBER(38,0)'], ['NOTE', 'VARCHAR']] });
1516

17+
// A DuckDB as the page starts one: the DuckDB-WASM build it serves, the extensions it loads from the
18+
// site, then the sample schema's tables and rows. A fresh one per case, so no earlier case has
19+
// already loaded an extension a later case needs.
20+
async function pageDuckDB() {
21+
const { SCHEMA: PAGE_SCHEMA, ROWS } = await import('../public/playground/examples.js');
22+
const { DUCKDB_DIR, DUCKDB_EXTENSIONS } = await import('../public/playground/duckdb-version.js');
23+
const require = createRequire(import.meta.url);
24+
const duckdb = require('@duckdb/duckdb-wasm/dist/duckdb-node-blocking.cjs');
25+
const dist = dirname(require.resolve('@duckdb/duckdb-wasm/dist/duckdb-node-blocking.cjs'));
26+
const bundles = { base: { mainModule: join(dist, 'duckdb-base.wasm'), mainWorker: join(dist, 'duckdb-node-base.worker.cjs') } };
27+
const db = await duckdb.createDuckDB(bundles, new duckdb.VoidLogger(), duckdb.NODE_RUNTIME);
28+
await db.instantiate();
29+
const conn = db.connect();
30+
conn.query(`SET custom_extension_repository = '${fileURLToPath(new URL(`../public/playground/${DUCKDB_DIR}extensions`, import.meta.url))}'`);
31+
for (const extension of DUCKDB_EXTENSIONS) conn.query(`LOAD ${extension}`);
32+
for (const table of JSON.parse(fixture_tables(PAGE_SCHEMA)).tables) conn.query(table);
33+
conn.query(ROWS);
34+
// Every value as text, as the page reads them.
35+
return (sql) => conn.query(`SELECT COLUMNS(*)::VARCHAR FROM (${sql}) AS result`).toArray().map((row) => Object.values(row.toJSON()));
36+
}
37+
1638
const marked = (html) => [...html.matchAll(/<mark[^>]*>(.*?)<\/mark>/g)].map((match) => match[1]);
1739

1840
test('spans after non-ASCII text underline the characters the diagnostic names', () => {
@@ -86,39 +108,52 @@ test('the conversion budget is timed in the browser and refuses with SQ0006 when
86108
});
87109

88110
test('every example chip stops at the stage its label names, the Run stage included', async () => {
89-
const { EXAMPLE_GROUPS, EXAMPLES, CAPTIONS, ROWS, SCHEMA: PAGE_SCHEMA } = await import('../public/playground/examples.js');
111+
const { EXAMPLE_GROUPS, EXAMPLES, CAPTIONS, SCHEMA: PAGE_SCHEMA } = await import('../public/playground/examples.js');
90112
const chips = EXAMPLE_GROUPS.flatMap((group) => group.examples);
91113
assert.deepEqual(chips.map((chip) => chip.key).sort(), Object.keys(EXAMPLES).sort());
92114
assert.equal(new Set(chips.map((chip) => chip.slug)).size, chips.length);
93-
// Translated examples run as the page runs them: the page's fixture tables and rows, in the
94-
// DuckDB-WASM build the page serves, with every value read as text.
95-
const require = createRequire(import.meta.url);
96-
const duckdb = require('@duckdb/duckdb-wasm/dist/duckdb-node-blocking.cjs');
97-
const dist = dirname(require.resolve('@duckdb/duckdb-wasm/dist/duckdb-node-blocking.cjs'));
98-
const bundles = { base: { mainModule: join(dist, 'duckdb-base.wasm'), mainWorker: join(dist, 'duckdb-node-base.worker.cjs') } };
99-
const db = await duckdb.createDuckDB(bundles, new duckdb.VoidLogger(), duckdb.NODE_RUNTIME);
100-
await db.instantiate();
101-
const conn = db.connect();
102-
for (const table of JSON.parse(fixture_tables(PAGE_SCHEMA)).tables) conn.query(table);
103-
conn.query(ROWS);
104-
const stops = (sql) => {
115+
// Translated examples run as the page runs them, each in a fresh DuckDB.
116+
const stops = async (sql) => {
105117
const report = JSON.parse(analyze(sql, PAGE_SCHEMA));
106118
const failed = ['parse', 'checks', 'translate'].find((stage) => report[stage].status !== 'pass');
107119
if (failed) return failed;
120+
const run = await pageDuckDB();
108121
try {
109-
conn.query(`SELECT COLUMNS(*)::VARCHAR FROM (${report.translate.statements[0].sql}) AS result`);
122+
run(report.translate.statements[0].sql);
110123
return null;
111124
} catch {
112125
return 'run';
113126
}
114127
};
115-
for (const chip of chips) assert.equal(stops(EXAMPLES[chip.key]), chip.stops, chip.name);
128+
for (const chip of chips) assert.equal(await stops(EXAMPLES[chip.key]), chip.stops, chip.name);
116129
// The one deliberate refusal is a known divergence, explained by its caption.
117130
const refused = JSON.parse(analyze(EXAMPLES.refused, PAGE_SCHEMA)).translate;
118131
assert.deepEqual(refused.diagnostics.map((d) => d.code), ['SQ0008']);
119132
assert.ok(CAPTIONS.refused);
120133
});
121134

135+
test('date arithmetic against CURRENT_DATE runs in the in-browser DuckDB, on its first statement', async () => {
136+
const { SCHEMA: PAGE_SCHEMA } = await import('../public/playground/examples.js');
137+
const cases = [
138+
// Last 30 days: the fixture's orders are older, and stay so.
139+
['SELECT order_id FROM orders WHERE ordered_at >= DATEADD(day, -30, CURRENT_DATE)', []],
140+
// Every sample customer signed up more than 30 days ago.
141+
['SELECT customer_id FROM customers WHERE DATEDIFF(day, signup_date, CURRENT_DATE) > 30 ORDER BY customer_id', [['10'], ['11'], ['12']]],
142+
// A month later than a month's last day is the next month's last day when it is shorter.
143+
[
144+
"SELECT DATEADD(month, 1, DATE '2024-01-31') AS leap, DATEADD(month, -1, DATE '2026-03-31') AS feb FROM orders " +
145+
'WHERE ordered_at < DATEADD(year, 1, CURRENT_DATE) AND order_id = 1',
146+
[['2024-02-29', '2026-02-28']],
147+
],
148+
];
149+
for (const [sql, rows] of cases) {
150+
const report = JSON.parse(analyze(sql, PAGE_SCHEMA));
151+
assert.equal(report.translate.status, 'pass', sql);
152+
const run = await pageDuckDB();
153+
assert.deepEqual(run(report.translate.statements[0].sql), rows, sql);
154+
}
155+
});
156+
122157
test('a run error from the in-browser DuckDB is told apart from an error about the rows', () => {
123158
// What the preview build of DuckDB-WASM reported for a translated DATEADD that native DuckDB 2.0 runs.
124159
assert.ok(previewLimit("Binder Error: No function matches the given name and argument types '+(DATE, INTERVAL)'."));

0 commit comments

Comments
 (0)