diff --git a/cli/package.json b/cli/package.json index d5d53ff..8848aae 100644 --- a/cli/package.json +++ b/cli/package.json @@ -1,6 +1,6 @@ { "name": "@codeplans/cli", - "version": "0.7.7", + "version": "0.7.8", "description": "Command-line tool for CodePlans: script workspace setups, work with code plans, tasks and work items from your terminal.", "license": "MIT", "type": "module", diff --git a/lib/db/index.ts b/lib/db/index.ts index 2dc590a..ceb8273 100644 --- a/lib/db/index.ts +++ b/lib/db/index.ts @@ -4,13 +4,18 @@ import { resolveDbSsl, sqliteFilePath } from '@/lib/runtime-env' import { getEnterpriseHooks } from '@/lib/ee/registry' import type { PostgresJsDatabase } from 'drizzle-orm/postgres-js' import type * as PgSchema from './schema.pg' +import { queueWrites, WriteQueue } from './sqlite-writes' // TypeScript always sees db as the postgres type for full IDE / type-check support. // In SQLite mode the runtime object is a libsql Drizzle instance but the column names // and table names are identical, so all queries work correctly. type Db = PostgresJsDatabase -function createDb(rawUrl: string): Db { +// `writes` is set for a local SQLite file, where writes must run one at a time +// (see lib/db/sqlite-writes.ts). +type Handle = { instance: Db; writes: WriteQueue | null } + +function createDb(rawUrl: string): Handle { if (config.db.provider === 'sqlite') { const { createClient } = require('@libsql/client') as typeof import('@libsql/client') const { drizzle } = require('drizzle-orm/libsql') as typeof import('drizzle-orm/libsql') @@ -28,18 +33,23 @@ function createDb(rawUrl: string): Db { mkdirSync(dirname(file), { recursive: true }) } // DATABASE_AUTH_TOKEN is only needed for a hosted libsql server (Turso). - const client = createClient({ url, authToken: process.env.DATABASE_AUTH_TOKEN || undefined }) - // WAL allows concurrent readers alongside a writer; busy_timeout retries - // instead of immediately throwing SQLITE_BUSY when a write lock is held. + // `timeout` is the busy timeout (ms) for every connection in libsql's + // pool: a write retries instead of failing with SQLITE_BUSY while another + // process holds the lock. + let client = createClient({ url, authToken: process.env.DATABASE_AUTH_TOKEN || undefined, timeout: 5000 }) + // WAL allows concurrent readers alongside a writer. It's stored in the + // database file, so setting it on one connection covers them all. client.execute('PRAGMA journal_mode=WAL') - client.execute('PRAGMA busy_timeout=5000') - return drizzle(client, { schema }) as unknown as PostgresJsDatabase + // A hosted libsql server (Turso) arbitrates writes itself. + const writes = url === ':memory:' || /^file:/i.test(url) ? new WriteQueue() : null + if (writes) client = queueWrites(client, writes) + return { instance: drizzle(client, { schema }) as unknown as PostgresJsDatabase, writes } } else { const postgres = require('postgres') as typeof import('postgres') const { drizzle } = require('drizzle-orm/postgres-js') as typeof import('drizzle-orm/postgres-js') const schema = require('./schema.pg') as typeof PgSchema const client = postgres(rawUrl, { ssl: resolveDbSsl(rawUrl, process.env.DB_SSL) ? 'require' : false, prepare: false }) - return drizzle(client, { schema }) + return { instance: drizzle(client, { schema }), writes: null } } } @@ -47,15 +57,15 @@ function createDb(rawUrl: string): Db { // them before any SQL runs. const WRITE_METHODS = new Set(['insert', 'update', 'delete', 'transaction', 'run', 'execute', 'batch']) -const opened = new Map() +const opened = new Map() -function openDb(url: string): Db { - let instance = opened.get(url) - if (!instance) { - instance = createDb(url) - opened.set(url, instance) +function openDb(url: string): Handle { + let handle = opened.get(url) + if (!handle) { + handle = createDb(url) + opened.set(url, handle) } - return instance + return handle } /** @@ -64,10 +74,10 @@ function openDb(url: string): Db { * through the `workspaceDatabase` hook (lib/ee/types.ts); it is opened once and * reused. */ -function currentDb(): { instance: Db; readOnly: string | null } { +function currentDb(): Handle & { readOnly: string | null } { const workspace = getEnterpriseHooks().workspaceDatabase() - if (!workspace) return { instance: openDb(config.db.url), readOnly: null } - return { instance: openDb(workspace.url), readOnly: workspace.readOnly ?? null } + if (!workspace) return { ...openDb(config.db.url), readOnly: null } + return { ...openDb(workspace.url), readOnly: workspace.readOnly ?? null } } /** Thrown when a read-only workspace tries to change data. */ @@ -83,10 +93,13 @@ export class ReadOnlyDatabaseError extends Error { // database, opened on first use, so behavior is the same as a plain instance. export const db: Db = new Proxy({} as Db, { get(_target, prop) { - const { instance, readOnly } = currentDb() + const { instance, writes, readOnly } = currentDb() if (readOnly && WRITE_METHODS.has(prop)) { return () => { throw new ReadOnlyDatabaseError(readOnly) } } + if (writes && prop === 'transaction') { + return (...args: Parameters) => writes.transaction(() => instance.transaction(...args)) + } const value = Reflect.get(instance, prop, instance) return typeof value === 'function' ? value.bind(instance) : value }, diff --git a/lib/db/sqlite-writes.ts b/lib/db/sqlite-writes.ts new file mode 100644 index 0000000..ef98b54 --- /dev/null +++ b/lib/db/sqlite-writes.ts @@ -0,0 +1,86 @@ +import { AsyncLocalStorage } from 'node:async_hooks' +import type { Client, InStatement } from '@libsql/client' + +/** Thrown when code inside db.transaction() writes through `db` instead of `tx`. */ +export class NestedWriteError extends Error { + constructor() { + super('Write through db inside db.transaction(); use the transaction (tx) passed to the callback') + this.name = 'NestedWriteError' + } +} + +/** + * Lets one write at a time run against a local SQLite file. + * + * libsql's local driver runs each statement synchronously, but a transaction + * spans awaits. While one is open it holds the file's write lock on its own + * pooled connection, so a write that arrives meanwhile on another connection + * busy-waits synchronously, blocking the event loop the transaction needs in + * order to finish, and fails with SQLITE_BUSY when the timeout runs out. + * Queueing writes behind open transactions avoids that. Reads don't queue: + * WAL lets them run beside a writer. + */ +export class WriteQueue { + private tail: Promise = Promise.resolve() + private readonly scope = new AsyncLocalStorage<{ open: boolean }>() + + /** Runs fn once every write queued before it has finished. */ + async run(fn: () => Promise | T): Promise { + const previous = this.tail + let release!: () => void + this.tail = new Promise((resolve) => { release = resolve }) + await previous + try { + return await fn() + } finally { + release() + } + } + + /** Runs a whole transaction as one queued write. */ + transaction(fn: () => Promise): Promise { + if (this.inTransaction()) return Promise.reject(new NestedWriteError()) + return this.run(() => { + const state = { open: true } + return this.scope.run(state, async () => { + try { + return await fn() + } finally { + // Work started inside the callback that outlives it queues normally. + state.open = false + } + }) + }) + } + + inTransaction(): boolean { + return this.scope.getStore()?.open === true + } +} + +const isRead = (stmt: InStatement) => /^\s*select\b/i.test(typeof stmt === 'string' ? stmt : stmt.sql) + +/** + * Sends a local libsql client's writes through `queue`. db.transaction() must + * go through queue.transaction() (see lib/db/index.ts); the client's own + * transaction() then runs inside that queued slot. + */ +export function queueWrites(client: Client, queue: WriteQueue): Client { + const execute = client.execute.bind(client) as (...args: unknown[]) => ReturnType + const batch = client.batch.bind(client) + const migrate = client.migrate.bind(client) + const executeMultiple = client.executeMultiple.bind(client) + + const write = (fn: () => Promise): Promise => + queue.inTransaction() ? Promise.reject(new NestedWriteError()) : queue.run(fn) + + Object.assign(client, { + execute: (...args: unknown[]) => + isRead(args[0] as InStatement) ? execute(...args) : write(() => execute(...args)), + batch: (stmts: Parameters[0], mode?: Parameters[1]) => + mode === 'read' ? batch(stmts, mode) : write(() => batch(stmts, mode)), + migrate: (stmts: Parameters[0]) => write(() => migrate(stmts)), + executeMultiple: (sql: string) => write(() => executeMultiple(sql)), + }) + return client +} diff --git a/package.json b/package.json index bede5cc..9347be3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "codeplans", - "version": "0.7.7", + "version": "0.7.8", "description": "Manage and track coordinated changes across your software architecture.", "author": "Sai Prakash ", "homepage": "https://codeplans.ai", @@ -27,7 +27,7 @@ "dependencies": { "@anthropic-ai/sdk": "0.111.0", "@hookform/resolvers": "^3.9.1", - "@libsql/client": "^0.17.3", + "@libsql/client": "^0.18.0", "@radix-ui/react-accordion": "1.2.12", "@radix-ui/react-alert-dialog": "1.1.15", "@radix-ui/react-aspect-ratio": "1.1.8", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index be88f2a..b3cf08c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -54,8 +54,8 @@ importers: specifier: ^3.9.1 version: 3.10.0(react-hook-form@7.71.1(react@19.2.4)) '@libsql/client': - specifier: ^0.17.3 - version: 0.17.3 + specifier: ^0.18.0 + version: 0.18.0 '@radix-ui/react-accordion': specifier: 1.2.12 version: 1.2.12(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) @@ -181,7 +181,7 @@ importers: version: 4.1.0 drizzle-orm: specifier: ^0.45.2 - version: 0.45.2(@libsql/client@0.17.3)(postgres@3.4.8) + version: 0.45.2(@libsql/client@0.18.0)(postgres@3.4.8) embla-carousel-react: specifier: 8.6.0 version: 8.6.0(react@19.2.4) @@ -1034,105 +1034,89 @@ packages: resolution: {integrity: sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==} cpu: [arm64] os: [linux] - libc: [glibc] '@img/sharp-libvips-linux-arm@1.2.4': resolution: {integrity: sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==} cpu: [arm] os: [linux] - libc: [glibc] '@img/sharp-libvips-linux-ppc64@1.2.4': resolution: {integrity: sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==} cpu: [ppc64] os: [linux] - libc: [glibc] '@img/sharp-libvips-linux-riscv64@1.2.4': resolution: {integrity: sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==} cpu: [riscv64] os: [linux] - libc: [glibc] '@img/sharp-libvips-linux-s390x@1.2.4': resolution: {integrity: sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==} cpu: [s390x] os: [linux] - libc: [glibc] '@img/sharp-libvips-linux-x64@1.2.4': resolution: {integrity: sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==} cpu: [x64] os: [linux] - libc: [glibc] '@img/sharp-libvips-linuxmusl-arm64@1.2.4': resolution: {integrity: sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==} cpu: [arm64] os: [linux] - libc: [musl] '@img/sharp-libvips-linuxmusl-x64@1.2.4': resolution: {integrity: sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==} cpu: [x64] os: [linux] - libc: [musl] '@img/sharp-linux-arm64@0.34.5': resolution: {integrity: sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm64] os: [linux] - libc: [glibc] '@img/sharp-linux-arm@0.34.5': resolution: {integrity: sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm] os: [linux] - libc: [glibc] '@img/sharp-linux-ppc64@0.34.5': resolution: {integrity: sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [ppc64] os: [linux] - libc: [glibc] '@img/sharp-linux-riscv64@0.34.5': resolution: {integrity: sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [riscv64] os: [linux] - libc: [glibc] '@img/sharp-linux-s390x@0.34.5': resolution: {integrity: sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [s390x] os: [linux] - libc: [glibc] '@img/sharp-linux-x64@0.34.5': resolution: {integrity: sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [x64] os: [linux] - libc: [glibc] '@img/sharp-linuxmusl-arm64@0.34.5': resolution: {integrity: sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm64] os: [linux] - libc: [musl] '@img/sharp-linuxmusl-x64@0.34.5': resolution: {integrity: sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [x64] os: [linux] - libc: [musl] '@img/sharp-wasm32@0.34.5': resolution: {integrity: sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==} @@ -1173,11 +1157,11 @@ packages: '@jridgewell/trace-mapping@0.3.31': resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} - '@libsql/client@0.17.3': - resolution: {integrity: sha512-HXk9wiAoJbKFbyBH4O+aEhN6ir5ERXuXvwE5OD2eR4/5RUa3Pw/8L9zrnVdU+iNJitRvisPWaIwmhkO3bH7giA==} + '@libsql/client@0.18.0': + resolution: {integrity: sha512-zMCCo58vv7w27j7+cnt/GW+X6/Rih6cHZ86PKi0AxCMOqMfrVgG9OZ5lW2bDMJhSeOjsv2szH2nJF3A9l36UmA==} - '@libsql/core@0.17.3': - resolution: {integrity: sha512-2UjK1i7JBkMduJo4WdvvBxMMvVJ31pArBZNONyz/GCJJAH+1UHat2X6vn10S/WpY5fKzIT98WqYFl2vzWRLOfg==} + '@libsql/core@0.18.0': + resolution: {integrity: sha512-N8RR2CIpcgtbKZnXs2KVpw2lJQfinUj3I56e7qRDRpK18Hty0Rr8nr00VmFb4R/DsqOanPB/TaP3+QAQLZeXMg==} '@libsql/darwin-arm64@0.5.29': resolution: {integrity: sha512-K+2RIB1OGFPYQbfay48GakLhqf3ArcbHqPFu7EZiaUcRgFcdw8RoltsMyvbj5ix2fY0HV3Q3Ioa/ByvQdaSM0A==} @@ -1276,28 +1260,24 @@ packages: engines: {node: '>= 10'} cpu: [arm64] os: [linux] - libc: [glibc] '@next/swc-linux-arm64-musl@16.2.0': resolution: {integrity: sha512-1ffhC6KY5qWLg5miMlKJp3dZbXelEfjuXt1qcp5WzSCQy36CV3y+JT7OC1WSFKizGQCDOcQbfkH/IjZP3cdRNA==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] - libc: [musl] '@next/swc-linux-x64-gnu@16.2.0': resolution: {integrity: sha512-FmbDcZQ8yJRq93EJSL6xaE0KK/Rslraf8fj1uViGxg7K4CKBCRYSubILJPEhjSgZurpcPQq12QNOJQ0DRJl6Hg==} engines: {node: '>= 10'} cpu: [x64] os: [linux] - libc: [glibc] '@next/swc-linux-x64-musl@16.2.0': resolution: {integrity: sha512-HzjIHVkmGAwRbh/vzvoBWWEbb8BBZPxBvVbDQDvzHSf3D8RP/4vjw7MNLDXFF9Q1WEzeQyEj2zdxBtVAHu5Oyw==} engines: {node: '>= 10'} cpu: [x64] os: [linux] - libc: [musl] '@next/swc-win32-arm64-msvc@16.2.0': resolution: {integrity: sha512-UMiFNQf5H7+1ZsZPxEsA064WEuFbRNq/kEXyepbCnSErp4f5iut75dBA8UeerFIG3vDaQNOfCpevnERPp2V+nA==} @@ -2053,42 +2033,36 @@ packages: engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [linux] - libc: [glibc] '@rolldown/binding-linux-arm64-musl@1.0.3': resolution: {integrity: sha512-VWkUHwWriDciit80wleYwKILoR/KMvxh/IdwS/paX+ZgpuRpCrKLUdadJbc0NpBEiyhpYawsJ73j9aCvOH+f7Q==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [linux] - libc: [musl] '@rolldown/binding-linux-ppc64-gnu@1.0.3': resolution: {integrity: sha512-5f1laC0SlIR0yDbFCd8acUhvJIag6N3zC5P7oUPN6wX0aOma+uKJ0wBDH5aq7I1PVI2ttTlhJwzwRIBnLiSGEg==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [ppc64] os: [linux] - libc: [glibc] '@rolldown/binding-linux-s390x-gnu@1.0.3': resolution: {integrity: sha512-Iq4ko0r4XsgbrF/LunNgHtAGLRRVE2kXonAXQ/MV0mC6jQpMOhW1SvtZja2EhC/kd05++bP78dsqBeIQyYJ6Yg==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [s390x] os: [linux] - libc: [glibc] '@rolldown/binding-linux-x64-gnu@1.0.3': resolution: {integrity: sha512-B8m6tD5+/N5FeNQFbKlLA/2yVq9ycQP1SeedyEYYKWBNR3ZQbkvIUcNnDNM03lO1l5F2roiiFJGgvoLLyZXtSg==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [linux] - libc: [glibc] '@rolldown/binding-linux-x64-musl@1.0.3': resolution: {integrity: sha512-pSdpdUJHkuCxun9LE7jvgUB9qsRgaiyNNCX7m/AvHTcq67AiT/Yhoxvw5zPfhrM8k/BfP8ce/hMOpthKDpEUow==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [linux] - libc: [musl] '@rolldown/binding-openharmony-arm64@1.0.3': resolution: {integrity: sha512-OXXS3RKJgX2uLwM+gYyuH5omcH8fL1LJs96pZGgtetVCahON57+d4SJHzTgZiOjxgGkSnpXpOsWuPDGAKAigEg==} @@ -2166,28 +2140,24 @@ packages: engines: {node: '>= 20'} cpu: [arm64] os: [linux] - libc: [glibc] '@tailwindcss/oxide-linux-arm64-musl@4.2.0': resolution: {integrity: sha512-XKcSStleEVnbH6W/9DHzZv1YhjE4eSS6zOu2eRtYAIh7aV4o3vIBs+t/B15xlqoxt6ef/0uiqJVB6hkHjWD/0A==} engines: {node: '>= 20'} cpu: [arm64] os: [linux] - libc: [musl] '@tailwindcss/oxide-linux-x64-gnu@4.2.0': resolution: {integrity: sha512-/hlXCBqn9K6fi7eAM0RsobHwJYa5V/xzWspVTzxnX+Ft9v6n+30Pz8+RxCn7sQL/vRHHLS30iQPrHQunu6/vJA==} engines: {node: '>= 20'} cpu: [x64] os: [linux] - libc: [glibc] '@tailwindcss/oxide-linux-x64-musl@4.2.0': resolution: {integrity: sha512-lKUaygq4G7sWkhQbfdRRBkaq4LY39IriqBQ+Gk6l5nKq6Ay2M2ZZb1tlIyRNgZKS8cbErTwuYSor0IIULC0SHw==} engines: {node: '>= 20'} cpu: [x64] os: [linux] - libc: [musl] '@tailwindcss/oxide-wasm32-wasi@4.2.0': resolution: {integrity: sha512-xuDjhAsFdUuFP5W9Ze4k/o4AskUtI8bcAGU4puTYprr89QaYFmhYOPfP+d1pH+k9ets6RoE23BXZM1X1jJqoyw==} @@ -2641,61 +2611,51 @@ packages: resolution: {integrity: sha512-zJc0H99FEPoFfSrNpa91HYfxzfAJCr502oxNK1cfdC9hlaFI43RT+JFCann9JUgZmLzzntChHyn13Sgn9ljHNg==} cpu: [arm64] os: [linux] - libc: [glibc] '@unrs/resolver-binding-linux-arm64-musl@1.12.2': resolution: {integrity: sha512-KQ3Lki6l+Pz1k/eBipN41ES+YUK30beLGb9YqcB1O542cyLCNE6GaxrfcY3T6EezmGGk84wb5XyO9loTM9tkcA==} cpu: [arm64] os: [linux] - libc: [musl] '@unrs/resolver-binding-linux-loong64-gnu@1.12.2': resolution: {integrity: sha512-3SJGEh1DborhG6pyxvhPzCT4bbSIVihsvgJc13P1bHG7KLdNDaF9T3gsTwFc7Jw/5Y5/iWOjkEx7Zy0NvCGX3Q==} cpu: [loong64] os: [linux] - libc: [glibc] '@unrs/resolver-binding-linux-loong64-musl@1.12.2': resolution: {integrity: sha512-jiuG/Obbel7uw1PwHNFfrkiKhLAF6mnyZ6aWlOAVN9WqKm8v0OFGnciJIHu8+CMvXLQ8AD51LPzAoUfT21D5Ew==} cpu: [loong64] os: [linux] - libc: [musl] '@unrs/resolver-binding-linux-ppc64-gnu@1.12.2': resolution: {integrity: sha512-q7xRvVpmcfeL+LlZg8Pbbo6QaTZwDU5BaGZbwfhkEsXJn3Was8xYfE0RBH266xZt0rM6B7i8xAYIvjthuUIWHg==} cpu: [ppc64] os: [linux] - libc: [glibc] '@unrs/resolver-binding-linux-riscv64-gnu@1.12.2': resolution: {integrity: sha512-0CVdx6lcnT3Q9inOH8tsMIOJ6ImndllMjqJHg8RLVdB7Vq4SfkEXl9mCSsVNuNA4MCYycRicCUxPCabVHJRr6A==} cpu: [riscv64] os: [linux] - libc: [glibc] '@unrs/resolver-binding-linux-riscv64-musl@1.12.2': resolution: {integrity: sha512-iOwlRo9vnp6R6ohHQS11n0NnfdXx/omhkocmIfaPRpQhKZ+3BDMkkdRVh53qjkFkpPddf+FETA28NwGN7l5l+w==} cpu: [riscv64] os: [linux] - libc: [musl] '@unrs/resolver-binding-linux-s390x-gnu@1.12.2': resolution: {integrity: sha512-HYJtLfXq94q8iZNFT1lknx258wlkkWhZeUXJRqzKBBUJ00CvZ+N33zgbCqimLjsyw5Va6uUxhVa12mI+kaveEw==} cpu: [s390x] os: [linux] - libc: [glibc] '@unrs/resolver-binding-linux-x64-gnu@1.12.2': resolution: {integrity: sha512-mPsUhunKKDih5O96Y6enDQyHc1SqBPlY1E/SfMWDM3EdJ95Z9CArPeCVwCCqbP45ljvivdEk8Fxn+SIb1rDAJQ==} cpu: [x64] os: [linux] - libc: [glibc] '@unrs/resolver-binding-linux-x64-musl@1.12.2': resolution: {integrity: sha512-azrt6+5ydLd8Vt210AAFis/lZevSfPw93EJRIJG+xPu4WCJ8K0kppCTpMyLPcKT7H15M4Jnt2tMp5bOvCkRC6A==} cpu: [x64] os: [linux] - libc: [musl] '@unrs/resolver-binding-openharmony-arm64@1.12.2': resolution: {integrity: sha512-YZ9hP4O0X9PQb8eO980qmLNGH4zT3I9+SZTdt0Pr0YyuGQhYKoOZkV02VzrzyOZJ5xIJ3UFIenKkUkGg8GjgWQ==} @@ -4258,56 +4218,48 @@ packages: engines: {node: '>= 12.0.0'} cpu: [arm64] os: [linux] - libc: [glibc] lightningcss-linux-arm64-gnu@1.32.0: resolution: {integrity: sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ==} engines: {node: '>= 12.0.0'} cpu: [arm64] os: [linux] - libc: [glibc] lightningcss-linux-arm64-musl@1.31.1: resolution: {integrity: sha512-mVZ7Pg2zIbe3XlNbZJdjs86YViQFoJSpc41CbVmKBPiGmC4YrfeOyz65ms2qpAobVd7WQsbW4PdsSJEMymyIMg==} engines: {node: '>= 12.0.0'} cpu: [arm64] os: [linux] - libc: [musl] lightningcss-linux-arm64-musl@1.32.0: resolution: {integrity: sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==} engines: {node: '>= 12.0.0'} cpu: [arm64] os: [linux] - libc: [musl] lightningcss-linux-x64-gnu@1.31.1: resolution: {integrity: sha512-xGlFWRMl+0KvUhgySdIaReQdB4FNudfUTARn7q0hh/V67PVGCs3ADFjw+6++kG1RNd0zdGRlEKa+T13/tQjPMA==} engines: {node: '>= 12.0.0'} cpu: [x64] os: [linux] - libc: [glibc] lightningcss-linux-x64-gnu@1.32.0: resolution: {integrity: sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==} engines: {node: '>= 12.0.0'} cpu: [x64] os: [linux] - libc: [glibc] lightningcss-linux-x64-musl@1.31.1: resolution: {integrity: sha512-eowF8PrKHw9LpoZii5tdZwnBcYDxRw2rRCyvAXLi34iyeYfqCQNA9rmUM0ce62NlPhCvof1+9ivRaTY6pSKDaA==} engines: {node: '>= 12.0.0'} cpu: [x64] os: [linux] - libc: [musl] lightningcss-linux-x64-musl@1.32.0: resolution: {integrity: sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==} engines: {node: '>= 12.0.0'} cpu: [x64] os: [linux] - libc: [musl] lightningcss-win32-arm64-msvc@1.31.1: resolution: {integrity: sha512-aJReEbSEQzx1uBlQizAOBSjcmr9dCdL3XuC/6HLXAxmtErsj2ICo5yYggg1qOODQMtnjNQv2UHb9NpOuFtYe4w==} @@ -6222,9 +6174,9 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 - '@libsql/client@0.17.3': + '@libsql/client@0.18.0': dependencies: - '@libsql/core': 0.17.3 + '@libsql/core': 0.18.0 '@libsql/hrana-client': 0.10.0 js-base64: 3.7.8 libsql: 0.5.29 @@ -6233,7 +6185,7 @@ snapshots: - bufferutil - utf-8-validate - '@libsql/core@0.17.3': + '@libsql/core@0.18.0': dependencies: js-base64: 3.7.8 @@ -8393,9 +8345,9 @@ snapshots: esbuild: 0.25.12 tsx: 4.21.0 - drizzle-orm@0.45.2(@libsql/client@0.17.3)(postgres@3.4.8): + drizzle-orm@0.45.2(@libsql/client@0.18.0)(postgres@3.4.8): optionalDependencies: - '@libsql/client': 0.17.3 + '@libsql/client': 0.18.0 postgres: 3.4.8 dunder-proto@1.0.1: diff --git a/tests/lib/db/concurrent-writes.test.ts b/tests/lib/db/concurrent-writes.test.ts new file mode 100644 index 0000000..4e7e943 --- /dev/null +++ b/tests/lib/db/concurrent-writes.test.ts @@ -0,0 +1,114 @@ +import { describe, it, expect, beforeAll, beforeEach, afterEach } from 'vitest' +import { readdirSync } from 'node:fs' +import { sql, eq, inArray } from 'drizzle-orm' +import { runMigrations, seedFixtures, clearTables, F } from '@/tests/helpers/db' +import { db } from '@/lib/db' +import { users, workItems, specLinks } from '@/lib/db/schema.sqlite' +import { createWorkItem, createTask } from '@/lib/db/mutations' +import { createSpec, linkSpec } from '@/lib/db/specs' +import { search } from '@/lib/db/search' +import { NestedWriteError, WriteQueue } from '@/lib/db/sqlite-writes' + +// Regression for parallel MCP writes against a SQLite file: writes that arrived +// while a transaction was open failed with SQLITE_BUSY, because connections +// libsql opened after a transaction had no busy timeout, and each transaction +// left its connection open until garbage collection. + +beforeAll(runMigrations) +beforeEach(seedFixtures) +afterEach(clearTables) +const d = db as any +const busyTimeout = async () => ((await d.all(sql`PRAGMA busy_timeout`)) as { timeout: number }[])[0].timeout +const openFiles = () => readdirSync(process.platform === 'linux' ? '/proc/self/fd' : '/dev/fd').length +const item = (title: string) => + createWorkItem({ productId: F.productShared, type: 'bug', title, description: '', severity: 'low', tags: [] }, F.alice, 'agent') + +describe('parallel writes on SQLite', () => { + it('keeps the busy timeout on the connection libsql opens after a transaction', async () => { + await d.transaction(async (tx: any) => tx.select().from(users)) + expect(await busyTimeout()).toBe(5000) + }) + + it.skipIf(process.platform === 'win32')('returns transaction connections instead of leaving them open', async () => { + await d.transaction(async (tx: any) => tx.select().from(users)) + const before = openFiles() + for (let i = 0; i < 100; i++) { + await d.transaction(async (tx: any) => tx.update(users).set({ name: `Alice ${i}` }).where(eq(users.id, F.alice))) + } + // libsql 0.17 kept about two files open per transaction (~200 here). + expect(openFiles() - before).toBeLessThan(10) + }) + + it('persists a parallel batch of creates and spec links', async () => { + const spec = await createSpec({ productId: F.productShared, title: 'Parallel spec', body: 'body', specType: 'design' }, F.alice, 'agent') + const titles = Array.from({ length: 8 }, (_, i) => `Parallel item ${i}`) + const results = await Promise.allSettled([ + ...titles.map(item), + createTask({ codePlanId: F.planActive, title: 'Parallel task', description: '', priority: 'low', tags: [] }), + createSpec({ productId: F.productShared, title: 'Second parallel spec', body: 'body', specType: 'design' }, F.alice, 'agent'), + linkSpec(spec.id, 'code_plan', F.planActive, 'creates', F.alice), + linkSpec(spec.id, 'asset', F.assetApi, undefined, F.alice), + ]) + expect(results.filter((r) => r.status === 'rejected')).toEqual([]) + + const ids = results.slice(0, titles.length).map((r) => (r as PromiseFulfilledResult<{ id: string }>).value.id) + const stored = await d.select({ id: workItems.id }).from(workItems).where(inArray(workItems.id, ids)) + expect(stored).toHaveLength(titles.length) + const found = await search(F.alice, '"Parallel item"', { types: ['work_item'], limit: 50 }) + expect(found.results.map((r: { id: string }) => r.id).sort()).toEqual([...ids].sort()) + + // Linking the new items right afterwards finds every one of them. + await Promise.all(ids.map((id) => linkSpec(spec.id, 'work_item', id, undefined, F.alice))) + expect(await d.select().from(specLinks).where(eq(specLinks.specId, spec.id))).toHaveLength(2 + ids.length) + }) + + it('queues a write behind an open transaction instead of failing it', async () => { + let inside!: () => void + const entered = new Promise((resolve) => { inside = resolve }) + const tx = d.transaction(async (t: any) => { + inside() + await new Promise((resolve) => setTimeout(resolve, 50)) + await t.insert(users).values({ id: 'u-tx', email: 'tx@example.com', name: 'Tx' }) + }) + await entered + const outside = d.insert(users).values({ id: 'u-out', email: 'out@example.com', name: 'Out' }).returning() + await Promise.all([tx, outside]) + const rows = await d.select({ id: users.id }).from(users).where(inArray(users.id, ['u-tx', 'u-out'])) + expect(rows).toHaveLength(2) + }) + + it('lets reads run while a transaction is open', async () => { + let release!: () => void + const held = new Promise((resolve) => { release = resolve }) + const tx = d.transaction(async () => held) + expect(await d.select().from(users).where(eq(users.id, F.alice))).toHaveLength(1) + release() + await tx + }) + + it('rejects a write through db inside db.transaction rather than deadlocking', async () => { + await expect( + d.transaction(async () => d.insert(users).values({ id: 'u-nested', email: 'n@example.com', name: 'N' })), + ).rejects.toMatchObject({ cause: expect.any(NestedWriteError) }) + await expect(d.transaction(async () => d.transaction(async () => {}))).rejects.toThrow(NestedWriteError) + // The queue is free again afterwards. + await d.insert(users).values({ id: 'u-after', email: 'after@example.com', name: 'After' }) + }) +}) + +describe('WriteQueue', () => { + it('runs writes one at a time, in order, and survives a failing one', async () => { + const queue = new WriteQueue() + const order: string[] = [] + const step = (name: string, ms: number, fail = false) => + queue.run(async () => { + order.push(`${name}:start`) + await new Promise((resolve) => setTimeout(resolve, ms)) + order.push(`${name}:end`) + if (fail) throw new Error(name) + }) + const results = await Promise.allSettled([step('a', 20), step('b', 0, true), step('c', 0)]) + expect(results.map((r) => r.status)).toEqual(['fulfilled', 'rejected', 'fulfilled']) + expect(order).toEqual(['a:start', 'a:end', 'b:start', 'b:end', 'c:start', 'c:end']) + }) +})