diff --git a/.changeset/powersync-managed-tables.md b/.changeset/powersync-managed-tables.md new file mode 100644 index 0000000000..95041888f0 --- /dev/null +++ b/.changeset/powersync-managed-tables.md @@ -0,0 +1,5 @@ +--- +'@tanstack/powersync-db-collection': minor +--- + +Support PowerSync managed tables (`direct: true`). The PowerSync SDK's diff trigger machinery only supports managed views: its trigger payloads read the single JSON `data` column of the `ps_data__` view, which does not exist for a managed table (the real columns live on the table itself), so collections over managed tables failed with `no such table: main.ps_data__`. The collection now detects the table's `direct` flag and creates its own tracking table and `AFTER INSERT/UPDATE/DELETE` triggers on the real table, with payloads built via `json_object` over the real columns and trigger `WHEN` clauses compiled against `NEW.`/`OLD.` row aliases for on-demand sync. Eager initial loads, sync-driven change tracking, optimistic mutation tracking, and disposal cleanup match the view-based path; managed views are untouched. Managed tables require `@powersync/common` >= 2.5.0 (dev dependency updated, `@powersync/node` test dependency bumped to `^1.3.0`). `trackMetadata` is not supported for managed tables yet (the SDK does not materialize the metadata columns) and configuring that combination now throws. diff --git a/docs/collections/powersync-collection.md b/docs/collections/powersync-collection.md index 472a7e6926..264b31ba42 100644 --- a/docs/collections/powersync-collection.md +++ b/docs/collections/powersync-collection.md @@ -506,6 +506,40 @@ class Connector implements PowerSyncBackendConnector { **Note**: If metadata is provided to an operation but the table doesn't have `trackMetadata: true`, a warning will be logged and the metadata will be ignored. +### Managed Tables (`direct: true`) + +PowerSync supports [managed tables](https://docs.powersync.com/client-sdks/advanced/performance-optimization#managed-views-vs-managed-tables), which store synced rows as real SQLite columns instead of a JSON `data` column behind a view: + +```typescript +const APP_SCHEMA = new Schema({ + documents: new Table( + { + name: column.text, + author: column.text, + }, + { + // Store data as real SQLite rows + direct: true, + }, + ), +}) +``` + +The collection supports managed tables everywhere the view-based layout is supported: + +- Eager mode loads rows from the real columns. +- On-demand mode compiles live query predicates into trigger `WHEN` clauses against the real `NEW.`/`OLD.` columns. +- Optimistic mutations write to the managed table and are tracked exactly once (no double writes). + +PowerSync's own diff trigger machinery does not support managed tables yet (its trigger payloads assume the JSON `data` column of a managed view). This collection handles that itself: for managed tables it creates the tracking table and `AFTER INSERT/UPDATE/DELETE` triggers on the real table, with payloads built from the real columns. The tracking table shape, trigger naming and cleanup contract match the SDK's behavior, so disposal leaves no leaked triggers or tables. + +Caveats: + +- Managed tables require a PowerSync SDK with managed table support (`@powersync/common` >= 2.5.0 and a matching platform SDK such as `@powersync/web` 2.6+ or `@powersync/node` 1.3+). +- `trackMetadata: true` is not supported for managed tables yet (the SDK does not materialize the metadata columns). Configuring this combination throws an error. +- Like the view-based path, tracking triggers and the tracking table are `TEMP` per database connection, so each tab tracks its own diffs. +- Reopening a database that stored a table as a view with a managed-table schema migrates the storage; the collection recreates its triggers against the real table and loads the migrated rows. + ## Configuration Options The `powerSyncCollectionOptions` function accepts the following options: diff --git a/packages/powersync-db-collection/package.json b/packages/powersync-db-collection/package.json index 2a5935e517..1838ce0e4d 100644 --- a/packages/powersync-db-collection/package.json +++ b/packages/powersync-db-collection/package.json @@ -63,7 +63,7 @@ }, "devDependencies": { "@powersync/common": "2.5.0", - "@powersync/node": "0.21.0", + "@powersync/node": "^1.3.0", "@types/debug": "^4.1.13", "@vitest/coverage-istanbul": "^5.1.0", "better-sqlite3": "^12.11.1" diff --git a/packages/powersync-db-collection/src/direct-triggers.ts b/packages/powersync-db-collection/src/direct-triggers.ts new file mode 100644 index 0000000000..869baf54bb --- /dev/null +++ b/packages/powersync-db-collection/src/direct-triggers.ts @@ -0,0 +1,281 @@ +import { DiffTriggerOperation, LogLevels } from '@powersync/common' +import type { + CommonPowerSyncDatabase, + LockContext, + TriggerRemoveCallback, +} from '@powersync/common' + +/** + * Options for {@link createDirectDiffTrigger}. Mirrors the SDK's + * `CreateDiffTriggerOptions`, restricted to the shapes the collection + * adapter uses. + */ +export type DirectDiffTriggerOptions = { + /** + * The SQLite table to trigger on. For a managed (`direct: true`) table this + * is the table's schema name: the SDK materializes managed tables as real + * tables under their schema name and does not create a `ps_data__` + * object for them (verified against `@powersync/node` 1.3.0 / + * `@powersync/common` 2.5.0). + */ + source: string + /** + * The tracking table the triggers write diff records into. Created here as + * a TEMP table and dropped on removal. + */ + destination: string + /** + * The real columns to track. Unlike the view-based path there is no JSON + * `data` column, so payloads are built with `json_object` over these + * columns directly. An empty array tracks only the row id. + */ + columns: Array + setupContext?: LockContext + when: Partial> + hooks?: { + /** + * Executed inside a write lock before the tracking table is created. + */ + beforeCreate?: (context: LockContext) => Promise + } +} + +/** + * The SQLite table shape used by the SDK's `TriggerManagerImpl`. The diff + * flush path (`SELECT * FROM `) reads exactly these columns, so + * adapter-managed triggers for managed tables must produce the same shape. + */ +const TRACKING_TABLE_COLUMNS = ` + operation_id INTEGER PRIMARY KEY AUTOINCREMENT, + id TEXT, + operation TEXT, + timestamp TEXT, + value TEXT, + previous_value TEXT +` + +/** + * Builds the JSON payload fragment for a trigger from the real columns of a + * managed table. This replaces the SDK's `NEW.data` / `json_extract(NEW.data, + * '$.')` fragments, which assume the single JSON `data` column of a + * managed view. + */ +const jsonFragment = (columns: Array, source: `NEW` | `OLD`) => { + if (columns.length == 0) { + // Don't track any columns except for the id + return `'{}'` + } + return `json_object(${columns + .map((column) => `'${column}', ${source}."${column}"`) + .join(', ')})` +} + +/** + * Mirrors the SDK's trigger naming scheme (`__ps_temp_trigger_____`) + * so adapter-managed triggers are recognizable and collision-free alongside + * SDK-created ones. + */ +const generateTriggerName = ( + operation: DiffTriggerOperation, + destinationTable: string, + triggerId: string, +) => + `__ps_temp_trigger_${operation.toLowerCase()}__${destinationTable}__${triggerId}` + +/** + * Resolves the unique id used in trigger names, preferring the setup + * transaction's UUID provider so ids are stable across multi-tab + * coordination contexts (same contract as the SDK's trigger manager). + */ +async function getUUID(context: LockContext | CommonPowerSyncDatabase) { + const { id: uuid } = await context.get<{ id: string }>(`SELECT uuid() as id`) + // Replace dashes with underscores for SQLite table/trigger name compatibility + return uuid.replace(/-/g, `_`) +} + +/** + * Creates the tracking table and `AFTER INSERT/UPDATE/DELETE` diff triggers + * for a managed (`direct: true`) PowerSync table. + * + * The SDK's `TriggerManagerImpl` only supports managed views: its trigger + * payloads read `NEW.data` / `json_extract(NEW.data, '$.')`, and it + * triggers on `ps_data__`, which does not exist for a managed table + * (the real columns live on the table itself). Until the SDK gains support, + * the adapter creates equivalent TEMP triggers itself: + * - payloads are built with `json_object('column', NEW."column", ...)` over + * the real columns (`OLD` for the previous value), + * - the tracking table shape, trigger naming, `when` clauses, `beforeCreate` + * hook timing and cleanup contract (drop triggers + tracking table) match + * `TriggerManagerImpl.createDiffTrigger`, + * - triggers are always TEMP. The SDK's storage-backed (persisted) mode and + * its claim-based cleanup never apply here, because the adapter never + * passes `useStorage` to the SDK either. + * + * @returns A callback to remove the trigger and drop the tracking table. + */ +export async function createDirectDiffTrigger( + database: CommonPowerSyncDatabase, + options: DirectDiffTriggerOptions, +): Promise { + await database.waitForReady() + const { source, destination, columns, when, hooks, setupContext } = options + const operations = Object.keys(when) as Array + if (operations.length == 0) { + throw new Error( + `At least one WHEN operation must be specified for the trigger.`, + ) + } + + // The adapter's tracking resources are TEMP: each connection (tab) owns its + // own tracking table and triggers, and they disappear with the connection. + const tableTriggerTypeClause = `TEMP` + + const whenClauses = Object.fromEntries( + Object.entries(when).map(([operation, filter]) => [ + operation, + `WHEN ${filter}`, + ]), + ) + + const triggerIds: Array = [] + const id = await getUUID(setupContext ?? database) + + const disposeWarningListener = database.registerListener({ + schemaChanged: () => { + database.logger.log({ + level: LogLevels.warn, + message: `The PowerSync schema has changed while previously configured triggers are still operational. This might cause unexpected results.`, + }) + }, + }) + + /** + * Declare the cleanup function early since if any of the init steps fail, + * we need to ensure we can cleanup the created resources. + * We unfortunately cannot rely on transaction rollback. + */ + const cleanup = async (removeOptions?: { + context?: LockContext + }): Promise => { + disposeWarningListener() + const context = removeOptions?.context + const doCleanup = async (tx: LockContext) => { + for (const triggerId of triggerIds) { + await tx.execute(`DROP TRIGGER IF EXISTS ${triggerId}; `) + } + await tx.execute(`DROP TABLE IF EXISTS ${destination};`) + } + if (context) { + await doCleanup(context) + } else { + await database.writeLock(doCleanup) + } + } + + const setup = async (tx: LockContext) => { + // Allow user code to execute in this lock context before the trigger is created. + await hooks?.beforeCreate?.(tx) + await tx.execute( + `CREATE ${tableTriggerTypeClause} TABLE ${destination} ( + ${TRACKING_TABLE_COLUMNS} + )`, + ) + + if (operations.includes(DiffTriggerOperation.INSERT)) { + const insertTriggerId = generateTriggerName( + DiffTriggerOperation.INSERT, + destination, + id, + ) + triggerIds.push(insertTriggerId) + + await tx.execute(` + CREATE ${tableTriggerTypeClause} TRIGGER ${insertTriggerId} AFTER INSERT ON ${source} ${ + whenClauses[DiffTriggerOperation.INSERT] + } BEGIN + INSERT INTO + ${destination} (id, operation, timestamp, value) + VALUES + ( + NEW.id, + 'INSERT', + strftime ('%Y-%m-%dT%H:%M:%fZ', 'now'), + ${jsonFragment(columns, 'NEW')} + ); + + END + `) + } + + if (operations.includes(DiffTriggerOperation.UPDATE)) { + const updateTriggerId = generateTriggerName( + DiffTriggerOperation.UPDATE, + destination, + id, + ) + triggerIds.push(updateTriggerId) + + await tx.execute(` + CREATE ${tableTriggerTypeClause} TRIGGER ${updateTriggerId} AFTER + UPDATE ON ${source} ${whenClauses[DiffTriggerOperation.UPDATE]} BEGIN + INSERT INTO + ${destination} (id, operation, timestamp, value, previous_value) + VALUES + ( + NEW.id, + 'UPDATE', + strftime ('%Y-%m-%dT%H:%M:%fZ', 'now'), + ${jsonFragment(columns, 'NEW')}, + ${jsonFragment(columns, 'OLD')} + ); + + END; + `) + } + + if (operations.includes(DiffTriggerOperation.DELETE)) { + const deleteTriggerId = generateTriggerName( + DiffTriggerOperation.DELETE, + destination, + id, + ) + triggerIds.push(deleteTriggerId) + + await tx.execute(` + CREATE ${tableTriggerTypeClause} TRIGGER ${deleteTriggerId} AFTER DELETE ON ${source} ${ + whenClauses[DiffTriggerOperation.DELETE] + } BEGIN + INSERT INTO + ${destination} (id, operation, timestamp, value) + VALUES + ( + OLD.id, + 'DELETE', + strftime ('%Y-%m-%dT%H:%M:%fZ', 'now'), + ${jsonFragment(columns, 'OLD')} + ); + + END; + `) + } + } + + try { + if (setupContext) { + await setup(setupContext) + } else { + await database.writeLock(setup) + } + return cleanup + } catch (error) { + try { + await cleanup(setupContext ? { context: setupContext } : undefined) + } catch (cleanupError) { + throw new AggregateError( + [error, cleanupError], + `Error during operation and cleanup`, + ) + } + throw error + } +} diff --git a/packages/powersync-db-collection/src/powersync.ts b/packages/powersync-db-collection/src/powersync.ts index dfb52c15f4..9895e13689 100644 --- a/packages/powersync-db-collection/src/powersync.ts +++ b/packages/powersync-db-collection/src/powersync.ts @@ -1,5 +1,6 @@ import { DiffTriggerOperation, LogLevels, sanitizeSQL } from '@powersync/common' import { or, whenSyncAccepted, withCollectionConfigFactory } from '@tanstack/db' +import { createDirectDiffTrigger } from './direct-triggers' import { compileSQLite } from './sqlite-compiler' import { PendingOperationStore } from './PendingOperationStore' import { PowerSyncTransactor } from './PowerSyncTransactor' @@ -241,6 +242,12 @@ export function powerSyncCollectionOptions< ) } +/** + * Builds the collection config shared by every `powerSyncCollectionOptions` + * call: wires the eager/on-demand sync implementation (diff-trigger based), + * the optimistic-mutation transactor, key derivation, and utils metadata, + * branching per storage kind (managed views vs managed tables). + */ function createPowerSyncCollectionConfig< TTable extends Table, TSchema extends StandardSchemaV1 = never, @@ -270,6 +277,25 @@ function createPowerSyncCollectionConfig< const { viewName, trackMetadata: metadataIsTracked } = table + /** + * Managed (`direct: true`) tables are stored as real SQLite tables with + * real columns instead of a view over a JSON `data` column. The SDK's + * trigger machinery only supports the view layout, so the adapter creates + * the tracking triggers itself for these tables. See + * {@link createDirectDiffTrigger}. + */ + const isManagedTable = table.direct === true + + if (isManagedTable && metadataIsTracked) { + // The PowerSync SDK does not materialize the `_metadata`/`_deleted` + // columns for managed tables, so the metadata persistence SQL the + // transactor relies on would fail at write time. Fail at configuration + // time instead. + throw new Error( + `PowerSync table ${viewName} is a managed table (direct: true) with trackMetadata: true. The PowerSync SDK does not support metadata columns for managed tables yet.`, + ) + } + /** * Deserializes data from the incoming sync stream */ @@ -421,6 +447,13 @@ function createPowerSyncCollectionConfig< } } + /** + * Establishes change tracking for the collection's table: creates the + * tracking table + AFTER triggers (SDK trigger manager for managed + * views, {@link createDirectDiffTrigger} for managed tables), wiring + * the `beforeCreate` hook that performs the initial eager load. + * Returns the disposal callback for teardown. + */ async function createDiffTrigger( options: { setupContext?: LockContext @@ -439,6 +472,55 @@ function createPowerSyncCollectionConfig< ) { const { setupContext, snapshot, when, writeType, batchQuery } = options + /** + * A load of every row is one snapshot transaction: during a + * reload it replaces the kept rows, so a row the table no + * longer has goes. A subset load commits each batch. A + * transaction begins only once its rows are read, so a batch + * that fails leaves none open. + */ + const beforeCreate = async (context: LockContext) => { + const batches: Array> = [] + const apply = () => { + begin({ snapshot }) + for (const row of batches.splice(0).flat()) { + write({ + type: writeType(row.id), + value: deserializeSyncRow(row), + }) + } + appliedReceipts.push(commit()) + } + let currentBatchCount = syncBatchSize + let cursor = 0 + while (currentBatchCount == syncBatchSize) { + const batchItems = await batchQuery(context, syncBatchSize, cursor) + currentBatchCount = batchItems.length + cursor += currentBatchCount + batches.push(batchItems) + if (!snapshot) apply() + } + if (snapshot) apply() + database.logger.log({ + level: LogLevels.info, + message: `Sync is ready for ${viewName} into ${trackedTableName}`, + }) + } + + if (isManagedTable) { + // Managed tables are real SQLite tables: the SDK's trigger manager + // cannot track them, so the adapter builds the tracking table and + // triggers itself. + return await createDirectDiffTrigger(database, { + source: viewName, + destination: trackedTableName, + columns: table.columns.map((column) => column.name), + setupContext, + when, + hooks: { beforeCreate }, + }) + } + return await database.triggers.createDiffTrigger({ source: viewName, destination: trackedTableName, @@ -446,42 +528,7 @@ function createPowerSyncCollectionConfig< setupContext, when, hooks: { - beforeCreate: async (context) => { - // A load of every row is one snapshot transaction: during a - // reload it replaces the kept rows, so a row the table no - // longer has goes. A subset load commits each batch. A - // transaction begins only once its rows are read, so a batch - // that fails leaves none open. - const batches: Array> = [] - const apply = () => { - begin({ snapshot }) - for (const row of batches.splice(0).flat()) { - write({ - type: writeType(row.id), - value: deserializeSyncRow(row), - }) - } - appliedReceipts.push(commit()) - } - let currentBatchCount = syncBatchSize - let cursor = 0 - while (currentBatchCount == syncBatchSize) { - const batchItems = await batchQuery( - context, - syncBatchSize, - cursor, - ) - currentBatchCount = batchItems.length - cursor += currentBatchCount - batches.push(batchItems) - if (!snapshot) apply() - } - if (snapshot) apply() - database.logger.log({ - level: LogLevels.info, - message: `Sync is ready for ${viewName} into ${trackedTableName}`, - }) - }, + beforeCreate, }, }) } @@ -609,8 +656,12 @@ function createPowerSyncCollectionConfig< } } - // Eager mode. - // Registers a diff trigger for the entire table. + /** + * Eager sync mode: registers a diff trigger for the entire table and + * performs the initial full-table load through its `beforeCreate` + * hook (SDK trigger manager for managed views, adapter-built + * triggers for managed tables). + */ function runEagerSync() { let onUnload: CleanupFn | void | null = null let onUnloadStarted = false @@ -713,8 +764,11 @@ function createPowerSyncCollectionConfig< } } - // On-demand mode. - // Registers a diff trigger for the active WHERE expressions. + /** + * On-demand sync mode: registers a diff trigger whose `when` clauses + * cover only the active live-query WHERE expressions, loading (and + * evicting) rows as the collection's data boundary changes. + */ function runOnDemandSync() { type DemandRecord = { options: LoadSubsetOptions @@ -794,13 +848,23 @@ function createPowerSyncCollectionConfig< active.length === 1 ? active[0] : or(active[0], active[1], ...active.slice(2)) + /** + * Chooses compile options per storage kind: managed views + * keep rows in a JSON `data` column, managed tables expose + * real columns, so trigger `when` clauses compile against + * NEW./OLD. row aliases instead. + */ + const rowCompileOptions = (rowPrefix: 'NEW' | 'OLD') => + isManagedTable + ? { rowPrefix } + : { jsonColumn: `${rowPrefix}.data` } const compiledNewData = compileSQLite( { where: combinedWhere }, - { jsonColumn: 'NEW.data' }, + rowCompileOptions(`NEW`), ) const compiledOldData = compileSQLite( { where: combinedWhere }, - { jsonColumn: 'OLD.data' }, + rowCompileOptions(`OLD`), ) const compiledView = compileSQLite({ where: combinedWhere }) const newDataWhenClause = toInlinedWhereClause(compiledNewData) diff --git a/packages/powersync-db-collection/src/sqlite-compiler.ts b/packages/powersync-db-collection/src/sqlite-compiler.ts index d5d0274965..99fb4dd59a 100644 --- a/packages/powersync-db-collection/src/sqlite-compiler.ts +++ b/packages/powersync-db-collection/src/sqlite-compiler.ts @@ -24,6 +24,13 @@ export interface CompileSQLiteOptions { * as a direct column in the tracked table. */ jsonColumn?: string + /** + * When set, column references emit `.""` instead of + * `""`. This is used for managed (`direct: true`) tables, where + * trigger `when` clauses must reference the real columns of the NEW or OLD + * row instead of a JSON `data` column. Mutually exclusive with `jsonColumn`. + */ + rowPrefix?: 'NEW' | 'OLD' } /** @@ -105,6 +112,8 @@ function compileExpression( return `${prefix}.${quoteIdentifier(columnName)}` } else if (compileOptions?.jsonColumn) { return `json_extract(${compileOptions.jsonColumn}, '$.${columnName}')` + } else if (compileOptions?.rowPrefix) { + return `${compileOptions.rowPrefix}.${quoteIdentifier(columnName)}` } return quoteIdentifier(columnName) } diff --git a/packages/powersync-db-collection/tests/managed-tables.test.ts b/packages/powersync-db-collection/tests/managed-tables.test.ts new file mode 100644 index 0000000000..3ca5a896b9 --- /dev/null +++ b/packages/powersync-db-collection/tests/managed-tables.test.ts @@ -0,0 +1,594 @@ +import { randomUUID } from 'node:crypto' +import { tmpdir } from 'node:os' +import { PowerSyncDatabase, Schema, Table, column } from '@powersync/node' +import { + and, + createCollection, + createLiveQueryCollection, + eq, + gt, +} from '@tanstack/db' +import { describe, expect, it, onTestFinished, vi } from 'vitest' +import { powerSyncCollectionOptions } from '../src' +import { TEST_DATABASE_IMPLEMENTATION } from './test-db-implementation' +import type { CommonPowerSyncDatabase } from '@powersync/node' + +// The same table shape declared once as a managed view and once as a managed +// table, to test reopening a database across the SDK's storage migration. +const VIEW_STORAGE_SCHEMA = new Schema({ + documents: new Table({ + name: column.text, + author: column.text, + created_at: column.text, + }), +}) + +const MANAGED_SCHEMA = new Schema({ + documents: new Table( + { + name: column.text, + author: column.text, + created_at: column.text, + }, + { + direct: true, + }, + ), + products: new Table( + { + name: column.text, + price: column.integer, + category: column.text, + }, + { + direct: true, + }, + ), +}) + +const MIXED_SCHEMA = new Schema({ + documents: new Table( + { + name: column.text, + }, + { + direct: true, + }, + ), + // The same schema also holds a JSON-backed view. + users: new Table({ + name: column.text, + }), +}) + +const MANAGED_METADATA_SCHEMA = new Schema({ + documents: new Table( + { + name: column.text, + }, + { + direct: true, + trackMetadata: true, + }, + ), +}) + +const describePowerSync = TEST_DATABASE_IMPLEMENTATION + ? describe + : describe.skip + +describePowerSync(`Managed Tables (direct)`, () => { + /** + * Builds a real PowerSync database (better-sqlite3 / node:sqlite backend) + * for the given schema, with a unique temp directory per call. + */ + async function createDatabase(schema: Schema) { + const db = new PowerSyncDatabase({ + database: { + dbFilename: `test-managed-${randomUUID()}.sqlite`, + dbLocation: tmpdir(), + implementation: TEST_DATABASE_IMPLEMENTATION, + }, + schema, + }) + onTestFinished(async () => { + await db.disconnectAndClear() + await db.close() + }) + // Initial clear in case a test might have failed + await db.disconnectAndClear() + return db + } + + /** + * Inserts the canonical row set used by the managed-tables scenarios + * through the SDK's write path (so triggers observe real operations). + */ + async function createTestData(db: CommonPowerSyncDatabase) { + await db.execute(` + INSERT into documents (id, name, author) + VALUES + (uuid(), 'one', 'a'), + (uuid(), 'two', 'b'), + (uuid(), 'three', 'c') + `) + } + + const sortedNames = (rows: Array<{ name?: string | null }>) => + rows.map((row) => row.name).sort() + + /** + * TEMP tracking resources live on the write connection, so they are read + * under the write lock. Returns whether the tracking table still exists and + * which diff rows and triggers remain. + */ + /** + * Reads a table's tracking-table contents (operation/id/value rows) + * for asserting exactly which diffs the adapter triggers recorded. + */ + async function readTrackingState( + db: CommonPowerSyncDatabase, + trackedTableName: string, + ) { + return await db.writeLock(async (tx) => { + const triggers = await tx.getAll<{ name: string }>( + `SELECT name FROM sqlite_temp_master WHERE type = 'trigger' AND name LIKE ?`, + [`%${trackedTableName}%`], + ) + const table = await tx.getOptional<{ count: number }>( + `SELECT COUNT(*) as count FROM sqlite_temp_master WHERE type = 'table' AND name = ?`, + [trackedTableName], + ) + const tableExists = table != null && table.count > 0 + const remaining = tableExists + ? await tx.getAll(`SELECT * FROM ${trackedTableName}`) + : [] + return { + triggers: triggers.map((t) => t.name), + tableExists, + remaining, + } + }) + } + + describe(`sync`, () => { + it(`should initialize and fetch initial data from real columns`, async () => { + const db = await createDatabase(MANAGED_SCHEMA) + await createTestData(db) + const collection = createCollection( + powerSyncCollectionOptions({ + database: db, + table: MANAGED_SCHEMA.props.documents, + }), + ) + onTestFinished(() => collection.cleanup({ dependents: true })) + + await collection.stateWhenReady() + + // Verify the collection state contains our items + expect(collection.size).toBe(3) + // Sort by name since keys are random UUIDs + expect(sortedNames(collection.toArray)).deep.equals([ + `one`, + `three`, + `two`, + ]) + // The rows were read from the real columns, not a JSON data column + const withAuthors = collection.toArray + .map((entry) => ({ name: entry.name!, author: entry.author! })) + .sort((a, b) => a.name.localeCompare(b.name)) + expect(withAuthors.map((entry) => entry.author)).toEqual([`a`, `c`, `b`]) + }) + + it(`should update when data syncs`, async () => { + const db = await createDatabase(MANAGED_SCHEMA) + await createTestData(db) + + const collection = createCollection( + powerSyncCollectionOptions({ + database: db, + table: MANAGED_SCHEMA.props.documents, + }), + ) + onTestFinished(() => collection.cleanup({ dependents: true })) + + await collection.stateWhenReady() + + // Verify the collection state contains our items + expect(collection.size).toBe(3) + + // Make an insert, simulates a sync from another client + await db.execute(` + INSERT into documents (id, name, author) + VALUES + (uuid(), 'four', 'd') + `) + + // The collection should update + await vi.waitFor( + () => { + expect(collection.size).toBe(4) + expect(sortedNames(collection.toArray)).deep.equals([ + `four`, + `one`, + `three`, + `two`, + ]) + }, + { timeout: 1000 }, + ) + + await db.execute(` + DELETE from documents + WHERE name = 'two' + `) + + // The collection should update + await vi.waitFor( + () => { + expect(collection.size).toBe(3) + expect(sortedNames(collection.toArray)).deep.equals([ + `four`, + `one`, + `three`, + ]) + }, + { timeout: 1000 }, + ) + + await db.execute(` + UPDATE documents + SET name = 'updated' + WHERE name = 'one' + `) + + // The collection should update + await vi.waitFor( + () => { + expect(collection.size).toBe(3) + expect(sortedNames(collection.toArray)).deep.equals([ + `four`, + `three`, + `updated`, + ]) + }, + { timeout: 1000 }, + ) + }) + + it(`should propagate collection mutations to PowerSync without double writing`, async () => { + const db = await createDatabase(MANAGED_SCHEMA) + await createTestData(db) + + const options = powerSyncCollectionOptions({ + database: db, + table: MANAGED_SCHEMA.props.documents, + }) + const collection = createCollection(options) + onTestFinished(() => collection.cleanup({ dependents: true })) + await collection.stateWhenReady() + + expect(collection.size).toBe(3) + const { trackedTableName } = options.utils.getMeta() + + const id = randomUUID() + const tx = collection.insert({ + id, + name: `new`, + author: `somebody`, + }) + + // The insert should optimistically update the collection + const newDoc = collection.get(id) + expect(newDoc).toBeDefined() + expect(newDoc!.name).toBe(`new`) + + await tx.when(`settled`) + // The item should now be present in PowerSync + const documents = await db.getAll<{ id: string; name: string }>( + `SELECT id, name FROM documents`, + ) + expect(documents.find((doc) => doc.id === id)!.name).toBe(`new`) + // The optimistic mutation was observed exactly once through the + // tracking triggers: the diff records were flushed, not re-applied. + const trackingState = await readTrackingState(db, trackedTableName) + expect(trackingState.remaining).toEqual([]) + + // Now do an update + await collection + .update(id, (d) => (d.name = `updatedNew`)) + .when(`settled`) + + const updatedDoc = collection.get(id) + expect(updatedDoc).toBeDefined() + expect(updatedDoc!.name).toBe(`updatedNew`) + // Only the updated field should be updated + expect(updatedDoc!.author).toBe(`somebody`) + + await collection.delete(id).when(`settled`) + + // There should be a crud entries for this + const crudEntries = await db.getAll<{ op: string }>( + `SELECT data ->> 'op' AS op FROM ps_crud ORDER BY id`, + ) + + // The three seeded rows arrive as PUTs (via the SDK's managed table + // write triggers), followed by our PUT, PATCH and DELETE. + expect(crudEntries.length).toBe(6) + expect(crudEntries.map((e) => e.op)).toEqual([ + `PUT`, + `PUT`, + `PUT`, + `PUT`, + `PATCH`, + `DELETE`, + ]) + // The deleted row is gone from SQLite and the collection + const afterDelete = await db.getAll<{ id: string }>( + `SELECT id FROM documents`, + ) + expect(afterDelete.find((doc) => doc.id === id)).toBeUndefined() + expect(collection.get(id)).toBeUndefined() + expect(collection.size).toBe(3) + }) + + it(`should handle on-demand subsets with real column when clauses`, async () => { + const db = await createDatabase(MANAGED_SCHEMA) + await db.execute(` + INSERT INTO products (id, name, price, category) + VALUES + (uuid(), 'Cheap A', 25, 'electronics'), + (uuid(), 'Expensive B', 150, 'electronics'), + (uuid(), 'Cheap C', 30, 'clothing'), + (uuid(), 'Expensive D', 200, 'clothing') + `) + + const collection = createCollection( + powerSyncCollectionOptions({ + database: db, + table: MANAGED_SCHEMA.props.products, + syncMode: `on-demand`, + }), + ) + onTestFinished(() => collection.cleanup({ dependents: true })) + + const live = createLiveQueryCollection({ + query: (q) => + q + .from({ product: collection }) + .where(({ product }) => + and(gt(product.price, 100), eq(product.category, `electronics`)), + ), + }) + + try { + await live.preload() + await vi.waitFor( + () => { + expect(collection.size).toBe(1) + expect(collection.toArray[0]!.name).toBe(`Expensive B`) + }, + { timeout: 1000 }, + ) + + // A row change outside the tracked when clauses must not be tracked + await db.execute( + `UPDATE products SET price = 300 WHERE name = 'Cheap C'`, + ) + await new Promise((resolve) => setTimeout(resolve, 100)) + expect(collection.size).toBe(1) + + // A row change inside the tracked when clauses must arrive + await db.execute( + `UPDATE products SET price = 400 WHERE name = 'Expensive B'`, + ) + await vi.waitFor( + () => { + const row = collection.toArray.find( + (candidate) => candidate.name === `Expensive B`, + ) + expect(row!.price).toBe(400) + }, + { timeout: 1000 }, + ) + } finally { + await live.cleanup() + } + + // Dropping the demand evicts the rows and stops tracking + await vi.waitFor( + () => { + expect(collection.size).toBe(0) + }, + { timeout: 1000 }, + ) + }) + }) + + describe(`Lifecycle`, () => { + it(`should cleanup tracking triggers and tables`, async () => { + const db = await createDatabase(MANAGED_SCHEMA) + const collectionOptions = powerSyncCollectionOptions({ + database: db, + table: MANAGED_SCHEMA.props.documents, + }) + + const meta = collectionOptions.utils.getMeta() + + const collection = createCollection(collectionOptions) + await collection.stateWhenReady() + const active = await readTrackingState(db, meta.trackedTableName) + // Three diff triggers (insert/update/delete) plus the tracking table + expect(active.triggers.length).toBe(3) + expect(active.tableExists).toBe(true) + + await collection.cleanup({ dependents: true }) + + // It seems that even though `cleanup` is async, the sync disposer cannot be async + // We wait for the table to be deleted + await vi.waitFor( + async () => { + const disposed = await readTrackingState(db, meta.trackedTableName) + expect(disposed.triggers.length).toBe(0) + expect(disposed.tableExists).toBe(false) + }, + { timeout: 1000 }, + ) + + // Nothing leaked into persisted storage either + const persisted = await db.getAll<{ name: string }>( + `SELECT name FROM sqlite_master WHERE name LIKE '%_tracking_%' OR name LIKE '__ps_temp_trigger_%'`, + ) + expect(persisted).toEqual([]) + }) + + it(`should reject managed tables that track metadata`, async () => { + const db = await createDatabase(MANAGED_METADATA_SCHEMA) + expect(() => + powerSyncCollectionOptions({ + database: db, + table: MANAGED_METADATA_SCHEMA.props.documents, + }), + ).toThrowError(/trackMetadata/) + }) + }) + + describe(`Mixed storage`, () => { + it(`should track managed tables and views in the same database`, async () => { + const db = await createDatabase(MIXED_SCHEMA) + + // Seeds run before the collections are created: the eager baseline + // load must observe them, as anything committed after it but before + // trigger creation can be missed. + await db.execute( + `INSERT INTO documents (id, name) VALUES (uuid(), 'doc-one')`, + ) + await db.execute( + `INSERT INTO users (id, name) VALUES (uuid(), 'user-one')`, + ) + + const documentsCollection = createCollection( + powerSyncCollectionOptions({ + database: db, + table: MIXED_SCHEMA.props.documents, + }), + ) + onTestFinished(() => documentsCollection.cleanup({ dependents: true })) + + const usersCollection = createCollection( + powerSyncCollectionOptions({ + database: db, + table: MIXED_SCHEMA.props.users, + }), + ) + onTestFinished(() => usersCollection.cleanup({ dependents: true })) + + await documentsCollection.stateWhenReady() + await usersCollection.stateWhenReady() + + expect(documentsCollection.size).toBe(1) + expect(usersCollection.size).toBe(1) + + // Both tracking paths observe new writes independently + await documentsCollection + .insert({ id: `doc-two`, name: `doc-two` }) + .when(`settled`) + await usersCollection + .insert({ id: `user-two`, name: `user-two` }) + .when(`settled`) + + expect(documentsCollection.size).toBe(2) + expect(usersCollection.size).toBe(2) + + const documentNames = await db.getAll<{ name: string }>( + `SELECT name FROM documents ORDER BY name`, + ) + const userNames = await db.getAll<{ name: string }>( + `SELECT name FROM users ORDER BY name`, + ) + expect(documentNames.map((row) => row.name)).toEqual([ + `doc-one`, + `doc-two`, + ]) + expect(userNames.map((row) => row.name)).toEqual([`user-one`, `user-two`]) + }) + + it(`should load migrated rows when reopening a view-stored database as managed tables`, async () => { + const dbFilename = `test-migration-${randomUUID()}.sqlite` + + // First run: the table is stored as a managed view. + const originalDb = new PowerSyncDatabase({ + database: { + dbFilename, + dbLocation: tmpdir(), + implementation: TEST_DATABASE_IMPLEMENTATION, + }, + schema: VIEW_STORAGE_SCHEMA, + }) + onTestFinished(async () => { + await originalDb.disconnectAndClear().catch(() => undefined) + await originalDb.close().catch(() => undefined) + }) + await originalDb.disconnectAndClear() + await originalDb.execute(` + INSERT into documents (id, name, author) + VALUES + ('migrated-1', 'first', 'a'), + ('migrated-2', 'second', 'b') + `) + await originalDb.close() + + // Second run: the schema marks the table as managed, so the SDK + // migrates the storage from the JSON view to a real table. + const migratedDb = new PowerSyncDatabase({ + database: { + dbFilename, + dbLocation: tmpdir(), + implementation: TEST_DATABASE_IMPLEMENTATION, + }, + schema: MANAGED_SCHEMA, + }) + onTestFinished(async () => { + await migratedDb.disconnectAndClear().catch(() => undefined) + await migratedDb.close().catch(() => undefined) + }) + // A clear would wipe the migrated rows, so the database is only awaited + // for readiness here. + await migratedDb.waitForReady() + + const storageType = await migratedDb.get<{ type: string }>( + `SELECT type FROM sqlite_master WHERE name = 'documents'`, + ) + expect(storageType.type).toBe(`table`) + + const collection = createCollection( + powerSyncCollectionOptions({ + database: migratedDb, + table: MANAGED_SCHEMA.props.documents, + }), + ) + onTestFinished(() => collection.cleanup({ dependents: true })) + await collection.stateWhenReady() + + // The adapter's triggers were recreated against the now-real table and + // the eager load read the migrated rows. + expect(collection.size).toBe(2) + const rows = [...collection.toArray].sort((a, b) => + a.id.localeCompare(b.id), + ) + expect(rows.map((row) => row.id)).toEqual([`migrated-1`, `migrated-2`]) + expect(rows.map((row) => row.author)).toEqual([`a`, `b`]) + + // The triggers track the migrated table + await migratedDb.execute( + `INSERT INTO documents (id, name, author) VALUES ('migrated-3', 'third', 'c')`, + ) + await vi.waitFor( + () => { + expect(collection.size).toBe(3) + }, + { timeout: 1000 }, + ) + }) + }) +}) diff --git a/packages/powersync-db-collection/tests/sqlite-compiler.test.ts b/packages/powersync-db-collection/tests/sqlite-compiler.test.ts index 59c7d5d81d..4ee48bc275 100644 --- a/packages/powersync-db-collection/tests/sqlite-compiler.test.ts +++ b/packages/powersync-db-collection/tests/sqlite-compiler.test.ts @@ -326,4 +326,26 @@ describe(`SQLite Compiler`, () => { expect(result.params).toEqual([]) }) }) + + describe(`managed table row aliases`, () => { + it(`should prefix column references with the row alias`, () => { + const result = compileSQLite( + { where: func(`eq`, [ref([`name`]), val(`test`)]) }, + { rowPrefix: 'NEW' }, + ) + + expect(result.where).toBe(`NEW."name" = ?`) + expect(result.params).toEqual([`test`]) + }) + + it(`should prefix the id column like any other real column`, () => { + const result = compileSQLite( + { where: func(`eq`, [ref([`id`]), val(`abc`)]) }, + { rowPrefix: 'OLD' }, + ) + + expect(result.where).toBe(`OLD."id" = ?`) + expect(result.params).toEqual([`abc`]) + }) + }) }) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b2008d5527..c6870e7aa5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1582,8 +1582,8 @@ importers: specifier: 2.5.0 version: 2.5.0 '@powersync/node': - specifier: 0.21.0 - version: 0.21.0(@powersync/common@2.5.0)(better-sqlite3@12.11.1) + specifier: ^1.3.0 + version: 1.3.0(@powersync/common@2.5.0)(better-sqlite3@12.11.1) '@types/debug': specifier: ^4.1.13 version: 4.1.13 @@ -5364,19 +5364,19 @@ packages: '@powersync/common@2.5.0': resolution: {integrity: sha512-kMlAfrQddGIAFuO7qCyYxf2yxAKdCrwGany9VcHs8Rx9f5KuqxqaCikzq/SC4Evf/f/sTS5eIwczZPPQboTjxw==} - '@powersync/node@0.21.0': - resolution: {integrity: sha512-mNDQ7dXmwdEf/jUXdN5IfFiI6hKczdoxqGzu3BKnuRhzeX2utWDZiW44Hnsihuqtryw1U2g5DATifQcx/gmlsg==} + '@powersync/node@1.3.0': + resolution: {integrity: sha512-04vhLTCHdj5WIwaVk6sGdPi1bVHDuiwXqWN3eQR1Qvqgp099RtCj1KXitwTbKWiqojQjxkwo+vK8c9xiHfcuWQ==} peerDependencies: - '@powersync/common': ^2.1.0 - better-sqlite3: 12.x + '@powersync/common': ^2.5.0 + better-sqlite3: 12.x || 13.x peerDependenciesMeta: better-sqlite3: optional: true - '@powersync/shared-internals@1.1.1': - resolution: {integrity: sha512-EiYxTbqbD6lfpDERLyECBsORDfrrgLSf4nmQ35Ug4cUNKFBwO4tGPZaquGeXbr4O+uJk0QFE/f6+33MLCDurIA==} + '@powersync/shared-internals@1.5.0': + resolution: {integrity: sha512-dIaMenW6gJNyKzD93d4X4CiBtJxyJ63pvK2CYGjqBtVqubGBt9d+jg6V6JTKw4VLPjFCZh7/zrXMV34wl7kzGw==} peerDependencies: - '@powersync/common': 2.0.0 + '@powersync/common': ^2.5.0 '@publint/pack@0.1.7': resolution: {integrity: sha512-4EDEmvxWtgsCnnVeBvtFIFZtUhPPt1+bA9JrSwU4Sa//6oKtzCSlGGXYJr44OD9aGISymbieJ4mCKHUygUDU+g==} @@ -17457,16 +17457,16 @@ snapshots: '@powersync/common@2.5.0': {} - '@powersync/node@0.21.0(@powersync/common@2.5.0)(better-sqlite3@12.11.1)': + '@powersync/node@1.3.0(@powersync/common@2.5.0)(better-sqlite3@12.11.1)': dependencies: '@powersync/common': 2.5.0 - '@powersync/shared-internals': 1.1.1(@powersync/common@2.5.0) + '@powersync/shared-internals': 1.5.0(@powersync/common@2.5.0) comlink: 4.4.2 - undici: 7.24.4 + undici: 7.30.1 optionalDependencies: better-sqlite3: 12.11.1 - '@powersync/shared-internals@1.1.1(@powersync/common@2.5.0)': + '@powersync/shared-internals@1.5.0(@powersync/common@2.5.0)': dependencies: '@powersync/common': 2.5.0