diff --git a/.changeset/powersync-numeric-keys.md b/.changeset/powersync-numeric-keys.md new file mode 100644 index 0000000000..7c98778f90 --- /dev/null +++ b/.changeset/powersync-numeric-keys.md @@ -0,0 +1,5 @@ +--- +'@tanstack/powersync-db-collection': patch +--- + +Support numeric collection keys. Set `numericIds: true` and have `deserializationSchema` convert `id` to a number, for a backend with integer primary keys; the collection key is then the deserialized number. On-demand loads, change tracking, and eviction compare `id` as a number, and local updates and deletes use the key. TypeScript requires `numericIds` exactly when the schema outputs a number `id`. Releasing an on-demand subset no longer revalidates its rows, so a row that fails validation no longer blocks the release. diff --git a/docs/collections/powersync-collection.md b/docs/collections/powersync-collection.md index 472a7e6926..a3a04b73c8 100644 --- a/docs/collections/powersync-collection.md +++ b/docs/collections/powersync-collection.md @@ -399,6 +399,42 @@ const task = tasksCollection.get("task-1") task.due_date.getTime() // OK - TypeScript knows this is a Date ``` +### Integer Primary Keys + +SQLite stores every PowerSync `id` as text. If your backend uses integer primary keys, convert `id` to a number in `deserializationSchema` and set `numericIds: true`. The collection key is then a number, so integer foreign keys match it, for example in `eq(part.id, line.partId)`. Predicates on `id`, such as `gt(product.id, 2)`, compare numbers in on-demand loads, change tracking, and eviction. + +```typescript +const APP_SCHEMA = new Schema({ + products: new Table({ + name: column.text, + }), +}) + +const productSchema = z.object({ + id: z.number(), + name: z.string().nullable(), +}) + +const products = createCollection( + powerSyncCollectionOptions({ + database: db, + table: APP_SCHEMA.props.products, + schema: productSchema, + deserializationSchema: productSchema.extend({ + id: z.coerce.number(), // SQLite TEXT '5' to the key 5 + }), + numericIds: true, + onDeserializationError: (error) => { + // Present fatal error + }, + }) +) + +products.get(5) +``` + +TypeScript requires `numericIds: true` when the schema outputs a number `id`, and rejects it otherwise. An `id` that may be either a string or a number is not supported. Equality and `in` predicates on a numeric `id` use SQLite's `id` index; range predicates such as `gt` scan the table. Numeric IDs must be safe JavaScript integers, and converting the number back to text must reproduce the stored ID exactly. Keep IDs as strings if they can exceed the safe integer range or contain leading zeros. Updates and deletes use the text form of the key to find the SQLite row. + ### Optimistic Updates Updates to the collection are applied optimistically to the local state first, then synchronized with PowerSync and the backend. If an error occurs during sync, the changes are automatically rolled back. @@ -526,6 +562,9 @@ interface PowerSyncCollectionConfig { [Key in keyof TOutput]?: (value: TOutput[Key]) => SQLiteCompatibleType } + // Integer primary keys: required when the schema outputs a number id + numericIds?: boolean + // Performance tuning syncBatchSize?: number // Control batch size for initial sync, defaults to 1000 } diff --git a/packages/powersync-db-collection/src/PowerSyncTransactor.ts b/packages/powersync-db-collection/src/PowerSyncTransactor.ts index d78b0a5c54..c1f1f8c628 100644 --- a/packages/powersync-db-collection/src/PowerSyncTransactor.ts +++ b/packages/powersync-db-collection/src/PowerSyncTransactor.ts @@ -2,7 +2,7 @@ import { LogLevels, sanitizeSQL } from '@powersync/common' import { LoadSubsetOperationAbortedError, isFailedStatus } from '@tanstack/db' import DebugModule from 'debug' import { PendingOperationStore } from './PendingOperationStore' -import { asPowerSyncRecord, mapOperationToPowerSync } from './helpers' +import { mapOperationToPowerSync } from './helpers' import type { CommonPowerSyncDatabase, LockContext } from '@powersync/common' import type { PendingMutation, Transaction } from '@tanstack/db' import type { PendingOperation } from './PendingOperationStore' @@ -247,7 +247,7 @@ export class PowerSyncTransactor { SET ${keys.map((key) => `${key} = ?`).join(`, `)} WHERE id = ? `, - [...queryParameters, asPowerSyncRecord(mutation.original).id], + [...queryParameters, String(mutation.key)], ) return @@ -278,14 +278,14 @@ export class PowerSyncTransactor { ` UPDATE ${tableName} SET _deleted = TRUE, _metadata = ? WHERE id = ? `, - [metadataValue, asPowerSyncRecord(mutation.original).id], + [metadataValue, String(mutation.key)], ) } else { await context.execute( ` DELETE FROM ${tableName} WHERE id = ? `, - [asPowerSyncRecord(mutation.original).id], + [String(mutation.key)], ) } }, diff --git a/packages/powersync-db-collection/src/definitions.ts b/packages/powersync-db-collection/src/definitions.ts index b57253a485..d05bd43c36 100644 --- a/packages/powersync-db-collection/src/definitions.ts +++ b/packages/powersync-db-collection/src/definitions.ts @@ -26,6 +26,30 @@ export type InferPowerSyncOutputType< ? ExtractedTable : InferSchemaOutput +/** + * The collection key: the type of the output row's `id`. It is a string + * unless a schema converts `id`, for example to a number. + */ +export type InferPowerSyncKey = TOutput extends { id: infer TId } + ? TId extends number + ? number + : string + : string + +/** + * SQLite stores every PowerSync `id` as text. Set `numericIds: true` when the + * schema converts `id` to a number, as for a backend with integer primary + * keys. Predicates on `id` then compare numbers, and the collection key is + * the number. The option is required exactly when the output `id` is a number. + * An `id` that may be a string or a number fits neither mode and is rejected. + */ +export type NumericIdsConfig = + InferPowerSyncKey extends number + ? { numericIds: true } + : InferPowerSyncKey extends string + ? { numericIds?: false } + : { numericIds: never } + /** * A mapping type for custom serialization of object properties to SQLite-compatible values. * @@ -208,7 +232,7 @@ export type BasePowerSyncCollectionConfig< > = Omit< BaseCollectionConfig< InferPowerSyncOutputType, - string, + InferPowerSyncKey>, TSchema >, `onInsert` | `onUpdate` | `onDelete` | `getKey` | `syncMode` @@ -230,7 +254,8 @@ export type BasePowerSyncCollectionConfig< * streaming of initial results, at the cost of more query calls. */ syncBatchSize?: number -} & (EagerSyncHooks | OnDemandSyncHooks) +} & NumericIdsConfig> & + (EagerSyncHooks | OnDemandSyncHooks) /** * Configuration interface for PowerSync collection options. @@ -306,7 +331,7 @@ export type EnhancedPowerSyncCollectionConfig< TSchema extends StandardSchemaV1 = never, > = CollectionConfig< OutputType, - string, + InferPowerSyncKey, TSchema, PowerSyncCollectionUtils > & { diff --git a/packages/powersync-db-collection/src/helpers.ts b/packages/powersync-db-collection/src/helpers.ts index 7bbb8e8b67..8d94823122 100644 --- a/packages/powersync-db-collection/src/helpers.ts +++ b/packages/powersync-db-collection/src/helpers.ts @@ -2,10 +2,12 @@ import { DiffTriggerOperation } from '@powersync/common' import type { ExtractColumnValueType, Table } from '@powersync/common' /** - * All PowerSync table records include a UUID `id` column. + * All PowerSync table records include an `id` column. SQLite stores it as + * text; a deserialization schema may convert it to a number, as for a + * backend with integer primary keys. */ export type PowerSyncRecord = { - id: string + id: string | number [key: string]: unknown } @@ -64,14 +66,7 @@ export type OptionalExtractedTable = OptionalIfUndefined<{ */ export type AnyTableColumnType = { [K in keyof ExtractedTableColumns]: unknown -} & { id: string } - -export function asPowerSyncRecord(record: any): PowerSyncRecord { - if (typeof record.id !== `string`) { - throw new Error(`Record must have a string id field`) - } - return record as PowerSyncRecord -} +} & { id: string | number } /** * Maps {@link DiffTriggerOperation} to TanstackDB operations diff --git a/packages/powersync-db-collection/src/powersync.ts b/packages/powersync-db-collection/src/powersync.ts index 2c557ccde7..42339c072b 100644 --- a/packages/powersync-db-collection/src/powersync.ts +++ b/packages/powersync-db-collection/src/powersync.ts @@ -9,7 +9,7 @@ import { compileSQLite } from './sqlite-compiler' import { PendingOperationStore } from './PendingOperationStore' import { PowerSyncTransactor } from './PowerSyncTransactor' import { DEFAULT_BATCH_SIZE } from './definitions' -import { asPowerSyncRecord, mapOperation } from './helpers' +import { mapOperation } from './helpers' import { convertTableToSchema } from './schema' import { serializeForSQLite } from './serialization' import type { @@ -30,6 +30,7 @@ import type { ConfigWithSQLiteInputType, ConfigWithSQLiteTypes, EnhancedPowerSyncCollectionConfig, + InferPowerSyncKey, InferPowerSyncOutputType, PowerSyncCollectionConfig, PowerSyncCollectionUtils, @@ -256,6 +257,7 @@ function createPowerSyncCollectionConfig< schema: inputSchema, syncBatchSize = DEFAULT_BATCH_SIZE, syncMode = 'eager', + numericIds = false, ...restConfig } = config @@ -272,6 +274,7 @@ function createPowerSyncCollectionConfig< // The collection output type type OutputType = InferPowerSyncOutputType + type PowerSyncKey = InferPowerSyncKey const { viewName, trackMetadata: metadataIsTracked } = table @@ -282,6 +285,18 @@ function createPowerSyncCollectionConfig< const validationSchema = deserializationSchema || schema const validation = validationSchema[`~standard`].validate(value) if (`value` in validation) { + // A numeric key must name exactly one stored id, so eviction and + // writes can derive one from the other. + const id = (validation.value as { id?: unknown }).id + if ( + numericIds && + !(Number.isSafeInteger(id) && String(id) === value.id) + ) { + const message = `A numeric PowerSync id must round-trip as a canonical safe integer` + database.logger.log({ level: LogLevels.error, message }) + onDeserializationError?.({ issues: [{ path: [`id`], message }] }) + throw new Error(message) + } return validation.value } else if (`issues` in validation) { const issueMessage = `Failed to validate incoming data for ${viewName}. Issues: ${validation.issues.map((issue) => `${issue.path} - ${issue.message}`)}` @@ -328,7 +343,7 @@ function createPowerSyncCollectionConfig< * Notice that this describes the Sync between the local SQLite table * and the in-memory tanstack-db collection. */ - const sync: SyncConfig = { + const sync: SyncConfig = { sync: (params) => { const persistence = params.metadata === undefined @@ -442,9 +457,9 @@ function createPowerSyncCollectionConfig< // holds. snapshot?: boolean afterRead?: () => Promise - onInitialRow?: (value: OutputType) => void + onInitialRow?: (key: PowerSyncKey) => void when: Record - writeType: (rowId: string) => OperationType + writeType: (key: PowerSyncKey) => OperationType batchQuery: ( lockContext: LockContext, batchSize: number, @@ -473,16 +488,21 @@ function createPowerSyncCollectionConfig< beforeCreate: async (context) => { // A retained reload uses one snapshot across its source reads. // Other initial loads and subset loads keep batch commits. A - // transaction begins only once its rows are read, so a batch - // that fails leaves none open. + // transaction begins only once its rows are read, deserialized, + // and keyed, so a batch that fails leaves none open. const batches: Array> = [] const apply = () => { + const entries = batches + .splice(0) + .flat() + .map((row) => { + const value = deserializeSyncRow(row) + return [getKey(value), value] as const + }) begin({ snapshot }) - for (const row of batches.splice(0).flat()) { - const type = writeType(row.id) - const value = deserializeSyncRow(row) - onInitialRow?.(value) - write({ type, value }) + for (const [key, value] of entries) { + onInitialRow?.(key) + write({ type: writeType(key), value }) } const applied = commit() if (applied !== true) void applied.catch(() => undefined) @@ -549,18 +569,20 @@ function createPowerSyncCollectionConfig< } try { - begin() const operations = await context.getAll( `SELECT * FROM ${trackedTableName} ORDER BY operation_id ASC`, ) const pendingOperations: Array = [] - for (const op of operations) { + const changes = operations.map((op): Parameters[0] => { const { id, operation, timestamp, value } = op const parsedValue = deserializeSyncRow({ id, ...JSON.parse(value), }) + // write() derives the key inside the transaction; a key getKey + // rejects must fail here, before the transaction opens. + getKey(parsedValue) const parsedPreviousValue = op.operation == DiffTriggerOperation.UPDATE ? deserializeSyncRow({ @@ -568,18 +590,20 @@ function createPowerSyncCollectionConfig< ...JSON.parse(op.previous_value), }) : undefined - write({ - type: mapOperation(operation), - value: parsedValue, - previousValue: parsedPreviousValue, - }) pendingOperations.push({ id, operation, timestamp, tableName: viewName, }) - } + return { + type: mapOperation(operation), + value: parsedValue, + previousValue: parsedPreviousValue, + } + }) + begin() + for (const change of changes) write(change) // clear the current operations await context.execute(`DELETE FROM ${trackedTableName}`) @@ -650,7 +674,9 @@ function createPowerSyncCollectionConfig< const appliedReceipts: Array = [] const listed = - persistence && !replacesRetainedRows ? new Set() : undefined + persistence && !replacesRetainedRows + ? new Set() + : undefined await establishTracking( { // Retained reloads replace atomically; fresh persisted startup @@ -667,7 +693,7 @@ function createPowerSyncCollectionConfig< // Copy omitted keys before committing: a synchronous // commit can change the accepted-row iterator's backing map. // Source rows remain batched; only omitted keys are retained. - const gone: Array = [] + const gone: Array = [] for (const [ key, ] of collection._state.acceptedSyncedEntries()) { @@ -698,11 +724,11 @@ function createPowerSyncCollectionConfig< [DiffTriggerOperation.DELETE]: `TRUE`, }, onInitialRow: listed - ? (value) => { - listed.add(asPowerSyncRecord(value).id) + ? (key) => { + listed.add(key) } : undefined, - writeType: (_rowId: string) => `insert`, + writeType: () => `insert`, batchQuery: ( lockContext: LockContext, batchSize: number, @@ -863,18 +889,15 @@ function createPowerSyncCollectionConfig< active.length === 1 ? active[0] : or(active[0], active[1], ...active.slice(2)) - const compiledNewData = compileSQLite( - { where: combinedWhere }, - { jsonColumn: 'NEW.data' }, + const newDataWhenClause = compileWhere( + combinedWhere, + 'NEW.data', ) - const compiledOldData = compileSQLite( - { where: combinedWhere }, - { jsonColumn: 'OLD.data' }, + const oldDataWhenClause = compileWhere( + combinedWhere, + 'OLD.data', ) - const compiledView = compileSQLite({ where: combinedWhere }) - const newDataWhenClause = toInlinedWhereClause(compiledNewData) - const oldDataWhenClause = toInlinedWhereClause(compiledOldData) - const viewWhereClause = toInlinedWhereClause(compiledView) + const viewWhereClause = compileWhere(combinedWhere) await establishTracking( { setupContext: ctx, @@ -883,22 +906,19 @@ function createPowerSyncCollectionConfig< [DiffTriggerOperation.UPDATE]: `(${newDataWhenClause}) OR (${oldDataWhenClause})`, [DiffTriggerOperation.DELETE]: oldDataWhenClause, }, - writeType: (rowId: string) => - collection.has(rowId) ? `update` : `insert`, + writeType: (key) => { + missing?.delete(key) + return collection.has(key) ? `update` : `insert` + }, batchQuery: ( lockContext: LockContext, batchSize: number, cursor: number, ) => - lockContext - .getAll( - `SELECT * FROM ${viewName} WHERE ${viewWhereClause} LIMIT ? OFFSET ?`, - [batchSize, cursor], - ) - .then((rows) => { - for (const row of rows) missing?.delete(row.id) - return rows - }), + lockContext.getAll( + `SELECT * FROM ${viewName} WHERE ${viewWhereClause} LIMIT ? OFFSET ?`, + [batchSize, cursor], + ), }, appliedReceipts, ) @@ -1004,10 +1024,16 @@ function createPowerSyncCollectionConfig< return task } - const toInlinedWhereClause = (compiled: { - where?: string - params: Array - }): string => { + // Loading, tracking, and eviction all compile predicates here, so + // they compare `id` the same way. + const compileWhere = ( + where: LoadSubsetOptions[`where`], + jsonColumn?: string, + ): string => { + const compiled = compileSQLite( + { where }, + { jsonColumn, numericId: numericIds }, + ) if (!compiled.where) return 'TRUE' const sqlParts = compiled.where.split('?') return sanitizeSQL( @@ -1080,8 +1106,7 @@ function createPowerSyncCollectionConfig< const performPhysicalRelease = async ( options: LoadSubsetOptions, ): Promise => { - const compiledDeparting = compileSQLite({ where: options.where }) - const departingWhereSQL = toInlinedWhereClause(compiledDeparting) + const departingWhereSQL = compileWhere(options.where) let rowsToEvict: Array<{ id: string }> for (;;) { if (stopped) return @@ -1095,10 +1120,7 @@ function createPowerSyncCollectionConfig< active.length === 1 ? active[0]! : or(active[0], active[1], ...active.slice(2)) - const compiledRemaining = compileSQLite({ - where: combinedRemaining, - }) - const remainingWhereSQL = toInlinedWhereClause(compiledRemaining) + const remainingWhereSQL = compileWhere(combinedRemaining) evictionSQL = `SELECT id FROM ${viewName} WHERE (${departingWhereSQL}) AND NOT (${remainingWhereSQL})` } @@ -1107,11 +1129,16 @@ function createPowerSyncCollectionConfig< if (stopped) return if (trackingRevision === revision) break } - if (rowsToEvict.length > 0) { + // Eviction does not revalidate rows. A loaded row's key is its stored + // id, or with numericIds the number that round-trips to it; a row + // with any other id was never loaded. + const keys = rowsToEvict.flatMap(({ id }) => { + const key = numericIds ? Number(id) : id + return String(key) === id ? [key as PowerSyncKey] : [] + }) + if (keys.length > 0) { begin() - for (const { id } of rowsToEvict) { - write({ type: `delete`, key: id }) - } + for (const key of keys) write({ type: `delete`, key }) void commit() } await rebuildTracking() @@ -1251,7 +1278,17 @@ function createPowerSyncCollectionConfig< getSyncMetadata: undefined, } - const getKey = (record: OutputType) => asPowerSyncRecord(record).id + const getKey = (record: OutputType): PowerSyncKey => { + const { id } = record as { id?: unknown } + if (numericIds ? !Number.isSafeInteger(id) : typeof id !== `string`) { + throw new Error( + numericIds + ? `With numericIds, a record id must be a safe integer` + : `Record must have a string id field; set numericIds for number ids`, + ) + } + return id as PowerSyncKey + } // Keep adapter hooks and serialization options out of the core config. const { diff --git a/packages/powersync-db-collection/src/sqlite-compiler.ts b/packages/powersync-db-collection/src/sqlite-compiler.ts index d5d0274965..a07fc969db 100644 --- a/packages/powersync-db-collection/src/sqlite-compiler.ts +++ b/packages/powersync-db-collection/src/sqlite-compiler.ts @@ -1,4 +1,5 @@ -import type { IR, LoadSubsetOptions } from '@tanstack/db' +import { IR } from '@tanstack/db' +import type { LoadSubsetOptions } from '@tanstack/db' /** * Result of compiling LoadSubsetOptions to SQLite @@ -24,6 +25,11 @@ export interface CompileSQLiteOptions { * as a direct column in the tracked table. */ jsonColumn?: string + /** + * When set, `id` references compile to `CAST( AS INTEGER)`, so a + * collection with numeric ids compares them as numbers, not as text. + */ + numericId?: boolean } /** @@ -100,10 +106,14 @@ function compileExpression( // JSON `data` object, so we skip json_extract. However, when compiling for // trigger WHEN clauses we still need the OLD./NEW. prefix. Extract it from // the jsonColumn option. - if (compileOptions?.jsonColumn && columnName === `id`) { - const prefix = compileOptions.jsonColumn.split('.')[0]! - return `${prefix}.${quoteIdentifier(columnName)}` - } else if (compileOptions?.jsonColumn) { + if (columnName === `id`) { + const prefix = compileOptions?.jsonColumn?.split('.')[0] + const id = prefix + ? `${prefix}.${quoteIdentifier(columnName)}` + : quoteIdentifier(columnName) + return compileOptions?.numericId ? `CAST(${id} AS INTEGER)` : id + } + if (compileOptions?.jsonColumn) { return `json_extract(${compileOptions.jsonColumn}, '$.${columnName}')` } return quoteIdentifier(columnName) @@ -183,9 +193,35 @@ function compileFunction( } } + // A numeric id equals a number exactly when its stored text is the number's + // canonical form, which every loaded numeric id is. Comparing that text, + // instead of CAST(id), lets SQLite look the id up in its primary key index. + const [column, value] = args + const idText = + compileOptions?.numericId && + (name === `eq` || name === `in`) && + column?.type === `ref` && + column.path.join() === `id` && + value?.type === `val` + // Compile arguments - const compiledArgs = args.map((arg: IR.BasicExpression) => - compileExpression(arg, params, compileOptions), + const compiledArgs = ( + idText + ? [ + column, + new IR.Value( + Array.isArray(value.value) + ? value.value.map(String) + : String(value.value), + ), + ] + : args + ).map((arg: IR.BasicExpression) => + compileExpression( + arg, + params, + idText ? { ...compileOptions, numericId: false } : compileOptions, + ), ) // Handle different operator types diff --git a/packages/powersync-db-collection/tests/numeric-keys.test.ts b/packages/powersync-db-collection/tests/numeric-keys.test.ts new file mode 100644 index 0000000000..7d946d7d56 --- /dev/null +++ b/packages/powersync-db-collection/tests/numeric-keys.test.ts @@ -0,0 +1,376 @@ +import { randomUUID } from 'node:crypto' +import { tmpdir } from 'node:os' +import { + LogLevels, + PowerSyncDatabase, + Schema, + Table, + column, +} from '@powersync/node' +import { IR, createCollection } from '@tanstack/db' +import { describe, expect, it, onTestFinished, vi } from 'vitest' +import { z } from 'zod' +import { compileSQLite, powerSyncCollectionOptions } from '../src' +import type { PowerSyncLogger } from '@powersync/node' + +// SQLite stores every PowerSync id as text. A backend with integer primary +// keys sets `numericIds` and converts `id` to a number in +// `deserializationSchema`, so the collection key is the deserialized number +// and integer foreign keys match it. Every path that writes a key must use +// that number, and every path that filters by `id` must compare numbers. + +const APP_SCHEMA = new Schema({ + products: new Table({ name: column.text, category: column.text }), +}) + +const row = z.object({ + id: z.number(), + name: z.string().nullable(), + category: z.string().nullable(), +}) +// Rows read from SQLite carry the id as text. +const sqliteRow = row.extend({ id: z.coerce.number() }) + +const categoryEquals = (category: string) => + new IR.Func(`eq`, [ + new IR.PropRef([`category`]), + new IR.Value(category), + ]) + +const idGreaterThan = (id: number) => + new IR.Func(`gt`, [new IR.PropRef([`id`]), new IR.Value(id)]) +const idEquals = (id: number) => + new IR.Func(`eq`, [new IR.PropRef([`id`]), new IR.Value(id)]) +const idIn = (ids: Array) => + new IR.Func(`in`, [new IR.PropRef([`id`]), new IR.Value(ids)]) + +async function createDatabase() { + const errors: Array = [] + const logger: PowerSyncLogger = { + log({ level, message }) { + if (level >= LogLevels.error) errors.push(message) + }, + } + const db = new PowerSyncDatabase({ + database: { + dbFilename: `test-numeric-keys-${randomUUID()}.sqlite`, + dbLocation: tmpdir(), + implementation: { type: `node:sqlite` }, + }, + schema: APP_SCHEMA, + logger, + }) + onTestFinished(async () => { + await new Promise((resolve) => setTimeout(resolve, 100)) + await db.disconnectAndClear() + await db.close() + }) + await db.disconnectAndClear() + await db.execute( + `INSERT INTO products (id, name, category) VALUES ('1', 'A', 'x'), ('2', 'B', 'x'), ('3', 'C', 'y')`, + ) + return { db, errors } +} + +function createNumericCollection( + db: PowerSyncDatabase, + syncMode: `eager` | `on-demand`, +) { + const collection = createCollection( + powerSyncCollectionOptions({ + database: db, + table: APP_SCHEMA.props.products, + schema: row, + deserializationSchema: sqliteRow, + onDeserializationError: () => {}, + numericIds: true, + syncMode, + }), + ) + onTestFinished(() => collection.cleanup()) + return collection +} + +describe(`numeric collection keys`, () => { + it.each([`007`, `9007199254740993`])( + `rejects a stored ID that cannot round-trip as a numeric key (%s)`, + async (id) => { + const { db } = await createDatabase() + await db.execute( + `INSERT INTO products (id, name, category) VALUES (?, 'D', 'invalid')`, + [id], + ) + const collection = createNumericCollection(db, `on-demand`) + await expect( + Promise.resolve( + collection._sync.loadSubset({ where: categoryEquals(`invalid`) }), + ), + ).rejects.toThrow(/canonical safe integer/) + expect(collection.size).toBe(0) + }, + ) + + it(`syncs and writes rows keyed by a deserialized number`, async () => { + const { db, errors } = await createDatabase() + const collection = createNumericCollection(db, `eager`) + await collection.stateWhenReady() + expect([...collection.keys()].sort()).toEqual([1, 2, 3]) + + await db.execute(`UPDATE products SET name = 'A2' WHERE id = '1'`) + await db.execute(`DELETE FROM products WHERE id = '3'`) + await vi.waitFor(() => { + expect(collection.get(1)?.name).toBe(`A2`) + expect(collection.has(3)).toBe(false) + }) + + await collection + .update(1, (draft) => { + draft.name = `A3` + }) + .when(`settled`) + await collection.delete(2).when(`settled`) + expect(await db.getAll(`SELECT id, name FROM products`)).toEqual([ + { id: `1`, name: `A3` }, + ]) + expect(errors).toEqual([]) + }) + + it(`reloads held rows as updates and evicts them by numeric key`, async () => { + const { db, errors } = await createDatabase() + const collection = createNumericCollection(db, `on-demand`) + const first = { where: categoryEquals(`x`) } + await collection._sync.loadSubset(first) + expect([...collection.keys()].sort()).toEqual([1, 2]) + + // Tracking for the second load re-reads every active predicate, so it + // re-reads the held rows: each must be an update, not an insert. + const second = { where: categoryEquals(`y`) } + await collection._sync.loadSubset(second) + expect([...collection.keys()].sort()).toEqual([1, 2, 3]) + + collection._sync.unloadSubset(second) + await vi.waitFor(() => + expect([...collection.keys()].sort()).toEqual([1, 2]), + ) + collection._sync.unloadSubset(first) + await vi.waitFor(() => expect(collection.size).toBe(0)) + expect(errors).toEqual([]) + }) + + // As text, '10' < '2', and a stored text id is never equal to a number. + // The subset load, the trigger that tracks later writes, and eviction must + // all compare ids as numbers. Each predicate below matches 3, 10, and 100 + // among the ids 1, 2, 3, 10, then 0 and 100. + it.each([ + [`gt(id, 2)`, idGreaterThan(2), [3, 10]], + [`in(id, [3, 10, 100])`, idIn([3, 10, 100]), [3, 10]], + [`eq(id, 100)`, idEquals(100), []], + ])( + `compares ids as numbers when loading, tracking, and evicting: %s`, + async (_, where, loaded) => { + const { db, errors } = await createDatabase() + await db.execute( + `INSERT INTO products (id, name, category) VALUES ('10', 'E', 'z')`, + ) + const collection = createNumericCollection(db, `on-demand`) + const subset = { where } + await collection._sync.loadSubset(subset) + expect([...collection.keys()].sort((a, b) => a - b)).toEqual(loaded) + + // One write, so once 100 arrives the tracking has judged 0 as well. + await db.execute( + `INSERT INTO products (id, name, category) VALUES ('0', 'F', 'z'), ('100', 'G', 'z')`, + ) + await vi.waitFor(() => expect(collection.has(100)).toBe(true)) + expect(collection.has(0)).toBe(false) + + collection._sync.unloadSubset(subset) + await vi.waitFor(() => expect(collection.size).toBe(0)) + expect(errors).toEqual([]) + }, + ) + + // `id` is SQLite's text primary key. Equality on a numeric id compares the + // stored text, so a load by id, such as a join's `in`, uses the index. + // A numeric range needs the CAST and scans. + it.each([ + [`eq(id, 5)`, idEquals(5)], + [`in(id, [1, 2])`, idIn([1, 2])], + ])( + `looks up numeric id equality through the id index: %s`, + async (_, where) => { + const { db } = await createDatabase() + const compiled = compileSQLite({ where }, { numericId: true }) + const plan = await db.getAll<{ detail: string }>( + `EXPLAIN QUERY PLAN SELECT * FROM products WHERE ${compiled.where}`, + compiled.params, + ) + expect(plan.map(({ detail }) => detail).join(` | `)).toMatch( + /USING INDEX/, + ) + }, + ) + + it(`keeps held rows after a tracking outage`, async () => { + vi.useFakeTimers() + const { db } = await createDatabase() + const collection = createNumericCollection(db, `on-demand`) + const first = { where: categoryEquals(`y`) } + const second = { where: categoryEquals(`x`) } + try { + await collection._sync.loadSubset(first) + await collection._sync.loadSubset(second) + const trigger = vi + .spyOn(db.triggers, `createDiffTrigger`) + .mockRejectedValueOnce(new Error(`release rebuild failed`)) + collection._sync.unloadSubset(first) + await vi.waitFor(() => + expect(collection.health.failureReason?.message).toBe( + `release rebuild failed`, + ), + ) + await vi.advanceTimersByTimeAsync(1_000) + await vi.waitFor(() => + expect(trigger.mock.calls.length).toBeGreaterThan(1), + ) + await vi.waitFor(() => expect(collection.health.hasErrors).toBe(false)) + // The rebuild reads the rows still in SQLite; a row it reads is kept. + expect([...collection.keys()].sort()).toEqual([1, 2]) + } finally { + await collection.cleanup() + await vi.runOnlyPendingTimersAsync() + vi.useRealTimers() + } + }) +}) + +// A load deserializes every row, and deserialization can fail. A sync +// transaction opens only after every row is deserialized, so a failing row +// leaves no open transaction behind. A release does not deserialize: it +// derives each key from the stored id, so a row that fails validation cannot +// keep the release from evicting the rows it loaded. +describe(`a row that fails deserialization`, () => { + // A row fails validation while its stored id is in `invalid`. + function createGatedCollection( + db: PowerSyncDatabase, + ids: `numeric` | `string`, + ) { + const invalid = new Set() + const valid = (id: string | number) => !invalid.has(String(id)) + const base = { + database: db, + table: APP_SCHEMA.props.products, + onDeserializationError: () => {}, + syncMode: `on-demand` as const, + } + const numeric = row.extend({ id: z.coerce.number().refine(valid) }) + const text = row.extend({ id: z.string().refine(valid) }) + const collection = + ids === `numeric` + ? createCollection( + powerSyncCollectionOptions({ + ...base, + schema: row, + deserializationSchema: numeric, + numericIds: true, + }), + ) + : createCollection( + powerSyncCollectionOptions({ + ...base, + schema: text, + deserializationSchema: text, + }), + ) + onTestFinished(() => collection.cleanup()) + const openTransactions = () => + collection._state.pendingSyncedTransactions.filter( + (transaction) => !transaction.committed, + ).length + return { collection, invalid, openTransactions } + } + + it(`fails a load without leaving a sync transaction open`, async () => { + const { db } = await createDatabase() + const { collection, invalid, openTransactions } = createGatedCollection( + db, + `numeric`, + ) + invalid.add(`2`) + await expect( + Promise.resolve( + collection._sync.loadSubset({ where: categoryEquals(`x`) }), + ), + ).rejects.toThrow() + expect(openTransactions()).toBe(0) + expect(collection.size).toBe(0) + + invalid.clear() + await collection._sync.loadSubset({ where: categoryEquals(`x`) }) + expect(collection.size).toBe(2) + }) + + // A JavaScript caller can omit numericIds for a schema that outputs number + // ids. Each row then deserializes but getKey rejects its key. The key is + // derived before a sync transaction opens, on the load path and on the path + // that syncs tracked writes. + it(`rejects a key without leaving a sync transaction open`, async () => { + const { db, errors } = await createDatabase() + const options = { + database: db, + table: APP_SCHEMA.props.products, + schema: row, + deserializationSchema: sqliteRow, + onDeserializationError: () => {}, + syncMode: `on-demand` as const, + } + // @ts-expect-error a number id needs numericIds + const config = powerSyncCollectionOptions(options) + const collection = createCollection(config) + onTestFinished(() => collection.cleanup()) + const openTransactions = () => + collection._state.pendingSyncedTransactions.filter( + (transaction) => !transaction.committed, + ).length + + // A tracked write. The subset starts empty, so its load succeeds. + await collection._sync.loadSubset({ where: categoryEquals(`new`) }) + await db.execute( + `INSERT INTO products (id, name, category) VALUES ('4', 'D', 'new')`, + ) + await vi.waitFor(() => + expect(errors).toContain( + `An error has been detected in the sync handler`, + ), + ) + expect(openTransactions()).toBe(0) + + // A load of existing rows. + await expect( + Promise.resolve( + collection._sync.loadSubset({ where: categoryEquals(`x`) }), + ), + ).rejects.toThrow(/set numericIds/) + expect(openTransactions()).toBe(0) + }) + + it.each([`numeric`, `string`] as const)( + `releases a subset that holds a permanently invalid row (%s ids)`, + async (ids) => { + const { db, errors } = await createDatabase() + const { collection, invalid, openTransactions } = createGatedCollection( + db, + ids, + ) + const subset = { where: categoryEquals(`x`) } + await collection._sync.loadSubset(subset) + expect(collection.size).toBe(2) + + invalid.add(`2`) + collection._sync.unloadSubset(subset) + await vi.waitFor(() => expect(collection.size).toBe(0)) + expect(openTransactions()).toBe(0) + expect(errors).toEqual([]) + }, + ) +}) diff --git a/packages/powersync-db-collection/tests/powersync.test-d.ts b/packages/powersync-db-collection/tests/powersync.test-d.ts index 4e481946be..838a677fd2 100644 --- a/packages/powersync-db-collection/tests/powersync.test-d.ts +++ b/packages/powersync-db-collection/tests/powersync.test-d.ts @@ -90,4 +90,47 @@ describe(`PowerSync collection type tests`, () => { }, }) }) + + it(`requires numericIds exactly when the schema outputs a number id`, () => { + const schema = z.object({ + id: z.number(), + name: z.string().nullable(), + author: z.string().nullable(), + created_at: z.string().nullable(), + }) + const base = { + database: {} as PowerSyncDatabase, + table: APP_SCHEMA.props.documents, + schema, + deserializationSchema: schema.extend({ id: z.coerce.number() }), + onDeserializationError: () => {}, + } + + const options = powerSyncCollectionOptions({ ...base, numericIds: true }) + expectTypeOf(createCollection(options).get) + .parameter(0) + .toEqualTypeOf() + + // @ts-expect-error a number id needs numericIds + powerSyncCollectionOptions(base) + + powerSyncCollectionOptions({ + database: {} as PowerSyncDatabase, + table: APP_SCHEMA.props.documents, + // @ts-expect-error a text id cannot use numericIds + numericIds: true, + }) + + // An id that may be either type fits neither mode. + const mixed = schema.extend({ id: z.union([z.string(), z.number()]) }) + const mixedBase = { + ...base, + schema: mixed, + deserializationSchema: mixed.extend({ id: z.string() }), + } + // @ts-expect-error a mixed id cannot omit numericIds + powerSyncCollectionOptions(mixedBase) + // @ts-expect-error a mixed id cannot use numericIds + powerSyncCollectionOptions({ ...mixedBase, numericIds: true }) + }) })