From 95ae1795877fdf1c24199c151e5a181aecf1e472 Mon Sep 17 00:00:00 2001 From: Jaime Resano Date: Tue, 10 Nov 2026 00:12:08 +0100 Subject: [PATCH 1/6] feat(powersync-db-collection): support numeric collection keys A deserializationSchema may convert `id` to a number, for a backend with integer primary keys. The collection key is now the deserialized row's id, and every path that writes a key derives it from the deserialized row: initial and on-demand loads, the reload's update/insert choice, the tracking-outage sweep, and unload eviction. Local updates and deletes write the key back to SQLite as text. Co-Authored-By: Claude Opus 5.5 --- .changeset/powersync-numeric-keys.md | 5 + docs/collections/powersync-collection.md | 27 +++ .../src/PowerSyncTransactor.ts | 8 +- .../src/definitions.ts | 14 +- .../powersync-db-collection/src/helpers.ts | 15 +- .../powersync-db-collection/src/powersync.ts | 57 +++--- .../tests/numeric-keys.test.ts | 163 ++++++++++++++++++ 7 files changed, 247 insertions(+), 42 deletions(-) create mode 100644 .changeset/powersync-numeric-keys.md create mode 100644 packages/powersync-db-collection/tests/numeric-keys.test.ts diff --git a/.changeset/powersync-numeric-keys.md b/.changeset/powersync-numeric-keys.md new file mode 100644 index 0000000000..fedfe9869b --- /dev/null +++ b/.changeset/powersync-numeric-keys.md @@ -0,0 +1,5 @@ +--- +'@tanstack/powersync-db-collection': patch +--- + +Support numeric collection keys. A `deserializationSchema` may convert `id` to a number, for a backend with integer primary keys; the collection key is then the deserialized number. Sync, on-demand reloads, eviction, and local updates and deletes all use that key. diff --git a/docs/collections/powersync-collection.md b/docs/collections/powersync-collection.md index 472a7e6926..ae6a2aea8b 100644 --- a/docs/collections/powersync-collection.md +++ b/docs/collections/powersync-collection.md @@ -399,6 +399,33 @@ const task = tasksCollection.get("task-1") task.due_date.getTime() // OK - TypeScript knows this is a Date ``` +### Integer Primary Keys + +SQLite stores every PowerSync `id` as text. If your backend uses integer primary keys, convert `id` to a number in `deserializationSchema`. The collection key is then a number, so integer foreign keys match it, for example in `eq(part.id, line.partId)`. + +```typescript +const productSchema = z.object({ + id: z.coerce.number(), // SQLite TEXT '5' to the key 5 + name: z.string().nullable(), +}) + +const products = createCollection( + powerSyncCollectionOptions({ + database: db, + table: APP_SCHEMA.props.products, + schema: productSchema, + deserializationSchema: productSchema, + onDeserializationError: (error) => { + // Present fatal error + }, + }) +) + +products.get(5) +``` + +Updates and deletes write the key back as text. The PowerSync service converts integer `id` columns to text the same way when it syncs them. + ### Optimistic Updates Updates to the collection are applied optimistically to the local state first, then synchronized with PowerSync and the backend. If an error occurs during sync, the changes are automatically rolled back. diff --git a/packages/powersync-db-collection/src/PowerSyncTransactor.ts b/packages/powersync-db-collection/src/PowerSyncTransactor.ts index d78b0a5c54..c1f1f8c628 100644 --- a/packages/powersync-db-collection/src/PowerSyncTransactor.ts +++ b/packages/powersync-db-collection/src/PowerSyncTransactor.ts @@ -2,7 +2,7 @@ import { LogLevels, sanitizeSQL } from '@powersync/common' import { LoadSubsetOperationAbortedError, isFailedStatus } from '@tanstack/db' import DebugModule from 'debug' import { PendingOperationStore } from './PendingOperationStore' -import { asPowerSyncRecord, mapOperationToPowerSync } from './helpers' +import { mapOperationToPowerSync } from './helpers' import type { CommonPowerSyncDatabase, LockContext } from '@powersync/common' import type { PendingMutation, Transaction } from '@tanstack/db' import type { PendingOperation } from './PendingOperationStore' @@ -247,7 +247,7 @@ export class PowerSyncTransactor { SET ${keys.map((key) => `${key} = ?`).join(`, `)} WHERE id = ? `, - [...queryParameters, asPowerSyncRecord(mutation.original).id], + [...queryParameters, String(mutation.key)], ) return @@ -278,14 +278,14 @@ export class PowerSyncTransactor { ` UPDATE ${tableName} SET _deleted = TRUE, _metadata = ? WHERE id = ? `, - [metadataValue, asPowerSyncRecord(mutation.original).id], + [metadataValue, String(mutation.key)], ) } else { await context.execute( ` DELETE FROM ${tableName} WHERE id = ? `, - [asPowerSyncRecord(mutation.original).id], + [String(mutation.key)], ) } }, diff --git a/packages/powersync-db-collection/src/definitions.ts b/packages/powersync-db-collection/src/definitions.ts index b57253a485..fd5f4a2604 100644 --- a/packages/powersync-db-collection/src/definitions.ts +++ b/packages/powersync-db-collection/src/definitions.ts @@ -26,6 +26,16 @@ export type InferPowerSyncOutputType< ? ExtractedTable : InferSchemaOutput +/** + * The collection key: the type of the output row's `id`. It is a string + * unless a schema converts `id`, for example to a number. + */ +export type InferPowerSyncKey = TOutput extends { id: infer TId } + ? TId extends number + ? number + : string + : string + /** * A mapping type for custom serialization of object properties to SQLite-compatible values. * @@ -208,7 +218,7 @@ export type BasePowerSyncCollectionConfig< > = Omit< BaseCollectionConfig< InferPowerSyncOutputType, - string, + InferPowerSyncKey>, TSchema >, `onInsert` | `onUpdate` | `onDelete` | `getKey` | `syncMode` @@ -306,7 +316,7 @@ export type EnhancedPowerSyncCollectionConfig< TSchema extends StandardSchemaV1 = never, > = CollectionConfig< OutputType, - string, + InferPowerSyncKey, TSchema, PowerSyncCollectionUtils > & { diff --git a/packages/powersync-db-collection/src/helpers.ts b/packages/powersync-db-collection/src/helpers.ts index 7bbb8e8b67..8d94823122 100644 --- a/packages/powersync-db-collection/src/helpers.ts +++ b/packages/powersync-db-collection/src/helpers.ts @@ -2,10 +2,12 @@ import { DiffTriggerOperation } from '@powersync/common' import type { ExtractColumnValueType, Table } from '@powersync/common' /** - * All PowerSync table records include a UUID `id` column. + * All PowerSync table records include an `id` column. SQLite stores it as + * text; a deserialization schema may convert it to a number, as for a + * backend with integer primary keys. */ export type PowerSyncRecord = { - id: string + id: string | number [key: string]: unknown } @@ -64,14 +66,7 @@ export type OptionalExtractedTable = OptionalIfUndefined<{ */ export type AnyTableColumnType = { [K in keyof ExtractedTableColumns]: unknown -} & { id: string } - -export function asPowerSyncRecord(record: any): PowerSyncRecord { - if (typeof record.id !== `string`) { - throw new Error(`Record must have a string id field`) - } - return record as PowerSyncRecord -} +} & { id: string | number } /** * Maps {@link DiffTriggerOperation} to TanstackDB operations diff --git a/packages/powersync-db-collection/src/powersync.ts b/packages/powersync-db-collection/src/powersync.ts index dfb52c15f4..dad1044247 100644 --- a/packages/powersync-db-collection/src/powersync.ts +++ b/packages/powersync-db-collection/src/powersync.ts @@ -4,7 +4,7 @@ import { compileSQLite } from './sqlite-compiler' import { PendingOperationStore } from './PendingOperationStore' import { PowerSyncTransactor } from './PowerSyncTransactor' import { DEFAULT_BATCH_SIZE } from './definitions' -import { asPowerSyncRecord, mapOperation } from './helpers' +import { mapOperation } from './helpers' import { convertTableToSchema } from './schema' import { serializeForSQLite } from './serialization' import type { @@ -25,6 +25,7 @@ import type { ConfigWithSQLiteInputType, ConfigWithSQLiteTypes, EnhancedPowerSyncCollectionConfig, + InferPowerSyncKey, InferPowerSyncOutputType, PowerSyncCollectionConfig, PowerSyncCollectionUtils, @@ -267,6 +268,7 @@ function createPowerSyncCollectionConfig< // The collection output type type OutputType = InferPowerSyncOutputType + type PowerSyncKey = InferPowerSyncKey const { viewName, trackMetadata: metadataIsTracked } = table @@ -323,7 +325,7 @@ function createPowerSyncCollectionConfig< * Notice that this describes the Sync between the local SQLite table * and the in-memory tanstack-db collection. */ - const sync: SyncConfig = { + const sync: SyncConfig = { sync: (params) => { const { begin, @@ -428,7 +430,7 @@ function createPowerSyncCollectionConfig< // holds. snapshot?: boolean when: Record - writeType: (rowId: string) => OperationType + writeType: (key: PowerSyncKey) => OperationType batchQuery: ( lockContext: LockContext, batchSize: number, @@ -456,10 +458,10 @@ function createPowerSyncCollectionConfig< const apply = () => { begin({ snapshot }) for (const row of batches.splice(0).flat()) { - write({ - type: writeType(row.id), - value: deserializeSyncRow(row), - }) + // The key comes from the deserialized row: a schema may + // convert SQLite's text id to a number. + const value = deserializeSyncRow(row) + write({ type: writeType(getKey(value)), value }) } appliedReceipts.push(commit()) } @@ -633,7 +635,7 @@ function createPowerSyncCollectionConfig< [DiffTriggerOperation.UPDATE]: `TRUE`, [DiffTriggerOperation.DELETE]: `TRUE`, }, - writeType: (_rowId: string) => `insert`, + writeType: () => `insert`, batchQuery: ( lockContext: LockContext, batchSize: number, @@ -814,22 +816,19 @@ function createPowerSyncCollectionConfig< [DiffTriggerOperation.UPDATE]: `(${newDataWhenClause}) OR (${oldDataWhenClause})`, [DiffTriggerOperation.DELETE]: oldDataWhenClause, }, - writeType: (rowId: string) => - collection.has(rowId) ? `update` : `insert`, + writeType: (key) => { + missing?.delete(key) + return collection.has(key) ? `update` : `insert` + }, batchQuery: ( lockContext: LockContext, batchSize: number, cursor: number, ) => - lockContext - .getAll( - `SELECT * FROM ${viewName} WHERE ${viewWhereClause} LIMIT ? OFFSET ?`, - [batchSize, cursor], - ) - .then((rows) => { - for (const row of rows) missing?.delete(row.id) - return rows - }), + lockContext.getAll( + `SELECT * FROM ${viewName} WHERE ${viewWhereClause} LIMIT ? OFFSET ?`, + [batchSize, cursor], + ), }, appliedReceipts, ) @@ -1013,14 +1012,14 @@ function createPowerSyncCollectionConfig< ): Promise => { const compiledDeparting = compileSQLite({ where: options.where }) const departingWhereSQL = toInlinedWhereClause(compiledDeparting) - let rowsToEvict: Array<{ id: string }> + let rowsToEvict: Array for (;;) { if (stopped) return const revision = trackingRevision const active = activeWhereExpressions() let evictionSQL: string if (active.length === 0) { - evictionSQL = `SELECT id FROM ${viewName} WHERE ${departingWhereSQL}` + evictionSQL = `SELECT * FROM ${viewName} WHERE ${departingWhereSQL}` } else { const combinedRemaining = active.length === 1 @@ -1030,18 +1029,18 @@ function createPowerSyncCollectionConfig< where: combinedRemaining, }) const remainingWhereSQL = toInlinedWhereClause(compiledRemaining) - evictionSQL = `SELECT id FROM ${viewName} WHERE (${departingWhereSQL}) AND NOT (${remainingWhereSQL})` + evictionSQL = `SELECT * FROM ${viewName} WHERE (${departingWhereSQL}) AND NOT (${remainingWhereSQL})` } - rowsToEvict = await database.getAll<{ id: string }>(evictionSQL) + rowsToEvict = await database.getAll(evictionSQL) // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- cleanup can run during the query if (stopped) return if (trackingRevision === revision) break } if (rowsToEvict.length > 0) { begin() - for (const { id } of rowsToEvict) { - write({ type: `delete`, key: id }) + for (const row of rowsToEvict) { + write({ type: `delete`, key: getKey(deserializeSyncRow(row)) }) } void commit() } @@ -1182,7 +1181,13 @@ function createPowerSyncCollectionConfig< getSyncMetadata: undefined, } - const getKey = (record: OutputType) => asPowerSyncRecord(record).id + const getKey = (record: OutputType): PowerSyncKey => { + const { id } = record as { id?: unknown } + if (typeof id !== `string` && typeof id !== `number`) { + throw new Error(`Record must have a string or number id field`) + } + return id as PowerSyncKey + } // Keep adapter hooks and serialization options out of the core config. const { diff --git a/packages/powersync-db-collection/tests/numeric-keys.test.ts b/packages/powersync-db-collection/tests/numeric-keys.test.ts new file mode 100644 index 0000000000..9d72bbfce9 --- /dev/null +++ b/packages/powersync-db-collection/tests/numeric-keys.test.ts @@ -0,0 +1,163 @@ +import { randomUUID } from 'node:crypto' +import { tmpdir } from 'node:os' +import { + LogLevels, + PowerSyncDatabase, + Schema, + Table, + column, +} from '@powersync/node' +import { IR, createCollection } from '@tanstack/db' +import { describe, expect, it, onTestFinished, vi } from 'vitest' +import { z } from 'zod' +import { powerSyncCollectionOptions } from '../src' +import type { PowerSyncLogger } from '@powersync/node' + +// SQLite stores every PowerSync id as text. A backend with integer primary +// keys converts `id` to a number in `deserializationSchema`, so the +// collection key is the deserialized number and integer foreign keys match +// it. Every path that writes a key must derive it from the deserialized row, +// not from SQLite's text id. + +const APP_SCHEMA = new Schema({ + products: new Table({ name: column.text, category: column.text }), +}) + +const row = z.object({ + id: z.coerce.number(), + name: z.string().nullable(), + category: z.string().nullable(), +}) + +const categoryEquals = (category: string) => + new IR.Func(`eq`, [ + new IR.PropRef([`category`]), + new IR.Value(category), + ]) + +async function createDatabase() { + const errors: Array = [] + const logger: PowerSyncLogger = { + log({ level, message }) { + if (level >= LogLevels.error) errors.push(message) + }, + } + const db = new PowerSyncDatabase({ + database: { + dbFilename: `test-numeric-keys-${randomUUID()}.sqlite`, + dbLocation: tmpdir(), + implementation: { type: `node:sqlite` }, + }, + schema: APP_SCHEMA, + logger, + }) + onTestFinished(async () => { + await new Promise((resolve) => setTimeout(resolve, 100)) + await db.disconnectAndClear() + await db.close() + }) + await db.disconnectAndClear() + await db.execute( + `INSERT INTO products (id, name, category) VALUES ('1', 'A', 'x'), ('2', 'B', 'x'), ('3', 'C', 'y')`, + ) + return { db, errors } +} + +function createNumericCollection( + db: PowerSyncDatabase, + syncMode: `eager` | `on-demand`, +) { + const collection = createCollection( + powerSyncCollectionOptions({ + database: db, + table: APP_SCHEMA.props.products, + schema: row, + deserializationSchema: row, + onDeserializationError: () => {}, + syncMode, + }), + ) + onTestFinished(() => collection.cleanup()) + return collection +} + +describe(`numeric collection keys`, () => { + it(`syncs and writes rows keyed by a deserialized number`, async () => { + const { db, errors } = await createDatabase() + const collection = createNumericCollection(db, `eager`) + await collection.stateWhenReady() + expect([...collection.keys()].sort()).toEqual([1, 2, 3]) + + await db.execute(`UPDATE products SET name = 'A2' WHERE id = '1'`) + await db.execute(`DELETE FROM products WHERE id = '3'`) + await vi.waitFor(() => { + expect(collection.get(1)?.name).toBe(`A2`) + expect(collection.has(3)).toBe(false) + }) + + await collection + .update(1, (draft) => { + draft.name = `A3` + }) + .when(`settled`) + await collection.delete(2).when(`settled`) + expect(await db.getAll(`SELECT id, name FROM products`)).toEqual([ + { id: `1`, name: `A3` }, + ]) + expect(errors).toEqual([]) + }) + + it(`reloads held rows as updates and evicts them by numeric key`, async () => { + const { db, errors } = await createDatabase() + const collection = createNumericCollection(db, `on-demand`) + const first = { where: categoryEquals(`x`) } + await collection._sync.loadSubset(first) + expect([...collection.keys()].sort()).toEqual([1, 2]) + + // Tracking for the second load re-reads every active predicate, so it + // re-reads the held rows: each must be an update, not an insert. + const second = { where: categoryEquals(`y`) } + await collection._sync.loadSubset(second) + expect([...collection.keys()].sort()).toEqual([1, 2, 3]) + + collection._sync.unloadSubset(second) + await vi.waitFor(() => + expect([...collection.keys()].sort()).toEqual([1, 2]), + ) + collection._sync.unloadSubset(first) + await vi.waitFor(() => expect(collection.size).toBe(0)) + expect(errors).toEqual([]) + }) + + it(`keeps held rows after a tracking outage`, async () => { + vi.useFakeTimers() + const { db } = await createDatabase() + const collection = createNumericCollection(db, `on-demand`) + const first = { where: categoryEquals(`y`) } + const second = { where: categoryEquals(`x`) } + try { + await collection._sync.loadSubset(first) + await collection._sync.loadSubset(second) + const trigger = vi + .spyOn(db.triggers, `createDiffTrigger`) + .mockRejectedValueOnce(new Error(`release rebuild failed`)) + collection._sync.unloadSubset(first) + await vi.waitFor(() => + expect(collection.health.failureReason?.message).toBe( + `release rebuild failed`, + ), + ) + await vi.advanceTimersByTimeAsync(1_000) + await vi.waitFor(() => + expect(trigger.mock.calls.length).toBeGreaterThan(1), + ) + await vi.waitFor(() => expect(collection.health.hasErrors).toBe(false)) + // The rebuild reads the rows still in SQLite; a row it reads is kept. + expect([...collection.keys()].sort()).toEqual([1, 2]) + } finally { + await collection.cleanup() + await vi.runOnlyPendingTimersAsync() + vi.useRealTimers() + } + }) +}) From 2e2728564cb6d69828382b25f158fe01798c7703 Mon Sep 17 00:00:00 2001 From: Jaime Resano Date: Tue, 10 Nov 2026 00:45:15 +0100 Subject: [PATCH 2/6] fix(powersync-db-collection): derive keys before opening a sync transaction Eviction and load batches deserialized each row after begin(). A row that failed deserialization, or had an id that is neither a string nor a number, threw inside the loop and left the sync transaction open; each release retry then opened another. Derive every key first, then open the transaction and write. Co-Authored-By: Claude Opus 5.5 --- .../powersync-db-collection/src/powersync.ts | 29 ++++--- .../tests/numeric-keys.test.ts | 76 +++++++++++++++++++ 2 files changed, 95 insertions(+), 10 deletions(-) diff --git a/packages/powersync-db-collection/src/powersync.ts b/packages/powersync-db-collection/src/powersync.ts index dad1044247..8808f3d3e0 100644 --- a/packages/powersync-db-collection/src/powersync.ts +++ b/packages/powersync-db-collection/src/powersync.ts @@ -452,16 +452,22 @@ function createPowerSyncCollectionConfig< // A load of every row is one snapshot transaction: during a // reload it replaces the kept rows, so a row the table no // longer has goes. A subset load commits each batch. A - // transaction begins only once its rows are read, so a batch - // that fails leaves none open. + // transaction begins only once its rows are read and + // deserialized, so a batch that fails leaves none open. const batches: Array> = [] const apply = () => { + // The key comes from the deserialized row: a schema may + // convert SQLite's text id to a number. + const entries = batches + .splice(0) + .flat() + .map((row) => { + const value = deserializeSyncRow(row) + return [getKey(value), value] as const + }) begin({ snapshot }) - for (const row of batches.splice(0).flat()) { - // The key comes from the deserialized row: a schema may - // convert SQLite's text id to a number. - const value = deserializeSyncRow(row) - write({ type: writeType(getKey(value)), value }) + for (const [key, value] of entries) { + write({ type: writeType(key), value }) } appliedReceipts.push(commit()) } @@ -1038,10 +1044,13 @@ function createPowerSyncCollectionConfig< if (trackingRevision === revision) break } if (rowsToEvict.length > 0) { + // Derive every key before the transaction opens: a row that fails + // deserialization must not leave it open. + const keys = rowsToEvict.map((row) => + getKey(deserializeSyncRow(row)), + ) begin() - for (const row of rowsToEvict) { - write({ type: `delete`, key: getKey(deserializeSyncRow(row)) }) - } + for (const key of keys) write({ type: `delete`, key }) void commit() } await rebuildTracking() diff --git a/packages/powersync-db-collection/tests/numeric-keys.test.ts b/packages/powersync-db-collection/tests/numeric-keys.test.ts index 9d72bbfce9..0b971081a9 100644 --- a/packages/powersync-db-collection/tests/numeric-keys.test.ts +++ b/packages/powersync-db-collection/tests/numeric-keys.test.ts @@ -161,3 +161,79 @@ describe(`numeric collection keys`, () => { } }) }) + +// Deriving a key deserializes the row, and deserialization can fail. A sync +// transaction opens only after every key is derived, so a failing row leaves +// no open transaction behind. +describe(`a row that fails deserialization`, () => { + function createGatedCollection(db: PowerSyncDatabase) { + const gate = { valid: true } + const gated = row.extend({ + id: z.coerce.number().refine(() => gate.valid), + }) + const collection = createCollection( + powerSyncCollectionOptions({ + database: db, + table: APP_SCHEMA.props.products, + schema: gated, + deserializationSchema: gated, + onDeserializationError: () => {}, + syncMode: `on-demand`, + }), + ) + const openTransactions = () => + collection._state.pendingSyncedTransactions.filter( + (transaction) => !transaction.committed, + ).length + return { collection, gate, openTransactions } + } + + it(`fails a load without leaving a sync transaction open`, async () => { + const { db } = await createDatabase() + const { collection, gate, openTransactions } = createGatedCollection(db) + try { + gate.valid = false + await expect( + Promise.resolve( + collection._sync.loadSubset({ where: categoryEquals(`x`) }), + ), + ).rejects.toThrow() + expect(openTransactions()).toBe(0) + expect(collection.size).toBe(0) + + gate.valid = true + await collection._sync.loadSubset({ where: categoryEquals(`x`) }) + expect([...collection.keys()].sort()).toEqual([1, 2]) + } finally { + await collection.cleanup() + } + }) + + it(`retries an eviction without leaving a sync transaction open`, async () => { + vi.useFakeTimers() + const { db, errors } = await createDatabase() + const { collection, gate, openTransactions } = createGatedCollection(db) + const first = { where: categoryEquals(`x`) } + try { + await collection._sync.loadSubset(first) + gate.valid = false + collection._sync.unloadSubset(first) + await vi.waitFor(() => + expect(errors).toContainEqual( + expect.stringContaining(`Could not release subset tracking`), + ), + ) + expect(openTransactions()).toBe(0) + expect([...collection.keys()].sort()).toEqual([1, 2]) + + gate.valid = true + await vi.advanceTimersByTimeAsync(1_000) + await vi.waitFor(() => expect(collection.size).toBe(0)) + expect(openTransactions()).toBe(0) + } finally { + await collection.cleanup() + await vi.runOnlyPendingTimersAsync() + vi.useRealTimers() + } + }) +}) From 50a34810a4a3854a238c204f4abfa999322dffbe Mon Sep 17 00:00:00 2001 From: Tanner Linsley Date: Tue, 10 Nov 2026 15:23:03 -0700 Subject: [PATCH 3/6] fix(powersync): reject numeric IDs that cannot round-trip Law: numeric collection keys must preserve the exact SQLite row identity. Reach: real SQLite subset loads with leading-zero IDs and IDs beyond the safe integer range. RED: both loaded successfully before the guard. GREEN: all seven numeric key tests pass with the guard. Mutant: removing the guard restores both failures. Limits: this does not exercise numeric SQL predicates or release with a persistently invalid row. --- docs/collections/powersync-collection.md | 2 +- .../powersync-db-collection/src/powersync.ts | 30 ++++++++++++++----- .../tests/numeric-keys.test.ts | 18 +++++++++++ 3 files changed, 41 insertions(+), 9 deletions(-) diff --git a/docs/collections/powersync-collection.md b/docs/collections/powersync-collection.md index ae6a2aea8b..8b2531970c 100644 --- a/docs/collections/powersync-collection.md +++ b/docs/collections/powersync-collection.md @@ -424,7 +424,7 @@ const products = createCollection( products.get(5) ``` -Updates and deletes write the key back as text. The PowerSync service converts integer `id` columns to text the same way when it syncs them. +Numeric IDs must be safe JavaScript integers, and converting the number back to text must reproduce the stored ID exactly. Keep IDs as strings if they can exceed the safe integer range or contain leading zeros. Updates and deletes use the text form of the key to find the SQLite row. ### Optimistic Updates diff --git a/packages/powersync-db-collection/src/powersync.ts b/packages/powersync-db-collection/src/powersync.ts index 8808f3d3e0..b6026124ad 100644 --- a/packages/powersync-db-collection/src/powersync.ts +++ b/packages/powersync-db-collection/src/powersync.ts @@ -279,6 +279,16 @@ function createPowerSyncCollectionConfig< const validationSchema = deserializationSchema || schema const validation = validationSchema[`~standard`].validate(value) if (`value` in validation) { + const id = (validation.value as { id?: unknown }).id + if ( + typeof id === `number` && + (!Number.isSafeInteger(id) || String(id) !== value.id) + ) { + const message = `A numeric PowerSync id must round-trip as a canonical safe integer` + database.logger.log({ level: LogLevels.error, message }) + onDeserializationError?.({ issues: [{ path: [`id`], message }] }) + throw new Error(message) + } return validation.value } else if (`issues` in validation) { const issueMessage = `Failed to validate incoming data for ${viewName}. Issues: ${validation.issues.map((issue) => `${issue.path} - ${issue.message}`)}` @@ -531,13 +541,12 @@ function createPowerSyncCollectionConfig< } try { - begin() const operations = await context.getAll( `SELECT * FROM ${trackedTableName} ORDER BY operation_id ASC`, ) const pendingOperations: Array = [] - for (const op of operations) { + const changes = operations.map((op) => { const { id, operation, timestamp, value } = op const parsedValue = deserializeSyncRow({ id, @@ -550,18 +559,20 @@ function createPowerSyncCollectionConfig< ...JSON.parse(op.previous_value), }) : undefined - write({ - type: mapOperation(operation), - value: parsedValue, - previousValue: parsedPreviousValue, - }) pendingOperations.push({ id, operation, timestamp, tableName: viewName, }) - } + return { + type: mapOperation(operation), + value: parsedValue, + previousValue: parsedPreviousValue, + } + }) + begin() + for (const change of changes) write(change) // clear the current operations await context.execute(`DELETE FROM ${trackedTableName}`) @@ -1195,6 +1206,9 @@ function createPowerSyncCollectionConfig< if (typeof id !== `string` && typeof id !== `number`) { throw new Error(`Record must have a string or number id field`) } + if (typeof id === `number` && !Number.isSafeInteger(id)) { + throw new Error(`A numeric PowerSync id must be a safe integer`) + } return id as PowerSyncKey } diff --git a/packages/powersync-db-collection/tests/numeric-keys.test.ts b/packages/powersync-db-collection/tests/numeric-keys.test.ts index 0b971081a9..d2cb55eb23 100644 --- a/packages/powersync-db-collection/tests/numeric-keys.test.ts +++ b/packages/powersync-db-collection/tests/numeric-keys.test.ts @@ -82,6 +82,24 @@ function createNumericCollection( } describe(`numeric collection keys`, () => { + it.each([`007`, `9007199254740993`])( + `rejects a stored ID that cannot round-trip as a numeric key (%s)`, + async (id) => { + const { db } = await createDatabase() + await db.execute( + `INSERT INTO products (id, name, category) VALUES (?, 'D', 'invalid')`, + [id], + ) + const collection = createNumericCollection(db, `on-demand`) + await expect( + Promise.resolve( + collection._sync.loadSubset({ where: categoryEquals(`invalid`) }), + ), + ).rejects.toThrow(/canonical safe integer/) + expect(collection.size).toBe(0) + }, + ) + it(`syncs and writes rows keyed by a deserialized number`, async () => { const { db, errors } = await createDatabase() const collection = createNumericCollection(db, `eager`) From 7bd76b652588d4f1f61e2eaa21ddc35b5585df0b Mon Sep 17 00:00:00 2001 From: Tanner Linsley Date: Tue, 10 Nov 2026 15:24:06 -0700 Subject: [PATCH 4/6] fix(powersync): preserve sync write types when staging changes Law: staged diff writes retain the Collection sync write contract. Reach: PowerSync declaration generation checks the staged changes. RED: TS2345 widened the operation to string. GREEN: the PowerSync build passes. Mutant: removing the callback return type restores TS2345. Limits: compile-time regression only. --- packages/powersync-db-collection/src/powersync.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/powersync-db-collection/src/powersync.ts b/packages/powersync-db-collection/src/powersync.ts index b6026124ad..8c11c5854a 100644 --- a/packages/powersync-db-collection/src/powersync.ts +++ b/packages/powersync-db-collection/src/powersync.ts @@ -546,7 +546,7 @@ function createPowerSyncCollectionConfig< ) const pendingOperations: Array = [] - const changes = operations.map((op) => { + const changes = operations.map((op): Parameters[0] => { const { id, operation, timestamp, value } = op const parsedValue = deserializeSyncRow({ id, From 59ee2d34d704c0230ce09a63b09462cf6a7dcf86 Mon Sep 17 00:00:00 2001 From: Jaime Resano Date: Wed, 11 Nov 2026 14:58:24 +0100 Subject: [PATCH 5/6] fix(powersync-db-collection): explicit numericIds mode, release without revalidation - Add `numericIds`. TypeScript requires it exactly when the schema outputs a number `id`. With it, every `id` reference compiles to `CAST(id AS INTEGER)`, and the subset load, trigger WHEN clauses, and eviction compile through one helper, so `gt(id, 2)` matches 10 on every path. Without it, SQL is unchanged. - Release no longer deserializes rows. It selects stored ids and derives each key from the id (or the number that round-trips to it), so a permanently invalid row cannot block eviction of the rows a subset loaded. Co-Authored-By: Claude Opus 5.5 --- .changeset/powersync-numeric-keys.md | 2 +- docs/collections/powersync-collection.md | 11 +- .../src/definitions.ts | 14 +- .../powersync-db-collection/src/powersync.ts | 79 ++++---- .../src/sqlite-compiler.ts | 17 +- .../tests/numeric-keys.test.ts | 169 +++++++++++------- .../tests/powersync.test-d.ts | 31 ++++ 7 files changed, 212 insertions(+), 111 deletions(-) diff --git a/.changeset/powersync-numeric-keys.md b/.changeset/powersync-numeric-keys.md index fedfe9869b..7c98778f90 100644 --- a/.changeset/powersync-numeric-keys.md +++ b/.changeset/powersync-numeric-keys.md @@ -2,4 +2,4 @@ '@tanstack/powersync-db-collection': patch --- -Support numeric collection keys. A `deserializationSchema` may convert `id` to a number, for a backend with integer primary keys; the collection key is then the deserialized number. Sync, on-demand reloads, eviction, and local updates and deletes all use that key. +Support numeric collection keys. Set `numericIds: true` and have `deserializationSchema` convert `id` to a number, for a backend with integer primary keys; the collection key is then the deserialized number. On-demand loads, change tracking, and eviction compare `id` as a number, and local updates and deletes use the key. TypeScript requires `numericIds` exactly when the schema outputs a number `id`. Releasing an on-demand subset no longer revalidates its rows, so a row that fails validation no longer blocks the release. diff --git a/docs/collections/powersync-collection.md b/docs/collections/powersync-collection.md index 8b2531970c..55621384b1 100644 --- a/docs/collections/powersync-collection.md +++ b/docs/collections/powersync-collection.md @@ -401,11 +401,11 @@ task.due_date.getTime() // OK - TypeScript knows this is a Date ### Integer Primary Keys -SQLite stores every PowerSync `id` as text. If your backend uses integer primary keys, convert `id` to a number in `deserializationSchema`. The collection key is then a number, so integer foreign keys match it, for example in `eq(part.id, line.partId)`. +SQLite stores every PowerSync `id` as text. If your backend uses integer primary keys, convert `id` to a number in `deserializationSchema` and set `numericIds: true`. The collection key is then a number, so integer foreign keys match it, for example in `eq(part.id, line.partId)`. Predicates on `id`, such as `gt(product.id, 2)`, compare numbers in on-demand loads, change tracking, and eviction. ```typescript const productSchema = z.object({ - id: z.coerce.number(), // SQLite TEXT '5' to the key 5 + id: z.number(), name: z.string().nullable(), }) @@ -414,7 +414,10 @@ const products = createCollection( database: db, table: APP_SCHEMA.props.products, schema: productSchema, - deserializationSchema: productSchema, + deserializationSchema: productSchema.extend({ + id: z.coerce.number(), // SQLite TEXT '5' to the key 5 + }), + numericIds: true, onDeserializationError: (error) => { // Present fatal error }, @@ -424,7 +427,7 @@ const products = createCollection( products.get(5) ``` -Numeric IDs must be safe JavaScript integers, and converting the number back to text must reproduce the stored ID exactly. Keep IDs as strings if they can exceed the safe integer range or contain leading zeros. Updates and deletes use the text form of the key to find the SQLite row. +TypeScript requires `numericIds: true` when the schema outputs a number `id`, and rejects it otherwise. Numeric IDs must be safe JavaScript integers, and converting the number back to text must reproduce the stored ID exactly. Keep IDs as strings if they can exceed the safe integer range or contain leading zeros. Updates and deletes use the text form of the key to find the SQLite row. ### Optimistic Updates diff --git a/packages/powersync-db-collection/src/definitions.ts b/packages/powersync-db-collection/src/definitions.ts index fd5f4a2604..7777169fb2 100644 --- a/packages/powersync-db-collection/src/definitions.ts +++ b/packages/powersync-db-collection/src/definitions.ts @@ -36,6 +36,17 @@ export type InferPowerSyncKey = TOutput extends { id: infer TId } : string : string +/** + * SQLite stores every PowerSync `id` as text. Set `numericIds: true` when the + * schema converts `id` to a number, as for a backend with integer primary + * keys. Predicates on `id` then compare numbers, and the collection key is + * the number. The option is required exactly when the output `id` is a number. + */ +export type NumericIdsConfig = + InferPowerSyncKey extends number + ? { numericIds: true } + : { numericIds?: false } + /** * A mapping type for custom serialization of object properties to SQLite-compatible values. * @@ -240,7 +251,8 @@ export type BasePowerSyncCollectionConfig< * streaming of initial results, at the cost of more query calls. */ syncBatchSize?: number -} & (EagerSyncHooks | OnDemandSyncHooks) +} & NumericIdsConfig> & + (EagerSyncHooks | OnDemandSyncHooks) /** * Configuration interface for PowerSync collection options. diff --git a/packages/powersync-db-collection/src/powersync.ts b/packages/powersync-db-collection/src/powersync.ts index 4b4bcb0b47..ae38de2abf 100644 --- a/packages/powersync-db-collection/src/powersync.ts +++ b/packages/powersync-db-collection/src/powersync.ts @@ -257,6 +257,7 @@ function createPowerSyncCollectionConfig< schema: inputSchema, syncBatchSize = DEFAULT_BATCH_SIZE, syncMode = 'eager', + numericIds = false, ...restConfig } = config @@ -284,10 +285,12 @@ function createPowerSyncCollectionConfig< const validationSchema = deserializationSchema || schema const validation = validationSchema[`~standard`].validate(value) if (`value` in validation) { + // A numeric key must name exactly one stored id, so eviction and + // writes can derive one from the other. const id = (validation.value as { id?: unknown }).id if ( - typeof id === `number` && - (!Number.isSafeInteger(id) || String(id) !== value.id) + numericIds && + !(Number.isSafeInteger(id) && String(id) === value.id) ) { const message = `A numeric PowerSync id must round-trip as a canonical safe integer` database.logger.log({ level: LogLevels.error, message }) @@ -878,18 +881,15 @@ function createPowerSyncCollectionConfig< active.length === 1 ? active[0] : or(active[0], active[1], ...active.slice(2)) - const compiledNewData = compileSQLite( - { where: combinedWhere }, - { jsonColumn: 'NEW.data' }, + const newDataWhenClause = compileWhere( + combinedWhere, + 'NEW.data', ) - const compiledOldData = compileSQLite( - { where: combinedWhere }, - { jsonColumn: 'OLD.data' }, + const oldDataWhenClause = compileWhere( + combinedWhere, + 'OLD.data', ) - const compiledView = compileSQLite({ where: combinedWhere }) - const newDataWhenClause = toInlinedWhereClause(compiledNewData) - const oldDataWhenClause = toInlinedWhereClause(compiledOldData) - const viewWhereClause = toInlinedWhereClause(compiledView) + const viewWhereClause = compileWhere(combinedWhere) await establishTracking( { setupContext: ctx, @@ -1016,10 +1016,16 @@ function createPowerSyncCollectionConfig< return task } - const toInlinedWhereClause = (compiled: { - where?: string - params: Array - }): string => { + // Loading, tracking, and eviction all compile predicates here, so + // they compare `id` the same way. + const compileWhere = ( + where: LoadSubsetOptions[`where`], + jsonColumn?: string, + ): string => { + const compiled = compileSQLite( + { where }, + { jsonColumn, numericId: numericIds }, + ) if (!compiled.where) return 'TRUE' const sqlParts = compiled.where.split('?') return sanitizeSQL( @@ -1092,39 +1098,37 @@ function createPowerSyncCollectionConfig< const performPhysicalRelease = async ( options: LoadSubsetOptions, ): Promise => { - const compiledDeparting = compileSQLite({ where: options.where }) - const departingWhereSQL = toInlinedWhereClause(compiledDeparting) - let rowsToEvict: Array + const departingWhereSQL = compileWhere(options.where) + let rowsToEvict: Array<{ id: string }> for (;;) { if (stopped) return const revision = trackingRevision const active = activeWhereExpressions() let evictionSQL: string if (active.length === 0) { - evictionSQL = `SELECT * FROM ${viewName} WHERE ${departingWhereSQL}` + evictionSQL = `SELECT id FROM ${viewName} WHERE ${departingWhereSQL}` } else { const combinedRemaining = active.length === 1 ? active[0]! : or(active[0], active[1], ...active.slice(2)) - const compiledRemaining = compileSQLite({ - where: combinedRemaining, - }) - const remainingWhereSQL = toInlinedWhereClause(compiledRemaining) - evictionSQL = `SELECT * FROM ${viewName} WHERE (${departingWhereSQL}) AND NOT (${remainingWhereSQL})` + const remainingWhereSQL = compileWhere(combinedRemaining) + evictionSQL = `SELECT id FROM ${viewName} WHERE (${departingWhereSQL}) AND NOT (${remainingWhereSQL})` } - rowsToEvict = await database.getAll(evictionSQL) + rowsToEvict = await database.getAll<{ id: string }>(evictionSQL) // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- cleanup can run during the query if (stopped) return if (trackingRevision === revision) break } - if (rowsToEvict.length > 0) { - // Derive every key before the transaction opens: a row that fails - // deserialization must not leave it open. - const keys = rowsToEvict.map((row) => - getKey(deserializeSyncRow(row)), - ) + // Eviction does not revalidate rows. A loaded row's key is its stored + // id, or with numericIds the number that round-trips to it; a row + // with any other id was never loaded. + const keys = rowsToEvict.flatMap(({ id }) => { + const key = numericIds ? Number(id) : id + return String(key) === id ? [key as PowerSyncKey] : [] + }) + if (keys.length > 0) { begin() for (const key of keys) write({ type: `delete`, key }) void commit() @@ -1268,11 +1272,12 @@ function createPowerSyncCollectionConfig< const getKey = (record: OutputType): PowerSyncKey => { const { id } = record as { id?: unknown } - if (typeof id !== `string` && typeof id !== `number`) { - throw new Error(`Record must have a string or number id field`) - } - if (typeof id === `number` && !Number.isSafeInteger(id)) { - throw new Error(`A numeric PowerSync id must be a safe integer`) + if (numericIds ? !Number.isSafeInteger(id) : typeof id !== `string`) { + throw new Error( + numericIds + ? `With numericIds, a record id must be a safe integer` + : `Record must have a string id field; set numericIds for number ids`, + ) } return id as PowerSyncKey } diff --git a/packages/powersync-db-collection/src/sqlite-compiler.ts b/packages/powersync-db-collection/src/sqlite-compiler.ts index d5d0274965..7b1c8c1819 100644 --- a/packages/powersync-db-collection/src/sqlite-compiler.ts +++ b/packages/powersync-db-collection/src/sqlite-compiler.ts @@ -24,6 +24,11 @@ export interface CompileSQLiteOptions { * as a direct column in the tracked table. */ jsonColumn?: string + /** + * When set, `id` references compile to `CAST( AS INTEGER)`, so a + * collection with numeric ids compares them as numbers, not as text. + */ + numericId?: boolean } /** @@ -100,10 +105,14 @@ function compileExpression( // JSON `data` object, so we skip json_extract. However, when compiling for // trigger WHEN clauses we still need the OLD./NEW. prefix. Extract it from // the jsonColumn option. - if (compileOptions?.jsonColumn && columnName === `id`) { - const prefix = compileOptions.jsonColumn.split('.')[0]! - return `${prefix}.${quoteIdentifier(columnName)}` - } else if (compileOptions?.jsonColumn) { + if (columnName === `id`) { + const prefix = compileOptions?.jsonColumn?.split('.')[0] + const id = prefix + ? `${prefix}.${quoteIdentifier(columnName)}` + : quoteIdentifier(columnName) + return compileOptions?.numericId ? `CAST(${id} AS INTEGER)` : id + } + if (compileOptions?.jsonColumn) { return `json_extract(${compileOptions.jsonColumn}, '$.${columnName}')` } return quoteIdentifier(columnName) diff --git a/packages/powersync-db-collection/tests/numeric-keys.test.ts b/packages/powersync-db-collection/tests/numeric-keys.test.ts index d2cb55eb23..15b79ce8df 100644 --- a/packages/powersync-db-collection/tests/numeric-keys.test.ts +++ b/packages/powersync-db-collection/tests/numeric-keys.test.ts @@ -14,20 +14,22 @@ import { powerSyncCollectionOptions } from '../src' import type { PowerSyncLogger } from '@powersync/node' // SQLite stores every PowerSync id as text. A backend with integer primary -// keys converts `id` to a number in `deserializationSchema`, so the -// collection key is the deserialized number and integer foreign keys match -// it. Every path that writes a key must derive it from the deserialized row, -// not from SQLite's text id. +// keys sets `numericIds` and converts `id` to a number in +// `deserializationSchema`, so the collection key is the deserialized number +// and integer foreign keys match it. Every path that writes a key must use +// that number, and every path that filters by `id` must compare numbers. const APP_SCHEMA = new Schema({ products: new Table({ name: column.text, category: column.text }), }) const row = z.object({ - id: z.coerce.number(), + id: z.number(), name: z.string().nullable(), category: z.string().nullable(), }) +// Rows read from SQLite carry the id as text. +const sqliteRow = row.extend({ id: z.coerce.number() }) const categoryEquals = (category: string) => new IR.Func(`eq`, [ @@ -35,6 +37,9 @@ const categoryEquals = (category: string) => new IR.Value(category), ]) +const idGreaterThan = (id: number) => + new IR.Func(`gt`, [new IR.PropRef([`id`]), new IR.Value(id)]) + async function createDatabase() { const errors: Array = [] const logger: PowerSyncLogger = { @@ -72,8 +77,9 @@ function createNumericCollection( database: db, table: APP_SCHEMA.props.products, schema: row, - deserializationSchema: row, + deserializationSchema: sqliteRow, onDeserializationError: () => {}, + numericIds: true, syncMode, }), ) @@ -147,6 +153,30 @@ describe(`numeric collection keys`, () => { expect(errors).toEqual([]) }) + // As text, '10' < '2'. The subset load, the trigger that tracks later + // writes, and eviction must all compare ids as numbers. + it(`compares ids as numbers when loading, tracking, and evicting`, async () => { + const { db, errors } = await createDatabase() + await db.execute( + `INSERT INTO products (id, name, category) VALUES ('10', 'E', 'z')`, + ) + const collection = createNumericCollection(db, `on-demand`) + const subset = { where: idGreaterThan(2) } + await collection._sync.loadSubset(subset) + expect([...collection.keys()].sort((a, b) => a - b)).toEqual([3, 10]) + + // One write, so once 100 arrives the tracking has judged 0 as well. + await db.execute( + `INSERT INTO products (id, name, category) VALUES ('0', 'F', 'z'), ('100', 'G', 'z')`, + ) + await vi.waitFor(() => expect(collection.has(100)).toBe(true)) + expect(collection.has(0)).toBe(false) + + collection._sync.unloadSubset(subset) + await vi.waitFor(() => expect(collection.size).toBe(0)) + expect(errors).toEqual([]) + }) + it(`keeps held rows after a tracking outage`, async () => { vi.useFakeTimers() const { db } = await createDatabase() @@ -180,78 +210,89 @@ describe(`numeric collection keys`, () => { }) }) -// Deriving a key deserializes the row, and deserialization can fail. A sync -// transaction opens only after every key is derived, so a failing row leaves -// no open transaction behind. +// A load deserializes every row, and deserialization can fail. A sync +// transaction opens only after every row is deserialized, so a failing row +// leaves no open transaction behind. A release does not deserialize: it +// derives each key from the stored id, so a row that fails validation cannot +// keep the release from evicting the rows it loaded. describe(`a row that fails deserialization`, () => { - function createGatedCollection(db: PowerSyncDatabase) { - const gate = { valid: true } - const gated = row.extend({ - id: z.coerce.number().refine(() => gate.valid), - }) - const collection = createCollection( - powerSyncCollectionOptions({ - database: db, - table: APP_SCHEMA.props.products, - schema: gated, - deserializationSchema: gated, - onDeserializationError: () => {}, - syncMode: `on-demand`, - }), - ) + // A row fails validation while its stored id is in `invalid`. + function createGatedCollection( + db: PowerSyncDatabase, + ids: `numeric` | `string`, + ) { + const invalid = new Set() + const valid = (id: string | number) => !invalid.has(String(id)) + const base = { + database: db, + table: APP_SCHEMA.props.products, + onDeserializationError: () => {}, + syncMode: `on-demand` as const, + } + const numeric = row.extend({ id: z.coerce.number().refine(valid) }) + const text = row.extend({ id: z.string().refine(valid) }) + const collection = + ids === `numeric` + ? createCollection( + powerSyncCollectionOptions({ + ...base, + schema: row, + deserializationSchema: numeric, + numericIds: true, + }), + ) + : createCollection( + powerSyncCollectionOptions({ + ...base, + schema: text, + deserializationSchema: text, + }), + ) + onTestFinished(() => collection.cleanup()) const openTransactions = () => collection._state.pendingSyncedTransactions.filter( (transaction) => !transaction.committed, ).length - return { collection, gate, openTransactions } + return { collection, invalid, openTransactions } } it(`fails a load without leaving a sync transaction open`, async () => { const { db } = await createDatabase() - const { collection, gate, openTransactions } = createGatedCollection(db) - try { - gate.valid = false - await expect( - Promise.resolve( - collection._sync.loadSubset({ where: categoryEquals(`x`) }), - ), - ).rejects.toThrow() - expect(openTransactions()).toBe(0) - expect(collection.size).toBe(0) + const { collection, invalid, openTransactions } = createGatedCollection( + db, + `numeric`, + ) + invalid.add(`2`) + await expect( + Promise.resolve( + collection._sync.loadSubset({ where: categoryEquals(`x`) }), + ), + ).rejects.toThrow() + expect(openTransactions()).toBe(0) + expect(collection.size).toBe(0) - gate.valid = true - await collection._sync.loadSubset({ where: categoryEquals(`x`) }) - expect([...collection.keys()].sort()).toEqual([1, 2]) - } finally { - await collection.cleanup() - } + invalid.clear() + await collection._sync.loadSubset({ where: categoryEquals(`x`) }) + expect(collection.size).toBe(2) }) - it(`retries an eviction without leaving a sync transaction open`, async () => { - vi.useFakeTimers() - const { db, errors } = await createDatabase() - const { collection, gate, openTransactions } = createGatedCollection(db) - const first = { where: categoryEquals(`x`) } - try { - await collection._sync.loadSubset(first) - gate.valid = false - collection._sync.unloadSubset(first) - await vi.waitFor(() => - expect(errors).toContainEqual( - expect.stringContaining(`Could not release subset tracking`), - ), + it.each([`numeric`, `string`] as const)( + `releases a subset that holds a permanently invalid row (%s ids)`, + async (ids) => { + const { db, errors } = await createDatabase() + const { collection, invalid, openTransactions } = createGatedCollection( + db, + ids, ) - expect(openTransactions()).toBe(0) - expect([...collection.keys()].sort()).toEqual([1, 2]) + const subset = { where: categoryEquals(`x`) } + await collection._sync.loadSubset(subset) + expect(collection.size).toBe(2) - gate.valid = true - await vi.advanceTimersByTimeAsync(1_000) + invalid.add(`2`) + collection._sync.unloadSubset(subset) await vi.waitFor(() => expect(collection.size).toBe(0)) expect(openTransactions()).toBe(0) - } finally { - await collection.cleanup() - await vi.runOnlyPendingTimersAsync() - vi.useRealTimers() - } - }) + expect(errors).toEqual([]) + }, + ) }) diff --git a/packages/powersync-db-collection/tests/powersync.test-d.ts b/packages/powersync-db-collection/tests/powersync.test-d.ts index 4e481946be..9f4d7b93e5 100644 --- a/packages/powersync-db-collection/tests/powersync.test-d.ts +++ b/packages/powersync-db-collection/tests/powersync.test-d.ts @@ -90,4 +90,35 @@ describe(`PowerSync collection type tests`, () => { }, }) }) + + it(`requires numericIds exactly when the schema outputs a number id`, () => { + const schema = z.object({ + id: z.number(), + name: z.string().nullable(), + author: z.string().nullable(), + created_at: z.string().nullable(), + }) + const base = { + database: {} as PowerSyncDatabase, + table: APP_SCHEMA.props.documents, + schema, + deserializationSchema: schema.extend({ id: z.coerce.number() }), + onDeserializationError: () => {}, + } + + const options = powerSyncCollectionOptions({ ...base, numericIds: true }) + expectTypeOf(createCollection(options).get) + .parameter(0) + .toEqualTypeOf() + + // @ts-expect-error a number id needs numericIds + powerSyncCollectionOptions(base) + + powerSyncCollectionOptions({ + database: {} as PowerSyncDatabase, + table: APP_SCHEMA.props.documents, + // @ts-expect-error a text id cannot use numericIds + numericIds: true, + }) + }) }) From 91097e0a2e02ad45dca748434dba569f5699f89c Mon Sep 17 00:00:00 2001 From: Jaime Resano Date: Wed, 11 Nov 2026 19:34:17 +0100 Subject: [PATCH 6/6] fix(powersync-db-collection): address review of numeric ids - Reject a schema whose `id` may be a string or a number: it fits neither `numericIds` mode, and its numeric rows would fail `getKey` at runtime. - Derive every key before a sync transaction opens, on the load path and on the path that syncs tracked writes, so a key `getKey` rejects leaves no transaction open. - Compare numeric ids with `eq` and `in` as their canonical text, so SQLite uses the `id` primary key index instead of scanning `CAST(id AS INTEGER)`. Range predicates keep the CAST. - Define the `products` table in the integer-key docs example and list `numericIds` among the configuration options. Co-Authored-By: Claude Opus 5.5 --- docs/collections/powersync-collection.md | 11 +- .../src/definitions.ts | 5 +- .../powersync-db-collection/src/powersync.ts | 18 ++- .../src/sqlite-compiler.ts | 33 ++++- .../tests/numeric-keys.test.ts | 122 ++++++++++++++---- .../tests/powersync.test-d.ts | 12 ++ 6 files changed, 169 insertions(+), 32 deletions(-) diff --git a/docs/collections/powersync-collection.md b/docs/collections/powersync-collection.md index 55621384b1..a3a04b73c8 100644 --- a/docs/collections/powersync-collection.md +++ b/docs/collections/powersync-collection.md @@ -404,6 +404,12 @@ task.due_date.getTime() // OK - TypeScript knows this is a Date SQLite stores every PowerSync `id` as text. If your backend uses integer primary keys, convert `id` to a number in `deserializationSchema` and set `numericIds: true`. The collection key is then a number, so integer foreign keys match it, for example in `eq(part.id, line.partId)`. Predicates on `id`, such as `gt(product.id, 2)`, compare numbers in on-demand loads, change tracking, and eviction. ```typescript +const APP_SCHEMA = new Schema({ + products: new Table({ + name: column.text, + }), +}) + const productSchema = z.object({ id: z.number(), name: z.string().nullable(), @@ -427,7 +433,7 @@ const products = createCollection( products.get(5) ``` -TypeScript requires `numericIds: true` when the schema outputs a number `id`, and rejects it otherwise. Numeric IDs must be safe JavaScript integers, and converting the number back to text must reproduce the stored ID exactly. Keep IDs as strings if they can exceed the safe integer range or contain leading zeros. Updates and deletes use the text form of the key to find the SQLite row. +TypeScript requires `numericIds: true` when the schema outputs a number `id`, and rejects it otherwise. An `id` that may be either a string or a number is not supported. Equality and `in` predicates on a numeric `id` use SQLite's `id` index; range predicates such as `gt` scan the table. Numeric IDs must be safe JavaScript integers, and converting the number back to text must reproduce the stored ID exactly. Keep IDs as strings if they can exceed the safe integer range or contain leading zeros. Updates and deletes use the text form of the key to find the SQLite row. ### Optimistic Updates @@ -556,6 +562,9 @@ interface PowerSyncCollectionConfig { [Key in keyof TOutput]?: (value: TOutput[Key]) => SQLiteCompatibleType } + // Integer primary keys: required when the schema outputs a number id + numericIds?: boolean + // Performance tuning syncBatchSize?: number // Control batch size for initial sync, defaults to 1000 } diff --git a/packages/powersync-db-collection/src/definitions.ts b/packages/powersync-db-collection/src/definitions.ts index 7777169fb2..d05bd43c36 100644 --- a/packages/powersync-db-collection/src/definitions.ts +++ b/packages/powersync-db-collection/src/definitions.ts @@ -41,11 +41,14 @@ export type InferPowerSyncKey = TOutput extends { id: infer TId } * schema converts `id` to a number, as for a backend with integer primary * keys. Predicates on `id` then compare numbers, and the collection key is * the number. The option is required exactly when the output `id` is a number. + * An `id` that may be a string or a number fits neither mode and is rejected. */ export type NumericIdsConfig = InferPowerSyncKey extends number ? { numericIds: true } - : { numericIds?: false } + : InferPowerSyncKey extends string + ? { numericIds?: false } + : { numericIds: never } /** * A mapping type for custom serialization of object properties to SQLite-compatible values. diff --git a/packages/powersync-db-collection/src/powersync.ts b/packages/powersync-db-collection/src/powersync.ts index ae38de2abf..42339c072b 100644 --- a/packages/powersync-db-collection/src/powersync.ts +++ b/packages/powersync-db-collection/src/powersync.ts @@ -488,14 +488,19 @@ function createPowerSyncCollectionConfig< beforeCreate: async (context) => { // A retained reload uses one snapshot across its source reads. // Other initial loads and subset loads keep batch commits. A - // transaction begins only once its rows are read and - // deserialized, so a batch that fails leaves none open. + // transaction begins only once its rows are read, deserialized, + // and keyed, so a batch that fails leaves none open. const batches: Array> = [] const apply = () => { - const values = batches.splice(0).flat().map(deserializeSyncRow) + const entries = batches + .splice(0) + .flat() + .map((row) => { + const value = deserializeSyncRow(row) + return [getKey(value), value] as const + }) begin({ snapshot }) - for (const value of values) { - const key = getKey(value) + for (const [key, value] of entries) { onInitialRow?.(key) write({ type: writeType(key), value }) } @@ -575,6 +580,9 @@ function createPowerSyncCollectionConfig< id, ...JSON.parse(value), }) + // write() derives the key inside the transaction; a key getKey + // rejects must fail here, before the transaction opens. + getKey(parsedValue) const parsedPreviousValue = op.operation == DiffTriggerOperation.UPDATE ? deserializeSyncRow({ diff --git a/packages/powersync-db-collection/src/sqlite-compiler.ts b/packages/powersync-db-collection/src/sqlite-compiler.ts index 7b1c8c1819..a07fc969db 100644 --- a/packages/powersync-db-collection/src/sqlite-compiler.ts +++ b/packages/powersync-db-collection/src/sqlite-compiler.ts @@ -1,4 +1,5 @@ -import type { IR, LoadSubsetOptions } from '@tanstack/db' +import { IR } from '@tanstack/db' +import type { LoadSubsetOptions } from '@tanstack/db' /** * Result of compiling LoadSubsetOptions to SQLite @@ -192,9 +193,35 @@ function compileFunction( } } + // A numeric id equals a number exactly when its stored text is the number's + // canonical form, which every loaded numeric id is. Comparing that text, + // instead of CAST(id), lets SQLite look the id up in its primary key index. + const [column, value] = args + const idText = + compileOptions?.numericId && + (name === `eq` || name === `in`) && + column?.type === `ref` && + column.path.join() === `id` && + value?.type === `val` + // Compile arguments - const compiledArgs = args.map((arg: IR.BasicExpression) => - compileExpression(arg, params, compileOptions), + const compiledArgs = ( + idText + ? [ + column, + new IR.Value( + Array.isArray(value.value) + ? value.value.map(String) + : String(value.value), + ), + ] + : args + ).map((arg: IR.BasicExpression) => + compileExpression( + arg, + params, + idText ? { ...compileOptions, numericId: false } : compileOptions, + ), ) // Handle different operator types diff --git a/packages/powersync-db-collection/tests/numeric-keys.test.ts b/packages/powersync-db-collection/tests/numeric-keys.test.ts index 15b79ce8df..7d946d7d56 100644 --- a/packages/powersync-db-collection/tests/numeric-keys.test.ts +++ b/packages/powersync-db-collection/tests/numeric-keys.test.ts @@ -10,7 +10,7 @@ import { import { IR, createCollection } from '@tanstack/db' import { describe, expect, it, onTestFinished, vi } from 'vitest' import { z } from 'zod' -import { powerSyncCollectionOptions } from '../src' +import { compileSQLite, powerSyncCollectionOptions } from '../src' import type { PowerSyncLogger } from '@powersync/node' // SQLite stores every PowerSync id as text. A backend with integer primary @@ -39,6 +39,10 @@ const categoryEquals = (category: string) => const idGreaterThan = (id: number) => new IR.Func(`gt`, [new IR.PropRef([`id`]), new IR.Value(id)]) +const idEquals = (id: number) => + new IR.Func(`eq`, [new IR.PropRef([`id`]), new IR.Value(id)]) +const idIn = (ids: Array) => + new IR.Func(`in`, [new IR.PropRef([`id`]), new IR.Value(ids)]) async function createDatabase() { const errors: Array = [] @@ -153,29 +157,59 @@ describe(`numeric collection keys`, () => { expect(errors).toEqual([]) }) - // As text, '10' < '2'. The subset load, the trigger that tracks later - // writes, and eviction must all compare ids as numbers. - it(`compares ids as numbers when loading, tracking, and evicting`, async () => { - const { db, errors } = await createDatabase() - await db.execute( - `INSERT INTO products (id, name, category) VALUES ('10', 'E', 'z')`, - ) - const collection = createNumericCollection(db, `on-demand`) - const subset = { where: idGreaterThan(2) } - await collection._sync.loadSubset(subset) - expect([...collection.keys()].sort((a, b) => a - b)).toEqual([3, 10]) + // As text, '10' < '2', and a stored text id is never equal to a number. + // The subset load, the trigger that tracks later writes, and eviction must + // all compare ids as numbers. Each predicate below matches 3, 10, and 100 + // among the ids 1, 2, 3, 10, then 0 and 100. + it.each([ + [`gt(id, 2)`, idGreaterThan(2), [3, 10]], + [`in(id, [3, 10, 100])`, idIn([3, 10, 100]), [3, 10]], + [`eq(id, 100)`, idEquals(100), []], + ])( + `compares ids as numbers when loading, tracking, and evicting: %s`, + async (_, where, loaded) => { + const { db, errors } = await createDatabase() + await db.execute( + `INSERT INTO products (id, name, category) VALUES ('10', 'E', 'z')`, + ) + const collection = createNumericCollection(db, `on-demand`) + const subset = { where } + await collection._sync.loadSubset(subset) + expect([...collection.keys()].sort((a, b) => a - b)).toEqual(loaded) - // One write, so once 100 arrives the tracking has judged 0 as well. - await db.execute( - `INSERT INTO products (id, name, category) VALUES ('0', 'F', 'z'), ('100', 'G', 'z')`, - ) - await vi.waitFor(() => expect(collection.has(100)).toBe(true)) - expect(collection.has(0)).toBe(false) + // One write, so once 100 arrives the tracking has judged 0 as well. + await db.execute( + `INSERT INTO products (id, name, category) VALUES ('0', 'F', 'z'), ('100', 'G', 'z')`, + ) + await vi.waitFor(() => expect(collection.has(100)).toBe(true)) + expect(collection.has(0)).toBe(false) - collection._sync.unloadSubset(subset) - await vi.waitFor(() => expect(collection.size).toBe(0)) - expect(errors).toEqual([]) - }) + collection._sync.unloadSubset(subset) + await vi.waitFor(() => expect(collection.size).toBe(0)) + expect(errors).toEqual([]) + }, + ) + + // `id` is SQLite's text primary key. Equality on a numeric id compares the + // stored text, so a load by id, such as a join's `in`, uses the index. + // A numeric range needs the CAST and scans. + it.each([ + [`eq(id, 5)`, idEquals(5)], + [`in(id, [1, 2])`, idIn([1, 2])], + ])( + `looks up numeric id equality through the id index: %s`, + async (_, where) => { + const { db } = await createDatabase() + const compiled = compileSQLite({ where }, { numericId: true }) + const plan = await db.getAll<{ detail: string }>( + `EXPLAIN QUERY PLAN SELECT * FROM products WHERE ${compiled.where}`, + compiled.params, + ) + expect(plan.map(({ detail }) => detail).join(` | `)).toMatch( + /USING INDEX/, + ) + }, + ) it(`keeps held rows after a tracking outage`, async () => { vi.useFakeTimers() @@ -276,6 +310,50 @@ describe(`a row that fails deserialization`, () => { expect(collection.size).toBe(2) }) + // A JavaScript caller can omit numericIds for a schema that outputs number + // ids. Each row then deserializes but getKey rejects its key. The key is + // derived before a sync transaction opens, on the load path and on the path + // that syncs tracked writes. + it(`rejects a key without leaving a sync transaction open`, async () => { + const { db, errors } = await createDatabase() + const options = { + database: db, + table: APP_SCHEMA.props.products, + schema: row, + deserializationSchema: sqliteRow, + onDeserializationError: () => {}, + syncMode: `on-demand` as const, + } + // @ts-expect-error a number id needs numericIds + const config = powerSyncCollectionOptions(options) + const collection = createCollection(config) + onTestFinished(() => collection.cleanup()) + const openTransactions = () => + collection._state.pendingSyncedTransactions.filter( + (transaction) => !transaction.committed, + ).length + + // A tracked write. The subset starts empty, so its load succeeds. + await collection._sync.loadSubset({ where: categoryEquals(`new`) }) + await db.execute( + `INSERT INTO products (id, name, category) VALUES ('4', 'D', 'new')`, + ) + await vi.waitFor(() => + expect(errors).toContain( + `An error has been detected in the sync handler`, + ), + ) + expect(openTransactions()).toBe(0) + + // A load of existing rows. + await expect( + Promise.resolve( + collection._sync.loadSubset({ where: categoryEquals(`x`) }), + ), + ).rejects.toThrow(/set numericIds/) + expect(openTransactions()).toBe(0) + }) + it.each([`numeric`, `string`] as const)( `releases a subset that holds a permanently invalid row (%s ids)`, async (ids) => { diff --git a/packages/powersync-db-collection/tests/powersync.test-d.ts b/packages/powersync-db-collection/tests/powersync.test-d.ts index 9f4d7b93e5..838a677fd2 100644 --- a/packages/powersync-db-collection/tests/powersync.test-d.ts +++ b/packages/powersync-db-collection/tests/powersync.test-d.ts @@ -120,5 +120,17 @@ describe(`PowerSync collection type tests`, () => { // @ts-expect-error a text id cannot use numericIds numericIds: true, }) + + // An id that may be either type fits neither mode. + const mixed = schema.extend({ id: z.union([z.string(), z.number()]) }) + const mixedBase = { + ...base, + schema: mixed, + deserializationSchema: mixed.extend({ id: z.string() }), + } + // @ts-expect-error a mixed id cannot omit numericIds + powerSyncCollectionOptions(mixedBase) + // @ts-expect-error a mixed id cannot use numericIds + powerSyncCollectionOptions({ ...mixedBase, numericIds: true }) }) })