|
| 1 | +import { |
| 2 | + connectReaderClient, |
| 3 | + type HotStandby, |
| 4 | + hotStandbyTest, |
| 5 | + type ReaderClientDriver, |
| 6 | +} from "@internal/testcontainers"; |
| 7 | +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; |
| 8 | +import { describe, expect } from "vitest"; |
| 9 | +import { PostgresRunStore } from "./PostgresRunStore.js"; |
| 10 | +import { markReadReplicaClient } from "./readReplicaClient.js"; |
| 11 | +import { |
| 12 | + replicaRecoveryConflictSqlstate, |
| 13 | + withReplicaRecoveryConflictRetry, |
| 14 | +} from "./replicaRecoveryConflictRetry.js"; |
| 15 | +import { seedRecoveryConflictRun } from "./testFixtures/recoveryConflictRun.js"; |
| 16 | + |
| 17 | +// The same retry the plain-Postgres tests drive with a gate, here against cancellations a real hot |
| 18 | +// standby raises during WAL replay. |
| 19 | + |
| 20 | +const DRIVERS: ReaderClientDriver[] = ["engine", "adapter"]; |
| 21 | + |
| 22 | +async function caught(promise: Promise<unknown>): Promise<unknown> { |
| 23 | + try { |
| 24 | + await promise; |
| 25 | + } catch (error) { |
| 26 | + return error; |
| 27 | + } |
| 28 | + throw new Error("expected the read to be cancelled, but it succeeded"); |
| 29 | +} |
| 30 | + |
| 31 | +async function poll(what: string, check: () => boolean, timeoutMs = 5_000) { |
| 32 | + const deadline = Date.now() + timeoutMs; |
| 33 | + while (!check()) { |
| 34 | + if (Date.now() > deadline) throw new Error(`timed out waiting for ${what}`); |
| 35 | + await new Promise((r) => setTimeout(r, 10)); |
| 36 | + } |
| 37 | +} |
| 38 | + |
| 39 | +async function seed(hotStandby: HotStandby) { |
| 40 | + const writer = connectReaderClient(hotStandby.primaryUrl, "engine"); |
| 41 | + try { |
| 42 | + return await seedRecoveryConflictRun(writer.client); |
| 43 | + } finally { |
| 44 | + await writer.disconnect(); |
| 45 | + await hotStandby.waitForReplay(); |
| 46 | + } |
| 47 | +} |
| 48 | + |
| 49 | +function wrappedReplica(client: RunOpsPrismaClient) { |
| 50 | + const retries: Array<{ model?: string; operation: string; attempt: number; sqlstate?: string }> = |
| 51 | + []; |
| 52 | + const replica = markReadReplicaClient( |
| 53 | + withReplicaRecoveryConflictRetry(client, { |
| 54 | + maxAttempts: 3, |
| 55 | + backoffMinMs: 25, |
| 56 | + backoffMaxMs: 100, |
| 57 | + onRetry: (info) => retries.push(info), |
| 58 | + }) |
| 59 | + ); |
| 60 | + return { replica, retries }; |
| 61 | +} |
| 62 | + |
| 63 | +describe("replica recovery-conflict retry on a real hot standby", () => { |
| 64 | + hotStandbyTest( |
| 65 | + "a lock conflict (40001) is recognised in every shape, and a run read through the store recovers", |
| 66 | + { timeout: 300_000 }, |
| 67 | + async ({ hotStandby }) => { |
| 68 | + const run = await seed(hotStandby); |
| 69 | + |
| 70 | + for (const driver of DRIVERS) { |
| 71 | + const reader = connectReaderClient(hotStandby.standbyUrl, driver); |
| 72 | + const bare = reader.client; |
| 73 | + try { |
| 74 | + // Connect first, so the read under test is the one that blocks. |
| 75 | + await bare.taskRun.findFirst({ where: { id: run.runId }, select: { id: true } }); |
| 76 | + |
| 77 | + // The bare client surfaces the cancellation in the shapes the classifier expects. |
| 78 | + for (const kind of ["model", "raw"] as const) { |
| 79 | + const before = await hotStandby.conflicts(); |
| 80 | + const conflict = await hotStandby.armLockConflict("TaskRun"); |
| 81 | + try { |
| 82 | + const cancelled = caught( |
| 83 | + kind === "model" |
| 84 | + ? bare.taskRun.findFirst({ where: { id: run.runId }, select: { id: true } }) |
| 85 | + : bare.$queryRawUnsafe(`SELECT id FROM "TaskRun" LIMIT 1`) |
| 86 | + ); |
| 87 | + await conflict.waitForBlockedRead(); |
| 88 | + await conflict.fire(); |
| 89 | + const error = await cancelled; |
| 90 | + |
| 91 | + expect(error, `${driver} ${kind}`).toMatchObject( |
| 92 | + kind === "model" |
| 93 | + ? { name: "PrismaClientKnownRequestError", code: "P2034" } |
| 94 | + : { |
| 95 | + name: "PrismaClientKnownRequestError", |
| 96 | + code: "P2010", |
| 97 | + meta: { code: "40001" }, |
| 98 | + } |
| 99 | + ); |
| 100 | + expect(replicaRecoveryConflictSqlstate(error), `${driver} ${kind}`).toBe("40001"); |
| 101 | + expect((await hotStandby.conflicts()).lock, driver).toBe(before.lock + 1); |
| 102 | + } finally { |
| 103 | + await conflict.release(); |
| 104 | + } |
| 105 | + } |
| 106 | + |
| 107 | + // Through the wrapped replica and the store, the cancelled read is retried. The retry |
| 108 | + // blocks on the still-armed lock and completes once it is released. |
| 109 | + const { replica, retries } = wrappedReplica(bare); |
| 110 | + const store = new PostgresRunStore({ |
| 111 | + prisma: bare as never, |
| 112 | + readOnlyPrisma: replica as never, |
| 113 | + schemaVariant: "dedicated", |
| 114 | + }); |
| 115 | + const before = await hotStandby.conflicts(); |
| 116 | + const conflict = await hotStandby.armLockConflict("TaskRun"); |
| 117 | + try { |
| 118 | + const found = store.findRun({ id: run.runId }, { select: { id: true } }); |
| 119 | + await conflict.waitForBlockedRead(); |
| 120 | + await conflict.fire(); |
| 121 | + await poll(`${driver}: the retry`, () => retries.length === 1); |
| 122 | + await conflict.release(); |
| 123 | + |
| 124 | + await expect(found, driver).resolves.toEqual({ id: run.runId }); |
| 125 | + expect(retries, driver).toEqual([ |
| 126 | + expect.objectContaining({ model: "TaskRun", attempt: 1, sqlstate: "40001" }), |
| 127 | + ]); |
| 128 | + expect((await hotStandby.conflicts()).lock, driver).toBe(before.lock + 1); |
| 129 | + } finally { |
| 130 | + await conflict.release(); |
| 131 | + } |
| 132 | + } finally { |
| 133 | + await reader.disconnect(); |
| 134 | + } |
| 135 | + } |
| 136 | + } |
| 137 | + ); |
| 138 | + |
| 139 | + // The backend-side buffer deadlock: replay already waits on a buffer pin the read's transaction |
| 140 | + // holds (an open cursor) when the read starts waiting on a lock replay holds. A plain model read |
| 141 | + // can't be driven into this ordering on demand, so this checks the 40P01 shape a real standby |
| 142 | + // produces, and that a read inside a transaction is not retried. |
| 143 | + hotStandbyTest( |
| 144 | + "a buffer deadlock with replay (40P01) is recognised, and is not retried inside a transaction", |
| 145 | + { timeout: 300_000 }, |
| 146 | + async ({ hotStandby }) => { |
| 147 | + await seed(hotStandby); |
| 148 | + await hotStandby.primarySql([ |
| 149 | + "CREATE TABLE hot_standby_pin (id int PRIMARY KEY, v int NOT NULL)", |
| 150 | + "INSERT INTO hot_standby_pin SELECT i, 0 FROM generate_series(1, 20) i", |
| 151 | + ]); |
| 152 | + // Long enough for the standby's deadlock check to run before replay gives up waiting. |
| 153 | + await hotStandby.setMaxStandbyStreamingDelay(30_000); |
| 154 | + |
| 155 | + let n = 0; |
| 156 | + for (const driver of DRIVERS) { |
| 157 | + for (const kind of ["model", "raw"] as const) { |
| 158 | + const label = `${driver} ${kind}`; |
| 159 | + // Fresh unfrozen tuples, so the VACUUM FREEZE below logs a freeze that needs the pin. |
| 160 | + await hotStandby.primarySql("UPDATE hot_standby_pin SET v = v + 1"); |
| 161 | + await hotStandby.waitForReplay(); |
| 162 | + |
| 163 | + const reader = connectReaderClient(hotStandby.standbyUrl, driver); |
| 164 | + const { replica, retries } = wrappedReplica(reader.client); |
| 165 | + const gid = `pin_${++n}`; |
| 166 | + try { |
| 167 | + const error = await caught( |
| 168 | + replica.$transaction( |
| 169 | + async (tx) => { |
| 170 | + await tx.$executeRawUnsafe( |
| 171 | + "DECLARE pin CURSOR FOR SELECT id FROM hot_standby_pin" |
| 172 | + ); |
| 173 | + await tx.$queryRawUnsafe("FETCH FORWARD 1 FROM pin"); |
| 174 | + await hotStandby.primarySql([ |
| 175 | + "BEGIN", |
| 176 | + `LOCK TABLE "TaskRun" IN ACCESS EXCLUSIVE MODE`, |
| 177 | + `PREPARE TRANSACTION '${gid}'`, |
| 178 | + ]); |
| 179 | + // Replay takes the lock, then waits on our pin to apply the freeze. |
| 180 | + await hotStandby.primarySql("VACUUM FREEZE hot_standby_pin"); |
| 181 | + await hotStandby.waitForReplayBlockedOnPin(); |
| 182 | + await (kind === "model" |
| 183 | + ? tx.taskRun.findFirst({ select: { id: true } }) |
| 184 | + : tx.$queryRawUnsafe(`SELECT id FROM "TaskRun" LIMIT 1`)); |
| 185 | + }, |
| 186 | + { timeout: 20_000, maxWait: 10_000 } |
| 187 | + ) |
| 188 | + ); |
| 189 | + |
| 190 | + expect(error, label).toMatchObject( |
| 191 | + kind === "model" |
| 192 | + ? { name: "PrismaClientUnknownRequestError" } |
| 193 | + : { name: "PrismaClientKnownRequestError", code: "P2010", meta: { code: "40P01" } } |
| 194 | + ); |
| 195 | + expect(String((error as Error).message), label).toMatch(/conflict with recovery/); |
| 196 | + expect(replicaRecoveryConflictSqlstate(error), label).toBe("40P01"); |
| 197 | + expect(retries, label).toEqual([]); |
| 198 | + } finally { |
| 199 | + await hotStandby.primarySql(`COMMIT PREPARED '${gid}'`).catch(() => {}); |
| 200 | + await reader.disconnect(); |
| 201 | + } |
| 202 | + } |
| 203 | + } |
| 204 | + } |
| 205 | + ); |
| 206 | +}); |
0 commit comments