diff --git a/.agents/skills/pgpm-projections/SKILL.md b/.agents/skills/pgpm-projections/SKILL.md index 54b2c8df2d..2b5f6197cc 100644 --- a/.agents/skills/pgpm-projections/SKILL.md +++ b/.agents/skills/pgpm-projections/SKILL.md @@ -117,30 +117,52 @@ pgpm diff --verify way to keep dependents intact. - Every emitted change has deploy, revert, and verify. Steps pgpm cannot derive (a DROP it does not know, an inverse that does not exist such as - `ALTER TYPE .. ADD VALUE`) are a hard error naming the object; only - `--allow-incomplete` emits them, with a `-- pgpm:` marker and a warning. + `ALTER TYPE .. ADD VALUE`, an enum value added and used by another step of + the same migration — Postgres cannot use it before the adding transaction + commits, so deploy with `--no-tx` or split the migration — a composite + attribute inserted mid-list, which `ADD ATTRIBUTE` can only append) are a + hard error naming the object; only `--allow-incomplete` emits them, with a + `-- pgpm:` marker and a warning. - Steps that can lose data (DROP TABLE/COLUMN/SCHEMA/SEQUENCE, any column type change, a type/domain/sequence that must be dropped and recreated) are a hard error listing each step; only `--allow-data-loss` emits them. Renames are never inferred from a drop/add pair: `--rename schema.table[.column]=…` emits `ALTER .. RENAME` with its inverse (and rebuilds dependents that name - the column around it); an undeclared rename is a refused column drop. + the column around it); an undeclared rename is a refused column drop. A + table rename also renames the constraints (and their indexes) Postgres named + after the old table (`old_pkey` → `new_pkey`), rewrites qualified references + in views, policies, comments, grants and `regclass` literals, and keeps a + changed function body (text Postgres does not rewrite) restorable by folding + its inverse into the rename's. A column of a table renamed in the same diff + is declared with either name: `--rename app.old=app.new --rename + app.old.col=app.new.name`. `--rename` never moves a column to another table + or schema. - Column attribute changes emit exact ALTERs with exact inverses: `SET/DROP DEFAULT`, `SET/DROP NOT NULL`, `ADD/DROP IDENTITY`, `ALTER COLUMN TYPE .. USING col::type` (or the `--using` expression; the inverse always casts back - with `col::oldtype`), generated-expression changes as DROP + ADD COLUMN + with `col::oldtype`; a `--using` value must be one expression, no top-level + comma), generated-expression changes as DROP + ADD COLUMN (derived data). Column-level `DEFERRABLE` / `INITIALLY DEFERRED` fold into their constraint. - Objects with dependents alter in place instead of DROP + CREATE: enum values - append (`ALTER TYPE .. ADD VALUE`), domains (`ALTER DOMAIN` constraints / - default / NOT NULL), sequences (`ALTER SEQUENCE`), composite types (`ADD / - DROP / ALTER ATTRIBUTE`). A function whose signature uses a dropped or + append (`ALTER TYPE .. ADD VALUE`, `BEFORE`/`AFTER` only a value that already + exists), domains (`ALTER DOMAIN` constraints / default / NOT NULL), sequences + (`ALTER SEQUENCE`, including the `START` Postgres derives from a dropped + `MINVALUE`/`MAXVALUE`), composite types (`ADD / DROP / ALTER ATTRIBUTE`). A function whose signature uses a dropped or altered type is dropped and recreated around it (its body is validated against the type's current shape in both directions). - COMMENT ON, GRANT/REVOKE, OWNER TO and ALTER DEFAULT PRIVILEGES are keyed state on their object: the diff emits the exact `COMMENT` / `GRANT` / `REVOKE` / `ALTER DEFAULT PRIVILEGES` with the previous state as inverse, - never a rebuild of the object. + never a rebuild of the object. An explicit `REVOKE .. FROM PUBLIC` is state + too: dropping it re-grants only what PUBLIC holds by default (EXECUTE on + functions, USAGE on types, CONNECT/TEMPORARY on databases; nothing on + tables, sequences or schemas), never `GRANT ALL`, so a table's or schema's + dropped revoke emits no GRANT at all; the inverse of an added revoke + likewise restores only that default (a comment when there is none). A table rename rewrites + `old.col` references only where `old` names the table itself, never another + relation's alias; a view/policy rebuilt around a renamed table is dropped + under the old name and recreated on revert from its text as written. - A module compares equal to the live database it deployed: type spellings (`integer`/`int4`, `timestamp with time zone`/`timestamptz`, …), catalog default casts (`'a'::app.status`, `nextval('s'::regclass)`, `'x'::text`), diff --git a/pgpm/cli/__tests__/diff-followups-e2e.test.ts b/pgpm/cli/__tests__/diff-followups-e2e.test.ts new file mode 100644 index 0000000000..80ea5fde8c --- /dev/null +++ b/pgpm/cli/__tests__/diff-followups-e2e.test.ts @@ -0,0 +1,274 @@ +/** + * e2e for the `pgpm diff` review follow-ups against live Postgres: enum values + * inserted at the head of the list, a composite attribute inserted mid-list, + * a removed MINVALUE (implied START), a removed `REVOKE ALL FROM PUBLIC`, a + * column rename on a table renamed in the same diff, and references to a + * renamed table from comments, views, policies and functions. Each migration + * deploys on v1 to the catalog of v2 and reverts back to v1. + * + * PREREQUISITES: a running PostgreSQL instance via standard PG* env vars. + */ +import { diffCatalogSnapshots, loadModule, snapshotCatalog, withoutColumnOrder } from '@pgpmjs/transform'; +import * as fs from 'fs'; +import * as path from 'path'; +import { teardownPgPools } from 'pg-cache'; + +import { CLIDeployTestFixture } from '../test-utils'; + +jest.setTimeout(240000); + +afterAll(async () => { + await teardownPgPools(); +}); + +const WS = 'fu-ws'; + +const writeModule = (moduleDir: string, name: string, sql: string) => { + fs.mkdirSync(path.join(moduleDir, 'deploy'), { recursive: true }); + fs.mkdirSync(path.join(moduleDir, 'revert'), { recursive: true }); + fs.mkdirSync(path.join(moduleDir, 'verify'), { recursive: true }); + fs.writeFileSync( + path.join(moduleDir, 'pgpm.plan'), + `%syntax-version=1.0.0\n%project=${name}\n%uri=https://github.com/test/${name}\n\ninit 2024-01-01T00:00:00Z test # add init\n` + ); + fs.writeFileSync( + path.join(moduleDir, `${name}.control`), + `comment = 'Diff follow-ups e2e module'\ndefault_version = '0.0.1'\nrelocatable = false\nsuperuser = false\n` + ); + fs.writeFileSync(path.join(moduleDir, 'deploy', 'init.sql'), `-- Deploy init to pg\n\nBEGIN;\n\n${sql}\n\nCOMMIT;\n`); + fs.writeFileSync(path.join(moduleDir, 'revert', 'init.sql'), `-- Revert init from pg\n\nBEGIN;\n\nDROP SCHEMA fu CASCADE;\n\nCOMMIT;\n`); + fs.writeFileSync(path.join(moduleDir, 'verify', 'init.sql'), `-- Verify init on pg\n\nBEGIN;\n\nROLLBACK;\n`); +}; + +interface Case { + name: string; + v1: string; + v2: string; + flags: string; + rows?: string; + deployed: RegExp[]; + notDeployed?: RegExp[]; + /** Catalog differences that remain after the revert (Postgres cannot remove enum values). */ + revertResidual?: RegExp[]; + /** Catalog differences that remain after the forward migration (documented, reported as incomplete). */ + forwardResidual?: RegExp[]; +} + +const CASES: Case[] = [ + { + name: 'enum-head', + v1: `CREATE SCHEMA fu; CREATE TYPE fu.status AS ENUM ('a'); CREATE TABLE fu.t (id int PRIMARY KEY, s fu.status);`, + v2: `CREATE SCHEMA fu; CREATE TYPE fu.status AS ENUM ('x', 'y', 'a'); CREATE TABLE fu.t (id int PRIMARY KEY, s fu.status);`, + flags: '--allow-incomplete', + deployed: [/ADD VALUE 'x' BEFORE 'a'/, /ADD VALUE 'y' AFTER 'x'/], + revertResidual: [/types: fu\.status differs/] + }, + { + name: 'composite-mid', + v1: `CREATE SCHEMA fu; CREATE TYPE fu.pt AS (a int, b int); CREATE TABLE fu.t (id int PRIMARY KEY, p fu.pt);`, + v2: `CREATE SCHEMA fu; CREATE TYPE fu.pt AS (a int, c int, b int); CREATE TABLE fu.t (id int PRIMARY KEY, p fu.pt);`, + flags: '--allow-incomplete', + deployed: [/ALTER TYPE fu\.pt ADD ATTRIBUTE c/], + // ADD ATTRIBUTE appends: the attribute order differs from v2 and pgpm says so (incomplete). + forwardResidual: [/types: fu\.pt differs/] + }, + { + name: 'sequence-minvalue', + v1: `CREATE SCHEMA fu; CREATE SEQUENCE fu.s MINVALUE 5 MAXVALUE 1000 CACHE 10;`, + v2: `CREATE SCHEMA fu; CREATE SEQUENCE fu.s;`, + flags: '', + deployed: [/NO MINVALUE/, /START (WITH )?1\b/] + }, + { + name: 'revoke-public', + v1: [ + 'CREATE SCHEMA fu; CREATE TABLE fu.t (id int PRIMARY KEY); REVOKE ALL ON TABLE fu.t FROM PUBLIC;', + 'CREATE FUNCTION fu.f() RETURNS int LANGUAGE sql AS $$ SELECT 1 $$; REVOKE ALL ON FUNCTION fu.f() FROM PUBLIC;' + ].join('\n'), + v2: `CREATE SCHEMA fu; CREATE TABLE fu.t (id int PRIMARY KEY); CREATE FUNCTION fu.f() RETURNS int LANGUAGE sql AS $$ SELECT 1 $$;`, + flags: '', + deployed: [/GRANT EXECUTE ON FUNCTION fu\.f\(\) TO PUBLIC/], + notDeployed: [/GRANT ALL/] + }, + { + name: 'rename-table-and-column', + v1: `CREATE SCHEMA fu; CREATE TABLE fu.old (id int PRIMARY KEY, col text);`, + v2: `CREATE SCHEMA fu; CREATE TABLE fu.new (id int PRIMARY KEY, name text);`, + flags: '--rename fu.old=fu.new --rename fu.old.col=fu.new.name', + rows: `INSERT INTO fu.old VALUES (1, 'kept')`, + deployed: [/ALTER TABLE fu\.old RENAME TO new/, /RENAME (COLUMN )?col TO name/], + notDeployed: [/DROP COLUMN/, /DROP TABLE/] + }, + { + name: 'rename-references', + v1: [ + 'CREATE SCHEMA fu; CREATE TABLE fu.old (id int PRIMARY KEY, col text);', + "COMMENT ON TABLE fu.old IS 'the table'; COMMENT ON COLUMN fu.old.col IS 'the col';", + 'GRANT SELECT ON TABLE fu.old TO PUBLIC;', + 'ALTER TABLE fu.old ENABLE ROW LEVEL SECURITY; CREATE POLICY p ON fu.old FOR SELECT USING (true);', + 'CREATE VIEW fu.v AS SELECT old.id, old.col FROM fu.old;', + 'CREATE FUNCTION fu.f() RETURNS bigint LANGUAGE sql AS $$ SELECT count(*) FROM fu.old $$;' + ].join('\n'), + v2: [ + 'CREATE SCHEMA fu; CREATE TABLE fu.new (id int PRIMARY KEY, col text);', + "COMMENT ON TABLE fu.new IS 'the table'; COMMENT ON COLUMN fu.new.col IS 'the col';", + 'GRANT SELECT ON TABLE fu.new TO PUBLIC;', + 'ALTER TABLE fu.new ENABLE ROW LEVEL SECURITY; CREATE POLICY p ON fu.new FOR SELECT USING (true);', + 'CREATE VIEW fu.v AS SELECT new.id, new.col FROM fu.new;', + 'CREATE FUNCTION fu.f() RETURNS bigint LANGUAGE sql AS $$ SELECT count(*) FROM fu.new $$;' + ].join('\n'), + flags: '--rename fu.old=fu.new', + deployed: [/ALTER TABLE fu\.old RENAME TO new/, /CREATE OR REPLACE FUNCTION fu\.f\(\)/], + notDeployed: [/COMMENT ON/, /DROP VIEW/, /DROP POLICY/, /REVOKE/] + }, + { + name: 'rename-rebuild', + v1: 'CREATE SCHEMA fu; CREATE TABLE fu.old (id int PRIMARY KEY, col text); CREATE VIEW fu.v AS SELECT old.id, old.col FROM fu.old;', + v2: 'CREATE SCHEMA fu; CREATE TABLE fu.new (id int PRIMARY KEY, col int); CREATE VIEW fu.v AS SELECT new.id, new.col FROM fu.new;', + flags: '--rename fu.old=fu.new --allow-data-loss', + deployed: [/ALTER TABLE fu\.old RENAME TO new/, /DROP VIEW fu\.v/, /ALTER COLUMN col TYPE int/, /CREATE VIEW fu\.v/] + }, + { + name: 'rename-alias', + v1: 'CREATE SCHEMA fu; CREATE TABLE fu.old (id int PRIMARY KEY); CREATE TABLE fu.other (id int PRIMARY KEY, n int); CREATE VIEW fu.v AS SELECT old.id, old.n FROM fu.other AS old;', + v2: 'CREATE SCHEMA fu; CREATE TABLE fu.new (id int PRIMARY KEY); CREATE TABLE fu.other (id int PRIMARY KEY, n int); CREATE VIEW fu.v AS SELECT old.id, old.n FROM fu.other AS old;', + flags: '--rename fu.old=fu.new', + deployed: [/ALTER TABLE fu\.old RENAME TO new/], + notDeployed: [/VIEW/] + } +]; + +describe('pgpm diff e2e: review follow-ups', () => { + let fixture: CLIDeployTestFixture; + let wsDir: string; + + beforeAll(async () => { + await loadModule(); + fixture = new CLIDeployTestFixture(); + wsDir = path.join(fixture.tempFixtureDir, WS); + fs.mkdirSync(wsDir, { recursive: true }); + fs.writeFileSync(path.join(wsDir, 'pgpm.json'), '{\n "packages": [\n "*"\n ]\n}'); + }); + + afterAll(async () => { + await fixture.cleanup(); + }); + + for (const c of CASES) { + it(`${c.name}: migrates v1 to v2 and reverts to v1`, async () => { + const v1 = `${c.name}-v1`; + const v2 = `${c.name}-v2`; + const mig = `${c.name}-mig`; + writeModule(path.join(wsDir, v1), v1, c.v1); + writeModule(path.join(wsDir, v2), v2, c.v2); + const v2Db = await fixture.setupTestDatabase(); + const testDb = await fixture.setupTestDatabase(); + for (const [db, pkg] of [[v2Db, v2], [testDb, v1]] as const) { + await fixture.runTerminalCommands(` + cd ${WS}/${pkg} + pgpm deploy --database ${db.name} --package ${pkg} --yes + `, { database: db.name }); + } + const v1Catalog = withoutColumnOrder(await snapshotCatalog(testDb)); + if (c.rows) await testDb.query(c.rows); + + await fixture.runTerminalCommands(` + cd ${WS} + pgpm diff ${v1} ${v2} ${c.flags} --emit-migration . --pkg ${mig} + `, {}); + const migrationDir = path.join(wsDir, mig); + const deployed = fs.readdirSync(path.join(migrationDir, 'deploy'), { recursive: true }) + .map(String).filter(f => f.endsWith('.sql')) + .map(f => fs.readFileSync(path.join(migrationDir, 'deploy', f), 'utf-8')).join('\n'); + for (const re of c.deployed) expect(deployed).toMatch(re); + for (const re of c.notDeployed ?? []) expect(deployed).not.toMatch(re); + + await fixture.runTerminalCommands(` + cd ${WS}/${mig} + pgpm deploy --database ${testDb.name} --package ${mig} --yes + `, { database: testDb.name }); + const forward = diffCatalogSnapshots(withoutColumnOrder(await snapshotCatalog(testDb)), withoutColumnOrder(await snapshotCatalog(v2Db))); + expect(forward).toEqual((c.forwardResidual ?? []).map(re => expect.stringMatching(re))); + if (c.rows) { + const rows = await testDb.query('SELECT id, name FROM fu.new ORDER BY id'); + expect(rows.rows).toEqual([{ id: 1, name: 'kept' }]); + } + + await fixture.runTerminalCommands(` + cd ${WS}/${mig} + pgpm revert --database ${testDb.name} --package ${mig} --yes --no-recursive + `, { database: testDb.name }); + const back = diffCatalogSnapshots(withoutColumnOrder(await snapshotCatalog(testDb)), v1Catalog); + expect(back).toEqual((c.revertResidual ?? []).map(re => expect.stringMatching(re))); + }); + } + + it('an added REVOKE .. FROM PUBLIC reverts to the privileges PUBLIC actually held (none on a schema, EXECUTE on a function)', async () => { + const v1 = 'revoke-added-v1'; + const v2 = 'revoke-added-v2'; + const base = 'CREATE SCHEMA fu; CREATE FUNCTION fu.f() RETURNS int LANGUAGE sql AS $$ SELECT 1 $$;'; + writeModule(path.join(wsDir, v1), v1, base); + writeModule(path.join(wsDir, v2), v2, `${base} REVOKE ALL ON SCHEMA fu FROM PUBLIC; REVOKE ALL ON FUNCTION fu.f() FROM PUBLIC;`); + const testDb = await fixture.setupTestDatabase(); + await fixture.runTerminalCommands(` + cd ${WS}/${v1} + pgpm deploy --database ${testDb.name} --package ${v1} --yes + `, { database: testDb.name }); + // Effective privileges: a NULL acl is the owner's default, spelled out. + const acls = async (): Promise => (await testDb.query(` + SELECT 'schema' AS obj, a.grantee::regrole::text AS grantee, a.privilege_type + FROM pg_namespace n, aclexplode(coalesce(n.nspacl, acldefault('n', n.nspowner))) a WHERE n.nspname = 'fu' + UNION ALL + SELECT 'function', a.grantee::regrole::text, a.privilege_type + FROM pg_proc p, aclexplode(coalesce(p.proacl, acldefault('f', p.proowner))) a + WHERE p.proname = 'f' AND p.pronamespace = 'fu'::regnamespace + ORDER BY 1, 2, 3`)).rows; + const before = await acls(); + await fixture.runTerminalCommands(` + cd ${WS} + pgpm diff ${v1} ${v2} --emit-migration . --pkg revoke-added-mig + `, {}); + await fixture.runTerminalCommands(` + cd ${WS}/revoke-added-mig + pgpm deploy --database ${testDb.name} --package revoke-added-mig --yes + `, { database: testDb.name }); + expect(await acls()).not.toEqual(before); + await fixture.runTerminalCommands(` + cd ${WS}/revoke-added-mig + pgpm revert --database ${testDb.name} --package revoke-added-mig --yes --no-recursive + `, { database: testDb.name }); + expect(await acls()).toEqual(before); + }); + + it('an enum value added and used in the same migration is named: Postgres cannot use it before the transaction commits', async () => { + const v1 = 'enum-used-v1'; + const v2 = 'enum-used-v2'; + writeModule(path.join(wsDir, v1), v1, `CREATE SCHEMA fu; CREATE TYPE fu.status AS ENUM ('a', 'b'); CREATE TABLE fu.t (id int PRIMARY KEY, s fu.status NOT NULL DEFAULT 'a');`); + writeModule(path.join(wsDir, v2), v2, `CREATE SCHEMA fu; CREATE TYPE fu.status AS ENUM ('a', 'b', 'c'); CREATE TABLE fu.t (id int PRIMARY KEY, s fu.status NOT NULL DEFAULT 'c');`); + await fixture.runTerminalCommands(` + cd ${WS} + pgpm diff ${v1} ${v2} --allow-incomplete --emit-migration . --pkg enum-used-mig + `, {}); + const deployed = fs.readdirSync(path.join(wsDir, 'enum-used-mig', 'deploy'), { recursive: true }) + .map(String).filter(f => f.endsWith('.sql')) + .map(f => fs.readFileSync(path.join(wsDir, 'enum-used-mig', 'deploy', f), 'utf-8')).join('\n'); + expect(deployed).toMatch(/--no-tx/); + // Without a transaction per deploy, the migration applies and reaches v2. + const v2Db = await fixture.setupTestDatabase(); + const testDb = await fixture.setupTestDatabase(); + await fixture.runTerminalCommands(` + cd ${WS}/${v2} + pgpm deploy --database ${v2Db.name} --package ${v2} --yes + `, { database: v2Db.name }); + await fixture.runTerminalCommands(` + cd ${WS}/${v1} + pgpm deploy --database ${testDb.name} --package ${v1} --yes + `, { database: testDb.name }); + await fixture.runTerminalCommands(` + cd ${WS}/enum-used-mig + pgpm deploy --database ${testDb.name} --package enum-used-mig --yes --no-tx + `, { database: testDb.name }); + const forward = diffCatalogSnapshots(withoutColumnOrder(await snapshotCatalog(testDb)), withoutColumnOrder(await snapshotCatalog(v2Db))); + expect(forward).toEqual([]); + }); +}); diff --git a/pgpm/cli/test-utils/CLIDeployTestFixture.ts b/pgpm/cli/test-utils/CLIDeployTestFixture.ts index 2543eae216..04db074833 100644 --- a/pgpm/cli/test-utils/CLIDeployTestFixture.ts +++ b/pgpm/cli/test-utils/CLIDeployTestFixture.ts @@ -223,7 +223,9 @@ export class CLIDeployTestFixture extends TestFixture { argv[baseKey] = false; i += 1; } else if (i + 1 < tokens.length && !tokens[i + 1].startsWith('--')) { - argv[key] = tokens[i + 1]; + // Repeated flags accumulate, as minimist does (`--rename a=b --rename c=d`). + const prior = argv[key]; + argv[key] = prior === undefined ? tokens[i + 1] : [...(Array.isArray(prior) ? prior : [prior]), tokens[i + 1]]; i += 2; } else { argv[key] = true; diff --git a/pgpm/transform/__tests__/semantic-diff-hardening.test.ts b/pgpm/transform/__tests__/semantic-diff-hardening.test.ts index bc4e45b310..2a6c7825cf 100644 --- a/pgpm/transform/__tests__/semantic-diff-hardening.test.ts +++ b/pgpm/transform/__tests__/semantic-diff-hardening.test.ts @@ -7,7 +7,7 @@ */ import { loadModule } from 'plpgsql-parser'; -import { diffSchemas as diffSchemasTyped, SemanticDiffOptions } from '../src/semantic-diff-driver'; +import { diffSchemas as diffSchemasTyped, parseRenameDeclaration, SemanticDiffOptions } from '../src/semantic-diff-driver'; // Options are typed loosely so this file compiles against the pre-fix driver too. const diffSchemas = (from: string, to: string, options?: Record): ReturnType => @@ -253,3 +253,180 @@ describe('diffSchemas: module vs dump representations', () => { expect(diffSchemas(from, to).identical).toBe(true); }); }); + +describe('diffSchemas: review follow-ups', () => { + it('inserts several enum values at the head referencing only labels that exist at that point', () => { + const from = `${S}\nCREATE TYPE app.status AS ENUM ('a');`; + const to = `${S}\nCREATE TYPE app.status AS ENUM ('x', 'y', 'a');`; + const deploy = norm(sql(diffSchemas(from, to, lax), 'deploy')); + expect(deploy).toMatch(/ADD VALUE 'x' BEFORE 'a'/); + expect(deploy).toMatch(/ADD VALUE 'y' AFTER 'x'/); + expect(deploy).not.toMatch(/BEFORE 'y'/); + }); + + it('names an enum value that is added and used in the same migration (Postgres rejects it in one transaction)', () => { + const from = `${S}\nCREATE TYPE app.status AS ENUM ('a');\nCREATE TABLE app.t (id int PRIMARY KEY, s app.status NOT NULL DEFAULT 'a');`; + const to = `${S}\nCREATE TYPE app.status AS ENUM ('a', 'c');\nCREATE TABLE app.t (id int PRIMARY KEY, s app.status NOT NULL DEFAULT 'c');`; + expect(() => diffSchemas(from, to)).toThrow(/enum value 'c'.*same migration/); + const result = diffSchemas(from, to, lax); + expect(result.warnings.join('\n')).toMatch(/enum value 'c'.*same migration.*--no-tx/); + expect(sql(result, 'deploy')).toMatch(/--no-tx/); + }); + + it('reports a composite attribute inserted mid-list as incomplete (ADD ATTRIBUTE appends)', () => { + const from = `${S}\nCREATE TYPE app.pt AS (a int, b int);`; + const to = `${S}\nCREATE TYPE app.pt AS (a int, c int, b int);`; + expect(() => diffSchemas(from, to)).toThrow(/app\.pt.*attribute "c".*append/); + const result = diffSchemas(from, to, lax); + expect(norm(sql(result, 'deploy'))).toMatch(/ALTER TYPE app\.pt ADD ATTRIBUTE c int(eger)?/); + expect(norm(sql(result, 'revert'))).toMatch(/ALTER TYPE app\.pt DROP ATTRIBUTE c/); + expect(result.warnings.join('\n')).toMatch(/attribute "c".*append/); + }); + + it('resets a removed MINVALUE/MAXVALUE together with the START it implied', () => { + const from = `${S}\nCREATE SEQUENCE app.s MINVALUE 5 MAXVALUE 1000 CACHE 10;`; + const to = `${S}\nCREATE SEQUENCE app.s;`; + const result = diffSchemas(from, to); + expect(result.warnings).toEqual([]); + const deploy = norm(sql(result, 'deploy')); + const revert = norm(sql(result, 'revert')); + expect(deploy).toMatch(/ALTER SEQUENCE app\.s .*NO MINVALUE/); + expect(deploy).toMatch(/NO MAXVALUE/); + expect(deploy).toMatch(/START (WITH )?1\b/); + expect(deploy).toMatch(/CACHE 1\b/); + expect(revert).toMatch(/MINVALUE 5/); + expect(revert).toMatch(/MAXVALUE 1000/); + expect(revert).toMatch(/START (WITH )?5\b/); + expect(revert).toMatch(/CACHE 10/); + }); + + it('rejects a --using expression with a top-level comma or a parse error, naming the flag', () => { + const from = `${S}\nCREATE TABLE app.t (id int PRIMARY KEY, code text);`; + const to = `${S}\nCREATE TABLE app.t (id int PRIMARY KEY, code int);`; + expect(() => diffSchemas(from, to, { ...lax, casts: { 'app.t.code': 'code::int, id' } })).toThrow(/--using.*single expression/); + expect(() => diffSchemas(from, to, { ...lax, casts: { 'app.t.code': 'nullif(code' } })).toThrow(/--using.*could not be parsed/); + const ok = diffSchemas(from, to, { ...lax, casts: { 'app.t.code': "nullif(code, '')::int" } }); + expect(norm(sql(ok, 'deploy'))).toMatch(/USING .*nullif ?\(code, ''\).*int/i); + }); + + it('does not re-grant when an explicit REVOKE ALL FROM PUBLIC disappears on an object PUBLIC has no default rights to', () => { + const from = `${S}\nCREATE TABLE app.t (id int PRIMARY KEY);\nREVOKE ALL ON TABLE app.t FROM PUBLIC;\nCREATE FUNCTION app.f() RETURNS int LANGUAGE sql AS $$ SELECT 1 $$;\nREVOKE ALL ON FUNCTION app.f() FROM PUBLIC;`; + const to = `${S}\nCREATE TABLE app.t (id int PRIMARY KEY);\nCREATE FUNCTION app.f() RETURNS int LANGUAGE sql AS $$ SELECT 1 $$;`; + const result = diffSchemas(from, to); + const deploy = norm(sql(result, 'deploy')); + expect(deploy).not.toMatch(/GRANT ALL ON (TABLE )?app\.t/); + expect(deploy).toMatch(/GRANT EXECUTE ON FUNCTION app\.f\(\) TO PUBLIC/); + expect(deploy).not.toMatch(/GRANT ALL/); + expect(norm(sql(result, 'revert'))).toMatch(/REVOKE EXECUTE ON FUNCTION app\.f\(\) FROM PUBLIC/); + // The other direction (adding the explicit revoke) is one REVOKE per object, not two. + const back = diffSchemas(to, from); + const backDeploy = norm(sql(back, 'deploy')); + expect(backDeploy.match(/REVOKE ALL ON (TABLE )?app\.t FROM PUBLIC/g)).toHaveLength(1); + expect(backDeploy.match(/REVOKE ALL ON FUNCTION app\.f\(\) FROM PUBLIC/g)).toHaveLength(1); + }); + + it('restores only what PUBLIC holds by default around an explicit revoke: EXECUTE on functions, CONNECT/TEMPORARY on databases, nothing on schemas', () => { + const from = `${S}\nCREATE FUNCTION app.f() RETURNS int LANGUAGE sql AS $$ SELECT 1 $$;`; + const to = `${from}\nREVOKE ALL ON SCHEMA app FROM PUBLIC;\nREVOKE ALL ON FUNCTION app.f() FROM PUBLIC;\nREVOKE ALL ON DATABASE mydb FROM PUBLIC;`; + const added = diffSchemas(from, to); + const deploy = norm(sql(added, 'deploy')); + expect(deploy).toMatch(/REVOKE ALL ON SCHEMA app FROM PUBLIC/); + expect(deploy).toMatch(/REVOKE ALL ON FUNCTION app\.f\(\) FROM PUBLIC/); + expect(deploy).toMatch(/REVOKE ALL ON DATABASE mydb FROM PUBLIC/); + const revert = norm(sql(added, 'revert')); + expect(revert).toMatch(/GRANT EXECUTE ON FUNCTION app\.f\(\) TO PUBLIC/); + expect(revert).toMatch(/GRANT CONNECT, TEMPORARY ON DATABASE mydb TO PUBLIC/); + expect(revert).not.toMatch(/GRANT ALL/); + expect(revert).not.toMatch(/ON SCHEMA app TO PUBLIC/); + const dropped = norm(sql(diffSchemas(to, from), 'deploy')); + expect(dropped).toMatch(/GRANT EXECUTE ON FUNCTION app\.f\(\) TO PUBLIC/); + expect(dropped).toMatch(/GRANT CONNECT, TEMPORARY ON DATABASE mydb TO PUBLIC/); + expect(dropped).not.toMatch(/GRANT ALL/); + expect(dropped).not.toMatch(/SCHEMA app/); + }); + + it('emits a single REVOKE when a grant is replaced by an explicit revoke', () => { + const from = `${S}\nCREATE ROLE app_user;\nCREATE TABLE app.t (id int PRIMARY KEY);\nGRANT SELECT ON TABLE app.t TO app_user;`; + const to = `${S}\nCREATE ROLE app_user;\nCREATE TABLE app.t (id int PRIMARY KEY);\nREVOKE SELECT ON TABLE app.t FROM app_user;`; + const deploy = norm(sql(diffSchemas(from, to), 'deploy')); + expect(deploy.match(/REVOKE SELECT ON (TABLE )?app\.t FROM app_user/g)).toHaveLength(1); + expect(deploy).not.toMatch(/GRANT/); + }); + + it('renames a column on a table renamed in the same diff (declared with either table name)', () => { + const from = `${S}\nCREATE TABLE app.old (id int PRIMARY KEY, col text);`; + const to = `${S}\nCREATE TABLE app.new (id int PRIMARY KEY, name text);`; + for (const columnSpec of ['app.old.col=app.new.name', 'app.old.col=app.old.name']) { + const result = diffSchemas(from, to, { renames: ['app.old=app.new', columnSpec].map(parseRenameDeclaration) }); + const deploy = norm(sql(result, 'deploy')); + expect(deploy).toMatch(/ALTER TABLE app\.old RENAME TO new/); + expect(deploy).toMatch(/ALTER TABLE app\.new RENAME (COLUMN )?col TO name/); + expect(deploy).not.toMatch(/DROP COLUMN/); + const revert = norm(sql(result, 'revert')); + expect(revert).toMatch(/ALTER TABLE app\.new RENAME (COLUMN )?name TO col/); + expect(revert).toMatch(/ALTER TABLE app\.new RENAME TO old/); + } + expect(() => diffSchemas(from, to, { renames: ['app.old=app.new', 'app.old.col=app.other.name'].map(parseRenameDeclaration) })).toThrow(/cannot move/); + }); + + it('follows a table rename through comments, grants, views, policies and function bodies', () => { + const body = (t: string) => [ + S, + `CREATE TABLE app.${t} (id int PRIMARY KEY, col text);`, + `COMMENT ON TABLE app.${t} IS 'the table';`, + `COMMENT ON COLUMN app.${t}.col IS 'the col';`, + `GRANT SELECT ON TABLE app.${t} TO PUBLIC;`, + `ALTER TABLE app.${t} ENABLE ROW LEVEL SECURITY;`, + `CREATE POLICY p ON app.${t} FOR SELECT USING (true);`, + `CREATE VIEW app.v AS SELECT ${t}.id, ${t}.col FROM app.${t};`, + `CREATE FUNCTION app.f() RETURNS bigint LANGUAGE sql AS $$ SELECT count(*) FROM app.${t} $$;` + ].join('\n'); + const result = diffSchemas(body('old'), body('new'), { renames: [parseRenameDeclaration('app.old=app.new')] }); + expect(result.warnings).toEqual([]); + const deploy = norm(sql(result, 'deploy')); + expect(deploy).toMatch(/ALTER TABLE app\.old RENAME TO new/); + expect(deploy).toMatch(/CREATE OR REPLACE FUNCTION app\.f\(\)/); + expect(deploy).toMatch(/FROM app\.new/); + expect(deploy).not.toMatch(/COMMENT ON/); + expect(deploy).not.toMatch(/REVOKE|GRANT/); + expect(deploy).not.toMatch(/DROP VIEW|CREATE (OR REPLACE )?VIEW/); + expect(deploy).not.toMatch(/POLICY/); + }); + + it('rebuilds a view around a breaking change on a renamed table and reverts it in order with its original text', () => { + const from = `${S}\nCREATE TABLE app.old (id int PRIMARY KEY, col text);\nCREATE VIEW app.v AS SELECT old.id, old.col FROM app.old;`; + const to = `${S}\nCREATE TABLE app.new (id int PRIMARY KEY, col int);\nCREATE VIEW app.v AS SELECT new.id, new.col FROM app.new;`; + const result = diffSchemas(from, to, { allowDataLoss: true, renames: [parseRenameDeclaration('app.old=app.new')] }); + const create = result.changes.find(c => /CREATE VIEW/.test(c.deploy))!; + expect(norm(create.revert)).toMatch(/DROP VIEW app\.v/); + const drop = result.changes.find(c => /DROP VIEW/.test(c.deploy))!; + expect(norm(drop.revert)).toMatch(/CREATE VIEW app\.v AS SELECT old\.id, old\.col FROM app\.old/); + const rename = result.changes.find(c => /RENAME TO new/.test(c.deploy))!; + expect(rename.revert).not.toMatch(/VIEW/); + const revert = norm([...result.changes].reverse().map(c => c.revert).join('\n')); + expect(revert.indexOf('DROP VIEW app.v')).toBeLessThan(revert.indexOf('TYPE text')); + expect(revert.indexOf('RENAME TO old')).toBeLessThan(revert.indexOf('CREATE VIEW app.v')); + }); + + it('leaves a table alias that happens to be named like the renamed table alone', () => { + const body = (t: string) => [ + S, + `CREATE TABLE app.${t} (id int PRIMARY KEY);`, + 'CREATE TABLE app.other (id int PRIMARY KEY, n int);', + 'CREATE VIEW app.v AS SELECT old.id, old.n FROM app.other AS old;' + ].join('\n'); + const result = diffSchemas(body('old'), body('new'), { renames: [parseRenameDeclaration('app.old=app.new')] }); + expect(norm(sql(result, 'deploy'))).toMatch(/ALTER TABLE app\.old RENAME TO new/); + expect(sql(result, 'deploy')).not.toMatch(/VIEW/); + expect(sql(result, 'revert')).not.toMatch(/VIEW/); + }); + + it('orders a consumer of a new enum value after the ADD VALUE that introduces it', () => { + const from = `${S}\nCREATE TYPE app.status AS ENUM ('a', 'b');\nCREATE TABLE app.t (id int PRIMARY KEY, s app.status NOT NULL DEFAULT 'a');`; + const to = `${S}\nCREATE TYPE app.status AS ENUM ('a', 'b', 'c');\nCREATE TABLE app.t (id int PRIMARY KEY, s app.status NOT NULL DEFAULT 'c');`; + const result = diffSchemas(from, to, { allowIncomplete: true }); + const added = result.changes.find(c => /ADD VALUE 'c'/.test(c.deploy))!; + const consumer = result.changes.find(c => /SET DEFAULT 'c'/.test(c.deploy))!; + expect(consumer.dependencies).toContain(added.name); + }); +}); diff --git a/pgpm/transform/src/annotations.ts b/pgpm/transform/src/annotations.ts index d999f7b00c..301900fdea 100644 --- a/pgpm/transform/src/annotations.ts +++ b/pgpm/transform/src/annotations.ts @@ -181,11 +181,40 @@ const defaultPrivilegesNode = (isGrant: boolean, options: unknown, objtype: stri }; }; -/** Privilege-set delta as GRANT/REVOKE statements with exact inverses. */ +/** + * Privileges PUBLIC holds by default (what an explicit REVOKE .. FROM PUBLIC + * actually takes away). Per the GRANT docs: none on tables, columns, + * sequences, schemas, foreign servers/wrappers, large objects, tablespaces or + * parameters; CONNECT and TEMPORARY on databases; EXECUTE on functions and + * procedures; USAGE on languages and types (including domains). + */ +const PUBLIC_DEFAULT_PRIVILEGES: Record = { + OBJECT_DATABASE: ['connect', 'temporary'], + OBJECT_FUNCTION: ['execute'], + OBJECT_PROCEDURE: ['execute'], + OBJECT_ROUTINE: ['execute'], + OBJECT_TYPE: ['usage'], + OBJECT_DOMAIN: ['usage'], + OBJECT_LANGUAGE: ['usage'] +}; + +const isPublicRole = (grantee: unknown): boolean => + (grantee as { RoleSpec?: { roletype?: string } } | undefined)?.RoleSpec?.roletype === 'ROLESPEC_PUBLIC'; + +/** + * Privilege-set delta as GRANT/REVOKE statements with exact inverses. + * `-token` entries are explicit revocations: adding one is a REVOKE (once, even + * when the same token was also held); dropping one only re-grants what the + * grantee holds by default (PUBLIC on functions/types/databases), since no + * grant existed. The inverse of an added revoke restores exactly that default, + * never ALL: a revoke of a privilege nobody held has nothing to put back. + */ function diffPrivileges( from: Set, to: Set, - build: (isGrant: boolean, tokens: string[]) => Node + build: (isGrant: boolean, tokens: string[]) => Node, + objtype: string, + grantee: unknown ): AnnotationStep[] { const steps: AnnotationStep[] = []; const positive = (set: Set): string[] => [...set].filter(t => !t.startsWith('-')).sort(); @@ -196,8 +225,10 @@ function diffPrivileges( const revoked = fromPos.filter(t => !toPos.includes(t)); const fromNeg = negative(from); const toNeg = negative(to); - const newlyRevoked = toNeg.filter(t => !fromNeg.includes(t)); - const unrevoked = fromNeg.filter(t => !toNeg.includes(t)); + const newlyRevoked = toNeg.filter(t => !fromNeg.includes(t) && !revoked.includes(t)); + const byDefault = isPublicRole(grantee) ? PUBLIC_DEFAULT_PRIVILEGES[objtype] ?? [] : []; + const heldByDefault = (tokens: string[]): string[] => tokens.includes('all') ? byDefault : tokens.filter(t => byDefault.includes(t)); + const unrevoked = heldByDefault(fromNeg.filter(t => !toNeg.includes(t) && !granted.includes(t))); if (revoked.length > 0) { steps.push({ sql: deparse(build(false, revoked)), inverse: deparse(build(true, revoked)) }); } @@ -205,7 +236,11 @@ function diffPrivileges( steps.push({ sql: deparse(build(true, granted)), inverse: deparse(build(false, granted)) }); } if (newlyRevoked.length > 0) { - steps.push({ sql: deparse(build(false, newlyRevoked)), inverse: deparse(build(true, newlyRevoked)) }); + const restore = heldByDefault(newlyRevoked); + steps.push({ + sql: deparse(build(false, newlyRevoked)), + inverse: restore.length > 0 ? deparse(build(true, restore)) : '-- pgpm: nothing to restore; the grantee held none of these privileges' + }); } if (unrevoked.length > 0) { steps.push({ sql: deparse(build(true, unrevoked)), inverse: deparse(build(false, unrevoked)) }); @@ -231,7 +266,7 @@ export function diffAnnotations(from: AnnotationState, to: AnnotationState, path const prior = from.grants.get(key)?.privs ?? new Set(); const entry = to.grants.get(key) ?? from.grants.get(key)!; const next = to.grants.get(key)?.privs ?? new Set(); - steps.push(...diffPrivileges(prior, next, (isGrant, tokens) => grantNode(isGrant, entry.objtype, entry.objects, tokens, entry.grantee))); + steps.push(...diffPrivileges(prior, next, (isGrant, tokens) => grantNode(isGrant, entry.objtype, entry.objects, tokens, entry.grantee), entry.objtype, entry.grantee)); } for (const key of new Set([...from.defaultPrivileges.keys(), ...to.defaultPrivileges.keys()])) { @@ -239,7 +274,7 @@ export function diffAnnotations(from: AnnotationState, to: AnnotationState, path const entry = to.defaultPrivileges.get(key) ?? from.defaultPrivileges.get(key)!; const next = to.defaultPrivileges.get(key)?.privs ?? new Set(); steps.push(...diffPrivileges(prior, next, (isGrant, tokens) => - defaultPrivilegesNode(isGrant, entry.options, entry.objtype, tokens, entry.grantee))); + defaultPrivilegesNode(isGrant, entry.options, entry.objtype, tokens, entry.grantee), entry.objtype, entry.grantee)); } for (const key of new Set([...from.owners.keys(), ...to.owners.keys()])) { diff --git a/pgpm/transform/src/object-alters.ts b/pgpm/transform/src/object-alters.ts index c50b6a7269..e7818801c4 100644 --- a/pgpm/transform/src/object-alters.ts +++ b/pgpm/transform/src/object-alters.ts @@ -42,8 +42,10 @@ export function alterEnum(from: Node, to: Node, label: string): AlterStep[] | nu const steps: AlterStep[] = []; toVals.forEach((v, i) => { if (fromVals.includes(v)) return; + // Values are added in target order, so the previous target value always + // exists by now; a value at the head goes BEFORE the first pre-existing one. const after = i > 0 ? toVals[i - 1] : undefined; - const before = after === undefined && toVals.length > 1 ? toVals[1] : undefined; + const before = after === undefined ? toVals.slice(1).find(x => fromVals.includes(x)) : undefined; const node: Node = { AlterEnumStmt: { typeName: to.typeName, newVal: v } }; if (after !== undefined) Object.assign(node.AlterEnumStmt as Node, { newValNeighbor: after, newValIsAfter: true }); else if (before !== undefined) Object.assign(node.AlterEnumStmt as Node, { newValNeighbor: before, newValIsAfter: false }); @@ -121,7 +123,31 @@ const SEQUENCE_DEFAULTS: Record = { owned_by: { DefElem: { defname: 'owned_by', arg: { List: { items: [{ String: { sval: 'none' } }] } }, defaction: 'DEFELEM_UNSPEC' } } }; -/** One ALTER SEQUENCE carrying every changed option (removed options reset to their defaults). */ +const sequenceOptionValue = (node: Node, name: string): number | undefined => { + for (const item of (node.options as { DefElem?: Node }[]) ?? []) { + const d = item.DefElem; + if (!d || d.defname !== name) continue; + const arg = d.arg as { Integer?: { ival?: number }; Float?: { fval?: string } } | undefined; + if (arg?.Integer) return arg.Integer.ival ?? 0; + if (arg?.Float) return Number(arg.Float.fval); + return undefined; + } + return undefined; +}; + +/** START WITH as the catalog records it: explicit, else MINVALUE (ascending) / MAXVALUE (descending), else 1 / -1. */ +const effectiveSequenceStart = (node: Node): number => { + const explicit = sequenceOptionValue(node, 'start'); + if (explicit !== undefined) return explicit; + const increment = sequenceOptionValue(node, 'increment') ?? 1; + return increment >= 0 ? sequenceOptionValue(node, 'minvalue') ?? 1 : sequenceOptionValue(node, 'maxvalue') ?? -1; +}; + +/** + * One ALTER SEQUENCE carrying every changed option (removed options reset to + * their defaults). START is compared as its effective value, since a removed + * MINVALUE/MAXVALUE also moves the start the catalog derived from it. + */ export function alterSequence(from: Node, to: Node): AlterStep[] | null { const read = (node: Node): Map => { const out = new Map(); @@ -139,12 +165,19 @@ export function alterSequence(from: Node, to: Node): AlterStep[] | null { for (const name of new Set([...a.keys(), ...b.keys()])) { const prior = a.get(name); const next = b.get(name); - if (stablePrint(prior) === stablePrint(next)) continue; + if (name === 'start' || stablePrint(prior) === stablePrint(next)) continue; const reset = SEQUENCE_DEFAULTS[name]; if (!reset && (!prior || !next)) return null; forward.push(next ?? reset); backward.push(prior ?? reset); } + const startFrom = effectiveSequenceStart(from); + const startTo = effectiveSequenceStart(to); + if (startFrom !== startTo) { + const start = (value: number): Node => ({ DefElem: { defname: 'start', arg: { Integer: { ival: value } }, defaction: 'DEFELEM_UNSPEC' } }); + forward.push(start(startTo)); + backward.push(start(startFrom)); + } if (forward.length === 0) return []; const alter = (options: Node[]): string => deparse({ AlterSeqStmt: { sequence: to.sequence, options } }); @@ -177,10 +210,17 @@ export function alterComposite(from: Node, to: Node, label: string): AlterStep[] destructive: `${label}: DROP ATTRIBUTE ${name}` }); } + const targetOrder = [...b.keys()]; for (const [name, def] of b) { const prior = a.get(name); if (!prior) { - steps.push({ sql: alter({ subtype: 'AT_AddColumn', def: { ColumnDef: def } }), inverse: alter({ subtype: 'AT_DropColumn', name }) }); + const displaced = targetOrder.slice(targetOrder.indexOf(name) + 1).find(k => a.has(k)); + steps.push({ + sql: alter({ subtype: 'AT_AddColumn', def: { ColumnDef: def } }), + inverse: alter({ subtype: 'AT_DropColumn', name }), + incomplete: displaced === undefined ? undefined : + `${label}: attribute "${name}" is inserted before "${displaced}" but ALTER TYPE .. ADD ATTRIBUTE appends; the attribute order differs from the target (recreate the type to match it)` + }); } else if (stablePrint(prior) !== stablePrint(def)) { const retype = (d: Node): string => alter({ subtype: 'AT_AlterColumnType', name, def: { ColumnDef: { typeName: d.typeName, collClause: d.collClause } } }); diff --git a/pgpm/transform/src/semantic-diff-driver.ts b/pgpm/transform/src/semantic-diff-driver.ts index 778fba93d5..1f52f11f5f 100644 --- a/pgpm/transform/src/semantic-diff-driver.ts +++ b/pgpm/transform/src/semantic-diff-driver.ts @@ -75,6 +75,8 @@ export interface RenameDeclaration { schema: string | null; /** The table, for column renames. */ table?: string; + /** For column renames declared with the table's new name (renamed in the same diff). */ + toTable?: string; from: string; to: string; } @@ -90,10 +92,14 @@ export function parseRenameDeclaration(spec: string): RenameDeclaration { if (a.length !== b.length || a.length < 2 || a.length > 3 || a.some(x => !x) || b.some(x => !x)) { throw new DiffPolicyError('rename', `pgpm diff: --rename expects schema.table=schema.table or schema.table.column=schema.table.column, got "${spec}"`); } - if (a.slice(0, -1).join('.') !== b.slice(0, -1).join('.')) { - throw new DiffPolicyError('rename', `pgpm diff: --rename cannot move "${lhs}" to another ${a.length === 3 ? 'table' : 'schema'} ("${rhs}")`); + if (a[0] !== b[0]) { + throw new DiffPolicyError('rename', `pgpm diff: --rename cannot move "${lhs}" to another schema ("${rhs}")`); + } + if (a.length === 3) { + const decl: RenameDeclaration = { kind: 'column', schema: a[0], table: a[1], from: a[2], to: b[2] }; + if (a[1] !== b[1]) decl.toTable = b[1]; + return decl; } - if (a.length === 3) return { kind: 'column', schema: a[0], table: a[1], from: a[2], to: b[2] }; return { kind: 'table', schema: a[0], from: a[1], to: b[1] }; } @@ -158,6 +164,8 @@ interface ObjectUnit { /** Statement indexes in source order. */ statements: number[]; texts: string[]; + /** The texts as written on the from side, before a declared rename rewrote them (what a revert recreates). */ + inverseTexts?: string[]; } interface SideProgram { @@ -314,6 +322,8 @@ interface TableShape { extras: Map; /** COMMENT / GRANT / OWNER state attached to the table. */ annotations: AnnotationState; + /** Constraints authored without a name; Postgres derives theirs from the table name. */ + implicitConstraints: { node: ConstraintNode; column?: string }[]; } /** Constraint kinds that can move between column, table-elt, and ALTER form. */ @@ -373,7 +383,7 @@ function normalizeColumnConstraint(c: ConstraintNode & Record, * the catalog rendering of the same table. */ function tableShape(unit: ObjectUnit): TableShape { - const shape: TableShape = { relation: null, columns: new Map(), extras: new Map(), annotations: emptyAnnotations() }; + const shape: TableShape = { relation: null, columns: new Map(), extras: new Map(), annotations: emptyAnnotations(), implicitConstraints: [] }; const table = unit.identity.name; const rawColumns: { colname: string; def: Record }[] = []; const constraints: { node: ConstraintNode & Record; column?: string }[] = []; @@ -484,6 +494,7 @@ function tableShape(unit: ObjectUnit): TableShape { } for (const { node, column } of constraints) { + if (!node.conname) shape.implicitConstraints.push({ node, column }); const canonical: Record = { ...node }; canonical.conname = node.conname ?? defaultConstraintName(table, node, column) ?? undefined; if (column && (node.keys ?? []).length === 0 && node.contype !== 'CONSTR_FOREIGN' && node.contype !== 'CONSTR_CHECK') { @@ -523,6 +534,50 @@ function tableShape(unit: ObjectUnit): TableShape { const deparseStmt = (node: Record): string => `${Deparser.deparse(node, { pretty: false })};`; +/** Postgres' NAMEDATALEN - 1: longer derived names are truncated by the server and cannot be predicted here. */ +const MAX_IDENTIFIER = 63; + +/** + * Renaming a table does not rename the constraints (and their indexes) whose + * names Postgres derived from the old table name; a fresh deploy of the target + * would derive them from the new one. Rename each to keep the catalogs equal. + */ +function implicitConstraintRenames( + shape: TableShape, + relation: Record, + fromTable: string, + toTable: string, + incomplete: string[] +): TableAlteration[] { + const steps: TableAlteration[] = []; + const seen = new Map(); + const pairs = shape.implicitConstraints.map(({ node, column }) => ({ + from: defaultConstraintName(fromTable, node, column), + to: defaultConstraintName(toTable, node, column) + })); + for (const { from } of pairs) if (from) seen.set(from, (seen.get(from) ?? 0) + 1); + const label = relation.schemaname ? `${relation.schemaname}.${toTable}` : toTable; + for (const { from, to } of pairs) { + if (!from || !to || from === to) continue; + if (seen.get(from)! > 1 || from.length > MAX_IDENTIFIER || to.length > MAX_IDENTIFIER) { + incomplete.push(`${label}: constraint "${from}" was named by Postgres from the old table name and cannot be renamed predictably (duplicate or truncated derivation); name it explicitly`); + continue; + } + const renameConstraint = (subname: string, newname: string): string => + deparseStmt({ RenameStmt: { renameType: 'OBJECT_TABCONSTRAINT', relationType: 'OBJECT_TABLE', relation: { ...relation, relname: toTable }, subname, newname, behavior: 'DROP_RESTRICT' } }); + steps.push({ + sql: renameConstraint(from, to), + inverse: renameConstraint(to, from), + phase: 'alter', + sub: { kind: 'constraint', schema: (relation.schemaname as string | undefined) ?? null, table: toTable, name: to }, + keyColumns: [], + breaking: false, + rename: true + }); + } + return steps; +} + /** Where in the migration an alteration belongs (tie-break only; edges rule). */ type Phase = 'drop' | 'remove' | 'alter' | 'add' | 'create'; const PHASE_RANK: Record = { drop: 0, remove: 1, alter: 2, add: 3, create: 4 }; @@ -579,10 +634,18 @@ function columnAttributes(def: Record): ColumnAttributes { /** The expression node of `SELECT `. */ function parseExpression(expr: string): unknown { - const stmts = parseSync(`SELECT ${expr}`).sql.stmts; - const select = stmts[0]?.stmt as unknown as { SelectStmt?: { targetList?: { ResTarget?: { val?: unknown } }[] } }; - const val = select?.SelectStmt?.targetList?.[0]?.ResTarget?.val; - if (stmts.length !== 1 || !val) throw new DiffPolicyError('incomplete', `pgpm diff: --using expression is not a single expression: ${expr}`); + let stmts: { stmt?: unknown }[]; + try { + stmts = parseSync(`SELECT ${expr}`).sql.stmts; + } catch (e) { + throw new DiffPolicyError('incomplete', `pgpm diff: --using expression could not be parsed: ${expr} (${(e as Error).message})`); + } + const select = stmts[0]?.stmt as { SelectStmt?: { targetList?: { ResTarget?: { val?: unknown } }[] } } | undefined; + const targets = select?.SelectStmt?.targetList ?? []; + const val = targets[0]?.ResTarget?.val; + if (stmts.length !== 1 || targets.length !== 1 || !val) { + throw new DiffPolicyError('incomplete', `pgpm diff: --using expression is not a single expression (one value, no top-level comma): ${expr}`); + } return cleanTree(val); } @@ -1016,7 +1079,43 @@ interface Atom { const objectKey = (ref: { schema: string | null; name: string }): string => `${ref.schema ?? ''}.${ref.name}`; /** Rewrite every reference to relation `schema.from` in `texts` to `schema.to` (AST-level). */ +/** COMMENT/SECURITY LABEL object kinds whose name list ends with one element below the relation. */ +const RELATION_SUBOBJECT_KINDS = new Set(['OBJECT_COLUMN', 'OBJECT_TABCONSTRAINT', 'OBJECT_TRIGGER', 'OBJECT_POLICY', 'OBJECT_RULE']); +const RELATION_KINDS = new Set(['OBJECT_TABLE', 'OBJECT_VIEW', 'OBJECT_MATVIEW', 'OBJECT_SEQUENCE', 'OBJECT_INDEX', 'OBJECT_FOREIGN_TABLE']); + +/** + * Rewrite every reference to relation `schema.from` as `to`: RangeVars + * (tables, FK targets, policies, grants, composite typevars), qualified + * column references, COMMENT ON name lists and `::regclass` literals. + * Function bodies are strings and stay as written: Postgres does not rewrite + * them on rename either, so a body naming the old relation diffs as a + * replacement against the target. + */ +const objectLabelOf = (id: ObjectIdentity): string => `${id.schema ? `${id.schema}.` : ''}${id.table ? `${id.table}.` : ''}${id.name}`; + function renameRelation(texts: string[], schema: string | null, from: string, to: string): string[] { + const svalOf = (x: unknown): string | undefined => (x as { String?: { sval?: string } })?.String?.sval; + const schemaMatches = (name: string | undefined | null): boolean => name === undefined || name === null || name === schema; + // `old.col` names the relation only where the statement reads `old` itself, + // not where `old` is another relation's alias (`FROM app.other AS old`). + let qualifiesRelation = false; + const scanRelations = (node: unknown, found: { named: boolean; aliased: boolean }): void => { + if (Array.isArray(node)) { node.forEach(x => scanRelations(x, found)); return; } + if (!node || typeof node !== 'object') return; + const rv = node as { relname?: unknown; schemaname?: string; alias?: { aliasname?: string } }; + if (typeof rv.relname === 'string') { + if (rv.alias?.aliasname === from) found.aliased = true; + else if (rv.relname === from && schemaMatches(rv.schemaname)) found.named = true; + } + for (const value of Object.values(node)) scanRelations(value, found); + }; + // `[schema?, relation, ...depth trailing names]` + const renameNames = (items: unknown[], depth: number): unknown[] => { + const at = items.length - 1 - depth; + if (at < 0 || svalOf(items[at]) !== from) return items; + if (at > 1 || (at === 1 && !schemaMatches(svalOf(items[0])))) return items; + return items.map((x, i) => (i === at ? { String: { sval: to } } : x)); + }; const walk = (node: unknown): unknown => { if (Array.isArray(node)) return node.map(walk); if (!node || typeof node !== 'object') return node; @@ -1024,20 +1123,53 @@ function renameRelation(texts: string[], schema: string | null, from: string, to const out: Record = {}; for (const [key, value] of Object.entries(rec)) { const rv = value as { relname?: string; schemaname?: string } | undefined; - if ( - (key === 'RangeVar' || key === 'relation' || key === 'pktable' || key === 'typevar') && - rv && typeof rv === 'object' && rv.relname === from && ((rv.schemaname ?? null) === schema || !rv.schemaname) - ) { + if (rv && typeof rv === 'object' && typeof rv.relname === 'string' && rv.relname === from && schemaMatches(rv.schemaname)) { out[key] = walk({ ...rv, relname: to }); continue; } + if (key === 'ColumnRef' && qualifiesRelation) { + const cr = value as { fields?: unknown[] }; + if (cr.fields && cr.fields.length >= 2) { + out[key] = { ...cr, fields: renameNames(cr.fields, 1) }; + continue; + } + } + if (key === 'CommentStmt' || key === 'SecLabelStmt') { + const c = value as { objtype?: string; object?: { List?: { items?: unknown[] } } }; + const items = c.object?.List?.items; + const depth = RELATION_SUBOBJECT_KINDS.has(c.objtype ?? '') ? 1 : RELATION_KINDS.has(c.objtype ?? '') ? 0 : -1; + if (items && depth >= 0) { + out[key] = { ...c, object: { List: { items: renameNames(items, depth) } } }; + continue; + } + } + if (key === 'TypeCast') { + const tc = value as { arg?: { A_Const?: { sval?: { sval?: string } } }; typeName?: { names?: unknown[] } }; + const names = tc.typeName?.names ?? []; + const literal = tc.arg?.A_Const?.sval?.sval; + if (svalOf(names[names.length - 1]) === 'regclass' && literal !== undefined) { + const parts = literal.split('.'); + const renamed = renameNames(parts.map(x => ({ String: { sval: x } })), 0).map(svalOf).join('.'); + if (renamed !== literal) { + out[key] = { ...tc, arg: { A_Const: { sval: { sval: renamed } } } }; + continue; + } + } + } out[key] = walk(value); } return out; }; - return texts.map(text => - parseSync(text).sql.stmts.map(st => deparseStmt(walk(st.stmt) as Record)).join('\n') - ); + const mentions = new RegExp(`\\b${from.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b`); + return texts.map(text => { + if (!mentions.test(text)) return text; + return parseSync(text).sql.stmts.map(st => { + const found = { named: false, aliased: false }; + scanRelations(st.stmt, found); + qualifiesRelation = found.named && !found.aliased; + return deparseStmt(walk(st.stmt) as Record); + }).join('\n'); + }); } /** @@ -1068,6 +1200,7 @@ export function diffSchemas( // ---- declared table renames: re-key the from-side unit under its new name. const renamedTables = new Map(); + const renamedTo = new Map(); for (const rename of renames.filter(r => r.kind === 'table')) { const oldKey = identityKey({ kind: 'table', schema: rename.schema, name: rename.from }); const newKey = identityKey({ kind: 'table', schema: rename.schema, name: rename.to }); @@ -1083,15 +1216,43 @@ export function diffSchemas( const renamed: ObjectUnit = { identity: { ...unit.identity, name: rename.to }, statements: unit.statements, - texts: renameRelation(unit.texts, rename.schema, rename.from, rename.to) + texts: renameRelation(unit.texts, rename.schema, rename.from, rename.to), + inverseTexts: unit.inverseTexts ?? unit.texts }; a.units.set(newKey, renamed); a.order[a.order.indexOf(oldKey)] = newKey; renamedTables.set(newKey, { from: rename.from, schema: rename.schema }); + renamedTo.set(oldKey, { key: newKey, name: rename.to }); + // Postgres follows the rename in views, policies, grants and comments; so does the from side. + for (const [key, other] of [...a.units]) { + if (key === newKey) continue; + const texts = renameRelation(other.texts, rename.schema, rename.from, rename.to); + const scoped = other.identity.table === rename.from && other.identity.schema === rename.schema; + if (!scoped && texts.every((t, i) => t === other.texts[i])) continue; + const identity = scoped ? { ...other.identity, table: rename.to } : other.identity; + const movedKey = identityKey(identity); + if (movedKey !== key) { + if (a.units.has(movedKey)) { + throw new DiffPolicyError('rename', `pgpm diff: cannot apply rename ${rename.schema ?? ''}.${rename.from} -> ${rename.to}: "${objectLabelOf(identity)}" already exists on the source`); + } + a.units.delete(key); + a.order[a.order.indexOf(key)] = movedKey; + } + a.units.set(movedKey, { ...other, identity, texts, inverseTexts: other.inverseTexts ?? other.texts }); + } } const columnRenames = new Map>(); for (const rename of renames.filter(r => r.kind === 'column')) { - const key = identityKey({ kind: 'table', schema: rename.schema, name: rename.table! }); + const declared = identityKey({ kind: 'table', schema: rename.schema, name: rename.table! }); + const target = renamedTo.get(declared); + const key = target?.key ?? declared; + const tableName = target?.name ?? rename.table!; + if (rename.toTable !== undefined && rename.toTable !== tableName && rename.toTable !== rename.table) { + throw new DiffPolicyError( + 'rename', + `pgpm diff: --rename cannot move "${rename.schema}.${rename.table}.${rename.from}" to another table ("${rename.schema}.${rename.toTable}.${rename.to}")` + ); + } if (!columnRenames.has(key)) columnRenames.set(key, new Map()); columnRenames.get(key)!.set(rename.from, rename.to); } @@ -1186,8 +1347,10 @@ export function diffSchemas( const objectLabel = (id: ObjectIdentity): string => `${id.schema ? `${id.schema}.` : ''}${id.name}`; // Drop + create pair for a unit's statements (replacements, rebuilds, removals). + // A drop runs before the alter phase (so before a table rename) and its + // inverse after the rename is undone: both speak the from side as written. const dropOf = (unit: ObjectUnit, path: string, rank: number): Atom | null => { - const recreate = unit.texts.join('\n'); + const recreate = (unit.inverseTexts ?? unit.texts).join('\n'); const drop = inverseOf(recreate, path); if (!drop.trim() || drop.trim().startsWith('--')) { incomplete.push(`${path}: no DROP derivable for kind "${unit.identity.kind}" (${objectLabel(unit.identity)}); drop it manually`); @@ -1256,7 +1419,7 @@ export function diffSchemas( keyColumns: [], breaking: false, rename: true - }); + }, ...implicitConstraintRenames(shapeFrom, relation, renamed.from, unit.identity.name, incomplete)); } const columnsTouched = diff.columnsAdded!.length + diff.columnsRemoved!.length + diff.columnsModified!.length > 0; @@ -1288,6 +1451,8 @@ export function diffSchemas( }); // Every other step of the table addresses it (or its columns) by the new name. for (const r of renameAtoms) for (const x of stepAtoms) x.deps.add(r.id); + const tableRename = renameAtoms.find(r => r.sub?.kind === 'table'); + if (tableRename) for (const r of renameAtoms) if (r !== tableRename) r.deps.add(tableRename.id); return; } if (unitFingerprint(prior) === unitFingerprint(unit)) return; @@ -1354,8 +1519,9 @@ export function diffSchemas( if (signature !== null && signature === functionSignature(pa.definition) && !dependencyChanged) { // Same name, parameters and return type: replace in place. Dependents // (views, triggers, defaults) keep pointing at the same function. + const inverseDefinition = prior.inverseTexts ? splitUnit({ ...prior, texts: prior.inverseTexts }).definition : pa.definition; const atom = addAtom({ - unit: key, name: path, phase: 'create', sql: asReplace(pb.definition), inverse: asReplace(pa.definition), + unit: key, name: path, phase: 'create', sql: asReplace(pb.definition), inverse: asReplace(inverseDefinition), sub: null, keyColumns: [], breaking: false, rank: toRank * 1000 }); track(key, atom); @@ -1528,6 +1694,21 @@ export function diffSchemas( } } + // ---- enum values added and used in the same migration: Postgres refuses to + // use a new value before the transaction that added it commits (55P04). + for (const atom of atoms) { + const added = /^ALTER TYPE (\S+) ADD VALUE ('(?:[^']|'')*')/.exec(atom.sql); + if (!added) continue; + const users = atoms.filter(x => x !== atom && x.phase !== 'drop' && !/^ALTER TYPE \S+ ADD VALUE /.test(x.sql) && x.sql.includes(added[2])); + if (users.length === 0) continue; + const note = `enum value ${added[2].replace(/\s+/g, ' ')} of ${added[1]} is added and used in the same migration; Postgres cannot use a new enum value before the transaction that added it commits (deploy with --no-tx, or split the migration)`; + incomplete.push(`${atom.name}: ${note}`); + for (const user of users) { + user.sql = `-- pgpm: ${note}\n${user.sql}`; + user.deps.add(atom.id); + } + } + // ---- policy: refuse data loss / incompleteness unless accepted explicitly. if (destructive.length > 0 && !options.allowDataLoss) { throw new DiffPolicyError( @@ -1544,6 +1725,22 @@ export function diffSchemas( warnings.push(...destructive.map(x => `data loss accepted: ${x}`)); warnings.push(...incomplete.map(x => `incomplete: ${x}`)); + // An object replaced in place whose new definition needs the table's new + // name (function bodies are text; Postgres does not rewrite them on RENAME) + // needs its old definition restored only once the table carries its old + // name again. The reverse order would recreate it first, so the rename's + // inverse restores it. A drop/create pair is already ordered: its create's + // inverse drops, and the drop's inverse recreates after the rename is undone. + for (const renameAtom of atoms) { + if (!renameAtom.rename || renameAtom.sub?.kind !== 'table') continue; + for (const atom of atoms) { + if (atom.phase !== 'create' || atom.added || !atom.inverse || !atom.deps.has(renameAtom.id)) continue; + if (unitPhase(atom.unit, 'drop').length > 0) continue; + renameAtom.inverse = `${renameAtom.inverse}\n${atom.inverse}`; + atom.inverse = ''; + } + } + // ---- order: stable topological sort over atoms (phase, then rank, break ties). const order = topoOrder(atoms, atom => `${atom.name} (${atom.phase})`);