diff --git a/graphile/graphile-function-source/package.json b/graphile/graphile-function-source/package.json index cc7d136847..29e04c5568 100644 --- a/graphile/graphile-function-source/package.json +++ b/graphile/graphile-function-source/package.json @@ -38,7 +38,7 @@ "url": "https://github.com/constructive-io/constructive/issues" }, "dependencies": { - "plpgsql-parser": "^18.6.2" + "plpgsql-parser": "^18.6.3" }, "peerDependencies": { "@dataplan/pg": "^1.1.1", diff --git a/graphile/graphile-sql-expression-validator/package.json b/graphile/graphile-sql-expression-validator/package.json index aa5fd9bbdf..c738fe4d9d 100644 --- a/graphile/graphile-sql-expression-validator/package.json +++ b/graphile/graphile-sql-expression-validator/package.json @@ -42,8 +42,8 @@ "url": "https://github.com/constructive-io/constructive/issues" }, "dependencies": { - "pgsql-deparser": "^18.4.2", - "pgsql-parser": "^18.3.2" + "pgsql-deparser": "^18.4.3", + "pgsql-parser": "^18.3.3" }, "peerDependencies": { "grafast": "^1.1.1", diff --git a/packages/csv-to-pg/package.json b/packages/csv-to-pg/package.json index 5d89af7b6e..aad5eddde4 100644 --- a/packages/csv-to-pg/package.json +++ b/packages/csv-to-pg/package.json @@ -34,7 +34,7 @@ "devDependencies": { "@types/js-yaml": "^4.0.9", "makage": "^0.8.0", - "pgsql-parser": "^18.3.2" + "pgsql-parser": "^18.3.3" }, "keywords": [ "csv", @@ -44,10 +44,10 @@ ], "dependencies": { "@pgsql/types": "^18.0.0", - "@pgsql/utils": "^18.3.2", + "@pgsql/utils": "^18.3.3", "csv-parser": "^3.2.1", "inquirerer": "^4.9.4", "js-yaml": "^4.1.0", - "pgsql-deparser": "^18.4.2" + "pgsql-deparser": "^18.4.3" } } diff --git a/packages/node-type-registry/package.json b/packages/node-type-registry/package.json index bd0cb40904..f1b713a70b 100644 --- a/packages/node-type-registry/package.json +++ b/packages/node-type-registry/package.json @@ -34,8 +34,8 @@ "@babel/generator": "^7.29.0", "@babel/types": "^7.29.0", "@pgsql/types": "^18.0.0", - "@pgsql/utils": "^18.3.2", - "pgsql-deparser": "^18.4.2", + "@pgsql/utils": "^18.3.3", + "pgsql-deparser": "^18.4.3", "schema-typescript": "^0.14.4" }, "devDependencies": { diff --git a/packages/safegres/package.json b/packages/safegres/package.json index 016f869504..0e6756a10e 100644 --- a/packages/safegres/package.json +++ b/packages/safegres/package.json @@ -59,16 +59,16 @@ "dependencies": { "@inquirerer/utils": "^3.3.13", "@pgpmjs/logger": "workspace:^", - "@pgsql/lint": "^18.3.3", - "@pgsql/traverse": "^18.8.2", + "@pgsql/lint": "^18.3.5", + "@pgsql/traverse": "^18.8.3", "@pgsql/types": "^18.0.0", "confstash": "^0.1.2", "inquirerer": "^4.9.4", "libpg-query": "^18.2.0", "pg": "^8.21.0", "pg-env": "workspace:^", - "pgsql-deparser": "^18.4.2", - "pgsql-parser": "^18.3.2", + "pgsql-deparser": "^18.4.3", + "pgsql-parser": "^18.3.3", "yanse": "^0.2.2" }, "peerDependencies": { diff --git a/pgpm/cli/__tests__/diff-constructs-e2e.test.ts b/pgpm/cli/__tests__/diff-constructs-e2e.test.ts new file mode 100644 index 0000000000..ff966d67a8 --- /dev/null +++ b/pgpm/cli/__tests__/diff-constructs-e2e.test.ts @@ -0,0 +1,541 @@ +/** + * e2e for `pgpm diff` over real-world PostgreSQL constructs mined from the + * pgsql-parser fixtures (kitchen-sink + upstream regression SQL): partitions, + * inheritance, exclusion constraints, index variants, generated columns, + * trigger variants, event triggers, policies, domains, range types, + * collations, extensions, operators, casts, aggregates, procedures, function + * variants, view options, identifiers, defaults, storage, reloptions, + * sequences, identity columns and comments. Each case deploys v1 (with rows + * where it matters), deploys the emitted migration, must reach the catalog + * of a fresh v2, and must revert back to the catalog of v1. + * + * PREREQUISITES: a running PostgreSQL instance via standard PG* env vars. + */ +import { diffCatalogSnapshots, loadModule, snapshotCatalog, withoutColumnOrder } from '@pgpmjs/transform'; +import * as fs from 'fs'; +import * as path from 'path'; +import { teardownPgPools } from 'pg-cache'; + +import { CLIDeployTestFixture } from '../test-utils'; + +jest.setTimeout(240000); + +afterAll(async () => { + await teardownPgPools(); +}); + +// Each case opens two databases; releasing their pools per case keeps the +// connection count flat when jest runs this suite next to the other diff suites. +afterEach(async () => { + await teardownPgPools(); +}); + +const WS = 'cx-ws'; + +const writeModule = (moduleDir: string, name: string, sql: string, revert: string) => { + fs.mkdirSync(path.join(moduleDir, 'deploy'), { recursive: true }); + fs.mkdirSync(path.join(moduleDir, 'revert'), { recursive: true }); + fs.mkdirSync(path.join(moduleDir, 'verify'), { recursive: true }); + fs.writeFileSync( + path.join(moduleDir, 'pgpm.plan'), + `%syntax-version=1.0.0\n%project=${name}\n%uri=https://github.com/test/${name}\n\ninit 2024-01-01T00:00:00Z test # add init\n` + ); + fs.writeFileSync( + path.join(moduleDir, `${name}.control`), + `comment = 'Diff constructs e2e module'\ndefault_version = '0.0.1'\nrelocatable = false\nsuperuser = false\n` + ); + fs.writeFileSync(path.join(moduleDir, 'deploy', 'init.sql'), `-- Deploy init to pg\n\nBEGIN;\n\n${sql}\n\nCOMMIT;\n`); + fs.writeFileSync(path.join(moduleDir, 'revert', 'init.sql'), `-- Revert init from pg\n\nBEGIN;\n\n${revert}\nDROP SCHEMA fx CASCADE;\n\nCOMMIT;\n`); + fs.writeFileSync(path.join(moduleDir, 'verify', 'init.sql'), `-- Verify init on pg\n\nBEGIN;\n\nROLLBACK;\n`); +}; + +interface Case { + name: string; + v1: string; + v2: string; + flags?: string; + rows?: string; + /** Extra revert statements for objects outside schema fx (extensions, event triggers, casts). */ + revert?: string; + /** Cluster-level roles the case references; created by the test before deploying. */ + roles?: string[]; + /** Row check after the forward migration: query and expected rows. */ + check?: { sql: string; rows: Record[] }; + deployed?: RegExp[]; + notDeployed?: RegExp[]; + forwardResidual?: RegExp[]; + revertResidual?: RegExp[]; +} + +const S = 'CREATE SCHEMA fx;'; + +export const CASES: Case[] = [ + // ---- partitioned tables + { + name: 'partition-add', + v1: `${S} CREATE TABLE fx.m (id int NOT NULL, k int NOT NULL, PRIMARY KEY (id, k)) PARTITION BY LIST (k); CREATE TABLE fx.m_1 PARTITION OF fx.m FOR VALUES IN (1);`, + v2: `${S} CREATE TABLE fx.m (id int NOT NULL, k int NOT NULL, PRIMARY KEY (id, k)) PARTITION BY LIST (k); CREATE TABLE fx.m_1 PARTITION OF fx.m FOR VALUES IN (1); CREATE TABLE fx.m_2 PARTITION OF fx.m FOR VALUES IN (2);`, + rows: 'INSERT INTO fx.m VALUES (1, 1)', + check: { sql: 'SELECT id, k FROM fx.m ORDER BY id', rows: [{ id: 1, k: 1 }] } + }, + { + name: 'partition-drop', + v1: `${S} CREATE TABLE fx.m (id int NOT NULL, k int NOT NULL) PARTITION BY RANGE (k); CREATE TABLE fx.m_a PARTITION OF fx.m FOR VALUES FROM (0) TO (10); CREATE TABLE fx.m_b PARTITION OF fx.m FOR VALUES FROM (10) TO (20);`, + v2: `${S} CREATE TABLE fx.m (id int NOT NULL, k int NOT NULL) PARTITION BY RANGE (k); CREATE TABLE fx.m_a PARTITION OF fx.m FOR VALUES FROM (0) TO (10);`, + flags: '--allow-data-loss' + }, + { + name: 'partition-bound-change', + v1: `${S} CREATE TABLE fx.m (id int NOT NULL, k int NOT NULL) PARTITION BY RANGE (k); CREATE TABLE fx.m_a PARTITION OF fx.m FOR VALUES FROM (0) TO (10);`, + v2: `${S} CREATE TABLE fx.m (id int NOT NULL, k int NOT NULL) PARTITION BY RANGE (k); CREATE TABLE fx.m_a PARTITION OF fx.m FOR VALUES FROM (0) TO (20);`, + rows: 'INSERT INTO fx.m VALUES (1, 5)', + check: { sql: 'SELECT id, k FROM fx.m ORDER BY id', rows: [{ id: 1, k: 5 }] } + }, + { + name: 'partition-default-and-hash', + v1: `${S} CREATE TABLE fx.h (id int NOT NULL) PARTITION BY HASH (id); CREATE TABLE fx.h_0 PARTITION OF fx.h FOR VALUES WITH (MODULUS 2, REMAINDER 0); CREATE TABLE fx.h_1 PARTITION OF fx.h FOR VALUES WITH (MODULUS 2, REMAINDER 1);`, + v2: `${S} CREATE TABLE fx.h (id int NOT NULL) PARTITION BY HASH (id); CREATE TABLE fx.h_0 PARTITION OF fx.h FOR VALUES WITH (MODULUS 2, REMAINDER 0); CREATE TABLE fx.h_1 PARTITION OF fx.h FOR VALUES WITH (MODULUS 2, REMAINDER 1); CREATE TABLE fx.l (id int NOT NULL, k text) PARTITION BY LIST (k); CREATE TABLE fx.l_d PARTITION OF fx.l DEFAULT;` + }, + { + name: 'partition-add-column-to-parent', + v1: `${S} CREATE TABLE fx.m (id int NOT NULL, k int NOT NULL) PARTITION BY LIST (k); CREATE TABLE fx.m_1 PARTITION OF fx.m FOR VALUES IN (1);`, + v2: `${S} CREATE TABLE fx.m (id int NOT NULL, k int NOT NULL, note text) PARTITION BY LIST (k); CREATE TABLE fx.m_1 PARTITION OF fx.m FOR VALUES IN (1);`, + rows: 'INSERT INTO fx.m VALUES (1, 1)', + check: { sql: 'SELECT id, k, note FROM fx.m ORDER BY id', rows: [{ id: 1, k: 1, note: null }] } + }, + // ---- inheritance + { + name: 'inherits-add-child', + v1: `${S} CREATE TABLE fx.base (id int PRIMARY KEY, kind text);`, + v2: `${S} CREATE TABLE fx.base (id int PRIMARY KEY, kind text); CREATE TABLE fx.child (extra text) INHERITS (fx.base);` + }, + { + name: 'inherits-parent-column-add', + v1: `${S} CREATE TABLE fx.base (id int PRIMARY KEY); CREATE TABLE fx.child (extra text) INHERITS (fx.base);`, + v2: `${S} CREATE TABLE fx.base (id int PRIMARY KEY, kind text); CREATE TABLE fx.child (extra text) INHERITS (fx.base);`, + rows: "INSERT INTO fx.child VALUES (1, 'e')", + check: { sql: 'SELECT id, kind, extra FROM fx.child', rows: [{ id: 1, kind: null, extra: 'e' }] } + }, + // ---- constraints + { + name: 'exclusion-constraint', + v1: `${S} CREATE TABLE fx.r (id int PRIMARY KEY, during int4range NOT NULL);`, + v2: `${S} CREATE TABLE fx.r (id int PRIMARY KEY, during int4range NOT NULL, CONSTRAINT r_no_overlap EXCLUDE USING gist (during WITH &&));`, + rows: "INSERT INTO fx.r VALUES (1, '[1,5)')" + }, + { + name: 'check-no-inherit-and-not-valid', + v1: `${S} CREATE TABLE fx.t (id int PRIMARY KEY, n int);`, + v2: `${S} CREATE TABLE fx.t (id int PRIMARY KEY, n int, CONSTRAINT n_pos CHECK (n > 0) NO INHERIT);`, + rows: 'INSERT INTO fx.t VALUES (1, 3)' + }, + { + name: 'fk-actions-change', + v1: `${S} CREATE TABLE fx.p (id int PRIMARY KEY); CREATE TABLE fx.c (id int PRIMARY KEY, p_id int REFERENCES fx.p (id));`, + v2: `${S} CREATE TABLE fx.p (id int PRIMARY KEY); CREATE TABLE fx.c (id int PRIMARY KEY, p_id int REFERENCES fx.p (id) ON DELETE CASCADE ON UPDATE RESTRICT DEFERRABLE INITIALLY DEFERRED);`, + rows: 'INSERT INTO fx.p VALUES (1); INSERT INTO fx.c VALUES (1, 1)' + }, + { + name: 'fk-match-full-composite', + v1: `${S} CREATE TABLE fx.p (a int, b int, PRIMARY KEY (a, b)); CREATE TABLE fx.c (id int PRIMARY KEY, a int, b int);`, + v2: `${S} CREATE TABLE fx.p (a int, b int, PRIMARY KEY (a, b)); CREATE TABLE fx.c (id int PRIMARY KEY, a int, b int, FOREIGN KEY (a, b) REFERENCES fx.p (a, b) MATCH FULL);` + }, + { + name: 'unique-nulls-not-distinct', + v1: `${S} CREATE TABLE fx.t (id int PRIMARY KEY, code text UNIQUE);`, + v2: `${S} CREATE TABLE fx.t (id int PRIMARY KEY, code text UNIQUE NULLS NOT DISTINCT);` + }, + { + name: 'named-not-null-constraint', + v1: `${S} CREATE TABLE fx.t (id int PRIMARY KEY, n int);`, + v2: `${S} CREATE TABLE fx.t (id int PRIMARY KEY, n int CONSTRAINT t_n_nn NOT NULL);`, + rows: 'INSERT INTO fx.t VALUES (1, 1)' + }, + // ---- indexes + { + name: 'index-partial-expression-include', + v1: `${S} CREATE TABLE fx.t (id int PRIMARY KEY, name text, active boolean, score int); CREATE INDEX t_name_idx ON fx.t (name);`, + v2: `${S} CREATE TABLE fx.t (id int PRIMARY KEY, name text, active boolean, score int); CREATE INDEX t_name_idx ON fx.t (lower(name)) INCLUDE (score) WHERE active;` + }, + { + name: 'index-opclass-collation-order', + v1: `${S} CREATE TABLE fx.t (id int PRIMARY KEY, name text, n int);`, + v2: `${S} CREATE TABLE fx.t (id int PRIMARY KEY, name text, n int); CREATE INDEX t_name_pat ON fx.t (name COLLATE "C" text_pattern_ops, n DESC NULLS LAST); CREATE UNIQUE INDEX t_n_uq ON fx.t (n) WITH (fillfactor = 70);` + }, + { + name: 'index-method-gin-brin', + v1: `${S} CREATE TABLE fx.t (id int PRIMARY KEY, tags text[], ts timestamptz);`, + v2: `${S} CREATE TABLE fx.t (id int PRIMARY KEY, tags text[], ts timestamptz); CREATE INDEX t_tags ON fx.t USING gin (tags); CREATE INDEX t_ts ON fx.t USING brin (ts);` + }, + // ---- generated columns + { + name: 'generated-column-add-change-drop', + v1: `${S} CREATE TABLE fx.t (id int PRIMARY KEY, a int, b int, s int GENERATED ALWAYS AS (a + b) STORED);`, + v2: `${S} CREATE TABLE fx.t (id int PRIMARY KEY, a int, b int, s int GENERATED ALWAYS AS (a * b) STORED, d int GENERATED ALWAYS AS (a - b) STORED);`, + flags: '--allow-data-loss', + rows: 'INSERT INTO fx.t (id, a, b) VALUES (1, 2, 3)', + check: { sql: 'SELECT id, s, d FROM fx.t', rows: [{ id: 1, s: 6, d: -1 }] } + }, + { + name: 'generated-virtual-pg18', + v1: `${S} CREATE TABLE fx.t (id int PRIMARY KEY, a int);`, + v2: `${S} CREATE TABLE fx.t (id int PRIMARY KEY, a int, v int GENERATED ALWAYS AS (a * 2) VIRTUAL);`, + rows: 'INSERT INTO fx.t VALUES (1, 4)', + check: { sql: 'SELECT id, v FROM fx.t', rows: [{ id: 1, v: 8 }] } + }, + // ---- triggers + { + name: 'trigger-when-and-transition-tables', + v1: [ + S, 'CREATE TABLE fx.t (id int PRIMARY KEY, n int); CREATE TABLE fx.log (n int);', + 'CREATE FUNCTION fx.tg() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN INSERT INTO fx.log SELECT NEW.n; RETURN NEW; END $$;', + 'CREATE TRIGGER t_tg AFTER UPDATE ON fx.t FOR EACH ROW EXECUTE FUNCTION fx.tg();' + ].join('\n'), + v2: [ + S, 'CREATE TABLE fx.t (id int PRIMARY KEY, n int); CREATE TABLE fx.log (n int);', + 'CREATE FUNCTION fx.tg() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN INSERT INTO fx.log SELECT n FROM newrows; RETURN NULL; END $$;', + 'CREATE TRIGGER t_tg AFTER UPDATE ON fx.t REFERENCING NEW TABLE AS newrows FOR EACH STATEMENT WHEN (pg_trigger_depth() = 0) EXECUTE FUNCTION fx.tg();' + ].join('\n') + }, + { + name: 'constraint-trigger-deferrable', + v1: [S, 'CREATE TABLE fx.t (id int PRIMARY KEY); CREATE FUNCTION fx.tg() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RETURN NEW; END $$;'].join('\n'), + v2: [S, 'CREATE TABLE fx.t (id int PRIMARY KEY); CREATE FUNCTION fx.tg() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RETURN NEW; END $$;', + 'CREATE CONSTRAINT TRIGGER t_ct AFTER INSERT ON fx.t DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION fx.tg();'].join('\n') + }, + { + name: 'trigger-instead-of-view-and-truncate', + v1: [S, 'CREATE TABLE fx.t (id int PRIMARY KEY); CREATE VIEW fx.v AS SELECT id FROM fx.t; CREATE FUNCTION fx.tg() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RETURN NEW; END $$;'].join('\n'), + v2: [S, 'CREATE TABLE fx.t (id int PRIMARY KEY); CREATE VIEW fx.v AS SELECT id FROM fx.t; CREATE FUNCTION fx.tg() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RETURN NEW; END $$;', + 'CREATE TRIGGER v_io INSTEAD OF INSERT ON fx.v FOR EACH ROW EXECUTE FUNCTION fx.tg();', + 'CREATE TRIGGER t_trunc AFTER TRUNCATE ON fx.t FOR EACH STATEMENT EXECUTE FUNCTION fx.tg();', + 'ALTER TABLE fx.t DISABLE TRIGGER t_trunc;'].join('\n') + }, + { + name: 'event-trigger', + v1: `${S} CREATE FUNCTION fx.evt() RETURNS event_trigger LANGUAGE plpgsql AS $$ BEGIN NULL; END $$;`, + v2: `${S} CREATE FUNCTION fx.evt() RETURNS event_trigger LANGUAGE plpgsql AS $$ BEGIN NULL; END $$; CREATE EVENT TRIGGER fx_evt ON ddl_command_end WHEN TAG IN ('CREATE TABLE') EXECUTE FUNCTION fx.evt();`, + revert: 'DROP EVENT TRIGGER IF EXISTS fx_evt;' + }, + // ---- policies + { + name: 'policy-restrictive-roles-check', + v1: `${S} CREATE TABLE fx.t (id int PRIMARY KEY, owner text); ALTER TABLE fx.t ENABLE ROW LEVEL SECURITY; CREATE POLICY p ON fx.t FOR SELECT USING (owner = current_user);`, + v2: `${S} CREATE TABLE fx.t (id int PRIMARY KEY, owner text); ALTER TABLE fx.t ENABLE ROW LEVEL SECURITY; ALTER TABLE fx.t FORCE ROW LEVEL SECURITY; CREATE POLICY p ON fx.t AS RESTRICTIVE FOR ALL TO PUBLIC USING (owner = current_user) WITH CHECK (owner IS NOT NULL);` + }, + { + name: 'policy-command-change-and-second-policy', + v1: `${S} CREATE TABLE fx.t (id int PRIMARY KEY); ALTER TABLE fx.t ENABLE ROW LEVEL SECURITY; CREATE POLICY p ON fx.t FOR INSERT TO fx_role WITH CHECK (true);`, + v2: `${S} CREATE TABLE fx.t (id int PRIMARY KEY); ALTER TABLE fx.t ENABLE ROW LEVEL SECURITY; CREATE POLICY p ON fx.t FOR UPDATE TO fx_role USING (id > 0); CREATE POLICY q ON fx.t FOR DELETE USING (false);`, + roles: ['fx_role'] + }, + // ---- types + { + name: 'domain-multi-constraints', + v1: `${S} CREATE DOMAIN fx.email AS text CHECK (VALUE ~ '@'); CREATE TABLE fx.t (id int PRIMARY KEY, e fx.email);`, + v2: `${S} CREATE DOMAIN fx.email AS text NOT NULL DEFAULT 'a@b' CONSTRAINT email_at CHECK (VALUE ~ '@') CONSTRAINT email_len CHECK (length(VALUE) < 100) COLLATE "C"; CREATE TABLE fx.t (id int PRIMARY KEY, e fx.email);`, + flags: '--allow-data-loss' + }, + { + name: 'range-and-multirange-type', + v1: `${S} CREATE TABLE fx.t (id int PRIMARY KEY);`, + v2: `${S} CREATE TYPE fx.floatrange AS RANGE (subtype = float8, subtype_diff = float8mi, multirange_type_name = fx.floatmultirange); CREATE TABLE fx.t (id int PRIMARY KEY, r fx.floatrange, mr fx.floatmultirange);` + }, + { + name: 'enum-rename-value', + v1: `${S} CREATE TYPE fx.c AS ENUM ('red', 'grn'); CREATE TABLE fx.t (id int PRIMARY KEY, c fx.c);`, + v2: `${S} CREATE TYPE fx.c AS ENUM ('red', 'green'); CREATE TABLE fx.t (id int PRIMARY KEY, c fx.c);`, + flags: '--allow-data-loss --allow-incomplete', + rows: "INSERT INTO fx.t VALUES (1, 'red')" + }, + { + name: 'composite-attribute-type-change', + v1: `${S} CREATE TYPE fx.pt AS (x int, y int); CREATE TABLE fx.t (id int PRIMARY KEY, p fx.pt, ps fx.pt[]);`, + v2: `${S} CREATE TYPE fx.pt AS (x bigint, y int, label text COLLATE "C"); CREATE TABLE fx.t (id int PRIMARY KEY, p fx.pt, ps fx.pt[]);`, + flags: '--allow-data-loss', + rows: 'INSERT INTO fx.t VALUES (1, ROW(1, 2), ARRAY[ROW(3, 4)::fx.pt]), (2, NULL, NULL)', + // NULLs survive the jsonb bridge as NULLs, not as empty records/arrays. + check: { + sql: 'SELECT id, (p).x AS x, (p).label AS label, (ps[1]).x AS px, ps IS NULL AS ps_null FROM fx.t ORDER BY id', + rows: [{ id: 1, x: '1', label: null, px: '3', ps_null: false }, { id: 2, x: null, label: null, px: null, ps_null: true }] + } + }, + { + name: 'shell-type-io-functions', + v1: `${S}`, + v2: `${S} CREATE TYPE fx.pt AS (x int, y int); CREATE TYPE fx.clr AS ENUM ('r'); CREATE DOMAIN fx.posint AS int CHECK (VALUE > 0); CREATE TABLE fx.t (id fx.posint PRIMARY KEY, p fx.pt DEFAULT ROW(0, 0), c fx.clr DEFAULT 'r', tags text[] DEFAULT '{}', ints int[] DEFAULT ARRAY[1, 2]);` + }, + // ---- collations, extensions, operators, casts, aggregates + { + name: 'collation-create-and-column-collate', + v1: `${S} CREATE TABLE fx.t (id int PRIMARY KEY, name text);`, + v2: `${S} CREATE COLLATION fx.nocase (provider = icu, locale = 'und-u-ks-level2', deterministic = false); CREATE TABLE fx.t (id int PRIMARY KEY, name text COLLATE fx.nocase);`, + flags: '--allow-data-loss', + rows: "INSERT INTO fx.t VALUES (1, 'A')", + check: { sql: 'SELECT id, name FROM fx.t', rows: [{ id: 1, name: 'A' }] } + }, + { + name: 'extension-add-drop-quoted', + v1: `CREATE EXTENSION IF NOT EXISTS pgcrypto; ${S} CREATE TABLE fx.t (id uuid PRIMARY KEY DEFAULT gen_random_uuid());`, + v2: `CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; CREATE EXTENSION IF NOT EXISTS citext SCHEMA public; ${S} CREATE TABLE fx.t (id uuid PRIMARY KEY DEFAULT uuid_generate_v4(), e citext);`, + revert: 'DROP EXTENSION IF EXISTS pgcrypto; DROP EXTENSION IF EXISTS "uuid-ossp"; DROP EXTENSION IF EXISTS citext;' + }, + { + name: 'operator-and-opclass', + v1: `${S} CREATE FUNCTION fx.add3(int, int) RETURNS int LANGUAGE sql IMMUTABLE AS $$ SELECT $1 + $2 + 3 $$;`, + v2: `${S} CREATE FUNCTION fx.add3(int, int) RETURNS int LANGUAGE sql IMMUTABLE AS $$ SELECT $1 + $2 + 3 $$; CREATE OPERATOR fx.### (LEFTARG = int, RIGHTARG = int, FUNCTION = fx.add3);` + }, + { + name: 'cast-add', + v1: `${S} CREATE TYPE fx.pt AS (x int, y int); CREATE FUNCTION fx.pt_to_text(fx.pt) RETURNS text LANGUAGE sql IMMUTABLE AS $$ SELECT format('%s,%s', $1.x, $1.y) $$;`, + v2: `${S} CREATE TYPE fx.pt AS (x int, y int); CREATE FUNCTION fx.pt_to_text(fx.pt) RETURNS text LANGUAGE sql IMMUTABLE AS $$ SELECT format('%s,%s', $1.x, $1.y) $$; CREATE CAST (fx.pt AS text) WITH FUNCTION fx.pt_to_text(fx.pt) AS ASSIGNMENT;`, + revert: '' + }, + { + name: 'aggregate-add-and-change', + v1: `${S} CREATE FUNCTION fx.sum_sfunc(int, int) RETURNS int LANGUAGE sql IMMUTABLE AS $$ SELECT $1 + $2 $$; CREATE AGGREGATE fx.mysum (int) (SFUNC = fx.sum_sfunc, STYPE = int, INITCOND = '0');`, + v2: `${S} CREATE FUNCTION fx.sum_sfunc(int, int) RETURNS int LANGUAGE sql IMMUTABLE AS $$ SELECT $1 + $2 $$; CREATE FUNCTION fx.sum_final(int) RETURNS text LANGUAGE sql IMMUTABLE AS $$ SELECT $1::text $$; CREATE AGGREGATE fx.mysum (int) (SFUNC = fx.sum_sfunc, STYPE = int, FINALFUNC = fx.sum_final, INITCOND = '0', PARALLEL = SAFE);` + }, + // ---- functions and procedures + { + name: 'procedure-body-change', + v1: `${S} CREATE TABLE fx.t (id int PRIMARY KEY); CREATE PROCEDURE fx.ins(p int) LANGUAGE plpgsql AS $$ BEGIN INSERT INTO fx.t VALUES (p); END $$;`, + v2: `${S} CREATE TABLE fx.t (id int PRIMARY KEY); CREATE PROCEDURE fx.ins(p int) LANGUAGE plpgsql AS $$ BEGIN INSERT INTO fx.t VALUES (p); COMMIT; END $$;` + }, + { + name: 'function-overloads-one-changes', + v1: `${S} CREATE FUNCTION fx.f(int) RETURNS int LANGUAGE sql AS $$ SELECT $1 $$; CREATE FUNCTION fx.f(text) RETURNS text LANGUAGE sql AS $$ SELECT $1 $$;`, + v2: `${S} CREATE FUNCTION fx.f(int) RETURNS int LANGUAGE sql AS $$ SELECT $1 + 1 $$; CREATE FUNCTION fx.f(text) RETURNS text LANGUAGE sql AS $$ SELECT $1 $$; CREATE FUNCTION fx.f(int, int) RETURNS int LANGUAGE sql AS $$ SELECT $1 + $2 $$;`, + deployed: [/CREATE OR REPLACE FUNCTION fx\.f\(int/] + }, + { + name: 'function-default-args-out-params-returns-table', + v1: `${S} CREATE FUNCTION fx.g(a int, b int DEFAULT 1) RETURNS int LANGUAGE sql AS $$ SELECT a + b $$; CREATE FUNCTION fx.o(IN a int, OUT dbl int, OUT tpl int) LANGUAGE sql AS $$ SELECT a * 2, a * 3 $$;`, + v2: `${S} CREATE FUNCTION fx.g(a int, b int DEFAULT 2) RETURNS int LANGUAGE sql AS $$ SELECT a + b $$; CREATE FUNCTION fx.o(IN a int, OUT dbl int, OUT tpl int, OUT quad int) LANGUAGE sql AS $$ SELECT a * 2, a * 3, a * 4 $$; CREATE FUNCTION fx.rt(n int) RETURNS TABLE (i int, sq int) LANGUAGE sql AS $$ SELECT g, g * g FROM generate_series(1, n) g $$; CREATE FUNCTION fx.so(n int) RETURNS SETOF int LANGUAGE sql AS $$ SELECT generate_series(1, n) $$;` + }, + { + name: 'function-security-definer-search-path-volatility', + v1: `${S} CREATE FUNCTION fx.s() RETURNS int LANGUAGE sql VOLATILE AS $$ SELECT 1 $$;`, + v2: `${S} CREATE FUNCTION fx.s() RETURNS int LANGUAGE sql STABLE SECURITY DEFINER SET search_path = fx, pg_temp PARALLEL SAFE COST 10 AS $$ SELECT 1 $$;`, + deployed: [/CREATE OR REPLACE FUNCTION fx\.s\(\)/] + }, + { + name: 'function-return-type-change-with-view-dependent', + v1: `${S} CREATE FUNCTION fx.r() RETURNS int LANGUAGE sql AS $$ SELECT 1 $$; CREATE VIEW fx.v AS SELECT fx.r() AS r;`, + v2: `${S} CREATE FUNCTION fx.r() RETURNS bigint LANGUAGE sql AS $$ SELECT 1 $$; CREATE VIEW fx.v AS SELECT fx.r() AS r;` + }, + { + name: 'function-sql-standard-body', + v1: `${S} CREATE FUNCTION fx.k(a int) RETURNS int LANGUAGE sql BEGIN ATOMIC SELECT a * 2; END;`, + v2: `${S} CREATE FUNCTION fx.k(a int) RETURNS int LANGUAGE sql BEGIN ATOMIC SELECT a * 3; END;` + }, + // ---- views + { + name: 'view-check-option-security-invoker', + v1: `${S} CREATE TABLE fx.t (id int PRIMARY KEY, n int); CREATE VIEW fx.v AS SELECT id, n FROM fx.t WHERE n > 0;`, + v2: `${S} CREATE TABLE fx.t (id int PRIMARY KEY, n int); CREATE VIEW fx.v WITH (security_invoker = true, security_barrier = true) AS SELECT id, n FROM fx.t WHERE n > 0 WITH CASCADED CHECK OPTION;` + }, + { + name: 'view-recursive-and-column-list', + v1: `${S} CREATE TABLE fx.t (id int PRIMARY KEY, parent int);`, + v2: `${S} CREATE TABLE fx.t (id int PRIMARY KEY, parent int); CREATE RECURSIVE VIEW fx.tree (id, depth) AS SELECT id, 0 FROM fx.t WHERE parent IS NULL UNION ALL SELECT t.id, tree.depth + 1 FROM fx.t t JOIN tree ON t.parent = tree.id; CREATE VIEW fx.named (a, b) AS SELECT id, parent FROM fx.t;` + }, + { + name: 'matview-with-index-and-no-data', + v1: `${S} CREATE TABLE fx.t (id int PRIMARY KEY, n int);`, + v2: `${S} CREATE TABLE fx.t (id int PRIMARY KEY, n int); CREATE MATERIALIZED VIEW fx.mv WITH (fillfactor = 90) AS SELECT id, n FROM fx.t WITH NO DATA; CREATE UNIQUE INDEX mv_id ON fx.mv (id);` + }, + { + name: 'matview-definition-change-keeps-index', + v1: `${S} CREATE TABLE fx.t (id int PRIMARY KEY, n int); CREATE MATERIALIZED VIEW fx.mv AS SELECT id, n FROM fx.t; CREATE UNIQUE INDEX mv_id ON fx.mv (id);`, + v2: `${S} CREATE TABLE fx.t (id int PRIMARY KEY, n int); CREATE MATERIALIZED VIEW fx.mv AS SELECT id, n, n * 2 AS dbl FROM fx.t; CREATE UNIQUE INDEX mv_id ON fx.mv (id);` + }, + // ---- identifiers + { + name: 'quoted-mixed-case-unicode-reserved', + v1: `${S} CREATE TABLE fx."MyTable" ("Id" int PRIMARY KEY, "order" text, "tëst" int); CREATE TABLE fx."user" (id int PRIMARY KEY);`, + v2: `${S} CREATE TABLE fx."MyTable" ("Id" int PRIMARY KEY, "order" text NOT NULL DEFAULT 'x', "tëst" bigint, "select" boolean); CREATE TABLE fx."user" (id int PRIMARY KEY, "group" text); CREATE INDEX "MyTable_order_Idx" ON fx."MyTable" ("order"); CREATE VIEW fx."View One" AS SELECT "Id", "order" FROM fx."MyTable";`, + flags: '--allow-data-loss', + rows: `INSERT INTO fx."MyTable" VALUES (1, 'o', 2)`, + check: { sql: 'SELECT "Id", "order", "tëst" FROM fx."MyTable"', rows: [{ Id: 1, order: 'o', tëst: '2' }] } + }, + { + name: 'quoted-schema-and-dollar-names', + v1: `CREATE SCHEMA "Fx Two"; ${S} CREATE TABLE "Fx Two".t (id int PRIMARY KEY);`, + v2: `CREATE SCHEMA "Fx Two"; ${S} CREATE TABLE "Fx Two".t (id int PRIMARY KEY, a$b int); CREATE TABLE "Fx Two"."t 2" (id int PRIMARY KEY REFERENCES "Fx Two".t (id));`, + revert: 'DROP SCHEMA "Fx Two" CASCADE;' + }, + // ---- storage, reloptions, persistence + { + name: 'column-storage-compression', + v1: `${S} CREATE TABLE fx.t (id int PRIMARY KEY, body text);`, + v2: `${S} CREATE TABLE fx.t (id int PRIMARY KEY, body text STORAGE EXTERNAL COMPRESSION pglz);` + }, + { + name: 'table-reloptions-and-unlogged', + v1: `${S} CREATE TABLE fx.t (id int PRIMARY KEY) WITH (fillfactor = 70); CREATE TABLE fx.u (id int PRIMARY KEY);`, + v2: `${S} CREATE TABLE fx.t (id int PRIMARY KEY) WITH (fillfactor = 90, autovacuum_enabled = false); CREATE UNLOGGED TABLE fx.u (id int PRIMARY KEY);` + }, + // ---- sequences and identity + { + name: 'sequence-owned-by-and-options', + v1: `${S} CREATE TABLE fx.t (id int PRIMARY KEY); CREATE SEQUENCE fx.s AS int INCREMENT BY 2 OWNED BY fx.t.id;`, + v2: `${S} CREATE SEQUENCE fx.s AS bigint INCREMENT BY 5 START WITH 100 CYCLE; CREATE TABLE fx.t (id int PRIMARY KEY, n int DEFAULT nextval('fx.s')); ALTER SEQUENCE fx.s OWNED BY fx.t.n;` + }, + { + name: 'identity-always-vs-by-default', + v1: `${S} CREATE TABLE fx.t (id int GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, n int);`, + v2: `${S} CREATE TABLE fx.t (id int GENERATED ALWAYS AS IDENTITY (START WITH 10 INCREMENT BY 2) PRIMARY KEY, n int GENERATED BY DEFAULT AS IDENTITY);`, + rows: 'INSERT INTO fx.t (n) VALUES (5)' + }, + { + name: 'serial-to-identity', + v1: `${S} CREATE TABLE fx.t (id serial PRIMARY KEY);`, + v2: `${S} CREATE TABLE fx.t (id int GENERATED ALWAYS AS IDENTITY PRIMARY KEY);`, + flags: '--allow-data-loss --allow-incomplete' + }, + // ---- comments and grants on every kind + { + name: 'comments-every-object-kind', + v1: [S, 'CREATE TABLE fx.t (id int PRIMARY KEY, n int CONSTRAINT n_pos CHECK (n > 0)); CREATE INDEX t_n ON fx.t (n); CREATE SEQUENCE fx.s; CREATE DOMAIN fx.d AS int; CREATE TYPE fx.e AS ENUM (\'a\'); CREATE VIEW fx.v AS SELECT id FROM fx.t;', + 'CREATE FUNCTION fx.tg() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RETURN NEW; END $$; CREATE TRIGGER t_tg BEFORE INSERT ON fx.t FOR EACH ROW EXECUTE FUNCTION fx.tg();', + 'ALTER TABLE fx.t ENABLE ROW LEVEL SECURITY; CREATE POLICY p ON fx.t USING (true);'].join('\n'), + v2: [S, 'CREATE TABLE fx.t (id int PRIMARY KEY, n int CONSTRAINT n_pos CHECK (n > 0)); CREATE INDEX t_n ON fx.t (n); CREATE SEQUENCE fx.s; CREATE DOMAIN fx.d AS int; CREATE TYPE fx.e AS ENUM (\'a\'); CREATE VIEW fx.v AS SELECT id FROM fx.t;', + 'CREATE FUNCTION fx.tg() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RETURN NEW; END $$; CREATE TRIGGER t_tg BEFORE INSERT ON fx.t FOR EACH ROW EXECUTE FUNCTION fx.tg();', + 'ALTER TABLE fx.t ENABLE ROW LEVEL SECURITY; CREATE POLICY p ON fx.t USING (true);', + "COMMENT ON SCHEMA fx IS 'schema'; COMMENT ON TABLE fx.t IS 'table'; COMMENT ON COLUMN fx.t.n IS 'col'; COMMENT ON CONSTRAINT n_pos ON fx.t IS 'con'; COMMENT ON INDEX fx.t_n IS 'idx';", + "COMMENT ON SEQUENCE fx.s IS 'seq'; COMMENT ON DOMAIN fx.d IS 'dom'; COMMENT ON TYPE fx.e IS 'enum'; COMMENT ON VIEW fx.v IS 'view'; COMMENT ON FUNCTION fx.tg() IS 'fn';", + "COMMENT ON TRIGGER t_tg ON fx.t IS 'trg'; COMMENT ON POLICY p ON fx.t IS 'pol';"].join('\n'), + notDeployed: [/DROP /] + }, + { + name: 'grants-non-table-objects', + v1: `${S} CREATE TABLE fx.t (id int PRIMARY KEY); CREATE SEQUENCE fx.s; CREATE FUNCTION fx.f() RETURNS int LANGUAGE sql AS $$ SELECT 1 $$; CREATE TYPE fx.e AS ENUM ('a');`, + v2: `${S} CREATE TABLE fx.t (id int PRIMARY KEY); CREATE SEQUENCE fx.s; CREATE FUNCTION fx.f() RETURNS int LANGUAGE sql AS $$ SELECT 1 $$; CREATE TYPE fx.e AS ENUM ('a'); + GRANT USAGE ON SCHEMA fx TO fx_reader; GRANT SELECT (id) ON fx.t TO fx_reader; GRANT USAGE, SELECT ON SEQUENCE fx.s TO fx_reader; REVOKE EXECUTE ON FUNCTION fx.f() FROM PUBLIC; GRANT EXECUTE ON FUNCTION fx.f() TO fx_reader; GRANT USAGE ON TYPE fx.e TO fx_reader; GRANT SELECT ON ALL TABLES IN SCHEMA fx TO fx_reader; ALTER DEFAULT PRIVILEGES IN SCHEMA fx GRANT SELECT ON TABLES TO fx_reader;`, + roles: ['fx_reader'], + notDeployed: [/DROP /] + }, + { + name: 'owner-change', + v1: `${S} CREATE TABLE fx.t (id int PRIMARY KEY); CREATE FUNCTION fx.f() RETURNS int LANGUAGE sql AS $$ SELECT 1 $$;`, + v2: `${S} CREATE TABLE fx.t (id int PRIMARY KEY); CREATE FUNCTION fx.f() RETURNS int LANGUAGE sql AS $$ SELECT 1 $$; ALTER TABLE fx.t OWNER TO fx_owner; ALTER FUNCTION fx.f() OWNER TO fx_owner; ALTER SCHEMA fx OWNER TO fx_owner;`, + roles: ['fx_owner'], + flags: '--allow-incomplete', + notDeployed: [/DROP /], + // v1 declares no owner, so the inverse of OWNER TO is not derivable: the + // revert is reported incomplete and leaves fx_owner (and its implicit + // privileges) in place. + revertResidual: [ + /^grants: fx\.f\(\):fx_owner:EXECUTE only in original$/, + ...['DELETE', 'INSERT', 'REFERENCES', 'SELECT', 'TRIGGER', 'TRUNCATE', 'UPDATE', 'USAGE'].map(p => new RegExp(`^grants: fx\\.t:fx_owner:${p} only in original$`)), + /^grants: fx:fx_owner:CREATE only in original$/, + /^grants: fx:fx_owner:USAGE only in original$/, + /^owners: function:fx\.f\(\) differs\n\s+original:\s+fx_owner\n\s+transformed:\s+postgres$/, + /^owners: relation:fx\.t differs\n\s+original:\s+fx_owner\n\s+transformed:\s+postgres$/, + /^owners: schema:fx differs\n\s+original:\s+fx_owner\n\s+transformed:\s+postgres$/, + /^owners: type:fx\.t differs\n\s+original:\s+fx_owner\n\s+transformed:\s+postgres$/ + ] + }, + // ---- misc statements + { + name: 'rule-and-statistics', + v1: `${S} CREATE TABLE fx.t (id int PRIMARY KEY, a int, b int);`, + v2: `${S} CREATE TABLE fx.t (id int PRIMARY KEY, a int, b int); CREATE STATISTICS fx.t_ab (dependencies) ON a, b FROM fx.t; CREATE RULE t_noop AS ON DELETE TO fx.t DO INSTEAD NOTHING;` + }, + { + name: 'column-type-change-with-index-and-view', + v1: `${S} CREATE TABLE fx.t (id int PRIMARY KEY, code varchar(10), n int); CREATE INDEX t_code ON fx.t (code); CREATE VIEW fx.v AS SELECT code, n FROM fx.t;`, + v2: `${S} CREATE TABLE fx.t (id int PRIMARY KEY, code text, n numeric(12,2)); CREATE INDEX t_code ON fx.t (code); CREATE VIEW fx.v AS SELECT code, n FROM fx.t;`, + flags: '--allow-data-loss', + rows: "INSERT INTO fx.t VALUES (1, 'ab', 3)", + check: { sql: 'SELECT code, n FROM fx.t', rows: [{ code: 'ab', n: '3.00' }] } + }, + { + name: 'array-and-composite-column-types', + v1: `${S} CREATE TYPE fx.pt AS (x int, y int); CREATE TABLE fx.t (id int PRIMARY KEY, pts fx.pt[], m int[][], d numeric(5,2)[]);`, + v2: `${S} CREATE TYPE fx.pt AS (x int, y int); CREATE TABLE fx.t (id int PRIMARY KEY, pts fx.pt[] NOT NULL DEFAULT '{}', m int[][], d numeric(5,2)[], ranges int4range[] DEFAULT ARRAY['[1,2)'::int4range]);`, + rows: 'INSERT INTO fx.t VALUES (1, ARRAY[ROW(1,2)::fx.pt], NULL, NULL)' + } +]; + +describe('pgpm diff e2e: real-world constructs', () => { + let fixture: CLIDeployTestFixture; + let wsDir: string; + + beforeAll(async () => { + await loadModule(); + fixture = new CLIDeployTestFixture(); + wsDir = path.join(fixture.tempFixtureDir, WS); + fs.mkdirSync(wsDir, { recursive: true }); + fs.writeFileSync(path.join(wsDir, 'pgpm.json'), '{\n "packages": [\n "*"\n ]\n}'); + // `pgpm diff` exits the process on a policy error; surface it as a test failure instead. + jest.spyOn(process, 'exit').mockImplementation(((code?: number) => { + throw new Error(`process.exit(${code})`); + }) as never); + }); + + afterAll(async () => { + await fixture.cleanup(); + }); + + const only = process.env.DIFF_CASES ? new Set(process.env.DIFF_CASES.split(',')) : null; + const dials = (process.env.DIFF_DIALS ?? '').split(';').filter(Boolean); + const variants = dials.length > 0 ? dials : ['']; + + for (const c of CASES) { + if (only && !only.has(c.name)) continue; + for (const dial of variants) { + const suffix = dial ? `-${dial.replace(/[^a-z]+/g, '-').replace(/^-|-$/g, '')}` : ''; + it(`${c.name}${suffix}: migrates v1 to v2 and reverts to v1`, async () => { + const v1 = `${c.name}-v1`; + const v2 = `${c.name}-v2`; + const mig = `${c.name}${suffix}-mig`; + writeModule(path.join(wsDir, v1), v1, c.v1, c.revert ?? ''); + writeModule(path.join(wsDir, v2), v2, c.v2, c.revert ?? ''); + const v2Db = await fixture.setupTestDatabase(); + const testDb = await fixture.setupTestDatabase(); + for (const role of c.roles ?? []) { + await testDb.query(`DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '${role}') THEN CREATE ROLE ${role} NOLOGIN; END IF; END $$`); + } + for (const [db, pkg] of [[v2Db, v2], [testDb, v1]] as const) { + await fixture.runTerminalCommands(` + cd ${WS}/${pkg} + pgpm deploy --database ${db.name} --package ${pkg} --yes + `, { database: db.name }); + } + const v1Catalog = withoutColumnOrder(await snapshotCatalog(testDb)); + if (c.rows) await testDb.query(c.rows); + + await fixture.runTerminalCommands(` + cd ${WS} + pgpm diff ${v1} ${v2} ${c.flags ?? ''} ${dial} --emit-migration . --pkg ${mig} + `, {}); + const migrationDir = path.join(wsDir, mig); + const deployed = fs.readdirSync(path.join(migrationDir, 'deploy'), { recursive: true }) + .map(String).filter(f => f.endsWith('.sql')) + .map(f => fs.readFileSync(path.join(migrationDir, 'deploy', f), 'utf-8')).join('\n'); + for (const re of c.deployed ?? []) expect(deployed).toMatch(re); + for (const re of c.notDeployed ?? []) expect(deployed).not.toMatch(re); + + await fixture.runTerminalCommands(` + cd ${WS}/${mig} + pgpm deploy --database ${testDb.name} --package ${mig} --yes + `, { database: testDb.name }); + const forward = diffCatalogSnapshots(withoutColumnOrder(await snapshotCatalog(testDb)), withoutColumnOrder(await snapshotCatalog(v2Db))); + expect(forward).toEqual((c.forwardResidual ?? []).map(re => expect.stringMatching(re))); + if (c.check) { + const rows = await testDb.query(c.check.sql); + expect(rows.rows).toEqual(c.check.rows); + } + + await fixture.runTerminalCommands(` + cd ${WS}/${mig} + pgpm revert --database ${testDb.name} --package ${mig} --yes --no-recursive + `, { database: testDb.name }); + const back = diffCatalogSnapshots(withoutColumnOrder(await snapshotCatalog(testDb)), v1Catalog); + expect(back).toEqual((c.revertResidual ?? []).map(re => expect.stringMatching(re))); + }); + } + } +}); diff --git a/pgpm/cli/__tests__/diff-generated-roundtrip.test.ts b/pgpm/cli/__tests__/diff-generated-roundtrip.test.ts index 2dce4c9bcd..502cb90964 100644 --- a/pgpm/cli/__tests__/diff-generated-roundtrip.test.ts +++ b/pgpm/cli/__tests__/diff-generated-roundtrip.test.ts @@ -26,6 +26,12 @@ afterAll(async () => { await teardownPgPools(); }); +// Each seed opens three databases; releasing their pools per test keeps the +// connection count flat when jest runs this suite next to the other diff suites. +afterEach(async () => { + await teardownPgPools(); +}); + // ---------------------------------------------------------------- generator /** mulberry32: small, fast, deterministic. */ @@ -47,6 +53,8 @@ interface Column { type: ColumnType; notNull: boolean; def?: string; + storage?: 'external'; + comment?: string; } interface Table { @@ -55,17 +63,24 @@ interface Table { fk?: { column: string; ref: string }; check?: { column: string }; unique?: string; - index?: { column: string; unique: boolean; partial: boolean }; + index?: { column: string; unique: boolean; partial: boolean; include: boolean }; grants: string[]; policy: boolean; trigger: boolean; + triggerDisabled: boolean; comment?: string; + unlogged: boolean; + fillfactor?: number; + identity: boolean; + viewSecurityInvoker: boolean; } interface Schema { enumValues: string[]; seqIncrement: number; fnBody: string; + /** Whether `gen.total(scale int)` exists alongside `gen.total()`. */ + overload: boolean; tables: Table[]; } @@ -99,7 +114,9 @@ const makeColumn = (rng: () => number, name: string): Column => { const type = pick(rng, COLUMN_TYPES); const notNull = rng() < 0.4; const def = rng() < 0.5 ? defaultFor(type) : undefined; - return { name, type, notNull, def }; + const storage = type === 'text' && rng() < 0.3 ? 'external' as const : undefined; + const comment = rng() < 0.2 ? `column ${name}` : undefined; + return { name, type, notNull, def, storage, comment }; }; function generateV1(rng: () => number): Schema { @@ -115,8 +132,14 @@ function generateV1(rng: () => number): Schema { grants: rng() < 0.7 ? ['SELECT'] : [], policy: rng() < 0.5, trigger: rng() < 0.5, - comment: rng() < 0.5 ? `table ${t}` : undefined + triggerDisabled: false, + comment: rng() < 0.5 ? `table ${t}` : undefined, + unlogged: false, + fillfactor: rng() < 0.3 ? 70 : undefined, + identity: rng() < 0.3, + viewSecurityInvoker: rng() < 0.3 }; + if (table.trigger && rng() < 0.3) table.triggerDisabled = true; if (t > 0 && rng() < 0.7) { columns.push({ name: 'ref_id', type: 'int', notNull: false }); table.fk = { column: 'ref_id', ref: `t${Math.floor(rng() * t)}` }; @@ -128,11 +151,14 @@ function generateV1(rng: () => number): Schema { if (distinct.length > 0 && rng() < 0.5) table.unique = pick(rng, distinct).name; if (rng() < 0.7) { const unique = distinct.length > 0 && rng() < 0.3; - table.index = { column: pick(rng, unique ? distinct : plain).name, unique, partial: rng() < 0.3 }; + table.index = { column: pick(rng, unique ? distinct : plain).name, unique, partial: rng() < 0.3, include: rng() < 0.3 }; } tables.push(table); } - return { enumValues: ['a', 'b'], seqIncrement: 1, fnBody: 'count(*)', tables }; + // A permanent table's FK may only reference a permanent table. + const referenced = new Set(tables.map(t => t.fk?.ref)); + for (const table of tables) if (!referenced.has(table.name) && rng() < 0.25) table.unlogged = true; + return { enumValues: ['a', 'b'], seqIncrement: 1, fnBody: 'count(*)', overload: rng() < 0.4, tables }; } const clone = (x: T): T => JSON.parse(JSON.stringify(x)); @@ -161,7 +187,9 @@ function mutate(rng: () => number, v1: Schema): { v2: Schema; mutation: Mutation const ops = [ 'addColumn', 'dropColumn', 'changeDefault', 'toggleNotNull', 'widenType', 'toggleIndex', 'changeFn', 'togglePolicy', 'changeGrants', 'addEnumValue', 'addTable', 'dropTable', 'toggleCheck', 'changeSeq', - 'toggleTrigger', 'changeComment', 'renameColumn' + 'toggleTrigger', 'changeComment', 'renameColumn', + 'toggleUnlogged', 'toggleReloptions', 'toggleIdentity', 'toggleTriggerState', 'toggleIndexInclude', 'toggleViewOption', + 'toggleStorage', 'toggleOverload', 'changeColumnComment' ] as const; const applied: string[] = []; let guard = 0; @@ -206,7 +234,7 @@ function mutate(rng: () => number, v1: Schema): { v2: Schema; mutation: Mutation if (table.index) { const flippable = UNIQUE_SAFE.includes(table.columns.find(c => c.name === table.index!.column)?.type ?? 'boolean'); table.index = rng() < 0.5 || !flippable ? undefined : { ...table.index, unique: !table.index.unique }; - } else if (free.length > 0) table.index = { column: pick(rng, free).name, unique: false, partial: true }; + } else if (free.length > 0) table.index = { column: pick(rng, free).name, unique: false, partial: true, include: false }; else continue; break; } @@ -219,7 +247,8 @@ function mutate(rng: () => number, v1: Schema): { v2: Schema; mutation: Mutation name: `t${v2.tables.length}`, columns: [makeColumn(rng, 'c0'), { name: 'ref_id', type: 'int', notNull: false }], fk: { column: 'ref_id', ref: v2.tables[0].name }, - grants: ['SELECT'], policy: false, trigger: false + grants: ['SELECT'], policy: false, trigger: false, triggerDisabled: false, + unlogged: v2.tables[0].unlogged, identity: false, viewSecurityInvoker: false }; v2.tables.push(t); break; @@ -252,6 +281,38 @@ function mutate(rng: () => number, v1: Schema): { v2: Schema; mutation: Mutation mutation.renames.push(`gen.${table.name}.${from}=gen.${table.name}.${col.name}`); break; } + case 'toggleUnlogged': { + if (v2.tables.some(t => t.fk?.ref === table.name)) continue; + table.unlogged = !table.unlogged; + break; + } + case 'toggleReloptions': table.fillfactor = table.fillfactor ? undefined : 70; break; + case 'toggleIdentity': table.identity = !table.identity; break; + case 'toggleTriggerState': { + if (!table.trigger) continue; + table.triggerDisabled = !table.triggerDisabled; + break; + } + case 'toggleIndexInclude': { + if (!table.index) continue; + table.index.include = !table.index.include; + break; + } + case 'toggleViewOption': table.viewSecurityInvoker = !table.viewSecurityInvoker; break; + case 'toggleStorage': { + const texts = free.filter(c => c.type === 'text' || c.type === 'varchar'); + if (texts.length === 0) continue; + const col = pick(rng, texts); + col.storage = col.storage ? undefined : 'external'; + break; + } + case 'toggleOverload': v2.overload = !v2.overload; break; + case 'changeColumnComment': { + if (free.length === 0) continue; + const col = pick(rng, free); + col.comment = col.comment ? undefined : 'changed'; + break; + } } applied.push(op); } @@ -264,7 +325,7 @@ function renderSql(schema: Schema): string { out.push(`CREATE TYPE gen.mood AS ENUM (${schema.enumValues.map(v => `'${v}'`).join(', ')});`); out.push(`CREATE SEQUENCE gen.seq INCREMENT ${schema.seqIncrement};`); schema.tables.forEach((table, i) => { - const cols = [' id int PRIMARY KEY']; + const cols = [` id int${table.identity ? ' GENERATED BY DEFAULT AS IDENTITY' : ''} PRIMARY KEY`]; for (const c of table.columns) { cols.push(` ${c.name} ${c.type}${c.notNull ? ' NOT NULL' : ''}${c.def ? ` DEFAULT ${c.def}` : ''}`); } @@ -272,9 +333,15 @@ function renderSql(schema: Schema): string { if (table.fk) cols.push(` CONSTRAINT ${table.name}_ref_fk FOREIGN KEY (${table.fk.column}) REFERENCES gen.${table.fk.ref} (id)`); if (table.check) cols.push(` CONSTRAINT ${table.name}_${table.check.column}_pos CHECK (${table.check.column} > 0)`); if (table.unique) cols.push(` CONSTRAINT ${table.name}_${table.unique}_key UNIQUE (${table.unique})`); - out.push(`CREATE TABLE gen.${table.name} (\n${cols.join(',\n')}\n);`); + const withClause = table.fillfactor ? ` WITH (fillfactor = ${table.fillfactor})` : ''; + out.push(`CREATE ${table.unlogged ? 'UNLOGGED ' : ''}TABLE gen.${table.name} (\n${cols.join(',\n')}\n)${withClause};`); + for (const c of table.columns) { + if (c.storage) out.push(`ALTER TABLE gen.${table.name} ALTER COLUMN ${c.name} SET STORAGE ${c.storage};`); + if (c.comment) out.push(`COMMENT ON COLUMN gen.${table.name}.${c.name} IS '${c.comment}';`); + } if (table.index) { - out.push(`CREATE ${table.index.unique ? 'UNIQUE ' : ''}INDEX ${table.name}_${table.index.column}_idx ON gen.${table.name} (${table.index.column})${table.index.partial ? ' WHERE id > 0' : ''};`); + const include = table.index.include ? ' INCLUDE (id)' : ''; + out.push(`CREATE ${table.index.unique ? 'UNIQUE ' : ''}INDEX ${table.name}_${table.index.column}_idx ON gen.${table.name} (${table.index.column})${include}${table.index.partial ? ' WHERE id > 0' : ''};`); } if (table.comment) out.push(`COMMENT ON TABLE gen.${table.name} IS '${table.comment}';`); if (table.grants.length > 0) out.push(`GRANT ${table.grants.join(', ')} ON gen.${table.name} TO PUBLIC;`); @@ -282,13 +349,18 @@ function renderSql(schema: Schema): string { out.push(`ALTER TABLE gen.${table.name} ENABLE ROW LEVEL SECURITY;`); out.push(`CREATE POLICY ${table.name}_read ON gen.${table.name} FOR SELECT USING (true);`); } - out.push(`CREATE VIEW gen.${table.name}_v AS SELECT id${table.columns.map(c => `, ${c.name}`).join('')} FROM gen.${table.name};`); + const viewOptions = table.viewSecurityInvoker ? ' WITH (security_invoker = true)' : ''; + out.push(`CREATE VIEW gen.${table.name}_v${viewOptions} AS SELECT id${table.columns.map(c => `, ${c.name}`).join('')} FROM gen.${table.name};`); if (table.trigger) { out.push(`CREATE FUNCTION gen.${table.name}_touch() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RETURN NEW; END $$;`); out.push(`CREATE TRIGGER ${table.name}_touch BEFORE INSERT ON gen.${table.name} FOR EACH ROW EXECUTE FUNCTION gen.${table.name}_touch();`); + if (table.triggerDisabled) out.push(`ALTER TABLE gen.${table.name} DISABLE TRIGGER ${table.name}_touch;`); } }); out.push(`CREATE FUNCTION gen.total() RETURNS bigint LANGUAGE sql AS $$ SELECT ${schema.fnBody} FROM gen.${schema.tables[0].name} $$;`); + if (schema.overload) { + out.push(`CREATE FUNCTION gen.total(scale int) RETURNS bigint LANGUAGE sql AS $$ SELECT ${schema.fnBody} * scale FROM gen.${schema.tables[0].name} $$;`); + } return out.join('\n'); } @@ -319,7 +391,7 @@ const writeModule = (moduleDir: string, name: string, sql: string) => { }; const WS = 'gen-ws'; -const SEEDS = [11, 22, 33, 44, 55, 66, 77, 88]; +const SEEDS = [11, 22, 33, 44, 55, 66, 77, 88, 99, 111, 122, 133, 144, 155, 166, 177]; const DIALS: [string, string][] = [ ['object', 'object'], ['atomic', 'alteration'], diff --git a/pgpm/cli/package.json b/pgpm/cli/package.json index 4016d52a1d..639a3647d0 100644 --- a/pgpm/cli/package.json +++ b/pgpm/cli/package.json @@ -65,7 +65,7 @@ "js-yaml": "^4.1.0", "pg-cache": "workspace:^", "pg-env": "workspace:^", - "pgsql-deparser": "^18.4.2", + "pgsql-deparser": "^18.4.3", "semver": "^7.8.5", "shelljs": "^0.10.0", "yanse": "^0.2.2" diff --git a/pgpm/core/package.json b/pgpm/core/package.json index d16764439c..b5a63163bc 100644 --- a/pgpm/core/package.json +++ b/pgpm/core/package.json @@ -68,8 +68,8 @@ "pg": "^8.21.0", "pg-cache": "workspace:^", "pg-env": "workspace:^", - "pgsql-deparser": "^18.4.2", - "pgsql-parser": "^18.3.2", + "pgsql-deparser": "^18.4.3", + "pgsql-parser": "^18.3.3", "semver": "^7.8.5", "yaml": "^2.9.0", "yanse": "^0.2.2" diff --git a/pgpm/slice/package.json b/pgpm/slice/package.json index 5589c483e4..5583c6ed42 100644 --- a/pgpm/slice/package.json +++ b/pgpm/slice/package.json @@ -45,6 +45,6 @@ "@pgpmjs/ast": "workspace:^", "@pgpmjs/transform": "workspace:^", "minimatch": "^10.2.6", - "plpgsql-parser": "^18.6.2" + "plpgsql-parser": "^18.6.3" } } diff --git a/pgpm/transform/__tests__/semantic-diff-hardening.test.ts b/pgpm/transform/__tests__/semantic-diff-hardening.test.ts index 2a6c7825cf..7ddc03240f 100644 --- a/pgpm/transform/__tests__/semantic-diff-hardening.test.ts +++ b/pgpm/transform/__tests__/semantic-diff-hardening.test.ts @@ -430,3 +430,30 @@ describe('diffSchemas: review follow-ups', () => { expect(consumer.dependencies).toContain(added.name); }); }); + +describe('diffSchemas: overload identities', () => { + it('treats varchar(10) and varchar as the same function overload (typmods are not part of a signature)', () => { + const fn = (arg: string, body: string): string => + `CREATE FUNCTION app.f(${arg}) RETURNS text LANGUAGE sql AS $$ SELECT ${body} $$;`; + const from = `${S}\n${fn('v varchar(10)', '$1::text')}\n${fn('i int', '$1::text')}`; + const to = `${S}\n${fn('v varchar', 'upper($1::text)')}\n${fn('i int', '$1::text')}`; + const result = diffSchemas(from, to); + const deploy = norm(sql(result, 'deploy')); + expect(deploy).not.toMatch(/DROP FUNCTION/); + expect(deploy).toMatch(/CREATE OR REPLACE FUNCTION app\.f\(v varchar\)/); + expect(deploy).not.toMatch(/app\.f\(i int\)/); + }); + + it('keeps operators with the same symbol but different operand types apart', () => { + const fns = `CREATE FUNCTION app.add_i(int, int) RETURNS int LANGUAGE sql AS $$ SELECT $1 + $2 $$; + CREATE FUNCTION app.add_t(text, text) RETURNS text LANGUAGE sql AS $$ SELECT $1 || $2 $$;`; + const opInt = 'CREATE OPERATOR app.### (LEFTARG = int, RIGHTARG = int, FUNCTION = app.add_i);'; + const opText = 'CREATE OPERATOR app.### (LEFTARG = text, RIGHTARG = text, FUNCTION = app.add_t);'; + const result = diffSchemas(`${S}\n${fns}\n${opInt}\n${opText}`, `${S}\n${fns}\n${opInt}`, lax); + const deploy = norm(sql(result, 'deploy')); + expect(deploy.match(/DROP OPERATOR/g)).toHaveLength(1); + expect(deploy).toMatch(/DROP OPERATOR app\.### ?\(text, text\)/); + expect(deploy).not.toMatch(/CREATE OPERATOR/); + expect(norm(sql(result, 'revert'))).toMatch(/CREATE OPERATOR app\.### \(LEFTARG = text/); + }); +}); diff --git a/pgpm/transform/package.json b/pgpm/transform/package.json index 201ff437ab..22100b2c02 100644 --- a/pgpm/transform/package.json +++ b/pgpm/transform/package.json @@ -45,9 +45,9 @@ "dependencies": { "@pgpmjs/ast": "workspace:^", "@pgpmjs/naming-spec": "workspace:^", - "@pgsql/scripts": "^18.5.2", - "@pgsql/semantics": "^18.2.2", - "@pgsql/transform": "^18.18.2", - "plpgsql-parser": "^18.6.2" + "@pgsql/scripts": "^18.5.3", + "@pgsql/semantics": "^18.2.3", + "@pgsql/transform": "^18.18.3", + "plpgsql-parser": "^18.6.3" } } diff --git a/pgpm/transform/src/annotations.ts b/pgpm/transform/src/annotations.ts index 301900fdea..897e12cd59 100644 --- a/pgpm/transform/src/annotations.ts +++ b/pgpm/transform/src/annotations.ts @@ -34,7 +34,7 @@ export interface AnnotationState { * `insert*` for WITH GRANT OPTION, `all`) and `-token` for privileges * explicitly revoked (e.g. pg_dump's `REVOKE ALL ... FROM PUBLIC`). */ - grants: Map }>; + grants: Map }>; /** `${objtype}:${object}` → OWNER TO statement template with its RoleSpec. */ owners: Map; /** `${options}:${objtype}:${grantee}` → default-privilege tokens (same scheme as grants). */ @@ -88,12 +88,13 @@ export function absorbAnnotation(state: AnnotationState, stmt: Node): boolean { } if (stmt.GrantStmt) { const g = stmt.GrantStmt as Node; - if (g.targtype !== 'ACL_TARGET_OBJECT') return false; + if (g.targtype !== 'ACL_TARGET_OBJECT' && g.targtype !== 'ACL_TARGET_ALL_IN_SCHEMA') return false; const objects = cleanTree(g.objects); for (const grantee of (g.grantees as unknown[]) ?? []) { const clean = cleanTree(grantee); - const key = `${g.objtype}:${stablePrint(objects)}:${stablePrint(clean)}`; - const entry = state.grants.get(key) ?? { objtype: g.objtype as string, objects, grantee: clean, privs: new Set() }; + const key = `${g.targtype}:${g.objtype}:${stablePrint(objects)}:${stablePrint(clean)}`; + const entry = state.grants.get(key) + ?? { targtype: g.targtype as string, objtype: g.objtype as string, objects, grantee: clean, privs: new Set() }; applyAction(entry.privs, g); state.grants.set(key, entry); } @@ -146,7 +147,7 @@ export interface AnnotationStep { incomplete?: string; } -const grantNode = (isGrant: boolean, objtype: string, objects: unknown, tokens: string[], grantee: unknown): Node => { +const grantNode = (isGrant: boolean, objtype: string, objects: unknown, tokens: string[], grantee: unknown, targtype = 'ACL_TARGET_OBJECT'): Node => { const option = tokens.some(t => t.endsWith('*')); const names = tokens.map(t => t.replace(/\*$/, '')); const privileges = names.includes('all') @@ -160,7 +161,7 @@ const grantNode = (isGrant: boolean, objtype: string, objects: unknown, tokens: return { GrantStmt: { is_grant: isGrant || undefined, - targtype: 'ACL_TARGET_OBJECT', + targtype, objtype, objects, privileges, @@ -266,7 +267,7 @@ export function diffAnnotations(from: AnnotationState, to: AnnotationState, path const prior = from.grants.get(key)?.privs ?? new Set(); const entry = to.grants.get(key) ?? from.grants.get(key)!; const next = to.grants.get(key)?.privs ?? new Set(); - steps.push(...diffPrivileges(prior, next, (isGrant, tokens) => grantNode(isGrant, entry.objtype, entry.objects, tokens, entry.grantee), entry.objtype, entry.grantee)); + steps.push(...diffPrivileges(prior, next, (isGrant, tokens) => grantNode(isGrant, entry.objtype, entry.objects, tokens, entry.grantee, entry.targtype), entry.objtype, entry.grantee)); } for (const key of new Set([...from.defaultPrivileges.keys(), ...to.defaultPrivileges.keys()])) { diff --git a/pgpm/transform/src/catalog-check.ts b/pgpm/transform/src/catalog-check.ts index be0f3d6a3e..01b14b893c 100644 --- a/pgpm/transform/src/catalog-check.ts +++ b/pgpm/transform/src/catalog-check.ts @@ -24,14 +24,17 @@ export interface CatalogSnapshot { triggers: Record; policies: Record; rls: Record; + /** Table/view grants plus schema, sequence, function and type ACLs (defaults made explicit). */ grants: Record; /** Enums (labels in order), domains, composite and range types. */ types: Record; sequences: Record; /** Views and materialized views (definition). */ views: Record; - /** COMMENT ON for schemas, relations, columns, functions and types. */ + /** Every COMMENT ON in user schemas (plus schema-less objects such as extensions and event triggers), keyed by catalog and object description. */ comments: Record; + /** Owners of schemas, relations, functions and types. */ + owners: Record; defaultPrivileges: Record; } @@ -56,16 +59,27 @@ export const snapshotCatalog = async (db: CatalogQueryable): Promise (SELECT current_user) + UNION ALL + SELECT target, g.grantee::regrole::text, g.privilege_type + FROM ( + SELECT n.nspname AS target, coalesce(n.nspacl, acldefault('n', n.nspowner)) AS acl + FROM pg_namespace n WHERE n.nspname NOT IN ${SYSTEM_SCHEMAS} AND n.nspname NOT LIKE 'pg\\_temp\\_%' AND n.nspname NOT LIKE 'pg\\_toast\\_temp\\_%' + UNION ALL + SELECT n.nspname || '.' || cl.relname, coalesce(cl.relacl, acldefault('s', cl.relowner)) + FROM pg_class cl JOIN pg_namespace n ON n.oid = cl.relnamespace + WHERE cl.relkind = 'S' AND n.nspname NOT IN ${SYSTEM_SCHEMAS} + UNION ALL + SELECT n.nspname || '.' || p.proname || '(' || pg_get_function_identity_arguments(p.oid) || ')', coalesce(p.proacl, acldefault('f', p.proowner)) + FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace WHERE n.nspname NOT IN ${SYSTEM_SCHEMAS} + UNION ALL + SELECT n.nspname || '.' || t.typname, coalesce(t.typacl, acldefault('T', t.typowner)) + FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace + WHERE n.nspname NOT IN ${SYSTEM_SCHEMAS} AND t.typtype IN ('e', 'd', 'c', 'r') + AND NOT EXISTS (SELECT 1 FROM pg_type el WHERE el.typarray = t.oid) + ) o, aclexplode(o.acl) g + WHERE g.grantee <> (SELECT oid FROM pg_roles WHERE rolname = current_user) `); const types = await db.query(` SELECT n.nspname AS schema, t.typname, t.typtype, @@ -152,24 +191,28 @@ export const snapshotCatalog = async (db: CatalogQueryable): Promise 0 AND NOT a.attisdropped AND col_description(cl.oid, a.attnum) IS NOT NULL + WHERE n.nspname NOT IN ${SYSTEM_SCHEMAS} AND cl.relkind IN ('r', 'p', 'v', 'm', 'S', 'f') UNION ALL - SELECT 'function', n.nspname || '.' || p.proname || '(' || pg_get_function_identity_arguments(p.oid) || ')', obj_description(p.oid, 'pg_proc') - FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace - WHERE n.nspname NOT IN ${SYSTEM_SCHEMAS} AND obj_description(p.oid, 'pg_proc') IS NOT NULL + SELECT 'function', n.nspname || '.' || p.proname || '(' || pg_get_function_identity_arguments(p.oid) || ')', pg_get_userbyid(p.proowner) + FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace WHERE n.nspname NOT IN ${SYSTEM_SCHEMAS} UNION ALL - SELECT 'type', n.nspname || '.' || t.typname, obj_description(t.oid, 'pg_type') + SELECT 'type', n.nspname || '.' || t.typname, pg_get_userbyid(t.typowner) FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace - WHERE n.nspname NOT IN ${SYSTEM_SCHEMAS} AND obj_description(t.oid, 'pg_type') IS NOT NULL + WHERE n.nspname NOT IN ${SYSTEM_SCHEMAS} AND t.typtype IN ('e', 'd', 'c', 'r') + AND NOT EXISTS (SELECT 1 FROM pg_type el WHERE el.typarray = t.oid) `); const defaultPrivileges = await db.query(` SELECT pg_get_userbyid(d.defaclrole) AS role, n.nspname AS schema, d.defaclobjtype AS objtype, @@ -181,7 +224,11 @@ export const snapshotCatalog = async (db: CatalogQueryable): Promise r.nspname, () => 'schema'), - tables: record(tables.rows, r => `${r.table_schema}.${r.table_name}`, () => 'table'), + tables: record( + tables.rows, + r => `${r.table_schema}.${r.table_name}`, + r => `persistence=${r.relpersistence} options=${r.reloptions ?? ''} bound=${r.partbound ?? ''} inherits=${r.inherits ?? ''}` + ), columns: record( columns.rows, r => `${r.table_schema}.${r.table_name}.${r.column_name}`, @@ -189,7 +236,8 @@ export const snapshotCatalog = async (db: CatalogQueryable): Promise `${r.schema}.${r.table}.${r.conname}`, r => r.def), indexes: record(indexes.rows, r => `${r.schemaname}.${r.tablename}.${r.indexname}`, r => r.indexdef), @@ -208,7 +256,7 @@ export const snapshotCatalog = async (db: CatalogQueryable): Promise `${r.table_schema}.${r.table_name}:${r.grantee}:${r.privilege_type}`, + r => `${r.target}:${r.grantee}:${r.privilege_type}`, () => 'grant' ), types: record(types.rows, r => `${r.schema}.${r.typname}`, r => `${r.typtype} ${r.def ?? ''}`), @@ -219,6 +267,7 @@ export const snapshotCatalog = async (db: CatalogQueryable): Promise `${r.schema}.${r.relname}`, r => `${r.relkind} ${r.def}`), comments: record(comments.rows, r => `${r.kind}:${r.target}`, r => r.comment), + owners: record(owners.rows, r => `${r.kind}:${r.target}`, r => r.owner), defaultPrivileges: record( defaultPrivileges.rows, r => `${r.role}:${r.schema ?? ''}:${r.objtype}`, diff --git a/pgpm/transform/src/facts.ts b/pgpm/transform/src/facts.ts index 17189a8eab..293fe6d263 100644 --- a/pgpm/transform/src/facts.ts +++ b/pgpm/transform/src/facts.ts @@ -10,6 +10,15 @@ * upstream classifies them as `other` with no creates, so they fell into * `misc/statements` and nothing could depend on them. They now create the * relation they name (kind `view` for materialized views, `table` for CTAS). + * - `CREATE EXTENSION`: upstream identifies it but records no creates, so the + * granularity driver had no object to file the statement under and dropped + * it from the program. + * - Schema objects upstream leaves unidentified — collations, aggregates, + * operators (`DefineStmt`), casts, extended statistics, rules, operator + * classes/families, publications, event triggers, foreign tables, foreign + * servers and FDWs. They create the object they name (naming-spec kind + * `other`, so they get their own unit, path and inverse instead of riding + * along with whatever statement preceded them). */ import type { StatementFacts } from '@pgsql/semantics'; import { classifyStatements as upstreamClassify } from '@pgsql/semantics'; @@ -18,11 +27,52 @@ import { identityOf as upstreamIdentityOf } from '@pgsql/transform'; import { parseSync } from 'plpgsql-parser'; import { sliceStatementBytes, sqlSourceBytes } from './byte-slice'; +import { canonicalTypeNames } from './type-canon'; interface IntoRelation { schemaname?: string; relname?: string } +interface Completion { schema: string | null; name: string; kind: StatementFacts['kind']; identity: ObjectIdentityKind } -/** The relation a `CreateTableAsStmt` creates, and the kind pgpm files it under. */ -function createTableAsTarget(text: string): { schema: string | null; name: string; kind: 'view' | 'table' } | null { +type ObjectIdentityKind = ObjectIdentity['kind']; + +const strings = (items: unknown[] | undefined): string[] => + (items ?? []).map(n => (n as { String?: { sval?: string } }).String?.sval).filter((s): s is string => typeof s === 'string'); + +/** Operator symbols spelled out so their paths are distinct and filesystem-safe. */ +const OPERATOR_CHARS: Record = { + '+': 'plus', '-': 'minus', '*': 'star', '/': 'slash', '<': 'lt', '>': 'gt', '=': 'eq', '~': 'tilde', + '!': 'bang', '@': 'at', '#': 'hash', '%': 'pct', '^': 'caret', '&': 'amp', '|': 'pipe', '`': 'tick', '?': 'q' +}; +const operatorName = (symbol: string): string => + `op_${[...symbol].map(c => OPERATOR_CHARS[c] ?? c).join('_')}`; + +interface TypeNameNode { names?: { String?: { sval?: string } }[]; arrayBounds?: unknown[] } + +/** `int4`, `text[]`, ...: a type as it names an overload, filesystem-safe. */ +const typeLabel = (type: TypeNameNode): string => { + const names = strings(canonicalTypeNames((type.names ?? []) as any) as any); + const base = names[names.length - 1] ?? 'unknown'; + return `${base}${type.arrayBounds && type.arrayBounds.length > 0 ? '_array' : ''}`.replace(/[^a-zA-Z0-9_]/g, '_'); +}; + +const qualified = (names: string[]): { schema: string | null; name: string } | null => + names.length === 0 ? null : { schema: names.length > 1 ? names[names.length - 2] : null, name: names[names.length - 1] }; + +const DEFINE_KINDS: Record = { + OBJECT_TYPE: { kind: 'type', identity: 'type' }, + OBJECT_COLLATION: { kind: 'other', identity: 'other' }, + OBJECT_AGGREGATE: { kind: 'other', identity: 'other' }, + OBJECT_OPERATOR: { kind: 'other', identity: 'other' }, + OBJECT_TSCONFIGURATION: { kind: 'other', identity: 'other' }, + OBJECT_TSDICTIONARY: { kind: 'other', identity: 'other' }, + OBJECT_TSPARSER: { kind: 'other', identity: 'other' }, + OBJECT_TSTEMPLATE: { kind: 'other', identity: 'other' } +}; + +const other = (target: { schema: string | null; name: string } | null, kind: StatementFacts['kind'] = 'other'): Completion | null => + target ? { ...target, kind, identity: 'other' } : null; + +/** The object a statement upstream leaves unidentified creates, and the kind pgpm files it under. */ +function completionFor(text: string): Completion | null { let stmts; try { stmts = parseSync(text).sql.stmts; @@ -30,14 +80,93 @@ function createTableAsTarget(text: string): { schema: string | null; name: strin return null; } if (stmts.length !== 1) return null; - const stmt = stmts[0].stmt as unknown as { CreateTableAsStmt?: { into?: { rel?: IntoRelation }; objtype?: string } }; - const rel = stmt.CreateTableAsStmt?.into?.rel; - if (!rel?.relname) return null; - return { - schema: rel.schemaname ?? null, - name: rel.relname, - kind: stmt.CreateTableAsStmt?.objtype === 'OBJECT_MATVIEW' ? 'view' : 'table' - }; + const stmt = stmts[0].stmt as unknown as Record; + const [tag] = Object.keys(stmt); + const node = stmt[tag]; + switch (tag) { + case 'CreateTableAsStmt': { + const rel = node.into?.rel as IntoRelation | undefined; + if (!rel?.relname) return null; + const kind = node.objtype === 'OBJECT_MATVIEW' ? 'view' : 'table'; + return { schema: rel.schemaname ?? null, name: rel.relname, kind, identity: kind }; + } + case 'CreateExtensionStmt': + return node.extname ? { schema: null, name: node.extname, kind: 'extension', identity: 'extension' } : null; + case 'DefineStmt': { + const spec = DEFINE_KINDS[node.kind]; + const target = qualified(strings(node.defnames)); + if (!spec || !target) return null; + // Operators and aggregates overload on their argument types, so the + // types are part of the identity (and the path). + if (node.kind === 'OBJECT_OPERATOR') { + const operand = (which: string): string => { + const def = (node.definition ?? []).find((d: any) => d.DefElem?.defname?.toLowerCase() === which)?.DefElem; + return def?.arg?.TypeName ? typeLabel(def.arg.TypeName) : 'none'; + }; + target.name = `${operatorName(target.name)}__${operand('leftarg')}_${operand('rightarg')}`; + } else if (node.kind === 'OBJECT_AGGREGATE') { + const params = (node.args?.[0]?.List?.items ?? []) as { FunctionParameter?: { argType?: TypeNameNode } }[]; + const labels = params.map(p => p.FunctionParameter?.argType).filter((t): t is TypeNameNode => !!t).map(typeLabel); + target.name = `${target.name}__${labels.length > 0 ? labels.join('_') : 'star'}`; + } + return { ...target, ...spec }; + } + case 'CreateCastStmt': { + const src = qualified(strings(node.sourcetype?.names)); + const tgt = qualified(strings(node.targettype?.names)); + return src && tgt ? other({ schema: null, name: `cast_${src.name}_as_${tgt.name}` }) : null; + } + case 'CreateStatsStmt': + return other(qualified(strings(node.defnames))); + case 'RuleStmt': { + const rel = node.relation as IntoRelation | undefined; + return rel?.relname && node.rulename + ? other({ schema: rel.schemaname ?? null, name: `${rel.relname}.rule.${node.rulename}` }) + : null; + } + case 'CreateOpClassStmt': { + const target = qualified(strings(node.opclassname)); + return target ? other({ schema: target.schema, name: `opclass_${node.amname}_${target.name}` }) : null; + } + case 'CreateOpFamilyStmt': { + const target = qualified(strings(node.opfamilyname)); + return target ? other({ schema: target.schema, name: `opfamily_${node.amname}_${target.name}` }) : null; + } + case 'CreatePublicationStmt': + return node.pubname ? other({ schema: null, name: `publication_${node.pubname}` }) : null; + case 'CreateSubscriptionStmt': + return node.subname ? other({ schema: null, name: `subscription_${node.subname}` }) : null; + case 'CreateEventTrigStmt': + return node.trigname ? other({ schema: null, name: `event_trigger_${node.trigname}` }) : null; + case 'CreateForeignTableStmt': { + const rel = node.base?.relation as IntoRelation | undefined; + return rel?.relname ? other({ schema: rel.schemaname ?? null, name: rel.relname }) : null; + } + case 'CreateForeignServerStmt': + return node.servername ? other({ schema: null, name: `server_${node.servername}` }) : null; + case 'CreateFdwStmt': + return node.fdwname ? other({ schema: null, name: `fdw_${node.fdwname}` }) : null; + default: + return null; + } +} + +/** Node tags `completionFor` identifies; others never need a parse. */ +const COMPLETED_TAGS = new Set([ + 'CreateTableAsStmt', 'CreateExtensionStmt', 'DefineStmt', 'CreateCastStmt', 'CreateStatsStmt', 'RuleStmt', + 'CreateOpClassStmt', 'CreateOpFamilyStmt', 'CreatePublicationStmt', 'CreateSubscriptionStmt', 'CreateEventTrigStmt', + 'CreateForeignTableStmt', 'CreateForeignServerStmt', 'CreateFdwStmt' +]); + +/** Identity kind of a completed statement, from its (completed) facts. */ +function completedIdentityKind(f: StatementFacts): ObjectIdentityKind | null { + if (!COMPLETED_TAGS.has(f.nodeTag) || f.creates.length === 0) return null; + switch (f.nodeTag) { + case 'CreateTableAsStmt': return f.kind === 'view' ? 'view' : 'table'; + case 'CreateExtensionStmt': return 'extension'; + case 'DefineStmt': return f.kind === 'type' ? 'type' : 'other'; + default: return 'other'; + } } /** `classifyStatements` with pgpm's completions applied. */ @@ -45,9 +174,9 @@ export function classifyStatements(sql: string): StatementFacts[] { const facts = upstreamClassify(sql); let bytes: Buffer | null = null; return facts.map(f => { - if (f.nodeTag !== 'CreateTableAsStmt' || f.creates.length > 0) return f; + if (!COMPLETED_TAGS.has(f.nodeTag) || f.creates.length > 0) return f; bytes ??= sqlSourceBytes(sql); - const target = createTableAsTarget(sliceStatementBytes(bytes, f.span.start, f.span.len)); + const target = completionFor(sliceStatementBytes(bytes, f.span.start, f.span.len)); if (!target) return f; return { ...f, kind: target.kind, creates: [{ schema: target.schema, name: target.name }] }; }); @@ -57,8 +186,7 @@ export function classifyStatements(sql: string): StatementFacts[] { export function identityOf(f: StatementFacts): ObjectIdentity | null { const upstream = upstreamIdentityOf(f); if (upstream) return upstream; - if (f.nodeTag === 'CreateTableAsStmt' && f.creates.length > 0) { - return { kind: f.kind === 'view' ? 'view' : 'table', schema: f.creates[0].schema, name: f.creates[0].name }; - } - return null; + const kind = completedIdentityKind(f); + if (!kind) return null; + return { kind, schema: f.creates[0].schema, name: f.creates[0].name }; } diff --git a/pgpm/transform/src/granularity-driver.ts b/pgpm/transform/src/granularity-driver.ts index 6cae7c6d31..f08d052343 100644 --- a/pgpm/transform/src/granularity-driver.ts +++ b/pgpm/transform/src/granularity-driver.ts @@ -111,7 +111,7 @@ export interface RestructureModuleOptions { * (creating) statement. Defaults to {@link defaultChangeName}: naming spec * v1 paths (`identityOf` + `pathFor`). */ - changeName?: (facts: StatementFacts) => string; + changeName?: (facts: StatementFacts, text: string) => string; } export interface RestructureModuleResult { @@ -183,6 +183,7 @@ export function restructureChanges( options: RestructureModuleOptions ): RestructureModuleResult { const nameFor = options.changeName ?? defaultChangeName; + const textOf = (f: StatementFacts): string => sliceStatementBytes(sqlBytes, f.span.start, f.span.len).trim(); const subNameFor = options.subObjectName ?? ((identity: NamingIdentity): string => pathFor(identity)); const alteration = options.changeGranularity === 'alteration'; @@ -348,13 +349,13 @@ export function restructureChanges( moved = true; }); if (!moved) { - const names = [...cyclic].map(g => groupSubName[g] ?? nameFor(groupFacts[g])).sort(); + const names = [...cyclic].map(g => groupSubName[g] ?? nameFor(groupFacts[g], textOf(groupFacts[g]))).sort(); throw new Error(`restructure: dependency cycle among changes; cannot order: ${names.join(', ')}`); } groupDeps = buildGroupDeps(); } - const groupNames = groupFacts.map((f, g) => groupSubName[g] ?? nameFor(f)); + const groupNames = groupFacts.map((f, g) => groupSubName[g] ?? nameFor(f, textOf(f))); // Slice statement text per group and collect each group's facts for // revert/verify generation. diff --git a/pgpm/transform/src/semantic-diff-driver.ts b/pgpm/transform/src/semantic-diff-driver.ts index 1f52f11f5f..9117ba16c4 100644 --- a/pgpm/transform/src/semantic-diff-driver.ts +++ b/pgpm/transform/src/semantic-diff-driver.ts @@ -44,7 +44,7 @@ import { sliceStatementBytes, sqlSourceBytes } from './byte-slice'; import { ConstraintNode, defaultConstraintName } from './constraint-names'; import { classifyStatements } from './facts'; import { identityOf } from './facts'; -import { ChangeGranularity, GranularityChange, restructureChanges } from './granularity-driver'; +import { ChangeGranularity, defaultChangeName, GranularityChange, restructureChanges } from './granularity-driver'; import { objectKeyOf, unitKeyOf } from './keys'; import { alterInPlace } from './object-alters'; import { topologicalOrder } from './order'; @@ -205,7 +205,13 @@ function annotationHost(text: string): string | null { if (objects.length !== 1) return null; const first = objects[0] as Record; if (first.RangeVar) return relation(first.RangeVar); + if (stmt.GrantStmt.targtype === 'ACL_TARGET_ALL_IN_SCHEMA' && first.String) return `.${(first.String as { sval: string }).sval}`; if (stmt.GrantStmt.objtype === 'OBJECT_SCHEMA' && first.String) return `.${(first.String as { sval: string }).sval}`; + const names = first.ObjectWithArgs + ? ((first.ObjectWithArgs as { objname?: { String?: { sval?: string } }[] }).objname ?? []).map(x => x.String?.sval ?? '') + : first.List ? items(first) : []; + if (names.length === 2) return `${names[0]}.${names[1]}`; + if (names.length === 1) return `.${names[0]}`; return null; } if (stmt.AlterDefaultPrivilegesStmt) { @@ -233,7 +239,66 @@ function annotationHost(text: string): string | null { * reference, falling back to the preceding statement's unit; truly homeless * statements are reported via `warnings`. */ -function readSide(sql: string, warnings: string[], label: string): SideProgram { +/** + * Argument-type signature of a `CREATE FUNCTION`/`PROCEDURE` text — the IN, + * INOUT and VARIADIC parameter types that make an overload distinct. Null for + * anything else. + */ +const signatureCache = new Map(); + +/** + * The input argument types of a `CREATE FUNCTION` statement as Postgres + * identifies the function: canonical type names, no typmods (`varchar(10)` + * and `varchar` are the same overload), OUT and TABLE parameters excluded. + */ +function functionArgSignature(text: string): string | null { + if (signatureCache.has(text)) return signatureCache.get(text)!; + let stmts; + try { + stmts = parseSync(text).sql.stmts; + } catch { + signatureCache.set(text, null); + return null; + } + const fn = stmts.length === 1 ? (stmts[0].stmt as unknown as { CreateFunctionStmt?: { parameters?: { FunctionParameter?: { mode?: string; argType?: unknown } }[] } }).CreateFunctionStmt : undefined; + const sig = !fn ? null : (fn.parameters ?? []) + .map(p => p.FunctionParameter) + .filter(p => p && p.mode !== 'FUNC_PARAM_OUT' && p.mode !== 'FUNC_PARAM_TABLE') + .map(p => Deparser.deparse({ TypeName: { ...canonicalizeTypes(cleanTree(p!.argType)), typmods: undefined } } as any, { pretty: false })) + .join(', '); + signatureCache.set(text, sig); + return sig; +} + +/** + * Functions declared with more than one argument signature across both + * sides, keyed `schema.name`. Each overload of such a function is its own + * object: identified (and pathed) as `name(argtypes)`. + */ +function overloadedFunctions(sides: string[]): Set { + const signatures = new Map>(); + for (const sql of sides) { + const bytes = sqlSourceBytes(sql); + for (const f of classifyStatements(sql)) { + if (f.nodeTag !== 'CreateFunctionStmt' || !f.creates[0]) continue; + const sig = functionArgSignature(sliceStatementBytes(bytes, f.span.start, f.span.len)); + if (sig === null) continue; + const name = `${f.creates[0].schema ?? ''}.${f.creates[0].name}`; + if (!signatures.has(name)) signatures.set(name, new Set()); + signatures.get(name)!.add(sig); + } + } + return new Set([...signatures].filter(([, sigs]) => sigs.size > 1).map(([name]) => name)); +} + +/** The identity an overloaded function's statement groups under: `name(argtypes)`. */ +function overloadIdentity(identity: ObjectIdentity, text: string, overloaded: Set): ObjectIdentity { + if (identity.kind !== 'function' || !overloaded.has(`${identity.schema ?? ''}.${identity.name}`)) return identity; + const sig = functionArgSignature(text); + return sig === null ? identity : { ...identity, name: `${identity.name}(${sig})` }; +} + +function readSide(sql: string, warnings: string[], label: string, overloaded: Set): SideProgram { const facts = classifyStatements(sql); const graph = buildStatementGraph(facts); const units = new Map(); @@ -248,6 +313,7 @@ function readSide(sql: string, warnings: string[], label: string): SideProgram { if (!text) return; let identity = groupingIdentity(f); + if (identity && f.nodeTag === 'CreateFunctionStmt') identity = overloadIdentity(identity, text, overloaded); if (identity && identity.kind === 'other') { const asTable = identityKey({ kind: 'table', schema: identity.schema, name: identity.name }); if (units.has(asTable)) identity = units.get(asTable)!.identity; @@ -304,6 +370,112 @@ function groupingIdentity(f: StatementFacts): ObjectIdentity | null { return identity; } +/** + * Fold an `ALTER SEQUENCE` (OWNED BY, increment, ...) into the unit's + * `CREATE SEQUENCE` node so the sequence compares as one definition whether + * its options were authored inline or as a later ALTER. Later options win. + */ +function foldSequenceAlter(nodes: Record[], definition: string[], stmt: Record): boolean { + const alter = stmt.AlterSeqStmt as { options?: { DefElem?: { defname?: string } }[] } | undefined; + if (!alter) return false; + const at = nodes.findIndex(n => n.CreateSeqStmt); + if (at < 0) return false; + const create = nodes[at].CreateSeqStmt as { options?: { DefElem?: { defname?: string } }[] }; + const overridden = new Set((alter.options ?? []).map(o => o.DefElem?.defname)); + const merged = { + CreateSeqStmt: { + ...create, + options: [...(create.options ?? []).filter(o => !overridden.has(o.DefElem?.defname)), ...(alter.options ?? [])] + } + }; + nodes[at] = merged; + definition[at] = deparseStmt(merged); + return true; +} + +/** A table column whose type is (an array of) the given user type. */ +interface DependentColumn { + relation: unknown; + table: string; + schema: string | null; + name: string; + def: Record; + toDef: Record | undefined; + array: boolean; +} + +/** Columns of the side's tables declared with the type (or arrays of it). */ +function typeDependentColumns(side: SideProgram, other: SideProgram, type: ObjectIdentity): DependentColumn[] { + const key = typeKey({ names: [type.schema, type.name].filter((x): x is string => x !== null).map(sval => ({ String: { sval } })) } as any); + const out: DependentColumn[] = []; + for (const unitKey of side.order) { + const unit = side.units.get(unitKey)!; + if (unit.identity.kind !== 'table') continue; + const shape = tableShape(unit); + const toShape = other.units.has(unitKey) ? tableShape(other.units.get(unitKey)!) : null; + for (const [name, col] of shape.columns) { + const colKey = typeKey((col.def as Record).typeName as any); + if (colKey !== key && colKey !== `${key}[]`) continue; + out.push({ + relation: shape.relation, + table: unit.identity.name, + schema: unit.identity.schema, + name, + def: col.def as Record, + toDef: toShape?.columns.get(name)?.def as Record | undefined, + array: colKey !== key + }); + } + } + return out; +} + +/** + * Postgres refuses to alter or drop a type a table column uses, so the + * column is parked in a bridge type (jsonb for composites, text otherwise) + * around the type's change and cast back afterwards. Values the new type + * rejects are lost, which the caller reports as destructive. + */ +function columnRebridge(col: DependentColumn, composite: boolean): { pre: { sql: string; inverse: string }[]; post: { sql: string; inverse: string }[] } { + const ident = (name: string): string => /^[a-z_][a-z0-9_]*$/.test(name) ? name : `"${name.replace(/"/g, '""')}"`; + const c = ident(col.name); + const typeName = col.def.typeName as Record; + const element: Record = { ...typeName, arrayBounds: undefined }; + const elementType = Deparser.deparse({ TypeName: element } as any, { pretty: false }); + const bridge = { names: [{ String: { sval: composite ? 'jsonb' : 'text' } }] }; + const out = composite ? `to_jsonb(${c})` : `${c}::text`; + // A USING expression may not contain a subquery, so an array of records + // is rebuilt by a session-local helper that lives only for the statement. + const helper = composite && col.array ? 'pg_temp.pgpm_jsonb_array' : null; + const back = !composite + ? `${c}::${Deparser.deparse({ TypeName: typeName } as any, { pretty: false })}` + : helper + ? `${helper}(NULL::${elementType}, ${c})` + : `jsonb_populate_record(NULL::${elementType}, ${c})`; + const alter = (cmd: Record): string => + deparseStmt({ AlterTableStmt: { relation: col.relation, objtype: 'OBJECT_TABLE', cmds: [{ AlterTableCmd: { behavior: 'DROP_RESTRICT', ...cmd } }] } }); + const retype = (target: unknown, using: string, collClause?: unknown): string => { + const stmt = alter({ subtype: 'AT_AlterColumnType', name: col.name, def: { ColumnDef: { typeName: target, collClause, raw_default: parseExpression(using) } } }); + if (!helper || using !== back) return stmt; + return [ + `CREATE FUNCTION ${helper}(base anyelement, j jsonb) RETURNS anyarray LANGUAGE sql IMMUTABLE AS $$ SELECT CASE WHEN j IS NULL THEN NULL ELSE ARRAY(SELECT jsonb_populate_record(base, e) FROM jsonb_array_elements(j) AS e) END $$;`, + stmt, + `DROP FUNCTION ${helper}(anyelement, jsonb);` + ].join('\n'); + }; + const setDefault = (expr: unknown): string => alter({ subtype: 'AT_ColumnDefault', name: col.name, def: expr }); + const dropDefault = alter({ subtype: 'AT_ColumnDefault', name: col.name }); + const fromDefault = columnAttributes(col.def).def; + const toDefault = columnAttributes(col.toDef ?? col.def).def; + const pre: { sql: string; inverse: string }[] = []; + const post: { sql: string; inverse: string }[] = []; + if (fromDefault) pre.push({ sql: dropDefault, inverse: setDefault(fromDefault) }); + pre.push({ sql: retype(bridge, out), inverse: retype(typeName, back, col.def.collClause) }); + post.push({ sql: retype(typeName, back, col.def.collClause), inverse: retype(bridge, out) }); + if (toDefault) post.push({ sql: setDefault(toDefault), inverse: dropDefault }); + return { pre, post }; +} + /** Normalized-AST fingerprint of one statement (formatting/position-proof). */ function fingerprint(text: string): string { const parsed = parseSync(text); @@ -318,12 +490,29 @@ interface TableShape { relation: unknown; /** column name → ColumnDef as authored (emitted), its canonical form (compared) and print key. */ columns: Map; + /** serial columns → the implied owned sequence (pg_dump's int + nextval + sequence). */ + serials: Map; /** normalized table-level constraints/extras, keyed by their print. */ extras: Map; /** COMMENT / GRANT / OWNER state attached to the table. */ annotations: AnnotationState; /** Constraints authored without a name; Postgres derives theirs from the table name. */ implicitConstraints: { node: ConstraintNode; column?: string }[]; + /** Table-level settings folded from CREATE TABLE and the ALTER TABLE commands that change them. */ + props: TableProps; +} + +interface TableProps { + /** relpersistence: 'p' (logged) or 'u' (unlogged). */ + persistence: string; + /** reloption name → DefElem. */ + options: Map; + /** INHERITS parents, keyed by print. */ + inherits: Map; + /** PARTITION OF parent and bound, when the table is a partition. */ + partition: { parent: unknown; bound: unknown } | null; + /** PARTITION BY clause of a partitioned table. */ + partspec: unknown; } /** Constraint kinds that can move between column, table-elt, and ALTER form. */ @@ -382,8 +571,28 @@ function normalizeColumnConstraint(c: ConstraintNode & Record, * default-expression casts are canonicalized so a module compares equal to * the catalog rendering of the same table. */ +const SERIAL_TYPES: Record = { + smallserial: 'int2', serial2: 'int2', serial: 'int4', serial4: 'int4', bigserial: 'int8', serial8: 'int8' +}; + +/** The serial pseudo-type a column is declared with, if any (`serial`, `bigserial`, ...). */ +function serialTypeName(typeName: unknown): string | null { + const names = ((typeName as { names?: { String?: { sval?: string } }[] } | undefined)?.names ?? []).map(n => n.String?.sval ?? ''); + const bare = names.length === 1 ? names[0] : names.length === 2 && names[0] === 'pg_catalog' ? names[1] : null; + return bare !== null && bare in SERIAL_TYPES ? bare : null; +} + function tableShape(unit: ObjectUnit): TableShape { - const shape: TableShape = { relation: null, columns: new Map(), extras: new Map(), annotations: emptyAnnotations(), implicitConstraints: [] }; + const props: TableProps = { persistence: 'p', options: new Map(), inherits: new Map(), partition: null, partspec: null }; + const shape: TableShape = { relation: null, columns: new Map(), serials: new Map(), extras: new Map(), annotations: emptyAnnotations(), implicitConstraints: [], props }; + const setOptions = (items: unknown[] | undefined, keep: boolean): void => { + for (const o of items ?? []) { + const name = (o as { DefElem?: { defname?: string } }).DefElem?.defname; + if (!name) continue; + if (keep) props.options.set(name, cleanTree(o)); + else props.options.delete(name); + } + }; const table = unit.identity.name; const rawColumns: { colname: string; def: Record }[] = []; const constraints: { node: ConstraintNode & Record; column?: string }[] = []; @@ -457,6 +666,18 @@ function tableShape(unit: ObjectUnit): TableShape { const stmt = raw.stmt as unknown as Record>; if (stmt.CreateStmt) { shape.relation = stmt.CreateStmt.relation; + props.persistence = (stmt.CreateStmt.relation as { relpersistence?: string }).relpersistence ?? 'p'; + setOptions(stmt.CreateStmt.options as unknown[] | undefined, true); + const parents = (stmt.CreateStmt.inhRelations as unknown[] | undefined) ?? []; + if (stmt.CreateStmt.partbound) { + props.partition = { parent: cleanTree(parents[0]), bound: cleanTree(stmt.CreateStmt.partbound) }; + } else { + for (const parent of parents) { + const clean = cleanTree(parent); + props.inherits.set(stablePrint(clean), clean); + } + } + if (stmt.CreateStmt.partspec) props.partspec = cleanTree(stmt.CreateStmt.partspec); const elts = (stmt.CreateStmt.tableElts as Record[] | undefined) ?? []; for (const elt of elts) { if (elt.ColumnDef) addColumn(elt.ColumnDef as Record); @@ -483,6 +704,14 @@ function tableShape(unit: ObjectUnit): TableShape { absorbAnnotation(shape.annotations, { AlterTableStmt: { relation: stmt.AlterTableStmt.relation, objtype: stmt.AlterTableStmt.objtype, cmds: [cmd] } }); + } else if (at.subtype === 'AT_SetRelOptions' || at.subtype === 'AT_ResetRelOptions') { + setOptions((def as { List?: { items?: unknown[] } } | undefined)?.List?.items, at.subtype === 'AT_SetRelOptions'); + } else if (at.subtype === 'AT_SetUnLogged' || at.subtype === 'AT_SetLogged') { + props.persistence = at.subtype === 'AT_SetUnLogged' ? 'u' : 'p'; + } else if (at.subtype === 'AT_AddInherit' || at.subtype === 'AT_DropInherit') { + const clean = cleanTree(def); + if (at.subtype === 'AT_AddInherit') props.inherits.set(stablePrint(clean), clean); + else props.inherits.delete(stablePrint(clean)); } else if (!foldColumnCommand(at)) { addExtra({ AlterTableCmd: at }); } @@ -510,7 +739,24 @@ function tableShape(unit: ObjectUnit): TableShape { addExtra({ Constraint: canonical }); } - for (const { colname, def } of rawColumns) { + for (const { colname, def: declared } of rawColumns) { + // `serial` is shorthand for an int column with an owned sequence and a + // nextval default; it is unfolded so it compares with that spelling. + const serialAs = SERIAL_TYPES[serialTypeName(declared.typeName) ?? '']; + const sequence = serialAs + ? { schema: unit.identity.schema, name: `${unit.identity.name}_${colname}_seq`, as: serialAs } + : null; + const def = sequence + ? { + ...declared, + typeName: { names: [{ String: { sval: 'pg_catalog' } }, { String: { sval: sequence.as } }], typemod: -1 }, + constraints: [ + ...((declared.constraints as unknown[] | undefined) ?? []), + { Constraint: { contype: 'CONSTR_DEFAULT', raw_expr: parseExpression(`nextval('${[sequence.schema, sequence.name].filter(Boolean).join('.')}'::regclass)`) } } + ] + } + : declared; + if (sequence) shape.serials.set(colname, sequence); const typeName = canonicalizeTypes(cleanTree(def.typeName)); const residual = ((def.constraints as unknown[] | undefined) ?? []).map(c => normalizeColumnConstraint((c as { Constraint: ConstraintNode & Record }).Constraint, typeName) @@ -518,7 +764,7 @@ function tableShape(unit: ObjectUnit): TableShape { const hasNotNull = residual.some( c => (c as { Constraint?: ConstraintNode }).Constraint?.contype === 'CONSTR_NOTNULL' ); - if (notNullImplied.has(colname) && !hasNotNull) { + if ((notNullImplied.has(colname) || sequence) && !hasNotNull) { residual.push({ Constraint: { contype: 'CONSTR_NOTNULL' } }); } const sorted = residual @@ -804,38 +1050,149 @@ function diffTable( typeChanged)); } + const defaultSteps: TableAlteration[] = []; if (stablePrint(a.def) !== stablePrint(b.def)) { const setDefault = (expr: unknown): string => alter(expr === undefined ? { subtype: 'AT_ColumnDefault', name } : { subtype: 'AT_ColumnDefault', name, def: expr }); - steps.push(attr(name, 'alter', setDefault(bRaw.def), setDefault(aRaw.def))); - } - if (a.notNull !== b.notNull) { - steps.push(attr(name, 'alter', - alter({ subtype: b.notNull ? 'AT_SetNotNull' : 'AT_DropNotNull', name }), - alter({ subtype: a.notNull ? 'AT_SetNotNull' : 'AT_DropNotNull', name }))); + defaultSteps.push(attr(name, 'alter', setDefault(bRaw.def), setDefault(aRaw.def))); } + const identitySteps: TableAlteration[] = []; if (stablePrint(a.identity) !== stablePrint(b.identity)) { const add = (c: Record): string => alter({ subtype: 'AT_AddIdentity', name, def: { Constraint: c } }); const drop = alter({ subtype: 'AT_DropIdentity', name }); - if (a.identity) steps.push(attr(name, 'alter', drop, add(aRaw.identity!))); - if (b.identity) steps.push(attr(name, 'alter', add(bRaw.identity!), drop)); + if (a.identity) identitySteps.push(attr(name, 'alter', drop, add(aRaw.identity!))); + if (b.identity) identitySteps.push(attr(name, 'alter', add(bRaw.identity!), drop)); } + // A serial's owned sequence comes and goes with the spelling: dropped + // after its default is gone (and before an identity claims its name), or + // created after an identity is dropped and before the default needs it. + const serialFrom = from.serials.get(renamedFrom.get(name) ?? name) ?? null; + const serialTo = to.serials.get(name) ?? null; + const sequenceSql = (seq: { schema: string | null; name: string; as: string }): { create: string; drop: string } => { + const items = (parts: (string | null)[]): unknown[] => parts.filter((x): x is string => x !== null).map(sval => ({ String: { sval } })); + return { + create: deparseStmt({ CreateSeqStmt: { sequence: { schemaname: seq.schema ?? undefined, relname: seq.name }, options: [ + { DefElem: { defname: 'as', arg: { TypeName: { names: items(['pg_catalog', seq.as]) } }, defaction: 'DEFELEM_UNSPEC' } }, + { DefElem: { defname: 'owned_by', arg: { List: { items: items([schema, table, name]) } }, defaction: 'DEFELEM_UNSPEC' } } + ] } }), + drop: deparseStmt({ DropStmt: { objects: [{ List: { items: items([seq.schema, seq.name]) } }], removeType: 'OBJECT_SEQUENCE', behavior: 'DROP_RESTRICT' } }) + }; + }; + // NOT NULL is required before an identity is added and may only be + // dropped after one is gone. + const notNullStep = a.notNull !== b.notNull + ? attr(name, 'alter', + alter({ subtype: b.notNull ? 'AT_SetNotNull' : 'AT_DropNotNull', name }), + alter({ subtype: a.notNull ? 'AT_SetNotNull' : 'AT_DropNotNull', name })) + : null; + if (notNullStep && b.notNull) steps.push(notNullStep); + if (serialFrom && !serialTo) { + const seq = sequenceSql(serialFrom); + ctx.destructive.push(`${qualified}: DROP SEQUENCE ${[serialFrom.schema, serialFrom.name].filter(Boolean).join('.')} (the serial's sequence and its current value are lost)`); + steps.push(...defaultSteps, attr(name, 'alter', seq.drop, seq.create), ...identitySteps); + } else if (serialTo && !serialFrom) { + const seq = sequenceSql(serialTo); + steps.push(...identitySteps, attr(name, 'alter', seq.create, seq.drop), ...defaultSteps); + } else { + steps.push(...defaultSteps, ...identitySteps); + } + if (notNullStep && !b.notNull) steps.push(notNullStep); if (stablePrint(a.other) !== stablePrint(b.other)) { ctx.incomplete.push(`${diff.path}: column "${name}" changed in a way pgpm cannot express as ALTER COLUMN (${a.other.length + b.other.length} unsupported constraint(s)); alter it manually`); } + // Storage and compression are plain per-column settings; "default" puts + // either back to what the column type implies. + const setting = (def: unknown, field: 'storage_name' | 'compression'): string => + ((def as Record)[field] ?? 'default').toLowerCase(); + for (const [field, subtype] of [['storage_name', 'AT_SetStorage'], ['compression', 'AT_SetCompression']] as const) { + const was = setting(prior.canon, field); + const now = setting(col.canon, field); + if (was === now) continue; + const set = (value: string): string => alter({ subtype, name, def: { String: { sval: value } } }); + steps.push(attr(name, 'alter', set(now), set(was))); + } const rest = (def: unknown): string => - stablePrint({ ...(def as Record), colname: null, typeName: null, collClause: null, constraints: null, is_local: null, location: null }); + stablePrint({ + ...(def as Record), colname: null, typeName: null, collClause: null, constraints: null, + is_local: null, location: null, storage_name: null, compression: null + }); if (rest(prior.canon) !== rest(col.canon)) { - ctx.incomplete.push(`${diff.path}: column "${name}" changed storage/compression/options pgpm cannot express as ALTER COLUMN; alter it manually`); + ctx.incomplete.push(`${diff.path}: column "${name}" changed options pgpm cannot express as ALTER COLUMN; alter it manually`); + } + } + // ---- table-level properties: persistence, reloptions, inheritance and + // partition bound all change in place with exact inverses. + const alterCmds = (rel: unknown, cmds: Record[]): string => + deparseStmt({ + AlterTableStmt: { relation: rel, cmds: cmds.map(cmd => ({ AlterTableCmd: { behavior: 'DROP_RESTRICT', ...cmd } })), objtype: 'OBJECT_TABLE' } + }); + const prop = (sql: string, inverse: string): TableAlteration => ({ sql, inverse, phase: 'alter', sub: null, keyColumns: [], breaking: false }); + if (from.props.persistence !== to.props.persistence) { + const persistence = (p: string): string => alter({ subtype: p === 'u' ? 'AT_SetUnLogged' : 'AT_SetLogged' }); + steps.push(prop(persistence(to.props.persistence), persistence(from.props.persistence))); + } + { + const set = (items: unknown[]): Record => ({ subtype: 'AT_SetRelOptions', def: { List: { items } } }); + const reset = (names: string[]): Record => + ({ subtype: 'AT_ResetRelOptions', def: { List: { items: names.map(defname => ({ DefElem: { defname, defaction: 'DEFELEM_UNSPEC' } })) } } }); + const changed = [...to.props.options].filter(([k, v]) => stablePrint(from.props.options.get(k)) !== stablePrint(v)); + const removed = [...from.props.options].filter(([k]) => !to.props.options.has(k)); + const forward: Record[] = []; + const backward: Record[] = []; + if (changed.length > 0) { + forward.push(set(changed.map(([, v]) => v))); + const restore = changed.filter(([k]) => from.props.options.has(k)).map(([k]) => from.props.options.get(k)); + const undo = changed.filter(([k]) => !from.props.options.has(k)).map(([k]) => k); + if (restore.length > 0) backward.push(set(restore)); + if (undo.length > 0) backward.push(reset(undo)); + } + if (removed.length > 0) { + forward.push(reset(removed.map(([k]) => k))); + backward.push(set(removed.map(([, v]) => v))); } + if (forward.length > 0) steps.push(prop(alterCmds(relation, forward), alterCmds(relation, backward))); } + for (const [print, parent] of from.props.inherits) { + if (to.props.inherits.has(print)) continue; + steps.push(prop(alter({ subtype: 'AT_DropInherit', def: parent }), alter({ subtype: 'AT_AddInherit', def: parent }))); + } + for (const [print, parent] of to.props.inherits) { + if (from.props.inherits.has(print)) continue; + steps.push(prop(alter({ subtype: 'AT_AddInherit', def: parent }), alter({ subtype: 'AT_DropInherit', def: parent }))); + } + if (stablePrint(from.props.partition) !== stablePrint(to.props.partition)) { + const parentOf = (p: { parent: unknown }): unknown => (p.parent as { RangeVar?: unknown }).RangeVar ?? p.parent; + const detach = (p: { parent: unknown }): string => + alterCmds(parentOf(p), [{ subtype: 'AT_DetachPartition', def: { PartitionCmd: { name: relation } } }]); + const attach = (p: { parent: unknown; bound: unknown }): string => + alterCmds(parentOf(p), [{ subtype: 'AT_AttachPartition', def: { PartitionCmd: { name: relation, bound: p.bound } } }]); + if (from.props.partition) steps.push(prop(detach(from.props.partition), attach(from.props.partition))); + if (to.props.partition) steps.push(prop(attach(to.props.partition), detach(to.props.partition))); + } + if (stablePrint(from.props.partspec) !== stablePrint(to.props.partspec)) { + ctx.incomplete.push(`${diff.path}: PARTITION BY changed; Postgres cannot re-partition a table in place, rebuild it manually`); + } + // Self-inverse ALTER TABLE subtypes: a removed extra with one of these // undoes with the paired subtype rather than a DROP CONSTRAINT. const INVERSE_SUBTYPE: Record = { AT_EnableRowSecurity: 'AT_DisableRowSecurity', AT_DisableRowSecurity: 'AT_EnableRowSecurity', AT_ForceRowSecurity: 'AT_NoForceRowSecurity', - AT_NoForceRowSecurity: 'AT_ForceRowSecurity' + AT_NoForceRowSecurity: 'AT_ForceRowSecurity', + AT_EnableTrig: 'AT_DisableTrig', + AT_DisableTrig: 'AT_EnableTrig', + AT_EnableAlwaysTrig: 'AT_EnableTrig', + AT_EnableReplicaTrig: 'AT_EnableTrig', + AT_EnableTrigAll: 'AT_DisableTrigAll', + AT_DisableTrigAll: 'AT_EnableTrigAll', + AT_EnableTrigUser: 'AT_DisableTrigUser', + AT_DisableTrigUser: 'AT_EnableTrigUser' }; + /** The trigger an ENABLE/DISABLE TRIGGER command addresses (it must exist first). */ + const triggerOf = (cmd: Record): NamingIdentity | null => + /Trig$/.test(cmd.subtype as string) && typeof cmd.name === 'string' ? { kind: 'trigger', schema, name: cmd.name, table } : null; + const invert = (cmd: Record): Record => + cmd.name === undefined ? { subtype: INVERSE_SUBTYPE[cmd.subtype as string] } : { subtype: INVERSE_SUBTYPE[cmd.subtype as string], name: cmd.name }; // Removed extras drop before added ones: a changed constraint that keeps // its name must DROP CONSTRAINT before the replacement ADD CONSTRAINT. @@ -856,11 +1213,12 @@ function diffTable( breaking: false }); } else if (subtype && INVERSE_SUBTYPE[subtype]) { + const cmd = (extra as { AlterTableCmd: Record }).AlterTableCmd; steps.push({ - sql: alter({ subtype: INVERSE_SUBTYPE[subtype] }), - inverse: alter({ subtype }), + sql: alter(invert(cmd)), + inverse: alter(cmd), phase: 'remove', - sub: null, + sub: triggerOf(cmd), keyColumns: [], breaking: false }); @@ -895,14 +1253,14 @@ function diffTable( if (at) { const subtype = at.subtype as string; const inverse = INVERSE_SUBTYPE[subtype] - ? alter({ subtype: INVERSE_SUBTYPE[subtype] }) + ? alter(invert(at)) : conname ? alter({ subtype: 'AT_DropConstraint', name: conname }) : ''; if (!inverse) ctx.incomplete.push(`${diff.path}: added table element has no derivable inverse (${describeExtra(extra)})`); steps.push({ sql: alter(at), inverse: inverse || `-- pgpm: no inverse for ${describeExtra(extra)}`, phase: 'add', - sub: conname ? { kind: 'constraint', schema, name: conname, table } : null, + sub: conname ? { kind: 'constraint', schema, name: conname, table } : triggerOf(at), keyColumns: constraintKeys(extra), breaking: false }); @@ -1017,9 +1375,14 @@ function functionSignature(texts: string[]): string | null { const nodes = parseSync(texts[0]).sql.stmts.map(s => s.stmt as unknown as Record>); const fn = nodes.length === 1 ? nodes[0].CreateFunctionStmt : undefined; if (!fn) return null; + // Postgres discards typmods from parameter types (f(varchar(10)) is f(varchar)). + const parameters = ((fn.parameters ?? []) as { FunctionParameter?: { argType?: Record } }[]) + .map(p => p.FunctionParameter + ? { FunctionParameter: { ...p.FunctionParameter, argType: p.FunctionParameter.argType ? { ...p.FunctionParameter.argType, typmods: undefined } : undefined } } + : p); return stablePrint(cleanTree({ funcname: fn.funcname, - parameters: fn.parameters, + parameters: canonicalizeTypes(parameters), returnType: fn.returnType, is_procedure: fn.is_procedure })); @@ -1193,8 +1556,9 @@ export function diffSchemas( const destructive: string[] = []; /** Steps with no derivable DROP/inverse; refused unless `allowIncomplete`. */ const incomplete: string[] = []; - const a = readSide(from, warnings, 'from'); - const b = readSide(to, warnings, 'to'); + const overloaded = overloadedFunctions([from, to]); + const a = readSide(from, warnings, 'from', overloaded); + const b = readSide(to, warnings, 'to', overloaded); const renames = options.renames ?? []; const casts = options.casts ?? {}; @@ -1375,6 +1739,7 @@ export function diffSchemas( for (const raw of parseSync(text).sql.stmts) { const stmt = raw.stmt as unknown as Record; if (absorbAnnotation(annotations, stmt)) continue; + if (foldSequenceAlter(nodes, definition, stmt)) continue; definition.push(deparseStmt(stmt)); nodes.push(stmt); } @@ -1480,8 +1845,8 @@ export function diffSchemas( const pa = splitUnit(prior); const pb = splitUnit(unit); const annotationSteps = diffAnnotations(pa.annotations, pb.annotations, path); - const emitSteps = (list: { sql: string; inverse: string; incomplete?: string; destructive?: string }[], after?: Atom, breaking = false): void => { - list.forEach((step, i) => { + const emitSteps = (list: { sql: string; inverse: string; incomplete?: string; destructive?: string }[], after?: Atom, breaking = false): Atom[] => { + return list.map((step, i) => { if (step.incomplete) incomplete.push(`${path}: ${step.incomplete}`); if (step.destructive) destructive.push(step.destructive); const atom = addAtom({ @@ -1490,8 +1855,36 @@ export function diffSchemas( }); if (after) atom.deps.add(after.id); track(key, atom); + return atom; }); }; + // Columns typed with this type: parked in a bridge type around steps + // Postgres refuses while the type is in use (see `columnRebridge`). + const dependentColumns = unit.identity.kind === 'type' || unit.identity.kind === 'other' + ? typeDependentColumns(a, b, unit.identity) + : []; + const bridgeColumns = (middle: Atom[], composite: boolean): void => { + for (const col of dependentColumns) { + const { pre, post } = columnRebridge(col, composite); + const sub: NamingIdentity = { kind: 'column', schema: col.schema, table: col.table, name: col.name }; + destructive.push(`${[col.schema, col.table, col.name].filter(Boolean).join('.')}: cast through ${composite ? 'jsonb' : 'text'} around ${objectLabel(unit.identity)} (values the new type rejects are lost)`); + let last: Atom | null = null; + pre.forEach((step, i) => { + const atom = addAtom({ unit: key, name: path, phase: 'alter', sql: step.sql, inverse: step.inverse, sub, keyColumns: [], breaking: true, rank: toRank * 1000 - 100 + i }); + if (last) atom.deps.add(last.id); + last = atom; + track(key, atom); + }); + for (const m of middle) if (last) m.deps.add(last.id); + post.forEach((step, i) => { + const atom = addAtom({ unit: key, name: path, phase: 'alter', sql: step.sql, inverse: step.inverse, sub, keyColumns: [], breaking: false, rank: toRank * 1000 + 900 + i }); + for (const m of middle) atom.deps.add(m.id); + if (last) atom.deps.add(last.id); + last = atom; + track(key, atom); + }); + } + }; const definitionChanged = definitionFingerprint(pa.definition) !== definitionFingerprint(pb.definition); if (!definitionChanged) { // Representation-only differences (e.g. pg_dump's FOR ROLE) yield no steps. @@ -1505,7 +1898,10 @@ export function diffSchemas( if (steps.length + annotationSteps.length > 0) modified(); // Shape changes (enum values, domain rules, attributes) invalidate // dependents the way a drop does; annotations do not. - emitSteps(steps, undefined, true); + const atoms = emitSteps(steps, undefined, true); + // ADD/DROP ATTRIBUTE work with dependent columns; ALTER ATTRIBUTE ... TYPE does not. + const retypes = atoms.filter(a => /\bALTER ATTRIBUTE\b/.test(a.sql) || /\bALTER ATTRIBUTE\b/.test(a.inverse)); + if (pa.nodes[0].CompositeTypeStmt && retypes.length > 0) bridgeColumns(retypes, true); emitSteps(annotationSteps); continue; } @@ -1540,6 +1936,7 @@ export function diffSchemas( track(key, drop); } track(key, create); + bridgeColumns(drop ? [drop, create] : [create], Boolean(pa.nodes[0]?.CompositeTypeStmt)); } // ---- from-side: unchanged dependents that a breaking step would invalidate @@ -1606,6 +2003,11 @@ export function diffSchemas( { granularity: options.granularity ?? 'object', changeGranularity: alteration ? 'alteration' : undefined, + changeName: (facts, text) => { + const identity = identityOf(facts); + if (!identity) return defaultChangeName(facts); + return pathFor(overloadIdentity(identity, text, overloaded), { style }); + }, subObjectName: identity => pathFor(identity, { style }) } ); @@ -1694,6 +2096,61 @@ export function diffSchemas( } } + // ENABLE/DISABLE TRIGGER addresses a trigger by name: it follows the + // trigger's creation and precedes its drop. + for (const atom of atoms) { + if (atom.sub?.kind !== 'trigger') continue; + const triggerKey = identityKey(atom.sub as ObjectIdentity); + const created = createAtomByName.get(pathFor(atom.sub, { style })); + if (created) atom.deps.add(created.id); + for (const create of unitPhase(triggerKey, 'create')) if (create.unit !== atom.unit) atom.deps.add(create.id); + for (const drop of unitPhase(triggerKey, 'drop')) drop.deps.add(atom.id); + } + + // Extensions are cluster-level providers of types, functions and operators + // that references do not resolve to (`uuid_generate_v4()` names no schema): + // creating one precedes every other step and dropping one follows them. + const isExtension = (atom: Atom): boolean => atom.name.startsWith('extensions/'); + for (const ext of atoms) { + if (!isExtension(ext)) continue; + for (const other of atoms) { + if (other === ext || isExtension(other)) continue; + if (ext.phase === 'drop') ext.deps.add(other.id); + else other.deps.add(ext.id); + } + } + + // Collations are referenced by `COLLATE` clauses that resolve to no tracked + // object: a column that starts using one follows its creation, and a + // collation is dropped only after every step whose inverse still uses it. + const collationRef = /^(CREATE|DROP) COLLATION (?:IF EXISTS )?(?:("?)([^".\s]+)\2\.)?("?)([^"\s(;]+)\4/; + for (const atom of atoms) { + const m = collationRef.exec(atom.sql); + if (!m) continue; + const bare = m[5].replace(/^"|"$/g, ''); + const uses = new RegExp(`\\bCOLLATE\\s+(?:"?${m[3] ?? '[^".\\s]+'}"?\\.)?"?${bare.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}"?(?![\\w"])`, 'i'); + for (const other of atoms) { + if (other === atom) continue; + if (m[1] === 'CREATE' && uses.test(other.sql)) other.deps.add(atom.id); + if (m[1] === 'DROP' && uses.test(other.inverse)) atom.deps.add(other.id); + } + } + + // `ALTER SEQUENCE .. OWNED BY table.column` names a column: it follows that + // column's addition and precedes its drop. + const ownedBy = /\bOWNED BY (?:"?([^".\s]+)"?\.)?"?([^".\s]+)"?\."?([^".\s;]+)"?/; + for (const atom of atoms) { + const m = /^ALTER SEQUENCE\b/.test(atom.sql) ? ownedBy.exec(atom.sql) : null; + if (!m) continue; + const owner = (x: Atom): boolean => + x.sub?.kind === 'column' && x.sub.table === m[2] && x.sub.name === m[3] && (m[1] === undefined || (x.sub.schema ?? 'public') === m[1]); + for (const other of atoms) { + if (other === atom || !owner(other)) continue; + if (other.phase === 'add') atom.deps.add(other.id); + if (other.phase === 'remove' && other.breaking) other.deps.add(atom.id); + } + } + // ---- enum values added and used in the same migration: Postgres refuses to // use a new value before the transaction that added it commits (55P04). for (const atom of atoms) { @@ -1940,9 +2397,14 @@ export function coverChanges( for (const obj of diff.objects) { deltaByKey.set(identityKey(obj.identity), obj.delta); } + const overloaded = new Set( + diff.objects + .filter(obj => obj.identity.kind === 'function' && /\)$/.test(obj.identity.name)) + .map(obj => `${obj.identity.schema ?? ''}.${obj.identity.name.replace(/\(.*\)$/, '')}`) + ); return changes.map(change => { const sideWarnings: string[] = []; - const side = readSide(change.deploy, sideWarnings, change.name); + const side = readSide(change.deploy, sideWarnings, change.name, overloaded); const objects: ChangeObjectCoverage[] = side.order.map(key => { const unit = side.units.get(key)!; return { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 01e8390376..4d3fa920cb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -734,8 +734,8 @@ importers: specifier: 16.13.0 version: 16.13.0 plpgsql-parser: - specifier: ^18.6.2 - version: 18.6.2 + specifier: ^18.6.3 + version: 18.6.3 postgraphile: specifier: 5.1.4 version: 5.1.4(abb1e7d37c806d345dec450464477780) @@ -1602,11 +1602,11 @@ importers: specifier: 16.13.0 version: 16.13.0 pgsql-deparser: - specifier: ^18.4.2 - version: 18.4.2 + specifier: ^18.4.3 + version: 18.4.3 pgsql-parser: - specifier: ^18.3.2 - version: 18.3.2 + specifier: ^18.3.3 + version: 18.3.3 devDependencies: '@types/node': specifier: ^22.19.11 @@ -2707,8 +2707,8 @@ importers: specifier: ^18.0.0 version: 18.0.0 '@pgsql/utils': - specifier: ^18.3.2 - version: 18.3.2 + specifier: ^18.3.3 + version: 18.3.3 csv-parser: specifier: ^3.2.1 version: 3.2.1 @@ -2719,8 +2719,8 @@ importers: specifier: ^4.1.0 version: 4.1.1 pgsql-deparser: - specifier: ^18.4.2 - version: 18.4.2 + specifier: ^18.4.3 + version: 18.4.3 devDependencies: '@types/js-yaml': specifier: ^4.0.9 @@ -2729,8 +2729,8 @@ importers: specifier: ^0.8.0 version: 0.8.0 pgsql-parser: - specifier: ^18.3.2 - version: 18.3.2 + specifier: ^18.3.3 + version: 18.3.3 publishDirectory: dist packages/errors: @@ -2809,11 +2809,11 @@ importers: specifier: ^18.0.0 version: 18.0.0 '@pgsql/utils': - specifier: ^18.3.2 - version: 18.3.2 + specifier: ^18.3.3 + version: 18.3.3 pgsql-deparser: - specifier: ^18.4.2 - version: 18.4.2 + specifier: ^18.4.3 + version: 18.4.3 schema-typescript: specifier: ^0.14.4 version: 0.14.4 @@ -2936,11 +2936,11 @@ importers: specifier: workspace:^ version: link:../../pgpm/logger/dist '@pgsql/lint': - specifier: ^18.3.3 - version: 18.3.3 + specifier: ^18.3.5 + version: 18.3.5 '@pgsql/traverse': - specifier: ^18.8.2 - version: 18.8.2 + specifier: ^18.8.3 + version: 18.8.3 '@pgsql/types': specifier: ^18.0.0 version: 18.0.0 @@ -2960,11 +2960,11 @@ importers: specifier: workspace:^ version: link:../../postgres/pg-env/dist pgsql-deparser: - specifier: ^18.4.2 - version: 18.4.2 + specifier: ^18.4.3 + version: 18.4.3 pgsql-parser: - specifier: ^18.3.2 - version: 18.3.2 + specifier: ^18.3.3 + version: 18.3.3 yanse: specifier: ^0.2.2 version: 0.2.2 @@ -3174,8 +3174,8 @@ importers: specifier: workspace:^ version: link:../../postgres/pg-env/dist pgsql-deparser: - specifier: ^18.4.2 - version: 18.4.2 + specifier: ^18.4.3 + version: 18.4.3 semver: specifier: ^7.8.5 version: 7.8.5 @@ -3278,11 +3278,11 @@ importers: specifier: workspace:^ version: link:../../postgres/pg-env/dist pgsql-deparser: - specifier: ^18.4.2 - version: 18.4.2 + specifier: ^18.4.3 + version: 18.4.3 pgsql-parser: - specifier: ^18.3.2 - version: 18.3.2 + specifier: ^18.3.3 + version: 18.3.3 semver: specifier: ^7.8.5 version: 7.8.5 @@ -3476,8 +3476,8 @@ importers: specifier: ^10.2.6 version: 10.2.6 plpgsql-parser: - specifier: ^18.6.2 - version: 18.6.2 + specifier: ^18.6.3 + version: 18.6.3 devDependencies: makage: specifier: ^0.8.0 @@ -3493,17 +3493,17 @@ importers: specifier: workspace:^ version: link:../naming-spec/dist '@pgsql/scripts': - specifier: ^18.5.2 - version: 18.5.2 + specifier: ^18.5.3 + version: 18.5.3 '@pgsql/semantics': - specifier: ^18.2.2 - version: 18.2.2 + specifier: ^18.2.3 + version: 18.2.3 '@pgsql/transform': - specifier: ^18.18.2 - version: 18.18.2 + specifier: ^18.18.3 + version: 18.18.3 plpgsql-parser: - specifier: ^18.6.2 - version: 18.6.2 + specifier: ^18.6.3 + version: 18.6.3 devDependencies: '@pgpmjs/bundle': specifier: workspace:^ @@ -3634,8 +3634,8 @@ importers: specifier: ^1.32.5 version: 1.32.5 pgsql-deparser: - specifier: ^18.4.2 - version: 18.4.2 + specifier: ^18.4.3 + version: 18.4.3 publishDirectory: dist postgres/pg-cache: @@ -3922,8 +3922,8 @@ importers: specifier: workspace:^ version: link:../pg-ast/dist pgsql-deparser: - specifier: ^18.4.2 - version: 18.4.2 + specifier: ^18.4.3 + version: 18.4.3 query-spec: specifier: workspace:^ version: link:../../packages/query-spec/dist @@ -6918,10 +6918,10 @@ packages: } engines: { node: '>=20.0.0' } - '@pgsql/lint@18.3.3': + '@pgsql/lint@18.3.5': resolution: { - integrity: sha512-k2tPbO2V1GH8yfcFf+U5zVvu+Ju126R5iKeBhpOcmmpPg+ID4eP+h0gtWhcz7UowFAulhw4qBi6khJX1eT3PBg==, + integrity: sha512-K1d60bQ8DCSu1KSPImL4Dfly+Ss83eTuI5GL2PED/dlP9BSjp9OZoD81qzmFmskCFgwyzdJBBLO+g53phM2bww==, } hasBin: true @@ -6931,28 +6931,28 @@ packages: integrity: sha512-muCDdPu/bM/f9d7IquBNfkMgiRPehv6vvjdhSMxXeVKwJKPX9xIscDu11ju15s7UeEXoWntE8zrSaR9wQ9wgCg==, } - '@pgsql/scripts@18.5.2': + '@pgsql/scripts@18.5.3': resolution: { - integrity: sha512-ZwjbUv9GnSQy3pwhMazRj+9F9FjrH1fHTO7RLu1Uu7GKxVEopTx1kJBFqP4ombLbbeJuSwBt1stAN5eRbNbjCA==, + integrity: sha512-w09pfpezYR8LH9GXZzX6KjDAKH1/CsnKRDcvHLG7OcU+p/dcH7pUoxLRRKAgSjMCc1DPbRq9D1IX5US2/Ol32Q==, } - '@pgsql/semantics@18.2.2': + '@pgsql/semantics@18.2.3': resolution: { - integrity: sha512-pOtQIhRbOraJSvfNQ6L4HRqZQtARbXA1jYN9g8Jpzo2RFvA/tQe2GD+WAwy3/0ou495rwgrY7a0F5KTo6rdm7Q==, + integrity: sha512-jXmyUbRDxpVFXV6WRdIsIwtjM1BwasKaupXkn73xYFTn71QzATkcL2X7SMfnyCykvZhWRdV/DFLCW9gc9sbF0g==, } - '@pgsql/transform@18.18.2': + '@pgsql/transform@18.18.3': resolution: { - integrity: sha512-sLMEEODaMcrxIYRGAZtAbOzMzHjfOUgJ65qq3FCaRz3MRQe8gkM7t6bvPG1MmmS8Ggc+jGz4Rj3/jPkXQjTn8A==, + integrity: sha512-Vk7DxfHN8OqGmD0098WkdI7zIna7GF+pANeqgd3i4cd7I4UvFVhOcsB1r11w6SVmlUlLMt+fMLILQds6BpihAA==, } - '@pgsql/traverse@18.8.2': + '@pgsql/traverse@18.8.3': resolution: { - integrity: sha512-gOf/vG/pLDSZQdC6JkKEVCO0/K5K/PqbPhyVV9v7EpZDo+gk2C2rREbN8s48tnd7eGNpDHbeHUPRPkESi03ksg==, + integrity: sha512-ai/plrUraOuy9aWaZy+xJ+hi70dPa6QDh73K+2u2EIcR+Dgx5MGvJtanpfH/b/hoj5XzsbMOWXJhXm/Q34OmHw==, } '@pgsql/types@18.0.0': @@ -6961,10 +6961,10 @@ packages: integrity: sha512-jtAY4JU7jdVYZ/Vv3zlZqVP+FRWZokyCLKPPCnAApiaJAElpyyiMC3HRiquBu29kSrVs+rkqZD50SUIb/zyEGw==, } - '@pgsql/utils@18.3.2': + '@pgsql/utils@18.3.3': resolution: { - integrity: sha512-dJp+D7vVEA83Ox+WwtQGZRUkpuATDUADxTxFxg+3fn5e22nsKldrSU80Cm2aJIPwSJTsfDLUpXrOxwVMTdrU9A==, + integrity: sha512-V5/Y6i9c5mPUglcJqOYixT8DvUO8maXsJNlF5o5CLSFZAv5iP3a0UR9JRPTv3vObBbhzldnX8pmOuVIGzUh3wg==, } '@pkgjs/parseargs@0.11.0': @@ -13627,16 +13627,16 @@ packages: integrity: sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==, } - pgsql-deparser@18.4.2: + pgsql-deparser@18.4.3: resolution: { - integrity: sha512-BDo6BpyJAYKvwoiGUQSDnJ2pZjUE9odiXTkpP0U6FqRHke+UjJU2iYJticcagGOp+Q9TOX6OO6pi795dNRRC9A==, + integrity: sha512-84ijKETOYAvUaefqwoNTy/RCPdW8Zu0fZ0WeH28+2JGP6D4ZFdg7KXMkFBGk15JiTViIj3Tn174jeYiUywFWcQ==, } - pgsql-parser@18.3.2: + pgsql-parser@18.3.3: resolution: { - integrity: sha512-3dXvFMHk9A3dZQIsHLhFvh7WSBbPbE4t0v4SmQ/CUfeEUlu7Wa9dc57uK2RTsbhLOWlNv0l7SmCRlh3xbXBiRw==, + integrity: sha512-H/pM6yTnErfhORJayJOP4Oe1FcjglVY2f13pSV/F4SLijfHNlyqJ6L505F+pWEidykS1oRWdqtFE8v9skdIz9w==, } picocolors@1.1.1: @@ -13731,16 +13731,16 @@ packages: engines: { node: '>=20' } hasBin: true - plpgsql-deparser@18.3.2: + plpgsql-deparser@18.3.3: resolution: { - integrity: sha512-rX2JV7miuq9nqwAjEiiNM5Me52G8YQ8I1kdI/78Sj8d9Z385LfjmVlZ2992OiBQn5KhQA7rLo2exfiALPZat5g==, + integrity: sha512-TgPX5pYEhe6fG3gNfvZdsIG13hglP85XPVgDxrvaLrCNXsBHHLP8iHvkPotAxcjQTNkzBs6Lzlq4yAkzhMsudw==, } - plpgsql-parser@18.6.2: + plpgsql-parser@18.6.3: resolution: { - integrity: sha512-+KmRFYrlSmzamI0+k9AYAT1j9BsWCDVTtvDRVwKbwGQWsTrcW8ko4TW3z+ErMUJ1qA1VSSJ6XWicfgLgzODQ7g==, + integrity: sha512-q4+Ct2ZwH9qbjslT8tbhm9QewT2gkGkeIwNB/wAGCuW/TKw0GPX1ADAueBaf0j7S0sDO/69q7fjbpmcGsNvcrg==, } pluralize@7.0.0: @@ -18372,54 +18372,54 @@ snapshots: tslib: 2.8.1 tsyringe: 4.10.0 - '@pgsql/lint@18.3.3': + '@pgsql/lint@18.3.5': dependencies: - '@pgsql/traverse': 18.8.2 + '@pgsql/traverse': 18.8.3 chalk: 4.1.2 git-changed: 0.3.0 libpg-query: 18.2.1 minimist: 1.2.8 - pgsql-parser: 18.3.2 + pgsql-parser: 18.3.3 transitivePeerDependencies: - supports-color '@pgsql/quotes@18.2.4': {} - '@pgsql/scripts@18.5.2': + '@pgsql/scripts@18.5.3': dependencies: '@pgsql/quotes': 18.2.4 - '@pgsql/transform': 18.18.2 - plpgsql-parser: 18.6.2 + '@pgsql/transform': 18.18.3 + plpgsql-parser: 18.6.3 transitivePeerDependencies: - supports-color - '@pgsql/semantics@18.2.2': + '@pgsql/semantics@18.2.3': dependencies: - '@pgsql/traverse': 18.8.2 - plpgsql-parser: 18.6.2 + '@pgsql/traverse': 18.8.3 + plpgsql-parser: 18.6.3 transitivePeerDependencies: - supports-color - '@pgsql/transform@18.18.2': + '@pgsql/transform@18.18.3': dependencies: '@pgsql/quotes': 18.2.4 - '@pgsql/semantics': 18.2.2 - '@pgsql/traverse': 18.8.2 - plpgsql-parser: 18.6.2 + '@pgsql/semantics': 18.2.3 + '@pgsql/traverse': 18.8.3 + plpgsql-parser: 18.6.3 transitivePeerDependencies: - supports-color - '@pgsql/traverse@18.8.2': + '@pgsql/traverse@18.8.3': dependencies: '@pgsql/types': 18.0.0 pg-proto-parser: 1.32.5 - plpgsql-deparser: 18.3.2 + plpgsql-deparser: 18.3.3 transitivePeerDependencies: - supports-color '@pgsql/types@18.0.0': {} - '@pgsql/utils@18.3.2': + '@pgsql/utils@18.3.3': dependencies: '@pgsql/types': 18.0.0 nested-obj: 0.2.3 @@ -23022,16 +23022,16 @@ snapshots: dependencies: split2: 4.2.0 - pgsql-deparser@18.4.2: + pgsql-deparser@18.4.3: dependencies: '@pgsql/quotes': 18.2.4 '@pgsql/types': 18.0.0 - pgsql-parser@18.3.2: + pgsql-parser@18.3.3: dependencies: '@pgsql/types': 18.0.0 libpg-query: 18.2.1 - pgsql-deparser: 18.4.2 + pgsql-deparser: 18.4.3 picocolors@1.1.1: {} @@ -23065,19 +23065,19 @@ snapshots: optionalDependencies: fsevents: 2.3.2 - plpgsql-deparser@18.3.2: + plpgsql-deparser@18.3.3: dependencies: '@pgsql/types': 18.0.0 libpg-query: 18.2.1 - pgsql-deparser: 18.4.2 + pgsql-deparser: 18.4.3 - plpgsql-parser@18.6.2: + plpgsql-parser@18.6.3: dependencies: - '@pgsql/traverse': 18.8.2 + '@pgsql/traverse': 18.8.3 '@pgsql/types': 18.0.0 libpg-query: 18.2.1 - pgsql-deparser: 18.4.2 - plpgsql-deparser: 18.3.2 + pgsql-deparser: 18.4.3 + plpgsql-deparser: 18.3.3 transitivePeerDependencies: - supports-color diff --git a/postgres/pg-ast/package.json b/postgres/pg-ast/package.json index 4d449b5586..6bb2307c91 100644 --- a/postgres/pg-ast/package.json +++ b/postgres/pg-ast/package.json @@ -39,7 +39,7 @@ "devDependencies": { "makage": "^0.8.0", "pg-proto-parser": "^1.32.5", - "pgsql-deparser": "^18.4.2" + "pgsql-deparser": "^18.4.3" }, "dependencies": { "@pgsql/types": "^18.0.0", diff --git a/postgres/query-builder/package.json b/postgres/query-builder/package.json index 4dd3dbb7a7..ae620b0044 100644 --- a/postgres/query-builder/package.json +++ b/postgres/query-builder/package.json @@ -42,7 +42,7 @@ "dependencies": { "@pgsql/quotes": "^18.2.4", "pg-ast": "workspace:^", - "pgsql-deparser": "^18.4.2", + "pgsql-deparser": "^18.4.3", "query-spec": "workspace:^" } }