diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 0683767..4277d3c 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -60,6 +60,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Requires `nowo-tech/transaction-doctrine` ^1.8.1 (fixes GHSA-9wh3-m3m5-jffx and GHSA-65w2-9r4h-848g). +### Fixed + +- **Foreign keys kept when collating the identifier columns (audit XI, latent).** `ExactColumnsCollationListener` + (1.20.1) replaced the entry table on `postGenerateSchemaTable` (`setClassTable()`, DBAL 4.5+); ORM 3.7–3.8 then added + the table's foreign keys to the object it had replaced, so any foreign key starting on the entry table, and its index, + would have been missing from the generated schema on MySQL / MariaDB. The entry table has none today, so no schema was + affected. The listener now runs on `postGenerateSchema`, once every foreign key exists: on DBAL 4.5+ the table is + rebuilt with its editor (foreign keys and indexes included) and swapped in with `GenerateSchemaEventArgs::setSchema()`; + on older DBAL the column options are set in place. The platform is the one of the schema tool's entity manager, and a + manager that does not map the entry is left alone. Same collations and fallback warning as before. + `postGenerateSchemaTable()` stays as a deprecated no-op, and the `ManagerRegistry` constructor argument is unused + (optional). Tests: a foreign key and an index on the entry table are kept, foreign keys and indexes with and without + the listener on MySQL 8.4, MySQL 8.0, MySQL 5.7, MariaDB and SQLite platforms, and the created tables on MySQL 8.4 / + MariaDB 11.4. +- The rollup tables (`nowo_http_log_daily`, `nowo_http_log_event_seen`) get their primary key through + `Table::addPrimaryKeyConstraint()` where doctrine/dbal has it (4.3+), with `setPrimaryKey()` kept for older DBAL, so + building the schema no longer raises the `Table::setPrimaryKey()` deprecation (doctrine/dbal#6867). Same key. Test: + no doctrine/dbal#6867 deprecation while the tables are added. + ## [1.23.0] - 2026-11-19 ### Changed diff --git a/src/Doctrine/ExactColumnsCollationListener.php b/src/Doctrine/ExactColumnsCollationListener.php index f1755ac..7908c4b 100644 --- a/src/Doctrine/ExactColumnsCollationListener.php +++ b/src/Doctrine/ExactColumnsCollationListener.php @@ -4,7 +4,9 @@ namespace Nowo\HttpLogBundle\Doctrine; -use Doctrine\ORM\EntityManagerInterface; +use Doctrine\DBAL\Schema\Schema; +use Doctrine\DBAL\Schema\Table; +use Doctrine\ORM\Tools\Event\GenerateSchemaEventArgs; use Doctrine\ORM\Tools\Event\GenerateSchemaTableEventArgs; use Doctrine\Persistence\ManagerRegistry; use Nowo\HttpLogBundle\Entity\HttpLogEntry; @@ -22,40 +24,75 @@ * filters and the indexes do not fold `Bob`, `bób` or `bob ` onto `bob` either. Applied to the generated schema (schema * tool, `doctrine:migrations:diff`); existing tables need the `ALTER TABLE` of UPGRADING. * - * Stateless (REQ-RUNTIME-002): the platform is read from the registry on each event. + * 1.23.1 (audit XI): applied on `postGenerateSchema`, once the schema tool has added every foreign key, by table name. + * 1.20.1–1.23.0 replaced the entry table on `postGenerateSchemaTable` (`setClassTable()`, DBAL 4.5+), and ORM 3.7–3.8 + * then added the table's foreign keys to the object it had replaced, so any foreign key starting on it (and its index) + * was lost on MySQL / MariaDB (none exist today: latent). Now: + * + * - DBAL 4.5+: the table is rebuilt with its table editor (foreign keys and indexes included) and swapped into a new + * schema given back with `GenerateSchemaEventArgs::setSchema()`; + * - older DBAL: the column's platform options are set in place (no editor exists there). + * + * Stateless (REQ-RUNTIME-002): the platform is the one of the schema tool's entity manager, read on each event; a + * manager that does not map the entry is left alone. */ final readonly class ExactColumnsCollationListener { /** Columns of the entry table that hold an identifier (those that are mapped get the collation). */ public const COLUMNS = ['user_identifier', 'client_ip', 'provider', 'credential_kind', 'credential_id', 'tenant_key', 'project_key']; + /** + * @param ManagerRegistry|null $doctrine unused since 1.23.1 (the platform comes from the schema tool's entity + * manager); kept for BC + */ public function __construct( - private ManagerRegistry $doctrine, + ?ManagerRegistry $doctrine = null, private ?LoggerInterface $logger = null, ) { + unset($doctrine); } - public function postGenerateSchemaTable(GenerateSchemaTableEventArgs $args): void + /** + * Gives the identifier columns of the entry table their exact-match collation in the generated schema, keeping every + * foreign key and index of the table. + */ + public function postGenerateSchema(GenerateSchemaEventArgs $args): void { - if ($args->getClassMetadata()->getName() !== HttpLogEntry::class) { + $manager = $args->getEntityManager(); + if ($manager->getMetadataFactory()->isTransient(HttpLogEntry::class)) { + return; + } + $schema = $args->getSchema(); + $name = $manager->getClassMetadata(HttpLogEntry::class)->getTableName(); + if (!$schema->hasTable($name)) { return; } - $manager = $this->doctrine->getManagerForClass(HttpLogEntry::class); - $collation = $manager instanceof EntityManagerInterface ? ExactCollation::forConnection($manager->getConnection(), $this->logger) : null; + $collation = ExactCollation::forConnection($manager->getConnection(), $this->logger); if ($collation === null) { return; } - $original = $table = $args->getClassTable(); - // @phpstan-ignore function.alreadyNarrowedType, function.alreadyNarrowedType (Table::edit() from doctrine/dbal 4.5, setClassTable() from doctrine/orm 3.6) - $editor = method_exists($table, 'edit') && method_exists($args, 'setClassTable'); + $original = $table = $schema->getTable($name); + // Table::edit() and Schema::edit() exist only from doctrine/dbal 4.5. + // @phpstan-ignore function.alreadyNarrowedType, function.alreadyNarrowedType + $editor = method_exists(Table::class, 'edit') && method_exists(Schema::class, 'edit'); foreach (self::COLUMNS as $column) { if ($table->hasColumn($column)) { $table = ExactCollation::collate($table, $column, $collation, $editor) ?? $table; } } if ($table !== $original) { - $args->setClassTable($table); + $args->setSchema($schema->edit()->dropTable($original->getObjectName())->addTable($table)->create()); } } + + /** + * No longer subscribed (1.23.1, audit XI): replacing the table there lost its foreign keys. Kept for BC; does nothing. + * + * @deprecated since 1.23.1, the collation is applied on postGenerateSchema ({@see self::postGenerateSchema()}) + */ + public function postGenerateSchemaTable(GenerateSchemaTableEventArgs $args): void + { + unset($args); + } } diff --git a/src/Resources/config/services.yaml b/src/Resources/config/services.yaml index 932c69b..32cf639 100644 --- a/src/Resources/config/services.yaml +++ b/src/Resources/config/services.yaml @@ -46,7 +46,7 @@ services: arguments: $logger: '@?logger' tags: - - { name: doctrine.event_listener, event: postGenerateSchemaTable } + - { name: doctrine.event_listener, event: postGenerateSchema } - { name: monolog.logger, channel: nowo_http_log } # REQ-ORM-002: removed by the extension when doctrine.table_prefix and doctrine.table_name_prefix keep their defaults. diff --git a/src/Rollup/RollupTables.php b/src/Rollup/RollupTables.php index dc01f68..ddf59eb 100644 --- a/src/Rollup/RollupTables.php +++ b/src/Rollup/RollupTables.php @@ -4,12 +4,15 @@ namespace Nowo\HttpLogBundle\Rollup; +use Doctrine\DBAL\Schema\PrimaryKeyConstraint; use Doctrine\DBAL\Schema\Schema; +use Doctrine\DBAL\Schema\Table; use Doctrine\DBAL\Types\Types; use Nowo\HttpLogBundle\Doctrine\ExactCollation; use Nowo\HttpLogBundle\Doctrine\TablePrefixListener; use function is_string; +use function method_exists; /** * Names and DBAL schema of the two rollup tables (HL-1 §8.3): `daily` (one bucket @@ -117,7 +120,7 @@ public function addTo(Schema $schema, bool|string $mysql = false): void $daily->addColumn('duration_ms_max', Types::INTEGER, ['default' => 0]); $daily->addColumn('first_seen_at', Types::DATETIME_MUTABLE); $daily->addColumn('last_seen_at', Types::DATETIME_MUTABLE); - $daily->setPrimaryKey(['id']); + $this->primaryKey($daily, 'id'); $daily->addUniqueIndex(self::KEY_COLUMNS, $this->index(self::UNIQUE_KEY)); foreach (self::DAILY_INDEXES as $name => $columns) { $daily->addIndex($columns, $this->index($name)); @@ -128,8 +131,30 @@ public function addTo(Schema $schema, bool|string $mysql = false): void $seen = $schema->createTable($this->eventSeen); $seen->addColumn('event_id', Types::STRING, ['length' => 32, 'fixed' => true]); $seen->addColumn('day', Types::DATE_MUTABLE); - $seen->setPrimaryKey(['event_id']); + $this->primaryKey($seen, 'event_id'); $seen->addIndex(['day'], $this->index(self::SEEN_INDEX)); } } + + /** + * Sets the primary key of `$table` on `$column`: through a primary key constraint where DBAL has one (4.3+; + * `Table::setPrimaryKey()` is deprecated there, doctrine/dbal#6867), else with `setPrimaryKey()` (older DBAL). Same + * key either way (`primary`, clustered). + * + * @param non-empty-string $column + */ + private function primaryKey(Table $table, string $column): void + { + // @phpstan-ignore function.alreadyNarrowedType (Table::addPrimaryKeyConstraint() exists only from doctrine/dbal 4.3) + if (method_exists($table, 'addPrimaryKeyConstraint')) { + $table->addPrimaryKeyConstraint(PrimaryKeyConstraint::editor()->setUnquotedColumnNames($column)->create()); + + return; + } + + // doctrine/dbal < 4.3 + // @codeCoverageIgnoreStart + $table->setPrimaryKey([$column]); + // @codeCoverageIgnoreEnd + } } diff --git a/tests/Integration/Doctrine/ExactCollationTest.php b/tests/Integration/Doctrine/ExactCollationTest.php index b59e59e..1675acd 100644 --- a/tests/Integration/Doctrine/ExactCollationTest.php +++ b/tests/Integration/Doctrine/ExactCollationTest.php @@ -16,10 +16,13 @@ use Doctrine\DBAL\Schema\Schema; use Doctrine\DBAL\Schema\Table; use Doctrine\DBAL\Types\Types; +use Doctrine\ORM\EntityManager; use Doctrine\ORM\Mapping\ClassMetadata; +use Doctrine\ORM\ORMSetup; +use Doctrine\ORM\Tools\Event\GenerateSchemaEventArgs; use Doctrine\ORM\Tools\Event\GenerateSchemaTableEventArgs; use Doctrine\ORM\Tools\SchemaTool; -use Doctrine\Persistence\ManagerRegistry; +use Doctrine\Persistence\Mapping\Driver\MappingDriverChain; use Nowo\HttpLogBundle\Doctrine\ExactCollation; use Nowo\HttpLogBundle\Doctrine\ExactColumnsCollationListener; use Nowo\HttpLogBundle\Entity\HttpLogEntry; @@ -40,19 +43,30 @@ use PHPUnit\Framework\TestCase; use Psr\Log\AbstractLogger; use ReflectionMethod; -use stdClass; use Stringable; use Symfony\Component\Clock\MockClock; +use function array_filter; use function array_keys; +use function array_map; +use function array_values; use function count; +use function getenv; use function implode; use function is_string; +use function ksort; +use function method_exists; use function preg_match; use function preg_match_all; +use function sort; use function sprintf; +use function str_replace; +use function str_starts_with; +use function strtolower; use function strtr; +use const PHP_VERSION_ID; + /** * REQ-SEC-006 (1.20.1): the identifier columns of the entry and rollup tables get a binary NO PAD collation on MySQL / * MariaDB (`utf8mb4_0900_bin` / `utf8mb4_nopad_bin`, `utf8mb4_bin` plus a warning on MySQL < 8.0.17), and look-alike @@ -201,19 +215,119 @@ public function otherTablesAndManagersAreLeftAlone(): void $sqlite = new UsageTestDatabase(createSchema: false, rollup: true); self::assertStringNotContainsString('COLLATE', implode("\n", (new SchemaTool($sqlite->entityManager))->getCreateSchemaSql([$sqlite->entityManager->getClassMetadata(HttpLogEntry::class)]))); + // A manager that does not map the entry (audit XI: the schema tool's own manager decides). + $configuration = ORMSetup::createAttributeMetadataConfiguration([], true); + $configuration->setMetadataDriverImpl(new MappingDriverChain()); + if (PHP_VERSION_ID >= 80400) { + $configuration->enableNativeLazyObjects(true); + } + $other = new EntityManager(DriverManager::getConnection(['driver' => 'pdo_mysql', 'serverVersion' => '8.4.0'], $configuration), $configuration); + $schema = $this->entrySchema(); + $args = new GenerateSchemaEventArgs($other, $schema); + (new ExactColumnsCollationListener())->postGenerateSchema($args); + self::assertSame($schema, $args->getSchema(), 'no mapping for the entry'); + self::assertFalse($schema->getTable('nowo_http_log_entry')->getColumn('user_identifier')->hasPlatformOption('collation')); + + // The entry is mapped, but the generated schema does not hold its table. $db = new UsageTestDatabase(connectionParams: ['driver' => 'pdo_mysql', 'serverVersion' => '8.4.0'], createSchema: false); + $empty = new Schema(); + $args = new GenerateSchemaEventArgs($db->entityManager, $empty); + (new ExactColumnsCollationListener($db))->postGenerateSchema($args); + self::assertSame($empty, $args->getSchema(), 'no entry table'); + self::assertFalse($db->connection()->isConnected()); + } + + /** + * Audit XI (1.23.1): a foreign key and an index starting on the entry table are kept (1.20.1–1.23.0 replaced the + * table on postGenerateSchemaTable, and ORM 3.7–3.8 then added its foreign keys to the replaced object). + */ + #[Test] + public function aForeignKeyAndAnIndexOfTheEntryTableAreKept(): void + { + $db = new UsageTestDatabase(connectionParams: ['driver' => 'pdo_mysql', 'serverVersion' => '8.4.0'], createSchema: false); + $schema = $this->entrySchema(); + $parent = $schema->createTable('host_account'); + $parent->addColumn('id', Types::INTEGER); + $parent->setPrimaryKey(['id']); + $entry = $schema->getTable('nowo_http_log_entry'); + $entry->addColumn('account_id', Types::INTEGER, ['notnull' => false]); + $entry->addIndex(['account_id'], 'idx_account'); + $entry->addForeignKeyConstraint('host_account', ['account_id'], ['id'], ['onDelete' => 'SET NULL'], 'fk_account'); + $before = $this->shape($schema); + + $args = new GenerateSchemaEventArgs($db->entityManager, $schema); + (new ExactColumnsCollationListener())->postGenerateSchema($args); + + self::assertSame($before, $this->shape($args->getSchema())); + self::assertContains('account_id->host_account(id) SET NULL', $this->shape($args->getSchema())['nowo_http_log_entry']['fks']); + self::assertSame('utf8mb4_0900_bin', $args->getSchema()->getTable('nowo_http_log_entry')->getColumn('user_identifier')->getPlatformOption('collation')); + self::assertFalse($args->getSchema()->getTable('nowo_http_log_entry')->getColumn('account_id')->hasPlatformOption('collation')); + } + + /** + * Audit XI (1.23.1): the schema generated with the listener has every foreign key and index of the one generated + * without it, on every platform. + * + * @param array $params + */ + #[Test] + #[DataProvider('connections')] + public function theGeneratedSchemaKeepsEveryForeignKeyAndIndex(array $params, ?string $expected, bool $warns): void + { + unset($expected, $warns); + if (($params['driver'] ?? null) === 'pdo_mysql' && !(new ReflectionMethod(Connection::class, 'getServerVersion'))->isPublic()) { + // doctrine/dbal 3 reads the default table options from the server while it generates the schema + $url = getenv('HTTPLOG_TEST_MYSQL_URL'); + if (!is_string($url) || $url === '') { + self::markTestSkipped('doctrine/dbal 3 needs HTTPLOG_TEST_MYSQL_URL to generate a MySQL schema.'); + } + $params += RollupTestPlatforms::params('HTTPLOG_TEST_MYSQL_URL'); + } + $generate = static function (bool $listener) use ($params): Schema { + $db = new UsageTestDatabase(true, connectionParams: $params, createSchema: false, rollup: true, exactCollation: $listener); + + return (new SchemaTool($db->entityManager))->getSchemaFromMetadata([$db->entityManager->getClassMetadata(HttpLogEntry::class)]); + }; + + self::assertSame($this->shape($generate(false)), $this->shape($generate(true))); + } + + /** + * Audit XI on MySQL 8.4 / MariaDB 11.4 (HTTPLOG_TEST_MYSQL_URL): the entry table created with the listener has every + * foreign key and secondary index of the schema generated without it, and the identifier columns are exact. + */ + #[Test] + public function theCreatedEntryTableKeepsEveryForeignKeyAndIndex(): void + { + $db = RollupTestPlatforms::open('HTTPLOG_TEST_MYSQL_URL'); + $connection = $db->connection(); + $expected = $this->shape((new SchemaTool( + (new UsageTestDatabase(true, connectionParams: RollupTestPlatforms::params('HTTPLOG_TEST_MYSQL_URL'), createSchema: false, rollup: true, exactCollation: false))->entityManager, + ))->getSchemaFromMetadata([$db->entityManager->getClassMetadata(HttpLogEntry::class)])); + $schemaManager = $connection->createSchemaManager(); + foreach (['nowo_http_log_entry', 'nowo_http_log_daily', 'nowo_http_log_event_seen'] as $name) { + // @phpstan-ignore function.alreadyNarrowedType (introspectTableByUnquotedName() exists from DBAL 4.3; DBAL 3 has introspectTable()) + $table = method_exists($schemaManager, 'introspectTableByUnquotedName') ? $schemaManager->introspectTableByUnquotedName($name) : $schemaManager->introspectTable($name); + $actual = $this->shape(new Schema([$table]))[$name]; + self::assertSame($expected[$name]['fks'], $actual['fks'], $name); + self::assertSame($this->secondary($expected[$name]['indexes']), $this->secondary($actual['indexes']), $name); + } + self::assertSame( + ExactCollation::forConnection($connection), + $connection->fetchOne("SELECT COLLATION_NAME FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'nowo_http_log_entry' AND COLUMN_NAME = 'user_identifier'"), + ); + } + + #[Test] + public function theFormerTableEventDoesNothing(): void + { $table = new Table('nowo_http_log_entry'); $table->addColumn('user_identifier', Types::STRING, ['length' => 255]); + $args = new GenerateSchemaTableEventArgs(new ClassMetadata(HttpLogEntry::class), new Schema(), $table); - $registry = $this->createMock(ManagerRegistry::class); - $registry->method('getManagerForClass')->willReturn(null); - $args = new GenerateSchemaTableEventArgs($db->entityManager->getClassMetadata(HttpLogEntry::class), new Schema(), $table); - (new ExactColumnsCollationListener($registry))->postGenerateSchemaTable($args); - self::assertSame($table, $args->getClassTable(), 'no manager for the entry'); + (new ExactColumnsCollationListener())->postGenerateSchemaTable($args); - $other = new GenerateSchemaTableEventArgs(new ClassMetadata(stdClass::class), new Schema(), $table); - (new ExactColumnsCollationListener($db))->postGenerateSchemaTable($other); - self::assertSame($table, $other->getClassTable(), 'another entity'); + self::assertSame($table, $args->getClassTable()); self::assertFalse($table->getColumn('user_identifier')->hasPlatformOption('collation')); } @@ -262,6 +376,53 @@ private function exporter(UsageTestDatabase $db): HttpLogPersonalDataExporter } /** @return AbstractLogger&object{records: list} */ + /** The generated schema of the entry table alone (SQLite: no server needed on doctrine/dbal 3). */ + private function entrySchema(): Schema + { + $db = new UsageTestDatabase(true, createSchema: false, exactCollation: false); + + return (new SchemaTool($db->entityManager))->getSchemaFromMetadata([$db->entityManager->getClassMetadata(HttpLogEntry::class)]); + } + + /** + * Foreign keys (local columns, foreign table and columns, ON DELETE) and indexes (columns, uniqueness) per table. + * + * @return array, indexes: list}> + */ + private function shape(Schema $schema): array + { + $shape = []; + foreach ($schema->getTables() as $table) { + $fks = []; + foreach ($table->getForeignKeys() as $fk) { + $fks[] = implode(',', $fk->getLocalColumns()) . '->' . $fk->getForeignTableName() . '(' . implode(',', $fk->getForeignColumns()) . ') ' . ($fk->getOption('onDelete') ?? '-'); + } + $indexes = []; + foreach ($table->getIndexes() as $index) { + $indexes[] = ($index->isPrimary() ? 'primary ' : ($index->isUnique() ? 'unique ' : '')) . implode(',', array_map(strtolower(...), $index->getColumns())); + } + // Introspected names come quoted ("user_id", `user_id`); generated ones do not. + $fks = str_replace(['"', '`'], '', $fks); + $indexes = str_replace(['"', '`'], '', $indexes); + sort($fks); + sort($indexes); + $shape[strtolower(str_replace(['"', '`'], '', $table->getName()))] = ['fks' => $fks, 'indexes' => $indexes]; + } + ksort($shape); + + return $shape; + } + + /** + * @param list $indexes + * + * @return list + */ + private function secondary(array $indexes): array + { + return array_values(array_filter($indexes, static fn (string $index): bool => !str_starts_with($index, 'primary '))); + } + private function logger(): AbstractLogger { return new class extends AbstractLogger { diff --git a/tests/Integration/Rollup/RollupTablesPrimaryKeyTest.php b/tests/Integration/Rollup/RollupTablesPrimaryKeyTest.php new file mode 100644 index 0000000..b5a5493 --- /dev/null +++ b/tests/Integration/Rollup/RollupTablesPrimaryKeyTest.php @@ -0,0 +1,57 @@ +addTo($schema, 'utf8mb4_0900_bin'); + + self::assertSame($before, Deprecation::getTriggeredDeprecations()[$link] ?? 0); + } finally { + Deprecation::disable(); + } + self::assertSame(['id'], $this->primaryKey($schema->getTable('nowo_http_log_daily'))); + self::assertSame(['event_id'], $this->primaryKey($schema->getTable('nowo_http_log_event_seen'))); + } + + /** + * @return list + */ + private function primaryKey(Table $table): array + { + // @phpstan-ignore function.alreadyNarrowedType (getPrimaryKeyConstraint() exists only from doctrine/dbal 4.3) + if (method_exists($table, 'getPrimaryKeyConstraint')) { + $constraint = $table->getPrimaryKeyConstraint(); + self::assertNotNull($constraint); + + return array_map(static fn (UnqualifiedName $name): string => $name->toString(), $constraint->getColumnNames()); + } + + return $table->getPrimaryKey()?->getColumns() ?? []; // @codeCoverageIgnore + } +} diff --git a/tests/Support/UsageTestDatabase.php b/tests/Support/UsageTestDatabase.php index 83c67e9..88455e9 100644 --- a/tests/Support/UsageTestDatabase.php +++ b/tests/Support/UsageTestDatabase.php @@ -65,7 +65,7 @@ public function __construct( $events->addEventListener(ToolEvents::postGenerateSchema, new RollupSchemaListener($this->rollupTables)); } if ($exactCollation) { - $events->addEventListener(ToolEvents::postGenerateSchemaTable, new ExactColumnsCollationListener($this, $logger)); + $events->addEventListener(ToolEvents::postGenerateSchema, new ExactColumnsCollationListener($this, $logger)); } if ($usageColumns) { $events->addEventListener(Events::loadClassMetadata, new UsageColumnsListener());