diff --git a/Dockerfile b/Dockerfile index 58a42e8d..c922149e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,6 +28,7 @@ RUN set -ex \ COPY ./src /app/src COPY ./tests /app/src/tests +COPY ./bin /app/bin COPY --from=composer /app/vendor /app/vendor diff --git a/README.md b/README.md index c021e60f..5dc2aefc 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,14 @@ $transfer->run( ); ``` +## Appwrite database recovery + +Appwrite database destinations use a `ProvisioningOwner` made from a stable logical migration identifier and a fresh attempt identifier for every execution. The required `getRecoverableOwner` callback is the recovery authority for an existing database whose status is `provisioning` or `failed`. + +A database status is local to that resource. It does not prove that the migration attempt which owns it has stopped, because an import can continue with other resources after recording a database failure. The callback must therefore consult the caller's authoritative operation lifecycle and return the exact stored owner only after that attempt is terminal. Return `null` while it is active or unknown; recovery then fails closed. This rule also applies when the retry uses the same logical migration identifier. + +The standalone CLI requires `--migration-id` and a fresh `--migration-attempt-id`. Recovering an incomplete database additionally requires both `--recover-migration-id` and `--recover-migration-attempt-id` for the exact terminal prior attempt. + ## Supported Resources Chart Sources: diff --git a/bin/MigrationCLI.php b/bin/MigrationCLI.php index 96cd99dd..77af9d25 100644 --- a/bin/MigrationCLI.php +++ b/bin/MigrationCLI.php @@ -9,9 +9,9 @@ use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Database; use Utopia\Database\Document; -use Utopia\Database\Validator\Authorization; use Utopia\Migration\Destination; use Utopia\Migration\Destinations\Appwrite as DestinationsAppwrite; +use Utopia\Migration\Destinations\Appwrite\ProvisioningOwner; use Utopia\Migration\Destinations\Local; use Utopia\Migration\Source; use Utopia\Migration\Sources\Appwrite; @@ -19,6 +19,9 @@ use Utopia\Migration\Sources\NHost; use Utopia\Migration\Sources\Supabase; use Utopia\Migration\Transfer; +use Utopia\Query\Schema\ColumnType; +use Utopia\Query\Schema\IndexType; +use Utopia\Query\Schema\Order; /** * Migrations CLI Tool @@ -31,6 +34,9 @@ class MigrationCLI protected mixed $destination; + /** @var list */ + private readonly array $arguments; + protected const STRUCTURE = [ '$collection' => 'databases', '$id' => 'collections', @@ -38,7 +44,7 @@ class MigrationCLI 'attributes' => [ [ '$id' => 'databaseInternalId', - 'type' => Database::VAR_STRING, + 'type' => ColumnType::String->value, 'format' => '', 'size' => Database::LENGTH_KEY, 'signed' => true, @@ -49,7 +55,7 @@ class MigrationCLI ], [ '$id' => 'databaseId', - 'type' => Database::VAR_STRING, + 'type' => ColumnType::String->value, 'signed' => true, 'size' => Database::LENGTH_KEY, 'format' => '', @@ -60,7 +66,7 @@ class MigrationCLI ], [ '$id' => 'name', - 'type' => Database::VAR_STRING, + 'type' => ColumnType::String->value, 'size' => Database::LENGTH_KEY, 'required' => true, 'signed' => true, @@ -69,7 +75,7 @@ class MigrationCLI ], [ '$id' => 'enabled', - 'type' => Database::VAR_BOOLEAN, + 'type' => ColumnType::Boolean->value, 'signed' => true, 'size' => 0, 'format' => '', @@ -80,7 +86,7 @@ class MigrationCLI ], [ '$id' => 'documentSecurity', - 'type' => Database::VAR_BOOLEAN, + 'type' => ColumnType::Boolean->value, 'signed' => true, 'size' => 0, 'format' => '', @@ -91,7 +97,7 @@ class MigrationCLI ], [ '$id' => 'attributes', - 'type' => Database::VAR_STRING, + 'type' => ColumnType::String->value, 'size' => 1000000, 'required' => false, 'signed' => true, @@ -100,7 +106,7 @@ class MigrationCLI ], [ '$id' => 'indexes', - 'type' => Database::VAR_STRING, + 'type' => ColumnType::String->value, 'size' => 1000000, 'required' => false, 'signed' => true, @@ -109,7 +115,7 @@ class MigrationCLI ], [ '$id' => 'search', - 'type' => Database::VAR_STRING, + 'type' => ColumnType::String->value, 'format' => '', 'size' => 16384, 'signed' => true, @@ -122,35 +128,63 @@ class MigrationCLI 'indexes' => [ [ '$id' => '_fulltext_search', - 'type' => Database::INDEX_FULLTEXT, + 'type' => IndexType::Fulltext->value, 'attributes' => ['search'], 'lengths' => [], 'orders' => [], ], [ '$id' => '_key_name', - 'type' => Database::INDEX_KEY, + 'type' => IndexType::Key->value, 'attributes' => ['name'], 'lengths' => [Database::LENGTH_KEY], - 'orders' => [Database::ORDER_ASC], + 'orders' => [Order::Asc->value], ], [ '$id' => '_key_enabled', - 'type' => Database::INDEX_KEY, + 'type' => IndexType::Key->value, 'attributes' => ['enabled'], 'lengths' => [], - 'orders' => [Database::ORDER_ASC], + 'orders' => [Order::Asc->value], ], [ '$id' => '_key_documentSecurity', - 'type' => Database::INDEX_KEY, + 'type' => IndexType::Key->value, 'attributes' => ['documentSecurity'], 'lengths' => [], - 'orders' => [Database::ORDER_ASC], + 'orders' => [Order::Asc->value], ], ], ]; + /** @param list $arguments */ + public function __construct(array $arguments = []) + { + $this->arguments = $arguments; + } + + public static function getHelp(): string + { + return <<<'HELP' +Usage: php bin/MigrationCLI.php [options] + +Options: + -h, --help Show this help. + --migration-id= Stable logical owner identifier for this migration. + Required for Appwrite; reuse it for retries. + --migration-attempt-id= + Required for Appwrite; use a fresh attempt for every + execution or retry. + --recover-migration-id= + --recover-migration-attempt-id= + Together attest that the exact prior migration attempt is terminal + and allow recovery of its provisioning or failed databases. + Recovery is refused by default. + Resource status alone never proves an attempt terminal. + +HELP; + } + /** * Prints the current status of migrations as a table after wiping the screen */ @@ -248,12 +282,22 @@ public function getDestination(): Destination { switch ($_ENV['DESTINATION_PROVIDER']) { case 'appwrite': + $database = $this->getDatabase('destination'); + $recoverableOwner = $this->getRecoverableOwner(); + return new DestinationsAppwrite( - $_ENV['DESTINATION_APPWRITE_TEST_PROJECT'], - $_ENV['DESTINATION_APPWRITE_TEST_ENDPOINT'], - $_ENV['DESTINATION_APPWRITE_TEST_KEY'], - $this->getDatabase('destination'), - self::STRUCTURE + project: $_ENV['DESTINATION_APPWRITE_TEST_PROJECT'], + endpoint: $_ENV['DESTINATION_APPWRITE_TEST_ENDPOINT'], + key: $_ENV['DESTINATION_APPWRITE_TEST_KEY'], + dbForProject: $database, + getDatabasesDB: static fn (Document $document): Database => $database, + collectionStructure: self::STRUCTURE, + dbForPlatform: $database, + projectInternalId: $_ENV['DESTINATION_APPWRITE_TEST_PROJECT_INTERNAL_ID'], + owner: new ProvisioningOwner($this->getMigrationId(), $this->getMigrationAttemptId()), + // The standalone operator supplies the fixed pair only after confirming the prior + // lifecycle owner is terminal. The destination independently compares it with the row. + getRecoverableOwner: static fn (Document $document): ?ProvisioningOwner => $recoverableOwner, ); case 'local': return new Local('./localBackup'); @@ -262,6 +306,62 @@ public function getDestination(): Destination } } + private function getMigrationId(): string + { + foreach ($this->arguments as $argument) { + if (! \str_starts_with($argument, '--migration-id=')) { + continue; + } + + $migrationId = \substr($argument, \strlen('--migration-id=')); + if ($migrationId !== '') { + return $migrationId; + } + } + + throw new \InvalidArgumentException('--migration-id is required for an Appwrite destination'); + } + + private function getMigrationAttemptId(): string + { + foreach ($this->arguments as $argument) { + if (! \str_starts_with($argument, '--migration-attempt-id=')) { + continue; + } + + $attemptId = \substr($argument, \strlen('--migration-attempt-id=')); + if ($attemptId !== '') { + return $attemptId; + } + } + + throw new \InvalidArgumentException('--migration-attempt-id is required for an Appwrite destination'); + } + + private function getRecoverableOwner(): ?ProvisioningOwner + { + $migrationId = null; + $attemptId = null; + + foreach ($this->arguments as $argument) { + if (\str_starts_with($argument, '--recover-migration-id=')) { + $value = \substr($argument, \strlen('--recover-migration-id=')); + $migrationId = $value !== '' ? $value : null; + } + + if (\str_starts_with($argument, '--recover-migration-attempt-id=')) { + $value = \substr($argument, \strlen('--recover-migration-attempt-id=')); + $attemptId = $value !== '' ? $value : null; + } + } + + if ($migrationId === null || $attemptId === null) { + return null; + } + + return new ProvisioningOwner($migrationId, $attemptId); + } + public function getDatabase(string $type): Database { Database::addFilter( @@ -280,7 +380,7 @@ function (mixed $value, Document $document, Database $database) { $attributeType = $attribute->getAttribute('type'); switch ($attributeType) { - case Database::VAR_RELATIONSHIP: + case ColumnType::Relationship->value: $options = $attribute->getAttribute('options'); foreach ($options as $key => $value) { $attribute->setAttribute($key, $value); @@ -288,7 +388,7 @@ function (mixed $value, Document $document, Database $database) { $attribute->removeAttribute('options'); break; - case Database::VAR_STRING: + case ColumnType::String->value: $filters = $attribute->getAttribute('filters', []); $attribute->setAttribute('encrypt', in_array('encrypt', $filters)); break; @@ -394,14 +494,14 @@ function (mixed $value, Document $attribute) { $database ->setDatabase('appwrite') ->setNamespace('_' . $_ENV[$prefix . 'NAMESPACE']); + $database->getAuthorization()->disable(); return $database; } public function start(): void { - $dotenv = Dotenv::createImmutable(__DIR__); - $dotenv->load(); + $this->loadEnvironment(); /** * Initialise All Source Adapters @@ -423,15 +523,30 @@ public function start(): void /** * Run Transfer */ - Authorization::skip(fn () => $this->transfer->run( + $this->transfer->run( $this->source->getSupportedResources(), function () { $this->drawFrame(); } - )); + ); + + $this->destination->success(); + } + + protected function loadEnvironment(): void + { + $dotenv = Dotenv::createImmutable(__DIR__); + $dotenv->load(); } } -$instance = new MigrationCLI(); -$instance->start(); -$instance->drawFrame(); +if (\realpath($_SERVER['SCRIPT_FILENAME'] ?? '') === __FILE__) { + $arguments = $_SERVER['argv'] ?? []; + if (\in_array('-h', $arguments, true) || \in_array('--help', $arguments, true)) { + echo MigrationCLI::getHelp(); + } else { + $instance = new MigrationCLI($arguments); + $instance->start(); + $instance->drawFrame(); + } +} diff --git a/composer.json b/composer.json index 1f8625c4..5a4857ca 100644 --- a/composer.json +++ b/composer.json @@ -10,7 +10,8 @@ "migration" ], "license": "MIT", - "minimum-stability": "stable", + "minimum-stability": "dev", + "prefer-stable": true, "autoload": { "psr-4": { "Utopia\\Migration\\": "src/Migration" @@ -25,16 +26,16 @@ "test": "./vendor/bin/phpunit", "lint": "./vendor/bin/pint --test", "format": "./vendor/bin/pint", - "check": "./vendor/bin/phpstan analyse --level 3 src tests --memory-limit 2G" + "check": "./vendor/bin/phpstan analyse --level 3 src tests bin --memory-limit 2G" }, "require": { "php": ">=8.5", "ext-curl": "*", "ext-openssl": "*", "appwrite/appwrite": "^27.0", - "utopia-php/database": "^7.0.0", - "utopia-php/storage": "4.*", - "utopia-php/dsn": "0.2.*", + "utopia-php/database": "dev-feat-query-lib as 7.0.0", + "utopia-php/storage": "^4.0", + "utopia-php/dsn": "^0.2", "halaxa/json-machine": "^1.2" }, "require-dev": { @@ -44,6 +45,20 @@ "laravel/pint": "1.*", "phpstan/phpstan": "1.*" }, + "repositories": [ + { + "type": "vcs", + "url": "https://github.com/utopia-php/database.git" + }, + { + "type": "vcs", + "url": "https://github.com/utopia-php/query.git" + }, + { + "type": "vcs", + "url": "https://github.com/utopia-php/async.git" + } + ], "config": { "allow-plugins": { "php-http/discovery": true, diff --git a/composer.lock b/composer.lock index ee1336d9..743d6eb1 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "fea66e8c192e5b3d1dc56c953be0c2eb", + "content-hash": "2004ea1027b967193c5a9247cff21c35", "packages": [ { "name": "adhocore/jwt", @@ -248,23 +248,23 @@ }, { "name": "google/protobuf", - "version": "v5.35.1", + "version": "v5.36.0", "source": { "type": "git", "url": "https://github.com/protocolbuffers/protobuf-php.git", - "reference": "55bb4a7d6739b5af0927b96213c1371a3afb7cfb" + "reference": "9c105104b54709ecd902494ab340ed2122789b2d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/55bb4a7d6739b5af0927b96213c1371a3afb7cfb", - "reference": "55bb4a7d6739b5af0927b96213c1371a3afb7cfb", + "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/9c105104b54709ecd902494ab340ed2122789b2d", + "reference": "9c105104b54709ecd902494ab340ed2122789b2d", "shasum": "" }, "require": { "php": ">=8.2.0" }, "require-dev": { - "phpunit/phpunit": ">=11.5.0 <12.0.0" + "phpunit/phpunit": ">=11.5.50 <12.0.0" }, "suggest": { "ext-bcmath": "Need to support JSON deserialization" @@ -286,9 +286,9 @@ "proto" ], "support": { - "source": "https://github.com/protocolbuffers/protobuf-php/tree/v5.35.1" + "source": "https://github.com/protocolbuffers/protobuf-php/tree/v5.36.0" }, - "time": "2026-06-11T21:19:23+00:00" + "time": "2026-08-20T13:06:50+00:00" }, { "name": "halaxa/json-machine", @@ -986,6 +986,71 @@ }, "time": "2026-01-21T04:14:03+00:00" }, + { + "name": "opis/closure", + "version": "4.5.0", + "source": { + "type": "git", + "url": "https://github.com/opis/closure.git", + "reference": "b97e42b95bb72d87507f5e2d137ceb239aea8d6b" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/opis/closure/zipball/b97e42b95bb72d87507f5e2d137ceb239aea8d6b", + "reference": "b97e42b95bb72d87507f5e2d137ceb239aea8d6b", + "shasum": "" + }, + "require": { + "php": "^8.0" + }, + "require-dev": { + "phpunit/phpunit": "^9.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "4.x-dev" + } + }, + "autoload": { + "files": [ + "src/functions.php" + ], + "psr-4": { + "Opis\\Closure\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Marius Sarca", + "email": "marius.sarca@gmail.com" + }, + { + "name": "Sorin Sarca", + "email": "sarca_sorin@hotmail.com" + } + ], + "description": "A library that can be used to serialize closures (anonymous functions) and arbitrary data.", + "homepage": "https://opis.io/closure", + "keywords": [ + "anonymous classes", + "anonymous functions", + "closure", + "function", + "serializable", + "serialization", + "serialize" + ], + "support": { + "issues": "https://github.com/opis/closure/issues", + "source": "https://github.com/opis/closure/tree/4.5.0" + }, + "time": "2026-03-05T13:32:42+00:00" + }, { "name": "php-http/discovery", "version": "1.20.0", @@ -1555,16 +1620,16 @@ }, { "name": "symfony/http-client", - "version": "v7.4.14", + "version": "v7.4.16", "source": { "type": "git", "url": "https://github.com/symfony/http-client.git", - "reference": "f6bc6b5a54ff5afac4725cacec9bf2f52eb15920" + "reference": "c513ed0ba5d1784a6b55fc84190dbe4451b12f41" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client/zipball/f6bc6b5a54ff5afac4725cacec9bf2f52eb15920", - "reference": "f6bc6b5a54ff5afac4725cacec9bf2f52eb15920", + "url": "https://api.github.com/repos/symfony/http-client/zipball/c513ed0ba5d1784a6b55fc84190dbe4451b12f41", + "reference": "c513ed0ba5d1784a6b55fc84190dbe4451b12f41", "shasum": "" }, "require": { @@ -1632,7 +1697,7 @@ "http" ], "support": { - "source": "https://github.com/symfony/http-client/tree/v7.4.14" + "source": "https://github.com/symfony/http-client/tree/v7.4.16" }, "funding": [ { @@ -1652,7 +1717,7 @@ "type": "tidelift" } ], - "time": "2026-06-16T11:50:14+00:00" + "time": "2026-07-29T16:20:51+00:00" }, { "name": "symfony/http-client-contracts", @@ -1903,16 +1968,16 @@ }, { "name": "symfony/polyfill-php83", - "version": "v1.38.2", + "version": "v1.41.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-php83.git", - "reference": "796a26abb75ce49f3a84433cd81bf1009d73d5f8" + "reference": "5ea99087fb99c273a9b9236ed4c31e78b16103c6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-php83/zipball/796a26abb75ce49f3a84433cd81bf1009d73d5f8", - "reference": "796a26abb75ce49f3a84433cd81bf1009d73d5f8", + "url": "https://api.github.com/repos/symfony/polyfill-php83/zipball/5ea99087fb99c273a9b9236ed4c31e78b16103c6", + "reference": "5ea99087fb99c273a9b9236ed4c31e78b16103c6", "shasum": "" }, "require": { @@ -1959,7 +2024,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-php83/tree/v1.38.2" + "source": "https://github.com/symfony/polyfill-php83/tree/v1.41.0" }, "funding": [ { @@ -1979,20 +2044,20 @@ "type": "tidelift" } ], - "time": "2026-05-27T06:51:48+00:00" + "time": "2026-07-01T12:47:55+00:00" }, { "name": "symfony/polyfill-php85", - "version": "v1.38.1", + "version": "v1.41.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-php85.git", - "reference": "ba2ba04f3352cfa2dcbbcb90aee13ed967f505b1" + "reference": "255fab485aaa1006ed411040c42aecd7b5302d7a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-php85/zipball/ba2ba04f3352cfa2dcbbcb90aee13ed967f505b1", - "reference": "ba2ba04f3352cfa2dcbbcb90aee13ed967f505b1", + "url": "https://api.github.com/repos/symfony/polyfill-php85/zipball/255fab485aaa1006ed411040c42aecd7b5302d7a", + "reference": "255fab485aaa1006ed411040c42aecd7b5302d7a", "shasum": "" }, "require": { @@ -2039,7 +2104,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-php85/tree/v1.38.1" + "source": "https://github.com/symfony/polyfill-php85/tree/v1.41.0" }, "funding": [ { @@ -2059,7 +2124,7 @@ "type": "tidelift" } ], - "time": "2026-05-26T02:25:22+00:00" + "time": "2026-07-01T12:47:55+00:00" }, { "name": "symfony/service-contracts", @@ -2200,35 +2265,152 @@ }, "time": "2025-06-29T15:42:06+00:00" }, + { + "name": "utopia-php/async", + "version": "0.1.1", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/async.git", + "reference": "3ee4fc3d505113d0d6050f35f5bf85e706866a22" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/async/zipball/3ee4fc3d505113d0d6050f35f5bf85e706866a22", + "reference": "3ee4fc3d505113d0d6050f35f5bf85e706866a22", + "shasum": "" + }, + "require": { + "opis/closure": "4.*", + "php": ">=8.1" + }, + "require-dev": { + "amphp/amp": "3.*", + "amphp/parallel": "2.*", + "amphp/process": "^2.0", + "laravel/pint": "1.*", + "phpstan/phpstan": "2.*", + "phpunit/phpunit": "11.5.45", + "react/child-process": "0.*", + "react/event-loop": "1.*", + "swoole/ide-helper": "*" + }, + "suggest": { + "amphp/amp": "Required for Amp promise adapter", + "amphp/parallel": "Required for Amp parallel adapter", + "ext-ev": "Required for ReactPHP event loop (recommended for best performance)", + "ext-parallel": "Required for parallel adapter (requires PHP ZTS build)", + "ext-sockets": "Required for Swoole Process adapter", + "ext-swoole": "Required for Swoole Thread and Process adapters (recommended for best performance)", + "react/child-process": "Required for ReactPHP parallel adapter", + "react/event-loop": "Required for ReactPHP promise and parallel adapters" + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Async\\": "src/" + } + }, + "autoload-dev": { + "psr-4": { + "Utopia\\Tests\\": "tests/" + } + }, + "scripts": { + "test-unit": [ + "vendor/bin/phpunit tests/Unit --exclude-group no-swoole" + ], + "test-promise-sync": [ + "vendor/bin/phpunit tests/E2e/Promise/SyncTest.php" + ], + "test-promise-swoole": [ + "vendor/bin/phpunit tests/E2e/Promise/Swoole" + ], + "test-promise-amp": [ + "vendor/bin/phpunit tests/E2e/Promise/Amp" + ], + "test-promise-react": [ + "vendor/bin/phpunit tests/E2e/Promise/React" + ], + "test-parallel-sync": [ + "vendor/bin/phpunit tests/E2e/Parallel/Sync" + ], + "test-parallel-swoole-thread": [ + "vendor/bin/phpunit tests/E2e/Parallel/Swoole/ThreadTest.php" + ], + "test-parallel-swoole-process": [ + "vendor/bin/phpunit tests/E2e/Parallel/Swoole/ProcessTest.php" + ], + "test-parallel-amp": [ + "vendor/bin/phpunit tests/E2e/Parallel/Amp" + ], + "test-parallel-react": [ + "vendor/bin/phpunit tests/E2e/Parallel/React" + ], + "test-parallel-ext": [ + "php -n -d extension=parallel.so -d extension=sockets.so vendor/bin/phpunit tests/E2e/Parallel/Parallel" + ], + "test-e2e": [ + "vendor/bin/phpunit tests/E2e --exclude-group ext-parallel" + ], + "test": [ + "@test-unit", + "@test-e2e", + "@test-parallel-ext" + ], + "lint": [ + "vendor/bin/pint" + ], + "format": [ + "php -d memory_limit=4G vendor/bin/pint" + ], + "check": [ + "vendor/bin/phpstan analyse src tests --level=max --memory-limit=4G" + ] + }, + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Appwrite Team", + "email": "team@appwrite.io" + } + ], + "description": "High-performance concurrent + parallel library with Promise and Parallel execution support for PHP.", + "support": { + "source": "https://github.com/utopia-php/async/tree/0.1.1", + "issues": "https://github.com/utopia-php/async/issues" + }, + "time": "2026-06-08T05:10:34+00:00" + }, { "name": "utopia-php/cache", - "version": "4.0.0", + "version": "4.0.2", "source": { "type": "git", "url": "https://github.com/utopia-php/cache.git", - "reference": "2ab50440114a7699b8a017ee2ecd8bcd9d54ac5e" + "reference": "92e02dab63606234b993b841ebf4c58845dd4620" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/cache/zipball/2ab50440114a7699b8a017ee2ecd8bcd9d54ac5e", - "reference": "2ab50440114a7699b8a017ee2ecd8bcd9d54ac5e", + "url": "https://api.github.com/repos/utopia-php/cache/zipball/92e02dab63606234b993b841ebf4c58845dd4620", + "reference": "92e02dab63606234b993b841ebf4c58845dd4620", "shasum": "" }, "require": { "ext-json": "*", - "ext-memcached": "*", - "ext-redis": "*", - "php": ">=8.3", - "utopia-php/circuit-breaker": "0.3.*", - "utopia-php/pools": "2.*", - "utopia-php/telemetry": "*" + "php": ">=8.4", + "utopia-php/circuit-breaker": "^0.3", + "utopia-php/pools": "^2.0", + "utopia-php/telemetry": "^0.4" }, "require-dev": { - "laravel/pint": "1.2.*", - "phpstan/phpstan": "^1.12", - "phpunit/phpunit": "^9.3", - "swoole/ide-helper": "^6.0", - "vimeo/psalm": "4.13.1" + "swoole/ide-helper": "^6.0" + }, + "suggest": { + "ext-memcached": "Required for the Memcached and Hazelcast adapters.", + "ext-redis": "Required for the Redis, RedisCluster and Sharding adapters.", + "ext-swoole": "Required for the multiplexing Redis adapter (>=6.0)." }, "type": "library", "autoload": { @@ -2240,6 +2422,12 @@ "license": [ "MIT" ], + "authors": [ + { + "name": "Team Appwrite", + "email": "team@appwrite.io" + } + ], "description": "A simple cache library to manage application cache storing, loading and purging", "keywords": [ "cache", @@ -2250,32 +2438,29 @@ ], "support": { "issues": "https://github.com/utopia-php/cache/issues", - "source": "https://github.com/utopia-php/cache/tree/4.0.0" + "source": "https://github.com/utopia-php/cache/tree/4.0.2" }, - "time": "2026-07-31T13:04:45+00:00" + "time": "2026-08-12T07:48:59+00:00" }, { "name": "utopia-php/circuit-breaker", - "version": "0.3.1", + "version": "0.3.2", "source": { "type": "git", "url": "https://github.com/utopia-php/circuit-breaker.git", - "reference": "db5d77f6c99ebce2ee81bd8ed4ae8f41bd2b0828" + "reference": "5fbc3802471b0d1b4260bd9f5544514e6929b481" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/circuit-breaker/zipball/db5d77f6c99ebce2ee81bd8ed4ae8f41bd2b0828", - "reference": "db5d77f6c99ebce2ee81bd8ed4ae8f41bd2b0828", + "url": "https://api.github.com/repos/utopia-php/circuit-breaker/zipball/5fbc3802471b0d1b4260bd9f5544514e6929b481", + "reference": "5fbc3802471b0d1b4260bd9f5544514e6929b481", "shasum": "" }, "require": { "php": ">=8.2" }, "require-dev": { - "laravel/pint": "^1.29", - "phpstan/phpstan": "^2.1", - "phpunit/phpunit": "^10.0", - "utopia-php/telemetry": "^0.4" + "utopia-php/telemetry": "^0.4.6" }, "suggest": { "ext-opentelemetry": "Required by utopia-php/telemetry when using OpenTelemetry metrics.", @@ -2287,7 +2472,7 @@ "type": "library", "autoload": { "psr-4": { - "Utopia\\CircuitBreaker\\": "src/CircuitBreaker" + "Utopia\\CircuitBreaker\\": "src/CircuitBreaker/" } }, "notification-url": "https://packagist.org/downloads/", @@ -2312,9 +2497,9 @@ ], "support": { "issues": "https://github.com/utopia-php/circuit-breaker/issues", - "source": "https://github.com/utopia-php/circuit-breaker/tree/0.3.1" + "source": "https://github.com/utopia-php/circuit-breaker/tree/0.3.2" }, - "time": "2026-05-29T12:12:23+00:00" + "time": "2026-08-05T18:07:20+00:00" }, { "name": "utopia-php/client", @@ -2423,16 +2608,16 @@ }, { "name": "utopia-php/database", - "version": "7.0.0", + "version": "dev-feat-query-lib", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "40185b100f92c402a189d6f4f0962c63bbe5726b" + "reference": "5c7226f01f58ff3ea45071441c66123b64d3fa2c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/40185b100f92c402a189d6f4f0962c63bbe5726b", - "reference": "40185b100f92c402a189d6f4f0962c63bbe5726b", + "url": "https://api.github.com/repos/utopia-php/database/zipball/5c7226f01f58ff3ea45071441c66123b64d3fa2c", + "reference": "5c7226f01f58ff3ea45071441c66123b64d3fa2c", "shasum": "" }, "require": { @@ -2441,21 +2626,29 @@ "ext-pdo": "*", "ext-redis": "*", "php": ">=8.5", - "utopia-php/cache": "^4.0.0", + "utopia-php/async": "^0.1", + "utopia-php/cache": "^4.0 || ^5.0", "utopia-php/console": "0.1.*", "utopia-php/mongo": "1.*", "utopia-php/pools": "2.*", - "utopia-php/validators": "0.3.*" + "utopia-php/query": "0.6.*", + "utopia-php/validators": "^0.6" }, "require-dev": { + "brianium/paratest": "7.20.*", "fakerphp/faker": "1.23.*", "laravel/pint": "*", - "pcov/clobber": "2.*", - "phpstan/phpstan": "1.*", - "phpunit/phpunit": "9.*", + "phpstan/phpstan": "2.1.*", + "phpstan/phpstan-phpunit": "2.0.*", + "phpunit/phpunit": "12.5.*", "rregeer/phpunit-coverage-check": "0.3.*", "swoole/ide-helper": "5.1.3", - "utopia-php/cli": "0.22.*" + "utopia-php/cli": "^0.22" + }, + "suggest": { + "ext-pdo": "Needed to support MariaDB, MySQL or SQLite Database Adapter", + "ext-redis": "Needed to support Redis Cache Adapter", + "mongodb/mongodb": "Needed to support MongoDB Database Adapter" }, "type": "library", "autoload": { @@ -2463,7 +2656,38 @@ "Utopia\\Database\\": "src/Database" } }, - "notification-url": "https://packagist.org/downloads/", + "autoload-dev": { + "psr-4": { + "Tests\\E2E\\": "tests/e2e", + "Tests\\Unit\\": "tests/unit" + } + }, + "scripts": { + "build": [ + "Composer\\Config::disableProcessTimeout", + "docker compose build" + ], + "start": [ + "Composer\\Config::disableProcessTimeout", + "docker compose up -d" + ], + "test": [ + "Composer\\Config::disableProcessTimeout", + "docker compose exec tests vendor/bin/paratest --configuration phpunit.xml --functional --processes 4" + ], + "lint": [ + "php -d memory_limit=2G ./vendor/bin/pint --test" + ], + "format": [ + "php -d memory_limit=2G ./vendor/bin/pint" + ], + "check": [ + "./vendor/bin/phpstan analyse --memory-limit 2G" + ], + "coverage": [ + "./vendor/bin/coverage-check ./tmp/clover.xml 90" + ] + }, "license": [ "MIT" ], @@ -2476,10 +2700,10 @@ "utopia" ], "support": { - "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/7.0.0" + "source": "https://github.com/utopia-php/database/tree/feat-query-lib", + "issues": "https://github.com/utopia-php/database/issues" }, - "time": "2026-07-31T13:33:10+00:00" + "time": "2026-08-31T22:50:42+00:00" }, { "name": "utopia-php/dsn", @@ -2530,16 +2754,16 @@ }, { "name": "utopia-php/mongo", - "version": "1.4.0", + "version": "1.5.3", "source": { "type": "git", "url": "https://github.com/utopia-php/mongo.git", - "reference": "78818fd295f2829aaad5b74c9094e8c6f603550d" + "reference": "be29ee2d84b9f7efdfc6fea5b4b2d4bf95ff5ec4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/mongo/zipball/78818fd295f2829aaad5b74c9094e8c6f603550d", - "reference": "78818fd295f2829aaad5b74c9094e8c6f603550d", + "url": "https://api.github.com/repos/utopia-php/mongo/zipball/be29ee2d84b9f7efdfc6fea5b4b2d4bf95ff5ec4", + "reference": "be29ee2d84b9f7efdfc6fea5b4b2d4bf95ff5ec4", "shasum": "" }, "require": { @@ -2585,27 +2809,27 @@ ], "support": { "issues": "https://github.com/utopia-php/mongo/issues", - "source": "https://github.com/utopia-php/mongo/tree/1.4.0" + "source": "https://github.com/utopia-php/mongo/tree/1.5.3" }, - "time": "2026-07-30T05:24:58+00:00" + "time": "2026-08-13T01:55:11+00:00" }, { "name": "utopia-php/pools", - "version": "2.0.1", + "version": "2.0.2", "source": { "type": "git", "url": "https://github.com/utopia-php/pools.git", - "reference": "48905dac1b8f8050c2e07bdda32a018ca33982fc" + "reference": "86d43fcacd125232c0743e8c22695faf97285cf3" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/pools/zipball/48905dac1b8f8050c2e07bdda32a018ca33982fc", - "reference": "48905dac1b8f8050c2e07bdda32a018ca33982fc", + "url": "https://api.github.com/repos/utopia-php/pools/zipball/86d43fcacd125232c0743e8c22695faf97285cf3", + "reference": "86d43fcacd125232c0743e8c22695faf97285cf3", "shasum": "" }, "require": { "php": ">=8.4", - "utopia-php/telemetry": "^0.4" + "utopia-php/telemetry": "^0.4.6" }, "require-dev": { "swoole/ide-helper": "6.*" @@ -2641,9 +2865,9 @@ ], "support": { "issues": "https://github.com/utopia-php/pools/issues", - "source": "https://github.com/utopia-php/pools/tree/2.0.1" + "source": "https://github.com/utopia-php/pools/tree/2.0.2" }, - "time": "2026-07-31T12:19:18+00:00" + "time": "2026-08-05T18:07:20+00:00" }, { "name": "utopia-php/psr7", @@ -2691,6 +2915,86 @@ }, "time": "2026-07-06T12:40:23+00:00" }, + { + "name": "utopia-php/query", + "version": "0.6.0", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/query.git", + "reference": "abaebb2f3426bdbc6f44bab04254a668de937148" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/query/zipball/abaebb2f3426bdbc6f44bab04254a668de937148", + "reference": "abaebb2f3426bdbc6f44bab04254a668de937148", + "shasum": "" + }, + "require": { + "php": ">=8.4" + }, + "require-dev": { + "brianium/paratest": "*", + "laravel/pint": "*", + "mongodb/mongodb": "^2.0", + "phpstan/phpstan": "*", + "phpunit/phpcov": "*", + "phpunit/phpunit": "^12.0" + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Query\\": "src/Query" + } + }, + "autoload-dev": { + "psr-4": { + "Tests\\Query\\": "tests/Query", + "Tests\\Integration\\": "tests/Integration" + } + }, + "scripts": { + "test": [ + "vendor/bin/paratest --testsuite Query --processes=auto --exclude-group=performance" + ], + "test:coverage": [ + "vendor/bin/paratest --testsuite Query --processes=auto --exclude-group=performance --coverage-php coverage/unit.cov" + ], + "test:performance": [ + "vendor/bin/phpunit --testsuite Query --group=performance" + ], + "test:integration": [ + "vendor/bin/phpunit --testsuite Integration" + ], + "test:integration:coverage": [ + "vendor/bin/phpunit --testsuite Integration --coverage-php coverage/integration.cov" + ], + "lint": [ + "php -d memory_limit=2G ./vendor/bin/pint --test" + ], + "format": [ + "php -d memory_limit=2G ./vendor/bin/pint" + ], + "check": [ + "./vendor/bin/phpstan analyse --level max src tests --memory-limit 2G" + ] + }, + "license": [ + "MIT" + ], + "description": "A simple library providing a query abstraction for filtering, ordering, and pagination", + "keywords": [ + "framework", + "php", + "query", + "upf", + "utopia" + ], + "support": { + "source": "https://github.com/utopia-php/query/tree/0.6.0", + "issues": "https://github.com/utopia-php/query/issues" + }, + "time": "2026-08-21T11:03:36+00:00" + }, { "name": "utopia-php/span", "version": "4.1.0", @@ -2733,16 +3037,16 @@ }, { "name": "utopia-php/storage", - "version": "4.0.1", + "version": "4.0.5", "source": { "type": "git", "url": "https://github.com/utopia-php/storage.git", - "reference": "9684da5ab161ae9375d4f47541ef825451d69f2a" + "reference": "593e732644ac809df18ae45b5561acf0a0c5e1be" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/storage/zipball/9684da5ab161ae9375d4f47541ef825451d69f2a", - "reference": "9684da5ab161ae9375d4f47541ef825451d69f2a", + "url": "https://api.github.com/repos/utopia-php/storage/zipball/593e732644ac809df18ae45b5561acf0a0c5e1be", + "reference": "593e732644ac809df18ae45b5561acf0a0c5e1be", "shasum": "" }, "require": { @@ -2754,8 +3058,8 @@ "psr/http-message": "^1.1 || ^2.0", "utopia-php/client": "0.2.* || 0.3.*", "utopia-php/psr7": "0.2.*", - "utopia-php/telemetry": "^0.4", - "utopia-php/validators": "0.3.*" + "utopia-php/telemetry": "^0.4.6", + "utopia-php/validators": "^0.6" }, "type": "library", "autoload": { @@ -2777,22 +3081,22 @@ ], "support": { "issues": "https://github.com/utopia-php/storage/issues", - "source": "https://github.com/utopia-php/storage/tree/4.0.1" + "source": "https://github.com/utopia-php/storage/tree/4.0.5" }, - "time": "2026-07-31T09:20:57+00:00" + "time": "2026-08-27T06:13:20+00:00" }, { "name": "utopia-php/telemetry", - "version": "0.4.5", + "version": "0.4.6", "source": { "type": "git", "url": "https://github.com/utopia-php/telemetry.git", - "reference": "139943bffcd4f6dd8fb9ed247f946a1d151b006a" + "reference": "f96778a01792c32df0876fe1f38a79b9588445d8" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/telemetry/zipball/139943bffcd4f6dd8fb9ed247f946a1d151b006a", - "reference": "139943bffcd4f6dd8fb9ed247f946a1d151b006a", + "url": "https://api.github.com/repos/utopia-php/telemetry/zipball/f96778a01792c32df0876fe1f38a79b9588445d8", + "reference": "f96778a01792c32df0876fe1f38a79b9588445d8", "shasum": "" }, "require": { @@ -2828,22 +3132,22 @@ ], "support": { "issues": "https://github.com/utopia-php/telemetry/issues", - "source": "https://github.com/utopia-php/telemetry/tree/0.4.5" + "source": "https://github.com/utopia-php/telemetry/tree/0.4.6" }, - "time": "2026-07-08T11:07:25+00:00" + "time": "2026-08-05T17:56:48+00:00" }, { "name": "utopia-php/validators", - "version": "0.3.1", + "version": "0.6.0", "source": { "type": "git", "url": "https://github.com/utopia-php/validators.git", - "reference": "d9c2269ebd2596a09681ccd2fd133eeedfcdf9ba" + "reference": "7afdde56a7a635f0cb3d8a5e22ebbef40baf31dc" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/validators/zipball/d9c2269ebd2596a09681ccd2fd133eeedfcdf9ba", - "reference": "d9c2269ebd2596a09681ccd2fd133eeedfcdf9ba", + "url": "https://api.github.com/repos/utopia-php/validators/zipball/7afdde56a7a635f0cb3d8a5e22ebbef40baf31dc", + "reference": "7afdde56a7a635f0cb3d8a5e22ebbef40baf31dc", "shasum": "" }, "require": { @@ -2868,9 +3172,9 @@ ], "support": { "issues": "https://github.com/utopia-php/validators/issues", - "source": "https://github.com/utopia-php/validators/tree/0.3.1" + "source": "https://github.com/utopia-php/validators/tree/0.6.0" }, - "time": "2026-07-14T11:55:48+00:00" + "time": "2026-08-27T04:46:08+00:00" } ], "packages-dev": [ @@ -5148,10 +5452,19 @@ "time": "2026-07-06T19:11:50+00:00" } ], - "aliases": [], - "minimum-stability": "stable", - "stability-flags": {}, - "prefer-stable": false, + "aliases": [ + { + "package": "utopia-php/database", + "version": "dev-feat-query-lib", + "alias": "7.0.0", + "alias_normalized": "7.0.0.0" + } + ], + "minimum-stability": "dev", + "stability-flags": { + "utopia-php/database": 20 + }, + "prefer-stable": true, "prefer-lowest": false, "platform": { "php": ">=8.5", diff --git a/phpunit.xml b/phpunit.xml index fad0deca..ce3ac877 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -2,7 +2,7 @@ diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index a8eb59d0..531712f2 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -24,22 +24,32 @@ use Appwrite\Services\Teams; use Appwrite\Services\Users; use Override; +use Utopia\Database\Adapter\Feature\Spatial; +use Utopia\Database\Attribute as UtopiaAttribute; +use Utopia\Database\Capability; +use Utopia\Database\Collection; use Utopia\Database\Database as UtopiaDatabase; use Utopia\Database\DateTime; use Utopia\Database\Document as UtopiaDocument; use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Exception\Authorization as AuthorizationException; +use Utopia\Database\Exception\Conflict as ConflictException; use Utopia\Database\Exception\Duplicate as DuplicateException; use Utopia\Database\Exception\Limit as LimitException; use Utopia\Database\Exception\Structure as StructureException; use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Permission; use Utopia\Database\Helpers\Role; +use Utopia\Database\Index as UtopiaIndex; use Utopia\Database\Query; +use Utopia\Database\Relationship as UtopiaRelationship; +use Utopia\Database\RelationSide; +use Utopia\Database\RelationType; use Utopia\Database\Validator\Index as IndexValidator; use Utopia\Database\Validator\Structure; use Utopia\Database\Validator\UID; use Utopia\Migration\Destination; +use Utopia\Migration\Destinations\Appwrite\ProvisioningOwner; use Utopia\Migration\Exception; use Utopia\Migration\Resource; use Utopia\Migration\Resources\Auth\AuthMethods; @@ -79,6 +89,10 @@ use Utopia\Migration\Resources\Storage\File; use Utopia\Migration\Resources\Templates\EmailTemplate; use Utopia\Migration\Transfer; +use Utopia\Query\Schema\ColumnType; +use Utopia\Query\Schema\ForeignKeyAction; +use Utopia\Query\Schema\IndexType; +use Utopia\Query\Schema\Order; class Appwrite extends Destination { @@ -152,6 +166,19 @@ class Appwrite extends Destination */ protected $getDatabaseDSN; + /** Immutable owner written with every provisioning transition initiated by this destination. */ + private readonly ProvisioningOwner $owner; + + /** + * Resolves the authoritative terminal owner of an incomplete database. + * Database status is resource-local and never proves the overall migration + * attempt terminal. Callers must derive this from their authoritative + * operation lifecycle; null means active or unknown and fails closed. + * + * @var callable(UtopiaDocument $database): ?ProvisioningOwner + */ + private $getRecoverableOwner; + /** * @var array */ @@ -198,6 +225,8 @@ class Appwrite extends Destination * @param UtopiaDatabase $dbForProject * @param callable(UtopiaDocument $database):UtopiaDatabase $getDatabasesDB * @param array> $collectionStructure + * @param ProvisioningOwner $owner Immutable logical migration and execution-attempt identifiers for databases provisioned by this destination. + * @param callable(UtopiaDocument $database): ?ProvisioningOwner $getRecoverableOwner Returns the exact authoritative terminal owner for an existing `provisioning` or `failed` database. Resource status alone is not lifecycle proof; return null while the owning migration attempt is active or unknown. * @param OnDuplicate $onDuplicate Behavior when a row with an existing $id is encountered. * @param (callable(Database $resource): string)|null $getDatabaseDSN Resolver for the destination's `_databases.database` value. Pass when the destination project's DSN differs from the source's, so the destination row carries its own DSN instead of inheriting the source's. * @param array>> $collectionStructures Per-database-type metadata collection structures (e.g. `['vectorsdb' => ...]`), used instead of $collectionStructure when the imported database's type has an entry. Types with an entry also get type-specific metadata written (e.g. vectorsdb collection `dimension`). @@ -211,6 +240,8 @@ public function __construct( protected array $collectionStructure, protected UtopiaDatabase $dbForPlatform, protected string $projectInternalId, + ProvisioningOwner $owner, + callable $getRecoverableOwner, protected OnDuplicate $onDuplicate = OnDuplicate::Fail, ?callable $getDatabaseDSN = null, protected array $collectionStructures = [], @@ -235,6 +266,8 @@ public function __construct( $this->getDatabasesDB = $getDatabasesDB; $this->getDatabaseDSN = $getDatabaseDSN; + $this->owner = $owner; + $this->getRecoverableOwner = $getRecoverableOwner; } /** @@ -275,7 +308,11 @@ private function getSupportForDatabaseStatus(): bool return $this->databaseStatusSupported = false; } - /** Orphan cleanup runs only after a successful migration — a mid-run throw preserves the destination as-is. */ + /** + * Transfer resources without committing terminal destination state. The + * caller must first persist its own finalization claim, then invoke + * success() while that generation is still authoritative. + */ #[Override] public function run( array $resources, @@ -285,9 +322,17 @@ public function run( ): void { $this->resetRunState(); parent::run($resources, $callback, $rootResourceId, $rootResourceType); - // parent::run() returning means every resource transferred, so the databases are usable. - // Flip status before the orphan sweep so a cleanup failure can't strand them in `provisioning`. - $this->markProvisionedDatabasesReady(); + } + + /** Finalize destination state only after the caller has fenced terminal ownership. */ + #[Override] + public function success(): void + { + // Flip status before the orphan sweep so a cleanup failure can't strand databases in `provisioning`. + if (! $this->markProvisionedDatabasesReady()) { + return; + } + $this->cleanupOverwriteOrphans(); } @@ -300,29 +345,84 @@ private function resetRunState(): void $this->provisioningDatabases = []; } - private function markProvisionedDatabasesReady(): void + private function markProvisionedDatabasesReady(): bool { + $ready = true; foreach (\array_keys($this->provisioningDatabases) as $databaseId) { try { - $this->setDatabaseStatus($databaseId, self::DATABASE_STATUS_READY); - } catch (\Throwable) { - // Best-effort: a transient error on one database must not strand the rest - // in provisioning or block the orphan-cleanup sweep that follows. + if (! $this->setDatabaseStatus($databaseId, self::DATABASE_STATUS_READY)) { + throw new DatabaseException('Database provisioning owner changed before finalization'); + } + } catch (\Throwable $error) { + $ready = false; + $this->addError(new Exception( + resourceName: Resource::TYPE_DATABASE, + resourceGroup: Transfer::GROUP_DATABASES, + resourceId: $databaseId, + message: $error->getMessage(), + code: $error->getCode(), + previous: $error, + )); } } + + return $ready; } - private function setDatabaseStatus(string $databaseId, string $status): void + private function setDatabaseStatus(string $databaseId, string $status): bool { if (! $this->getSupportForDatabaseStatus()) { - return; + return true; } - $this->dbForProject->updateDocument( - self::META_DATABASES, - $databaseId, - new UtopiaDocument(['status' => $status]), - ); + try { + $database = $this->dbForProject->getDocument( + self::META_DATABASES, + $databaseId, + forUpdate: true, + ); + $owner = $this->getProvisioningOwner($database); + if ( + $database->isEmpty() + || $database->getAttribute('status') !== self::DATABASE_STATUS_PROVISIONING + || $owner === null + || ! $owner->equals($this->owner) + ) { + return false; + } + + $version = $database->getVersion(); + if ($version === null) { + throw new DatabaseException('Database provisioning ownership requires a document version'); + } + + $updated = $this->dbForProject->updateDocument( + self::META_DATABASES, + $databaseId, + new UtopiaDocument(['status' => $status]), + expectedVersion: $version, + ); + + return ! $updated->isEmpty(); + } catch (ConflictException) { + return false; + } + } + + private function getProvisioningOwner(UtopiaDocument $database): ?ProvisioningOwner + { + $migrationId = $database->getAttribute('migrationId'); + $attemptId = $database->getAttribute('migrationAttemptId'); + if ( + ! \is_string($migrationId) + || $migrationId === '' + || ! \is_string($attemptId) + || $attemptId === '' + ) { + return null; + } + + return new ProvisioningOwner($migrationId, $attemptId); } /** Best-effort transition to `failed`; a secondary error here must not mask the caller's original throw. */ @@ -671,43 +771,83 @@ protected function createDatabase(Database $resource): bool $createdAt = $this->normalizeDateTime($resource->getCreatedAt()); $updatedAt = $this->normalizeDateTime($resource->getUpdatedAt(), $createdAt); - if ($this->onDuplicate !== OnDuplicate::Fail) { - $existing = $this->dbForProject->getDocument(self::META_DATABASES, $resource->getId()); - $action = $this->onDuplicate->resolveSchemaAction( - !$existing->isEmpty(), - $updatedAt, - $existing->getUpdatedAt(), - ); - - $isFailed = ! $existing->isEmpty() - && $this->getSupportForDatabaseStatus() - && $existing->getAttribute('status') === self::DATABASE_STATUS_FAILED; - - if ($isFailed) { - // A prior run created the metadata document but left the database unusable (its backing - // collection may be missing). Force Overwrite — regardless of timestamps or spec match — - // so the recovery path recreates the collection instead of skipping it forever. - $action = SchemaAction::Overwrite; - } elseif ($action !== SchemaAction::Create && $this->databaseSpecMatches($existing, $resource)) { - // Spec match → skip work. Create excluded; nothing on dest to match against. - $action = SchemaAction::Skip; + $existing = $this->dbForProject->getDocument(self::META_DATABASES, $resource->getId()); + $supportsStatus = $this->getSupportForDatabaseStatus(); + $status = $supportsStatus && ! $existing->isEmpty() + ? $existing->getAttribute('status') + : null; + $isIncomplete = \in_array( + $status, + [self::DATABASE_STATUS_PROVISIONING, self::DATABASE_STATUS_FAILED], + true, + ); + $expectedOwner = null; + if ($isIncomplete) { + $snapshotOwner = $this->getProvisioningOwner($existing); + $expectedOwner = ($this->getRecoverableOwner)($existing); + if ( + $snapshotOwner === null + || ! $expectedOwner instanceof ProvisioningOwner + || ! $snapshotOwner->equals($expectedOwner) + || $this->owner->attemptId === $expectedOwner->attemptId + ) { + throw new DatabaseException('Database '.$resource->getId().' recovery requires exact terminal-owner attestation and a fresh attempt'); } + } - $earlyReturn = match ($action) { - SchemaAction::Skip => (function () use ($resource, $existing): bool { - $resource->setSequence($existing->getSequence()); - $resource->setStatus(Resource::STATUS_SKIPPED, 'Already exists on destination'); - // Recover a database left in `provisioning` by a prior failed run: the spec matches so - // we skip re-import, but the end-of-run sweep should still flip it to `ready`. + if ($this->onDuplicate !== OnDuplicate::Fail || $isIncomplete) { + /** @var array{action: SchemaAction, database: UtopiaDocument, incomplete: bool} $claim */ + $claim = $this->dbForProject->withTransaction(function () use ( + $resource, + $updatedAt, + $supportsStatus, + $status, + $isIncomplete, + $expectedOwner, + ): array { + $locked = $this->dbForProject->getDocument( + self::META_DATABASES, + $resource->getId(), + forUpdate: true, + ); + $lockedStatus = $supportsStatus && ! $locked->isEmpty() + ? $locked->getAttribute('status') + : null; + $lockedIncomplete = \in_array( + $lockedStatus, + [self::DATABASE_STATUS_PROVISIONING, self::DATABASE_STATUS_FAILED], + true, + ); + + if ($isIncomplete) { + $lockedOwner = $this->getProvisioningOwner($locked); if ( - $this->getSupportForDatabaseStatus() - && $existing->getAttribute('status') === self::DATABASE_STATUS_PROVISIONING + $lockedStatus !== $status + || $lockedOwner === null + || ! $expectedOwner instanceof ProvisioningOwner + || ! $lockedOwner->equals($expectedOwner) ) { - $this->provisioningDatabases[$resource->getId()] = true; + throw new DatabaseException('Database '.$resource->getId().' recovery owner changed before it could be claimed'); } - return false; - })(), - SchemaAction::Overwrite => (function () use ($resource, $existing, $updatedAt, $isFailed): bool { + } elseif ($lockedIncomplete) { + throw new DatabaseException('Database '.$resource->getId().' requires terminal migration attestation before recovery'); + } + + $action = $this->onDuplicate->resolveSchemaAction( + ! $locked->isEmpty(), + $updatedAt, + $locked->getUpdatedAt(), + ); + + if ($isIncomplete) { + // A prior run created the metadata document but left the database unusable. Force an + // overwrite after the locked ownership check so retries can recreate its collection. + $action = SchemaAction::Overwrite; + } elseif ($action !== SchemaAction::Create && $this->databaseSpecMatches($locked, $resource)) { + $action = SchemaAction::Skip; + } + + if ($action === SchemaAction::Overwrite) { $document = [ 'name' => $resource->getDatabaseName(), 'search' => implode(' ', [$resource->getId(), $resource->getDatabaseName()]), @@ -718,52 +858,68 @@ protected function createDatabase(Database $resource): bool '$updatedAt' => $updatedAt, ]; - if ($this->getSupportForDatabaseStatus()) { + if ($supportsStatus) { $document['status'] = self::DATABASE_STATUS_PROVISIONING; + $document['migrationId'] = $this->owner->migrationId; + $document['migrationAttemptId'] = $this->owner->attemptId; } - $this->dbForProject->updateDocument(self::META_DATABASES, $existing->getId(), new UtopiaDocument($document)); - $resource->setSequence($existing->getSequence()); - - // Only a `failed` database can be missing its backing collection (a prior run wrote the - // metadata document but threw before createCollection). Recreate it so we never flip a - // database to ready with no collection behind it. A healthy overwrite already has its - // collection, so we skip the lookup entirely. - if ($isFailed && $this->dbForProject->getCollection($this->databaseCollectionId($existing))->isEmpty()) { - try { - $structure = $this->collectionStructureFor($resource); - - $columns = \array_map( - fn ($attr) => new UtopiaDocument($attr), - $structure['attributes'] - ); - - $indexes = \array_map( - fn ($index) => new UtopiaDocument($index), - $structure['indexes'] - ); - - $this->dbForProject->createCollection( - $this->databaseCollectionId($existing), - $columns, - $indexes - ); - } catch (\Throwable $e) { - $this->markDatabaseFailed($resource->getId()); - throw $e; - } + $version = $locked->getVersion(); + if ($version === null) { + throw new DatabaseException('Database provisioning ownership requires a document version'); } + $updated = $this->dbForProject->updateDocument( + self::META_DATABASES, + $locked->getId(), + new UtopiaDocument($document), + expectedVersion: $version, + ); + if ($updated->isEmpty()) { + throw new DatabaseException('Database '.$resource->getId().' provisioning owner changed before it could be claimed'); + } + } + + return [ + 'action' => $action, + 'database' => $locked, + 'incomplete' => $isIncomplete, + ]; + }); + + $action = $claim['action']; + $existing = $claim['database']; + $isIncomplete = $claim['incomplete']; - if ($this->getSupportForDatabaseStatus()) { - $this->provisioningDatabases[$resource->getId()] = true; + if ($action === SchemaAction::Skip) { + $resource->setSequence($existing->getSequence()); + $resource->setStatus(Resource::STATUS_SKIPPED, 'Already exists on destination'); + return false; + } + + if ($action === SchemaAction::Overwrite) { + $resource->setSequence($existing->getSequence()); + + // The claim transaction commits before inspecting or creating the backing collection. + if ($isIncomplete && $this->dbForProject->getCollection($this->databaseCollectionId($existing))->isEmpty()) { + try { + $structure = $this->collectionStructureFor($resource); + + $this->dbForProject->createCollection(new Collection( + id: $this->databaseCollectionId($existing), + attributes: $this->schemaAttributes($structure['attributes'] ?? []), + indexes: $this->schemaIndexes($structure['indexes'] ?? []), + )); + } catch (\Throwable $e) { + $this->markDatabaseFailed($resource->getId()); + throw $e; } + } - return true; - })(), - SchemaAction::Create => null, - }; - if ($earlyReturn !== null) { - return $earlyReturn; + if ($supportsStatus) { + $this->provisioningDatabases[$resource->getId()] = true; + } + + return true; } } @@ -785,32 +941,28 @@ protected function createDatabase(Database $resource): bool // source leaves status untouched so the collection default applies. Never copy the source's state. if ($this->getSupportForDatabaseStatus()) { $document['status'] = self::DATABASE_STATUS_PROVISIONING; + $document['migrationId'] = $this->owner->migrationId; + $document['migrationAttemptId'] = $this->owner->attemptId; } $database = $this->dbForProject->createDocument(self::META_DATABASES, new UtopiaDocument($document)); - $resource->setSequence($database->getSequence()); - try { - $structure = $this->collectionStructureFor($resource); + $database = $this->dbForProject->getDocument(self::META_DATABASES, $database->getId()); - $columns = \array_map( - fn ($attr) => new UtopiaDocument($attr), - $structure['attributes'] - ); + if ($database->isEmpty()) { + throw new DatabaseException('Failed to reload created database '.$resource->getId()); + } - $indexes = \array_map( - fn ($index) => new UtopiaDocument($index), - $structure['indexes'] - ); + $resource->setSequence($database->getSequence()); + $structure = $this->collectionStructureFor($resource); - $this->dbForProject->createCollection( - $this->databaseCollectionId($database), - $columns, - $indexes - ); + $this->dbForProject->createCollection(new Collection( + id: $this->databaseCollectionId($database), + attributes: $this->schemaAttributes($structure['attributes'] ?? []), + indexes: $this->schemaIndexes($structure['indexes'] ?? []), + )); } catch (\Throwable $e) { - // The metadata document exists but the database isn't usable; mark it failed before propagating. $this->markDatabaseFailed($resource->getId()); throw $e; } @@ -933,11 +1085,11 @@ protected function createEntity(Table $resource): bool $resource->setSequence($table->getSequence()); - $dbForDatabases->createCollection( - $this->tableCollectionId($database, $table), + $dbForDatabases->createCollection(new Collection( + id: $this->tableCollectionId($database, $table), permissions: $resource->getPermissions(), - documentSecurity: $resource->getRowSecurity() - ); + documentSecurity: $resource->getRowSecurity(), + )); return true; } @@ -955,32 +1107,7 @@ protected function createField(Column|Attribute $resource): bool } // column will be matching attribute as well // column type will be matching attribute type as well - $type = match ($resource->getType()) { - Column::TYPE_DATETIME => UtopiaDatabase::VAR_DATETIME, - Column::TYPE_BOOLEAN => UtopiaDatabase::VAR_BOOLEAN, - Column::TYPE_INTEGER => UtopiaDatabase::VAR_INTEGER, - Column::TYPE_BIG_INT => UtopiaDatabase::VAR_BIGINT, - Column::TYPE_FLOAT => UtopiaDatabase::VAR_FLOAT, - Column::TYPE_RELATIONSHIP => UtopiaDatabase::VAR_RELATIONSHIP, - - Column::TYPE_STRING, - Column::TYPE_IP, - Column::TYPE_EMAIL, - Column::TYPE_URL, - Column::TYPE_ENUM => UtopiaDatabase::VAR_STRING, - - Column::TYPE_POINT => UtopiaDatabase::VAR_POINT, - Column::TYPE_LINE => UtopiaDatabase::VAR_LINESTRING, - Column::TYPE_POLYGON => UtopiaDatabase::VAR_POLYGON, - Column::TYPE_TEXT => UtopiaDatabase::VAR_TEXT, - Column::TYPE_VARCHAR => UtopiaDatabase::VAR_VARCHAR, - Column::TYPE_MEDIUMTEXT => UtopiaDatabase::VAR_MEDIUMTEXT, - Column::TYPE_LONGTEXT => UtopiaDatabase::VAR_LONGTEXT, - Column::TYPE_OBJECT => UtopiaDatabase::VAR_OBJECT, - Column::TYPE_VECTOR => UtopiaDatabase::VAR_VECTOR, - - default => throw new \Exception('Invalid resource type ' . $resource->getType(), Exception::CODE_VALIDATION), - }; + $type = $this->schemaColumnType($resource); $database = $this->dbForProject->getDocument( self::META_DATABASES, @@ -1015,7 +1142,7 @@ protected function createField(Column|Attribute $resource): bool } if (!empty($resource->getFormat())) { - if (!Structure::hasFormat($resource->getFormat(), $type)) { + if (!Structure::hasFormat($resource->getFormat(), ColumnType::from($type))) { $resource->setStatus(Resource::STATUS_ERROR, "Format {$resource->getFormat()} not available for column type {$type}"); $this->addError(new Exception( resourceName: $resource->getName(), @@ -1049,8 +1176,9 @@ protected function createField(Column|Attribute $resource): bool return false; } - if ($type === UtopiaDatabase::VAR_RELATIONSHIP) { - $resource->getOptions()['side'] = UtopiaDatabase::RELATION_SIDE_PARENT; + $relatedTable = null; + if ($type === ColumnType::Relationship->value) { + $resource->getOptions()['side'] = RelationSide::Parent->value; $relatedTable = $this->dbForProject->getDocument( $this->databaseCollectionId($database), $resource->getOptions()['relatedCollection'] @@ -1073,7 +1201,7 @@ protected function createField(Column|Attribute $resource): bool $this->trackOrphanCandidate($database, $table, 'attributeKeys', $resource->getKey(), $dbForDatabases); - $isRelationship = $type === UtopiaDatabase::VAR_RELATIONSHIP; + $isRelationship = $type === ColumnType::Relationship->value; // Source emits both sides of a two-way; processing one side reconciles both. Partner skip. $twoWayPairKey = $this->twoWayPairKey($database, $table, $resource, $type); @@ -1145,7 +1273,19 @@ protected function createField(Column|Attribute $resource): bool '$updatedAt' => $updatedAt, ]); - $this->dbForProject->checkAttribute($table, $column); + $this->dbForProject->checkAttribute($table, UtopiaAttribute::fromArray([ + 'key' => $resource->getKey(), + 'type' => $type, + 'size' => $resource->getSize(), + 'required' => $resource->isRequired(), + 'signed' => $resource->isSigned(), + 'default' => $resource->getDefault(), + 'array' => $resource->isArray(), + 'format' => $resource->getFormat() !== '' ? $resource->getFormat() : null, + 'formatOptions' => $resource->getFormatOptions(), + 'filters' => $resource->getFilters(), + 'options' => $resource->getOptions() !== [] ? $resource->getOptions() : null, + ])); $column = $this->dbForProject->createDocument(self::META_ATTRIBUTES, $column); } catch (DuplicateException $e) { @@ -1178,11 +1318,11 @@ protected function createField(Column|Attribute $resource): bool $twoWayKey = null; - if ($type === UtopiaDatabase::VAR_RELATIONSHIP && $options['twoWay']) { + if ($type === ColumnType::Relationship->value && $options['twoWay'] && $relatedTable !== null) { $twoWayKey = $options['twoWayKey']; $options['relatedCollection'] = $table->getId(); $options['twoWayKey'] = $resource->getKey(); - $options['side'] = UtopiaDatabase::RELATION_SIDE_CHILD; + $options['side'] = RelationSide::Child->value; try { $twoWayAttribute = new UtopiaDocument([ @@ -1241,16 +1381,25 @@ protected function createField(Column|Attribute $resource): bool try { switch ($type) { - case UtopiaDatabase::VAR_RELATIONSHIP: + case ColumnType::Relationship->value: + if ($relatedTable === null) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: 'Related table not found', + ); + } if (!$dbForDatabases->createRelationship( - collection: $this->tableCollectionId($database, $table), - // @phpstan-ignore-next-line — $relatedTable is set when type is VAR_RELATIONSHIP. - relatedCollection: $this->tableCollectionId($database, $relatedTable), - type: $options['relationType'], - twoWay: $options['twoWay'], - id: $resource->getKey(), - twoWayKey: $options['twoWay'] ? $twoWayKey : $options['twoWayKey'] ?? null, - onDelete: $options['onDelete'], + new UtopiaRelationship( + collection: $this->tableCollectionId($database, $table), + relatedCollection: $this->tableCollectionId($database, $relatedTable), + type: RelationType::from($options['relationType']), + twoWay: $options['twoWay'], + key: $resource->getKey(), + twoWayKey: (string) ($options['twoWay'] ? $twoWayKey : $options['twoWayKey'] ?? ''), + onDelete: ForeignKeyAction::from($options['onDelete']), + ) )) { throw new Exception( resourceName: $resource->getName(), @@ -1263,16 +1412,18 @@ protected function createField(Column|Attribute $resource): bool default: if (!$dbForDatabases->createAttribute( $this->tableCollectionId($database, $table), - $resource->getKey(), - $type, - $resource->getSize(), - $resource->isRequired(), - $resource->getDefault(), - $resource->isSigned(), - $resource->isArray(), - $resource->getFormat(), - $resource->getFormatOptions(), - $resource->getFilters(), + new UtopiaAttribute( + key: $resource->getKey(), + type: ColumnType::from($type), + size: $resource->getSize(), + required: $resource->isRequired(), + default: $resource->getDefault(), + signed: $resource->isSigned(), + array: $resource->isArray(), + format: $resource->getFormat() !== '' ? $resource->getFormat() : null, + formatOptions: $resource->getFormatOptions(), + filters: $resource->getFilters(), + ), )) { throw new \Exception('Failed to create Column', Exception::CODE_INTERNAL); } @@ -1287,8 +1438,7 @@ protected function createField(Column|Attribute $resource): bool throw $e; } - if ($type === UtopiaDatabase::VAR_RELATIONSHIP && $options['twoWay']) { - // @phpstan-ignore-next-line — $relatedTable is set when type is VAR_RELATIONSHIP. + if ($type === ColumnType::Relationship->value && $options['twoWay'] && $relatedTable !== null) { $this->dbForProject->purgeCachedDocument($this->databaseCollectionId($database), $relatedTable->getId()); } @@ -1311,6 +1461,69 @@ private function collectionStructureFor(Database $resource): array return $this->collectionStructures[$resource->getType()] ?? $this->collectionStructure; } + private function schemaColumnType(Column|Attribute $resource): string + { + return match ($resource->getType()) { + Column::TYPE_DATETIME => ColumnType::Datetime->value, + Column::TYPE_BOOLEAN => ColumnType::Boolean->value, + Column::TYPE_INTEGER => ColumnType::Integer->value, + Column::TYPE_BIG_INT => ColumnType::BigInteger->value, + Column::TYPE_FLOAT => ColumnType::Double->value, + Column::TYPE_RELATIONSHIP => ColumnType::Relationship->value, + Column::TYPE_STRING, + Column::TYPE_IP, + Column::TYPE_EMAIL, + Column::TYPE_URL, + Column::TYPE_ENUM => ColumnType::String->value, + Column::TYPE_POINT => ColumnType::Point->value, + Column::TYPE_LINE => ColumnType::Linestring->value, + Column::TYPE_POLYGON => ColumnType::Polygon->value, + Column::TYPE_TEXT => ColumnType::Text->value, + Column::TYPE_VARCHAR => ColumnType::Varchar->value, + Column::TYPE_MEDIUMTEXT => ColumnType::MediumText->value, + Column::TYPE_LONGTEXT => ColumnType::LongText->value, + Column::TYPE_OBJECT => ColumnType::Object->value, + Column::TYPE_VECTOR => ColumnType::Vector->value, + default => throw new \Exception('Invalid resource type ' . $resource->getType(), Exception::CODE_VALIDATION), + }; + } + + /** + * @param array $attributes + * @return array + */ + private function schemaAttributes(array $attributes): array + { + return \array_map(function (mixed $attr): UtopiaAttribute { + if ($attr instanceof UtopiaAttribute) { + return $attr; + } + if ($attr instanceof UtopiaDocument) { + return UtopiaAttribute::fromArray($attr->getArrayCopy()); + } + + return UtopiaAttribute::fromArray(\is_array($attr) ? $attr : []); + }, $attributes); + } + + /** + * @param array $indexes + * @return array + */ + private function schemaIndexes(array $indexes): array + { + return \array_map(function (mixed $index): UtopiaIndex { + if ($index instanceof UtopiaIndex) { + return $index; + } + if ($index instanceof UtopiaDocument) { + return UtopiaIndex::fromArray($index->getArrayCopy()); + } + + return UtopiaIndex::fromArray(\is_array($index) ? $index : []); + }, $indexes); + } + /** * Type-specific metadata for an imported entity's collection document. VectorsDB collections * carry a required `dimension`; archives written before it was serialized have none, so a @@ -1342,7 +1555,7 @@ private function entityTypeMetadata(Table $resource): array private function syncVectorDimension(Column|Attribute $resource, string $type, UtopiaDocument $database, UtopiaDocument $table): void { if ( - $type !== UtopiaDatabase::VAR_VECTOR + $type !== ColumnType::Vector->value || $resource->getKey() !== self::VECTORSDB_EMBEDDINGS_KEY || $resource->getTable()->getDatabase()->getType() !== Resource::TYPE_DATABASE_VECTORSDB || !isset($this->collectionStructures[Resource::TYPE_DATABASE_VECTORSDB]) @@ -1461,7 +1674,7 @@ protected function createIndex(Index $resource): bool // Lengths hidden by default $lengths = []; - if ($dbForDatabases->getAdapter()->getSupportForAttributes()) { + if ($dbForDatabases->getAdapter()->supports(Capability::DefinedAttributes)) { $this->validateFieldsForIndexes($resource, $table, $lengths); } @@ -1487,24 +1700,27 @@ protected function createIndex(Index $resource): bool $tableColumns = $table->getAttribute('attributes', []); $tableIndexes = $table->getAttribute('indexes', []); + $adapter = $dbForDatabases->getAdapter(); $validator = new IndexValidator( $tableColumns, $tableIndexes, - $dbForDatabases->getAdapter()->getMaxIndexLength(), - $dbForDatabases->getAdapter()->getInternalIndexesKeys(), - $dbForDatabases->getAdapter()->getSupportForIndexArray(), - $dbForDatabases->getAdapter()->getSupportForSpatialIndexNull(), - $dbForDatabases->getAdapter()->getSupportForSpatialIndexOrder(), - $dbForDatabases->getAdapter()->getSupportForVectors(), - $dbForDatabases->getAdapter()->getSupportForAttributes(), - $dbForDatabases->getAdapter()->getSupportForMultipleFulltextIndexes(), - $dbForDatabases->getAdapter()->getSupportForIdenticalIndexes(), - $dbForDatabases->getAdapter()->getSupportForObjectIndexes(), - $dbForDatabases->getAdapter()->getSupportForTrigramIndex(), - $dbForDatabases->getAdapter()->getSupportForSpatialAttributes(), - $dbForDatabases->getAdapter()->getSupportForIndex(), - $dbForDatabases->getAdapter()->getSupportForUniqueIndex(), - $dbForDatabases->getAdapter()->getSupportForFulltextIndex() + $adapter->getMaxIndexLength(), + $adapter->getInternalIndexesKeys(), + $adapter->supports(Capability::IndexArray), + $adapter->supports(Capability::SpatialIndexNull), + $adapter->supports(Capability::SpatialIndexOrder), + $adapter->supports(Capability::Vectors), + $adapter->supports(Capability::DefinedAttributes), + $adapter->supports(Capability::MultipleFulltextIndexes), + $adapter->supports(Capability::IdenticalIndexes), + $adapter->supports(Capability::ObjectIndexes), + $adapter->supports(Capability::TrigramIndex), + $adapter instanceof Spatial, + $adapter->supports(Capability::Index), + $adapter->supports(Capability::UniqueIndex), + $adapter->supports(Capability::Fulltext), + $adapter->supports(Capability::TTLIndexes), + $adapter->supports(Capability::Objects), ); if (!$validator->isValid($index)) { @@ -1523,11 +1739,21 @@ protected function createIndex(Index $resource): bool try { $result = $dbForDatabases->createIndex( $this->tableCollectionId($database, $table), - $resource->getKey(), - $resource->getType(), - $resource->getColumns(), - $lengths, - $resource->getOrders() + new UtopiaIndex( + key: $resource->getKey(), + type: IndexType::from($resource->getType()), + attributes: $resource->getColumns(), + lengths: $lengths, + // Sources hand back the 'ASC'/'DESC' strings they were stored + // as, while UtopiaIndex takes Order cases and rejects anything + // else with an InvalidArgumentException -- which is not a + // Migration Exception, so the transfer would abort instead of + // recording a failed index. + orders: \array_map( + static fn (mixed $order): ?Order => Order::tryFrom(\is_string($order) ? \strtoupper($order) : ''), + $resource->getOrders(), + ), + ), ); if (!$result) { @@ -1642,17 +1868,19 @@ protected function createRecord(Row $resource, bool $isLast): bool $resource->getTable()->getId(), ); // Strip row payload fields the table doesn't declare — guards against orphans surviving in source archives. - if ($dbForDatabases->getAdapter()->getSupportForAttributes()) { + if ($dbForDatabases->getAdapter()->supports(Capability::DefinedAttributes)) { foreach ($this->rowBuffer as $row) { foreach ($row as $key => $value) { if (\str_starts_with($key, '$')) { continue; } - /** @var \Utopia\Database\Document $attribute */ $found = false; foreach ($table->getAttribute('attributes', []) as $attribute) { - if ($attribute->getAttribute('key') == $key) { + $attrKey = $attribute instanceof UtopiaAttribute + ? $attribute->key + : $attribute->getAttribute('key'); + if ($attrKey == $key) { $found = true; break; } @@ -1710,7 +1938,7 @@ protected function createRecord(Row $resource, bool $isLast): bool return true; } - /** Relationships route through deleteRelationship since deleteAttribute throws for VAR_RELATIONSHIP. */ + /** Relationships route through deleteRelationship since deleteAttribute throws for relationship columns. */ private function dropAttributeForRecreate( UtopiaDocument $database, UtopiaDocument $table, @@ -1833,7 +2061,7 @@ private function updateRelationshipInPlace( $dbForDatabases->updateRelationship( collection: $this->tableCollectionId($database, $table), id: $resource->getKey(), - onDelete: (string) ($sourceOptions['onDelete'] ?? ''), + onDelete: ForeignKeyAction::from((string) ($sourceOptions['onDelete'] ?? ForeignKeyAction::Restrict->value)), ); } @@ -2021,7 +2249,7 @@ private function twoWayPairKey( Column|Attribute $resource, string $type, ): ?string { - if ($type !== UtopiaDatabase::VAR_RELATIONSHIP) { + if ($type !== ColumnType::Relationship->value) { return null; } $options = $resource->getOptions(); @@ -2201,7 +2429,7 @@ private function dropOrphanAttribute( $options = $attrDoc->getAttribute('options', []); $collectionId = $this->tableCollectionId($database, $table); - if ($type === UtopiaDatabase::VAR_RELATIONSHIP) { + if ($type === ColumnType::Relationship->value) { $this->bestEffort(fn () => $dbForDatabases->deleteRelationship($collectionId, $key)); } else { $this->bestEffort(fn () => $dbForDatabases->deleteAttribute($collectionId, $key)); @@ -2210,7 +2438,7 @@ private function dropOrphanAttribute( $this->dbForProject->purgeCachedDocument($this->databaseCollectionId($database), $table->getId()); $dbForDatabases->purgeCachedCollection($collectionId); - if ($type !== UtopiaDatabase::VAR_RELATIONSHIP) { + if ($type !== ColumnType::Relationship->value) { return; } $partner = $this->resolveTwoWayPartner($database, $options); @@ -3859,13 +4087,19 @@ private function validateFieldsForIndexes(Index $resource, UtopiaDocument $table $tableColumns = $table->getAttribute('attributes', []); $oldColumns = \array_map( - fn ($attr) => $attr->getArrayCopy(), + function ($attr) { + if ($attr instanceof UtopiaAttribute) { + return $attr->toDocument()->getArrayCopy(); + } + + return $attr->getArrayCopy(); + }, $tableColumns ); $oldColumns[] = [ 'key' => '$id', - 'type' => UtopiaDatabase::VAR_STRING, + 'type' => ColumnType::String->value, 'status' => 'available', 'required' => true, 'array' => false, @@ -3875,7 +4109,7 @@ private function validateFieldsForIndexes(Index $resource, UtopiaDocument $table $oldColumns[] = [ 'key' => '$createdAt', - 'type' => UtopiaDatabase::VAR_DATETIME, + 'type' => ColumnType::Datetime->value, 'status' => 'available', 'signed' => false, 'required' => false, @@ -3886,7 +4120,7 @@ private function validateFieldsForIndexes(Index $resource, UtopiaDocument $table $oldColumns[] = [ 'key' => '$updatedAt', - 'type' => UtopiaDatabase::VAR_DATETIME, + 'type' => ColumnType::Datetime->value, 'status' => 'available', 'signed' => false, 'required' => false, @@ -3915,7 +4149,7 @@ private function validateFieldsForIndexes(Index $resource, UtopiaDocument $table $columnType = $oldColumns[$columnIndex]['type']; $columnArray = $oldColumns[$columnIndex]['array'] ?? false; - if ($columnType === UtopiaDatabase::VAR_RELATIONSHIP) { + if ($columnType === ColumnType::Relationship->value || $columnType === ColumnType::Relationship) { throw new Exception( resourceName: $resource->getName(), resourceGroup: $resource->getGroup(), diff --git a/src/Migration/Destinations/Appwrite/ProvisioningOwner.php b/src/Migration/Destinations/Appwrite/ProvisioningOwner.php new file mode 100644 index 00000000..7a27879d --- /dev/null +++ b/src/Migration/Destinations/Appwrite/ProvisioningOwner.php @@ -0,0 +1,24 @@ +migrationId === $owner->migrationId + && $this->attemptId === $owner->attemptId; + } +} diff --git a/src/Migration/Resources/Database/Column.php b/src/Migration/Resources/Database/Column.php index cc056504..a9df216f 100644 --- a/src/Migration/Resources/Database/Column.php +++ b/src/Migration/Resources/Database/Column.php @@ -86,6 +86,10 @@ public static function resolve(array $column): array $type = \is_string($column['type'] ?? null) ? $column['type'] : ''; $format = \is_string($column['format'] ?? null) ? $column['format'] : ''; + if ($type === 'biginteger') { + $type = self::TYPE_BIG_INT; + } + if (isset(self::FORMAT_SIZES[$type])) { $format = $type; $type = self::TYPE_STRING; diff --git a/src/Migration/Resources/Database/Columns/Relationship.php b/src/Migration/Resources/Database/Columns/Relationship.php index b530c317..6cf1c736 100644 --- a/src/Migration/Resources/Database/Columns/Relationship.php +++ b/src/Migration/Resources/Database/Columns/Relationship.php @@ -2,9 +2,10 @@ namespace Utopia\Migration\Resources\Database\Columns; -use Utopia\Database\Database; +use Utopia\Database\RelationSide; use Utopia\Migration\Resources\Database\Column; use Utopia\Migration\Resources\Database\Table; +use Utopia\Query\Schema\ForeignKeyAction; class Relationship extends Column { @@ -15,8 +16,8 @@ public function __construct( string $relationType, bool $twoWay = false, ?string $twoWayKey = null, - string $onDelete = Database::RELATION_MUTATE_RESTRICT, - string $side = Database::RELATION_SIDE_PARENT, + string $onDelete = ForeignKeyAction::Restrict->value, + string $side = RelationSide::Parent->value, string $createdAt = '', string $updatedAt = '' ) { diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index 61443459..34ed9a54 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -22,6 +22,7 @@ use Utopia\Database\Database as UtopiaDatabase; use Utopia\Database\DateTime as UtopiaDateTime; use Utopia\Database\Document as UtopiaDocument; +use Utopia\Database\RelationSide; use Utopia\Migration\Exception; use Utopia\Migration\Resource; use Utopia\Migration\Resources\Auth\AuthMethods; @@ -88,6 +89,7 @@ use Utopia\Migration\Sources\Appwrite\Reader\API as APIReader; use Utopia\Migration\Sources\Appwrite\Reader\Database as DatabaseReader; use Utopia\Migration\Transfer; +use Utopia\Query\Schema\ColumnType; class Appwrite extends Source { @@ -1283,8 +1285,8 @@ private function exportFields(string $entityType, int $batchSize): void foreach ($response as $column) { if ( - $column['type'] === UtopiaDatabase::VAR_RELATIONSHIP - && $column['side'] === UtopiaDatabase::RELATION_SIDE_CHILD + $column['type'] === ColumnType::Relationship->value + && $column['side'] === RelationSide::Child->value ) { continue; } diff --git a/src/Migration/Sources/Appwrite/Reader/Database.php b/src/Migration/Sources/Appwrite/Reader/Database.php index 547ec1e6..701eafdb 100644 --- a/src/Migration/Sources/Appwrite/Reader/Database.php +++ b/src/Migration/Sources/Appwrite/Reader/Database.php @@ -2,6 +2,7 @@ namespace Utopia\Migration\Sources\Appwrite\Reader; +use Utopia\Database\Capability; use Utopia\Database\Database as UtopiaDatabase; use Utopia\Database\Document as UtopiaDocument; use Utopia\Database\Exception as DatabaseException; @@ -16,6 +17,7 @@ use Utopia\Migration\Resources\Database\Row as RowResource; use Utopia\Migration\Resources\Database\Table as TableResource; use Utopia\Migration\Sources\Appwrite\Reader; +use Utopia\Query\Schema\ColumnType; /** * @implements Reader @@ -259,13 +261,18 @@ public function listColumns(TableResource $resource, array $queries = []): array } foreach ($columns as $column) { - if ($column['type'] !== UtopiaDatabase::VAR_RELATIONSHIP) { + if ($column['type'] !== ColumnType::Relationship->value) { continue; } $options = $column['options']; - foreach ($options as $key => $value) { - $column[$key] = $value; + if ($options instanceof UtopiaDocument) { + $options = $options->getArrayCopy(); + } + if (\is_array($options)) { + foreach ($options as $key => $value) { + $column[$key] = $value; + } } unset($column['options']); @@ -492,7 +499,7 @@ public function queryLimit(int $limit): Query public function getSupportForAttributes(): bool { - return $this->dbForProject->getAdapter()->getSupportForAttributes(); + return $this->dbForProject->getAdapter()->supports(Capability::DefinedAttributes); } /** diff --git a/src/Migration/Sources/CSV.php b/src/Migration/Sources/CSV.php index 0b6b725c..5ca8ae49 100644 --- a/src/Migration/Sources/CSV.php +++ b/src/Migration/Sources/CSV.php @@ -3,6 +3,8 @@ namespace Utopia\Migration\Sources; use Utopia\Database\Database as UtopiaDatabase; +use Utopia\Database\RelationSide; +use Utopia\Database\RelationType; use Utopia\Migration\Exception; use Utopia\Migration\Resource; use Utopia\Migration\Resource as UtopiaResource; @@ -232,7 +234,7 @@ private function exportRows(int $batchSize): void if ( $type === Column::TYPE_RELATIONSHIP && - $relationSide === UtopiaDatabase::RELATION_SIDE_CHILD + $relationSide === RelationSide::Child->value ) { continue; } @@ -244,8 +246,8 @@ private function exportRows(int $batchSize): void if ( $type === Column::TYPE_RELATIONSHIP && - $relationType === UtopiaDatabase::RELATION_MANY_TO_MANY && - $relationSide === UtopiaDatabase::RELATION_SIDE_PARENT + $relationType === RelationType::ManyToMany->value && + $relationSide === RelationSide::Parent->value ) { $manyToManyKeys[$key] = true; } diff --git a/tests/Migration/E2E/Destinations/AppwriteDatabaseConcurrencyTest.php b/tests/Migration/E2E/Destinations/AppwriteDatabaseConcurrencyTest.php new file mode 100644 index 00000000..d6dcd349 --- /dev/null +++ b/tests/Migration/E2E/Destinations/AppwriteDatabaseConcurrencyTest.php @@ -0,0 +1,400 @@ +> */ + public array $databaseWrites = []; + + public ?Closure $beforeBackingCollectionCreate = null; + + public bool $throwAfterBackingCollectionCallback = false; + + #[Override] + public function updateDocument(string $collection, string $id, UtopiaDocument $document, ?int $expectedVersion = null): UtopiaDocument + { + if ($collection === 'databases') { + $this->databaseWrites[] = $document->getArrayCopy(); + } + + return parent::updateDocument($collection, $id, $document, $expectedVersion); + } + + #[Override] + public function createCollection(Collection $collection): Collection + { + if ( + $this->beforeBackingCollectionCreate !== null + && \str_starts_with($collection->getId(), 'database_') + ) { + $callback = $this->beforeBackingCollectionCreate; + $this->beforeBackingCollectionCreate = null; + $callback(); + + if ($this->throwAfterBackingCollectionCallback) { + throw new DatabaseException('backing collection creation failed'); + } + } + + return parent::createCollection($collection); + } +} + +final class AppwriteDatabaseConcurrencyTest extends TestCase +{ + public function testStaleIncompleteClaimLosesAfterAnotherSuccessorCommits(): void + { + foreach (['provisioning', 'failed'] as $status) { + [$second, $third, $path] = $this->createSharedDatabases(); + + try { + $this->seedDatabase($second, $status, 'migration-first', 'attempt-first'); + $winner = null; + $loser = $this->createDestination( + $second, + 'migration-second', + 'attempt-second', + function (UtopiaDocument $snapshot) use ($third, &$winner): ProvisioningOwner { + $winner = $this->createDestination( + $third, + 'migration-third', + 'attempt-third', + static fn (UtopiaDocument $document): ProvisioningOwner => new ProvisioningOwner( + 'migration-first', + 'attempt-first', + ), + ); + $this->runTransfer($third, $winner); + + return new ProvisioningOwner('migration-first', 'attempt-first'); + }, + ); + $second->databaseWrites = []; + + $this->runTransfer($second, $loser); + + $database = $this->getDatabaseDocument($third); + $this->assertInstanceOf(AppwriteDestination::class, $winner); + $this->assertSame([], $this->errorMessages($winner)); + $this->assertNotSame([], $this->errorMessages($loser)); + $this->assertSame([], $second->databaseWrites); + $this->assertSame('ready', $database->getAttribute('status')); + $this->assertSame('migration-third', $database->getAttribute('migrationId')); + $this->assertSame('attempt-third', $database->getAttribute('migrationAttemptId')); + } finally { + $this->removeSQLiteFiles($path); + } + } + } + + public function testStaleOwnerCannotMarkDatabaseReadyAfterTakeover(): void + { + [$second, $third, $path] = $this->createSharedDatabases(); + + try { + $this->seedDatabase($second, 'provisioning', 'migration-shared', 'attempt-first'); + $successor = null; + $destination = $this->createDestination( + $second, + 'migration-shared', + 'attempt-second', + static fn (UtopiaDocument $snapshot): ProvisioningOwner => new ProvisioningOwner( + 'migration-shared', + 'attempt-first', + ), + ); + + $this->runTransfer( + $second, + $destination, + function () use ($third, &$successor): void { + $successor = $this->createDestination( + $third, + 'migration-shared', + 'attempt-third', + static fn (UtopiaDocument $snapshot): ProvisioningOwner => new ProvisioningOwner( + 'migration-shared', + 'attempt-second', + ), + ); + $this->claimWithoutTerminalTransition($third, $successor); + }, + ); + + $database = $this->getDatabaseDocument($third); + $readyWrites = \array_values(\array_filter( + $second->databaseWrites, + static fn (array $document): bool => ($document['status'] ?? null) === 'ready', + )); + $this->assertInstanceOf(AppwriteDestination::class, $successor); + $this->assertSame( + ['Database provisioning owner changed before finalization'], + $this->errorMessages($destination), + ); + $this->assertSame([], $this->errorMessages($successor)); + $this->assertSame([], $readyWrites); + $this->assertSame('provisioning', $database->getAttribute('status')); + $this->assertSame('migration-shared', $database->getAttribute('migrationId')); + $this->assertSame('attempt-third', $database->getAttribute('migrationAttemptId')); + } finally { + $this->removeSQLiteFiles($path); + } + } + + public function testStaleOwnerCannotMarkDatabaseFailedAfterTakeover(): void + { + [$second, $third, $path] = $this->createSharedDatabases(); + + try { + $this->seedDatabase($second, 'failed', 'migration-shared', 'attempt-first'); + $successor = null; + $second->throwAfterBackingCollectionCallback = true; + $second->beforeBackingCollectionCreate = function () use ($third, &$successor): void { + $successor = $this->createDestination( + $third, + 'migration-shared', + 'attempt-third', + static fn (UtopiaDocument $snapshot): ProvisioningOwner => new ProvisioningOwner( + 'migration-shared', + 'attempt-second', + ), + ); + $this->claimWithoutTerminalTransition($third, $successor); + }; + $destination = $this->createDestination( + $second, + 'migration-shared', + 'attempt-second', + static fn (UtopiaDocument $snapshot): ProvisioningOwner => new ProvisioningOwner( + 'migration-shared', + 'attempt-first', + ), + ); + + $this->runTransfer($second, $destination); + + $database = $this->getDatabaseDocument($third); + $failedWrites = \array_values(\array_filter( + $second->databaseWrites, + static fn (array $document): bool => ($document['status'] ?? null) === 'failed', + )); + $this->assertInstanceOf(AppwriteDestination::class, $successor); + $this->assertNotSame([], $this->errorMessages($destination)); + $this->assertSame([], $this->errorMessages($successor)); + $this->assertSame([], $failedWrites); + $this->assertSame('provisioning', $database->getAttribute('status')); + $this->assertSame('migration-shared', $database->getAttribute('migrationId')); + $this->assertSame('attempt-third', $database->getAttribute('migrationAttemptId')); + } finally { + $this->removeSQLiteFiles($path); + } + } + + private function createDestination( + UtopiaDatabase $database, + string $migrationId, + string $migrationAttemptId, + callable $getRecoverableOwner, + ): AppwriteDestination { + return new AppwriteDestination( + project: 'destination-project', + endpoint: 'http://example.test/v1', + key: 'test-key', + dbForProject: $database, + getDatabasesDB: static fn (UtopiaDocument $document): UtopiaDatabase => $database, + collectionStructure: ['attributes' => [], 'indexes' => []], + dbForPlatform: $database, + projectInternalId: '1', + owner: new ProvisioningOwner($migrationId, $migrationAttemptId), + getRecoverableOwner: $getRecoverableOwner, + onDuplicate: OnDuplicate::Fail, + ); + } + + private function runTransfer( + UtopiaDatabase $database, + AppwriteDestination $destination, + ?callable $callback = null, + ): void { + $source = new class () extends MockSource { + #[Override] + public function supportsDatabaseStatus(): bool + { + return true; + } + }; + $source->pushMockResource(new DatabaseResource( + id: 'database', + name: 'Database', + type: 'tablesdb', + database: 'source-dsn', + databaseStatus: 'ready', + )); + + $transfer = new Transfer($source, $destination); + $database->getAuthorization()->skip( + static function () use ($callback, $destination, $transfer): void { + $transfer->run( + [Resource::TYPE_DATABASE], + $callback ?? static function (): void { + }, + ); + $destination->success(); + }, + ); + } + + private function claimWithoutTerminalTransition( + UtopiaDatabase $database, + AppwriteDestination $destination, + ): void { + try { + $this->runTransfer( + $database, + $destination, + static function (): void { + throw new \RuntimeException('stop after provisioning claim'); + }, + ); + $this->fail('Expected transfer callback to stop before the terminal transition'); + } catch (\RuntimeException $error) { + $this->assertSame('stop after provisioning claim', $error->getMessage()); + } + } + + /** @return array{RecordingSQLiteProjectDatabase, RecordingSQLiteProjectDatabase, string} */ + private function createSharedDatabases(): array + { + $path = \tempnam(\sys_get_temp_dir(), 'migration-owner-'); + if ($path === false) { + throw new \RuntimeException('Failed to create SQLite test database'); + } + + $attributes = SQLite::getPDOAttributes(); + $attributes[PDO::ATTR_PERSISTENT] = false; + $secondConnection = new SQLiteConnection('sqlite:'.$path, null, null, $attributes); + $thirdConnection = new SQLiteConnection('sqlite:'.$path, null, null, $attributes); + $secondConnection->exec('PRAGMA journal_mode = WAL'); + $secondConnection->exec('PRAGMA busy_timeout = 1000'); + $thirdConnection->exec('PRAGMA busy_timeout = 1000'); + + $second = new RecordingSQLiteProjectDatabase( + new SQLite($secondConnection), + new Cache(new None()), + ); + $third = new RecordingSQLiteProjectDatabase( + new SQLite($thirdConnection), + new Cache(new None()), + ); + $namespace = 'migration_owner_'.\uniqid(); + foreach ([$second, $third] as $database) { + $database + ->setDatabase('appwrite') + ->setNamespace($namespace); + } + + $second->create(); + $second->createCollection(new Collection( + id: 'databases', + attributes: [ + $this->attribute('name', ColumnType::String, required: true, size: 256), + $this->attribute('enabled', ColumnType::Boolean, default: true), + $this->attribute('search', ColumnType::String, size: 16384), + $this->attribute('originalId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('type', ColumnType::String, default: 'tablesdb', size: 128), + $this->attribute('database', ColumnType::String, size: 2000), + $this->attribute('status', ColumnType::String, size: 16), + $this->attribute('migrationId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('migrationAttemptId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), + ], + )); + + return [$second, $third, $path]; + } + + private function seedDatabase( + UtopiaDatabase $database, + string $status, + string $migrationId, + string $migrationAttemptId, + ): UtopiaDocument { + return $database->getAuthorization()->skip( + static fn (): UtopiaDocument => $database->createDocument('databases', new UtopiaDocument([ + '$id' => 'database', + 'name' => 'Database', + 'enabled' => true, + 'search' => 'database Database', + 'originalId' => null, + 'type' => 'tablesdb', + 'database' => '', + 'status' => $status, + 'migrationId' => $migrationId, + 'migrationAttemptId' => $migrationAttemptId, + ])), + ); + } + + private function attribute( + string $id, + ColumnType $type, + bool $required = false, + mixed $default = null, + int $size = 0, + ): UtopiaAttribute { + return new UtopiaAttribute( + key: $id, + type: $type, + size: $size, + required: $required, + default: $default, + ); + } + + private function getDatabaseDocument(UtopiaDatabase $database): UtopiaDocument + { + return $database->getAuthorization()->skip( + static fn (): UtopiaDocument => $database->getDocument('databases', 'database'), + ); + } + + /** @return list */ + private function errorMessages(AppwriteDestination $destination): array + { + return \array_map( + static fn (\Throwable $error): string => $error->getMessage(), + $destination->getErrors(), + ); + } + + private function removeSQLiteFiles(string $path): void + { + foreach ([$path, $path.'-wal', $path.'-shm'] as $file) { + if (\is_file($file)) { + \unlink($file); + } + } + } +} diff --git a/tests/Migration/Unit/Destinations/AppwriteCheckAttributeTest.php b/tests/Migration/Unit/Destinations/AppwriteCheckAttributeTest.php new file mode 100644 index 00000000..7825f7e9 --- /dev/null +++ b/tests/Migration/Unit/Destinations/AppwriteCheckAttributeTest.php @@ -0,0 +1,66 @@ +assertStringContainsString( + "checkAttribute(\$table, UtopiaAttribute::fromArray([", + $source, + ); + $this->assertStringContainsString( + "'key' => \$resource->getKey(),", + $source, + ); + $this->assertStringNotContainsString( + 'checkAttribute($table, $column)', + $source, + ); + + $column = new UtopiaDocument([ + '$id' => '1_2_title', + 'key' => 'title', + 'type' => 'string', + 'size' => 128, + 'required' => true, + 'signed' => true, + 'default' => null, + 'array' => false, + 'format' => '', + 'formatOptions' => [], + 'filters' => [], + 'options' => [], + ]); + + $fromDocumentCopy = UtopiaAttribute::fromArray($column->getArrayCopy()); + $this->assertSame('1_2_title', $fromDocumentCopy->key); + + $attribute = UtopiaAttribute::fromArray([ + 'key' => 'title', + 'type' => 'string', + 'size' => 128, + 'required' => true, + 'signed' => true, + 'default' => null, + 'array' => false, + 'format' => null, + 'formatOptions' => [], + 'filters' => [], + 'options' => null, + ]); + + $this->assertInstanceOf(UtopiaAttribute::class, $attribute); + $this->assertSame('title', $attribute->key); + $this->assertSame(ColumnType::String, $attribute->type); + $this->assertSame(128, $attribute->size); + } +} diff --git a/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php b/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php index 1e125acd..6779aa6d 100644 --- a/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php +++ b/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php @@ -7,15 +7,41 @@ use Utopia\Cache\Adapter\Memory as MemoryCache; use Utopia\Cache\Cache; use Utopia\Database\Adapter\Memory as MemoryAdapter; +use Utopia\Database\Attribute as UtopiaAttribute; +use Utopia\Database\Capability; +use Utopia\Database\Collection; use Utopia\Database\Database as UtopiaDatabase; use Utopia\Database\Document as UtopiaDocument; +use Utopia\Database\Exception as DatabaseException; use Utopia\Migration\Destinations\Appwrite as AppwriteDestination; +use Utopia\Migration\Destinations\Appwrite\ProvisioningOwner; use Utopia\Migration\Destinations\OnDuplicate; +use Utopia\Migration\Exception as MigrationException; use Utopia\Migration\Resource; use Utopia\Migration\Resources\Database\Database as DatabaseResource; use Utopia\Migration\Transfer; +use Utopia\Query\Schema\ColumnType; use Utopia\Tests\Unit\Adapters\MockSource; +class ReplicaMemoryAdapter extends MemoryAdapter +{ + /** @return array */ + #[Override] + public function capabilities(): array + { + return [...parent::capabilities(), Capability::TransactionRetries]; + } +} + +final class StandaloneMemoryAdapter extends ReplicaMemoryAdapter +{ + #[Override] + public function withTransaction(callable $callback): mixed + { + return $callback(); + } +} + class CountingAppwriteDestination extends AppwriteDestination { public int $runCount = 0; @@ -32,6 +58,162 @@ public function run( } } +final class ReloadFailingProjectDatabase extends UtopiaDatabase +{ + public bool $failNextDatabasesRead = false; + + #[Override] + public function getDocument(string $collection, string $id, array $queries = [], bool $forUpdate = false): UtopiaDocument + { + $document = parent::getDocument($collection, $id, $queries, $forUpdate); + if ($this->failNextDatabasesRead && $collection === 'databases' && !$document->isEmpty()) { + $this->failNextDatabasesRead = false; + + return new UtopiaDocument(); + } + + return $document; + } +} + +/** + * Fails the reload and every status write while the metadata store is unavailable, + * which is the only way production reaches a document stranded in `provisioning`: + * markDatabaseFailed() swallows its own error so it cannot mask the original throw. + */ +class StrandedProvisioningProjectDatabase extends UtopiaDatabase +{ + public bool $failNextDatabasesRead = false; + + public bool $failDatabasesWrites = false; + + #[Override] + public function getDocument(string $collection, string $id, array $queries = [], bool $forUpdate = false): UtopiaDocument + { + $document = parent::getDocument($collection, $id, $queries, $forUpdate); + if ($this->failNextDatabasesRead && $collection === 'databases' && !$document->isEmpty()) { + $this->failNextDatabasesRead = false; + + return new UtopiaDocument(); + } + + return $document; + } + + #[Override] + public function updateDocument(string $collection, string $id, UtopiaDocument $document, ?int $expectedVersion = null): UtopiaDocument + { + if ($this->failDatabasesWrites && $collection === 'databases') { + throw new DatabaseException('metadata store unavailable'); + } + + return parent::updateDocument($collection, $id, $document, $expectedVersion); + } +} + +final class InterleavingProjectDatabase extends UtopiaDatabase +{ + public bool $interceptNextDatabasesReload = false; + + /** @var (callable(): void)|null */ + public $onDatabasesReload = null; + + #[Override] + public function getDocument(string $collection, string $id, array $queries = [], bool $forUpdate = false): UtopiaDocument + { + $document = parent::getDocument($collection, $id, $queries, $forUpdate); + if ( + $this->interceptNextDatabasesReload + && $collection === 'databases' + && ! $document->isEmpty() + ) { + $this->interceptNextDatabasesReload = false; + ($this->onDatabasesReload)(); + } + + return $document; + } +} + +class RecordingProjectDatabase extends UtopiaDatabase +{ + /** @var list, expectedVersion?: int|null}> */ + public array $databaseWrites = []; + + public bool $failReadyWrite = false; + + public bool $disappearBeforeGuardedWrite = false; + + /** @var list */ + public array $failReadyWrites = []; + + #[Override] + public function createDocument(string $collection, UtopiaDocument $document): UtopiaDocument + { + if ($collection === 'databases') { + $this->databaseWrites[] = [ + 'operation' => 'create', + 'id' => $document->getId(), + 'document' => $document->getArrayCopy(), + ]; + } + + return parent::createDocument($collection, $document); + } + + #[Override] + public function updateDocument(string $collection, string $id, UtopiaDocument $document, ?int $expectedVersion = null): UtopiaDocument + { + if ($collection === 'databases') { + $this->databaseWrites[] = [ + 'operation' => 'update', + 'id' => $id, + 'document' => $document->getArrayCopy(), + 'expectedVersion' => $expectedVersion, + ]; + } + + if ( + ($this->failReadyWrite || \in_array($id, $this->failReadyWrites, true)) + && $collection === 'databases' + && $document->getAttribute('status') === 'ready' + ) { + throw new DatabaseException('ready status unavailable'); + } + + if ( + $this->disappearBeforeGuardedWrite + && $collection === 'databases' + && $expectedVersion !== null + ) { + $this->disappearBeforeGuardedWrite = false; + parent::deleteDocument($collection, $id); + + return new UtopiaDocument(); + } + + return parent::updateDocument($collection, $id, $document, $expectedVersion); + } +} + +final class FinalizerInterleavingProjectDatabase extends RecordingProjectDatabase +{ + public ?\Closure $beforeFinalizerRead = null; + + #[Override] + public function getDocument(string $collection, string $id, array $queries = [], bool $forUpdate = false): UtopiaDocument + { + $document = parent::getDocument($collection, $id, $queries, $forUpdate); + if ($forUpdate && $collection === 'databases' && $this->beforeFinalizerRead !== null) { + $callback = $this->beforeFinalizerRead; + $this->beforeFinalizerRead = null; + $callback(); + } + + return $document; + } +} + final class AppwriteDatabaseStatusTest extends TestCase { public function testDatabaseCreationOmitsStatusThroughLegacyAndExplicitEntrypoints(): void @@ -46,13 +228,21 @@ public function testDatabaseCreationOmitsStatusThroughLegacyAndExplicitEntrypoin $this->assertSame(1, $destination->runCount); $this->assertFalse($created->isEmpty()); $this->assertArrayNotHasKey('status', $created->getArrayCopy()); + $this->assertFalse( + $database->getCollection('database_'.$created->getSequence())->isEmpty(), + 'Metadata collection must use the persisted database sequence', + ); } } public function testDatabaseCreationPreservesLifecycleThroughLegacyAndExplicitEntrypoints(): void { foreach ([false, true] as $explicit) { - $database = $this->createProjectDatabase(withStatus: true); + $database = new UtopiaDatabase( + new StandaloneMemoryAdapter(), + new Cache(new MemoryCache()), + ); + $this->createProjectDatabase(withStatus: true, database: $database); $destination = $this->runDatabaseTransfer($database, $explicit); @@ -61,12 +251,716 @@ public function testDatabaseCreationPreservesLifecycleThroughLegacyAndExplicitEn $this->assertSame(1, $destination->runCount); $this->assertFalse($created->isEmpty()); $this->assertSame('ready', $created->getAttribute('status')); + $this->assertFalse( + $database->getCollection('database_'.$created->getSequence())->isEmpty(), + 'Metadata collection must use the persisted database sequence', + ); + } + } + + public function testCreatePersistsProvisioningAndOwnerAtomically(): void + { + $database = new RecordingProjectDatabase(new MemoryAdapter(), new Cache(new MemoryCache())); + $this->createProjectDatabase(withStatus: true, database: $database); + + $destination = $this->runDatabaseTransfer( + $database, + explicit: false, + migrationId: 'migration-create', + migrationAttemptId: 'attempt-create', + ); + + $this->assertSame([], $this->errorMessages($destination)); + $this->assertSame('create', $database->databaseWrites[0]['operation']); + $this->assertSame('provisioning', $database->databaseWrites[0]['document']['status']); + $this->assertSame('migration-create', $database->databaseWrites[0]['document']['migrationId']); + $this->assertSame('attempt-create', $database->databaseWrites[0]['document']['migrationAttemptId']); + } + + public function testStandaloneUniqueCreateCannotFinalizeAfterOwnerMismatch(): void + { + $database = new FinalizerInterleavingProjectDatabase( + new StandaloneMemoryAdapter(), + new Cache(new MemoryCache()), + ); + $this->createProjectDatabase(withStatus: true, database: $database); + $database->beforeFinalizerRead = static function () use ($database): void { + $database->updateDocument( + 'databases', + 'database', + new UtopiaDocument([ + 'status' => 'provisioning', + 'migrationId' => 'migration-successor', + 'migrationAttemptId' => 'attempt-successor', + ]), + ); + }; + + $destination = $this->runDatabaseTransfer( + $database, + explicit: false, + migrationId: 'migration-create', + migrationAttemptId: 'attempt-create', + ); + + $terminalWrites = \array_values(\array_filter( + $database->databaseWrites, + static fn (array $write): bool => \in_array( + $write['document']['status'] ?? null, + ['ready', 'failed'], + true, + ), + )); + $created = $this->getDatabaseDocument($database); + $this->assertSame(['Database provisioning owner changed before finalization'], $this->errorMessages($destination)); + $this->assertCount(1, $terminalWrites); + $this->assertIsInt($terminalWrites[0]['expectedVersion']); + $this->assertSame('provisioning', $created->getAttribute('status')); + $this->assertSame('migration-successor', $created->getAttribute('migrationId')); + $this->assertSame('attempt-successor', $created->getAttribute('migrationAttemptId')); + } + + public function testAuthorizedOverwritePersistsProvisioningAndNewOwnerAtomically(): void + { + $database = new RecordingProjectDatabase(new ReplicaMemoryAdapter(), new Cache(new MemoryCache())); + $this->createProjectDatabase(withStatus: true, database: $database); + $this->seedDatabase( + $database, + status: 'provisioning', + migrationId: 'migration-old', + migrationAttemptId: 'attempt-old', + ); + $database->databaseWrites = []; + + $destination = $this->runDatabaseTransfer( + $database, + explicit: false, + migrationId: 'migration-new', + migrationAttemptId: 'attempt-new', + getRecoverableOwner: static fn (UtopiaDocument $database): ProvisioningOwner => new ProvisioningOwner( + 'migration-old', + 'attempt-old', + ), + ); + + $this->assertSame([], $this->errorMessages($destination)); + $write = $database->databaseWrites[0] ?? null; + $this->assertNotNull($write); + $this->assertSame('update', $write['operation']); + $this->assertSame('provisioning', $write['document']['status']); + $this->assertSame('migration-new', $write['document']['migrationId']); + $this->assertSame('attempt-new', $write['document']['migrationAttemptId']); + } + + public function testActiveProvisioningRefusalRetainsExistingOwner(): void + { + $database = new RecordingProjectDatabase(new StandaloneMemoryAdapter(), new Cache(new MemoryCache())); + $this->createProjectDatabase(withStatus: true, database: $database); + $this->seedDatabase( + $database, + status: 'provisioning', + migrationId: 'migration-active', + migrationAttemptId: 'attempt-active', + ); + $database->databaseWrites = []; + + $destination = $this->runDatabaseTransfer( + $database, + explicit: false, + migrationId: 'migration-colliding', + migrationAttemptId: 'attempt-colliding', + getRecoverableOwner: static fn (UtopiaDocument $database): ?ProvisioningOwner => null, + ); + + $existing = $this->getDatabaseDocument($database); + $this->assertNotSame([], $this->errorMessages($destination)); + $this->assertSame([], $database->databaseWrites); + $this->assertSame('provisioning', $existing->getAttribute('status')); + $this->assertSame('migration-active', $existing->getAttribute('migrationId')); + $this->assertSame('attempt-active', $existing->getAttribute('migrationAttemptId')); + } + + public function testMissingOrMismatchedOwnerRefusesRecovery(): void + { + foreach ([null, 'migration-existing'] as $owner) { + $database = new RecordingProjectDatabase(new StandaloneMemoryAdapter(), new Cache(new MemoryCache())); + $this->createProjectDatabase(withStatus: true, database: $database); + $this->seedDatabase( + $database, + status: 'provisioning', + migrationId: $owner, + migrationAttemptId: 'attempt-existing', + ); + $database->databaseWrites = []; + + $destination = $this->runDatabaseTransfer( + $database, + explicit: false, + migrationId: 'migration-new', + migrationAttemptId: 'attempt-new', + getRecoverableOwner: static fn (UtopiaDocument $database): ProvisioningOwner => new ProvisioningOwner( + 'migration-terminal', + 'attempt-existing', + ), + ); + + $existing = $this->getDatabaseDocument($database); + $this->assertNotSame([], $this->errorMessages($destination)); + $this->assertSame([], $database->databaseWrites); + $this->assertSame('provisioning', $existing->getAttribute('status')); + $this->assertSame($owner, $existing->getAttribute('migrationId')); + $this->assertSame('attempt-existing', $existing->getAttribute('migrationAttemptId')); + } + } + + public function testMissingOrMismatchedAttemptRefusesRecovery(): void + { + foreach ([null, 'attempt-existing'] as $attemptId) { + $database = new RecordingProjectDatabase(new StandaloneMemoryAdapter(), new Cache(new MemoryCache())); + $this->createProjectDatabase(withStatus: true, database: $database); + $this->seedDatabase( + $database, + status: 'provisioning', + migrationId: 'migration-existing', + migrationAttemptId: $attemptId, + ); + $database->databaseWrites = []; + + $destination = $this->runDatabaseTransfer( + $database, + explicit: false, + migrationId: 'migration-new', + migrationAttemptId: 'attempt-new', + getRecoverableOwner: static fn (UtopiaDocument $database): ProvisioningOwner => new ProvisioningOwner( + 'migration-existing', + 'attempt-terminal', + ), + ); + + $existing = $this->getDatabaseDocument($database); + $this->assertNotSame([], $this->errorMessages($destination)); + $this->assertSame([], $database->databaseWrites); + $this->assertSame('provisioning', $existing->getAttribute('status')); + $this->assertSame('migration-existing', $existing->getAttribute('migrationId')); + $this->assertSame($attemptId, $existing->getAttribute('migrationAttemptId')); + } + } + + public function testRecoveryRefusesReusingThePriorAttempt(): void + { + $database = new RecordingProjectDatabase(new StandaloneMemoryAdapter(), new Cache(new MemoryCache())); + $this->createProjectDatabase(withStatus: true, database: $database); + $this->seedDatabase( + $database, + status: 'failed', + migrationId: 'migration-shared', + migrationAttemptId: 'attempt-shared', + ); + $database->databaseWrites = []; + + $destination = $this->runDatabaseTransfer( + $database, + explicit: false, + migrationId: 'migration-shared', + migrationAttemptId: 'attempt-shared', + getRecoverableOwner: static fn (UtopiaDocument $database): ProvisioningOwner => new ProvisioningOwner( + 'migration-shared', + 'attempt-shared', + ), + ); + + $existing = $this->getDatabaseDocument($database); + $this->assertNotSame([], $this->errorMessages($destination)); + $this->assertSame([], $database->databaseWrites); + $this->assertSame('failed', $existing->getAttribute('status')); + $this->assertSame('migration-shared', $existing->getAttribute('migrationId')); + $this->assertSame('attempt-shared', $existing->getAttribute('migrationAttemptId')); + } + + public function testSameMigrationFailedRetryRequiresTerminalAttestation(): void + { + $database = new RecordingProjectDatabase(new ReplicaMemoryAdapter(), new Cache(new MemoryCache())); + $this->createProjectDatabase(withStatus: true, database: $database); + $this->seedDatabase( + $database, + status: 'failed', + migrationId: 'migration-shared', + migrationAttemptId: 'attempt-active', + ); + $database->databaseWrites = []; + + $destination = $this->runDatabaseTransfer( + $database, + explicit: false, + migrationId: 'migration-shared', + migrationAttemptId: 'attempt-retry', + getRecoverableOwner: static fn (UtopiaDocument $database): ?ProvisioningOwner => null, + ); + + $existing = $this->getDatabaseDocument($database); + $this->assertNotSame([], $this->errorMessages($destination)); + $this->assertSame([], $database->databaseWrites); + $this->assertSame('failed', $existing->getAttribute('status')); + $this->assertSame('migration-shared', $existing->getAttribute('migrationId')); + $this->assertSame('attempt-active', $existing->getAttribute('migrationAttemptId')); + } + + public function testPreExistingRecoveryUsesDocumentVersionCompareAndSet(): void + { + foreach ([new MemoryAdapter(), new StandaloneMemoryAdapter()] as $adapter) { + $database = new RecordingProjectDatabase($adapter, new Cache(new MemoryCache())); + $this->createProjectDatabase(withStatus: true, database: $database); + $this->seedDatabase( + $database, + status: 'failed', + migrationId: 'migration-old', + migrationAttemptId: 'attempt-old', + ); + $database->databaseWrites = []; + + $destination = $this->runDatabaseTransfer( + $database, + explicit: false, + migrationId: 'migration-old', + migrationAttemptId: 'attempt-new', + getRecoverableOwner: static fn (UtopiaDocument $database): ProvisioningOwner => new ProvisioningOwner( + 'migration-old', + 'attempt-old', + ), + ); + + $existing = $this->getDatabaseDocument($database); + $this->assertSame([], $this->errorMessages($destination)); + $this->assertNotSame([], $database->databaseWrites); + $this->assertSame('ready', $existing->getAttribute('status')); + $this->assertSame('migration-old', $existing->getAttribute('migrationId')); + $this->assertSame('attempt-new', $existing->getAttribute('migrationAttemptId')); + } + } + + public function testHealthyOverwriteUsesDocumentVersionCompareAndSet(): void + { + foreach ([new MemoryAdapter(), new StandaloneMemoryAdapter()] as $adapter) { + $database = new RecordingProjectDatabase($adapter, new Cache(new MemoryCache())); + $this->createProjectDatabase(withStatus: true, database: $database); + $this->seedDatabase( + $database, + status: 'ready', + migrationId: 'migration-old', + migrationAttemptId: 'attempt-old', + ); + $database->getAuthorization()->skip( + static fn (): UtopiaDocument => $database->updateDocument( + 'databases', + 'database', + new UtopiaDocument(['name' => 'Old database']), + ), + ); + $database->databaseWrites = []; + + $destination = $this->runDatabaseTransfer( + $database, + explicit: false, + migrationId: 'migration-new', + migrationAttemptId: 'attempt-new', + onDuplicate: OnDuplicate::Overwrite, + resourceUpdatedAt: '2099-01-01T00:00:00.000+00:00', + ); + + $existing = $this->getDatabaseDocument($database); + $this->assertSame([], $this->errorMessages($destination)); + $this->assertNotSame([], $database->databaseWrites); + $this->assertSame('ready', $existing->getAttribute('status')); + $this->assertSame('migration-new', $existing->getAttribute('migrationId')); + $this->assertSame('attempt-new', $existing->getAttribute('migrationAttemptId')); + } + } + + public function testFailedDatabaseRecoveryRequiresExactTerminalOwner(): void + { + foreach ( + [ + 'no attestation' => ['migration-terminal', null], + 'missing owner' => [null, 'migration-terminal'], + 'mismatched owner' => ['migration-existing', 'migration-terminal'], + 'unknown terminal owner' => ['migration-existing', null], + ] as [$owner, $recoverableMigrationId] + ) { + $database = new RecordingProjectDatabase(new StandaloneMemoryAdapter(), new Cache(new MemoryCache())); + $this->createProjectDatabase(withStatus: true, database: $database); + $this->seedDatabase( + $database, + status: 'failed', + migrationId: $owner, + migrationAttemptId: 'attempt-existing', + ); + $database->databaseWrites = []; + + $destination = $this->runDatabaseTransfer( + $database, + explicit: false, + migrationId: 'migration-new', + migrationAttemptId: 'attempt-new', + getRecoverableOwner: static fn (UtopiaDocument $database): ?ProvisioningOwner => $recoverableMigrationId === null + ? null + : new ProvisioningOwner($recoverableMigrationId, 'attempt-existing'), + ); + + $existing = $this->getDatabaseDocument($database); + $this->assertNotSame([], $this->errorMessages($destination)); + $this->assertSame([], $database->databaseWrites); + $this->assertSame('failed', $existing->getAttribute('status')); + $this->assertSame($owner, $existing->getAttribute('migrationId')); + $this->assertSame('attempt-existing', $existing->getAttribute('migrationAttemptId')); } } - private function createProjectDatabase(bool $withStatus): UtopiaDatabase + public function testReadyWriteFailureRetainsProvisioningOwner(): void + { + $database = new RecordingProjectDatabase(new MemoryAdapter(), new Cache(new MemoryCache())); + $this->createProjectDatabase(withStatus: true, database: $database); + $database->failReadyWrite = true; + + $destination = $this->runDatabaseTransfer( + $database, + explicit: false, + migrationId: 'migration-ready-failure', + migrationAttemptId: 'attempt-ready-failure', + ); + + $created = $this->getDatabaseDocument($database); + $errors = $destination->getErrors(); + $this->assertCount(1, $errors); + $this->assertSame(Resource::TYPE_DATABASE, $errors[0]->getResourceName()); + $this->assertSame(Transfer::GROUP_DATABASES, $errors[0]->getResourceGroup()); + $this->assertSame('database', $errors[0]->getResourceId()); + $this->assertSame('ready status unavailable', $errors[0]->getMessage()); + $this->assertInstanceOf(DatabaseException::class, $errors[0]->getPrevious()); + $this->assertSame('provisioning', $created->getAttribute('status')); + $this->assertSame('migration-ready-failure', $created->getAttribute('migrationId')); + $this->assertSame('attempt-ready-failure', $created->getAttribute('migrationAttemptId')); + } + + public function testDatabaseDisappearanceBeforeReadyWriteLosesFinalizationOwnership(): void + { + $database = new RecordingProjectDatabase(new MemoryAdapter(), new Cache(new MemoryCache())); + $this->createProjectDatabase(withStatus: true, database: $database); + $database->disappearBeforeGuardedWrite = true; + + $destination = $this->runDatabaseTransfer( + $database, + explicit: false, + migrationId: 'migration-disappearing-ready', + migrationAttemptId: 'attempt-disappearing-ready', + ); + + $readyWrites = \array_values(\array_filter( + $database->databaseWrites, + static fn (array $write): bool => ($write['document']['status'] ?? null) === 'ready', + )); + + $this->assertSame(['Database provisioning owner changed before finalization'], $this->errorMessages($destination)); + $this->assertCount(1, $readyWrites); + $this->assertIsInt($readyWrites[0]['expectedVersion']); + $this->assertTrue($this->getDatabaseDocument($database)->isEmpty()); + } + + public function testDatabaseDisappearanceBeforeRecoveryClaimLosesOwnership(): void + { + $database = new RecordingProjectDatabase(new StandaloneMemoryAdapter(), new Cache(new MemoryCache())); + $this->createProjectDatabase(withStatus: true, database: $database); + $seeded = $this->seedDatabase( + $database, + status: 'failed', + migrationId: 'migration-old', + migrationAttemptId: 'attempt-old', + ); + $database->databaseWrites = []; + $database->disappearBeforeGuardedWrite = true; + + $destination = $this->runDatabaseTransfer( + $database, + explicit: false, + migrationId: 'migration-new', + migrationAttemptId: 'attempt-new', + getRecoverableOwner: static fn (UtopiaDocument $database): ProvisioningOwner => new ProvisioningOwner( + 'migration-old', + 'attempt-old', + ), + success: false, + ); + + $claimWrites = \array_values(\array_filter( + $database->databaseWrites, + static fn (array $write): bool => ($write['document']['status'] ?? null) === 'provisioning', + )); + + $this->assertSame( + ['Database database provisioning owner changed before it could be claimed'], + $this->errorMessages($destination), + ); + $this->assertCount(1, $claimWrites); + $this->assertSame( + [true], + \array_map( + static fn (array $write): bool => \is_int($write['expectedVersion'] ?? null), + $claimWrites, + ), + ); + $this->assertTrue($this->getDatabaseDocument($database)->isEmpty()); + $this->assertTrue($database->getCollection('database_'.$seeded->getSequence())->isEmpty()); + } + + public function testDatabaseFinalizersRunOnlyAfterSuccess(): void + { + $database = new RecordingProjectDatabase(new MemoryAdapter(), new Cache(new MemoryCache())); + $this->createProjectDatabase(withStatus: true, database: $database); + + $destination = $this->runDatabaseTransfer( + $database, + explicit: false, + migrationId: 'migration-finalizing', + migrationAttemptId: 'attempt-finalizing', + success: false, + ); + + $provisioning = $this->getDatabaseDocument($database); + $this->assertSame('provisioning', $provisioning->getAttribute('status')); + $this->assertSame('migration-finalizing', $provisioning->getAttribute('migrationId')); + $this->assertSame('attempt-finalizing', $provisioning->getAttribute('migrationAttemptId')); + + $database->getAuthorization()->skip($destination->success(...)); + + $this->assertSame('ready', $this->getDatabaseDocument($database)->getAttribute('status')); + } + + public function testReadyFinalizationAttemptsEveryDatabaseAndReportsEveryFailure(): void { - $database = new UtopiaDatabase( + $database = new RecordingProjectDatabase(new MemoryAdapter(), new Cache(new MemoryCache())); + $this->createProjectDatabase(withStatus: true, database: $database); + $database->failReadyWrites = ['database-first', 'database-third']; + + $destination = $this->runDatabaseTransfer( + $database, + explicit: false, + migrationId: 'migration-finalizers', + migrationAttemptId: 'attempt-finalizers', + databaseIds: ['database-first', 'database-second', 'database-third'], + ); + + $readyAttempts = \array_values(\array_map( + static fn (array $write): string => $write['id'], + \array_filter( + $database->databaseWrites, + static fn (array $write): bool => ($write['document']['status'] ?? null) === 'ready', + ), + )); + $errors = $destination->getErrors(); + $errorIds = \array_map( + static fn (MigrationException $error): string => $error->getResourceId(), + $errors, + ); + + $this->assertSame(['database-first', 'database-second', 'database-third'], $readyAttempts); + $this->assertSame(['database-first', 'database-third'], $errorIds); + $this->assertSame('provisioning', $this->getDatabaseDocument($database, 'database-first')->getAttribute('status')); + $this->assertSame('ready', $this->getDatabaseDocument($database, 'database-second')->getAttribute('status')); + $this->assertSame('provisioning', $this->getDatabaseDocument($database, 'database-third')->getAttribute('status')); + } + + public function testReloadFailureMarksTheDatabaseFailed(): void + { + $database = new ReloadFailingProjectDatabase( + new StandaloneMemoryAdapter(), + new Cache(new MemoryCache()), + ); + $this->createProjectDatabase(withStatus: true, database: $database); + $database->failNextDatabasesRead = true; + + $destination = $this->runDatabaseTransfer($database, explicit: false); + + $created = $this->getDatabaseDocument($database); + $this->assertNotSame([], $this->errorMessages($destination)); + $this->assertStringContainsString('Failed to reload created database', $this->errorMessages($destination)[0]); + $this->assertSame('failed', $created->getAttribute('status')); + $this->assertSame('migration-current', $created->getAttribute('migrationId')); + $this->assertSame('attempt-current', $created->getAttribute('migrationAttemptId')); + $this->assertTrue( + $database->getCollection('database_'.$created->getSequence())->isEmpty(), + 'A reload failure must not leave a backing collection behind the metadata document', + ); + } + + public function testFailedDatabaseRetrySucceedsUnderOnDuplicateFail(): void + { + $database = new ReloadFailingProjectDatabase( + new ReplicaMemoryAdapter(), + new Cache(new MemoryCache()), + ); + $this->createProjectDatabase(withStatus: true, database: $database); + $database->failNextDatabasesRead = true; + + $this->runDatabaseTransfer($database, explicit: false); + + $failed = $this->getDatabaseDocument($database); + $this->assertSame('failed', $failed->getAttribute('status')); + $this->assertTrue( + $database->getCollection('database_'.$failed->getSequence())->isEmpty(), + ); + + $destination = $this->runDatabaseTransfer( + $database, + explicit: false, + migrationId: 'migration-current', + migrationAttemptId: 'attempt-recovery', + getRecoverableOwner: static fn (UtopiaDocument $existing): ProvisioningOwner => new ProvisioningOwner( + 'migration-current', + 'attempt-current', + ), + ); + + $recovered = $this->getDatabaseDocument($database); + $this->assertSame([], $this->errorMessages($destination)); + $this->assertSame('ready', $recovered->getAttribute('status')); + $this->assertSame($failed->getSequence(), $recovered->getSequence()); + $this->assertSame('migration-current', $recovered->getAttribute('migrationId')); + $this->assertSame('attempt-recovery', $recovered->getAttribute('migrationAttemptId')); + $this->assertFalse( + $database->getCollection('database_'.$recovered->getSequence())->isEmpty(), + 'A Fail retry must recreate the backing collection for a previously failed database', + ); + } + + public function testProvisioningDatabaseRetrySucceedsUnderOnDuplicateFail(): void + { + $database = new StrandedProvisioningProjectDatabase( + new ReplicaMemoryAdapter(), + new Cache(new MemoryCache()), + ); + $this->createProjectDatabase(withStatus: true, database: $database); + $database->failNextDatabasesRead = true; + $database->failDatabasesWrites = true; + + $this->runDatabaseTransfer($database, explicit: false); + + $stranded = $this->getDatabaseDocument($database); + $this->assertSame( + 'provisioning', + $stranded->getAttribute('status'), + 'The reload threw and the write that would record the failure threw too, so the document is stranded mid-provision', + ); + $this->assertTrue( + $database->getCollection('database_'.$stranded->getSequence())->isEmpty(), + 'The backing collection was never created, so the database is unusable', + ); + $database->failDatabasesWrites = false; + + $destination = $this->runDatabaseTransfer( + $database, + explicit: false, + migrationId: 'migration-recovery', + migrationAttemptId: 'attempt-recovery', + getRecoverableOwner: static fn (UtopiaDocument $existing): ProvisioningOwner => new ProvisioningOwner( + 'migration-current', + 'attempt-current', + ), + ); + + $recovered = $this->getDatabaseDocument($database); + $this->assertSame([], $this->errorMessages($destination)); + $this->assertSame('ready', $recovered->getAttribute('status')); + $this->assertSame($stranded->getSequence(), $recovered->getSequence()); + $this->assertSame('migration-recovery', $recovered->getAttribute('migrationId')); + $this->assertSame('attempt-recovery', $recovered->getAttribute('migrationAttemptId')); + $this->assertFalse( + $database->getCollection('database_'.$recovered->getSequence())->isEmpty(), + 'A Fail retry must recover a database stranded in provisioning, not keep colliding with its metadata id', + ); + } + + public function testConcurrentMigrationDoesNotRecoverAnActivelyProvisioningDatabase(): void + { + $database = new InterleavingProjectDatabase( + new MemoryAdapter(), + new Cache(new MemoryCache()), + ); + $this->createProjectDatabase(withStatus: true, database: $database); + + $second = null; + $statusDuringOverlap = null; + $collectionExistsDuringOverlap = null; + $database->onDatabasesReload = function () use ( + $database, + &$second, + &$statusDuringOverlap, + &$collectionExistsDuringOverlap, + ): void { + $second = $this->runDatabaseTransfer( + $database, + explicit: false, + migrationId: 'migration-second', + migrationAttemptId: 'attempt-second', + ); + $provisioning = $this->getDatabaseDocument($database); + $statusDuringOverlap = $provisioning->getAttribute('status'); + $collectionExistsDuringOverlap = ! $database + ->getCollection('database_'.$provisioning->getSequence()) + ->isEmpty(); + }; + $database->interceptNextDatabasesReload = true; + + $first = $this->runDatabaseTransfer( + $database, + explicit: false, + migrationId: 'migration-first', + migrationAttemptId: 'attempt-first', + ); + + $created = $this->getDatabaseDocument($database); + $this->assertSame([], $this->errorMessages($first)); + $this->assertInstanceOf(CountingAppwriteDestination::class, $second); + $this->assertNotSame([], $this->errorMessages($second)); + $this->assertStringContainsString('terminal-owner attestation', $this->errorMessages($second)[0]); + $this->assertSame('provisioning', $statusDuringOverlap); + $this->assertFalse($collectionExistsDuringOverlap); + $this->assertSame('ready', $created->getAttribute('status')); + $this->assertSame('migration-first', $created->getAttribute('migrationId')); + $this->assertSame('attempt-first', $created->getAttribute('migrationAttemptId')); + $this->assertFalse($database->getCollection('database_'.$created->getSequence())->isEmpty()); + } + + public function testAuthorizedOverwriteReplacesTerminalOwner(): void + { + $database = new RecordingProjectDatabase(new ReplicaMemoryAdapter(), new Cache(new MemoryCache())); + $this->createProjectDatabase(withStatus: true, database: $database); + $this->seedDatabase( + $database, + status: 'provisioning', + migrationId: 'migration-terminal', + migrationAttemptId: 'attempt-terminal', + ); + + $destination = $this->runDatabaseTransfer( + $database, + explicit: false, + migrationId: 'migration-successor', + migrationAttemptId: 'attempt-successor', + getRecoverableOwner: static fn (UtopiaDocument $database): ProvisioningOwner => new ProvisioningOwner( + 'migration-terminal', + 'attempt-terminal', + ), + ); + + $recovered = $this->getDatabaseDocument($database); + $this->assertSame([], $this->errorMessages($destination)); + $this->assertSame('ready', $recovered->getAttribute('status')); + $this->assertSame('migration-successor', $recovered->getAttribute('migrationId')); + $this->assertSame('attempt-successor', $recovered->getAttribute('migrationAttemptId')); + } + + private function createProjectDatabase(bool $withStatus, ?UtopiaDatabase $database = null): UtopiaDatabase + { + $database ??= new UtopiaDatabase( new MemoryAdapter(), new Cache(new MemoryCache()), ); @@ -76,44 +970,83 @@ private function createProjectDatabase(bool $withStatus): UtopiaDatabase $database->create(); $attributes = [ - $this->attribute('name', UtopiaDatabase::VAR_STRING, required: true, size: 256), - $this->attribute('enabled', UtopiaDatabase::VAR_BOOLEAN, default: true), - $this->attribute('search', UtopiaDatabase::VAR_STRING, size: 16384), - $this->attribute('originalId', UtopiaDatabase::VAR_STRING, size: UtopiaDatabase::LENGTH_KEY), - $this->attribute('type', UtopiaDatabase::VAR_STRING, default: 'tablesdb', size: 128), - $this->attribute('database', UtopiaDatabase::VAR_STRING, size: 2000), + $this->attribute('name', ColumnType::String, required: true, size: 256), + $this->attribute('enabled', ColumnType::Boolean, default: true), + $this->attribute('search', ColumnType::String, size: 16384), + $this->attribute('originalId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('type', ColumnType::String, default: 'tablesdb', size: 128), + $this->attribute('database', ColumnType::String, size: 2000), ]; if ($withStatus) { - $attributes[] = $this->attribute('status', UtopiaDatabase::VAR_STRING, size: 16); + $attributes[] = $this->attribute('status', ColumnType::String, size: 16); + $attributes[] = $this->attribute('migrationId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY); + $attributes[] = $this->attribute('migrationAttemptId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY); } - $database->createCollection('databases', $attributes); + $database->createCollection(new Collection( + id: 'databases', + attributes: $attributes, + )); return $database; } + private function seedDatabase( + UtopiaDatabase $database, + string $status, + ?string $migrationId, + ?string $migrationAttemptId, + ): UtopiaDocument { + $document = [ + '$id' => 'database', + 'name' => 'Database', + 'enabled' => true, + 'search' => 'database Database', + 'originalId' => null, + 'type' => 'tablesdb', + 'database' => '', + 'status' => $status, + ]; + if ($migrationId !== null) { + $document['migrationId'] = $migrationId; + } + if ($migrationAttemptId !== null) { + $document['migrationAttemptId'] = $migrationAttemptId; + } + + return $database->getAuthorization()->skip( + static fn (): UtopiaDocument => $database->createDocument('databases', new UtopiaDocument($document)), + ); + } + private function attribute( string $id, - string $type, + ColumnType $type, bool $required = false, mixed $default = null, int $size = 0, - ): UtopiaDocument { - return new UtopiaDocument([ - '$id' => $id, - 'type' => $type, - 'size' => $size, - 'required' => $required, - 'default' => $default, - 'array' => false, - 'signed' => true, - 'filters' => [], - ]); + ): UtopiaAttribute { + return new UtopiaAttribute( + key: $id, + type: $type, + size: $size, + required: $required, + default: $default, + ); } - private function runDatabaseTransfer(UtopiaDatabase $database, bool $explicit): CountingAppwriteDestination - { + private function runDatabaseTransfer( + UtopiaDatabase $database, + bool $explicit, + string $migrationId = 'migration-current', + string $migrationAttemptId = 'attempt-current', + ?callable $getRecoverableOwner = null, + OnDuplicate $onDuplicate = OnDuplicate::Fail, + string $resourceUpdatedAt = '', + array $databaseIds = ['database'], + bool $success = true, + ): CountingAppwriteDestination { $source = new class () extends MockSource { #[Override] public function supportsDatabaseStatus(): bool @@ -121,13 +1054,16 @@ public function supportsDatabaseStatus(): bool return true; } }; - $source->pushMockResource(new DatabaseResource( - id: 'database', - name: 'Database', - type: 'tablesdb', - database: 'source-dsn', - databaseStatus: 'ready', - )); + foreach ($databaseIds as $databaseId) { + $source->pushMockResource(new DatabaseResource( + id: $databaseId, + name: 'Database', + updatedAt: $resourceUpdatedAt, + type: 'tablesdb', + database: 'source-dsn', + databaseStatus: 'ready', + )); + } $destination = new CountingAppwriteDestination( project: 'destination-project', @@ -138,12 +1074,15 @@ public function supportsDatabaseStatus(): bool collectionStructure: ['attributes' => [], 'indexes' => []], dbForPlatform: $database, projectInternalId: '1', - onDuplicate: OnDuplicate::Fail, + owner: new ProvisioningOwner($migrationId, $migrationAttemptId), + getRecoverableOwner: $getRecoverableOwner + ?? static fn (UtopiaDocument $document): ?ProvisioningOwner => null, + onDuplicate: $onDuplicate, ); $transfer = new Transfer($source, $destination); $database->getAuthorization()->skip( - static function () use ($explicit, $transfer): void { + static function () use ($destination, $explicit, $success, $transfer): void { if ($explicit) { $transfer->runWithResourceSelector( [Resource::TYPE_DATABASE], @@ -157,24 +1096,27 @@ static function (): void { parentResourceType: '', ); - return; + } else { + $transfer->run( + [Resource::TYPE_DATABASE], + static function (): void { + }, + ); } - $transfer->run( - [Resource::TYPE_DATABASE], - static function (): void { - }, - ); + if ($success) { + $destination->success(); + } }, ); return $destination; } - private function getDatabaseDocument(UtopiaDatabase $database): UtopiaDocument + private function getDatabaseDocument(UtopiaDatabase $database, string $databaseId = 'database'): UtopiaDocument { return $database->getAuthorization()->skip( - static fn (): UtopiaDocument => $database->getDocument('databases', 'database'), + static fn (): UtopiaDocument => $database->getDocument('databases', $databaseId), ); } diff --git a/tests/Migration/Unit/Destinations/AppwriteDestinationDsnTest.php b/tests/Migration/Unit/Destinations/AppwriteDestinationDsnTest.php index fa22151c..e76ebef2 100644 --- a/tests/Migration/Unit/Destinations/AppwriteDestinationDsnTest.php +++ b/tests/Migration/Unit/Destinations/AppwriteDestinationDsnTest.php @@ -7,6 +7,7 @@ use Utopia\Database\Database as UtopiaDatabase; use Utopia\Database\Document as UtopiaDocument; use Utopia\Migration\Destinations\Appwrite as AppwriteDestination; +use Utopia\Migration\Destinations\Appwrite\ProvisioningOwner; use Utopia\Migration\Destinations\OnDuplicate; use Utopia\Migration\Resources\Database\Database as DatabaseResource; @@ -73,6 +74,8 @@ private function makeDestination(?callable $getDatabaseDSN): AppwriteDestination collectionStructure: ['attributes' => [], 'indexes' => []], dbForPlatform: $this->createStub(UtopiaDatabase::class), projectInternalId: '1', + owner: new ProvisioningOwner('migration-test', 'attempt-test'), + getRecoverableOwner: static fn (UtopiaDocument $database): ?ProvisioningOwner => null, onDuplicate: OnDuplicate::Fail, getDatabaseDSN: $getDatabaseDSN, ); diff --git a/tests/Migration/Unit/Destinations/AppwriteIndexLengthsTest.php b/tests/Migration/Unit/Destinations/AppwriteIndexLengthsTest.php index 362f6e79..85ea4e88 100644 --- a/tests/Migration/Unit/Destinations/AppwriteIndexLengthsTest.php +++ b/tests/Migration/Unit/Destinations/AppwriteIndexLengthsTest.php @@ -8,9 +8,12 @@ use Utopia\Cache\Adapter\Memory as MemoryCache; use Utopia\Cache\Cache; use Utopia\Database\Adapter\Memory as MemoryAdapter; +use Utopia\Database\Attribute as UtopiaAttribute; +use Utopia\Database\Collection; use Utopia\Database\Database as UtopiaDatabase; use Utopia\Database\Document as UtopiaDocument; use Utopia\Migration\Destinations\Appwrite as AppwriteDestination; +use Utopia\Migration\Destinations\Appwrite\ProvisioningOwner; use Utopia\Migration\Destinations\OnDuplicate; use Utopia\Migration\Resource; use Utopia\Migration\Resources\Database\Columns\Text; @@ -18,6 +21,8 @@ use Utopia\Migration\Resources\Database\Index; use Utopia\Migration\Resources\Database\Table; use Utopia\Migration\Transfer; +use Utopia\Query\Schema\ColumnType; +use Utopia\Query\Schema\Order; use Utopia\Tests\Unit\Adapters\MockSource; /** @@ -175,7 +180,7 @@ private function transferIndex( type: 'key', columns: ['reference', 'channel'], lengths: $withLengths, - orders: ['ASC', 'ASC'], + orders: [Order::Asc->value, Order::Asc->value], createdAt: $updatedAt, updatedAt: $updatedAt, ); @@ -188,19 +193,21 @@ private function transferIndex( getDatabasesDB: static fn (UtopiaDocument $document): UtopiaDatabase => $database, collectionStructure: [ 'attributes' => [ - $this->attributeArray('databaseInternalId', UtopiaDatabase::VAR_STRING, size: UtopiaDatabase::LENGTH_KEY), - $this->attributeArray('databaseId', UtopiaDatabase::VAR_STRING, size: UtopiaDatabase::LENGTH_KEY), - $this->attributeArray('name', UtopiaDatabase::VAR_STRING, size: 256), - $this->attributeArray('enabled', UtopiaDatabase::VAR_BOOLEAN, default: true), - $this->attributeArray('documentSecurity', UtopiaDatabase::VAR_BOOLEAN, default: false), - $this->attributeArray('search', UtopiaDatabase::VAR_STRING, size: 16384), - $this->attributeArray('attributes', UtopiaDatabase::VAR_STRING, size: 16384, filters: ['subQueryAttributes']), - $this->attributeArray('indexes', UtopiaDatabase::VAR_STRING, size: 16384, filters: ['subQueryIndexes']), + $this->attributeArray('databaseInternalId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), + $this->attributeArray('databaseId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), + $this->attributeArray('name', ColumnType::String, size: 256), + $this->attributeArray('enabled', ColumnType::Boolean, default: true), + $this->attributeArray('documentSecurity', ColumnType::Boolean, default: false), + $this->attributeArray('search', ColumnType::String, size: 16384), + $this->attributeArray('attributes', ColumnType::String, size: 16384, filters: ['subQueryAttributes']), + $this->attributeArray('indexes', ColumnType::String, size: 16384, filters: ['subQueryIndexes']), ], 'indexes' => [], ], dbForPlatform: $database, projectInternalId: '1', + owner: new ProvisioningOwner('migration-test', 'attempt-test'), + getRecoverableOwner: static fn (UtopiaDocument $document): ?ProvisioningOwner => null, onDuplicate: $onDuplicate, ); @@ -244,48 +251,57 @@ private function projectDatabase(): UtopiaDatabase ->setNamespace('_project'); $database->create(); - $database->createCollection('databases', [ - $this->attribute('name', UtopiaDatabase::VAR_STRING, required: true, size: 256), - $this->attribute('enabled', UtopiaDatabase::VAR_BOOLEAN, default: true), - $this->attribute('search', UtopiaDatabase::VAR_STRING, size: 16384), - $this->attribute('originalId', UtopiaDatabase::VAR_STRING, size: UtopiaDatabase::LENGTH_KEY), - $this->attribute('type', UtopiaDatabase::VAR_STRING, default: 'tablesdb', size: 128), - $this->attribute('database', UtopiaDatabase::VAR_STRING, size: 2000), - ]); - - $database->createCollection('attributes', [ - $this->attribute('key', UtopiaDatabase::VAR_STRING, size: 256), - $this->attribute('databaseInternalId', UtopiaDatabase::VAR_STRING, size: UtopiaDatabase::LENGTH_KEY), - $this->attribute('databaseId', UtopiaDatabase::VAR_STRING, size: UtopiaDatabase::LENGTH_KEY), - $this->attribute('collectionInternalId', UtopiaDatabase::VAR_STRING, size: UtopiaDatabase::LENGTH_KEY), - $this->attribute('collectionId', UtopiaDatabase::VAR_STRING, size: UtopiaDatabase::LENGTH_KEY), - $this->attribute('type', UtopiaDatabase::VAR_STRING, size: 256), - $this->attribute('status', UtopiaDatabase::VAR_STRING, size: 64), - $this->attribute('size', UtopiaDatabase::VAR_INTEGER), - $this->attribute('required', UtopiaDatabase::VAR_BOOLEAN, default: false), - $this->attribute('signed', UtopiaDatabase::VAR_BOOLEAN, default: true), - $this->attribute('default', UtopiaDatabase::VAR_STRING, size: 16384), - $this->attribute('array', UtopiaDatabase::VAR_BOOLEAN, default: false), - $this->attribute('format', UtopiaDatabase::VAR_STRING, size: 64), - $this->attribute('formatOptions', UtopiaDatabase::VAR_STRING, size: 16384, filters: ['json']), - $this->attribute('filters', UtopiaDatabase::VAR_STRING, size: 64, array: true), - $this->attribute('options', UtopiaDatabase::VAR_STRING, size: 16384, filters: ['json']), - $this->attribute('error', UtopiaDatabase::VAR_STRING, size: 2048), - ]); - - $database->createCollection('indexes', [ - $this->attribute('key', UtopiaDatabase::VAR_STRING, size: 256), - $this->attribute('status', UtopiaDatabase::VAR_STRING, size: 64), - $this->attribute('databaseInternalId', UtopiaDatabase::VAR_STRING, size: UtopiaDatabase::LENGTH_KEY), - $this->attribute('databaseId', UtopiaDatabase::VAR_STRING, size: UtopiaDatabase::LENGTH_KEY), - $this->attribute('collectionInternalId', UtopiaDatabase::VAR_STRING, size: UtopiaDatabase::LENGTH_KEY), - $this->attribute('collectionId', UtopiaDatabase::VAR_STRING, size: UtopiaDatabase::LENGTH_KEY), - $this->attribute('type', UtopiaDatabase::VAR_STRING, size: 16), - $this->attribute('attributes', UtopiaDatabase::VAR_STRING, size: 256, array: true), - $this->attribute('lengths', UtopiaDatabase::VAR_INTEGER, array: true), - $this->attribute('orders', UtopiaDatabase::VAR_STRING, size: 4, array: true), - $this->attribute('error', UtopiaDatabase::VAR_STRING, size: 2048), - ]); + $database->createCollection(new Collection( + id: 'databases', + attributes: [ + $this->attribute('name', ColumnType::String, required: true, size: 256), + $this->attribute('enabled', ColumnType::Boolean, default: true), + $this->attribute('search', ColumnType::String, size: 16384), + $this->attribute('originalId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('type', ColumnType::String, default: 'tablesdb', size: 128), + $this->attribute('database', ColumnType::String, size: 2000), + ], + )); + + $database->createCollection(new Collection( + id: 'attributes', + attributes: [ + $this->attribute('key', ColumnType::String, size: 256), + $this->attribute('databaseInternalId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('databaseId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('collectionInternalId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('collectionId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('type', ColumnType::String, size: 256), + $this->attribute('status', ColumnType::String, size: 64), + $this->attribute('size', ColumnType::Integer), + $this->attribute('required', ColumnType::Boolean, default: false), + $this->attribute('signed', ColumnType::Boolean, default: true), + $this->attribute('default', ColumnType::String, size: 16384), + $this->attribute('array', ColumnType::Boolean, default: false), + $this->attribute('format', ColumnType::String, size: 64), + $this->attribute('formatOptions', ColumnType::String, size: 16384, filters: ['json']), + $this->attribute('filters', ColumnType::String, size: 64, array: true), + $this->attribute('options', ColumnType::String, size: 16384, filters: ['json']), + $this->attribute('error', ColumnType::String, size: 2048), + ], + )); + + $database->createCollection(new Collection( + id: 'indexes', + attributes: [ + $this->attribute('key', ColumnType::String, size: 256), + $this->attribute('status', ColumnType::String, size: 64), + $this->attribute('databaseInternalId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('databaseId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('collectionInternalId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('collectionId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('type', ColumnType::String, size: 16), + $this->attribute('attributes', ColumnType::String, size: 256, array: true), + $this->attribute('lengths', ColumnType::Integer, array: true), + $this->attribute('orders', ColumnType::String, size: 4, array: true), + $this->attribute('error', ColumnType::String, size: 2048), + ], + )); return $database; } @@ -296,7 +312,7 @@ private function projectDatabase(): UtopiaDatabase */ private function attributeArray( string $id, - string $type, + ColumnType $type, bool $required = false, mixed $default = null, int $size = 0, @@ -305,7 +321,7 @@ private function attributeArray( ): array { return [ '$id' => $id, - 'type' => $type, + 'type' => $type->value, 'size' => $size, 'required' => $required, 'default' => $default, @@ -320,14 +336,22 @@ private function attributeArray( */ private function attribute( string $id, - string $type, + ColumnType $type, bool $required = false, mixed $default = null, int $size = 0, bool $array = false, array $filters = [], - ): UtopiaDocument { - return new UtopiaDocument($this->attributeArray($id, $type, $required, $default, $size, $array, $filters)); + ): UtopiaAttribute { + return new UtopiaAttribute( + key: $id, + type: $type, + size: $size, + required: $required, + default: $default, + array: $array, + filters: $filters, + ); } /** diff --git a/tests/Migration/Unit/Destinations/ProvisioningOwnerTest.php b/tests/Migration/Unit/Destinations/ProvisioningOwnerTest.php new file mode 100644 index 00000000..54100d30 --- /dev/null +++ b/tests/Migration/Unit/Destinations/ProvisioningOwnerTest.php @@ -0,0 +1,36 @@ + */ + public static function emptyIdentifiers(): array + { + return [ + 'migration identifier' => ['', 'attempt'], + 'attempt identifier' => ['migration', ''], + ]; + } + + #[DataProvider('emptyIdentifiers')] + public function testIdentifiersMustBeNonEmpty(string $migrationId, string $attemptId): void + { + $this->expectException(\InvalidArgumentException::class); + + new ProvisioningOwner($migrationId, $attemptId); + } + + public function testEqualityRequiresTheExactPair(): void + { + $owner = new ProvisioningOwner('migration', 'attempt'); + + $this->assertTrue($owner->equals(new ProvisioningOwner('migration', 'attempt'))); + $this->assertFalse($owner->equals(new ProvisioningOwner('migration-other', 'attempt'))); + $this->assertFalse($owner->equals(new ProvisioningOwner('migration', 'attempt-other'))); + } +} diff --git a/tests/Migration/Unit/MigrationCLITest.php b/tests/Migration/Unit/MigrationCLITest.php new file mode 100644 index 00000000..497098e0 --- /dev/null +++ b/tests/Migration/Unit/MigrationCLITest.php @@ -0,0 +1,345 @@ + $arguments */ + public function __construct( + array $arguments, + private readonly Database $database, + private readonly ?Source $injectedSource = null, + ) { + parent::__construct($arguments); + } + + #[Override] + public function getDatabase(string $type): Database + { + $this->database->getAuthorization()->disable(); + + return $this->database; + } + + #[Override] + public function getSource(): Source + { + return $this->injectedSource ?? parent::getSource(); + } + + #[Override] + public function drawFrame(): void + { + } + + #[Override] + protected function loadEnvironment(): void + { + } + + /** @return list<\Utopia\Migration\Exception> */ + public function getErrors(): array + { + return $this->destination->getErrors(); + } +} + +final class TransactionalMemoryAdapter extends MemoryAdapter +{ + /** @return array */ + #[Override] + public function capabilities(): array + { + return [...parent::capabilities(), Capability::TransactionRetries]; + } +} + +#[BackupGlobals(true)] +final class MigrationCLITest extends TestCase +{ + public function testHelpExplainsExplicitMigrationRecoveryAttestation(): void + { + $this->assertStringContainsString('--recover-migration-id=', \MigrationCLI::getHelp()); + $this->assertStringContainsString('--recover-migration-attempt-id=', \MigrationCLI::getHelp()); + $this->assertStringNotContainsString('--recover-provisioning', \MigrationCLI::getHelp()); + $this->assertStringContainsString('--migration-id=', \MigrationCLI::getHelp()); + $this->assertStringContainsString('--migration-attempt-id=', \MigrationCLI::getHelp()); + $this->assertStringContainsString('reuse it for retries', \MigrationCLI::getHelp()); + $this->assertStringContainsString('fresh attempt', \MigrationCLI::getHelp()); + $this->assertStringContainsString('prior migration attempt is terminal', \MigrationCLI::getHelp()); + $this->assertStringContainsString('refused by default', \MigrationCLI::getHelp()); + $this->assertStringContainsString('Resource status alone never proves', \MigrationCLI::getHelp()); + } + + public function testIncompleteDatabaseRecoveryRequiresExactTerminalMigrationIdentifier(): void + { + $cases = [ + 'absent' => [[], false], + 'bare migration' => [['--recover-migration-id', '--recover-migration-attempt-id=attempt-terminal'], false], + 'bare attempt' => [['--recover-migration-id=migration-terminal', '--recover-migration-attempt-id'], false], + 'empty migration' => [['--recover-migration-id=', '--recover-migration-attempt-id=attempt-terminal'], false], + 'empty attempt' => [['--recover-migration-id=migration-terminal', '--recover-migration-attempt-id='], false], + 'migration only' => [['--recover-migration-id=migration-terminal'], false], + 'attempt only' => [['--recover-migration-attempt-id=attempt-terminal'], false], + 'migration mismatch' => [['--recover-migration-id=migration-other', '--recover-migration-attempt-id=attempt-terminal'], false], + 'attempt mismatch' => [['--recover-migration-id=migration-terminal', '--recover-migration-attempt-id=attempt-other'], false], + 'retired unsafe option' => [['--recover-provisioning'], false], + 'exact' => [[ + '--recover-migration-id=migration-terminal', + '--recover-migration-attempt-id=attempt-terminal', + ], true], + ]; + + foreach (['provisioning', 'failed'] as $status) { + foreach ($cases as [$recoveryArguments, $recover]) { + $database = $this->createProjectDatabase($status); + $arguments = [ + 'MigrationCLI.php', + '--migration-id=migration-current', + '--migration-attempt-id=attempt-current', + ...$recoveryArguments, + ]; + $cli = new TestMigrationCLI($arguments, $database); + + $destination = $cli->getDestination(); + $this->runTransfer($database, $destination); + + $created = $database->getAuthorization()->skip( + static fn (): Document => $database->getDocument('databases', 'database'), + ); + + if (! $recover) { + $this->assertNotSame([], $destination->getErrors()); + $this->assertSame($status, $created->getAttribute('status')); + $this->assertSame('migration-terminal', $created->getAttribute('migrationId')); + $this->assertSame('attempt-terminal', $created->getAttribute('migrationAttemptId')); + $this->assertTrue($database->getCollection('database_'.$created->getSequence())->isEmpty()); + continue; + } + + $this->assertSame([], $destination->getErrors()); + $this->assertSame('ready', $created->getAttribute('status')); + $this->assertSame('migration-current', $created->getAttribute('migrationId')); + $this->assertSame('attempt-current', $created->getAttribute('migrationAttemptId')); + $this->assertFalse($database->getCollection('database_'.$created->getSequence())->isEmpty()); + } + } + } + + public function testAppwriteDestinationRequiresMigrationIdentifier(): void + { + $cli = new TestMigrationCLI(['MigrationCLI.php'], $this->createProjectDatabase()); + + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('--migration-id is required for an Appwrite destination'); + + $cli->getDestination(); + } + + public function testAppwriteDestinationRequiresMigrationAttemptIdentifier(): void + { + $cli = new TestMigrationCLI( + ['MigrationCLI.php', '--migration-id=migration-current'], + $this->createProjectDatabase(), + ); + + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('--migration-attempt-id is required for an Appwrite destination'); + + $cli->getDestination(); + } + + public function testStartFinalizesSuccessfulAppwriteDatabase(): void + { + $database = $this->createProjectDatabase(status: null); + $source = $this->createSource(new DatabaseResource( + id: 'database', + name: 'Database', + type: 'tablesdb', + database: 'source-dsn', + databaseStatus: 'ready', + )); + $cli = new TestMigrationCLI( + [ + 'MigrationCLI.php', + '--migration-id=migration-current', + '--migration-attempt-id=attempt-current', + ], + $database, + $source, + ); + + $cli->start(); + + $created = $database->getDocument('databases', 'database'); + $this->assertSame([], $cli->getErrors()); + $this->assertSame('ready', $created->getAttribute('status')); + $this->assertSame('migration-current', $created->getAttribute('migrationId')); + $this->assertSame('attempt-current', $created->getAttribute('migrationAttemptId')); + $this->assertFalse($database->getCollection('database_'.$created->getSequence())->isEmpty()); + } + + public function testStartFinalizesSuccessfulResourcesWhenDestinationHasErrors(): void + { + $database = $this->createProjectDatabase(status: null); + $valid = new DatabaseResource( + id: 'database', + name: 'Database', + type: 'tablesdb', + database: 'source-dsn', + databaseStatus: 'ready', + ); + $invalid = new DatabaseResource( + id: 'invalid id', + name: 'Invalid database', + type: 'tablesdb', + database: 'source-dsn', + databaseStatus: 'ready', + ); + $cli = new TestMigrationCLI( + [ + 'MigrationCLI.php', + '--migration-id=migration-current', + '--migration-attempt-id=attempt-current', + ], + $database, + $this->createSource($valid, $invalid), + ); + + $cli->start(); + + $created = $database->getDocument('databases', 'database'); + $errors = $cli->getErrors(); + $this->assertCount(1, $errors); + $this->assertSame('invalid id', $errors[0]->getResourceId()); + $this->assertSame(Transfer::GROUP_DATABASES, $errors[0]->getResourceGroup()); + $this->assertSame(Resource::STATUS_SUCCESS, $valid->getStatus()); + $this->assertSame(Resource::STATUS_ERROR, $invalid->getStatus()); + $this->assertSame('ready', $created->getAttribute('status')); + $this->assertSame('migration-current', $created->getAttribute('migrationId')); + $this->assertSame('attempt-current', $created->getAttribute('migrationAttemptId')); + $this->assertFalse($database->getCollection('database_'.$created->getSequence())->isEmpty()); + } + + private function createProjectDatabase(?string $status = 'provisioning'): Database + { + $database = new Database(new TransactionalMemoryAdapter(), new Cache(new MemoryCache())); + $database + ->setDatabase('appwrite') + ->setNamespace('_project'); + $database->create(); + $database->createCollection(new Collection( + id: 'databases', + attributes: [ + new Attribute(key: 'name', type: ColumnType::String, size: 256, required: true), + new Attribute(key: 'enabled', type: ColumnType::Boolean, default: true), + new Attribute(key: 'search', type: ColumnType::String, size: 16384), + new Attribute(key: 'originalId', type: ColumnType::String, size: Database::LENGTH_KEY), + new Attribute(key: 'type', type: ColumnType::String, size: 128, default: 'tablesdb'), + new Attribute(key: 'database', type: ColumnType::String, size: 2000), + new Attribute(key: 'status', type: ColumnType::String, size: 16), + new Attribute(key: 'migrationId', type: ColumnType::String, size: Database::LENGTH_KEY), + new Attribute(key: 'migrationAttemptId', type: ColumnType::String, size: Database::LENGTH_KEY), + ], + )); + if ($status !== null) { + $database->getAuthorization()->skip( + static fn (): Document => $database->createDocument('databases', new Document([ + '$id' => 'database', + 'name' => 'Database', + 'enabled' => true, + 'search' => 'database Database', + 'originalId' => null, + 'type' => 'tablesdb', + 'database' => '', + 'status' => $status, + 'migrationId' => 'migration-terminal', + 'migrationAttemptId' => 'attempt-terminal', + ])), + ); + } + + return $database; + } + + private function createSource(DatabaseResource ...$resources): MockSource + { + $source = new class () extends MockSource { + /** @return list */ + #[Override] + public static function getSupportedResources(): array + { + return [Resource::TYPE_DATABASE]; + } + + #[Override] + public function supportsDatabaseStatus(): bool + { + return true; + } + }; + foreach ($resources as $resource) { + $source->pushMockResource($resource); + } + + return $source; + } + + private function runTransfer(Database $database, Destination $destination): void + { + $source = new class () extends MockSource { + #[Override] + public function supportsDatabaseStatus(): bool + { + return true; + } + }; + $source->pushMockResource(new DatabaseResource( + id: 'database', + name: 'Database', + type: 'tablesdb', + database: 'source-dsn', + databaseStatus: 'ready', + )); + + $transfer = new Transfer($source, $destination); + $database->getAuthorization()->skip( + static function () use ($destination, $transfer): void { + $transfer->run([Resource::TYPE_DATABASE], static function (): void { + }); + $destination->success(); + }, + ); + } + + protected function setUp(): void + { + parent::setUp(); + $_ENV['DESTINATION_PROVIDER'] = 'appwrite'; + $_ENV['DESTINATION_APPWRITE_TEST_PROJECT'] = 'destination-project'; + $_ENV['DESTINATION_APPWRITE_TEST_ENDPOINT'] = 'http://example.test/v1'; + $_ENV['DESTINATION_APPWRITE_TEST_KEY'] = 'test-key'; + $_ENV['DESTINATION_APPWRITE_TEST_PROJECT_INTERNAL_ID'] = '1'; + } +} diff --git a/tests/Migration/Unit/Resources/ColumnTest.php b/tests/Migration/Unit/Resources/ColumnTest.php index 045f155e..fc2d7967 100644 --- a/tests/Migration/Unit/Resources/ColumnTest.php +++ b/tests/Migration/Unit/Resources/ColumnTest.php @@ -106,4 +106,12 @@ public function testUnsizedAndUnknownDefinitionsResolveToZero(): void Column::resolve(['key' => 'unknown']), ); } + + public function testQueryLibBigIntegerAliasBecomesBigint(): void + { + $this->assertSame( + ['type' => Column::TYPE_BIG_INT, 'format' => '', 'size' => 0], + Column::resolve(['key' => 'total', 'type' => 'biginteger']), + ); + } } diff --git a/tests/bootstrap.php b/tests/bootstrap.php new file mode 100644 index 00000000..17c58c63 --- /dev/null +++ b/tests/bootstrap.php @@ -0,0 +1,7 @@ +