From a259e294c5c39ac80c68ef4e090c5ad80e73d2c7 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 14 Aug 2026 16:34:48 +1200 Subject: [PATCH 01/32] feat: adapt migration schema calls to query-lib value objects Published 2.0 still used Database::VAR_* and positional createAttribute/createIndex, which feat-query-lib removed. Rebase onto main and pass Attribute/Index/Relationship VOs plus ColumnType/IndexType so Appwrite can pin this branch as 2.0.0. --- bin/MigrationCLI.php | 36 +- composer.json | 15 +- composer.lock | 475 ++++++++++++++---- src/Migration/Destinations/Appwrite.php | 276 +++++----- .../Database/Columns/Relationship.php | 7 +- src/Migration/Sources/Appwrite.php | 6 +- .../Sources/Appwrite/Reader/Database.php | 15 +- src/Migration/Sources/CSV.php | 8 +- .../AppwriteDatabaseStatusTest.php | 37 +- .../Destinations/AppwriteIndexLengthsTest.php | 104 ++-- 10 files changed, 673 insertions(+), 306 deletions(-) diff --git a/bin/MigrationCLI.php b/bin/MigrationCLI.php index 96cd99dd..e6681148 100644 --- a/bin/MigrationCLI.php +++ b/bin/MigrationCLI.php @@ -19,6 +19,8 @@ use Utopia\Migration\Sources\NHost; use Utopia\Migration\Sources\Supabase; use Utopia\Migration\Transfer; +use Utopia\Query\Schema\ColumnType; +use Utopia\Query\Schema\IndexType; /** * Migrations CLI Tool @@ -38,7 +40,7 @@ class MigrationCLI 'attributes' => [ [ '$id' => 'databaseInternalId', - 'type' => Database::VAR_STRING, + 'type' => ColumnType::String->value, 'format' => '', 'size' => Database::LENGTH_KEY, 'signed' => true, @@ -49,7 +51,7 @@ class MigrationCLI ], [ '$id' => 'databaseId', - 'type' => Database::VAR_STRING, + 'type' => ColumnType::String->value, 'signed' => true, 'size' => Database::LENGTH_KEY, 'format' => '', @@ -60,7 +62,7 @@ class MigrationCLI ], [ '$id' => 'name', - 'type' => Database::VAR_STRING, + 'type' => ColumnType::String->value, 'size' => Database::LENGTH_KEY, 'required' => true, 'signed' => true, @@ -69,7 +71,7 @@ class MigrationCLI ], [ '$id' => 'enabled', - 'type' => Database::VAR_BOOLEAN, + 'type' => ColumnType::Boolean->value, 'signed' => true, 'size' => 0, 'format' => '', @@ -80,7 +82,7 @@ class MigrationCLI ], [ '$id' => 'documentSecurity', - 'type' => Database::VAR_BOOLEAN, + 'type' => ColumnType::Boolean->value, 'signed' => true, 'size' => 0, 'format' => '', @@ -91,7 +93,7 @@ class MigrationCLI ], [ '$id' => 'attributes', - 'type' => Database::VAR_STRING, + 'type' => ColumnType::String->value, 'size' => 1000000, 'required' => false, 'signed' => true, @@ -100,7 +102,7 @@ class MigrationCLI ], [ '$id' => 'indexes', - 'type' => Database::VAR_STRING, + 'type' => ColumnType::String->value, 'size' => 1000000, 'required' => false, 'signed' => true, @@ -109,7 +111,7 @@ class MigrationCLI ], [ '$id' => 'search', - 'type' => Database::VAR_STRING, + 'type' => ColumnType::String->value, 'format' => '', 'size' => 16384, 'signed' => true, @@ -122,31 +124,31 @@ class MigrationCLI 'indexes' => [ [ '$id' => '_fulltext_search', - 'type' => Database::INDEX_FULLTEXT, + 'type' => IndexType::Fulltext->value, 'attributes' => ['search'], 'lengths' => [], 'orders' => [], ], [ '$id' => '_key_name', - 'type' => Database::INDEX_KEY, + 'type' => IndexType::Key->value, 'attributes' => ['name'], 'lengths' => [Database::LENGTH_KEY], - 'orders' => [Database::ORDER_ASC], + 'orders' => ['ASC'], ], [ '$id' => '_key_enabled', - 'type' => Database::INDEX_KEY, + 'type' => IndexType::Key->value, 'attributes' => ['enabled'], 'lengths' => [], - 'orders' => [Database::ORDER_ASC], + 'orders' => ['ASC'], ], [ '$id' => '_key_documentSecurity', - 'type' => Database::INDEX_KEY, + 'type' => IndexType::Key->value, 'attributes' => ['documentSecurity'], 'lengths' => [], - 'orders' => [Database::ORDER_ASC], + 'orders' => ['ASC'], ], ], ]; @@ -280,7 +282,7 @@ function (mixed $value, Document $document, Database $database) { $attributeType = $attribute->getAttribute('type'); switch ($attributeType) { - case Database::VAR_RELATIONSHIP: + case ColumnType::Relationship->value: $options = $attribute->getAttribute('options'); foreach ($options as $key => $value) { $attribute->setAttribute($key, $value); @@ -288,7 +290,7 @@ function (mixed $value, Document $document, Database $database) { $attribute->removeAttribute('options'); break; - case Database::VAR_STRING: + case ColumnType::String->value: $filters = $attribute->getAttribute('filters', []); $attribute->setAttribute('encrypt', in_array('encrypt', $filters)); break; diff --git a/composer.json b/composer.json index 1f8625c4..78ad8ffb 100644 --- a/composer.json +++ b/composer.json @@ -10,7 +10,8 @@ "migration" ], "license": "MIT", - "minimum-stability": "stable", + "minimum-stability": "dev", + "prefer-stable": true, "autoload": { "psr-4": { "Utopia\\Migration\\": "src/Migration" @@ -32,7 +33,7 @@ "ext-curl": "*", "ext-openssl": "*", "appwrite/appwrite": "^27.0", - "utopia-php/database": "^7.0.0", + "utopia-php/database": "dev-feat-query-lib as 7.0.0", "utopia-php/storage": "4.*", "utopia-php/dsn": "0.2.*", "halaxa/json-machine": "^1.2" @@ -44,6 +45,16 @@ "laravel/pint": "1.*", "phpstan/phpstan": "1.*" }, + "repositories": [ + { + "type": "vcs", + "url": "https://github.com/utopia-php/database.git" + }, + { + "type": "vcs", + "url": "https://github.com/utopia-php/async.git" + } + ], "config": { "allow-plugins": { "php-http/discovery": true, diff --git a/composer.lock b/composer.lock index ee1336d9..1ca9e57b 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "fea66e8c192e5b3d1dc56c953be0c2eb", + "content-hash": "e20f678bfb397f89a6cfa0083988f539", "packages": [ { "name": "adhocore/jwt", @@ -986,6 +986,71 @@ }, "time": "2026-01-21T04:14:03+00:00" }, + { + "name": "opis/closure", + "version": "4.5.0", + "source": { + "type": "git", + "url": "https://github.com/opis/closure.git", + "reference": "b97e42b95bb72d87507f5e2d137ceb239aea8d6b" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/opis/closure/zipball/b97e42b95bb72d87507f5e2d137ceb239aea8d6b", + "reference": "b97e42b95bb72d87507f5e2d137ceb239aea8d6b", + "shasum": "" + }, + "require": { + "php": "^8.0" + }, + "require-dev": { + "phpunit/phpunit": "^9.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "4.x-dev" + } + }, + "autoload": { + "files": [ + "src/functions.php" + ], + "psr-4": { + "Opis\\Closure\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Marius Sarca", + "email": "marius.sarca@gmail.com" + }, + { + "name": "Sorin Sarca", + "email": "sarca_sorin@hotmail.com" + } + ], + "description": "A library that can be used to serialize closures (anonymous functions) and arbitrary data.", + "homepage": "https://opis.io/closure", + "keywords": [ + "anonymous classes", + "anonymous functions", + "closure", + "function", + "serializable", + "serialization", + "serialize" + ], + "support": { + "issues": "https://github.com/opis/closure/issues", + "source": "https://github.com/opis/closure/tree/4.5.0" + }, + "time": "2026-03-05T13:32:42+00:00" + }, { "name": "php-http/discovery", "version": "1.20.0", @@ -1555,16 +1620,16 @@ }, { "name": "symfony/http-client", - "version": "v7.4.14", + "version": "v7.4.16", "source": { "type": "git", "url": "https://github.com/symfony/http-client.git", - "reference": "f6bc6b5a54ff5afac4725cacec9bf2f52eb15920" + "reference": "c513ed0ba5d1784a6b55fc84190dbe4451b12f41" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client/zipball/f6bc6b5a54ff5afac4725cacec9bf2f52eb15920", - "reference": "f6bc6b5a54ff5afac4725cacec9bf2f52eb15920", + "url": "https://api.github.com/repos/symfony/http-client/zipball/c513ed0ba5d1784a6b55fc84190dbe4451b12f41", + "reference": "c513ed0ba5d1784a6b55fc84190dbe4451b12f41", "shasum": "" }, "require": { @@ -1632,7 +1697,7 @@ "http" ], "support": { - "source": "https://github.com/symfony/http-client/tree/v7.4.14" + "source": "https://github.com/symfony/http-client/tree/v7.4.16" }, "funding": [ { @@ -1652,7 +1717,7 @@ "type": "tidelift" } ], - "time": "2026-06-16T11:50:14+00:00" + "time": "2026-07-29T16:20:51+00:00" }, { "name": "symfony/http-client-contracts", @@ -1903,16 +1968,16 @@ }, { "name": "symfony/polyfill-php83", - "version": "v1.38.2", + "version": "v1.41.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-php83.git", - "reference": "796a26abb75ce49f3a84433cd81bf1009d73d5f8" + "reference": "5ea99087fb99c273a9b9236ed4c31e78b16103c6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-php83/zipball/796a26abb75ce49f3a84433cd81bf1009d73d5f8", - "reference": "796a26abb75ce49f3a84433cd81bf1009d73d5f8", + "url": "https://api.github.com/repos/symfony/polyfill-php83/zipball/5ea99087fb99c273a9b9236ed4c31e78b16103c6", + "reference": "5ea99087fb99c273a9b9236ed4c31e78b16103c6", "shasum": "" }, "require": { @@ -1959,7 +2024,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-php83/tree/v1.38.2" + "source": "https://github.com/symfony/polyfill-php83/tree/v1.41.0" }, "funding": [ { @@ -1979,20 +2044,20 @@ "type": "tidelift" } ], - "time": "2026-05-27T06:51:48+00:00" + "time": "2026-07-01T12:47:55+00:00" }, { "name": "symfony/polyfill-php85", - "version": "v1.38.1", + "version": "v1.41.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-php85.git", - "reference": "ba2ba04f3352cfa2dcbbcb90aee13ed967f505b1" + "reference": "255fab485aaa1006ed411040c42aecd7b5302d7a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-php85/zipball/ba2ba04f3352cfa2dcbbcb90aee13ed967f505b1", - "reference": "ba2ba04f3352cfa2dcbbcb90aee13ed967f505b1", + "url": "https://api.github.com/repos/symfony/polyfill-php85/zipball/255fab485aaa1006ed411040c42aecd7b5302d7a", + "reference": "255fab485aaa1006ed411040c42aecd7b5302d7a", "shasum": "" }, "require": { @@ -2039,7 +2104,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-php85/tree/v1.38.1" + "source": "https://github.com/symfony/polyfill-php85/tree/v1.41.0" }, "funding": [ { @@ -2059,7 +2124,7 @@ "type": "tidelift" } ], - "time": "2026-05-26T02:25:22+00:00" + "time": "2026-07-01T12:47:55+00:00" }, { "name": "symfony/service-contracts", @@ -2200,35 +2265,152 @@ }, "time": "2025-06-29T15:42:06+00:00" }, + { + "name": "utopia-php/async", + "version": "0.1.1", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/async.git", + "reference": "3ee4fc3d505113d0d6050f35f5bf85e706866a22" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/async/zipball/3ee4fc3d505113d0d6050f35f5bf85e706866a22", + "reference": "3ee4fc3d505113d0d6050f35f5bf85e706866a22", + "shasum": "" + }, + "require": { + "opis/closure": "4.*", + "php": ">=8.1" + }, + "require-dev": { + "amphp/amp": "3.*", + "amphp/parallel": "2.*", + "amphp/process": "^2.0", + "laravel/pint": "1.*", + "phpstan/phpstan": "2.*", + "phpunit/phpunit": "11.5.45", + "react/child-process": "0.*", + "react/event-loop": "1.*", + "swoole/ide-helper": "*" + }, + "suggest": { + "amphp/amp": "Required for Amp promise adapter", + "amphp/parallel": "Required for Amp parallel adapter", + "ext-ev": "Required for ReactPHP event loop (recommended for best performance)", + "ext-parallel": "Required for parallel adapter (requires PHP ZTS build)", + "ext-sockets": "Required for Swoole Process adapter", + "ext-swoole": "Required for Swoole Thread and Process adapters (recommended for best performance)", + "react/child-process": "Required for ReactPHP parallel adapter", + "react/event-loop": "Required for ReactPHP promise and parallel adapters" + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Async\\": "src/" + } + }, + "autoload-dev": { + "psr-4": { + "Utopia\\Tests\\": "tests/" + } + }, + "scripts": { + "test-unit": [ + "vendor/bin/phpunit tests/Unit --exclude-group no-swoole" + ], + "test-promise-sync": [ + "vendor/bin/phpunit tests/E2e/Promise/SyncTest.php" + ], + "test-promise-swoole": [ + "vendor/bin/phpunit tests/E2e/Promise/Swoole" + ], + "test-promise-amp": [ + "vendor/bin/phpunit tests/E2e/Promise/Amp" + ], + "test-promise-react": [ + "vendor/bin/phpunit tests/E2e/Promise/React" + ], + "test-parallel-sync": [ + "vendor/bin/phpunit tests/E2e/Parallel/Sync" + ], + "test-parallel-swoole-thread": [ + "vendor/bin/phpunit tests/E2e/Parallel/Swoole/ThreadTest.php" + ], + "test-parallel-swoole-process": [ + "vendor/bin/phpunit tests/E2e/Parallel/Swoole/ProcessTest.php" + ], + "test-parallel-amp": [ + "vendor/bin/phpunit tests/E2e/Parallel/Amp" + ], + "test-parallel-react": [ + "vendor/bin/phpunit tests/E2e/Parallel/React" + ], + "test-parallel-ext": [ + "php -n -d extension=parallel.so -d extension=sockets.so vendor/bin/phpunit tests/E2e/Parallel/Parallel" + ], + "test-e2e": [ + "vendor/bin/phpunit tests/E2e --exclude-group ext-parallel" + ], + "test": [ + "@test-unit", + "@test-e2e", + "@test-parallel-ext" + ], + "lint": [ + "vendor/bin/pint" + ], + "format": [ + "php -d memory_limit=4G vendor/bin/pint" + ], + "check": [ + "vendor/bin/phpstan analyse src tests --level=max --memory-limit=4G" + ] + }, + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Appwrite Team", + "email": "team@appwrite.io" + } + ], + "description": "High-performance concurrent + parallel library with Promise and Parallel execution support for PHP.", + "support": { + "source": "https://github.com/utopia-php/async/tree/0.1.1", + "issues": "https://github.com/utopia-php/async/issues" + }, + "time": "2026-06-08T05:10:34+00:00" + }, { "name": "utopia-php/cache", - "version": "4.0.0", + "version": "4.0.2", "source": { "type": "git", "url": "https://github.com/utopia-php/cache.git", - "reference": "2ab50440114a7699b8a017ee2ecd8bcd9d54ac5e" + "reference": "92e02dab63606234b993b841ebf4c58845dd4620" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/cache/zipball/2ab50440114a7699b8a017ee2ecd8bcd9d54ac5e", - "reference": "2ab50440114a7699b8a017ee2ecd8bcd9d54ac5e", + "url": "https://api.github.com/repos/utopia-php/cache/zipball/92e02dab63606234b993b841ebf4c58845dd4620", + "reference": "92e02dab63606234b993b841ebf4c58845dd4620", "shasum": "" }, "require": { "ext-json": "*", - "ext-memcached": "*", - "ext-redis": "*", - "php": ">=8.3", - "utopia-php/circuit-breaker": "0.3.*", - "utopia-php/pools": "2.*", - "utopia-php/telemetry": "*" + "php": ">=8.4", + "utopia-php/circuit-breaker": "^0.3", + "utopia-php/pools": "^2.0", + "utopia-php/telemetry": "^0.4" }, "require-dev": { - "laravel/pint": "1.2.*", - "phpstan/phpstan": "^1.12", - "phpunit/phpunit": "^9.3", - "swoole/ide-helper": "^6.0", - "vimeo/psalm": "4.13.1" + "swoole/ide-helper": "^6.0" + }, + "suggest": { + "ext-memcached": "Required for the Memcached and Hazelcast adapters.", + "ext-redis": "Required for the Redis, RedisCluster and Sharding adapters.", + "ext-swoole": "Required for the multiplexing Redis adapter (>=6.0)." }, "type": "library", "autoload": { @@ -2240,6 +2422,12 @@ "license": [ "MIT" ], + "authors": [ + { + "name": "Team Appwrite", + "email": "team@appwrite.io" + } + ], "description": "A simple cache library to manage application cache storing, loading and purging", "keywords": [ "cache", @@ -2250,32 +2438,29 @@ ], "support": { "issues": "https://github.com/utopia-php/cache/issues", - "source": "https://github.com/utopia-php/cache/tree/4.0.0" + "source": "https://github.com/utopia-php/cache/tree/4.0.2" }, - "time": "2026-07-31T13:04:45+00:00" + "time": "2026-08-12T07:48:59+00:00" }, { "name": "utopia-php/circuit-breaker", - "version": "0.3.1", + "version": "0.3.2", "source": { "type": "git", "url": "https://github.com/utopia-php/circuit-breaker.git", - "reference": "db5d77f6c99ebce2ee81bd8ed4ae8f41bd2b0828" + "reference": "5fbc3802471b0d1b4260bd9f5544514e6929b481" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/circuit-breaker/zipball/db5d77f6c99ebce2ee81bd8ed4ae8f41bd2b0828", - "reference": "db5d77f6c99ebce2ee81bd8ed4ae8f41bd2b0828", + "url": "https://api.github.com/repos/utopia-php/circuit-breaker/zipball/5fbc3802471b0d1b4260bd9f5544514e6929b481", + "reference": "5fbc3802471b0d1b4260bd9f5544514e6929b481", "shasum": "" }, "require": { "php": ">=8.2" }, "require-dev": { - "laravel/pint": "^1.29", - "phpstan/phpstan": "^2.1", - "phpunit/phpunit": "^10.0", - "utopia-php/telemetry": "^0.4" + "utopia-php/telemetry": "^0.4.6" }, "suggest": { "ext-opentelemetry": "Required by utopia-php/telemetry when using OpenTelemetry metrics.", @@ -2287,7 +2472,7 @@ "type": "library", "autoload": { "psr-4": { - "Utopia\\CircuitBreaker\\": "src/CircuitBreaker" + "Utopia\\CircuitBreaker\\": "src/CircuitBreaker/" } }, "notification-url": "https://packagist.org/downloads/", @@ -2312,9 +2497,9 @@ ], "support": { "issues": "https://github.com/utopia-php/circuit-breaker/issues", - "source": "https://github.com/utopia-php/circuit-breaker/tree/0.3.1" + "source": "https://github.com/utopia-php/circuit-breaker/tree/0.3.2" }, - "time": "2026-05-29T12:12:23+00:00" + "time": "2026-08-05T18:07:20+00:00" }, { "name": "utopia-php/client", @@ -2423,16 +2608,16 @@ }, { "name": "utopia-php/database", - "version": "7.0.0", + "version": "dev-feat-query-lib", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "40185b100f92c402a189d6f4f0962c63bbe5726b" + "reference": "e593b78c587e1f1d66b644979f5c4ff11dffcd31" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/40185b100f92c402a189d6f4f0962c63bbe5726b", - "reference": "40185b100f92c402a189d6f4f0962c63bbe5726b", + "url": "https://api.github.com/repos/utopia-php/database/zipball/e593b78c587e1f1d66b644979f5c4ff11dffcd31", + "reference": "e593b78c587e1f1d66b644979f5c4ff11dffcd31", "shasum": "" }, "require": { @@ -2441,29 +2626,67 @@ "ext-pdo": "*", "ext-redis": "*", "php": ">=8.5", - "utopia-php/cache": "^4.0.0", + "utopia-php/async": "0.1.*", + "utopia-php/cache": "4.*", "utopia-php/console": "0.1.*", "utopia-php/mongo": "1.*", "utopia-php/pools": "2.*", - "utopia-php/validators": "0.3.*" + "utopia-php/query": "0.4.*", + "utopia-php/validators": "0.4.*" }, "require-dev": { + "brianium/paratest": "7.20.*", "fakerphp/faker": "1.23.*", "laravel/pint": "*", - "pcov/clobber": "2.*", - "phpstan/phpstan": "1.*", - "phpunit/phpunit": "9.*", + "phpstan/phpstan": "2.1.*", + "phpunit/phpunit": "12.5.*", "rregeer/phpunit-coverage-check": "0.3.*", "swoole/ide-helper": "5.1.3", "utopia-php/cli": "0.22.*" }, + "suggest": { + "ext-pdo": "Needed to support MariaDB, MySQL or SQLite Database Adapter", + "ext-redis": "Needed to support Redis Cache Adapter", + "mongodb/mongodb": "Needed to support MongoDB Database Adapter" + }, "type": "library", "autoload": { "psr-4": { "Utopia\\Database\\": "src/Database" } }, - "notification-url": "https://packagist.org/downloads/", + "autoload-dev": { + "psr-4": { + "Tests\\E2E\\": "tests/e2e", + "Tests\\Unit\\": "tests/unit" + } + }, + "scripts": { + "build": [ + "Composer\\Config::disableProcessTimeout", + "docker compose build" + ], + "start": [ + "Composer\\Config::disableProcessTimeout", + "docker compose up -d" + ], + "test": [ + "Composer\\Config::disableProcessTimeout", + "docker compose exec tests vendor/bin/paratest --configuration phpunit.xml --functional --processes 4" + ], + "lint": [ + "php -d memory_limit=2G ./vendor/bin/pint --test" + ], + "format": [ + "php -d memory_limit=2G ./vendor/bin/pint" + ], + "check": [ + "./vendor/bin/phpstan analyse --memory-limit 2G" + ], + "coverage": [ + "./vendor/bin/coverage-check ./tmp/clover.xml 90" + ] + }, "license": [ "MIT" ], @@ -2476,10 +2699,10 @@ "utopia" ], "support": { - "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/7.0.0" + "source": "https://github.com/utopia-php/database/tree/feat-query-lib", + "issues": "https://github.com/utopia-php/database/issues" }, - "time": "2026-07-31T13:33:10+00:00" + "time": "2026-08-14T02:54:13+00:00" }, { "name": "utopia-php/dsn", @@ -2530,16 +2753,16 @@ }, { "name": "utopia-php/mongo", - "version": "1.4.0", + "version": "1.5.3", "source": { "type": "git", "url": "https://github.com/utopia-php/mongo.git", - "reference": "78818fd295f2829aaad5b74c9094e8c6f603550d" + "reference": "be29ee2d84b9f7efdfc6fea5b4b2d4bf95ff5ec4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/mongo/zipball/78818fd295f2829aaad5b74c9094e8c6f603550d", - "reference": "78818fd295f2829aaad5b74c9094e8c6f603550d", + "url": "https://api.github.com/repos/utopia-php/mongo/zipball/be29ee2d84b9f7efdfc6fea5b4b2d4bf95ff5ec4", + "reference": "be29ee2d84b9f7efdfc6fea5b4b2d4bf95ff5ec4", "shasum": "" }, "require": { @@ -2585,27 +2808,27 @@ ], "support": { "issues": "https://github.com/utopia-php/mongo/issues", - "source": "https://github.com/utopia-php/mongo/tree/1.4.0" + "source": "https://github.com/utopia-php/mongo/tree/1.5.3" }, - "time": "2026-07-30T05:24:58+00:00" + "time": "2026-08-13T01:55:11+00:00" }, { "name": "utopia-php/pools", - "version": "2.0.1", + "version": "2.0.2", "source": { "type": "git", "url": "https://github.com/utopia-php/pools.git", - "reference": "48905dac1b8f8050c2e07bdda32a018ca33982fc" + "reference": "86d43fcacd125232c0743e8c22695faf97285cf3" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/pools/zipball/48905dac1b8f8050c2e07bdda32a018ca33982fc", - "reference": "48905dac1b8f8050c2e07bdda32a018ca33982fc", + "url": "https://api.github.com/repos/utopia-php/pools/zipball/86d43fcacd125232c0743e8c22695faf97285cf3", + "reference": "86d43fcacd125232c0743e8c22695faf97285cf3", "shasum": "" }, "require": { "php": ">=8.4", - "utopia-php/telemetry": "^0.4" + "utopia-php/telemetry": "^0.4.6" }, "require-dev": { "swoole/ide-helper": "6.*" @@ -2641,9 +2864,9 @@ ], "support": { "issues": "https://github.com/utopia-php/pools/issues", - "source": "https://github.com/utopia-php/pools/tree/2.0.1" + "source": "https://github.com/utopia-php/pools/tree/2.0.2" }, - "time": "2026-07-31T12:19:18+00:00" + "time": "2026-08-05T18:07:20+00:00" }, { "name": "utopia-php/psr7", @@ -2691,6 +2914,55 @@ }, "time": "2026-07-06T12:40:23+00:00" }, + { + "name": "utopia-php/query", + "version": "0.4.0", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/query.git", + "reference": "c334515035a2ab0aa49176eeca96de3e1c096e22" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/query/zipball/c334515035a2ab0aa49176eeca96de3e1c096e22", + "reference": "c334515035a2ab0aa49176eeca96de3e1c096e22", + "shasum": "" + }, + "require": { + "php": ">=8.4" + }, + "require-dev": { + "brianium/paratest": "*", + "laravel/pint": "*", + "mongodb/mongodb": "^2.0", + "phpstan/phpstan": "*", + "phpunit/phpcov": "*", + "phpunit/phpunit": "^12.0" + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Query\\": "src/Query" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "description": "A simple library providing a query abstraction for filtering, ordering, and pagination", + "keywords": [ + "framework", + "php", + "query", + "upf", + "utopia" + ], + "support": { + "issues": "https://github.com/utopia-php/query/issues", + "source": "https://github.com/utopia-php/query/tree/0.4.0" + }, + "time": "2026-08-14T01:42:36+00:00" + }, { "name": "utopia-php/span", "version": "4.1.0", @@ -2733,16 +3005,16 @@ }, { "name": "utopia-php/storage", - "version": "4.0.1", + "version": "4.0.3", "source": { "type": "git", "url": "https://github.com/utopia-php/storage.git", - "reference": "9684da5ab161ae9375d4f47541ef825451d69f2a" + "reference": "f8b122e753c01f30f774d390d7a36f030dfaef46" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/storage/zipball/9684da5ab161ae9375d4f47541ef825451d69f2a", - "reference": "9684da5ab161ae9375d4f47541ef825451d69f2a", + "url": "https://api.github.com/repos/utopia-php/storage/zipball/f8b122e753c01f30f774d390d7a36f030dfaef46", + "reference": "f8b122e753c01f30f774d390d7a36f030dfaef46", "shasum": "" }, "require": { @@ -2754,8 +3026,8 @@ "psr/http-message": "^1.1 || ^2.0", "utopia-php/client": "0.2.* || 0.3.*", "utopia-php/psr7": "0.2.*", - "utopia-php/telemetry": "^0.4", - "utopia-php/validators": "0.3.*" + "utopia-php/telemetry": "^0.4.6", + "utopia-php/validators": "^0.4" }, "type": "library", "autoload": { @@ -2777,22 +3049,22 @@ ], "support": { "issues": "https://github.com/utopia-php/storage/issues", - "source": "https://github.com/utopia-php/storage/tree/4.0.1" + "source": "https://github.com/utopia-php/storage/tree/4.0.3" }, - "time": "2026-07-31T09:20:57+00:00" + "time": "2026-08-10T04:51:03+00:00" }, { "name": "utopia-php/telemetry", - "version": "0.4.5", + "version": "0.4.6", "source": { "type": "git", "url": "https://github.com/utopia-php/telemetry.git", - "reference": "139943bffcd4f6dd8fb9ed247f946a1d151b006a" + "reference": "f96778a01792c32df0876fe1f38a79b9588445d8" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/telemetry/zipball/139943bffcd4f6dd8fb9ed247f946a1d151b006a", - "reference": "139943bffcd4f6dd8fb9ed247f946a1d151b006a", + "url": "https://api.github.com/repos/utopia-php/telemetry/zipball/f96778a01792c32df0876fe1f38a79b9588445d8", + "reference": "f96778a01792c32df0876fe1f38a79b9588445d8", "shasum": "" }, "require": { @@ -2828,22 +3100,22 @@ ], "support": { "issues": "https://github.com/utopia-php/telemetry/issues", - "source": "https://github.com/utopia-php/telemetry/tree/0.4.5" + "source": "https://github.com/utopia-php/telemetry/tree/0.4.6" }, - "time": "2026-07-08T11:07:25+00:00" + "time": "2026-08-05T17:56:48+00:00" }, { "name": "utopia-php/validators", - "version": "0.3.1", + "version": "0.4.2", "source": { "type": "git", "url": "https://github.com/utopia-php/validators.git", - "reference": "d9c2269ebd2596a09681ccd2fd133eeedfcdf9ba" + "reference": "a5b7b78b789e5af0a30f96da8355bbf48fb56699" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/validators/zipball/d9c2269ebd2596a09681ccd2fd133eeedfcdf9ba", - "reference": "d9c2269ebd2596a09681ccd2fd133eeedfcdf9ba", + "url": "https://api.github.com/repos/utopia-php/validators/zipball/a5b7b78b789e5af0a30f96da8355bbf48fb56699", + "reference": "a5b7b78b789e5af0a30f96da8355bbf48fb56699", "shasum": "" }, "require": { @@ -2868,9 +3140,9 @@ ], "support": { "issues": "https://github.com/utopia-php/validators/issues", - "source": "https://github.com/utopia-php/validators/tree/0.3.1" + "source": "https://github.com/utopia-php/validators/tree/0.4.2" }, - "time": "2026-07-14T11:55:48+00:00" + "time": "2026-08-11T07:43:49+00:00" } ], "packages-dev": [ @@ -5148,10 +5420,19 @@ "time": "2026-07-06T19:11:50+00:00" } ], - "aliases": [], - "minimum-stability": "stable", - "stability-flags": {}, - "prefer-stable": false, + "aliases": [ + { + "package": "utopia-php/database", + "version": "dev-feat-query-lib", + "alias": "7.0.0", + "alias_normalized": "7.0.0.0" + } + ], + "minimum-stability": "dev", + "stability-flags": { + "utopia-php/database": 20 + }, + "prefer-stable": true, "prefer-lowest": false, "platform": { "php": ">=8.5", diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index a8eb59d0..f2ed2873 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -24,6 +24,9 @@ use Appwrite\Services\Teams; use Appwrite\Services\Users; use Override; +use Utopia\Database\Adapter\Feature\Spatial; +use Utopia\Database\Attribute as UtopiaAttribute; +use Utopia\Database\Capability; use Utopia\Database\Database as UtopiaDatabase; use Utopia\Database\DateTime; use Utopia\Database\Document as UtopiaDocument; @@ -35,7 +38,11 @@ use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Permission; use Utopia\Database\Helpers\Role; +use Utopia\Database\Index as UtopiaIndex; use Utopia\Database\Query; +use Utopia\Database\Relationship as UtopiaRelationship; +use Utopia\Database\RelationSide; +use Utopia\Database\RelationType; use Utopia\Database\Validator\Index as IndexValidator; use Utopia\Database\Validator\Structure; use Utopia\Database\Validator\UID; @@ -79,6 +86,9 @@ use Utopia\Migration\Resources\Storage\File; use Utopia\Migration\Resources\Templates\EmailTemplate; use Utopia\Migration\Transfer; +use Utopia\Query\Schema\ColumnType; +use Utopia\Query\Schema\ForeignKeyAction; +use Utopia\Query\Schema\IndexType; class Appwrite extends Destination { @@ -733,20 +743,10 @@ protected function createDatabase(Database $resource): bool try { $structure = $this->collectionStructureFor($resource); - $columns = \array_map( - fn ($attr) => new UtopiaDocument($attr), - $structure['attributes'] - ); - - $indexes = \array_map( - fn ($index) => new UtopiaDocument($index), - $structure['indexes'] - ); - $this->dbForProject->createCollection( $this->databaseCollectionId($existing), - $columns, - $indexes + $this->schemaAttributes($structure['attributes'] ?? []), + $this->schemaIndexes($structure['indexes'] ?? []) ); } catch (\Throwable $e) { $this->markDatabaseFailed($resource->getId()); @@ -794,20 +794,10 @@ protected function createDatabase(Database $resource): bool try { $structure = $this->collectionStructureFor($resource); - $columns = \array_map( - fn ($attr) => new UtopiaDocument($attr), - $structure['attributes'] - ); - - $indexes = \array_map( - fn ($index) => new UtopiaDocument($index), - $structure['indexes'] - ); - $this->dbForProject->createCollection( $this->databaseCollectionId($database), - $columns, - $indexes + $this->schemaAttributes($structure['attributes'] ?? []), + $this->schemaIndexes($structure['indexes'] ?? []) ); } catch (\Throwable $e) { // The metadata document exists but the database isn't usable; mark it failed before propagating. @@ -955,32 +945,7 @@ protected function createField(Column|Attribute $resource): bool } // column will be matching attribute as well // column type will be matching attribute type as well - $type = match ($resource->getType()) { - Column::TYPE_DATETIME => UtopiaDatabase::VAR_DATETIME, - Column::TYPE_BOOLEAN => UtopiaDatabase::VAR_BOOLEAN, - Column::TYPE_INTEGER => UtopiaDatabase::VAR_INTEGER, - Column::TYPE_BIG_INT => UtopiaDatabase::VAR_BIGINT, - Column::TYPE_FLOAT => UtopiaDatabase::VAR_FLOAT, - Column::TYPE_RELATIONSHIP => UtopiaDatabase::VAR_RELATIONSHIP, - - Column::TYPE_STRING, - Column::TYPE_IP, - Column::TYPE_EMAIL, - Column::TYPE_URL, - Column::TYPE_ENUM => UtopiaDatabase::VAR_STRING, - - Column::TYPE_POINT => UtopiaDatabase::VAR_POINT, - Column::TYPE_LINE => UtopiaDatabase::VAR_LINESTRING, - Column::TYPE_POLYGON => UtopiaDatabase::VAR_POLYGON, - Column::TYPE_TEXT => UtopiaDatabase::VAR_TEXT, - Column::TYPE_VARCHAR => UtopiaDatabase::VAR_VARCHAR, - Column::TYPE_MEDIUMTEXT => UtopiaDatabase::VAR_MEDIUMTEXT, - Column::TYPE_LONGTEXT => UtopiaDatabase::VAR_LONGTEXT, - Column::TYPE_OBJECT => UtopiaDatabase::VAR_OBJECT, - Column::TYPE_VECTOR => UtopiaDatabase::VAR_VECTOR, - - default => throw new \Exception('Invalid resource type ' . $resource->getType(), Exception::CODE_VALIDATION), - }; + $type = $this->schemaColumnType($resource); $database = $this->dbForProject->getDocument( self::META_DATABASES, @@ -1015,7 +980,7 @@ protected function createField(Column|Attribute $resource): bool } if (!empty($resource->getFormat())) { - if (!Structure::hasFormat($resource->getFormat(), $type)) { + if (!Structure::hasFormat($resource->getFormat(), ColumnType::from($type))) { $resource->setStatus(Resource::STATUS_ERROR, "Format {$resource->getFormat()} not available for column type {$type}"); $this->addError(new Exception( resourceName: $resource->getName(), @@ -1049,8 +1014,9 @@ protected function createField(Column|Attribute $resource): bool return false; } - if ($type === UtopiaDatabase::VAR_RELATIONSHIP) { - $resource->getOptions()['side'] = UtopiaDatabase::RELATION_SIDE_PARENT; + $relatedTable = null; + if ($type === ColumnType::Relationship->value) { + $resource->getOptions()['side'] = RelationSide::Parent->value; $relatedTable = $this->dbForProject->getDocument( $this->databaseCollectionId($database), $resource->getOptions()['relatedCollection'] @@ -1073,7 +1039,7 @@ protected function createField(Column|Attribute $resource): bool $this->trackOrphanCandidate($database, $table, 'attributeKeys', $resource->getKey(), $dbForDatabases); - $isRelationship = $type === UtopiaDatabase::VAR_RELATIONSHIP; + $isRelationship = $type === ColumnType::Relationship->value; // Source emits both sides of a two-way; processing one side reconciles both. Partner skip. $twoWayPairKey = $this->twoWayPairKey($database, $table, $resource, $type); @@ -1178,11 +1144,11 @@ protected function createField(Column|Attribute $resource): bool $twoWayKey = null; - if ($type === UtopiaDatabase::VAR_RELATIONSHIP && $options['twoWay']) { + if ($type === ColumnType::Relationship->value && $options['twoWay'] && $relatedTable !== null) { $twoWayKey = $options['twoWayKey']; $options['relatedCollection'] = $table->getId(); $options['twoWayKey'] = $resource->getKey(); - $options['side'] = UtopiaDatabase::RELATION_SIDE_CHILD; + $options['side'] = RelationSide::Child->value; try { $twoWayAttribute = new UtopiaDocument([ @@ -1241,16 +1207,25 @@ protected function createField(Column|Attribute $resource): bool try { switch ($type) { - case UtopiaDatabase::VAR_RELATIONSHIP: + case ColumnType::Relationship->value: + if ($relatedTable === null) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: 'Related table not found', + ); + } if (!$dbForDatabases->createRelationship( - collection: $this->tableCollectionId($database, $table), - // @phpstan-ignore-next-line — $relatedTable is set when type is VAR_RELATIONSHIP. - relatedCollection: $this->tableCollectionId($database, $relatedTable), - type: $options['relationType'], - twoWay: $options['twoWay'], - id: $resource->getKey(), - twoWayKey: $options['twoWay'] ? $twoWayKey : $options['twoWayKey'] ?? null, - onDelete: $options['onDelete'], + new UtopiaRelationship( + collection: $this->tableCollectionId($database, $table), + relatedCollection: $this->tableCollectionId($database, $relatedTable), + type: RelationType::from($options['relationType']), + twoWay: $options['twoWay'], + key: $resource->getKey(), + twoWayKey: (string) ($options['twoWay'] ? $twoWayKey : $options['twoWayKey'] ?? ''), + onDelete: ForeignKeyAction::from($options['onDelete']), + ) )) { throw new Exception( resourceName: $resource->getName(), @@ -1263,16 +1238,18 @@ protected function createField(Column|Attribute $resource): bool default: if (!$dbForDatabases->createAttribute( $this->tableCollectionId($database, $table), - $resource->getKey(), - $type, - $resource->getSize(), - $resource->isRequired(), - $resource->getDefault(), - $resource->isSigned(), - $resource->isArray(), - $resource->getFormat(), - $resource->getFormatOptions(), - $resource->getFilters(), + new UtopiaAttribute( + key: $resource->getKey(), + type: ColumnType::from($type), + size: $resource->getSize(), + required: $resource->isRequired(), + default: $resource->getDefault(), + signed: $resource->isSigned(), + array: $resource->isArray(), + format: $resource->getFormat() !== '' ? $resource->getFormat() : null, + formatOptions: $resource->getFormatOptions(), + filters: $resource->getFilters(), + ), )) { throw new \Exception('Failed to create Column', Exception::CODE_INTERNAL); } @@ -1287,8 +1264,7 @@ protected function createField(Column|Attribute $resource): bool throw $e; } - if ($type === UtopiaDatabase::VAR_RELATIONSHIP && $options['twoWay']) { - // @phpstan-ignore-next-line — $relatedTable is set when type is VAR_RELATIONSHIP. + if ($type === ColumnType::Relationship->value && $options['twoWay'] && $relatedTable !== null) { $this->dbForProject->purgeCachedDocument($this->databaseCollectionId($database), $relatedTable->getId()); } @@ -1311,6 +1287,69 @@ private function collectionStructureFor(Database $resource): array return $this->collectionStructures[$resource->getType()] ?? $this->collectionStructure; } + private function schemaColumnType(Column|Attribute $resource): string + { + return match ($resource->getType()) { + Column::TYPE_DATETIME => ColumnType::Datetime->value, + Column::TYPE_BOOLEAN => ColumnType::Boolean->value, + Column::TYPE_INTEGER => ColumnType::Integer->value, + Column::TYPE_BIG_INT => ColumnType::BigInteger->value, + Column::TYPE_FLOAT => ColumnType::Double->value, + Column::TYPE_RELATIONSHIP => ColumnType::Relationship->value, + Column::TYPE_STRING, + Column::TYPE_IP, + Column::TYPE_EMAIL, + Column::TYPE_URL, + Column::TYPE_ENUM => ColumnType::String->value, + Column::TYPE_POINT => ColumnType::Point->value, + Column::TYPE_LINE => ColumnType::Linestring->value, + Column::TYPE_POLYGON => ColumnType::Polygon->value, + Column::TYPE_TEXT => ColumnType::Text->value, + Column::TYPE_VARCHAR => ColumnType::Varchar->value, + Column::TYPE_MEDIUMTEXT => ColumnType::MediumText->value, + Column::TYPE_LONGTEXT => ColumnType::LongText->value, + Column::TYPE_OBJECT => ColumnType::Object->value, + Column::TYPE_VECTOR => ColumnType::Vector->value, + default => throw new \Exception('Invalid resource type ' . $resource->getType(), Exception::CODE_VALIDATION), + }; + } + + /** + * @param array $attributes + * @return array + */ + private function schemaAttributes(array $attributes): array + { + return \array_map(function (mixed $attr): UtopiaAttribute { + if ($attr instanceof UtopiaAttribute) { + return $attr; + } + if ($attr instanceof UtopiaDocument) { + return UtopiaAttribute::fromDocument($attr); + } + + return UtopiaAttribute::fromArray(\is_array($attr) ? $attr : []); + }, $attributes); + } + + /** + * @param array $indexes + * @return array + */ + private function schemaIndexes(array $indexes): array + { + return \array_map(function (mixed $index): UtopiaIndex { + if ($index instanceof UtopiaIndex) { + return $index; + } + if ($index instanceof UtopiaDocument) { + return UtopiaIndex::fromDocument($index); + } + + return UtopiaIndex::fromArray(\is_array($index) ? $index : []); + }, $indexes); + } + /** * Type-specific metadata for an imported entity's collection document. VectorsDB collections * carry a required `dimension`; archives written before it was serialized have none, so a @@ -1342,7 +1381,7 @@ private function entityTypeMetadata(Table $resource): array private function syncVectorDimension(Column|Attribute $resource, string $type, UtopiaDocument $database, UtopiaDocument $table): void { if ( - $type !== UtopiaDatabase::VAR_VECTOR + $type !== ColumnType::Vector->value || $resource->getKey() !== self::VECTORSDB_EMBEDDINGS_KEY || $resource->getTable()->getDatabase()->getType() !== Resource::TYPE_DATABASE_VECTORSDB || !isset($this->collectionStructures[Resource::TYPE_DATABASE_VECTORSDB]) @@ -1461,7 +1500,7 @@ protected function createIndex(Index $resource): bool // Lengths hidden by default $lengths = []; - if ($dbForDatabases->getAdapter()->getSupportForAttributes()) { + if ($dbForDatabases->getAdapter()->supports(Capability::DefinedAttributes)) { $this->validateFieldsForIndexes($resource, $table, $lengths); } @@ -1487,24 +1526,27 @@ protected function createIndex(Index $resource): bool $tableColumns = $table->getAttribute('attributes', []); $tableIndexes = $table->getAttribute('indexes', []); + $adapter = $dbForDatabases->getAdapter(); $validator = new IndexValidator( $tableColumns, $tableIndexes, - $dbForDatabases->getAdapter()->getMaxIndexLength(), - $dbForDatabases->getAdapter()->getInternalIndexesKeys(), - $dbForDatabases->getAdapter()->getSupportForIndexArray(), - $dbForDatabases->getAdapter()->getSupportForSpatialIndexNull(), - $dbForDatabases->getAdapter()->getSupportForSpatialIndexOrder(), - $dbForDatabases->getAdapter()->getSupportForVectors(), - $dbForDatabases->getAdapter()->getSupportForAttributes(), - $dbForDatabases->getAdapter()->getSupportForMultipleFulltextIndexes(), - $dbForDatabases->getAdapter()->getSupportForIdenticalIndexes(), - $dbForDatabases->getAdapter()->getSupportForObjectIndexes(), - $dbForDatabases->getAdapter()->getSupportForTrigramIndex(), - $dbForDatabases->getAdapter()->getSupportForSpatialAttributes(), - $dbForDatabases->getAdapter()->getSupportForIndex(), - $dbForDatabases->getAdapter()->getSupportForUniqueIndex(), - $dbForDatabases->getAdapter()->getSupportForFulltextIndex() + $adapter->getMaxIndexLength(), + $adapter->getInternalIndexesKeys(), + $adapter->supports(Capability::IndexArray), + $adapter->supports(Capability::SpatialIndexNull), + $adapter->supports(Capability::SpatialIndexOrder), + $adapter->supports(Capability::Vectors), + $adapter->supports(Capability::DefinedAttributes), + $adapter->supports(Capability::MultipleFulltextIndexes), + $adapter->supports(Capability::IdenticalIndexes), + $adapter->supports(Capability::ObjectIndexes), + $adapter->supports(Capability::TrigramIndex), + $adapter instanceof Spatial, + $adapter->supports(Capability::Index), + $adapter->supports(Capability::UniqueIndex), + $adapter->supports(Capability::Fulltext), + $adapter->supports(Capability::TTLIndexes), + $adapter->supports(Capability::Objects), ); if (!$validator->isValid($index)) { @@ -1523,11 +1565,13 @@ protected function createIndex(Index $resource): bool try { $result = $dbForDatabases->createIndex( $this->tableCollectionId($database, $table), - $resource->getKey(), - $resource->getType(), - $resource->getColumns(), - $lengths, - $resource->getOrders() + new UtopiaIndex( + key: $resource->getKey(), + type: IndexType::from($resource->getType()), + attributes: $resource->getColumns(), + lengths: $lengths, + orders: $resource->getOrders(), + ), ); if (!$result) { @@ -1642,17 +1686,19 @@ protected function createRecord(Row $resource, bool $isLast): bool $resource->getTable()->getId(), ); // Strip row payload fields the table doesn't declare — guards against orphans surviving in source archives. - if ($dbForDatabases->getAdapter()->getSupportForAttributes()) { + if ($dbForDatabases->getAdapter()->supports(Capability::DefinedAttributes)) { foreach ($this->rowBuffer as $row) { foreach ($row as $key => $value) { if (\str_starts_with($key, '$')) { continue; } - /** @var \Utopia\Database\Document $attribute */ $found = false; foreach ($table->getAttribute('attributes', []) as $attribute) { - if ($attribute->getAttribute('key') == $key) { + $attrKey = $attribute instanceof UtopiaAttribute + ? $attribute->key + : $attribute->getAttribute('key'); + if ($attrKey == $key) { $found = true; break; } @@ -1710,7 +1756,7 @@ protected function createRecord(Row $resource, bool $isLast): bool return true; } - /** Relationships route through deleteRelationship since deleteAttribute throws for VAR_RELATIONSHIP. */ + /** Relationships route through deleteRelationship since deleteAttribute throws for relationship columns. */ private function dropAttributeForRecreate( UtopiaDocument $database, UtopiaDocument $table, @@ -1833,7 +1879,7 @@ private function updateRelationshipInPlace( $dbForDatabases->updateRelationship( collection: $this->tableCollectionId($database, $table), id: $resource->getKey(), - onDelete: (string) ($sourceOptions['onDelete'] ?? ''), + onDelete: ForeignKeyAction::from((string) ($sourceOptions['onDelete'] ?? ForeignKeyAction::Restrict->value)), ); } @@ -2021,7 +2067,7 @@ private function twoWayPairKey( Column|Attribute $resource, string $type, ): ?string { - if ($type !== UtopiaDatabase::VAR_RELATIONSHIP) { + if ($type !== ColumnType::Relationship->value) { return null; } $options = $resource->getOptions(); @@ -2201,7 +2247,7 @@ private function dropOrphanAttribute( $options = $attrDoc->getAttribute('options', []); $collectionId = $this->tableCollectionId($database, $table); - if ($type === UtopiaDatabase::VAR_RELATIONSHIP) { + if ($type === ColumnType::Relationship->value) { $this->bestEffort(fn () => $dbForDatabases->deleteRelationship($collectionId, $key)); } else { $this->bestEffort(fn () => $dbForDatabases->deleteAttribute($collectionId, $key)); @@ -2210,7 +2256,7 @@ private function dropOrphanAttribute( $this->dbForProject->purgeCachedDocument($this->databaseCollectionId($database), $table->getId()); $dbForDatabases->purgeCachedCollection($collectionId); - if ($type !== UtopiaDatabase::VAR_RELATIONSHIP) { + if ($type !== ColumnType::Relationship->value) { return; } $partner = $this->resolveTwoWayPartner($database, $options); @@ -3859,13 +3905,19 @@ private function validateFieldsForIndexes(Index $resource, UtopiaDocument $table $tableColumns = $table->getAttribute('attributes', []); $oldColumns = \array_map( - fn ($attr) => $attr->getArrayCopy(), + function ($attr) { + if ($attr instanceof UtopiaAttribute) { + return $attr->toDocument()->getArrayCopy(); + } + + return $attr->getArrayCopy(); + }, $tableColumns ); $oldColumns[] = [ 'key' => '$id', - 'type' => UtopiaDatabase::VAR_STRING, + 'type' => ColumnType::String->value, 'status' => 'available', 'required' => true, 'array' => false, @@ -3875,7 +3927,7 @@ private function validateFieldsForIndexes(Index $resource, UtopiaDocument $table $oldColumns[] = [ 'key' => '$createdAt', - 'type' => UtopiaDatabase::VAR_DATETIME, + 'type' => ColumnType::Datetime->value, 'status' => 'available', 'signed' => false, 'required' => false, @@ -3886,7 +3938,7 @@ private function validateFieldsForIndexes(Index $resource, UtopiaDocument $table $oldColumns[] = [ 'key' => '$updatedAt', - 'type' => UtopiaDatabase::VAR_DATETIME, + 'type' => ColumnType::Datetime->value, 'status' => 'available', 'signed' => false, 'required' => false, @@ -3915,7 +3967,7 @@ private function validateFieldsForIndexes(Index $resource, UtopiaDocument $table $columnType = $oldColumns[$columnIndex]['type']; $columnArray = $oldColumns[$columnIndex]['array'] ?? false; - if ($columnType === UtopiaDatabase::VAR_RELATIONSHIP) { + if ($columnType === ColumnType::Relationship->value || $columnType === ColumnType::Relationship) { throw new Exception( resourceName: $resource->getName(), resourceGroup: $resource->getGroup(), diff --git a/src/Migration/Resources/Database/Columns/Relationship.php b/src/Migration/Resources/Database/Columns/Relationship.php index b530c317..6cf1c736 100644 --- a/src/Migration/Resources/Database/Columns/Relationship.php +++ b/src/Migration/Resources/Database/Columns/Relationship.php @@ -2,9 +2,10 @@ namespace Utopia\Migration\Resources\Database\Columns; -use Utopia\Database\Database; +use Utopia\Database\RelationSide; use Utopia\Migration\Resources\Database\Column; use Utopia\Migration\Resources\Database\Table; +use Utopia\Query\Schema\ForeignKeyAction; class Relationship extends Column { @@ -15,8 +16,8 @@ public function __construct( string $relationType, bool $twoWay = false, ?string $twoWayKey = null, - string $onDelete = Database::RELATION_MUTATE_RESTRICT, - string $side = Database::RELATION_SIDE_PARENT, + string $onDelete = ForeignKeyAction::Restrict->value, + string $side = RelationSide::Parent->value, string $createdAt = '', string $updatedAt = '' ) { diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index 61443459..34ed9a54 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -22,6 +22,7 @@ use Utopia\Database\Database as UtopiaDatabase; use Utopia\Database\DateTime as UtopiaDateTime; use Utopia\Database\Document as UtopiaDocument; +use Utopia\Database\RelationSide; use Utopia\Migration\Exception; use Utopia\Migration\Resource; use Utopia\Migration\Resources\Auth\AuthMethods; @@ -88,6 +89,7 @@ use Utopia\Migration\Sources\Appwrite\Reader\API as APIReader; use Utopia\Migration\Sources\Appwrite\Reader\Database as DatabaseReader; use Utopia\Migration\Transfer; +use Utopia\Query\Schema\ColumnType; class Appwrite extends Source { @@ -1283,8 +1285,8 @@ private function exportFields(string $entityType, int $batchSize): void foreach ($response as $column) { if ( - $column['type'] === UtopiaDatabase::VAR_RELATIONSHIP - && $column['side'] === UtopiaDatabase::RELATION_SIDE_CHILD + $column['type'] === ColumnType::Relationship->value + && $column['side'] === RelationSide::Child->value ) { continue; } diff --git a/src/Migration/Sources/Appwrite/Reader/Database.php b/src/Migration/Sources/Appwrite/Reader/Database.php index 547ec1e6..701eafdb 100644 --- a/src/Migration/Sources/Appwrite/Reader/Database.php +++ b/src/Migration/Sources/Appwrite/Reader/Database.php @@ -2,6 +2,7 @@ namespace Utopia\Migration\Sources\Appwrite\Reader; +use Utopia\Database\Capability; use Utopia\Database\Database as UtopiaDatabase; use Utopia\Database\Document as UtopiaDocument; use Utopia\Database\Exception as DatabaseException; @@ -16,6 +17,7 @@ use Utopia\Migration\Resources\Database\Row as RowResource; use Utopia\Migration\Resources\Database\Table as TableResource; use Utopia\Migration\Sources\Appwrite\Reader; +use Utopia\Query\Schema\ColumnType; /** * @implements Reader @@ -259,13 +261,18 @@ public function listColumns(TableResource $resource, array $queries = []): array } foreach ($columns as $column) { - if ($column['type'] !== UtopiaDatabase::VAR_RELATIONSHIP) { + if ($column['type'] !== ColumnType::Relationship->value) { continue; } $options = $column['options']; - foreach ($options as $key => $value) { - $column[$key] = $value; + if ($options instanceof UtopiaDocument) { + $options = $options->getArrayCopy(); + } + if (\is_array($options)) { + foreach ($options as $key => $value) { + $column[$key] = $value; + } } unset($column['options']); @@ -492,7 +499,7 @@ public function queryLimit(int $limit): Query public function getSupportForAttributes(): bool { - return $this->dbForProject->getAdapter()->getSupportForAttributes(); + return $this->dbForProject->getAdapter()->supports(Capability::DefinedAttributes); } /** diff --git a/src/Migration/Sources/CSV.php b/src/Migration/Sources/CSV.php index 0b6b725c..5ca8ae49 100644 --- a/src/Migration/Sources/CSV.php +++ b/src/Migration/Sources/CSV.php @@ -3,6 +3,8 @@ namespace Utopia\Migration\Sources; use Utopia\Database\Database as UtopiaDatabase; +use Utopia\Database\RelationSide; +use Utopia\Database\RelationType; use Utopia\Migration\Exception; use Utopia\Migration\Resource; use Utopia\Migration\Resource as UtopiaResource; @@ -232,7 +234,7 @@ private function exportRows(int $batchSize): void if ( $type === Column::TYPE_RELATIONSHIP && - $relationSide === UtopiaDatabase::RELATION_SIDE_CHILD + $relationSide === RelationSide::Child->value ) { continue; } @@ -244,8 +246,8 @@ private function exportRows(int $batchSize): void if ( $type === Column::TYPE_RELATIONSHIP && - $relationType === UtopiaDatabase::RELATION_MANY_TO_MANY && - $relationSide === UtopiaDatabase::RELATION_SIDE_PARENT + $relationType === RelationType::ManyToMany->value && + $relationSide === RelationSide::Parent->value ) { $manyToManyKeys[$key] = true; } diff --git a/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php b/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php index 1e125acd..00f34bdf 100644 --- a/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php +++ b/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php @@ -7,6 +7,7 @@ use Utopia\Cache\Adapter\Memory as MemoryCache; use Utopia\Cache\Cache; use Utopia\Database\Adapter\Memory as MemoryAdapter; +use Utopia\Database\Attribute as UtopiaAttribute; use Utopia\Database\Database as UtopiaDatabase; use Utopia\Database\Document as UtopiaDocument; use Utopia\Migration\Destinations\Appwrite as AppwriteDestination; @@ -14,6 +15,7 @@ use Utopia\Migration\Resource; use Utopia\Migration\Resources\Database\Database as DatabaseResource; use Utopia\Migration\Transfer; +use Utopia\Query\Schema\ColumnType; use Utopia\Tests\Unit\Adapters\MockSource; class CountingAppwriteDestination extends AppwriteDestination @@ -76,16 +78,16 @@ private function createProjectDatabase(bool $withStatus): UtopiaDatabase $database->create(); $attributes = [ - $this->attribute('name', UtopiaDatabase::VAR_STRING, required: true, size: 256), - $this->attribute('enabled', UtopiaDatabase::VAR_BOOLEAN, default: true), - $this->attribute('search', UtopiaDatabase::VAR_STRING, size: 16384), - $this->attribute('originalId', UtopiaDatabase::VAR_STRING, size: UtopiaDatabase::LENGTH_KEY), - $this->attribute('type', UtopiaDatabase::VAR_STRING, default: 'tablesdb', size: 128), - $this->attribute('database', UtopiaDatabase::VAR_STRING, size: 2000), + $this->attribute('name', ColumnType::String, required: true, size: 256), + $this->attribute('enabled', ColumnType::Boolean, default: true), + $this->attribute('search', ColumnType::String, size: 16384), + $this->attribute('originalId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('type', ColumnType::String, default: 'tablesdb', size: 128), + $this->attribute('database', ColumnType::String, size: 2000), ]; if ($withStatus) { - $attributes[] = $this->attribute('status', UtopiaDatabase::VAR_STRING, size: 16); + $attributes[] = $this->attribute('status', ColumnType::String, size: 16); } $database->createCollection('databases', $attributes); @@ -95,21 +97,18 @@ private function createProjectDatabase(bool $withStatus): UtopiaDatabase private function attribute( string $id, - string $type, + ColumnType $type, bool $required = false, mixed $default = null, int $size = 0, - ): UtopiaDocument { - return new UtopiaDocument([ - '$id' => $id, - 'type' => $type, - 'size' => $size, - 'required' => $required, - 'default' => $default, - 'array' => false, - 'signed' => true, - 'filters' => [], - ]); + ): UtopiaAttribute { + return new UtopiaAttribute( + key: $id, + type: $type, + size: $size, + required: $required, + default: $default, + ); } private function runDatabaseTransfer(UtopiaDatabase $database, bool $explicit): CountingAppwriteDestination diff --git a/tests/Migration/Unit/Destinations/AppwriteIndexLengthsTest.php b/tests/Migration/Unit/Destinations/AppwriteIndexLengthsTest.php index 362f6e79..e7a70388 100644 --- a/tests/Migration/Unit/Destinations/AppwriteIndexLengthsTest.php +++ b/tests/Migration/Unit/Destinations/AppwriteIndexLengthsTest.php @@ -8,6 +8,7 @@ use Utopia\Cache\Adapter\Memory as MemoryCache; use Utopia\Cache\Cache; use Utopia\Database\Adapter\Memory as MemoryAdapter; +use Utopia\Database\Attribute as UtopiaAttribute; use Utopia\Database\Database as UtopiaDatabase; use Utopia\Database\Document as UtopiaDocument; use Utopia\Migration\Destinations\Appwrite as AppwriteDestination; @@ -18,6 +19,7 @@ use Utopia\Migration\Resources\Database\Index; use Utopia\Migration\Resources\Database\Table; use Utopia\Migration\Transfer; +use Utopia\Query\Schema\ColumnType; use Utopia\Tests\Unit\Adapters\MockSource; /** @@ -188,14 +190,14 @@ private function transferIndex( getDatabasesDB: static fn (UtopiaDocument $document): UtopiaDatabase => $database, collectionStructure: [ 'attributes' => [ - $this->attributeArray('databaseInternalId', UtopiaDatabase::VAR_STRING, size: UtopiaDatabase::LENGTH_KEY), - $this->attributeArray('databaseId', UtopiaDatabase::VAR_STRING, size: UtopiaDatabase::LENGTH_KEY), - $this->attributeArray('name', UtopiaDatabase::VAR_STRING, size: 256), - $this->attributeArray('enabled', UtopiaDatabase::VAR_BOOLEAN, default: true), - $this->attributeArray('documentSecurity', UtopiaDatabase::VAR_BOOLEAN, default: false), - $this->attributeArray('search', UtopiaDatabase::VAR_STRING, size: 16384), - $this->attributeArray('attributes', UtopiaDatabase::VAR_STRING, size: 16384, filters: ['subQueryAttributes']), - $this->attributeArray('indexes', UtopiaDatabase::VAR_STRING, size: 16384, filters: ['subQueryIndexes']), + $this->attributeArray('databaseInternalId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), + $this->attributeArray('databaseId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), + $this->attributeArray('name', ColumnType::String, size: 256), + $this->attributeArray('enabled', ColumnType::Boolean, default: true), + $this->attributeArray('documentSecurity', ColumnType::Boolean, default: false), + $this->attributeArray('search', ColumnType::String, size: 16384), + $this->attributeArray('attributes', ColumnType::String, size: 16384, filters: ['subQueryAttributes']), + $this->attributeArray('indexes', ColumnType::String, size: 16384, filters: ['subQueryIndexes']), ], 'indexes' => [], ], @@ -245,46 +247,46 @@ private function projectDatabase(): UtopiaDatabase $database->create(); $database->createCollection('databases', [ - $this->attribute('name', UtopiaDatabase::VAR_STRING, required: true, size: 256), - $this->attribute('enabled', UtopiaDatabase::VAR_BOOLEAN, default: true), - $this->attribute('search', UtopiaDatabase::VAR_STRING, size: 16384), - $this->attribute('originalId', UtopiaDatabase::VAR_STRING, size: UtopiaDatabase::LENGTH_KEY), - $this->attribute('type', UtopiaDatabase::VAR_STRING, default: 'tablesdb', size: 128), - $this->attribute('database', UtopiaDatabase::VAR_STRING, size: 2000), + $this->attribute('name', ColumnType::String, required: true, size: 256), + $this->attribute('enabled', ColumnType::Boolean, default: true), + $this->attribute('search', ColumnType::String, size: 16384), + $this->attribute('originalId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('type', ColumnType::String, default: 'tablesdb', size: 128), + $this->attribute('database', ColumnType::String, size: 2000), ]); $database->createCollection('attributes', [ - $this->attribute('key', UtopiaDatabase::VAR_STRING, size: 256), - $this->attribute('databaseInternalId', UtopiaDatabase::VAR_STRING, size: UtopiaDatabase::LENGTH_KEY), - $this->attribute('databaseId', UtopiaDatabase::VAR_STRING, size: UtopiaDatabase::LENGTH_KEY), - $this->attribute('collectionInternalId', UtopiaDatabase::VAR_STRING, size: UtopiaDatabase::LENGTH_KEY), - $this->attribute('collectionId', UtopiaDatabase::VAR_STRING, size: UtopiaDatabase::LENGTH_KEY), - $this->attribute('type', UtopiaDatabase::VAR_STRING, size: 256), - $this->attribute('status', UtopiaDatabase::VAR_STRING, size: 64), - $this->attribute('size', UtopiaDatabase::VAR_INTEGER), - $this->attribute('required', UtopiaDatabase::VAR_BOOLEAN, default: false), - $this->attribute('signed', UtopiaDatabase::VAR_BOOLEAN, default: true), - $this->attribute('default', UtopiaDatabase::VAR_STRING, size: 16384), - $this->attribute('array', UtopiaDatabase::VAR_BOOLEAN, default: false), - $this->attribute('format', UtopiaDatabase::VAR_STRING, size: 64), - $this->attribute('formatOptions', UtopiaDatabase::VAR_STRING, size: 16384, filters: ['json']), - $this->attribute('filters', UtopiaDatabase::VAR_STRING, size: 64, array: true), - $this->attribute('options', UtopiaDatabase::VAR_STRING, size: 16384, filters: ['json']), - $this->attribute('error', UtopiaDatabase::VAR_STRING, size: 2048), + $this->attribute('key', ColumnType::String, size: 256), + $this->attribute('databaseInternalId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('databaseId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('collectionInternalId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('collectionId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('type', ColumnType::String, size: 256), + $this->attribute('status', ColumnType::String, size: 64), + $this->attribute('size', ColumnType::Integer), + $this->attribute('required', ColumnType::Boolean, default: false), + $this->attribute('signed', ColumnType::Boolean, default: true), + $this->attribute('default', ColumnType::String, size: 16384), + $this->attribute('array', ColumnType::Boolean, default: false), + $this->attribute('format', ColumnType::String, size: 64), + $this->attribute('formatOptions', ColumnType::String, size: 16384, filters: ['json']), + $this->attribute('filters', ColumnType::String, size: 64, array: true), + $this->attribute('options', ColumnType::String, size: 16384, filters: ['json']), + $this->attribute('error', ColumnType::String, size: 2048), ]); $database->createCollection('indexes', [ - $this->attribute('key', UtopiaDatabase::VAR_STRING, size: 256), - $this->attribute('status', UtopiaDatabase::VAR_STRING, size: 64), - $this->attribute('databaseInternalId', UtopiaDatabase::VAR_STRING, size: UtopiaDatabase::LENGTH_KEY), - $this->attribute('databaseId', UtopiaDatabase::VAR_STRING, size: UtopiaDatabase::LENGTH_KEY), - $this->attribute('collectionInternalId', UtopiaDatabase::VAR_STRING, size: UtopiaDatabase::LENGTH_KEY), - $this->attribute('collectionId', UtopiaDatabase::VAR_STRING, size: UtopiaDatabase::LENGTH_KEY), - $this->attribute('type', UtopiaDatabase::VAR_STRING, size: 16), - $this->attribute('attributes', UtopiaDatabase::VAR_STRING, size: 256, array: true), - $this->attribute('lengths', UtopiaDatabase::VAR_INTEGER, array: true), - $this->attribute('orders', UtopiaDatabase::VAR_STRING, size: 4, array: true), - $this->attribute('error', UtopiaDatabase::VAR_STRING, size: 2048), + $this->attribute('key', ColumnType::String, size: 256), + $this->attribute('status', ColumnType::String, size: 64), + $this->attribute('databaseInternalId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('databaseId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('collectionInternalId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('collectionId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('type', ColumnType::String, size: 16), + $this->attribute('attributes', ColumnType::String, size: 256, array: true), + $this->attribute('lengths', ColumnType::Integer, array: true), + $this->attribute('orders', ColumnType::String, size: 4, array: true), + $this->attribute('error', ColumnType::String, size: 2048), ]); return $database; @@ -296,7 +298,7 @@ private function projectDatabase(): UtopiaDatabase */ private function attributeArray( string $id, - string $type, + ColumnType $type, bool $required = false, mixed $default = null, int $size = 0, @@ -305,7 +307,7 @@ private function attributeArray( ): array { return [ '$id' => $id, - 'type' => $type, + 'type' => $type->value, 'size' => $size, 'required' => $required, 'default' => $default, @@ -320,14 +322,22 @@ private function attributeArray( */ private function attribute( string $id, - string $type, + ColumnType $type, bool $required = false, mixed $default = null, int $size = 0, bool $array = false, array $filters = [], - ): UtopiaDocument { - return new UtopiaDocument($this->attributeArray($id, $type, $required, $default, $size, $array, $filters)); + ): UtopiaAttribute { + return new UtopiaAttribute( + key: $id, + type: $type, + size: $size, + required: $required, + default: $default, + array: $array, + filters: $filters, + ); } /** From d32d19600d8f4b16e9ed8bc697f7ce195151d2b6 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 14 Aug 2026 23:55:54 +1200 Subject: [PATCH 02/32] fix: accept query-lib biginteger as API bigint Appwrite stores ColumnType::BigInteger as biginteger. CSV export resolved that as an unsupported column type and wrote no rows. --- src/Migration/Resources/Database/Column.php | 4 ++++ tests/Migration/Unit/Resources/ColumnTest.php | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/src/Migration/Resources/Database/Column.php b/src/Migration/Resources/Database/Column.php index cc056504..a9df216f 100644 --- a/src/Migration/Resources/Database/Column.php +++ b/src/Migration/Resources/Database/Column.php @@ -86,6 +86,10 @@ public static function resolve(array $column): array $type = \is_string($column['type'] ?? null) ? $column['type'] : ''; $format = \is_string($column['format'] ?? null) ? $column['format'] : ''; + if ($type === 'biginteger') { + $type = self::TYPE_BIG_INT; + } + if (isset(self::FORMAT_SIZES[$type])) { $format = $type; $type = self::TYPE_STRING; diff --git a/tests/Migration/Unit/Resources/ColumnTest.php b/tests/Migration/Unit/Resources/ColumnTest.php index 045f155e..fc2d7967 100644 --- a/tests/Migration/Unit/Resources/ColumnTest.php +++ b/tests/Migration/Unit/Resources/ColumnTest.php @@ -106,4 +106,12 @@ public function testUnsizedAndUnknownDefinitionsResolveToZero(): void Column::resolve(['key' => 'unknown']), ); } + + public function testQueryLibBigIntegerAliasBecomesBigint(): void + { + $this->assertSame( + ['type' => Column::TYPE_BIG_INT, 'format' => '', 'size' => 0], + Column::resolve(['key' => 'total', 'type' => 'biginteger']), + ); + } } From 47e2597bc204c60c920891295a88bfa8e025743f Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Sat, 15 Aug 2026 01:47:48 +1200 Subject: [PATCH 03/32] fix: create collection metadata from the persisted database sequence createDocument can return an empty Mongo sequence while a subsequent getDocument has the ObjectId. Creating database_{seq} from the create return left table import looking up a collection that did not exist. --- src/Migration/Destinations/Appwrite.php | 5 +++++ .../Unit/Destinations/AppwriteDatabaseStatusTest.php | 8 ++++++++ 2 files changed, 13 insertions(+) diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index f2ed2873..75fb2219 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -788,6 +788,11 @@ protected function createDatabase(Database $resource): bool } $database = $this->dbForProject->createDocument(self::META_DATABASES, new UtopiaDocument($document)); + $database = $this->dbForProject->getDocument(self::META_DATABASES, $database->getId()); + + if ($database->isEmpty()) { + throw new DatabaseException('Failed to reload created database '.$resource->getId()); + } $resource->setSequence($database->getSequence()); diff --git a/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php b/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php index 00f34bdf..bde31198 100644 --- a/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php +++ b/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php @@ -48,6 +48,10 @@ public function testDatabaseCreationOmitsStatusThroughLegacyAndExplicitEntrypoin $this->assertSame(1, $destination->runCount); $this->assertFalse($created->isEmpty()); $this->assertArrayNotHasKey('status', $created->getArrayCopy()); + $this->assertFalse( + $database->getCollection('database_'.$created->getSequence())->isEmpty(), + 'Metadata collection must use the persisted database sequence', + ); } } @@ -63,6 +67,10 @@ public function testDatabaseCreationPreservesLifecycleThroughLegacyAndExplicitEn $this->assertSame(1, $destination->runCount); $this->assertFalse($created->isEmpty()); $this->assertSame('ready', $created->getAttribute('status')); + $this->assertFalse( + $database->getCollection('database_'.$created->getSequence())->isEmpty(), + 'Metadata collection must use the persisted database sequence', + ); } } From cce7c66ae91c6639290f5050f11e9264c8da033b Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 21 Aug 2026 14:09:35 +1200 Subject: [PATCH 04/32] (fix): allow migrating HuggingFace OAuth2 providers Appwrite main added huggingface as a project OAuth2 provider. Without an allow-list entry, Appwrite-to-Appwrite migrations fail on that provider even when the rest of the transfer succeeded. --- .../Resources/Auth/OAuth2/OAuth2Provider.php | 1 + .../Unit/Resources/OAuth2ProviderTest.php | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/src/Migration/Resources/Auth/OAuth2/OAuth2Provider.php b/src/Migration/Resources/Auth/OAuth2/OAuth2Provider.php index e029dbdf..54c2ee00 100644 --- a/src/Migration/Resources/Auth/OAuth2/OAuth2Provider.php +++ b/src/Migration/Resources/Auth/OAuth2/OAuth2Provider.php @@ -49,6 +49,7 @@ final class OAuth2Provider extends Resource 'github' => ['clientId' => ['target' => self::TARGET_APP_ID]], 'gitlab' => ['clientId' => ['target' => self::TARGET_APP_ID], 'endpoint' => ['target' => self::TARGET_SECRET]], 'google' => ['clientId' => ['target' => self::TARGET_APP_ID], 'prompt' => ['target' => self::TARGET_SECRET]], + 'huggingface' => ['clientId' => ['target' => self::TARGET_APP_ID]], 'keycloak' => [ 'clientId' => ['target' => self::TARGET_APP_ID], 'endpoint' => ['target' => self::TARGET_SECRET, 'key' => 'keycloakDomain'], diff --git a/tests/Migration/Unit/Resources/OAuth2ProviderTest.php b/tests/Migration/Unit/Resources/OAuth2ProviderTest.php index c24a06cc..c74b009b 100644 --- a/tests/Migration/Unit/Resources/OAuth2ProviderTest.php +++ b/tests/Migration/Unit/Resources/OAuth2ProviderTest.php @@ -25,6 +25,24 @@ public function testFromArrayAppwrite(): void $this->assertTrue($provider->isConfigured()); } + public function testFromArrayHuggingFace(): void + { + $provider = OAuth2Provider::fromArray('huggingface', [ + 'id' => 'huggingface', + 'enabled' => true, + 'clientId' => 'client-123', + 'clientSecret' => 'super-secret', + ]); + + $this->assertNotNull($provider); + $this->assertEquals('huggingface', $provider->getProviderKey()); + $this->assertTrue($provider->getEnabled()); + $this->assertEquals(['clientId' => 'client-123'], $provider->getSettings()); + $this->assertEquals('client-123', $provider->getDestinationAppId()); + $this->assertEquals([], $provider->getDestinationSecretFields()); + $this->assertTrue($provider->isConfigured()); + } + public function testFromArrayNeverCopiesSecrets(): void { foreach (\array_keys(OAuth2Provider::PROVIDERS) as $providerKey) { From 0144e335ae44114077e3b9966d13997835165d33 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 21 Aug 2026 16:06:41 +1200 Subject: [PATCH 05/32] chore: pin utopia-php/database to Appwrite's feat-query-lib SHA Appwrite #11649 locks database at 5719edd. Staying on e593b78 would only prove the schema VO calls against an older query-lib surface. --- composer.lock | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/composer.lock b/composer.lock index 1ca9e57b..d4cae1bf 100644 --- a/composer.lock +++ b/composer.lock @@ -248,23 +248,23 @@ }, { "name": "google/protobuf", - "version": "v5.35.1", + "version": "v5.36.0", "source": { "type": "git", "url": "https://github.com/protocolbuffers/protobuf-php.git", - "reference": "55bb4a7d6739b5af0927b96213c1371a3afb7cfb" + "reference": "9c105104b54709ecd902494ab340ed2122789b2d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/55bb4a7d6739b5af0927b96213c1371a3afb7cfb", - "reference": "55bb4a7d6739b5af0927b96213c1371a3afb7cfb", + "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/9c105104b54709ecd902494ab340ed2122789b2d", + "reference": "9c105104b54709ecd902494ab340ed2122789b2d", "shasum": "" }, "require": { "php": ">=8.2.0" }, "require-dev": { - "phpunit/phpunit": ">=11.5.0 <12.0.0" + "phpunit/phpunit": ">=11.5.50 <12.0.0" }, "suggest": { "ext-bcmath": "Need to support JSON deserialization" @@ -286,9 +286,9 @@ "proto" ], "support": { - "source": "https://github.com/protocolbuffers/protobuf-php/tree/v5.35.1" + "source": "https://github.com/protocolbuffers/protobuf-php/tree/v5.36.0" }, - "time": "2026-06-11T21:19:23+00:00" + "time": "2026-08-20T13:06:50+00:00" }, { "name": "halaxa/json-machine", @@ -2612,12 +2612,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "e593b78c587e1f1d66b644979f5c4ff11dffcd31" + "reference": "5719eddbd77bb6b9a9916aec64ee3a8b0bc45c0f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/e593b78c587e1f1d66b644979f5c4ff11dffcd31", - "reference": "e593b78c587e1f1d66b644979f5c4ff11dffcd31", + "url": "https://api.github.com/repos/utopia-php/database/zipball/5719eddbd77bb6b9a9916aec64ee3a8b0bc45c0f", + "reference": "5719eddbd77bb6b9a9916aec64ee3a8b0bc45c0f", "shasum": "" }, "require": { @@ -2632,7 +2632,7 @@ "utopia-php/mongo": "1.*", "utopia-php/pools": "2.*", "utopia-php/query": "0.4.*", - "utopia-php/validators": "0.4.*" + "utopia-php/validators": "0.4.* || 0.5.*" }, "require-dev": { "brianium/paratest": "7.20.*", @@ -2702,7 +2702,7 @@ "source": "https://github.com/utopia-php/database/tree/feat-query-lib", "issues": "https://github.com/utopia-php/database/issues" }, - "time": "2026-08-14T02:54:13+00:00" + "time": "2026-08-20T11:46:41+00:00" }, { "name": "utopia-php/dsn", From a11db206fd09bd047f36496226ab95ecbf30fbad Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 21 Aug 2026 16:16:04 +1200 Subject: [PATCH 06/32] test: stub Swoole Coroutine when the extension is missing utopia-php/database feat-query-lib keys silenced events with Coroutine::getCid(). The CI image is vanilla PHP, so Memory-adapter tests fatalled before they could run. --- phpunit.xml | 2 +- tests/bootstrap.php | 7 +++++++ tests/stubs/SwooleCoroutine.php | 11 +++++++++++ 3 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 tests/bootstrap.php create mode 100644 tests/stubs/SwooleCoroutine.php diff --git a/phpunit.xml b/phpunit.xml index fad0deca..ce3ac877 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -2,7 +2,7 @@ diff --git a/tests/bootstrap.php b/tests/bootstrap.php new file mode 100644 index 00000000..17c58c63 --- /dev/null +++ b/tests/bootstrap.php @@ -0,0 +1,7 @@ + Date: Fri, 21 Aug 2026 16:16:04 +1200 Subject: [PATCH 07/32] fix: mark databases failed when metadata reload misses createDocument can persist a row whose subsequent getDocument is empty. That throw sat outside the failed-status handler, so a later skip could flip the unusable database to ready without a backing collection. --- src/Migration/Destinations/Appwrite.php | 13 +++--- .../AppwriteDatabaseStatusTest.php | 43 ++++++++++++++++++- 2 files changed, 47 insertions(+), 9 deletions(-) diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index 75fb2219..43c273a5 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -788,15 +788,15 @@ protected function createDatabase(Database $resource): bool } $database = $this->dbForProject->createDocument(self::META_DATABASES, new UtopiaDocument($document)); - $database = $this->dbForProject->getDocument(self::META_DATABASES, $database->getId()); - if ($database->isEmpty()) { - throw new DatabaseException('Failed to reload created database '.$resource->getId()); - } + try { + $database = $this->dbForProject->getDocument(self::META_DATABASES, $database->getId()); - $resource->setSequence($database->getSequence()); + if ($database->isEmpty()) { + throw new DatabaseException('Failed to reload created database '.$resource->getId()); + } - try { + $resource->setSequence($database->getSequence()); $structure = $this->collectionStructureFor($resource); $this->dbForProject->createCollection( @@ -805,7 +805,6 @@ protected function createDatabase(Database $resource): bool $this->schemaIndexes($structure['indexes'] ?? []) ); } catch (\Throwable $e) { - // The metadata document exists but the database isn't usable; mark it failed before propagating. $this->markDatabaseFailed($resource->getId()); throw $e; } diff --git a/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php b/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php index bde31198..071cd591 100644 --- a/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php +++ b/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php @@ -34,6 +34,24 @@ public function run( } } +final class ReloadFailingProjectDatabase extends UtopiaDatabase +{ + public bool $failNextDatabasesRead = false; + + #[Override] + public function getDocument(string $collection, string $id, array $queries = [], bool $forUpdate = false): UtopiaDocument + { + $document = parent::getDocument($collection, $id, $queries, $forUpdate); + if ($this->failNextDatabasesRead && $collection === 'databases' && !$document->isEmpty()) { + $this->failNextDatabasesRead = false; + + return new UtopiaDocument(); + } + + return $document; + } +} + final class AppwriteDatabaseStatusTest extends TestCase { public function testDatabaseCreationOmitsStatusThroughLegacyAndExplicitEntrypoints(): void @@ -74,9 +92,30 @@ public function testDatabaseCreationPreservesLifecycleThroughLegacyAndExplicitEn } } - private function createProjectDatabase(bool $withStatus): UtopiaDatabase + public function testReloadFailureMarksTheDatabaseFailed(): void + { + $database = new ReloadFailingProjectDatabase( + new MemoryAdapter(), + new Cache(new MemoryCache()), + ); + $this->createProjectDatabase(withStatus: true, database: $database); + $database->failNextDatabasesRead = true; + + $destination = $this->runDatabaseTransfer($database, explicit: false); + + $created = $this->getDatabaseDocument($database); + $this->assertNotSame([], $this->errorMessages($destination)); + $this->assertStringContainsString('Failed to reload created database', $this->errorMessages($destination)[0]); + $this->assertSame('failed', $created->getAttribute('status')); + $this->assertTrue( + $database->getCollection('database_'.$created->getSequence())->isEmpty(), + 'A reload failure must not leave a backing collection behind the metadata document', + ); + } + + private function createProjectDatabase(bool $withStatus, ?UtopiaDatabase $database = null): UtopiaDatabase { - $database = new UtopiaDatabase( + $database ??= new UtopiaDatabase( new MemoryAdapter(), new Cache(new MemoryCache()), ); From b67ace1a95907ed8dd2ec2ef8215de145b63cb36 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 21 Aug 2026 17:11:39 +1200 Subject: [PATCH 08/32] (chore): use caret ranges for Utopia dependencies Asterisk wildcards on utopia-php packages are replaced with equivalent caret constraints so Composer ranges stay consistent. --- composer.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/composer.json b/composer.json index 78ad8ffb..c2edd013 100644 --- a/composer.json +++ b/composer.json @@ -34,8 +34,8 @@ "ext-openssl": "*", "appwrite/appwrite": "^27.0", "utopia-php/database": "dev-feat-query-lib as 7.0.0", - "utopia-php/storage": "4.*", - "utopia-php/dsn": "0.2.*", + "utopia-php/storage": "^4.0", + "utopia-php/dsn": "^0.2", "halaxa/json-machine": "^1.2" }, "require-dev": { From 7a3a60e4f5516ba860f1e970a09b0582f66b51b4 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 21 Aug 2026 17:23:34 +1200 Subject: [PATCH 09/32] (chore): refresh lockfile after caret Utopia constraints Keep composer.json and composer.lock in sync so `composer validate` passes, and pin utopia-php/database to the current query-lib HEAD. --- composer.lock | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/composer.lock b/composer.lock index d4cae1bf..591fcbb1 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "e20f678bfb397f89a6cfa0083988f539", + "content-hash": "669e0b5ccd20bfaf8cacb8461b605d49", "packages": [ { "name": "adhocore/jwt", @@ -2612,12 +2612,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "5719eddbd77bb6b9a9916aec64ee3a8b0bc45c0f" + "reference": "798f294c1615102c92f8e8f7f8b3d18f36ad083b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/5719eddbd77bb6b9a9916aec64ee3a8b0bc45c0f", - "reference": "5719eddbd77bb6b9a9916aec64ee3a8b0bc45c0f", + "url": "https://api.github.com/repos/utopia-php/database/zipball/798f294c1615102c92f8e8f7f8b3d18f36ad083b", + "reference": "798f294c1615102c92f8e8f7f8b3d18f36ad083b", "shasum": "" }, "require": { @@ -2626,13 +2626,13 @@ "ext-pdo": "*", "ext-redis": "*", "php": ">=8.5", - "utopia-php/async": "0.1.*", - "utopia-php/cache": "4.*", - "utopia-php/console": "0.1.*", - "utopia-php/mongo": "1.*", - "utopia-php/pools": "2.*", - "utopia-php/query": "0.4.*", - "utopia-php/validators": "0.4.* || 0.5.*" + "utopia-php/async": "^0.1", + "utopia-php/cache": "^4.0", + "utopia-php/console": "^0.1", + "utopia-php/mongo": "^1.0", + "utopia-php/pools": "^2.0", + "utopia-php/query": "^0.4", + "utopia-php/validators": "^0.4 || ^0.5" }, "require-dev": { "brianium/paratest": "7.20.*", @@ -2642,7 +2642,7 @@ "phpunit/phpunit": "12.5.*", "rregeer/phpunit-coverage-check": "0.3.*", "swoole/ide-helper": "5.1.3", - "utopia-php/cli": "0.22.*" + "utopia-php/cli": "^0.22" }, "suggest": { "ext-pdo": "Needed to support MariaDB, MySQL or SQLite Database Adapter", @@ -2702,7 +2702,7 @@ "source": "https://github.com/utopia-php/database/tree/feat-query-lib", "issues": "https://github.com/utopia-php/database/issues" }, - "time": "2026-08-20T11:46:41+00:00" + "time": "2026-08-21T05:22:01+00:00" }, { "name": "utopia-php/dsn", From f2de906c9a84b9c911658d446434ccc9acce7c24 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 21 Aug 2026 18:52:50 +1200 Subject: [PATCH 10/32] (feat): pass Collection to createCollection Database::createCollection no longer accepts a string id. --- composer.lock | 8 ++++---- src/Migration/Destinations/Appwrite.php | 19 ++++--------------- .../AppwriteDatabaseStatusTest.php | 3 ++- .../Destinations/AppwriteIndexLengthsTest.php | 13 +++++++------ 4 files changed, 17 insertions(+), 26 deletions(-) diff --git a/composer.lock b/composer.lock index 591fcbb1..fabd1850 100644 --- a/composer.lock +++ b/composer.lock @@ -2612,12 +2612,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "798f294c1615102c92f8e8f7f8b3d18f36ad083b" + "reference": "808f90bff1a7ccec78cc2eabebd30c2454a9c896" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/798f294c1615102c92f8e8f7f8b3d18f36ad083b", - "reference": "798f294c1615102c92f8e8f7f8b3d18f36ad083b", + "url": "https://api.github.com/repos/utopia-php/database/zipball/808f90bff1a7ccec78cc2eabebd30c2454a9c896", + "reference": "808f90bff1a7ccec78cc2eabebd30c2454a9c896", "shasum": "" }, "require": { @@ -2702,7 +2702,7 @@ "source": "https://github.com/utopia-php/database/tree/feat-query-lib", "issues": "https://github.com/utopia-php/database/issues" }, - "time": "2026-08-21T05:22:01+00:00" + "time": "2026-08-21T06:50:33+00:00" }, { "name": "utopia-php/dsn", diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index 43c273a5..c23ea0dc 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -27,6 +27,7 @@ use Utopia\Database\Adapter\Feature\Spatial; use Utopia\Database\Attribute as UtopiaAttribute; use Utopia\Database\Capability; +use Utopia\Database\Collection; use Utopia\Database\Database as UtopiaDatabase; use Utopia\Database\DateTime; use Utopia\Database\Document as UtopiaDocument; @@ -743,11 +744,7 @@ protected function createDatabase(Database $resource): bool try { $structure = $this->collectionStructureFor($resource); - $this->dbForProject->createCollection( - $this->databaseCollectionId($existing), - $this->schemaAttributes($structure['attributes'] ?? []), - $this->schemaIndexes($structure['indexes'] ?? []) - ); + $this->dbForProject->createCollection(new Collection(id: $this->databaseCollectionId($existing), attributes: $this->schemaAttributes($structure['attributes'] ?? []), indexes: $this->schemaIndexes($structure['indexes'] ?? []))); } catch (\Throwable $e) { $this->markDatabaseFailed($resource->getId()); throw $e; @@ -799,11 +796,7 @@ protected function createDatabase(Database $resource): bool $resource->setSequence($database->getSequence()); $structure = $this->collectionStructureFor($resource); - $this->dbForProject->createCollection( - $this->databaseCollectionId($database), - $this->schemaAttributes($structure['attributes'] ?? []), - $this->schemaIndexes($structure['indexes'] ?? []) - ); + $this->dbForProject->createCollection(new Collection(id: $this->databaseCollectionId($database), attributes: $this->schemaAttributes($structure['attributes'] ?? []), indexes: $this->schemaIndexes($structure['indexes'] ?? []))); } catch (\Throwable $e) { $this->markDatabaseFailed($resource->getId()); throw $e; @@ -927,11 +920,7 @@ protected function createEntity(Table $resource): bool $resource->setSequence($table->getSequence()); - $dbForDatabases->createCollection( - $this->tableCollectionId($database, $table), - permissions: $resource->getPermissions(), - documentSecurity: $resource->getRowSecurity() - ); + $dbForDatabases->createCollection(new Collection(id: $this->tableCollectionId($database, $table), permissions: $resource->getPermissions(), documentSecurity: $resource->getRowSecurity())); return true; } diff --git a/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php b/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php index 071cd591..2081838c 100644 --- a/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php +++ b/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php @@ -8,6 +8,7 @@ use Utopia\Cache\Cache; use Utopia\Database\Adapter\Memory as MemoryAdapter; use Utopia\Database\Attribute as UtopiaAttribute; +use Utopia\Database\Collection; use Utopia\Database\Database as UtopiaDatabase; use Utopia\Database\Document as UtopiaDocument; use Utopia\Migration\Destinations\Appwrite as AppwriteDestination; @@ -137,7 +138,7 @@ private function createProjectDatabase(bool $withStatus, ?UtopiaDatabase $databa $attributes[] = $this->attribute('status', ColumnType::String, size: 16); } - $database->createCollection('databases', $attributes); + $database->createCollection(new Collection(id: 'databases', attributes: $attributes)); return $database; } diff --git a/tests/Migration/Unit/Destinations/AppwriteIndexLengthsTest.php b/tests/Migration/Unit/Destinations/AppwriteIndexLengthsTest.php index e7a70388..084ed576 100644 --- a/tests/Migration/Unit/Destinations/AppwriteIndexLengthsTest.php +++ b/tests/Migration/Unit/Destinations/AppwriteIndexLengthsTest.php @@ -9,6 +9,7 @@ use Utopia\Cache\Cache; use Utopia\Database\Adapter\Memory as MemoryAdapter; use Utopia\Database\Attribute as UtopiaAttribute; +use Utopia\Database\Collection; use Utopia\Database\Database as UtopiaDatabase; use Utopia\Database\Document as UtopiaDocument; use Utopia\Migration\Destinations\Appwrite as AppwriteDestination; @@ -246,16 +247,16 @@ private function projectDatabase(): UtopiaDatabase ->setNamespace('_project'); $database->create(); - $database->createCollection('databases', [ + $database->createCollection(new Collection(id: 'databases', attributes: [ $this->attribute('name', ColumnType::String, required: true, size: 256), $this->attribute('enabled', ColumnType::Boolean, default: true), $this->attribute('search', ColumnType::String, size: 16384), $this->attribute('originalId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), $this->attribute('type', ColumnType::String, default: 'tablesdb', size: 128), $this->attribute('database', ColumnType::String, size: 2000), - ]); + ])); - $database->createCollection('attributes', [ + $database->createCollection(new Collection(id: 'attributes', attributes: [ $this->attribute('key', ColumnType::String, size: 256), $this->attribute('databaseInternalId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), $this->attribute('databaseId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), @@ -273,9 +274,9 @@ private function projectDatabase(): UtopiaDatabase $this->attribute('filters', ColumnType::String, size: 64, array: true), $this->attribute('options', ColumnType::String, size: 16384, filters: ['json']), $this->attribute('error', ColumnType::String, size: 2048), - ]); + ])); - $database->createCollection('indexes', [ + $database->createCollection(new Collection(id: 'indexes', attributes: [ $this->attribute('key', ColumnType::String, size: 256), $this->attribute('status', ColumnType::String, size: 64), $this->attribute('databaseInternalId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), @@ -287,7 +288,7 @@ private function projectDatabase(): UtopiaDatabase $this->attribute('lengths', ColumnType::Integer, array: true), $this->attribute('orders', ColumnType::String, size: 4, array: true), $this->attribute('error', ColumnType::String, size: 2048), - ]); + ])); return $database; } From 3fccb0c29d824410c9803936d0d8eae38cb49ee8 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 21 Aug 2026 22:24:19 +1200 Subject: [PATCH 11/32] (fix): pass Attribute models to checkAttribute Database::checkAttribute now requires Attribute. Build schema models from the resource key so metadata document IDs are not used as attribute keys. --- src/Migration/Destinations/Appwrite.php | 18 ++++- .../AppwriteCheckAttributeTest.php | 66 +++++++++++++++++++ 2 files changed, 81 insertions(+), 3 deletions(-) create mode 100644 tests/Migration/Unit/Destinations/AppwriteCheckAttributeTest.php diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index c23ea0dc..23e91dce 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -1104,7 +1104,19 @@ protected function createField(Column|Attribute $resource): bool '$updatedAt' => $updatedAt, ]); - $this->dbForProject->checkAttribute($table, $column); + $this->dbForProject->checkAttribute($table, UtopiaAttribute::fromArray([ + 'key' => $resource->getKey(), + 'type' => $type, + 'size' => $resource->getSize(), + 'required' => $resource->isRequired(), + 'signed' => $resource->isSigned(), + 'default' => $resource->getDefault(), + 'array' => $resource->isArray(), + 'format' => $resource->getFormat() !== '' ? $resource->getFormat() : null, + 'formatOptions' => $resource->getFormatOptions(), + 'filters' => $resource->getFilters(), + 'options' => $resource->getOptions() !== [] ? $resource->getOptions() : null, + ])); $column = $this->dbForProject->createDocument(self::META_ATTRIBUTES, $column); } catch (DuplicateException $e) { @@ -1318,7 +1330,7 @@ private function schemaAttributes(array $attributes): array return $attr; } if ($attr instanceof UtopiaDocument) { - return UtopiaAttribute::fromDocument($attr); + return UtopiaAttribute::fromArray($attr->getArrayCopy()); } return UtopiaAttribute::fromArray(\is_array($attr) ? $attr : []); @@ -1336,7 +1348,7 @@ private function schemaIndexes(array $indexes): array return $index; } if ($index instanceof UtopiaDocument) { - return UtopiaIndex::fromDocument($index); + return UtopiaIndex::fromArray($index->getArrayCopy()); } return UtopiaIndex::fromArray(\is_array($index) ? $index : []); diff --git a/tests/Migration/Unit/Destinations/AppwriteCheckAttributeTest.php b/tests/Migration/Unit/Destinations/AppwriteCheckAttributeTest.php new file mode 100644 index 00000000..7825f7e9 --- /dev/null +++ b/tests/Migration/Unit/Destinations/AppwriteCheckAttributeTest.php @@ -0,0 +1,66 @@ +assertStringContainsString( + "checkAttribute(\$table, UtopiaAttribute::fromArray([", + $source, + ); + $this->assertStringContainsString( + "'key' => \$resource->getKey(),", + $source, + ); + $this->assertStringNotContainsString( + 'checkAttribute($table, $column)', + $source, + ); + + $column = new UtopiaDocument([ + '$id' => '1_2_title', + 'key' => 'title', + 'type' => 'string', + 'size' => 128, + 'required' => true, + 'signed' => true, + 'default' => null, + 'array' => false, + 'format' => '', + 'formatOptions' => [], + 'filters' => [], + 'options' => [], + ]); + + $fromDocumentCopy = UtopiaAttribute::fromArray($column->getArrayCopy()); + $this->assertSame('1_2_title', $fromDocumentCopy->key); + + $attribute = UtopiaAttribute::fromArray([ + 'key' => 'title', + 'type' => 'string', + 'size' => 128, + 'required' => true, + 'signed' => true, + 'default' => null, + 'array' => false, + 'format' => null, + 'formatOptions' => [], + 'filters' => [], + 'options' => null, + ]); + + $this->assertInstanceOf(UtopiaAttribute::class, $attribute); + $this->assertSame('title', $attribute->key); + $this->assertSame(ColumnType::String, $attribute->type); + $this->assertSame(128, $attribute->size); + } +} From 3a5fb77ac4ab5f93a09fd1c8537434e11a3d16d0 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 21 Aug 2026 22:24:34 +1200 Subject: [PATCH 12/32] (fix): pass Attribute models into checkAttribute Appwrite E2E migrations failed because checkAttribute now requires Attribute, and the destination still handed it a metadata Document. --- composer.lock | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/composer.lock b/composer.lock index fabd1850..3f7ce063 100644 --- a/composer.lock +++ b/composer.lock @@ -2612,12 +2612,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "808f90bff1a7ccec78cc2eabebd30c2454a9c896" + "reference": "196db02d6e2eadb4d42c192d5a0f260c06b3e1db" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/808f90bff1a7ccec78cc2eabebd30c2454a9c896", - "reference": "808f90bff1a7ccec78cc2eabebd30c2454a9c896", + "url": "https://api.github.com/repos/utopia-php/database/zipball/196db02d6e2eadb4d42c192d5a0f260c06b3e1db", + "reference": "196db02d6e2eadb4d42c192d5a0f260c06b3e1db", "shasum": "" }, "require": { @@ -2631,7 +2631,7 @@ "utopia-php/console": "^0.1", "utopia-php/mongo": "^1.0", "utopia-php/pools": "^2.0", - "utopia-php/query": "^0.4", + "utopia-php/query": "^0.5", "utopia-php/validators": "^0.4 || ^0.5" }, "require-dev": { @@ -2702,7 +2702,7 @@ "source": "https://github.com/utopia-php/database/tree/feat-query-lib", "issues": "https://github.com/utopia-php/database/issues" }, - "time": "2026-08-21T06:50:33+00:00" + "time": "2026-08-21T09:27:18+00:00" }, { "name": "utopia-php/dsn", @@ -2916,16 +2916,16 @@ }, { "name": "utopia-php/query", - "version": "0.4.0", + "version": "0.5.0", "source": { "type": "git", "url": "https://github.com/utopia-php/query.git", - "reference": "c334515035a2ab0aa49176eeca96de3e1c096e22" + "reference": "802821c6fb0470410e1f0e65561cc16224c343c0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/query/zipball/c334515035a2ab0aa49176eeca96de3e1c096e22", - "reference": "c334515035a2ab0aa49176eeca96de3e1c096e22", + "url": "https://api.github.com/repos/utopia-php/query/zipball/802821c6fb0470410e1f0e65561cc16224c343c0", + "reference": "802821c6fb0470410e1f0e65561cc16224c343c0", "shasum": "" }, "require": { @@ -2959,9 +2959,9 @@ ], "support": { "issues": "https://github.com/utopia-php/query/issues", - "source": "https://github.com/utopia-php/query/tree/0.4.0" + "source": "https://github.com/utopia-php/query/tree/0.5.0" }, - "time": "2026-08-14T01:42:36+00:00" + "time": "2026-08-21T06:16:10+00:00" }, { "name": "utopia-php/span", From 18c09d40a53baa31e59abf9b52698c78c0f9b731 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 21 Aug 2026 22:36:21 +1200 Subject: [PATCH 13/32] (fix): recover failed databases on OnDuplicate::Fail retries A reload failure leaves a metadata document in `failed` with no backing collection. Recovery only ran when onDuplicate was not Fail, so the default policy retried createDocument against the existing ID and stranded the database. --- src/Migration/Destinations/Appwrite.php | 16 +++++----- .../AppwriteDatabaseStatusTest.php | 29 +++++++++++++++++++ 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index 23e91dce..3a2dc781 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -682,22 +682,22 @@ protected function createDatabase(Database $resource): bool $createdAt = $this->normalizeDateTime($resource->getCreatedAt()); $updatedAt = $this->normalizeDateTime($resource->getUpdatedAt(), $createdAt); - if ($this->onDuplicate !== OnDuplicate::Fail) { - $existing = $this->dbForProject->getDocument(self::META_DATABASES, $resource->getId()); + $existing = $this->dbForProject->getDocument(self::META_DATABASES, $resource->getId()); + $isFailed = ! $existing->isEmpty() + && $this->getSupportForDatabaseStatus() + && $existing->getAttribute('status') === self::DATABASE_STATUS_FAILED; + + if ($this->onDuplicate !== OnDuplicate::Fail || $isFailed) { $action = $this->onDuplicate->resolveSchemaAction( !$existing->isEmpty(), $updatedAt, $existing->getUpdatedAt(), ); - $isFailed = ! $existing->isEmpty() - && $this->getSupportForDatabaseStatus() - && $existing->getAttribute('status') === self::DATABASE_STATUS_FAILED; - if ($isFailed) { // A prior run created the metadata document but left the database unusable (its backing - // collection may be missing). Force Overwrite — regardless of timestamps or spec match — - // so the recovery path recreates the collection instead of skipping it forever. + // collection may be missing). Force Overwrite — regardless of timestamps, spec match, or + // OnDuplicate::Fail — so retries recreate the collection instead of hitting the existing ID. $action = SchemaAction::Overwrite; } elseif ($action !== SchemaAction::Create && $this->databaseSpecMatches($existing, $resource)) { // Spec match → skip work. Create excluded; nothing on dest to match against. diff --git a/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php b/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php index 2081838c..05ee2ae9 100644 --- a/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php +++ b/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php @@ -114,6 +114,35 @@ public function testReloadFailureMarksTheDatabaseFailed(): void ); } + public function testFailedDatabaseRetrySucceedsUnderOnDuplicateFail(): void + { + $database = new ReloadFailingProjectDatabase( + new MemoryAdapter(), + new Cache(new MemoryCache()), + ); + $this->createProjectDatabase(withStatus: true, database: $database); + $database->failNextDatabasesRead = true; + + $this->runDatabaseTransfer($database, explicit: false); + + $failed = $this->getDatabaseDocument($database); + $this->assertSame('failed', $failed->getAttribute('status')); + $this->assertTrue( + $database->getCollection('database_'.$failed->getSequence())->isEmpty(), + ); + + $destination = $this->runDatabaseTransfer($database, explicit: false); + + $recovered = $this->getDatabaseDocument($database); + $this->assertSame([], $this->errorMessages($destination)); + $this->assertSame('ready', $recovered->getAttribute('status')); + $this->assertSame($failed->getSequence(), $recovered->getSequence()); + $this->assertFalse( + $database->getCollection('database_'.$recovered->getSequence())->isEmpty(), + 'A Fail retry must recreate the backing collection for a previously failed database', + ); + } + private function createProjectDatabase(bool $withStatus, ?UtopiaDatabase $database = null): UtopiaDatabase { $database ??= new UtopiaDatabase( From 04ab893e1a9ab69967ec8d7ab7a4eb4e699ee1ce Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 21 Aug 2026 22:53:45 +1200 Subject: [PATCH 14/32] (refactor): use Schema\Order and wrap Collection constructors Index types already used IndexType; column direction was still a raw ASC string. Collection constructors with multiple named params were also jammed on one line. --- bin/MigrationCLI.php | 7 +- composer.json | 4 + composer.lock | 59 +++++++++--- src/Migration/Destinations/Appwrite.php | 18 +++- .../AppwriteDatabaseStatusTest.php | 5 +- .../Destinations/AppwriteIndexLengthsTest.php | 96 ++++++++++--------- 6 files changed, 125 insertions(+), 64 deletions(-) diff --git a/bin/MigrationCLI.php b/bin/MigrationCLI.php index e6681148..3d472faf 100644 --- a/bin/MigrationCLI.php +++ b/bin/MigrationCLI.php @@ -21,6 +21,7 @@ use Utopia\Migration\Transfer; use Utopia\Query\Schema\ColumnType; use Utopia\Query\Schema\IndexType; +use Utopia\Query\Schema\Order; /** * Migrations CLI Tool @@ -134,21 +135,21 @@ class MigrationCLI 'type' => IndexType::Key->value, 'attributes' => ['name'], 'lengths' => [Database::LENGTH_KEY], - 'orders' => ['ASC'], + 'orders' => [Order::Asc->value], ], [ '$id' => '_key_enabled', 'type' => IndexType::Key->value, 'attributes' => ['enabled'], 'lengths' => [], - 'orders' => ['ASC'], + 'orders' => [Order::Asc->value], ], [ '$id' => '_key_documentSecurity', 'type' => IndexType::Key->value, 'attributes' => ['documentSecurity'], 'lengths' => [], - 'orders' => ['ASC'], + 'orders' => [Order::Asc->value], ], ], ]; diff --git a/composer.json b/composer.json index c2edd013..27ef7b47 100644 --- a/composer.json +++ b/composer.json @@ -50,6 +50,10 @@ "type": "vcs", "url": "https://github.com/utopia-php/database.git" }, + { + "type": "vcs", + "url": "https://github.com/utopia-php/query.git" + }, { "type": "vcs", "url": "https://github.com/utopia-php/async.git" diff --git a/composer.lock b/composer.lock index 3f7ce063..780b84b5 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "669e0b5ccd20bfaf8cacb8461b605d49", + "content-hash": "2004ea1027b967193c5a9247cff21c35", "packages": [ { "name": "adhocore/jwt", @@ -2612,12 +2612,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "196db02d6e2eadb4d42c192d5a0f260c06b3e1db" + "reference": "32823ded323e35c4bd5029cf904cc8b269b62b06" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/196db02d6e2eadb4d42c192d5a0f260c06b3e1db", - "reference": "196db02d6e2eadb4d42c192d5a0f260c06b3e1db", + "url": "https://api.github.com/repos/utopia-php/database/zipball/32823ded323e35c4bd5029cf904cc8b269b62b06", + "reference": "32823ded323e35c4bd5029cf904cc8b269b62b06", "shasum": "" }, "require": { @@ -2631,7 +2631,7 @@ "utopia-php/console": "^0.1", "utopia-php/mongo": "^1.0", "utopia-php/pools": "^2.0", - "utopia-php/query": "^0.5", + "utopia-php/query": "dev-feat-schema-order as 0.5.0", "utopia-php/validators": "^0.4 || ^0.5" }, "require-dev": { @@ -2702,7 +2702,7 @@ "source": "https://github.com/utopia-php/database/tree/feat-query-lib", "issues": "https://github.com/utopia-php/database/issues" }, - "time": "2026-08-21T09:27:18+00:00" + "time": "2026-08-21T10:48:05+00:00" }, { "name": "utopia-php/dsn", @@ -2916,16 +2916,16 @@ }, { "name": "utopia-php/query", - "version": "0.5.0", + "version": "dev-feat-schema-order", "source": { "type": "git", "url": "https://github.com/utopia-php/query.git", - "reference": "802821c6fb0470410e1f0e65561cc16224c343c0" + "reference": "08b8e70214ae2d9315762afc1a6135c6864f7f7c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/query/zipball/802821c6fb0470410e1f0e65561cc16224c343c0", - "reference": "802821c6fb0470410e1f0e65561cc16224c343c0", + "url": "https://api.github.com/repos/utopia-php/query/zipball/08b8e70214ae2d9315762afc1a6135c6864f7f7c", + "reference": "08b8e70214ae2d9315762afc1a6135c6864f7f7c", "shasum": "" }, "require": { @@ -2945,7 +2945,38 @@ "Utopia\\Query\\": "src/Query" } }, - "notification-url": "https://packagist.org/downloads/", + "autoload-dev": { + "psr-4": { + "Tests\\Query\\": "tests/Query", + "Tests\\Integration\\": "tests/Integration" + } + }, + "scripts": { + "test": [ + "vendor/bin/paratest --testsuite Query --processes=auto --exclude-group=performance" + ], + "test:coverage": [ + "vendor/bin/paratest --testsuite Query --processes=auto --exclude-group=performance --coverage-php coverage/unit.cov" + ], + "test:performance": [ + "vendor/bin/phpunit --testsuite Query --group=performance" + ], + "test:integration": [ + "vendor/bin/phpunit --testsuite Integration" + ], + "test:integration:coverage": [ + "vendor/bin/phpunit --testsuite Integration --coverage-php coverage/integration.cov" + ], + "lint": [ + "php -d memory_limit=2G ./vendor/bin/pint --test" + ], + "format": [ + "php -d memory_limit=2G ./vendor/bin/pint" + ], + "check": [ + "./vendor/bin/phpstan analyse --level max src tests --memory-limit 2G" + ] + }, "license": [ "MIT" ], @@ -2958,10 +2989,10 @@ "utopia" ], "support": { - "issues": "https://github.com/utopia-php/query/issues", - "source": "https://github.com/utopia-php/query/tree/0.5.0" + "source": "https://github.com/utopia-php/query/tree/feat-schema-order", + "issues": "https://github.com/utopia-php/query/issues" }, - "time": "2026-08-21T06:16:10+00:00" + "time": "2026-08-21T10:44:38+00:00" }, { "name": "utopia-php/span", diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index 3a2dc781..ce082875 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -744,7 +744,11 @@ protected function createDatabase(Database $resource): bool try { $structure = $this->collectionStructureFor($resource); - $this->dbForProject->createCollection(new Collection(id: $this->databaseCollectionId($existing), attributes: $this->schemaAttributes($structure['attributes'] ?? []), indexes: $this->schemaIndexes($structure['indexes'] ?? []))); + $this->dbForProject->createCollection(new Collection( + id: $this->databaseCollectionId($existing), + attributes: $this->schemaAttributes($structure['attributes'] ?? []), + indexes: $this->schemaIndexes($structure['indexes'] ?? []), + )); } catch (\Throwable $e) { $this->markDatabaseFailed($resource->getId()); throw $e; @@ -796,7 +800,11 @@ protected function createDatabase(Database $resource): bool $resource->setSequence($database->getSequence()); $structure = $this->collectionStructureFor($resource); - $this->dbForProject->createCollection(new Collection(id: $this->databaseCollectionId($database), attributes: $this->schemaAttributes($structure['attributes'] ?? []), indexes: $this->schemaIndexes($structure['indexes'] ?? []))); + $this->dbForProject->createCollection(new Collection( + id: $this->databaseCollectionId($database), + attributes: $this->schemaAttributes($structure['attributes'] ?? []), + indexes: $this->schemaIndexes($structure['indexes'] ?? []), + )); } catch (\Throwable $e) { $this->markDatabaseFailed($resource->getId()); throw $e; @@ -920,7 +928,11 @@ protected function createEntity(Table $resource): bool $resource->setSequence($table->getSequence()); - $dbForDatabases->createCollection(new Collection(id: $this->tableCollectionId($database, $table), permissions: $resource->getPermissions(), documentSecurity: $resource->getRowSecurity())); + $dbForDatabases->createCollection(new Collection( + id: $this->tableCollectionId($database, $table), + permissions: $resource->getPermissions(), + documentSecurity: $resource->getRowSecurity(), + )); return true; } diff --git a/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php b/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php index 05ee2ae9..40a56178 100644 --- a/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php +++ b/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php @@ -167,7 +167,10 @@ private function createProjectDatabase(bool $withStatus, ?UtopiaDatabase $databa $attributes[] = $this->attribute('status', ColumnType::String, size: 16); } - $database->createCollection(new Collection(id: 'databases', attributes: $attributes)); + $database->createCollection(new Collection( + id: 'databases', + attributes: $attributes, + )); return $database; } diff --git a/tests/Migration/Unit/Destinations/AppwriteIndexLengthsTest.php b/tests/Migration/Unit/Destinations/AppwriteIndexLengthsTest.php index 084ed576..2cb3dd1a 100644 --- a/tests/Migration/Unit/Destinations/AppwriteIndexLengthsTest.php +++ b/tests/Migration/Unit/Destinations/AppwriteIndexLengthsTest.php @@ -21,6 +21,7 @@ use Utopia\Migration\Resources\Database\Table; use Utopia\Migration\Transfer; use Utopia\Query\Schema\ColumnType; +use Utopia\Query\Schema\Order; use Utopia\Tests\Unit\Adapters\MockSource; /** @@ -178,7 +179,7 @@ private function transferIndex( type: 'key', columns: ['reference', 'channel'], lengths: $withLengths, - orders: ['ASC', 'ASC'], + orders: [Order::Asc->value, Order::Asc->value], createdAt: $updatedAt, updatedAt: $updatedAt, ); @@ -247,48 +248,57 @@ private function projectDatabase(): UtopiaDatabase ->setNamespace('_project'); $database->create(); - $database->createCollection(new Collection(id: 'databases', attributes: [ - $this->attribute('name', ColumnType::String, required: true, size: 256), - $this->attribute('enabled', ColumnType::Boolean, default: true), - $this->attribute('search', ColumnType::String, size: 16384), - $this->attribute('originalId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), - $this->attribute('type', ColumnType::String, default: 'tablesdb', size: 128), - $this->attribute('database', ColumnType::String, size: 2000), - ])); - - $database->createCollection(new Collection(id: 'attributes', attributes: [ - $this->attribute('key', ColumnType::String, size: 256), - $this->attribute('databaseInternalId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), - $this->attribute('databaseId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), - $this->attribute('collectionInternalId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), - $this->attribute('collectionId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), - $this->attribute('type', ColumnType::String, size: 256), - $this->attribute('status', ColumnType::String, size: 64), - $this->attribute('size', ColumnType::Integer), - $this->attribute('required', ColumnType::Boolean, default: false), - $this->attribute('signed', ColumnType::Boolean, default: true), - $this->attribute('default', ColumnType::String, size: 16384), - $this->attribute('array', ColumnType::Boolean, default: false), - $this->attribute('format', ColumnType::String, size: 64), - $this->attribute('formatOptions', ColumnType::String, size: 16384, filters: ['json']), - $this->attribute('filters', ColumnType::String, size: 64, array: true), - $this->attribute('options', ColumnType::String, size: 16384, filters: ['json']), - $this->attribute('error', ColumnType::String, size: 2048), - ])); - - $database->createCollection(new Collection(id: 'indexes', attributes: [ - $this->attribute('key', ColumnType::String, size: 256), - $this->attribute('status', ColumnType::String, size: 64), - $this->attribute('databaseInternalId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), - $this->attribute('databaseId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), - $this->attribute('collectionInternalId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), - $this->attribute('collectionId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), - $this->attribute('type', ColumnType::String, size: 16), - $this->attribute('attributes', ColumnType::String, size: 256, array: true), - $this->attribute('lengths', ColumnType::Integer, array: true), - $this->attribute('orders', ColumnType::String, size: 4, array: true), - $this->attribute('error', ColumnType::String, size: 2048), - ])); + $database->createCollection(new Collection( + id: 'databases', + attributes: [ + $this->attribute('name', ColumnType::String, required: true, size: 256), + $this->attribute('enabled', ColumnType::Boolean, default: true), + $this->attribute('search', ColumnType::String, size: 16384), + $this->attribute('originalId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('type', ColumnType::String, default: 'tablesdb', size: 128), + $this->attribute('database', ColumnType::String, size: 2000), + ], + )); + + $database->createCollection(new Collection( + id: 'attributes', + attributes: [ + $this->attribute('key', ColumnType::String, size: 256), + $this->attribute('databaseInternalId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('databaseId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('collectionInternalId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('collectionId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('type', ColumnType::String, size: 256), + $this->attribute('status', ColumnType::String, size: 64), + $this->attribute('size', ColumnType::Integer), + $this->attribute('required', ColumnType::Boolean, default: false), + $this->attribute('signed', ColumnType::Boolean, default: true), + $this->attribute('default', ColumnType::String, size: 16384), + $this->attribute('array', ColumnType::Boolean, default: false), + $this->attribute('format', ColumnType::String, size: 64), + $this->attribute('formatOptions', ColumnType::String, size: 16384, filters: ['json']), + $this->attribute('filters', ColumnType::String, size: 64, array: true), + $this->attribute('options', ColumnType::String, size: 16384, filters: ['json']), + $this->attribute('error', ColumnType::String, size: 2048), + ], + )); + + $database->createCollection(new Collection( + id: 'indexes', + attributes: [ + $this->attribute('key', ColumnType::String, size: 256), + $this->attribute('status', ColumnType::String, size: 64), + $this->attribute('databaseInternalId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('databaseId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('collectionInternalId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('collectionId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('type', ColumnType::String, size: 16), + $this->attribute('attributes', ColumnType::String, size: 256, array: true), + $this->attribute('lengths', ColumnType::Integer, array: true), + $this->attribute('orders', ColumnType::String, size: 4, array: true), + $this->attribute('error', ColumnType::String, size: 2048), + ], + )); return $database; } From bf52c2c0408ab6a7d03091983a1c150e26fde631 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 27 Aug 2026 13:47:10 +1200 Subject: [PATCH 15/32] chore(deps): drop the deleted query branch from the lock and re-pin database The lock held utopia-php/query at dev-feat-schema-order, a branch that no longer exists on the remote, so the resolution only survived as long as nobody resolved it again. database's branch requires query 0.6.*, which is released, so this takes the release. storage 4.0.4 comes along because 4.0.3 capped utopia-php/validators at ^0.4 while database requires ^0.5; 4.0.4 dropped the validators dependency outright. database has required ^0.5 on main as well as on the branch, so this was already true before the query-lib work and only surfaced now that the lock moved. Co-Authored-By: Claude Opus 5 --- composer.lock | 59 ++++++++++++++++++++++++++------------------------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/composer.lock b/composer.lock index 780b84b5..a7b7e8a8 100644 --- a/composer.lock +++ b/composer.lock @@ -2612,12 +2612,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "32823ded323e35c4bd5029cf904cc8b269b62b06" + "reference": "c6ae48dbf0104ff4b6cb8a0ac173ab0dfcd23ef7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/32823ded323e35c4bd5029cf904cc8b269b62b06", - "reference": "32823ded323e35c4bd5029cf904cc8b269b62b06", + "url": "https://api.github.com/repos/utopia-php/database/zipball/c6ae48dbf0104ff4b6cb8a0ac173ab0dfcd23ef7", + "reference": "c6ae48dbf0104ff4b6cb8a0ac173ab0dfcd23ef7", "shasum": "" }, "require": { @@ -2627,18 +2627,19 @@ "ext-redis": "*", "php": ">=8.5", "utopia-php/async": "^0.1", - "utopia-php/cache": "^4.0", - "utopia-php/console": "^0.1", - "utopia-php/mongo": "^1.0", - "utopia-php/pools": "^2.0", - "utopia-php/query": "dev-feat-schema-order as 0.5.0", - "utopia-php/validators": "^0.4 || ^0.5" + "utopia-php/cache": "^4.0 || ^5.0", + "utopia-php/console": "0.1.*", + "utopia-php/mongo": "1.*", + "utopia-php/pools": "2.*", + "utopia-php/query": "0.6.*", + "utopia-php/validators": "^0.5" }, "require-dev": { "brianium/paratest": "7.20.*", "fakerphp/faker": "1.23.*", "laravel/pint": "*", "phpstan/phpstan": "2.1.*", + "phpstan/phpstan-phpunit": "2.0.*", "phpunit/phpunit": "12.5.*", "rregeer/phpunit-coverage-check": "0.3.*", "swoole/ide-helper": "5.1.3", @@ -2702,7 +2703,7 @@ "source": "https://github.com/utopia-php/database/tree/feat-query-lib", "issues": "https://github.com/utopia-php/database/issues" }, - "time": "2026-08-21T10:48:05+00:00" + "time": "2026-08-27T01:20:20+00:00" }, { "name": "utopia-php/dsn", @@ -2916,16 +2917,16 @@ }, { "name": "utopia-php/query", - "version": "dev-feat-schema-order", + "version": "0.6.0", "source": { "type": "git", "url": "https://github.com/utopia-php/query.git", - "reference": "08b8e70214ae2d9315762afc1a6135c6864f7f7c" + "reference": "abaebb2f3426bdbc6f44bab04254a668de937148" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/query/zipball/08b8e70214ae2d9315762afc1a6135c6864f7f7c", - "reference": "08b8e70214ae2d9315762afc1a6135c6864f7f7c", + "url": "https://api.github.com/repos/utopia-php/query/zipball/abaebb2f3426bdbc6f44bab04254a668de937148", + "reference": "abaebb2f3426bdbc6f44bab04254a668de937148", "shasum": "" }, "require": { @@ -2989,10 +2990,10 @@ "utopia" ], "support": { - "source": "https://github.com/utopia-php/query/tree/feat-schema-order", + "source": "https://github.com/utopia-php/query/tree/0.6.0", "issues": "https://github.com/utopia-php/query/issues" }, - "time": "2026-08-21T10:44:38+00:00" + "time": "2026-08-21T11:03:36+00:00" }, { "name": "utopia-php/span", @@ -3036,16 +3037,16 @@ }, { "name": "utopia-php/storage", - "version": "4.0.3", + "version": "4.0.4", "source": { "type": "git", "url": "https://github.com/utopia-php/storage.git", - "reference": "f8b122e753c01f30f774d390d7a36f030dfaef46" + "reference": "4be424e24022b7f25a4a0a60a0f4dbf45fccc02d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/storage/zipball/f8b122e753c01f30f774d390d7a36f030dfaef46", - "reference": "f8b122e753c01f30f774d390d7a36f030dfaef46", + "url": "https://api.github.com/repos/utopia-php/storage/zipball/4be424e24022b7f25a4a0a60a0f4dbf45fccc02d", + "reference": "4be424e24022b7f25a4a0a60a0f4dbf45fccc02d", "shasum": "" }, "require": { @@ -3058,7 +3059,7 @@ "utopia-php/client": "0.2.* || 0.3.*", "utopia-php/psr7": "0.2.*", "utopia-php/telemetry": "^0.4.6", - "utopia-php/validators": "^0.4" + "utopia-php/validators": "^0.5" }, "type": "library", "autoload": { @@ -3080,9 +3081,9 @@ ], "support": { "issues": "https://github.com/utopia-php/storage/issues", - "source": "https://github.com/utopia-php/storage/tree/4.0.3" + "source": "https://github.com/utopia-php/storage/tree/4.0.4" }, - "time": "2026-08-10T04:51:03+00:00" + "time": "2026-08-14T06:49:12+00:00" }, { "name": "utopia-php/telemetry", @@ -3137,16 +3138,16 @@ }, { "name": "utopia-php/validators", - "version": "0.4.2", + "version": "0.5.0", "source": { "type": "git", "url": "https://github.com/utopia-php/validators.git", - "reference": "a5b7b78b789e5af0a30f96da8355bbf48fb56699" + "reference": "ea6c1ad13019c8a088866cf422c232928de5a25e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/validators/zipball/a5b7b78b789e5af0a30f96da8355bbf48fb56699", - "reference": "a5b7b78b789e5af0a30f96da8355bbf48fb56699", + "url": "https://api.github.com/repos/utopia-php/validators/zipball/ea6c1ad13019c8a088866cf422c232928de5a25e", + "reference": "ea6c1ad13019c8a088866cf422c232928de5a25e", "shasum": "" }, "require": { @@ -3171,9 +3172,9 @@ ], "support": { "issues": "https://github.com/utopia-php/validators/issues", - "source": "https://github.com/utopia-php/validators/tree/0.4.2" + "source": "https://github.com/utopia-php/validators/tree/0.5.0" }, - "time": "2026-08-11T07:43:49+00:00" + "time": "2026-08-14T05:12:07+00:00" } ], "packages-dev": [ From 24e0f032760a4b619f711a42107cdb94c6dccc87 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 27 Aug 2026 13:59:29 +1200 Subject: [PATCH 16/32] fix(destination): map index orders onto Order before building the index The Appwrite destination passed the 'ASC'/'DESC' strings a source hands back straight into Utopia\Database\Index, which takes Order cases and rejects anything else. The resulting InvalidArgumentException is not a Migration Exception, so instead of recording a failed index the transfer aborted. This was live before the lock re-pin and simply could not be seen: the lock held utopia-php/query at a deleted branch whose Index took plain strings, so CI never built an index against the contract the released library actually has. AppwriteIndexLengthsTest covers it -- two of its cases pass ['ASC', 'ASC'] and went red on the first run against the re-pinned lock. Co-Authored-By: Claude Opus 5 --- src/Migration/Destinations/Appwrite.php | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index ce082875..72f426b4 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -90,6 +90,7 @@ use Utopia\Query\Schema\ColumnType; use Utopia\Query\Schema\ForeignKeyAction; use Utopia\Query\Schema\IndexType; +use Utopia\Query\Schema\Order; class Appwrite extends Destination { @@ -1587,7 +1588,15 @@ protected function createIndex(Index $resource): bool type: IndexType::from($resource->getType()), attributes: $resource->getColumns(), lengths: $lengths, - orders: $resource->getOrders(), + // Sources hand back the 'ASC'/'DESC' strings they were stored + // as, while UtopiaIndex takes Order cases and rejects anything + // else with an InvalidArgumentException -- which is not a + // Migration Exception, so the transfer would abort instead of + // recording a failed index. + orders: \array_map( + static fn (mixed $order): ?Order => Order::tryFrom(\is_string($order) ? \strtoupper($order) : ''), + $resource->getOrders(), + ), ), ); From ff6f1d583d4d53924a1d3f7c7461d1aec8cbf3e1 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 27 Aug 2026 15:32:09 +1200 Subject: [PATCH 17/32] fix(destination): let a database stranded in provisioning be retried Recovery was gated on `failed` alone, but `provisioning` is reachable as a terminal state on its own. markDatabaseFailed() deliberately swallows its own error so a secondary failure cannot mask the caller's throw, so a metadata store that is down for both the reload and the status write leaves the document in `provisioning` with no backing collection. Under the default OnDuplicate::Fail that document was unrecoverable: the gate opened only for `failed`, so every retry fell through to createDocument and hit "Document already exists", and the backing collection was never created. The `provisioning` handling further down, in the Skip branch, sat inside the same gate and so was unreachable in exactly the case it was written for. Both states mean the same thing -- a prior run created the metadata and did not finish -- so the predicate now covers both. The collection is still only recreated when it is actually missing, which was already the guard. testProvisioningDatabaseRetrySucceedsUnderOnDuplicateFail drives the real writer rather than seeding a status: it fails the reload and then the status write, asserts the document is stranded in `provisioning` with no collection, and then asserts the retry recovers it. Seen red as "Document already exists". Found by Greptile on #222. Co-Authored-By: Claude Opus 5 --- src/Migration/Destinations/Appwrite.php | 22 ++++-- .../AppwriteDatabaseStatusTest.php | 73 +++++++++++++++++++ 2 files changed, 89 insertions(+), 6 deletions(-) diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index 72f426b4..44c4fe39 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -684,18 +684,28 @@ protected function createDatabase(Database $resource): bool $updatedAt = $this->normalizeDateTime($resource->getUpdatedAt(), $createdAt); $existing = $this->dbForProject->getDocument(self::META_DATABASES, $resource->getId()); - $isFailed = ! $existing->isEmpty() + // Both states mean a prior run created the metadata document and never finished. + // `provisioning` is reachable on its own: markDatabaseFailed() swallows its own + // error so it cannot mask the caller's throw, so a metadata store that is down + // for the reload and the status write strands the document there. Recovering only + // `failed` left those stranded documents unretryable -- every retry collided with + // the existing metadata id and never created the backing collection. + $isIncomplete = ! $existing->isEmpty() && $this->getSupportForDatabaseStatus() - && $existing->getAttribute('status') === self::DATABASE_STATUS_FAILED; + && \in_array( + $existing->getAttribute('status'), + [self::DATABASE_STATUS_FAILED, self::DATABASE_STATUS_PROVISIONING], + true, + ); - if ($this->onDuplicate !== OnDuplicate::Fail || $isFailed) { + if ($this->onDuplicate !== OnDuplicate::Fail || $isIncomplete) { $action = $this->onDuplicate->resolveSchemaAction( !$existing->isEmpty(), $updatedAt, $existing->getUpdatedAt(), ); - if ($isFailed) { + if ($isIncomplete) { // A prior run created the metadata document but left the database unusable (its backing // collection may be missing). Force Overwrite — regardless of timestamps, spec match, or // OnDuplicate::Fail — so retries recreate the collection instead of hitting the existing ID. @@ -719,7 +729,7 @@ protected function createDatabase(Database $resource): bool } return false; })(), - SchemaAction::Overwrite => (function () use ($resource, $existing, $updatedAt, $isFailed): bool { + SchemaAction::Overwrite => (function () use ($resource, $existing, $updatedAt, $isIncomplete): bool { $document = [ 'name' => $resource->getDatabaseName(), 'search' => implode(' ', [$resource->getId(), $resource->getDatabaseName()]), @@ -741,7 +751,7 @@ protected function createDatabase(Database $resource): bool // metadata document but threw before createCollection). Recreate it so we never flip a // database to ready with no collection behind it. A healthy overwrite already has its // collection, so we skip the lookup entirely. - if ($isFailed && $this->dbForProject->getCollection($this->databaseCollectionId($existing))->isEmpty()) { + if ($isIncomplete && $this->dbForProject->getCollection($this->databaseCollectionId($existing))->isEmpty()) { try { $structure = $this->collectionStructureFor($resource); diff --git a/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php b/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php index 40a56178..e2fb3eb9 100644 --- a/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php +++ b/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php @@ -11,6 +11,7 @@ use Utopia\Database\Collection; use Utopia\Database\Database as UtopiaDatabase; use Utopia\Database\Document as UtopiaDocument; +use Utopia\Database\Exception as DatabaseException; use Utopia\Migration\Destinations\Appwrite as AppwriteDestination; use Utopia\Migration\Destinations\OnDuplicate; use Utopia\Migration\Resource; @@ -53,6 +54,43 @@ public function getDocument(string $collection, string $id, array $queries = [], } } +/** + * Fails the reload and then the status write that would record the failure, which + * is the only way production reaches a document stranded in `provisioning`: + * markDatabaseFailed() swallows its own error so it cannot mask the original throw. + */ +class StrandedProvisioningProjectDatabase extends UtopiaDatabase +{ + public bool $failNextDatabasesRead = false; + + public bool $failNextDatabasesWrite = false; + + #[Override] + public function getDocument(string $collection, string $id, array $queries = [], bool $forUpdate = false): UtopiaDocument + { + $document = parent::getDocument($collection, $id, $queries, $forUpdate); + if ($this->failNextDatabasesRead && $collection === 'databases' && !$document->isEmpty()) { + $this->failNextDatabasesRead = false; + + return new UtopiaDocument(); + } + + return $document; + } + + #[Override] + public function updateDocument(string $collection, string $id, UtopiaDocument $document): UtopiaDocument + { + if ($this->failNextDatabasesWrite && $collection === 'databases') { + $this->failNextDatabasesWrite = false; + + throw new DatabaseException('metadata store unavailable'); + } + + return parent::updateDocument($collection, $id, $document); + } +} + final class AppwriteDatabaseStatusTest extends TestCase { public function testDatabaseCreationOmitsStatusThroughLegacyAndExplicitEntrypoints(): void @@ -143,6 +181,41 @@ public function testFailedDatabaseRetrySucceedsUnderOnDuplicateFail(): void ); } + public function testProvisioningDatabaseRetrySucceedsUnderOnDuplicateFail(): void + { + $database = new StrandedProvisioningProjectDatabase( + new MemoryAdapter(), + new Cache(new MemoryCache()), + ); + $this->createProjectDatabase(withStatus: true, database: $database); + $database->failNextDatabasesRead = true; + $database->failNextDatabasesWrite = true; + + $this->runDatabaseTransfer($database, explicit: false); + + $stranded = $this->getDatabaseDocument($database); + $this->assertSame( + 'provisioning', + $stranded->getAttribute('status'), + 'The reload threw and the write that would record the failure threw too, so the document is stranded mid-provision', + ); + $this->assertTrue( + $database->getCollection('database_'.$stranded->getSequence())->isEmpty(), + 'The backing collection was never created, so the database is unusable', + ); + + $destination = $this->runDatabaseTransfer($database, explicit: false); + + $recovered = $this->getDatabaseDocument($database); + $this->assertSame([], $this->errorMessages($destination)); + $this->assertSame('ready', $recovered->getAttribute('status')); + $this->assertSame($stranded->getSequence(), $recovered->getSequence()); + $this->assertFalse( + $database->getCollection('database_'.$recovered->getSequence())->isEmpty(), + 'A Fail retry must recover a database stranded in provisioning, not keep colliding with its metadata id', + ); + } + private function createProjectDatabase(bool $withStatus, ?UtopiaDatabase $database = null): UtopiaDatabase { $database ??= new UtopiaDatabase( From a40d6a3d36586203ded2b198503ebbc9cd1f0879 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Sat, 29 Aug 2026 22:42:39 +1200 Subject: [PATCH 18/32] (chore): align database dependency with query-lib head --- composer.lock | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/composer.lock b/composer.lock index a7b7e8a8..d88fa05a 100644 --- a/composer.lock +++ b/composer.lock @@ -2612,12 +2612,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "c6ae48dbf0104ff4b6cb8a0ac173ab0dfcd23ef7" + "reference": "6d71e52ff72e0d908e9d45a3fe318bd43edf9d6f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/c6ae48dbf0104ff4b6cb8a0ac173ab0dfcd23ef7", - "reference": "c6ae48dbf0104ff4b6cb8a0ac173ab0dfcd23ef7", + "url": "https://api.github.com/repos/utopia-php/database/zipball/6d71e52ff72e0d908e9d45a3fe318bd43edf9d6f", + "reference": "6d71e52ff72e0d908e9d45a3fe318bd43edf9d6f", "shasum": "" }, "require": { @@ -2632,7 +2632,7 @@ "utopia-php/mongo": "1.*", "utopia-php/pools": "2.*", "utopia-php/query": "0.6.*", - "utopia-php/validators": "^0.5" + "utopia-php/validators": "^0.6" }, "require-dev": { "brianium/paratest": "7.20.*", @@ -2703,7 +2703,7 @@ "source": "https://github.com/utopia-php/database/tree/feat-query-lib", "issues": "https://github.com/utopia-php/database/issues" }, - "time": "2026-08-27T01:20:20+00:00" + "time": "2026-08-28T02:21:00+00:00" }, { "name": "utopia-php/dsn", @@ -3037,16 +3037,16 @@ }, { "name": "utopia-php/storage", - "version": "4.0.4", + "version": "4.0.5", "source": { "type": "git", "url": "https://github.com/utopia-php/storage.git", - "reference": "4be424e24022b7f25a4a0a60a0f4dbf45fccc02d" + "reference": "593e732644ac809df18ae45b5561acf0a0c5e1be" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/storage/zipball/4be424e24022b7f25a4a0a60a0f4dbf45fccc02d", - "reference": "4be424e24022b7f25a4a0a60a0f4dbf45fccc02d", + "url": "https://api.github.com/repos/utopia-php/storage/zipball/593e732644ac809df18ae45b5561acf0a0c5e1be", + "reference": "593e732644ac809df18ae45b5561acf0a0c5e1be", "shasum": "" }, "require": { @@ -3059,7 +3059,7 @@ "utopia-php/client": "0.2.* || 0.3.*", "utopia-php/psr7": "0.2.*", "utopia-php/telemetry": "^0.4.6", - "utopia-php/validators": "^0.5" + "utopia-php/validators": "^0.6" }, "type": "library", "autoload": { @@ -3081,9 +3081,9 @@ ], "support": { "issues": "https://github.com/utopia-php/storage/issues", - "source": "https://github.com/utopia-php/storage/tree/4.0.4" + "source": "https://github.com/utopia-php/storage/tree/4.0.5" }, - "time": "2026-08-14T06:49:12+00:00" + "time": "2026-08-27T06:13:20+00:00" }, { "name": "utopia-php/telemetry", @@ -3138,16 +3138,16 @@ }, { "name": "utopia-php/validators", - "version": "0.5.0", + "version": "0.6.0", "source": { "type": "git", "url": "https://github.com/utopia-php/validators.git", - "reference": "ea6c1ad13019c8a088866cf422c232928de5a25e" + "reference": "7afdde56a7a635f0cb3d8a5e22ebbef40baf31dc" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/validators/zipball/ea6c1ad13019c8a088866cf422c232928de5a25e", - "reference": "ea6c1ad13019c8a088866cf422c232928de5a25e", + "url": "https://api.github.com/repos/utopia-php/validators/zipball/7afdde56a7a635f0cb3d8a5e22ebbef40baf31dc", + "reference": "7afdde56a7a635f0cb3d8a5e22ebbef40baf31dc", "shasum": "" }, "require": { @@ -3172,9 +3172,9 @@ ], "support": { "issues": "https://github.com/utopia-php/validators/issues", - "source": "https://github.com/utopia-php/validators/tree/0.5.0" + "source": "https://github.com/utopia-php/validators/tree/0.6.0" }, - "time": "2026-08-14T05:12:07+00:00" + "time": "2026-08-27T04:46:08+00:00" } ], "packages-dev": [ From b38b3c76fd1d8787fbed7a8cd1013dc8b790322f Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Sun, 30 Aug 2026 01:43:34 +1200 Subject: [PATCH 19/32] (chore): align database dependency with ready query-lib head --- composer.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/composer.lock b/composer.lock index d88fa05a..fba3f9bf 100644 --- a/composer.lock +++ b/composer.lock @@ -2612,12 +2612,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "6d71e52ff72e0d908e9d45a3fe318bd43edf9d6f" + "reference": "6a3149819c55b08802cafc9f33324dca9299fe2b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/6d71e52ff72e0d908e9d45a3fe318bd43edf9d6f", - "reference": "6d71e52ff72e0d908e9d45a3fe318bd43edf9d6f", + "url": "https://api.github.com/repos/utopia-php/database/zipball/6a3149819c55b08802cafc9f33324dca9299fe2b", + "reference": "6a3149819c55b08802cafc9f33324dca9299fe2b", "shasum": "" }, "require": { @@ -2703,7 +2703,7 @@ "source": "https://github.com/utopia-php/database/tree/feat-query-lib", "issues": "https://github.com/utopia-php/database/issues" }, - "time": "2026-08-28T02:21:00+00:00" + "time": "2026-08-29T13:33:39+00:00" }, { "name": "utopia-php/dsn", From 4c3a89ae4bd5fb9b3b4fe701fc4c45e8788d5d17 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Sun, 30 Aug 2026 01:55:51 +1200 Subject: [PATCH 20/32] (fix): prevent concurrent provisioning recovery --- src/Migration/Destinations/Appwrite.php | 52 +++++++------ .../AppwriteDatabaseStatusTest.php | 77 ++++++++++++++++++- 2 files changed, 101 insertions(+), 28 deletions(-) diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index 44c4fe39..2772e258 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -164,6 +164,15 @@ class Appwrite extends Destination */ protected $getDatabaseDSN; + /** + * Confirms that the operation which owns a `provisioning` database is no + * longer active. Callers must derive this from their operation lifecycle; + * without that proof, recovery fails closed. + * + * @var (callable(UtopiaDocument $database): bool)|null + */ + private $canRecoverDatabase; + /** * @var array */ @@ -213,6 +222,7 @@ class Appwrite extends Destination * @param OnDuplicate $onDuplicate Behavior when a row with an existing $id is encountered. * @param (callable(Database $resource): string)|null $getDatabaseDSN Resolver for the destination's `_databases.database` value. Pass when the destination project's DSN differs from the source's, so the destination row carries its own DSN instead of inheriting the source's. * @param array>> $collectionStructures Per-database-type metadata collection structures (e.g. `['vectorsdb' => ...]`), used instead of $collectionStructure when the imported database's type has an entry. Types with an entry also get type-specific metadata written (e.g. vectorsdb collection `dimension`). + * @param (callable(UtopiaDocument $database): bool)|null $canRecoverDatabase Returns true only when the operation which owns an existing `provisioning` database is confirmed terminal. Null refuses recovery so concurrent provisioning cannot be overwritten. */ public function __construct( string $project, @@ -226,6 +236,7 @@ public function __construct( protected OnDuplicate $onDuplicate = OnDuplicate::Fail, ?callable $getDatabaseDSN = null, protected array $collectionStructures = [], + ?callable $canRecoverDatabase = null, ) { $this->projectId = $project; $this->endpoint = $endpoint; @@ -247,6 +258,7 @@ public function __construct( $this->getDatabasesDB = $getDatabasesDB; $this->getDatabaseDSN = $getDatabaseDSN; + $this->canRecoverDatabase = $canRecoverDatabase; } /** @@ -684,19 +696,18 @@ protected function createDatabase(Database $resource): bool $updatedAt = $this->normalizeDateTime($resource->getUpdatedAt(), $createdAt); $existing = $this->dbForProject->getDocument(self::META_DATABASES, $resource->getId()); - // Both states mean a prior run created the metadata document and never finished. - // `provisioning` is reachable on its own: markDatabaseFailed() swallows its own - // error so it cannot mask the caller's throw, so a metadata store that is down - // for the reload and the status write strands the document there. Recovering only - // `failed` left those stranded documents unretryable -- every retry collided with - // the existing metadata id and never created the backing collection. - $isIncomplete = ! $existing->isEmpty() - && $this->getSupportForDatabaseStatus() - && \in_array( - $existing->getAttribute('status'), - [self::DATABASE_STATUS_FAILED, self::DATABASE_STATUS_PROVISIONING], - true, - ); + $supportsStatus = ! $existing->isEmpty() && $this->getSupportForDatabaseStatus(); + $status = $supportsStatus ? $existing->getAttribute('status') : null; + $isProvisioning = $status === self::DATABASE_STATUS_PROVISIONING; + $canRecoverProvisioning = $isProvisioning + && $this->canRecoverDatabase !== null + && ($this->canRecoverDatabase)($existing); + + if ($isProvisioning && ! $canRecoverProvisioning) { + throw new DatabaseException('Database '.$resource->getId().' is already being provisioned'); + } + + $isIncomplete = $status === self::DATABASE_STATUS_FAILED || $canRecoverProvisioning; if ($this->onDuplicate !== OnDuplicate::Fail || $isIncomplete) { $action = $this->onDuplicate->resolveSchemaAction( @@ -719,14 +730,6 @@ protected function createDatabase(Database $resource): bool SchemaAction::Skip => (function () use ($resource, $existing): bool { $resource->setSequence($existing->getSequence()); $resource->setStatus(Resource::STATUS_SKIPPED, 'Already exists on destination'); - // Recover a database left in `provisioning` by a prior failed run: the spec matches so - // we skip re-import, but the end-of-run sweep should still flip it to `ready`. - if ( - $this->getSupportForDatabaseStatus() - && $existing->getAttribute('status') === self::DATABASE_STATUS_PROVISIONING - ) { - $this->provisioningDatabases[$resource->getId()] = true; - } return false; })(), SchemaAction::Overwrite => (function () use ($resource, $existing, $updatedAt, $isIncomplete): bool { @@ -747,10 +750,9 @@ protected function createDatabase(Database $resource): bool $this->dbForProject->updateDocument(self::META_DATABASES, $existing->getId(), new UtopiaDocument($document)); $resource->setSequence($existing->getSequence()); - // Only a `failed` database can be missing its backing collection (a prior run wrote the - // metadata document but threw before createCollection). Recreate it so we never flip a - // database to ready with no collection behind it. A healthy overwrite already has its - // collection, so we skip the lookup entirely. + // An incomplete database can be missing its backing collection. Recreate it so we never + // flip a database to ready with no collection behind it. A healthy overwrite already has + // its collection, so we skip the lookup entirely. if ($isIncomplete && $this->dbForProject->getCollection($this->databaseCollectionId($existing))->isEmpty()) { try { $structure = $this->collectionStructureFor($resource); diff --git a/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php b/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php index e2fb3eb9..42ad9eec 100644 --- a/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php +++ b/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php @@ -91,6 +91,30 @@ public function updateDocument(string $collection, string $id, UtopiaDocument $d } } +final class InterleavingProjectDatabase extends UtopiaDatabase +{ + public bool $interceptNextDatabasesReload = false; + + /** @var (callable(): void)|null */ + public $onDatabasesReload = null; + + #[Override] + public function getDocument(string $collection, string $id, array $queries = [], bool $forUpdate = false): UtopiaDocument + { + $document = parent::getDocument($collection, $id, $queries, $forUpdate); + if ( + $this->interceptNextDatabasesReload + && $collection === 'databases' + && ! $document->isEmpty() + ) { + $this->interceptNextDatabasesReload = false; + ($this->onDatabasesReload)(); + } + + return $document; + } +} + final class AppwriteDatabaseStatusTest extends TestCase { public function testDatabaseCreationOmitsStatusThroughLegacyAndExplicitEntrypoints(): void @@ -204,7 +228,11 @@ public function testProvisioningDatabaseRetrySucceedsUnderOnDuplicateFail(): voi 'The backing collection was never created, so the database is unusable', ); - $destination = $this->runDatabaseTransfer($database, explicit: false); + $destination = $this->runDatabaseTransfer( + $database, + explicit: false, + canRecoverDatabase: static fn (UtopiaDocument $existing): bool => true, + ); $recovered = $this->getDatabaseDocument($database); $this->assertSame([], $this->errorMessages($destination)); @@ -216,6 +244,45 @@ public function testProvisioningDatabaseRetrySucceedsUnderOnDuplicateFail(): voi ); } + public function testConcurrentMigrationDoesNotRecoverAnActivelyProvisioningDatabase(): void + { + $database = new InterleavingProjectDatabase( + new MemoryAdapter(), + new Cache(new MemoryCache()), + ); + $this->createProjectDatabase(withStatus: true, database: $database); + + $second = null; + $statusDuringOverlap = null; + $collectionExistsDuringOverlap = null; + $database->onDatabasesReload = function () use ( + $database, + &$second, + &$statusDuringOverlap, + &$collectionExistsDuringOverlap, + ): void { + $second = $this->runDatabaseTransfer($database, explicit: false); + $provisioning = $this->getDatabaseDocument($database); + $statusDuringOverlap = $provisioning->getAttribute('status'); + $collectionExistsDuringOverlap = ! $database + ->getCollection('database_'.$provisioning->getSequence()) + ->isEmpty(); + }; + $database->interceptNextDatabasesReload = true; + + $first = $this->runDatabaseTransfer($database, explicit: false); + + $created = $this->getDatabaseDocument($database); + $this->assertSame([], $this->errorMessages($first)); + $this->assertInstanceOf(CountingAppwriteDestination::class, $second); + $this->assertNotSame([], $this->errorMessages($second)); + $this->assertStringContainsString('already being provisioned', $this->errorMessages($second)[0]); + $this->assertSame('provisioning', $statusDuringOverlap); + $this->assertFalse($collectionExistsDuringOverlap); + $this->assertSame('ready', $created->getAttribute('status')); + $this->assertFalse($database->getCollection('database_'.$created->getSequence())->isEmpty()); + } + private function createProjectDatabase(bool $withStatus, ?UtopiaDatabase $database = null): UtopiaDatabase { $database ??= new UtopiaDatabase( @@ -264,8 +331,11 @@ private function attribute( ); } - private function runDatabaseTransfer(UtopiaDatabase $database, bool $explicit): CountingAppwriteDestination - { + private function runDatabaseTransfer( + UtopiaDatabase $database, + bool $explicit, + ?callable $canRecoverDatabase = null, + ): CountingAppwriteDestination { $source = new class () extends MockSource { #[Override] public function supportsDatabaseStatus(): bool @@ -291,6 +361,7 @@ public function supportsDatabaseStatus(): bool dbForPlatform: $database, projectInternalId: '1', onDuplicate: OnDuplicate::Fail, + canRecoverDatabase: $canRecoverDatabase, ); $transfer = new Transfer($source, $destination); From 519e57393271040ed1ca7fb39fb96b584b97cd7c Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Sun, 30 Aug 2026 02:07:10 +1200 Subject: [PATCH 21/32] (fix): require explicit CLI provisioning recovery --- Dockerfile | 1 + bin/MigrationCLI.php | 64 ++++++++-- composer.json | 2 +- tests/Migration/Unit/MigrationCLITest.php | 144 ++++++++++++++++++++++ 4 files changed, 199 insertions(+), 12 deletions(-) create mode 100644 tests/Migration/Unit/MigrationCLITest.php diff --git a/Dockerfile b/Dockerfile index 58a42e8d..c922149e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,6 +28,7 @@ RUN set -ex \ COPY ./src /app/src COPY ./tests /app/src/tests +COPY ./bin /app/bin COPY --from=composer /app/vendor /app/vendor diff --git a/bin/MigrationCLI.php b/bin/MigrationCLI.php index 3d472faf..688149b5 100644 --- a/bin/MigrationCLI.php +++ b/bin/MigrationCLI.php @@ -9,7 +9,6 @@ use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Database; use Utopia\Database\Document; -use Utopia\Database\Validator\Authorization; use Utopia\Migration\Destination; use Utopia\Migration\Destinations\Appwrite as DestinationsAppwrite; use Utopia\Migration\Destinations\Local; @@ -34,6 +33,9 @@ class MigrationCLI protected mixed $destination; + /** @var list */ + private readonly array $arguments; + protected const STRUCTURE = [ '$collection' => 'databases', '$id' => 'collections', @@ -154,6 +156,26 @@ class MigrationCLI ], ]; + /** @param list $arguments */ + public function __construct(array $arguments = []) + { + $this->arguments = $arguments; + } + + public static function getHelp(): string + { + return <<<'HELP' +Usage: php bin/MigrationCLI.php [options] + +Options: + -h, --help Show this help. + --recover-provisioning Attest that no active migration owns an existing + provisioning database and allow its recovery. + Recovery is refused by default. + +HELP; + } + /** * Prints the current status of migrations as a table after wiping the screen */ @@ -251,12 +273,24 @@ public function getDestination(): Destination { switch ($_ENV['DESTINATION_PROVIDER']) { case 'appwrite': + $database = $this->getDatabase('destination'); + return new DestinationsAppwrite( - $_ENV['DESTINATION_APPWRITE_TEST_PROJECT'], - $_ENV['DESTINATION_APPWRITE_TEST_ENDPOINT'], - $_ENV['DESTINATION_APPWRITE_TEST_KEY'], - $this->getDatabase('destination'), - self::STRUCTURE + project: $_ENV['DESTINATION_APPWRITE_TEST_PROJECT'], + endpoint: $_ENV['DESTINATION_APPWRITE_TEST_ENDPOINT'], + key: $_ENV['DESTINATION_APPWRITE_TEST_KEY'], + dbForProject: $database, + getDatabasesDB: static fn (Document $document): Database => $database, + collectionStructure: self::STRUCTURE, + dbForPlatform: $database, + projectInternalId: $_ENV['DESTINATION_APPWRITE_TEST_PROJECT_INTERNAL_ID'], + // The standalone operator sets this flag only after confirming the lifecycle owner is + // terminal. Without that explicit attestation, provisioning recovery fails closed. + canRecoverDatabase: fn (Document $document): bool => \in_array( + '--recover-provisioning', + $this->arguments, + true, + ), ); case 'local': return new Local('./localBackup'); @@ -397,6 +431,7 @@ function (mixed $value, Document $attribute) { $database ->setDatabase('appwrite') ->setNamespace('_' . $_ENV[$prefix . 'NAMESPACE']); + $database->getAuthorization()->disable(); return $database; } @@ -426,15 +461,22 @@ public function start(): void /** * Run Transfer */ - Authorization::skip(fn () => $this->transfer->run( + $this->transfer->run( $this->source->getSupportedResources(), function () { $this->drawFrame(); } - )); + ); } } -$instance = new MigrationCLI(); -$instance->start(); -$instance->drawFrame(); +if (\realpath($_SERVER['SCRIPT_FILENAME'] ?? '') === __FILE__) { + $arguments = $_SERVER['argv'] ?? []; + if (\in_array('-h', $arguments, true) || \in_array('--help', $arguments, true)) { + echo MigrationCLI::getHelp(); + } else { + $instance = new MigrationCLI($arguments); + $instance->start(); + $instance->drawFrame(); + } +} diff --git a/composer.json b/composer.json index 27ef7b47..5a4857ca 100644 --- a/composer.json +++ b/composer.json @@ -26,7 +26,7 @@ "test": "./vendor/bin/phpunit", "lint": "./vendor/bin/pint --test", "format": "./vendor/bin/pint", - "check": "./vendor/bin/phpstan analyse --level 3 src tests --memory-limit 2G" + "check": "./vendor/bin/phpstan analyse --level 3 src tests bin --memory-limit 2G" }, "require": { "php": ">=8.5", diff --git a/tests/Migration/Unit/MigrationCLITest.php b/tests/Migration/Unit/MigrationCLITest.php new file mode 100644 index 00000000..27c9d05e --- /dev/null +++ b/tests/Migration/Unit/MigrationCLITest.php @@ -0,0 +1,144 @@ + $arguments */ + public function __construct(array $arguments, private readonly Database $database) + { + parent::__construct($arguments); + } + + #[Override] + public function getDatabase(string $type): Database + { + return $this->database; + } +} + +#[BackupGlobals(true)] +final class MigrationCLITest extends TestCase +{ + public function testHelpExplainsExplicitProvisioningRecoveryAttestation(): void + { + $this->assertStringContainsString('--recover-provisioning', \MigrationCLI::getHelp()); + $this->assertStringContainsString('no active migration', \MigrationCLI::getHelp()); + $this->assertStringContainsString('refused by default', \MigrationCLI::getHelp()); + } + + public function testProvisioningRecoveryRequiresExplicitOperatorAttestation(): void + { + foreach ([false, true] as $recover) { + $database = $this->createProjectDatabase(); + $arguments = $recover ? ['MigrationCLI.php', '--recover-provisioning'] : ['MigrationCLI.php']; + $cli = new TestMigrationCLI($arguments, $database); + + $destination = $cli->getDestination(); + $this->runTransfer($database, $destination); + + $created = $database->getAuthorization()->skip( + static fn (): Document => $database->getDocument('databases', 'database'), + ); + + if (! $recover) { + $this->assertNotSame([], $destination->getErrors()); + $this->assertSame('provisioning', $created->getAttribute('status')); + $this->assertTrue($database->getCollection('database_'.$created->getSequence())->isEmpty()); + continue; + } + + $this->assertSame([], $destination->getErrors()); + $this->assertSame('ready', $created->getAttribute('status')); + $this->assertFalse($database->getCollection('database_'.$created->getSequence())->isEmpty()); + } + } + + private function createProjectDatabase(): Database + { + $database = new Database(new MemoryAdapter(), new Cache(new MemoryCache())); + $database + ->setDatabase('appwrite') + ->setNamespace('_project'); + $database->create(); + $database->createCollection(new Collection( + id: 'databases', + attributes: [ + new Attribute(key: 'name', type: ColumnType::String, size: 256, required: true), + new Attribute(key: 'enabled', type: ColumnType::Boolean, default: true), + new Attribute(key: 'search', type: ColumnType::String, size: 16384), + new Attribute(key: 'originalId', type: ColumnType::String, size: Database::LENGTH_KEY), + new Attribute(key: 'type', type: ColumnType::String, size: 128, default: 'tablesdb'), + new Attribute(key: 'database', type: ColumnType::String, size: 2000), + new Attribute(key: 'status', type: ColumnType::String, size: 16), + ], + )); + $database->getAuthorization()->skip( + static fn (): Document => $database->createDocument('databases', new Document([ + '$id' => 'database', + 'name' => 'Database', + 'enabled' => true, + 'search' => 'database Database', + 'originalId' => null, + 'type' => 'tablesdb', + 'database' => '', + 'status' => 'provisioning', + ])), + ); + + return $database; + } + + private function runTransfer(Database $database, Destination $destination): void + { + $source = new class () extends MockSource { + #[Override] + public function supportsDatabaseStatus(): bool + { + return true; + } + }; + $source->pushMockResource(new DatabaseResource( + id: 'database', + name: 'Database', + type: 'tablesdb', + database: 'source-dsn', + databaseStatus: 'ready', + )); + + $transfer = new Transfer($source, $destination); + $database->getAuthorization()->skip( + static fn () => $transfer->run([Resource::TYPE_DATABASE], static function (): void { + }), + ); + } + + protected function setUp(): void + { + parent::setUp(); + $_ENV['DESTINATION_PROVIDER'] = 'appwrite'; + $_ENV['DESTINATION_APPWRITE_TEST_PROJECT'] = 'destination-project'; + $_ENV['DESTINATION_APPWRITE_TEST_ENDPOINT'] = 'http://example.test/v1'; + $_ENV['DESTINATION_APPWRITE_TEST_KEY'] = 'test-key'; + $_ENV['DESTINATION_APPWRITE_TEST_PROJECT_INTERNAL_ID'] = '1'; + } +} From 8c2776da637ed67332a18062f59827819811bc5d Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Sun, 30 Aug 2026 02:12:25 +1200 Subject: [PATCH 22/32] (fix): require provisioning lifecycle policy --- src/Migration/Destinations/Appwrite.php | 7 +++---- .../Unit/Destinations/AppwriteDatabaseStatusTest.php | 3 ++- .../Unit/Destinations/AppwriteDestinationDsnTest.php | 1 + .../Unit/Destinations/AppwriteIndexLengthsTest.php | 1 + 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index 2772e258..1592cdff 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -169,7 +169,7 @@ class Appwrite extends Destination * longer active. Callers must derive this from their operation lifecycle; * without that proof, recovery fails closed. * - * @var (callable(UtopiaDocument $database): bool)|null + * @var callable(UtopiaDocument $database): bool */ private $canRecoverDatabase; @@ -219,10 +219,10 @@ class Appwrite extends Destination * @param UtopiaDatabase $dbForProject * @param callable(UtopiaDocument $database):UtopiaDatabase $getDatabasesDB * @param array> $collectionStructure + * @param callable(UtopiaDocument $database): bool $canRecoverDatabase Returns true only when the operation which owns an existing `provisioning` database is confirmed terminal. * @param OnDuplicate $onDuplicate Behavior when a row with an existing $id is encountered. * @param (callable(Database $resource): string)|null $getDatabaseDSN Resolver for the destination's `_databases.database` value. Pass when the destination project's DSN differs from the source's, so the destination row carries its own DSN instead of inheriting the source's. * @param array>> $collectionStructures Per-database-type metadata collection structures (e.g. `['vectorsdb' => ...]`), used instead of $collectionStructure when the imported database's type has an entry. Types with an entry also get type-specific metadata written (e.g. vectorsdb collection `dimension`). - * @param (callable(UtopiaDocument $database): bool)|null $canRecoverDatabase Returns true only when the operation which owns an existing `provisioning` database is confirmed terminal. Null refuses recovery so concurrent provisioning cannot be overwritten. */ public function __construct( string $project, @@ -233,10 +233,10 @@ public function __construct( protected array $collectionStructure, protected UtopiaDatabase $dbForPlatform, protected string $projectInternalId, + callable $canRecoverDatabase, protected OnDuplicate $onDuplicate = OnDuplicate::Fail, ?callable $getDatabaseDSN = null, protected array $collectionStructures = [], - ?callable $canRecoverDatabase = null, ) { $this->projectId = $project; $this->endpoint = $endpoint; @@ -700,7 +700,6 @@ protected function createDatabase(Database $resource): bool $status = $supportsStatus ? $existing->getAttribute('status') : null; $isProvisioning = $status === self::DATABASE_STATUS_PROVISIONING; $canRecoverProvisioning = $isProvisioning - && $this->canRecoverDatabase !== null && ($this->canRecoverDatabase)($existing); if ($isProvisioning && ! $canRecoverProvisioning) { diff --git a/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php b/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php index 42ad9eec..ecb0a488 100644 --- a/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php +++ b/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php @@ -360,8 +360,9 @@ public function supportsDatabaseStatus(): bool collectionStructure: ['attributes' => [], 'indexes' => []], dbForPlatform: $database, projectInternalId: '1', + canRecoverDatabase: $canRecoverDatabase + ?? static fn (UtopiaDocument $document): bool => false, onDuplicate: OnDuplicate::Fail, - canRecoverDatabase: $canRecoverDatabase, ); $transfer = new Transfer($source, $destination); diff --git a/tests/Migration/Unit/Destinations/AppwriteDestinationDsnTest.php b/tests/Migration/Unit/Destinations/AppwriteDestinationDsnTest.php index fa22151c..e1747b10 100644 --- a/tests/Migration/Unit/Destinations/AppwriteDestinationDsnTest.php +++ b/tests/Migration/Unit/Destinations/AppwriteDestinationDsnTest.php @@ -73,6 +73,7 @@ private function makeDestination(?callable $getDatabaseDSN): AppwriteDestination collectionStructure: ['attributes' => [], 'indexes' => []], dbForPlatform: $this->createStub(UtopiaDatabase::class), projectInternalId: '1', + canRecoverDatabase: static fn (UtopiaDocument $database): bool => false, onDuplicate: OnDuplicate::Fail, getDatabaseDSN: $getDatabaseDSN, ); diff --git a/tests/Migration/Unit/Destinations/AppwriteIndexLengthsTest.php b/tests/Migration/Unit/Destinations/AppwriteIndexLengthsTest.php index 2cb3dd1a..3bb9a3ff 100644 --- a/tests/Migration/Unit/Destinations/AppwriteIndexLengthsTest.php +++ b/tests/Migration/Unit/Destinations/AppwriteIndexLengthsTest.php @@ -205,6 +205,7 @@ private function transferIndex( ], dbForPlatform: $database, projectInternalId: '1', + canRecoverDatabase: static fn (UtopiaDocument $document): bool => false, onDuplicate: $onDuplicate, ); From 9d94e93776c368f89fdd6e55186840166d3cc370 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Sun, 30 Aug 2026 11:46:38 +1200 Subject: [PATCH 23/32] (fix): persist database provisioning owner --- bin/MigrationCLI.php | 33 ++- src/Migration/Destinations/Appwrite.php | 36 ++- .../AppwriteDatabaseStatusTest.php | 206 +++++++++++++++++- .../AppwriteDestinationDsnTest.php | 3 +- .../Destinations/AppwriteIndexLengthsTest.php | 3 +- tests/Migration/Unit/MigrationCLITest.php | 21 +- 6 files changed, 279 insertions(+), 23 deletions(-) diff --git a/bin/MigrationCLI.php b/bin/MigrationCLI.php index 688149b5..d1834d28 100644 --- a/bin/MigrationCLI.php +++ b/bin/MigrationCLI.php @@ -169,6 +169,8 @@ public static function getHelp(): string Options: -h, --help Show this help. + --migration-id= Stable owner identifier for this migration. + Required for Appwrite; reuse it for retries. --recover-provisioning Attest that no active migration owns an existing provisioning database and allow its recovery. Recovery is refused by default. @@ -284,13 +286,18 @@ public function getDestination(): Destination collectionStructure: self::STRUCTURE, dbForPlatform: $database, projectInternalId: $_ENV['DESTINATION_APPWRITE_TEST_PROJECT_INTERNAL_ID'], + migrationId: $this->getMigrationId(), // The standalone operator sets this flag only after confirming the lifecycle owner is // terminal. Without that explicit attestation, provisioning recovery fails closed. - canRecoverDatabase: fn (Document $document): bool => \in_array( - '--recover-provisioning', - $this->arguments, - true, - ), + getRecoverableMigrationId: function (Document $document): ?string { + if (! \in_array('--recover-provisioning', $this->arguments, true)) { + return null; + } + + $migrationId = $document->getAttribute('migrationId'); + + return \is_string($migrationId) && $migrationId !== '' ? $migrationId : null; + }, ); case 'local': return new Local('./localBackup'); @@ -299,6 +306,22 @@ public function getDestination(): Destination } } + private function getMigrationId(): string + { + foreach ($this->arguments as $argument) { + if (! \str_starts_with($argument, '--migration-id=')) { + continue; + } + + $migrationId = \substr($argument, \strlen('--migration-id=')); + if ($migrationId !== '') { + return $migrationId; + } + } + + throw new \InvalidArgumentException('--migration-id is required for an Appwrite destination'); + } + public function getDatabase(string $type): Database { Database::addFilter( diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index 1592cdff..eaf39ef1 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -164,14 +164,17 @@ class Appwrite extends Destination */ protected $getDatabaseDSN; + /** Immutable owner written with every provisioning transition initiated by this destination. */ + private readonly string $migrationId; + /** - * Confirms that the operation which owns a `provisioning` database is no - * longer active. Callers must derive this from their operation lifecycle; - * without that proof, recovery fails closed. + * Resolves the authoritative terminal owner of a `provisioning` database. + * Callers must derive this from their operation lifecycle; null means the + * owner is active or unknown and recovery fails closed. * - * @var callable(UtopiaDocument $database): bool + * @var callable(UtopiaDocument $database): ?string */ - private $canRecoverDatabase; + private $getRecoverableMigrationId; /** * @var array @@ -219,7 +222,8 @@ class Appwrite extends Destination * @param UtopiaDatabase $dbForProject * @param callable(UtopiaDocument $database):UtopiaDatabase $getDatabasesDB * @param array> $collectionStructure - * @param callable(UtopiaDocument $database): bool $canRecoverDatabase Returns true only when the operation which owns an existing `provisioning` database is confirmed terminal. + * @param string $migrationId Immutable identifier for the migration that will own databases provisioned by this destination. + * @param callable(UtopiaDocument $database): ?string $getRecoverableMigrationId Returns the authoritative terminal migration identifier for an existing `provisioning` database, or null when its owner is active or unknown. * @param OnDuplicate $onDuplicate Behavior when a row with an existing $id is encountered. * @param (callable(Database $resource): string)|null $getDatabaseDSN Resolver for the destination's `_databases.database` value. Pass when the destination project's DSN differs from the source's, so the destination row carries its own DSN instead of inheriting the source's. * @param array>> $collectionStructures Per-database-type metadata collection structures (e.g. `['vectorsdb' => ...]`), used instead of $collectionStructure when the imported database's type has an entry. Types with an entry also get type-specific metadata written (e.g. vectorsdb collection `dimension`). @@ -233,7 +237,8 @@ public function __construct( protected array $collectionStructure, protected UtopiaDatabase $dbForPlatform, protected string $projectInternalId, - callable $canRecoverDatabase, + string $migrationId, + callable $getRecoverableMigrationId, protected OnDuplicate $onDuplicate = OnDuplicate::Fail, ?callable $getDatabaseDSN = null, protected array $collectionStructures = [], @@ -258,7 +263,11 @@ public function __construct( $this->getDatabasesDB = $getDatabasesDB; $this->getDatabaseDSN = $getDatabaseDSN; - $this->canRecoverDatabase = $canRecoverDatabase; + if ($migrationId === '') { + throw new \InvalidArgumentException('Migration identifier must not be empty'); + } + $this->migrationId = $migrationId; + $this->getRecoverableMigrationId = $getRecoverableMigrationId; } /** @@ -699,8 +708,15 @@ protected function createDatabase(Database $resource): bool $supportsStatus = ! $existing->isEmpty() && $this->getSupportForDatabaseStatus(); $status = $supportsStatus ? $existing->getAttribute('status') : null; $isProvisioning = $status === self::DATABASE_STATUS_PROVISIONING; + $existingMigrationId = $isProvisioning ? $existing->getAttribute('migrationId') : null; + $recoverableMigrationId = $isProvisioning + ? ($this->getRecoverableMigrationId)($existing) + : null; $canRecoverProvisioning = $isProvisioning - && ($this->canRecoverDatabase)($existing); + && \is_string($existingMigrationId) + && $existingMigrationId !== '' + && \is_string($recoverableMigrationId) + && $existingMigrationId === $recoverableMigrationId; if ($isProvisioning && ! $canRecoverProvisioning) { throw new DatabaseException('Database '.$resource->getId().' is already being provisioned'); @@ -744,6 +760,7 @@ protected function createDatabase(Database $resource): bool if ($this->getSupportForDatabaseStatus()) { $document['status'] = self::DATABASE_STATUS_PROVISIONING; + $document['migrationId'] = $this->migrationId; } $this->dbForProject->updateDocument(self::META_DATABASES, $existing->getId(), new UtopiaDocument($document)); @@ -798,6 +815,7 @@ protected function createDatabase(Database $resource): bool // source leaves status untouched so the collection default applies. Never copy the source's state. if ($this->getSupportForDatabaseStatus()) { $document['status'] = self::DATABASE_STATUS_PROVISIONING; + $document['migrationId'] = $this->migrationId; } $database = $this->dbForProject->createDocument(self::META_DATABASES, new UtopiaDocument($document)); diff --git a/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php b/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php index ecb0a488..ea6c06c6 100644 --- a/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php +++ b/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php @@ -115,6 +115,48 @@ public function getDocument(string $collection, string $id, array $queries = [], } } +final class RecordingProjectDatabase extends UtopiaDatabase +{ + /** @var list}> */ + public array $databaseWrites = []; + + public bool $failReadyWrite = false; + + #[Override] + public function createDocument(string $collection, UtopiaDocument $document): UtopiaDocument + { + if ($collection === 'databases') { + $this->databaseWrites[] = [ + 'operation' => 'create', + 'document' => $document->getArrayCopy(), + ]; + } + + return parent::createDocument($collection, $document); + } + + #[Override] + public function updateDocument(string $collection, string $id, UtopiaDocument $document): UtopiaDocument + { + if ($collection === 'databases') { + $this->databaseWrites[] = [ + 'operation' => 'update', + 'document' => $document->getArrayCopy(), + ]; + } + + if ( + $this->failReadyWrite + && $collection === 'databases' + && $document->getAttribute('status') === 'ready' + ) { + throw new DatabaseException('ready status unavailable'); + } + + return parent::updateDocument($collection, $id, $document); + } +} + final class AppwriteDatabaseStatusTest extends TestCase { public function testDatabaseCreationOmitsStatusThroughLegacyAndExplicitEntrypoints(): void @@ -155,6 +197,101 @@ public function testDatabaseCreationPreservesLifecycleThroughLegacyAndExplicitEn } } + public function testCreatePersistsProvisioningAndOwnerAtomically(): void + { + $database = new RecordingProjectDatabase(new MemoryAdapter(), new Cache(new MemoryCache())); + $this->createProjectDatabase(withStatus: true, database: $database); + + $destination = $this->runDatabaseTransfer( + $database, + explicit: false, + migrationId: 'migration-create', + ); + + $this->assertSame([], $this->errorMessages($destination)); + $this->assertSame('create', $database->databaseWrites[0]['operation']); + $this->assertSame('provisioning', $database->databaseWrites[0]['document']['status']); + $this->assertSame('migration-create', $database->databaseWrites[0]['document']['migrationId']); + } + + public function testAuthorizedOverwritePersistsProvisioningAndNewOwnerAtomically(): void + { + $database = new RecordingProjectDatabase(new MemoryAdapter(), new Cache(new MemoryCache())); + $this->createProjectDatabase(withStatus: true, database: $database); + $this->seedDatabase($database, status: 'provisioning', migrationId: 'migration-old'); + $database->databaseWrites = []; + + $destination = $this->runDatabaseTransfer( + $database, + explicit: false, + migrationId: 'migration-new', + getRecoverableMigrationId: static fn (UtopiaDocument $database): ?string => 'migration-old', + ); + + $this->assertSame([], $this->errorMessages($destination)); + $write = $database->databaseWrites[0] ?? null; + $this->assertNotNull($write); + $this->assertSame('update', $write['operation']); + $this->assertSame('provisioning', $write['document']['status']); + $this->assertSame('migration-new', $write['document']['migrationId']); + } + + public function testActiveProvisioningRefusalRetainsExistingOwner(): void + { + $database = $this->createProjectDatabase(withStatus: true); + $this->seedDatabase($database, status: 'provisioning', migrationId: 'migration-active'); + + $destination = $this->runDatabaseTransfer( + $database, + explicit: false, + migrationId: 'migration-colliding', + getRecoverableMigrationId: static fn (UtopiaDocument $database): ?string => null, + ); + + $existing = $this->getDatabaseDocument($database); + $this->assertNotSame([], $this->errorMessages($destination)); + $this->assertSame('provisioning', $existing->getAttribute('status')); + $this->assertSame('migration-active', $existing->getAttribute('migrationId')); + } + + public function testMissingOrMismatchedOwnerRefusesRecovery(): void + { + foreach ([null, 'migration-existing'] as $owner) { + $database = $this->createProjectDatabase(withStatus: true); + $this->seedDatabase($database, status: 'provisioning', migrationId: $owner); + + $destination = $this->runDatabaseTransfer( + $database, + explicit: false, + migrationId: 'migration-new', + getRecoverableMigrationId: static fn (UtopiaDocument $database): ?string => 'migration-terminal', + ); + + $existing = $this->getDatabaseDocument($database); + $this->assertNotSame([], $this->errorMessages($destination)); + $this->assertSame('provisioning', $existing->getAttribute('status')); + $this->assertSame($owner, $existing->getAttribute('migrationId')); + } + } + + public function testReadyWriteFailureRetainsProvisioningOwner(): void + { + $database = new RecordingProjectDatabase(new MemoryAdapter(), new Cache(new MemoryCache())); + $this->createProjectDatabase(withStatus: true, database: $database); + $database->failReadyWrite = true; + + $destination = $this->runDatabaseTransfer( + $database, + explicit: false, + migrationId: 'migration-ready-failure', + ); + + $created = $this->getDatabaseDocument($database); + $this->assertSame([], $this->errorMessages($destination)); + $this->assertSame('provisioning', $created->getAttribute('status')); + $this->assertSame('migration-ready-failure', $created->getAttribute('migrationId')); + } + public function testReloadFailureMarksTheDatabaseFailed(): void { $database = new ReloadFailingProjectDatabase( @@ -170,6 +307,7 @@ public function testReloadFailureMarksTheDatabaseFailed(): void $this->assertNotSame([], $this->errorMessages($destination)); $this->assertStringContainsString('Failed to reload created database', $this->errorMessages($destination)[0]); $this->assertSame('failed', $created->getAttribute('status')); + $this->assertSame('migration-current', $created->getAttribute('migrationId')); $this->assertTrue( $database->getCollection('database_'.$created->getSequence())->isEmpty(), 'A reload failure must not leave a backing collection behind the metadata document', @@ -231,13 +369,15 @@ public function testProvisioningDatabaseRetrySucceedsUnderOnDuplicateFail(): voi $destination = $this->runDatabaseTransfer( $database, explicit: false, - canRecoverDatabase: static fn (UtopiaDocument $existing): bool => true, + migrationId: 'migration-recovery', + getRecoverableMigrationId: static fn (UtopiaDocument $existing): ?string => 'migration-current', ); $recovered = $this->getDatabaseDocument($database); $this->assertSame([], $this->errorMessages($destination)); $this->assertSame('ready', $recovered->getAttribute('status')); $this->assertSame($stranded->getSequence(), $recovered->getSequence()); + $this->assertSame('migration-recovery', $recovered->getAttribute('migrationId')); $this->assertFalse( $database->getCollection('database_'.$recovered->getSequence())->isEmpty(), 'A Fail retry must recover a database stranded in provisioning, not keep colliding with its metadata id', @@ -261,7 +401,11 @@ public function testConcurrentMigrationDoesNotRecoverAnActivelyProvisioningDatab &$statusDuringOverlap, &$collectionExistsDuringOverlap, ): void { - $second = $this->runDatabaseTransfer($database, explicit: false); + $second = $this->runDatabaseTransfer( + $database, + explicit: false, + migrationId: 'migration-second', + ); $provisioning = $this->getDatabaseDocument($database); $statusDuringOverlap = $provisioning->getAttribute('status'); $collectionExistsDuringOverlap = ! $database @@ -270,7 +414,11 @@ public function testConcurrentMigrationDoesNotRecoverAnActivelyProvisioningDatab }; $database->interceptNextDatabasesReload = true; - $first = $this->runDatabaseTransfer($database, explicit: false); + $first = $this->runDatabaseTransfer( + $database, + explicit: false, + migrationId: 'migration-first', + ); $created = $this->getDatabaseDocument($database); $this->assertSame([], $this->errorMessages($first)); @@ -280,9 +428,28 @@ public function testConcurrentMigrationDoesNotRecoverAnActivelyProvisioningDatab $this->assertSame('provisioning', $statusDuringOverlap); $this->assertFalse($collectionExistsDuringOverlap); $this->assertSame('ready', $created->getAttribute('status')); + $this->assertSame('migration-first', $created->getAttribute('migrationId')); $this->assertFalse($database->getCollection('database_'.$created->getSequence())->isEmpty()); } + public function testAuthorizedOverwriteReplacesTerminalOwner(): void + { + $database = $this->createProjectDatabase(withStatus: true); + $this->seedDatabase($database, status: 'provisioning', migrationId: 'migration-terminal'); + + $destination = $this->runDatabaseTransfer( + $database, + explicit: false, + migrationId: 'migration-successor', + getRecoverableMigrationId: static fn (UtopiaDocument $database): ?string => 'migration-terminal', + ); + + $recovered = $this->getDatabaseDocument($database); + $this->assertSame([], $this->errorMessages($destination)); + $this->assertSame('ready', $recovered->getAttribute('status')); + $this->assertSame('migration-successor', $recovered->getAttribute('migrationId')); + } + private function createProjectDatabase(bool $withStatus, ?UtopiaDatabase $database = null): UtopiaDatabase { $database ??= new UtopiaDatabase( @@ -305,6 +472,7 @@ private function createProjectDatabase(bool $withStatus, ?UtopiaDatabase $databa if ($withStatus) { $attributes[] = $this->attribute('status', ColumnType::String, size: 16); + $attributes[] = $this->attribute('migrationId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY); } $database->createCollection(new Collection( @@ -315,6 +483,30 @@ private function createProjectDatabase(bool $withStatus, ?UtopiaDatabase $databa return $database; } + private function seedDatabase( + UtopiaDatabase $database, + string $status, + ?string $migrationId, + ): UtopiaDocument { + $document = [ + '$id' => 'database', + 'name' => 'Database', + 'enabled' => true, + 'search' => 'database Database', + 'originalId' => null, + 'type' => 'tablesdb', + 'database' => '', + 'status' => $status, + ]; + if ($migrationId !== null) { + $document['migrationId'] = $migrationId; + } + + return $database->getAuthorization()->skip( + static fn (): UtopiaDocument => $database->createDocument('databases', new UtopiaDocument($document)), + ); + } + private function attribute( string $id, ColumnType $type, @@ -334,7 +526,8 @@ private function attribute( private function runDatabaseTransfer( UtopiaDatabase $database, bool $explicit, - ?callable $canRecoverDatabase = null, + string $migrationId = 'migration-current', + ?callable $getRecoverableMigrationId = null, ): CountingAppwriteDestination { $source = new class () extends MockSource { #[Override] @@ -360,8 +553,9 @@ public function supportsDatabaseStatus(): bool collectionStructure: ['attributes' => [], 'indexes' => []], dbForPlatform: $database, projectInternalId: '1', - canRecoverDatabase: $canRecoverDatabase - ?? static fn (UtopiaDocument $document): bool => false, + migrationId: $migrationId, + getRecoverableMigrationId: $getRecoverableMigrationId + ?? static fn (UtopiaDocument $document): ?string => null, onDuplicate: OnDuplicate::Fail, ); diff --git a/tests/Migration/Unit/Destinations/AppwriteDestinationDsnTest.php b/tests/Migration/Unit/Destinations/AppwriteDestinationDsnTest.php index e1747b10..39ea6917 100644 --- a/tests/Migration/Unit/Destinations/AppwriteDestinationDsnTest.php +++ b/tests/Migration/Unit/Destinations/AppwriteDestinationDsnTest.php @@ -73,7 +73,8 @@ private function makeDestination(?callable $getDatabaseDSN): AppwriteDestination collectionStructure: ['attributes' => [], 'indexes' => []], dbForPlatform: $this->createStub(UtopiaDatabase::class), projectInternalId: '1', - canRecoverDatabase: static fn (UtopiaDocument $database): bool => false, + migrationId: 'migration-test', + getRecoverableMigrationId: static fn (UtopiaDocument $database): ?string => null, onDuplicate: OnDuplicate::Fail, getDatabaseDSN: $getDatabaseDSN, ); diff --git a/tests/Migration/Unit/Destinations/AppwriteIndexLengthsTest.php b/tests/Migration/Unit/Destinations/AppwriteIndexLengthsTest.php index 3bb9a3ff..14b0aefa 100644 --- a/tests/Migration/Unit/Destinations/AppwriteIndexLengthsTest.php +++ b/tests/Migration/Unit/Destinations/AppwriteIndexLengthsTest.php @@ -205,7 +205,8 @@ private function transferIndex( ], dbForPlatform: $database, projectInternalId: '1', - canRecoverDatabase: static fn (UtopiaDocument $document): bool => false, + migrationId: 'migration-test', + getRecoverableMigrationId: static fn (UtopiaDocument $document): ?string => null, onDuplicate: $onDuplicate, ); diff --git a/tests/Migration/Unit/MigrationCLITest.php b/tests/Migration/Unit/MigrationCLITest.php index 27c9d05e..38391ea3 100644 --- a/tests/Migration/Unit/MigrationCLITest.php +++ b/tests/Migration/Unit/MigrationCLITest.php @@ -42,6 +42,8 @@ final class MigrationCLITest extends TestCase public function testHelpExplainsExplicitProvisioningRecoveryAttestation(): void { $this->assertStringContainsString('--recover-provisioning', \MigrationCLI::getHelp()); + $this->assertStringContainsString('--migration-id=', \MigrationCLI::getHelp()); + $this->assertStringContainsString('reuse it for retries', \MigrationCLI::getHelp()); $this->assertStringContainsString('no active migration', \MigrationCLI::getHelp()); $this->assertStringContainsString('refused by default', \MigrationCLI::getHelp()); } @@ -50,7 +52,10 @@ public function testProvisioningRecoveryRequiresExplicitOperatorAttestation(): v { foreach ([false, true] as $recover) { $database = $this->createProjectDatabase(); - $arguments = $recover ? ['MigrationCLI.php', '--recover-provisioning'] : ['MigrationCLI.php']; + $arguments = ['MigrationCLI.php', '--migration-id=migration-current']; + if ($recover) { + $arguments[] = '--recover-provisioning'; + } $cli = new TestMigrationCLI($arguments, $database); $destination = $cli->getDestination(); @@ -63,16 +68,28 @@ public function testProvisioningRecoveryRequiresExplicitOperatorAttestation(): v if (! $recover) { $this->assertNotSame([], $destination->getErrors()); $this->assertSame('provisioning', $created->getAttribute('status')); + $this->assertSame('migration-terminal', $created->getAttribute('migrationId')); $this->assertTrue($database->getCollection('database_'.$created->getSequence())->isEmpty()); continue; } $this->assertSame([], $destination->getErrors()); $this->assertSame('ready', $created->getAttribute('status')); + $this->assertSame('migration-current', $created->getAttribute('migrationId')); $this->assertFalse($database->getCollection('database_'.$created->getSequence())->isEmpty()); } } + public function testAppwriteDestinationRequiresMigrationIdentifier(): void + { + $cli = new TestMigrationCLI(['MigrationCLI.php'], $this->createProjectDatabase()); + + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('--migration-id is required for an Appwrite destination'); + + $cli->getDestination(); + } + private function createProjectDatabase(): Database { $database = new Database(new MemoryAdapter(), new Cache(new MemoryCache())); @@ -90,6 +107,7 @@ private function createProjectDatabase(): Database new Attribute(key: 'type', type: ColumnType::String, size: 128, default: 'tablesdb'), new Attribute(key: 'database', type: ColumnType::String, size: 2000), new Attribute(key: 'status', type: ColumnType::String, size: 16), + new Attribute(key: 'migrationId', type: ColumnType::String, size: Database::LENGTH_KEY), ], )); $database->getAuthorization()->skip( @@ -102,6 +120,7 @@ private function createProjectDatabase(): Database 'type' => 'tablesdb', 'database' => '', 'status' => 'provisioning', + 'migrationId' => 'migration-terminal', ])), ); From 06f41d0671f595a3a8139e539d6e13326cf9a515 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 1 Sep 2026 04:47:43 +1200 Subject: [PATCH 24/32] (fix): fence database provisioning attempts --- bin/MigrationCLI.php | 69 ++- composer.lock | 8 +- src/Migration/Destinations/Appwrite.php | 320 ++++++++++---- .../Appwrite/ProvisioningOwner.php | 24 ++ .../AppwriteDatabaseConcurrencyTest.php | 394 +++++++++++++++++ .../AppwriteDatabaseStatusTest.php | 399 ++++++++++++++++-- .../AppwriteDestinationDsnTest.php | 5 +- .../Destinations/AppwriteIndexLengthsTest.php | 5 +- .../Destinations/ProvisioningOwnerTest.php | 36 ++ tests/Migration/Unit/MigrationCLITest.php | 119 ++++-- 10 files changed, 1212 insertions(+), 167 deletions(-) create mode 100644 src/Migration/Destinations/Appwrite/ProvisioningOwner.php create mode 100644 tests/Migration/E2E/Destinations/AppwriteDatabaseConcurrencyTest.php create mode 100644 tests/Migration/Unit/Destinations/ProvisioningOwnerTest.php diff --git a/bin/MigrationCLI.php b/bin/MigrationCLI.php index d1834d28..9070cf39 100644 --- a/bin/MigrationCLI.php +++ b/bin/MigrationCLI.php @@ -11,6 +11,7 @@ use Utopia\Database\Document; use Utopia\Migration\Destination; use Utopia\Migration\Destinations\Appwrite as DestinationsAppwrite; +use Utopia\Migration\Destinations\Appwrite\ProvisioningOwner; use Utopia\Migration\Destinations\Local; use Utopia\Migration\Source; use Utopia\Migration\Sources\Appwrite; @@ -169,10 +170,15 @@ public static function getHelp(): string Options: -h, --help Show this help. - --migration-id= Stable owner identifier for this migration. + --migration-id= Stable logical owner identifier for this migration. Required for Appwrite; reuse it for retries. - --recover-provisioning Attest that no active migration owns an existing - provisioning database and allow its recovery. + --migration-attempt-id= + Required for Appwrite; use a fresh attempt for every + execution or retry. + --recover-migration-id= + --recover-migration-attempt-id= + Together attest that the exact prior migration attempt is terminal + and allow recovery of its provisioning or failed databases. Recovery is refused by default. HELP; @@ -276,6 +282,7 @@ public function getDestination(): Destination switch ($_ENV['DESTINATION_PROVIDER']) { case 'appwrite': $database = $this->getDatabase('destination'); + $recoverableOwner = $this->getRecoverableOwner(); return new DestinationsAppwrite( project: $_ENV['DESTINATION_APPWRITE_TEST_PROJECT'], @@ -286,18 +293,10 @@ public function getDestination(): Destination collectionStructure: self::STRUCTURE, dbForPlatform: $database, projectInternalId: $_ENV['DESTINATION_APPWRITE_TEST_PROJECT_INTERNAL_ID'], - migrationId: $this->getMigrationId(), - // The standalone operator sets this flag only after confirming the lifecycle owner is - // terminal. Without that explicit attestation, provisioning recovery fails closed. - getRecoverableMigrationId: function (Document $document): ?string { - if (! \in_array('--recover-provisioning', $this->arguments, true)) { - return null; - } - - $migrationId = $document->getAttribute('migrationId'); - - return \is_string($migrationId) && $migrationId !== '' ? $migrationId : null; - }, + owner: new ProvisioningOwner($this->getMigrationId(), $this->getMigrationAttemptId()), + // The standalone operator supplies the fixed pair only after confirming the prior + // lifecycle owner is terminal. The destination independently compares it with the row. + getRecoverableOwner: static fn (Document $document): ?ProvisioningOwner => $recoverableOwner, ); case 'local': return new Local('./localBackup'); @@ -322,6 +321,46 @@ private function getMigrationId(): string throw new \InvalidArgumentException('--migration-id is required for an Appwrite destination'); } + private function getMigrationAttemptId(): string + { + foreach ($this->arguments as $argument) { + if (! \str_starts_with($argument, '--migration-attempt-id=')) { + continue; + } + + $attemptId = \substr($argument, \strlen('--migration-attempt-id=')); + if ($attemptId !== '') { + return $attemptId; + } + } + + throw new \InvalidArgumentException('--migration-attempt-id is required for an Appwrite destination'); + } + + private function getRecoverableOwner(): ?ProvisioningOwner + { + $migrationId = null; + $attemptId = null; + + foreach ($this->arguments as $argument) { + if (\str_starts_with($argument, '--recover-migration-id=')) { + $value = \substr($argument, \strlen('--recover-migration-id=')); + $migrationId = $value !== '' ? $value : null; + } + + if (\str_starts_with($argument, '--recover-migration-attempt-id=')) { + $value = \substr($argument, \strlen('--recover-migration-attempt-id=')); + $attemptId = $value !== '' ? $value : null; + } + } + + if ($migrationId === null || $attemptId === null) { + return null; + } + + return new ProvisioningOwner($migrationId, $attemptId); + } + public function getDatabase(string $type): Database { Database::addFilter( diff --git a/composer.lock b/composer.lock index fba3f9bf..a50b3100 100644 --- a/composer.lock +++ b/composer.lock @@ -2612,12 +2612,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "6a3149819c55b08802cafc9f33324dca9299fe2b" + "reference": "745378d6c0124b7af31d69d2715cb07aa88fe9af" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/6a3149819c55b08802cafc9f33324dca9299fe2b", - "reference": "6a3149819c55b08802cafc9f33324dca9299fe2b", + "url": "https://api.github.com/repos/utopia-php/database/zipball/745378d6c0124b7af31d69d2715cb07aa88fe9af", + "reference": "745378d6c0124b7af31d69d2715cb07aa88fe9af", "shasum": "" }, "require": { @@ -2703,7 +2703,7 @@ "source": "https://github.com/utopia-php/database/tree/feat-query-lib", "issues": "https://github.com/utopia-php/database/issues" }, - "time": "2026-08-29T13:33:39+00:00" + "time": "2026-08-31T16:19:44+00:00" }, { "name": "utopia-php/dsn", diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index eaf39ef1..b5867410 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -48,6 +48,7 @@ use Utopia\Database\Validator\Structure; use Utopia\Database\Validator\UID; use Utopia\Migration\Destination; +use Utopia\Migration\Destinations\Appwrite\ProvisioningOwner; use Utopia\Migration\Exception; use Utopia\Migration\Resource; use Utopia\Migration\Resources\Auth\AuthMethods; @@ -165,16 +166,19 @@ class Appwrite extends Destination protected $getDatabaseDSN; /** Immutable owner written with every provisioning transition initiated by this destination. */ - private readonly string $migrationId; + private readonly ProvisioningOwner $owner; /** - * Resolves the authoritative terminal owner of a `provisioning` database. + * Resolves the authoritative terminal owner of an incomplete database. * Callers must derive this from their operation lifecycle; null means the * owner is active or unknown and recovery fails closed. * - * @var callable(UtopiaDocument $database): ?string + * @var callable(UtopiaDocument $database): ?ProvisioningOwner */ - private $getRecoverableMigrationId; + private $getRecoverableOwner; + + /** @var array */ + private array $createdDatabaseOwners = []; /** * @var array @@ -222,8 +226,8 @@ class Appwrite extends Destination * @param UtopiaDatabase $dbForProject * @param callable(UtopiaDocument $database):UtopiaDatabase $getDatabasesDB * @param array> $collectionStructure - * @param string $migrationId Immutable identifier for the migration that will own databases provisioned by this destination. - * @param callable(UtopiaDocument $database): ?string $getRecoverableMigrationId Returns the authoritative terminal migration identifier for an existing `provisioning` database, or null when its owner is active or unknown. + * @param ProvisioningOwner $owner Immutable logical migration and execution-attempt identifiers for databases provisioned by this destination. + * @param callable(UtopiaDocument $database): ?ProvisioningOwner $getRecoverableOwner Returns the exact authoritative terminal owner for an existing `provisioning` or `failed` database, or null when its owner is active or unknown. * @param OnDuplicate $onDuplicate Behavior when a row with an existing $id is encountered. * @param (callable(Database $resource): string)|null $getDatabaseDSN Resolver for the destination's `_databases.database` value. Pass when the destination project's DSN differs from the source's, so the destination row carries its own DSN instead of inheriting the source's. * @param array>> $collectionStructures Per-database-type metadata collection structures (e.g. `['vectorsdb' => ...]`), used instead of $collectionStructure when the imported database's type has an entry. Types with an entry also get type-specific metadata written (e.g. vectorsdb collection `dimension`). @@ -237,8 +241,8 @@ public function __construct( protected array $collectionStructure, protected UtopiaDatabase $dbForPlatform, protected string $projectInternalId, - string $migrationId, - callable $getRecoverableMigrationId, + ProvisioningOwner $owner, + callable $getRecoverableOwner, protected OnDuplicate $onDuplicate = OnDuplicate::Fail, ?callable $getDatabaseDSN = null, protected array $collectionStructures = [], @@ -263,11 +267,8 @@ public function __construct( $this->getDatabasesDB = $getDatabasesDB; $this->getDatabaseDSN = $getDatabaseDSN; - if ($migrationId === '') { - throw new \InvalidArgumentException('Migration identifier must not be empty'); - } - $this->migrationId = $migrationId; - $this->getRecoverableMigrationId = $getRecoverableMigrationId; + $this->owner = $owner; + $this->getRecoverableOwner = $getRecoverableOwner; } /** @@ -331,6 +332,7 @@ private function resetRunState(): void $this->orphansByTable = []; $this->processedTwoWayPairs = []; $this->provisioningDatabases = []; + $this->createdDatabaseOwners = []; } private function markProvisionedDatabasesReady(): void @@ -351,11 +353,113 @@ private function setDatabaseStatus(string $databaseId, string $status): void return; } + $transition = $this->dbForProject->withTransaction(function () use ($databaseId, $status): ?bool { + $database = $this->dbForProject->getDocument( + self::META_DATABASES, + $databaseId, + forUpdate: true, + ); + $owner = $this->getProvisioningOwner($database); + if ( + $database->isEmpty() + || $database->getAttribute('status') !== self::DATABASE_STATUS_PROVISIONING + || $owner === null + || ! $owner->equals($this->owner) + ) { + return false; + } + + if (! $this->supportsAtomicOwnerMutation()) { + return null; + } + + $this->dbForProject->updateDocument( + self::META_DATABASES, + $databaseId, + new UtopiaDocument(['status' => $status]), + ); + + return true; + }); + + if ($transition === true) { + unset($this->createdDatabaseOwners[$databaseId]); + return; + } + + if ($transition === false) { + unset($this->createdDatabaseOwners[$databaseId]); + return; + } + + $createdOwner = $this->createdDatabaseOwners[$databaseId] ?? null; + if ($createdOwner === null || ! $createdOwner->equals($this->owner)) { + return; + } + + // Adapters without an atomic claim primitive may finalize only the exact row this + // destination instance just created. forUpdate disables caches even when no lock exists. + $database = $this->dbForProject->getDocument( + self::META_DATABASES, + $databaseId, + forUpdate: true, + ); + $owner = $this->getProvisioningOwner($database); + if ( + $database->isEmpty() + || $database->getAttribute('status') !== self::DATABASE_STATUS_PROVISIONING + || $owner === null + || ! $owner->equals($createdOwner) + ) { + unset($this->createdDatabaseOwners[$databaseId]); + return; + } + $this->dbForProject->updateDocument( self::META_DATABASES, $databaseId, new UtopiaDocument(['status' => $status]), ); + unset($this->createdDatabaseOwners[$databaseId]); + } + + private function getProvisioningOwner(UtopiaDocument $database): ?ProvisioningOwner + { + $migrationId = $database->getAttribute('migrationId'); + $attemptId = $database->getAttribute('migrationAttemptId'); + if ( + ! \is_string($migrationId) + || $migrationId === '' + || ! \is_string($attemptId) + || $attemptId === '' + ) { + return null; + } + + return new ProvisioningOwner($migrationId, $attemptId); + } + + /** + * This fence depends on the exact utopia-php/database pin in composer.lock: + * SQL exposes UpdateLock or TransactionRetries, SQLite uses BEGIN IMMEDIATE, + * and replica Mongo exposes TransactionRetries only while a session is active. + */ + private function supportsAtomicOwnerMutation(): bool + { + $adapter = $this->dbForProject->getAdapter(); + + return $adapter->inTransaction() + && ( + $adapter->supports(Capability::UpdateLock) + || $adapter->supports(Capability::TransactionRetries) + ); + } + + private function requireAtomicOwnerMutation(): void + { + if (! $this->supportsAtomicOwnerMutation()) { + throw new DatabaseException('Database provisioning ownership requires an atomic transaction with update locks or conflict retries'); + } } /** Best-effort transition to `failed`; a secondary error here must not mask the caller's original throw. */ @@ -705,49 +809,82 @@ protected function createDatabase(Database $resource): bool $updatedAt = $this->normalizeDateTime($resource->getUpdatedAt(), $createdAt); $existing = $this->dbForProject->getDocument(self::META_DATABASES, $resource->getId()); - $supportsStatus = ! $existing->isEmpty() && $this->getSupportForDatabaseStatus(); - $status = $supportsStatus ? $existing->getAttribute('status') : null; - $isProvisioning = $status === self::DATABASE_STATUS_PROVISIONING; - $existingMigrationId = $isProvisioning ? $existing->getAttribute('migrationId') : null; - $recoverableMigrationId = $isProvisioning - ? ($this->getRecoverableMigrationId)($existing) + $supportsStatus = $this->getSupportForDatabaseStatus(); + $status = $supportsStatus && ! $existing->isEmpty() + ? $existing->getAttribute('status') : null; - $canRecoverProvisioning = $isProvisioning - && \is_string($existingMigrationId) - && $existingMigrationId !== '' - && \is_string($recoverableMigrationId) - && $existingMigrationId === $recoverableMigrationId; - - if ($isProvisioning && ! $canRecoverProvisioning) { - throw new DatabaseException('Database '.$resource->getId().' is already being provisioned'); + $isIncomplete = \in_array( + $status, + [self::DATABASE_STATUS_PROVISIONING, self::DATABASE_STATUS_FAILED], + true, + ); + $expectedOwner = null; + if ($isIncomplete) { + $snapshotOwner = $this->getProvisioningOwner($existing); + $expectedOwner = ($this->getRecoverableOwner)($existing); + if ( + $snapshotOwner === null + || ! $expectedOwner instanceof ProvisioningOwner + || ! $snapshotOwner->equals($expectedOwner) + || $this->owner->attemptId === $expectedOwner->attemptId + ) { + throw new DatabaseException('Database '.$resource->getId().' recovery owner is active, unknown, mismatched, or reuses the prior attempt'); + } } - $isIncomplete = $status === self::DATABASE_STATUS_FAILED || $canRecoverProvisioning; - if ($this->onDuplicate !== OnDuplicate::Fail || $isIncomplete) { - $action = $this->onDuplicate->resolveSchemaAction( - !$existing->isEmpty(), + /** @var array{action: SchemaAction, database: UtopiaDocument, incomplete: bool} $claim */ + $claim = $this->dbForProject->withTransaction(function () use ( + $resource, $updatedAt, - $existing->getUpdatedAt(), - ); + $supportsStatus, + $status, + $isIncomplete, + $expectedOwner, + ): array { + $locked = $this->dbForProject->getDocument( + self::META_DATABASES, + $resource->getId(), + forUpdate: true, + ); + $lockedStatus = $supportsStatus && ! $locked->isEmpty() + ? $locked->getAttribute('status') + : null; + $lockedIncomplete = \in_array( + $lockedStatus, + [self::DATABASE_STATUS_PROVISIONING, self::DATABASE_STATUS_FAILED], + true, + ); - if ($isIncomplete) { - // A prior run created the metadata document but left the database unusable (its backing - // collection may be missing). Force Overwrite — regardless of timestamps, spec match, or - // OnDuplicate::Fail — so retries recreate the collection instead of hitting the existing ID. - $action = SchemaAction::Overwrite; - } elseif ($action !== SchemaAction::Create && $this->databaseSpecMatches($existing, $resource)) { - // Spec match → skip work. Create excluded; nothing on dest to match against. - $action = SchemaAction::Skip; - } + if ($isIncomplete) { + $lockedOwner = $this->getProvisioningOwner($locked); + if ( + $lockedStatus !== $status + || $lockedOwner === null + || ! $expectedOwner instanceof ProvisioningOwner + || ! $lockedOwner->equals($expectedOwner) + ) { + throw new DatabaseException('Database '.$resource->getId().' recovery owner changed before it could be claimed'); + } + } elseif ($lockedIncomplete) { + throw new DatabaseException('Database '.$resource->getId().' requires terminal migration attestation before recovery'); + } - $earlyReturn = match ($action) { - SchemaAction::Skip => (function () use ($resource, $existing): bool { - $resource->setSequence($existing->getSequence()); - $resource->setStatus(Resource::STATUS_SKIPPED, 'Already exists on destination'); - return false; - })(), - SchemaAction::Overwrite => (function () use ($resource, $existing, $updatedAt, $isIncomplete): bool { + $action = $this->onDuplicate->resolveSchemaAction( + ! $locked->isEmpty(), + $updatedAt, + $locked->getUpdatedAt(), + ); + + if ($isIncomplete) { + // A prior run created the metadata document but left the database unusable. Force an + // overwrite after the locked ownership check so retries can recreate its collection. + $action = SchemaAction::Overwrite; + } elseif ($action !== SchemaAction::Create && $this->databaseSpecMatches($locked, $resource)) { + $action = SchemaAction::Skip; + } + + if ($action === SchemaAction::Overwrite) { $document = [ 'name' => $resource->getDatabaseName(), 'search' => implode(' ', [$resource->getId(), $resource->getDatabaseName()]), @@ -758,42 +895,63 @@ protected function createDatabase(Database $resource): bool '$updatedAt' => $updatedAt, ]; - if ($this->getSupportForDatabaseStatus()) { + if ($supportsStatus) { $document['status'] = self::DATABASE_STATUS_PROVISIONING; - $document['migrationId'] = $this->migrationId; + $document['migrationId'] = $this->owner->migrationId; + $document['migrationAttemptId'] = $this->owner->attemptId; } - $this->dbForProject->updateDocument(self::META_DATABASES, $existing->getId(), new UtopiaDocument($document)); - $resource->setSequence($existing->getSequence()); + // This gate must remain inside the transaction and immediately before the + // existing-row mutation so the locked ownership evidence cannot go stale. + $this->requireAtomicOwnerMutation(); + $this->dbForProject->updateDocument( + self::META_DATABASES, + $locked->getId(), + new UtopiaDocument($document), + ); + } - // An incomplete database can be missing its backing collection. Recreate it so we never - // flip a database to ready with no collection behind it. A healthy overwrite already has - // its collection, so we skip the lookup entirely. - if ($isIncomplete && $this->dbForProject->getCollection($this->databaseCollectionId($existing))->isEmpty()) { - try { - $structure = $this->collectionStructureFor($resource); - - $this->dbForProject->createCollection(new Collection( - id: $this->databaseCollectionId($existing), - attributes: $this->schemaAttributes($structure['attributes'] ?? []), - indexes: $this->schemaIndexes($structure['indexes'] ?? []), - )); - } catch (\Throwable $e) { - $this->markDatabaseFailed($resource->getId()); - throw $e; - } - } + return [ + 'action' => $action, + 'database' => $locked, + 'incomplete' => $isIncomplete, + ]; + }); + + $action = $claim['action']; + $existing = $claim['database']; + $isIncomplete = $claim['incomplete']; + + if ($action === SchemaAction::Skip) { + $resource->setSequence($existing->getSequence()); + $resource->setStatus(Resource::STATUS_SKIPPED, 'Already exists on destination'); + return false; + } - if ($this->getSupportForDatabaseStatus()) { - $this->provisioningDatabases[$resource->getId()] = true; + if ($action === SchemaAction::Overwrite) { + $resource->setSequence($existing->getSequence()); + + // The claim transaction commits before inspecting or creating the backing collection. + if ($isIncomplete && $this->dbForProject->getCollection($this->databaseCollectionId($existing))->isEmpty()) { + try { + $structure = $this->collectionStructureFor($resource); + + $this->dbForProject->createCollection(new Collection( + id: $this->databaseCollectionId($existing), + attributes: $this->schemaAttributes($structure['attributes'] ?? []), + indexes: $this->schemaIndexes($structure['indexes'] ?? []), + )); + } catch (\Throwable $e) { + $this->markDatabaseFailed($resource->getId()); + throw $e; } + } - return true; - })(), - SchemaAction::Create => null, - }; - if ($earlyReturn !== null) { - return $earlyReturn; + if ($supportsStatus) { + $this->provisioningDatabases[$resource->getId()] = true; + } + + return true; } } @@ -815,10 +973,14 @@ protected function createDatabase(Database $resource): bool // source leaves status untouched so the collection default applies. Never copy the source's state. if ($this->getSupportForDatabaseStatus()) { $document['status'] = self::DATABASE_STATUS_PROVISIONING; - $document['migrationId'] = $this->migrationId; + $document['migrationId'] = $this->owner->migrationId; + $document['migrationAttemptId'] = $this->owner->attemptId; } $database = $this->dbForProject->createDocument(self::META_DATABASES, new UtopiaDocument($document)); + if ($supportsStatus) { + $this->createdDatabaseOwners[$database->getId()] = $this->owner; + } try { $database = $this->dbForProject->getDocument(self::META_DATABASES, $database->getId()); diff --git a/src/Migration/Destinations/Appwrite/ProvisioningOwner.php b/src/Migration/Destinations/Appwrite/ProvisioningOwner.php new file mode 100644 index 00000000..7a27879d --- /dev/null +++ b/src/Migration/Destinations/Appwrite/ProvisioningOwner.php @@ -0,0 +1,24 @@ +migrationId === $owner->migrationId + && $this->attemptId === $owner->attemptId; + } +} diff --git a/tests/Migration/E2E/Destinations/AppwriteDatabaseConcurrencyTest.php b/tests/Migration/E2E/Destinations/AppwriteDatabaseConcurrencyTest.php new file mode 100644 index 00000000..d614a5ff --- /dev/null +++ b/tests/Migration/E2E/Destinations/AppwriteDatabaseConcurrencyTest.php @@ -0,0 +1,394 @@ +> */ + public array $databaseWrites = []; + + public ?Closure $beforeBackingCollectionCreate = null; + + public bool $throwAfterBackingCollectionCallback = false; + + #[Override] + public function updateDocument(string $collection, string $id, UtopiaDocument $document): UtopiaDocument + { + if ($collection === 'databases') { + $this->databaseWrites[] = $document->getArrayCopy(); + } + + return parent::updateDocument($collection, $id, $document); + } + + #[Override] + public function createCollection(Collection $collection): Collection + { + if ( + $this->beforeBackingCollectionCreate !== null + && \str_starts_with($collection->getId(), 'database_') + ) { + $callback = $this->beforeBackingCollectionCreate; + $this->beforeBackingCollectionCreate = null; + $callback(); + + if ($this->throwAfterBackingCollectionCallback) { + throw new DatabaseException('backing collection creation failed'); + } + } + + return parent::createCollection($collection); + } +} + +final class AppwriteDatabaseConcurrencyTest extends TestCase +{ + public function testStaleIncompleteClaimLosesAfterAnotherSuccessorCommits(): void + { + foreach (['provisioning', 'failed'] as $status) { + [$second, $third, $path] = $this->createSharedDatabases(); + + try { + $this->seedDatabase($second, $status, 'migration-shared', 'attempt-first'); + $winner = null; + $loser = $this->createDestination( + $second, + 'migration-shared', + 'attempt-second', + function (UtopiaDocument $snapshot) use ($third, &$winner): ProvisioningOwner { + $winner = $this->createDestination( + $third, + 'migration-shared', + 'attempt-third', + static fn (UtopiaDocument $document): ProvisioningOwner => new ProvisioningOwner( + 'migration-shared', + 'attempt-first', + ), + ); + $this->runTransfer($third, $winner); + + return new ProvisioningOwner('migration-shared', 'attempt-first'); + }, + ); + $second->databaseWrites = []; + + $this->runTransfer($second, $loser); + + $database = $this->getDatabaseDocument($third); + $this->assertInstanceOf(AppwriteDestination::class, $winner); + $this->assertSame([], $this->errorMessages($winner)); + $this->assertNotSame([], $this->errorMessages($loser)); + $this->assertSame([], $second->databaseWrites); + $this->assertSame('ready', $database->getAttribute('status')); + $this->assertSame('migration-shared', $database->getAttribute('migrationId')); + $this->assertSame('attempt-third', $database->getAttribute('migrationAttemptId')); + } finally { + $this->removeSQLiteFiles($path); + } + } + } + + public function testStaleOwnerCannotMarkDatabaseReadyAfterTakeover(): void + { + [$second, $third, $path] = $this->createSharedDatabases(); + + try { + $this->seedDatabase($second, 'provisioning', 'migration-shared', 'attempt-first'); + $successor = null; + $destination = $this->createDestination( + $second, + 'migration-shared', + 'attempt-second', + static fn (UtopiaDocument $snapshot): ProvisioningOwner => new ProvisioningOwner( + 'migration-shared', + 'attempt-first', + ), + ); + + $this->runTransfer( + $second, + $destination, + function () use ($third, &$successor): void { + $successor = $this->createDestination( + $third, + 'migration-shared', + 'attempt-third', + static fn (UtopiaDocument $snapshot): ProvisioningOwner => new ProvisioningOwner( + 'migration-shared', + 'attempt-second', + ), + ); + $this->claimWithoutTerminalTransition($third, $successor); + }, + ); + + $database = $this->getDatabaseDocument($third); + $readyWrites = \array_values(\array_filter( + $second->databaseWrites, + static fn (array $document): bool => ($document['status'] ?? null) === 'ready', + )); + $this->assertInstanceOf(AppwriteDestination::class, $successor); + $this->assertSame([], $this->errorMessages($destination)); + $this->assertSame([], $this->errorMessages($successor)); + $this->assertSame([], $readyWrites); + $this->assertSame('provisioning', $database->getAttribute('status')); + $this->assertSame('migration-shared', $database->getAttribute('migrationId')); + $this->assertSame('attempt-third', $database->getAttribute('migrationAttemptId')); + } finally { + $this->removeSQLiteFiles($path); + } + } + + public function testStaleOwnerCannotMarkDatabaseFailedAfterTakeover(): void + { + [$second, $third, $path] = $this->createSharedDatabases(); + + try { + $this->seedDatabase($second, 'failed', 'migration-shared', 'attempt-first'); + $successor = null; + $second->throwAfterBackingCollectionCallback = true; + $second->beforeBackingCollectionCreate = function () use ($third, &$successor): void { + $successor = $this->createDestination( + $third, + 'migration-shared', + 'attempt-third', + static fn (UtopiaDocument $snapshot): ProvisioningOwner => new ProvisioningOwner( + 'migration-shared', + 'attempt-second', + ), + ); + $this->claimWithoutTerminalTransition($third, $successor); + }; + $destination = $this->createDestination( + $second, + 'migration-shared', + 'attempt-second', + static fn (UtopiaDocument $snapshot): ProvisioningOwner => new ProvisioningOwner( + 'migration-shared', + 'attempt-first', + ), + ); + + $this->runTransfer($second, $destination); + + $database = $this->getDatabaseDocument($third); + $failedWrites = \array_values(\array_filter( + $second->databaseWrites, + static fn (array $document): bool => ($document['status'] ?? null) === 'failed', + )); + $this->assertInstanceOf(AppwriteDestination::class, $successor); + $this->assertNotSame([], $this->errorMessages($destination)); + $this->assertSame([], $this->errorMessages($successor)); + $this->assertSame([], $failedWrites); + $this->assertSame('provisioning', $database->getAttribute('status')); + $this->assertSame('migration-shared', $database->getAttribute('migrationId')); + $this->assertSame('attempt-third', $database->getAttribute('migrationAttemptId')); + } finally { + $this->removeSQLiteFiles($path); + } + } + + private function createDestination( + UtopiaDatabase $database, + string $migrationId, + string $migrationAttemptId, + callable $getRecoverableOwner, + ): AppwriteDestination { + return new AppwriteDestination( + project: 'destination-project', + endpoint: 'http://example.test/v1', + key: 'test-key', + dbForProject: $database, + getDatabasesDB: static fn (UtopiaDocument $document): UtopiaDatabase => $database, + collectionStructure: ['attributes' => [], 'indexes' => []], + dbForPlatform: $database, + projectInternalId: '1', + owner: new ProvisioningOwner($migrationId, $migrationAttemptId), + getRecoverableOwner: $getRecoverableOwner, + onDuplicate: OnDuplicate::Fail, + ); + } + + private function runTransfer( + UtopiaDatabase $database, + AppwriteDestination $destination, + ?callable $callback = null, + ): void { + $source = new class () extends MockSource { + #[Override] + public function supportsDatabaseStatus(): bool + { + return true; + } + }; + $source->pushMockResource(new DatabaseResource( + id: 'database', + name: 'Database', + type: 'tablesdb', + database: 'source-dsn', + databaseStatus: 'ready', + )); + + $transfer = new Transfer($source, $destination); + $database->getAuthorization()->skip( + static fn () => $transfer->run( + [Resource::TYPE_DATABASE], + $callback ?? static function (): void { + }, + ), + ); + } + + private function claimWithoutTerminalTransition( + UtopiaDatabase $database, + AppwriteDestination $destination, + ): void { + try { + $this->runTransfer( + $database, + $destination, + static function (): void { + throw new \RuntimeException('stop after provisioning claim'); + }, + ); + $this->fail('Expected transfer callback to stop before the terminal transition'); + } catch (\RuntimeException $error) { + $this->assertSame('stop after provisioning claim', $error->getMessage()); + } + } + + /** @return array{RecordingSQLiteProjectDatabase, RecordingSQLiteProjectDatabase, string} */ + private function createSharedDatabases(): array + { + $path = \tempnam(\sys_get_temp_dir(), 'migration-owner-'); + if ($path === false) { + throw new \RuntimeException('Failed to create SQLite test database'); + } + + $attributes = SQLite::getPDOAttributes(); + $attributes[PDO::ATTR_PERSISTENT] = false; + $secondConnection = new SQLiteConnection('sqlite:'.$path, null, null, $attributes); + $thirdConnection = new SQLiteConnection('sqlite:'.$path, null, null, $attributes); + $secondConnection->exec('PRAGMA journal_mode = WAL'); + $secondConnection->exec('PRAGMA busy_timeout = 1000'); + $thirdConnection->exec('PRAGMA busy_timeout = 1000'); + + $second = new RecordingSQLiteProjectDatabase( + new SQLite($secondConnection), + new Cache(new None()), + ); + $third = new RecordingSQLiteProjectDatabase( + new SQLite($thirdConnection), + new Cache(new None()), + ); + $namespace = 'migration_owner_'.\uniqid(); + foreach ([$second, $third] as $database) { + $database + ->setDatabase('appwrite') + ->setNamespace($namespace); + } + + $second->create(); + $second->createCollection(new Collection( + id: 'databases', + attributes: [ + $this->attribute('name', ColumnType::String, required: true, size: 256), + $this->attribute('enabled', ColumnType::Boolean, default: true), + $this->attribute('search', ColumnType::String, size: 16384), + $this->attribute('originalId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('type', ColumnType::String, default: 'tablesdb', size: 128), + $this->attribute('database', ColumnType::String, size: 2000), + $this->attribute('status', ColumnType::String, size: 16), + $this->attribute('migrationId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), + $this->attribute('migrationAttemptId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY), + ], + )); + + return [$second, $third, $path]; + } + + private function seedDatabase( + UtopiaDatabase $database, + string $status, + string $migrationId, + string $migrationAttemptId, + ): UtopiaDocument { + return $database->getAuthorization()->skip( + static fn (): UtopiaDocument => $database->createDocument('databases', new UtopiaDocument([ + '$id' => 'database', + 'name' => 'Database', + 'enabled' => true, + 'search' => 'database Database', + 'originalId' => null, + 'type' => 'tablesdb', + 'database' => '', + 'status' => $status, + 'migrationId' => $migrationId, + 'migrationAttemptId' => $migrationAttemptId, + ])), + ); + } + + private function attribute( + string $id, + ColumnType $type, + bool $required = false, + mixed $default = null, + int $size = 0, + ): UtopiaAttribute { + return new UtopiaAttribute( + key: $id, + type: $type, + size: $size, + required: $required, + default: $default, + ); + } + + private function getDatabaseDocument(UtopiaDatabase $database): UtopiaDocument + { + return $database->getAuthorization()->skip( + static fn (): UtopiaDocument => $database->getDocument('databases', 'database'), + ); + } + + /** @return list */ + private function errorMessages(AppwriteDestination $destination): array + { + return \array_map( + static fn (\Throwable $error): string => $error->getMessage(), + $destination->getErrors(), + ); + } + + private function removeSQLiteFiles(string $path): void + { + foreach ([$path, $path.'-wal', $path.'-shm'] as $file) { + if (\is_file($file)) { + \unlink($file); + } + } + } +} diff --git a/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php b/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php index ea6c06c6..5d322120 100644 --- a/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php +++ b/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php @@ -8,11 +8,13 @@ use Utopia\Cache\Cache; use Utopia\Database\Adapter\Memory as MemoryAdapter; use Utopia\Database\Attribute as UtopiaAttribute; +use Utopia\Database\Capability; use Utopia\Database\Collection; use Utopia\Database\Database as UtopiaDatabase; use Utopia\Database\Document as UtopiaDocument; use Utopia\Database\Exception as DatabaseException; use Utopia\Migration\Destinations\Appwrite as AppwriteDestination; +use Utopia\Migration\Destinations\Appwrite\ProvisioningOwner; use Utopia\Migration\Destinations\OnDuplicate; use Utopia\Migration\Resource; use Utopia\Migration\Resources\Database\Database as DatabaseResource; @@ -20,6 +22,25 @@ use Utopia\Query\Schema\ColumnType; use Utopia\Tests\Unit\Adapters\MockSource; +class ReplicaMemoryAdapter extends MemoryAdapter +{ + /** @return array */ + #[Override] + public function capabilities(): array + { + return [...parent::capabilities(), Capability::TransactionRetries]; + } +} + +final class StandaloneMemoryAdapter extends ReplicaMemoryAdapter +{ + #[Override] + public function withTransaction(callable $callback): mixed + { + return $callback(); + } +} + class CountingAppwriteDestination extends AppwriteDestination { public int $runCount = 0; @@ -55,15 +76,15 @@ public function getDocument(string $collection, string $id, array $queries = [], } /** - * Fails the reload and then the status write that would record the failure, which - * is the only way production reaches a document stranded in `provisioning`: + * Fails the reload and every status write while the metadata store is unavailable, + * which is the only way production reaches a document stranded in `provisioning`: * markDatabaseFailed() swallows its own error so it cannot mask the original throw. */ class StrandedProvisioningProjectDatabase extends UtopiaDatabase { public bool $failNextDatabasesRead = false; - public bool $failNextDatabasesWrite = false; + public bool $failDatabasesWrites = false; #[Override] public function getDocument(string $collection, string $id, array $queries = [], bool $forUpdate = false): UtopiaDocument @@ -81,9 +102,7 @@ public function getDocument(string $collection, string $id, array $queries = [], #[Override] public function updateDocument(string $collection, string $id, UtopiaDocument $document): UtopiaDocument { - if ($this->failNextDatabasesWrite && $collection === 'databases') { - $this->failNextDatabasesWrite = false; - + if ($this->failDatabasesWrites && $collection === 'databases') { throw new DatabaseException('metadata store unavailable'); } @@ -115,7 +134,7 @@ public function getDocument(string $collection, string $id, array $queries = [], } } -final class RecordingProjectDatabase extends UtopiaDatabase +class RecordingProjectDatabase extends UtopiaDatabase { /** @var list}> */ public array $databaseWrites = []; @@ -157,6 +176,24 @@ public function updateDocument(string $collection, string $id, UtopiaDocument $d } } +final class FinalizerInterleavingProjectDatabase extends RecordingProjectDatabase +{ + public ?\Closure $beforeFinalizerRead = null; + + #[Override] + public function getDocument(string $collection, string $id, array $queries = [], bool $forUpdate = false): UtopiaDocument + { + $document = parent::getDocument($collection, $id, $queries, $forUpdate); + if ($forUpdate && $collection === 'databases' && $this->beforeFinalizerRead !== null) { + $callback = $this->beforeFinalizerRead; + $this->beforeFinalizerRead = null; + $callback(); + } + + return $document; + } +} + final class AppwriteDatabaseStatusTest extends TestCase { public function testDatabaseCreationOmitsStatusThroughLegacyAndExplicitEntrypoints(): void @@ -181,7 +218,11 @@ public function testDatabaseCreationOmitsStatusThroughLegacyAndExplicitEntrypoin public function testDatabaseCreationPreservesLifecycleThroughLegacyAndExplicitEntrypoints(): void { foreach ([false, true] as $explicit) { - $database = $this->createProjectDatabase(withStatus: true); + $database = new UtopiaDatabase( + new StandaloneMemoryAdapter(), + new Cache(new MemoryCache()), + ); + $this->createProjectDatabase(withStatus: true, database: $database); $destination = $this->runDatabaseTransfer($database, $explicit); @@ -206,26 +247,79 @@ public function testCreatePersistsProvisioningAndOwnerAtomically(): void $database, explicit: false, migrationId: 'migration-create', + migrationAttemptId: 'attempt-create', ); $this->assertSame([], $this->errorMessages($destination)); $this->assertSame('create', $database->databaseWrites[0]['operation']); $this->assertSame('provisioning', $database->databaseWrites[0]['document']['status']); $this->assertSame('migration-create', $database->databaseWrites[0]['document']['migrationId']); + $this->assertSame('attempt-create', $database->databaseWrites[0]['document']['migrationAttemptId']); + } + + public function testStandaloneUniqueCreateCannotFinalizeAfterOwnerMismatch(): void + { + $database = new FinalizerInterleavingProjectDatabase( + new StandaloneMemoryAdapter(), + new Cache(new MemoryCache()), + ); + $this->createProjectDatabase(withStatus: true, database: $database); + $database->beforeFinalizerRead = static function () use ($database): void { + $database->updateDocument( + 'databases', + 'database', + new UtopiaDocument([ + 'status' => 'provisioning', + 'migrationId' => 'migration-successor', + 'migrationAttemptId' => 'attempt-successor', + ]), + ); + }; + + $destination = $this->runDatabaseTransfer( + $database, + explicit: false, + migrationId: 'migration-create', + migrationAttemptId: 'attempt-create', + ); + + $terminalWrites = \array_values(\array_filter( + $database->databaseWrites, + static fn (array $write): bool => \in_array( + $write['document']['status'] ?? null, + ['ready', 'failed'], + true, + ), + )); + $created = $this->getDatabaseDocument($database); + $this->assertSame([], $this->errorMessages($destination)); + $this->assertSame([], $terminalWrites); + $this->assertSame('provisioning', $created->getAttribute('status')); + $this->assertSame('migration-successor', $created->getAttribute('migrationId')); + $this->assertSame('attempt-successor', $created->getAttribute('migrationAttemptId')); } public function testAuthorizedOverwritePersistsProvisioningAndNewOwnerAtomically(): void { - $database = new RecordingProjectDatabase(new MemoryAdapter(), new Cache(new MemoryCache())); + $database = new RecordingProjectDatabase(new ReplicaMemoryAdapter(), new Cache(new MemoryCache())); $this->createProjectDatabase(withStatus: true, database: $database); - $this->seedDatabase($database, status: 'provisioning', migrationId: 'migration-old'); + $this->seedDatabase( + $database, + status: 'provisioning', + migrationId: 'migration-old', + migrationAttemptId: 'attempt-old', + ); $database->databaseWrites = []; $destination = $this->runDatabaseTransfer( $database, explicit: false, migrationId: 'migration-new', - getRecoverableMigrationId: static fn (UtopiaDocument $database): ?string => 'migration-old', + migrationAttemptId: 'attempt-new', + getRecoverableOwner: static fn (UtopiaDocument $database): ProvisioningOwner => new ProvisioningOwner( + 'migration-old', + 'attempt-old', + ), ); $this->assertSame([], $this->errorMessages($destination)); @@ -234,43 +328,241 @@ public function testAuthorizedOverwritePersistsProvisioningAndNewOwnerAtomically $this->assertSame('update', $write['operation']); $this->assertSame('provisioning', $write['document']['status']); $this->assertSame('migration-new', $write['document']['migrationId']); + $this->assertSame('attempt-new', $write['document']['migrationAttemptId']); } public function testActiveProvisioningRefusalRetainsExistingOwner(): void { - $database = $this->createProjectDatabase(withStatus: true); - $this->seedDatabase($database, status: 'provisioning', migrationId: 'migration-active'); + $database = new RecordingProjectDatabase(new StandaloneMemoryAdapter(), new Cache(new MemoryCache())); + $this->createProjectDatabase(withStatus: true, database: $database); + $this->seedDatabase( + $database, + status: 'provisioning', + migrationId: 'migration-active', + migrationAttemptId: 'attempt-active', + ); + $database->databaseWrites = []; $destination = $this->runDatabaseTransfer( $database, explicit: false, migrationId: 'migration-colliding', - getRecoverableMigrationId: static fn (UtopiaDocument $database): ?string => null, + migrationAttemptId: 'attempt-colliding', + getRecoverableOwner: static fn (UtopiaDocument $database): ?ProvisioningOwner => null, ); $existing = $this->getDatabaseDocument($database); $this->assertNotSame([], $this->errorMessages($destination)); + $this->assertSame([], $database->databaseWrites); $this->assertSame('provisioning', $existing->getAttribute('status')); $this->assertSame('migration-active', $existing->getAttribute('migrationId')); + $this->assertSame('attempt-active', $existing->getAttribute('migrationAttemptId')); } public function testMissingOrMismatchedOwnerRefusesRecovery(): void { foreach ([null, 'migration-existing'] as $owner) { - $database = $this->createProjectDatabase(withStatus: true); - $this->seedDatabase($database, status: 'provisioning', migrationId: $owner); + $database = new RecordingProjectDatabase(new StandaloneMemoryAdapter(), new Cache(new MemoryCache())); + $this->createProjectDatabase(withStatus: true, database: $database); + $this->seedDatabase( + $database, + status: 'provisioning', + migrationId: $owner, + migrationAttemptId: 'attempt-existing', + ); + $database->databaseWrites = []; $destination = $this->runDatabaseTransfer( $database, explicit: false, migrationId: 'migration-new', - getRecoverableMigrationId: static fn (UtopiaDocument $database): ?string => 'migration-terminal', + migrationAttemptId: 'attempt-new', + getRecoverableOwner: static fn (UtopiaDocument $database): ProvisioningOwner => new ProvisioningOwner( + 'migration-terminal', + 'attempt-existing', + ), ); $existing = $this->getDatabaseDocument($database); $this->assertNotSame([], $this->errorMessages($destination)); + $this->assertSame([], $database->databaseWrites); $this->assertSame('provisioning', $existing->getAttribute('status')); $this->assertSame($owner, $existing->getAttribute('migrationId')); + $this->assertSame('attempt-existing', $existing->getAttribute('migrationAttemptId')); + } + } + + public function testMissingOrMismatchedAttemptRefusesRecovery(): void + { + foreach ([null, 'attempt-existing'] as $attemptId) { + $database = new RecordingProjectDatabase(new StandaloneMemoryAdapter(), new Cache(new MemoryCache())); + $this->createProjectDatabase(withStatus: true, database: $database); + $this->seedDatabase( + $database, + status: 'provisioning', + migrationId: 'migration-existing', + migrationAttemptId: $attemptId, + ); + $database->databaseWrites = []; + + $destination = $this->runDatabaseTransfer( + $database, + explicit: false, + migrationId: 'migration-new', + migrationAttemptId: 'attempt-new', + getRecoverableOwner: static fn (UtopiaDocument $database): ProvisioningOwner => new ProvisioningOwner( + 'migration-existing', + 'attempt-terminal', + ), + ); + + $existing = $this->getDatabaseDocument($database); + $this->assertNotSame([], $this->errorMessages($destination)); + $this->assertSame([], $database->databaseWrites); + $this->assertSame('provisioning', $existing->getAttribute('status')); + $this->assertSame('migration-existing', $existing->getAttribute('migrationId')); + $this->assertSame($attemptId, $existing->getAttribute('migrationAttemptId')); + } + } + + public function testRecoveryRefusesReusingThePriorAttempt(): void + { + $database = new RecordingProjectDatabase(new StandaloneMemoryAdapter(), new Cache(new MemoryCache())); + $this->createProjectDatabase(withStatus: true, database: $database); + $this->seedDatabase( + $database, + status: 'failed', + migrationId: 'migration-shared', + migrationAttemptId: 'attempt-shared', + ); + $database->databaseWrites = []; + + $destination = $this->runDatabaseTransfer( + $database, + explicit: false, + migrationId: 'migration-shared', + migrationAttemptId: 'attempt-shared', + getRecoverableOwner: static fn (UtopiaDocument $database): ProvisioningOwner => new ProvisioningOwner( + 'migration-shared', + 'attempt-shared', + ), + ); + + $existing = $this->getDatabaseDocument($database); + $this->assertNotSame([], $this->errorMessages($destination)); + $this->assertSame([], $database->databaseWrites); + $this->assertSame('failed', $existing->getAttribute('status')); + $this->assertSame('migration-shared', $existing->getAttribute('migrationId')); + $this->assertSame('attempt-shared', $existing->getAttribute('migrationAttemptId')); + } + + public function testPreExistingRecoveryRequiresAtomicMutationCapability(): void + { + foreach ([new MemoryAdapter(), new StandaloneMemoryAdapter()] as $adapter) { + $database = new RecordingProjectDatabase($adapter, new Cache(new MemoryCache())); + $this->createProjectDatabase(withStatus: true, database: $database); + $this->seedDatabase( + $database, + status: 'failed', + migrationId: 'migration-old', + migrationAttemptId: 'attempt-old', + ); + $database->databaseWrites = []; + + $destination = $this->runDatabaseTransfer( + $database, + explicit: false, + migrationId: 'migration-new', + migrationAttemptId: 'attempt-new', + getRecoverableOwner: static fn (UtopiaDocument $database): ProvisioningOwner => new ProvisioningOwner( + 'migration-old', + 'attempt-old', + ), + ); + + $existing = $this->getDatabaseDocument($database); + $this->assertNotSame([], $this->errorMessages($destination)); + $this->assertSame([], $database->databaseWrites); + $this->assertSame('failed', $existing->getAttribute('status')); + $this->assertSame('migration-old', $existing->getAttribute('migrationId')); + $this->assertSame('attempt-old', $existing->getAttribute('migrationAttemptId')); + } + } + + public function testHealthyOverwriteRequiresAtomicMutationCapability(): void + { + foreach ([new MemoryAdapter(), new StandaloneMemoryAdapter()] as $adapter) { + $database = new RecordingProjectDatabase($adapter, new Cache(new MemoryCache())); + $this->createProjectDatabase(withStatus: true, database: $database); + $this->seedDatabase( + $database, + status: 'ready', + migrationId: 'migration-old', + migrationAttemptId: 'attempt-old', + ); + $database->getAuthorization()->skip( + static fn (): UtopiaDocument => $database->updateDocument( + 'databases', + 'database', + new UtopiaDocument(['name' => 'Old database']), + ), + ); + $database->databaseWrites = []; + + $destination = $this->runDatabaseTransfer( + $database, + explicit: false, + migrationId: 'migration-new', + migrationAttemptId: 'attempt-new', + onDuplicate: OnDuplicate::Overwrite, + resourceUpdatedAt: '2099-01-01T00:00:00.000+00:00', + ); + + $existing = $this->getDatabaseDocument($database); + $this->assertNotSame([], $this->errorMessages($destination)); + $this->assertSame([], $database->databaseWrites); + $this->assertSame('ready', $existing->getAttribute('status')); + $this->assertSame('migration-old', $existing->getAttribute('migrationId')); + $this->assertSame('attempt-old', $existing->getAttribute('migrationAttemptId')); + } + } + + public function testFailedDatabaseRecoveryRequiresExactTerminalOwner(): void + { + foreach ( + [ + 'no attestation' => ['migration-terminal', null], + 'missing owner' => [null, 'migration-terminal'], + 'mismatched owner' => ['migration-existing', 'migration-terminal'], + 'unknown terminal owner' => ['migration-existing', null], + ] as [$owner, $recoverableMigrationId] + ) { + $database = new RecordingProjectDatabase(new StandaloneMemoryAdapter(), new Cache(new MemoryCache())); + $this->createProjectDatabase(withStatus: true, database: $database); + $this->seedDatabase( + $database, + status: 'failed', + migrationId: $owner, + migrationAttemptId: 'attempt-existing', + ); + $database->databaseWrites = []; + + $destination = $this->runDatabaseTransfer( + $database, + explicit: false, + migrationId: 'migration-new', + migrationAttemptId: 'attempt-new', + getRecoverableOwner: static fn (UtopiaDocument $database): ?ProvisioningOwner => $recoverableMigrationId === null + ? null + : new ProvisioningOwner($recoverableMigrationId, 'attempt-existing'), + ); + + $existing = $this->getDatabaseDocument($database); + $this->assertNotSame([], $this->errorMessages($destination)); + $this->assertSame([], $database->databaseWrites); + $this->assertSame('failed', $existing->getAttribute('status')); + $this->assertSame($owner, $existing->getAttribute('migrationId')); + $this->assertSame('attempt-existing', $existing->getAttribute('migrationAttemptId')); } } @@ -284,18 +576,20 @@ public function testReadyWriteFailureRetainsProvisioningOwner(): void $database, explicit: false, migrationId: 'migration-ready-failure', + migrationAttemptId: 'attempt-ready-failure', ); $created = $this->getDatabaseDocument($database); $this->assertSame([], $this->errorMessages($destination)); $this->assertSame('provisioning', $created->getAttribute('status')); $this->assertSame('migration-ready-failure', $created->getAttribute('migrationId')); + $this->assertSame('attempt-ready-failure', $created->getAttribute('migrationAttemptId')); } public function testReloadFailureMarksTheDatabaseFailed(): void { $database = new ReloadFailingProjectDatabase( - new MemoryAdapter(), + new StandaloneMemoryAdapter(), new Cache(new MemoryCache()), ); $this->createProjectDatabase(withStatus: true, database: $database); @@ -308,6 +602,7 @@ public function testReloadFailureMarksTheDatabaseFailed(): void $this->assertStringContainsString('Failed to reload created database', $this->errorMessages($destination)[0]); $this->assertSame('failed', $created->getAttribute('status')); $this->assertSame('migration-current', $created->getAttribute('migrationId')); + $this->assertSame('attempt-current', $created->getAttribute('migrationAttemptId')); $this->assertTrue( $database->getCollection('database_'.$created->getSequence())->isEmpty(), 'A reload failure must not leave a backing collection behind the metadata document', @@ -317,7 +612,7 @@ public function testReloadFailureMarksTheDatabaseFailed(): void public function testFailedDatabaseRetrySucceedsUnderOnDuplicateFail(): void { $database = new ReloadFailingProjectDatabase( - new MemoryAdapter(), + new ReplicaMemoryAdapter(), new Cache(new MemoryCache()), ); $this->createProjectDatabase(withStatus: true, database: $database); @@ -331,12 +626,23 @@ public function testFailedDatabaseRetrySucceedsUnderOnDuplicateFail(): void $database->getCollection('database_'.$failed->getSequence())->isEmpty(), ); - $destination = $this->runDatabaseTransfer($database, explicit: false); + $destination = $this->runDatabaseTransfer( + $database, + explicit: false, + migrationId: 'migration-recovery', + migrationAttemptId: 'attempt-recovery', + getRecoverableOwner: static fn (UtopiaDocument $existing): ProvisioningOwner => new ProvisioningOwner( + 'migration-current', + 'attempt-current', + ), + ); $recovered = $this->getDatabaseDocument($database); $this->assertSame([], $this->errorMessages($destination)); $this->assertSame('ready', $recovered->getAttribute('status')); $this->assertSame($failed->getSequence(), $recovered->getSequence()); + $this->assertSame('migration-recovery', $recovered->getAttribute('migrationId')); + $this->assertSame('attempt-recovery', $recovered->getAttribute('migrationAttemptId')); $this->assertFalse( $database->getCollection('database_'.$recovered->getSequence())->isEmpty(), 'A Fail retry must recreate the backing collection for a previously failed database', @@ -346,12 +652,12 @@ public function testFailedDatabaseRetrySucceedsUnderOnDuplicateFail(): void public function testProvisioningDatabaseRetrySucceedsUnderOnDuplicateFail(): void { $database = new StrandedProvisioningProjectDatabase( - new MemoryAdapter(), + new ReplicaMemoryAdapter(), new Cache(new MemoryCache()), ); $this->createProjectDatabase(withStatus: true, database: $database); $database->failNextDatabasesRead = true; - $database->failNextDatabasesWrite = true; + $database->failDatabasesWrites = true; $this->runDatabaseTransfer($database, explicit: false); @@ -365,12 +671,17 @@ public function testProvisioningDatabaseRetrySucceedsUnderOnDuplicateFail(): voi $database->getCollection('database_'.$stranded->getSequence())->isEmpty(), 'The backing collection was never created, so the database is unusable', ); + $database->failDatabasesWrites = false; $destination = $this->runDatabaseTransfer( $database, explicit: false, migrationId: 'migration-recovery', - getRecoverableMigrationId: static fn (UtopiaDocument $existing): ?string => 'migration-current', + migrationAttemptId: 'attempt-recovery', + getRecoverableOwner: static fn (UtopiaDocument $existing): ProvisioningOwner => new ProvisioningOwner( + 'migration-current', + 'attempt-current', + ), ); $recovered = $this->getDatabaseDocument($database); @@ -378,6 +689,7 @@ public function testProvisioningDatabaseRetrySucceedsUnderOnDuplicateFail(): voi $this->assertSame('ready', $recovered->getAttribute('status')); $this->assertSame($stranded->getSequence(), $recovered->getSequence()); $this->assertSame('migration-recovery', $recovered->getAttribute('migrationId')); + $this->assertSame('attempt-recovery', $recovered->getAttribute('migrationAttemptId')); $this->assertFalse( $database->getCollection('database_'.$recovered->getSequence())->isEmpty(), 'A Fail retry must recover a database stranded in provisioning, not keep colliding with its metadata id', @@ -405,6 +717,7 @@ public function testConcurrentMigrationDoesNotRecoverAnActivelyProvisioningDatab $database, explicit: false, migrationId: 'migration-second', + migrationAttemptId: 'attempt-second', ); $provisioning = $this->getDatabaseDocument($database); $statusDuringOverlap = $provisioning->getAttribute('status'); @@ -418,36 +731,49 @@ public function testConcurrentMigrationDoesNotRecoverAnActivelyProvisioningDatab $database, explicit: false, migrationId: 'migration-first', + migrationAttemptId: 'attempt-first', ); $created = $this->getDatabaseDocument($database); $this->assertSame([], $this->errorMessages($first)); $this->assertInstanceOf(CountingAppwriteDestination::class, $second); $this->assertNotSame([], $this->errorMessages($second)); - $this->assertStringContainsString('already being provisioned', $this->errorMessages($second)[0]); + $this->assertStringContainsString('recovery owner', $this->errorMessages($second)[0]); $this->assertSame('provisioning', $statusDuringOverlap); $this->assertFalse($collectionExistsDuringOverlap); $this->assertSame('ready', $created->getAttribute('status')); $this->assertSame('migration-first', $created->getAttribute('migrationId')); + $this->assertSame('attempt-first', $created->getAttribute('migrationAttemptId')); $this->assertFalse($database->getCollection('database_'.$created->getSequence())->isEmpty()); } public function testAuthorizedOverwriteReplacesTerminalOwner(): void { - $database = $this->createProjectDatabase(withStatus: true); - $this->seedDatabase($database, status: 'provisioning', migrationId: 'migration-terminal'); + $database = new RecordingProjectDatabase(new ReplicaMemoryAdapter(), new Cache(new MemoryCache())); + $this->createProjectDatabase(withStatus: true, database: $database); + $this->seedDatabase( + $database, + status: 'provisioning', + migrationId: 'migration-terminal', + migrationAttemptId: 'attempt-terminal', + ); $destination = $this->runDatabaseTransfer( $database, explicit: false, migrationId: 'migration-successor', - getRecoverableMigrationId: static fn (UtopiaDocument $database): ?string => 'migration-terminal', + migrationAttemptId: 'attempt-successor', + getRecoverableOwner: static fn (UtopiaDocument $database): ProvisioningOwner => new ProvisioningOwner( + 'migration-terminal', + 'attempt-terminal', + ), ); $recovered = $this->getDatabaseDocument($database); $this->assertSame([], $this->errorMessages($destination)); $this->assertSame('ready', $recovered->getAttribute('status')); $this->assertSame('migration-successor', $recovered->getAttribute('migrationId')); + $this->assertSame('attempt-successor', $recovered->getAttribute('migrationAttemptId')); } private function createProjectDatabase(bool $withStatus, ?UtopiaDatabase $database = null): UtopiaDatabase @@ -473,6 +799,7 @@ private function createProjectDatabase(bool $withStatus, ?UtopiaDatabase $databa if ($withStatus) { $attributes[] = $this->attribute('status', ColumnType::String, size: 16); $attributes[] = $this->attribute('migrationId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY); + $attributes[] = $this->attribute('migrationAttemptId', ColumnType::String, size: UtopiaDatabase::LENGTH_KEY); } $database->createCollection(new Collection( @@ -487,6 +814,7 @@ private function seedDatabase( UtopiaDatabase $database, string $status, ?string $migrationId, + ?string $migrationAttemptId, ): UtopiaDocument { $document = [ '$id' => 'database', @@ -501,6 +829,9 @@ private function seedDatabase( if ($migrationId !== null) { $document['migrationId'] = $migrationId; } + if ($migrationAttemptId !== null) { + $document['migrationAttemptId'] = $migrationAttemptId; + } return $database->getAuthorization()->skip( static fn (): UtopiaDocument => $database->createDocument('databases', new UtopiaDocument($document)), @@ -527,7 +858,10 @@ private function runDatabaseTransfer( UtopiaDatabase $database, bool $explicit, string $migrationId = 'migration-current', - ?callable $getRecoverableMigrationId = null, + string $migrationAttemptId = 'attempt-current', + ?callable $getRecoverableOwner = null, + OnDuplicate $onDuplicate = OnDuplicate::Fail, + string $resourceUpdatedAt = '', ): CountingAppwriteDestination { $source = new class () extends MockSource { #[Override] @@ -539,6 +873,7 @@ public function supportsDatabaseStatus(): bool $source->pushMockResource(new DatabaseResource( id: 'database', name: 'Database', + updatedAt: $resourceUpdatedAt, type: 'tablesdb', database: 'source-dsn', databaseStatus: 'ready', @@ -553,10 +888,10 @@ public function supportsDatabaseStatus(): bool collectionStructure: ['attributes' => [], 'indexes' => []], dbForPlatform: $database, projectInternalId: '1', - migrationId: $migrationId, - getRecoverableMigrationId: $getRecoverableMigrationId - ?? static fn (UtopiaDocument $document): ?string => null, - onDuplicate: OnDuplicate::Fail, + owner: new ProvisioningOwner($migrationId, $migrationAttemptId), + getRecoverableOwner: $getRecoverableOwner + ?? static fn (UtopiaDocument $document): ?ProvisioningOwner => null, + onDuplicate: $onDuplicate, ); $transfer = new Transfer($source, $destination); diff --git a/tests/Migration/Unit/Destinations/AppwriteDestinationDsnTest.php b/tests/Migration/Unit/Destinations/AppwriteDestinationDsnTest.php index 39ea6917..e76ebef2 100644 --- a/tests/Migration/Unit/Destinations/AppwriteDestinationDsnTest.php +++ b/tests/Migration/Unit/Destinations/AppwriteDestinationDsnTest.php @@ -7,6 +7,7 @@ use Utopia\Database\Database as UtopiaDatabase; use Utopia\Database\Document as UtopiaDocument; use Utopia\Migration\Destinations\Appwrite as AppwriteDestination; +use Utopia\Migration\Destinations\Appwrite\ProvisioningOwner; use Utopia\Migration\Destinations\OnDuplicate; use Utopia\Migration\Resources\Database\Database as DatabaseResource; @@ -73,8 +74,8 @@ private function makeDestination(?callable $getDatabaseDSN): AppwriteDestination collectionStructure: ['attributes' => [], 'indexes' => []], dbForPlatform: $this->createStub(UtopiaDatabase::class), projectInternalId: '1', - migrationId: 'migration-test', - getRecoverableMigrationId: static fn (UtopiaDocument $database): ?string => null, + owner: new ProvisioningOwner('migration-test', 'attempt-test'), + getRecoverableOwner: static fn (UtopiaDocument $database): ?ProvisioningOwner => null, onDuplicate: OnDuplicate::Fail, getDatabaseDSN: $getDatabaseDSN, ); diff --git a/tests/Migration/Unit/Destinations/AppwriteIndexLengthsTest.php b/tests/Migration/Unit/Destinations/AppwriteIndexLengthsTest.php index 14b0aefa..85ea4e88 100644 --- a/tests/Migration/Unit/Destinations/AppwriteIndexLengthsTest.php +++ b/tests/Migration/Unit/Destinations/AppwriteIndexLengthsTest.php @@ -13,6 +13,7 @@ use Utopia\Database\Database as UtopiaDatabase; use Utopia\Database\Document as UtopiaDocument; use Utopia\Migration\Destinations\Appwrite as AppwriteDestination; +use Utopia\Migration\Destinations\Appwrite\ProvisioningOwner; use Utopia\Migration\Destinations\OnDuplicate; use Utopia\Migration\Resource; use Utopia\Migration\Resources\Database\Columns\Text; @@ -205,8 +206,8 @@ private function transferIndex( ], dbForPlatform: $database, projectInternalId: '1', - migrationId: 'migration-test', - getRecoverableMigrationId: static fn (UtopiaDocument $document): ?string => null, + owner: new ProvisioningOwner('migration-test', 'attempt-test'), + getRecoverableOwner: static fn (UtopiaDocument $document): ?ProvisioningOwner => null, onDuplicate: $onDuplicate, ); diff --git a/tests/Migration/Unit/Destinations/ProvisioningOwnerTest.php b/tests/Migration/Unit/Destinations/ProvisioningOwnerTest.php new file mode 100644 index 00000000..54100d30 --- /dev/null +++ b/tests/Migration/Unit/Destinations/ProvisioningOwnerTest.php @@ -0,0 +1,36 @@ + */ + public static function emptyIdentifiers(): array + { + return [ + 'migration identifier' => ['', 'attempt'], + 'attempt identifier' => ['migration', ''], + ]; + } + + #[DataProvider('emptyIdentifiers')] + public function testIdentifiersMustBeNonEmpty(string $migrationId, string $attemptId): void + { + $this->expectException(\InvalidArgumentException::class); + + new ProvisioningOwner($migrationId, $attemptId); + } + + public function testEqualityRequiresTheExactPair(): void + { + $owner = new ProvisioningOwner('migration', 'attempt'); + + $this->assertTrue($owner->equals(new ProvisioningOwner('migration', 'attempt'))); + $this->assertFalse($owner->equals(new ProvisioningOwner('migration-other', 'attempt'))); + $this->assertFalse($owner->equals(new ProvisioningOwner('migration', 'attempt-other'))); + } +} diff --git a/tests/Migration/Unit/MigrationCLITest.php b/tests/Migration/Unit/MigrationCLITest.php index 38391ea3..d9ae8131 100644 --- a/tests/Migration/Unit/MigrationCLITest.php +++ b/tests/Migration/Unit/MigrationCLITest.php @@ -11,6 +11,7 @@ use Utopia\Cache\Cache; use Utopia\Database\Adapter\Memory as MemoryAdapter; use Utopia\Database\Attribute; +use Utopia\Database\Capability; use Utopia\Database\Collection; use Utopia\Database\Database; use Utopia\Database\Document; @@ -36,47 +37,84 @@ public function getDatabase(string $type): Database } } +final class TransactionalMemoryAdapter extends MemoryAdapter +{ + /** @return array */ + #[Override] + public function capabilities(): array + { + return [...parent::capabilities(), Capability::TransactionRetries]; + } +} + #[BackupGlobals(true)] final class MigrationCLITest extends TestCase { - public function testHelpExplainsExplicitProvisioningRecoveryAttestation(): void + public function testHelpExplainsExplicitMigrationRecoveryAttestation(): void { - $this->assertStringContainsString('--recover-provisioning', \MigrationCLI::getHelp()); + $this->assertStringContainsString('--recover-migration-id=', \MigrationCLI::getHelp()); + $this->assertStringContainsString('--recover-migration-attempt-id=', \MigrationCLI::getHelp()); + $this->assertStringNotContainsString('--recover-provisioning', \MigrationCLI::getHelp()); $this->assertStringContainsString('--migration-id=', \MigrationCLI::getHelp()); + $this->assertStringContainsString('--migration-attempt-id=', \MigrationCLI::getHelp()); $this->assertStringContainsString('reuse it for retries', \MigrationCLI::getHelp()); - $this->assertStringContainsString('no active migration', \MigrationCLI::getHelp()); + $this->assertStringContainsString('fresh attempt', \MigrationCLI::getHelp()); + $this->assertStringContainsString('prior migration attempt is terminal', \MigrationCLI::getHelp()); $this->assertStringContainsString('refused by default', \MigrationCLI::getHelp()); } - public function testProvisioningRecoveryRequiresExplicitOperatorAttestation(): void + public function testIncompleteDatabaseRecoveryRequiresExactTerminalMigrationIdentifier(): void { - foreach ([false, true] as $recover) { - $database = $this->createProjectDatabase(); - $arguments = ['MigrationCLI.php', '--migration-id=migration-current']; - if ($recover) { - $arguments[] = '--recover-provisioning'; - } - $cli = new TestMigrationCLI($arguments, $database); - - $destination = $cli->getDestination(); - $this->runTransfer($database, $destination); - - $created = $database->getAuthorization()->skip( - static fn (): Document => $database->getDocument('databases', 'database'), - ); - - if (! $recover) { - $this->assertNotSame([], $destination->getErrors()); - $this->assertSame('provisioning', $created->getAttribute('status')); - $this->assertSame('migration-terminal', $created->getAttribute('migrationId')); - $this->assertTrue($database->getCollection('database_'.$created->getSequence())->isEmpty()); - continue; + $cases = [ + 'absent' => [[], false], + 'bare migration' => [['--recover-migration-id', '--recover-migration-attempt-id=attempt-terminal'], false], + 'bare attempt' => [['--recover-migration-id=migration-terminal', '--recover-migration-attempt-id'], false], + 'empty migration' => [['--recover-migration-id=', '--recover-migration-attempt-id=attempt-terminal'], false], + 'empty attempt' => [['--recover-migration-id=migration-terminal', '--recover-migration-attempt-id='], false], + 'migration only' => [['--recover-migration-id=migration-terminal'], false], + 'attempt only' => [['--recover-migration-attempt-id=attempt-terminal'], false], + 'migration mismatch' => [['--recover-migration-id=migration-other', '--recover-migration-attempt-id=attempt-terminal'], false], + 'attempt mismatch' => [['--recover-migration-id=migration-terminal', '--recover-migration-attempt-id=attempt-other'], false], + 'retired unsafe option' => [['--recover-provisioning'], false], + 'exact' => [[ + '--recover-migration-id=migration-terminal', + '--recover-migration-attempt-id=attempt-terminal', + ], true], + ]; + + foreach (['provisioning', 'failed'] as $status) { + foreach ($cases as [$recoveryArguments, $recover]) { + $database = $this->createProjectDatabase($status); + $arguments = [ + 'MigrationCLI.php', + '--migration-id=migration-current', + '--migration-attempt-id=attempt-current', + ...$recoveryArguments, + ]; + $cli = new TestMigrationCLI($arguments, $database); + + $destination = $cli->getDestination(); + $this->runTransfer($database, $destination); + + $created = $database->getAuthorization()->skip( + static fn (): Document => $database->getDocument('databases', 'database'), + ); + + if (! $recover) { + $this->assertNotSame([], $destination->getErrors()); + $this->assertSame($status, $created->getAttribute('status')); + $this->assertSame('migration-terminal', $created->getAttribute('migrationId')); + $this->assertSame('attempt-terminal', $created->getAttribute('migrationAttemptId')); + $this->assertTrue($database->getCollection('database_'.$created->getSequence())->isEmpty()); + continue; + } + + $this->assertSame([], $destination->getErrors()); + $this->assertSame('ready', $created->getAttribute('status')); + $this->assertSame('migration-current', $created->getAttribute('migrationId')); + $this->assertSame('attempt-current', $created->getAttribute('migrationAttemptId')); + $this->assertFalse($database->getCollection('database_'.$created->getSequence())->isEmpty()); } - - $this->assertSame([], $destination->getErrors()); - $this->assertSame('ready', $created->getAttribute('status')); - $this->assertSame('migration-current', $created->getAttribute('migrationId')); - $this->assertFalse($database->getCollection('database_'.$created->getSequence())->isEmpty()); } } @@ -90,9 +128,22 @@ public function testAppwriteDestinationRequiresMigrationIdentifier(): void $cli->getDestination(); } - private function createProjectDatabase(): Database + public function testAppwriteDestinationRequiresMigrationAttemptIdentifier(): void + { + $cli = new TestMigrationCLI( + ['MigrationCLI.php', '--migration-id=migration-current'], + $this->createProjectDatabase(), + ); + + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('--migration-attempt-id is required for an Appwrite destination'); + + $cli->getDestination(); + } + + private function createProjectDatabase(string $status = 'provisioning'): Database { - $database = new Database(new MemoryAdapter(), new Cache(new MemoryCache())); + $database = new Database(new TransactionalMemoryAdapter(), new Cache(new MemoryCache())); $database ->setDatabase('appwrite') ->setNamespace('_project'); @@ -108,6 +159,7 @@ private function createProjectDatabase(): Database new Attribute(key: 'database', type: ColumnType::String, size: 2000), new Attribute(key: 'status', type: ColumnType::String, size: 16), new Attribute(key: 'migrationId', type: ColumnType::String, size: Database::LENGTH_KEY), + new Attribute(key: 'migrationAttemptId', type: ColumnType::String, size: Database::LENGTH_KEY), ], )); $database->getAuthorization()->skip( @@ -119,8 +171,9 @@ private function createProjectDatabase(): Database 'originalId' => null, 'type' => 'tablesdb', 'database' => '', - 'status' => 'provisioning', + 'status' => $status, 'migrationId' => 'migration-terminal', + 'migrationAttemptId' => 'attempt-terminal', ])), ); From dbcc4410f2ad72e115a35c2badc4b6cfa0680c87 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 1 Sep 2026 05:00:41 +1200 Subject: [PATCH 25/32] (fix): recover terminal migration failures --- bin/MigrationCLI.php | 1 + src/Migration/Destinations/Appwrite.php | 15 +++-- .../AppwriteDatabaseConcurrencyTest.php | 63 +++++++++++++++++-- .../AppwriteDatabaseStatusTest.php | 14 +---- tests/Migration/Unit/MigrationCLITest.php | 1 + 5 files changed, 73 insertions(+), 21 deletions(-) diff --git a/bin/MigrationCLI.php b/bin/MigrationCLI.php index 9070cf39..53425f4a 100644 --- a/bin/MigrationCLI.php +++ b/bin/MigrationCLI.php @@ -180,6 +180,7 @@ public static function getHelp(): string Together attest that the exact prior migration attempt is terminal and allow recovery of its provisioning or failed databases. Recovery is refused by default. + Same-migration failed retries need only a fresh attempt identifier. HELP; } diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index b5867410..a669f9de 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -170,8 +170,10 @@ class Appwrite extends Destination /** * Resolves the authoritative terminal owner of an incomplete database. - * Callers must derive this from their operation lifecycle; null means the - * owner is active or unknown and recovery fails closed. + * Callers must derive this from their operation lifecycle for provisioning + * databases and failed databases owned by another logical migration. A + * failed database owned by this logical migration is already terminal and + * can be retried with a fresh attempt. Null otherwise fails closed. * * @var callable(UtopiaDocument $database): ?ProvisioningOwner */ @@ -227,7 +229,7 @@ class Appwrite extends Destination * @param callable(UtopiaDocument $database):UtopiaDatabase $getDatabasesDB * @param array> $collectionStructure * @param ProvisioningOwner $owner Immutable logical migration and execution-attempt identifiers for databases provisioned by this destination. - * @param callable(UtopiaDocument $database): ?ProvisioningOwner $getRecoverableOwner Returns the exact authoritative terminal owner for an existing `provisioning` or `failed` database, or null when its owner is active or unknown. + * @param callable(UtopiaDocument $database): ?ProvisioningOwner $getRecoverableOwner Returns the exact authoritative terminal owner for an existing `provisioning` database or a `failed` database owned by another logical migration. A same-migration `failed` database is terminal and needs only a fresh attempt. Return null when the owner is active or unknown. * @param OnDuplicate $onDuplicate Behavior when a row with an existing $id is encountered. * @param (callable(Database $resource): string)|null $getDatabaseDSN Resolver for the destination's `_databases.database` value. Pass when the destination project's DSN differs from the source's, so the destination row carries its own DSN instead of inheriting the source's. * @param array>> $collectionStructures Per-database-type metadata collection structures (e.g. `['vectorsdb' => ...]`), used instead of $collectionStructure when the imported database's type has an entry. Types with an entry also get type-specific metadata written (e.g. vectorsdb collection `dimension`). @@ -821,7 +823,12 @@ protected function createDatabase(Database $resource): bool $expectedOwner = null; if ($isIncomplete) { $snapshotOwner = $this->getProvisioningOwner($existing); - $expectedOwner = ($this->getRecoverableOwner)($existing); + $isSameMigrationFailure = $status === self::DATABASE_STATUS_FAILED + && $snapshotOwner !== null + && $snapshotOwner->migrationId === $this->owner->migrationId; + $expectedOwner = $isSameMigrationFailure + ? $snapshotOwner + : ($this->getRecoverableOwner)($existing); if ( $snapshotOwner === null || ! $expectedOwner instanceof ProvisioningOwner diff --git a/tests/Migration/E2E/Destinations/AppwriteDatabaseConcurrencyTest.php b/tests/Migration/E2E/Destinations/AppwriteDatabaseConcurrencyTest.php index d614a5ff..9e715679 100644 --- a/tests/Migration/E2E/Destinations/AppwriteDatabaseConcurrencyTest.php +++ b/tests/Migration/E2E/Destinations/AppwriteDatabaseConcurrencyTest.php @@ -31,8 +31,22 @@ final class RecordingSQLiteProjectDatabase extends UtopiaDatabase public ?Closure $beforeBackingCollectionCreate = null; + public ?Closure $beforeTransaction = null; + public bool $throwAfterBackingCollectionCallback = false; + #[Override] + public function withTransaction(callable $callback): mixed + { + if ($this->beforeTransaction !== null) { + $beforeTransaction = $this->beforeTransaction; + $this->beforeTransaction = null; + $beforeTransaction(); + } + + return parent::withTransaction($callback); + } + #[Override] public function updateDocument(string $collection, string $id, UtopiaDocument $document): UtopiaDocument { @@ -65,31 +79,68 @@ public function createCollection(Collection $collection): Collection final class AppwriteDatabaseConcurrencyTest extends TestCase { + public function testSameMigrationFailedFreshAttemptsHaveOneWinner(): void + { + [$second, $third, $path] = $this->createSharedDatabases(); + + try { + $this->seedDatabase($second, 'failed', 'migration-shared', 'attempt-first'); + $winner = $this->createDestination( + $third, + 'migration-shared', + 'attempt-third', + static fn (UtopiaDocument $snapshot): ?ProvisioningOwner => null, + ); + $second->beforeTransaction = function () use ($third, $winner): void { + $this->runTransfer($third, $winner); + }; + $loser = $this->createDestination( + $second, + 'migration-shared', + 'attempt-second', + static fn (UtopiaDocument $snapshot): ?ProvisioningOwner => null, + ); + $second->databaseWrites = []; + + $this->runTransfer($second, $loser); + + $database = $this->getDatabaseDocument($third); + $this->assertSame([], $this->errorMessages($winner)); + $this->assertNotSame([], $this->errorMessages($loser)); + $this->assertSame([], $second->databaseWrites); + $this->assertSame('ready', $database->getAttribute('status')); + $this->assertSame('migration-shared', $database->getAttribute('migrationId')); + $this->assertSame('attempt-third', $database->getAttribute('migrationAttemptId')); + } finally { + $this->removeSQLiteFiles($path); + } + } + public function testStaleIncompleteClaimLosesAfterAnotherSuccessorCommits(): void { foreach (['provisioning', 'failed'] as $status) { [$second, $third, $path] = $this->createSharedDatabases(); try { - $this->seedDatabase($second, $status, 'migration-shared', 'attempt-first'); + $this->seedDatabase($second, $status, 'migration-first', 'attempt-first'); $winner = null; $loser = $this->createDestination( $second, - 'migration-shared', + 'migration-second', 'attempt-second', function (UtopiaDocument $snapshot) use ($third, &$winner): ProvisioningOwner { $winner = $this->createDestination( $third, - 'migration-shared', + 'migration-third', 'attempt-third', static fn (UtopiaDocument $document): ProvisioningOwner => new ProvisioningOwner( - 'migration-shared', + 'migration-first', 'attempt-first', ), ); $this->runTransfer($third, $winner); - return new ProvisioningOwner('migration-shared', 'attempt-first'); + return new ProvisioningOwner('migration-first', 'attempt-first'); }, ); $second->databaseWrites = []; @@ -102,7 +153,7 @@ function (UtopiaDocument $snapshot) use ($third, &$winner): ProvisioningOwner { $this->assertNotSame([], $this->errorMessages($loser)); $this->assertSame([], $second->databaseWrites); $this->assertSame('ready', $database->getAttribute('status')); - $this->assertSame('migration-shared', $database->getAttribute('migrationId')); + $this->assertSame('migration-third', $database->getAttribute('migrationId')); $this->assertSame('attempt-third', $database->getAttribute('migrationAttemptId')); } finally { $this->removeSQLiteFiles($path); diff --git a/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php b/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php index 5d322120..8c8a3aea 100644 --- a/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php +++ b/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php @@ -472,12 +472,8 @@ public function testPreExistingRecoveryRequiresAtomicMutationCapability(): void $destination = $this->runDatabaseTransfer( $database, explicit: false, - migrationId: 'migration-new', + migrationId: 'migration-old', migrationAttemptId: 'attempt-new', - getRecoverableOwner: static fn (UtopiaDocument $database): ProvisioningOwner => new ProvisioningOwner( - 'migration-old', - 'attempt-old', - ), ); $existing = $this->getDatabaseDocument($database); @@ -629,19 +625,15 @@ public function testFailedDatabaseRetrySucceedsUnderOnDuplicateFail(): void $destination = $this->runDatabaseTransfer( $database, explicit: false, - migrationId: 'migration-recovery', + migrationId: 'migration-current', migrationAttemptId: 'attempt-recovery', - getRecoverableOwner: static fn (UtopiaDocument $existing): ProvisioningOwner => new ProvisioningOwner( - 'migration-current', - 'attempt-current', - ), ); $recovered = $this->getDatabaseDocument($database); $this->assertSame([], $this->errorMessages($destination)); $this->assertSame('ready', $recovered->getAttribute('status')); $this->assertSame($failed->getSequence(), $recovered->getSequence()); - $this->assertSame('migration-recovery', $recovered->getAttribute('migrationId')); + $this->assertSame('migration-current', $recovered->getAttribute('migrationId')); $this->assertSame('attempt-recovery', $recovered->getAttribute('migrationAttemptId')); $this->assertFalse( $database->getCollection('database_'.$recovered->getSequence())->isEmpty(), diff --git a/tests/Migration/Unit/MigrationCLITest.php b/tests/Migration/Unit/MigrationCLITest.php index d9ae8131..b04c9ae6 100644 --- a/tests/Migration/Unit/MigrationCLITest.php +++ b/tests/Migration/Unit/MigrationCLITest.php @@ -61,6 +61,7 @@ public function testHelpExplainsExplicitMigrationRecoveryAttestation(): void $this->assertStringContainsString('fresh attempt', \MigrationCLI::getHelp()); $this->assertStringContainsString('prior migration attempt is terminal', \MigrationCLI::getHelp()); $this->assertStringContainsString('refused by default', \MigrationCLI::getHelp()); + $this->assertStringContainsString('Same-migration failed retries', \MigrationCLI::getHelp()); } public function testIncompleteDatabaseRecoveryRequiresExactTerminalMigrationIdentifier(): void From e7d88ded53c5326a12b1dc11df3354772a7198b0 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 1 Sep 2026 05:09:04 +1200 Subject: [PATCH 26/32] (fix): require terminal migration attestation --- README.md | 8 +++ bin/MigrationCLI.php | 2 +- src/Migration/Destinations/Appwrite.php | 18 +++---- .../AppwriteDatabaseConcurrencyTest.php | 51 ------------------- .../AppwriteDatabaseStatusTest.php | 38 +++++++++++++- tests/Migration/Unit/MigrationCLITest.php | 2 +- 6 files changed, 53 insertions(+), 66 deletions(-) diff --git a/README.md b/README.md index c021e60f..5dc2aefc 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,14 @@ $transfer->run( ); ``` +## Appwrite database recovery + +Appwrite database destinations use a `ProvisioningOwner` made from a stable logical migration identifier and a fresh attempt identifier for every execution. The required `getRecoverableOwner` callback is the recovery authority for an existing database whose status is `provisioning` or `failed`. + +A database status is local to that resource. It does not prove that the migration attempt which owns it has stopped, because an import can continue with other resources after recording a database failure. The callback must therefore consult the caller's authoritative operation lifecycle and return the exact stored owner only after that attempt is terminal. Return `null` while it is active or unknown; recovery then fails closed. This rule also applies when the retry uses the same logical migration identifier. + +The standalone CLI requires `--migration-id` and a fresh `--migration-attempt-id`. Recovering an incomplete database additionally requires both `--recover-migration-id` and `--recover-migration-attempt-id` for the exact terminal prior attempt. + ## Supported Resources Chart Sources: diff --git a/bin/MigrationCLI.php b/bin/MigrationCLI.php index 53425f4a..7f71cca2 100644 --- a/bin/MigrationCLI.php +++ b/bin/MigrationCLI.php @@ -180,7 +180,7 @@ public static function getHelp(): string Together attest that the exact prior migration attempt is terminal and allow recovery of its provisioning or failed databases. Recovery is refused by default. - Same-migration failed retries need only a fresh attempt identifier. + Resource status alone never proves an attempt terminal. HELP; } diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index a669f9de..ea05fe22 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -170,10 +170,9 @@ class Appwrite extends Destination /** * Resolves the authoritative terminal owner of an incomplete database. - * Callers must derive this from their operation lifecycle for provisioning - * databases and failed databases owned by another logical migration. A - * failed database owned by this logical migration is already terminal and - * can be retried with a fresh attempt. Null otherwise fails closed. + * Database status is resource-local and never proves the overall migration + * attempt terminal. Callers must derive this from their authoritative + * operation lifecycle; null means active or unknown and fails closed. * * @var callable(UtopiaDocument $database): ?ProvisioningOwner */ @@ -229,7 +228,7 @@ class Appwrite extends Destination * @param callable(UtopiaDocument $database):UtopiaDatabase $getDatabasesDB * @param array> $collectionStructure * @param ProvisioningOwner $owner Immutable logical migration and execution-attempt identifiers for databases provisioned by this destination. - * @param callable(UtopiaDocument $database): ?ProvisioningOwner $getRecoverableOwner Returns the exact authoritative terminal owner for an existing `provisioning` database or a `failed` database owned by another logical migration. A same-migration `failed` database is terminal and needs only a fresh attempt. Return null when the owner is active or unknown. + * @param callable(UtopiaDocument $database): ?ProvisioningOwner $getRecoverableOwner Returns the exact authoritative terminal owner for an existing `provisioning` or `failed` database. Resource status alone is not lifecycle proof; return null while the owning migration attempt is active or unknown. * @param OnDuplicate $onDuplicate Behavior when a row with an existing $id is encountered. * @param (callable(Database $resource): string)|null $getDatabaseDSN Resolver for the destination's `_databases.database` value. Pass when the destination project's DSN differs from the source's, so the destination row carries its own DSN instead of inheriting the source's. * @param array>> $collectionStructures Per-database-type metadata collection structures (e.g. `['vectorsdb' => ...]`), used instead of $collectionStructure when the imported database's type has an entry. Types with an entry also get type-specific metadata written (e.g. vectorsdb collection `dimension`). @@ -823,19 +822,14 @@ protected function createDatabase(Database $resource): bool $expectedOwner = null; if ($isIncomplete) { $snapshotOwner = $this->getProvisioningOwner($existing); - $isSameMigrationFailure = $status === self::DATABASE_STATUS_FAILED - && $snapshotOwner !== null - && $snapshotOwner->migrationId === $this->owner->migrationId; - $expectedOwner = $isSameMigrationFailure - ? $snapshotOwner - : ($this->getRecoverableOwner)($existing); + $expectedOwner = ($this->getRecoverableOwner)($existing); if ( $snapshotOwner === null || ! $expectedOwner instanceof ProvisioningOwner || ! $snapshotOwner->equals($expectedOwner) || $this->owner->attemptId === $expectedOwner->attemptId ) { - throw new DatabaseException('Database '.$resource->getId().' recovery owner is active, unknown, mismatched, or reuses the prior attempt'); + throw new DatabaseException('Database '.$resource->getId().' recovery requires exact terminal-owner attestation and a fresh attempt'); } } diff --git a/tests/Migration/E2E/Destinations/AppwriteDatabaseConcurrencyTest.php b/tests/Migration/E2E/Destinations/AppwriteDatabaseConcurrencyTest.php index 9e715679..960858a5 100644 --- a/tests/Migration/E2E/Destinations/AppwriteDatabaseConcurrencyTest.php +++ b/tests/Migration/E2E/Destinations/AppwriteDatabaseConcurrencyTest.php @@ -31,22 +31,8 @@ final class RecordingSQLiteProjectDatabase extends UtopiaDatabase public ?Closure $beforeBackingCollectionCreate = null; - public ?Closure $beforeTransaction = null; - public bool $throwAfterBackingCollectionCallback = false; - #[Override] - public function withTransaction(callable $callback): mixed - { - if ($this->beforeTransaction !== null) { - $beforeTransaction = $this->beforeTransaction; - $this->beforeTransaction = null; - $beforeTransaction(); - } - - return parent::withTransaction($callback); - } - #[Override] public function updateDocument(string $collection, string $id, UtopiaDocument $document): UtopiaDocument { @@ -79,43 +65,6 @@ public function createCollection(Collection $collection): Collection final class AppwriteDatabaseConcurrencyTest extends TestCase { - public function testSameMigrationFailedFreshAttemptsHaveOneWinner(): void - { - [$second, $third, $path] = $this->createSharedDatabases(); - - try { - $this->seedDatabase($second, 'failed', 'migration-shared', 'attempt-first'); - $winner = $this->createDestination( - $third, - 'migration-shared', - 'attempt-third', - static fn (UtopiaDocument $snapshot): ?ProvisioningOwner => null, - ); - $second->beforeTransaction = function () use ($third, $winner): void { - $this->runTransfer($third, $winner); - }; - $loser = $this->createDestination( - $second, - 'migration-shared', - 'attempt-second', - static fn (UtopiaDocument $snapshot): ?ProvisioningOwner => null, - ); - $second->databaseWrites = []; - - $this->runTransfer($second, $loser); - - $database = $this->getDatabaseDocument($third); - $this->assertSame([], $this->errorMessages($winner)); - $this->assertNotSame([], $this->errorMessages($loser)); - $this->assertSame([], $second->databaseWrites); - $this->assertSame('ready', $database->getAttribute('status')); - $this->assertSame('migration-shared', $database->getAttribute('migrationId')); - $this->assertSame('attempt-third', $database->getAttribute('migrationAttemptId')); - } finally { - $this->removeSQLiteFiles($path); - } - } - public function testStaleIncompleteClaimLosesAfterAnotherSuccessorCommits(): void { foreach (['provisioning', 'failed'] as $status) { diff --git a/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php b/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php index 8c8a3aea..7114a505 100644 --- a/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php +++ b/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php @@ -456,6 +456,34 @@ public function testRecoveryRefusesReusingThePriorAttempt(): void $this->assertSame('attempt-shared', $existing->getAttribute('migrationAttemptId')); } + public function testSameMigrationFailedRetryRequiresTerminalAttestation(): void + { + $database = new RecordingProjectDatabase(new ReplicaMemoryAdapter(), new Cache(new MemoryCache())); + $this->createProjectDatabase(withStatus: true, database: $database); + $this->seedDatabase( + $database, + status: 'failed', + migrationId: 'migration-shared', + migrationAttemptId: 'attempt-active', + ); + $database->databaseWrites = []; + + $destination = $this->runDatabaseTransfer( + $database, + explicit: false, + migrationId: 'migration-shared', + migrationAttemptId: 'attempt-retry', + getRecoverableOwner: static fn (UtopiaDocument $database): ?ProvisioningOwner => null, + ); + + $existing = $this->getDatabaseDocument($database); + $this->assertNotSame([], $this->errorMessages($destination)); + $this->assertSame([], $database->databaseWrites); + $this->assertSame('failed', $existing->getAttribute('status')); + $this->assertSame('migration-shared', $existing->getAttribute('migrationId')); + $this->assertSame('attempt-active', $existing->getAttribute('migrationAttemptId')); + } + public function testPreExistingRecoveryRequiresAtomicMutationCapability(): void { foreach ([new MemoryAdapter(), new StandaloneMemoryAdapter()] as $adapter) { @@ -474,6 +502,10 @@ public function testPreExistingRecoveryRequiresAtomicMutationCapability(): void explicit: false, migrationId: 'migration-old', migrationAttemptId: 'attempt-new', + getRecoverableOwner: static fn (UtopiaDocument $database): ProvisioningOwner => new ProvisioningOwner( + 'migration-old', + 'attempt-old', + ), ); $existing = $this->getDatabaseDocument($database); @@ -627,6 +659,10 @@ public function testFailedDatabaseRetrySucceedsUnderOnDuplicateFail(): void explicit: false, migrationId: 'migration-current', migrationAttemptId: 'attempt-recovery', + getRecoverableOwner: static fn (UtopiaDocument $existing): ProvisioningOwner => new ProvisioningOwner( + 'migration-current', + 'attempt-current', + ), ); $recovered = $this->getDatabaseDocument($database); @@ -730,7 +766,7 @@ public function testConcurrentMigrationDoesNotRecoverAnActivelyProvisioningDatab $this->assertSame([], $this->errorMessages($first)); $this->assertInstanceOf(CountingAppwriteDestination::class, $second); $this->assertNotSame([], $this->errorMessages($second)); - $this->assertStringContainsString('recovery owner', $this->errorMessages($second)[0]); + $this->assertStringContainsString('terminal-owner attestation', $this->errorMessages($second)[0]); $this->assertSame('provisioning', $statusDuringOverlap); $this->assertFalse($collectionExistsDuringOverlap); $this->assertSame('ready', $created->getAttribute('status')); diff --git a/tests/Migration/Unit/MigrationCLITest.php b/tests/Migration/Unit/MigrationCLITest.php index b04c9ae6..71f88aac 100644 --- a/tests/Migration/Unit/MigrationCLITest.php +++ b/tests/Migration/Unit/MigrationCLITest.php @@ -61,7 +61,7 @@ public function testHelpExplainsExplicitMigrationRecoveryAttestation(): void $this->assertStringContainsString('fresh attempt', \MigrationCLI::getHelp()); $this->assertStringContainsString('prior migration attempt is terminal', \MigrationCLI::getHelp()); $this->assertStringContainsString('refused by default', \MigrationCLI::getHelp()); - $this->assertStringContainsString('Same-migration failed retries', \MigrationCLI::getHelp()); + $this->assertStringContainsString('Resource status alone never proves', \MigrationCLI::getHelp()); } public function testIncompleteDatabaseRecoveryRequiresExactTerminalMigrationIdentifier(): void From 3f42d20cfa6a2adf2007dc495650cd368013b574 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 1 Sep 2026 08:22:03 +1200 Subject: [PATCH 27/32] (fix): surface database finalizer failures --- src/Migration/Destinations/Appwrite.php | 12 ++- .../AppwriteDatabaseStatusTest.php | 75 +++++++++++++++---- 2 files changed, 71 insertions(+), 16 deletions(-) diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index ea05fe22..0cafda2f 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -341,9 +341,15 @@ private function markProvisionedDatabasesReady(): void foreach (\array_keys($this->provisioningDatabases) as $databaseId) { try { $this->setDatabaseStatus($databaseId, self::DATABASE_STATUS_READY); - } catch (\Throwable) { - // Best-effort: a transient error on one database must not strand the rest - // in provisioning or block the orphan-cleanup sweep that follows. + } catch (\Throwable $error) { + $this->addError(new Exception( + resourceName: Resource::TYPE_DATABASE, + resourceGroup: Transfer::GROUP_DATABASES, + resourceId: $databaseId, + message: $error->getMessage(), + code: $error->getCode(), + previous: $error, + )); } } } diff --git a/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php b/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php index 7114a505..d580da2d 100644 --- a/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php +++ b/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php @@ -16,6 +16,7 @@ use Utopia\Migration\Destinations\Appwrite as AppwriteDestination; use Utopia\Migration\Destinations\Appwrite\ProvisioningOwner; use Utopia\Migration\Destinations\OnDuplicate; +use Utopia\Migration\Exception as MigrationException; use Utopia\Migration\Resource; use Utopia\Migration\Resources\Database\Database as DatabaseResource; use Utopia\Migration\Transfer; @@ -136,17 +137,21 @@ public function getDocument(string $collection, string $id, array $queries = [], class RecordingProjectDatabase extends UtopiaDatabase { - /** @var list}> */ + /** @var list}> */ public array $databaseWrites = []; public bool $failReadyWrite = false; + /** @var list */ + public array $failReadyWrites = []; + #[Override] public function createDocument(string $collection, UtopiaDocument $document): UtopiaDocument { if ($collection === 'databases') { $this->databaseWrites[] = [ 'operation' => 'create', + 'id' => $document->getId(), 'document' => $document->getArrayCopy(), ]; } @@ -160,12 +165,13 @@ public function updateDocument(string $collection, string $id, UtopiaDocument $d if ($collection === 'databases') { $this->databaseWrites[] = [ 'operation' => 'update', + 'id' => $id, 'document' => $document->getArrayCopy(), ]; } if ( - $this->failReadyWrite + ($this->failReadyWrite || \in_array($id, $this->failReadyWrites, true)) && $collection === 'databases' && $document->getAttribute('status') === 'ready' ) { @@ -608,12 +614,52 @@ public function testReadyWriteFailureRetainsProvisioningOwner(): void ); $created = $this->getDatabaseDocument($database); - $this->assertSame([], $this->errorMessages($destination)); + $errors = $destination->getErrors(); + $this->assertCount(1, $errors); + $this->assertSame(Resource::TYPE_DATABASE, $errors[0]->getResourceName()); + $this->assertSame(Transfer::GROUP_DATABASES, $errors[0]->getResourceGroup()); + $this->assertSame('database', $errors[0]->getResourceId()); + $this->assertSame('ready status unavailable', $errors[0]->getMessage()); + $this->assertInstanceOf(DatabaseException::class, $errors[0]->getPrevious()); $this->assertSame('provisioning', $created->getAttribute('status')); $this->assertSame('migration-ready-failure', $created->getAttribute('migrationId')); $this->assertSame('attempt-ready-failure', $created->getAttribute('migrationAttemptId')); } + public function testReadyFinalizationAttemptsEveryDatabaseAndReportsEveryFailure(): void + { + $database = new RecordingProjectDatabase(new MemoryAdapter(), new Cache(new MemoryCache())); + $this->createProjectDatabase(withStatus: true, database: $database); + $database->failReadyWrites = ['database-first', 'database-third']; + + $destination = $this->runDatabaseTransfer( + $database, + explicit: false, + migrationId: 'migration-finalizers', + migrationAttemptId: 'attempt-finalizers', + databaseIds: ['database-first', 'database-second', 'database-third'], + ); + + $readyAttempts = \array_values(\array_map( + static fn (array $write): string => $write['id'], + \array_filter( + $database->databaseWrites, + static fn (array $write): bool => ($write['document']['status'] ?? null) === 'ready', + ), + )); + $errors = $destination->getErrors(); + $errorIds = \array_map( + static fn (MigrationException $error): string => $error->getResourceId(), + $errors, + ); + + $this->assertSame(['database-first', 'database-second', 'database-third'], $readyAttempts); + $this->assertSame(['database-first', 'database-third'], $errorIds); + $this->assertSame('provisioning', $this->getDatabaseDocument($database, 'database-first')->getAttribute('status')); + $this->assertSame('ready', $this->getDatabaseDocument($database, 'database-second')->getAttribute('status')); + $this->assertSame('provisioning', $this->getDatabaseDocument($database, 'database-third')->getAttribute('status')); + } + public function testReloadFailureMarksTheDatabaseFailed(): void { $database = new ReloadFailingProjectDatabase( @@ -890,6 +936,7 @@ private function runDatabaseTransfer( ?callable $getRecoverableOwner = null, OnDuplicate $onDuplicate = OnDuplicate::Fail, string $resourceUpdatedAt = '', + array $databaseIds = ['database'], ): CountingAppwriteDestination { $source = new class () extends MockSource { #[Override] @@ -898,14 +945,16 @@ public function supportsDatabaseStatus(): bool return true; } }; - $source->pushMockResource(new DatabaseResource( - id: 'database', - name: 'Database', - updatedAt: $resourceUpdatedAt, - type: 'tablesdb', - database: 'source-dsn', - databaseStatus: 'ready', - )); + foreach ($databaseIds as $databaseId) { + $source->pushMockResource(new DatabaseResource( + id: $databaseId, + name: 'Database', + updatedAt: $resourceUpdatedAt, + type: 'tablesdb', + database: 'source-dsn', + databaseStatus: 'ready', + )); + } $destination = new CountingAppwriteDestination( project: 'destination-project', @@ -952,10 +1001,10 @@ static function (): void { return $destination; } - private function getDatabaseDocument(UtopiaDatabase $database): UtopiaDocument + private function getDatabaseDocument(UtopiaDatabase $database, string $databaseId = 'database'): UtopiaDocument { return $database->getAuthorization()->skip( - static fn (): UtopiaDocument => $database->getDocument('databases', 'database'), + static fn (): UtopiaDocument => $database->getDocument('databases', $databaseId), ); } From ec7db1f9b103be056c137cb38174d252db6dc449 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 1 Sep 2026 10:28:07 +1200 Subject: [PATCH 28/32] (fix): fence database migration finalization --- composer.lock | 8 +- src/Migration/Destinations/Appwrite.php | 123 ++++++------------ .../AppwriteDatabaseConcurrencyTest.php | 22 ++-- .../AppwriteDatabaseStatusTest.php | 77 +++++++---- tests/Migration/Unit/MigrationCLITest.php | 7 +- 5 files changed, 115 insertions(+), 122 deletions(-) diff --git a/composer.lock b/composer.lock index a50b3100..f7bceff9 100644 --- a/composer.lock +++ b/composer.lock @@ -2612,12 +2612,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "745378d6c0124b7af31d69d2715cb07aa88fe9af" + "reference": "93cf45fe3a28b1cc46f7850b1bbddfa67d518144" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/745378d6c0124b7af31d69d2715cb07aa88fe9af", - "reference": "745378d6c0124b7af31d69d2715cb07aa88fe9af", + "url": "https://api.github.com/repos/utopia-php/database/zipball/93cf45fe3a28b1cc46f7850b1bbddfa67d518144", + "reference": "93cf45fe3a28b1cc46f7850b1bbddfa67d518144", "shasum": "" }, "require": { @@ -2703,7 +2703,7 @@ "source": "https://github.com/utopia-php/database/tree/feat-query-lib", "issues": "https://github.com/utopia-php/database/issues" }, - "time": "2026-08-31T16:19:44+00:00" + "time": "2026-08-31T22:07:53+00:00" }, { "name": "utopia-php/dsn", diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index 0cafda2f..294dab08 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -33,6 +33,7 @@ use Utopia\Database\Document as UtopiaDocument; use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Exception\Authorization as AuthorizationException; +use Utopia\Database\Exception\Conflict as ConflictException; use Utopia\Database\Exception\Duplicate as DuplicateException; use Utopia\Database\Exception\Limit as LimitException; use Utopia\Database\Exception\Structure as StructureException; @@ -178,9 +179,6 @@ class Appwrite extends Destination */ private $getRecoverableOwner; - /** @var array */ - private array $createdDatabaseOwners = []; - /** * @var array */ @@ -310,7 +308,11 @@ private function getSupportForDatabaseStatus(): bool return $this->databaseStatusSupported = false; } - /** Orphan cleanup runs only after a successful migration — a mid-run throw preserves the destination as-is. */ + /** + * Transfer resources without committing terminal destination state. The + * caller must first persist its own finalization claim, then invoke + * success() while that generation is still authoritative. + */ #[Override] public function run( array $resources, @@ -320,9 +322,17 @@ public function run( ): void { $this->resetRunState(); parent::run($resources, $callback, $rootResourceId, $rootResourceType); - // parent::run() returning means every resource transferred, so the databases are usable. - // Flip status before the orphan sweep so a cleanup failure can't strand them in `provisioning`. - $this->markProvisionedDatabasesReady(); + } + + /** Finalize destination state only after the caller has fenced terminal ownership. */ + #[Override] + public function success(): void + { + // Flip status before the orphan sweep so a cleanup failure can't strand databases in `provisioning`. + if (! $this->markProvisionedDatabasesReady()) { + return; + } + $this->cleanupOverwriteOrphans(); } @@ -333,15 +343,18 @@ private function resetRunState(): void $this->orphansByTable = []; $this->processedTwoWayPairs = []; $this->provisioningDatabases = []; - $this->createdDatabaseOwners = []; } - private function markProvisionedDatabasesReady(): void + private function markProvisionedDatabasesReady(): bool { + $ready = true; foreach (\array_keys($this->provisioningDatabases) as $databaseId) { try { - $this->setDatabaseStatus($databaseId, self::DATABASE_STATUS_READY); + if (! $this->setDatabaseStatus($databaseId, self::DATABASE_STATUS_READY)) { + throw new DatabaseException('Database provisioning owner changed before finalization'); + } } catch (\Throwable $error) { + $ready = false; $this->addError(new Exception( resourceName: Resource::TYPE_DATABASE, resourceGroup: Transfer::GROUP_DATABASES, @@ -352,15 +365,17 @@ private function markProvisionedDatabasesReady(): void )); } } + + return $ready; } - private function setDatabaseStatus(string $databaseId, string $status): void + private function setDatabaseStatus(string $databaseId, string $status): bool { if (! $this->getSupportForDatabaseStatus()) { - return; + return true; } - $transition = $this->dbForProject->withTransaction(function () use ($databaseId, $status): ?bool { + try { $database = $this->dbForProject->getDocument( self::META_DATABASES, $databaseId, @@ -376,58 +391,22 @@ private function setDatabaseStatus(string $databaseId, string $status): void return false; } - if (! $this->supportsAtomicOwnerMutation()) { - return null; + $version = $database->getVersion(); + if ($version === null) { + throw new DatabaseException('Database provisioning ownership requires a document version'); } $this->dbForProject->updateDocument( self::META_DATABASES, $databaseId, new UtopiaDocument(['status' => $status]), + expectedVersion: $version, ); return true; - }); - - if ($transition === true) { - unset($this->createdDatabaseOwners[$databaseId]); - return; - } - - if ($transition === false) { - unset($this->createdDatabaseOwners[$databaseId]); - return; - } - - $createdOwner = $this->createdDatabaseOwners[$databaseId] ?? null; - if ($createdOwner === null || ! $createdOwner->equals($this->owner)) { - return; - } - - // Adapters without an atomic claim primitive may finalize only the exact row this - // destination instance just created. forUpdate disables caches even when no lock exists. - $database = $this->dbForProject->getDocument( - self::META_DATABASES, - $databaseId, - forUpdate: true, - ); - $owner = $this->getProvisioningOwner($database); - if ( - $database->isEmpty() - || $database->getAttribute('status') !== self::DATABASE_STATUS_PROVISIONING - || $owner === null - || ! $owner->equals($createdOwner) - ) { - unset($this->createdDatabaseOwners[$databaseId]); - return; + } catch (ConflictException) { + return false; } - - $this->dbForProject->updateDocument( - self::META_DATABASES, - $databaseId, - new UtopiaDocument(['status' => $status]), - ); - unset($this->createdDatabaseOwners[$databaseId]); } private function getProvisioningOwner(UtopiaDocument $database): ?ProvisioningOwner @@ -446,29 +425,6 @@ private function getProvisioningOwner(UtopiaDocument $database): ?ProvisioningOw return new ProvisioningOwner($migrationId, $attemptId); } - /** - * This fence depends on the exact utopia-php/database pin in composer.lock: - * SQL exposes UpdateLock or TransactionRetries, SQLite uses BEGIN IMMEDIATE, - * and replica Mongo exposes TransactionRetries only while a session is active. - */ - private function supportsAtomicOwnerMutation(): bool - { - $adapter = $this->dbForProject->getAdapter(); - - return $adapter->inTransaction() - && ( - $adapter->supports(Capability::UpdateLock) - || $adapter->supports(Capability::TransactionRetries) - ); - } - - private function requireAtomicOwnerMutation(): void - { - if (! $this->supportsAtomicOwnerMutation()) { - throw new DatabaseException('Database provisioning ownership requires an atomic transaction with update locks or conflict retries'); - } - } - /** Best-effort transition to `failed`; a secondary error here must not mask the caller's original throw. */ private function markDatabaseFailed(string $databaseId): void { @@ -908,13 +864,15 @@ protected function createDatabase(Database $resource): bool $document['migrationAttemptId'] = $this->owner->attemptId; } - // This gate must remain inside the transaction and immediately before the - // existing-row mutation so the locked ownership evidence cannot go stale. - $this->requireAtomicOwnerMutation(); + $version = $locked->getVersion(); + if ($version === null) { + throw new DatabaseException('Database provisioning ownership requires a document version'); + } $this->dbForProject->updateDocument( self::META_DATABASES, $locked->getId(), new UtopiaDocument($document), + expectedVersion: $version, ); } @@ -985,9 +943,6 @@ protected function createDatabase(Database $resource): bool } $database = $this->dbForProject->createDocument(self::META_DATABASES, new UtopiaDocument($document)); - if ($supportsStatus) { - $this->createdDatabaseOwners[$database->getId()] = $this->owner; - } try { $database = $this->dbForProject->getDocument(self::META_DATABASES, $database->getId()); diff --git a/tests/Migration/E2E/Destinations/AppwriteDatabaseConcurrencyTest.php b/tests/Migration/E2E/Destinations/AppwriteDatabaseConcurrencyTest.php index 960858a5..d6dcd349 100644 --- a/tests/Migration/E2E/Destinations/AppwriteDatabaseConcurrencyTest.php +++ b/tests/Migration/E2E/Destinations/AppwriteDatabaseConcurrencyTest.php @@ -34,13 +34,13 @@ final class RecordingSQLiteProjectDatabase extends UtopiaDatabase public bool $throwAfterBackingCollectionCallback = false; #[Override] - public function updateDocument(string $collection, string $id, UtopiaDocument $document): UtopiaDocument + public function updateDocument(string $collection, string $id, UtopiaDocument $document, ?int $expectedVersion = null): UtopiaDocument { if ($collection === 'databases') { $this->databaseWrites[] = $document->getArrayCopy(); } - return parent::updateDocument($collection, $id, $document); + return parent::updateDocument($collection, $id, $document, $expectedVersion); } #[Override] @@ -150,7 +150,10 @@ function () use ($third, &$successor): void { static fn (array $document): bool => ($document['status'] ?? null) === 'ready', )); $this->assertInstanceOf(AppwriteDestination::class, $successor); - $this->assertSame([], $this->errorMessages($destination)); + $this->assertSame( + ['Database provisioning owner changed before finalization'], + $this->errorMessages($destination), + ); $this->assertSame([], $this->errorMessages($successor)); $this->assertSame([], $readyWrites); $this->assertSame('provisioning', $database->getAttribute('status')); @@ -253,11 +256,14 @@ public function supportsDatabaseStatus(): bool $transfer = new Transfer($source, $destination); $database->getAuthorization()->skip( - static fn () => $transfer->run( - [Resource::TYPE_DATABASE], - $callback ?? static function (): void { - }, - ), + static function () use ($callback, $destination, $transfer): void { + $transfer->run( + [Resource::TYPE_DATABASE], + $callback ?? static function (): void { + }, + ); + $destination->success(); + }, ); } diff --git a/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php b/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php index d580da2d..464234e2 100644 --- a/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php +++ b/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php @@ -101,13 +101,13 @@ public function getDocument(string $collection, string $id, array $queries = [], } #[Override] - public function updateDocument(string $collection, string $id, UtopiaDocument $document): UtopiaDocument + public function updateDocument(string $collection, string $id, UtopiaDocument $document, ?int $expectedVersion = null): UtopiaDocument { if ($this->failDatabasesWrites && $collection === 'databases') { throw new DatabaseException('metadata store unavailable'); } - return parent::updateDocument($collection, $id, $document); + return parent::updateDocument($collection, $id, $document, $expectedVersion); } } @@ -137,7 +137,7 @@ public function getDocument(string $collection, string $id, array $queries = [], class RecordingProjectDatabase extends UtopiaDatabase { - /** @var list}> */ + /** @var list, expectedVersion?: int|null}> */ public array $databaseWrites = []; public bool $failReadyWrite = false; @@ -160,13 +160,14 @@ public function createDocument(string $collection, UtopiaDocument $document): Ut } #[Override] - public function updateDocument(string $collection, string $id, UtopiaDocument $document): UtopiaDocument + public function updateDocument(string $collection, string $id, UtopiaDocument $document, ?int $expectedVersion = null): UtopiaDocument { if ($collection === 'databases') { $this->databaseWrites[] = [ 'operation' => 'update', 'id' => $id, 'document' => $document->getArrayCopy(), + 'expectedVersion' => $expectedVersion, ]; } @@ -178,7 +179,7 @@ public function updateDocument(string $collection, string $id, UtopiaDocument $d throw new DatabaseException('ready status unavailable'); } - return parent::updateDocument($collection, $id, $document); + return parent::updateDocument($collection, $id, $document, $expectedVersion); } } @@ -298,8 +299,9 @@ public function testStandaloneUniqueCreateCannotFinalizeAfterOwnerMismatch(): vo ), )); $created = $this->getDatabaseDocument($database); - $this->assertSame([], $this->errorMessages($destination)); - $this->assertSame([], $terminalWrites); + $this->assertSame(['Database provisioning owner changed before finalization'], $this->errorMessages($destination)); + $this->assertCount(1, $terminalWrites); + $this->assertIsInt($terminalWrites[0]['expectedVersion']); $this->assertSame('provisioning', $created->getAttribute('status')); $this->assertSame('migration-successor', $created->getAttribute('migrationId')); $this->assertSame('attempt-successor', $created->getAttribute('migrationAttemptId')); @@ -490,7 +492,7 @@ public function testSameMigrationFailedRetryRequiresTerminalAttestation(): void $this->assertSame('attempt-active', $existing->getAttribute('migrationAttemptId')); } - public function testPreExistingRecoveryRequiresAtomicMutationCapability(): void + public function testPreExistingRecoveryUsesDocumentVersionCompareAndSet(): void { foreach ([new MemoryAdapter(), new StandaloneMemoryAdapter()] as $adapter) { $database = new RecordingProjectDatabase($adapter, new Cache(new MemoryCache())); @@ -515,15 +517,15 @@ public function testPreExistingRecoveryRequiresAtomicMutationCapability(): void ); $existing = $this->getDatabaseDocument($database); - $this->assertNotSame([], $this->errorMessages($destination)); - $this->assertSame([], $database->databaseWrites); - $this->assertSame('failed', $existing->getAttribute('status')); + $this->assertSame([], $this->errorMessages($destination)); + $this->assertNotSame([], $database->databaseWrites); + $this->assertSame('ready', $existing->getAttribute('status')); $this->assertSame('migration-old', $existing->getAttribute('migrationId')); - $this->assertSame('attempt-old', $existing->getAttribute('migrationAttemptId')); + $this->assertSame('attempt-new', $existing->getAttribute('migrationAttemptId')); } } - public function testHealthyOverwriteRequiresAtomicMutationCapability(): void + public function testHealthyOverwriteUsesDocumentVersionCompareAndSet(): void { foreach ([new MemoryAdapter(), new StandaloneMemoryAdapter()] as $adapter) { $database = new RecordingProjectDatabase($adapter, new Cache(new MemoryCache())); @@ -553,11 +555,11 @@ public function testHealthyOverwriteRequiresAtomicMutationCapability(): void ); $existing = $this->getDatabaseDocument($database); - $this->assertNotSame([], $this->errorMessages($destination)); - $this->assertSame([], $database->databaseWrites); + $this->assertSame([], $this->errorMessages($destination)); + $this->assertNotSame([], $database->databaseWrites); $this->assertSame('ready', $existing->getAttribute('status')); - $this->assertSame('migration-old', $existing->getAttribute('migrationId')); - $this->assertSame('attempt-old', $existing->getAttribute('migrationAttemptId')); + $this->assertSame('migration-new', $existing->getAttribute('migrationId')); + $this->assertSame('attempt-new', $existing->getAttribute('migrationAttemptId')); } } @@ -626,6 +628,29 @@ public function testReadyWriteFailureRetainsProvisioningOwner(): void $this->assertSame('attempt-ready-failure', $created->getAttribute('migrationAttemptId')); } + public function testDatabaseFinalizersRunOnlyAfterSuccess(): void + { + $database = new RecordingProjectDatabase(new MemoryAdapter(), new Cache(new MemoryCache())); + $this->createProjectDatabase(withStatus: true, database: $database); + + $destination = $this->runDatabaseTransfer( + $database, + explicit: false, + migrationId: 'migration-finalizing', + migrationAttemptId: 'attempt-finalizing', + success: false, + ); + + $provisioning = $this->getDatabaseDocument($database); + $this->assertSame('provisioning', $provisioning->getAttribute('status')); + $this->assertSame('migration-finalizing', $provisioning->getAttribute('migrationId')); + $this->assertSame('attempt-finalizing', $provisioning->getAttribute('migrationAttemptId')); + + $database->getAuthorization()->skip($destination->success(...)); + + $this->assertSame('ready', $this->getDatabaseDocument($database)->getAttribute('status')); + } + public function testReadyFinalizationAttemptsEveryDatabaseAndReportsEveryFailure(): void { $database = new RecordingProjectDatabase(new MemoryAdapter(), new Cache(new MemoryCache())); @@ -937,6 +962,7 @@ private function runDatabaseTransfer( OnDuplicate $onDuplicate = OnDuplicate::Fail, string $resourceUpdatedAt = '', array $databaseIds = ['database'], + bool $success = true, ): CountingAppwriteDestination { $source = new class () extends MockSource { #[Override] @@ -973,7 +999,7 @@ public function supportsDatabaseStatus(): bool $transfer = new Transfer($source, $destination); $database->getAuthorization()->skip( - static function () use ($explicit, $transfer): void { + static function () use ($destination, $explicit, $success, $transfer): void { if ($explicit) { $transfer->runWithResourceSelector( [Resource::TYPE_DATABASE], @@ -987,14 +1013,17 @@ static function (): void { parentResourceType: '', ); - return; + } else { + $transfer->run( + [Resource::TYPE_DATABASE], + static function (): void { + }, + ); } - $transfer->run( - [Resource::TYPE_DATABASE], - static function (): void { - }, - ); + if ($success) { + $destination->success(); + } }, ); diff --git a/tests/Migration/Unit/MigrationCLITest.php b/tests/Migration/Unit/MigrationCLITest.php index 71f88aac..10052bca 100644 --- a/tests/Migration/Unit/MigrationCLITest.php +++ b/tests/Migration/Unit/MigrationCLITest.php @@ -200,8 +200,11 @@ public function supportsDatabaseStatus(): bool $transfer = new Transfer($source, $destination); $database->getAuthorization()->skip( - static fn () => $transfer->run([Resource::TYPE_DATABASE], static function (): void { - }), + static function () use ($destination, $transfer): void { + $transfer->run([Resource::TYPE_DATABASE], static function (): void { + }); + $destination->success(); + }, ); } From 97863f303a190ad15f604c2daa11e90ac889bfe4 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 1 Sep 2026 11:02:30 +1200 Subject: [PATCH 29/32] (fix): reject empty guarded database writes --- src/Migration/Destinations/Appwrite.php | 9 +- .../AppwriteDatabaseStatusTest.php | 83 +++++++++++++++++++ 2 files changed, 89 insertions(+), 3 deletions(-) diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index 294dab08..531712f2 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -396,14 +396,14 @@ private function setDatabaseStatus(string $databaseId, string $status): bool throw new DatabaseException('Database provisioning ownership requires a document version'); } - $this->dbForProject->updateDocument( + $updated = $this->dbForProject->updateDocument( self::META_DATABASES, $databaseId, new UtopiaDocument(['status' => $status]), expectedVersion: $version, ); - return true; + return ! $updated->isEmpty(); } catch (ConflictException) { return false; } @@ -868,12 +868,15 @@ protected function createDatabase(Database $resource): bool if ($version === null) { throw new DatabaseException('Database provisioning ownership requires a document version'); } - $this->dbForProject->updateDocument( + $updated = $this->dbForProject->updateDocument( self::META_DATABASES, $locked->getId(), new UtopiaDocument($document), expectedVersion: $version, ); + if ($updated->isEmpty()) { + throw new DatabaseException('Database '.$resource->getId().' provisioning owner changed before it could be claimed'); + } } return [ diff --git a/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php b/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php index 464234e2..6779aa6d 100644 --- a/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php +++ b/tests/Migration/Unit/Destinations/AppwriteDatabaseStatusTest.php @@ -142,6 +142,8 @@ class RecordingProjectDatabase extends UtopiaDatabase public bool $failReadyWrite = false; + public bool $disappearBeforeGuardedWrite = false; + /** @var list */ public array $failReadyWrites = []; @@ -179,6 +181,17 @@ public function updateDocument(string $collection, string $id, UtopiaDocument $d throw new DatabaseException('ready status unavailable'); } + if ( + $this->disappearBeforeGuardedWrite + && $collection === 'databases' + && $expectedVersion !== null + ) { + $this->disappearBeforeGuardedWrite = false; + parent::deleteDocument($collection, $id); + + return new UtopiaDocument(); + } + return parent::updateDocument($collection, $id, $document, $expectedVersion); } } @@ -628,6 +641,76 @@ public function testReadyWriteFailureRetainsProvisioningOwner(): void $this->assertSame('attempt-ready-failure', $created->getAttribute('migrationAttemptId')); } + public function testDatabaseDisappearanceBeforeReadyWriteLosesFinalizationOwnership(): void + { + $database = new RecordingProjectDatabase(new MemoryAdapter(), new Cache(new MemoryCache())); + $this->createProjectDatabase(withStatus: true, database: $database); + $database->disappearBeforeGuardedWrite = true; + + $destination = $this->runDatabaseTransfer( + $database, + explicit: false, + migrationId: 'migration-disappearing-ready', + migrationAttemptId: 'attempt-disappearing-ready', + ); + + $readyWrites = \array_values(\array_filter( + $database->databaseWrites, + static fn (array $write): bool => ($write['document']['status'] ?? null) === 'ready', + )); + + $this->assertSame(['Database provisioning owner changed before finalization'], $this->errorMessages($destination)); + $this->assertCount(1, $readyWrites); + $this->assertIsInt($readyWrites[0]['expectedVersion']); + $this->assertTrue($this->getDatabaseDocument($database)->isEmpty()); + } + + public function testDatabaseDisappearanceBeforeRecoveryClaimLosesOwnership(): void + { + $database = new RecordingProjectDatabase(new StandaloneMemoryAdapter(), new Cache(new MemoryCache())); + $this->createProjectDatabase(withStatus: true, database: $database); + $seeded = $this->seedDatabase( + $database, + status: 'failed', + migrationId: 'migration-old', + migrationAttemptId: 'attempt-old', + ); + $database->databaseWrites = []; + $database->disappearBeforeGuardedWrite = true; + + $destination = $this->runDatabaseTransfer( + $database, + explicit: false, + migrationId: 'migration-new', + migrationAttemptId: 'attempt-new', + getRecoverableOwner: static fn (UtopiaDocument $database): ProvisioningOwner => new ProvisioningOwner( + 'migration-old', + 'attempt-old', + ), + success: false, + ); + + $claimWrites = \array_values(\array_filter( + $database->databaseWrites, + static fn (array $write): bool => ($write['document']['status'] ?? null) === 'provisioning', + )); + + $this->assertSame( + ['Database database provisioning owner changed before it could be claimed'], + $this->errorMessages($destination), + ); + $this->assertCount(1, $claimWrites); + $this->assertSame( + [true], + \array_map( + static fn (array $write): bool => \is_int($write['expectedVersion'] ?? null), + $claimWrites, + ), + ); + $this->assertTrue($this->getDatabaseDocument($database)->isEmpty()); + $this->assertTrue($database->getCollection('database_'.$seeded->getSequence())->isEmpty()); + } + public function testDatabaseFinalizersRunOnlyAfterSuccess(): void { $database = new RecordingProjectDatabase(new MemoryAdapter(), new Cache(new MemoryCache())); From e56b9644f49e72edde622905cede91e5b6d0eb89 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 1 Sep 2026 11:04:34 +1200 Subject: [PATCH 30/32] (fix): finalize standalone migrations --- bin/MigrationCLI.php | 13 +- tests/Migration/Unit/MigrationCLITest.php | 156 +++++++++++++++++++--- 2 files changed, 150 insertions(+), 19 deletions(-) diff --git a/bin/MigrationCLI.php b/bin/MigrationCLI.php index 7f71cca2..ea41c6ab 100644 --- a/bin/MigrationCLI.php +++ b/bin/MigrationCLI.php @@ -501,8 +501,7 @@ function (mixed $value, Document $attribute) { public function start(): void { - $dotenv = Dotenv::createImmutable(__DIR__); - $dotenv->load(); + $this->loadEnvironment(); /** * Initialise All Source Adapters @@ -530,6 +529,16 @@ function () { $this->drawFrame(); } ); + + if ($this->source->getErrors() === [] && $this->destination->getErrors() === []) { + $this->destination->success(); + } + } + + protected function loadEnvironment(): void + { + $dotenv = Dotenv::createImmutable(__DIR__); + $dotenv->load(); } } diff --git a/tests/Migration/Unit/MigrationCLITest.php b/tests/Migration/Unit/MigrationCLITest.php index 10052bca..a23297d6 100644 --- a/tests/Migration/Unit/MigrationCLITest.php +++ b/tests/Migration/Unit/MigrationCLITest.php @@ -18,6 +18,7 @@ use Utopia\Migration\Destination; use Utopia\Migration\Resource; use Utopia\Migration\Resources\Database\Database as DatabaseResource; +use Utopia\Migration\Source; use Utopia\Migration\Transfer; use Utopia\Query\Schema\ColumnType; use Utopia\Tests\Unit\Adapters\MockSource; @@ -25,16 +26,43 @@ final class TestMigrationCLI extends \MigrationCLI { /** @param list $arguments */ - public function __construct(array $arguments, private readonly Database $database) - { + public function __construct( + array $arguments, + private readonly Database $database, + private readonly ?Source $injectedSource = null, + ) { parent::__construct($arguments); } #[Override] public function getDatabase(string $type): Database { + $this->database->getAuthorization()->disable(); + return $this->database; } + + #[Override] + public function getSource(): Source + { + return $this->injectedSource ?? parent::getSource(); + } + + #[Override] + public function drawFrame(): void + { + } + + #[Override] + protected function loadEnvironment(): void + { + } + + /** @return list<\Throwable> */ + public function getErrors(): array + { + return $this->destination->getErrors(); + } } final class TransactionalMemoryAdapter extends MemoryAdapter @@ -142,7 +170,76 @@ public function testAppwriteDestinationRequiresMigrationAttemptIdentifier(): voi $cli->getDestination(); } - private function createProjectDatabase(string $status = 'provisioning'): Database + public function testStartFinalizesSuccessfulAppwriteDatabase(): void + { + $database = $this->createProjectDatabase(status: null); + $source = $this->createSource(new DatabaseResource( + id: 'database', + name: 'Database', + type: 'tablesdb', + database: 'source-dsn', + databaseStatus: 'ready', + )); + $cli = new TestMigrationCLI( + [ + 'MigrationCLI.php', + '--migration-id=migration-current', + '--migration-attempt-id=attempt-current', + ], + $database, + $source, + ); + + $cli->start(); + + $created = $database->getDocument('databases', 'database'); + $this->assertSame([], $cli->getErrors()); + $this->assertSame('ready', $created->getAttribute('status')); + $this->assertSame('migration-current', $created->getAttribute('migrationId')); + $this->assertSame('attempt-current', $created->getAttribute('migrationAttemptId')); + $this->assertFalse($database->getCollection('database_'.$created->getSequence())->isEmpty()); + } + + public function testStartDoesNotFinalizeWhenDestinationHasErrors(): void + { + $database = $this->createProjectDatabase(status: null); + $valid = new DatabaseResource( + id: 'database', + name: 'Database', + type: 'tablesdb', + database: 'source-dsn', + databaseStatus: 'ready', + ); + $invalid = new DatabaseResource( + id: 'invalid id', + name: 'Invalid database', + type: 'tablesdb', + database: 'source-dsn', + databaseStatus: 'ready', + ); + $cli = new TestMigrationCLI( + [ + 'MigrationCLI.php', + '--migration-id=migration-current', + '--migration-attempt-id=attempt-current', + ], + $database, + $this->createSource($valid, $invalid), + ); + + $cli->start(); + + $created = $database->getDocument('databases', 'database'); + $this->assertNotSame([], $cli->getErrors()); + $this->assertSame(Resource::STATUS_SUCCESS, $valid->getStatus()); + $this->assertSame(Resource::STATUS_ERROR, $invalid->getStatus()); + $this->assertSame('provisioning', $created->getAttribute('status')); + $this->assertSame('migration-current', $created->getAttribute('migrationId')); + $this->assertSame('attempt-current', $created->getAttribute('migrationAttemptId')); + $this->assertFalse($database->getCollection('database_'.$created->getSequence())->isEmpty()); + } + + private function createProjectDatabase(?string $status = 'provisioning'): Database { $database = new Database(new TransactionalMemoryAdapter(), new Cache(new MemoryCache())); $database @@ -163,24 +260,49 @@ private function createProjectDatabase(string $status = 'provisioning'): Databas new Attribute(key: 'migrationAttemptId', type: ColumnType::String, size: Database::LENGTH_KEY), ], )); - $database->getAuthorization()->skip( - static fn (): Document => $database->createDocument('databases', new Document([ - '$id' => 'database', - 'name' => 'Database', - 'enabled' => true, - 'search' => 'database Database', - 'originalId' => null, - 'type' => 'tablesdb', - 'database' => '', - 'status' => $status, - 'migrationId' => 'migration-terminal', - 'migrationAttemptId' => 'attempt-terminal', - ])), - ); + if ($status !== null) { + $database->getAuthorization()->skip( + static fn (): Document => $database->createDocument('databases', new Document([ + '$id' => 'database', + 'name' => 'Database', + 'enabled' => true, + 'search' => 'database Database', + 'originalId' => null, + 'type' => 'tablesdb', + 'database' => '', + 'status' => $status, + 'migrationId' => 'migration-terminal', + 'migrationAttemptId' => 'attempt-terminal', + ])), + ); + } return $database; } + private function createSource(DatabaseResource ...$resources): MockSource + { + $source = new class () extends MockSource { + /** @return list */ + #[Override] + public static function getSupportedResources(): array + { + return [Resource::TYPE_DATABASE]; + } + + #[Override] + public function supportsDatabaseStatus(): bool + { + return true; + } + }; + foreach ($resources as $resource) { + $source->pushMockResource($resource); + } + + return $source; + } + private function runTransfer(Database $database, Destination $destination): void { $source = new class () extends MockSource { From 60b7a5231596dc6bc2af6760c19002f94fe943b1 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 1 Sep 2026 11:05:15 +1200 Subject: [PATCH 31/32] (chore): repin guarded database writes --- composer.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/composer.lock b/composer.lock index f7bceff9..743d6eb1 100644 --- a/composer.lock +++ b/composer.lock @@ -2612,12 +2612,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "93cf45fe3a28b1cc46f7850b1bbddfa67d518144" + "reference": "5c7226f01f58ff3ea45071441c66123b64d3fa2c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/93cf45fe3a28b1cc46f7850b1bbddfa67d518144", - "reference": "93cf45fe3a28b1cc46f7850b1bbddfa67d518144", + "url": "https://api.github.com/repos/utopia-php/database/zipball/5c7226f01f58ff3ea45071441c66123b64d3fa2c", + "reference": "5c7226f01f58ff3ea45071441c66123b64d3fa2c", "shasum": "" }, "require": { @@ -2703,7 +2703,7 @@ "source": "https://github.com/utopia-php/database/tree/feat-query-lib", "issues": "https://github.com/utopia-php/database/issues" }, - "time": "2026-08-31T22:07:53+00:00" + "time": "2026-08-31T22:50:42+00:00" }, { "name": "utopia-php/dsn", From 1724bf3a4f53446e15c2bac901d3673ce6fea937 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 1 Sep 2026 11:39:23 +1200 Subject: [PATCH 32/32] (fix): finalize partial standalone migrations --- bin/MigrationCLI.php | 4 +--- tests/Migration/Unit/MigrationCLITest.php | 11 +++++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/bin/MigrationCLI.php b/bin/MigrationCLI.php index ea41c6ab..77af9d25 100644 --- a/bin/MigrationCLI.php +++ b/bin/MigrationCLI.php @@ -530,9 +530,7 @@ function () { } ); - if ($this->source->getErrors() === [] && $this->destination->getErrors() === []) { - $this->destination->success(); - } + $this->destination->success(); } protected function loadEnvironment(): void diff --git a/tests/Migration/Unit/MigrationCLITest.php b/tests/Migration/Unit/MigrationCLITest.php index a23297d6..497098e0 100644 --- a/tests/Migration/Unit/MigrationCLITest.php +++ b/tests/Migration/Unit/MigrationCLITest.php @@ -58,7 +58,7 @@ protected function loadEnvironment(): void { } - /** @return list<\Throwable> */ + /** @return list<\Utopia\Migration\Exception> */ public function getErrors(): array { return $this->destination->getErrors(); @@ -200,7 +200,7 @@ public function testStartFinalizesSuccessfulAppwriteDatabase(): void $this->assertFalse($database->getCollection('database_'.$created->getSequence())->isEmpty()); } - public function testStartDoesNotFinalizeWhenDestinationHasErrors(): void + public function testStartFinalizesSuccessfulResourcesWhenDestinationHasErrors(): void { $database = $this->createProjectDatabase(status: null); $valid = new DatabaseResource( @@ -230,10 +230,13 @@ public function testStartDoesNotFinalizeWhenDestinationHasErrors(): void $cli->start(); $created = $database->getDocument('databases', 'database'); - $this->assertNotSame([], $cli->getErrors()); + $errors = $cli->getErrors(); + $this->assertCount(1, $errors); + $this->assertSame('invalid id', $errors[0]->getResourceId()); + $this->assertSame(Transfer::GROUP_DATABASES, $errors[0]->getResourceGroup()); $this->assertSame(Resource::STATUS_SUCCESS, $valid->getStatus()); $this->assertSame(Resource::STATUS_ERROR, $invalid->getStatus()); - $this->assertSame('provisioning', $created->getAttribute('status')); + $this->assertSame('ready', $created->getAttribute('status')); $this->assertSame('migration-current', $created->getAttribute('migrationId')); $this->assertSame('attempt-current', $created->getAttribute('migrationAttemptId')); $this->assertFalse($database->getCollection('database_'.$created->getSequence())->isEmpty());