From a9da54a1817d8a11b2ac2abe84bded943cf27c7e Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Thu, 29 Oct 2026 08:43:31 +0100 Subject: [PATCH 01/11] enh: Split off face embeddings into a deduplicated table fixes #1588 Assisted-by: ClaudeCode:claude-opus-5-5 Signed-off-by: Marcel Klehr --- .../Images/ClusteringFaceClassifier.php | 30 +-- lib/Dav/Faces/FacePhoto.php | 7 +- lib/Db/FaceDetection.php | 22 ++- lib/Db/FaceDetectionMapper.php | 157 ++++++++++++---- lib/Db/FaceDetectionWithTitle.php | 4 +- lib/Db/FaceEmbedding.php | 60 ++++++ lib/Db/FaceEmbeddingMapper.php | 68 +++++++ .../Version013002000Date20261028094721.php | 80 ++++++++ .../Version013002000Date20261028094821.php | 176 ++++++++++++++++++ lib/Service/FsActionService.php | 41 +--- lib/TaskProcessing/TaskResultListener.php | 28 ++- psalm-baseline.xml | 19 +- tests/ClusterTest.php | 2 +- 13 files changed, 592 insertions(+), 102 deletions(-) create mode 100644 lib/Db/FaceEmbedding.php create mode 100644 lib/Db/FaceEmbeddingMapper.php create mode 100644 lib/Migration/Version013002000Date20261028094721.php create mode 100644 lib/Migration/Version013002000Date20261028094821.php diff --git a/lib/Classifiers/Images/ClusteringFaceClassifier.php b/lib/Classifiers/Images/ClusteringFaceClassifier.php index 81eff74a6..9a49b1294 100644 --- a/lib/Classifiers/Images/ClusteringFaceClassifier.php +++ b/lib/Classifiers/Images/ClusteringFaceClassifier.php @@ -9,8 +9,9 @@ use OCA\Recognize\BackgroundJobs\ClusterFacesJob; use OCA\Recognize\Classifiers\Classifier; -use OCA\Recognize\Db\FaceDetection; use OCA\Recognize\Db\FaceDetectionMapper; +use OCA\Recognize\Db\FaceEmbedding; +use OCA\Recognize\Db\FaceEmbeddingMapper; use OCA\Recognize\Service\Logger; use OCA\Recognize\Service\QueueService; use OCP\AppFramework\Services\IAppConfig; @@ -41,6 +42,7 @@ public function __construct( Logger $logger, IAppConfig $config, private FaceDetectionMapper $faceDetections, + private FaceEmbeddingMapper $faceEmbeddings, QueueService $queue, IRootFolder $rootFolder, private IUserMountCache $userMountCache, private IJobList $jobList, ITempManager $tempManager, IPreview $previewProvider, private IManager $shareManager) { @@ -73,7 +75,7 @@ public function classify(array $queueFiles): void { $filteredQueueFiles = []; foreach ($queueFiles as $queueFile) { try { - $facesByFileCount = count($this->faceDetections->findByFileId($queueFile->getFileId())); + $facesByFileCount = $this->faceEmbeddings->countByFileId($queueFile->getFileId()); } catch (Exception $e) { $this->logger->debug('finding faces by file '.$queueFile->getFileId().' failed', ['exception' => $e]); $facesByFileCount = 1; @@ -114,19 +116,25 @@ public function classify(array $queueFiles): void { $userIds = []; } + $embedding = new FaceEmbedding(); + $embedding->setX($face['x']); + $embedding->setY($face['y']); + $embedding->setWidth($face['width']); + $embedding->setHeight($face['height']); + $embedding->setVector($face['vector']); + $embedding->setFileId($queueFile->getFileId()); + try { + $this->faceEmbeddings->insert($embedding); + } catch (\Throwable $e) { + $this->logger->error('Could not store face embedding in database', ['exception' => $e]); + continue; + } + // Insert face detection for all users with access foreach ($userIds as $userId) { $this->logger->debug('preparing face detection for user ' . $userId); - $faceDetection = new FaceDetection(); - $faceDetection->setX($face['x']); - $faceDetection->setY($face['y']); - $faceDetection->setWidth($face['width']); - $faceDetection->setHeight($face['height']); - $faceDetection->setVector($face['vector']); - $faceDetection->setFileId($queueFile->getFileId()); - $faceDetection->setUserId($userId); try { - $this->faceDetections->insert($faceDetection); + $this->faceDetections->insertForUser($embedding, $userId); } catch (\Throwable $e) { $this->logger->error('Could not store face detection in database', ['exception' => $e]); continue; diff --git a/lib/Dav/Faces/FacePhoto.php b/lib/Dav/Faces/FacePhoto.php index f30715343..d48370816 100644 --- a/lib/Dav/Faces/FacePhoto.php +++ b/lib/Dav/Faces/FacePhoto.php @@ -46,11 +46,12 @@ public function getName(): string { #[Override] public function delete(): void { - $detections = $this->detectionMapper->findByClusterId($this->faceDetection->getClusterId()); - if (count($detections) > 1) { + $detections = $this->detectionMapper->findByClusterIdWithVector($this->faceDetection->getClusterId()); + $vectors = array_filter($detections, fn (FaceDetection $detection) => $detection->getId() === $this->faceDetection->getId()); + if (count($detections) > 1 && count($vectors) > 0) { $centroid = FaceClusterAnalyzer::calculateCentroidOfDetections($detections); $distance = new Euclidean(); - $distanceValue = $distance->compute($centroid, $this->faceDetection->getVector()); + $distanceValue = $distance->compute($centroid, reset($vectors)->getVector()); // Set threshold to avoid recreating the same mistake $this->faceDetection->setThreshold($distanceValue); } diff --git a/lib/Db/FaceDetection.php b/lib/Db/FaceDetection.php index 3c7589a83..0f13f6b49 100644 --- a/lib/Db/FaceDetection.php +++ b/lib/Db/FaceDetection.php @@ -29,6 +29,11 @@ * @method int|null getClusterId() * @method float getThreshold() * @method setThreshold(float $threshold) + * @method int|null getEmbeddingId() + * @method setEmbeddingId(int $embeddingId) + * + * The face vector is stored once per file in {@see FaceEmbedding} and only populated + * on detections loaded with {@see FaceDetectionMapper} methods that join it in. */ class FaceDetection extends Entity { protected $fileId; @@ -37,17 +42,18 @@ class FaceDetection extends Entity { protected $y; protected $height; protected $width; + protected $embeddingId; protected $faceVector; protected $clusterId; protected $threshold; /** * @var string[] */ - public static $columns = ['id', 'user_id', 'file_id', 'x', 'y', 'height', 'width', 'face_vector', 'cluster_id', 'threshold']; + public static $columns = ['id', 'user_id', 'file_id', 'x', 'y', 'height', 'width', 'embedding_id', 'cluster_id', 'threshold']; /** * @var string[] */ - public static $fields = ['id', 'userId', 'fileId', 'x', 'y', 'height', 'width', 'faceVector', 'clusterId', 'threshold']; + public static $fields = ['id', 'userId', 'fileId', 'x', 'y', 'height', 'width', 'embeddingId', 'clusterId', 'threshold']; public function __construct() { // add types in constructor @@ -57,6 +63,7 @@ public function __construct() { $this->addType('y', 'float'); $this->addType('height', 'float'); $this->addType('width', 'float'); + $this->addType('embeddingId', 'integer'); $this->addType('faceVector', 'json'); $this->addType('clusterId', 'integer'); $this->addType('threshold', 'float'); @@ -65,7 +72,7 @@ public function __construct() { public function toArray(): array { $array = []; foreach (static::$fields as $field) { - if ($field === 'faceVector') { + if ($field === 'embeddingId') { continue; } $array[$field] = $this->{$field}; @@ -79,4 +86,13 @@ public function getVector(): array { public function setVector(array $vector): void { $this->setter('faceVector', [$vector]); } + + /** + * The vector is not a column of the detections table + */ + public function getUpdatedFields(): array { + $fields = parent::getUpdatedFields(); + unset($fields['faceVector']); + return $fields; + } } diff --git a/lib/Db/FaceDetectionMapper.php b/lib/Db/FaceDetectionMapper.php index 2651c67c8..234e998f6 100644 --- a/lib/Db/FaceDetectionMapper.php +++ b/lib/Db/FaceDetectionMapper.php @@ -24,12 +24,44 @@ final class FaceDetectionMapper extends QBMapper { private IConfig $config; private SettingsService $settingsService; + private FaceEmbeddingMapper $embeddings; - public function __construct(IDBConnection $db, IConfig $config, SettingsService $settingsService) { + public function __construct(IDBConnection $db, IConfig $config, SettingsService $settingsService, FaceEmbeddingMapper $embeddings) { parent::__construct($db, 'recognize_face_detections', FaceDetection::class); $this->db = $db; $this->config = $config; $this->settingsService = $settingsService; + $this->embeddings = $embeddings; + } + + /** + * Selects detections together with the face vector of their embedding + */ + private function selectWithVector(IQueryBuilder $qb): IQueryBuilder { + return $qb->select(array_map(fn ($col) => 'd.'.$col, FaceDetection::$columns)) + ->addSelect('e.face_vector') + ->from('recognize_face_detections', 'd') + ->innerJoin('d', 'recognize_face_embeddings', 'e', $qb->expr()->eq('d.embedding_id', 'e.id')); + } + + /** + * Stores the vector of a detection that has no embedding yet as a new embedding + * + * @throws \OCP\DB\Exception + */ + private function ensureEmbedding(FaceDetection $detection): void { + if ($detection->getEmbeddingId() !== null) { + return; + } + $embedding = new FaceEmbedding(); + $embedding->setFileId($detection->getFileId()); + $embedding->setX($detection->getX()); + $embedding->setY($detection->getY()); + $embedding->setHeight($detection->getHeight()); + $embedding->setWidth($detection->getWidth()); + $embedding->setVector($detection->getVector()); + $this->embeddings->insert($embedding); + $detection->setEmbeddingId($embedding->getId()); } private function getMinDetectionSize(): float { @@ -55,20 +87,20 @@ public function find(int $id): FaceDetection { } /** + * Inserts a detection for a user, unless that user already has a detection for the same embedding. + * If the detection has no embedding yet, its vector is stored as a new embedding. + * * @param FaceDetection $entity * @throws \OCP\DB\Exception */ public function insert(Entity $entity): FaceDetection { /** @var FaceDetection $entity */ + $this->ensureEmbedding($entity); $qb = $this->db->getQueryBuilder(); $qb->select(FaceDetection::$columns) ->from('recognize_face_detections') - ->where($qb->expr()->eq('file_id', $qb->createPositionalParameter($entity->getFileId(), IQueryBuilder::PARAM_INT))) - ->andWhere($qb->expr()->eq('user_id', $qb->createPositionalParameter($entity->getUserId(), IQueryBuilder::PARAM_STR))) - ->andWhere($qb->expr()->eq('x', $qb->createPositionalParameter($entity->getX(), IQueryBuilder::PARAM_INT))) - ->andWhere($qb->expr()->eq('y', $qb->createPositionalParameter($entity->getY(), IQueryBuilder::PARAM_INT))) - ->andWhere($qb->expr()->eq('height', $qb->createPositionalParameter($entity->getHeight(), IQueryBuilder::PARAM_INT))) - ->andWhere($qb->expr()->eq('width', $qb->createPositionalParameter($entity->getWidth(), IQueryBuilder::PARAM_INT))); + ->where($qb->expr()->eq('embedding_id', $qb->createPositionalParameter($entity->getEmbeddingId(), IQueryBuilder::PARAM_INT))) + ->andWhere($qb->expr()->eq('user_id', $qb->createPositionalParameter($entity->getUserId(), IQueryBuilder::PARAM_STR))); $duplicates = $this->findEntities($qb); if (empty($duplicates)) { @@ -83,9 +115,27 @@ public function insert(Entity $entity): FaceDetection { */ public function insertWithoutDeduplication(Entity $entity): FaceDetection { /** @var FaceDetection $entity */ + $this->ensureEmbedding($entity); return parent::insert($entity); } + /** + * Creates the per-user detection for an embedding + * + * @throws \OCP\DB\Exception + */ + public function insertForUser(FaceEmbedding $embedding, string $userId): FaceDetection { + $detection = new FaceDetection(); + $detection->setEmbeddingId($embedding->getId()); + $detection->setFileId($embedding->getFileId()); + $detection->setX($embedding->getX()); + $detection->setY($embedding->getY()); + $detection->setHeight($embedding->getHeight()); + $detection->setWidth($embedding->getWidth()); + $detection->setUserId($userId); + return parent::insert($detection); + } + /** * @throws \OCP\DB\Exception */ @@ -93,6 +143,20 @@ public function deleteAll(): void { $qb = $this->db->getQueryBuilder(); $qb->delete('recognize_face_detections') ->executeStatement(); + $this->embeddings->deleteAll(); + } + + /** + * Deletes all detections of a file for all users, including the embeddings + * + * @throws \OCP\DB\Exception + */ + public function deleteByFileId(int $fileId): void { + $qb = $this->db->getQueryBuilder(); + $qb->delete('recognize_face_detections') + ->where($qb->expr()->eq('file_id', $qb->createPositionalParameter($fileId, IQueryBuilder::PARAM_INT))) + ->executeStatement(); + $this->embeddings->deleteByFileId($fileId); } /** @@ -108,6 +172,18 @@ public function findByClusterId(int $clusterId) : array { return $this->findEntities($qb); } + /** + * @param int $clusterId + * @return list<\OCA\Recognize\Db\FaceDetection> + * @throws \OCP\DB\Exception + */ + public function findByClusterIdWithVector(int $clusterId) : array { + $qb = $this->db->getQueryBuilder(); + $this->selectWithVector($qb) + ->where($qb->expr()->eq('d.cluster_id', $qb->createPositionalParameter($clusterId))); + return $this->findEntities($qb); + } + /** * @throws \OCP\DB\Exception * @return list<\OCA\Recognize\Db\FaceDetection> @@ -146,21 +222,33 @@ public function findByFileIdAndUser(int $fileId, string $userId): array { } /** + * Gives the passed users a detection for every face found in the file that they don't have yet + * + * @param int $fileId + * @param list $userIds + * @return list The users that received new detections * @throws \OCP\DB\Exception */ - public function copyDetectionsForFileFromUserToUser(int $fileId, string $fromUser, string $toUser) : void { - $detections = $this->findByFileIdAndUser($fileId, $fromUser); - foreach ($detections as $detection) { - $detectionCopy = new FaceDetection(); - $detectionCopy->setX($detection->getX()); - $detectionCopy->setY($detection->getY()); - $detectionCopy->setVector($detection->getVector()); - $detectionCopy->setFileId($detection->getFileId()); - $detectionCopy->setHeight($detection->getHeight()); - $detectionCopy->setWidth($detection->getWidth()); - $detectionCopy->setUserId($toUser); - $this->insert($detectionCopy); + public function addUsersToFile(int $fileId, array $userIds) : array { + $embeddings = $this->embeddings->findByFileId($fileId); + if (count($embeddings) === 0 || count($userIds) === 0) { + return []; } + $hasDetection = []; + foreach ($this->findByFileId($fileId) as $detection) { + $hasDetection[$detection->getUserId()][(int)$detection->getEmbeddingId()] = true; + } + $addedUsers = []; + foreach ($userIds as $userId) { + foreach ($embeddings as $embedding) { + if ($hasDetection[$userId][$embedding->getId()] ?? false) { + continue; + } + $this->insertForUser($embedding, $userId); + $addedUsers[$userId] = true; + } + } + return array_map('strval', array_keys($addedUsers)); } /** @@ -249,12 +337,11 @@ public function findUserIds() :array { */ public function findUnclusteredByUserId(string $userId, int $limit = 0, float $minHeight = 0, float $minWidth = 0) : array { $qb = $this->db->getQueryBuilder(); - $qb->select(FaceDetection::$columns) - ->from('recognize_face_detections') - ->where($qb->expr()->eq('user_id', $qb->createPositionalParameter($userId))) - ->andWhere($qb->expr()->gte('height', $qb->createPositionalParameter($minHeight))) - ->andWhere($qb->expr()->gte('width', $qb->createPositionalParameter($minWidth))) - ->andWhere($qb->expr()->isNull('cluster_id')); + $this->selectWithVector($qb) + ->where($qb->expr()->eq('d.user_id', $qb->createPositionalParameter($userId))) + ->andWhere($qb->expr()->gte('d.height', $qb->createPositionalParameter($minHeight))) + ->andWhere($qb->expr()->gte('d.width', $qb->createPositionalParameter($minWidth))) + ->andWhere($qb->expr()->isNull('d.cluster_id')); if ($limit > 0) { $qb->setMaxResults($limit); } @@ -285,11 +372,10 @@ public function findRejectedByUserId(string $userId) : array { */ public function findClusterSample(int $clusterId, int $n, int $minHeight = 0, int $minWidth = 0): array { $qb = $this->db->getQueryBuilder(); - $qb->select(FaceDetection::$columns) - ->from('recognize_face_detections', 'd') - ->where($qb->expr()->eq('cluster_id', $qb->createPositionalParameter($clusterId))) - ->andWhere($qb->expr()->gte('height', $qb->createPositionalParameter($minHeight))) - ->andWhere($qb->expr()->gte('width', $qb->createPositionalParameter($minWidth))) + $this->selectWithVector($qb) + ->where($qb->expr()->eq('d.cluster_id', $qb->createPositionalParameter($clusterId))) + ->andWhere($qb->expr()->gte('d.height', $qb->createPositionalParameter($minHeight))) + ->andWhere($qb->expr()->gte('d.width', $qb->createPositionalParameter($minWidth))) ->orderBy( $qb->createFunction( $this->config->getSystemValue('dbtype', 'sqlite') === 'mysql' @@ -311,12 +397,11 @@ public function findClusterSample(int $clusterId, int $n, int $minHeight = 0, in */ public function sampleRejectedDetectionsByUserId(string $userId, int $n, float $minHeight = 0, float $minWidth = 0): array { $qb = $this->db->getQueryBuilder(); - $qb->select(FaceDetection::$columns) - ->from('recognize_face_detections', 'd') - ->where($qb->expr()->eq('cluster_id', $qb->createPositionalParameter(-1))) - ->andWhere($qb->expr()->eq('user_id', $qb->createPositionalParameter($userId))) - ->andWhere($qb->expr()->gte('height', $qb->createPositionalParameter($minHeight))) - ->andWhere($qb->expr()->gte('width', $qb->createPositionalParameter($minWidth))) + $this->selectWithVector($qb) + ->where($qb->expr()->eq('d.cluster_id', $qb->createPositionalParameter(-1))) + ->andWhere($qb->expr()->eq('d.user_id', $qb->createPositionalParameter($userId))) + ->andWhere($qb->expr()->gte('d.height', $qb->createPositionalParameter($minHeight))) + ->andWhere($qb->expr()->gte('d.width', $qb->createPositionalParameter($minWidth))) ->orderBy( $qb->createFunction( $this->config->getSystemValue('dbtype', 'sqlite') === 'mysql' diff --git a/lib/Db/FaceDetectionWithTitle.php b/lib/Db/FaceDetectionWithTitle.php index 3a0b8a864..f0bb4e2dc 100644 --- a/lib/Db/FaceDetectionWithTitle.php +++ b/lib/Db/FaceDetectionWithTitle.php @@ -15,8 +15,8 @@ final class FaceDetectionWithTitle extends FaceDetection { protected $title; - public static $columns = ['id', 'user_id', 'file_id', 'x', 'y', 'height', 'width', 'face_vector', 'cluster_id', 'title']; - public static $fields = ['id', 'userId', 'fileId', 'x', 'y', 'height', 'width', 'faceVector', 'clusterId', 'title']; + public static $columns = ['id', 'user_id', 'file_id', 'x', 'y', 'height', 'width', 'embedding_id', 'cluster_id', 'title']; + public static $fields = ['id', 'userId', 'fileId', 'x', 'y', 'height', 'width', 'embeddingId', 'clusterId', 'title']; public function __construct() { parent::__construct(); diff --git a/lib/Db/FaceEmbedding.php b/lib/Db/FaceEmbedding.php new file mode 100644 index 000000000..2caef0189 --- /dev/null +++ b/lib/Db/FaceEmbedding.php @@ -0,0 +1,60 @@ +addType('fileId', 'integer'); + $this->addType('x', 'float'); + $this->addType('y', 'float'); + $this->addType('height', 'float'); + $this->addType('width', 'float'); + $this->addType('faceVector', 'json'); + } + + public function getVector(): array { + return $this->getter('faceVector'); + } + public function setVector(array $vector): void { + $this->setter('faceVector', [$vector]); + } +} diff --git a/lib/Db/FaceEmbeddingMapper.php b/lib/Db/FaceEmbeddingMapper.php new file mode 100644 index 000000000..cd861b1fb --- /dev/null +++ b/lib/Db/FaceEmbeddingMapper.php @@ -0,0 +1,68 @@ + + */ +final class FaceEmbeddingMapper extends QBMapper { + public function __construct(IDBConnection $db) { + parent::__construct($db, 'recognize_face_embeddings', FaceEmbedding::class); + $this->db = $db; + } + + /** + * @throws \OCP\DB\Exception + * @return list<\OCA\Recognize\Db\FaceEmbedding> + */ + public function findByFileId(int $fileId): array { + $qb = $this->db->getQueryBuilder(); + $qb->select(FaceEmbedding::$columns) + ->from('recognize_face_embeddings') + ->where($qb->expr()->eq('file_id', $qb->createPositionalParameter($fileId, IQueryBuilder::PARAM_INT))); + return $this->findEntities($qb); + } + + /** + * @throws \OCP\DB\Exception + */ + public function countByFileId(int $fileId): int { + $qb = $this->db->getQueryBuilder(); + $qb->select($qb->func()->count('id')) + ->from('recognize_face_embeddings') + ->where($qb->expr()->eq('file_id', $qb->createPositionalParameter($fileId, IQueryBuilder::PARAM_INT))); + $result = $qb->executeQuery(); + /** @var int|string $count */ + $count = $result->fetch(\PDO::FETCH_COLUMN); + $result->closeCursor(); + return (int)$count; + } + + /** + * @throws \OCP\DB\Exception + */ + public function deleteByFileId(int $fileId): void { + $qb = $this->db->getQueryBuilder(); + $qb->delete('recognize_face_embeddings') + ->where($qb->expr()->eq('file_id', $qb->createPositionalParameter($fileId, IQueryBuilder::PARAM_INT))) + ->executeStatement(); + } + + /** + * @throws \OCP\DB\Exception + */ + public function deleteAll(): void { + $qb = $this->db->getQueryBuilder(); + $qb->delete('recognize_face_embeddings') + ->executeStatement(); + } +} diff --git a/lib/Migration/Version013002000Date20261028094721.php b/lib/Migration/Version013002000Date20261028094721.php new file mode 100644 index 000000000..3bd509437 --- /dev/null +++ b/lib/Migration/Version013002000Date20261028094721.php @@ -0,0 +1,80 @@ +hasTable('recognize_face_embeddings')) { + $table = $schema->createTable('recognize_face_embeddings'); + $table->addColumn('id', Types::BIGINT, [ + 'autoincrement' => true, + 'notnull' => true, + 'length' => 64, + ]); + $table->addColumn('file_id', Types::BIGINT, [ + 'notnull' => true, + 'length' => 64, + ]); + $table->addColumn('x', Types::FLOAT, [ + 'notnull' => false, + ]); + $table->addColumn('y', Types::FLOAT, [ + 'notnull' => false, + ]); + $table->addColumn('height', Types::FLOAT, [ + 'notnull' => false, + ]); + $table->addColumn('width', Types::FLOAT, [ + 'notnull' => false, + ]); + $table->addColumn('face_vector', Types::TEXT, [ + 'notnull' => true, + ]); + $table->setPrimaryKey(['id'], 'recognize_faceemb_id'); + $table->addIndex(['file_id'], 'recognize_faceemb_file'); + $changed = true; + } + + if ($schema->hasTable('recognize_face_detections')) { + $table = $schema->getTable('recognize_face_detections'); + if (!$table->hasColumn('embedding_id')) { + $table->addColumn('embedding_id', Types::BIGINT, [ + 'notnull' => false, + 'length' => 64, + ]); + $table->addIndex(['embedding_id'], 'recognize_facedet_embedding'); + $changed = true; + } + } + return $changed ? $schema : null; + } +} diff --git a/lib/Migration/Version013002000Date20261028094821.php b/lib/Migration/Version013002000Date20261028094821.php new file mode 100644 index 000000000..ab25ce868 --- /dev/null +++ b/lib/Migration/Version013002000Date20261028094821.php @@ -0,0 +1,176 @@ +hasTable('recognize_face_detections') || !$schema->hasTable('recognize_face_embeddings')) { + return; + } + $table = $schema->getTable('recognize_face_detections'); + if (!$table->hasColumn('face_vector') || !$table->hasColumn('embedding_id')) { + return; + } + + $qb = $this->db->getQueryBuilder(); + $qb->select($qb->func()->count('id')) + ->from('recognize_face_detections') + ->where($qb->expr()->isNull('embedding_id')) + ->andWhere($qb->expr()->isNotNull('file_id')); + $result = $qb->executeQuery(); + $total = (int)$result->fetchOne(); + $result->closeCursor(); + if ($total === 0) { + return; + } + + $output->info('Deduplicating ' . $total . ' face detections'); + $output->startProgress($total); + + $lastFileId = -1; + while (true) { + $qb = $this->db->getQueryBuilder(); + $qb->selectDistinct('file_id') + ->from('recognize_face_detections') + ->where($qb->expr()->isNull('embedding_id')) + ->andWhere($qb->expr()->gt('file_id', $qb->createNamedParameter($lastFileId, IQueryBuilder::PARAM_INT))) + ->orderBy('file_id') + ->setMaxResults(self::FILE_BATCH_SIZE); + $result = $qb->executeQuery(); + $fileIds = array_map('intval', $result->fetchAll(\PDO::FETCH_COLUMN)); + $result->closeCursor(); + if (count($fileIds) === 0) { + break; + } + $lastFileId = max($fileIds); + + $this->db->beginTransaction(); + try { + $output->advance($this->migrateFiles($fileIds)); + $this->db->commit(); + } catch (\Throwable $e) { + $this->db->rollBack(); + throw $e; + } + } + $output->finishProgress(); + } + + /** + * @param list $fileIds + * @return int The number of migrated detections + * @throws \OCP\DB\Exception + */ + private function migrateFiles(array $fileIds): int { + // Group the detections of these files by face; all copies of a face share file and bounding box + $qb = $this->db->getQueryBuilder(); + $qb->select('id', 'file_id', 'x', 'y', 'height', 'width') + ->from('recognize_face_detections') + ->where($qb->expr()->isNull('embedding_id')) + ->andWhere($qb->expr()->in('file_id', $qb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY))) + ->orderBy('id'); + $result = $qb->executeQuery(); + /** @var array> $detectionIdsByFace */ + $detectionIdsByFace = []; + $count = 0; + while ($row = $result->fetch()) { + $key = $this->faceKey($row); + $detectionIdsByFace[$key][] = (int)$row['id']; + $count++; + } + $result->closeCursor(); + + // Reuse embeddings created by a previous, interrupted run + $qb = $this->db->getQueryBuilder(); + $qb->select('id', 'file_id', 'x', 'y', 'height', 'width') + ->from('recognize_face_embeddings') + ->where($qb->expr()->in('file_id', $qb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY))); + $result = $qb->executeQuery(); + /** @var array $embeddingIdByFace */ + $embeddingIdByFace = []; + while ($row = $result->fetch()) { + $embeddingIdByFace[$this->faceKey($row)] = (int)$row['id']; + } + $result->closeCursor(); + + // Copy one detection per face within the database, so neither the vectors nor + // the float coordinates have to make a lossy round trip through PHP + $missingFaces = array_diff_key($detectionIdsByFace, $embeddingIdByFace); + $copyQuery = 'INSERT INTO `*PREFIX*recognize_face_embeddings` (`file_id`, `x`, `y`, `height`, `width`, `face_vector`) ' + . 'SELECT `file_id`, `x`, `y`, `height`, `width`, `face_vector` FROM `*PREFIX*recognize_face_detections` WHERE `id` = ?'; + foreach ($missingFaces as $key => $detectionIds) { + $this->db->executeStatement($copyQuery, [$detectionIds[0]], [IQueryBuilder::PARAM_INT]); + /** @psalm-suppress DeprecatedMethod The query builder cannot express INSERT ... SELECT */ + $embeddingIdByFace[$key] = $this->db->lastInsertId('*PREFIX*recognize_face_embeddings'); + } + + foreach ($detectionIdsByFace as $key => $detectionIds) { + foreach (array_chunk($detectionIds, self::QUERY_CHUNK_SIZE) as $chunk) { + $qb = $this->db->getQueryBuilder(); + $qb->update('recognize_face_detections') + ->set('embedding_id', $qb->createNamedParameter($embeddingIdByFace[$key], IQueryBuilder::PARAM_INT)) + ->where($qb->expr()->in('id', $qb->createNamedParameter($chunk, IQueryBuilder::PARAM_INT_ARRAY))); + $qb->executeStatement(); + } + } + + return $count; + } + + /** + * @param array $row + */ + private function faceKey(array $row): string { + return implode('|', [(int)$row['file_id'], (float)$row['x'], (float)$row['y'], (float)$row['height'], (float)$row['width']]); + } + + /** + * @param IOutput $output + * @param Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper` + * @param array $options + * + * @return ?ISchemaWrapper + * @throws SchemaException + */ + public function changeSchema(IOutput $output, Closure $schemaClosure, array $options) { + /** @var ISchemaWrapper $schema */ + $schema = $schemaClosure(); + + if ($schema->hasTable('recognize_face_detections')) { + $table = $schema->getTable('recognize_face_detections'); + if ($table->hasColumn('face_vector') && $table->hasColumn('embedding_id')) { + $table->dropColumn('face_vector'); + return $schema; + } + } + return null; + } +} diff --git a/lib/Service/FsActionService.php b/lib/Service/FsActionService.php index 23bd2b643..2fa470f58 100644 --- a/lib/Service/FsActionService.php +++ b/lib/Service/FsActionService.php @@ -125,7 +125,7 @@ public function processActions(array $actions): void { break; } try { - $this->onMove($action->getOwner(), $action->getAddedUsers(), $action->getTargetUsers(), $node); + $this->onMove($action->getAddedUsers(), $action->getTargetUsers(), $node); } catch (Exception|InvalidPathException|NotFoundException $e) { $this->logger->warning('Failed to process move action: ' . $e->getMessage() . ' Continuing.', ['exception' => $e]); } @@ -168,24 +168,7 @@ private function onAccessUpdate(int $storageId, int $rootId): void { $files = $this->storageService->getFilesInMount($storageId, $rootId, [ClusteringFaceClassifier::MODEL_NAME], 0, 0); $userIdsToScheduleClustering = []; foreach ($files as $fileInfo) { - $node = $this->rootFolder->getFirstNodeById($fileInfo['fileid']) ?: null; - $ownerId = $node?->getOwner()?->getUID(); - if ($ownerId === null) { - continue; - } - $detectionsForFile = $this->faceDetectionMapper->findByFileId($fileInfo['fileid']); - $userHasDetectionForFile = []; - foreach ($detectionsForFile as $detection) { - $userHasDetectionForFile[$detection->getUserId()] = true; - } - foreach ($userIds as $userId) { - if ($userId === $ownerId) { - continue; - } - if ($userHasDetectionForFile[$userId] ?? false) { - continue; - } - $this->faceDetectionMapper->copyDetectionsForFileFromUserToUser($fileInfo['fileid'], $ownerId, $userId); + foreach ($this->faceDetectionMapper->addUsersToFile($fileInfo['fileid'], $userIds) as $userId) { $userIdsToScheduleClustering[$userId] = true; } $this->faceDetectionMapper->removeDetectionsForFileFromUsersNotInList($fileInfo['fileid'], $userIds); @@ -304,14 +287,7 @@ public function isFileIgnored(Node $node) : bool { public function onDeletion(int $nodeId, ?array $mimeTypes = null): void { // Try Deleting possibly existing face detections try { - /** - * @var \OCA\Recognize\Db\FaceDetection[] $faceDetections - */ - $faceDetections = $this->faceDetectionMapper->findByFileId($nodeId); - foreach ($faceDetections as $detection) { - $this->logger->debug('Delete face detection ' . $detection->getId()); - $this->faceDetectionMapper->delete($detection); - } + $this->faceDetectionMapper->deleteByFileId($nodeId); } catch (Exception $e) { $this->logger->warning($e->getMessage(), ['exception' => $e]); } @@ -325,32 +301,27 @@ public function onDeletion(int $nodeId, ?array $mimeTypes = null): void { } /** - * @param string $ownerId * @param list $usersToAdd * @param list $targetUserIds * @param Node $node * @return void * @throws Exception|InvalidPathException|NotFoundException */ - private function onMove(string $ownerId, array $usersToAdd, array $targetUserIds, Node $node): void { + private function onMove(array $usersToAdd, array $targetUserIds, Node $node): void { if ($node instanceof Folder) { try { foreach ($node->getDirectoryListing() as $n) { if (!in_array($n->getMimetype(), Constants::IMAGE_FORMATS)) { continue; } - $this->onMove($ownerId, $usersToAdd, $targetUserIds, $n); + $this->onMove($usersToAdd, $targetUserIds, $n); } } catch (NotFoundException|Exception|InvalidPathException $e) { $this->logger->warning('Error in recognize file listener', ['exception' => $e]); } return; } - foreach ($usersToAdd as $userId) { - if (count($this->faceDetectionMapper->findByFileIdAndUser($node->getId(), $userId)) > 0) { - continue; - } - $this->faceDetectionMapper->copyDetectionsForFileFromUserToUser($node->getId(), $ownerId, $userId); + foreach ($this->faceDetectionMapper->addUsersToFile($node->getId(), $usersToAdd) as $userId) { $this->jobList->add(ClusterFacesJob::class, ['userId' => $userId]); } $this->faceDetectionMapper->removeDetectionsForFileFromUsersNotInList($node->getId(), $targetUserIds); diff --git a/lib/TaskProcessing/TaskResultListener.php b/lib/TaskProcessing/TaskResultListener.php index bfcad9f6f..aa703595f 100644 --- a/lib/TaskProcessing/TaskResultListener.php +++ b/lib/TaskProcessing/TaskResultListener.php @@ -19,6 +19,8 @@ use OCA\Recognize\Classifiers\Video\MovinetClassifier; use OCA\Recognize\Db\FaceDetection; use OCA\Recognize\Db\FaceDetectionMapper; +use OCA\Recognize\Db\FaceEmbedding; +use OCA\Recognize\Db\FaceEmbeddingMapper; use OCA\Recognize\Db\QueueFile; use OCA\Recognize\Service\QueueService; use OCA\Recognize\Service\TagManager; @@ -39,7 +41,7 @@ * Stores results of TaskProcessing tasks scheduled by this app: * * - image / video / audio classification → assigns category tags via {@see TagManager} - * - face recognition → inserts {@see FaceDetection} rows and schedules a {@see ClusterFacesJob} + * - face recognition → inserts a {@see FaceEmbedding} per face, a {@see FaceDetection} per face and user with access, and schedules a {@see ClusterFacesJob} * * @template-implements IEventListener */ @@ -48,6 +50,7 @@ public function __construct( private LoggerInterface $logger, private TagManager $tagManager, private FaceDetectionMapper $faceDetections, + private FaceEmbeddingMapper $faceEmbeddings, private IUserMountCache $userMountCache, private IAppConfig $config, private IJobList $jobList, @@ -225,17 +228,22 @@ private function applyFaceResults(array $fileIds, array $results): void { $this->logger->warning('Face entry without embedding vector for file ' . $fileId); continue; } + $embedding = new FaceEmbedding(); + $embedding->setFileId($fileId); + $embedding->setX((float)($face['x'] ?? 0)); + $embedding->setY((float)($face['y'] ?? 0)); + $embedding->setWidth((float)($face['width'] ?? 0)); + $embedding->setHeight((float)($face['height'] ?? 0)); + $embedding->setVector($vector); + try { + $this->faceEmbeddings->insert($embedding); + } catch (\Throwable $e) { + $this->logger->error('Could not store face embedding for file ' . $fileId, ['exception' => $e]); + continue; + } foreach ($userIds as $userId) { - $detection = new FaceDetection(); - $detection->setFileId($fileId); - $detection->setUserId($userId); - $detection->setX((float)($face['x'] ?? 0)); - $detection->setY((float)($face['y'] ?? 0)); - $detection->setWidth((float)($face['width'] ?? 0)); - $detection->setHeight((float)($face['height'] ?? 0)); - $detection->setVector($vector); try { - $this->faceDetections->insert($detection); + $this->faceDetections->insertForUser($embedding, $userId); } catch (\Throwable $e) { $this->logger->error('Could not store face detection for file ' . $fileId, ['exception' => $e]); continue; diff --git a/psalm-baseline.xml b/psalm-baseline.xml index 40afa6065..bdf706cff 100644 --- a/psalm-baseline.xml +++ b/psalm-baseline.xml @@ -614,7 +614,7 @@ - faceDetection->getVector()]]> + getVector()]]> getFile()->getSize()]]> @@ -688,6 +688,7 @@ + @@ -715,6 +716,22 @@ + + + + + + + + + + + getter('faceVector')]]> + + + + + diff --git a/tests/ClusterTest.php b/tests/ClusterTest.php index 8d31bcbad..7bbcfe2a8 100644 --- a/tests/ClusterTest.php +++ b/tests/ClusterTest.php @@ -116,7 +116,7 @@ public function testClusterThresholds() { $detections = $this->faceDetectionMapper->findByClusterId($clusters[0]->getId()); self::assertCount(self::INITIAL_DETECTIONS_PER_CLUSTER, $detections); - $detections = $this->faceDetectionMapper->findByClusterId($clusters[1]->getId()); + $detections = $this->faceDetectionMapper->findByClusterIdWithVector($clusters[1]->getId()); self::assertCount(self::INITIAL_DETECTIONS_PER_CLUSTER, $detections); $detectionToRemove = $detections[0]; From 13d8cfec4748fae48cde17ddb75d7a3723a3013b Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Thu, 29 Oct 2026 09:08:02 +0100 Subject: [PATCH 02/11] fix: Make migrations run on oracle Assisted-by: ClaudeCode:claude-opus-5-5 Signed-off-by: Marcel Klehr --- lib/Db/FaceDetectionMapper.php | 2 +- lib/Db/FaceEmbeddingMapper.php | 10 ++-- .../Version013002000Date20261028094721.php | 6 +-- .../Version013002000Date20261028094821.php | 47 ++++++++++++------- 4 files changed, 39 insertions(+), 26 deletions(-) diff --git a/lib/Db/FaceDetectionMapper.php b/lib/Db/FaceDetectionMapper.php index 234e998f6..53420de00 100644 --- a/lib/Db/FaceDetectionMapper.php +++ b/lib/Db/FaceDetectionMapper.php @@ -41,7 +41,7 @@ private function selectWithVector(IQueryBuilder $qb): IQueryBuilder { return $qb->select(array_map(fn ($col) => 'd.'.$col, FaceDetection::$columns)) ->addSelect('e.face_vector') ->from('recognize_face_detections', 'd') - ->innerJoin('d', 'recognize_face_embeddings', 'e', $qb->expr()->eq('d.embedding_id', 'e.id')); + ->innerJoin('d', 'recognize_face_vectors', 'e', $qb->expr()->eq('d.embedding_id', 'e.id')); } /** diff --git a/lib/Db/FaceEmbeddingMapper.php b/lib/Db/FaceEmbeddingMapper.php index cd861b1fb..dc6eba7cd 100644 --- a/lib/Db/FaceEmbeddingMapper.php +++ b/lib/Db/FaceEmbeddingMapper.php @@ -16,7 +16,7 @@ */ final class FaceEmbeddingMapper extends QBMapper { public function __construct(IDBConnection $db) { - parent::__construct($db, 'recognize_face_embeddings', FaceEmbedding::class); + parent::__construct($db, 'recognize_face_vectors', FaceEmbedding::class); $this->db = $db; } @@ -27,7 +27,7 @@ public function __construct(IDBConnection $db) { public function findByFileId(int $fileId): array { $qb = $this->db->getQueryBuilder(); $qb->select(FaceEmbedding::$columns) - ->from('recognize_face_embeddings') + ->from('recognize_face_vectors') ->where($qb->expr()->eq('file_id', $qb->createPositionalParameter($fileId, IQueryBuilder::PARAM_INT))); return $this->findEntities($qb); } @@ -38,7 +38,7 @@ public function findByFileId(int $fileId): array { public function countByFileId(int $fileId): int { $qb = $this->db->getQueryBuilder(); $qb->select($qb->func()->count('id')) - ->from('recognize_face_embeddings') + ->from('recognize_face_vectors') ->where($qb->expr()->eq('file_id', $qb->createPositionalParameter($fileId, IQueryBuilder::PARAM_INT))); $result = $qb->executeQuery(); /** @var int|string $count */ @@ -52,7 +52,7 @@ public function countByFileId(int $fileId): int { */ public function deleteByFileId(int $fileId): void { $qb = $this->db->getQueryBuilder(); - $qb->delete('recognize_face_embeddings') + $qb->delete('recognize_face_vectors') ->where($qb->expr()->eq('file_id', $qb->createPositionalParameter($fileId, IQueryBuilder::PARAM_INT))) ->executeStatement(); } @@ -62,7 +62,7 @@ public function deleteByFileId(int $fileId): void { */ public function deleteAll(): void { $qb = $this->db->getQueryBuilder(); - $qb->delete('recognize_face_embeddings') + $qb->delete('recognize_face_vectors') ->executeStatement(); } } diff --git a/lib/Migration/Version013002000Date20261028094721.php b/lib/Migration/Version013002000Date20261028094721.php index 3bd509437..04c594469 100644 --- a/lib/Migration/Version013002000Date20261028094721.php +++ b/lib/Migration/Version013002000Date20261028094721.php @@ -15,7 +15,7 @@ use OCP\Migration\SimpleMigrationStep; /** - * Adds the recognize_face_embeddings table, which stores each face vector once per file + * Adds the recognize_face_vectors table, which stores each face vector once per file * instead of once per user with access to the file */ final class Version013002000Date20261028094721 extends SimpleMigrationStep { @@ -33,8 +33,8 @@ public function changeSchema(IOutput $output, Closure $schemaClosure, array $opt $schema = $schemaClosure(); $changed = false; - if (!$schema->hasTable('recognize_face_embeddings')) { - $table = $schema->createTable('recognize_face_embeddings'); + if (!$schema->hasTable('recognize_face_vectors')) { + $table = $schema->createTable('recognize_face_vectors'); $table->addColumn('id', Types::BIGINT, [ 'autoincrement' => true, 'notnull' => true, diff --git a/lib/Migration/Version013002000Date20261028094821.php b/lib/Migration/Version013002000Date20261028094821.php index ab25ce868..16cb8f9cf 100644 --- a/lib/Migration/Version013002000Date20261028094821.php +++ b/lib/Migration/Version013002000Date20261028094821.php @@ -17,7 +17,7 @@ /** * Moves the face vectors from recognize_face_detections, where they were duplicated for every user - * with access to a file, into recognize_face_embeddings, then drops the vector column + * with access to a file, into recognize_face_vectors, then drops the vector column */ final class Version013002000Date20261028094821 extends SimpleMigrationStep { private const FILE_BATCH_SIZE = 500; @@ -32,7 +32,7 @@ public function preSchemaChange(IOutput $output, Closure $schemaClosure, array $ /** @var ISchemaWrapper $schema */ $schema = $schemaClosure(); - if (!$schema->hasTable('recognize_face_detections') || !$schema->hasTable('recognize_face_embeddings')) { + if (!$schema->hasTable('recognize_face_detections') || !$schema->hasTable('recognize_face_vectors')) { return; } $table = $schema->getTable('recognize_face_detections'); @@ -109,27 +109,20 @@ private function migrateFiles(array $fileIds): int { $result->closeCursor(); // Reuse embeddings created by a previous, interrupted run - $qb = $this->db->getQueryBuilder(); - $qb->select('id', 'file_id', 'x', 'y', 'height', 'width') - ->from('recognize_face_embeddings') - ->where($qb->expr()->in('file_id', $qb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY))); - $result = $qb->executeQuery(); - /** @var array $embeddingIdByFace */ - $embeddingIdByFace = []; - while ($row = $result->fetch()) { - $embeddingIdByFace[$this->faceKey($row)] = (int)$row['id']; - } - $result->closeCursor(); + $embeddingIdByFace = $this->findEmbeddingIds($fileIds); // Copy one detection per face within the database, so neither the vectors nor // the float coordinates have to make a lossy round trip through PHP $missingFaces = array_diff_key($detectionIdsByFace, $embeddingIdByFace); - $copyQuery = 'INSERT INTO `*PREFIX*recognize_face_embeddings` (`file_id`, `x`, `y`, `height`, `width`, `face_vector`) ' + $copyQuery = 'INSERT INTO `*PREFIX*recognize_face_vectors` (`file_id`, `x`, `y`, `height`, `width`, `face_vector`) ' . 'SELECT `file_id`, `x`, `y`, `height`, `width`, `face_vector` FROM `*PREFIX*recognize_face_detections` WHERE `id` = ?'; - foreach ($missingFaces as $key => $detectionIds) { + foreach ($missingFaces as $detectionIds) { $this->db->executeStatement($copyQuery, [$detectionIds[0]], [IQueryBuilder::PARAM_INT]); - /** @psalm-suppress DeprecatedMethod The query builder cannot express INSERT ... SELECT */ - $embeddingIdByFace[$key] = $this->db->lastInsertId('*PREFIX*recognize_face_embeddings'); + } + // Read the ids back instead of using lastInsertId(), which fails on Oracle for table + // names longer than the sequence names Doctrine generates + if (count($missingFaces) > 0) { + $embeddingIdByFace = $this->findEmbeddingIds($fileIds); } foreach ($detectionIdsByFace as $key => $detectionIds) { @@ -145,6 +138,26 @@ private function migrateFiles(array $fileIds): int { return $count; } + /** + * @param list $fileIds + * @return array + * @throws \OCP\DB\Exception + */ + private function findEmbeddingIds(array $fileIds): array { + $qb = $this->db->getQueryBuilder(); + $qb->select('id', 'file_id', 'x', 'y', 'height', 'width') + ->from('recognize_face_vectors') + ->where($qb->expr()->in('file_id', $qb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY))); + $result = $qb->executeQuery(); + /** @var array $embeddingIdByFace */ + $embeddingIdByFace = []; + while ($row = $result->fetch()) { + $embeddingIdByFace[$this->faceKey($row)] = (int)$row['id']; + } + $result->closeCursor(); + return $embeddingIdByFace; + } + /** * @param array $row */ From b90d4af815654c1b7962d58671924d72dd0e8b7c Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Thu, 29 Oct 2026 09:15:10 +0100 Subject: [PATCH 03/11] fix: Harden migration against null vectors Assisted-by: ClaudeCode:claude-opus-5-5 Signed-off-by: Marcel Klehr --- lib/Migration/Version013002000Date20261028094821.php | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib/Migration/Version013002000Date20261028094821.php b/lib/Migration/Version013002000Date20261028094821.php index 16cb8f9cf..5f27108c7 100644 --- a/lib/Migration/Version013002000Date20261028094821.php +++ b/lib/Migration/Version013002000Date20261028094821.php @@ -40,6 +40,17 @@ public function preSchemaChange(IOutput $output, Closure $schemaClosure, array $ return; } + // Detections without a vector can't be clustered and can't be copied into the + // recognize_face_vectors table, where the vector is mandatory + $qb = $this->db->getQueryBuilder(); + $qb->delete('recognize_face_detections') + ->where($qb->expr()->isNull('face_vector')) + ->andWhere($qb->expr()->isNull('embedding_id')); + $deleted = $qb->executeStatement(); + if ($deleted > 0) { + $output->info('Removed ' . $deleted . ' face detections without a face vector'); + } + $qb = $this->db->getQueryBuilder(); $qb->select($qb->func()->count('id')) ->from('recognize_face_detections') From 328a03fbd6cf8deca787d9d684392824c3f2c057 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Thu, 29 Oct 2026 09:24:19 +0100 Subject: [PATCH 04/11] fix: Minor issues Assisted-by: ClaudeCode:claude-opus-5-5 Signed-off-by: Marcel Klehr --- lib/Dav/Faces/FacePhoto.php | 7 ++++--- lib/Db/FaceDetectionMapper.php | 28 ++++++++++++++++++++++++---- tests/ClusterTest.php | 7 ++++--- 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/lib/Dav/Faces/FacePhoto.php b/lib/Dav/Faces/FacePhoto.php index d48370816..dd2208270 100644 --- a/lib/Dav/Faces/FacePhoto.php +++ b/lib/Dav/Faces/FacePhoto.php @@ -47,11 +47,12 @@ public function getName(): string { #[Override] public function delete(): void { $detections = $this->detectionMapper->findByClusterIdWithVector($this->faceDetection->getClusterId()); - $vectors = array_filter($detections, fn (FaceDetection $detection) => $detection->getId() === $this->faceDetection->getId()); - if (count($detections) > 1 && count($vectors) > 0) { + // $this->faceDetection was loaded without its vector, so take it from the cluster's detections + $thisDetection = current(array_filter($detections, fn (FaceDetection $detection) => $detection->getId() === $this->faceDetection->getId())); + if (count($detections) > 1 && $thisDetection !== false) { $centroid = FaceClusterAnalyzer::calculateCentroidOfDetections($detections); $distance = new Euclidean(); - $distanceValue = $distance->compute($centroid, reset($vectors)->getVector()); + $distanceValue = $distance->compute($centroid, $thisDetection->getVector()); // Set threshold to avoid recreating the same mistake $this->faceDetection->setThreshold($distanceValue); } diff --git a/lib/Db/FaceDetectionMapper.php b/lib/Db/FaceDetectionMapper.php index 53420de00..75f38c20e 100644 --- a/lib/Db/FaceDetectionMapper.php +++ b/lib/Db/FaceDetectionMapper.php @@ -47,12 +47,21 @@ private function selectWithVector(IQueryBuilder $qb): IQueryBuilder { /** * Stores the vector of a detection that has no embedding yet as a new embedding * + * @param bool $reuseExisting Use an existing embedding of the same file with the same bounding box instead, if there is one * @throws \OCP\DB\Exception */ - private function ensureEmbedding(FaceDetection $detection): void { + private function ensureEmbedding(FaceDetection $detection, bool $reuseExisting): void { if ($detection->getEmbeddingId() !== null) { return; } + if ($reuseExisting) { + foreach ($this->embeddings->findByFileId($detection->getFileId()) as $existing) { + if (self::sameBoundingBox($existing, $detection)) { + $detection->setEmbeddingId($existing->getId()); + return; + } + } + } $embedding = new FaceEmbedding(); $embedding->setFileId($detection->getFileId()); $embedding->setX($detection->getX()); @@ -64,6 +73,16 @@ private function ensureEmbedding(FaceDetection $detection): void { $detection->setEmbeddingId($embedding->getId()); } + /** + * Compares with a tolerance, as not all databases store doubles losslessly + */ + private static function sameBoundingBox(FaceEmbedding $embedding, FaceDetection $detection): bool { + return abs($embedding->getX() - $detection->getX()) < 1e-9 + && abs($embedding->getY() - $detection->getY()) < 1e-9 + && abs($embedding->getHeight() - $detection->getHeight()) < 1e-9 + && abs($embedding->getWidth() - $detection->getWidth()) < 1e-9; + } + private function getMinDetectionSize(): float { if ($this->settingsService->getSetting('taskprocessing.enabled') === 'true') { return ImageFaceRecognitionClassifier::MIN_DETECTION_SIZE; @@ -88,14 +107,15 @@ public function find(int $id): FaceDetection { /** * Inserts a detection for a user, unless that user already has a detection for the same embedding. - * If the detection has no embedding yet, its vector is stored as a new embedding. + * If the detection has no embedding yet, it is linked to the embedding of the same face in the same file, + * or its vector is stored as a new embedding if there is none. * * @param FaceDetection $entity * @throws \OCP\DB\Exception */ public function insert(Entity $entity): FaceDetection { /** @var FaceDetection $entity */ - $this->ensureEmbedding($entity); + $this->ensureEmbedding($entity, true); $qb = $this->db->getQueryBuilder(); $qb->select(FaceDetection::$columns) ->from('recognize_face_detections') @@ -115,7 +135,7 @@ public function insert(Entity $entity): FaceDetection { */ public function insertWithoutDeduplication(Entity $entity): FaceDetection { /** @var FaceDetection $entity */ - $this->ensureEmbedding($entity); + $this->ensureEmbedding($entity, false); return parent::insert($entity); } diff --git a/tests/ClusterTest.php b/tests/ClusterTest.php index 7bbcfe2a8..49d6d2672 100644 --- a/tests/ClusterTest.php +++ b/tests/ClusterTest.php @@ -47,7 +47,8 @@ public function setUp(): void { } $detections = $this->faceDetectionMapper->findByUserId(self::TEST_USER1); foreach ($detections as $detection) { - $this->faceDetectionMapper->delete($detection); + // Also removes the embeddings, so inserted detections don't get linked to the vectors of a previous test + $this->faceDetectionMapper->deleteByFileId($detection->getFileId()); } // Generate artificial clusters along just one dimension @@ -248,7 +249,7 @@ public function testClusterTemptClusterMerging() { $vector = self::getNullVector(); $vector[0] = $clusterValue + 0.001 * $i; $detection->setVector($vector); - $detection->setFileId($i); + $detection->setFileId(2 * self::INITIAL_DETECTIONS_PER_CLUSTER + $i); $detection->setHeight(0.5); $detection->setWidth(0.5); $this->faceDetectionMapper->insert($detection); @@ -277,7 +278,7 @@ public function testClusterTemptClusterMerging() { $vector = self::getNullVector(); $vector[0] = $clusterValue + 0.001 * $i; $detection->setVector($vector); - $detection->setFileId($i); + $detection->setFileId(3 * self::INITIAL_DETECTIONS_PER_CLUSTER + $i); $detection->setHeight(0.5); $detection->setWidth(0.5); $this->faceDetectionMapper->insert($detection); From 92a8e5ee3aecacaba0a0f17cba78a908dd58dc1f Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Thu, 29 Oct 2026 10:18:28 +0100 Subject: [PATCH 05/11] fix: Fix psalm issues Assisted-by: ClaudeCode:claude-opus-5-5 Signed-off-by: Marcel Klehr --- lib/Dav/Faces/FacePhoto.php | 4 +++- psalm-baseline.xml | 3 --- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/Dav/Faces/FacePhoto.php b/lib/Dav/Faces/FacePhoto.php index dd2208270..9d2d90d6a 100644 --- a/lib/Dav/Faces/FacePhoto.php +++ b/lib/Dav/Faces/FacePhoto.php @@ -52,7 +52,9 @@ public function delete(): void { if (count($detections) > 1 && $thisDetection !== false) { $centroid = FaceClusterAnalyzer::calculateCentroidOfDetections($detections); $distance = new Euclidean(); - $distanceValue = $distance->compute($centroid, $thisDetection->getVector()); + /** @var list $vector */ + $vector = $thisDetection->getVector(); + $distanceValue = $distance->compute($centroid, $vector); // Set threshold to avoid recreating the same mistake $this->faceDetection->setThreshold($distanceValue); } diff --git a/psalm-baseline.xml b/psalm-baseline.xml index bdf706cff..6e2a7f46d 100644 --- a/psalm-baseline.xml +++ b/psalm-baseline.xml @@ -613,9 +613,6 @@ - - getVector()]]> - getFile()->getSize()]]> From 823c301b80d2475c882a737ffa0ec74ee259f32a Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Mon, 2 Nov 2026 08:24:36 +0100 Subject: [PATCH 06/11] fix(TaskResultListener): Do not produce duplicates Assisted-by: ClaudeCode:claude-opus-5.5 Signed-off-by: Marcel Klehr --- lib/Db/FaceEmbedding.php | 10 ++++++ lib/TaskProcessing/TaskResultListener.php | 38 ++++++++++++++++++++--- 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/lib/Db/FaceEmbedding.php b/lib/Db/FaceEmbedding.php index 2caef0189..fdcb279df 100644 --- a/lib/Db/FaceEmbedding.php +++ b/lib/Db/FaceEmbedding.php @@ -57,4 +57,14 @@ public function getVector(): array { public function setVector(array $vector): void { $this->setter('faceVector', [$vector]); } + + /** + * Compares with a tolerance, as not all databases store doubles losslessly + */ + public function hasSameBoundingBox(FaceEmbedding $other): bool { + return abs($this->getX() - $other->getX()) < 1e-9 + && abs($this->getY() - $other->getY()) < 1e-9 + && abs($this->getHeight() - $other->getHeight()) < 1e-9 + && abs($this->getWidth() - $other->getWidth()) < 1e-9; + } } diff --git a/lib/TaskProcessing/TaskResultListener.php b/lib/TaskProcessing/TaskResultListener.php index aa703595f..f3f6cc31e 100644 --- a/lib/TaskProcessing/TaskResultListener.php +++ b/lib/TaskProcessing/TaskResultListener.php @@ -203,6 +203,18 @@ private function applyFaceResults(array $fileIds, array $results): void { } $raw = (string)$results[$i]; $userIds = $this->getUsersWithFileAccess($fileId); + // The file may have been processed before (e.g. it was edited and queued again), + // so reuse faces that are already stored instead of storing them a second time + try { + $existingEmbeddings = $this->faceEmbeddings->findByFileId($fileId); + $hasDetection = []; + foreach ($this->faceDetections->findByFileId($fileId) as $detection) { + $hasDetection[$detection->getUserId()][(int)$detection->getEmbeddingId()] = true; + } + } catch (\Throwable $e) { + $this->logger->error('Could not look up existing faces for file ' . $fileId, ['exception' => $e]); + continue; + } foreach (explode("\n", $raw) as $line) { $line = trim($line); if ($line === '') { @@ -235,13 +247,29 @@ private function applyFaceResults(array $fileIds, array $results): void { $embedding->setWidth((float)($face['width'] ?? 0)); $embedding->setHeight((float)($face['height'] ?? 0)); $embedding->setVector($vector); - try { - $this->faceEmbeddings->insert($embedding); - } catch (\Throwable $e) { - $this->logger->error('Could not store face embedding for file ' . $fileId, ['exception' => $e]); - continue; + $existingEmbedding = null; + foreach ($existingEmbeddings as $candidate) { + if ($candidate->hasSameBoundingBox($embedding)) { + $existingEmbedding = $candidate; + break; + } + } + if ($existingEmbedding !== null) { + $embedding = $existingEmbedding; + } else { + try { + $this->faceEmbeddings->insert($embedding); + } catch (\Throwable $e) { + $this->logger->error('Could not store face embedding for file ' . $fileId, ['exception' => $e]); + continue; + } + $existingEmbeddings[] = $embedding; } foreach ($userIds as $userId) { + if ($hasDetection[$userId][$embedding->getId()] ?? false) { + continue; + } + $hasDetection[$userId][$embedding->getId()] = true; try { $this->faceDetections->insertForUser($embedding, $userId); } catch (\Throwable $e) { From cdd499be4c3b8b1462a408f5a278c146108bf3be Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Mon, 2 Nov 2026 08:25:49 +0100 Subject: [PATCH 07/11] fix(FaceDetectionMapper): Do not count faces without embeddings Assisted-by: ClaudeCode:claude-opus-5.5 Signed-off-by: Marcel Klehr --- lib/Db/FaceDetectionMapper.php | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/lib/Db/FaceDetectionMapper.php b/lib/Db/FaceDetectionMapper.php index 75f38c20e..d26e35462 100644 --- a/lib/Db/FaceDetectionMapper.php +++ b/lib/Db/FaceDetectionMapper.php @@ -472,11 +472,13 @@ public function findDetectionForPreviewImageByClusterId(int $clusterId) : FaceDe public function countUnclustered(): int { $minDetectionSize = $this->getMinDetectionSize(); $qb = $this->db->getQueryBuilder(); - $qb->select($qb->func()->count('id')) - ->from('recognize_face_detections') - ->where($qb->expr()->isNull('cluster_id')) - ->andWhere($qb->expr()->gte('height', $qb->createPositionalParameter($minDetectionSize))) - ->andWhere($qb->expr()->gte('width', $qb->createPositionalParameter($minDetectionSize))); + // Only count detections that can actually be clustered, i.e. those with an embedding (see findUnclusteredByUserId) + $qb->select($qb->func()->count('d.id')) + ->from('recognize_face_detections', 'd') + ->innerJoin('d', 'recognize_face_vectors', 'e', $qb->expr()->eq('d.embedding_id', 'e.id')) + ->where($qb->expr()->isNull('d.cluster_id')) + ->andWhere($qb->expr()->gte('d.height', $qb->createPositionalParameter($minDetectionSize))) + ->andWhere($qb->expr()->gte('d.width', $qb->createPositionalParameter($minDetectionSize))); $result = $qb->executeQuery(); /** @var int|string $count */ $count = $result->fetch(\PDO::FETCH_COLUMN); @@ -491,11 +493,14 @@ public function countUnclustered(): int { public function getUsersForUnclustered(): array { $minDetectionSize = $this->getMinDetectionSize(); $qb = $this->db->getQueryBuilder(); - $qb->selectDistinct('user_id') - ->from('recognize_face_detections') - ->where($qb->expr()->isNull('cluster_id')) - ->andWhere($qb->expr()->gte('height', $qb->createPositionalParameter($minDetectionSize))) - ->andWhere($qb->expr()->gte('width', $qb->createPositionalParameter($minDetectionSize))); + // Only consider detections that can actually be clustered, i.e. those with an embedding (see findUnclusteredByUserId), + // otherwise ClusterFacesJob would be scheduled for these users forever + $qb->selectDistinct('d.user_id') + ->from('recognize_face_detections', 'd') + ->innerJoin('d', 'recognize_face_vectors', 'e', $qb->expr()->eq('d.embedding_id', 'e.id')) + ->where($qb->expr()->isNull('d.cluster_id')) + ->andWhere($qb->expr()->gte('d.height', $qb->createPositionalParameter($minDetectionSize))) + ->andWhere($qb->expr()->gte('d.width', $qb->createPositionalParameter($minDetectionSize))); $result = $qb->executeQuery(); /** @var array $users */ $users = $result->fetchAll(\PDO::FETCH_COLUMN); From c8fda39e7247e5946e03fccaa7a189d1c74db03e Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Mon, 2 Nov 2026 08:28:44 +0100 Subject: [PATCH 08/11] fix(Migration): Fail early instead of losing data + no duplicate copies Assisted-by: ClaudeCode:claude-opus-5.5 Signed-off-by: Marcel Klehr --- .../Version013002000Date20261028094821.php | 83 ++++++++++++++----- 1 file changed, 62 insertions(+), 21 deletions(-) diff --git a/lib/Migration/Version013002000Date20261028094821.php b/lib/Migration/Version013002000Date20261028094821.php index 5f27108c7..020d4ff6d 100644 --- a/lib/Migration/Version013002000Date20261028094821.php +++ b/lib/Migration/Version013002000Date20261028094821.php @@ -22,6 +22,8 @@ final class Version013002000Date20261028094821 extends SimpleMigrationStep { private const FILE_BATCH_SIZE = 500; private const QUERY_CHUNK_SIZE = 1000; + // Coordinates are fractions of the image size, so this is far below the size of a pixel + private const COORDINATE_TOLERANCE = 1e-6; public function __construct( private IDBConnection $db, @@ -99,6 +101,8 @@ public function preSchemaChange(IOutput $output, Closure $schemaClosure, array $ * @param list $fileIds * @return int The number of migrated detections * @throws \OCP\DB\Exception + * @throws \RuntimeException If a face could not be linked to its embedding; this aborts the migration + * before the face_vector column is dropped, so no vectors are lost */ private function migrateFiles(array $fileIds): int { // Group the detections of these files by face; all copies of a face share file and bounding box @@ -109,38 +113,49 @@ private function migrateFiles(array $fileIds): int { ->andWhere($qb->expr()->in('file_id', $qb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY))) ->orderBy('id'); $result = $qb->executeQuery(); - /** @var array> $detectionIdsByFace */ - $detectionIdsByFace = []; + /** @var array}> $faces */ + $faces = []; $count = 0; while ($row = $result->fetch()) { - $key = $this->faceKey($row); - $detectionIdsByFace[$key][] = (int)$row['id']; + $face = $this->normalizeFace($row); + $key = implode('|', $face); + $faces[$key] ??= ['face' => $face, 'detectionIds' => []]; + $faces[$key]['detectionIds'][] = (int)$row['id']; $count++; } $result->closeCursor(); // Reuse embeddings created by a previous, interrupted run - $embeddingIdByFace = $this->findEmbeddingIds($fileIds); + $embeddingsByFile = $this->findEmbeddings($fileIds); // Copy one detection per face within the database, so neither the vectors nor // the float coordinates have to make a lossy round trip through PHP - $missingFaces = array_diff_key($detectionIdsByFace, $embeddingIdByFace); $copyQuery = 'INSERT INTO `*PREFIX*recognize_face_vectors` (`file_id`, `x`, `y`, `height`, `width`, `face_vector`) ' . 'SELECT `file_id`, `x`, `y`, `height`, `width`, `face_vector` FROM `*PREFIX*recognize_face_detections` WHERE `id` = ?'; - foreach ($missingFaces as $detectionIds) { - $this->db->executeStatement($copyQuery, [$detectionIds[0]], [IQueryBuilder::PARAM_INT]); + $copied = false; + foreach ($faces as $face) { + if ($this->findMatchingEmbeddingId($face['face'], $embeddingsByFile) === null) { + $this->db->executeStatement($copyQuery, [$face['detectionIds'][0]], [IQueryBuilder::PARAM_INT]); + $copied = true; + // Placeholder until the ids are read back below, so a face within the tolerance isn't copied twice + $embeddingsByFile[$face['face']['file_id']][] = ['id' => 0, 'face' => $face['face']]; + } } // Read the ids back instead of using lastInsertId(), which fails on Oracle for table // names longer than the sequence names Doctrine generates - if (count($missingFaces) > 0) { - $embeddingIdByFace = $this->findEmbeddingIds($fileIds); + if ($copied) { + $embeddingsByFile = $this->findEmbeddings($fileIds); } - foreach ($detectionIdsByFace as $key => $detectionIds) { - foreach (array_chunk($detectionIds, self::QUERY_CHUNK_SIZE) as $chunk) { + foreach ($faces as $face) { + $embeddingId = $this->findMatchingEmbeddingId($face['face'], $embeddingsByFile); + if ($embeddingId === null) { + throw new \RuntimeException('Could not find the copied face embedding for file ' . $face['face']['file_id'] . ' (bounding box ' . implode(', ', array_slice($face['face'], 1)) . ')'); + } + foreach (array_chunk($face['detectionIds'], self::QUERY_CHUNK_SIZE) as $chunk) { $qb = $this->db->getQueryBuilder(); $qb->update('recognize_face_detections') - ->set('embedding_id', $qb->createNamedParameter($embeddingIdByFace[$key], IQueryBuilder::PARAM_INT)) + ->set('embedding_id', $qb->createNamedParameter($embeddingId, IQueryBuilder::PARAM_INT)) ->where($qb->expr()->in('id', $qb->createNamedParameter($chunk, IQueryBuilder::PARAM_INT_ARRAY))); $qb->executeStatement(); } @@ -151,29 +166,55 @@ private function migrateFiles(array $fileIds): int { /** * @param list $fileIds - * @return array + * @return array> The embeddings by file id * @throws \OCP\DB\Exception */ - private function findEmbeddingIds(array $fileIds): array { + private function findEmbeddings(array $fileIds): array { $qb = $this->db->getQueryBuilder(); $qb->select('id', 'file_id', 'x', 'y', 'height', 'width') ->from('recognize_face_vectors') ->where($qb->expr()->in('file_id', $qb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY))); $result = $qb->executeQuery(); - /** @var array $embeddingIdByFace */ - $embeddingIdByFace = []; + $embeddingsByFile = []; while ($row = $result->fetch()) { - $embeddingIdByFace[$this->faceKey($row)] = (int)$row['id']; + $face = $this->normalizeFace($row); + $embeddingsByFile[$face['file_id']][] = ['id' => (int)$row['id'], 'face' => $face]; } $result->closeCursor(); - return $embeddingIdByFace; + return $embeddingsByFile; + } + + /** + * Compares the coordinates with a tolerance instead of exactly, as the copied floats + * are not guaranteed to be read back with the exact same representation on all databases + * + * @param array{file_id: int, x: float, y: float, height: float, width: float} $face + * @param array> $embeddingsByFile + */ + private function findMatchingEmbeddingId(array $face, array $embeddingsByFile): ?int { + foreach ($embeddingsByFile[$face['file_id']] ?? [] as $embedding) { + if (abs($embedding['face']['x'] - $face['x']) < self::COORDINATE_TOLERANCE + && abs($embedding['face']['y'] - $face['y']) < self::COORDINATE_TOLERANCE + && abs($embedding['face']['height'] - $face['height']) < self::COORDINATE_TOLERANCE + && abs($embedding['face']['width'] - $face['width']) < self::COORDINATE_TOLERANCE) { + return $embedding['id']; + } + } + return null; } /** * @param array $row + * @return array{file_id: int, x: float, y: float, height: float, width: float} */ - private function faceKey(array $row): string { - return implode('|', [(int)$row['file_id'], (float)$row['x'], (float)$row['y'], (float)$row['height'], (float)$row['width']]); + private function normalizeFace(array $row): array { + return [ + 'file_id' => (int)$row['file_id'], + 'x' => (float)$row['x'], + 'y' => (float)$row['y'], + 'height' => (float)$row['height'], + 'width' => (float)$row['width'], + ]; } /** From 663ccf4591672c81fc3cd83c29a5a6b996925fc9 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Mon, 2 Nov 2026 08:30:06 +0100 Subject: [PATCH 09/11] fix(Migration): Remove face detections without a fileId Assisted-by: ClaudeCode:claude-opus-5.5 Signed-off-by: Marcel Klehr --- .../Version013002000Date20261028094821.php | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/Migration/Version013002000Date20261028094821.php b/lib/Migration/Version013002000Date20261028094821.php index 020d4ff6d..431339cc9 100644 --- a/lib/Migration/Version013002000Date20261028094821.php +++ b/lib/Migration/Version013002000Date20261028094821.php @@ -53,11 +53,21 @@ public function preSchemaChange(IOutput $output, Closure $schemaClosure, array $ $output->info('Removed ' . $deleted . ' face detections without a face vector'); } + // Detections without a file can't be shown or linked to an embedding, which belongs to a file, + // so they would lose their vector when the column is dropped and linger as unusable rows + $qb = $this->db->getQueryBuilder(); + $qb->delete('recognize_face_detections') + ->where($qb->expr()->isNull('file_id')) + ->andWhere($qb->expr()->isNull('embedding_id')); + $deleted = $qb->executeStatement(); + if ($deleted > 0) { + $output->info('Removed ' . $deleted . ' face detections without a file'); + } + $qb = $this->db->getQueryBuilder(); $qb->select($qb->func()->count('id')) ->from('recognize_face_detections') - ->where($qb->expr()->isNull('embedding_id')) - ->andWhere($qb->expr()->isNotNull('file_id')); + ->where($qb->expr()->isNull('embedding_id')); $result = $qb->executeQuery(); $total = (int)$result->fetchOne(); $result->closeCursor(); From 877dfab9e31c101ee2493eb0be7fbbf3bb6df9dd Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Mon, 2 Nov 2026 08:33:37 +0100 Subject: [PATCH 10/11] fix(ClusteringFaceClassifier): Use a transaction per file Assisted-by: ClaudeCode:claude-opus-5.5 Signed-off-by: Marcel Klehr --- .../Images/ClusteringFaceClassifier.php | 89 +++++----- lib/TaskProcessing/TaskResultListener.php | 153 ++++++++++-------- 2 files changed, 136 insertions(+), 106 deletions(-) diff --git a/lib/Classifiers/Images/ClusteringFaceClassifier.php b/lib/Classifiers/Images/ClusteringFaceClassifier.php index 9a49b1294..6e164dcae 100644 --- a/lib/Classifiers/Images/ClusteringFaceClassifier.php +++ b/lib/Classifiers/Images/ClusteringFaceClassifier.php @@ -23,6 +23,7 @@ use OCP\Files\IRootFolder; use OCP\Files\Node; use OCP\Files\NotFoundException; +use OCP\IDBConnection; use OCP\IPreview; use OCP\ITempManager; use OCP\Share\IManager; @@ -45,7 +46,8 @@ public function __construct( private FaceEmbeddingMapper $faceEmbeddings, QueueService $queue, IRootFolder $rootFolder, private IUserMountCache $userMountCache, private IJobList $jobList, ITempManager $tempManager, IPreview $previewProvider, - private IManager $shareManager) { + private IManager $shareManager, + private IDBConnection $db) { parent::__construct($logger, $config, $rootFolder, $queue, $tempManager, $previewProvider); } @@ -99,52 +101,61 @@ public function classify(array $queueFiles): void { */ foreach ($classifierProcess as $queueFile => $faces) { $this->logger->debug('Face results for ' . $queueFile->getFileId() . ' are in'); - foreach ($faces as $face) { - if ($face['score'] < self::MIN_FACE_RECOGNITION_SCORE) { - $this->logger->debug('Face score too low. continuing with next face.'); - continue; - } - if (abs($face['angle']['roll']) > self::MAX_FACE_ROLL || abs($face['angle']['yaw']) > self::MAX_FACE_YAW) { - $this->logger->debug('Face is not straight. continuing with next face.'); - continue; - } - try { - $node = $this->rootFolder->getFirstNodeById($queueFile->getFileId()); - $userIds = $node !== null ? $this->getUsersWithFileAccess($node) : []; - } catch (InvalidPathException|NotFoundException $e) { - $userIds = []; - } + try { + $node = $this->rootFolder->getFirstNodeById($queueFile->getFileId()); + $userIds = $node !== null ? $this->getUsersWithFileAccess($node) : []; + } catch (InvalidPathException|NotFoundException $e) { + $userIds = []; + } - $embedding = new FaceEmbedding(); - $embedding->setX($face['x']); - $embedding->setY($face['y']); - $embedding->setWidth($face['width']); - $embedding->setHeight($face['height']); - $embedding->setVector($face['vector']); - $embedding->setFileId($queueFile->getFileId()); - try { + // Store the faces of a file all at once or not at all: files that already have faces are + // never analysed again, so a partially stored file would stay incomplete forever + $storedFaces = 0; + $this->db->beginTransaction(); + try { + foreach ($faces as $face) { + if ($face['score'] < self::MIN_FACE_RECOGNITION_SCORE) { + $this->logger->debug('Face score too low. continuing with next face.'); + continue; + } + if (abs($face['angle']['roll']) > self::MAX_FACE_ROLL || abs($face['angle']['yaw']) > self::MAX_FACE_YAW) { + $this->logger->debug('Face is not straight. continuing with next face.'); + continue; + } + + $embedding = new FaceEmbedding(); + $embedding->setX($face['x']); + $embedding->setY($face['y']); + $embedding->setWidth($face['width']); + $embedding->setHeight($face['height']); + $embedding->setVector($face['vector']); + $embedding->setFileId($queueFile->getFileId()); $this->faceEmbeddings->insert($embedding); - } catch (\Throwable $e) { - $this->logger->error('Could not store face embedding in database', ['exception' => $e]); - continue; - } - // Insert face detection for all users with access - foreach ($userIds as $userId) { - $this->logger->debug('preparing face detection for user ' . $userId); - try { + // Insert face detection for all users with access + foreach ($userIds as $userId) { + $this->logger->debug('preparing face detection for user ' . $userId); $this->faceDetections->insertForUser($embedding, $userId); - } catch (\Throwable $e) { - $this->logger->error('Could not store face detection in database', ['exception' => $e]); - continue; } - $this->logger->debug('scheduling ClusterFacesJob for user ' . $userId); - $this->jobList->add(ClusterFacesJob::class, ['userId' => $userId]); + $storedFaces++; } - $this->config->setAppValueString(self::MODEL_NAME . '.status', 'true', lazy: true); - $this->config->setAppValueString(self::MODEL_NAME . '.lastFile', (string)time(), lazy: true); + $this->db->commit(); + } catch (\Throwable $e) { + $this->db->rollBack(); + $this->logger->error('Could not store faces of file ' . $queueFile->getFileId() . ' in database, discarding them so the file can be analysed again', ['exception' => $e]); + continue; + } + + if ($storedFaces === 0) { + continue; + } + foreach ($userIds as $userId) { + $this->logger->debug('scheduling ClusterFacesJob for user ' . $userId); + $this->jobList->add(ClusterFacesJob::class, ['userId' => $userId]); } + $this->config->setAppValueString(self::MODEL_NAME . '.status', 'true', lazy: true); + $this->config->setAppValueString(self::MODEL_NAME . '.lastFile', (string)time(), lazy: true); } $this->logger->debug('face classifier end'); } diff --git a/lib/TaskProcessing/TaskResultListener.php b/lib/TaskProcessing/TaskResultListener.php index f3f6cc31e..343be64f1 100644 --- a/lib/TaskProcessing/TaskResultListener.php +++ b/lib/TaskProcessing/TaskResultListener.php @@ -30,6 +30,7 @@ use OCP\EventDispatcher\IEventListener; use OCP\Files\Config\ICachedMountInfo; use OCP\Files\Config\IUserMountCache; +use OCP\IDBConnection; use OCP\IUserManager; use OCP\IUserSession; use OCP\TaskProcessing\Events\TaskFailedEvent; @@ -57,6 +58,7 @@ public function __construct( private QueueService $queue, private IUserSession $userSession, private IUserManager $userManager, + private IDBConnection $db, ) { } @@ -201,92 +203,109 @@ private function applyFaceResults(array $fileIds, array $results): void { if (!isset($results[$i])) { continue; } - $raw = (string)$results[$i]; $userIds = $this->getUsersWithFileAccess($fileId); - // The file may have been processed before (e.g. it was edited and queued again), - // so reuse faces that are already stored instead of storing them a second time + $faces = $this->parseFaces($fileId, (string)$results[$i]); + + // Store the faces of a file all at once or not at all, so a failure can't leave + // a face stored for only some of the users with access + /** @var array $usersWithNewDetections */ + $usersWithNewDetections = []; + $this->db->beginTransaction(); try { + // The file may have been processed before (e.g. it was edited and queued again), + // so reuse faces that are already stored instead of storing them a second time $existingEmbeddings = $this->faceEmbeddings->findByFileId($fileId); $hasDetection = []; foreach ($this->faceDetections->findByFileId($fileId) as $detection) { $hasDetection[$detection->getUserId()][(int)$detection->getEmbeddingId()] = true; } - } catch (\Throwable $e) { - $this->logger->error('Could not look up existing faces for file ' . $fileId, ['exception' => $e]); - continue; - } - foreach (explode("\n", $raw) as $line) { - $line = trim($line); - if ($line === '') { - continue; - } - try { - $face = json_decode($line, true, 512, JSON_THROW_ON_ERROR); - } catch (\JsonException $e) { - $this->logger->warning('Invalid face JSON for file ' . $fileId, ['exception' => $e]); - continue; - } - if (!is_array($face)) { - continue; - } - if (isset($face['score']) && (float)$face['score'] < ImageFaceRecognitionClassifier::MIN_FACE_RECOGNITION_SCORE) { - continue; - } - // Accept either a full face object {x,y,width,height,score,vector,angle} - // or a bare embedding vector (list of numbers). - $isBareVector = array_is_list($face) && count($face) > 0 && is_numeric($face[0]); - $vector = $isBareVector ? $face : ($face['vector'] ?? null); - if (!is_array($vector)) { - $this->logger->warning('Face entry without embedding vector for file ' . $fileId); - continue; - } - $embedding = new FaceEmbedding(); - $embedding->setFileId($fileId); - $embedding->setX((float)($face['x'] ?? 0)); - $embedding->setY((float)($face['y'] ?? 0)); - $embedding->setWidth((float)($face['width'] ?? 0)); - $embedding->setHeight((float)($face['height'] ?? 0)); - $embedding->setVector($vector); - $existingEmbedding = null; - foreach ($existingEmbeddings as $candidate) { - if ($candidate->hasSameBoundingBox($embedding)) { - $existingEmbedding = $candidate; - break; + + foreach ($faces as $embedding) { + $existingEmbedding = null; + foreach ($existingEmbeddings as $candidate) { + if ($candidate->hasSameBoundingBox($embedding)) { + $existingEmbedding = $candidate; + break; + } } - } - if ($existingEmbedding !== null) { - $embedding = $existingEmbedding; - } else { - try { + if ($existingEmbedding !== null) { + $embedding = $existingEmbedding; + } else { $this->faceEmbeddings->insert($embedding); - } catch (\Throwable $e) { - $this->logger->error('Could not store face embedding for file ' . $fileId, ['exception' => $e]); - continue; - } - $existingEmbeddings[] = $embedding; - } - foreach ($userIds as $userId) { - if ($hasDetection[$userId][$embedding->getId()] ?? false) { - continue; + $existingEmbeddings[] = $embedding; } - $hasDetection[$userId][$embedding->getId()] = true; - try { + foreach ($userIds as $userId) { + if ($hasDetection[$userId][$embedding->getId()] ?? false) { + continue; + } + $hasDetection[$userId][$embedding->getId()] = true; $this->faceDetections->insertForUser($embedding, $userId); - } catch (\Throwable $e) { - $this->logger->error('Could not store face detection for file ' . $fileId, ['exception' => $e]); - continue; - } - if (!isset($scheduledClusterJobsFor[$userId])) { - $this->jobList->add(ClusterFacesJob::class, ['userId' => $userId]); - $scheduledClusterJobsFor[$userId] = true; + $usersWithNewDetections[$userId] = true; } } + $this->db->commit(); + } catch (\Throwable $e) { + $this->db->rollBack(); + $this->logger->error('Could not store faces of file ' . $fileId . ' in database', ['exception' => $e]); + continue; + } + + foreach (array_keys($usersWithNewDetections) as $userId) { + $userId = (string)$userId; + if (!isset($scheduledClusterJobsFor[$userId])) { + $this->jobList->add(ClusterFacesJob::class, ['userId' => $userId]); + $scheduledClusterJobsFor[$userId] = true; + } } $this->config->setAppValueString($model . '.status', 'true', lazy: true); $this->config->setAppValueString($model . '.lastFile', (string)time(), lazy: true); } } + /** + * Parses the faces from the output of a face recognition task, one JSON object per line + * + * @return list + */ + private function parseFaces(int $fileId, string $raw): array { + $faces = []; + foreach (explode("\n", $raw) as $line) { + $line = trim($line); + if ($line === '') { + continue; + } + try { + $face = json_decode($line, true, 512, JSON_THROW_ON_ERROR); + } catch (\JsonException $e) { + $this->logger->warning('Invalid face JSON for file ' . $fileId, ['exception' => $e]); + continue; + } + if (!is_array($face)) { + continue; + } + if (isset($face['score']) && (float)$face['score'] < ImageFaceRecognitionClassifier::MIN_FACE_RECOGNITION_SCORE) { + continue; + } + // Accept either a full face object {x,y,width,height,score,vector,angle} + // or a bare embedding vector (list of numbers). + $isBareVector = array_is_list($face) && count($face) > 0 && is_numeric($face[0]); + $vector = $isBareVector ? $face : ($face['vector'] ?? null); + if (!is_array($vector)) { + $this->logger->warning('Face entry without embedding vector for file ' . $fileId); + continue; + } + $embedding = new FaceEmbedding(); + $embedding->setFileId($fileId); + $embedding->setX((float)($face['x'] ?? 0)); + $embedding->setY((float)($face['y'] ?? 0)); + $embedding->setWidth((float)($face['width'] ?? 0)); + $embedding->setHeight((float)($face['height'] ?? 0)); + $embedding->setVector($vector); + $faces[] = $embedding; + } + return $faces; + } + private function enqueueForLandmarks(int $fileId): void { $mounts = $this->userMountCache->getMountsForFileId($fileId); if (count($mounts) === 0) { From a621c6eaffc7ac7a346fb9f45aa99ba6fcba2b47 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Mon, 2 Nov 2026 08:36:49 +0100 Subject: [PATCH 11/11] fix(ensureEmbedding): Throw a clear error in case there is no embedding or vector Assisted-by: ClaudeCode:claude-opus-5.5 Signed-off-by: Marcel Klehr --- lib/Db/FaceDetection.php | 7 +++++++ lib/Db/FaceDetectionMapper.php | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/lib/Db/FaceDetection.php b/lib/Db/FaceDetection.php index 0f13f6b49..e0c8a8b3a 100644 --- a/lib/Db/FaceDetection.php +++ b/lib/Db/FaceDetection.php @@ -83,6 +83,13 @@ public function toArray(): array { public function getVector(): array { return $this->getter('faceVector'); } + + /** + * Whether the vector is set; most queries don't load it, as it is stored with the embedding + */ + public function hasVector(): bool { + return is_array($this->faceVector); + } public function setVector(array $vector): void { $this->setter('faceVector', [$vector]); } diff --git a/lib/Db/FaceDetectionMapper.php b/lib/Db/FaceDetectionMapper.php index d26e35462..c7c3a0cef 100644 --- a/lib/Db/FaceDetectionMapper.php +++ b/lib/Db/FaceDetectionMapper.php @@ -49,6 +49,7 @@ private function selectWithVector(IQueryBuilder $qb): IQueryBuilder { * * @param bool $reuseExisting Use an existing embedding of the same file with the same bounding box instead, if there is one * @throws \OCP\DB\Exception + * @throws \InvalidArgumentException If the detection has neither an embedding nor a vector to create one from */ private function ensureEmbedding(FaceDetection $detection, bool $reuseExisting): void { if ($detection->getEmbeddingId() !== null) { @@ -62,6 +63,9 @@ private function ensureEmbedding(FaceDetection $detection, bool $reuseExisting): } } } + if (!$detection->hasVector()) { + throw new \InvalidArgumentException('Cannot store a face detection for file ' . $detection->getFileId() . ' without an embedding or a face vector'); + } $embedding = new FaceEmbedding(); $embedding->setFileId($detection->getFileId()); $embedding->setX($detection->getX());