diff --git a/lib/Classifiers/Images/ClusteringFaceClassifier.php b/lib/Classifiers/Images/ClusteringFaceClassifier.php index 81eff74a6..6e164dcae 100644 --- a/lib/Classifiers/Images/ClusteringFaceClassifier.php +++ b/lib/Classifiers/Images/ClusteringFaceClassifier.php @@ -9,8 +9,9 @@ use OCA\Recognize\BackgroundJobs\ClusterFacesJob; use OCA\Recognize\Classifiers\Classifier; -use OCA\Recognize\Db\FaceDetection; use OCA\Recognize\Db\FaceDetectionMapper; +use OCA\Recognize\Db\FaceEmbedding; +use OCA\Recognize\Db\FaceEmbeddingMapper; use OCA\Recognize\Service\Logger; use OCA\Recognize\Service\QueueService; use OCP\AppFramework\Services\IAppConfig; @@ -22,6 +23,7 @@ use OCP\Files\IRootFolder; use OCP\Files\Node; use OCP\Files\NotFoundException; +use OCP\IDBConnection; use OCP\IPreview; use OCP\ITempManager; use OCP\Share\IManager; @@ -41,9 +43,11 @@ public function __construct( Logger $logger, IAppConfig $config, private FaceDetectionMapper $faceDetections, + private FaceEmbeddingMapper $faceEmbeddings, QueueService $queue, IRootFolder $rootFolder, private IUserMountCache $userMountCache, private IJobList $jobList, ITempManager $tempManager, IPreview $previewProvider, - private IManager $shareManager) { + private IManager $shareManager, + private IDBConnection $db) { parent::__construct($logger, $config, $rootFolder, $queue, $tempManager, $previewProvider); } @@ -73,7 +77,7 @@ public function classify(array $queueFiles): void { $filteredQueueFiles = []; foreach ($queueFiles as $queueFile) { try { - $facesByFileCount = count($this->faceDetections->findByFileId($queueFile->getFileId())); + $facesByFileCount = $this->faceEmbeddings->countByFileId($queueFile->getFileId()); } catch (Exception $e) { $this->logger->debug('finding faces by file '.$queueFile->getFileId().' failed', ['exception' => $e]); $facesByFileCount = 1; @@ -97,46 +101,61 @@ public function classify(array $queueFiles): void { */ foreach ($classifierProcess as $queueFile => $faces) { $this->logger->debug('Face results for ' . $queueFile->getFileId() . ' are in'); - foreach ($faces as $face) { - if ($face['score'] < self::MIN_FACE_RECOGNITION_SCORE) { - $this->logger->debug('Face score too low. continuing with next face.'); - continue; - } - if (abs($face['angle']['roll']) > self::MAX_FACE_ROLL || abs($face['angle']['yaw']) > self::MAX_FACE_YAW) { - $this->logger->debug('Face is not straight. continuing with next face.'); - continue; - } - try { - $node = $this->rootFolder->getFirstNodeById($queueFile->getFileId()); - $userIds = $node !== null ? $this->getUsersWithFileAccess($node) : []; - } catch (InvalidPathException|NotFoundException $e) { - $userIds = []; - } + try { + $node = $this->rootFolder->getFirstNodeById($queueFile->getFileId()); + $userIds = $node !== null ? $this->getUsersWithFileAccess($node) : []; + } catch (InvalidPathException|NotFoundException $e) { + $userIds = []; + } - // Insert face detection for all users with access - foreach ($userIds as $userId) { - $this->logger->debug('preparing face detection for user ' . $userId); - $faceDetection = new FaceDetection(); - $faceDetection->setX($face['x']); - $faceDetection->setY($face['y']); - $faceDetection->setWidth($face['width']); - $faceDetection->setHeight($face['height']); - $faceDetection->setVector($face['vector']); - $faceDetection->setFileId($queueFile->getFileId()); - $faceDetection->setUserId($userId); - try { - $this->faceDetections->insert($faceDetection); - } catch (\Throwable $e) { - $this->logger->error('Could not store face detection in database', ['exception' => $e]); + // Store the faces of a file all at once or not at all: files that already have faces are + // never analysed again, so a partially stored file would stay incomplete forever + $storedFaces = 0; + $this->db->beginTransaction(); + try { + foreach ($faces as $face) { + if ($face['score'] < self::MIN_FACE_RECOGNITION_SCORE) { + $this->logger->debug('Face score too low. continuing with next face.'); + continue; + } + if (abs($face['angle']['roll']) > self::MAX_FACE_ROLL || abs($face['angle']['yaw']) > self::MAX_FACE_YAW) { + $this->logger->debug('Face is not straight. continuing with next face.'); continue; } - $this->logger->debug('scheduling ClusterFacesJob for user ' . $userId); - $this->jobList->add(ClusterFacesJob::class, ['userId' => $userId]); + + $embedding = new FaceEmbedding(); + $embedding->setX($face['x']); + $embedding->setY($face['y']); + $embedding->setWidth($face['width']); + $embedding->setHeight($face['height']); + $embedding->setVector($face['vector']); + $embedding->setFileId($queueFile->getFileId()); + $this->faceEmbeddings->insert($embedding); + + // Insert face detection for all users with access + foreach ($userIds as $userId) { + $this->logger->debug('preparing face detection for user ' . $userId); + $this->faceDetections->insertForUser($embedding, $userId); + } + $storedFaces++; } - $this->config->setAppValueString(self::MODEL_NAME . '.status', 'true', lazy: true); - $this->config->setAppValueString(self::MODEL_NAME . '.lastFile', (string)time(), lazy: true); + $this->db->commit(); + } catch (\Throwable $e) { + $this->db->rollBack(); + $this->logger->error('Could not store faces of file ' . $queueFile->getFileId() . ' in database, discarding them so the file can be analysed again', ['exception' => $e]); + continue; + } + + if ($storedFaces === 0) { + continue; + } + foreach ($userIds as $userId) { + $this->logger->debug('scheduling ClusterFacesJob for user ' . $userId); + $this->jobList->add(ClusterFacesJob::class, ['userId' => $userId]); } + $this->config->setAppValueString(self::MODEL_NAME . '.status', 'true', lazy: true); + $this->config->setAppValueString(self::MODEL_NAME . '.lastFile', (string)time(), lazy: true); } $this->logger->debug('face classifier end'); } diff --git a/lib/Dav/Faces/FacePhoto.php b/lib/Dav/Faces/FacePhoto.php index f30715343..9d2d90d6a 100644 --- a/lib/Dav/Faces/FacePhoto.php +++ b/lib/Dav/Faces/FacePhoto.php @@ -46,11 +46,15 @@ public function getName(): string { #[Override] public function delete(): void { - $detections = $this->detectionMapper->findByClusterId($this->faceDetection->getClusterId()); - if (count($detections) > 1) { + $detections = $this->detectionMapper->findByClusterIdWithVector($this->faceDetection->getClusterId()); + // $this->faceDetection was loaded without its vector, so take it from the cluster's detections + $thisDetection = current(array_filter($detections, fn (FaceDetection $detection) => $detection->getId() === $this->faceDetection->getId())); + if (count($detections) > 1 && $thisDetection !== false) { $centroid = FaceClusterAnalyzer::calculateCentroidOfDetections($detections); $distance = new Euclidean(); - $distanceValue = $distance->compute($centroid, $this->faceDetection->getVector()); + /** @var list $vector */ + $vector = $thisDetection->getVector(); + $distanceValue = $distance->compute($centroid, $vector); // Set threshold to avoid recreating the same mistake $this->faceDetection->setThreshold($distanceValue); } diff --git a/lib/Db/FaceDetection.php b/lib/Db/FaceDetection.php index 3c7589a83..e0c8a8b3a 100644 --- a/lib/Db/FaceDetection.php +++ b/lib/Db/FaceDetection.php @@ -29,6 +29,11 @@ * @method int|null getClusterId() * @method float getThreshold() * @method setThreshold(float $threshold) + * @method int|null getEmbeddingId() + * @method setEmbeddingId(int $embeddingId) + * + * The face vector is stored once per file in {@see FaceEmbedding} and only populated + * on detections loaded with {@see FaceDetectionMapper} methods that join it in. */ class FaceDetection extends Entity { protected $fileId; @@ -37,17 +42,18 @@ class FaceDetection extends Entity { protected $y; protected $height; protected $width; + protected $embeddingId; protected $faceVector; protected $clusterId; protected $threshold; /** * @var string[] */ - public static $columns = ['id', 'user_id', 'file_id', 'x', 'y', 'height', 'width', 'face_vector', 'cluster_id', 'threshold']; + public static $columns = ['id', 'user_id', 'file_id', 'x', 'y', 'height', 'width', 'embedding_id', 'cluster_id', 'threshold']; /** * @var string[] */ - public static $fields = ['id', 'userId', 'fileId', 'x', 'y', 'height', 'width', 'faceVector', 'clusterId', 'threshold']; + public static $fields = ['id', 'userId', 'fileId', 'x', 'y', 'height', 'width', 'embeddingId', 'clusterId', 'threshold']; public function __construct() { // add types in constructor @@ -57,6 +63,7 @@ public function __construct() { $this->addType('y', 'float'); $this->addType('height', 'float'); $this->addType('width', 'float'); + $this->addType('embeddingId', 'integer'); $this->addType('faceVector', 'json'); $this->addType('clusterId', 'integer'); $this->addType('threshold', 'float'); @@ -65,7 +72,7 @@ public function __construct() { public function toArray(): array { $array = []; foreach (static::$fields as $field) { - if ($field === 'faceVector') { + if ($field === 'embeddingId') { continue; } $array[$field] = $this->{$field}; @@ -76,7 +83,23 @@ public function toArray(): array { public function getVector(): array { return $this->getter('faceVector'); } + + /** + * Whether the vector is set; most queries don't load it, as it is stored with the embedding + */ + public function hasVector(): bool { + return is_array($this->faceVector); + } public function setVector(array $vector): void { $this->setter('faceVector', [$vector]); } + + /** + * The vector is not a column of the detections table + */ + public function getUpdatedFields(): array { + $fields = parent::getUpdatedFields(); + unset($fields['faceVector']); + return $fields; + } } diff --git a/lib/Db/FaceDetectionMapper.php b/lib/Db/FaceDetectionMapper.php index 2651c67c8..c7c3a0cef 100644 --- a/lib/Db/FaceDetectionMapper.php +++ b/lib/Db/FaceDetectionMapper.php @@ -24,12 +24,67 @@ final class FaceDetectionMapper extends QBMapper { private IConfig $config; private SettingsService $settingsService; + private FaceEmbeddingMapper $embeddings; - public function __construct(IDBConnection $db, IConfig $config, SettingsService $settingsService) { + public function __construct(IDBConnection $db, IConfig $config, SettingsService $settingsService, FaceEmbeddingMapper $embeddings) { parent::__construct($db, 'recognize_face_detections', FaceDetection::class); $this->db = $db; $this->config = $config; $this->settingsService = $settingsService; + $this->embeddings = $embeddings; + } + + /** + * Selects detections together with the face vector of their embedding + */ + private function selectWithVector(IQueryBuilder $qb): IQueryBuilder { + return $qb->select(array_map(fn ($col) => 'd.'.$col, FaceDetection::$columns)) + ->addSelect('e.face_vector') + ->from('recognize_face_detections', 'd') + ->innerJoin('d', 'recognize_face_vectors', 'e', $qb->expr()->eq('d.embedding_id', 'e.id')); + } + + /** + * Stores the vector of a detection that has no embedding yet as a new embedding + * + * @param bool $reuseExisting Use an existing embedding of the same file with the same bounding box instead, if there is one + * @throws \OCP\DB\Exception + * @throws \InvalidArgumentException If the detection has neither an embedding nor a vector to create one from + */ + private function ensureEmbedding(FaceDetection $detection, bool $reuseExisting): void { + if ($detection->getEmbeddingId() !== null) { + return; + } + if ($reuseExisting) { + foreach ($this->embeddings->findByFileId($detection->getFileId()) as $existing) { + if (self::sameBoundingBox($existing, $detection)) { + $detection->setEmbeddingId($existing->getId()); + return; + } + } + } + if (!$detection->hasVector()) { + throw new \InvalidArgumentException('Cannot store a face detection for file ' . $detection->getFileId() . ' without an embedding or a face vector'); + } + $embedding = new FaceEmbedding(); + $embedding->setFileId($detection->getFileId()); + $embedding->setX($detection->getX()); + $embedding->setY($detection->getY()); + $embedding->setHeight($detection->getHeight()); + $embedding->setWidth($detection->getWidth()); + $embedding->setVector($detection->getVector()); + $this->embeddings->insert($embedding); + $detection->setEmbeddingId($embedding->getId()); + } + + /** + * Compares with a tolerance, as not all databases store doubles losslessly + */ + private static function sameBoundingBox(FaceEmbedding $embedding, FaceDetection $detection): bool { + return abs($embedding->getX() - $detection->getX()) < 1e-9 + && abs($embedding->getY() - $detection->getY()) < 1e-9 + && abs($embedding->getHeight() - $detection->getHeight()) < 1e-9 + && abs($embedding->getWidth() - $detection->getWidth()) < 1e-9; } private function getMinDetectionSize(): float { @@ -55,20 +110,21 @@ public function find(int $id): FaceDetection { } /** + * Inserts a detection for a user, unless that user already has a detection for the same embedding. + * If the detection has no embedding yet, it is linked to the embedding of the same face in the same file, + * or its vector is stored as a new embedding if there is none. + * * @param FaceDetection $entity * @throws \OCP\DB\Exception */ public function insert(Entity $entity): FaceDetection { /** @var FaceDetection $entity */ + $this->ensureEmbedding($entity, true); $qb = $this->db->getQueryBuilder(); $qb->select(FaceDetection::$columns) ->from('recognize_face_detections') - ->where($qb->expr()->eq('file_id', $qb->createPositionalParameter($entity->getFileId(), IQueryBuilder::PARAM_INT))) - ->andWhere($qb->expr()->eq('user_id', $qb->createPositionalParameter($entity->getUserId(), IQueryBuilder::PARAM_STR))) - ->andWhere($qb->expr()->eq('x', $qb->createPositionalParameter($entity->getX(), IQueryBuilder::PARAM_INT))) - ->andWhere($qb->expr()->eq('y', $qb->createPositionalParameter($entity->getY(), IQueryBuilder::PARAM_INT))) - ->andWhere($qb->expr()->eq('height', $qb->createPositionalParameter($entity->getHeight(), IQueryBuilder::PARAM_INT))) - ->andWhere($qb->expr()->eq('width', $qb->createPositionalParameter($entity->getWidth(), IQueryBuilder::PARAM_INT))); + ->where($qb->expr()->eq('embedding_id', $qb->createPositionalParameter($entity->getEmbeddingId(), IQueryBuilder::PARAM_INT))) + ->andWhere($qb->expr()->eq('user_id', $qb->createPositionalParameter($entity->getUserId(), IQueryBuilder::PARAM_STR))); $duplicates = $this->findEntities($qb); if (empty($duplicates)) { @@ -83,9 +139,27 @@ public function insert(Entity $entity): FaceDetection { */ public function insertWithoutDeduplication(Entity $entity): FaceDetection { /** @var FaceDetection $entity */ + $this->ensureEmbedding($entity, false); return parent::insert($entity); } + /** + * Creates the per-user detection for an embedding + * + * @throws \OCP\DB\Exception + */ + public function insertForUser(FaceEmbedding $embedding, string $userId): FaceDetection { + $detection = new FaceDetection(); + $detection->setEmbeddingId($embedding->getId()); + $detection->setFileId($embedding->getFileId()); + $detection->setX($embedding->getX()); + $detection->setY($embedding->getY()); + $detection->setHeight($embedding->getHeight()); + $detection->setWidth($embedding->getWidth()); + $detection->setUserId($userId); + return parent::insert($detection); + } + /** * @throws \OCP\DB\Exception */ @@ -93,6 +167,20 @@ public function deleteAll(): void { $qb = $this->db->getQueryBuilder(); $qb->delete('recognize_face_detections') ->executeStatement(); + $this->embeddings->deleteAll(); + } + + /** + * Deletes all detections of a file for all users, including the embeddings + * + * @throws \OCP\DB\Exception + */ + public function deleteByFileId(int $fileId): void { + $qb = $this->db->getQueryBuilder(); + $qb->delete('recognize_face_detections') + ->where($qb->expr()->eq('file_id', $qb->createPositionalParameter($fileId, IQueryBuilder::PARAM_INT))) + ->executeStatement(); + $this->embeddings->deleteByFileId($fileId); } /** @@ -108,6 +196,18 @@ public function findByClusterId(int $clusterId) : array { return $this->findEntities($qb); } + /** + * @param int $clusterId + * @return list<\OCA\Recognize\Db\FaceDetection> + * @throws \OCP\DB\Exception + */ + public function findByClusterIdWithVector(int $clusterId) : array { + $qb = $this->db->getQueryBuilder(); + $this->selectWithVector($qb) + ->where($qb->expr()->eq('d.cluster_id', $qb->createPositionalParameter($clusterId))); + return $this->findEntities($qb); + } + /** * @throws \OCP\DB\Exception * @return list<\OCA\Recognize\Db\FaceDetection> @@ -146,21 +246,33 @@ public function findByFileIdAndUser(int $fileId, string $userId): array { } /** + * Gives the passed users a detection for every face found in the file that they don't have yet + * + * @param int $fileId + * @param list $userIds + * @return list The users that received new detections * @throws \OCP\DB\Exception */ - public function copyDetectionsForFileFromUserToUser(int $fileId, string $fromUser, string $toUser) : void { - $detections = $this->findByFileIdAndUser($fileId, $fromUser); - foreach ($detections as $detection) { - $detectionCopy = new FaceDetection(); - $detectionCopy->setX($detection->getX()); - $detectionCopy->setY($detection->getY()); - $detectionCopy->setVector($detection->getVector()); - $detectionCopy->setFileId($detection->getFileId()); - $detectionCopy->setHeight($detection->getHeight()); - $detectionCopy->setWidth($detection->getWidth()); - $detectionCopy->setUserId($toUser); - $this->insert($detectionCopy); + public function addUsersToFile(int $fileId, array $userIds) : array { + $embeddings = $this->embeddings->findByFileId($fileId); + if (count($embeddings) === 0 || count($userIds) === 0) { + return []; + } + $hasDetection = []; + foreach ($this->findByFileId($fileId) as $detection) { + $hasDetection[$detection->getUserId()][(int)$detection->getEmbeddingId()] = true; } + $addedUsers = []; + foreach ($userIds as $userId) { + foreach ($embeddings as $embedding) { + if ($hasDetection[$userId][$embedding->getId()] ?? false) { + continue; + } + $this->insertForUser($embedding, $userId); + $addedUsers[$userId] = true; + } + } + return array_map('strval', array_keys($addedUsers)); } /** @@ -249,12 +361,11 @@ public function findUserIds() :array { */ public function findUnclusteredByUserId(string $userId, int $limit = 0, float $minHeight = 0, float $minWidth = 0) : array { $qb = $this->db->getQueryBuilder(); - $qb->select(FaceDetection::$columns) - ->from('recognize_face_detections') - ->where($qb->expr()->eq('user_id', $qb->createPositionalParameter($userId))) - ->andWhere($qb->expr()->gte('height', $qb->createPositionalParameter($minHeight))) - ->andWhere($qb->expr()->gte('width', $qb->createPositionalParameter($minWidth))) - ->andWhere($qb->expr()->isNull('cluster_id')); + $this->selectWithVector($qb) + ->where($qb->expr()->eq('d.user_id', $qb->createPositionalParameter($userId))) + ->andWhere($qb->expr()->gte('d.height', $qb->createPositionalParameter($minHeight))) + ->andWhere($qb->expr()->gte('d.width', $qb->createPositionalParameter($minWidth))) + ->andWhere($qb->expr()->isNull('d.cluster_id')); if ($limit > 0) { $qb->setMaxResults($limit); } @@ -285,11 +396,10 @@ public function findRejectedByUserId(string $userId) : array { */ public function findClusterSample(int $clusterId, int $n, int $minHeight = 0, int $minWidth = 0): array { $qb = $this->db->getQueryBuilder(); - $qb->select(FaceDetection::$columns) - ->from('recognize_face_detections', 'd') - ->where($qb->expr()->eq('cluster_id', $qb->createPositionalParameter($clusterId))) - ->andWhere($qb->expr()->gte('height', $qb->createPositionalParameter($minHeight))) - ->andWhere($qb->expr()->gte('width', $qb->createPositionalParameter($minWidth))) + $this->selectWithVector($qb) + ->where($qb->expr()->eq('d.cluster_id', $qb->createPositionalParameter($clusterId))) + ->andWhere($qb->expr()->gte('d.height', $qb->createPositionalParameter($minHeight))) + ->andWhere($qb->expr()->gte('d.width', $qb->createPositionalParameter($minWidth))) ->orderBy( $qb->createFunction( $this->config->getSystemValue('dbtype', 'sqlite') === 'mysql' @@ -311,12 +421,11 @@ public function findClusterSample(int $clusterId, int $n, int $minHeight = 0, in */ public function sampleRejectedDetectionsByUserId(string $userId, int $n, float $minHeight = 0, float $minWidth = 0): array { $qb = $this->db->getQueryBuilder(); - $qb->select(FaceDetection::$columns) - ->from('recognize_face_detections', 'd') - ->where($qb->expr()->eq('cluster_id', $qb->createPositionalParameter(-1))) - ->andWhere($qb->expr()->eq('user_id', $qb->createPositionalParameter($userId))) - ->andWhere($qb->expr()->gte('height', $qb->createPositionalParameter($minHeight))) - ->andWhere($qb->expr()->gte('width', $qb->createPositionalParameter($minWidth))) + $this->selectWithVector($qb) + ->where($qb->expr()->eq('d.cluster_id', $qb->createPositionalParameter(-1))) + ->andWhere($qb->expr()->eq('d.user_id', $qb->createPositionalParameter($userId))) + ->andWhere($qb->expr()->gte('d.height', $qb->createPositionalParameter($minHeight))) + ->andWhere($qb->expr()->gte('d.width', $qb->createPositionalParameter($minWidth))) ->orderBy( $qb->createFunction( $this->config->getSystemValue('dbtype', 'sqlite') === 'mysql' @@ -367,11 +476,13 @@ public function findDetectionForPreviewImageByClusterId(int $clusterId) : FaceDe public function countUnclustered(): int { $minDetectionSize = $this->getMinDetectionSize(); $qb = $this->db->getQueryBuilder(); - $qb->select($qb->func()->count('id')) - ->from('recognize_face_detections') - ->where($qb->expr()->isNull('cluster_id')) - ->andWhere($qb->expr()->gte('height', $qb->createPositionalParameter($minDetectionSize))) - ->andWhere($qb->expr()->gte('width', $qb->createPositionalParameter($minDetectionSize))); + // Only count detections that can actually be clustered, i.e. those with an embedding (see findUnclusteredByUserId) + $qb->select($qb->func()->count('d.id')) + ->from('recognize_face_detections', 'd') + ->innerJoin('d', 'recognize_face_vectors', 'e', $qb->expr()->eq('d.embedding_id', 'e.id')) + ->where($qb->expr()->isNull('d.cluster_id')) + ->andWhere($qb->expr()->gte('d.height', $qb->createPositionalParameter($minDetectionSize))) + ->andWhere($qb->expr()->gte('d.width', $qb->createPositionalParameter($minDetectionSize))); $result = $qb->executeQuery(); /** @var int|string $count */ $count = $result->fetch(\PDO::FETCH_COLUMN); @@ -386,11 +497,14 @@ public function countUnclustered(): int { public function getUsersForUnclustered(): array { $minDetectionSize = $this->getMinDetectionSize(); $qb = $this->db->getQueryBuilder(); - $qb->selectDistinct('user_id') - ->from('recognize_face_detections') - ->where($qb->expr()->isNull('cluster_id')) - ->andWhere($qb->expr()->gte('height', $qb->createPositionalParameter($minDetectionSize))) - ->andWhere($qb->expr()->gte('width', $qb->createPositionalParameter($minDetectionSize))); + // Only consider detections that can actually be clustered, i.e. those with an embedding (see findUnclusteredByUserId), + // otherwise ClusterFacesJob would be scheduled for these users forever + $qb->selectDistinct('d.user_id') + ->from('recognize_face_detections', 'd') + ->innerJoin('d', 'recognize_face_vectors', 'e', $qb->expr()->eq('d.embedding_id', 'e.id')) + ->where($qb->expr()->isNull('d.cluster_id')) + ->andWhere($qb->expr()->gte('d.height', $qb->createPositionalParameter($minDetectionSize))) + ->andWhere($qb->expr()->gte('d.width', $qb->createPositionalParameter($minDetectionSize))); $result = $qb->executeQuery(); /** @var array $users */ $users = $result->fetchAll(\PDO::FETCH_COLUMN); diff --git a/lib/Db/FaceDetectionWithTitle.php b/lib/Db/FaceDetectionWithTitle.php index 3a0b8a864..f0bb4e2dc 100644 --- a/lib/Db/FaceDetectionWithTitle.php +++ b/lib/Db/FaceDetectionWithTitle.php @@ -15,8 +15,8 @@ final class FaceDetectionWithTitle extends FaceDetection { protected $title; - public static $columns = ['id', 'user_id', 'file_id', 'x', 'y', 'height', 'width', 'face_vector', 'cluster_id', 'title']; - public static $fields = ['id', 'userId', 'fileId', 'x', 'y', 'height', 'width', 'faceVector', 'clusterId', 'title']; + public static $columns = ['id', 'user_id', 'file_id', 'x', 'y', 'height', 'width', 'embedding_id', 'cluster_id', 'title']; + public static $fields = ['id', 'userId', 'fileId', 'x', 'y', 'height', 'width', 'embeddingId', 'clusterId', 'title']; public function __construct() { parent::__construct(); diff --git a/lib/Db/FaceEmbedding.php b/lib/Db/FaceEmbedding.php new file mode 100644 index 000000000..fdcb279df --- /dev/null +++ b/lib/Db/FaceEmbedding.php @@ -0,0 +1,70 @@ +addType('fileId', 'integer'); + $this->addType('x', 'float'); + $this->addType('y', 'float'); + $this->addType('height', 'float'); + $this->addType('width', 'float'); + $this->addType('faceVector', 'json'); + } + + public function getVector(): array { + return $this->getter('faceVector'); + } + public function setVector(array $vector): void { + $this->setter('faceVector', [$vector]); + } + + /** + * Compares with a tolerance, as not all databases store doubles losslessly + */ + public function hasSameBoundingBox(FaceEmbedding $other): bool { + return abs($this->getX() - $other->getX()) < 1e-9 + && abs($this->getY() - $other->getY()) < 1e-9 + && abs($this->getHeight() - $other->getHeight()) < 1e-9 + && abs($this->getWidth() - $other->getWidth()) < 1e-9; + } +} diff --git a/lib/Db/FaceEmbeddingMapper.php b/lib/Db/FaceEmbeddingMapper.php new file mode 100644 index 000000000..dc6eba7cd --- /dev/null +++ b/lib/Db/FaceEmbeddingMapper.php @@ -0,0 +1,68 @@ + + */ +final class FaceEmbeddingMapper extends QBMapper { + public function __construct(IDBConnection $db) { + parent::__construct($db, 'recognize_face_vectors', FaceEmbedding::class); + $this->db = $db; + } + + /** + * @throws \OCP\DB\Exception + * @return list<\OCA\Recognize\Db\FaceEmbedding> + */ + public function findByFileId(int $fileId): array { + $qb = $this->db->getQueryBuilder(); + $qb->select(FaceEmbedding::$columns) + ->from('recognize_face_vectors') + ->where($qb->expr()->eq('file_id', $qb->createPositionalParameter($fileId, IQueryBuilder::PARAM_INT))); + return $this->findEntities($qb); + } + + /** + * @throws \OCP\DB\Exception + */ + public function countByFileId(int $fileId): int { + $qb = $this->db->getQueryBuilder(); + $qb->select($qb->func()->count('id')) + ->from('recognize_face_vectors') + ->where($qb->expr()->eq('file_id', $qb->createPositionalParameter($fileId, IQueryBuilder::PARAM_INT))); + $result = $qb->executeQuery(); + /** @var int|string $count */ + $count = $result->fetch(\PDO::FETCH_COLUMN); + $result->closeCursor(); + return (int)$count; + } + + /** + * @throws \OCP\DB\Exception + */ + public function deleteByFileId(int $fileId): void { + $qb = $this->db->getQueryBuilder(); + $qb->delete('recognize_face_vectors') + ->where($qb->expr()->eq('file_id', $qb->createPositionalParameter($fileId, IQueryBuilder::PARAM_INT))) + ->executeStatement(); + } + + /** + * @throws \OCP\DB\Exception + */ + public function deleteAll(): void { + $qb = $this->db->getQueryBuilder(); + $qb->delete('recognize_face_vectors') + ->executeStatement(); + } +} diff --git a/lib/Migration/Version013002000Date20261028094721.php b/lib/Migration/Version013002000Date20261028094721.php new file mode 100644 index 000000000..04c594469 --- /dev/null +++ b/lib/Migration/Version013002000Date20261028094721.php @@ -0,0 +1,80 @@ +hasTable('recognize_face_vectors')) { + $table = $schema->createTable('recognize_face_vectors'); + $table->addColumn('id', Types::BIGINT, [ + 'autoincrement' => true, + 'notnull' => true, + 'length' => 64, + ]); + $table->addColumn('file_id', Types::BIGINT, [ + 'notnull' => true, + 'length' => 64, + ]); + $table->addColumn('x', Types::FLOAT, [ + 'notnull' => false, + ]); + $table->addColumn('y', Types::FLOAT, [ + 'notnull' => false, + ]); + $table->addColumn('height', Types::FLOAT, [ + 'notnull' => false, + ]); + $table->addColumn('width', Types::FLOAT, [ + 'notnull' => false, + ]); + $table->addColumn('face_vector', Types::TEXT, [ + 'notnull' => true, + ]); + $table->setPrimaryKey(['id'], 'recognize_faceemb_id'); + $table->addIndex(['file_id'], 'recognize_faceemb_file'); + $changed = true; + } + + if ($schema->hasTable('recognize_face_detections')) { + $table = $schema->getTable('recognize_face_detections'); + if (!$table->hasColumn('embedding_id')) { + $table->addColumn('embedding_id', Types::BIGINT, [ + 'notnull' => false, + 'length' => 64, + ]); + $table->addIndex(['embedding_id'], 'recognize_facedet_embedding'); + $changed = true; + } + } + return $changed ? $schema : null; + } +} diff --git a/lib/Migration/Version013002000Date20261028094821.php b/lib/Migration/Version013002000Date20261028094821.php new file mode 100644 index 000000000..431339cc9 --- /dev/null +++ b/lib/Migration/Version013002000Date20261028094821.php @@ -0,0 +1,251 @@ +hasTable('recognize_face_detections') || !$schema->hasTable('recognize_face_vectors')) { + return; + } + $table = $schema->getTable('recognize_face_detections'); + if (!$table->hasColumn('face_vector') || !$table->hasColumn('embedding_id')) { + return; + } + + // Detections without a vector can't be clustered and can't be copied into the + // recognize_face_vectors table, where the vector is mandatory + $qb = $this->db->getQueryBuilder(); + $qb->delete('recognize_face_detections') + ->where($qb->expr()->isNull('face_vector')) + ->andWhere($qb->expr()->isNull('embedding_id')); + $deleted = $qb->executeStatement(); + if ($deleted > 0) { + $output->info('Removed ' . $deleted . ' face detections without a face vector'); + } + + // Detections without a file can't be shown or linked to an embedding, which belongs to a file, + // so they would lose their vector when the column is dropped and linger as unusable rows + $qb = $this->db->getQueryBuilder(); + $qb->delete('recognize_face_detections') + ->where($qb->expr()->isNull('file_id')) + ->andWhere($qb->expr()->isNull('embedding_id')); + $deleted = $qb->executeStatement(); + if ($deleted > 0) { + $output->info('Removed ' . $deleted . ' face detections without a file'); + } + + $qb = $this->db->getQueryBuilder(); + $qb->select($qb->func()->count('id')) + ->from('recognize_face_detections') + ->where($qb->expr()->isNull('embedding_id')); + $result = $qb->executeQuery(); + $total = (int)$result->fetchOne(); + $result->closeCursor(); + if ($total === 0) { + return; + } + + $output->info('Deduplicating ' . $total . ' face detections'); + $output->startProgress($total); + + $lastFileId = -1; + while (true) { + $qb = $this->db->getQueryBuilder(); + $qb->selectDistinct('file_id') + ->from('recognize_face_detections') + ->where($qb->expr()->isNull('embedding_id')) + ->andWhere($qb->expr()->gt('file_id', $qb->createNamedParameter($lastFileId, IQueryBuilder::PARAM_INT))) + ->orderBy('file_id') + ->setMaxResults(self::FILE_BATCH_SIZE); + $result = $qb->executeQuery(); + $fileIds = array_map('intval', $result->fetchAll(\PDO::FETCH_COLUMN)); + $result->closeCursor(); + if (count($fileIds) === 0) { + break; + } + $lastFileId = max($fileIds); + + $this->db->beginTransaction(); + try { + $output->advance($this->migrateFiles($fileIds)); + $this->db->commit(); + } catch (\Throwable $e) { + $this->db->rollBack(); + throw $e; + } + } + $output->finishProgress(); + } + + /** + * @param list $fileIds + * @return int The number of migrated detections + * @throws \OCP\DB\Exception + * @throws \RuntimeException If a face could not be linked to its embedding; this aborts the migration + * before the face_vector column is dropped, so no vectors are lost + */ + private function migrateFiles(array $fileIds): int { + // Group the detections of these files by face; all copies of a face share file and bounding box + $qb = $this->db->getQueryBuilder(); + $qb->select('id', 'file_id', 'x', 'y', 'height', 'width') + ->from('recognize_face_detections') + ->where($qb->expr()->isNull('embedding_id')) + ->andWhere($qb->expr()->in('file_id', $qb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY))) + ->orderBy('id'); + $result = $qb->executeQuery(); + /** @var array}> $faces */ + $faces = []; + $count = 0; + while ($row = $result->fetch()) { + $face = $this->normalizeFace($row); + $key = implode('|', $face); + $faces[$key] ??= ['face' => $face, 'detectionIds' => []]; + $faces[$key]['detectionIds'][] = (int)$row['id']; + $count++; + } + $result->closeCursor(); + + // Reuse embeddings created by a previous, interrupted run + $embeddingsByFile = $this->findEmbeddings($fileIds); + + // Copy one detection per face within the database, so neither the vectors nor + // the float coordinates have to make a lossy round trip through PHP + $copyQuery = 'INSERT INTO `*PREFIX*recognize_face_vectors` (`file_id`, `x`, `y`, `height`, `width`, `face_vector`) ' + . 'SELECT `file_id`, `x`, `y`, `height`, `width`, `face_vector` FROM `*PREFIX*recognize_face_detections` WHERE `id` = ?'; + $copied = false; + foreach ($faces as $face) { + if ($this->findMatchingEmbeddingId($face['face'], $embeddingsByFile) === null) { + $this->db->executeStatement($copyQuery, [$face['detectionIds'][0]], [IQueryBuilder::PARAM_INT]); + $copied = true; + // Placeholder until the ids are read back below, so a face within the tolerance isn't copied twice + $embeddingsByFile[$face['face']['file_id']][] = ['id' => 0, 'face' => $face['face']]; + } + } + // Read the ids back instead of using lastInsertId(), which fails on Oracle for table + // names longer than the sequence names Doctrine generates + if ($copied) { + $embeddingsByFile = $this->findEmbeddings($fileIds); + } + + foreach ($faces as $face) { + $embeddingId = $this->findMatchingEmbeddingId($face['face'], $embeddingsByFile); + if ($embeddingId === null) { + throw new \RuntimeException('Could not find the copied face embedding for file ' . $face['face']['file_id'] . ' (bounding box ' . implode(', ', array_slice($face['face'], 1)) . ')'); + } + foreach (array_chunk($face['detectionIds'], self::QUERY_CHUNK_SIZE) as $chunk) { + $qb = $this->db->getQueryBuilder(); + $qb->update('recognize_face_detections') + ->set('embedding_id', $qb->createNamedParameter($embeddingId, IQueryBuilder::PARAM_INT)) + ->where($qb->expr()->in('id', $qb->createNamedParameter($chunk, IQueryBuilder::PARAM_INT_ARRAY))); + $qb->executeStatement(); + } + } + + return $count; + } + + /** + * @param list $fileIds + * @return array> The embeddings by file id + * @throws \OCP\DB\Exception + */ + private function findEmbeddings(array $fileIds): array { + $qb = $this->db->getQueryBuilder(); + $qb->select('id', 'file_id', 'x', 'y', 'height', 'width') + ->from('recognize_face_vectors') + ->where($qb->expr()->in('file_id', $qb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY))); + $result = $qb->executeQuery(); + $embeddingsByFile = []; + while ($row = $result->fetch()) { + $face = $this->normalizeFace($row); + $embeddingsByFile[$face['file_id']][] = ['id' => (int)$row['id'], 'face' => $face]; + } + $result->closeCursor(); + return $embeddingsByFile; + } + + /** + * Compares the coordinates with a tolerance instead of exactly, as the copied floats + * are not guaranteed to be read back with the exact same representation on all databases + * + * @param array{file_id: int, x: float, y: float, height: float, width: float} $face + * @param array> $embeddingsByFile + */ + private function findMatchingEmbeddingId(array $face, array $embeddingsByFile): ?int { + foreach ($embeddingsByFile[$face['file_id']] ?? [] as $embedding) { + if (abs($embedding['face']['x'] - $face['x']) < self::COORDINATE_TOLERANCE + && abs($embedding['face']['y'] - $face['y']) < self::COORDINATE_TOLERANCE + && abs($embedding['face']['height'] - $face['height']) < self::COORDINATE_TOLERANCE + && abs($embedding['face']['width'] - $face['width']) < self::COORDINATE_TOLERANCE) { + return $embedding['id']; + } + } + return null; + } + + /** + * @param array $row + * @return array{file_id: int, x: float, y: float, height: float, width: float} + */ + private function normalizeFace(array $row): array { + return [ + 'file_id' => (int)$row['file_id'], + 'x' => (float)$row['x'], + 'y' => (float)$row['y'], + 'height' => (float)$row['height'], + 'width' => (float)$row['width'], + ]; + } + + /** + * @param IOutput $output + * @param Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper` + * @param array $options + * + * @return ?ISchemaWrapper + * @throws SchemaException + */ + public function changeSchema(IOutput $output, Closure $schemaClosure, array $options) { + /** @var ISchemaWrapper $schema */ + $schema = $schemaClosure(); + + if ($schema->hasTable('recognize_face_detections')) { + $table = $schema->getTable('recognize_face_detections'); + if ($table->hasColumn('face_vector') && $table->hasColumn('embedding_id')) { + $table->dropColumn('face_vector'); + return $schema; + } + } + return null; + } +} diff --git a/lib/Service/FsActionService.php b/lib/Service/FsActionService.php index 23bd2b643..2fa470f58 100644 --- a/lib/Service/FsActionService.php +++ b/lib/Service/FsActionService.php @@ -125,7 +125,7 @@ public function processActions(array $actions): void { break; } try { - $this->onMove($action->getOwner(), $action->getAddedUsers(), $action->getTargetUsers(), $node); + $this->onMove($action->getAddedUsers(), $action->getTargetUsers(), $node); } catch (Exception|InvalidPathException|NotFoundException $e) { $this->logger->warning('Failed to process move action: ' . $e->getMessage() . ' Continuing.', ['exception' => $e]); } @@ -168,24 +168,7 @@ private function onAccessUpdate(int $storageId, int $rootId): void { $files = $this->storageService->getFilesInMount($storageId, $rootId, [ClusteringFaceClassifier::MODEL_NAME], 0, 0); $userIdsToScheduleClustering = []; foreach ($files as $fileInfo) { - $node = $this->rootFolder->getFirstNodeById($fileInfo['fileid']) ?: null; - $ownerId = $node?->getOwner()?->getUID(); - if ($ownerId === null) { - continue; - } - $detectionsForFile = $this->faceDetectionMapper->findByFileId($fileInfo['fileid']); - $userHasDetectionForFile = []; - foreach ($detectionsForFile as $detection) { - $userHasDetectionForFile[$detection->getUserId()] = true; - } - foreach ($userIds as $userId) { - if ($userId === $ownerId) { - continue; - } - if ($userHasDetectionForFile[$userId] ?? false) { - continue; - } - $this->faceDetectionMapper->copyDetectionsForFileFromUserToUser($fileInfo['fileid'], $ownerId, $userId); + foreach ($this->faceDetectionMapper->addUsersToFile($fileInfo['fileid'], $userIds) as $userId) { $userIdsToScheduleClustering[$userId] = true; } $this->faceDetectionMapper->removeDetectionsForFileFromUsersNotInList($fileInfo['fileid'], $userIds); @@ -304,14 +287,7 @@ public function isFileIgnored(Node $node) : bool { public function onDeletion(int $nodeId, ?array $mimeTypes = null): void { // Try Deleting possibly existing face detections try { - /** - * @var \OCA\Recognize\Db\FaceDetection[] $faceDetections - */ - $faceDetections = $this->faceDetectionMapper->findByFileId($nodeId); - foreach ($faceDetections as $detection) { - $this->logger->debug('Delete face detection ' . $detection->getId()); - $this->faceDetectionMapper->delete($detection); - } + $this->faceDetectionMapper->deleteByFileId($nodeId); } catch (Exception $e) { $this->logger->warning($e->getMessage(), ['exception' => $e]); } @@ -325,32 +301,27 @@ public function onDeletion(int $nodeId, ?array $mimeTypes = null): void { } /** - * @param string $ownerId * @param list $usersToAdd * @param list $targetUserIds * @param Node $node * @return void * @throws Exception|InvalidPathException|NotFoundException */ - private function onMove(string $ownerId, array $usersToAdd, array $targetUserIds, Node $node): void { + private function onMove(array $usersToAdd, array $targetUserIds, Node $node): void { if ($node instanceof Folder) { try { foreach ($node->getDirectoryListing() as $n) { if (!in_array($n->getMimetype(), Constants::IMAGE_FORMATS)) { continue; } - $this->onMove($ownerId, $usersToAdd, $targetUserIds, $n); + $this->onMove($usersToAdd, $targetUserIds, $n); } } catch (NotFoundException|Exception|InvalidPathException $e) { $this->logger->warning('Error in recognize file listener', ['exception' => $e]); } return; } - foreach ($usersToAdd as $userId) { - if (count($this->faceDetectionMapper->findByFileIdAndUser($node->getId(), $userId)) > 0) { - continue; - } - $this->faceDetectionMapper->copyDetectionsForFileFromUserToUser($node->getId(), $ownerId, $userId); + foreach ($this->faceDetectionMapper->addUsersToFile($node->getId(), $usersToAdd) as $userId) { $this->jobList->add(ClusterFacesJob::class, ['userId' => $userId]); } $this->faceDetectionMapper->removeDetectionsForFileFromUsersNotInList($node->getId(), $targetUserIds); diff --git a/lib/TaskProcessing/TaskResultListener.php b/lib/TaskProcessing/TaskResultListener.php index bfcad9f6f..343be64f1 100644 --- a/lib/TaskProcessing/TaskResultListener.php +++ b/lib/TaskProcessing/TaskResultListener.php @@ -19,6 +19,8 @@ use OCA\Recognize\Classifiers\Video\MovinetClassifier; use OCA\Recognize\Db\FaceDetection; use OCA\Recognize\Db\FaceDetectionMapper; +use OCA\Recognize\Db\FaceEmbedding; +use OCA\Recognize\Db\FaceEmbeddingMapper; use OCA\Recognize\Db\QueueFile; use OCA\Recognize\Service\QueueService; use OCA\Recognize\Service\TagManager; @@ -28,6 +30,7 @@ use OCP\EventDispatcher\IEventListener; use OCP\Files\Config\ICachedMountInfo; use OCP\Files\Config\IUserMountCache; +use OCP\IDBConnection; use OCP\IUserManager; use OCP\IUserSession; use OCP\TaskProcessing\Events\TaskFailedEvent; @@ -39,7 +42,7 @@ * Stores results of TaskProcessing tasks scheduled by this app: * * - image / video / audio classification → assigns category tags via {@see TagManager} - * - face recognition → inserts {@see FaceDetection} rows and schedules a {@see ClusterFacesJob} + * - face recognition → inserts a {@see FaceEmbedding} per face, a {@see FaceDetection} per face and user with access, and schedules a {@see ClusterFacesJob} * * @template-implements IEventListener */ @@ -48,12 +51,14 @@ public function __construct( private LoggerInterface $logger, private TagManager $tagManager, private FaceDetectionMapper $faceDetections, + private FaceEmbeddingMapper $faceEmbeddings, private IUserMountCache $userMountCache, private IAppConfig $config, private IJobList $jobList, private QueueService $queue, private IUserSession $userSession, private IUserManager $userManager, + private IDBConnection $db, ) { } @@ -198,59 +203,109 @@ private function applyFaceResults(array $fileIds, array $results): void { if (!isset($results[$i])) { continue; } - $raw = (string)$results[$i]; $userIds = $this->getUsersWithFileAccess($fileId); - foreach (explode("\n", $raw) as $line) { - $line = trim($line); - if ($line === '') { - continue; - } - try { - $face = json_decode($line, true, 512, JSON_THROW_ON_ERROR); - } catch (\JsonException $e) { - $this->logger->warning('Invalid face JSON for file ' . $fileId, ['exception' => $e]); - continue; - } - if (!is_array($face)) { - continue; - } - if (isset($face['score']) && (float)$face['score'] < ImageFaceRecognitionClassifier::MIN_FACE_RECOGNITION_SCORE) { - continue; - } - // Accept either a full face object {x,y,width,height,score,vector,angle} - // or a bare embedding vector (list of numbers). - $isBareVector = array_is_list($face) && count($face) > 0 && is_numeric($face[0]); - $vector = $isBareVector ? $face : ($face['vector'] ?? null); - if (!is_array($vector)) { - $this->logger->warning('Face entry without embedding vector for file ' . $fileId); - continue; + $faces = $this->parseFaces($fileId, (string)$results[$i]); + + // Store the faces of a file all at once or not at all, so a failure can't leave + // a face stored for only some of the users with access + /** @var array $usersWithNewDetections */ + $usersWithNewDetections = []; + $this->db->beginTransaction(); + try { + // The file may have been processed before (e.g. it was edited and queued again), + // so reuse faces that are already stored instead of storing them a second time + $existingEmbeddings = $this->faceEmbeddings->findByFileId($fileId); + $hasDetection = []; + foreach ($this->faceDetections->findByFileId($fileId) as $detection) { + $hasDetection[$detection->getUserId()][(int)$detection->getEmbeddingId()] = true; } - foreach ($userIds as $userId) { - $detection = new FaceDetection(); - $detection->setFileId($fileId); - $detection->setUserId($userId); - $detection->setX((float)($face['x'] ?? 0)); - $detection->setY((float)($face['y'] ?? 0)); - $detection->setWidth((float)($face['width'] ?? 0)); - $detection->setHeight((float)($face['height'] ?? 0)); - $detection->setVector($vector); - try { - $this->faceDetections->insert($detection); - } catch (\Throwable $e) { - $this->logger->error('Could not store face detection for file ' . $fileId, ['exception' => $e]); - continue; + + foreach ($faces as $embedding) { + $existingEmbedding = null; + foreach ($existingEmbeddings as $candidate) { + if ($candidate->hasSameBoundingBox($embedding)) { + $existingEmbedding = $candidate; + break; + } } - if (!isset($scheduledClusterJobsFor[$userId])) { - $this->jobList->add(ClusterFacesJob::class, ['userId' => $userId]); - $scheduledClusterJobsFor[$userId] = true; + if ($existingEmbedding !== null) { + $embedding = $existingEmbedding; + } else { + $this->faceEmbeddings->insert($embedding); + $existingEmbeddings[] = $embedding; + } + foreach ($userIds as $userId) { + if ($hasDetection[$userId][$embedding->getId()] ?? false) { + continue; + } + $hasDetection[$userId][$embedding->getId()] = true; + $this->faceDetections->insertForUser($embedding, $userId); + $usersWithNewDetections[$userId] = true; } } + $this->db->commit(); + } catch (\Throwable $e) { + $this->db->rollBack(); + $this->logger->error('Could not store faces of file ' . $fileId . ' in database', ['exception' => $e]); + continue; + } + + foreach (array_keys($usersWithNewDetections) as $userId) { + $userId = (string)$userId; + if (!isset($scheduledClusterJobsFor[$userId])) { + $this->jobList->add(ClusterFacesJob::class, ['userId' => $userId]); + $scheduledClusterJobsFor[$userId] = true; + } } $this->config->setAppValueString($model . '.status', 'true', lazy: true); $this->config->setAppValueString($model . '.lastFile', (string)time(), lazy: true); } } + /** + * Parses the faces from the output of a face recognition task, one JSON object per line + * + * @return list + */ + private function parseFaces(int $fileId, string $raw): array { + $faces = []; + foreach (explode("\n", $raw) as $line) { + $line = trim($line); + if ($line === '') { + continue; + } + try { + $face = json_decode($line, true, 512, JSON_THROW_ON_ERROR); + } catch (\JsonException $e) { + $this->logger->warning('Invalid face JSON for file ' . $fileId, ['exception' => $e]); + continue; + } + if (!is_array($face)) { + continue; + } + if (isset($face['score']) && (float)$face['score'] < ImageFaceRecognitionClassifier::MIN_FACE_RECOGNITION_SCORE) { + continue; + } + // Accept either a full face object {x,y,width,height,score,vector,angle} + // or a bare embedding vector (list of numbers). + $isBareVector = array_is_list($face) && count($face) > 0 && is_numeric($face[0]); + $vector = $isBareVector ? $face : ($face['vector'] ?? null); + if (!is_array($vector)) { + $this->logger->warning('Face entry without embedding vector for file ' . $fileId); + continue; + } + $embedding = new FaceEmbedding(); + $embedding->setFileId($fileId); + $embedding->setX((float)($face['x'] ?? 0)); + $embedding->setY((float)($face['y'] ?? 0)); + $embedding->setWidth((float)($face['width'] ?? 0)); + $embedding->setHeight((float)($face['height'] ?? 0)); + $embedding->setVector($vector); + $faces[] = $embedding; + } + return $faces; + } + private function enqueueForLandmarks(int $fileId): void { $mounts = $this->userMountCache->getMountsForFileId($fileId); if (count($mounts) === 0) { diff --git a/psalm-baseline.xml b/psalm-baseline.xml index 40afa6065..6e2a7f46d 100644 --- a/psalm-baseline.xml +++ b/psalm-baseline.xml @@ -613,9 +613,6 @@ - - faceDetection->getVector()]]> - getFile()->getSize()]]> @@ -688,6 +685,7 @@ + @@ -715,6 +713,22 @@ + + + + + + + + + + + getter('faceVector')]]> + + + + + diff --git a/tests/ClusterTest.php b/tests/ClusterTest.php index 8d31bcbad..49d6d2672 100644 --- a/tests/ClusterTest.php +++ b/tests/ClusterTest.php @@ -47,7 +47,8 @@ public function setUp(): void { } $detections = $this->faceDetectionMapper->findByUserId(self::TEST_USER1); foreach ($detections as $detection) { - $this->faceDetectionMapper->delete($detection); + // Also removes the embeddings, so inserted detections don't get linked to the vectors of a previous test + $this->faceDetectionMapper->deleteByFileId($detection->getFileId()); } // Generate artificial clusters along just one dimension @@ -116,7 +117,7 @@ public function testClusterThresholds() { $detections = $this->faceDetectionMapper->findByClusterId($clusters[0]->getId()); self::assertCount(self::INITIAL_DETECTIONS_PER_CLUSTER, $detections); - $detections = $this->faceDetectionMapper->findByClusterId($clusters[1]->getId()); + $detections = $this->faceDetectionMapper->findByClusterIdWithVector($clusters[1]->getId()); self::assertCount(self::INITIAL_DETECTIONS_PER_CLUSTER, $detections); $detectionToRemove = $detections[0]; @@ -248,7 +249,7 @@ public function testClusterTemptClusterMerging() { $vector = self::getNullVector(); $vector[0] = $clusterValue + 0.001 * $i; $detection->setVector($vector); - $detection->setFileId($i); + $detection->setFileId(2 * self::INITIAL_DETECTIONS_PER_CLUSTER + $i); $detection->setHeight(0.5); $detection->setWidth(0.5); $this->faceDetectionMapper->insert($detection); @@ -277,7 +278,7 @@ public function testClusterTemptClusterMerging() { $vector = self::getNullVector(); $vector[0] = $clusterValue + 0.001 * $i; $detection->setVector($vector); - $detection->setFileId($i); + $detection->setFileId(3 * self::INITIAL_DETECTIONS_PER_CLUSTER + $i); $detection->setHeight(0.5); $detection->setWidth(0.5); $this->faceDetectionMapper->insert($detection);