Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
93 changes: 56 additions & 37 deletions lib/Classifiers/Images/ClusteringFaceClassifier.php
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,9 @@

use OCA\Recognize\BackgroundJobs\ClusterFacesJob;
use OCA\Recognize\Classifiers\Classifier;
use OCA\Recognize\Db\FaceDetection;
use OCA\Recognize\Db\FaceDetectionMapper;
use OCA\Recognize\Db\FaceEmbedding;
use OCA\Recognize\Db\FaceEmbeddingMapper;
use OCA\Recognize\Service\Logger;
use OCA\Recognize\Service\QueueService;
use OCP\AppFramework\Services\IAppConfig;
Expand All @@ -22,6 +23,7 @@
use OCP\Files\IRootFolder;
use OCP\Files\Node;
use OCP\Files\NotFoundException;
use OCP\IDBConnection;
use OCP\IPreview;
use OCP\ITempManager;
use OCP\Share\IManager;
Expand All @@ -41,9 +43,11 @@ public function __construct(
Logger $logger,
IAppConfig $config,
private FaceDetectionMapper $faceDetections,
private FaceEmbeddingMapper $faceEmbeddings,
QueueService $queue, IRootFolder $rootFolder,
private IUserMountCache $userMountCache, private IJobList $jobList, ITempManager $tempManager, IPreview $previewProvider,
private IManager $shareManager) {
private IManager $shareManager,
private IDBConnection $db) {
parent::__construct($logger, $config, $rootFolder, $queue, $tempManager, $previewProvider);
}

Expand Down Expand Up @@ -73,7 +77,7 @@ public function classify(array $queueFiles): void {
$filteredQueueFiles = [];
foreach ($queueFiles as $queueFile) {
try {
$facesByFileCount = count($this->faceDetections->findByFileId($queueFile->getFileId()));
$facesByFileCount = $this->faceEmbeddings->countByFileId($queueFile->getFileId());
} catch (Exception $e) {
$this->logger->debug('finding faces by file '.$queueFile->getFileId().' failed', ['exception' => $e]);
$facesByFileCount = 1;
Expand All @@ -97,46 +101,61 @@ public function classify(array $queueFiles): void {
*/
foreach ($classifierProcess as $queueFile => $faces) {
$this->logger->debug('Face results for ' . $queueFile->getFileId() . ' are in');
foreach ($faces as $face) {
if ($face['score'] < self::MIN_FACE_RECOGNITION_SCORE) {
$this->logger->debug('Face score too low. continuing with next face.');
continue;
}
if (abs($face['angle']['roll']) > self::MAX_FACE_ROLL || abs($face['angle']['yaw']) > self::MAX_FACE_YAW) {
$this->logger->debug('Face is not straight. continuing with next face.');
continue;
}

try {
$node = $this->rootFolder->getFirstNodeById($queueFile->getFileId());
$userIds = $node !== null ? $this->getUsersWithFileAccess($node) : [];
} catch (InvalidPathException|NotFoundException $e) {
$userIds = [];
}
try {
$node = $this->rootFolder->getFirstNodeById($queueFile->getFileId());
$userIds = $node !== null ? $this->getUsersWithFileAccess($node) : [];
} catch (InvalidPathException|NotFoundException $e) {
$userIds = [];
}

// Insert face detection for all users with access
foreach ($userIds as $userId) {
$this->logger->debug('preparing face detection for user ' . $userId);
$faceDetection = new FaceDetection();
$faceDetection->setX($face['x']);
$faceDetection->setY($face['y']);
$faceDetection->setWidth($face['width']);
$faceDetection->setHeight($face['height']);
$faceDetection->setVector($face['vector']);
$faceDetection->setFileId($queueFile->getFileId());
$faceDetection->setUserId($userId);
try {
$this->faceDetections->insert($faceDetection);
} catch (\Throwable $e) {
$this->logger->error('Could not store face detection in database', ['exception' => $e]);
// Store the faces of a file all at once or not at all: files that already have faces are
// never analysed again, so a partially stored file would stay incomplete forever
$storedFaces = 0;
$this->db->beginTransaction();
try {
foreach ($faces as $face) {
if ($face['score'] < self::MIN_FACE_RECOGNITION_SCORE) {
$this->logger->debug('Face score too low. continuing with next face.');
continue;
}
if (abs($face['angle']['roll']) > self::MAX_FACE_ROLL || abs($face['angle']['yaw']) > self::MAX_FACE_YAW) {
$this->logger->debug('Face is not straight. continuing with next face.');
continue;
}
$this->logger->debug('scheduling ClusterFacesJob for user ' . $userId);
$this->jobList->add(ClusterFacesJob::class, ['userId' => $userId]);

$embedding = new FaceEmbedding();
$embedding->setX($face['x']);
$embedding->setY($face['y']);
$embedding->setWidth($face['width']);
$embedding->setHeight($face['height']);
$embedding->setVector($face['vector']);
$embedding->setFileId($queueFile->getFileId());
$this->faceEmbeddings->insert($embedding);

// Insert face detection for all users with access
foreach ($userIds as $userId) {
$this->logger->debug('preparing face detection for user ' . $userId);
$this->faceDetections->insertForUser($embedding, $userId);
}
$storedFaces++;
}
$this->config->setAppValueString(self::MODEL_NAME . '.status', 'true', lazy: true);
$this->config->setAppValueString(self::MODEL_NAME . '.lastFile', (string)time(), lazy: true);
$this->db->commit();
} catch (\Throwable $e) {
$this->db->rollBack();
$this->logger->error('Could not store faces of file ' . $queueFile->getFileId() . ' in database, discarding them so the file can be analysed again', ['exception' => $e]);
continue;
}

if ($storedFaces === 0) {
continue;
}
foreach ($userIds as $userId) {
$this->logger->debug('scheduling ClusterFacesJob for user ' . $userId);
$this->jobList->add(ClusterFacesJob::class, ['userId' => $userId]);
}
$this->config->setAppValueString(self::MODEL_NAME . '.status', 'true', lazy: true);
$this->config->setAppValueString(self::MODEL_NAME . '.lastFile', (string)time(), lazy: true);
}
$this->logger->debug('face classifier end');
}
Expand Down
10 changes: 7 additions & 3 deletions lib/Dav/Faces/FacePhoto.php
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,15 @@ public function getName(): string {

#[Override]
public function delete(): void {
$detections = $this->detectionMapper->findByClusterId($this->faceDetection->getClusterId());
if (count($detections) > 1) {
$detections = $this->detectionMapper->findByClusterIdWithVector($this->faceDetection->getClusterId());
// $this->faceDetection was loaded without its vector, so take it from the cluster's detections
$thisDetection = current(array_filter($detections, fn (FaceDetection $detection) => $detection->getId() === $this->faceDetection->getId()));
if (count($detections) > 1 && $thisDetection !== false) {
$centroid = FaceClusterAnalyzer::calculateCentroidOfDetections($detections);
$distance = new Euclidean();
$distanceValue = $distance->compute($centroid, $this->faceDetection->getVector());
/** @var list<float> $vector */
$vector = $thisDetection->getVector();
$distanceValue = $distance->compute($centroid, $vector);
// Set threshold to avoid recreating the same mistake
$this->faceDetection->setThreshold($distanceValue);
}
Expand Down
29 changes: 26 additions & 3 deletions lib/Db/FaceDetection.php
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,11 @@
* @method int|null getClusterId()
* @method float getThreshold()
* @method setThreshold(float $threshold)
* @method int|null getEmbeddingId()
* @method setEmbeddingId(int $embeddingId)
*
* The face vector is stored once per file in {@see FaceEmbedding} and only populated
* on detections loaded with {@see FaceDetectionMapper} methods that join it in.
*/
class FaceDetection extends Entity {
protected $fileId;
Expand All @@ -37,17 +42,18 @@ class FaceDetection extends Entity {
protected $y;
protected $height;
protected $width;
protected $embeddingId;
protected $faceVector;
protected $clusterId;
protected $threshold;
/**
* @var string[]
*/
public static $columns = ['id', 'user_id', 'file_id', 'x', 'y', 'height', 'width', 'face_vector', 'cluster_id', 'threshold'];
public static $columns = ['id', 'user_id', 'file_id', 'x', 'y', 'height', 'width', 'embedding_id', 'cluster_id', 'threshold'];
/**
* @var string[]
*/
public static $fields = ['id', 'userId', 'fileId', 'x', 'y', 'height', 'width', 'faceVector', 'clusterId', 'threshold'];
public static $fields = ['id', 'userId', 'fileId', 'x', 'y', 'height', 'width', 'embeddingId', 'clusterId', 'threshold'];

public function __construct() {
// add types in constructor
Expand All @@ -57,6 +63,7 @@ public function __construct() {
$this->addType('y', 'float');
$this->addType('height', 'float');
$this->addType('width', 'float');
$this->addType('embeddingId', 'integer');
$this->addType('faceVector', 'json');
$this->addType('clusterId', 'integer');
$this->addType('threshold', 'float');
Expand All @@ -65,7 +72,7 @@ public function __construct() {
public function toArray(): array {
$array = [];
foreach (static::$fields as $field) {
if ($field === 'faceVector') {
if ($field === 'embeddingId') {
continue;
}
$array[$field] = $this->{$field};
Expand All @@ -76,7 +83,23 @@ public function toArray(): array {
public function getVector(): array {
return $this->getter('faceVector');
}

/**
* Whether the vector is set; most queries don't load it, as it is stored with the embedding
*/
public function hasVector(): bool {
return is_array($this->faceVector);
}
public function setVector(array $vector): void {
$this->setter('faceVector', [$vector]);
}

/**
* The vector is not a column of the detections table
*/
public function getUpdatedFields(): array {
$fields = parent::getUpdatedFields();
unset($fields['faceVector']);
return $fields;
}
}
Loading
Loading