diff --git a/.github/workflows/cluster-faces-test.yml b/.github/workflows/cluster-faces-test.yml index 0885ef9c..e0bc84be 100644 --- a/.github/workflows/cluster-faces-test.yml +++ b/.github/workflows/cluster-faces-test.yml @@ -25,7 +25,7 @@ jobs: matrix: php-versions: ['8.2'] databases: ['sqlite'] - server-versions: ['master'] + server-versions: ['stable30'] pure-js-mode: ['false'] name: Test cluster-faces command on ${{ matrix.server-versions }} wasm:${{ matrix.pure-js-mode }} diff --git a/.github/workflows/files-scan-test.yml b/.github/workflows/files-scan-test.yml index af7a5e91..4ecabeea 100644 --- a/.github/workflows/files-scan-test.yml +++ b/.github/workflows/files-scan-test.yml @@ -25,7 +25,7 @@ jobs: matrix: php-versions: ['8.2'] databases: ['sqlite', 'mysql', 'pgsql'] - server-versions: ['master'] + server-versions: ['stable30'] name: Test files:scan command on ${{ matrix.databases }}-${{ matrix.server-versions }} diff --git a/.github/workflows/full-run-test.yml b/.github/workflows/full-run-test.yml index a999d0e9..0a39561d 100644 --- a/.github/workflows/full-run-test.yml +++ b/.github/workflows/full-run-test.yml @@ -30,7 +30,7 @@ jobs: matrix: php-versions: ['8.2'] databases: ['sqlite'] - server-versions: ['master'] + server-versions: ['stable30'] pure-js-mode: ['false'] imagenet-enabled: ['true'] faces-enabled: ['true'] @@ -39,7 +39,7 @@ jobs: movinet-enabled: ['true'] include: # test pure-js once - - server-versions: master + - server-versions: stable30 databases: sqlite php-versions: 8.2 pure-js-mode: true diff --git a/.github/workflows/phpunit-mysql.yml b/.github/workflows/phpunit-mysql.yml index 66f1ffc3..b66526a2 100644 --- a/.github/workflows/phpunit-mysql.yml +++ b/.github/workflows/phpunit-mysql.yml @@ -40,7 +40,7 @@ jobs: strategy: matrix: php-versions: ['8.1', '8.2', '8.3'] - server-versions: ['master'] + server-versions: ['stable30'] services: mysql: diff --git a/.github/workflows/phpunit-pgsql.yml b/.github/workflows/phpunit-pgsql.yml index 4fabd638..48c42b11 100644 --- a/.github/workflows/phpunit-pgsql.yml +++ b/.github/workflows/phpunit-pgsql.yml @@ -41,7 +41,7 @@ jobs: fail-fast: false matrix: php-versions: ['8.2'] - server-versions: ['master'] + server-versions: ['stable30'] services: postgres: diff --git a/.github/workflows/phpunit-sqlite.yml b/.github/workflows/phpunit-sqlite.yml index c9a4bafe..d04b86cf 100644 --- a/.github/workflows/phpunit-sqlite.yml +++ b/.github/workflows/phpunit-sqlite.yml @@ -42,7 +42,7 @@ jobs: fail-fast: false matrix: php-versions: ['8.2'] - server-versions: ['master'] + server-versions: ['stable30'] steps: - name: Set app env diff --git a/.github/workflows/psalm.yml b/.github/workflows/psalm.yml index 3f4f0fb4..946f545f 100644 --- a/.github/workflows/psalm.yml +++ b/.github/workflows/psalm.yml @@ -31,7 +31,7 @@ jobs: strategy: matrix: php-versions: [ '8.1', '8.2', '8.3' ] - server-versions: [ 'dev-master' ] + server-versions: [ 'dev-stable30' ] fail-fast: false name: Nextcloud diff --git a/composer.json b/composer.json index 0c87f648..2fe8f7bd 100644 --- a/composer.json +++ b/composer.json @@ -12,7 +12,7 @@ } }, "require-dev": { - "nextcloud/ocp": "dev-master", + "nextcloud/ocp": "dev-stable30", "symfony/console": "^5.4", "symfony/process": "^5.2" }, diff --git a/lib/Hooks/FileListener.php b/lib/Hooks/FileListener.php index 54284016..459951c2 100644 --- a/lib/Hooks/FileListener.php +++ b/lib/Hooks/FileListener.php @@ -47,6 +47,7 @@ */ class FileListener implements IEventListener { private ?bool $movingFromIgnoredTerritory; + private ?array $movingDirFromIgnoredTerritory; /** @var list */ private array $sourceUserIds; private ?Node $source = null; @@ -64,6 +65,7 @@ public function __construct( private IJobList $jobList, ) { $this->movingFromIgnoredTerritory = null; + $this->movingDirFromIgnoredTerritory = null; $this->sourceUserIds = []; } @@ -139,16 +141,22 @@ public function handle(Event $event): void { } } if ($event instanceof BeforeNodeRenamedEvent) { + $this->movingFromIgnoredTerritory = null; + $this->movingDirFromIgnoredTerritory = []; if (in_array($event->getSource()->getName(), [...Constants::IGNORE_MARKERS_ALL, ...Constants::IGNORE_MARKERS_IMAGE, ...Constants::IGNORE_MARKERS_AUDIO, ...Constants::IGNORE_MARKERS_VIDEO], true)) { $this->resetIgnoreCache($event->getSource()); return; } // We try to remember whether the source node is in ignored territory // because after moving isIgnored doesn't work anymore :( - if ($this->isIgnored($event->getSource())) { - $this->movingFromIgnoredTerritory = true; + if ($event->getSource()->getType() !== FileInfo::TYPE_FOLDER) { + if ($this->isFileIgnored($event->getSource())) { + $this->movingFromIgnoredTerritory = true; + } else { + $this->movingFromIgnoredTerritory = false; + } } else { - $this->movingFromIgnoredTerritory = false; + $this->movingDirFromIgnoredTerritory = $this->getDirIgnores($event->getSource()); } $this->sourceUserIds = $this->getUsersWithFileAccess($event->getSource()); $this->source = $event->getSource(); @@ -175,23 +183,45 @@ public function handle(Event $event): void { $this->postDelete($event->getTarget()->getParent()); return; } - - if ($this->movingFromIgnoredTerritory) { - if ($this->isIgnored($event->getTarget())) { + if ($event->getTarget()->getType() !== FileInfo::TYPE_FOLDER) { + if ($this->movingFromIgnoredTerritory) { + if ($this->isFileIgnored($event->getTarget())) { + return; + } + $this->postInsert($event->getTarget()); + return; + } + if ($this->isFileIgnored($event->getTarget())) { + $this->postDelete($event->getTarget()); + return; + } + } else { + if ($this->movingDirFromIgnoredTerritory !== null && count($this->movingDirFromIgnoredTerritory) !== 0) { + $oldIgnores = $this->movingDirFromIgnoredTerritory; + $newIgnores = $this->getDirIgnores($event->getTarget()); + $diff1 = array_diff($newIgnores, $oldIgnores); + $diff2 = array_diff($oldIgnores, $newIgnores); + if (count($diff1) !== 0 || count($diff2) !== 0) { + if (count($diff1) !== 0) { + $this->postDelete($event->getTarget(), true, $diff1); + } + if (count($diff2) !== 0) { + $this->postInsert($event->getTarget(), true, $diff2); + } + } + return; + } + $ignoredMimeTypes = $this->getDirIgnores($event->getTarget()); + if (!empty($ignoredMimeTypes)) { + $this->postDelete($event->getTarget(), true, $ignoredMimeTypes); return; } - $this->postInsert($event->getTarget()); - return; - } - if ($this->isIgnored($event->getTarget())) { - $this->postDelete($event->getTarget()); - return; } $this->postRename($this->source ?? $event->getSource(), $event->getTarget()); return; } if ($event instanceof BeforeNodeDeletedEvent) { - $this->postDelete($event->getNode(), false); + $this->postDelete($event->getNode()); return; } if ($event instanceof NodeDeletedEvent) { @@ -246,7 +276,7 @@ public function handle(Event $event): void { } } - public function postDelete(Node $node, bool $recurse = true): void { + public function postDelete(Node $node, bool $recurse = true, ?array $mimeTypes = null): void { if ($node->getType() === FileInfo::TYPE_FOLDER) { if (!$recurse) { return; @@ -254,7 +284,7 @@ public function postDelete(Node $node, bool $recurse = true): void { try { /** @var Folder $node */ foreach ($node->getDirectoryListing() as $child) { - $this->postDelete($child); + $this->postDelete($child, true, $mimeTypes); } } catch (NotFoundException $e) { $this->logger->debug($e->getMessage(), ['exception' => $e]); @@ -262,6 +292,10 @@ public function postDelete(Node $node, bool $recurse = true): void { return; } + if ($mimeTypes !== null && !in_array($node->getMimetype(), $mimeTypes)) { + return; + } + // Try Deleting possibly existing face detections try { /** @@ -291,7 +325,7 @@ public function postDelete(Node $node, bool $recurse = true): void { /** * @throws \OCP\Files\InvalidPathException */ - public function postInsert(Node $node, bool $recurse = true): void { + public function postInsert(Node $node, bool $recurse = true, ?array $mimeTypes = null): void { if ($node->getType() === FileInfo::TYPE_FOLDER) { if (!$recurse) { return; @@ -309,6 +343,10 @@ public function postInsert(Node $node, bool $recurse = true): void { return; } + if ($mimeTypes !== null && !in_array($node->getMimetype(), $mimeTypes)) { + return; + } + $queueFile = new QueueFile(); if ($node->getMountPoint()->getNumericStorageId() === null) { return; @@ -316,7 +354,7 @@ public function postInsert(Node $node, bool $recurse = true): void { $queueFile->setStorageId($node->getMountPoint()->getNumericStorageId()); $queueFile->setRootId($node->getMountPoint()->getStorageRootId()); - if ($this->isIgnored($node)) { + if ($this->isFileIgnored($node)) { return; } @@ -401,7 +439,7 @@ private function copyFaceDetectionsForNode(string $ownerId, array $usersToAdd, a * @throws \OCP\Files\InvalidPathException * @throws \OCP\Files\NotFoundException */ - public function isIgnored(Node $node) : bool { + public function isFileIgnored(Node $node) : bool { $ignoreMarkers = []; $mimeType = $node->getMimetype(); $storageId = $node->getMountPoint()->getNumericStorageId(); @@ -419,6 +457,7 @@ public function isIgnored(Node $node) : bool { if (in_array($mimeType, Constants::AUDIO_FORMATS)) { $ignoreMarkers = array_merge($ignoreMarkers, Constants::IGNORE_MARKERS_AUDIO); } + if (count($ignoreMarkers) === 0) { return true; } @@ -426,17 +465,45 @@ public function isIgnored(Node $node) : bool { $ignoreMarkers = array_merge($ignoreMarkers, Constants::IGNORE_MARKERS_ALL); $ignoredPaths = $this->ignoreService->getIgnoredDirectories($storageId, $ignoreMarkers); - $relevantIgnorePaths = array_filter($ignoredPaths, static function (string $ignoredPath) use ($node) { - return stripos($node->getInternalPath(), $ignoredPath ? $ignoredPath . '/' : $ignoredPath) === 0; - }); - if (count($relevantIgnorePaths) > 0) { - return true; + foreach ($ignoredPaths as $ignoredPath) { + if (stripos($node->getInternalPath(), $ignoredPath ? $ignoredPath . '/' : $ignoredPath) === 0) { + return true; + } } - return false; } + /** + * @param \OCP\Files\Node $node + * @return array + * @throws Exception + */ + public function getDirIgnores(Node $node) : array { + $storageId = $node->getMountPoint()->getNumericStorageId(); + if ($storageId === null) { + return []; + } + + $ignoredMimeTypes = []; + foreach ([ + [Constants::IGNORE_MARKERS_IMAGE, Constants::IMAGE_FORMATS], + [Constants::IGNORE_MARKERS_VIDEO, Constants::VIDEO_FORMATS], + [Constants::IGNORE_MARKERS_AUDIO, Constants::AUDIO_FORMATS], + [Constants::IGNORE_MARKERS_ALL, array_merge(Constants::IMAGE_FORMATS, Constants::VIDEO_FORMATS, Constants::AUDIO_FORMATS)], + ] as $iteration) { + [$ignoreMarkers, $mimeTypes] = $iteration; + $ignoredPaths = $this->ignoreService->getIgnoredDirectories($storageId, $ignoreMarkers); + foreach ($ignoredPaths as $ignoredPath) { + if (stripos($node->getInternalPath(), $ignoredPath ? $ignoredPath . '/' : $ignoredPath) === 0) { + $ignoredMimeTypes = array_unique(array_merge($ignoredMimeTypes, $mimeTypes)); + } + } + } + + return $ignoredMimeTypes; + } + private function resetIgnoreCache(Node $node) : void { $storageId = $node->getMountPoint()->getNumericStorageId(); if ($storageId === null) { diff --git a/tests/ClassifierTest.php b/tests/ClassifierTest.php index 94aa5482..72c65ba5 100644 --- a/tests/ClassifierTest.php +++ b/tests/ClassifierTest.php @@ -184,6 +184,59 @@ public function testFileListener(string $ignoreFileName) : void { self::assertCount(0, $this->queue->getFromQueue(ImagenetClassifier::MODEL_NAME, $storageId, $rootId, 100), 'entry should have been removed from imagenet queue after moving it into ignored territory'); } + /** + * @dataProvider ignoreImageFilesProvider + * @return void + * @throws \OCP\DB\Exception + * @throws \OCP\Files\InvalidPathException + * @throws \OCP\Files\NotFoundException + * @throws \OCP\Files\NotPermittedException + */ + public function testFileListenerWithFolder(string $ignoreFileName) : void { + $this->config->setAppValueString('imagenet.enabled', 'true'); + $this->queue->clearQueue(ImagenetClassifier::MODEL_NAME); + + $folder = $this->userFolder->newFolder('/folder/'); + $this->testFile = $folder->newFile('/alpine.jpg', file_get_contents(__DIR__.'/res/alpine.JPG')); + $ignoreFolder = $this->userFolder->newFolder('/test/ignore/'); + $ignoredFolder = $ignoreFolder->newFolder('/folder/'); + $ignoreFile = $this->userFolder->newFile('/test/' . $ignoreFileName, ''); + $this->ignoredFile = $ignoredFolder->newFile('/alpine-2.jpg', file_get_contents(__DIR__.'/res/alpine.JPG')); + + $storageId = $this->testFile->getMountPoint()->getNumericStorageId(); + $rootId = $this->testFile->getMountPoint()->getStorageRootId(); + + self::assertCount(1, $this->queue->getFromQueue(ImagenetClassifier::MODEL_NAME, $storageId, $rootId, 100), 'one element should have been added to imagenet queue'); + + $folder->delete(); + + self::assertCount(0, $this->queue->getFromQueue(ImagenetClassifier::MODEL_NAME, $storageId, $rootId, 100), 'entry should have been removed from imagenet queue'); + + $ignoreFile->delete(); + + self::assertCount(1, $this->queue->getFromQueue(ImagenetClassifier::MODEL_NAME, $storageId, $rootId, 100), 'one element should have been added to imagenet queue after deleting ignore file'); + + $ignoreFile = $this->userFolder->newFile('/test/' . $ignoreFileName, ''); + + self::assertCount(0, $this->queue->getFromQueue(ImagenetClassifier::MODEL_NAME, $storageId, $rootId, 100), 'entry should have been removed from imagenet queue after creating ignore file'); + + $ignoredFolder->move($this->userFolder->getPath() . '/folder2'); + + self::assertCount(1, $this->queue->getFromQueue(ImagenetClassifier::MODEL_NAME, $storageId, $rootId, 100), 'one element should have been added to imagenet queue after moving it out of ignored territory'); + + $ignoreFile->move($this->userFolder->getPath() . '/' . $ignoreFileName); + + self::assertCount(0, $this->queue->getFromQueue(ImagenetClassifier::MODEL_NAME, $storageId, $rootId, 100), 'entry should have been removed from imagenet queue after moving ignore file'); + + $ignoreFile->move($this->userFolder->getPath() . '/test/' . $ignoreFileName); + + self::assertCount(1, $this->queue->getFromQueue(ImagenetClassifier::MODEL_NAME, $storageId, $rootId, 100), 'one element should have been added to imagenet queue after moving ignore file'); + + $ignoredFolder->move($this->userFolder->getPath() . '/test/ignore/folder'); + + self::assertCount(0, $this->queue->getFromQueue(ImagenetClassifier::MODEL_NAME, $storageId, $rootId, 100), 'entry should have been removed from imagenet queue after moving it into ignored territory'); + } + /** * @dataProvider ignoreImageFilesProvider * @return void