Merge pull request #31839 from nextcloud/feat/metadata-server

Add a metadata service to store file metadata
nextcloud · Apr 14, 2022 · 3ca7971 · 3ca7971
2 parents e827e0a + 1c7ecfc
commit 3ca7971
Show file tree

Hide file tree

Showing 85 changed files with 1,044 additions and 81 deletions.
diff --git a/3rdparty b/3rdparty
diff --git a/apps/accessibility/composer/autoload.php b/apps/accessibility/composer/autoload.php
@@ -2,6 +2,11 @@
 
 // autoload.php @generated by Composer
 
+if (PHP_VERSION_ID < 50600) {
+    echo 'Composer 2.3.0 dropped support for autoloading on PHP <5.6 and you are running '.PHP_VERSION.', please upgrade PHP or use Composer 2.2 LTS via "composer self-update --2.2". Aborting.'.PHP_EOL;
+    exit(1);
+}
+
 require_once __DIR__ . '/composer/autoload_real.php';
 
 return ComposerAutoloaderInitAccessibility::getLoader();
diff --git a/apps/accessibility/composer/composer/autoload_real.php b/apps/accessibility/composer/composer/autoload_real.php
@@ -27,7 +27,7 @@ public static function getLoader()
         spl_autoload_unregister(array('ComposerAutoloaderInitAccessibility', 'loadClassLoader'));
 
         require __DIR__ . '/autoload_static.php';
-        \Composer\Autoload\ComposerStaticInitAccessibility::getInitializer($loader)();
+        call_user_func(\Composer\Autoload\ComposerStaticInitAccessibility::getInitializer($loader));
 
         $loader->setClassMapAuthoritative(true);
         $loader->register(true);

diff --git a/apps/admin_audit/composer/autoload.php b/apps/admin_audit/composer/autoload.php
@@ -2,6 +2,11 @@
 
 // autoload.php @generated by Composer
 
+if (PHP_VERSION_ID < 50600) {
+    echo 'Composer 2.3.0 dropped support for autoloading on PHP <5.6 and you are running '.PHP_VERSION.', please upgrade PHP or use Composer 2.2 LTS via "composer self-update --2.2". Aborting.'.PHP_EOL;
+    exit(1);
+}
+
 require_once __DIR__ . '/composer/autoload_real.php';
 
 return ComposerAutoloaderInitAdminAudit::getLoader();
diff --git a/apps/admin_audit/composer/composer/autoload_real.php b/apps/admin_audit/composer/composer/autoload_real.php
@@ -27,7 +27,7 @@ public static function getLoader()
         spl_autoload_unregister(array('ComposerAutoloaderInitAdminAudit', 'loadClassLoader'));
 
         require __DIR__ . '/autoload_static.php';
-        \Composer\Autoload\ComposerStaticInitAdminAudit::getInitializer($loader)();
+        call_user_func(\Composer\Autoload\ComposerStaticInitAdminAudit::getInitializer($loader));
 
         $loader->setClassMapAuthoritative(true);
         $loader->register(true);

diff --git a/apps/cloud_federation_api/composer/autoload.php b/apps/cloud_federation_api/composer/autoload.php
@@ -2,6 +2,11 @@
 
 // autoload.php @generated by Composer
 
+if (PHP_VERSION_ID < 50600) {
+    echo 'Composer 2.3.0 dropped support for autoloading on PHP <5.6 and you are running '.PHP_VERSION.', please upgrade PHP or use Composer 2.2 LTS via "composer self-update --2.2". Aborting.'.PHP_EOL;
+    exit(1);
+}
+
 require_once __DIR__ . '/composer/autoload_real.php';
 
 return ComposerAutoloaderInitCloudFederationAPI::getLoader();
diff --git a/apps/cloud_federation_api/composer/composer/autoload_real.php b/apps/cloud_federation_api/composer/composer/autoload_real.php
@@ -27,7 +27,7 @@ public static function getLoader()
         spl_autoload_unregister(array('ComposerAutoloaderInitCloudFederationAPI', 'loadClassLoader'));
 
         require __DIR__ . '/autoload_static.php';
-        \Composer\Autoload\ComposerStaticInitCloudFederationAPI::getInitializer($loader)();
+        call_user_func(\Composer\Autoload\ComposerStaticInitCloudFederationAPI::getInitializer($loader));
 
         $loader->setClassMapAuthoritative(true);
         $loader->register(true);

diff --git a/apps/comments/composer/autoload.php b/apps/comments/composer/autoload.php
@@ -2,6 +2,11 @@
 
 // autoload.php @generated by Composer
 
+if (PHP_VERSION_ID < 50600) {
+    echo 'Composer 2.3.0 dropped support for autoloading on PHP <5.6 and you are running '.PHP_VERSION.', please upgrade PHP or use Composer 2.2 LTS via "composer self-update --2.2". Aborting.'.PHP_EOL;
+    exit(1);
+}
+
 require_once __DIR__ . '/composer/autoload_real.php';
 
 return ComposerAutoloaderInitComments::getLoader();
diff --git a/apps/comments/composer/composer/autoload_real.php b/apps/comments/composer/composer/autoload_real.php
@@ -27,7 +27,7 @@ public static function getLoader()
         spl_autoload_unregister(array('ComposerAutoloaderInitComments', 'loadClassLoader'));
 
         require __DIR__ . '/autoload_static.php';
-        \Composer\Autoload\ComposerStaticInitComments::getInitializer($loader)();
+        call_user_func(\Composer\Autoload\ComposerStaticInitComments::getInitializer($loader));
 
         $loader->setClassMapAuthoritative(true);
         $loader->register(true);

diff --git a/apps/contactsinteraction/composer/autoload.php b/apps/contactsinteraction/composer/autoload.php
@@ -2,6 +2,11 @@
 
 // autoload.php @generated by Composer
 
+if (PHP_VERSION_ID < 50600) {
+    echo 'Composer 2.3.0 dropped support for autoloading on PHP <5.6 and you are running '.PHP_VERSION.', please upgrade PHP or use Composer 2.2 LTS via "composer self-update --2.2". Aborting.'.PHP_EOL;
+    exit(1);
+}
+
 require_once __DIR__ . '/composer/autoload_real.php';
 
 return ComposerAutoloaderInitContactsInteraction::getLoader();
diff --git a/apps/contactsinteraction/composer/composer/autoload_real.php b/apps/contactsinteraction/composer/composer/autoload_real.php
@@ -27,7 +27,7 @@ public static function getLoader()
         spl_autoload_unregister(array('ComposerAutoloaderInitContactsInteraction', 'loadClassLoader'));
 
         require __DIR__ . '/autoload_static.php';
-        \Composer\Autoload\ComposerStaticInitContactsInteraction::getInitializer($loader)();
+        call_user_func(\Composer\Autoload\ComposerStaticInitContactsInteraction::getInitializer($loader));
 
         $loader->setClassMapAuthoritative(true);
         $loader->register(true);

diff --git a/apps/dav/composer/autoload.php b/apps/dav/composer/autoload.php
@@ -2,6 +2,11 @@
 
 // autoload.php @generated by Composer
 
+if (PHP_VERSION_ID < 50600) {
+    echo 'Composer 2.3.0 dropped support for autoloading on PHP <5.6 and you are running '.PHP_VERSION.', please upgrade PHP or use Composer 2.2 LTS via "composer self-update --2.2". Aborting.'.PHP_EOL;
+    exit(1);
+}
+
 require_once __DIR__ . '/composer/autoload_real.php';
 
 return ComposerAutoloaderInitDAV::getLoader();
diff --git a/apps/dav/composer/composer/autoload_real.php b/apps/dav/composer/composer/autoload_real.php
@@ -27,7 +27,7 @@ public static function getLoader()
         spl_autoload_unregister(array('ComposerAutoloaderInitDAV', 'loadClassLoader'));
 
         require __DIR__ . '/autoload_static.php';
-        \Composer\Autoload\ComposerStaticInitDAV::getInitializer($loader)();
+        call_user_func(\Composer\Autoload\ComposerStaticInitDAV::getInitializer($loader));
 
         $loader->setClassMapAuthoritative(true);
         $loader->register(true);

diff --git a/apps/dav/composer/composer/installed.php b/apps/dav/composer/composer/installed.php
@@ -5,7 +5,7 @@
         'type' => 'library',
         'install_path' => __DIR__ . '/../',
         'aliases' => array(),
-        'reference' => 'e2c675724fc4ea50f1275bf0027b96f277c32578',
+        'reference' => '9586920c0ec4016864a2219e838fb272127822d8',
         'name' => '__root__',
         'dev' => false,
     ),
@@ -16,7 +16,7 @@
             'type' => 'library',
             'install_path' => __DIR__ . '/../',
             'aliases' => array(),
-            'reference' => 'e2c675724fc4ea50f1275bf0027b96f277c32578',
+            'reference' => '9586920c0ec4016864a2219e838fb272127822d8',
             'dev_requirement' => false,
         ),
     ),

diff --git a/apps/dav/lib/Connector/Sabre/Directory.php b/apps/dav/lib/Connector/Sabre/Directory.php
@@ -34,6 +34,8 @@
 
 use OC\Files\Mount\MoveableMount;
 use OC\Files\View;
+use OC\Metadata\FileMetadata;
+use OC\Metadata\MetadataGroup;
 use OCA\DAV\Connector\Sabre\Exception\FileLocked;
 use OCA\DAV\Connector\Sabre\Exception\Forbidden;
 use OCA\DAV\Connector\Sabre\Exception\InvalidPath;
@@ -73,6 +75,9 @@ class Directory extends \OCA\DAV\Connector\Sabre\Node implements \Sabre\DAV\ICol
 	 */
 	private $tree;
 
+	/** @var array<string, array<int, FileMetadata>> */
+	private array $metadata = [];
+
 	/**
 	 * Sets up the node, expects a full path name
 	 *

diff --git a/apps/dav/lib/Connector/Sabre/File.php b/apps/dav/lib/Connector/Sabre/File.php
@@ -43,6 +43,7 @@
 use OC\Files\Filesystem;
 use OC\Files\Stream\HashWrapper;
 use OC\Files\View;
+use OC\Metadata\FileMetadata;
 use OCA\DAV\AppInfo\Application;
 use OCA\DAV\Connector\Sabre\Exception\EntityTooLarge;
 use OCA\DAV\Connector\Sabre\Exception\FileLocked;
@@ -80,6 +81,9 @@ class File extends Node implements IFile {
 
 	protected IL10N $l10n;
 
+	/** @var array<string, FileMetadata> */
+	private array $metadata = [];
+
 	/**
 	 * Sets up the node, expects a full path name
 	 *
@@ -757,4 +761,16 @@ public function hash(string $type) {
 	public function getNode(): \OCP\Files\File {
 		return $this->node;
 	}
+
+	public function getMetadata(string $group): FileMetadata {
+		return $this->metadata[$group];
+	}
+
+	public function setMetadata(string $group, FileMetadata $metadata): void {
+		$this->metadata[$group] = $metadata;
+	}
+
+	public function hasMetadata(string $group) {
+		return array_key_exists($group, $this->metadata);
+	}
 }
diff --git a/apps/dav/lib/Connector/Sabre/FilesPlugin.php b/apps/dav/lib/Connector/Sabre/FilesPlugin.php
@@ -34,13 +34,15 @@
 namespace OCA\DAV\Connector\Sabre;
 
 use OC\AppFramework\Http\Request;
+use OC\Metadata\IMetadataManager;
 use OCP\Constants;
 use OCP\Files\ForbiddenException;
 use OCP\Files\StorageNotAvailableException;
 use OCP\IConfig;
 use OCP\IPreview;
 use OCP\IRequest;
 use OCP\IUserSession;
+use Psr\Log\LoggerInterface;
 use Sabre\DAV\Exception\Forbidden;
 use Sabre\DAV\Exception\NotFound;
 use Sabre\DAV\IFile;
@@ -50,6 +52,7 @@
 use Sabre\DAV\Tree;
 use Sabre\HTTP\RequestInterface;
 use Sabre\HTTP\ResponseInterface;
+use Sabre\Uri;
 
 class FilesPlugin extends ServerPlugin {
 
@@ -79,6 +82,7 @@ class FilesPlugin extends ServerPlugin {
 	public const SHARE_NOTE = '{http://nextcloud.org/ns}note';
 	public const SUBFOLDER_COUNT_PROPERTYNAME = '{http://nextcloud.org/ns}contained-folder-count';
 	public const SUBFILE_COUNT_PROPERTYNAME = '{http://nextcloud.org/ns}contained-file-count';
+	public const FILE_METADATA_SIZE = '{http://nextcloud.org/ns}file-metadata-size';
 
 	/**
 	 * Reference to main server object
@@ -436,6 +440,29 @@ public function handleGetProperties(PropFind $propFind, \Sabre\DAV\INode $node)
 			$propFind->handle(self::UPLOAD_TIME_PROPERTYNAME, function () use ($node) {
 				return $node->getFileInfo()->getUploadTime();
 			});
+
+			if ($this->config->getSystemValueBool('enable_file_metadata', true)) {
+				$propFind->handle(self::FILE_METADATA_SIZE, function () use ($node) {
+					if (!str_starts_with($node->getFileInfo()->getMimetype(), 'image')) {
+						return json_encode((object)[]);
+					}
+
+					if ($node->hasMetadata('size')) {
+						$sizeMetadata = $node->getMetadata('size');
+					} else {
+						// This code path should not be called since we try to preload
+						// the metadata when loading the folder or the search results
+						// in one go
+						$metadataManager = \OC::$server->get(IMetadataManager::class);
+						$sizeMetadata = $metadataManager->fetchMetadataFor('size', [$node->getId()])[$node->getId()];
+
+						// TODO would be nice to display this in the profiler...
+						\OC::$server->get(LoggerInterface::class)->debug('Inefficient fetching of metadata');
+					}
+
+					return json_encode((object)$sizeMetadata->getMetadata());
+				});
+			}
 		}
 
 		if ($node instanceof Directory) {
@@ -448,6 +475,32 @@ public function handleGetProperties(PropFind $propFind, \Sabre\DAV\INode $node)
 			});
 
 			$requestProperties = $propFind->getRequestedProperties();
+
+			// TODO detect dynamically which metadata groups are requested and
+			// preload all of them and not just size
+			if ($this->config->getSystemValueBool('enable_file_metadata', true)
+				&& in_array(self::FILE_METADATA_SIZE, $requestProperties, true)) {
+				// Preloading of the metadata
+				$fileIds = [];
+				foreach ($node->getChildren() as $child) {
+					/** @var \OCP\Files\Node|Node $child */
+					if (str_starts_with($child->getFileInfo()->getMimeType(), 'image/')) {
+						/** @var File $child */
+						$fileIds[] = $child->getFileInfo()->getId();
+					}
+				}
+				/** @var IMetaDataManager $metadataManager */
+				$metadataManager = \OC::$server->get(IMetadataManager::class);
+				$preloadedMetadata = $metadataManager->fetchMetadataFor('size', $fileIds);
+				foreach ($node->getChildren() as $child) {
+					/** @var \OCP\Files\Node|Node $child */
+					if (str_starts_with($child->getFileInfo()->getMimeType(), 'image')) {
+						/** @var File $child */
+						$child->setMetadata('size', $preloadedMetadata[$child->getFileInfo()->getId()]);
+					}
+				}
+			}
+
 			if (in_array(self::SUBFILE_COUNT_PROPERTYNAME, $requestProperties, true)
 				|| in_array(self::SUBFOLDER_COUNT_PROPERTYNAME, $requestProperties, true)) {
 				$nbFiles = 0;