From 12fc796402cf3c4c2410c97018a1b38f1c418238 Mon Sep 17 00:00:00 2001 From: Christian Weiske Date: Tue, 14 Mar 2023 18:24:19 +0100 Subject: [PATCH] [FEATURE] Add "base folder" configuration setting The base folder allows us to use the same bucket for multiple projects by forcing every project/TYPO3 instance into an own prefix: ``` mybucket/ + project1/ | + file23.jpg | + subfolder/ | | + subfile42.png | + _processed_/ | + ... + project2/ + file2.jpg + _processed_/ + ... ``` The prefix is not visible in the TYPO3 UI (file list, file information) and gets added transparently to S3 requests, and is removed from S3 responses as well. My initial version used a simpler approach by just overriding getRootLevelFolder() and getParentFolderIdentifierOfIdentifier(), but this led to inconsistencies in the UI, and the processing folder could not be moved into the base folder because of inconsistencies inside the TYPO3 API. --- Classes/Driver/AmazonS3Driver.php | 93 ++++++++++++++++--- .../FlexForm/AmazonS3DriverFlexForm.xml | 10 ++ .../Private/Language/locallang_flexform.xlf | 6 ++ 3 files changed, 94 insertions(+), 15 deletions(-) diff --git a/Classes/Driver/AmazonS3Driver.php b/Classes/Driver/AmazonS3Driver.php index c5577ebe..de76a70f 100644 --- a/Classes/Driver/AmazonS3Driver.php +++ b/Classes/Driver/AmazonS3Driver.php @@ -78,6 +78,14 @@ class AmazonS3Driver extends AbstractHierarchicalFilesystemDriver implements Str */ protected $baseUrl = ''; + /** + * Folder that is used as root folder. + * Must be empty or have a trailing slash. + * + * @var string + */ + protected $baseFolder = ''; + /** * Stream wrapper protocol: Will be set in the constructor * @@ -217,6 +225,7 @@ public function processConfiguration() public function initialize() { $this->initializeBaseUrl() + ->initializeBaseFolder() ->initializeSettings() ->initializeClient(); $this->resetRequestCache(); @@ -237,7 +246,7 @@ public function getPublicUrl($identifier) { $uriParts = GeneralUtility::trimExplode('/', ltrim($identifier, '/'), true); $uriParts = array_map('rawurlencode', $uriParts); - return $this->baseUrl . '/' . implode('/', $uriParts); + return $this->baseUrl . '/' . $this->addBaseFolder(implode('/', $uriParts)); } /** @@ -393,14 +402,14 @@ public function addFile($localFilePath, $targetFolderIdentifier, $newFileName = if (filesize($localFilePath) === 0) { // Multipart uploader would fail to upload empty files $this->s3Client->upload( $this->configuration['bucket'], - $targetIdentifier, + $this->addBaseFolder($targetIdentifier), '' ); } else { $multipartUploadAdapter = GeneralUtility::makeInstance(MultipartUploaderAdapter::class, $this->s3Client); $multipartUploadAdapter->upload( $localFilePath, - $targetIdentifier, + $this->addBaseFolder($targetIdentifier), $this->configuration['bucket'], $this->getCacheControl($targetIdentifier) ); @@ -523,7 +532,7 @@ public function getFileForLocalProcessing($fileIdentifier, $writable = true) $temporaryPath = $this->getTemporaryPathForFile($fileIdentifier); $this->s3Client->getObject([ 'Bucket' => $this->configuration['bucket'], - 'Key' => $fileIdentifier, + 'Key' => $this->addBaseFolder($fileIdentifier), 'SaveAs' => $temporaryPath, ]); if (!is_file($temporaryPath)) { @@ -598,7 +607,7 @@ public function getFileContents($fileIdentifier) { $result = $this->s3Client->getObject([ 'Bucket' => $this->configuration['bucket'], - 'Key' => $fileIdentifier + 'Key' => $this->addBaseFolder($fileIdentifier) ]); return (string)$result['Body']; } @@ -1081,6 +1090,19 @@ protected function initializeBaseUrl() return $this; } + /** + * Set the $baseFolder variable from configuration + */ + protected function initializeBaseFolder(): self + { + $baseFolder = $this->configuration['baseFolder'] ?? ''; + if ($baseFolder != '') { + $baseFolder = rtrim($baseFolder, '/') . '/'; + } + $this->baseFolder = $baseFolder; + return $this; + } + /** * initializeSettings * @@ -1181,6 +1203,30 @@ protected function testConnection() } } + /** + * Prefix the given file/path identifier with the base folder. + * Used by internal functions that directly speak with S3Client + */ + protected function addBaseFolder(string $identifier): string + { + if ($this->baseFolder) { + $identifier = str_replace('//', '/', $this->baseFolder . $identifier); + } + return $identifier; + } + + /** + * Remove base folder prefix from a given S3 path. + * Used when returning information from S3. + */ + protected function removeBaseFolder(string $identifier): string + { + if ($this->baseFolder) { + $identifier = substr($identifier, strlen($this->baseFolder)); + } + return $identifier; + } + /** * @return \TYPO3\CMS\Core\Messaging\FlashMessageQueue */ @@ -1234,7 +1280,7 @@ protected function getMetaInfo($identifier) try { $metadata = $this->s3Client->headObject([ 'Bucket' => $this->configuration['bucket'], - 'Key' => $identifier + 'Key' => $this->addBaseFolder($identifier) ])->toArray(); $metaInfoDownloadAdapter = GeneralUtility::makeInstance(MetaInfoDownloadAdapter::class); $this->metaInfoCache[$identifier] = $metaInfoDownloadAdapter->getMetaInfoFromResponse($this, $identifier, $metadata); @@ -1313,7 +1359,7 @@ protected function getObjectPermissions($identifier) try { $response = $this->s3Client->getObjectAcl([ 'Bucket' => $this->configuration['bucket'], - 'Key' => $identifier + 'Key' => $this->addBaseFolder($identifier) ])->toArray(); // Until the SDK provides any useful information about folder permissions, we take full access for granted as long as one user with full access exists. @@ -1351,10 +1397,10 @@ protected function getObjectPermissions($identifier) */ protected function deleteObject(string $identifier): bool { - $this->s3Client->deleteObject(['Bucket' => $this->configuration['bucket'], 'Key' => $identifier]); + $this->s3Client->deleteObject(['Bucket' => $this->configuration['bucket'], 'Key' => $this->addBaseFolder($identifier)]); $this->flushMetaInfoCache($identifier); $this->resetRequestCache(); - return !$this->s3Client->doesObjectExist($this->configuration['bucket'], $identifier); + return !$this->s3Client->doesObjectExist($this->configuration['bucket'], $this->addBaseFolder($identifier)); } /** @@ -1382,7 +1428,7 @@ protected function createObject($identifier, $body = '', $overrideArgs = []) $this->normalizeIdentifier($identifier); $args = [ 'Bucket' => $this->configuration['bucket'], - 'Key' => $identifier, + 'Key' => $this->addBaseFolder($identifier), 'Body' => $body ]; $this->s3Client->putObject(array_merge_recursive($args, $overrideArgs)); @@ -1441,7 +1487,7 @@ protected function getStreamWrapperPath($file) throw new \RuntimeException('Type "' . gettype($file) . '" is not supported.', 1325191178); } $this->normalizeIdentifier($identifier); - return $basePath . $identifier; + return $basePath . $this->addBaseFolder($identifier); } /** @@ -1505,20 +1551,37 @@ protected function getListObjects($identifier, $overrideArgs = []) { $args = [ 'Bucket' => $this->configuration['bucket'] ?? '', - 'Prefix' => $identifier, + 'Prefix' => $this->addBaseFolder($identifier), ]; $result = $this->getCachedResponse('listObjectsV2', array_merge_recursive($args, $overrideArgs)); // Cache the given meta info $metaInfoDownloadAdapter = GeneralUtility::makeInstance(MetaInfoDownloadAdapter::class); if (isset($result['Contents']) && is_array($result['Contents'])) { - foreach ($result['Contents'] as $content) { + $baseFolderSelfKey = null; + foreach ($result['Contents'] as $key => &$content) { + $content['Key'] = $this->removeBaseFolder($content['Key']); + if ($content['Key'] === '') { + $baseFolderSelfKey = $key; + continue; + } $fileIdentifier = $identifier . $content['Key']; $this->normalizeIdentifier($fileIdentifier); if (!isset($this->metaInfoCache[$fileIdentifier])) { $this->metaInfoCache[$fileIdentifier] = $metaInfoDownloadAdapter->getMetaInfoFromResponse($this, $fileIdentifier, $content); } } + unset($content); + if ($baseFolderSelfKey !== null) { + unset($result['Contents'][$baseFolderSelfKey]); + } + } + + if (isset($result['CommonPrefixes'])) { + foreach ($result['CommonPrefixes'] as &$prefix) { + $prefix['Prefix'] = $this->removeBaseFolder($prefix['Prefix']); + } + unset($prefix); } if (isset($overrideArgs['MaxKeys']) && $overrideArgs['MaxKeys'] <= 1000) { @@ -1572,8 +1635,8 @@ protected function copyObject($identifier, $targetIdentifier) { $this->s3Client->copyObject([ 'Bucket' => $this->configuration['bucket'], - 'CopySource' => $this->configuration['bucket'] . '/' . $identifier, - 'Key' => $targetIdentifier, + 'CopySource' => $this->configuration['bucket'] . '/' . $this->addBaseFolder($identifier), + 'Key' => $this->addBaseFolder($targetIdentifier), 'CacheControl' => $this->getCacheControl($targetIdentifier) ]); $this->flushMetaInfoCache($targetIdentifier); diff --git a/Configuration/FlexForm/AmazonS3DriverFlexForm.xml b/Configuration/FlexForm/AmazonS3DriverFlexForm.xml index 4c2f0211..0846f54f 100644 --- a/Configuration/FlexForm/AmazonS3DriverFlexForm.xml +++ b/Configuration/FlexForm/AmazonS3DriverFlexForm.xml @@ -187,6 +187,16 @@ + + + + LLL:EXT:aus_driver_amazon_s3/Resources/Private/Language/locallang_flexform.xlf:driverConfiguration.baseFolder-description + + input + 30 + + + diff --git a/Resources/Private/Language/locallang_flexform.xlf b/Resources/Private/Language/locallang_flexform.xlf index 9799c7a4..5c1dbd5b 100644 --- a/Resources/Private/Language/locallang_flexform.xlf +++ b/Resources/Private/Language/locallang_flexform.xlf @@ -33,6 +33,12 @@ Without protocol - "Protocol" setting is used here. + + Base folder + + + Root folder to use as base for all file operations. Other files and folders outside this base folder are not visible. + Cache header: max age (in seconds, optional)