Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP] new s3 scanner #11712

Closed
wants to merge 1 commit into from
Closed

[WIP] new s3 scanner #11712

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 45 additions & 5 deletions apps/files_external/lib/amazons3.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -240,11 +240,7 @@ public function opendir($path) {

try {
$files = array();
$result = $this->getConnection()->getIterator('ListObjects', array(
'Bucket' => $this->bucket,
'Delimiter' => '/',
'Prefix' => $path
), array('return_prefixes' => true));
$result = $this->getIterator($path);

foreach ($result as $object) {
$file = basename(
Expand All @@ -265,6 +261,19 @@ public function opendir($path) {
}
}

public function getIterator($path = '') {
if ($this->isRoot($path)) {
$path = '';
} else if ($path !== '') {
$path .= '/';
}
return $this->getConnection()->getIterator('ListObjects', array(
'Bucket' => $this->bucket,
'Delimiter' => '/',
'Prefix' => $path
), array('return_prefixes' => true));
}

public function stat($path) {
$path = $this->normalizePath($path);

Expand Down Expand Up @@ -542,6 +551,33 @@ public function getId() {
return $this->id;
}

/**
* FIXME use permissions in bucket
* @param string $path
* @return int
*/
public function getPermissions($path) {
$permissions = 0;
$permissions |= \OCP\PERMISSION_CREATE;
$permissions |= \OCP\PERMISSION_READ;
$permissions |= \OCP\PERMISSION_UPDATE;
$permissions |= \OCP\PERMISSION_DELETE;
if (!\OC_Util::isSharingDisabledForUser()) {
$permissions |= \OCP\PERMISSION_SHARE;
}
return $permissions;
}

public function getScanner($path = '', $storage = null) {
if (!$storage) {
$storage = $this;
}
if (!isset($this->scanner)) {
$this->scanner = new \OCA\Files_External\Cache\Scanner($storage);
}
return $this->scanner;
}

/**
* Returns the connection
*
Expand Down Expand Up @@ -587,6 +623,10 @@ public function getConnection() {
return $this->connection;
}

public function getBucket() {
return $this->bucket;
}

public function writeBack($tmpFile) {
if (!isset(self::$tmpFiles[$tmpFile])) {
return false;
Expand Down
202 changes: 202 additions & 0 deletions apps/files_external/lib/cache/scanner.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,202 @@
<?php
/**
* Copyright (c) 2014 Jörn Friedrich Dreyer <[email protected]>
* This file is licensed under the Affero General Public License version 3 or
* later.
* See the COPYING-README file.
*/

namespace OCA\Files_External\Cache;

use OC\Files\Filesystem;
use OC\Hooks\BasicEmitter;
use OCP\Config;

/**
* Class Scanner
*/
class Scanner extends \OC\Files\Cache\Scanner {

public function __construct (\OC\Files\Storage\Storage $storage) {
parent::__construct($storage);
}

/**
* get all the metadata of a file or folder
* *
*
* @param string $path
* @return array an array of metadata of the file
*/
public function getData($object) {
$data = array();
if (isset($object['Key'])) {
$data['path'] = trim($object['Key'], '/');
if (isset($object['ContentType'])) {
$data['mimetype'] = $object['ContentType'];
} else if (substr($object['Key'],-1) === '/') {
$data['mimetype'] = 'httpd/unix-directory';
$data['etag'] = null;
} else {
$data['etag'] = $object['ETag'];
}
$data['mtime'] = strtotime($object['LastModified']);
if ($data['mtime'] === false) {
$data['mtime'] = -1;
}
$data['size'] = (int)$object['Size'];
$data['name'] = basename($data['path']);
$data['storage_mtime'] = $data['mtime'];
if (!isset($data['mimetype'])) {
$data['mimetype'] = \OC_Helper::getMimetypeDetector()->detectPath($data['name']);
}
$data['permissions'] = $this->storage->getPermissions($data['path']);
} else {
throw new \Exception('expected key in object');
}
return $data;
}

/**
* scan a folder and all it's children
*
* @param string $path
* @param bool $recursive
* @param int $reuse
* @return array an array of the meta data of the scanned file or folder
*/
public function scan($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1) {
$params = array(
'Bucket' => $this->storage->getBucket()
);

//mark all storage_mtimes as unknown
$stmt = \OC::$server->getDatabaseConnection()->prepare(
'UPDATE `*PREFIX*filecache` SET `storage_mtime` = ? WHERE `storage` = ?'
);
$stmt->execute(array(-1, $this->cache->getNumericStorageId()));

$data = $this->getData(array('Key' => '/'));
$rootData = $this->cache->get('');
if ($rootData) {
// remember rootId for size update after scanning
$rootId = $rootData['fileid'];
// Only update metadata that has changed
$newData = array_diff_assoc($data, $rootData);
} else {
$newData = $data;
}
if (!empty($newData)) {
// remember rootId for size update after scanning
$rootId = $this->addToCache('', $newData);
}

$previousParent = null;
$parentId = null;
$maxChildMTime = -1;
try {
// Since there are no real directories on S3, we fetch all objects
do {
// instead of the iterator, manually loop over the list ...
$objects = $this->storage->getConnection()->listObjects($params);
// ... so we get more than one directory
foreach ($objects['Contents'] as $object) {
$data = $this->getData($object);
if (/*$data['name'] === basename($path)
||*/ self::isPartialFile($data['name'])
|| Filesystem::isFileBlacklisted($data['name'])
) {
continue;
}

$parent = dirname($data['path']);
if ($parent === '.' or $parent === '/') {
$parent = '';
}
if ($data['mtime'] > $maxChildMTime) {
$maxChildMTime = $data['mtime'];
}
if ($previousParent !== $parent) {
if ($previousParent) {
$this->cache->calculateFolderSize($previousParent,
array('fileid' => $parentId, 'mimetype' => 'httpd/unix-directory', 'size' => -1)
);
}
//update mtime of previous parent in cache
$this->updateCache($previousParent,
array('mtime' => $maxChildMTime, 'storage_mtime' => $maxChildMTime)
);

// propagate mtime up the tree
$p = dirname($previousParent);
if ($p === '.') {
$p = '';
}
do {
$pData = $this->cache->get($p);
if ($maxChildMTime > $pData['mtime']) {
$this->updateCache($pData['path'],
array('mtime' => $maxChildMTime, 'storage_mtime' => $maxChildMTime)
);
}
$p = dirname($previousParent);
if ($p === '.') {
$p = '';
}
} while ($p !== '' || $maxChildMTime > $pData['mtime'] );

$maxChildMTime = -1;
$previousParent = $parent;
$parentId = (int)$this->cache->getId($parent);
//FIXME what if $parentId is -1?
}
$data['parent'] = $parentId;

$cacheData = $this->cache->get($data['path']);
if ($cacheData) {
// Only update metadata that has changed
$newData = array_diff_assoc($data, $cacheData);
} else {
$newData = $data;
}
if (!empty($newData)) {
$this->addToCache($data['path'], $newData);
$this->emit(
'\OC\Files\Cache\Scanner',
'postScanFile',
array($data['path'], $this->storageId)
);
\OC_Hook::emit(
'\OC\Files\Cache\Scanner',
'post_scan_file',
array('path' => $data['path'], 'storage' => $this->storageId)
);
}
// we reached the end when the list is no longer truncated
}
} while ($objects['IsTruncated']);
} catch (S3Exception $e) {
\OCP\Util::logException('files_external', $e);
return null;
}

// update root size
$this->cache->calculateFolderSize('',
array('fileid' => $rootId, 'mimetype' => 'httpd/unix-directory', 'size' => -1)
);

//delete all filecache entries where storage_mtimes has not been updated
$stmt = \OC::$server->getDatabaseConnection()->prepare(
'DELETE FROM `*PREFIX*filecache` WHERE `storage` = ? AND `storage_mtime` = ?'
);
$stmt->execute(array($this->cache->getNumericStorageId(), -1));

return $this->cache->get($path);
}
public function scanFile($file, $reuseExisting = 0) {

}
public function scanChildren($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1) {

}
}