Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP] introduction of interface IChunkHandler and a very first stupid implemen... #12160

Closed
wants to merge 4 commits into from
Closed

[WIP] introduction of interface IChunkHandler and a very first stupid implemen... #12160

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
1bee33e
introduction of interface IChunkHandler and a very first stupid imple…
DeepDiver1975 Nov 13, 2014
33fd130
adding unit tests for file chunking
DeepDiver1975 Nov 13, 2014
754cae8
Remove unused file_assemble
DeepDiver1975 Dec 8, 2014
174b14a
adding hook processing to chunked file upload
DeepDiver1975 Dec 8, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions lib/private/cache/file.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,19 @@

namespace OC\Cache;

use OC\Files\View;
use OCP\Files\Storage;

class File {
/**
* @var View|Storage
*/
protected $storage;

public function __construct($storage = null) {
$this->storage = $storage;
}

/**
* Returns the cache storage for the logged in user
* @return \OC\Files\View cache storage
Expand Down
89 changes: 75 additions & 14 deletions lib/private/connector/sabre/file.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -238,45 +238,67 @@ private function createFileChunked($data)
if (empty($info)) {
throw new \Sabre\DAV\Exception\NotImplemented();
}
$chunk_handler = new OC_FileChunking($info);
$bytesWritten = $chunk_handler->store($info['index'], $data);

// we first assembly the target file as a part file
$targetPath = $path . '/' . $info['name'];
if (isset($_SERVER['CONTENT_LENGTH'])) {
$expected = $_SERVER['CONTENT_LENGTH'];
} else {
$expected = -1;
}
$partFilePath = $path . '/' . $info['name'] . '.ocTransferId' . $info['transferid'] . '.part';
/** @var \OC\Files\Storage\Storage $storage */
list($storage,) = $this->fileView->resolvePath($partFilePath);
$storeData = $storage->getChunkHandler()->storeChunk($partFilePath, $info['index'], $info['chunkcount'], $expected, $data, $info['transferid']);
$bytesWritten = $storeData['bytesWritten'];

//detect aborted upload
if (isset ($_SERVER['REQUEST_METHOD']) && $_SERVER['REQUEST_METHOD'] === 'PUT' ) {
if (isset($_SERVER['CONTENT_LENGTH'])) {
$expected = $_SERVER['CONTENT_LENGTH'];
if ($bytesWritten != $expected) {
$chunk_handler->remove($info['index']);
throw new \Sabre\DAV\Exception\BadRequest(
'expected filesize ' . $expected . ' got ' . $bytesWritten);
}
}
}

if ($chunk_handler->isComplete()) {
if ($storeData['complete']) {

try {
// we first assembly the target file as a part file
$partFile = $path . '/' . $info['name'] . '.ocTransferId' . $info['transferid'] . '.part';
$chunk_handler->file_assemble($partFile);

// here is the final atomic rename
$targetPath = $path . '/' . $info['name'];
$renameOkay = $this->fileView->rename($partFile, $targetPath);
$fileExists = $this->fileView->file_exists($targetPath);
// trigger hooks for post processing
$targetFileExists = $storage->file_exists('/files' . $targetPath);
$run = $this->runPreHooks($targetFileExists, '/files' . $targetPath);
if (!$run) {
\OC::$server->getLogger()->error('Hook execution on {file} failed', array('app' => 'webdav', 'file' => $targetPath));
// delete part file
$storage->unlink('/files' . $partFilePath);
throw new \Sabre\DAV\Exception('Upload rejected');
}
$renameOkay = $storage->rename('/files' . $partFilePath, '/files' . $targetPath);
$fileExists = $storage->file_exists('/files' . $targetPath);
if ($renameOkay === false || $fileExists === false) {
\OC_Log::write('webdav', '\OC\Files\Filesystem::rename() failed', \OC_Log::ERROR);
\OC::$server->getLogger()->error('\OC\Files\Filesystem::rename() failed', array('app'=>'webdav'));
// only delete if an error occurred and the target file was already created
if ($fileExists) {
$this->fileView->unlink($targetPath);
$storage->unlink('/files' . $targetPath);
}
$partFileExists = $storage->file_exists('/files' . $partFilePath);
if ($partFileExists) {
$storage->unlink('/files' . $partFilePath);
}
throw new \Sabre\DAV\Exception('Could not rename part file assembled from chunks');
}

// trigger hooks for post processing
$this->runPostHooks($targetFileExists, '/files' . $targetPath);

// allow sync clients to send the mtime along in a header
$mtime = OC_Request::hasModificationTime();
if ($mtime !== false) {
if($this->fileView->touch($targetPath, $mtime)) {
// TODO: will this update the cache properly - e.g. smb where we cannot change the mtime ???
if($storage->touch('/files' . $targetPath, $mtime)) {
header('X-OC-MTime: accepted');
}
}
Expand All @@ -290,4 +312,43 @@ private function createFileChunked($data)

return null;
}

private function runPreHooks($fileExists, $path) {
$run = true;
if(!$fileExists) {
OC_Hook::emit(
\OC\Files\Filesystem::CLASSNAME,
\OC\Files\Filesystem::signal_create,
array(
\OC\Files\Filesystem::signal_param_path => $path,
\OC\Files\Filesystem::signal_param_run => &$run
)
);
}
OC_Hook::emit(
\OC\Files\Filesystem::CLASSNAME,
\OC\Files\Filesystem::signal_write,
array(
\OC\Files\Filesystem::signal_param_path => $path,
\OC\Files\Filesystem::signal_param_run => &$run
)
);

return $run;
}

private function runPostHooks($fileExists, $path) {
if(!$fileExists) {
OC_Hook::emit(
\OC\Files\Filesystem::CLASSNAME,
\OC\Files\Filesystem::signal_post_create,
array( \OC\Files\Filesystem::signal_param_path => $path)
);
}
OC_Hook::emit(
\OC\Files\Filesystem::CLASSNAME,
\OC\Files\Filesystem::signal_post_write,
array( \OC\Files\Filesystem::signal_param_path => $path)
);
}
}
104 changes: 5 additions & 99 deletions lib/private/filechunking.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,16 @@ static public function decodeName($name) {
/**
* @param string[] $info
*/
public function __construct($info) {
public function __construct($info, $cache = null) {
$this->info = $info;
$this->cache = $cache;
}

public function getPrefix() {
$name = $this->info['name'];
$transferid = $this->info['transferid'];
$transferId = $this->info['transferid'];

return $name.'-chunking-'.$transferid.'-';
return $name.'-chunking-'.$transferId.'-';
}

protected function getCache() {
Expand Down Expand Up @@ -72,7 +73,7 @@ public function isComplete() {
*
* @return integer assembled file size
*
* @throws \OC\InsufficientStorageException when file could not be fully
* @throws \OCP\Files\NotEnoughSpaceException when file could not be fully
* assembled due to lack of free space
*/
public function assemble($f) {
Expand Down Expand Up @@ -123,99 +124,4 @@ public function remove($index) {
$prefix = $this->getPrefix();
$cache->remove($prefix.$index);
}

public function signature_split($orgfile, $input) {
$info = unpack('n', fread($input, 2));
$blocksize = $info[1];
$this->info['transferid'] = mt_rand();
$count = 0;
$needed = array();
$cache = $this->getCache();
$prefix = $this->getPrefix();
while (!feof($orgfile)) {
$new_md5 = fread($input, 16);
if (feof($input)) {
break;
}
$data = fread($orgfile, $blocksize);
$org_md5 = md5($data, true);
if ($org_md5 == $new_md5) {
$cache->set($prefix.$count, $data);
} else {
$needed[] = $count;
}
$count++;
}
return array(
'transferid' => $this->info['transferid'],
'needed' => $needed,
'count' => $count,
);
}

/**
* Assembles the chunks into the file specified by the path.
* Also triggers the relevant hooks and proxies.
*
* @param string $path target path
*
* @return boolean assembled file size or false if file could not be created
*
* @throws \OC\InsufficientStorageException when file could not be fully
* assembled due to lack of free space
*/
public function file_assemble($path) {
$absolutePath = \OC\Files\Filesystem::normalizePath(\OC\Files\Filesystem::getView()->getAbsolutePath($path));
$data = '';
// use file_put_contents as method because that best matches what this function does
if (OC_FileProxy::runPreProxies('file_put_contents', $absolutePath, $data)
&& \OC\Files\Filesystem::isValidPath($path)) {
$path = \OC\Files\Filesystem::getView()->getRelativePath($absolutePath);
$exists = \OC\Files\Filesystem::file_exists($path);
$run = true;
if(!$exists) {
OC_Hook::emit(
\OC\Files\Filesystem::CLASSNAME,
\OC\Files\Filesystem::signal_create,
array(
\OC\Files\Filesystem::signal_param_path => $path,
\OC\Files\Filesystem::signal_param_run => &$run
)
);
}
OC_Hook::emit(
\OC\Files\Filesystem::CLASSNAME,
\OC\Files\Filesystem::signal_write,
array(
\OC\Files\Filesystem::signal_param_path => $path,
\OC\Files\Filesystem::signal_param_run => &$run
)
);
if(!$run) {
return false;
}
$target = \OC\Files\Filesystem::fopen($path, 'w');
if($target) {
$count = $this->assemble($target);
fclose($target);
if(!$exists) {
OC_Hook::emit(
\OC\Files\Filesystem::CLASSNAME,
\OC\Files\Filesystem::signal_post_create,
array( \OC\Files\Filesystem::signal_param_path => $path)
);
}
OC_Hook::emit(
\OC\Files\Filesystem::CLASSNAME,
\OC\Files\Filesystem::signal_post_write,
array( \OC\Files\Filesystem::signal_param_path => $path)
);
OC_FileProxy::runPostProxies('file_put_contents', $absolutePath, $count);
return $count > 0;
}else{
return false;
}
}
return false;
}
}
70 changes: 70 additions & 0 deletions lib/private/files/cachingchunkhandler.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
<?php
/**
* @author Thomas Müller
* @copyright 2014 Thomas Müller [email protected]
*
* This file is licensed under the Affero General Public License version 3 or
* later.
* See the COPYING-README file.
*/

namespace OC\Files;

use OCP\Files\Storage;

class CachingChunkHandler implements \OCP\Files\IChunkHandler {

/**
* @var Storage
*/
private $storage;

private $cache;

public function __construct(Storage $storage) {
$this->storage = $storage;
}

/**
* Write a chunk to a give file.
*
* @param string $fileName
* @param int $index
* @param int $numberOfChunk
* @param int $chunkSize
* @param string $data
* @return array
*/
function storeChunk($fileName, $index, $numberOfChunk, $chunkSize, $data, $transferId) {
$info = array(
'name' => $transferId,
'transferid' => $transferId,
'chunkcount' => $numberOfChunk,
''
);
$chunkHandler = new \OC_FileChunking($info, $this->cache);
$bytesWritten = $chunkHandler->store($index, $data);
if ($bytesWritten != $chunkSize) {
$chunkHandler->remove($index);
}
$complete = false;
$actualSize = $chunkHandler->getCurrentSize();

if ($chunkHandler->isComplete()) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see, so the last chunk would trigger assembling.

I'm not sure, I'd have moved this to a separate function but am not sure why. Maybe to give the caller more control of when they want to do the assembling.

If you decide to keep it this way, it might be a good idea to at least expose isComplete() on the interface.

$complete = true;
$f = $this->storage->fopen("/files" . $fileName, 'w');
$chunkHandler->assemble($f);
fclose($f);
}

return array(
'complete' => $complete,
'bytesWritten' => $bytesWritten,
'actualSize' => $actualSize
);
}

public function setFileCache($cache) {
$this->cache = $cache;
}
}
12 changes: 12 additions & 0 deletions lib/private/files/storage/common.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,10 @@

namespace OC\Files\Storage;

use OC\Files\CachingChunkHandler;
use OC\Files\Filesystem;
use OC\Files\Cache\Watcher;
use OCP\Files\IChunkHandler;

/**
* Storage backend class for providing common filesystem operation methods
Expand Down Expand Up @@ -437,4 +439,14 @@ protected function removeCachedFile($path) {
public function instanceOfStorage($class) {
return is_a($this, $class);
}

/**
* Returns the storage specific chunk handler
*
* @return IChunkHandler
*/
public function getChunkHandler() {
return new CachingChunkHandler($this);
}

}
11 changes: 11 additions & 0 deletions lib/private/files/storage/wrapper/wrapper.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@

namespace OC\Files\Storage\Wrapper;

use OCP\Files\IChunkHandler;

class Wrapper implements \OC\Files\Storage\Storage {
/**
* @var \OC\Files\Storage\Storage $storage
Expand Down Expand Up @@ -465,4 +467,13 @@ public function instanceOfStorage($class) {
public function __call($method, $args) {
return call_user_func_array(array($this->storage, $method), $args);
}

/**
* Returns the storage specific chunk handler
*
* @return IChunkHandler
*/
public function getChunkHandler() {
return $this->storage->getChunkHandler();
}
}
Loading