From a663692fa4b1fe66121d03b123ae39ef4ffc87a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20M=C3=BCller?= Date: Thu, 13 Nov 2014 15:17:12 +0100 Subject: [PATCH] introduction of interface IChunkHandler and a very first stupid implementation reusing the existing chunking mechanism --- lib/private/connector/sabre/file.php | 35 ++++++----- lib/private/filechunking.php | 37 ++--------- lib/private/files/cachingchunkhandler.php | 62 +++++++++++++++++++ lib/private/files/storage/common.php | 12 ++++ lib/private/files/storage/wrapper/wrapper.php | 11 ++++ lib/public/files/ichunkhandler.php | 30 +++++++++ lib/public/files/storage.php | 7 +++ 7 files changed, 147 insertions(+), 47 deletions(-) create mode 100644 lib/private/files/cachingchunkhandler.php create mode 100644 lib/public/files/ichunkhandler.php diff --git a/lib/private/connector/sabre/file.php b/lib/private/connector/sabre/file.php index d93b8e68eb65..bfc185039581 100644 --- a/lib/private/connector/sabre/file.php +++ b/lib/private/connector/sabre/file.php @@ -238,37 +238,43 @@ private function createFileChunked($data) if (empty($info)) { throw new \Sabre\DAV\Exception\NotImplemented(); } - $chunk_handler = new OC_FileChunking($info); - $bytesWritten = $chunk_handler->store($info['index'], $data); + + // we first assembly the target file as a part file + $targetPath = $path . '/' . $info['name']; + if (isset($_SERVER['CONTENT_LENGTH'])) { + $expected = $_SERVER['CONTENT_LENGTH']; + } else { + $expected = -1; + } + $partFilePath = $path . '/' . $info['name'] . '.ocTransferId' . $info['transferid'] . '.part'; + /** @var \OC\Files\Storage\Storage $storage */ + list($storage,) = $this->fileView->resolvePath($partFilePath); + $storeData = $storage->getChunkHandler()->storeChunk($partFilePath, $info['index'], $info['chunkcount'], $expected, $data, $info['transferid']); + $bytesWritten = $storeData['bytesWritten']; //detect aborted upload if (isset ($_SERVER['REQUEST_METHOD']) && $_SERVER['REQUEST_METHOD'] === 'PUT' ) { if (isset($_SERVER['CONTENT_LENGTH'])) { - $expected = $_SERVER['CONTENT_LENGTH']; if ($bytesWritten != $expected) { - $chunk_handler->remove($info['index']); throw new \Sabre\DAV\Exception\BadRequest( 'expected filesize ' . $expected . ' got ' . $bytesWritten); } } } - if ($chunk_handler->isComplete()) { + if ($storeData['complete']) { try { - // we first assembly the target file as a part file - $partFile = $path . '/' . $info['name'] . '.ocTransferId' . $info['transferid'] . '.part'; - $chunk_handler->file_assemble($partFile); // here is the final atomic rename - $targetPath = $path . '/' . $info['name']; - $renameOkay = $this->fileView->rename($partFile, $targetPath); - $fileExists = $this->fileView->file_exists($targetPath); + // TODO: will this properly trigger all hooks + $renameOkay = $storage->rename('/files' . $partFilePath, '/files' . $targetPath); + $fileExists = $storage->file_exists('/files' . $targetPath); if ($renameOkay === false || $fileExists === false) { - \OC_Log::write('webdav', '\OC\Files\Filesystem::rename() failed', \OC_Log::ERROR); + \OC::$server->getLogger()->error('\OC\Files\Filesystem::rename() failed', array('app'=>'webdav')); // only delete if an error occurred and the target file was already created if ($fileExists) { - $this->fileView->unlink($targetPath); + $storage->unlink('/files' . $targetPath); } throw new \Sabre\DAV\Exception('Could not rename part file assembled from chunks'); } @@ -276,7 +282,8 @@ private function createFileChunked($data) // allow sync clients to send the mtime along in a header $mtime = OC_Request::hasModificationTime(); if ($mtime !== false) { - if($this->fileView->touch($targetPath, $mtime)) { + // TODO: will this update the cache properly - e.g. smb where we cannot change the mtime ??? + if($storage->touch('/files' . $targetPath, $mtime)) { header('X-OC-MTime: accepted'); } } diff --git a/lib/private/filechunking.php b/lib/private/filechunking.php index 990499e40b49..1f95acef3f52 100644 --- a/lib/private/filechunking.php +++ b/lib/private/filechunking.php @@ -25,9 +25,9 @@ public function __construct($info) { public function getPrefix() { $name = $this->info['name']; - $transferid = $this->info['transferid']; + $transferId = $this->info['transferid']; - return $name.'-chunking-'.$transferid.'-'; + return $name.'-chunking-'.$transferId.'-'; } protected function getCache() { @@ -72,7 +72,7 @@ public function isComplete() { * * @return integer assembled file size * - * @throws \OC\InsufficientStorageException when file could not be fully + * @throws \OCP\Files\NotEnoughSpaceException when file could not be fully * assembled due to lack of free space */ public function assemble($f) { @@ -124,35 +124,6 @@ public function remove($index) { $cache->remove($prefix.$index); } - public function signature_split($orgfile, $input) { - $info = unpack('n', fread($input, 2)); - $blocksize = $info[1]; - $this->info['transferid'] = mt_rand(); - $count = 0; - $needed = array(); - $cache = $this->getCache(); - $prefix = $this->getPrefix(); - while (!feof($orgfile)) { - $new_md5 = fread($input, 16); - if (feof($input)) { - break; - } - $data = fread($orgfile, $blocksize); - $org_md5 = md5($data, true); - if ($org_md5 == $new_md5) { - $cache->set($prefix.$count, $data); - } else { - $needed[] = $count; - } - $count++; - } - return array( - 'transferid' => $this->info['transferid'], - 'needed' => $needed, - 'count' => $count, - ); - } - /** * Assembles the chunks into the file specified by the path. * Also triggers the relevant hooks and proxies. @@ -161,7 +132,7 @@ public function signature_split($orgfile, $input) { * * @return boolean assembled file size or false if file could not be created * - * @throws \OC\InsufficientStorageException when file could not be fully + * @throws \OCP\Files\NotEnoughSpaceException when file could not be fully * assembled due to lack of free space */ public function file_assemble($path) { diff --git a/lib/private/files/cachingchunkhandler.php b/lib/private/files/cachingchunkhandler.php new file mode 100644 index 000000000000..06c7be616d87 --- /dev/null +++ b/lib/private/files/cachingchunkhandler.php @@ -0,0 +1,62 @@ +storage = $storage; + } + + /** + * Write a chunk to a give file. + * + * @param string $fileName + * @param int $index + * @param int $numberOfChunk + * @param int $chunkSize + * @param string $data + * @return array + */ + function storeChunk($fileName, $index, $numberOfChunk, $chunkSize, $data, $transferId) { + $info = array( + 'name' => $transferId, + 'transferid' => $transferId, + 'chunkcount' => $numberOfChunk, + '' + ); + $chunkHandler = new \OC_FileChunking($info); + $bytesWritten = $chunkHandler->store($index, $data); + if ($bytesWritten != $chunkSize) { + $chunkHandler->remove($index); + } + $complete = false; + if ($chunkHandler->isComplete()) { + $complete = true; + $f = $this->storage->fopen("/files" . $fileName, 'w'); + $chunkHandler->assemble($f); + fclose($f); + } + + return array( + 'complete' => $complete, + 'bytesWritten' => $bytesWritten, + 'actualSize' => $chunkHandler->getCurrentSize() + ); + } +} diff --git a/lib/private/files/storage/common.php b/lib/private/files/storage/common.php index 518d3ec400c9..683f53dceca2 100644 --- a/lib/private/files/storage/common.php +++ b/lib/private/files/storage/common.php @@ -8,8 +8,10 @@ namespace OC\Files\Storage; +use OC\Files\CachingChunkHandler; use OC\Files\Filesystem; use OC\Files\Cache\Watcher; +use OCP\Files\IChunkHandler; /** * Storage backend class for providing common filesystem operation methods @@ -437,4 +439,14 @@ protected function removeCachedFile($path) { public function instanceOfStorage($class) { return is_a($this, $class); } + + /** + * Returns the storage specific chunk handler + * + * @return IChunkHandler + */ + public function getChunkHandler() { + return new CachingChunkHandler($this); + } + } diff --git a/lib/private/files/storage/wrapper/wrapper.php b/lib/private/files/storage/wrapper/wrapper.php index d899c88363f2..93575ddc7895 100644 --- a/lib/private/files/storage/wrapper/wrapper.php +++ b/lib/private/files/storage/wrapper/wrapper.php @@ -8,6 +8,8 @@ namespace OC\Files\Storage\Wrapper; +use OCP\Files\IChunkHandler; + class Wrapper implements \OC\Files\Storage\Storage { /** * @var \OC\Files\Storage\Storage $storage @@ -465,4 +467,13 @@ public function instanceOfStorage($class) { public function __call($method, $args) { return call_user_func_array(array($this->storage, $method), $args); } + + /** + * Returns the storage specific chunk handler + * + * @return IChunkHandler + */ + public function getChunkHandler() { + return $this->storage->getChunkHandler(); + } } diff --git a/lib/public/files/ichunkhandler.php b/lib/public/files/ichunkhandler.php new file mode 100644 index 000000000000..facc54eecd8f --- /dev/null +++ b/lib/public/files/ichunkhandler.php @@ -0,0 +1,30 @@ +