From 796b4f6ec0d360a2e88a0718dc6e511f7818d5ea Mon Sep 17 00:00:00 2001 From: Robin McCorkell Date: Fri, 23 Jan 2015 21:53:21 +0000 Subject: [PATCH] Store storage availability in database Storage status is saved in the database. Failed storages are rechecked every 10 minutes, while working storages are rechecked every request. Using the files_external app will recheck all external storages when the settings page is viewed, or whenever an external storage is saved. --- apps/files_external/lib/config.php | 4 +- db_structure.xml | 12 + lib/private/files/cache/storage.php | 48 ++- lib/private/files/storage/common.php | 17 + .../files/storage/wrapper/availability.php | 384 ++++++++++++++++++ lib/private/files/storage/wrapper/wrapper.php | 18 + lib/private/util.php | 5 + lib/public/files/storage.php | 21 + .../files/storage/wrapper/availability.php | 137 +++++++ version.php | 2 +- 10 files changed, 637 insertions(+), 11 deletions(-) create mode 100644 lib/private/files/storage/wrapper/availability.php create mode 100644 tests/lib/files/storage/wrapper/availability.php diff --git a/apps/files_external/lib/config.php b/apps/files_external/lib/config.php index ddfab4398792..ccd7469b3bde 100644 --- a/apps/files_external/lib/config.php +++ b/apps/files_external/lib/config.php @@ -454,7 +454,9 @@ private static function getBackendStatus($class, $options, $isPersonal) { if (class_exists($class)) { try { $storage = new $class($options); - return $storage->test($isPersonal); + $result = $storage->test($isPersonal); + $storage->setAvailability($result); + return $result; } catch (Exception $exception) { \OCP\Util::logException('files_external', $exception); return false; diff --git a/db_structure.xml b/db_structure.xml index eb6540047d6d..73d4a4989312 100644 --- a/db_structure.xml +++ b/db_structure.xml @@ -102,6 +102,18 @@ 4 + + available + boolean + true + true + + + + last_checked + integer + + storages_id_index true diff --git a/lib/private/files/cache/storage.php b/lib/private/files/cache/storage.php index d7d57811a7dd..078936f88e92 100644 --- a/lib/private/files/cache/storage.php +++ b/lib/private/files/cache/storage.php @@ -1,6 +1,7 @@ + * Copyright (c) 2015 Robin McCorkell * This file is licensed under the Affero General Public License version 3 or * later. * See the COPYING-README file. @@ -21,8 +22,9 @@ class Storage { /** * @param \OC\Files\Storage\Storage|string $storage + * @param bool $isAvailable */ - public function __construct($storage) { + public function __construct($storage, $isAvailable = true) { if ($storage instanceof \OC\Files\Storage\Storage) { $this->storageId = $storage->getId(); } else { @@ -30,17 +32,25 @@ public function __construct($storage) { } $this->storageId = self::adjustStorageId($this->storageId); - $sql = 'SELECT `numeric_id` FROM `*PREFIX*storages` WHERE `id` = ?'; - $result = \OC_DB::executeAudited($sql, array($this->storageId)); - if ($row = $result->fetchRow()) { + if ($row = self::getStorageById($this->storageId)) { $this->numericId = $row['numeric_id']; } else { - $sql = 'INSERT INTO `*PREFIX*storages` (`id`) VALUES(?)'; - \OC_DB::executeAudited($sql, array($this->storageId)); + $sql = 'INSERT INTO `*PREFIX*storages` (`id`, `available`) VALUES(?, ?)'; + \OC_DB::executeAudited($sql, array($this->storageId, $isAvailable)); $this->numericId = \OC_DB::insertid('*PREFIX*storages'); } } + /** + * @param string $storageId + * @return array|null + */ + public static function getStorageById($storageId) { + $sql = 'SELECT * FROM `*PREFIX*storages` WHERE `id` = ?'; + $result = \OC_DB::executeAudited($sql, array($storageId)); + return $result->fetchRow(); + } + /** * Adjusts the storage id to use md5 if too long * @param string $storageId storage id @@ -81,15 +91,35 @@ public static function getStorageId($numericId) { public static function getNumericStorageId($storageId) { $storageId = self::adjustStorageId($storageId); - $sql = 'SELECT `numeric_id` FROM `*PREFIX*storages` WHERE `id` = ?'; - $result = \OC_DB::executeAudited($sql, array($storageId)); - if ($row = $result->fetchRow()) { + if ($row = self::getStorageById($storageId)) { return $row['numeric_id']; } else { return null; } } + /** + * @return array|null [ available, last_checked ] + */ + public function getAvailability() { + if ($row = self::getStorageById($this->storageId)) { + return [ + 'available' => $row['available'], + 'last_checked' => $row['last_checked'] + ]; + } else { + return null; + } + } + + /** + * @param bool $isAvailable + */ + public function setAvailability($isAvailable) { + $sql = 'UPDATE `*PREFIX*storages` SET `available` = ?, `last_checked` = ? WHERE `id` = ?'; + \OC_DB::executeAudited($sql, array($isAvailable, time(), $this->storageId)); + } + /** * @param string $storageId * @return bool diff --git a/lib/private/files/storage/common.php b/lib/private/files/storage/common.php index b2bf41f751c8..3869922e27e0 100644 --- a/lib/private/files/storage/common.php +++ b/lib/private/files/storage/common.php @@ -451,4 +451,21 @@ public function getDirectDownload($path) { return []; } + /** + * Get availability of the storage + * + * @return array [ available, last_checked ] + */ + public function getAvailability() { + return $this->getStorageCache()->getAvailability(); + } + + /** + * Set availability of the storage + * + * @param bool $isAvailable + */ + public function setAvailability($isAvailable) { + $this->getStorageCache()->setAvailability($isAvailable); + } } diff --git a/lib/private/files/storage/wrapper/availability.php b/lib/private/files/storage/wrapper/availability.php new file mode 100644 index 000000000000..fdc22cdfd026 --- /dev/null +++ b/lib/private/files/storage/wrapper/availability.php @@ -0,0 +1,384 @@ + + * This file is licensed under the Affero General Public License version 3 or + * later. + * See the COPYING-README file. + */ + +namespace OC\Files\Storage\Wrapper; + +/** + * Availability checker for storages + * + * Throws a StorageNotAvailableException for storages with known failures + */ +class Availability extends Wrapper { + const RECHECK_TTL = 600; // 10 minutes + + /** + * @return bool + */ + private function updateAvailability() { + try { + $result = $this->test(); + } catch (\Exception $e) { + $result = false; + } + $this->setAvailability($result); + return $result; + } + + /** + * @return bool + */ + private function isAvailable() { + $availability = $this->getAvailability(); + if (!$availability['available']) { + // trigger a recheck if TTL reached + if ((time() - $availability['last_checked']) > self::RECHECK_TTL) { + return $this->updateAvailability(); + } + } + return $availability['available']; + } + + /** + * @throws \OCP\Files\StorageNotAvailableException + */ + private function checkAvailability() { + if (!$this->isAvailable()) { + throw new \OCP\Files\StorageNotAvailableException(); + } + } + + public function mkdir($path) { + $this->checkAvailability(); + try { + return parent::mkdir($path); + } catch (\OCP\Files\StorageNotAvailableException $e) { + $this->setAvailability(false); + throw $e; + } + } + + public function rmdir($path) { + $this->checkAvailability(); + try { + return parent::rmdir($path); + } catch (\OCP\Files\StorageNotAvailableException $e) { + $this->setAvailability(false); + throw $e; + } + } + + public function opendir($path) { + $this->checkAvailability(); + try { + return parent::opendir($path); + } catch (\OCP\Files\StorageNotAvailableException $e) { + $this->setAvailability(false); + throw $e; + } + } + + public function is_dir($path) { + $this->checkAvailability(); + try { + return parent::is_dir($path); + } catch (\OCP\Files\StorageNotAvailableException $e) { + $this->setAvailability(false); + throw $e; + } + } + + public function is_file($path) { + $this->checkAvailability(); + try { + return parent::is_file($path); + } catch (\OCP\Files\StorageNotAvailableException $e) { + $this->setAvailability(false); + throw $e; + } + } + + public function stat($path) { + $this->checkAvailability(); + try { + return parent::stat($path); + } catch (\OCP\Files\StorageNotAvailableException $e) { + $this->setAvailability(false); + throw $e; + } + } + + public function filetype($path) { + $this->checkAvailability(); + try { + return parent::filetype($path); + } catch (\OCP\Files\StorageNotAvailableException $e) { + $this->setAvailability(false); + throw $e; + } + } + + public function filesize($path) { + $this->checkAvailability(); + try { + return parent::filesize($path); + } catch (\OCP\Files\StorageNotAvailableException $e) { + $this->setAvailability(false); + throw $e; + } + } + + public function isCreatable($path) { + $this->checkAvailability(); + try { + return parent::isCreatable($path); + } catch (\OCP\Files\StorageNotAvailableException $e) { + $this->setAvailability(false); + throw $e; + } + } + + public function isReadable($path) { + $this->checkAvailability(); + try { + return parent::isReadable($path); + } catch (\OCP\Files\StorageNotAvailableException $e) { + $this->setAvailability(false); + throw $e; + } + } + + public function isUpdatable($path) { + $this->checkAvailability(); + try { + return parent::isUpdatable($path); + } catch (\OCP\Files\StorageNotAvailableException $e) { + $this->setAvailability(false); + throw $e; + } + } + + public function isDeletable($path) { + $this->checkAvailability(); + try { + return parent::isDeletable($path); + } catch (\OCP\Files\StorageNotAvailableException $e) { + $this->setAvailability(false); + throw $e; + } + } + + public function isSharable($path) { + $this->checkAvailability(); + try { + return parent::isSharable($path); + } catch (\OCP\Files\StorageNotAvailableException $e) { + $this->setAvailability(false); + throw $e; + } + } + + public function getPermissions($path) { + $this->checkAvailability(); + try { + return parent::getPermissions($path); + } catch (\OCP\Files\StorageNotAvailableException $e) { + $this->setAvailability(false); + throw $e; + } + } + + public function file_exists($path) { + $this->checkAvailability(); + try { + return parent::file_exists($path); + } catch (\OCP\Files\StorageNotAvailableException $e) { + $this->setAvailability(false); + throw $e; + } + } + + public function filemtime($path) { + $this->checkAvailability(); + try { + return parent::filemtime($path); + } catch (\OCP\Files\StorageNotAvailableException $e) { + $this->setAvailability(false); + throw $e; + } + } + + public function file_get_contents($path) { + $this->checkAvailability(); + try { + return parent::file_get_contents($path); + } catch (\OCP\Files\StorageNotAvailableException $e) { + $this->setAvailability(false); + throw $e; + } + } + + public function file_put_contents($path, $data) { + $this->checkAvailability(); + try { + return parent::file_put_contents($path, $data); + } catch (\OCP\Files\StorageNotAvailableException $e) { + $this->setAvailability(false); + throw $e; + } + } + + public function unlink($path) { + $this->checkAvailability(); + try { + return parent::unlink($path); + } catch (\OCP\Files\StorageNotAvailableException $e) { + $this->setAvailability(false); + throw $e; + } + } + + public function rename($path1, $path2) { + $this->checkAvailability(); + try { + return parent::rename($path1, $path2); + } catch (\OCP\Files\StorageNotAvailableException $e) { + $this->setAvailability(false); + throw $e; + } + } + + public function copy($path1, $path2) { + $this->checkAvailability(); + try { + return parent::copy($path1, $path2); + } catch (\OCP\Files\StorageNotAvailableException $e) { + $this->setAvailability(false); + throw $e; + } + } + + public function fopen($path, $mode) { + $this->checkAvailability(); + try { + return parent::fopen($path, $mode); + } catch (\OCP\Files\StorageNotAvailableException $e) { + $this->setAvailability(false); + throw $e; + } + } + + public function getMimeType($path) { + $this->checkAvailability(); + try { + return parent::getMimeType($path); + } catch (\OCP\Files\StorageNotAvailableException $e) { + $this->setAvailability(false); + throw $e; + } + } + + public function hash($type, $path, $raw = false) { + $this->checkAvailability(); + try { + return parent::hash($type, $path, $raw); + } catch (\OCP\Files\StorageNotAvailableException $e) { + $this->setAvailability(false); + throw $e; + } + } + + public function free_space($path) { + $this->checkAvailability(); + try { + return parent::free_space($path); + } catch (\OCP\Files\StorageNotAvailableException $e) { + $this->setAvailability(false); + throw $e; + } + } + + public function search($query) { + $this->checkAvailability(); + try { + return parent::search($query); + } catch (\OCP\Files\StorageNotAvailableException $e) { + $this->setAvailability(false); + throw $e; + } + } + + public function touch($path, $mtime = null) { + $this->checkAvailability(); + try { + return parent::touch($path, $mtime); + } catch (\OCP\Files\StorageNotAvailableException $e) { + $this->setAvailability(false); + throw $e; + } + } + + public function getLocalFile($path) { + $this->checkAvailability(); + try { + return parent::getLocalFile($path); + } catch (\OCP\Files\StorageNotAvailableException $e) { + $this->setAvailability(false); + throw $e; + } + } + + public function getLocalFolder($path) { + $this->checkAvailability(); + try { + return parent::getLocalFolder($path); + } catch (\OCP\Files\StorageNotAvailableException $e) { + $this->setAvailability(false); + throw $e; + } + } + + public function hasUpdated($path, $time) { + $this->checkAvailability(); + try { + return parent::hasUpdated($path, $time); + } catch (\OCP\Files\StorageNotAvailableException $e) { + $this->setAvailability(false); + throw $e; + } + } + + public function getOwner($path) { + $this->checkAvailability(); + try { + return parent::getOwner($path); + } catch (\OCP\Files\StorageNotAvailableException $e) { + $this->setAvailability(false); + throw $e; + } + } + + public function getETag($path) { + $this->checkAvailability(); + try { + return parent::getETag($path); + } catch (\OCP\Files\StorageNotAvailableException $e) { + $this->setAvailability(false); + throw $e; + } + } + + public function getDirectDownload($path) { + $this->checkAvailability(); + try { + return parent::getDirectDownload($path); + } catch (\OCP\Files\StorageNotAvailableException $e) { + $this->setAvailability(false); + throw $e; + } + } +} diff --git a/lib/private/files/storage/wrapper/wrapper.php b/lib/private/files/storage/wrapper/wrapper.php index ea9de2873611..efe3503ee108 100644 --- a/lib/private/files/storage/wrapper/wrapper.php +++ b/lib/private/files/storage/wrapper/wrapper.php @@ -477,4 +477,22 @@ public function __call($method, $args) { public function getDirectDownload($path) { return $this->storage->getDirectDownload($path); } + + /** + * Get availability of the storage + * + * @return array [ available, last_checked ] + */ + public function getAvailability() { + return $this->storage->getAvailability(); + } + + /** + * Set availability of the storage + * + * @param bool $isAvailable + */ + public function setAvailability($isAvailable) { + $this->storage->setAvailability($isAvailable); + } } diff --git a/lib/private/util.php b/lib/private/util.php index 9a01ca3ac959..6a6a28f57a15 100644 --- a/lib/private/util.php +++ b/lib/private/util.php @@ -129,6 +129,11 @@ public static function setupFS($user = '') { //trigger creation of user home and /files folder \OC::$server->getUserFolder($user); + // install storage availability wrapper + \OC\Files\Filesystem::addStorageWrapper('oc_availability', function ($mountPoint, $storage) { + return new \OC\Files\Storage\Wrapper\Availability(['storage' => $storage]); + }); + OC_Hook::emit('OC_Filesystem', 'setup', array('user' => $user, 'user_dir' => $userDir)); } \OC::$server->getEventLogger()->end('setup_fs'); diff --git a/lib/public/files/storage.php b/lib/public/files/storage.php index 36d5b800df6a..81b488741070 100644 --- a/lib/public/files/storage.php +++ b/lib/public/files/storage.php @@ -345,4 +345,25 @@ public function instanceOfStorage($class); * @return array */ public function getDirectDownload($path); + + /** + * Test a storage + * + * @return bool + */ + public function test(); + + /** + * Get availability of the storage + * + * @return array [ available, last_checked ] + */ + public function getAvailability(); + + /** + * Set availability of the storage + * + * @param bool $isAvailable + */ + public function setAvailability($isAvailable); } diff --git a/tests/lib/files/storage/wrapper/availability.php b/tests/lib/files/storage/wrapper/availability.php new file mode 100644 index 000000000000..41da3b9dde6f --- /dev/null +++ b/tests/lib/files/storage/wrapper/availability.php @@ -0,0 +1,137 @@ + + * This file is licensed under the Affero General Public License version 3 or + * later. + * See the COPYING-README file. + */ + +namespace Test\Files\Storage\Wrapper; + +class Availability extends \Test\TestCase { + protected function getWrapperInstance() { + $storage = $this->getMockBuilder('\OC\Files\Storage\Temporary') + ->disableOriginalConstructor() + ->getMock(); + $wrapper = new \OC\Files\Storage\Wrapper\Availability(['storage' => $storage]); + return [$storage, $wrapper]; + } + + /** + * Storage is available + */ + public function testAvailable() { + list($storage, $wrapper) = $this->getWrapperInstance(); + $storage->expects($this->once()) + ->method('getAvailability') + ->willReturn(['available' => true, 'last_checked' => 0]); + $storage->expects($this->never()) + ->method('test'); + $storage->expects($this->once()) + ->method('mkdir'); + + $wrapper->mkdir('foobar'); + } + + /** + * Storage marked unavailable, TTL not expired + * + * @expectedException \OCP\Files\StorageNotAvailableException + */ + public function testUnavailable() { + list($storage, $wrapper) = $this->getWrapperInstance(); + $storage->expects($this->once()) + ->method('getAvailability') + ->willReturn(['available' => false, 'last_checked' => time()]); + $storage->expects($this->never()) + ->method('test'); + $storage->expects($this->never()) + ->method('mkdir'); + + $wrapper->mkdir('foobar'); + } + + /** + * Storage marked unavailable, TTL expired + */ + public function testUnavailableRecheck() { + list($storage, $wrapper) = $this->getWrapperInstance(); + $storage->expects($this->once()) + ->method('getAvailability') + ->willReturn(['available' => false, 'last_checked' => 0]); + $storage->expects($this->once()) + ->method('test') + ->willReturn(true); + $storage->expects($this->once()) + ->method('setAvailability') + ->with($this->equalTo(true)); + $storage->expects($this->once()) + ->method('mkdir'); + + $wrapper->mkdir('foobar'); + } + + /** + * Storage marked available, but throws StorageNotAvailableException + * + * @expectedException \OCP\Files\StorageNotAvailableException + */ + public function testAvailableThrowStorageNotAvailable() { + list($storage, $wrapper) = $this->getWrapperInstance(); + $storage->expects($this->once()) + ->method('getAvailability') + ->willReturn(['available' => true, 'last_checked' => 0]); + $storage->expects($this->never()) + ->method('test'); + $storage->expects($this->once()) + ->method('mkdir') + ->will($this->throwException(new \OCP\Files\StorageNotAvailableException())); + $storage->expects($this->once()) + ->method('setAvailability') + ->with($this->equalTo(false)); + + $wrapper->mkdir('foobar'); + } + + /** + * Storage available, but call fails + * Method failure does not indicate storage unavailability + */ + public function testAvailableFailure() { + list($storage, $wrapper) = $this->getWrapperInstance(); + $storage->expects($this->once()) + ->method('getAvailability') + ->willReturn(['available' => true, 'last_checked' => 0]); + $storage->expects($this->never()) + ->method('test'); + $storage->expects($this->once()) + ->method('mkdir') + ->willReturn(false); + $storage->expects($this->never()) + ->method('setAvailability'); + + $wrapper->mkdir('foobar'); + } + + /** + * Storage available, but throws exception + * Standard exception does not indicate storage unavailability + * + * @expectedException \Exception + */ + public function testAvailableThrow() { + list($storage, $wrapper) = $this->getWrapperInstance(); + $storage->expects($this->once()) + ->method('getAvailability') + ->willReturn(['available' => true, 'last_checked' => 0]); + $storage->expects($this->never()) + ->method('test'); + $storage->expects($this->once()) + ->method('mkdir') + ->will($this->throwException(new \Exception())); + $storage->expects($this->never()) + ->method('setAvailability'); + + $wrapper->mkdir('foobar'); + } +} diff --git a/version.php b/version.php index e50220be11b3..07e4f1e88ce9 100644 --- a/version.php +++ b/version.php @@ -3,7 +3,7 @@ // We only can count up. The 4. digit is only for the internal patchlevel to trigger DB upgrades // between betas, final and RCs. This is _not_ the public version number. Reset minor/patchlevel // when updating major/minor version number. -$OC_Version=array(8, 0, 0, 7); +$OC_Version=array(8, 0, 0, 8); // The human readable string $OC_VersionString='8.0';