From 36e8e9ebf5f4703e943540ab7221263a0e245d78 Mon Sep 17 00:00:00 2001 From: Christian Kamm Date: Thu, 1 Oct 2015 11:39:09 +0200 Subject: [PATCH] Propagator: Download disk space checks #2939 * There's a critical 50 MB threshold under which syncs abort (OWNCLOUD_CRITICAL_FREE_SPACE) * The sync client always keeps 250 MB free (OWNCLOUD_FREE_SPACE) --- src/libsync/owncloudpropagator.cpp | 53 ++++++++++++++++++++++++++++++ src/libsync/owncloudpropagator.h | 31 +++++++++++++++++ src/libsync/propagatedownload.cpp | 50 ++++++++++++++++++++-------- src/libsync/propagatedownload.h | 5 ++- src/libsync/syncengine.cpp | 14 +------- 5 files changed, 125 insertions(+), 28 deletions(-) diff --git a/src/libsync/owncloudpropagator.cpp b/src/libsync/owncloudpropagator.cpp index c89cdb5c505..c1018622bdb 100644 --- a/src/libsync/owncloudpropagator.cpp +++ b/src/libsync/owncloudpropagator.cpp @@ -45,6 +45,32 @@ namespace OCC { +qint64 criticalFreeSpaceLimit() +{ + qint64 value = 50 * 1000 * 1000LL; + + static bool hasEnv = false; + static qint64 env = qgetenv("OWNCLOUD_CRITICAL_FREE_SPACE").toLongLong(&hasEnv); + if (hasEnv) { + value = env; + } + + return qBound(0LL, value, freeSpaceLimit()); +} + +qint64 freeSpaceLimit() +{ + qint64 value = 250 * 1000 * 1000LL; + + static bool hasEnv = false; + static qint64 env = qgetenv("OWNCLOUD_FREE_SPACE").toLongLong(&hasEnv); + if (hasEnv) { + value = env; + } + + return value; +} + OwncloudPropagator::~OwncloudPropagator() {} @@ -532,6 +558,24 @@ AccountPtr OwncloudPropagator::account() const return _account; } +OwncloudPropagator::DiskSpaceResult OwncloudPropagator::diskSpaceCheck() const +{ + const qint64 freeBytes = Utility::freeDiskSpace(_localDir); + if (freeBytes < 0) { + return DiskSpaceOk; + } + + if (freeBytes < criticalFreeSpaceLimit()) { + return DiskSpaceCritical; + } + + if (freeBytes - _rootJob->committedDiskSpace() < freeSpaceLimit()) { + return DiskSpaceFailure; + } + + return DiskSpaceOk; +} + // ================================================================================ PropagatorJob::JobParallelism PropagateDirectory::parallelism() @@ -660,6 +704,15 @@ void PropagateDirectory::finalize() emit finished(_hasError == SyncFileItem::NoStatus ? SyncFileItem::Success : _hasError); } +qint64 PropagateDirectory::committedDiskSpace() const +{ + qint64 needed = 0; + foreach (PropagatorJob* job, _subJobs) { + needed += job->committedDiskSpace(); + } + return needed; +} + CleanupPollsJob::~CleanupPollsJob() {} diff --git a/src/libsync/owncloudpropagator.h b/src/libsync/owncloudpropagator.h index ca22aab7add..e4f6dca896f 100644 --- a/src/libsync/owncloudpropagator.h +++ b/src/libsync/owncloudpropagator.h @@ -37,6 +37,17 @@ typedef struct ne_prop_result_set_s ne_prop_result_set; namespace OCC { +/** Free disk space threshold below which syncs will abort and not even start. + */ +qint64 criticalFreeSpaceLimit(); + +/** The client will not intentionally reduce the available free disk space below + * this limit. + * + * Uploads will still run and downloads that are small enough will continue too. + */ +qint64 freeSpaceLimit(); + class SyncJournalDb; class OwncloudPropagator; @@ -78,6 +89,13 @@ class PropagatorJob : public QObject { virtual JobParallelism parallelism() { return FullParallelism; } + /** The space that the running jobs need to complete but don't actually use yet. + * + * Note that this does *not* include the disk space that's already + * in use by running jobs for things like a download-in-progress. + */ + virtual qint64 committedDiskSpace() const { return 0; } + public slots: virtual void abort() {} @@ -203,6 +221,8 @@ class OWNCLOUDSYNC_EXPORT PropagateDirectory : public PropagatorJob { void finalize(); + qint64 committedDiskSpace() const Q_DECL_OVERRIDE; + private slots: bool possiblyRunNextJob(PropagatorJob *next) { if (next->_state == NotYetStarted) { @@ -321,6 +341,17 @@ class OwncloudPropagator : public QObject { AccountPtr account() const; + enum DiskSpaceResult + { + DiskSpaceOk, + DiskSpaceFailure, + DiskSpaceCritical + }; + + /** Checks whether there's enough disk space available to complete + * all jobs that are currently running. + */ + DiskSpaceResult diskSpaceCheck() const; private slots: diff --git a/src/libsync/propagatedownload.cpp b/src/libsync/propagatedownload.cpp index e1e4ed48951..2730ebf5975 100644 --- a/src/libsync/propagatedownload.cpp +++ b/src/libsync/propagatedownload.cpp @@ -346,6 +346,30 @@ void PropagateDownloadFileQNAM::start() FileSystem::setFileHidden(_tmpFile.fileName(), true); + _resumeStart = _tmpFile.size(); + if (_resumeStart > 0) { + if (_resumeStart == _item->_size) { + qDebug() << "File is already complete, no need to download"; + _tmpFile.close(); + downloadFinished(); + return; + } + } + + // If there's not enough space to fully download this file, stop. + const auto diskSpaceResult = _propagator->diskSpaceCheck(); + if (diskSpaceResult == OwncloudPropagator::DiskSpaceFailure) { + done(SyncFileItem::NormalError, + tr("The download would reduce free disk space below %1").arg( + Utility::octetsToString(freeSpaceLimit()))); + return; + } else if (diskSpaceResult == OwncloudPropagator::DiskSpaceCritical) { + done(SyncFileItem::FatalError, + tr("Free space on disk is less than %1").arg( + Utility::octetsToString(criticalFreeSpaceLimit()))); + return; + } + { SyncJournalDb::DownloadInfo pi; pi._etag = _item->_etag; @@ -355,24 +379,13 @@ void PropagateDownloadFileQNAM::start() _propagator->_journal->commit("download file start"); } - QMap headers; - quint64 startSize = _tmpFile.size(); - if (startSize > 0) { - if (startSize == _item->_size) { - qDebug() << "File is already complete, no need to download"; - _tmpFile.close(); - downloadFinished(); - return; - } - } - if (_item->_directDownloadUrl.isEmpty()) { // Normal job, download from oC instance _job = new GETFileJob(_propagator->account(), _propagator->_remoteFolder + _item->_file, - &_tmpFile, headers, expectedEtagForResume, startSize); + &_tmpFile, headers, expectedEtagForResume, _resumeStart); } else { // We were provided a direct URL, use that one qDebug() << Q_FUNC_INFO << "directDownloadUrl given for " << _item->_file << _item->_directDownloadUrl; @@ -384,7 +397,7 @@ void PropagateDownloadFileQNAM::start() QUrl url = QUrl::fromUserInput(_item->_directDownloadUrl); _job = new GETFileJob(_propagator->account(), url, - &_tmpFile, headers, expectedEtagForResume, startSize); + &_tmpFile, headers, expectedEtagForResume, _resumeStart); } _job->setBandwidthManager(&_propagator->_bandwidthManager); connect(_job, SIGNAL(finishedSignal()), this, SLOT(slotGetFinished())); @@ -393,6 +406,14 @@ void PropagateDownloadFileQNAM::start() _job->start(); } +qint64 PropagateDownloadFileQNAM::committedDiskSpace() const +{ + if (_state == Running) { + return qBound(0ULL, _item->_size - _resumeStart - _downloadProgress, _item->_size); + } + return 0; +} + const char owncloudCustomSoftErrorStringC[] = "owncloud-custom-soft-error-string"; void PropagateDownloadFileQNAM::slotGetFinished() { @@ -684,7 +705,8 @@ void PropagateDownloadFileQNAM::downloadFinished() void PropagateDownloadFileQNAM::slotDownloadProgress(qint64 received, qint64) { if (!_job) return; - emit progress(*_item, received + _job->resumeStart()); + _downloadProgress = received; + emit progress(*_item, _resumeStart + received); } diff --git a/src/libsync/propagatedownload.h b/src/libsync/propagatedownload.h index 85ea82da73f..e67b8d0eb8a 100644 --- a/src/libsync/propagatedownload.h +++ b/src/libsync/propagatedownload.h @@ -110,8 +110,9 @@ class PropagateDownloadFileQNAM : public PropagateItemJob { Q_OBJECT public: PropagateDownloadFileQNAM(OwncloudPropagator* propagator,const SyncFileItemPtr& item) - : PropagateItemJob(propagator, item) {} + : PropagateItemJob(propagator, item), _resumeStart(0), _downloadProgress(0) {} void start() Q_DECL_OVERRIDE; + qint64 committedDiskSpace() const Q_DECL_OVERRIDE; private slots: void slotGetFinished(); @@ -121,6 +122,8 @@ private slots: void slotChecksumFail( const QString& errMsg ); private: + quint64 _resumeStart; + qint64 _downloadProgress; QPointer _job; QFile _tmpFile; }; diff --git a/src/libsync/syncengine.cpp b/src/libsync/syncengine.cpp index e4bb8ce01dc..3d12eaec5bc 100644 --- a/src/libsync/syncengine.cpp +++ b/src/libsync/syncengine.cpp @@ -54,18 +54,6 @@ extern "C" const char *csync_instruction_str(enum csync_instructions_e instr); namespace OCC { -/* The minimum amount of space required to start a sync run */ -static qint64 minFreeSpace() -{ - static bool ok = false; - static qint64 freeSpace = qgetenv("OWNCLOUD_MIN_FREE_SPACE").toLongLong(&ok); - if (ok) { - return freeSpace; - } - - return 250 * 1000 * 1000LL; -} - bool SyncEngine::_syncRunning = false; SyncEngine::SyncEngine(AccountPtr account, CSYNC *ctx, const QString& localPath, @@ -612,7 +600,7 @@ void SyncEngine::startSync() } // Check free size on disk first. - const qint64 minFree = minFreeSpace(); + const qint64 minFree = criticalFreeSpaceLimit(); const qint64 freeBytes = Utility::freeDiskSpace(_localPath); if (freeBytes >= 0) { qDebug() << "There are" << freeBytes << "bytes available at" << _localPath