Group requests to ensure balance between bandwidth utilization and bookkeeping #5391 #5390 #4498 #1633 #4454

src/libsync/CMakeLists.txt

@@ -55,6 +55,7 @@ set(libsync_SRCS
     propagateremotedelete.cpp
     propagateremotemove.cpp
     propagateremotemkdir.cpp
+    propagatefiles.cpp
     syncengine.cpp
     syncfilestatus.cpp
     syncfilestatustracker.cpp

src/libsync/capabilities.cpp

@@ -116,6 +116,15 @@ bool Capabilities::chunkingNg() const
     return _capabilities["dav"].toMap()["chunking"].toByteArray() >= "1.0";
 }
 
+bool Capabilities::scheduling() const
+{
+    static const auto scheduling = qgetenv("OWNCLOUD_SCHEDULING");
+    if (scheduling == "0") return false;
+    if (scheduling == "1") return true;
+
+    return _capabilities["dav"].toMap()["scheduling"].toByteArray() >= "1.0";
+}
+
 bool Capabilities::chunkingParallelUploadDisabled() const
 {
     return _capabilities["dav"].toMap()["chunkingParallelUploadDisabled"].toBool();

src/libsync/capabilities.h

@@ -41,6 +41,7 @@ class OWNCLOUDSYNC_EXPORT Capabilities {
     int  sharePublicLinkExpireDateDays() const;
     bool shareResharing() const;
     bool chunkingNg() const;
+    bool scheduling() const;
 
     /// disable parallel upload in chunking
     bool chunkingParallelUploadDisabled() const;

src/libsync/configfile.cpp

@@ -53,6 +53,7 @@ static const char updateCheckIntervalC[] = "updateCheckInterval";
 static const char geometryC[] = "geometry";
 static const char timeoutC[] = "timeout";
 static const char chunkSizeC[] = "chunkSize";
+static const char smallFileSizeC[] = "smallFileSize";
 
 static const char proxyHostC[] = "Proxy/host";
 static const char proxyTypeC[] = "Proxy/type";
@@ -130,6 +131,12 @@ quint64 ConfigFile::chunkSize() const
     return settings.value(QLatin1String(chunkSizeC), 10*1000*1000).toLongLong(); // default to 10 MB
 }
 
+quint64 ConfigFile::smallFileSize() const
+{
+    QSettings settings(configFile(), QSettings::IniFormat);
+    return settings.value(QLatin1String(smallFileSizeC), 500*1000).toLongLong(); // default to 500 kB
+}
+
 void ConfigFile::setOptionalDesktopNotifications(bool show)
 {
     QSettings settings(configFile(), QSettings::IniFormat);

src/libsync/configfile.h

@@ -115,6 +115,7 @@ class OWNCLOUDSYNC_EXPORT ConfigFile
 
     int timeout() const;
     quint64 chunkSize() const;
+    quint64 smallFileSize() const;
 
     void saveGeometry(QWidget *w);
     void restoreGeometry(QWidget *w);

src/libsync/owncloudpropagator.cpp

@@ -313,6 +313,13 @@ void OwncloudPropagator::start(const SyncFileItemVector& items)
     directories.push(qMakePair(QString(), _rootJob.data()));
     QVector<PropagatorJob*> directoriesToRemove;
     QString removedDirectory;
+
+    // This needs to be changed before marging - capability to switch on/off scheduling, since
+    // server needs to decide if to do that - scheduling will also bring more features in the future
+    // TODO: change it before marging to -> bool enableScheduledRequests = account()->capabilities().scheduling();
+    bool enableScheduledRequests = true;
+
+    PropagateFiles* filesJob = new PropagateFiles(this);
     foreach(const SyncFileItemPtr &item, items) {
 
         if (!removedDirectory.isEmpty() && item->_file.startsWith(removedDirectory)) {
@@ -389,6 +396,9 @@ void OwncloudPropagator::start(const SyncFileItemVector& items)
                 currentDirJob->append(dir);
             }
             directories.push(qMakePair(item->destination() + "/" , dir));
+        } else if (enableScheduledRequests
+              && (item->_instruction == CSYNC_INSTRUCTION_NEW || item->_instruction == CSYNC_INSTRUCTION_SYNC)) {
+            filesJob->append(item);
         } else if (PropagateItemJob* current = createJob(item)) {
             if (item->_instruction == CSYNC_INSTRUCTION_TYPE_CHANGE) {
                 // will delete directories, so defer execution
@@ -400,6 +410,13 @@ void OwncloudPropagator::start(const SyncFileItemVector& items)
         }
     }
 
+    if (enableScheduledRequests && !filesJob->isEmpty()){
+        // This job has parallelism WaitForFinished to allow directoriesToRemove be last
+        _rootJob->append(filesJob);
+    } else {
+        delete filesJob;
+    }
+
     foreach(PropagatorJob* it, directoriesToRemove) {
         _rootJob->append(it);
     }
@@ -459,6 +476,20 @@ quint64 OwncloudPropagator::chunkSize()
     return chunkSize;
 }
 
+quint64 OwncloudPropagator::smallFileSize()
+{
+    // A small filesize item is a file whose transfer time
+    // typically will be lower than its bookkeeping time.
+    static uint smallFileSize;
+    if (!smallFileSize) {
+        smallFileSize = qgetenv("OWNCLOUD_SMALLFILE_SIZE").toUInt();
+        if (smallFileSize == 0) {
+            ConfigFile cfg;
+            smallFileSize = cfg.smallFileSize();
+        }
+    }
+    return smallFileSize;
+}
 
 bool OwncloudPropagator::localFileNameClash( const QString& relFile )
 {

src/libsync/owncloudpropagator.h

@@ -257,7 +257,6 @@ class PropagateIgnoreJob : public PropagateItemJob {
 class OwncloudPropagator : public QObject {
     Q_OBJECT
 
-    PropagateItemJob *createJob(const SyncFileItemPtr& item);
     QScopedPointer<PropagateDirectory> _rootJob;
 
 public:
@@ -322,6 +321,7 @@ class OwncloudPropagator : public QObject {
 
     /** returns the size of chunks in bytes  */
     static quint64 chunkSize();
+    static quint64 smallFileSize();
 
     AccountPtr account() const;
 
@@ -337,7 +337,11 @@ class OwncloudPropagator : public QObject {
      */
     DiskSpaceResult diskSpaceCheck() const;
 
+    PropagateItemJob *createJob(const SyncFileItemPtr& item);
 
+    int runningAtRootJob(){
+        return _rootJob.data()->_runningNow;
+    }
 
 private slots:
 
@@ -410,6 +414,106 @@ private slots:
     void slotPollFinished();
 };
 
+/**
+ * @brief The PropagateFiles class is a container class.
+ *
+ * It will also ensure proper bandwidth utilization vs bookkeeping balance
+ *
+ * @ingroup libsync
+ *
+ * State Machine:
+ *
+ *  _________________________________________________ ___________________________________________
+ * |                                                 |                                           |
+ * |                     (Empty DB items queue and populated Data items queue?)                  |
+ * |                       |                                             |                       |
+ * |                   Yes |                                             | No                    |
+ * |                       |                                             |                       |
+ * |<-----------[Schedule Data job]      (Empty Data items queue and populated DB items queue?)  |
+ * |                                                     |              |                        |
+ * |                                                  No |              | Yes                    |
+ * |                                                     |              |                        |
+ * |                                                     |      [Schedule DB job]--------------->|
+ * |                                                     |                                       |
+ * |                                                     |                                       |
+ * |                 (Populated Data items queue and populated DB items queue?)                  |
+ * |                                  |                           |                              |
+ * |                              Yes |                           | No                           |
+ * |                                  |                           |                              |
+ * | (Active running Data items number exceeded limit?)       [Finish - no items]                |
+ * |            |                        |                                                       |
+ * |        No  |                        | Yes                                                   |
+ * |            |                        |                                                       |
+ *  <---[Schedule Data job]          [Schedule DB job]------------------------------------------>
+ *
+ *
+ */
+class PropagateFiles : public PropagatorJob {
+    Q_OBJECT
+public:
+    QVector<PropagatorJob *> _subJobs;
+
+    QVector<SyncFileItemPtr> _syncDBItems; // Items which bookkeeping on the server is longer then the transfer of its payload
+
+    QVector<SyncFileItemPtr> _syncDataItems; // Items which transfer of the payload is longer then bookkeeping on the server
+
+    int _jobsFinished; // number of jobs that have completed
+    SyncFileItem::Status _hasError;  // NoStatus,  or NormalError / SoftError if there was an error
+    int _firstUnfinishedSubJob;
+    int _totalItems;
+    int _activeDBJobsNow;
+    int _activeDataJobsNow;
+
+    explicit PropagateFiles(OwncloudPropagator *propagator)
+        : PropagatorJob(propagator)
+        , _jobsFinished(0), _hasError(SyncFileItem::NoStatus), _firstUnfinishedSubJob(0), _totalItems(0), _activeDBJobsNow(0), _activeDataJobsNow(0) { }
+
+    virtual ~PropagateFiles() {
+        qDeleteAll(_subJobs);
+    }
+
+    bool isEmpty() {
+        return _syncDBItems.isEmpty() && _syncDataItems.isEmpty();
+    }
+
+    void append(const SyncFileItemPtr &item);
+    virtual bool scheduleNextJob() Q_DECL_OVERRIDE;
+    bool scheduleNewJob(QVector<SyncFileItemPtr> &syncJobs);
+    bool scheduleNextItem();
+
+    virtual void abort() Q_DECL_OVERRIDE {
+        foreach (PropagatorJob *n, _subJobs)
+            n->abort();
+    }
+
+    void finalize();
+
+    qint64 committedDiskSpace() const Q_DECL_OVERRIDE;
+
+    JobParallelism parallelism() Q_DECL_OVERRIDE { return OCC::PropagatorJob::WaitForFinished; }
+
+private slots:
+    bool possiblyRunNextJob(PropagatorJob *next) {
+        if (next->_state == NotYetStarted) {
+            connect(next, SIGNAL(finished(SyncFileItem::Status)), this, SLOT(slotSubJobFinished(SyncFileItem::Status)), Qt::QueuedConnection);
+            connect(next, SIGNAL(itemCompleted(const SyncFileItem &, const PropagatorJob &)),
+                    this, SIGNAL(itemCompleted(const SyncFileItem &, const PropagatorJob &)));
+            connect(next, SIGNAL(progress(const SyncFileItem &,quint64)), this, SIGNAL(progress(const SyncFileItem &,quint64)));
+            connect(next, SIGNAL(ready()), this, SIGNAL(ready()));
+
+            PropagateItemJob *job = qobject_cast<PropagateItemJob *>(next);
+            if(job->_item->_size <= propagator()->smallFileSize()){
+                _activeDBJobsNow++;
+            } else {
+                _activeDataJobsNow++;
+            }
+        }
+        return next->scheduleNextJob();
+    }
+
+    void slotSubJobFinished(SyncFileItem::Status status);
+};
+
 }
 
 #endif

src/libsync/propagatedownload.h

@@ -115,7 +115,7 @@ class PropagateDownloadFile : public PropagateItemJob {
     qint64 committedDiskSpace() const Q_DECL_OVERRIDE;
 
     // We think it might finish quickly because it is a small file.
-    bool isLikelyFinishedQuickly() Q_DECL_OVERRIDE { return _item->_size < 100*1024; }
+    bool isLikelyFinishedQuickly() Q_DECL_OVERRIDE { return _item->_size < propagator()->smallFileSize(); }
 
     /**
      * Whether an existing folder with the same name may be deleted before