Increase item limit in each tab to 100,000

Saves larger data in separate files.

Data from these files are loaded into memory only when needed.

The threshold value in bytes is stored as "item_data_threshold" hidden
option. Item data that are larger than this value will be stored in
separate files. The default value is 4096. Setting value to 0 would
store all non-empty item data to separate files. A negative value
disables the functionality and saves all into tab data file as before.

Cleans up data directories on idle and exit.

Files in synchronized directories now also work the same.

This change is not backwards-compatible. Going back to previous release
could mean losing some data. But going to a new release again would
parse the data correctly. Workaround is to set "item_data_threshold" to
a negative value and trigger storing data tab (e.g. moving an item back
and forth).

Fixes #1144

plugins/itemencrypted/itemencrypted.cpp

@@ -38,7 +38,7 @@ const QLatin1String dataFileHeaderV2("CopyQ_encrypted_tab v2");
 
 const QLatin1String configEncryptTabs("encrypt_tabs");
 
-const int maxItemCount = 10000;
+const int maxItemCount = 100'000;
 
 bool waitOrTerminate(QProcess *p, int timeoutMs)
 {

plugins/itemfakevim/itemfakevim.cpp

@@ -754,7 +754,7 @@ QVariant ItemFakeVimLoader::icon() const
 
 void ItemFakeVimLoader::setEnabled(bool enabled)
 {
-    m_enabled = enabled;
+    ItemLoaderInterface::setEnabled(enabled);
     updateCurrentlyEnabledState();
 }
 
@@ -811,7 +811,7 @@ void ItemFakeVimLoader::updateCurrentlyEnabledState()
     if ( qobject_cast<QGuiApplication*>(qApp) == nullptr )
         return;
 
-    const bool enable = m_enabled && m_reallyEnabled;
+    const bool enable = isEnabled() && m_reallyEnabled;
     if (m_currentlyEnabled == enable)
         return;
 

plugins/itemfakevim/itemfakevim.h

@@ -49,7 +49,6 @@ class ItemFakeVimLoader final : public QObject, public ItemLoaderInterface
 
     void wrapEditWidget(QObject *obj);
 
-    bool m_enabled = true;
     bool m_reallyEnabled = false;
     bool m_currentlyEnabled = false;
     QString m_sourceFileName;

plugins/itemsync/filewatcher.cpp

@@ -29,6 +29,79 @@ const QLatin1String mimePrivatePrefix(COPYQ_MIME_PREFIX_ITEMSYNC_PRIVATE);
 const QLatin1String mimeOldBaseName(COPYQ_MIME_PREFIX_ITEMSYNC_PRIVATE "old-basename");
 const QLatin1String mimeHashPrefix(COPYQ_MIME_PREFIX_ITEMSYNC_PRIVATE "hash");
 
+class SyncDataFile {
+public:
+    SyncDataFile() = default;
+
+    explicit SyncDataFile(const QString &path, const QString &format = QString())
+        : m_path(path)
+        , m_format(format)
+    {}
+
+    const QString &path() const { return m_path; }
+    void setPath(const QString &path) { m_path = path; }
+
+    const QString &format() const { return m_format; }
+    void setFormat(const QString &format) { m_format = format; }
+
+    qint64 size() const {
+        QFileInfo f(m_path);
+        return f.size();
+    }
+
+    QByteArray readAll() const
+    {
+        COPYQ_LOG_VERBOSE( QStringLiteral("ItemSync: Reading file: %1").arg(m_path) );
+
+        QFile f(m_path);
+        if ( !f.open(QIODevice::ReadOnly) )
+            return QByteArray();
+
+        if ( m_format.isEmpty() )
+            return f.readAll();
+
+        QDataStream stream(&f);
+        QVariantMap dataMap;
+        if ( !deserializeData(&stream, &dataMap) ) {
+            log( QStringLiteral("ItemSync: Failed to read file \"%1\": %2")
+                    .arg(m_path, f.errorString()), LogError );
+            return QByteArray();
+        }
+
+        return dataMap.value(m_format).toByteArray();
+    }
+
+private:
+    QString m_path;
+    QString m_format;
+};
+Q_DECLARE_METATYPE(SyncDataFile)
+
+QDataStream &operator<<(QDataStream &out, SyncDataFile value)
+{
+    return out << value.path() << value.format();
+}
+
+QDataStream &operator>>(QDataStream &in, SyncDataFile &value)
+{
+    QString path;
+    QString format;
+    in >> path >> format;
+    value.setPath(path);
+    value.setFormat(format);
+    return in;
+}
+
+void registerSyncDataFileConverter()
+{
+    QMetaType::registerConverter(&SyncDataFile::readAll);
+#if QT_VERSION < QT_VERSION_CHECK(6,0,0)
+    qRegisterMetaTypeStreamOperators<SyncDataFile>("SyncDataFile");
+#else
+    qRegisterMetaType<SyncDataFile>("SyncDataFile");
+#endif
+}
+
 struct Ext {
     Ext() : extension(), format() {}
 
@@ -61,7 +134,7 @@ const QLatin1String noteFileSuffix("_note.txt");
 const int defaultUpdateFocusItemsIntervalMs = 10000;
 const int batchItemUpdateIntervalMs = 100;
 
-const qint64 sizeLimit = 10 << 20;
+const qint64 sizeLimit = 50'000'000;
 
 FileFormat getFormatSettingsFromFileName(const QString &fileName,
                                          const QList<FileFormat> &formatSettings,
@@ -425,13 +498,15 @@ FileWatcher::FileWatcher(
         QAbstractItemModel *model,
         int maxItems,
         const QList<FileFormat> &formatSettings,
+        int itemDataThreshold,
         QObject *parent)
     : QObject(parent)
     , m_model(model)
     , m_formatSettings(formatSettings)
     , m_path(path)
     , m_valid(true)
     , m_maxItems(maxItems)
+    , m_itemDataThreshold(itemDataThreshold)
 {
     m_updateTimer.setSingleShot(true);
 
@@ -943,18 +1018,38 @@ void FileWatcher::updateDataAndWatchFile(const QDir &dir, const BaseNameExtensio
 
         const QString fileName = basePath + ext.extension;
 
-        QFile f( dir.absoluteFilePath(fileName) );
+        const QString path = dir.absoluteFilePath(fileName);
+        QFile f(path);
         if ( !f.open(QIODevice::ReadOnly) )
             continue;
 
         if ( ext.extension == dataFileSuffix ) {
             QDataStream stream(&f);
-            if ( deserializeData(&stream, dataMap) )
+            QVariantMap dataMap2;
+            if ( deserializeData(&stream, &dataMap2) ) {
+                for (auto it = dataMap2.constBegin(); it != dataMap2.constEnd(); ++it) {
+                    const QVariant &value = it.value();
+                    const qint64 size = value.type() == QVariant::ByteArray
+                        ? value.toByteArray().size()
+                        : value.value<SyncDataFile>().size();
+                    if (m_itemDataThreshold >= 0 && size > m_itemDataThreshold) {
+                        const QVariant syncDataFile = QVariant::fromValue(SyncDataFile(path, it.key()));
+                        dataMap->insert(it.key(), syncDataFile);
+                    } else {
+                        dataMap->insert(it.key(), value);
+                    }
+                }
+
                 mimeToExtension->insert(mimeUnknownFormats, dataFileSuffix);
+            }
         } else if ( f.size() > sizeLimit || ext.format.startsWith(mimeNoFormat)
                     || dataMap->contains(ext.format) )
         {
             mimeToExtension->insert(mimeNoFormat + ext.extension, ext.extension);
+        } else if ( m_itemDataThreshold >= 0 && f.size() > m_itemDataThreshold ) {
+            const QVariant value = QVariant::fromValue(SyncDataFile(path));
+            dataMap->insert(ext.format, value);
+            mimeToExtension->insert(ext.format, ext.extension);
         } else {
             dataMap->insert(ext.format, f.readAll());
             mimeToExtension->insert(ext.format, ext.extension);

plugins/itemsync/filewatcher.h

@@ -46,6 +46,11 @@ using BaseNameExtensionsList = QList<BaseNameExtensions>;
 
 using Hash = QByteArray;
 
+class SyncDataFile;
+QDataStream &operator<<(QDataStream &out, SyncDataFile value);
+QDataStream &operator>>(QDataStream &in, SyncDataFile &value);
+void registerSyncDataFileConverter();
+
 class FileWatcher final : public QObject {
 public:
     static QString getBaseName(const QModelIndex &index);
@@ -61,8 +66,15 @@ class FileWatcher final : public QObject {
 
     static Hash calculateHash(const QByteArray &bytes);
 
-    FileWatcher(const QString &path, const QStringList &paths, QAbstractItemModel *model,
-                int maxItems, const QList<FileFormat> &formatSettings, QObject *parent = nullptr);
+    FileWatcher(
+        const QString &path,
+        const QStringList &paths,
+        QAbstractItemModel *model,
+        int maxItems,
+        const QList<FileFormat> &formatSettings,
+        int itemDataThreshold,
+        QObject *parent = nullptr
+    );
 
     const QString &path() const { return m_path; }
 
@@ -127,6 +139,7 @@ class FileWatcher final : public QObject {
     QList<QPersistentModelIndex> m_batchIndexData;
     BaseNameExtensionsList m_fileList;
     int m_lastBatchIndex = -1;
+    int m_itemDataThreshold = -1;
 };
 
 #endif // FILEWATCHER_H

plugins/itemsync/itemsync.cpp

@@ -5,6 +5,7 @@
 
 #include "filewatcher.h"
 
+#include "common/appconfig.h"
 #include "common/compatibility.h"
 #include "common/contenttype.h"
 #include "common/log.h"
@@ -531,6 +532,7 @@ QString ItemSyncScriptable::selectedTabPath()
 
 ItemSyncLoader::ItemSyncLoader()
 {
+    registerSyncDataFileConverter();
 }
 
 ItemSyncLoader::~ItemSyncLoader() = default;
@@ -604,6 +606,12 @@ void ItemSyncLoader::loadSettings(const QSettings &settings)
         fixUserMimeType(&fileFormat.itemMime);
         m_formatSettings.append(fileFormat);
     }
+
+    const QSettings settingsTopLevel(settings.fileName(), settings.format());
+    m_itemDataThreshold = settingsTopLevel.value(
+        QStringLiteral("Options/") + Config::item_data_threshold::name(),
+        Config::item_data_threshold::defaultValue()
+    ).toInt();
 }
 
 QWidget *ItemSyncLoader::createSettingsWidget(QWidget *parent)
@@ -769,6 +777,7 @@ ItemSaverPtr ItemSyncLoader::loadItems(const QString &tabName, QAbstractItemMode
         return nullptr;
     }
 
-    auto *watcher = new FileWatcher(path, files, model, maxItems, m_formatSettings);
+    auto *watcher = new FileWatcher(
+        path, files, model, maxItems, m_formatSettings, m_itemDataThreshold);
     return std::make_shared<ItemSyncSaver>(tabPath, watcher);
 }

plugins/itemsync/itemsync.h

@@ -158,6 +158,7 @@ private slots:
     ItemSyncTabPaths m_tabPaths;
     QStringList m_tabPathsSaved;
     QList<FileFormat> m_formatSettings;
+    int m_itemDataThreshold = -1;
 };
 
 #endif // ITEMSYNC_H

plugins/itemsync/tests/itemsynctests.cpp

@@ -3,6 +3,7 @@
 #include "itemsynctests.h"
 
 #include "common/mimetypes.h"
+#include "common/sleeptimer.h"
 #include "tests/test_utils.h"
 
 #include <QDir>
@@ -768,3 +769,43 @@ void ItemSyncTests::avoidDuplicateItemsAddedFromClipboard()
     TEST( m_test->setClipboard("one") );
     WAIT_ON_OUTPUT(args << "read(0,1,2,3)", "one,two,,");
 }
+
+void ItemSyncTests::saveLargeItem()
+{
+    const auto tab = testTab(1);
+    const auto args = Args("tab") << tab;
+
+    const auto script = R"(
+        write(0, [{
+            'text/plain': '1234567890'.repeat(10000),
+            'application/x-copyq-test-data': 'abcdefghijklmnopqrstuvwxyz'.repeat(10000),
+        }])
+        )";
+    RUN(args << script, "");
+
+    for (int i = 0; i < 2; ++i) {
+        RUN(args << "read(0).left(20)", "12345678901234567890");
+        RUN(args << "read(0).length", "100000\n");
+        RUN(args << "getItem(0)[mimeText].left(20)", "12345678901234567890");
+        RUN(args << "getItem(0)[mimeText].length", "100000\n");
+        RUN(args << "getItem(0)['application/x-copyq-test-data'].left(26)", "abcdefghijklmnopqrstuvwxyz");
+        RUN(args << "getItem(0)['application/x-copyq-test-data'].length", "260000\n");
+        RUN("unload" << tab, tab + "\n");
+    }
+
+    RUN("show" << tab, "");
+    RUN("keys" << clipboardBrowserId << keyNameFor(QKeySequence::Copy), "");
+    WAIT_ON_OUTPUT("clipboard().left(20)", "12345678901234567890");
+    RUN("clipboard('application/x-copyq-test-data').left(26)", "abcdefghijklmnopqrstuvwxyz");
+    RUN("clipboard('application/x-copyq-test-data').length", "260000\n");
+
+    const auto tab2 = testTab(2);
+    const auto args2 = Args("tab") << tab2;
+    RUN("show" << tab2, "");
+    waitFor(waitMsPasteClipboard);
+    RUN("keys" << clipboardBrowserId << keyNameFor(QKeySequence::Paste), "");
+    RUN(args2 << "read(0).left(20)", "12345678901234567890");
+    RUN(args2 << "read(0).length", "100000\n");
+    RUN(args << "getItem(0)['application/x-copyq-test-data'].left(26)", "abcdefghijklmnopqrstuvwxyz");
+    RUN(args << "getItem(0)['application/x-copyq-test-data'].length", "260000\n");
+}

plugins/itemsync/tests/itemsynctests.h

@@ -52,6 +52,8 @@ private slots:
 
     void avoidDuplicateItemsAddedFromClipboard();
 
+    void saveLargeItem();
+
 private:
     TestInterfacePtr m_test;
 };

src/app/app.cpp

@@ -7,6 +7,7 @@
 #include "common/log.h"
 #include "common/settings.h"
 #include "common/textdata.h"
+#include "item/serialize.h"
 #include "platform/platformnativeinterface.h"
 #ifdef Q_OS_UNIX
 #   include "platform/unix/unixsignalhandler.h"
@@ -120,6 +121,8 @@ App::App(QCoreApplication *application,
     , m_started(false)
     , m_closed(false)
 {
+    registerDataFileConverter();
+
     QObject::connect(m_app, &QCoreApplication::aboutToQuit, [this]() { exit(); });
 
 #ifdef Q_OS_UNIX

src/app/clipboardclient.cpp

@@ -9,6 +9,7 @@
 #include "common/common.h"
 #include "common/log.h"
 #include "common/textdata.h"
+#include "item/itemfactory.h"
 #include "platform/platformnativeinterface.h"
 #include "scriptable/scriptable.h"
 #include "scriptable/scriptableproxy.h"
@@ -114,9 +115,14 @@ void ClipboardClient::onConnectionFailed()
 
 void ClipboardClient::start(const QStringList &arguments)
 {
+    ItemFactory itemFactory;
+    itemFactory.loadPlugins();
+    QSettings settings;
+    itemFactory.loadItemFactorySettings(&settings);
+
     QJSEngine engine;
     ScriptableProxy scriptableProxy(nullptr, nullptr);
-    Scriptable scriptable(&engine, &scriptableProxy);
+    Scriptable scriptable(&engine, &scriptableProxy, &itemFactory);
 
     const auto serverName = clipboardServerName();
     ClientSocket socket(serverName);