Add insertMany method

CHANGELOG.md

@@ -1,3 +1,11 @@
+## 1.4.3
+
+* Add `insertMany` method to handle batch inserts/updates
+  * This, combined with a new index, massively speeds up large inserts:
+    - Inserting 1000 entities with 2 indices is now 40x faster than a simple loop using `insert`s individually
+    - When updating 1000 existing entities with new values, the new implementation leads to an even greater 111x speed-up
+  * This further proves that the synchronous database approach can handle even large local databases and operations. If you need to insert even larger amounts of data without dropping a frame, there is [a solution for that](https://github.com/simolus3/sqlite3.dart/issues/260#issuecomment-2446618546) as well.
+
 ## 1.4.2
 
 * Add full example app

example/pubspec.lock

@@ -105,7 +105,7 @@ packages:
       path: ".."
       relative: true
     source: path
-    version: "1.4.2"
+    version: "1.4.3"
   leak_tracker:
     dependency: transitive
     description:

lib/src/index_entity_store.dart

@@ -251,6 +251,11 @@ class IndexedEntityStore<T, K> {
     );
   }
 
+  late final _entityInsertStatement = _database.prepare(
+    'REPLACE INTO `entity` (`type`, `key`, `value`) VALUES (?, ?, ?)',
+    persistent: true,
+  );
+
   /// Insert or updates the given entity in the database.
   ///
   /// In case an entity with the same primary already exists in the database, it will be updated.
@@ -259,8 +264,7 @@ class IndexedEntityStore<T, K> {
     _database.execute('BEGIN');
     assert(_database.autocommit == false);
 
-    _database.execute(
-      'REPLACE INTO `entity` (`type`, `key`, `value`) VALUES (?, ?, ?)',
+    _entityInsertStatement.execute(
       [_entityKey, _connector.getPrimaryKey(e), _connector.serialize(e)],
     );
 
@@ -271,15 +275,37 @@ class IndexedEntityStore<T, K> {
     _handleUpdate({_connector.getPrimaryKey(e)});
   }
 
-  void _updateIndexInternal(T e) {
-    _database.execute(
-      'DELETE FROM `index` WHERE `type` = ? AND `entity` = ?',
-      [_entityKey, _connector.getPrimaryKey(e)],
-    );
+  /// Insert or update many entities in a single batch
+  ///
+  /// Notification for changes will only fire after all changes have been written (this queries will get a single update after all writes are finished)
+  void insertMany(Iterable<T> entities) {
+    _database.execute('BEGIN');
+    assert(_database.autocommit == false);
+
+    final keys = <K>{};
+    for (final e in entities) {
+      _entityInsertStatement.execute(
+        [_entityKey, _connector.getPrimaryKey(e), _connector.serialize(e)],
+      );
 
+      _updateIndexInternal(e);
+
+      keys.add(_connector.getPrimaryKey(e));
+    }
+
+    _database.execute('COMMIT');
+
+    _handleUpdate(keys);
+  }
+
+  late final _insertIndexStatement = _database.prepare(
+    'INSERT INTO `index` (`type`, `entity`, `field`, `value`) VALUES (?, ?, ?, ?)',
+    persistent: true,
+  );
+
+  void _updateIndexInternal(T e) {
     for (final indexColumn in _indexColumns._indexColumns.values) {
-      _database.execute(
-        'INSERT INTO `index` (`type`, `entity`, `field`, `value`) VALUES (?, ?, ?, ?)',
+      _insertIndexStatement.execute(
         [
           _entityKey,
           _connector.getPrimaryKey(e),

lib/src/indexed_entity_database.dart

@@ -10,34 +10,62 @@ class IndexedEntityDabase {
       "SELECT name FROM sqlite_master WHERE type='table' AND name='entity';",
     );
     if (res.isEmpty) {
-      _database.execute('PRAGMA foreign_keys = ON');
+      debugPrint('Creating new DB');
 
-      _database.execute(
-        'CREATE TABLE `entity` ( `type` TEXT NOT NULL, `key` NOT NULL, `value`, PRIMARY KEY ( `type`, `key` ) )',
-      );
+      _initialDBSetup();
+      _v2Migration();
+    } else if (_dbVersion == 1) {
+      debugPrint('Migrating DB to v2');
 
-      _database.execute(
-        'CREATE TABLE `index` ( `type` TEXT NOT NULL, `entity` NOT NULL, `field` TEXT NOT NULL, `value`, '
-        ' FOREIGN KEY (`type`, `entity`) REFERENCES `entity` (`type`, `key`) ON DELETE CASCADE'
-        ')',
-      );
+      _v2Migration();
+    }
 
-      _database.execute(
-        'CREATE INDEX index_field_values '
-        'ON `index` ( `type`, `field`, `value` )',
-      );
+    assert(_dbVersion == 2);
+  }
 
-      _database.execute(
-        'CREATE TABLE `metadata` ( `key` TEXT NOT NULL, `value` )',
-      );
+  void _initialDBSetup() {
+    _database.execute('PRAGMA foreign_keys = ON');
 
-      _database.execute(
-        'INSERT INTO `metadata` ( `key`, `value` ) VALUES ( ?, ? )',
-        ['version', 1],
-      );
+    _database.execute(
+      'CREATE TABLE `entity` ( `type` TEXT NOT NULL, `key` NOT NULL, `value`, PRIMARY KEY ( `type`, `key` ) )',
+    );
 
-      debugPrint('New DB created');
-    }
+    _database.execute(
+      'CREATE TABLE `index` ( `type` TEXT NOT NULL, `entity` NOT NULL, `field` TEXT NOT NULL, `value`, '
+      ' FOREIGN KEY (`type`, `entity`) REFERENCES `entity` (`type`, `key`) ON DELETE CASCADE'
+      ')',
+    );
+
+    _database.execute(
+      'CREATE INDEX index_field_values '
+      'ON `index` ( `type`, `field`, `value` )',
+    );
+
+    _database.execute(
+      'CREATE TABLE `metadata` ( `key` TEXT NOT NULL, `value` )',
+    );
+
+    _database.execute(
+      'INSERT INTO `metadata` ( `key`, `value` ) VALUES ( ?, ? )',
+      ['version', 1],
+    );
+  }
+
+  int get _dbVersion => _database.select(
+        'SELECT `value` FROM `metadata` WHERE `key` = ?',
+        ['version'],
+      ).single['value'] as int;
+
+  void _v2Migration() {
+    _database.execute(
+      'CREATE UNIQUE INDEX index_type_entity_field_index '
+      'ON `index` ( `type`, `entity`, `field` )',
+    );
+
+    _database.execute(
+      'UPDATE `metadata` SET `value` = ? WHERE `key` = ?',
+      [2, 'version'],
+    );
   }
 
   factory IndexedEntityDabase.open(String path) {

pubspec.yaml

@@ -1,6 +1,6 @@
 name: indexed_entity_store
 description: A fast, simple, and synchronous entity store for Flutter applications.
-version: 1.4.2
+version: 1.4.3
 repository: https://github.com/LunaONE/indexed_entity_store
 
 environment:

test/indexed_entity_store_test.dart

@@ -977,6 +977,77 @@ void main() {
     expect(listSubscription.value, isEmpty);
     expect(fooStore.getAllOnce(), isEmpty);
   });
+
+  test(
+    'Performance',
+    () async {
+      final path =
+          '/tmp/index_entity_store_test_${FlutterTimeline.now}.sqlite3';
+
+      final db = IndexedEntityDabase.open(path);
+
+      final fooStore = db.entityStore(fooConnector);
+
+      expect(fooStore.getAllOnce(), isEmpty);
+
+      // Insert one row, so statements are prepared
+      fooStore.insert(
+        _FooEntity(id: 0, valueA: 'a', valueB: 1, valueC: true),
+      );
+
+      const batchSize = 1000;
+
+      // many
+      {
+        final sw2 = Stopwatch()..start();
+
+        for (var i = 1; i <= batchSize; i++) {
+          fooStore.insert(
+            _FooEntity(id: i, valueA: 'a', valueB: 1, valueC: true),
+          );
+        }
+
+        debugPrint(
+          'insert tooks ${(sw2.elapsedMicroseconds / 1000).toStringAsFixed(2)}ms',
+        );
+      }
+
+      // insertMany
+      {
+        final sw2 = Stopwatch()..start();
+
+        fooStore.insertMany(
+          [
+            for (var i = batchSize + 1; i <= batchSize * 2; i++)
+              _FooEntity(id: i, valueA: 'a', valueB: 1, valueC: true),
+          ],
+        );
+
+        debugPrint(
+          'insertMany tooks ${(sw2.elapsedMicroseconds / 1000).toStringAsFixed(2)}ms',
+        );
+      }
+
+      // many again (which needs to replace all existing entities and update the indices)
+      {
+        final sw2 = Stopwatch()..start();
+
+        fooStore.insertMany(
+          [
+            for (var i = batchSize + 1; i <= batchSize * 2; i++)
+              _FooEntity(id: i, valueA: 'aaaaaa', valueB: 111111, valueC: true),
+          ],
+        );
+
+        debugPrint(
+          'insertMany again tooks ${(sw2.elapsedMicroseconds / 1000).toStringAsFixed(2)}ms',
+        );
+      }
+
+      expect(fooStore.getAllOnce(), hasLength(batchSize * 2 + 1));
+    },
+    skip: !Platform.isMacOS, // only run locally for now
+  );
 }
 
 class _FooEntity {