writeMany: Use a single statement to insert the batch

CHANGELOG.md

@@ -1,6 +1,8 @@
 ## 2.0.1
 
 * Add tests for internal schema migrations
+* Speed up `writeMany` by defaulting to use a single statement for all inserts (as opposed to a single transactions with many individual inserts)
+  * One can revert to the previous behavior by setting `singleStatement: false` in the call
 
 ## 2.0.0
 

example/perf/write_many.dart

@@ -0,0 +1,169 @@
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:flutter/foundation.dart' show FlutterTimeline, debugPrint;
+import 'package:flutter/widgets.dart';
+import 'package:indexed_entity_store/indexed_entity_store.dart';
+import 'package:path_provider/path_provider.dart';
+
+void main() async {
+  WidgetsFlutterBinding.ensureInitialized();
+
+  for (final batchSize in [10, 100, 1000, 10000]) {
+    for (final singleStatement in [true, false]) {
+      for (final largeValue in [true, false]) {
+        debugPrint(
+          '\nCase: singleStatement=$singleStatement, largeValue=$largeValue, batchSize=$batchSize',
+        );
+
+        final path = (await getApplicationCacheDirectory())
+            .uri
+            .resolve('./index_entity_store_test_${FlutterTimeline.now}.sqlite3')
+            .toFilePath();
+
+        debugPrint(path);
+
+        final db = IndexedEntityDabase.open(path);
+
+        final fooStore = db.entityStore(fooConnector);
+
+        // Insert one row each, so statements are prepared
+        fooStore.write(
+          _FooEntity(id: 0, valueA: 'a', valueB: 1, valueC: true),
+        );
+        fooStore.writeMany(
+          [_FooEntity(id: 0, valueA: 'a', valueB: 1, valueC: true)],
+          singleStatement: singleStatement,
+        );
+        fooStore.delete(key: 0);
+
+        // many
+        {
+          final sw2 = Stopwatch()..start();
+
+          for (var i = 1; i <= batchSize; i++) {
+            fooStore.write(
+              _FooEntity(id: i, valueA: 'a', valueB: 1, valueC: true),
+            );
+          }
+
+          final durMs = (sw2.elapsedMicroseconds / 1000).toStringAsFixed(2);
+          debugPrint(
+            '$batchSize x `write` took ${durMs}ms',
+          );
+        }
+
+        // 10 kB
+        final valueA = largeValue ? 'a1' * 1024 * 10 : 'a1';
+        final valueA2 = largeValue ? 'a2' * 1024 * 10 : 'a2';
+
+        // writeMany
+        {
+          final sw2 = Stopwatch()..start();
+
+          fooStore.writeMany(
+            [
+              for (var i = batchSize + 1; i <= batchSize * 2; i++)
+                _FooEntity(
+                  id: i,
+                  valueA: valueA,
+                  valueB: 1,
+                  valueC: true,
+                ),
+            ],
+            singleStatement: singleStatement,
+          );
+
+          final durMs = (sw2.elapsedMicroseconds / 1000).toStringAsFixed(2);
+          debugPrint(
+            '`writeMany` took ${durMs}ms',
+          );
+        }
+
+        // writeMany again (which needs to replace all existing entities and update the indices)
+        {
+          final sw2 = Stopwatch()..start();
+
+          fooStore.writeMany(
+            [
+              for (var i = batchSize + 1; i <= batchSize * 2; i++)
+                _FooEntity(
+                  id: i,
+                  valueA: valueA2,
+                  valueB: 111111,
+                  valueC: true,
+                ),
+            ],
+            singleStatement: singleStatement,
+          );
+
+          final durMs = (sw2.elapsedMicroseconds / 1000).toStringAsFixed(2);
+          debugPrint(
+            '`writeMany` again took ${durMs}ms',
+          );
+        }
+
+        if (fooStore.queryOnce().length != (batchSize * 2)) {
+          throw 'unexpected store size';
+        }
+
+        db.dispose();
+
+        File(path).deleteSync();
+      }
+    }
+  }
+
+  exit(0);
+}
+
+class _FooEntity {
+  _FooEntity({
+    required this.id,
+    required this.valueA,
+    required this.valueB,
+    required this.valueC,
+  });
+
+  final int id;
+
+  /// indexed via `a`
+  final String valueA;
+
+  /// indexed via "b"
+  final int valueB;
+
+  /// not indexed
+  final bool valueC;
+
+  Map<String, dynamic> toJSON() {
+    return {
+      'id': id,
+      'valueA': valueA,
+      'valueB': valueB,
+      'valueC': valueC,
+    };
+  }
+
+  static _FooEntity fromJSON(Map<String, dynamic> json) {
+    return _FooEntity(
+      id: json['id'],
+      valueA: json['valueA'],
+      valueB: json['valueB'],
+      valueC: json['valueC'],
+    );
+  }
+}
+
+final fooConnector = IndexedEntityConnector<_FooEntity, int, String>(
+  entityKey: 'foo',
+  getPrimaryKey: (f) => f.id,
+  getIndices: (index) {
+    index((e) => e.valueA, as: 'a');
+    index((e) => e.valueB, as: 'b');
+  },
+  serialize: (f) => jsonEncode(f.toJSON()),
+  deserialize: (s) => _FooEntity.fromJSON(
+    jsonDecode(s) as Map<String, dynamic>,
+  ),
+);

example/run_perf.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+set +x
+
+fvm flutter run --release -d macos ./perf/write_many.dart

lib/src/index_entity_store.dart

@@ -241,28 +241,55 @@ class IndexedEntityStore<T, K> {
   /// Insert or update many entities in a single batch
   ///
   /// Notification for changes will only fire after all changes have been written (meaning queries will get a single update after all writes are finished)
-  void writeMany(Iterable<T> entities) {
+  void writeMany(
+    Iterable<T> entities, {
+    bool singleStatement = true,
+  }) {
     final keys = <K>{};
 
-    try {
-      _database.execute('BEGIN');
-      assert(_database.autocommit == false);
+    if (singleStatement) {
+      if (entities.isEmpty) {
+        return;
+      }
 
-      for (final e in entities) {
-        _entityInsertStatement.execute(
-          [_entityKey, _connector.getPrimaryKey(e), _connector.serialize(e)],
-        );
+      _database.execute(
+        [
+          'REPLACE INTO `entity` (`type`, `key`, `value`) '
+              ' VALUES (?1, ?, ?)',
+          // Add additional entry values for each further parameter
+          ', (?1, ?, ?)' * (entities.length - 1),
+        ].join(' '),
+        [
+          _entityKey,
+          for (final e in entities) ...[
+            _connector.getPrimaryKey(e),
+            _connector.serialize(e),
+          ],
+        ],
+      );
+    } else {
+      // transaction variant
 
-        _updateIndexInternal(e);
+      try {
+        _database.execute('BEGIN');
+        assert(_database.autocommit == false);
 
-        keys.add(_connector.getPrimaryKey(e));
-      }
+        for (final e in entities) {
+          _entityInsertStatement.execute(
+            [_entityKey, _connector.getPrimaryKey(e), _connector.serialize(e)],
+          );
 
-      _database.execute('COMMIT');
-    } catch (e) {
-      _database.execute('ROLLBACK');
+          _updateIndexInternal(e);
 
-      rethrow;
+          keys.add(_connector.getPrimaryKey(e));
+        }
+
+        _database.execute('COMMIT');
+      } catch (e) {
+        _database.execute('ROLLBACK');
+
+        rethrow;
+      }
     }
 
     _handleUpdate(keys);

lib/src/indexed_entity_database.dart

@@ -138,7 +138,7 @@ class IndexedEntityDabase {
     );
 
     // This index is needed to not pay a performance penalty for the new foreign key constraint (between entities)
-    // Otherwise the `insertMany` update duration would increase 5x (even without making use of the reference, passing `null`).
+    // Otherwise the `writeMany` update duration would increase 5x (even without making use of the reference, passing `null`).
     // With this index, even though it only tracks non-`null` references (which would be rare), overall insert performance stays the same as before.
     _database.execute(
       'CREATE INDEX index_field_values_FK '

test/indexed_entity_store_test.dart

@@ -1173,77 +1173,6 @@ void main() {
     expect(listSubscription.value, isEmpty);
     expect(fooStore.queryOnce(), isEmpty);
   });
-
-  test(
-    'Performance',
-    () async {
-      final path =
-          '/tmp/index_entity_store_test_${FlutterTimeline.now}.sqlite3';
-
-      final db = IndexedEntityDabase.open(path);
-
-      final fooStore = db.entityStore(fooConnector);
-
-      expect(fooStore.queryOnce(), isEmpty);
-
-      // Insert one row, so statements are prepared
-      fooStore.write(
-        _FooEntity(id: 0, valueA: 'a', valueB: 1, valueC: true),
-      );
-
-      const batchSize = 1000;
-
-      // many
-      {
-        final sw2 = Stopwatch()..start();
-
-        for (var i = 1; i <= batchSize; i++) {
-          fooStore.write(
-            _FooEntity(id: i, valueA: 'a', valueB: 1, valueC: true),
-          );
-        }
-
-        debugPrint(
-          'insert tooks ${(sw2.elapsedMicroseconds / 1000).toStringAsFixed(2)}ms',
-        );
-      }
-
-      // insertMany
-      {
-        final sw2 = Stopwatch()..start();
-
-        fooStore.writeMany(
-          [
-            for (var i = batchSize + 1; i <= batchSize * 2; i++)
-              _FooEntity(id: i, valueA: 'a', valueB: 1, valueC: true),
-          ],
-        );
-
-        debugPrint(
-          'insertMany tooks ${(sw2.elapsedMicroseconds / 1000).toStringAsFixed(2)}ms',
-        );
-      }
-
-      // many again (which needs to replace all existing entities and update the indices)
-      {
-        final sw2 = Stopwatch()..start();
-
-        fooStore.writeMany(
-          [
-            for (var i = batchSize + 1; i <= batchSize * 2; i++)
-              _FooEntity(id: i, valueA: 'aaaaaa', valueB: 111111, valueC: true),
-          ],
-        );
-
-        debugPrint(
-          'insertMany again tooks ${(sw2.elapsedMicroseconds / 1000).toStringAsFixed(2)}ms',
-        );
-      }
-
-      expect(fooStore.queryOnce(), hasLength(batchSize * 2 + 1));
-    },
-    skip: !Platform.isMacOS, // only run locally for now
-  );
 }
 
 class _FooEntity {