diff --git a/CHANGELOG.md b/CHANGELOG.md index 19e1eb7..816550a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ ## 2.0.1 * Add tests for internal schema migrations +* Speed up `writeMany` by defaulting to use a single statement for all inserts (as opposed to a single transactions with many individual inserts) + * One can revert to the previous behavior by setting `singleStatement: false` in the call ## 2.0.0 diff --git a/example/perf/write_many.dart b/example/perf/write_many.dart new file mode 100644 index 0000000..b674688 --- /dev/null +++ b/example/perf/write_many.dart @@ -0,0 +1,169 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:flutter/foundation.dart' show FlutterTimeline, debugPrint; +import 'package:flutter/widgets.dart'; +import 'package:indexed_entity_store/indexed_entity_store.dart'; +import 'package:path_provider/path_provider.dart'; + +void main() async { + WidgetsFlutterBinding.ensureInitialized(); + + for (final batchSize in [10, 100, 1000, 10000]) { + for (final singleStatement in [true, false]) { + for (final largeValue in [true, false]) { + debugPrint( + '\nCase: singleStatement=$singleStatement, largeValue=$largeValue, batchSize=$batchSize', + ); + + final path = (await getApplicationCacheDirectory()) + .uri + .resolve('./index_entity_store_test_${FlutterTimeline.now}.sqlite3') + .toFilePath(); + + debugPrint(path); + + final db = IndexedEntityDabase.open(path); + + final fooStore = db.entityStore(fooConnector); + + // Insert one row each, so statements are prepared + fooStore.write( + _FooEntity(id: 0, valueA: 'a', valueB: 1, valueC: true), + ); + fooStore.writeMany( + [_FooEntity(id: 0, valueA: 'a', valueB: 1, valueC: true)], + singleStatement: singleStatement, + ); + fooStore.delete(key: 0); + + // many + { + final sw2 = Stopwatch()..start(); + + for (var i = 1; i <= batchSize; i++) { + fooStore.write( + _FooEntity(id: i, valueA: 'a', valueB: 1, valueC: true), + ); + } + + final durMs = (sw2.elapsedMicroseconds / 1000).toStringAsFixed(2); + debugPrint( + '$batchSize x `write` took ${durMs}ms', + ); + } + + // 10 kB + final valueA = largeValue ? 'a1' * 1024 * 10 : 'a1'; + final valueA2 = largeValue ? 'a2' * 1024 * 10 : 'a2'; + + // writeMany + { + final sw2 = Stopwatch()..start(); + + fooStore.writeMany( + [ + for (var i = batchSize + 1; i <= batchSize * 2; i++) + _FooEntity( + id: i, + valueA: valueA, + valueB: 1, + valueC: true, + ), + ], + singleStatement: singleStatement, + ); + + final durMs = (sw2.elapsedMicroseconds / 1000).toStringAsFixed(2); + debugPrint( + '`writeMany` took ${durMs}ms', + ); + } + + // writeMany again (which needs to replace all existing entities and update the indices) + { + final sw2 = Stopwatch()..start(); + + fooStore.writeMany( + [ + for (var i = batchSize + 1; i <= batchSize * 2; i++) + _FooEntity( + id: i, + valueA: valueA2, + valueB: 111111, + valueC: true, + ), + ], + singleStatement: singleStatement, + ); + + final durMs = (sw2.elapsedMicroseconds / 1000).toStringAsFixed(2); + debugPrint( + '`writeMany` again took ${durMs}ms', + ); + } + + if (fooStore.queryOnce().length != (batchSize * 2)) { + throw 'unexpected store size'; + } + + db.dispose(); + + File(path).deleteSync(); + } + } + } + + exit(0); +} + +class _FooEntity { + _FooEntity({ + required this.id, + required this.valueA, + required this.valueB, + required this.valueC, + }); + + final int id; + + /// indexed via `a` + final String valueA; + + /// indexed via "b" + final int valueB; + + /// not indexed + final bool valueC; + + Map toJSON() { + return { + 'id': id, + 'valueA': valueA, + 'valueB': valueB, + 'valueC': valueC, + }; + } + + static _FooEntity fromJSON(Map json) { + return _FooEntity( + id: json['id'], + valueA: json['valueA'], + valueB: json['valueB'], + valueC: json['valueC'], + ); + } +} + +final fooConnector = IndexedEntityConnector<_FooEntity, int, String>( + entityKey: 'foo', + getPrimaryKey: (f) => f.id, + getIndices: (index) { + index((e) => e.valueA, as: 'a'); + index((e) => e.valueB, as: 'b'); + }, + serialize: (f) => jsonEncode(f.toJSON()), + deserialize: (s) => _FooEntity.fromJSON( + jsonDecode(s) as Map, + ), +); diff --git a/example/run_perf.sh b/example/run_perf.sh new file mode 100755 index 0000000..720154c --- /dev/null +++ b/example/run_perf.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +set +x + +fvm flutter run --release -d macos ./perf/write_many.dart diff --git a/lib/src/index_entity_store.dart b/lib/src/index_entity_store.dart index abab40d..2e8f858 100644 --- a/lib/src/index_entity_store.dart +++ b/lib/src/index_entity_store.dart @@ -241,28 +241,55 @@ class IndexedEntityStore { /// Insert or update many entities in a single batch /// /// Notification for changes will only fire after all changes have been written (meaning queries will get a single update after all writes are finished) - void writeMany(Iterable entities) { + void writeMany( + Iterable entities, { + bool singleStatement = true, + }) { final keys = {}; - try { - _database.execute('BEGIN'); - assert(_database.autocommit == false); + if (singleStatement) { + if (entities.isEmpty) { + return; + } - for (final e in entities) { - _entityInsertStatement.execute( - [_entityKey, _connector.getPrimaryKey(e), _connector.serialize(e)], - ); + _database.execute( + [ + 'REPLACE INTO `entity` (`type`, `key`, `value`) ' + ' VALUES (?1, ?, ?)', + // Add additional entry values for each further parameter + ', (?1, ?, ?)' * (entities.length - 1), + ].join(' '), + [ + _entityKey, + for (final e in entities) ...[ + _connector.getPrimaryKey(e), + _connector.serialize(e), + ], + ], + ); + } else { + // transaction variant - _updateIndexInternal(e); + try { + _database.execute('BEGIN'); + assert(_database.autocommit == false); - keys.add(_connector.getPrimaryKey(e)); - } + for (final e in entities) { + _entityInsertStatement.execute( + [_entityKey, _connector.getPrimaryKey(e), _connector.serialize(e)], + ); - _database.execute('COMMIT'); - } catch (e) { - _database.execute('ROLLBACK'); + _updateIndexInternal(e); - rethrow; + keys.add(_connector.getPrimaryKey(e)); + } + + _database.execute('COMMIT'); + } catch (e) { + _database.execute('ROLLBACK'); + + rethrow; + } } _handleUpdate(keys); diff --git a/lib/src/indexed_entity_database.dart b/lib/src/indexed_entity_database.dart index 84b2f4c..a72eb56 100644 --- a/lib/src/indexed_entity_database.dart +++ b/lib/src/indexed_entity_database.dart @@ -138,7 +138,7 @@ class IndexedEntityDabase { ); // This index is needed to not pay a performance penalty for the new foreign key constraint (between entities) - // Otherwise the `insertMany` update duration would increase 5x (even without making use of the reference, passing `null`). + // Otherwise the `writeMany` update duration would increase 5x (even without making use of the reference, passing `null`). // With this index, even though it only tracks non-`null` references (which would be rare), overall insert performance stays the same as before. _database.execute( 'CREATE INDEX index_field_values_FK ' diff --git a/test/indexed_entity_store_test.dart b/test/indexed_entity_store_test.dart index fe7e61a..b427fef 100644 --- a/test/indexed_entity_store_test.dart +++ b/test/indexed_entity_store_test.dart @@ -1173,77 +1173,6 @@ void main() { expect(listSubscription.value, isEmpty); expect(fooStore.queryOnce(), isEmpty); }); - - test( - 'Performance', - () async { - final path = - '/tmp/index_entity_store_test_${FlutterTimeline.now}.sqlite3'; - - final db = IndexedEntityDabase.open(path); - - final fooStore = db.entityStore(fooConnector); - - expect(fooStore.queryOnce(), isEmpty); - - // Insert one row, so statements are prepared - fooStore.write( - _FooEntity(id: 0, valueA: 'a', valueB: 1, valueC: true), - ); - - const batchSize = 1000; - - // many - { - final sw2 = Stopwatch()..start(); - - for (var i = 1; i <= batchSize; i++) { - fooStore.write( - _FooEntity(id: i, valueA: 'a', valueB: 1, valueC: true), - ); - } - - debugPrint( - 'insert tooks ${(sw2.elapsedMicroseconds / 1000).toStringAsFixed(2)}ms', - ); - } - - // insertMany - { - final sw2 = Stopwatch()..start(); - - fooStore.writeMany( - [ - for (var i = batchSize + 1; i <= batchSize * 2; i++) - _FooEntity(id: i, valueA: 'a', valueB: 1, valueC: true), - ], - ); - - debugPrint( - 'insertMany tooks ${(sw2.elapsedMicroseconds / 1000).toStringAsFixed(2)}ms', - ); - } - - // many again (which needs to replace all existing entities and update the indices) - { - final sw2 = Stopwatch()..start(); - - fooStore.writeMany( - [ - for (var i = batchSize + 1; i <= batchSize * 2; i++) - _FooEntity(id: i, valueA: 'aaaaaa', valueB: 111111, valueC: true), - ], - ); - - debugPrint( - 'insertMany again tooks ${(sw2.elapsedMicroseconds / 1000).toStringAsFixed(2)}ms', - ); - } - - expect(fooStore.queryOnce(), hasLength(batchSize * 2 + 1)); - }, - skip: !Platform.isMacOS, // only run locally for now - ); } class _FooEntity { diff --git a/test/performance_test.dart b/test/performance_test.dart new file mode 100644 index 0000000..b6f6396 --- /dev/null +++ b/test/performance_test.dart @@ -0,0 +1,161 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:flutter/foundation.dart'; +import 'package:flutter/widgets.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:indexed_entity_store/indexed_entity_store.dart'; + +void main() { + WidgetsFlutterBinding.ensureInitialized(); + + const singleStatement = true; + + test( + 'Performance', + () async { + final path = + '/tmp/index_entity_store_test_${FlutterTimeline.now}.sqlite3'; + + final db = IndexedEntityDabase.open(path); + + final fooStore = db.entityStore(fooConnector); + + expect(fooStore.queryOnce(), isEmpty); + + // Insert one row, so statements are prepared + fooStore.write( + _FooEntity(id: 0, valueA: 'a', valueB: 1, valueC: true), + ); + + const batchSize = 1000; + + // many + { + final sw2 = Stopwatch()..start(); + + for (var i = 1; i <= batchSize; i++) { + fooStore.write( + _FooEntity(id: i, valueA: 'a', valueB: 1, valueC: true), + ); + } + + debugPrint( + '$batchSize x `write` took ${(sw2.elapsedMicroseconds / 1000).toStringAsFixed(2)}ms', + ); + } + + // writeMany + { + final sw2 = Stopwatch()..start(); + + fooStore.writeMany( + [ + for (var i = batchSize + 1; i <= batchSize * 2; i++) + _FooEntity(id: i, valueA: 'a', valueB: 1, valueC: true), + ], + singleStatement: singleStatement, + ); + + debugPrint( + '`writeMany` took ${(sw2.elapsedMicroseconds / 1000).toStringAsFixed(2)}ms', + ); + } + + // writeMany again (which needs to replace all existing entities and update the indices) + { + final sw2 = Stopwatch()..start(); + + fooStore.writeMany( + [ + for (var i = batchSize + 1; i <= batchSize * 2; i++) + _FooEntity(id: i, valueA: 'aaaaaa', valueB: 111111, valueC: true), + ], + singleStatement: singleStatement, + ); + + debugPrint( + '`writeMany` again took ${(sw2.elapsedMicroseconds / 1000).toStringAsFixed(2)}ms', + ); + } + + expect(fooStore.queryOnce(), hasLength(batchSize * 2 + 1)); + }, + skip: !Platform.isMacOS, // only run locally for now + ); +} + +class _FooEntity { + _FooEntity({ + required this.id, + required this.valueA, + required this.valueB, + required this.valueC, + }); + + final int id; + + /// indexed via `a` + final String valueA; + + /// indexed via "b" + final int valueB; + + /// not indexed + final bool valueC; + + Map toJSON() { + return { + 'id': id, + 'valueA': valueA, + 'valueB': valueB, + 'valueC': valueC, + }; + } + + static _FooEntity fromJSON(Map json) { + return _FooEntity( + id: json['id'], + valueA: json['valueA'], + valueB: json['valueB'], + valueC: json['valueC'], + ); + } +} + +final fooConnector = IndexedEntityConnector<_FooEntity, int, String>( + entityKey: 'foo', + getPrimaryKey: (f) => f.id, + getIndices: (index) { + index((e) => e.valueA, as: 'a'); + index((e) => e.valueB, as: 'b'); + }, + serialize: (f) => jsonEncode(f.toJSON()), + deserialize: (s) => _FooEntity.fromJSON( + jsonDecode(s) as Map, + ), +); + +final fooConnectorWithIndexOnBAndC = + IndexedEntityConnector<_FooEntity, int, String>( + entityKey: fooConnector.entityKey, + getPrimaryKey: fooConnector.getPrimaryKey, + getIndices: (index) { + index((e) => e.valueB + 1000, as: 'b'); // updated index B + index((e) => e.valueC, as: 'c'); + }, + serialize: fooConnector.serialize, + deserialize: fooConnector.deserialize, +); + +final fooConnectorWithUniqueIndexOnBAndIndexOnC = + IndexedEntityConnector<_FooEntity, int, String>( + entityKey: fooConnector.entityKey, + getPrimaryKey: fooConnector.getPrimaryKey, + getIndices: (index) { + index((e) => e.valueB + 1000, as: 'b', unique: true); // updated index B + index((e) => e.valueC, as: 'c'); + }, + serialize: fooConnector.serialize, + deserialize: fooConnector.deserialize, +);