diff --git a/example/perf/write_many.dart b/example/perf/write_many.dart new file mode 100644 index 0000000..b674688 --- /dev/null +++ b/example/perf/write_many.dart @@ -0,0 +1,169 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:flutter/foundation.dart' show FlutterTimeline, debugPrint; +import 'package:flutter/widgets.dart'; +import 'package:indexed_entity_store/indexed_entity_store.dart'; +import 'package:path_provider/path_provider.dart'; + +void main() async { + WidgetsFlutterBinding.ensureInitialized(); + + for (final batchSize in [10, 100, 1000, 10000]) { + for (final singleStatement in [true, false]) { + for (final largeValue in [true, false]) { + debugPrint( + '\nCase: singleStatement=$singleStatement, largeValue=$largeValue, batchSize=$batchSize', + ); + + final path = (await getApplicationCacheDirectory()) + .uri + .resolve('./index_entity_store_test_${FlutterTimeline.now}.sqlite3') + .toFilePath(); + + debugPrint(path); + + final db = IndexedEntityDabase.open(path); + + final fooStore = db.entityStore(fooConnector); + + // Insert one row each, so statements are prepared + fooStore.write( + _FooEntity(id: 0, valueA: 'a', valueB: 1, valueC: true), + ); + fooStore.writeMany( + [_FooEntity(id: 0, valueA: 'a', valueB: 1, valueC: true)], + singleStatement: singleStatement, + ); + fooStore.delete(key: 0); + + // many + { + final sw2 = Stopwatch()..start(); + + for (var i = 1; i <= batchSize; i++) { + fooStore.write( + _FooEntity(id: i, valueA: 'a', valueB: 1, valueC: true), + ); + } + + final durMs = (sw2.elapsedMicroseconds / 1000).toStringAsFixed(2); + debugPrint( + '$batchSize x `write` took ${durMs}ms', + ); + } + + // 10 kB + final valueA = largeValue ? 'a1' * 1024 * 10 : 'a1'; + final valueA2 = largeValue ? 'a2' * 1024 * 10 : 'a2'; + + // writeMany + { + final sw2 = Stopwatch()..start(); + + fooStore.writeMany( + [ + for (var i = batchSize + 1; i <= batchSize * 2; i++) + _FooEntity( + id: i, + valueA: valueA, + valueB: 1, + valueC: true, + ), + ], + singleStatement: singleStatement, + ); + + final durMs = (sw2.elapsedMicroseconds / 1000).toStringAsFixed(2); + debugPrint( + '`writeMany` took ${durMs}ms', + ); + } + + // writeMany again (which needs to replace all existing entities and update the indices) + { + final sw2 = Stopwatch()..start(); + + fooStore.writeMany( + [ + for (var i = batchSize + 1; i <= batchSize * 2; i++) + _FooEntity( + id: i, + valueA: valueA2, + valueB: 111111, + valueC: true, + ), + ], + singleStatement: singleStatement, + ); + + final durMs = (sw2.elapsedMicroseconds / 1000).toStringAsFixed(2); + debugPrint( + '`writeMany` again took ${durMs}ms', + ); + } + + if (fooStore.queryOnce().length != (batchSize * 2)) { + throw 'unexpected store size'; + } + + db.dispose(); + + File(path).deleteSync(); + } + } + } + + exit(0); +} + +class _FooEntity { + _FooEntity({ + required this.id, + required this.valueA, + required this.valueB, + required this.valueC, + }); + + final int id; + + /// indexed via `a` + final String valueA; + + /// indexed via "b" + final int valueB; + + /// not indexed + final bool valueC; + + Map toJSON() { + return { + 'id': id, + 'valueA': valueA, + 'valueB': valueB, + 'valueC': valueC, + }; + } + + static _FooEntity fromJSON(Map json) { + return _FooEntity( + id: json['id'], + valueA: json['valueA'], + valueB: json['valueB'], + valueC: json['valueC'], + ); + } +} + +final fooConnector = IndexedEntityConnector<_FooEntity, int, String>( + entityKey: 'foo', + getPrimaryKey: (f) => f.id, + getIndices: (index) { + index((e) => e.valueA, as: 'a'); + index((e) => e.valueB, as: 'b'); + }, + serialize: (f) => jsonEncode(f.toJSON()), + deserialize: (s) => _FooEntity.fromJSON( + jsonDecode(s) as Map, + ), +); diff --git a/example/run_perf.sh b/example/run_perf.sh new file mode 100755 index 0000000..7ca72ec --- /dev/null +++ b/example/run_perf.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +set +x + +fvm flutter run --release -d macos ./perf/write_many.dart +# fvm flutter run -d macos ./perf/write_many.dart diff --git a/lib/src/index_entity_store.dart b/lib/src/index_entity_store.dart index abab40d..2e8f858 100644 --- a/lib/src/index_entity_store.dart +++ b/lib/src/index_entity_store.dart @@ -241,28 +241,55 @@ class IndexedEntityStore { /// Insert or update many entities in a single batch /// /// Notification for changes will only fire after all changes have been written (meaning queries will get a single update after all writes are finished) - void writeMany(Iterable entities) { + void writeMany( + Iterable entities, { + bool singleStatement = true, + }) { final keys = {}; - try { - _database.execute('BEGIN'); - assert(_database.autocommit == false); + if (singleStatement) { + if (entities.isEmpty) { + return; + } - for (final e in entities) { - _entityInsertStatement.execute( - [_entityKey, _connector.getPrimaryKey(e), _connector.serialize(e)], - ); + _database.execute( + [ + 'REPLACE INTO `entity` (`type`, `key`, `value`) ' + ' VALUES (?1, ?, ?)', + // Add additional entry values for each further parameter + ', (?1, ?, ?)' * (entities.length - 1), + ].join(' '), + [ + _entityKey, + for (final e in entities) ...[ + _connector.getPrimaryKey(e), + _connector.serialize(e), + ], + ], + ); + } else { + // transaction variant - _updateIndexInternal(e); + try { + _database.execute('BEGIN'); + assert(_database.autocommit == false); - keys.add(_connector.getPrimaryKey(e)); - } + for (final e in entities) { + _entityInsertStatement.execute( + [_entityKey, _connector.getPrimaryKey(e), _connector.serialize(e)], + ); - _database.execute('COMMIT'); - } catch (e) { - _database.execute('ROLLBACK'); + _updateIndexInternal(e); - rethrow; + keys.add(_connector.getPrimaryKey(e)); + } + + _database.execute('COMMIT'); + } catch (e) { + _database.execute('ROLLBACK'); + + rethrow; + } } _handleUpdate(keys); diff --git a/lib/src/indexed_entity_database.dart b/lib/src/indexed_entity_database.dart index 1702400..1f12591 100644 --- a/lib/src/indexed_entity_database.dart +++ b/lib/src/indexed_entity_database.dart @@ -136,7 +136,7 @@ class IndexedEntityDabase { ); // This index is needed to not pay a performance penalty for the new foreign key constraint (between entities) - // Otherwise the `insertMany` update duration would increase 5x (even without making use of the reference, passing `null`). + // Otherwise the `writeMany` update duration would increase 5x (even without making use of the reference, passing `null`). // With this index, even though it only tracks non-`null` references (which would be rare), overall insert performance stays the same as before. _database.execute( 'CREATE INDEX index_field_values_FK ' diff --git a/test/indexed_entity_store_test.dart b/test/indexed_entity_store_test.dart index fe7e61a..b427fef 100644 --- a/test/indexed_entity_store_test.dart +++ b/test/indexed_entity_store_test.dart @@ -1173,77 +1173,6 @@ void main() { expect(listSubscription.value, isEmpty); expect(fooStore.queryOnce(), isEmpty); }); - - test( - 'Performance', - () async { - final path = - '/tmp/index_entity_store_test_${FlutterTimeline.now}.sqlite3'; - - final db = IndexedEntityDabase.open(path); - - final fooStore = db.entityStore(fooConnector); - - expect(fooStore.queryOnce(), isEmpty); - - // Insert one row, so statements are prepared - fooStore.write( - _FooEntity(id: 0, valueA: 'a', valueB: 1, valueC: true), - ); - - const batchSize = 1000; - - // many - { - final sw2 = Stopwatch()..start(); - - for (var i = 1; i <= batchSize; i++) { - fooStore.write( - _FooEntity(id: i, valueA: 'a', valueB: 1, valueC: true), - ); - } - - debugPrint( - 'insert tooks ${(sw2.elapsedMicroseconds / 1000).toStringAsFixed(2)}ms', - ); - } - - // insertMany - { - final sw2 = Stopwatch()..start(); - - fooStore.writeMany( - [ - for (var i = batchSize + 1; i <= batchSize * 2; i++) - _FooEntity(id: i, valueA: 'a', valueB: 1, valueC: true), - ], - ); - - debugPrint( - 'insertMany tooks ${(sw2.elapsedMicroseconds / 1000).toStringAsFixed(2)}ms', - ); - } - - // many again (which needs to replace all existing entities and update the indices) - { - final sw2 = Stopwatch()..start(); - - fooStore.writeMany( - [ - for (var i = batchSize + 1; i <= batchSize * 2; i++) - _FooEntity(id: i, valueA: 'aaaaaa', valueB: 111111, valueC: true), - ], - ); - - debugPrint( - 'insertMany again tooks ${(sw2.elapsedMicroseconds / 1000).toStringAsFixed(2)}ms', - ); - } - - expect(fooStore.queryOnce(), hasLength(batchSize * 2 + 1)); - }, - skip: !Platform.isMacOS, // only run locally for now - ); } class _FooEntity { diff --git a/test/performance_test.dart b/test/performance_test.dart new file mode 100644 index 0000000..4ef2536 --- /dev/null +++ b/test/performance_test.dart @@ -0,0 +1,312 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:flutter/foundation.dart'; +import 'package:flutter/widgets.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:indexed_entity_store/indexed_entity_store.dart'; + +void main() { + WidgetsFlutterBinding.ensureInitialized(); + + const singleStatement = true; + + test( + 'Performance', + () async { + final path = + '/tmp/index_entity_store_test_${FlutterTimeline.now}.sqlite3'; + + final db = IndexedEntityDabase.open(path); + + final fooStore = db.entityStore(fooConnector); + + expect(fooStore.queryOnce(), isEmpty); + + // Insert one row, so statements are prepared + fooStore.write( + _FooEntity(id: 0, valueA: 'a', valueB: 1, valueC: true), + ); + + const batchSize = 1000; + + // many + { + final sw2 = Stopwatch()..start(); + + for (var i = 1; i <= batchSize; i++) { + fooStore.write( + _FooEntity(id: i, valueA: 'a', valueB: 1, valueC: true), + ); + } + + debugPrint( + '$batchSize x `write` took ${(sw2.elapsedMicroseconds / 1000).toStringAsFixed(2)}ms', + ); + } + + // writeMany + { + final sw2 = Stopwatch()..start(); + + fooStore.writeMany( + [ + for (var i = batchSize + 1; i <= batchSize * 2; i++) + _FooEntity(id: i, valueA: 'a', valueB: 1, valueC: true), + ], + singleStatement: singleStatement, + ); + + debugPrint( + '`writeMany` took ${(sw2.elapsedMicroseconds / 1000).toStringAsFixed(2)}ms', + ); + } + + // writeMany again (which needs to replace all existing entities and update the indices) + { + final sw2 = Stopwatch()..start(); + + fooStore.writeMany( + [ + for (var i = batchSize + 1; i <= batchSize * 2; i++) + _FooEntity(id: i, valueA: 'aaaaaa', valueB: 111111, valueC: true), + ], + singleStatement: singleStatement, + ); + + debugPrint( + '`writeMany` again took ${(sw2.elapsedMicroseconds / 1000).toStringAsFixed(2)}ms', + ); + } + + expect(fooStore.queryOnce(), hasLength(batchSize * 2 + 1)); + }, + skip: !Platform.isMacOS, // only run locally for now + ); +} + +class _FooEntity { + _FooEntity({ + required this.id, + required this.valueA, + required this.valueB, + required this.valueC, + }); + + final int id; + + /// indexed via `a` + final String valueA; + + /// indexed via "b" + final int valueB; + + /// not indexed + final bool valueC; + + Map toJSON() { + return { + 'id': id, + 'valueA': valueA, + 'valueB': valueB, + 'valueC': valueC, + }; + } + + static _FooEntity fromJSON(Map json) { + return _FooEntity( + id: json['id'], + valueA: json['valueA'], + valueB: json['valueB'], + valueC: json['valueC'], + ); + } +} + +final fooConnector = IndexedEntityConnector<_FooEntity, int, String>( + entityKey: 'foo', + getPrimaryKey: (f) => f.id, + getIndices: (index) { + index((e) => e.valueA, as: 'a'); + index((e) => e.valueB, as: 'b'); + }, + serialize: (f) => jsonEncode(f.toJSON()), + deserialize: (s) => _FooEntity.fromJSON( + jsonDecode(s) as Map, + ), +); + +final fooConnectorWithIndexOnBAndC = + IndexedEntityConnector<_FooEntity, int, String>( + entityKey: fooConnector.entityKey, + getPrimaryKey: fooConnector.getPrimaryKey, + getIndices: (index) { + index((e) => e.valueB + 1000, as: 'b'); // updated index B + index((e) => e.valueC, as: 'c'); + }, + serialize: fooConnector.serialize, + deserialize: fooConnector.deserialize, +); + +final fooConnectorWithUniqueIndexOnBAndIndexOnC = + IndexedEntityConnector<_FooEntity, int, String>( + entityKey: fooConnector.entityKey, + getPrimaryKey: fooConnector.getPrimaryKey, + getIndices: (index) { + index((e) => e.valueB + 1000, as: 'b', unique: true); // updated index B + index((e) => e.valueC, as: 'c'); + }, + serialize: fooConnector.serialize, + deserialize: fooConnector.deserialize, +); + +class _AllSupportedIndexTypes { + _AllSupportedIndexTypes({ + required this.string, + required this.stringOpt, + required this.number, + required this.numberOpt, + required this.integer, + required this.integerOpt, + required this.float, + required this.floatOpt, + required this.boolean, + required this.booleanOpt, + required this.dateTime, + required this.dateTimeOpt, + }); + + factory _AllSupportedIndexTypes.defaultIfNull({ + String? string, + String? stringOpt, + num? number, + num? numberOpt, + int? integer, + int? integerOpt, + double? float, + double? floatOpt, + bool? boolean, + bool? booleanOpt, + DateTime? dateTime, + DateTime? dateTimeOpt, + }) { + return _AllSupportedIndexTypes( + string: string ?? '', + stringOpt: stringOpt, + number: number ?? 0, + numberOpt: numberOpt, + integer: integer ?? 0, + integerOpt: integerOpt, + float: float ?? 0, + floatOpt: floatOpt, + boolean: boolean ?? false, + booleanOpt: booleanOpt, + dateTime: dateTime ?? DateTime.now(), + dateTimeOpt: dateTimeOpt, + ); + } + + final String string; + final String? stringOpt; + final num number; + final num? numberOpt; + final int integer; + final int? integerOpt; + final double float; + final double? floatOpt; + final bool boolean; + final bool? booleanOpt; + final DateTime dateTime; + final DateTime? dateTimeOpt; + + Map toJSON() { + return { + 'string': string, + 'stringOpt': stringOpt, + 'number': number, + 'numberOpt': numberOpt, + 'integer': integer, + 'integerOpt': integerOpt, + 'float': float, + 'floatOpt': floatOpt, + 'boolean': boolean, + 'booleanOpt': booleanOpt, + 'dateTime': dateTime.toIso8601String(), + 'dateTimeOpt': dateTimeOpt?.toIso8601String(), + }; + } + + static _AllSupportedIndexTypes fromJSON(Map json) { + return _AllSupportedIndexTypes( + string: json['string'], + stringOpt: json['stringOpt'], + number: json['number'], + numberOpt: json['numberOpt'], + integer: json['integer'], + integerOpt: json['integerOpt'], + float: json['float'], + floatOpt: json['floatOpt'], + boolean: json['boolean'], + booleanOpt: json['booleanOpt'], + dateTime: DateTime.parse(json['dateTime']), + dateTimeOpt: DateTime.tryParse(json['dateTimeOpt'] ?? ''), + ); + } +} + +extension on int { + String get name { + switch (this) { + case 0: + return 'zero'; + case 1: + return 'one'; + case 2: + return 'two'; + case 3: + return 'three'; + case 4: + return 'four'; + case 5: + return 'five'; + case 6: + return 'six'; + case 7: + return 'seven'; + case 8: + return 'eight'; + case 9: + return 'nine'; + + default: + throw '$this not mapped to a name'; + } + } +} + +/// A value wrapper class which only has object identity, and no value-based `==` implementation. +/// +/// This way we can test that change updates are already prevented on the store-layer and do not depend on the `ValueNotifier` preventing updates (due to the current and new value being equal). +class _ValueWrapper { + _ValueWrapper( + this.key, + this.value, + ); + + final int key; + final String value; + + Map toJSON() { + return { + 'key': key, + 'value': value, + }; + } + + static _ValueWrapper fromJSON(Map json) { + return _ValueWrapper( + json['key'], + json['value'], + ); + } +}