From 9afd1dfbfaf98c43acaa6feab2e124121b9d989f Mon Sep 17 00:00:00 2001 From: Timm Preetz Date: Thu, 31 Oct 2024 16:09:12 +0100 Subject: [PATCH] Add `insertMany` method --- CHANGELOG.md | 8 ++++ example/pubspec.lock | 2 +- lib/src/index_entity_store.dart | 44 +++++++++++++---- lib/src/indexed_entity_database.dart | 72 +++++++++++++++++++--------- pubspec.yaml | 2 +- test/indexed_entity_store_test.dart | 71 +++++++++++++++++++++++++++ 6 files changed, 166 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d996bd0..6b8e6f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## 1.4.3 + +* Add `insertMany` method to handle batch inserts/updates + * This, combined with a new index, massively speeds up large inserts: + - Inserting 1000 entities with 2 indices is now 40x faster than a simple loop using `insert`s individually + - When updating 1000 existing entities with new values, the new implementation leads to an even greater 111x speed-up + * This further proves that the synchronous database approach can handle even large local databases and operations. If you need to insert even larger amounts of data without dropping a frame, there is [a solution for that](https://github.com/simolus3/sqlite3.dart/issues/260#issuecomment-2446618546) as well. + ## 1.4.2 * Add full example app diff --git a/example/pubspec.lock b/example/pubspec.lock index af4b048..437d704 100644 --- a/example/pubspec.lock +++ b/example/pubspec.lock @@ -105,7 +105,7 @@ packages: path: ".." relative: true source: path - version: "1.4.2" + version: "1.4.3" leak_tracker: dependency: transitive description: diff --git a/lib/src/index_entity_store.dart b/lib/src/index_entity_store.dart index 11dacb1..7b7fd5e 100644 --- a/lib/src/index_entity_store.dart +++ b/lib/src/index_entity_store.dart @@ -251,6 +251,11 @@ class IndexedEntityStore { ); } + late final _entityInsertStatement = _database.prepare( + 'REPLACE INTO `entity` (`type`, `key`, `value`) VALUES (?, ?, ?)', + persistent: true, + ); + /// Insert or updates the given entity in the database. /// /// In case an entity with the same primary already exists in the database, it will be updated. @@ -259,8 +264,7 @@ class IndexedEntityStore { _database.execute('BEGIN'); assert(_database.autocommit == false); - _database.execute( - 'REPLACE INTO `entity` (`type`, `key`, `value`) VALUES (?, ?, ?)', + _entityInsertStatement.execute( [_entityKey, _connector.getPrimaryKey(e), _connector.serialize(e)], ); @@ -271,15 +275,37 @@ class IndexedEntityStore { _handleUpdate({_connector.getPrimaryKey(e)}); } - void _updateIndexInternal(T e) { - _database.execute( - 'DELETE FROM `index` WHERE `type` = ? AND `entity` = ?', - [_entityKey, _connector.getPrimaryKey(e)], - ); + /// Insert or update many entities in a single batch + /// + /// Notification for changes will only fire after all changes have been written (this queries will get a single update after all writes are finished) + void insertMany(Iterable entities) { + _database.execute('BEGIN'); + assert(_database.autocommit == false); + + final keys = {}; + for (final e in entities) { + _entityInsertStatement.execute( + [_entityKey, _connector.getPrimaryKey(e), _connector.serialize(e)], + ); + _updateIndexInternal(e); + + keys.add(_connector.getPrimaryKey(e)); + } + + _database.execute('COMMIT'); + + _handleUpdate(keys); + } + + late final _insertIndexStatement = _database.prepare( + 'INSERT INTO `index` (`type`, `entity`, `field`, `value`) VALUES (?, ?, ?, ?)', + persistent: true, + ); + + void _updateIndexInternal(T e) { for (final indexColumn in _indexColumns._indexColumns.values) { - _database.execute( - 'INSERT INTO `index` (`type`, `entity`, `field`, `value`) VALUES (?, ?, ?, ?)', + _insertIndexStatement.execute( [ _entityKey, _connector.getPrimaryKey(e), diff --git a/lib/src/indexed_entity_database.dart b/lib/src/indexed_entity_database.dart index 6fddf6b..b3247c3 100644 --- a/lib/src/indexed_entity_database.dart +++ b/lib/src/indexed_entity_database.dart @@ -10,34 +10,62 @@ class IndexedEntityDabase { "SELECT name FROM sqlite_master WHERE type='table' AND name='entity';", ); if (res.isEmpty) { - _database.execute('PRAGMA foreign_keys = ON'); + debugPrint('Creating new DB'); - _database.execute( - 'CREATE TABLE `entity` ( `type` TEXT NOT NULL, `key` NOT NULL, `value`, PRIMARY KEY ( `type`, `key` ) )', - ); + _initialDBSetup(); + _v2Migration(); + } else if (_dbVersion == 1) { + debugPrint('Migrating DB to v2'); - _database.execute( - 'CREATE TABLE `index` ( `type` TEXT NOT NULL, `entity` NOT NULL, `field` TEXT NOT NULL, `value`, ' - ' FOREIGN KEY (`type`, `entity`) REFERENCES `entity` (`type`, `key`) ON DELETE CASCADE' - ')', - ); + _v2Migration(); + } - _database.execute( - 'CREATE INDEX index_field_values ' - 'ON `index` ( `type`, `field`, `value` )', - ); + assert(_dbVersion == 2); + } - _database.execute( - 'CREATE TABLE `metadata` ( `key` TEXT NOT NULL, `value` )', - ); + void _initialDBSetup() { + _database.execute('PRAGMA foreign_keys = ON'); - _database.execute( - 'INSERT INTO `metadata` ( `key`, `value` ) VALUES ( ?, ? )', - ['version', 1], - ); + _database.execute( + 'CREATE TABLE `entity` ( `type` TEXT NOT NULL, `key` NOT NULL, `value`, PRIMARY KEY ( `type`, `key` ) )', + ); - debugPrint('New DB created'); - } + _database.execute( + 'CREATE TABLE `index` ( `type` TEXT NOT NULL, `entity` NOT NULL, `field` TEXT NOT NULL, `value`, ' + ' FOREIGN KEY (`type`, `entity`) REFERENCES `entity` (`type`, `key`) ON DELETE CASCADE' + ')', + ); + + _database.execute( + 'CREATE INDEX index_field_values ' + 'ON `index` ( `type`, `field`, `value` )', + ); + + _database.execute( + 'CREATE TABLE `metadata` ( `key` TEXT NOT NULL, `value` )', + ); + + _database.execute( + 'INSERT INTO `metadata` ( `key`, `value` ) VALUES ( ?, ? )', + ['version', 1], + ); + } + + int get _dbVersion => _database.select( + 'SELECT `value` FROM `metadata` WHERE `key` = ?', + ['version'], + ).single['value'] as int; + + void _v2Migration() { + _database.execute( + 'CREATE UNIQUE INDEX index_type_entity_field_index ' + 'ON `index` ( `type`, `entity`, `field` )', + ); + + _database.execute( + 'UPDATE `metadata` SET `value` = ? WHERE `key` = ?', + [2, 'version'], + ); } factory IndexedEntityDabase.open(String path) { diff --git a/pubspec.yaml b/pubspec.yaml index 0a629fe..bae9216 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -1,6 +1,6 @@ name: indexed_entity_store description: A fast, simple, and synchronous entity store for Flutter applications. -version: 1.4.2 +version: 1.4.3 repository: https://github.com/LunaONE/indexed_entity_store environment: diff --git a/test/indexed_entity_store_test.dart b/test/indexed_entity_store_test.dart index 9bc3304..b7dab2a 100644 --- a/test/indexed_entity_store_test.dart +++ b/test/indexed_entity_store_test.dart @@ -977,6 +977,77 @@ void main() { expect(listSubscription.value, isEmpty); expect(fooStore.getAllOnce(), isEmpty); }); + + test( + 'Performance', + () async { + final path = + '/tmp/index_entity_store_test_${FlutterTimeline.now}.sqlite3'; + + final db = IndexedEntityDabase.open(path); + + final fooStore = db.entityStore(fooConnector); + + expect(fooStore.getAllOnce(), isEmpty); + + // Insert one row, so statements are prepared + fooStore.insert( + _FooEntity(id: 0, valueA: 'a', valueB: 1, valueC: true), + ); + + const batchSize = 1000; + + // many + { + final sw2 = Stopwatch()..start(); + + for (var i = 1; i <= batchSize; i++) { + fooStore.insert( + _FooEntity(id: i, valueA: 'a', valueB: 1, valueC: true), + ); + } + + debugPrint( + 'insert tooks ${(sw2.elapsedMicroseconds / 1000).toStringAsFixed(2)}ms', + ); + } + + // insertMany + { + final sw2 = Stopwatch()..start(); + + fooStore.insertMany( + [ + for (var i = batchSize + 1; i <= batchSize * 2; i++) + _FooEntity(id: i, valueA: 'a', valueB: 1, valueC: true), + ], + ); + + debugPrint( + 'insertMany tooks ${(sw2.elapsedMicroseconds / 1000).toStringAsFixed(2)}ms', + ); + } + + // many again (which needs to replace all existing entities and update the indices) + { + final sw2 = Stopwatch()..start(); + + fooStore.insertMany( + [ + for (var i = batchSize + 1; i <= batchSize * 2; i++) + _FooEntity(id: i, valueA: 'aaaaaa', valueB: 111111, valueC: true), + ], + ); + + debugPrint( + 'insertMany again tooks ${(sw2.elapsedMicroseconds / 1000).toStringAsFixed(2)}ms', + ); + } + + expect(fooStore.getAllOnce(), hasLength(batchSize * 2 + 1)); + }, + skip: !Platform.isMacOS, // only run locally for now + ); } class _FooEntity {