diff --git a/yuuna/lib/src/dictionary/dictionary_operations_params.dart b/yuuna/lib/src/dictionary/dictionary_operations_params.dart index d14977487..220b492e5 100644 --- a/yuuna/lib/src/dictionary/dictionary_operations_params.dart +++ b/yuuna/lib/src/dictionary/dictionary_operations_params.dart @@ -58,6 +58,7 @@ class PrepareDictionaryParams extends IsolateParams { required this.dictionary, required this.dictionaryFormat, required this.workingDirectory, + required this.useSlowImport, required super.sendPort, }); @@ -69,6 +70,9 @@ class PrepareDictionaryParams extends IsolateParams { /// A working directory from which to extract dictionary data from. final Directory workingDirectory; + + /// Whether or not to use ACID-compliant importing. + final bool useSlowImport; } /// For isolate communication purposes. Used for dictionary deletion. diff --git a/yuuna/lib/src/dictionary/dictionary_utils.dart b/yuuna/lib/src/dictionary/dictionary_utils.dart index ecef3f20e..d223c153b 100644 --- a/yuuna/lib/src/dictionary/dictionary_utils.dart +++ b/yuuna/lib/src/dictionary/dictionary_utils.dart @@ -36,7 +36,7 @@ Future depositDictionaryDataHelper(PrepareDictionaryParams params) async { /// Create a new instance of Isar as this is a different isolate. final Isar database = await Isar.open( globalSchemas, - maxSizeMiB: 4096, + maxSizeMiB: 8192, ); /// Perform format-specific entity generation. @@ -108,130 +108,247 @@ Future depositDictionaryDataHelper(PrepareDictionaryParams params) async { } } - /// Write the [Dictionary] entity. - database.writeTxnSync(() { - database.dictionarys.putSync(params.dictionary); - }); - - /// Write [DictionaryTag] entities. - int tagCount = 0; - int tagTotal = tags.length; - partition(tags, 10000).forEach((batch) { + if (params.useSlowImport) { + /// Write the [Dictionary] entity. database.writeTxnSync(() { - database.dictionaryTags.putAllSync(batch); + database.dictionarys.putSync(params.dictionary); }); - tagCount += batch.length; - params.send(t.import_write_tag(count: tagCount, total: tagTotal)); - }); - - /// Write [DictionaryPitch] entities. - int pitchCount = 0; - int pitchTotal = pitchesByHeading.values.map((e) => e.length).sum; - partition>>( - pitchesByHeading.entries, 10000) - .forEach((batch) { - database.writeTxnSync(() { - for (MapEntry> pitchesForHeading in batch) { - DictionaryHeading heading = pitchesForHeading.key; - List pitches = pitchesForHeading.value; - database.dictionaryHeadings.putSync(heading); - database.dictionaryPitchs.putAllSync(pitches); - - pitchCount += pitches.length; - } + /// Write [DictionaryTag] entities. + int tagCount = 0; + int tagTotal = tags.length; + partition(tags, 10000).forEach((batch) { + database.writeTxnSync(() { + database.dictionaryTags.putAllSync(batch); + }); + tagCount += batch.length; + params.send(t.import_write_tag(count: tagCount, total: tagTotal)); }); - params.send(t.import_write_pitch(count: pitchCount, total: pitchTotal)); - }); - - /// Write [DictionaryFrequency] entities. - int frequencyCount = 0; - int frequencyTotal = frequenciesByHeading.values.map((e) => e.length).sum; - partition>>( - frequenciesByHeading.entries, 10000) - .forEach((batch) { - database.writeTxnSync(() { - for (MapEntry> frequenciesForHeading in batch) { - DictionaryHeading heading = frequenciesForHeading.key; - List frequencies = frequenciesForHeading.value; - - database.dictionaryHeadings.putSync(heading); - database.dictionaryFrequencys.putAllSync(frequencies); - - frequencyCount += frequencies.length; - } + /// Write [DictionaryPitch] entities. + int pitchCount = 0; + int pitchTotal = pitchesByHeading.values.map((e) => e.length).sum; + partition>>( + pitchesByHeading.entries, 10000) + .forEach((batch) { + database.writeTxnSync(() { + for (MapEntry> pitchesForHeading in batch) { + DictionaryHeading heading = pitchesForHeading.key; + List pitches = pitchesForHeading.value; + + database.dictionaryHeadings.putSync(heading); + database.dictionaryPitchs.putAllSync(pitches); + + pitchCount += pitches.length; + } + }); + + params.send(t.import_write_pitch(count: pitchCount, total: pitchTotal)); }); - params.send(t.import_write_frequency( - count: frequencyCount, total: frequencyTotal)); - }); - - /// Used to test the collision resistance of the FNV-1a algorithm used - /// for hashing dictionary headings to each have unique integer IDs. - /// This doesn't seem that heavy but we shouldn't instantiate millions - /// of elements at any given time, so this should be commented out for - /// a production release or when not debugging for collisions. - /// - /// For testing, a mix of Japanese bilingual and monolingual dictionaries - /// can be imported in sequence. The collision count should always be - /// zero. Interestingly, the Dart implementation of FNV-1a recommended by - /// Isar seems to produce less collisions than a MurmurHash V3 - /// implementation. In any case, the code below can be uncommented for - /// and hash algorithm comparison testing and research. - /// - /// The idea is to get the delta number of headings, but also take into - /// account the number of headings already in the database. - - // int headingsInDatabase = database.dictionaryHeadings.countSync(); - // int headingsToImportAlreadyInDatabase = database.dictionaryHeadings - // .getAllSync(entriesByHeading.keys.map((e) => e.id).toList()) - // .whereNotNull() - // .length; - // int headingsToImportNotInDatabase = - // entriesByHeading.keys.length - headingsToImportAlreadyInDatabase; - - // debugPrint('Headings In Database: $headingsInDatabase'); - // debugPrint( - // 'Headings To Import Already In Database: $headingsToImportAlreadyInDatabase'); - // debugPrint( - // 'Headings To Import Not In Database: $headingsToImportNotInDatabase'); - - /// Write [DictionaryEntry] entities. - int entryCount = 0; - int entryTotal = entriesByHeading.values.map((e) => e.length).sum; - partition>>( - entriesByHeading.entries, 10000) - .forEach((batch) { - database.writeTxnSync(() { - for (MapEntry> entriesForHeading in batch) { - DictionaryHeading heading = entriesForHeading.key; - List entries = entriesForHeading.value; - - database.dictionaryHeadings.putSync(heading); - database.dictionaryEntrys.putAllSync(entries); - - entryCount += entries.length; - } + /// Write [DictionaryFrequency] entities. + int frequencyCount = 0; + int frequencyTotal = frequenciesByHeading.values.map((e) => e.length).sum; + partition>>( + frequenciesByHeading.entries, 10000) + .forEach((batch) { + database.writeTxnSync(() { + for (MapEntry> frequenciesForHeading in batch) { + DictionaryHeading heading = frequenciesForHeading.key; + List frequencies = frequenciesForHeading.value; + + database.dictionaryHeadings.putSync(heading); + database.dictionaryFrequencys.putAllSync(frequencies); + + frequencyCount += frequencies.length; + } + }); + + params.send(t.import_write_frequency( + count: frequencyCount, total: frequencyTotal)); }); - params.send(t.import_write_entry(count: entryCount, total: entryTotal)); - }); + /// Used to test the collision resistance of the FNV-1a algorithm used + /// for hashing dictionary headings to each have unique integer IDs. + /// This doesn't seem that heavy but we shouldn't instantiate millions + /// of elements at any given time, so this should be commented out for + /// a production release or when not debugging for collisions. + /// + /// For testing, a mix of Japanese bilingual and monolingual dictionaries + /// can be imported in sequence. The collision count should always be + /// zero. Interestingly, the Dart implementation of FNV-1a recommended by + /// Isar seems to produce less collisions than a MurmurHash V3 + /// implementation. In any case, the code below can be uncommented for + /// and hash algorithm comparison testing and research. + /// + /// The idea is to get the delta number of headings, but also take into + /// account the number of headings already in the database. + + // int headingsInDatabase = database.dictionaryHeadings.countSync(); + // int headingsToImportAlreadyInDatabase = database.dictionaryHeadings + // .getAllSync(entriesByHeading.keys.map((e) => e.id).toList()) + // .whereNotNull() + // .length; + // int headingsToImportNotInDatabase = + // entriesByHeading.keys.length - headingsToImportAlreadyInDatabase; + + // debugPrint('Headings In Database: $headingsInDatabase'); + // debugPrint( + // 'Headings To Import Already In Database: $headingsToImportAlreadyInDatabase'); + // debugPrint( + // 'Headings To Import Not In Database: $headingsToImportNotInDatabase'); + + /// Write [DictionaryEntry] entities. + int entryCount = 0; + int entryTotal = entriesByHeading.values.map((e) => e.length).sum; + partition>>( + entriesByHeading.entries, 10000) + .forEach((batch) { + database.writeTxnSync(() { + for (MapEntry> entriesForHeading in batch) { + DictionaryHeading heading = entriesForHeading.key; + List entries = entriesForHeading.value; + + database.dictionaryHeadings.putSync(heading); + database.dictionaryEntrys.putAllSync(entries); + + entryCount += entries.length; + } + }); + + params.send(t.import_write_entry(count: entryCount, total: entryTotal)); + }); - /// Collision count should always be zero. + /// Collision count should always be zero. - // int newHeadingsInDatabase = database.dictionaryHeadings.countSync(); - // int collisionsFound = newHeadingsInDatabase - - // headingsInDatabase - - // headingsToImportNotInDatabase; - // debugPrint('New Headings In Database: $newHeadingsInDatabase'); - // debugPrint('Collisions Found: $collisionsFound'); + // int newHeadingsInDatabase = database.dictionaryHeadings.countSync(); + // int collisionsFound = newHeadingsInDatabase - + // headingsInDatabase - + // headingsToImportNotInDatabase; + // debugPrint('New Headings In Database: $newHeadingsInDatabase'); + // debugPrint('Collisions Found: $collisionsFound'); + } else { + /// Write as one transaction. If anything fails, no changes should occur. + database.writeTxnSync(() { + /// Write the [Dictionary] entity. + database.dictionarys.putSync(params.dictionary); + + /// Write [DictionaryTag] entities. + int tagCount = 0; + int tagTotal = tags.length; + database.dictionaryTags.putAllSync(tags); + partition(tags, 10000).forEach((batch) { + database.dictionaryTags.putAllSync(batch); + tagCount += batch.length; + params.send(t.import_write_tag(count: tagCount, total: tagTotal)); + }); + + /// Write [DictionaryPitch] entities. + int pitchCount = 0; + int pitchTotal = pitchesByHeading.values.map((e) => e.length).sum; + partition>>( + pitchesByHeading.entries, 10000) + .forEach((batch) { + for (MapEntry> pitchesForHeading in batch) { + DictionaryHeading heading = pitchesForHeading.key; + List pitches = pitchesForHeading.value; + + database.dictionaryHeadings.putSync(heading); + database.dictionaryPitchs.putAllSync(pitches); + pitchCount += pitches.length; + } + + params + .send(t.import_write_pitch(count: pitchCount, total: pitchTotal)); + }); + + /// Write [DictionaryFrequency] entities. + int frequencyCount = 0; + int frequencyTotal = + frequenciesByHeading.values.map((e) => e.length).sum; + partition>>( + frequenciesByHeading.entries, 10000) + .forEach((batch) { + for (MapEntry> frequenciesForHeading in batch) { + DictionaryHeading heading = frequenciesForHeading.key; + List frequencies = frequenciesForHeading.value; + + database.dictionaryHeadings.putSync(heading); + database.dictionaryFrequencys.putAllSync(frequencies); + frequencyCount += frequencies.length; + } + + params.send(t.import_write_frequency( + count: frequencyCount, total: frequencyTotal)); + }); + + /// Used to test the collision resistance of the FNV-1a algorithm used + /// for hashing dictionary headings to each have unique integer IDs. + /// This doesn't seem that heavy but we shouldn't instantiate millions + /// of elements at any given time, so this should be commented out for + /// a production release or when not debugging for collisions. + /// + /// For testing, a mix of Japanese bilingual and monolingual dictionaries + /// can be imported in sequence. The collision count should always be + /// zero. Interestingly, the Dart implementation of FNV-1a recommended by + /// Isar seems to produce less collisions than a MurmurHash V3 + /// implementation. In any case, the code below can be uncommented for + /// and hash algorithm comparison testing and research. + /// + /// The idea is to get the delta number of headings, but also take into + /// account the number of headings already in the database. + // int headingsInDatabase = database.dictionaryHeadings.countSync(); + // int headingsToImportAlreadyInDatabase = database.dictionaryHeadings + // .getAllSync(entriesByHeading.keys.map((e) => e.id).toList()) + // .whereNotNull() + // .length; + // int headingsToImportNotInDatabase = + // entriesByHeading.keys.length - headingsToImportAlreadyInDatabase; + + // debugPrint('Headings In Database: $headingsInDatabase'); + // debugPrint( + // 'Headings To Import Already In Database: $headingsToImportAlreadyInDatabase'); + // debugPrint( + // 'Headings To Import Not In Database: $headingsToImportNotInDatabase'); + + /// Write [DictionaryEntry] entities. + int entryCount = 0; + int entryTotal = entriesByHeading.values.map((e) => e.length).sum; + partition>>( + entriesByHeading.entries, 10000) + .forEach((batch) { + for (MapEntry> entriesForHeading in batch) { + DictionaryHeading heading = entriesForHeading.key; + List entries = entriesForHeading.value; + + database.dictionaryHeadings.putSync(heading); + database.dictionaryEntrys.putAllSync(entries); + entryCount += entries.length; + } + + params + .send(t.import_write_entry(count: entryCount, total: entryTotal)); + }); + + /// Collision count should always be zero. + // int newHeadingsInDatabase = database.dictionaryHeadings.countSync(); + // int collisionsFound = newHeadingsInDatabase - + // headingsInDatabase - + // headingsToImportNotInDatabase; + // debugPrint('New Headings In Database: $newHeadingsInDatabase'); + // debugPrint('Collisions Found: $collisionsFound'); + }); + } } catch (e, stackTrace) { params.send(stackTrace); params.send(e); + rethrow; } } @@ -240,7 +357,7 @@ Future preloadResult(int id) async { /// Create a new instance of Isar as this is a different isolate. final Isar database = await Isar.open( globalSchemas, - maxSizeMiB: 4096, + maxSizeMiB: 8192, ); DictionarySearchResult result = database.dictionarySearchResults.getSync(id)!; @@ -283,7 +400,7 @@ Future updateDictionaryHistoryHelper( ) async { final Isar database = await Isar.open( globalSchemas, - maxSizeMiB: 4096, + maxSizeMiB: 8192, ); DictionarySearchResult result = @@ -299,7 +416,7 @@ Future updateDictionaryHistoryHelper( Future deleteDictionariesHelper(DeleteDictionaryParams params) async { final Isar database = await Isar.open( globalSchemas, - maxSizeMiB: 4096, + maxSizeMiB: 8192, ); database.writeTxnSync(() { @@ -317,7 +434,7 @@ Future deleteDictionariesHelper(DeleteDictionaryParams params) async { Future deleteDictionaryHelper(DeleteDictionaryParams params) async { final Isar database = await Isar.open( globalSchemas, - maxSizeMiB: 4096, + maxSizeMiB: 8192, ); int id = params.dictionaryId!; diff --git a/yuuna/lib/src/language/implementations/english_language.dart b/yuuna/lib/src/language/implementations/english_language.dart index 40149ca50..14e996677 100644 --- a/yuuna/lib/src/language/implementations/english_language.dart +++ b/yuuna/lib/src/language/implementations/english_language.dart @@ -48,7 +48,7 @@ Future prepareSearchResultsEnglishLanguage( final Lemmatizer lemmatizer = Lemmatizer(); final Isar database = await Isar.open( globalSchemas, - maxSizeMiB: 4096, + maxSizeMiB: 8192, ); int bestLength = 0; diff --git a/yuuna/lib/src/language/implementations/japanese_language.dart b/yuuna/lib/src/language/implementations/japanese_language.dart index db6ad34d1..900bf2f2a 100644 --- a/yuuna/lib/src/language/implementations/japanese_language.dart +++ b/yuuna/lib/src/language/implementations/japanese_language.dart @@ -303,7 +303,7 @@ Future prepareSearchResultsJapaneseLanguage( final Isar database = await Isar.open( globalSchemas, - maxSizeMiB: 4096, + maxSizeMiB: 8192, ); Map uniqueHeadingsById = {}; diff --git a/yuuna/lib/src/models/app_model.dart b/yuuna/lib/src/models/app_model.dart index 5faa902fc..99b923ad3 100644 --- a/yuuna/lib/src/models/app_model.dart +++ b/yuuna/lib/src/models/app_model.dart @@ -18,7 +18,6 @@ import 'package:flutter_accessibility_service/accessibility_event.dart'; import 'package:flutter_accessibility_service/flutter_accessibility_service.dart'; import 'package:flutter_cache_manager/flutter_cache_manager.dart'; import 'package:flutter_charset_detector/flutter_charset_detector.dart'; -import 'package:flutter_exit_app/flutter_exit_app.dart'; import 'package:flutter_riverpod/flutter_riverpod.dart'; import 'package:flutter_vlc_player/flutter_vlc_player.dart'; import 'package:fluttertoast/fluttertoast.dart'; @@ -932,7 +931,7 @@ class AppModel with ChangeNotifier { /// Initialise persistent database. _database = await Isar.open( globalSchemas, - maxSizeMiB: 4096, + maxSizeMiB: 8192, ); /// Perform startup activities unnecessary to further initialisation here. @@ -1262,6 +1261,7 @@ class AppModel with ChangeNotifier { dictionary: dictionary, workingDirectory: workingDirectory, dictionaryFormat: dictionaryFormat, + useSlowImport: useSlowImport, sendPort: receivePort.sendPort, ); @@ -2091,7 +2091,7 @@ class AppModel with ChangeNotifier { await _audioHandler?.stop(); if (_shouldKillMediaOnPop) { - await FlutterExitApp.exitApp(); + SystemNavigator.pop(); } } diff --git a/yuuna/lib/src/pages/implementations/dictionary_dialog_page.dart b/yuuna/lib/src/pages/implementations/dictionary_dialog_page.dart index a71a48bb9..ffa5748b2 100644 --- a/yuuna/lib/src/pages/implementations/dictionary_dialog_page.dart +++ b/yuuna/lib/src/pages/implementations/dictionary_dialog_page.dart @@ -255,7 +255,7 @@ class _DictionaryDialogPageState extends BasePageState { ), const JidoujishoDivider(), buildImportDropdown(), - // buildSlowImportSwitch(), + buildSlowImportSwitch(), ], ), ), diff --git a/yuuna/pubspec.yaml b/yuuna/pubspec.yaml index 4ffdad656..f33a7a231 100644 --- a/yuuna/pubspec.yaml +++ b/yuuna/pubspec.yaml @@ -1,7 +1,7 @@ name: yuuna description: A full-featured immersion language learning suite for mobile. publish_to: 'none' -version: 2.4.4+45 +version: 2.4.5+46 environment: sdk: ">=2.17.0 <3.0.0"