From b6e60301bd82cedd9f35f01ede44d9ee6c3daf0b Mon Sep 17 00:00:00 2001
From: lrorpilla <leorafaelorpilla@gmail.com>
Date: Tue, 28 Mar 2023 14:15:37 +1000
Subject: [PATCH] Add back slow import toggle option

---
 .../dictionary_operations_params.dart         |   4 +
 .../lib/src/dictionary/dictionary_utils.dart  | 347 ++++++++++++------
 .../implementations/english_language.dart     |   2 +-
 .../implementations/japanese_language.dart    |   2 +-
 yuuna/lib/src/models/app_model.dart           |   6 +-
 .../dictionary_dialog_page.dart               |   2 +-
 yuuna/pubspec.yaml                            |   2 +-
 7 files changed, 243 insertions(+), 122 deletions(-)

diff --git a/yuuna/lib/src/dictionary/dictionary_operations_params.dart b/yuuna/lib/src/dictionary/dictionary_operations_params.dart
index d14977487..220b492e5 100644
--- a/yuuna/lib/src/dictionary/dictionary_operations_params.dart
+++ b/yuuna/lib/src/dictionary/dictionary_operations_params.dart
@@ -58,6 +58,7 @@ class PrepareDictionaryParams extends IsolateParams {
     required this.dictionary,
     required this.dictionaryFormat,
     required this.workingDirectory,
+    required this.useSlowImport,
     required super.sendPort,
   });
 
@@ -69,6 +70,9 @@ class PrepareDictionaryParams extends IsolateParams {
 
   /// A working directory from which to extract dictionary data from.
   final Directory workingDirectory;
+
+  /// Whether or not to use ACID-compliant importing.
+  final bool useSlowImport;
 }
 
 /// For isolate communication purposes. Used for dictionary deletion.
diff --git a/yuuna/lib/src/dictionary/dictionary_utils.dart b/yuuna/lib/src/dictionary/dictionary_utils.dart
index ecef3f20e..d223c153b 100644
--- a/yuuna/lib/src/dictionary/dictionary_utils.dart
+++ b/yuuna/lib/src/dictionary/dictionary_utils.dart
@@ -36,7 +36,7 @@ Future<void> depositDictionaryDataHelper(PrepareDictionaryParams params) async {
     /// Create a new instance of Isar as this is a different isolate.
     final Isar database = await Isar.open(
       globalSchemas,
-      maxSizeMiB: 4096,
+      maxSizeMiB: 8192,
     );
 
     /// Perform format-specific entity generation.
@@ -108,130 +108,247 @@ Future<void> depositDictionaryDataHelper(PrepareDictionaryParams params) async {
       }
     }
 
-    /// Write the [Dictionary] entity.
-    database.writeTxnSync(() {
-      database.dictionarys.putSync(params.dictionary);
-    });
-
-    /// Write [DictionaryTag] entities.
-    int tagCount = 0;
-    int tagTotal = tags.length;
-    partition<DictionaryTag>(tags, 10000).forEach((batch) {
+    if (params.useSlowImport) {
+      /// Write the [Dictionary] entity.
       database.writeTxnSync(() {
-        database.dictionaryTags.putAllSync(batch);
+        database.dictionarys.putSync(params.dictionary);
       });
-      tagCount += batch.length;
-      params.send(t.import_write_tag(count: tagCount, total: tagTotal));
-    });
-
-    /// Write [DictionaryPitch] entities.
-    int pitchCount = 0;
-    int pitchTotal = pitchesByHeading.values.map((e) => e.length).sum;
-    partition<MapEntry<DictionaryHeading, List<DictionaryPitch>>>(
-            pitchesByHeading.entries, 10000)
-        .forEach((batch) {
-      database.writeTxnSync(() {
-        for (MapEntry<DictionaryHeading,
-            List<DictionaryPitch>> pitchesForHeading in batch) {
-          DictionaryHeading heading = pitchesForHeading.key;
-          List<DictionaryPitch> pitches = pitchesForHeading.value;
 
-          database.dictionaryHeadings.putSync(heading);
-          database.dictionaryPitchs.putAllSync(pitches);
-
-          pitchCount += pitches.length;
-        }
+      /// Write [DictionaryTag] entities.
+      int tagCount = 0;
+      int tagTotal = tags.length;
+      partition<DictionaryTag>(tags, 10000).forEach((batch) {
+        database.writeTxnSync(() {
+          database.dictionaryTags.putAllSync(batch);
+        });
+        tagCount += batch.length;
+        params.send(t.import_write_tag(count: tagCount, total: tagTotal));
       });
 
-      params.send(t.import_write_pitch(count: pitchCount, total: pitchTotal));
-    });
-
-    /// Write [DictionaryFrequency] entities.
-    int frequencyCount = 0;
-    int frequencyTotal = frequenciesByHeading.values.map((e) => e.length).sum;
-    partition<MapEntry<DictionaryHeading, List<DictionaryFrequency>>>(
-            frequenciesByHeading.entries, 10000)
-        .forEach((batch) {
-      database.writeTxnSync(() {
-        for (MapEntry<DictionaryHeading,
-            List<DictionaryFrequency>> frequenciesForHeading in batch) {
-          DictionaryHeading heading = frequenciesForHeading.key;
-          List<DictionaryFrequency> frequencies = frequenciesForHeading.value;
-
-          database.dictionaryHeadings.putSync(heading);
-          database.dictionaryFrequencys.putAllSync(frequencies);
-
-          frequencyCount += frequencies.length;
-        }
+      /// Write [DictionaryPitch] entities.
+      int pitchCount = 0;
+      int pitchTotal = pitchesByHeading.values.map((e) => e.length).sum;
+      partition<MapEntry<DictionaryHeading, List<DictionaryPitch>>>(
+              pitchesByHeading.entries, 10000)
+          .forEach((batch) {
+        database.writeTxnSync(() {
+          for (MapEntry<DictionaryHeading,
+              List<DictionaryPitch>> pitchesForHeading in batch) {
+            DictionaryHeading heading = pitchesForHeading.key;
+            List<DictionaryPitch> pitches = pitchesForHeading.value;
+
+            database.dictionaryHeadings.putSync(heading);
+            database.dictionaryPitchs.putAllSync(pitches);
+
+            pitchCount += pitches.length;
+          }
+        });
+
+        params.send(t.import_write_pitch(count: pitchCount, total: pitchTotal));
       });
 
-      params.send(t.import_write_frequency(
-          count: frequencyCount, total: frequencyTotal));
-    });
-
-    /// Used to test the collision resistance of the FNV-1a algorithm used
-    /// for hashing dictionary headings to each have unique integer IDs.
-    /// This doesn't seem that heavy but we shouldn't instantiate millions
-    /// of elements at any given time, so this should be commented out for
-    /// a production release or when not debugging for collisions.
-    ///
-    /// For testing, a mix of Japanese bilingual and monolingual dictionaries
-    /// can be imported in sequence. The collision count should always be
-    /// zero. Interestingly, the Dart implementation of FNV-1a recommended by
-    /// Isar seems to produce less collisions than a MurmurHash V3
-    /// implementation. In any case, the code below can be uncommented for
-    /// and hash algorithm comparison testing and research.
-    ///
-    /// The idea is to get the delta number of headings, but also take into
-    /// account the number of headings already in the database.
-
-    // int headingsInDatabase = database.dictionaryHeadings.countSync();
-    // int headingsToImportAlreadyInDatabase = database.dictionaryHeadings
-    //     .getAllSync(entriesByHeading.keys.map((e) => e.id).toList())
-    //     .whereNotNull()
-    //     .length;
-    // int headingsToImportNotInDatabase =
-    //     entriesByHeading.keys.length - headingsToImportAlreadyInDatabase;
-
-    // debugPrint('Headings In Database: $headingsInDatabase');
-    // debugPrint(
-    //     'Headings To Import Already In Database: $headingsToImportAlreadyInDatabase');
-    // debugPrint(
-    //     'Headings To Import Not In Database: $headingsToImportNotInDatabase');
-
-    /// Write [DictionaryEntry] entities.
-    int entryCount = 0;
-    int entryTotal = entriesByHeading.values.map((e) => e.length).sum;
-    partition<MapEntry<DictionaryHeading, List<DictionaryEntry>>>(
-            entriesByHeading.entries, 10000)
-        .forEach((batch) {
-      database.writeTxnSync(() {
-        for (MapEntry<DictionaryHeading,
-            List<DictionaryEntry>> entriesForHeading in batch) {
-          DictionaryHeading heading = entriesForHeading.key;
-          List<DictionaryEntry> entries = entriesForHeading.value;
-
-          database.dictionaryHeadings.putSync(heading);
-          database.dictionaryEntrys.putAllSync(entries);
-
-          entryCount += entries.length;
-        }
+      /// Write [DictionaryFrequency] entities.
+      int frequencyCount = 0;
+      int frequencyTotal = frequenciesByHeading.values.map((e) => e.length).sum;
+      partition<MapEntry<DictionaryHeading, List<DictionaryFrequency>>>(
+              frequenciesByHeading.entries, 10000)
+          .forEach((batch) {
+        database.writeTxnSync(() {
+          for (MapEntry<DictionaryHeading,
+              List<DictionaryFrequency>> frequenciesForHeading in batch) {
+            DictionaryHeading heading = frequenciesForHeading.key;
+            List<DictionaryFrequency> frequencies = frequenciesForHeading.value;
+
+            database.dictionaryHeadings.putSync(heading);
+            database.dictionaryFrequencys.putAllSync(frequencies);
+
+            frequencyCount += frequencies.length;
+          }
+        });
+
+        params.send(t.import_write_frequency(
+            count: frequencyCount, total: frequencyTotal));
       });
 
-      params.send(t.import_write_entry(count: entryCount, total: entryTotal));
-    });
+      /// Used to test the collision resistance of the FNV-1a algorithm used
+      /// for hashing dictionary headings to each have unique integer IDs.
+      /// This doesn't seem that heavy but we shouldn't instantiate millions
+      /// of elements at any given time, so this should be commented out for
+      /// a production release or when not debugging for collisions.
+      ///
+      /// For testing, a mix of Japanese bilingual and monolingual dictionaries
+      /// can be imported in sequence. The collision count should always be
+      /// zero. Interestingly, the Dart implementation of FNV-1a recommended by
+      /// Isar seems to produce less collisions than a MurmurHash V3
+      /// implementation. In any case, the code below can be uncommented for
+      /// and hash algorithm comparison testing and research.
+      ///
+      /// The idea is to get the delta number of headings, but also take into
+      /// account the number of headings already in the database.
+
+      // int headingsInDatabase = database.dictionaryHeadings.countSync();
+      // int headingsToImportAlreadyInDatabase = database.dictionaryHeadings
+      //     .getAllSync(entriesByHeading.keys.map((e) => e.id).toList())
+      //     .whereNotNull()
+      //     .length;
+      // int headingsToImportNotInDatabase =
+      //     entriesByHeading.keys.length - headingsToImportAlreadyInDatabase;
+
+      // debugPrint('Headings In Database: $headingsInDatabase');
+      // debugPrint(
+      //     'Headings To Import Already In Database: $headingsToImportAlreadyInDatabase');
+      // debugPrint(
+      //     'Headings To Import Not In Database: $headingsToImportNotInDatabase');
+
+      /// Write [DictionaryEntry] entities.
+      int entryCount = 0;
+      int entryTotal = entriesByHeading.values.map((e) => e.length).sum;
+      partition<MapEntry<DictionaryHeading, List<DictionaryEntry>>>(
+              entriesByHeading.entries, 10000)
+          .forEach((batch) {
+        database.writeTxnSync(() {
+          for (MapEntry<DictionaryHeading,
+              List<DictionaryEntry>> entriesForHeading in batch) {
+            DictionaryHeading heading = entriesForHeading.key;
+            List<DictionaryEntry> entries = entriesForHeading.value;
+
+            database.dictionaryHeadings.putSync(heading);
+            database.dictionaryEntrys.putAllSync(entries);
+
+            entryCount += entries.length;
+          }
+        });
+
+        params.send(t.import_write_entry(count: entryCount, total: entryTotal));
+      });
 
-    /// Collision count should always be zero.
+      /// Collision count should always be zero.
 
-    // int newHeadingsInDatabase = database.dictionaryHeadings.countSync();
-    // int collisionsFound = newHeadingsInDatabase -
-    //     headingsInDatabase -
-    //     headingsToImportNotInDatabase;
-    // debugPrint('New Headings In Database: $newHeadingsInDatabase');
-    // debugPrint('Collisions Found: $collisionsFound');
+      // int newHeadingsInDatabase = database.dictionaryHeadings.countSync();
+      // int collisionsFound = newHeadingsInDatabase -
+      //     headingsInDatabase -
+      //     headingsToImportNotInDatabase;
+      // debugPrint('New Headings In Database: $newHeadingsInDatabase');
+      // debugPrint('Collisions Found: $collisionsFound');
+    } else {
+      /// Write as one transaction. If anything fails, no changes should occur.
+      database.writeTxnSync(() {
+        /// Write the [Dictionary] entity.
+        database.dictionarys.putSync(params.dictionary);
+
+        /// Write [DictionaryTag] entities.
+        int tagCount = 0;
+        int tagTotal = tags.length;
+        database.dictionaryTags.putAllSync(tags);
+        partition<DictionaryTag>(tags, 10000).forEach((batch) {
+          database.dictionaryTags.putAllSync(batch);
+          tagCount += batch.length;
+          params.send(t.import_write_tag(count: tagCount, total: tagTotal));
+        });
+
+        /// Write [DictionaryPitch] entities.
+        int pitchCount = 0;
+        int pitchTotal = pitchesByHeading.values.map((e) => e.length).sum;
+        partition<MapEntry<DictionaryHeading, List<DictionaryPitch>>>(
+                pitchesByHeading.entries, 10000)
+            .forEach((batch) {
+          for (MapEntry<DictionaryHeading,
+              List<DictionaryPitch>> pitchesForHeading in batch) {
+            DictionaryHeading heading = pitchesForHeading.key;
+            List<DictionaryPitch> pitches = pitchesForHeading.value;
+
+            database.dictionaryHeadings.putSync(heading);
+            database.dictionaryPitchs.putAllSync(pitches);
+            pitchCount += pitches.length;
+          }
+
+          params
+              .send(t.import_write_pitch(count: pitchCount, total: pitchTotal));
+        });
+
+        /// Write [DictionaryFrequency] entities.
+        int frequencyCount = 0;
+        int frequencyTotal =
+            frequenciesByHeading.values.map((e) => e.length).sum;
+        partition<MapEntry<DictionaryHeading, List<DictionaryFrequency>>>(
+                frequenciesByHeading.entries, 10000)
+            .forEach((batch) {
+          for (MapEntry<DictionaryHeading,
+              List<DictionaryFrequency>> frequenciesForHeading in batch) {
+            DictionaryHeading heading = frequenciesForHeading.key;
+            List<DictionaryFrequency> frequencies = frequenciesForHeading.value;
+
+            database.dictionaryHeadings.putSync(heading);
+            database.dictionaryFrequencys.putAllSync(frequencies);
+            frequencyCount += frequencies.length;
+          }
+
+          params.send(t.import_write_frequency(
+              count: frequencyCount, total: frequencyTotal));
+        });
+
+        /// Used to test the collision resistance of the FNV-1a algorithm used
+        /// for hashing dictionary headings to each have unique integer IDs.
+        /// This doesn't seem that heavy but we shouldn't instantiate millions
+        /// of elements at any given time, so this should be commented out for
+        /// a production release or when not debugging for collisions.
+        ///
+        /// For testing, a mix of Japanese bilingual and monolingual dictionaries
+        /// can be imported in sequence. The collision count should always be
+        /// zero. Interestingly, the Dart implementation of FNV-1a recommended by
+        /// Isar seems to produce less collisions than a MurmurHash V3
+        /// implementation. In any case, the code below can be uncommented for
+        /// and hash algorithm comparison testing and research.
+        ///
+        /// The idea is to get the delta number of headings, but also take into
+        /// account the number of headings already in the database.
+        // int headingsInDatabase = database.dictionaryHeadings.countSync();
+        // int headingsToImportAlreadyInDatabase = database.dictionaryHeadings
+        //     .getAllSync(entriesByHeading.keys.map((e) => e.id).toList())
+        //     .whereNotNull()
+        //     .length;
+        // int headingsToImportNotInDatabase =
+        //     entriesByHeading.keys.length - headingsToImportAlreadyInDatabase;
+
+        // debugPrint('Headings In Database: $headingsInDatabase');
+        // debugPrint(
+        //     'Headings To Import Already In Database: $headingsToImportAlreadyInDatabase');
+        // debugPrint(
+        //     'Headings To Import Not In Database: $headingsToImportNotInDatabase');
+
+        /// Write [DictionaryEntry] entities.
+        int entryCount = 0;
+        int entryTotal = entriesByHeading.values.map((e) => e.length).sum;
+        partition<MapEntry<DictionaryHeading, List<DictionaryEntry>>>(
+                entriesByHeading.entries, 10000)
+            .forEach((batch) {
+          for (MapEntry<DictionaryHeading,
+              List<DictionaryEntry>> entriesForHeading in batch) {
+            DictionaryHeading heading = entriesForHeading.key;
+            List<DictionaryEntry> entries = entriesForHeading.value;
+
+            database.dictionaryHeadings.putSync(heading);
+            database.dictionaryEntrys.putAllSync(entries);
+            entryCount += entries.length;
+          }
+
+          params
+              .send(t.import_write_entry(count: entryCount, total: entryTotal));
+        });
+
+        /// Collision count should always be zero.
+        // int newHeadingsInDatabase = database.dictionaryHeadings.countSync();
+        // int collisionsFound = newHeadingsInDatabase -
+        //     headingsInDatabase -
+        //     headingsToImportNotInDatabase;
+        // debugPrint('New Headings In Database: $newHeadingsInDatabase');
+        // debugPrint('Collisions Found: $collisionsFound');
+      });
+    }
   } catch (e, stackTrace) {
     params.send(stackTrace);
     params.send(e);
+    rethrow;
   }
 }
 
@@ -240,7 +357,7 @@ Future<void> preloadResult(int id) async {
   /// Create a new instance of Isar as this is a different isolate.
   final Isar database = await Isar.open(
     globalSchemas,
-    maxSizeMiB: 4096,
+    maxSizeMiB: 8192,
   );
   DictionarySearchResult result = database.dictionarySearchResults.getSync(id)!;
 
@@ -283,7 +400,7 @@ Future<void> updateDictionaryHistoryHelper(
 ) async {
   final Isar database = await Isar.open(
     globalSchemas,
-    maxSizeMiB: 4096,
+    maxSizeMiB: 8192,
   );
 
   DictionarySearchResult result =
@@ -299,7 +416,7 @@ Future<void> updateDictionaryHistoryHelper(
 Future<void> deleteDictionariesHelper(DeleteDictionaryParams params) async {
   final Isar database = await Isar.open(
     globalSchemas,
-    maxSizeMiB: 4096,
+    maxSizeMiB: 8192,
   );
 
   database.writeTxnSync(() {
@@ -317,7 +434,7 @@ Future<void> deleteDictionariesHelper(DeleteDictionaryParams params) async {
 Future<void> deleteDictionaryHelper(DeleteDictionaryParams params) async {
   final Isar database = await Isar.open(
     globalSchemas,
-    maxSizeMiB: 4096,
+    maxSizeMiB: 8192,
   );
 
   int id = params.dictionaryId!;
diff --git a/yuuna/lib/src/language/implementations/english_language.dart b/yuuna/lib/src/language/implementations/english_language.dart
index 40149ca50..14e996677 100644
--- a/yuuna/lib/src/language/implementations/english_language.dart
+++ b/yuuna/lib/src/language/implementations/english_language.dart
@@ -48,7 +48,7 @@ Future<int?> prepareSearchResultsEnglishLanguage(
   final Lemmatizer lemmatizer = Lemmatizer();
   final Isar database = await Isar.open(
     globalSchemas,
-    maxSizeMiB: 4096,
+    maxSizeMiB: 8192,
   );
 
   int bestLength = 0;
diff --git a/yuuna/lib/src/language/implementations/japanese_language.dart b/yuuna/lib/src/language/implementations/japanese_language.dart
index db6ad34d1..900bf2f2a 100644
--- a/yuuna/lib/src/language/implementations/japanese_language.dart
+++ b/yuuna/lib/src/language/implementations/japanese_language.dart
@@ -303,7 +303,7 @@ Future<int?> prepareSearchResultsJapaneseLanguage(
 
   final Isar database = await Isar.open(
     globalSchemas,
-    maxSizeMiB: 4096,
+    maxSizeMiB: 8192,
   );
 
   Map<int, DictionaryHeading> uniqueHeadingsById = {};
diff --git a/yuuna/lib/src/models/app_model.dart b/yuuna/lib/src/models/app_model.dart
index 5faa902fc..99b923ad3 100644
--- a/yuuna/lib/src/models/app_model.dart
+++ b/yuuna/lib/src/models/app_model.dart
@@ -18,7 +18,6 @@ import 'package:flutter_accessibility_service/accessibility_event.dart';
 import 'package:flutter_accessibility_service/flutter_accessibility_service.dart';
 import 'package:flutter_cache_manager/flutter_cache_manager.dart';
 import 'package:flutter_charset_detector/flutter_charset_detector.dart';
-import 'package:flutter_exit_app/flutter_exit_app.dart';
 import 'package:flutter_riverpod/flutter_riverpod.dart';
 import 'package:flutter_vlc_player/flutter_vlc_player.dart';
 import 'package:fluttertoast/fluttertoast.dart';
@@ -932,7 +931,7 @@ class AppModel with ChangeNotifier {
     /// Initialise persistent database.
     _database = await Isar.open(
       globalSchemas,
-      maxSizeMiB: 4096,
+      maxSizeMiB: 8192,
     );
 
     /// Perform startup activities unnecessary to further initialisation here.
@@ -1262,6 +1261,7 @@ class AppModel with ChangeNotifier {
         dictionary: dictionary,
         workingDirectory: workingDirectory,
         dictionaryFormat: dictionaryFormat,
+        useSlowImport: useSlowImport,
         sendPort: receivePort.sendPort,
       );
 
@@ -2091,7 +2091,7 @@ class AppModel with ChangeNotifier {
     await _audioHandler?.stop();
 
     if (_shouldKillMediaOnPop) {
-      await FlutterExitApp.exitApp();
+      SystemNavigator.pop();
     }
   }
 
diff --git a/yuuna/lib/src/pages/implementations/dictionary_dialog_page.dart b/yuuna/lib/src/pages/implementations/dictionary_dialog_page.dart
index a71a48bb9..ffa5748b2 100644
--- a/yuuna/lib/src/pages/implementations/dictionary_dialog_page.dart
+++ b/yuuna/lib/src/pages/implementations/dictionary_dialog_page.dart
@@ -255,7 +255,7 @@ class _DictionaryDialogPageState extends BasePageState {
                 ),
               const JidoujishoDivider(),
               buildImportDropdown(),
-              // buildSlowImportSwitch(),
+              buildSlowImportSwitch(),
             ],
           ),
         ),
diff --git a/yuuna/pubspec.yaml b/yuuna/pubspec.yaml
index 4ffdad656..f33a7a231 100644
--- a/yuuna/pubspec.yaml
+++ b/yuuna/pubspec.yaml
@@ -1,7 +1,7 @@
 name: yuuna
 description: A full-featured immersion language learning suite for mobile.
 publish_to: 'none'
-version: 2.4.4+45
+version: 2.4.5+46
 environment:
   sdk: ">=2.17.0 <3.0.0"