Refactor SmartEntities and SmartEntity classes for improved embedding…

… and error handling

- Replaced the _embed_queue_complete method with a direct call to _show_embed_completion_notice for clarity.
- Updated SmartEntity initialization to queue embedding if the vector is missing or mismatched with model dimensions.
- Enhanced error handling in AjsonMultiFileSourceDataAdapter to fix trailing comma issues in JSON parsing.
- Commented out the file removal logic to prevent accidental deletion of data files.

smart-entities/smart_entities.js

@@ -386,7 +386,7 @@ export class SmartEntities extends Collection {
         await this.process_save_queue();
       }
     }
-    if (!this.is_queue_halted) this._embed_queue_complete();
+    this._show_embed_completion_notice();
     this.process_save_queue();
   }
 
@@ -438,16 +438,6 @@ export class SmartEntities extends Collection {
     return Math.round(this.total_tokens / elapsed_time);
   }
 
-  /**
-   * Handles the completion of the embed queue processing.
-   * @private
-   * @returns {void}
-   */
-  _embed_queue_complete() {
-    this._show_embed_completion_notice();
-    this._reset_embed_queue_stats();
-  }
-
   /**
    * Resets the statistics related to embed queue processing.
    * @private
@@ -480,7 +470,6 @@ export class SmartEntities extends Collection {
         timeout: 0,
         button: { text: "Resume", callback: () => this.resume_embed_queue_processing(100) }
       });
-    this.env.save();
   }
 
   /**
@@ -490,7 +479,6 @@ export class SmartEntities extends Collection {
    */
   resume_embed_queue_processing(delay = 0) {
     console.log("resume_embed_queue_processing");
-    this.is_queue_halted = false;
     this.notices?.remove('embedding_paused');
     setTimeout(() => {
       this.embedded_total = 0;

smart-entities/smart_entity.js

@@ -41,11 +41,17 @@ export class SmartEntity extends CollectionItem {
 
   /**
    * Initializes the SmartEntity instance.
+   * Checks if the entity has a vector and if it matches the model dimensions.
+   * If not, it queues an embed.
+   * Removes embeddings for inactive models.
    * @returns {void}
    */
   init() {
     super.init();
-    if (!this.vec) {
+    if (!this.vec){
+      this.queue_embed();
+    }else if (this.vec.length !== this.embed_model.model_config.dims) {
+      this.vec = null;
       this.queue_embed();
     }
     // Only keep active model embeddings

smart-sources/adapters/data/ajson_multi_file.js

@@ -189,6 +189,12 @@ export class AjsonMultiFileSourceDataAdapter extends AjsonMultiFileItemDataAdapt
     try {
       data = JSON.parse(json_str);
     } catch (e) {
+      // if any lines do not end with a comma, add comma to end of line and try again
+      if(ajson.split('\n').some(line => !line.endsWith(','))) {
+        console.warn("fixing trailing comma error");
+        ajson = ajson.split('\n').map(line => line.endsWith(',') ? line : line + ',').join('\n');
+        return this._parse(ajson);
+      }
       console.warn("Error parsing multi-line JSON:", e);
       console.warn(this.item.key);
       return { final_states, rewrite_needed: true };
@@ -234,7 +240,7 @@ export class AjsonMultiFileSourceDataAdapter extends AjsonMultiFileItemDataAdapt
     } else {
       console.warn("No active items remain, removing file", data_path);
       // No active items remain, remove file
-      if (await this.fs.exists(data_path)) await this.fs.remove(data_path);
+      // if (await this.fs.exists(data_path)) await this.fs.remove(data_path);
     }
   }