diff --git a/smart-blocks/smart_block.js b/smart-blocks/smart_block.js index 2667b5de..9e2323b7 100644 --- a/smart-blocks/smart_block.js +++ b/smart-blocks/smart_block.js @@ -105,13 +105,6 @@ export class SmartBlock extends SmartEntity { if(!content) content = await this.read(); this._embed_input = this.breadcrumbs + "\n" + content; } - // PREVENT EMBEDDING BASED ON HASH - // likely better handled since reduces embed_batch size - if(this.vec){ - // falsy values filtered out in SmartEmbedModel.embed_batch - if(this.last_embed.hash === this.last_read.hash) return false; // Already embedded - this.last_embed.hash = this.last_read.hash; - } return this._embed_input; } @@ -278,7 +271,7 @@ export class SmartBlock extends SmartEntity { get last_embed() { return this.data.last_embed; } get last_read() { return this.data.last_read; } - + /** * Retrieves the sub-key of the block. * @readonly diff --git a/smart-entities/smart_entities.js b/smart-entities/smart_entities.js index 504a66c3..940eded5 100644 --- a/smart-entities/smart_entities.js +++ b/smart-entities/smart_entities.js @@ -377,6 +377,9 @@ export class SmartEntities extends Collection { console.error(e); console.error(`Error processing ${this.collection_key} embed queue: ` + JSON.stringify((e || {}), null, 2)); } + batch.forEach(item => { + item.embed_hash = item.read_hash; + }); this.embedded_total += batch.length; this.total_tokens += batch.reduce((acc, item) => acc + (item.tokens || 0), 0); this._show_embed_progress_notice(); diff --git a/smart-entities/smart_entity.js b/smart-entities/smart_entity.js index 4e73ceb0..7df9d164 100644 --- a/smart-entities/smart_entity.js +++ b/smart-entities/smart_entity.js @@ -143,6 +143,17 @@ export class SmartEntity extends CollectionItem { this.env.connections_cache[cache_key] = connections; } + get read_hash() { return this.data.last_read?.hash; } + set read_hash(hash) { + if(!this.data.last_read) this.data.last_read = {}; + this.data.last_read.hash = hash; + } + get embed_hash() { return this.data.last_embed?.hash; } + set embed_hash(hash) { + if(!this.data.last_embed) this.data.last_embed = {}; + this.data.last_embed.hash = hash; + } + /** * Gets the embed link for the entity. * @readonly @@ -208,7 +219,7 @@ export class SmartEntity extends CollectionItem { * @readonly * @returns {boolean} Always returns true. Can be overridden in child classes. */ - get should_embed() { return true; } // may override in child class + get should_embed() { return !this.vec; } // may override in child class /** * Sets the error for the embedding model. diff --git a/smart-sources/smart_source.js b/smart-sources/smart_source.js index b82485cb..598ea657 100644 --- a/smart-sources/smart_source.js +++ b/smart-sources/smart_source.js @@ -577,6 +577,9 @@ export class SmartSource extends SmartEntity { .filter(link_path => link_path); } get path() { return this.data.path; } + get should_embed() { + return !this.vec || !this.embed_hash || this.embed_hash !== this.read_hash; + } get smart_change_adapter() { return this.env.settings.is_obsidian_vault ? "obsidian_markdown" : "markdown"; } get source_adapters() { return this.collection.source_adapters; } get source_adapter() { diff --git a/smart-sources/smart_sources.js b/smart-sources/smart_sources.js index 85b30db1..bc206ea5 100644 --- a/smart-sources/smart_sources.js +++ b/smart-sources/smart_sources.js @@ -415,14 +415,15 @@ export class SmartSources extends SmartEntities { try{ const embed_blocks = this.block_collection.settings.embed_blocks; this._embed_queue = Object.values(this.items).reduce((acc, item) => { - if(item._queue_embed) acc.push(item); + if(item._queue_embed && item.should_embed) acc.push(item); if(embed_blocks) item.blocks.forEach(block => { if(block._queue_embed && block.should_embed) acc.push(block); }); return acc; }, []); + // console.log(this._embed_queue.map(item => item.key)); }catch(e){ - console.error(`Error getting embed queue: ` + JSON.stringify((e || {}), null, 2)); + console.error(`Error getting embed queue:`, e); } } return this._embed_queue;