From b03d20045679404cb4961c16b36e3bdcfcb24e47 Mon Sep 17 00:00:00 2001 From: Brian Joseph Petro Date: Thu, 19 Dec 2024 10:21:06 -0500 Subject: [PATCH] Added default export object for adapters to enhance modularity. - Refactored data adapter retrieval in Collection class. - Removed deprecated methods and classes, including MarkdownSourceAdapter and related tests, to streamline the codebase. - Updated tests to reflect changes in adapter structure and functionality, ensuring reliability across the codebase. --- smart-blocks/adapters/_adapter.js | 7 +- smart-blocks/adapters/markdown_block.js | 5 + smart-collections/adapters/_adapter.js | 5 + smart-collections/adapters/_file.js | 5 + .../adapters/ajson_multi_file.js | 7 +- smart-collections/collection.js | 19 +- smart-collections/item.js | 12 +- smart-collections/tests/collection.test.js | 7 +- smart-entities/adapters/_adapter.js | 5 + smart-entities/adapters/default.js | 7 +- smart-sources/adapters/_adapter.js | 47 +- smart-sources/adapters/_file.js | 7 +- smart-sources/adapters/_test.js | 27 - .../adapters/data/ajson_multi_file.js | 5 + smart-sources/adapters/data/sqlite.js | 5 + smart-sources/adapters/file.js | 10 - smart-sources/adapters/markdown.js | 372 -------------- smart-sources/adapters/markdown.test.js | 468 ------------------ smart-sources/adapters/markdown_source.js | 7 +- smart-sources/adapters/text.js | 7 +- 20 files changed, 89 insertions(+), 945 deletions(-) delete mode 100644 smart-sources/adapters/_test.js delete mode 100644 smart-sources/adapters/file.js delete mode 100644 smart-sources/adapters/markdown.js delete mode 100644 smart-sources/adapters/markdown.test.js diff --git a/smart-blocks/adapters/_adapter.js b/smart-blocks/adapters/_adapter.js index 0083e2da..7c074b5a 100644 --- a/smart-blocks/adapters/_adapter.js +++ b/smart-blocks/adapters/_adapter.js @@ -112,4 +112,9 @@ export class BlockContentAdapter { async create_hash(content) { return await create_hash(content); } -} \ No newline at end of file +} + +export default { + collection: null, // No collection adapter for this base file + item: BlockContentAdapter +}; \ No newline at end of file diff --git a/smart-blocks/adapters/markdown_block.js b/smart-blocks/adapters/markdown_block.js index 85a5c5d4..998e7e9e 100644 --- a/smart-blocks/adapters/markdown_block.js +++ b/smart-blocks/adapters/markdown_block.js @@ -170,3 +170,8 @@ export class MarkdownBlockContentAdapter extends BlockContentAdapter { await this.item.source.import(); } } + +export default { + collection: null, // No collection adapter needed for markdown blocks + item: MarkdownBlockContentAdapter +}; diff --git a/smart-collections/adapters/_adapter.js b/smart-collections/adapters/_adapter.js index e8bfc810..c98978e1 100644 --- a/smart-collections/adapters/_adapter.js +++ b/smart-collections/adapters/_adapter.js @@ -152,3 +152,8 @@ export class ItemDataAdapter { return this.item.collection.data_adapter; } } + +export default { + collection: CollectionDataAdapter, + item: ItemDataAdapter +}; \ No newline at end of file diff --git a/smart-collections/adapters/_file.js b/smart-collections/adapters/_file.js index 5a9e060d..6ff52d3f 100644 --- a/smart-collections/adapters/_file.js +++ b/smart-collections/adapters/_file.js @@ -37,3 +37,8 @@ export class FileItemDataAdapter extends ItemDataAdapter { return this.item.collection.data_fs || this.item.collection.env.data_fs; } } + +export default { + collection: FileCollectionDataAdapter, + item: FileItemDataAdapter +}; \ No newline at end of file diff --git a/smart-collections/adapters/ajson_multi_file.js b/smart-collections/adapters/ajson_multi_file.js index 93f0a08c..d9bffae7 100644 --- a/smart-collections/adapters/ajson_multi_file.js +++ b/smart-collections/adapters/ajson_multi_file.js @@ -350,4 +350,9 @@ export class AjsonMultiFileItemDataAdapter extends FileItemDataAdapter { } return ajson; } -} \ No newline at end of file +} + +export default { + collection: AjsonMultiFileCollectionDataAdapter, + item: AjsonMultiFileItemDataAdapter +}; \ No newline at end of file diff --git a/smart-collections/collection.js b/smart-collections/collection.js index 828a17a7..c3836dea 100644 --- a/smart-collections/collection.js +++ b/smart-collections/collection.js @@ -264,16 +264,21 @@ export class Collection { */ get data_adapter() { if (!this._data_adapter) { - const config = this.env.opts.collections?.[this.collection_key]; - const data_adapter_class = config?.data_adapter - ?? this.env.opts.collections?.smart_collections?.data_adapter; - if (!data_adapter_class) { - throw new Error(`No data adapter class found for ${this.collection_key} or smart_collections`); - } - this._data_adapter = new data_adapter_class(this); + const AdapterClass = this.get_adapter_class('data'); + this._data_adapter = new AdapterClass(this); } return this._data_adapter; } + get_adapter_class(type) { + const config = this.env.opts.collections?.[this.collection_key]; + const adapter_key = type + '_adapter'; + const adapter_module = config?.[adapter_key] + ?? this.env.opts.collections?.smart_collections?.[adapter_key] + ; + if(typeof adapter_module === 'function') return adapter_module; // backward compatibility + if(typeof adapter_module?.collection === 'function') return adapter_module.collection; + throw new Error(`No adapter class found for ${this.collection_key} or smart_collections`); + } /** * Data directory strategy for this collection. Defaults to 'multi'. diff --git a/smart-collections/item.js b/smart-collections/item.js index e7ebebe2..a2013822 100644 --- a/smart-collections/item.js +++ b/smart-collections/item.js @@ -81,16 +81,6 @@ export class CollectionItem { return create_uid(this.data); } - /** - * Ensures the item is fully loaded if it was queued for loading. - * @returns {Promise} - */ - async ensure_loaded() { - if (this._queue_load) { - await this.load(); - } - } - /** * Updates the item data and returns true if changed. * @param {Object} data @@ -159,7 +149,7 @@ export class CollectionItem { */ async load() { try { - await this.data_adapter.load(this); + await this.data_adapter.load_item(this); this.init(); } catch (err) { this._load_error = err; diff --git a/smart-collections/tests/collection.test.js b/smart-collections/tests/collection.test.js index 67c10b3a..dbf363e0 100644 --- a/smart-collections/tests/collection.test.js +++ b/smart-collections/tests/collection.test.js @@ -9,7 +9,7 @@ class TestItem extends CollectionItem { } // A mock data adapter to ensure data_adapter is always available. -class MockDataAdapter { +class MockCollectionDataAdapter { constructor(collection) { this.collection = collection; } @@ -19,6 +19,9 @@ class MockDataAdapter { async save_item(item) {} async load(item) {} } +const default_adapter_export = { + collection: MockCollectionDataAdapter +}; // Create an environment with the specified item type and mock data adapter function create_env_and_collection(itemType = CollectionItem) { @@ -31,7 +34,7 @@ function create_env_and_collection(itemType = CollectionItem) { opts: { collections: { test_items: { - data_adapter: MockDataAdapter + data_adapter: default_adapter_export } } }, diff --git a/smart-entities/adapters/_adapter.js b/smart-entities/adapters/_adapter.js index 8f67a403..da8003fd 100644 --- a/smart-entities/adapters/_adapter.js +++ b/smart-entities/adapters/_adapter.js @@ -147,3 +147,8 @@ export class EntityVectorAdapter { throw new Error('EntityVectorAdapter.delete_vec() not implemented'); } } + +export default { + collection: EntitiesVectorAdapter, + item: EntityVectorAdapter +}; \ No newline at end of file diff --git a/smart-entities/adapters/default.js b/smart-entities/adapters/default.js index 8bbbecf4..4aca18f3 100644 --- a/smart-entities/adapters/default.js +++ b/smart-entities/adapters/default.js @@ -316,4 +316,9 @@ export class DefaultEntityVectorAdapter extends EntityVectorAdapter { this.item.data.embeddings[this.item.embed_model_key].vec = vec; } -} \ No newline at end of file +} + +export default { + collection: DefaultEntitiesVectorAdapter, + item: DefaultEntityVectorAdapter +}; \ No newline at end of file diff --git a/smart-sources/adapters/_adapter.js b/smart-sources/adapters/_adapter.js index 2d5400db..a6c01058 100644 --- a/smart-sources/adapters/_adapter.js +++ b/smart-sources/adapters/_adapter.js @@ -1,45 +1,4 @@ import { create_hash } from "../utils/create_hash.js"; -/** - * @deprecated - */ -export class SourceAdapter { - constructor(item, opts = {}) { - this.item = item; - this.opts = opts; - } - get collection() { return this.item.collection; } - get env() { return this.collection.env; } - get smart_change() { return this.collection.smart_change; } - get block_collection() { return this.env.smart_blocks; } - get source_collection() { return this.env.smart_sources; } - - // override these methods in the adapter class - throw_not_implemented(method_name) { - throw new Error(`Method "${method_name}" is not implemented for file type "${this.item.file_type}" in "${this.constructor.name}".`); - } - // source methods - async import() { this.throw_not_implemented('import'); } - async append(content) { this.throw_not_implemented('append'); } - async update(full_content, opts = {}) { this.throw_not_implemented('update'); } - // async _update(content) { this.throw_not_implemented('_update'); } - async read(opts = {}) { this.throw_not_implemented('read'); } - // async _read() { this.throw_not_implemented('_read'); } - async remove() { this.throw_not_implemented('remove'); } - async move_to(entity_ref) { this.throw_not_implemented('move_to'); } - async merge(content, opts = {}) { this.throw_not_implemented('merge'); } - - // block methods - async block_append(content) { this.throw_not_implemented('block_append'); } - async block_update(full_content, opts = {}) { this.throw_not_implemented('block_update'); } - async _block_update(content) { this.throw_not_implemented('_block_update'); } - async block_read(opts = {}) { this.throw_not_implemented('block_read'); } - async _block_read() { this.throw_not_implemented('_block_read'); } - async block_remove() { this.throw_not_implemented('block_remove'); } - async block_move_to(entity_ref) { this.throw_not_implemented('block_move_to'); } - async block_merge(content, opts = {}) { this.throw_not_implemented('block_merge'); } - // HELPER METHODS - async create_hash(content) { return await create_hash(content); } -} export class SourceContentAdapter { constructor(item) { @@ -53,4 +12,8 @@ export class SourceContentAdapter { // HELPER METHODS get data() { return this.item.data; } async create_hash(content) { return await create_hash(content); } -} \ No newline at end of file +} + +export default { + item: SourceContentAdapter +}; \ No newline at end of file diff --git a/smart-sources/adapters/_file.js b/smart-sources/adapters/_file.js index d08c0e93..5369c431 100644 --- a/smart-sources/adapters/_file.js +++ b/smart-sources/adapters/_file.js @@ -235,4 +235,9 @@ export class FileSourceContentAdapter extends SourceContentAdapter { await this.update(new_content); } -} \ No newline at end of file +} + +export default { + collection: null, // No collection adapter for this base file + item: FileSourceContentAdapter +}; \ No newline at end of file diff --git a/smart-sources/adapters/_test.js b/smart-sources/adapters/_test.js deleted file mode 100644 index dc941cb2..00000000 --- a/smart-sources/adapters/_test.js +++ /dev/null @@ -1,27 +0,0 @@ -import { MarkdownSourceAdapter } from "./markdown.js"; - -export class SourceTestAdapter extends MarkdownSourceAdapter { - constructor(item) { - super(item); - this.content = this.item.collection.adapter.test_data[this.item.constructor.name+":"+this.item.data.path].content; - } - - get fs() { return this.source_collection.fs; } - get data() { return this.item.data; } - get file_path() { return this.item.data.path; } - get source() { return this.item.source ? this.item.source : this.item; } - - async _update(content) { - this.content = content; - } - - async _read() { - return this.content; - } - - async remove() { - this.content = ""; - this.item.delete(); - } - -} \ No newline at end of file diff --git a/smart-sources/adapters/data/ajson_multi_file.js b/smart-sources/adapters/data/ajson_multi_file.js index fe64ccd4..df9a3931 100644 --- a/smart-sources/adapters/data/ajson_multi_file.js +++ b/smart-sources/adapters/data/ajson_multi_file.js @@ -237,3 +237,8 @@ export class AjsonMultiFileSourceDataAdapter extends AjsonMultiFileItemDataAdapt return 'smart_sources'; } } + +export default { + collection: AjsonMultiFileSourcesDataAdapter, + item: AjsonMultiFileSourceDataAdapter +}; diff --git a/smart-sources/adapters/data/sqlite.js b/smart-sources/adapters/data/sqlite.js index fdeb3a58..9a44e088 100644 --- a/smart-sources/adapters/data/sqlite.js +++ b/smart-sources/adapters/data/sqlite.js @@ -261,3 +261,8 @@ export class SqliteSourceDataAdapter extends SqliteCollectionDataAdapter { } } } + +export default { + collection: SqliteSourceDataAdapter, + item: SqliteSourceDataAdapter +}; diff --git a/smart-sources/adapters/file.js b/smart-sources/adapters/file.js deleted file mode 100644 index 400c9046..00000000 --- a/smart-sources/adapters/file.js +++ /dev/null @@ -1,10 +0,0 @@ -import { SourceAdapter } from "./_adapter.js"; -export class FileSourceAdapter extends SourceAdapter { - async update(content) { - await this.fs.write(this.file_path, content); - } - async read() { - return await this.fs.read(this.file_path); - } - get file_path() { return this.item.file_path; } -} \ No newline at end of file diff --git a/smart-sources/adapters/markdown.js b/smart-sources/adapters/markdown.js deleted file mode 100644 index 076da2b5..00000000 --- a/smart-sources/adapters/markdown.js +++ /dev/null @@ -1,372 +0,0 @@ -import { increase_heading_depth } from "../utils/increase_heading_depth.js"; -import { FileSourceAdapter } from "./file.js"; -import { markdown_to_blocks } from "../blocks/markdown_to_blocks.js"; -import { get_markdown_links } from "../utils/get_markdown_links.js"; -import { get_line_range } from "../utils/get_line_range.js"; -import { block_read, block_update, block_destroy } from "../blocks/markdown_crud.js"; - -/** - * @module MarkdownSourceAdapter - * @deprecated - * @description Adapter that reads and writes markdown files as sources for Smart Connections. - */ -export class MarkdownSourceAdapter extends FileSourceAdapter { - get fs() { return this.source_collection.fs; } - get data() { return this.item.data; } - get source() { return this.item.source ? this.item.source : this.item; } - - async import() { - const content = await this._read(); - if (!content) return console.warn("No content to import for " + this.file_path); - - const hash = await this.create_hash(content); - if (this.data.blocks && this.data.hash === hash) { - console.log("File stats changed, but content is the same. Skipping import."); - return; - } - - this.data.hash = hash; // set import hash - this.data.last_read.hash = hash; - - const blocks_obj = markdown_to_blocks(content); - this.data.blocks = blocks_obj; - - const outlinks = get_markdown_links(content); - this.data.outlinks = outlinks; - - if (this.item.block_collection) { - for (const [sub_key, line_range] of Object.entries(blocks_obj)) { - const block_key = `${this.source.key}${sub_key}`; - const block_content = get_line_range(content, line_range[0], line_range[1]); - const block_outlinks = get_markdown_links(block_content); - - const block = await this.item.block_collection.create_or_update({ - key: block_key, - outlinks: block_outlinks, - size: block_content.length, - }); - - block._embed_input = `${block.breadcrumbs}\n${block_content}`; // improves perf by preventing extra read at embed-time - block.data.hash = await this.create_hash(block._embed_input); - } - } - } - - async append(content) { - if (this.smart_change) { - content = this.smart_change.wrap("content", { - before: "", - after: content, - adapter: this.item.smart_change_adapter - }); - } - const current_content = await this.read(); - const new_content = [ - current_content, - "", - content, - ].join("\n").trim(); - await this._update(new_content); - } - - async update(full_content, opts = {}) { - const { mode = "append_blocks" } = opts; - if (mode === "replace_all") { - await this._update(full_content); - } else if (mode === "replace_blocks") { - await this.merge(full_content, { mode: "replace_blocks" }); - } else if (mode === "append_blocks") { - await this.merge(full_content, { mode: "append_blocks" }); - } - } - async _update(content) { - await super.update(content); - } - - - async read(opts = {}) { - let content = await this._read(); - this.source.data.last_read.hash = await this.create_hash(content); - if (this.source.last_read.hash !== this.source.hash) { - this.source.loaded_at = null; - await this.source.import(); - } - if (opts.no_changes && this.smart_change) { - const unwrapped = this.smart_change.unwrap(content, { file_type: this.item.file_type }); - content = unwrapped[opts.no_changes === 'after' ? 'after' : 'before']; - } - if (opts.add_depth) { - content = increase_heading_depth(content, opts.add_depth); - } - - return content; - } - async _read() { - return await super.read(); - } - - - async remove() { - await this.fs.remove(this.file_path); - this.item.delete(); - } - - async move_to(entity_ref) { - const new_path = typeof entity_ref === "string" ? entity_ref : entity_ref.key; - if (!new_path) { - throw new Error("Invalid entity reference for move_to operation"); - } - - const current_content = await this.read(); - const [target_source_key, ...headings] = new_path.split("#"); - const target_source = this.env.smart_sources.get(target_source_key); - - if (headings.length > 0) { - const new_headings_content = this.construct_headings(headings); - const new_content = `${new_headings_content}\n${current_content}`; - await this._update(new_content); - } - - if (target_source) { - await this.merge(current_content, { mode: 'append_blocks' }); - } else { - await this.rename_and_import(target_source_key, current_content); - } - - if (this.item.key !== target_source_key) await this.remove(); - } - - construct_headings(headings) { - return headings.map((heading, i) => `${"#".repeat(i + 1)} ${heading}`).join("\n"); - } - - async rename_and_import(target_source_key, content) { - await this.fs.rename(this.file_path, target_source_key); - const new_source = await this.item.collection.create_or_update({ path: target_source_key, content }); - await new_source.import(); - } - - /** - * Merge content into the source - * @param {string} content - The content to merge into the source - * @param {Object} opts - Options for the merge operation - * @param {string} opts.mode - The mode to use for the merge operation. Defaults to 'append_blocks' (may also be 'replace_blocks') - */ - async merge(content, opts = {}) { - const { mode = 'append_blocks' } = opts; - const blocks_obj = markdown_to_blocks(content); - - if (typeof blocks_obj !== 'object' || Array.isArray(blocks_obj)) { - console.warn("merge error: Expected an object from markdown_to_blocks, but received:", blocks_obj); - throw new Error("merge error: markdown_to_blocks did not return an object as expected."); - } - const { new_blocks, new_with_parent_blocks, changed_blocks, same_blocks } = await this.get_changes(blocks_obj, content); - for(const block of new_blocks){ - await this.append(block.content); - } - for(const block of new_with_parent_blocks){ - const parent_block = this.env.smart_blocks.get(block.parent_key); - await parent_block.append(block.content); - } - for(const block of changed_blocks){ - const changed_block = this.item.block_collection.get(block.key); - if(mode === "replace_blocks"){ - await changed_block.update(block.content); - }else{ - await changed_block.append(block.content); - } - } - } - - async get_changes(blocks_obj, content) { - const new_blocks = []; - const new_with_parent_blocks = []; - const changed_blocks = []; - const same_blocks = []; - const existing_blocks = this.source.data.blocks || {}; - for (const [sub_key, line_range] of Object.entries(blocks_obj)) { - const has_existing = !!existing_blocks[sub_key]; - const block_key = `${this.source.key}${sub_key}`; - const block_content = get_line_range(content, line_range[0], line_range[1]); - if(!has_existing){ - new_blocks.push({ - key: block_key, - state: "new", - content: block_content, - }); - continue; - } - let has_parent; - let headings = sub_key.split("#"); - let parent_key; - while(!has_parent && headings.length > 0){ - headings.pop(); // remove the last heading - parent_key = headings.join("#"); - has_parent = !!existing_blocks[parent_key]; - } - if(has_parent){ - new_with_parent_blocks.push({ - key: block_key, - parent_key: `${this.source.key}${parent_key}`, - state: "new", - content: block_content, - }); - continue; - } - const block = this.item.env.smart_blocks.get(block_key); - const content_hash = await this.create_hash(block_content); - if(content_hash !== block.hash){ - changed_blocks.push({ - key: block_key, - state: "changed", - content: block_content, - }); - continue; - } - same_blocks.push({ - key: block_key, - state: "same", - content: block_content, - }); - } - return { - new_blocks, - new_with_parent_blocks, - changed_blocks, - same_blocks, - } - } - - async block_read(opts = {}) { - const source_content = await this.read(); - - try { - const block_content = block_read(source_content, this.item.sub_key); - const breadcrumbs = this.item.breadcrumbs; - const embed_input = breadcrumbs + "\n" + block_content; - const hash = await this.create_hash(embed_input); - - if (hash !== this.item.hash) { - this.item.source?.queue_import(); - return this._block_read(source_content, this.item.sub_key); - } - - return block_content; - } catch (error) { - console.warn("Error reading block:", error.message); - return "BLOCK NOT FOUND"; - } - } - _block_read(source_content, block_key){ - return block_read(source_content, block_key); - } - - async block_append(append_content) { - let all_lines = (await this.read()).split("\n"); - if(all_lines[this.item.line_start] === append_content.split("\n")[0]){ - append_content = append_content.split("\n").slice(1).join("\n"); - } - if(this.smart_change) append_content = this.smart_change.wrap("content", { before: "", after: append_content, adapter: this.item.smart_change_adapter }); - await this._block_append(append_content); - } - - async _block_append(append_content) { - let all_lines = (await this.read()).split("\n"); - const content_before = all_lines.slice(0, this.item.line_end + 1); - const content_after = all_lines.slice(this.item.line_end + 1); - const new_content = [ - ...content_before, - "", // add a blank line before appending - append_content, - ...content_after, - ].join("\n").trim(); - await this.item.source._update(new_content); - await this.item.source.import(); - } - - async block_update(new_block_content, opts = {}) { - if(this.smart_change) new_block_content = this.smart_change.wrap("content", { - before: await this.block_read({ no_changes: "before", headings: "last" }), - after: new_block_content, - adapter: this.item.smart_change_adapter - }); - await this._block_update(new_block_content); - } - - async _block_update(new_block_content) { - const full_content = await this.read(); - try { - const updated_content = block_update(full_content, this.item.sub_key, new_block_content); - await this.item.source._update(updated_content); - await this.item.source.import(); - } catch (error) { - console.warn("Error updating block:", error.message); - } - } - - async block_remove() { - const full_content = await this.read(); - try { - const updated_content = block_destroy(full_content, this.item.sub_key); - await this.item.source._update(updated_content); - await this.item.source.import(); - } catch (error) { - console.warn("Error removing block:", error.message); - } - this.item.delete(); - } - - async block_move_to(to_key) { - const to_collection_key = to_key.includes("#") ? "smart_blocks" : "smart_sources"; - const to_entity = this.env[to_collection_key].get(to_key); - let content = await this.block_read({ no_changes: "before", headings: "last" }); - try { - if(this.smart_change){ - const smart_change = this.smart_change.wrap('location', { - to_key: to_key, - before: await this.block_read({headings: 'last', no_change: 'before'}), - adapter: this.item.smart_change_adapter - }); - this._block_update(smart_change); - }else{ - this.block_remove(); - } - } catch (e) { - console.warn("error removing block: ", e); - } - try { - if(to_entity) { - if(this.smart_change){ - content = this.smart_change.wrap("location", { from_key: this.item.source.key, after: content, adapter: this.item.smart_change_adapter }); - await to_entity._append(content); - }else{ - await to_entity.append(content); - } - } else { - const target_source_key = to_key.split("#")[0]; - const target_source = this.env.smart_sources.get(target_source_key); - if (to_key.includes("#")) { - const headings = to_key.split("#").slice(1); - const new_headings_content = headings.map((heading, i) => `${"#".repeat(i + 1)} ${heading}`).join("\n"); - let new_content = [ - new_headings_content, - ...content.split("\n").slice(1) - ].join("\n").trim(); - if(this.smart_change) new_content = this.smart_change.wrap("location", { from_key: this.item.source.key, after: new_content, adapter: this.item.smart_change_adapter }); - if(target_source) await target_source._append(new_content); - else await this.env.smart_sources.create(target_source_key, new_content); - } else { - if(this.smart_change) content = this.smart_change.wrap("location", { from_key: this.item.source.key, after: content, adapter: this.item.smart_change_adapter }); - if(target_source) await target_source._append(content); - else await this.env.smart_sources.create(target_source_key, content); - } - } - } catch (e) { - console.warn("error moving block: ", e); - // return to original location - this.item.deleted = false; - await this.block_update(content); - } - await this.item.source.import(); - } -} \ No newline at end of file diff --git a/smart-sources/adapters/markdown.test.js b/smart-sources/adapters/markdown.test.js deleted file mode 100644 index fb127ad5..00000000 --- a/smart-sources/adapters/markdown.test.js +++ /dev/null @@ -1,468 +0,0 @@ -import test from 'ava'; -import { JsonSingleFileCollectionDataAdapter } from '../../smart-collections/adapters/json_single_file.js'; -import { SmartFsTestAdapter } from '../../smart-fs/adapters/_test.js'; -import { SourceTestAdapter } from './_test.js'; -import { MarkdownSourceAdapter } from '../adapters/markdown.js'; -import { SmartSource } from '../smart_source.js'; -import { SmartSources } from '../smart_sources.js'; -import { SmartBlock } from '../smart_block.js'; -import { SmartBlocks } from '../smart_blocks.js'; -// import { SmartDirectory } from '../smart_directory.js'; -// import { SmartDirectories } from '../smart_directories.js'; -import { SmartEnv } from '../../smart-environment/smart_env.js'; -import { SmartEmbedModel } from '../../smart-embed-model/smart_embed_model.js'; -import { SmartEmbedTransformersAdapter } from '../../smart-embed-model/adapters/transformers.js'; -import { SmartEmbedOpenAIAdapter } from '../../smart-embed-model/adapters/openai.js'; -import { SmartFs } from '../../smart-fs/smart_fs.js'; -import { SmartSettings } from '../../smart-settings/smart_settings.js'; -const __dirname = new URL('.', import.meta.url).pathname; - -class TestMain { - load_settings() { return {}; } - save_settings() {} - get settings() { return {}; } - get smart_env_config() { - return { - env_path: __dirname, - env_data_dir: 'test', - modules: { - // smart_embed_model: { - // class: SmartEmbedModel, - // adapters: { - // transformers: SmartEmbedTransformersAdapter, - // openai: SmartEmbedOpenAIAdapter, - // }, - // }, - smart_fs: { - class: SmartFs, - adapter: SmartFsTestAdapter, - }, - smart_settings: { - class: SmartSettings, - }, - }, - collections: { - smart_sources: { - class: SmartSources, - data_adapter: JsonSingleFileCollectionDataAdapter, - source_adapters: { - test: SourceTestAdapter, - md: MarkdownSourceAdapter, - default: MarkdownSourceAdapter - }, - }, - smart_blocks: SmartBlocks, - // smart_directories: SmartDirectories, - }, - item_types: { - SmartSource, - SmartBlock, - // SmartDirectory, - }, - }; - } -} - -export async function load_test_env(t) { - const main = new TestMain(); - const env = await SmartEnv.create(main, main.smart_env_config); - env.smart_sources.settings.smart_change = {}; - env.smart_sources.settings.smart_change.active = false; - t.context.env = env; - t.context.fs = env.smart_sources.fs; -} - -test.beforeEach(async t => { - await load_test_env(t); -}); - -test.serial('MarkdownSourceAdapter import method', async t => { - const env = t.context.env; - const fs = t.context.fs; - - // Create a markdown file - const content = `# Heading 1 -Some content under heading 1. - -## Heading 2 -Some content under heading 2. - -[Link to other file](other.md) -`; - - await fs.write('test.md', content); - await fs.load_files(); - - // Create or update the SmartSource - const source = await env.smart_sources.create_or_update({ path: 'test.md' }); - - // Import the content - await source.import(); - - // Check that the blocks are correctly parsed - t.truthy(source.data.blocks); - - // Check that outlinks are correctly set - t.truthy(source.data.outlinks); - t.is(source.data.outlinks.length, 1); - t.is(source.data.outlinks[0].target, 'other.md'); - - // Check that blocks are created in smart_blocks collection - const block1 = env.smart_blocks.get('test.md#Heading 1'); - t.truthy(block1); - const block2 = env.smart_blocks.get('test.md#Heading 1#Heading 2'); - t.truthy(block2); -}); - -test.serial('MarkdownSourceAdapter append method', async t => { - const env = t.context.env; - const fs = t.context.fs; - - // Initial content - const initial_content = '# Initial Heading\nInitial content.'; - await fs.write('append_test.md', initial_content); - await fs.load_files(); - - const source = await env.smart_sources.create_or_update({ path: 'append_test.md' }); - - // Append content - const append_content = '# Appended Heading\nAppended content.'; - await source.append(append_content); - - // Read the content - const content = await source.read(); - - // Check that content is appended - const expected_content = initial_content + '\n\n' + append_content; - t.is(content, expected_content, 'Content should be appended correctly.'); -}); - -test.serial('MarkdownSourceAdapter update method - replace_all mode', async t => { - const env = t.context.env; - const fs = t.context.fs; - - const initial_content = '# Heading\nInitial content.'; - await fs.write('update_test.md', initial_content); - await fs.load_files(); - - const source = await env.smart_sources.create_or_update({ path: 'update_test.md' }); - - const new_content = '# New Heading\nNew content.'; - await source.update(new_content, { mode: 'replace_all' }); - - const content = await source.read(); - t.is(content, new_content, 'Content should be replaced entirely.'); -}); - -test.serial('MarkdownSourceAdapter update method - merge_replace mode', async t => { - const env = t.context.env; - const fs = t.context.fs; - - const initial_content = '# Heading\nInitial content.\n## Subheading\nSubcontent.'; - await fs.write('update_test.md', initial_content); - await fs.load_files(); - - const source = await env.smart_sources.create_or_update({ path: 'update_test.md' }); - - const new_content = '# Heading\nUpdated content.\n## Subheading\nUpdated subcontent.'; - await source.update(new_content, { mode: 'merge_replace' }); - - const content = await source.read(); - - // Since 'merge_replace' should replace matching blocks - const expected_content = new_content; - t.is(content, expected_content, 'Content should have matching blocks replaced.'); -}); - -test.serial('MarkdownSourceAdapter merge method - append_blocks mode', async t => { - const env = t.context.env; - const fs = t.context.fs; - - const initial_content = '# Heading\nInitial content.'; - await fs.write('update_test.md', initial_content); - await fs.load_files(); - - const source = await env.smart_sources.create_or_update({ path: 'update_test.md' }); - await source.import(); - - const new_content = '# Heading\nAppended content.\n## New Subheading\nNew subcontent.'; - await source.merge(new_content, { mode: 'append_blocks' }); - - const content = await source.read(); - - const expected_content = '# Heading\nInitial content.\n\nAppended content.\n## New Subheading\nNew subcontent.'; - t.is(content, expected_content, 'Content should have new blocks appended.'); -}); - -test.serial('MarkdownSourceAdapter read method with no_changes option', async t => { - const env = t.context.env; - const fs = t.context.fs; - - const content_with_changes = `<<<<<<< HEAD -# Original Heading -======= -# New Heading ->>>>>>> -Some content`; - - await fs.write('changes_test.md', content_with_changes); - await fs.load_files(); - const source = await env.smart_sources.create_or_update({ path: 'changes_test.md' }); - - const content_before = await source.read({ no_changes: 'before' }); - t.is(content_before, '# Original Heading\nSome content', 'Should return content before changes'); - - const content_after = await source.read({ no_changes: 'after' }); - t.is(content_after, '# New Heading\nSome content', 'Should return content after changes'); - - const content_with_changes_result = await source.read({ no_changes: false }); - t.is(content_with_changes_result, content_with_changes, 'Should return content with change syntax'); -}); - -test.serial('MarkdownSourceAdapter read method with add_depth option', async t => { - const env = t.context.env; - const fs = t.context.fs; - - const content = `# Heading 1 -## Heading 2 -Some content`; - - await fs.write('depth_test.md', content); - await fs.load_files(); - const source = await env.smart_sources.create_or_update({ path: 'depth_test.md' }); - - const content_depth_1 = await source.read({ add_depth: 1 }); - const expected_depth_1 = `## Heading 1 -### Heading 2 -Some content`; - t.is(content_depth_1, expected_depth_1, 'Should increase heading depth by 1'); -}); - -test.serial('MarkdownSourceAdapter remove method', async t => { - const env = t.context.env; - const fs = t.context.fs; - - const content = 'Some content to remove.'; - await fs.write('remove_test.md', content); - await fs.load_files(); - - const source = await env.smart_sources.create_or_update({ path: 'remove_test.md' }); - - await source.remove(); - await env.smart_sources.process_save_queue(); - - const exists = await fs.exists('remove_test.md'); - t.false(exists, 'File should be removed'); - - // Ensure that source is deleted from smart_sources collection - const source_in_collection = env.smart_sources.get('remove_test.md'); - t.falsy(source_in_collection, 'Source should be removed from collection'); -}); - -test.serial('MarkdownSourceAdapter move_to method - move to new path', async t => { - const env = t.context.env; - const fs = t.context.fs; - - const content = 'Content to move.'; - await fs.write('move_from.md', content); - await fs.load_files(); - - const source = await env.smart_sources.create_or_update({ path: 'move_from.md' }); - - await source.move_to('move_to.md'); - - const exists_old = await fs.exists('move_from.md'); - t.false(exists_old, 'Old file should not exist'); - - const exists_new = await fs.exists('move_to.md'); - t.true(exists_new, 'New file should exist'); - - const new_content = await fs.read('move_to.md'); - t.is(new_content, content, 'Content should be moved correctly'); - - // Ensure that source is updated in smart_sources collection - const source_new = env.smart_sources.get('move_to.md'); - t.truthy(source_new, 'Source should be updated in collection'); -}); - -test.serial('MarkdownSourceAdapter move_to method - move to existing source (merge)', async t => { - const env = t.context.env; - const fs = t.context.fs; - - const content_from = 'Content from source.'; - const content_to = 'Content in destination source.'; - - await fs.write('move_from.md', content_from); - await fs.write('move_to.md', content_to); - await fs.load_files(); - - const source_from = await env.smart_sources.create_or_update({ path: 'move_from.md' }); - const source_to = await env.smart_sources.create_or_update({ path: 'move_to.md' }); - - await source_from.move_to('move_to.md'); - // process save queue to ensure changes are saved (deleted items removed from collection) - await env.smart_sources.process_save_queue(); - - const exists_old = await fs.exists('move_from.md'); - t.false(exists_old, 'Old file should not exist'); - - const new_content = await fs.read('move_to.md'); - const expected_content = content_to + '\n\n' + content_from; - t.is(new_content, expected_content, 'Content should be merged in destination'); - - // Ensure that source_from is removed from smart_sources collection - const source_from_in_collection = env.smart_sources.get('move_from.md'); - t.falsy(source_from_in_collection, 'Source from should be removed from collection'); -}); - -test.serial('MarkdownSourceAdapter block_read method', async t => { - const env = t.context.env; - const fs = t.context.fs; - - const content = `# Heading 1 -Content under heading 1. -## Heading 2 -Content under heading 2.`; - - await fs.write('block_read_test.md', content); - await fs.load_files(); - - const source = await env.smart_sources.create_or_update({ path: 'block_read_test.md' }); - await source.import(); - - const block = env.smart_blocks.get('block_read_test.md#Heading 1'); - - const block_content = await block.read(); - - - t.is(block_content, content, 'Block content should be read correctly.'); - const block2 = env.smart_blocks.get('block_read_test.md#Heading 1#Heading 2'); - const expected_block_content = `## Heading 2 -Content under heading 2.`; - t.is(await block2.read(), expected_block_content, 'Block content should be read correctly.'); -}); - -test.serial('MarkdownSourceAdapter block_append method', async t => { - const env = t.context.env; - const fs = t.context.fs; - - const content = `# Heading 1 -Content under heading 1.`; - - await fs.write('block_append_test.md', content); - await fs.load_files(); - - const source = await env.smart_sources.create_or_update({ path: 'block_append_test.md' }); - await source.import(); - - const block = env.smart_blocks.get('block_append_test.md#Heading 1'); - - const append_content = 'Additional content for heading 1.'; - await block.append(append_content); - - const block_content = await block.read(); - - const expected_block_content = `# Heading 1 -Content under heading 1. - -Additional content for heading 1.`; - - t.is(block_content, expected_block_content, 'Content should be appended to the block.'); -}); - -test.serial('MarkdownSourceAdapter block_update method', async t => { - const env = t.context.env; - const fs = t.context.fs; - - const content = `# Heading 1 -Content under heading 1.`; - - await fs.write('block_update_test.md', content); - await fs.load_files(); - - const source = await env.smart_sources.create_or_update({ path: 'block_update_test.md' }); - await source.import(); - - const block = env.smart_blocks.get('block_update_test.md#Heading 1'); - - const new_block_content = `# Heading 1 -Updated content under heading 1.`; - - await block.update(new_block_content); - - const block_content = await block.read(); - - t.is(block_content, new_block_content, 'Block content should be updated.'); -}); - -test.serial('MarkdownSourceAdapter block_remove method', async t => { - const env = t.context.env; - const fs = t.context.fs; - - const content = `# Heading 1 -Content under heading 1. -# Heading 2 -Content under heading 2.`; - - await fs.write('block_remove_test.md', content); - await fs.load_files(); - - const source = await env.smart_sources.create_or_update({ path: 'block_remove_test.md' }); - await source.import(); - - const block = env.smart_blocks.get('block_remove_test.md#Heading 1'); - - await block.remove(); - await env.smart_sources.process_save_queue(); - - const source_content = await source.read(); - - const expected_content = `# Heading 2 -Content under heading 2.`; - - t.is(source_content, expected_content, 'Block should be removed from source.'); - - const block_in_collection = env.smart_blocks.get('block_remove_test.md#Heading 1'); - t.falsy(block_in_collection, 'Block should be removed from collection.'); -}); - -test.serial('MarkdownSourceAdapter block_move_to method', async t => { - const env = t.context.env; - const fs = t.context.fs; - - const content_source = `# Heading 1 -Content under heading 1. -## Subheading -Subcontent.`; - - await fs.write('block_move_from.md', content_source); - - const content_target = `# Existing Heading -Existing content.`; - - await fs.write('block_move_to.md', content_target); - await fs.load_files(); - - const source = await env.smart_sources.create_or_update({ path: 'block_move_from.md' }); - await source.import(); - - const target = await env.smart_sources.create_or_update({ path: 'block_move_to.md' }); - - const block = env.smart_blocks.get('block_move_from.md#Heading 1#Subheading'); - - await block.move_to('block_move_to.md'); - await env.smart_sources.process_save_queue(); - // Check that the block is removed from source - const source_content = await source.read(); - const expected_source_content = `# Heading 1 -Content under heading 1.`; - t.is(source_content, expected_source_content, 'Block should be removed from source.'); - - // Check that the block content is appended to target - const target_content = await target.read(); - const expected_target_content = content_target + '\n\n## Subheading\nSubcontent.'; - t.is(target_content, expected_target_content, 'Block content should be appended to target.'); - // Ensure block is removed from collection - const block_in_collection = env.smart_blocks.get('block_move_from.md#Heading 1#Subheading'); - t.falsy(block_in_collection, 'Block should be removed from collection.'); -}); \ No newline at end of file diff --git a/smart-sources/adapters/markdown_source.js b/smart-sources/adapters/markdown_source.js index a5adec05..dbad21cc 100644 --- a/smart-sources/adapters/markdown_source.js +++ b/smart-sources/adapters/markdown_source.js @@ -102,4 +102,9 @@ export class MarkdownSourceContentAdapter extends FileSourceContentAdapter { } } -} \ No newline at end of file +} + +export default { + collection: null, // No collection adapter needed for markdown sources + item: MarkdownSourceContentAdapter +}; \ No newline at end of file diff --git a/smart-sources/adapters/text.js b/smart-sources/adapters/text.js index feeef7e4..5a1f3675 100644 --- a/smart-sources/adapters/text.js +++ b/smart-sources/adapters/text.js @@ -1,4 +1,9 @@ import { FileSourceContentAdapter } from "./_file.js"; export class TextSourceContentAdapter extends FileSourceContentAdapter { -} \ No newline at end of file +} + +export default { + collection: null, // No collection adapter needed for text sources + item: TextSourceContentAdapter +}; \ No newline at end of file