diff --git a/smart-blocks/smart_block.js b/smart-blocks/smart_block.js index 2c493c2d..acd0db91 100644 --- a/smart-blocks/smart_block.js +++ b/smart-blocks/smart_block.js @@ -280,7 +280,8 @@ export class SmartBlock extends SmartEntity { * @readonly * @returns {Array|undefined} An array containing the start and end lines or `undefined` if not set. */ - get lines() { return this.source?.data?.blocks?.[this.sub_key]; } + // get lines() { return this.source?.data?.blocks?.[this.sub_key]; } + get lines() { return this.data.lines; } /** * Retrieves the starting line number of the block. diff --git a/smart-sources/package.json b/smart-sources/package.json index 9d6a18d1..b6eae5a6 100644 --- a/smart-sources/package.json +++ b/smart-sources/package.json @@ -7,7 +7,7 @@ "description": "Easy to manage collections mapped to local or remote sources.", "main": "main.js", "scripts": { - "test_content": "test/test_content.sh", + "test_content": "node test/test_content.js", "test": "npx ava --verbose" }, "bin": { diff --git a/smart-sources/smart_source.js b/smart-sources/smart_source.js index 5f9d412a..8a392093 100644 --- a/smart-sources/smart_source.js +++ b/smart-sources/smart_source.js @@ -234,7 +234,7 @@ export class SmartSource extends SmartEntity { * @returns {Promise} */ async _update(content) { - await this.source_adapter._update(content); + await this.source_adapter.update(content); } /** diff --git a/smart-sources/test/markdown_source.test.js b/smart-sources/test/markdown_source.test.js index 573d4b4c..3c471a08 100644 --- a/smart-sources/test/markdown_source.test.js +++ b/smart-sources/test/markdown_source.test.js @@ -35,18 +35,27 @@ import { SmartBlocks } from 'smart-blocks/smart_blocks.js'; import { SmartBlock } from 'smart-blocks/smart_block.js'; import { SmartSettings } from 'smart-settings/smart_settings.js'; import { MarkdownSourceContentAdapter } from 'smart-sources/adapters/markdown_source.js'; +import { MarkdownBlockContentAdapter } from 'smart-blocks/adapters/markdown_block.js'; import ajson_data_adapter from 'smart-sources/adapters/data/ajson_multi_file.js'; +import path from 'path'; const VARIATIONS_DIR = 'test/test-content/variations'; +import { execSync } from 'child_process'; +import fs from 'fs'; async function create_test_env() { + // run adjacent test_content.js script and wait for it to finish (use execSync) + execSync('node test/test_content.js'); + await new Promise(resolve => setTimeout(resolve, 1000)); + const env_path = path.join(process.cwd(), 'test/test-content'); + // console.log(env_path); const env = await SmartEnv.create({ // Mock main or configuration object load_settings(){ return {}; }, save_settings(){}, get settings(){ return {}; } }, { - env_path: 'test/test-content', + env_path: env_path, modules: { smart_fs: { class: SmartFs, adapter: NodeFsSmartFsAdapter }, smart_settings: { class: SmartSettings }, @@ -64,6 +73,10 @@ async function create_test_env() { smart_blocks: { class: SmartBlocks, data_adapter: ajson_data_adapter, + block_adapters: { + md: MarkdownBlockContentAdapter, + default: MarkdownBlockContentAdapter, + } }, }, item_types: { @@ -84,13 +97,21 @@ async function create_test_env() { test.before(async t => { t.context.env = await create_test_env(); // Process initial import and save queues to ensure a stable state + await t.context.env.smart_sources.process_load_queue(); await t.context.env.smart_sources.process_source_import_queue(); await t.context.env.smart_sources.process_save_queue(); }); +test.after(async t => { + await new Promise(resolve => setTimeout(resolve, 1000)); + // delete test/test-content/variations directory + fs.rmdirSync('test/test-content/variations', { recursive: true }); +}); + test.serial('frontmatter_note.md: import and verify blocks', async t => { const { env } = t.context; const source = env.smart_sources.get('variations/frontmatter_note.md'); + // console.log(Object.keys(env.smart_sources.items)); t.truthy(source, 'frontmatter_note.md should be imported'); // Verify frontmatter block diff --git a/smart-sources/test/test-content/test-env/multi/append_source_md.ajson b/smart-sources/test/test-content/test-env/multi/append_source_md.ajson deleted file mode 100644 index 6cade70d..00000000 --- a/smart-sources/test/test-content/test-env/multi/append_source_md.ajson +++ /dev/null @@ -1 +0,0 @@ -"SmartSource:append_source.md": {"path":"append_source.md","embeddings":{},"embedding":{},"history":[],"class_name":"SmartSource","updated_field":"new_value","blocks":null}, \ No newline at end of file diff --git a/smart-sources/test/test-content/test-env/multi/my_source_md.ajson b/smart-sources/test/test-content/test-env/multi/my_source_md.ajson deleted file mode 100644 index 92dc4e71..00000000 --- a/smart-sources/test/test-content/test-env/multi/my_source_md.ajson +++ /dev/null @@ -1 +0,0 @@ -"SmartSource:my_source.md": {"path":"my_source.md","embeddings":{},"embedding":{},"history":[],"class_name":"SmartSource","blocks":null}, \ No newline at end of file diff --git a/smart-sources/test/test-content/test-env/multi/source_to_delete_md.ajson b/smart-sources/test/test-content/test-env/multi/source_to_delete_md.ajson deleted file mode 100644 index 337f1887..00000000 --- a/smart-sources/test/test-content/test-env/multi/source_to_delete_md.ajson +++ /dev/null @@ -1 +0,0 @@ -"SmartSource:source_to_delete.md": {"path":"source_to_delete.md","embeddings":{},"embedding":{},"history":[],"class_name":"SmartSource","blocks":null}, \ No newline at end of file diff --git a/smart-sources/test/test-content/test-env/multi/source_with_blocks_md.ajson b/smart-sources/test/test-content/test-env/multi/source_with_blocks_md.ajson deleted file mode 100644 index 12222c9c..00000000 --- a/smart-sources/test/test-content/test-env/multi/source_with_blocks_md.ajson +++ /dev/null @@ -1,7 +0,0 @@ -"SmartSource:source_with_blocks.md": {"path":"source_with_blocks.md","embeddings":{},"embedding":{},"history":[],"class_name":"SmartSource","blocks":{"#Source With Blocks":[1,12],"#Source With Blocks#{1}":[3,4],"#Source With Blocks#Section 1":[5,8],"#Source With Blocks#Section 1#{1}":[7,8],"#Source With Blocks#Section 2":[9,12],"#Source With Blocks#Section 2#{1}":[11,12]},"mtime":1733785360594,"size":158,"hash":"776f49e146417054088d7846865cfdd905dd3c7fdf0f89a3626ac285a687b1af","last_read_hash":"776f49e146417054088d7846865cfdd905dd3c7fdf0f89a3626ac285a687b1af","outlinks":[]}, -"SmartBlock:source_with_blocks.md#Source With Blocks#{1}": {"path":null,"embeddings":{},"embedding":{},"text":null,"length":0,"class_name":"SmartBlock","key":"source_with_blocks.md#Source With Blocks#{1}","outlinks":[],"size":51,"hash":"15634711965250ff9855eacbb56465f4719a26d10b34b98a04d789afa470808c"}, -"SmartBlock:source_with_blocks.md#Source With Blocks#Section 1": {"path":null,"embeddings":{},"embedding":{},"text":null,"length":0,"class_name":"SmartBlock","key":"source_with_blocks.md#Source With Blocks#Section 1","outlinks":[],"size":41,"hash":"c6de3c55a464452860f405d4e9bfe6618ff3d1415aebbc322bf25686a74c4fa4"}, -"SmartBlock:source_with_blocks.md#Source With Blocks#Section 1#{1}": {"path":null,"embeddings":{},"embedding":{},"text":null,"length":0,"class_name":"SmartBlock","key":"source_with_blocks.md#Source With Blocks#Section 1#{1}","outlinks":[],"size":27,"hash":"8b44e2205e7f4e56da32fa22f85a43c7ab2dced309629e172fd0c69af5342318"}, -"SmartBlock:source_with_blocks.md#Source With Blocks#Section 2": {"path":null,"embeddings":{},"embedding":{},"text":null,"length":0,"class_name":"SmartBlock","key":"source_with_blocks.md#Source With Blocks#Section 2","outlinks":[],"size":42,"hash":"00e5fc8301b4cb5837e6f9f9412a885f1b9bfeb7fdd3f9ad966b4b4b3e282dbd"}, -"SmartBlock:source_with_blocks.md#Source With Blocks#Section 2#{1}": {"path":null,"embeddings":{},"embedding":{},"text":null,"length":0,"class_name":"SmartBlock","key":"source_with_blocks.md#Source With Blocks#Section 2#{1}","outlinks":[],"size":28,"hash":"d237b6acb8e98b346971474b74e4beecfac2538d6340bb1ef333d975ecaefe52"}, -"SmartBlock:source_with_blocks.md#Source With Blocks": {"path":null,"embeddings":{},"embedding":{},"text":null,"length":0,"class_name":"SmartBlock","key":"source_with_blocks.md#Source With Blocks","outlinks":[],"size":158,"hash":"c9475bffe80cab2e64d747e698a6bf265a609b606a008cedb5ebd40709d86588"}, \ No newline at end of file diff --git a/smart-sources/test/test_content.js b/smart-sources/test/test_content.js new file mode 100644 index 00000000..dbe24e71 --- /dev/null +++ b/smart-sources/test/test_content.js @@ -0,0 +1,350 @@ +#!/usr/bin/env node + +/** + * @fileoverview + * This script sets up a variety of Markdown note files to test the SmartSources Markdown adapter and Block parser. + * It creates a structured folder with multiple Markdown files, each containing different patterns and quirks to ensure + * comprehensive integration testing of parsing, importing, block extraction, and CRUD operations. + * + * Directory structure: + * test/test-content/variations/ + * ├── frontmatter_note.md + * ├── nested_headings.md + * ├── code_blocks.md + * ├── no_headings.md + * ├── only_lists.md + * ├── repeated_headings.md + * ├── mixed_content.md + * ├── large_note.md + * ├── empty_note.md + * ├── special_chars_headings.md + * └── frontmatter_complex.md + */ + +"use strict"; + +/** + * @typedef {import('fs')} fs + */ + +import fs from "fs"; +import path from "path"; + +/** + * @description Creates a directory if it does not already exist. + * @param {string} dir_path The directory path to create. + */ +function ensure_dir_exists(dir_path) { + if (!fs.existsSync(dir_path)) { + fs.mkdirSync(dir_path, { recursive: true }); + } +} + +/** + * @description Writes a file with the given content. + * @param {string} file_path The file path where the content should be written. + * @param {string} content The content to write to the file. + */ +function write_file(file_path, content) { + fs.writeFileSync(file_path, content, { encoding: "utf8" }); +} + +/** + * @description Appends content to a file. + * @param {string} file_path The file path to append to. + * @param {string} content The content to append. + */ +function append_to_file(file_path, content) { + fs.appendFileSync(file_path, content, { encoding: "utf8" }); +} + +// Base directory +const base_dir = path.join("test", "test-content", "variations"); +ensure_dir_exists(base_dir); + +/////////////////////////////////////////////// +// 1. Note with frontmatter and headings +/////////////////////////////////////////////// +write_file( + path.join(base_dir, "frontmatter_note.md"), + `--- +title: "Test Frontmatter" +date: 2024-01-01 +tags: [test, markdown, frontmatter] +--- + +# Heading 1 + +Some introductory content. + +## Heading 2 + +More content under heading 2. + +### Heading 3 + +Deeper level heading content. +` +); + +// Integration test notes: +// - Test that frontmatter is parsed as a block (#---frontmatter---) and can be updated/destroyed. +// - Verify that headings and subheadings are correctly extracted as blocks. +// - Check that outlinks (if any) are parsed. +// - Confirm block CRUD operations (block_read, block_update, block_destroy) against frontmatter or headings. +// - Ensure that removing or rewriting frontmatter updates AJSON correctly. + +/////////////////////////////////////////////// +// 2. Note with deeply nested headings +/////////////////////////////////////////////// +write_file( + path.join(base_dir, "nested_headings.md"), + `# Level 1 + +## Level 2 + +### Level 3 + +#### Level 4 + +##### Level 5 + +###### Level 6 + +Content at the deepest heading level. +` +); + +// Integration test notes: +// - Verify that up to six levels of headings are correctly identified as blocks. +// - Check how block removal or updates propagate when dealing with deeply nested headings. +// - Test that markdown_to_blocks does not break with extreme heading nesting. +// - Validate that rewriting minimal files after import preserves heading structure. + +/////////////////////////////////////////////// +// 3. Note with code blocks +/////////////////////////////////////////////// +write_file( + path.join(base_dir, "code_blocks.md"), + `# Code Blocks Test + +Here is some code: + +\`\`\`javascript +function hello() { + console.log("Hello, world!"); +} +\`\`\` + +## Another Heading + +\`\`\`bash +echo "test" +\`\`\` + +` +); + +// Integration test notes: +// - Confirm that code blocks are treated as normal content lines and not misinterpreted as headings. +// - Verify block extraction does not fail due to code fences. +// - Test block updates within code fenced areas (append, update content inside code blocks). +// - Ensure code blocks do not cause JSON parse errors in AJSON. + +/////////////////////////////////////////////// +// 4. Note with no headings at all +/////////////////////////////////////////////// +write_file( + path.join(base_dir, "no_headings.md"), + `This file has no headings. + +Just plain text. + +Even multiple lines of it. + +` +); + +// Integration test notes: +// - Ensure that when no headings are present, a single root-level block (#) is created. +// - Test CRUD on the root block content. +// - Verify searching and embedding still works with a no-heading scenario. +// - Confirm rewriting minimal file still works with no headings. + +/////////////////////////////////////////////// +// 5. Note with only lists +/////////////////////////////////////////////// +write_file( + path.join(base_dir, "only_lists.md"), + `- First item + - Nested item +- Second item + +- Third item + - Another nested level + - Deeply nested +` +); + +// Integration test notes: +// - Ensure lists are converted into blocks as content blocks under root. +// - Check that each top-level list item creates a separate sub-block. +// - Validate block_destroy removes only that item’s lines. +// - Confirm that updates to list items reflect in AJSON. + +/////////////////////////////////////////////// +// 6. Note with repeated headings +/////////////////////////////////////////////// +write_file( + path.join(base_dir, "repeated_headings.md"), + `# Repeated +Content under the first occurrence. + +# Repeated +Second occurrence of the same heading text. + +## Repeated +Subheading also repeated multiple times. + +## Repeated +Another subheading with the same name. +` +); + +// Integration test notes: +// - Verify that repeated headings get suffixed with [2], [3], etc. for top-level duplicates. +// - Check that block references to repeated headings are stable. +// - Ensure import/export is correct and stable for repeated heading names. +// - Test block removal of one repeated heading does not affect others incorrectly. + +/////////////////////////////////////////////// +// 7. Note with mixed content (frontmatter, code, lists, headings) +/////////////////////////////////////////////// +write_file( + path.join(base_dir, "mixed_content.md"), + `--- +title: Mixed Content Test +--- + +# Main Heading + +Some intro text. + +- A list item +- Another list item + +\`\`\`python +print("Hello") +\`\`\` + +## Subheading + +More text under subheading. +` +); + +// Integration test notes: +// - Test complex scenario: frontmatter + headings + lists + code blocks all in one file. +// - Confirm block parsing remains correct with multiple content types. +// - Test partial updates: removing list block, updating code block content, rewriting frontmatter. +// - Validate stable AJSON after multiple round-trip import/export cycles. + +/////////////////////////////////////////////// +// 8. Large note +/////////////////////////////////////////////// +write_file( + path.join(base_dir, "large_note.md"), + `# Large Note + +` +); + +for (let i = 1; i <= 200; i++) { + append_to_file( + path.join(base_dir, "large_note.md"), + `Paragraph line ${i} with some content to increase note size.\n` + ); +} + +append_to_file(path.join(base_dir, "large_note.md"), `## Another Heading\n`); + +for (let i = 1; i <= 200; i++) { + append_to_file( + path.join(base_dir, "large_note.md"), + `Another big paragraph block line ${i}.\n` + ); +} + +// Integration test notes: +// - Tests performance and correctness with large file sizes. +// - Validate that line references remain stable after large content changes. +// - Check block updates and deletions still function at scale. +// - Ensure no memory or JSON parse issues with large content. + +/////////////////////////////////////////////// +// 9. Empty note +/////////////////////////////////////////////// +write_file( + path.join(base_dir, "empty_note.md"), + ` + +` +); + +// Integration test notes: +// - Confirm that an empty file still imports without errors. +// - Verify that no blocks are created (or a root block) and that CRUD ops handle empty gracefully. +// - Check that saving after load doesn't create malformed AJSON. + +/////////////////////////////////////////////// +// 10. Headings with special characters +/////////////////////////////////////////////// +write_file( + path.join(base_dir, "special_chars_headings.md"), + `# Heading with $Special_Char + +## Heading With Spaces and (Parentheses) + +### Heading_with_underscores_and_$variables + +Content under tricky headings. +` +); + +// Integration test notes: +// - Ensure special characters in headings do not break block keys. +// - Validate that AJSON keys are properly JSON-stringified and remain stable across imports. +// - Check CRUD operations on these special heading blocks. + +/////////////////////////////////////////////// +// 11. Complex frontmatter with lists, nested data +/////////////////////////////////////////////// +write_file( + path.join(base_dir, "frontmatter_complex.md"), + `--- +title: Complex Frontmatter +authors: + - name: Alice + role: Editor + - name: Bob + role: Reviewer +metadata: + tags: [complex, nested, frontmatter] + version: 2 +--- + +# Main Content + +This note tests complex nested frontmatter. +` +); + +// Integration test notes: +// - Test parsing and rewriting of complex, nested frontmatter data structures. +// - Confirm block_update on frontmatter accurately replaces nested YAML. +// - Validate block_destroy can remove frontmatter entirely and rewrite file. +// - Ensure stable keys and JSON parse logic in AJSON after complex frontmatter import. + +/////////////////////////////////////////////// +console.log(`All test notes have been created in ${base_dir}.`); +console.log("Use these notes to run integration tests on SmartSources Markdown adapter and block parser."); \ No newline at end of file diff --git a/smart-sources/test/test_content.sh b/smart-sources/test/test_content.sh deleted file mode 100644 index 44a8881a..00000000 --- a/smart-sources/test/test_content.sh +++ /dev/null @@ -1,284 +0,0 @@ -#!/usr/bin/env bash - -# This script sets up a variety of Markdown note files to test the SmartSources Markdown adapter and Block parser. -# It creates a structured folder with multiple Markdown files, each containing different patterns and quirks to ensure -# comprehensive integration testing of parsing, importing, block extraction, and CRUD operations. - -# Directory structure: -# test/test-content/variations/ -# ├── frontmatter_note.md -# ├── nested_headings.md -# ├── code_blocks.md -# ├── no_headings.md -# ├── only_lists.md -# ├── repeated_headings.md -# ├── mixed_content.md -# ├── large_note.md -# ├── empty_note.md -# ├── special_chars_headings.md -# └── frontmatter_complex.md - -set -e - -BASE_DIR="test/test-content/variations" - -# Create base directory -mkdir -p "$BASE_DIR" - -############################################# -# 1. Note with frontmatter and headings -############################################# -cat > "$BASE_DIR/frontmatter_note.md" < "$BASE_DIR/nested_headings.md" < "$BASE_DIR/code_blocks.md" < "$BASE_DIR/no_headings.md" < "$BASE_DIR/only_lists.md" < "$BASE_DIR/repeated_headings.md" < "$BASE_DIR/mixed_content.md" < "$BASE_DIR/large_note.md" <> "$BASE_DIR/large_note.md" -done - -echo "## Another Heading" >> "$BASE_DIR/large_note.md" -for i in {1..200}; do - echo "Another big paragraph block line $i." >> "$BASE_DIR/large_note.md" -done - -# Integration test notes: -# - Tests performance and correctness with large file sizes. -# - Validate that line references remain stable after large content changes. -# - Check block updates and deletions still function at scale. -# - Ensure no memory or JSON parse issues with large content. - -############################################# -# 9. Empty note -############################################# -cat > "$BASE_DIR/empty_note.md" < "$BASE_DIR/special_chars_headings.md" < "$BASE_DIR/frontmatter_complex.md" <