feat: replace relative path to path trace logic for included (#556)

diplodoc-platform · Nov 13, 2024 · a3d8f52 · a3d8f52
1 parent 72e7d7a
commit a3d8f52
Show file tree

Hide file tree

Showing 11 changed files with 98 additions and 69 deletions.
diff --git a/src/transform/md.ts b/src/transform/md.ts
@@ -5,7 +5,7 @@ import DefaultMarkdownIt from 'markdown-it';
 import attrs from 'markdown-it-attrs';
 
 import DefaultPlugins from './plugins';
-import DefaultPreprocessors from './preprocessors';
+import {preprocess} from './preprocessors';
 import {log} from './log';
 import makeHighlight from './highlight';
 import extractTitle from './title';
@@ -118,13 +118,10 @@ function initParser(
             needTitle,
             needFlatListHeadings = false,
             getPublicPath,
-            preprocessors = DefaultPreprocessors,
         } = options;
 
-        // Run input preprocessor
-        for (const preprocessor of preprocessors) {
-            input = preprocessor(input, pluginOptions, md);
-        }
+        // Run preprocessor
+        input = preprocess(input, pluginOptions, options, md);
 
         // Generate global href link
         const href = getPublicPath ? getPublicPath(options) : '';

diff --git a/src/transform/plugins/includes/collect.ts b/src/transform/plugins/includes/collect.ts
@@ -2,19 +2,20 @@ import {relative} from 'path';
 import {bold} from 'chalk';
 import {readFileSync} from 'fs';
 
-import {getRelativePath, isFileExists, resolveRelativePath} from '../../utilsFS';
+import {isFileExists, resolveRelativePath} from '../../utilsFS';
 
 import {IncludeCollectOpts} from './types';
 
 const includesPaths: string[] = [];
 
 function processRecursive(
+    relativePath: string,
     includePath: string,
     targetDestPath: string,
     options: IncludeCollectOpts,
 ) {
     const {path, log, copyFile, includedParentPath: includedParentPathNullable, included} = options;
-    const includedParentPath = includedParentPathNullable || path;
+    const includedParentPath = includedParentPathNullable || [];
 
     const includeOptions = {
         ...options,
@@ -30,21 +31,22 @@ function processRecursive(
             const content = contentProcessed ?? readFileSync(targetDestPath, 'utf8');
 
             if (content) {
-                const includedRelativePath = getRelativePath(includedParentPath, includePath);
+                const key = [...includedParentPath, relativePath];
+                const hash = key.join(':');
 
                 // The appendix is the map that protects from multiple include files
-                if (!options.appendix?.has(includedRelativePath)) {
+                if (!options.appendix?.has(hash)) {
                     // Recursive function to include the depth structure
                     const includeContent = collectRecursive(content, {
                         ...options,
                         path: includePath,
-                        includedParentPath,
+                        includedParentPath: key,
                     });
 
                     // Add to appendix set structure
                     options.appendix?.set(
-                        includedRelativePath,
-                        `{% included (${includedRelativePath}) %}\n${includeContent}\n{% endincluded %}`,
+                        relativePath,
+                        `{% included (${hash}) %}\n${includeContent}\n{% endincluded %}`,
                     );
                 }
             }
@@ -91,7 +93,7 @@ function collectRecursive(result: string, options: IncludeCollectOpts) {
 
         includesPaths.push(includePath);
 
-        processRecursive(includePath, targetDestPath, options);
+        processRecursive(relativePath, includePath, targetDestPath, options);
 
         includesPaths.pop();
     }

diff --git a/src/transform/plugins/includes/index.ts b/src/transform/plugins/includes/index.ts
@@ -2,13 +2,7 @@ import {bold} from 'chalk';
 import Token from 'markdown-it/lib/token';
 
 import {StateCore} from '../../typings';
-import {
-    GetFileTokensOpts,
-    getFileTokens,
-    getFullIncludePath,
-    isFileExists,
-    resolveRelativePath,
-} from '../../utilsFS';
+import {GetFileTokensOpts, getFileTokens, getFullIncludePath, isFileExists} from '../../utilsFS';
 import {findBlockTokens} from '../../utils';
 import {MarkdownItPluginCb, MarkdownItPluginOpts} from '../typings';
 
@@ -49,10 +43,6 @@ function unfoldIncludes(md: MarkdownItIncluded, state: StateCore, path: string,
                 const [, keyword /* description */, , includePath] = match;
 
                 const fullIncludePath = getFullIncludePath(includePath, root, path);
-                const relativeIncludePath = resolveRelativePath(path, includePath);
-
-                // Check the existed included store and extract it
-                const included = md.included?.[relativeIncludePath];
 
                 let pathname = fullIncludePath;
                 let hash = '';
@@ -68,10 +58,10 @@ function unfoldIncludes(md: MarkdownItIncluded, state: StateCore, path: string,
                     continue;
                 }
 
-                const fileTokens = getFileTokens(pathname, state, {
-                    ...options,
-                    content: included, // The content forces the function to use it instead of reading from the disk
-                });
+                // Check the existed included store and extract it
+                const included = md.included?.[pathname];
+
+                const fileTokens = getFileTokens(pathname, state, options, included);
 
                 let includedTokens;
                 if (hash) {

diff --git a/src/transform/plugins/includes/types.ts b/src/transform/plugins/includes/types.ts
@@ -12,7 +12,7 @@ export type IncludeCollectOpts = MarkdownItPluginOpts & {
     copyFile(path: string, dest: string, opts: IncludeCollectOpts): string | null | undefined;
     singlePage: Boolean;
     included: Boolean;
-    includedParentPath?: string;
+    includedParentPath?: string[];
     additionalIncludedList?: string[];
     appendix?: Map<string, string>;
 };
diff --git a/src/transform/plugins/links/index.ts b/src/transform/plugins/links/index.ts
@@ -16,6 +16,7 @@ import {
 import {getFileTokens, isFileExists} from '../../utilsFS';
 import {CacheContext, StateCore} from '../../typings';
 import {MarkdownItPluginCb, MarkdownItPluginOpts} from '../typings';
+import {MarkdownItIncluded} from '../includes/types';
 
 function getTitleFromTokens(tokens: Token[]) {
     let title = '';
@@ -51,27 +52,35 @@ type Options = {
     cache?: CacheContext;
 };
 
-const getTitle = (id: string | null, options: Options) => {
+const getTitle = (md: MarkdownItIncluded, id: string | null, options: Options) => {
     const {file, state, opts} = options;
 
-    const fileTokens = getFileTokens(file, state, {
-        ...opts,
-        disableLint: true,
-        disableTitleRefSubstitution: true,
-        disableCircularError: true,
-        inheritVars: false,
-    });
+    // Check the existed included store and extract it
+    const included = md.included?.[file];
+
+    const fileTokens = getFileTokens(
+        file,
+        state,
+        {
+            ...opts,
+            disableLint: true,
+            disableTitleRefSubstitution: true,
+            disableCircularError: true,
+            inheritVars: false,
+        },
+        included,
+    );
     const sourceTokens = id ? findBlockTokens(fileTokens, id) : fileTokens;
     return getTitleFromTokens(sourceTokens);
 };
 
-const addTitle = (options: Options) => {
+const addTitle = (md: MarkdownItIncluded, options: Options) => {
     const {hash, state, isEmptyLink, tokens, idx, nextToken, href, currentPath, log, cache} =
         options;
 
     const id = hash && hash.slice(1);
     const key = [id, path].join('::');
-    const title = cache?.get(key) ?? getTitle(id, options);
+    const title = cache?.get(key) ?? getTitle(md, id, options);
     cache?.set(key, title);
 
     if (title) {
@@ -111,7 +120,13 @@ function getDefaultPublicPath(
 }
 
 // eslint-disable-next-line complexity
-function processLink(state: StateCore, tokens: Token[], idx: number, opts: ProcOpts) {
+function processLink(
+    md: MarkdownItIncluded,
+    state: StateCore,
+    tokens: Token[],
+    idx: number,
+    opts: ProcOpts,
+) {
     const {
         path: startPath,
         root,
@@ -180,7 +195,7 @@ function processLink(state: StateCore, tokens: Token[], idx: number, opts: ProcO
         isPageFile &&
         !state.env.disableTitleRefSubstitution
     ) {
-        addTitle({
+        addTitle(md, {
             hash,
             file,
             state,
@@ -213,7 +228,7 @@ function processLink(state: StateCore, tokens: Token[], idx: number, opts: ProcO
     }
 }
 
-const index: MarkdownItPluginCb<ProcOpts & Options> = (md, opts) => {
+const index: MarkdownItPluginCb<ProcOpts & Options> = (md: MarkdownItIncluded, opts) => {
     const plugin = (state: StateCore) => {
         const tokens = state.tokens;
         let i = 0;
@@ -231,7 +246,7 @@ const index: MarkdownItPluginCb<ProcOpts & Options> = (md, opts) => {
                     const isYfmAnchor = tokenClass ? tokenClass.includes('yfm-anchor') : false;
 
                     if (isLinkOpenToken && !isYfmAnchor) {
-                        processLink(state, childrenTokens, j, opts);
+                        processLink(md, state, childrenTokens, j, opts);
                     }
 
                     j++;

diff --git a/src/transform/preprocessors.ts b/src/transform/preprocessors.ts
diff --git a/src/transform/preprocessors/included/index.ts b/src/transform/preprocessors/included/index.ts
@@ -26,13 +26,15 @@ const preprocessLine = (
         return false;
     }
 
-    const includePathRelative = match[1];
+    const includePathKey = match[1];
 
     // Protect from empty path
-    if (!includePathRelative) {
+    if (!includePathKey) {
         return false;
     }
 
+    const includePaths = includePathKey.split(':');
+
     // Read all content from top to bottom(!) char of the included block
     const data = [];
     let line = start;
@@ -55,7 +57,11 @@ const preprocessLine = (
         }
 
         // Normalize the path to absolute
-        const includePath = getFullIncludePath(includePathRelative, root, path);
+        let includePath = getFullIncludePath(includePaths[0], root, path);
+        for (let index = 1; index < includePaths.length; index++) {
+            const pathname = includePaths[index];
+            includePath = getFullIncludePath(pathname, root, includePath);
+        }
 
         // Store the included content
         md.included[includePath] = data.join('\n');

diff --git a/src/transform/preprocessors/index.ts b/src/transform/preprocessors/index.ts
@@ -0,0 +1,25 @@
+import type {MarkdownIt, MarkdownItPluginOpts, MarkdownItPreprocessorCb} from '../plugins/typings';
+import type {PluginOptions} from '../yfmlint';
+
+import included from './included';
+
+const defaultPreprocessors = [included] as MarkdownItPreprocessorCb[];
+
+export default defaultPreprocessors;
+
+export function preprocess(
+    content: string,
+    pluginOptions: MarkdownItPluginOpts | PluginOptions | unknown,
+    options?: Partial<MarkdownItPluginOpts> & {
+        preprocessors?: MarkdownItPreprocessorCb[];
+    },
+    md?: MarkdownIt,
+) {
+    const {preprocessors = defaultPreprocessors} = options ?? {};
+
+    for (const preprocessor of preprocessors) {
+        content = preprocessor(content, pluginOptions as MarkdownItPluginOpts, md);
+    }
+
+    return content;
+}
diff --git a/src/transform/utilsFS.ts b/src/transform/utilsFS.ts
@@ -7,6 +7,7 @@ import {join, parse, relative, resolve, sep} from 'path';
 import liquidSnippet from './liquid';
 import {StateCore} from './typings';
 import {defaultTransformLink} from './utils';
+import {preprocess} from './preprocessors';
 
 const filesCache: Record<string, string> = {};
 
@@ -39,7 +40,12 @@ export type GetFileTokensOpts = {
     content?: string;
 };
 
-export function getFileTokens(path: string, state: StateCore, options: GetFileTokensOpts) {
+export function getFileTokens(
+    path: string,
+    state: StateCore,
+    options: GetFileTokensOpts,
+    content?: string,
+) {
     const {
         getVarsPerFile,
         vars,
@@ -51,7 +57,6 @@ export function getFileTokens(path: string, state: StateCore, options: GetFileTo
         inheritVars = true,
         conditionsInCode,
     } = options;
-    let {content} = options;
 
     const builtVars = (getVarsPerFile && !inheritVars ? getVarsPerFile(path) : vars) || {};
 
@@ -77,6 +82,9 @@ export function getFileTokens(path: string, state: StateCore, options: GetFileTo
         sourceMap = liquidResult.sourceMap;
     }
 
+    // Run preprocessor
+    content = preprocess(content, options);
+
     if (!disableLint && lintMarkdown) {
         lintMarkdown({
             input: content,

diff --git a/src/transform/yfmlint/index.ts b/src/transform/yfmlint/index.ts
@@ -6,7 +6,7 @@ import union from 'lodash/union';
 import attrs from 'markdown-it-attrs';
 
 import {LogLevels, Logger} from '../log';
-import defaultPreprocessors from '../preprocessors';
+import {preprocess} from '../preprocessors';
 
 import baseDefaultLintConfig from './yfmlint';
 import {
@@ -29,13 +29,7 @@ const lintCache = new Set();
 
 function yfmlint(opts: Options) {
     let {input} = opts;
-    const {
-        plugins: customPlugins,
-        preprocessors = defaultPreprocessors,
-        pluginOptions,
-        customLintRules,
-        sourceMap,
-    } = opts;
+    const {plugins: customPlugins, pluginOptions, customLintRules, sourceMap} = opts;
     const {path = 'input', log} = pluginOptions;
 
     pluginOptions.isLintRun = true;
@@ -65,9 +59,8 @@ function yfmlint(opts: Options) {
     const plugins = customPlugins && [attrs, ...customPlugins];
     const preparedPlugins = plugins && plugins.map((plugin) => [plugin, pluginOptions]);
 
-    for (const preprocessor of preprocessors) {
-        input = preprocessor(input, pluginOptions);
-    }
+    // Run preprocessor
+    input = preprocess(input, pluginOptions, opts);
 
     let result;
     try {

diff --git a/test/mocks/include-included-3-deep.expect.md b/test/mocks/include-included-3-deep.expect.md
@@ -3,12 +3,12 @@ start main
 {% include [Text](included/file-1-deep.md) %}
 
 end main
-{% included (included/file-3.md) %}
+{% included (included/file-1-deep.md:file-2-deep.md:file-3.md) %}
 start file 3
 
 end file 3
 {% endincluded %}
-{% included (included/file-2-deep.md) %}
+{% included (included/file-1-deep.md:file-2-deep.md) %}
 start file 2
 
 {% include [Text](file-3.md) %}