feat: add merge included (#485)

src/transform/md.ts

@@ -1,8 +1,9 @@
-import type {EnvType, MarkdownIt, OptionsType} from './typings';
+import type {EnvType, MarkdownIt, MarkdownItPluginOpts, OptionsType} from './typings';
 import type Token from 'markdown-it/lib/token';
 
 import DefaultMarkdownIt from 'markdown-it';
 import DefaultPlugins from './plugins';
+import DefaultPreprocessors from './preprocessors';
 import {log} from './log';
 import makeHighlight from './highlight';
 import attrs from 'markdown-it-attrs';
@@ -11,10 +12,21 @@ import getHeadings from './headings';
 import sanitizeHtml from './sanitize';
 
 function initMarkdownit(options: OptionsType) {
-    const {allowHTML = false, linkify = false, breaks = true, highlightLangs = {}} = options;
+    const {
+        allowHTML = false,
+        linkify = false,
+        breaks = true,
+        highlightLangs = {},
+        disableRules = [],
+    } = options;
 
     const highlight = makeHighlight(highlightLangs);
     const md = new DefaultMarkdownIt({html: allowHTML, linkify, highlight, breaks}) as MarkdownIt;
+
+    if (disableRules?.length) {
+        md.disable(disableRules);
+    }
+
     const env = {
         // TODO: move md.meta directly to env
         get meta() {
@@ -38,38 +50,50 @@ function initMarkdownit(options: OptionsType) {
         title: '',
     } as EnvType;
 
-    initPlugins(md, options);
+    // Plugin options is the plugin context that remains during the build of one file
+    const pluginOptions = getPluginOptions(options);
+
+    // Init the plugins. Which install the md rules (core, block, ...)
+    initPlugins(md, options, pluginOptions);
 
-    const parse = initParser(md, options, env);
+    // Init preprocessor and MD parser
+    const parse = initParser(md, options, env, pluginOptions);
+
+    // Init render to HTML compiler
     const compile = initCompiler(md, options, env);
 
     return {parse, compile, env};
 }
 
-function initPlugins(md: MarkdownIt, options: OptionsType) {
+function getPluginOptions(options: OptionsType) {
     const {
         vars = {},
         path,
         extractTitle,
         conditionsInCode = false,
         disableLiquid = false,
-        linkify = false,
-        linkifyTlds,
-        leftDelimiter = '{',
-        rightDelimiter = '}',
-        plugins = DefaultPlugins,
         ...customOptions
     } = options;
 
-    const pluginOptions = {
+    return {
         ...customOptions,
         conditionsInCode,
         vars,
         path,
         extractTitle,
         disableLiquid,
         log,
-    };
+    } as MarkdownItPluginOpts;
+}
+
+function initPlugins(md: MarkdownIt, options: OptionsType, pluginOptions: MarkdownItPluginOpts) {
+    const {
+        linkify = false,
+        linkifyTlds,
+        leftDelimiter = '{',
+        rightDelimiter = '}',
+        plugins = DefaultPlugins,
+    } = options;
 
     // Need for ids of headers
     md.use(attrs, {leftDelimiter, rightDelimiter});
@@ -81,17 +105,30 @@ function initPlugins(md: MarkdownIt, options: OptionsType) {
     }
 }
 
-function initParser(md: MarkdownIt, options: OptionsType, env: EnvType) {
+function initParser(
+    md: MarkdownIt,
+    options: OptionsType,
+    env: EnvType,
+    pluginOptions: MarkdownItPluginOpts,
+) {
     return (input: string) => {
         const {
             extractTitle: extractTitleOption,
             needTitle,
             needFlatListHeadings = false,
             getPublicPath,
+            preprocessors = DefaultPreprocessors,
         } = options;
 
+        // Run input preprocessor
+        for (const preprocessor of preprocessors) {
+            input = preprocessor(input, pluginOptions, md);
+        }
+
+        // Generate global href link
         const href = getPublicPath ? getPublicPath(options) : '';
 
+        // Generate MD tokens
         let tokens = md.parse(input, env);
 
         if (extractTitleOption) {
@@ -121,12 +158,15 @@ function initCompiler(md: MarkdownIt, options: OptionsType, env: EnvType) {
     const {needToSanitizeHtml = true, renderInline = false, sanitizeOptions} = options;
 
     return (tokens: Token[]) => {
+        // Remove inline tokens if inline mode is activated
         if (renderInline) {
             tokens = tokens.filter((token) => token.type === 'inline');
         }
 
+        // Generate HTML
         const html = md.renderer.render(tokens, md.options, env);
 
+        // Sanitize the page
         return needToSanitizeHtml ? sanitizeHtml(html, sanitizeOptions) : html;
     };
 }

src/transform/plugins/includes/collect.ts

@@ -1,24 +1,39 @@
 import {relative} from 'path';
 import {bold} from 'chalk';
 
-import {isFileExists, resolveRelativePath} from '../../utilsFS';
+import {getRelativePath, isFileExists, resolveRelativePath} from '../../utilsFS';
 import {MarkdownItPluginOpts} from '../typings';
 
 const includesPaths: string[] = [];
 
 type Opts = MarkdownItPluginOpts & {
     destPath: string;
-    copyFile(path: string, dest: string, opts: Opts): void;
+    copyFile(path: string, dest: string, opts: Opts): string | null | undefined;
     singlePage: Boolean;
+    included: Boolean;
+    includedParentPath?: string;
 };
 
 const collect = (input: string, options: Opts) => {
-    const {root, path, destPath = '', log, copyFile, singlePage} = options;
+    const {
+        root,
+        path,
+        destPath = '',
+        log,
+        copyFile,
+        singlePage,
+        includedParentPath: includedParentPathNullable,
+        included,
+    } = options;
+    const includedParentPath = includedParentPathNullable || path;
+
     const INCLUDE_REGEXP = /{%\s*include\s*(notitle)?\s*\[(.+?)]\((.+?)\)\s*%}/g;
 
     let match,
         result = input;
 
+    const appendix: Map<string, string> = new Map();
+
     while ((match = INCLUDE_REGEXP.exec(result)) !== null) {
         let [, , , relativePath] = match;
         const [matchedInclude] = match;
@@ -55,19 +70,44 @@ const collect = (input: string, options: Opts) => {
         };
 
         try {
-            copyFile(includePath, targetDestPath, includeOptions);
+            const content = copyFile(includePath, targetDestPath, includeOptions);
+
+            // To reduce file reading we can include the file content into the generated content
+            if (included && content) {
+                const includedRelativePath = getRelativePath(includedParentPath, includePath);
+
+                // The appendix is the map that protects from multiple include files
+                if (!appendix.has(includedRelativePath)) {
+                    // Recursive function to include the depth structure
+                    const includeContent = collect(content, {
+                        ...options,
+                        path: includePath,
+                        includedParentPath,
+                    });
+                    // Add to appendix set structure
+                    appendix.set(
+                        includedRelativePath,
+                        `{% included (${includedRelativePath}) %}\n${includeContent}\n{% endincluded %}`,
+                    );
+                }
+            }
         } catch (e) {
             log.error(`No such file or has no access to ${bold(includePath)} in ${bold(path)}`);
         } finally {
             includesPaths.pop();
         }
     }
 
+    // Appendix should be appended to the end of the file (it supports depth structure, so the included files will have included as well)
+    if (appendix.size > 0) {
+        result += '\n' + [...appendix.values()].join('\n');
+    }
+
     if (singlePage) {
         return result;
     }
 
-    return null;
+    return result;
 };
 
 export = collect;

src/transform/plugins/includes/index.ts

@@ -1,10 +1,17 @@
 import {bold} from 'chalk';
+import Token from 'markdown-it/lib/token';
 
-import {GetFileTokensOpts, getFileTokens, getFullIncludePath, isFileExists} from '../../utilsFS';
+import {StateCore} from '../../typings';
+import {
+    GetFileTokensOpts,
+    getFileTokens,
+    getFullIncludePath,
+    isFileExists,
+    resolveRelativePath,
+} from '../../utilsFS';
 import {findBlockTokens} from '../../utils';
-import Token from 'markdown-it/lib/token';
 import {MarkdownItPluginCb, MarkdownItPluginOpts} from '../typings';
-import {StateCore} from 'src/transform/typings';
+import {MarkdownItIncluded} from './types';
 
 const INCLUDE_REGEXP = /^{%\s*include\s*(notitle)?\s*\[(.+?)]\((.+?)\)\s*%}$/;
 
@@ -20,7 +27,7 @@ type Options = MarkdownItPluginOpts &
         noReplaceInclude: boolean;
     };
 
-function unfoldIncludes(state: StateCore, path: string, options: Options) {
+function unfoldIncludes(md: MarkdownItIncluded, state: StateCore, path: string, options: Options) {
     const {root, notFoundCb, log, noReplaceInclude = false} = options;
     const {tokens} = state;
     let i = 0;
@@ -41,6 +48,11 @@ function unfoldIncludes(state: StateCore, path: string, options: Options) {
                 const [, keyword /* description */, , includePath] = match;
 
                 const fullIncludePath = getFullIncludePath(includePath, root, path);
+                const relativeIncludePath = resolveRelativePath(path, includePath);
+
+                // Check the existed included store and extract it
+                const included = md.included?.[relativeIncludePath];
+
                 let pathname = fullIncludePath;
                 let hash = '';
                 const hashIndex = fullIncludePath.lastIndexOf('#');
@@ -55,7 +67,10 @@ function unfoldIncludes(state: StateCore, path: string, options: Options) {
                     continue;
                 }
 
-                const fileTokens = getFileTokens(pathname, state, options);
+                const fileTokens = getFileTokens(pathname, state, {
+                    ...options,
+                    content: included, // The content forces the function to use it instead of reading from the disk
+                });
 
                 let includedTokens;
                 if (hash) {
@@ -92,7 +107,7 @@ function unfoldIncludes(state: StateCore, path: string, options: Options) {
     }
 }
 
-const index: MarkdownItPluginCb<Options> = (md, options) => {
+const index: MarkdownItPluginCb<Options> = (md: MarkdownItIncluded, options) => {
     const {path: optPath, log} = options;
 
     const plugin = (state: StateCore) => {
@@ -113,7 +128,7 @@ const index: MarkdownItPluginCb<Options> = (md, options) => {
         }
 
         env.includes.push(path);
-        unfoldIncludes(state, path, options);
+        unfoldIncludes(md, state, path, options);
         env.includes.pop();
     };
 

src/transform/plugins/includes/types.ts

@@ -0,0 +1,7 @@
+import {MarkdownIt} from '../../typings';
+
+export interface MarkdownItIncluded extends MarkdownIt {
+    included?: {
+        [key: string]: string;
+    };
+}

src/transform/plugins/typings.ts

@@ -1,16 +1 @@
-import type {Logger} from '../log';
-import type {CacheContext, MarkdownIt} from '../typings';
-
-export interface MarkdownItPluginOpts {
-    path: string;
-    log: Logger;
-    lang: 'ru' | 'en' | 'es' | 'fr' | 'cs' | 'ar' | 'he';
-    root: string;
-    rootPublicPath: string;
-    isLintRun: boolean;
-    cache?: CacheContext;
-}
-
-export type MarkdownItPluginCb<T extends {} = {}> = {
-    (md: MarkdownIt, opts: T & MarkdownItPluginOpts): void;
-};
+export * from '../typings'; // TODO: Remove in major release

src/transform/preprocessors.ts

@@ -0,0 +1,7 @@
+import type {MarkdownItPreprocessorCb} from './plugins/typings';
+
+import included from './preprocessors/included';
+
+const defaultPreprocessors = [included] as MarkdownItPreprocessorCb[];
+
+export = defaultPreprocessors;