feat: liquid markdown front matter separately from the rest of the co…

…ntent

src/transform/frontmatter/common.ts

@@ -0,0 +1,27 @@
+export type FrontMatter = {
+    [key: string]: unknown;
+    metadata?: Record<string, unknown>[];
+};
+
+export const frontMatterFence = '---';
+
+/**
+ * Temporary workaround to enable parsing YAML metadata from potentially
+ * Liquid-aware source files
+ * @param content Input string which could contain Liquid-style substitution syntax (which clashes with YAML
+ * object syntax)
+ * @returns String with `{}` escaped, ready to be parsed with `js-yaml`
+ */
+export const escapeLiquidSubstitutionSyntax = (content: string): string =>
+    content.replace(/{{/g, '(({{').replace(/}}/g, '}}))');
+
+/**
+ * Inverse of a workaround defined above.
+ * @see `escapeLiquidSubstitutionSyntax`
+ * @param escapedContent Input string with `{}` escaped with backslashes
+ * @returns Unescaped string
+ */
+export const unescapeLiquidSubstitutionSyntax = (escapedContent: string): string =>
+    escapedContent.replace(/\(\({{/g, '{{').replace(/}}\)\)/g, '}}');
+
+export const countLineAmount = (str: string) => str.split(/\r?\n/).length;

src/transform/frontmatter/emplace.ts

@@ -0,0 +1,24 @@
+import {dump} from 'js-yaml';
+
+import {FrontMatter, frontMatterFence} from './common';
+
+export const serializeFrontMatter = (frontMatter: FrontMatter) => {
+    const dumped = dump(frontMatter, {lineWidth: -1}).trim();
+
+    // This empty object check is a bit naive
+    // The other option would be to check if all own fields are `undefined`,
+    // since we exploit passing in `undefined` to remove a field quite a bit
+    if (dumped === '{}') {
+        return '';
+    }
+
+    return `${frontMatterFence}\n${dumped}\n${frontMatterFence}\n`;
+};
+
+export const emplaceSerializedFrontMatter = (
+    frontMatterStrippedContent: string,
+    frontMatter: string,
+) => `${frontMatter}${frontMatterStrippedContent}`;
+
+export const emplaceFrontMatter = (frontMatterStrippedContent: string, frontMatter: FrontMatter) =>
+    emplaceSerializedFrontMatter(frontMatterStrippedContent, serializeFrontMatter(frontMatter));

src/transform/frontmatter/extract.ts

@@ -0,0 +1,94 @@
+import {YAMLException, load} from 'js-yaml';
+
+import {log} from '../log';
+
+import {
+    FrontMatter,
+    countLineAmount,
+    escapeLiquidSubstitutionSyntax,
+    frontMatterFence,
+    unescapeLiquidSubstitutionSyntax,
+} from './common';
+import {transformFrontMatterValues} from './transformValues';
+
+type ParseExistingMetadataReturn = {
+    frontMatter: FrontMatter;
+    frontMatterStrippedContent: string;
+    frontMatterLineCount: number;
+};
+
+const matchMetadata = (fileContent: string) => {
+    if (!fileContent.startsWith(frontMatterFence)) {
+        return null;
+    }
+
+    // Search by format:
+    // ---
+    // metaName1: metaValue1
+    // metaName2: meta value2
+    // incorrectMetadata
+    // ---
+    const regexpMetadata = '(?<=-{3}\\r?\\n)((.*\\r?\\n)*?)(?=-{3}\\r?\\n)';
+    // Search by format:
+    // ---
+    // main content 123
+    const regexpFileContent = '-{3}\\r?\\n((.*[\r?\n]*)*)';
+
+    const regexpParseFileContent = new RegExp(`${regexpMetadata}${regexpFileContent}`, 'gm');
+
+    return regexpParseFileContent.exec(fileContent);
+};
+
+const duplicateKeysCompatibleLoad = (yaml: string, filePath: string | undefined) => {
+    try {
+        return load(yaml);
+    } catch (e) {
+        if (e instanceof YAMLException) {
+            const duplicateKeysDeprecationWarning = `
+                In ${filePath ?? '(unknown)'}: Encountered a YAML parsing exception when processing file metadata: ${e.reason}.
+                It's highly possible the input file contains duplicate mapping keys.
+                Will retry processing with necessary compatibility flags.
+                Please note that this behaviour is DEPRECATED and WILL be removed in a future version
+                without further notice, so the build WILL fail when supplied with YAML-incompatible meta.
+            `
+                .replace(/^\s+/gm, '')
+                .replace(/\n/g, ' ')
+                .trim();
+
+            log.warn(duplicateKeysDeprecationWarning);
+
+            return load(yaml, {json: true});
+        }
+
+        throw e;
+    }
+};
+
+export const separateAndExtractFrontMatter = (
+    fileContent: string,
+    filePath?: string,
+): ParseExistingMetadataReturn => {
+    const matches = matchMetadata(fileContent);
+
+    if (matches && matches.length > 0) {
+        const [, metadata, , metadataStrippedContent] = matches;
+
+        return {
+            frontMatter: transformFrontMatterValues(
+                duplicateKeysCompatibleLoad(
+                    escapeLiquidSubstitutionSyntax(metadata),
+                    filePath,
+                ) as FrontMatter,
+                (v) => (typeof v === 'string' ? unescapeLiquidSubstitutionSyntax(v) : v),
+            ),
+            frontMatterStrippedContent: metadataStrippedContent,
+            frontMatterLineCount: countLineAmount(metadata),
+        };
+    }
+
+    return {
+        frontMatter: {},
+        frontMatterStrippedContent: fileContent,
+        frontMatterLineCount: 0,
+    };
+};

src/transform/frontmatter/index.ts

@@ -0,0 +1,4 @@
+export * from './extract';
+export * from './emplace';
+export * from './transformValues';
+export {countLineAmount} from './common';

src/transform/frontmatter/transformValues.ts

@@ -0,0 +1,22 @@
+import {FrontMatter} from './common';
+
+export const transformFrontMatterValues = (
+    frontMatter: FrontMatter,
+    valueMapper: (v: unknown) => unknown,
+): FrontMatter => {
+    const transformInner = (something: unknown): unknown => {
+        if (typeof something === 'object' && something !== null) {
+            return Object.fromEntries(
+                Object.entries(something).map(([k, v]) => [k, transformInner(v)]),
+            );
+        }
+
+        if (Array.isArray(something)) {
+            return something.map((el) => transformInner(el));
+        }
+
+        return valueMapper(something);
+    };
+
+    return transformInner(frontMatter) as FrontMatter;
+};

src/transform/index.ts

@@ -3,7 +3,7 @@ import type {EnvType, OptionsType, OutputType} from './typings';
 import {bold} from 'chalk';
 
 import {log} from './log';
-import liquid from './liquid';
+import liquidSnippet from './liquid';
 import initMarkdownit from './md';
 
 function applyLiquid(input: string, options: OptionsType) {
@@ -15,7 +15,9 @@ function applyLiquid(input: string, options: OptionsType) {
         isLiquided = false,
     } = options;
 
-    return disableLiquid || isLiquided ? input : liquid(input, vars, path, {conditionsInCode});
+    return disableLiquid || isLiquided
+        ? input
+        : liquidSnippet(input, vars, path, {conditionsInCode});
 }
 
 function handleError(error: unknown, path?: string): never {

src/transform/liquid/index.ts

@@ -1,5 +1,13 @@
 import type {Dictionary} from 'lodash';
 
+import {
+    countLineAmount,
+    emplaceSerializedFrontMatter,
+    separateAndExtractFrontMatter,
+    serializeFrontMatter,
+    transformFrontMatterValues,
+} from '../frontmatter';
+
 import applySubstitutions from './substitutions';
 import {prepareSourceMap} from './sourceMap';
 import applyCycles from './cycles';
@@ -66,7 +74,7 @@ function repairCode(str: string, codes: string[]) {
     return replace(fence, fence, (code) => codes[Number(code)], str);
 }
 
-function liquid<
+function liquidSnippet<
     B extends boolean = false,
     C = B extends false ? string : {output: string; sourceMap: Dictionary<string>},
 >(
@@ -141,6 +149,67 @@ function liquid<
     return output as unknown as C;
 }
 
-// 'export default' instead of 'export = ' because of circular dependency with './cycles.ts'.
-// somehow it breaks import in './cycles.ts' and imports nothing
-export default liquid;
+type TransformSourceMapOptions = {
+    emplacedResultOffset: number;
+    emplacedSourceOffset: number;
+};
+
+function transformSourceMap(
+    sourceMap: Dictionary<string>,
+    {emplacedResultOffset, emplacedSourceOffset}: TransformSourceMapOptions,
+) {
+    return Object.fromEntries(
+        Object.entries(sourceMap).map(([lineInResult, lineInSource]) => [
+            (Number(lineInResult) + emplacedResultOffset).toString(),
+            (Number(lineInSource) + emplacedSourceOffset).toString(),
+        ]),
+    );
+}
+
+function liquidDocument<
+    B extends boolean = false,
+    C = B extends false ? string : {output: string; sourceMap: Dictionary<string>},
+>(
+    originInput: string,
+    vars: Record<string, unknown>,
+    path?: string,
+    settings?: ArgvSettings & {withSourceMap?: B},
+): C {
+    const {frontMatter, frontMatterStrippedContent, frontMatterLineCount} =
+        separateAndExtractFrontMatter(originInput, path);
+
+    const transformedFrontMatter = transformFrontMatterValues(frontMatter, (v) =>
+        typeof v === 'string'
+            ? liquidSnippet(v, vars, path, {...settings, withSourceMap: false})
+            : v,
+    );
+    const transformedAndSerialized = serializeFrontMatter(transformedFrontMatter);
+
+    // -1 comes from the fact that the last line in serialized FM is the same as the first line in stripped content
+    const resultFrontMatterOffset = Math.max(0, countLineAmount(transformedAndSerialized) - 1);
+    const sourceFrontMatterOffset = Math.max(0, frontMatterLineCount - 1);
+
+    const liquidProcessedContent = liquidSnippet(frontMatterStrippedContent, vars, path, settings);
+
+    // typeof check for better inference; the catch is that return of liquidSnippet can be an
+    // object even with source maps off, see `substitutions.test.ts`
+    return (settings?.withSourceMap && typeof liquidProcessedContent === 'object'
+        ? {
+              output: emplaceSerializedFrontMatter(
+                  liquidProcessedContent.output,
+                  transformedAndSerialized,
+              ),
+              sourceMap: transformSourceMap(liquidProcessedContent.sourceMap, {
+                  emplacedResultOffset: resultFrontMatterOffset,
+                  emplacedSourceOffset: sourceFrontMatterOffset,
+              }),
+          }
+        : emplaceSerializedFrontMatter(
+              liquidProcessedContent as string,
+              transformedAndSerialized,
+          )) as unknown as C;
+}
+
+// both default and named exports for convenience
+export {liquidDocument, liquidSnippet};
+export default liquidDocument;

src/transform/utilsFS.ts

@@ -4,7 +4,7 @@ import {readFileSync, statSync} from 'fs';
 import escapeRegExp from 'lodash/escapeRegExp';
 import {join, parse, relative, resolve, sep} from 'path';
 
-import liquid from './liquid';
+import liquidSnippet from './liquid';
 import {StateCore} from './typings';
 import {defaultTransformLink} from './utils';
 
@@ -68,7 +68,7 @@ export function getFileTokens(path: string, state: StateCore, options: GetFileTo
     let sourceMap;
 
     if (!disableLiquid) {
-        const liquidResult = liquid(content, builtVars, path, {
+        const liquidResult = liquidSnippet(content, builtVars, path, {
             withSourceMap: true,
             conditionsInCode,
         });

test/liquid/__snapshots__/frontmatter.test.ts.snap

@@ -0,0 +1,36 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`Liquid substitutions in front matter (formerly metadata) should not fail even when variables contain reserved characters (YAML multiline syntax) 1`] = `
+"---
+multiline: \\">- This should break, right?\\\\n\\\\tRight?\\"
+---
+Content"
+`;
+
+exports[`Liquid substitutions in front matter (formerly metadata) should not fail even when variables contain reserved characters (curly braces) 1`] = `
+"---
+braces: '{}'
+---
+Content"
+`;
+
+exports[`Liquid substitutions in front matter (formerly metadata) should not fail even when variables contain reserved characters (double quotes) 1`] = `
+"---
+quotes: '\\"When you arise in the morning, think of what a precious privilege it is to be alive - to breathe, to think, to enjoy, to love.\\" — Marcus Aurelius (allegedly)'
+---
+Content"
+`;
+
+exports[`Liquid substitutions in front matter (formerly metadata) should not fail even when variables contain reserved characters (single quotes) 1`] = `
+"---
+quotes: This isn't your typical substitution. It has single quotes.
+---
+Content"
+`;
+
+exports[`Liquid substitutions in front matter (formerly metadata) should not fail even when variables contain reserved characters (square brackets) 1`] = `
+"---
+brackets: '[]'
+---
+Content"
+`;