fix(widget-markdown): handle line anchors in shortcode tokenizer inst…

…ead of changing user patterns

packages/netlify-cms-core/src/valueObjects/EditorComponent.js

@@ -28,20 +28,7 @@ export default function createEditorComponent(config) {
     type,
     icon,
     widget,
-    // We allow consumers to specify line anchors (^ and $) without a
-    // multiline flag, but we need to be able to match in a multi-line
-    // context. In order to do so, we replace instances of line anchors
-    // with lookarounds that include \n's, so that non-multiline expressions
-    // still work as expected.
-    //
-    // Note: This is mocked in packages/netlify-cms-widget-markdown/src/serializers/__tests__/remarkShortcodes.spec.js;
-    // if you change it here, change it there as well!
-    pattern: new RegExp(
-      // We use a negative lookbehind so that we only replace carets
-      // that aren't a negation in a character set or escaped
-      pattern.source.replace(/(?<!\[|\\)\^/, '(?<=^|\n)').replace(/(?<!\\)\$/, '(?=$|\n)'),
-      pattern.flags,
-    ),
+    pattern,
     fromBlock: bind(fromBlock) || (() => ({})),
     toBlock: bind(toBlock) || (() => 'Plugin'),
     toPreview: bind(toPreview) || (!widget && (bind(toBlock) || (() => 'Plugin'))),

packages/netlify-cms-widget-markdown/src/serializers/__tests__/remarkShortcodes.spec.js

@@ -19,12 +19,7 @@ function EditorComponent({ id = 'foo', fromBlock = jest.fn(), pattern }) {
   return {
     id,
     fromBlock,
-    // The EditorComponent factory (packages/netlify-cms-core/src/valueObjects/EditorComponent.js)
-    // modifies incoming regex patterns as follows
-    pattern: new RegExp(
-      pattern.source.replace(/(?<!\[|\\)\^/, '(?<=^|\n)').replace(/(?<!\\)\$/, '(?=$|\n)'),
-      pattern.flags,
-    ),
+    pattern,
   };
 }
 
@@ -52,7 +47,7 @@ describe('remarkParseShortcodes', () => {
         expect.arrayContaining(['foo\n\nbar']),
       );
     });
-    it('should match out-of-order shortcodes', () => {
+    it('should match shortcodes based on order of occurrence in value', () => {
       const fooEditorComponent = EditorComponent({ id: 'foo', pattern: /foo/ });
       const barEditorComponent = EditorComponent({ id: 'bar', pattern: /bar/ });
       process(
@@ -64,7 +59,7 @@ describe('remarkParseShortcodes', () => {
       );
       expect(fooEditorComponent.fromBlock).toHaveBeenCalledWith(expect.arrayContaining(['foo']));
     });
-    it('should match out-of-order shortcodes with line-end tokens', () => {
+    it('should match shortcodes based on order of occurrence in value even when some use line anchors', () => {
       const barEditorComponent = EditorComponent({ id: 'bar', pattern: /bar/ });
       const bazEditorComponent = EditorComponent({ id: 'baz', pattern: /^baz$/ });
       process(

packages/netlify-cms-widget-markdown/src/serializers/remarkShortcodes.js

@@ -10,6 +10,33 @@ export function remarkParseShortcodes({ plugins }) {
 
 function createShortcodeTokenizer({ plugins }) {
   return function tokenizeShortcode(eat, value, silent) {
+    // Plugin patterns may rely on `^` and `$` tokens, even if they don't
+    // use the multiline flag. To support this, we fall back to searching
+    // through each line individually, trimming trailing whitespace and
+    // newlines, if we don't initially match on a pattern. We keep track of
+    // the starting position of each line so that we can sort correctly
+    // across the full multilen matches.
+    const trimmedLines = value
+      .split('\n\n')
+      .reduce((acc, line) => {
+        const [
+          { start: previousLineStart, originalLength: previousLineOriginalLength } = {
+            start: 0,
+            originalLength: 0,
+          },
+        ] = acc;
+        return [
+          {
+            line: line.trimEnd(),
+            start: previousLineStart + previousLineOriginalLength + 2,
+            originalLength: line.length,
+          },
+          ...acc,
+        ];
+      }, [])
+      .reverse()
+      .map(({ line, start }) => ({ line, start }));
+
     // Attempt to find a regex match for each plugin's pattern, and then
     // select the first by its occurence in `value`. This ensures we won't
     // skip a plugin that occurs later in the plugin registry, but earlier
@@ -18,7 +45,17 @@ function createShortcodeTokenizer({ plugins }) {
       plugins
         .toList()
         .map(plugin => ({
-          match: value.match(plugin.pattern),
+          match:
+            value.match(plugin.pattern) ||
+            trimmedLines
+              .map(({ line, start }) => {
+                const match = line.match(plugin.pattern);
+                if (match) {
+                  match.index += start;
+                }
+                return match;
+              })
+              .find(match => !!match),
           plugin,
         }))
         .filter(({ match }) => !!match)