Update to remark@13 (micromark)

This updates MDX to use and support remark@13, which comes with a new internal parser (micromark), and supports CommonMark. See <https://github.com/remarkjs/remark/releases/tag/13.0.0> for more information. In short, this means MDX parses markdown closer to what folks expect. And it means all latest plugins work (again). But it also means that parsing MDX syntax (JSX, expressions, ESM) got an update. See: <https://github.com/micromark/micromark-extension-mdxjs> and <https://github.com/syntax-tree/mdast-util-mdx> for the syntax. In short, this means MDX parsing is now JavaScript-aware: import/exports are actually parsed for being valid JavaScript. Expressions previously counted braces, but now can include braces in strings or comments or whatnot. This also means we can drop Babel (in a future PR) because we already have a JavaScript AST. This also deprecates the packages `remark-mdxjs` (which is now the default in `remark-mdx`), `remark-mdx-remove-exports`, and `remark-mdx-remove-imports`. Related to GH-704. Related to GH-1041. Closes GH-720. Closes GH-1028. Closes GH-1050. Closes GH-1081. Closes GH-1193. Closes GH-1238. Closes GH-1283. Closes GH-1316. Closes GH-1318. Closes GH-1341.
mdx-js · Dec 11, 2020 · 609d9f2 · 609d9f2
1 parent 8c922e4
commit 609d9f2
Show file tree

Hide file tree

Showing 90 changed files with 12,063 additions and 10,761 deletions.
diff --git a/.remarkrc.js b/.remarkrc.js
@@ -1,7 +1,6 @@
 module.exports = {
   plugins: [
     './packages/remark-mdx',
-    './packages/remark-mdxjs',
     'preset-wooorm',
     'preset-prettier',
     ['retext', false],

diff --git a/docs/advanced/typescript.mdx b/docs/advanced/typescript.mdx
@@ -15,8 +15,6 @@ improve the developer experience for TypeScript users.
 - `@mdx-js/runtime`
 - `@mdx-js/vue`
 - `remark-mdx`
-- `remark-mdx-remove-exports`
-- `remark-mdx-remove-imports`
 
 Include types, no additional setup needed.
 

diff --git a/packages/gatsby-theme-mdx/package.json b/packages/gatsby-theme-mdx/package.json
@@ -47,10 +47,9 @@
     "react-helmet": "6.1.0",
     "react-live": "2.2.2",
     "remark-emoji": "2.1.0",
-    "remark-mdx-remove-exports": "^2.0.0-next.8",
-    "remark-mdx-remove-imports": "^2.0.0-next.8",
     "remark-slug": "6.0.0",
-    "theme-ui": "0.3.4"
+    "theme-ui": "0.3.4",
+    "unist-util-remove": "^2.0.0"
   },
   "gitHead": "bf7deab69996449cb99c2217dff75e65855eb2c1"
 }
diff --git a/packages/gatsby-theme-mdx/src/components/playground-editor.js b/packages/gatsby-theme-mdx/src/components/playground-editor.js
@@ -5,18 +5,19 @@ import {MDXProvider, mdx as createElement} from '@mdx-js/react'
 import * as Rebass from '@rebass/emotion'
 import {ThemeContext} from '@emotion/core'
 import {css} from 'theme-ui'
-import removeImports from 'remark-mdx-remove-imports'
-import removeExports from 'remark-mdx-remove-exports'
+import remove from 'unist-util-remove'
 
 import CodeBlock from './code-block'
 
+const removeEsm = () => tree => remove(tree, 'mdxjsEsm')
+
 const transformCode = src => {
   let transpiledMDX = ''
 
   try {
     transpiledMDX = mdx.sync(src, {
       skipExport: true,
-      remarkPlugins: [removeImports, removeExports]
+      remarkPlugins: [removeEsm]
     })
   } catch (_e) {
     return ''

diff --git a/packages/mdx/index.js b/packages/mdx/index.js
@@ -1,7 +1,6 @@
 const unified = require('unified')
 const remarkParse = require('remark-parse')
 const remarkMdx = require('remark-mdx')
-const remarkMdxJs = require('remark-mdxjs')
 const squeeze = require('remark-squeeze-paragraphs')
 const mdxAstToMdxHast = require('./mdx-ast-to-mdx-hast')
 const mdxHastToJsx = require('./mdx-hast-to-jsx')
@@ -12,7 +11,6 @@ function createMdxAstCompiler(options = {}) {
   return unified()
     .use(remarkParse)
     .use(remarkMdx)
-    .use(remarkMdxJs)
     .use(squeeze)
     .use(options.remarkPlugins)
     .use(mdxAstToMdxHast)

diff --git a/packages/mdx/mdx-ast-to-mdx-hast.js b/packages/mdx/mdx-ast-to-mdx-hast.js
@@ -61,17 +61,20 @@ function mdxAstToMdxHast() {
         export(h, node) {
           return Object.assign({}, node, {type: 'export'})
         },
-        mdxBlockElement(h, node) {
+        mdxjsEsm(h, node) {
+          return Object.assign({}, node, {type: 'mdxjsEsm'})
+        },
+        mdxJsxFlowElement(h, node) {
           return Object.assign({}, node, {children: all(h, node)})
         },
-        mdxSpanElement(h, node) {
+        mdxJsxTextElement(h, node) {
           return Object.assign({}, node, {children: all(h, node)})
         },
-        mdxBlockExpression(h, node) {
-          return Object.assign({}, node, {type: 'mdxBlockExpression'})
+        mdxFlowExpression(h, node) {
+          return Object.assign({}, node, {type: 'mdxFlowExpression'})
         },
-        mdxSpanExpression(h, node) {
-          return Object.assign({}, node, {type: 'mdxSpanExpression'})
+        mdxTextExpression(h, node) {
+          return Object.assign({}, node, {type: 'mdxTextExpression'})
         }
       }
     })

diff --git a/packages/mdx/mdx-hast-to-jsx.js b/packages/mdx/mdx-hast-to-jsx.js
@@ -1,12 +1,17 @@
 const {transformSync} = require('@babel/core')
 const uniq = require('lodash.uniq')
-const {serializeTags} = require('remark-mdx/lib/serialize/mdx-element')
-const serializeMdxExpression = require('remark-mdx/lib/serialize/mdx-expression')
+const encode = require('stringify-entities/light')
 const toH = require('hast-to-hyperscript')
+const recast = require('recast')
 const BabelPluginApplyMdxProp = require('babel-plugin-apply-mdx-type-prop')
 const BabelPluginExtractImportNames = require('babel-plugin-extract-import-names')
 const BabelPluginExtractExportNames = require('babel-plugin-extract-export-names')
 
+// To do: `recast` might be heavy (have to check), and `astring` might be a good
+// alternative.
+// However, `astring` doesn’t support JSX.
+// When we start compiling JSX away, `astring` might be a good fit though.
+
 function toJSX(node, parentNode = {}, options = {}) {
   if (node.type === 'root') {
     return serializeRoot(node, options)
@@ -21,18 +26,14 @@ function toJSX(node, parentNode = {}, options = {}) {
     return serializeText(node, options, parentNode)
   }
 
-  if (node.type === 'mdxBlockExpression' || node.type === 'mdxSpanExpression') {
+  if (node.type === 'mdxFlowExpression' || node.type === 'mdxTextExpression') {
     return serializeMdxExpression(node)
   }
 
   // To do: pass `parentName` in?
-  if (node.type === 'mdxBlockElement' || node.type === 'mdxSpanElement') {
+  if (node.type === 'mdxJsxFlowElement' || node.type === 'mdxJsxTextElement') {
     return serializeComponent(node, options, parentNode)
   }
-
-  if (node.type === 'import' || node.type === 'export') {
-    return serializeEsSyntax(node)
-  }
 }
 
 function serializeRoot(node, options) {
@@ -42,31 +43,28 @@ function serializeRoot(node, options) {
     wrapExport
   } = options
 
-  const groups = {import: [], export: [], rest: []}
+  const groups = {mdxjsEsm: [], rest: []}
 
-  for (const child of node.children) {
-    groups[
-      child.type === 'import' || child.type === 'export' ? child.type : 'rest'
-    ].push(child)
-  }
+  node.children.forEach(child => {
+    groups[child.type === 'mdxjsEsm' ? child.type : 'rest'].push(child)
+  })
 
   // Find a default export, assumes there’s zero or one.
-  const defaultExport = groups.export.find(child => child.default)
-
-  const layout = defaultExport
-    ? defaultExport.value
-        .replace(/^export\s+default\s+/, '')
-        .replace(/;\s*$/, '')
-    : null
-
-  const importStatements = groups.import
-    .map(childNode => toJSX(childNode, node))
-    .join('\n')
-
-  const exportStatements = groups.export
-    .filter(child => !child.default)
-    .map(childNode => toJSX(childNode, node))
-    .join('\n')
+  const importStatements = []
+  const exportStatements = []
+  let layout
+
+  groups.mdxjsEsm.forEach(child => {
+    child.data.estree.body.forEach(eschild => {
+      if (eschild.type === 'ImportDeclaration') {
+        importStatements.push(recast.prettyPrint(eschild).code)
+      } else if (eschild.type === 'ExportDefaultDeclaration') {
+        layout = recast.prettyPrint(eschild.declaration).code
+      } else {
+        exportStatements.push(recast.prettyPrint(eschild).code)
+      }
+    })
+  })
 
   const doc = groups.rest
     .map(childNode => toJSX(childNode, node, options))
@@ -85,7 +83,9 @@ MDXContent.isMDXComponent = true`
   // Check JSX nodes against imports
   const babelPluginExtractImportNamesInstance = new BabelPluginExtractImportNames()
   const babelPluginExtractExportNamesInstance = new BabelPluginExtractExportNames()
-  const importsAndExports = [importStatements, exportStatements].join('\n')
+  const importsAndExports = []
+    .concat(importStatements, exportStatements)
+    .join('\n')
 
   transformSync(importsAndExports, {
     configFile: false,
@@ -115,15 +115,18 @@ MDXContent.isMDXComponent = true`
     ]
   }).code
 
-  const exportStatementsPostMdxTypeProps = transformSync(exportStatements, {
-    configFile: false,
-    babelrc: false,
-    plugins: [
-      require('@babel/plugin-syntax-jsx'),
-      require('@babel/plugin-syntax-object-rest-spread'),
-      babelPluginApplyMdxPropToExportsInstance.plugin
-    ]
-  }).code
+  const exportStatementsPostMdxTypeProps = transformSync(
+    exportStatements.join('\n'),
+    {
+      configFile: false,
+      babelrc: false,
+      plugins: [
+        require('@babel/plugin-syntax-jsx'),
+        require('@babel/plugin-syntax-object-rest-spread'),
+        babelPluginApplyMdxPropToExportsInstance.plugin
+      ]
+    }
+  ).code
 
   const allJsxNames = [
     ...babelPluginApplyMdxPropInstance.state.names,
@@ -187,12 +190,10 @@ function serializeElement(node, options, parentNode) {
 }
 
 function serializeComponent(node, options) {
+  const tags = serializeTags(node)
   let content = serializeChildren(node, options)
-  const tags = serializeTags(
-    Object.assign({}, node, {children: content ? ['!'] : []})
-  )
 
-  if (node.type === 'mdxBlockElement' && content) {
+  if (node.type === 'mdxJsxFlowElement' && content) {
     content = '\n' + content + '\n'
   }
 
@@ -212,10 +213,6 @@ function serializeText(node, options, parentNode) {
   return toTemplateLiteral(node.value)
 }
 
-function serializeEsSyntax(node) {
-  return node.value
-}
-
 function serializeChildren(node, options) {
   const children = node.children || []
   const childOptions = Object.assign({}, options, {
@@ -261,3 +258,91 @@ function compile(options = {}) {
 module.exports = compile
 compile.default = compile
 compile.toJSX = toJSX
+
+// To do: this is all extracted (and simplified) from `mdast-util-mdx-jsx` for
+// now.
+// We can remove it when we drop JSX!
+
+const eol = /\r?\n|\r/g
+
+function serializeTags(node) {
+  const selfClosing = node.name && (!node.children || !node.children.length)
+  const attributes = []
+  let index = -1
+  let attribute
+  let result
+
+  // None.
+  if (node.attributes && node.attributes.length) {
+    if (!node.name) {
+      throw new Error('Cannot serialize fragment w/ attributes')
+    }
+
+    while (++index < node.attributes.length) {
+      attribute = node.attributes[index]
+
+      if (attribute.type === 'mdxJsxExpressionAttribute') {
+        result = '{' + (attribute.value || '') + '}'
+      } else {
+        if (!attribute.name) {
+          throw new Error('Cannot serialize attribute w/o name')
+        }
+
+        result =
+          attribute.name +
+          (attribute.value == null
+            ? ''
+            : '=' +
+              (typeof attribute.value === 'object'
+                ? '{' + (attribute.value.value || '') + '}'
+                : '"' + encode(attribute.value, {subset: ['"']}) + '"'))
+      }
+
+      attributes.push(result)
+    }
+  }
+
+  return {
+    open:
+      '<' +
+      (node.name || '') +
+      (node.type === 'mdxJsxFlowElement' && attributes.length > 1
+        ? // Flow w/ multiple attributes.
+          '\n' + indent(attributes.join('\n')) + '\n'
+        : attributes.length // Text or flow w/ a single attribute.
+        ? ' ' + dedentStart(indent(attributes.join(' ')))
+        : '') +
+      (selfClosing ? '/' : '') +
+      '>',
+    close: selfClosing ? '' : '</' + (node.name || '') + '>'
+  }
+}
+
+function serializeMdxExpression(node) {
+  const value = node.value || ''
+  return '{' + (node.type === 'mdxFlowExpression' ? indent(value) : value) + '}'
+}
+
+function dedentStart(value) {
+  return value.replace(/^ +/, '')
+}
+
+function indent(value) {
+  const result = []
+  let start = 0
+  let match
+
+  while ((match = eol.exec(value))) {
+    one(value.slice(start, match.index))
+    result.push(match[0])
+    start = match.index + match[0].length
+  }
+
+  one(value.slice(start))
+
+  return result.join('')
+
+  function one(slice) {
+    result.push((slice ? '  ' : '') + slice)
+  }
+}
diff --git a/packages/mdx/package.json b/packages/mdx/package.json
@@ -52,18 +52,20 @@
     "detab": "2.0.3",
     "hast-to-hyperscript": "9.0.0",
     "lodash.uniq": "4.5.0",
-    "mdast-util-to-hast": "9.1.0",
+    "mdast-util-to-hast": "10.0.1",
+    "recast": "^0.20.4",
     "remark-mdx": "^2.0.0-next.8",
-    "remark-mdxjs": "^2.0.0-next.8",
-    "remark-parse": "8.0.2",
+    "remark-parse": "9.0.0",
     "remark-squeeze-paragraphs": "4.0.0",
+    "stringify-entities": "^3.1.0",
     "unified": "9.0.0",
     "unist-builder": "2.0.3"
   },
   "devDependencies": {
-    "remark-footnotes": "1.0.0",
+    "remark-footnotes": "3.0.0",
+    "remark-gfm": "1.0.0",
     "rehype-katex": "^4.0.0",
-    "remark-math": "^3.0.0"
+    "remark-math": "^4.0.0"
   },
   "gitHead": "bf7deab69996449cb99c2217dff75e65855eb2c1"
 }