feat(core, mdx-loader): deduplicate MDX compilation - siteConfig.future.experimental_faster.mdxCrossCompilerCache

packages/docusaurus-mdx-loader/src/createMDXLoader.ts

@@ -6,30 +6,49 @@
  */
 
 import {createProcessors} from './processor';
-import type {Options} from './loader';
+import type {Options} from './options';
 import type {RuleSetRule, RuleSetUseItem} from 'webpack';
 
-async function enhancedOptions(options: Options): Promise<Options> {
+type CreateOptions = {
+  useCrossCompilerCache?: boolean;
+};
+
+async function normalizeOptions(
+  optionsInput: Options & CreateOptions,
+): Promise<Options> {
   // Because Jest doesn't like ESM / createProcessors()
   if (process.env.N0DE_ENV === 'test' || process.env.JEST_WORKER_ID) {
-    return options;
+    return optionsInput;
   }
 
+  let options = optionsInput;
+
   // We create the processor earlier here, to avoid the lazy processor creating
   // Lazy creation messes-up with Rsdoctor ability to measure mdx-loader perf
-  const newOptions: Options = options.processors
-    ? options
-    : {...options, processors: await createProcessors({options})};
+  if (!options.processors) {
+    options = {...options, processors: await createProcessors({options})};
+  }
+
+  // Cross-compiler cache permits to compile client/server MDX only once
+  // We don't want to cache in dev mode (docusaurus start)
+  // We only have multiple compilers in production mode (docusaurus build)
+  // TODO wrong but good enough for now (example: "docusaurus build --dev")
+  if (options.useCrossCompilerCache && process.env.NODE_ENV === 'production') {
+    options = {
+      ...options,
+      crossCompilerCache: new Map(),
+    };
+  }
 
-  return newOptions;
+  return options;
 }
 
 export async function createMDXLoaderItem(
-  options: Options,
+  options: Options & CreateOptions,
 ): Promise<RuleSetUseItem> {
   return {
     loader: require.resolve('@docusaurus/mdx-loader'),
-    options: await enhancedOptions(options),
+    options: await normalizeOptions(options),
   };
 }
 
@@ -38,7 +57,7 @@ export async function createMDXLoaderRule({
   options,
 }: {
   include: RuleSetRule['include'];
-  options: Options;
+  options: Options & CreateOptions;
 }): Promise<RuleSetRule> {
   return {
     test: /\.mdx?$/i,

packages/docusaurus-mdx-loader/src/index.ts

@@ -37,5 +37,6 @@ export type LoadedMDXContent<FrontMatter, Metadata, Assets = undefined> = {
   (): JSX.Element;
 };
 
-export type {Options, MDXPlugin} from './loader';
+export type {MDXPlugin} from './loader';
 export type {MDXOptions} from './processor';
+export type {Options} from './options';

packages/docusaurus-mdx-loader/src/loader.ts

@@ -18,14 +18,8 @@ import {
   createAssetsExportCode,
   extractContentTitleData,
 } from './utils';
-import type {
-  SimpleProcessors,
-  MDXOptions,
-  SimpleProcessorResult,
-} from './processor';
-import type {ResolveMarkdownLink} from './remark/resolveMarkdownLinks';
-
-import type {MarkdownConfig} from '@docusaurus/types';
+import type {WebpackCompilerName} from '@docusaurus/utils';
+import type {Options} from './options';
 import type {LoaderContext} from 'webpack';
 
 // TODO as of April 2023, no way to import/re-export this ESM type easily :/
@@ -35,33 +29,17 @@ type Pluggable = any; // TODO fix this asap
 
 export type MDXPlugin = Pluggable;
 
-export type Options = Partial<MDXOptions> & {
-  markdownConfig: MarkdownConfig;
-  staticDirs: string[];
-  siteDir: string;
-  isMDXPartial?: (filePath: string) => boolean;
-  isMDXPartialFrontMatterWarningDisabled?: boolean;
-  removeContentTitle?: boolean;
-  metadataPath?: (filePath: string) => string;
-  createAssets?: (metadata: {
-    filePath: string;
-    frontMatter: {[key: string]: unknown};
-  }) => {[key: string]: unknown};
-  resolveMarkdownLink?: ResolveMarkdownLink;
-
-  // Will usually be created by "createMDXLoaderItem"
-  processors?: SimpleProcessors;
-};
-
-export async function mdxLoader(
-  this: LoaderContext<Options>,
-  fileContent: string,
-): Promise<void> {
-  const compilerName = getWebpackLoaderCompilerName(this);
-  const callback = this.async();
-  const filePath = this.resourcePath;
-  const options: Options = this.getOptions();
-
+async function loadMDX({
+  fileContent,
+  filePath,
+  options,
+  compilerName,
+}: {
+  fileContent: string;
+  filePath: string;
+  options: Options;
+  compilerName: WebpackCompilerName;
+}): Promise<string> {
   const {frontMatter} = await options.markdownConfig.parseFrontMatter({
     filePath,
     fileContent,
@@ -70,18 +48,13 @@ export async function mdxLoader(
 
   const hasFrontMatter = Object.keys(frontMatter).length > 0;
 
-  let result: SimpleProcessorResult;
-  try {
-    result = await compileToJSX({
-      fileContent,
-      filePath,
-      frontMatter,
-      options,
-      compilerName,
-    });
-  } catch (error) {
-    return callback(error as Error);
-  }
+  const result = await compileToJSX({
+    fileContent,
+    filePath,
+    frontMatter,
+    options,
+    compilerName,
+  });
 
   const contentTitle = extractContentTitleData(result.data);
 
@@ -97,7 +70,7 @@ ${JSON.stringify(frontMatter, null, 2)}`;
     if (!options.isMDXPartialFrontMatterWarningDisabled) {
       const shouldError = process.env.NODE_ENV === 'test' || process.env.CI;
       if (shouldError) {
-        return callback(new Error(errorMessage));
+        throw new Error(errorMessage);
       }
       logger.warn(errorMessage);
     }
@@ -146,5 +119,68 @@ ${exportsCode}
 ${result.content}
 `;
 
-  return callback(null, code);
+  return code;
+}
+
+// Note: we cache promises instead of strings
+// This is because client/server compilations might be triggered in parallel
+// When this happens for the same file, we don't want to compile it twice
+async function loadMDXWithCaching({
+  resource,
+  fileContent,
+  filePath,
+  options,
+  compilerName,
+}: {
+  resource: string; // path?query#hash
+  filePath: string; // path
+  fileContent: string;
+  options: Options;
+  compilerName: WebpackCompilerName;
+}): Promise<string> {
+  // Note we "resource" as cache key, not "filePath" nor "fileContent"
+  // This is because:
+  // - the same file can be compiled in different variants (blog.mdx?truncated)
+  // - the same content can be processed differently (versioned docs links)
+  const cacheKey = resource;
+
+  const cachedPromise = options.crossCompilerCache?.get(cacheKey);
+  if (cachedPromise) {
+    // We can clean up the cache and free memory here
+    // We know there are only 2 compilations for the same file
+    // Note: once we introduce RSCs we'll probably have 3 compilations
+    // Note: we can't use string keys in WeakMap
+    // But we could eventually use WeakRef for the values
+    options.crossCompilerCache?.delete(cacheKey);
+    return cachedPromise;
+  }
+  const promise = loadMDX({
+    fileContent,
+    filePath,
+    options,
+    compilerName,
+  });
+  options.crossCompilerCache?.set(cacheKey, promise);
+  return promise;
+}
+
+export async function mdxLoader(
+  this: LoaderContext<Options>,
+  fileContent: string,
+): Promise<void> {
+  const compilerName = getWebpackLoaderCompilerName(this);
+  const callback = this.async();
+  const options: Options = this.getOptions();
+  try {
+    const result = await loadMDXWithCaching({
+      resource: this.resource,
+      filePath: this.resourcePath,
+      fileContent,
+      options,
+      compilerName,
+    });
+    return callback(null, result);
+  } catch (error) {
+    return callback(error as Error);
+  }
 }

packages/docusaurus-mdx-loader/src/options.ts

@@ -0,0 +1,29 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import type {MDXOptions, SimpleProcessors} from './processor';
+import type {MarkdownConfig} from '@docusaurus/types';
+import type {ResolveMarkdownLink} from './remark/resolveMarkdownLinks';
+
+export type Options = Partial<MDXOptions> & {
+  markdownConfig: MarkdownConfig;
+  staticDirs: string[];
+  siteDir: string;
+  isMDXPartial?: (filePath: string) => boolean;
+  isMDXPartialFrontMatterWarningDisabled?: boolean;
+  removeContentTitle?: boolean;
+  metadataPath?: (filePath: string) => string;
+  createAssets?: (metadata: {
+    filePath: string;
+    frontMatter: {[key: string]: unknown};
+  }) => {[key: string]: unknown};
+  resolveMarkdownLink?: ResolveMarkdownLink;
+
+  // Will usually be created by "createMDXLoaderItem"
+  processors?: SimpleProcessors;
+  crossCompilerCache?: Map<string, Promise<string>>; // MDX => Promise<JSX> cache
+};

packages/docusaurus-mdx-loader/src/preprocessor.ts

@@ -11,7 +11,7 @@ import {
   admonitionTitleToDirectiveLabel,
 } from '@docusaurus/utils';
 import {normalizeAdmonitionOptions} from './remark/admonitions';
-import type {Options} from './loader';
+import type {Options} from './options';
 
 /**
  * Preprocess the string before passing it to MDX

packages/docusaurus-mdx-loader/src/processor.ts

@@ -20,7 +20,7 @@ import codeCompatPlugin from './remark/mdx1Compat/codeCompatPlugin';
 import {getFormat} from './format';
 import type {WebpackCompilerName} from '@docusaurus/utils';
 import type {MDXFrontMatter} from './frontMatter';
-import type {Options} from './loader';
+import type {Options} from './options';
 import type {AdmonitionOptions} from './remark/admonitions';
 
 // @ts-expect-error: TODO see https://github.com/microsoft/TypeScript/issues/49721

packages/docusaurus-mdx-loader/src/utils.ts

@@ -10,7 +10,7 @@ import {escapePath, type WebpackCompilerName} from '@docusaurus/utils';
 import {getProcessor, type SimpleProcessorResult} from './processor';
 import {validateMDXFrontMatter} from './frontMatter';
 import preprocessor from './preprocessor';
-import type {Options} from './loader';
+import type {Options} from './options';
 
 /**
  * Converts assets an object with Webpack require calls code.

packages/docusaurus-plugin-content-blog/src/__tests__/index.test.ts

@@ -13,6 +13,7 @@ import {
   getFileCommitDate,
   LAST_UPDATE_FALLBACK,
 } from '@docusaurus/utils';
+import {DEFAULT_FUTURE_CONFIG} from '@docusaurus/core/src/server/configValidation';
 import pluginContentBlog from '../index';
 import {validateOptions} from '../options';
 import type {
@@ -106,7 +107,7 @@ const getPlugin = async (
     baseUrl: '/',
     url: 'https://docusaurus.io',
     markdown,
-    future: {},
+    future: DEFAULT_FUTURE_CONFIG,
     staticDirectories: ['static'],
   } as DocusaurusConfig;
   return pluginContentBlog(

packages/docusaurus-plugin-content-blog/src/index.ts

@@ -21,10 +21,7 @@ import {
   resolveMarkdownLinkPathname,
 } from '@docusaurus/utils';
 import {getTagsFilePathsToWatch} from '@docusaurus/utils-validation';
-import {
-  createMDXLoaderItem,
-  type Options as MDXLoaderOptions,
-} from '@docusaurus/mdx-loader';
+import {createMDXLoaderItem} from '@docusaurus/mdx-loader';
 import {
   getBlogTags,
   paginateBlogPosts,
@@ -114,7 +111,9 @@ export default async function pluginContentBlog(
 
     const contentDirs = getContentPathList(contentPaths);
 
-    const loaderOptions: MDXLoaderOptions = {
+    const mdxLoaderItem = await createMDXLoaderItem({
+      useCrossCompilerCache:
+        siteConfig.future.experimental_faster.mdxCrossCompilerCache,
       admonitions,
       remarkPlugins,
       rehypePlugins,
@@ -168,7 +167,7 @@ export default async function pluginContentBlog(
         }
         return permalink;
       },
-    };
+    });
 
     function createBlogMarkdownLoader(): RuleSetUseItem {
       const markdownLoaderOptions: BlogMarkdownLoaderOptions = {
@@ -185,10 +184,7 @@ export default async function pluginContentBlog(
       include: contentDirs
         // Trailing slash is important, see https://github.com/facebook/docusaurus/pull/3970
         .map(addTrailingPathSeparator),
-      use: [
-        await createMDXLoaderItem(loaderOptions),
-        createBlogMarkdownLoader(),
-      ],
+      use: [mdxLoaderItem, createBlogMarkdownLoader()],
     };
   }