Support static file robots.txt and sitemap.xml as metadata route (#46963

)

Support top level static `robots.txt` and `sitemap.xml` as metadata
route in app directory. When those files are placed in top root
directory

Refactored a bit the page files matching logic, to reuse it between dev
server and build

Closes NEXT-267

packages/next/src/build/index.ts

@@ -132,8 +132,9 @@ import {
 import { webpackBuild } from './webpack-build'
 import { NextBuildContext } from './build-context'
 import { normalizePathSep } from '../shared/lib/page-path/normalize-path-sep'
-import { isAppRouteRoute } from '../lib/is-app-route-route'
+import { isAppRouteRoute, isMetadataRoute } from '../lib/is-app-route-route'
 import { createClientRouterFilter } from '../lib/create-client-router-filter'
+import { createValidFileMatcher } from '../server/lib/find-page-file'
 
 export type SsgRoute = {
   initialRevalidateSeconds: number | false
@@ -491,15 +492,17 @@ export default async function build(
 
       NextBuildContext.buildSpinner = buildSpinner
 
+      const validFileMatcher = createValidFileMatcher(
+        config.pageExtensions,
+        appDir
+      )
+
       const pagesPaths =
         !appDirOnly && pagesDir
           ? await nextBuildSpan
               .traceChild('collect-pages')
               .traceAsyncFn(() =>
-                recursiveReadDir(
-                  pagesDir,
-                  new RegExp(`\\.(?:${config.pageExtensions.join('|')})$`)
-                )
+                recursiveReadDir(pagesDir, validFileMatcher.isPageFile)
               )
           : []
 
@@ -509,12 +512,7 @@ export default async function build(
         appPaths = await nextBuildSpan
           .traceChild('collect-app-paths')
           .traceAsyncFn(() =>
-            recursiveReadDir(
-              appDir,
-              new RegExp(
-                `^(page|route)\\.(?:${config.pageExtensions.join('|')})$`
-              )
-            )
+            recursiveReadDir(appDir, validFileMatcher.isAppRouterPage)
           )
       }
 
@@ -2442,7 +2440,9 @@ export default async function build(
               appConfig.revalidate === 0 ||
               exportConfig.initialPageRevalidationMap[page] === 0
 
-            const isRouteHandler = isAppRouteRoute(originalAppPath)
+            const isRouteHandler =
+              isAppRouteRoute(originalAppPath) ||
+              isMetadataRoute(originalAppPath)
 
             routes.forEach((route) => {
               if (isDynamicRoute(page) && route === page) return

packages/next/src/build/webpack-config.ts

@@ -1708,6 +1708,7 @@ export default async function getBaseWebpackConfig(
         'next-app-loader',
         'next-font-loader',
         'next-invalid-import-error-loader',
+        'next-metadata-route-loader',
       ].reduce((alias, loader) => {
         // using multiple aliases to replace `resolveLoader.modules`
         alias[loader] = path.join(__dirname, 'webpack', 'loaders', loader)

packages/next/src/build/webpack/loaders/next-app-loader.ts

@@ -10,9 +10,26 @@ import { verifyRootLayout } from '../../../lib/verifyRootLayout'
 import * as Log from '../../../build/output/log'
 import { APP_DIR_ALIAS } from '../../../lib/constants'
 import { buildMetadata, discoverStaticMetadataFiles } from './metadata/discover'
+import {
+  isAppRouteRoute,
+  isMetadataRoute,
+} from '../../../lib/is-app-route-route'
+
+export type AppLoaderOptions = {
+  name: string
+  pagePath: string
+  appDir: string
+  appPaths: string[] | null
+  pageExtensions: string[]
+  basePath: string
+  assetPrefix: string
+  rootDir?: string
+  tsconfigPath?: string
+  isDev?: boolean
+}
+type AppLoader = webpack.LoaderDefinitionFunction<AppLoaderOptions>
 
 const isNotResolvedError = (err: any) => err.message.includes("Can't resolve")
-import { isAppRouteRoute } from '../../../lib/is-app-route-route'
 
 const FILE_TYPES = {
   layout: 'layout',
@@ -39,21 +56,29 @@ export type ComponentsType = {
 }
 
 async function createAppRouteCode({
+  name,
   pagePath,
   resolver,
 }: {
+  name: string
   pagePath: string
   resolver: PathResolver
 }): Promise<string> {
   // Split based on any specific path separators (both `/` and `\`)...
+  const routeName = name.split('/').pop()!
   const splittedPath = pagePath.split(/[\\/]/)
   // Then join all but the last part with the same separator, `/`...
   const segmentPath = splittedPath.slice(0, -1).join('/')
   // Then add the `/route` suffix...
-  const matchedPagePath = `${segmentPath}/route`
+  const matchedPagePath = `${segmentPath}/${routeName}`
+
   // This, when used with the resolver will give us the pathname to the built
   // route handler file.
-  const resolvedPagePath = await resolver(matchedPagePath)
+  let resolvedPagePath = (await resolver(matchedPagePath))!
+
+  if (isMetadataRoute(name)) {
+    resolvedPagePath = `next-metadata-route-loader!${resolvedPagePath}`
+  }
 
   // TODO: verify if other methods need to be injected
   // TODO: validate that the handler exports at least one of the supported methods
@@ -249,20 +274,6 @@ function createAbsolutePath(appDir: string, pathToTurnAbsolute: string) {
   )
 }
 
-export type AppLoaderOptions = {
-  name: string
-  pagePath: string
-  appDir: string
-  appPaths: string[] | null
-  pageExtensions: string[]
-  basePath: string
-  assetPrefix: string
-  rootDir?: string
-  tsconfigPath?: string
-  isDev?: boolean
-}
-type AppLoader = webpack.LoaderDefinitionFunction<AppLoaderOptions>
-
 const nextAppLoader: AppLoader = async function nextAppLoader() {
   const loaderOptions = this.getOptions() || {}
   const {
@@ -283,10 +294,12 @@ const nextAppLoader: AppLoader = async function nextAppLoader() {
   }
 
   const extensions = pageExtensions.map((extension) => `.${extension}`)
+
   const resolveOptions: any = {
     ...NODE_RESOLVE_OPTIONS,
     extensions,
   }
+
   const resolve = this.getResolve(resolveOptions)
 
   const normalizedAppPaths =
@@ -344,8 +357,8 @@ const nextAppLoader: AppLoader = async function nextAppLoader() {
     }
   }
 
-  if (isAppRouteRoute(name)) {
-    return createAppRouteCode({ pagePath, resolver })
+  if (isAppRouteRoute(name) || isMetadataRoute(name)) {
+    return createAppRouteCode({ name, pagePath, resolver })
   }
 
   const {

packages/next/src/build/webpack/loaders/next-metadata-route-loader.ts

@@ -0,0 +1,45 @@
+import type webpack from 'webpack'
+import path from 'path'
+
+const staticFileRegex = /[\\/](robots\.txt|sitemap\.xml)/
+
+function isStaticRoute(resourcePath: string) {
+  return staticFileRegex.test(resourcePath)
+}
+
+function getContentType(resourcePath: string) {
+  const filename = path.basename(resourcePath)
+  const [name] = filename.split('.')
+  if (name === 'sitemap') return 'application/xml'
+  if (name === 'robots') return 'text/plain'
+  return 'text/plain'
+}
+
+const nextMetadataRouterLoader: webpack.LoaderDefinitionFunction = function (
+  content: string
+) {
+  const { resourcePath } = this
+
+  const code = isStaticRoute(resourcePath)
+    ? `import { NextResponse } from 'next/server'
+
+const content = ${JSON.stringify(content)}
+const contentType = ${JSON.stringify(getContentType(resourcePath))}
+export function GET() {
+  return new NextResponse(content, {
+    status: 200,
+    headers: {
+      'Content-Type': contentType,
+    },
+  })
+}
+
+export const dynamic = 'force-static'
+`
+    : // TODO: handle the defined configs in routes file
+      `export { default as GET } from ${JSON.stringify(resourcePath)}`
+
+  return code
+}
+
+export default nextMetadataRouterLoader

packages/next/src/build/webpack/plugins/flight-client-entry-plugin.ts

@@ -31,7 +31,10 @@ import {
 } from '../loaders/utils'
 import { traverseModules } from '../utils'
 import { normalizePathSep } from '../../../shared/lib/page-path/normalize-path-sep'
-import { isAppRouteRoute } from '../../../lib/is-app-route-route'
+import {
+  isAppRouteRoute,
+  isMetadataRoute,
+} from '../../../lib/is-app-route-route'
 import { getProxiedPluginState } from '../../build-context'
 
 interface Options {
@@ -181,7 +184,8 @@ export class FlightClientEntryPlugin {
         if (
           name.startsWith('pages/') ||
           // Skip for route.js entries
-          (name.startsWith('app/') && isAppRouteRoute(name))
+          (name.startsWith('app/') &&
+            (isAppRouteRoute(name) || isMetadataRoute(name)))
         ) {
           continue
         }

packages/next/src/export/worker.ts

@@ -40,7 +40,7 @@ import { mockRequest } from '../server/lib/mock-request'
 import { RouteKind } from '../server/future/route-kind'
 import { NodeNextRequest, NodeNextResponse } from '../server/base-http/node'
 import { StaticGenerationContext } from '../server/future/route-handlers/app-route-route-handler'
-import { isAppRouteRoute } from '../lib/is-app-route-route'
+import { isAppRouteRoute, isMetadataRoute } from '../lib/is-app-route-route'
 
 loadRequireHook()
 
@@ -167,7 +167,8 @@ export default async function exportPage({
       let renderAmpPath = ampPath
       let query = { ...originalQuery }
       let params: { [key: string]: string | string[] } | undefined
-      const isRouteHandler = isAppDir && isAppRouteRoute(page)
+      const isRouteHandler =
+        isAppDir && (isAppRouteRoute(page) || isMetadataRoute(page))
 
       if (isAppDir) {
         outDir = join(distDir, 'server/app')

packages/next/src/lib/is-app-route-route.ts

@@ -1,3 +1,11 @@
 export function isAppRouteRoute(route: string): boolean {
   return route.endsWith('/route')
 }
+
+// TODO: support more metadata routes
+const staticMetadataRoutes = ['robots.txt', 'sitemap.xml']
+export function isMetadataRoute(route: string): boolean {
+  // Remove the 'app/' or '/' prefix, only check the route name since they're only allowed in root app directory
+  const filename = route.replace(/^app\//, '').replace(/^\//, '')
+  return staticMetadataRoutes.includes(filename)
+}

packages/next/src/lib/recursive-readdir.ts

@@ -8,10 +8,10 @@ import { join } from 'path'
 export async function recursiveReadDir(
   /** Directory to read */
   dir: string,
-  /** Filter for the file name, only the name part is considered, not the full path */
-  filter: RegExp,
-  /** Filter for the file name, only the name part is considered, not the full path */
-  ignore?: RegExp,
+  /** Filter for the file path */
+  filter: (absoluteFilePath: string) => boolean,
+  /** Filter for the file path */
+  ignore?: (absoluteFilePath: string) => boolean,
   /** This doesn't have to be provided, it's used for the recursion */
   arr: string[] = [],
   /** Used to replace the initial path, only the relative path is left, it's faster than path.relative. */
@@ -22,7 +22,8 @@ export async function recursiveReadDir(
   await Promise.all(
     result.map(async (part: Dirent) => {
       const absolutePath = join(dir, part.name)
-      if (ignore && ignore.test(part.name)) return
+      const relativePath = absolutePath.replace(rootDir, '')
+      if (ignore && ignore(absolutePath)) return
 
       // readdir does not follow symbolic links
       // if part is a symbolic link, follow it using stat
@@ -37,11 +38,11 @@ export async function recursiveReadDir(
         return
       }
 
-      if (!filter.test(part.name)) {
+      if (!filter(absolutePath)) {
         return
       }
 
-      arr.push(absolutePath.replace(rootDir, ''))
+      arr.push(relativePath)
     })
   )
 

packages/next/src/lib/typescript/getTypeScriptIntent.ts

@@ -29,11 +29,13 @@ export async function getTypeScriptIntent(
   // project for the user when we detect TypeScript files. So, we need to check
   // the `pages/` directory for a TypeScript file.
   // Checking all directories is too slow, so this is a happy medium.
+  const tsFilesRegex = /.*\.(ts|tsx)$/
+  const excludedRegex = /(node_modules|.*\.d\.ts$)/
   for (const dir of intentDirs) {
     const typescriptFiles = await recursiveReadDir(
       dir,
-      /.*\.(ts|tsx)$/,
-      /(node_modules|.*\.d\.ts)/
+      (name) => tsFilesRegex.test(name),
+      (name) => excludedRegex.test(name)
     )
     if (typescriptFiles.length) {
       return { firstTimeSetup: true }

packages/next/src/server/dev/next-dev-server.ts

@@ -48,7 +48,7 @@ import { eventCliSession } from '../../telemetry/events'
 import { Telemetry } from '../../telemetry/storage'
 import { setGlobal } from '../../trace'
 import HotReloader from './hot-reloader'
-import { findPageFile, isLayoutsLeafPage } from '../lib/find-page-file'
+import { createValidFileMatcher, findPageFile } from '../lib/find-page-file'
 import { getNodeOptionsWithoutInspect } from '../lib/utils'
 import {
   UnwrapPromise,
@@ -358,8 +358,9 @@ export default class DevServer extends Server {
       return
     }
 
-    const regexPageExtension = new RegExp(
-      `\\.+(?:${this.nextConfig.pageExtensions.join('|')})$`
+    const validFileMatcher = createValidFileMatcher(
+      this.nextConfig.pageExtensions,
+      this.appDir
     )
 
     let resolved = false
@@ -474,7 +475,7 @@ export default class DevServer extends Server {
 
           if (
             meta?.accuracy === undefined ||
-            !regexPageExtension.test(fileName)
+            !validFileMatcher.isPageFile(fileName)
           ) {
             continue
           }
@@ -543,7 +544,7 @@ export default class DevServer extends Server {
           }
 
           if (isAppPath) {
-            if (!isLayoutsLeafPage(fileName, this.nextConfig.pageExtensions)) {
+            if (!validFileMatcher.isAppRouterPage(fileName)) {
               continue
             }
 

packages/next/src/server/future/route-handlers/app-route-route-handler.ts

@@ -337,8 +337,8 @@ export class AppRouteRouteHandler implements RouteHandler<AppRouteRouteMatch> {
       RequestStore,
       RequestContext
     > = new RequestAsyncStorageWrapper(),
-    private readonly staticAsyncLocalStorageWrapper = new StaticGenerationAsyncStorageWrapper(),
-    private readonly moduleLoader: ModuleLoader = new NodeModuleLoader()
+    protected readonly staticAsyncLocalStorageWrapper = new StaticGenerationAsyncStorageWrapper(),
+    protected readonly moduleLoader: ModuleLoader = new NodeModuleLoader()
   ) {}
 
   private resolve(method: string, mod: AppRouteModule): AppRouteHandlerFn {
@@ -427,12 +427,12 @@ export class AppRouteRouteHandler implements RouteHandler<AppRouteRouteMatch> {
     // TODO-APP: this is temporarily used for edge.
     request?: Request
   ): Promise<Response> {
-    // This is added by the webpack loader, we load it directly from the module.
-    const { requestAsyncStorage, staticGenerationAsyncStorage } = module
-
     // Get the handler function for the given method.
     const handle = this.resolve(req.method, module)
 
+    // This is added by the webpack loader, we load it directly from the module.
+    const { requestAsyncStorage, staticGenerationAsyncStorage } = module
+
     const requestContext: RequestContext =
       process.env.NEXT_RUNTIME === 'edge'
         ? { req, res }

packages/next/src/server/future/route-kind.ts

@@ -13,7 +13,7 @@ export const enum RouteKind {
    */
   APP_PAGE = 'APP_PAGE',
   /**
-   * `APP_ROUTE` represents all the API routes that are under `app/` with the
+   * `APP_ROUTE` represents all the API routes and metadata routes that are under `app/` with the
    * filename of `route.{j,t}s{,x}`.
    */
   APP_ROUTE = 'APP_ROUTE',