Update tag handling for app cache (#53321)

Optimizes how we handle cache tags for soft tags (auto-added by Next.js) and normal tags (added manually) and adds differentiating between `revalidatePath('/blog/first')` and page/layout. Soft tags are not stored across cache entry and instead auto sent along when checking cache entries. This allows us to prevent storing exponential amounts of tags across cache entries while still having the relationship between them so that single path revalidation can work properly. x-ref: [slack thread](https://vercel.slack.com/archives/C042LHPJ1NX/p1690586837903309)
vercel · Aug 31, 2023 · 2eef775 · 2eef775
1 parent acbbf14
commit 2eef775
Show file tree

Hide file tree

Showing 25 changed files with 378 additions and 338 deletions.
diff --git a/packages/next/src/build/utils.ts b/packages/next/src/build/utils.ts
@@ -1241,7 +1241,7 @@ export async function buildAppStaticPaths({
   return StaticGenerationAsyncStorageWrapper.wrap(
     staticGenerationAsyncStorage,
     {
-      pathname: page,
+      urlPathname: page,
       renderOpts: {
         originalPathname: page,
         incrementalCache,

diff --git a/packages/next/src/client/components/app-router-headers.ts b/packages/next/src/client/components/app-router-headers.ts
@@ -4,7 +4,6 @@ export const ACTION = 'Next-Action' as const
 export const NEXT_ROUTER_STATE_TREE = 'Next-Router-State-Tree' as const
 export const NEXT_ROUTER_PREFETCH = 'Next-Router-Prefetch' as const
 export const NEXT_URL = 'Next-Url' as const
-export const FETCH_CACHE_HEADER = 'x-vercel-sc-headers' as const
 export const RSC_CONTENT_TYPE_HEADER = 'text/x-component' as const
 export const RSC_VARY_HEADER =
   `${RSC}, ${NEXT_ROUTER_STATE_TREE}, ${NEXT_ROUTER_PREFETCH}, ${NEXT_URL}` as const

diff --git a/packages/next/src/client/components/static-generation-async-storage.ts b/packages/next/src/client/components/static-generation-async-storage.ts
@@ -5,8 +5,8 @@ import { createAsyncLocalStorage } from './async-local-storage'
 
 export interface StaticGenerationStore {
   readonly isStaticGeneration: boolean
-  readonly pathname: string
-  readonly originalPathname?: string
+  readonly pagePath?: string
+  readonly urlPathname: string
   readonly incrementalCache?: IncrementalCache
   readonly isOnDemandRevalidate?: boolean
   readonly isPrerendering?: boolean

diff --git a/packages/next/src/export/worker.ts b/packages/next/src/export/worker.ts
@@ -25,7 +25,10 @@ import { isDynamicRoute } from '../shared/lib/router/utils/is-dynamic'
 import { getRouteMatcher } from '../shared/lib/router/utils/route-matcher'
 import { getRouteRegex } from '../shared/lib/router/utils/route-regex'
 import { normalizePagePath } from '../shared/lib/page-path/normalize-page-path'
-import { SERVER_PROPS_EXPORT_ERROR } from '../lib/constants'
+import {
+  NEXT_CACHE_TAGS_HEADER,
+  SERVER_PROPS_EXPORT_ERROR,
+} from '../lib/constants'
 import { requireFontManifest } from '../server/require'
 import { normalizeLocalePath } from '../shared/lib/i18n/normalize-locale-path'
 import { trace } from '../trace'
@@ -499,7 +502,7 @@ export default async function exportPage({
                 .fetchTags
 
               if (cacheTags) {
-                headers['x-next-cache-tags'] = cacheTags
+                headers[NEXT_CACHE_TAGS_HEADER] = cacheTags
               }
 
               if (!headers['content-type'] && body.type) {
@@ -554,7 +557,7 @@ export default async function exportPage({
               const cacheTags = (curRenderOpts as any).fetchTags
               const headers = cacheTags
                 ? {
-                    'x-next-cache-tags': cacheTags,
+                    [NEXT_CACHE_TAGS_HEADER]: cacheTags,
                   }
                 : undefined
 

diff --git a/packages/next/src/lib/constants.ts b/packages/next/src/lib/constants.ts
@@ -6,6 +6,14 @@ export const PRERENDER_REVALIDATE_HEADER = 'x-prerender-revalidate'
 export const PRERENDER_REVALIDATE_ONLY_GENERATED_HEADER =
   'x-prerender-revalidate-if-generated'
 
+export const NEXT_CACHE_TAGS_HEADER = 'x-next-cache-tags'
+export const NEXT_CACHE_SOFT_TAGS_HEADER = 'x-next-cache-soft-tags'
+export const NEXT_CACHE_REVALIDATED_TAGS_HEADER = 'x-next-revalidated-tags'
+export const NEXT_CACHE_REVALIDATE_TAG_TOKEN_HEADER =
+  'x-next-revalidate-tag-token'
+
+export const NEXT_CACHE_IMPLICIT_TAG_ID = '_N_T_'
+
 // in seconds
 export const CACHE_ONE_YEAR = 31536000
 

diff --git a/packages/next/src/server/app-render/action-handler.ts b/packages/next/src/server/app-render/action-handler.ts
@@ -32,6 +32,10 @@ import {
   getModifiedCookieValues,
 } from '../web/spec-extension/adapters/request-cookies'
 import { RequestStore } from '../../client/components/request-async-storage'
+import {
+  NEXT_CACHE_REVALIDATED_TAGS_HEADER,
+  NEXT_CACHE_REVALIDATE_TAG_TOKEN_HEADER,
+} from '../../lib/constants'
 
 function nodeToWebReadableStream(nodeReadable: import('stream').Readable) {
   if (process.env.NEXT_RUNTIME !== 'edge') {
@@ -178,11 +182,11 @@ async function createRedirectRenderResult(
 
     if (staticGenerationStore.revalidatedTags) {
       forwardedHeaders.set(
-        'x-next-revalidated-tags',
+        NEXT_CACHE_REVALIDATED_TAGS_HEADER,
         staticGenerationStore.revalidatedTags.join(',')
       )
       forwardedHeaders.set(
-        'x-next-revalidate-tag-token',
+        NEXT_CACHE_REVALIDATE_TAG_TOKEN_HEADER,
         staticGenerationStore.incrementalCache?.prerenderManifest?.preview
           ?.previewModeId || ''
       )

diff --git a/packages/next/src/server/app-render/app-render.tsx b/packages/next/src/server/app-render/app-render.tsx
@@ -1786,7 +1786,7 @@ export async function renderToHTMLOrFlight(
     () =>
       StaticGenerationAsyncStorageWrapper.wrap(
         staticGenerationAsyncStorage,
-        { pathname: pagePath, renderOpts },
+        { urlPathname: pathname, renderOpts },
         () => wrappedRender()
       )
   )

diff --git a/packages/next/src/server/async-storage/static-generation-async-storage-wrapper.ts b/packages/next/src/server/async-storage/static-generation-async-storage-wrapper.ts
@@ -4,7 +4,7 @@ import type { AsyncLocalStorage } from 'async_hooks'
 import type { IncrementalCache } from '../lib/incremental-cache'
 
 export type StaticGenerationContext = {
-  pathname: string
+  urlPathname: string
   renderOpts: {
     originalPathname?: string
     incrementalCache?: IncrementalCache
@@ -34,7 +34,7 @@ export const StaticGenerationAsyncStorageWrapper: AsyncStorageWrapper<
 > = {
   wrap<Result>(
     storage: AsyncLocalStorage<StaticGenerationStore>,
-    { pathname, renderOpts }: StaticGenerationContext,
+    { urlPathname, renderOpts }: StaticGenerationContext,
     callback: (store: StaticGenerationStore) => Result
   ): Result {
     /**
@@ -57,8 +57,8 @@ export const StaticGenerationAsyncStorageWrapper: AsyncStorageWrapper<
 
     const store: StaticGenerationStore = {
       isStaticGeneration,
-      pathname,
-      originalPathname: renderOpts.originalPathname,
+      urlPathname,
+      pagePath: renderOpts.originalPathname,
       incrementalCache:
         // we fallback to a global incremental cache for edge-runtime locally
         // so that it can access the fs cache without mocks

diff --git a/packages/next/src/server/base-server.ts b/packages/next/src/server/base-server.ts
@@ -113,7 +113,10 @@ import {
   fromNodeOutgoingHttpHeaders,
   toNodeOutgoingHttpHeaders,
 } from './web/utils'
-import { NEXT_QUERY_PARAM_PREFIX } from '../lib/constants'
+import {
+  NEXT_CACHE_TAGS_HEADER,
+  NEXT_QUERY_PARAM_PREFIX,
+} from '../lib/constants'
 import { normalizeLocalePath } from '../shared/lib/i18n/normalize-locale-path'
 import {
   NextRequestAdapter,
@@ -1997,7 +2000,7 @@ export default abstract class Server<ServerOptions extends Options = Options> {
             headers = toNodeOutgoingHttpHeaders(response.headers)
 
             if (cacheTags) {
-              headers['x-next-cache-tags'] = cacheTags
+              headers[NEXT_CACHE_TAGS_HEADER] = cacheTags
             }
 
             if (!headers['content-type'] && blob.type) {
@@ -2116,7 +2119,7 @@ export default abstract class Server<ServerOptions extends Options = Options> {
       const cacheTags = (renderOpts as any).fetchTags
       if (cacheTags) {
         headers = {
-          'x-next-cache-tags': cacheTags,
+          [NEXT_CACHE_TAGS_HEADER]: cacheTags,
         }
       }
 
@@ -2407,7 +2410,7 @@ export default abstract class Server<ServerOptions extends Options = Options> {
       const headers = { ...cachedData.headers }
 
       if (!(this.minimalMode && isSSG)) {
-        delete headers['x-next-cache-tags']
+        delete headers[NEXT_CACHE_TAGS_HEADER]
       }
 
       await sendResponse(
@@ -2424,11 +2427,11 @@ export default abstract class Server<ServerOptions extends Options = Options> {
         if (
           this.minimalMode &&
           isSSG &&
-          cachedData.headers?.['x-next-cache-tags']
+          cachedData.headers?.[NEXT_CACHE_TAGS_HEADER]
         ) {
           res.setHeader(
-            'x-next-cache-tags',
-            cachedData.headers['x-next-cache-tags'] as string
+            NEXT_CACHE_TAGS_HEADER,
+            cachedData.headers[NEXT_CACHE_TAGS_HEADER] as string
           )
         }
         if (isDataReq && typeof cachedData.pageData !== 'string') {

diff --git a/packages/next/src/server/future/route-modules/app-route/module.ts b/packages/next/src/server/future/route-modules/app-route/module.ts
@@ -264,12 +264,13 @@ export class AppRouteRouteModule extends RouteModule<
 
     // Get the context for the static generation.
     const staticGenerationContext: StaticGenerationContext = {
-      pathname: this.definition.pathname,
+      urlPathname: request.nextUrl.pathname,
       renderOpts:
         // If the staticGenerationContext is not provided then we default to
         // the default values.
         context.staticGenerationContext ?? {
           supportsDynamicHTML: false,
+          originalPathname: this.definition.pathname,
         },
     }
 

diff --git a/packages/next/src/server/image-optimizer.ts b/packages/next/src/server/image-optimizer.ts
@@ -321,7 +321,11 @@ export class ImageOptimizerCache {
   async set(
     cacheKey: string,
     value: IncrementalCacheValue | null,
-    revalidate?: number | false
+    {
+      revalidate,
+    }: {
+      revalidate?: number | false
+    }
   ) {
     if (value?.kind !== 'IMAGE') {
       throw new Error('invariant attempted to set non-image to image-cache')