tweak tags handling a bit more

packages/next/src/lib/constants.ts

@@ -7,6 +7,7 @@ export const PRERENDER_REVALIDATE_ONLY_GENERATED_HEADER =
   'x-prerender-revalidate-if-generated'
 
 export const NEXT_CACHE_TAGS_HEADER = 'x-next-cache-tags'
+export const NEXT_CACHE_SOFT_TAGS_HEADER = 'x-next-cache-soft-tags'
 export const NEXT_CACHE_REVALIDATED_TAGS_HEADER = 'x-next-revalidated-tags'
 export const NEXT_CACHE_REVALIDATE_TAG_TOKEN_HEADER =
   'x-next-revalidate-tag-token'

packages/next/src/server/lib/incremental-cache/fetch-cache.ts

@@ -1,7 +1,10 @@
 import type { CacheHandler, CacheHandlerContext, CacheHandlerValue } from './'
 
 import LRUCache from 'next/dist/compiled/lru-cache'
-import { CACHE_ONE_YEAR, NEXT_CACHE_TAGS_HEADER } from '../../../lib/constants'
+import {
+  CACHE_ONE_YEAR,
+  NEXT_CACHE_SOFT_TAGS_HEADER,
+} from '../../../lib/constants'
 
 let rateLimitedUntil = 0
 let memoryCache: LRUCache<string, CacheHandlerValue> | undefined
@@ -13,6 +16,7 @@ interface NextFetchCacheParams {
   fetchUrl?: string
 }
 
+const CACHE_TAGS_HEADER = 'x-vercel-cache-tags' as const
 const CACHE_HEADERS_HEADER = 'x-vercel-sc-headers' as const
 const CACHE_STATE_HEADER = 'x-vercel-cache-state' as const
 const CACHE_VERSION_HEADER = 'x-data-cache-version' as const
@@ -24,7 +28,6 @@ export default class FetchCache implements CacheHandler {
   private headers: Record<string, string>
   private cacheEndpoint?: string
   private debug: boolean
-  private revalidatedTags: string[]
 
   static isAvailable(ctx: {
     _requestHeaders: CacheHandlerContext['_requestHeaders']
@@ -37,7 +40,6 @@ export default class FetchCache implements CacheHandler {
   constructor(ctx: CacheHandlerContext) {
     this.debug = !!process.env.NEXT_PRIVATE_DEBUG_CACHE
     this.headers = {}
-    this.revalidatedTags = ctx.revalidatedTags
     this.headers[CACHE_VERSION_HEADER] = '2'
     this.headers['Content-Type'] = 'application/json'
 
@@ -144,18 +146,16 @@ export default class FetchCache implements CacheHandler {
 
   public async get(
     key: string,
-    {
-      fetchCache,
-      fetchIdx,
-      fetchUrl,
-      tags,
-    }: {
+    ctx: {
+      tags?: string[]
+      softTags?: string[]
       fetchCache?: boolean
       fetchUrl?: string
       fetchIdx?: number
-      tags?: string[]
     }
   ) {
+    const { tags, softTags, fetchCache, fetchIdx, fetchUrl } = ctx
+
     if (!fetchCache) return null
 
     if (Date.now() < rateLimitedUntil) {
@@ -189,8 +189,9 @@ export default class FetchCache implements CacheHandler {
             method: 'GET',
             headers: {
               ...this.headers,
-              [NEXT_CACHE_TAGS_HEADER]: tags?.join(','),
               [CACHE_FETCH_URL_HEADER]: fetchUrl,
+              [CACHE_TAGS_HEADER]: tags?.join(',') || '',
+              [NEXT_CACHE_SOFT_TAGS_HEADER]: softTags?.join(',') || '',
             } as any,
             next: fetchParams as NextFetchRequestConfig,
           }
@@ -236,13 +237,16 @@ export default class FetchCache implements CacheHandler {
               ? Date.now() - CACHE_ONE_YEAR
               : Date.now() - parseInt(age || '0', 10) * 1000,
         }
+
         if (this.debug) {
           console.log(
             `got fetch cache entry for ${key}, duration: ${
               Date.now() - start
             }ms, size: ${
               Object.keys(cached).length
-            }, cache-state: ${cacheState}`
+            }, cache-state: ${cacheState} tags: ${tags?.join(
+              ','
+            )} softTags: ${softTags?.join(',')}`
           )
         }
 
@@ -267,7 +271,9 @@ export default class FetchCache implements CacheHandler {
       fetchCache,
       fetchIdx,
       fetchUrl,
+      tags,
     }: {
+      tags?: string[]
       fetchCache?: boolean
       fetchUrl?: string
       fetchIdx?: number
@@ -301,7 +307,12 @@ export default class FetchCache implements CacheHandler {
           this.headers[CACHE_CONTROL_VALUE_HEADER] =
             data.data.headers['cache-control']
         }
-        const body = JSON.stringify(data)
+        const body = JSON.stringify({
+          ...data,
+          // we send the tags in the header instead
+          // of in the body here
+          tags: undefined,
+        })
 
         if (this.debug) {
           console.log('set cache', key)
@@ -318,7 +329,8 @@ export default class FetchCache implements CacheHandler {
             method: 'POST',
             headers: {
               ...this.headers,
-              '': fetchUrl || '',
+              [CACHE_FETCH_URL_HEADER]: fetchUrl || '',
+              [CACHE_TAGS_HEADER]: tags?.join(',') || '',
             },
             body: body,
             next: fetchParams as NextFetchRequestConfig,

packages/next/src/server/lib/incremental-cache/file-system-cache.ts

@@ -109,11 +109,13 @@ export default class FileSystemCache implements CacheHandler {
   public async get(
     key: string,
     {
-      fetchCache,
       tags,
+      softTags,
+      fetchCache,
     }: {
-      fetchCache?: boolean
       tags?: string[]
+      softTags?: string[]
+      fetchCache?: boolean
     } = {}
   ) {
     let data = memoryCache?.get(key)
@@ -163,6 +165,17 @@ export default class FileSystemCache implements CacheHandler {
             lastModified,
             value: parsedData,
           }
+
+          if (data.value?.kind === 'FETCH') {
+            const storedTags = data.value?.data?.tags
+
+            // update stored tags if a new one is being added
+            // TODO: remove this when we can send the tags
+            // via header on GET same as SET
+            if (!tags?.every((tag) => storedTags?.includes(tag))) {
+              await this.set(key, data.value, { tags })
+            }
+          }
         } else {
           const pageData = isAppPath
             ? (
@@ -251,7 +264,9 @@ export default class FileSystemCache implements CacheHandler {
     if (data && data?.value?.kind === 'FETCH') {
       this.loadTagsManifest()
 
-      const wasRevalidated = tags?.some((tag) => {
+      const combinedTags = [...(tags || []), ...(softTags || [])]
+
+      const wasRevalidated = combinedTags.some((tag) => {
         if (this.revalidatedTags.includes(tag)) {
           return true
         }
@@ -272,7 +287,13 @@ export default class FileSystemCache implements CacheHandler {
     return data || null
   }
 
-  public async set(key: string, data: CacheHandlerValue['value']) {
+  public async set(
+    key: string,
+    data: CacheHandlerValue['value'],
+    ctx: {
+      tags?: string[]
+    }
+  ) {
     memoryCache?.set(key, {
       value: data,
       lastModified: Date.now(),
@@ -327,7 +348,13 @@ export default class FileSystemCache implements CacheHandler {
         fetchCache: true,
       })
       await this.fs.mkdir(path.dirname(filePath))
-      await this.fs.writeFile(filePath, JSON.stringify(data))
+      await this.fs.writeFile(
+        filePath,
+        JSON.stringify({
+          ...data,
+          tags: ctx.tags,
+        })
+      )
     }
   }
 

packages/next/src/server/lib/incremental-cache/index.ts

@@ -397,6 +397,7 @@ export class IncrementalCache {
       fetchUrl?: string
       fetchIdx?: number
       tags?: string[]
+      softTags?: string[]
     } = {}
   ): Promise<IncrementalCacheEntry | null> {
     if (
@@ -431,9 +432,10 @@ export class IncrementalCache {
     const cacheData = await this.cacheHandler?.get(cacheKey, ctx)
 
     if (cacheData?.value?.kind === 'FETCH') {
+      const combinedTags = [...(ctx.tags || []), ...(ctx.softTags || [])]
       // if a tag was revalidated we don't return stale data
       if (
-        ctx.tags?.some((tag) => {
+        combinedTags.some((tag) => {
           return this.revalidatedTags?.includes(tag)
         })
       ) {
@@ -518,6 +520,7 @@ export class IncrementalCache {
       fetchCache?: boolean
       fetchUrl?: string
       fetchIdx?: number
+      tags?: string[]
     }
   ) {
     if (

packages/next/src/server/lib/patch-fetch.ts

@@ -203,11 +203,6 @@ export function patchFetch({
         }
         const implicitTags = addImplicitTags(staticGenerationStore)
 
-        for (const tag of implicitTags || []) {
-          if (!tags.includes(tag)) {
-            tags.push(tag)
-          }
-        }
         const isOnlyCache = staticGenerationStore.fetchCache === 'only-cache'
         const isForceCache = staticGenerationStore.fetchCache === 'force-cache'
         const isDefaultCache =
@@ -435,6 +430,7 @@ export function patchFetch({
                     revalidate,
                     fetchUrl,
                     fetchIdx,
+                    tags,
                   }
                 )
               } catch (err) {
@@ -468,6 +464,7 @@ export function patchFetch({
                 fetchUrl,
                 fetchIdx,
                 tags,
+                softTags: implicitTags,
               })
 
           if (entry) {

packages/next/src/server/response-cache/types.ts

@@ -20,6 +20,9 @@ export interface CachedFetchValue {
     body: string
     url: string
     status?: number
+    // tags are only present with file-system-cache
+    // fetch cache stores tags outside of cache entry
+    tags?: string[]
   }
   revalidate: number
 }

packages/next/src/server/web/spec-extension/unstable-cache.ts

@@ -19,26 +19,27 @@ export function unstable_cache<T extends Callback>(
   const staticGenerationAsyncStorage: StaticGenerationAsyncStorage =
     (fetch as any).__nextGetStaticStore?.() || _staticGenerationAsyncStorage
 
-  const store: undefined | StaticGenerationStore =
-    staticGenerationAsyncStorage?.getStore()
-
-  const incrementalCache:
-    | import('../../lib/incremental-cache').IncrementalCache
-    | undefined =
-    store?.incrementalCache || (globalThis as any).__incrementalCache
-
-  if (!incrementalCache) {
-    throw new Error(
-      `Invariant: incrementalCache missing in unstable_cache ${cb.toString()}`
-    )
-  }
   if (options.revalidate === 0) {
     throw new Error(
       `Invariant revalidate: 0 can not be passed to unstable_cache(), must be "false" or "> 0" ${cb.toString()}`
     )
   }
 
   const cachedCb = async (...args: any[]) => {
+    const store: undefined | StaticGenerationStore =
+      staticGenerationAsyncStorage?.getStore()
+
+    const incrementalCache:
+      | import('../../lib/incremental-cache').IncrementalCache
+      | undefined =
+      store?.incrementalCache || (globalThis as any).__incrementalCache
+
+    if (!incrementalCache) {
+      throw new Error(
+        `Invariant: incrementalCache missing in unstable_cache ${cb.toString()}`
+      )
+    }
+
     const joinedKey = `${cb.toString()}-${
       Array.isArray(keyParts) && keyParts.join(',')
     }-${JSON.stringify(args)}`
@@ -69,12 +70,6 @@ export function unstable_cache<T extends Callback>(
         }
         const implicitTags = addImplicitTags(store)
 
-        for (const tag of implicitTags) {
-          if (!tags.includes(tag)) {
-            tags.push(tag)
-          }
-        }
-
         const cacheKey = await incrementalCache?.fetchCacheKey(joinedKey)
         const cacheEntry =
           cacheKey &&
@@ -85,6 +80,7 @@ export function unstable_cache<T extends Callback>(
             fetchCache: true,
             revalidate: options.revalidate,
             tags,
+            softTags: implicitTags,
           }))
 
         const invokeCallback = async () => {
@@ -110,6 +106,7 @@ export function unstable_cache<T extends Callback>(
               {
                 revalidate: options.revalidate,
                 fetchCache: true,
+                tags,
               }
             )
           }

test/unit/incremental-cache/file-system-cache.test.ts

@@ -20,14 +20,18 @@ describe('FileSystemCache', () => {
       fileURLToPath(new URL('./images/icon.png', import.meta.url))
     )
 
-    await fsCache.set('icon.png', {
-      body: binary,
-      headers: {
-        'Content-Type': 'image/png',
+    await fsCache.set(
+      'icon.png',
+      {
+        body: binary,
+        headers: {
+          'Content-Type': 'image/png',
+        },
+        status: 200,
+        kind: 'ROUTE',
       },
-      status: 200,
-      kind: 'ROUTE',
-    })
+      {}
+    )
 
     expect((await fsCache.get('icon.png')).value).toEqual({
       body: binary,