diff --git a/packages/next/src/client/app-index.tsx b/packages/next/src/client/app-index.tsx index 93cb6d996cc06..bbab03a359092 100644 --- a/packages/next/src/client/app-index.tsx +++ b/packages/next/src/client/app-index.tsx @@ -43,7 +43,7 @@ const appElement: HTMLElement | Document | null = document const encoder = new TextEncoder() -let initialServerDataBuffer: string[] | undefined = undefined +let initialServerDataBuffer: (string | Uint8Array)[] | undefined = undefined let initialServerDataWriter: ReadableStreamDefaultController | undefined = undefined let initialServerDataLoaded = false @@ -56,6 +56,7 @@ function nextServerDataCallback( | [isBootStrap: 0] | [isNotBootstrap: 1, responsePartial: string] | [isFormState: 2, formState: any] + | [isBinary: 3, responseBase64Partial: string] ): void { if (seg[0] === 0) { initialServerDataBuffer = [] @@ -70,6 +71,22 @@ function nextServerDataCallback( } } else if (seg[0] === 2) { initialFormStateData = seg[1] + } else if (seg[0] === 3) { + if (!initialServerDataBuffer) + throw new Error('Unexpected server data: missing bootstrap script.') + + // Decode the base64 string back to binary data. + const binaryString = atob(seg[1]) + const decodedChunk = new Uint8Array(binaryString.length) + for (var i = 0; i < binaryString.length; i++) { + decodedChunk[i] = binaryString.charCodeAt(i) + } + + if (initialServerDataWriter) { + initialServerDataWriter.enqueue(decodedChunk) + } else { + initialServerDataBuffer.push(decodedChunk) + } } } @@ -89,7 +106,7 @@ function isStreamErrorOrUnfinished(ctr: ReadableStreamDefaultController) { function nextServerDataRegisterWriter(ctr: ReadableStreamDefaultController) { if (initialServerDataBuffer) { initialServerDataBuffer.forEach((val) => { - ctr.enqueue(encoder.encode(val)) + ctr.enqueue(typeof val === 'string' ? encoder.encode(val) : val) }) if (initialServerDataLoaded && !initialServerDataFlushed) { if (isStreamErrorOrUnfinished(ctr)) { diff --git a/packages/next/src/server/app-render/app-render.tsx b/packages/next/src/server/app-render/app-render.tsx index 3aab7b00c4968..54462a090e24f 100644 --- a/packages/next/src/server/app-render/app-render.tsx +++ b/packages/next/src/server/app-render/app-render.tsx @@ -10,7 +10,7 @@ import type { } from './types' import type { StaticGenerationStore } from '../../client/components/static-generation-async-storage.external' import type { RequestStore } from '../../client/components/request-async-storage.external' -import type { NextParsedUrlQuery } from '../request-meta' +import { getRequestMeta, type NextParsedUrlQuery } from '../request-meta' import type { LoaderTree } from '../lib/app-dir-module' import type { AppPageModule } from '../route-modules/app-page/module' import type { ClientReferenceManifest } from '../../build/webpack/plugins/flight-manifest-plugin' @@ -37,10 +37,8 @@ import { import { canSegmentBeOverridden } from '../../client/components/match-segments' import { stripInternalQueries } from '../internal-utils' import { - NEXT_ROUTER_PREFETCH_HEADER, NEXT_ROUTER_STATE_TREE, NEXT_URL, - RSC_HEADER, } from '../../client/components/app-router-headers' import { createMetadataComponents, @@ -400,7 +398,7 @@ function createFlightDataResolver(ctx: AppRenderContext) { // Generate the flight data and as soon as it can, convert it into a string. const promise = generateFlight(ctx) .then(async (result) => ({ - flightData: await result.toUnchunkedString(true), + flightData: await result.toUnchunkedBuffer(true), })) // Otherwise if it errored, return the error. .catch((err) => ({ err })) @@ -799,12 +797,11 @@ async function renderToHTMLOrFlightImpl( query = { ...query } stripInternalQueries(query) - const isRSCRequest = req.headers[RSC_HEADER.toLowerCase()] !== undefined - - const isPrefetchRSCRequest = - isRSCRequest && - req.headers[NEXT_ROUTER_PREFETCH_HEADER.toLowerCase()] !== undefined + const isRSCRequest = Boolean(getRequestMeta(req, 'isRSCRequest')) + const isPrefetchRSCRequest = Boolean( + getRequestMeta(req, 'isPrefetchRSCRequest') + ) /** * Router state provided from the client-side router. Used to handle rendering * from the common layout down. This value will be undefined if the request diff --git a/packages/next/src/server/app-render/use-flight-response.tsx b/packages/next/src/server/app-render/use-flight-response.tsx index 6069a71aff358..86831a58bd550 100644 --- a/packages/next/src/server/app-render/use-flight-response.tsx +++ b/packages/next/src/server/app-render/use-flight-response.tsx @@ -9,6 +9,7 @@ const isEdgeRuntime = process.env.NEXT_RUNTIME === 'edge' const INLINE_FLIGHT_PAYLOAD_BOOTSTRAP = 0 const INLINE_FLIGHT_PAYLOAD_DATA = 1 const INLINE_FLIGHT_PAYLOAD_FORM_STATE = 2 +const INLINE_FLIGHT_PAYLOAD_BINARY = 3 const flightResponses = new WeakMap, Promise>() const encoder = new TextEncoder() @@ -96,10 +97,8 @@ export function createInlinedDataReadableStream( ? `` + `${scriptStart}self.__next_f.push(${htmlInlinedData})` ) ) } diff --git a/packages/next/src/server/base-server.ts b/packages/next/src/server/base-server.ts index 81c50fbeaa407..7dbd10d9fece5 100644 --- a/packages/next/src/server/base-server.ts +++ b/packages/next/src/server/base-server.ts @@ -13,6 +13,8 @@ import type { ParsedUrlQuery } from 'querystring' import type { RenderOptsPartial as PagesRenderOptsPartial } from './render' import type { RenderOptsPartial as AppRenderOptsPartial } from './app-render/types' import type { + CachedAppPageValue, + CachedPageValue, ResponseCacheBase, ResponseCacheEntry, ResponseGenerator, @@ -2599,15 +2601,28 @@ export default abstract class Server< } // We now have a valid HTML result that we can return to the user. + if (isAppPath) { + return { + value: { + kind: 'APP_PAGE', + html: result, + headers, + rscData: metadata.flightData, + postponed: metadata.postponed, + status: res.statusCode, + } satisfies CachedAppPageValue, + revalidate: metadata.revalidate, + } + } + return { value: { kind: 'PAGE', html: result, pageData: metadata.pageData ?? metadata.flightData, - postponed: metadata.postponed, headers, status: isAppPath ? res.statusCode : undefined, - }, + } satisfies CachedPageValue, revalidate: metadata.revalidate, } } @@ -2720,7 +2735,6 @@ export default abstract class Server< value: { kind: 'PAGE', html: RenderResult.fromStatic(html), - postponed: undefined, status: undefined, headers: undefined, pageData: {}, @@ -2790,7 +2804,7 @@ export default abstract class Server< } const didPostpone = - cacheEntry.value?.kind === 'PAGE' && + cacheEntry.value?.kind === 'APP_PAGE' && typeof cacheEntry.value.postponed === 'string' if ( @@ -2883,9 +2897,23 @@ export default abstract class Server< // and the revalidate options. const onCacheEntry = getRequestMeta(req, 'onCacheEntry') if (onCacheEntry) { - const finished = await onCacheEntry(cacheEntry, { - url: getRequestMeta(req, 'initURL'), - }) + const finished = await onCacheEntry( + { + ...cacheEntry, + // TODO: remove this when upstream doesn't + // always expect this value to be "PAGE" + value: { + ...cacheEntry.value, + kind: + cacheEntry.value?.kind === 'APP_PAGE' + ? 'PAGE' + : cacheEntry.value?.kind, + }, + }, + { + url: getRequestMeta(req, 'initURL'), + } + ) if (finished) { // TODO: maybe we have to end the request? return null @@ -2954,7 +2982,7 @@ export default abstract class Server< }) ) return null - } else if (isAppPath) { + } else if (cachedData.kind === 'APP_PAGE') { // If the request has a postponed state and it's a resume request we // should error. if (cachedData.postponed && minimalPostponed) { @@ -3015,7 +3043,7 @@ export default abstract class Server< // return the generated payload if (isRSCRequest && !isPreviewMode) { // If this is a dynamic RSC request, then stream the response. - if (typeof cachedData.pageData !== 'string') { + if (typeof cachedData.rscData === 'undefined') { if (cachedData.postponed) { throw new Error('Invariant: Expected postponed to be undefined') } @@ -3036,7 +3064,7 @@ export default abstract class Server< // data. return { type: 'rsc', - body: RenderResult.fromStatic(cachedData.pageData), + body: RenderResult.fromStatic(cachedData.rscData), revalidate: cacheEntry.revalidate, } } @@ -3076,7 +3104,7 @@ export default abstract class Server< throw new Error('Invariant: expected a result to be returned') } - if (result.value?.kind !== 'PAGE') { + if (result.value?.kind !== 'APP_PAGE') { throw new Error( `Invariant: expected a page response, got ${result.value?.kind}` ) diff --git a/packages/next/src/server/lib/incremental-cache/fetch-cache.ts b/packages/next/src/server/lib/incremental-cache/fetch-cache.ts index be34cf90394ee..ee0431a4f218d 100644 --- a/packages/next/src/server/lib/incremental-cache/fetch-cache.ts +++ b/packages/next/src/server/lib/incremental-cache/fetch-cache.ts @@ -109,7 +109,10 @@ export default class FetchCache implements CacheHandler { } // rough estimate of size of cache value return ( - value.html.length + (JSON.stringify(value.pageData)?.length || 0) + value.html.length + + (JSON.stringify( + value.kind === 'APP_PAGE' ? value.rscData : value.pageData + )?.length || 0) ) }, }) diff --git a/packages/next/src/server/lib/incremental-cache/file-system-cache.ts b/packages/next/src/server/lib/incremental-cache/file-system-cache.ts index 859a5030f3a62..5268fa4e66a9b 100644 --- a/packages/next/src/server/lib/incremental-cache/file-system-cache.ts +++ b/packages/next/src/server/lib/incremental-cache/file-system-cache.ts @@ -69,7 +69,10 @@ export default class FileSystemCache implements CacheHandler { } // rough estimate of size of cache value return ( - value.html.length + (JSON.stringify(value.pageData)?.length || 0) + value.html.length + + (JSON.stringify( + value.kind === 'APP_PAGE' ? value.rscData : value.pageData + )?.length || 0) ) }, }) @@ -245,23 +248,6 @@ export default class FileSystemCache implements CacheHandler { } } } else { - const pageData = isAppPath - ? await this.fs.readFile( - this.getFilePath( - `${key}${ - isRoutePPREnabled ? RSC_PREFETCH_SUFFIX : RSC_SUFFIX - }`, - 'app' - ), - 'utf8' - ) - : JSON.parse( - await this.fs.readFile( - this.getFilePath(`${key}${NEXT_DATA_SUFFIX}`, 'pages'), - 'utf8' - ) - ) - let meta: RouteMetadata | undefined if (isAppPath) { @@ -275,16 +261,42 @@ export default class FileSystemCache implements CacheHandler { } catch {} } - data = { - lastModified: mtime.getTime(), - value: { - kind: 'PAGE', - html: fileData, - pageData, - postponed: meta?.postponed, - headers: meta?.headers, - status: meta?.status, - }, + if (isAppPath) { + const rscData = await this.fs.readFile( + this.getFilePath( + `${key}${isRoutePPREnabled ? RSC_PREFETCH_SUFFIX : RSC_SUFFIX}`, + 'app' + ) + ) + data = { + lastModified: mtime.getTime(), + value: { + kind: 'APP_PAGE', + html: fileData, + rscData, + postponed: meta?.postponed, + headers: meta?.headers, + status: meta?.status, + }, + } + } else { + const pageData = JSON.parse( + await this.fs.readFile( + this.getFilePath(`${key}${NEXT_DATA_SUFFIX}`, 'pages'), + 'utf8' + ) + ) + + data = { + lastModified: mtime.getTime(), + value: { + kind: 'PAGE', + html: fileData, + pageData, + headers: meta?.headers, + status: meta?.status, + }, + } } } @@ -296,7 +308,7 @@ export default class FileSystemCache implements CacheHandler { } } - if (data?.value?.kind === 'PAGE') { + if (data?.value?.kind === 'APP_PAGE' || data?.value?.kind === 'PAGE') { let cacheTags: undefined | string[] const tagsHeader = data.value.headers?.[NEXT_CACHE_TAGS_HEADER] @@ -380,8 +392,8 @@ export default class FileSystemCache implements CacheHandler { return } - if (data?.kind === 'PAGE') { - const isAppPath = typeof data.pageData === 'string' + if (data?.kind === 'PAGE' || data?.kind === 'APP_PAGE') { + const isAppPath = data.kind === 'APP_PAGE' const htmlPath = this.getFilePath( `${key}.html`, isAppPath ? 'app' : 'pages' @@ -400,14 +412,14 @@ export default class FileSystemCache implements CacheHandler { }`, isAppPath ? 'app' : 'pages' ), - isAppPath ? data.pageData : JSON.stringify(data.pageData) + isAppPath ? data.rscData : JSON.stringify(data.pageData) ) - if (data.headers || data.status) { + if (data.headers || data.status || (isAppPath && data.postponed)) { const meta: RouteMetadata = { headers: data.headers, status: data.status, - postponed: data.postponed, + postponed: isAppPath ? data.postponed : undefined, } await this.fs.writeFile( diff --git a/packages/next/src/server/render-result.ts b/packages/next/src/server/render-result.ts index 57bd89695dbfb..448a0431795e6 100644 --- a/packages/next/src/server/render-result.ts +++ b/packages/next/src/server/render-result.ts @@ -4,7 +4,9 @@ import type { FetchMetrics } from './base-http' import { chainStreams, + streamFromBuffer, streamFromString, + streamToBuffer, streamToString, } from './stream-utils/node-web-streams-helper' import { isAbortError, pipeToNodeResponse } from './pipe-readable' @@ -12,7 +14,7 @@ import { isAbortError, pipeToNodeResponse } from './pipe-readable' type ContentTypeOption = string | undefined export type AppPageRenderResultMetadata = { - flightData?: string + flightData?: Buffer revalidate?: Revalidate staticBailoutInfo?: { stack?: string @@ -50,6 +52,7 @@ export type RenderResultResponse = | ReadableStream[] | ReadableStream | string + | Buffer | null export type RenderResultOptions< @@ -89,7 +92,7 @@ export default class RenderResult< * @param value the static response value * @returns a new RenderResult instance */ - public static fromStatic(value: string) { + public static fromStatic(value: string | Buffer) { return new RenderResult(value, { metadata: {} }) } @@ -125,6 +128,26 @@ export default class RenderResult< return typeof this.response !== 'string' } + public toUnchunkedBuffer(stream?: false): Buffer + public toUnchunkedBuffer(stream: true): Promise + public toUnchunkedBuffer(stream = false): Promise | Buffer { + if (this.response === null) { + throw new Error('Invariant: null responses cannot be unchunked') + } + + if (typeof this.response !== 'string') { + if (!stream) { + throw new Error( + 'Invariant: dynamic responses cannot be unchunked. This is a bug in Next.js' + ) + } + + return streamToBuffer(this.readable) + } + + return Buffer.from(this.response) + } + /** * Returns the response if it is a string. If the page was dynamic, this will * return a promise if the `stream` option is true, or it will throw an error. @@ -164,6 +187,10 @@ export default class RenderResult< throw new Error('Invariant: static responses cannot be streamed') } + if (Buffer.isBuffer(this.response)) { + return streamFromBuffer(this.response) + } + // If the response is an array of streams, then chain them together. if (Array.isArray(this.response)) { return chainStreams(...this.response) @@ -191,6 +218,8 @@ export default class RenderResult< responses = [streamFromString(this.response)] } else if (Array.isArray(this.response)) { responses = this.response + } else if (Buffer.isBuffer(this.response)) { + responses = [streamFromBuffer(this.response)] } else { responses = [this.response] } diff --git a/packages/next/src/server/response-cache/types.ts b/packages/next/src/server/response-cache/types.ts index d797148ac9b09..955136c3a21cc 100644 --- a/packages/next/src/server/response-cache/types.ts +++ b/packages/next/src/server/response-cache/types.ts @@ -42,12 +42,22 @@ export interface CachedRedirectValue { props: Object } -interface CachedPageValue { - kind: 'PAGE' +export interface CachedAppPageValue { + kind: 'APP_PAGE' // this needs to be a RenderResult so since renderResponse // expects that type instead of a string html: RenderResult + rscData: Buffer | undefined + status: number | undefined postponed: string | undefined + headers: OutgoingHttpHeaders | undefined +} + +export interface CachedPageValue { + kind: 'PAGE' + // this needs to be a RenderResult so since renderResponse + // expects that type instead of a string + html: RenderResult pageData: Object status: number | undefined headers: OutgoingHttpHeaders | undefined @@ -71,13 +81,23 @@ export interface CachedImageValue { isStale?: boolean } -interface IncrementalCachedPageValue { +export interface IncrementalCachedAppPageValue { + kind: 'APP_PAGE' + // this needs to be a string since the cache expects to store + // the string value + html: string + rscData: Buffer | undefined + headers: OutgoingHttpHeaders | undefined + postponed: string | undefined + status: number | undefined +} + +export interface IncrementalCachedPageValue { kind: 'PAGE' // this needs to be a string since the cache expects to store // the string value html: string pageData: Object - postponed: string | undefined headers: OutgoingHttpHeaders | undefined status: number | undefined } @@ -94,6 +114,7 @@ export type IncrementalCacheEntry = { export type IncrementalCacheValue = | CachedRedirectValue | IncrementalCachedPageValue + | IncrementalCachedAppPageValue | CachedImageValue | CachedFetchValue | CachedRouteValue @@ -101,6 +122,7 @@ export type IncrementalCacheValue = export type ResponseCacheValue = | CachedRedirectValue | CachedPageValue + | CachedAppPageValue | CachedImageValue | CachedRouteValue diff --git a/packages/next/src/server/response-cache/utils.ts b/packages/next/src/server/response-cache/utils.ts index 174d6fa619a4a..18ec64c1dd052 100644 --- a/packages/next/src/server/response-cache/utils.ts +++ b/packages/next/src/server/response-cache/utils.ts @@ -12,12 +12,20 @@ export async function fromResponseCacheEntry( ? { kind: 'PAGE', html: await cacheEntry.value.html.toUnchunkedString(true), - postponed: cacheEntry.value.postponed, pageData: cacheEntry.value.pageData, headers: cacheEntry.value.headers, status: cacheEntry.value.status, } - : cacheEntry.value, + : cacheEntry.value?.kind === 'APP_PAGE' + ? { + kind: 'APP_PAGE', + html: await cacheEntry.value.html.toUnchunkedString(true), + postponed: cacheEntry.value.postponed, + rscData: cacheEntry.value.rscData, + headers: cacheEntry.value.headers, + status: cacheEntry.value.status, + } + : cacheEntry.value, } } @@ -42,10 +50,18 @@ export async function toResponseCacheEntry( kind: 'PAGE', html: RenderResult.fromStatic(response.value.html), pageData: response.value.pageData, - postponed: response.value.postponed, headers: response.value.headers, status: response.value.status, } - : response.value, + : response.value?.kind === 'APP_PAGE' + ? { + kind: 'APP_PAGE', + html: RenderResult.fromStatic(response.value.html), + rscData: response.value.rscData, + headers: response.value.headers, + status: response.value.status, + postponed: response.value.postponed, + } + : response.value, } } diff --git a/packages/next/src/server/stream-utils/node-web-streams-helper.ts b/packages/next/src/server/stream-utils/node-web-streams-helper.ts index a3d4ae8b924ee..8b051257c5868 100644 --- a/packages/next/src/server/stream-utils/node-web-streams-helper.ts +++ b/packages/next/src/server/stream-utils/node-web-streams-helper.ts @@ -73,6 +73,28 @@ export function streamFromString(str: string): ReadableStream { }) } +export function streamFromBuffer(chunk: Buffer): ReadableStream { + return new ReadableStream({ + start(controller) { + controller.enqueue(chunk) + controller.close() + }, + }) +} + +export async function streamToBuffer( + stream: ReadableStream +): Promise { + const buffers: Buffer[] = [] + + // @ts-expect-error TypeScript gets this wrong (https://nodejs.org/api/webstreams.html#async-iteration) + for await (const chunk of stream) { + buffers.push(chunk) + } + + return Buffer.concat(buffers) +} + export async function streamToString( stream: ReadableStream ): Promise { diff --git a/test/e2e/app-dir/binary/app/client.js b/test/e2e/app-dir/binary/app/client.js new file mode 100644 index 0000000000000..37f29ac669dc4 --- /dev/null +++ b/test/e2e/app-dir/binary/app/client.js @@ -0,0 +1,19 @@ +'use client' + +import { useEffect, useState } from 'react' + +export function Client({ binary, arbitrary }) { + const [hydrated, setHydrated] = useState(false) + + useEffect(() => { + setHydrated(true) + }, []) + + return ( + <> +
utf8 binary: {new TextDecoder().decode(binary)}
+
arbitrary binary: {String(arbitrary)}
+
hydrated: {String(hydrated)}
+ + ) +} diff --git a/test/e2e/app-dir/binary/app/layout.js b/test/e2e/app-dir/binary/app/layout.js new file mode 100644 index 0000000000000..8525f5f8c0b2a --- /dev/null +++ b/test/e2e/app-dir/binary/app/layout.js @@ -0,0 +1,12 @@ +export const metadata = { + title: 'Next.js', + description: 'Generated by Next.js', +} + +export default function RootLayout({ children }) { + return ( + + {children} + + ) +} diff --git a/test/e2e/app-dir/binary/app/page.js b/test/e2e/app-dir/binary/app/page.js new file mode 100644 index 0000000000000..5bbd22eedea98 --- /dev/null +++ b/test/e2e/app-dir/binary/app/page.js @@ -0,0 +1,8 @@ +import { Client } from './client' + +export default function Page() { + const binaryData = new Uint8Array([104, 101, 108, 108, 111]) + const nonUtf8BinaryData = new Uint8Array([0xff, 0, 1, 2, 3]) + + return +} diff --git a/test/e2e/app-dir/binary/next.config.js b/test/e2e/app-dir/binary/next.config.js new file mode 100644 index 0000000000000..5b7ed7e24f002 --- /dev/null +++ b/test/e2e/app-dir/binary/next.config.js @@ -0,0 +1,6 @@ +module.exports = { + experimental: { + // This ensures that we're running the experimental React. + taint: true, + }, +} diff --git a/test/e2e/app-dir/binary/rsc-binary.test.ts b/test/e2e/app-dir/binary/rsc-binary.test.ts new file mode 100644 index 0000000000000..ab400e325d5ce --- /dev/null +++ b/test/e2e/app-dir/binary/rsc-binary.test.ts @@ -0,0 +1,32 @@ +import { nextTestSetup } from 'e2e-utils' +import { check } from 'next-test-utils' + +describe('RSC binary serialization', () => { + const { next, skipped } = nextTestSetup({ + files: __dirname, + skipDeployment: true, + dependencies: { + react: '19.0.0-beta-04b058868c-20240508', + 'react-dom': '19.0.0-beta-04b058868c-20240508', + 'server-only': 'latest', + }, + }) + if (skipped) return + + afterEach(async () => { + await next.stop() + }) + + it('should correctly encode/decode binaries and hydrate', async function () { + const browser = await next.browser('/') + await check(async () => { + const content = await browser.elementByCss('body').text() + + return content.includes('utf8 binary: hello') && + content.includes('arbitrary binary: 255,0,1,2,3') && + content.includes('hydrated: true') + ? 'success' + : 'fail' + }, 'success') + }) +})