From 3b65593592b1b10949f58d5993669ef01fcc4af0 Mon Sep 17 00:00:00 2001 From: Felipe Barso <77860630+aprendendofelipe@users.noreply.github.com> Date: Mon, 17 Oct 2022 19:26:16 -0300 Subject: [PATCH] fix(stream): Allows body larger than 16 KiB with middleware (#41270) Fixes #39262 The solution is to call `stream.push(null)` to trigger the `end` event which allows `getRawBody` to run completely. ## Bug - [x] Related issues linked using `fixes #number` - [x] Integration tests added - [ ] Errors have a helpful link attached, see `contributing.md` ## Feature - [ ] Implements an existing feature request or RFC. Make sure the feature request has been accepted for implementation before opening a PR. - [ ] Related issues linked using `fixes #number` - [ ] Integration tests added - [ ] Documentation added - [ ] Telemetry added. In case of a feature if it's used or not. - [ ] Errors have a helpful link attached, see `contributing.md` ## Documentation / Examples - [x] Make sure the linting passes by running `pnpm lint` - [x] The "examples guidelines" are followed from [our contributing doc](https://github.com/vercel/next.js/blob/canary/contributing/examples/adding-examples.md) Co-authored-by: JJ Kasper --- packages/next/server/body-streams.ts | 10 +- .../index.test.ts | 265 ++++++++++++++++++ .../index.test.ts | 53 ++++ 3 files changed, 326 insertions(+), 2 deletions(-) create mode 100644 test/e2e/middleware-fetches-with-body/index.test.ts diff --git a/packages/next/server/body-streams.ts b/packages/next/server/body-streams.ts index 0951502f5b7a3..2053240d92351 100644 --- a/packages/next/server/body-streams.ts +++ b/packages/next/server/body-streams.ts @@ -76,8 +76,14 @@ export function getClonableBody( const input = buffered ?? readable const p1 = new PassThrough() const p2 = new PassThrough() - input.pipe(p1) - input.pipe(p2) + input.on('data', (chunk) => { + p1.push(chunk) + p2.push(chunk) + }) + input.on('end', () => { + p1.push(null) + p2.push(null) + }) buffered = p2 return p1 }, diff --git a/test/e2e/middleware-fetches-with-body/index.test.ts b/test/e2e/middleware-fetches-with-body/index.test.ts new file mode 100644 index 0000000000000..457775c4df2f3 --- /dev/null +++ b/test/e2e/middleware-fetches-with-body/index.test.ts @@ -0,0 +1,265 @@ +import { createNext } from 'e2e-utils' +import { NextInstance } from 'test/lib/next-modes/base' +import { fetchViaHTTP } from 'next-test-utils' + +describe('Middleware fetches with body', () => { + let next: NextInstance + + beforeAll(async () => { + next = await createNext({ + files: { + 'pages/api/default.js': ` + export default (req, res) => res.json({ body: req.body }) + `, + 'pages/api/size_limit_5kb.js': ` + export const config = { api: { bodyParser: { sizeLimit: '5kb' } } } + export default (req, res) => res.json({ body: req.body }) + `, + 'pages/api/size_limit_5mb.js': ` + export const config = { api: { bodyParser: { sizeLimit: '5mb' } } } + export default (req, res) => res.json({ body: req.body }) + `, + 'pages/api/body_parser_false.js': ` + export const config = { api: { bodyParser: false } } + + async function buffer(readable) { + const chunks = [] + for await (const chunk of readable) { + chunks.push(typeof chunk === 'string' ? Buffer.from(chunk) : chunk) + } + return Buffer.concat(chunks) + } + + export default async (req, res) => { + const buf = await buffer(req) + const rawBody = buf.toString('utf8'); + + res.json({ rawBody, body: req.body }) + } + `, + 'middleware.js': ` + import { NextResponse } from 'next/server'; + + export default async (req) => NextResponse.next(); + `, + }, + dependencies: {}, + }) + }) + afterAll(() => next.destroy()) + + describe('with default bodyParser sizeLimit (1mb)', () => { + it('should return 413 for body greater than 1mb', async () => { + const bodySize = 1024 * 1024 + 1 + const body = 'r'.repeat(bodySize) + + const res = await fetchViaHTTP( + next.url, + '/api/default', + {}, + { + body, + method: 'POST', + } + ) + + expect(res.status).toBe(413) + expect(res.statusText).toBe('Body exceeded 1mb limit') + }) + + it('should be able to send and return body size equal to 1mb', async () => { + const bodySize = 1024 * 1024 + const body = 'B1C2D3E4F5G6H7I8J9K0LaMbNcOdPeQf'.repeat(bodySize / 32) + + const res = await fetchViaHTTP( + next.url, + '/api/default', + {}, + { + body, + method: 'POST', + } + ) + const data = await res.json() + + expect(res.status).toBe(200) + expect(data.body.length).toBe(bodySize) + expect(data.body.split('B1C2D3E4F5G6H7I8J9K0LaMbNcOdPeQf').length).toBe( + bodySize / 32 + 1 + ) + }) + + it('should be able to send and return body greater than default highWaterMark (16KiB)', async () => { + const bodySize = 16 * 1024 + 1 + const body = + 'CD1E2F3G4H5I6J7K8L9M0NaObPcQdReS'.repeat(bodySize / 32) + 'C' + + const res = await fetchViaHTTP( + next.url, + '/api/default', + {}, + { + body, + method: 'POST', + } + ) + const data = await res.json() + + expect(res.status).toBe(200) + expect(data.body.length).toBe(bodySize) + expect(data.body.split('CD1E2F3G4H5I6J7K8L9M0NaObPcQdReS').length).toBe( + 512 + 1 + ) + }) + }) + + describe('with custom bodyParser sizeLimit (5kb)', () => { + it('should return 413 for body greater than 5kb', async () => { + const bodySize = 5 * 1024 + 1 + const body = 's'.repeat(bodySize) + + const res = await fetchViaHTTP( + next.url, + '/api/size_limit_5kb', + {}, + { + body, + method: 'POST', + } + ) + + expect(res.status).toBe(413) + expect(res.statusText).toBe('Body exceeded 5kb limit') + }) + + it('should be able to send and return body size equal to 5kb', async () => { + const bodySize = 5120 + const body = 'DEF1G2H3I4J5K6L7M8N9O0PaQbRcSdTe'.repeat(bodySize / 32) + + const res = await fetchViaHTTP( + next.url, + '/api/size_limit_5kb', + {}, + { + body, + method: 'POST', + } + ) + const data = await res.json() + + expect(res.status).toBe(200) + expect(data.body.length).toBe(bodySize) + expect(data.body.split('DEF1G2H3I4J5K6L7M8N9O0PaQbRcSdTe').length).toBe( + bodySize / 32 + 1 + ) + }) + }) + + describe('with custom bodyParser sizeLimit (5mb)', () => { + it('should return 413 for body equal to 10mb', async () => { + const bodySize = 10 * 1024 * 1024 + const body = 't'.repeat(bodySize) + + const res = await fetchViaHTTP( + next.url, + '/api/size_limit_5mb', + {}, + { + body, + method: 'POST', + } + ) + + expect(res.status).toBe(413) + expect(res.statusText).toBe('Body exceeded 5mb limit') + }) + + it('should return 413 for body greater than 5mb', async () => { + const bodySize = 5 * 1024 * 1024 + 1 + const body = 'u'.repeat(bodySize) + + const res = await fetchViaHTTP( + next.url, + '/api/size_limit_5mb', + {}, + { + body, + method: 'POST', + } + ) + + expect(res.status).toBe(413) + expect(res.statusText).toBe('Body exceeded 5mb limit') + }) + + it('should be able to send and return body size equal to 5mb', async () => { + const bodySize = 5 * 1024 * 1024 + const body = 'FGHI1J2K3L4M5N6O7P8Q9R0SaTbUcVdW'.repeat(bodySize / 32) + + const res = await fetchViaHTTP( + next.url, + '/api/size_limit_5mb', + {}, + { + body, + method: 'POST', + } + ) + const data = await res.json() + + expect(res.status).toBe(200) + expect(data.body.length).toBe(bodySize) + expect(data.body.split('FGHI1J2K3L4M5N6O7P8Q9R0SaTbUcVdW').length).toBe( + bodySize / 32 + 1 + ) + }) + }) + + describe('with bodyParser = false', () => { + it('should be able to send and return with body size equal to 16KiB', async () => { + const bodySize = 16 * 1024 + const body = 'HIJK1L2M3N4O5P6Q7R8S9T0UaVbWcXdY'.repeat(bodySize / 32) + + const res = await fetchViaHTTP( + next.url, + '/api/body_parser_false', + {}, + { + body, + method: 'POST', + } + ) + const data = await res.json() + + expect(res.status).toBe(200) + expect(data.body).toBeUndefined() + expect(data.rawBody.length).toBe(bodySize) + expect( + data.rawBody.split('HIJK1L2M3N4O5P6Q7R8S9T0UaVbWcXdY').length + ).toBe(bodySize / 32 + 1) + }) + + it('should be able to send and return with body greater than 16KiB', async () => { + const bodySize = 1024 * 1024 + const body = 'JKLM1N2O3P4Q5R6S7T8U9V0WaXbYcZdA'.repeat(bodySize / 32) + + const res = await fetchViaHTTP( + next.url, + '/api/body_parser_false', + {}, + { + body, + method: 'POST', + } + ) + const data = await res.json() + + expect(res.status).toBe(200) + expect(data.body).toBeUndefined() + expect(data.rawBody.length).toBe(bodySize) + expect( + data.rawBody.split('JKLM1N2O3P4Q5R6S7T8U9V0WaXbYcZdA').length + ).toBe(bodySize / 32 + 1) + }) + }) +}) diff --git a/test/production/reading-request-body-in-middleware/index.test.ts b/test/production/reading-request-body-in-middleware/index.test.ts index 88a92a98b05f0..cf9e5e0b412e6 100644 --- a/test/production/reading-request-body-in-middleware/index.test.ts +++ b/test/production/reading-request-body-in-middleware/index.test.ts @@ -101,6 +101,32 @@ describe('reading request body in middleware', () => { expect(response.headers.has('data')).toBe(false) }) + it('passes the body greater than 64KiB to the api endpoint', async () => { + const response = await fetchViaHTTP( + next.url, + '/api/hi', + { + next: '1', + }, + { + method: 'POST', + headers: { + 'content-type': 'application/json', + }, + body: JSON.stringify({ + foo: 'bar'.repeat(22 * 1024), + }), + } + ) + const data = await response.json() + expect(response.status).toEqual(200) + expect(data.foo.length).toBe(22 * 1024 * 3) + expect(data.foo.split('bar').length).toBe(22 * 1024 + 1) + expect(data.api).toBeTrue() + expect(response.headers.get('x-from-root-middleware')).toEqual('1') + expect(response.headers.has('data')).toBe(false) + }) + it('passes the body to the api endpoint when no body is consumed on middleware', async () => { const response = await fetchViaHTTP( next.url, @@ -127,4 +153,31 @@ describe('reading request body in middleware', () => { expect(response.headers.get('x-from-root-middleware')).toEqual('1') expect(response.headers.has('data')).toBe(false) }) + + it('passes the body greater than 64KiB to the api endpoint when no body is consumed on middleware', async () => { + const response = await fetchViaHTTP( + next.url, + '/api/hi', + { + next: '1', + no_reading: '1', + }, + { + method: 'POST', + headers: { + 'content-type': 'application/json', + }, + body: JSON.stringify({ + foo: 'bar'.repeat(22 * 1024), + }), + } + ) + const data = await response.json() + expect(response.status).toEqual(200) + expect(data.foo.length).toBe(22 * 1024 * 3) + expect(data.foo.split('bar').length).toBe(22 * 1024 + 1) + expect(data.api).toBeTrue() + expect(response.headers.get('x-from-root-middleware')).toEqual('1') + expect(response.headers.has('data')).toBe(false) + }) })