fix(stream): Allows body larger than 16 KiB with middleware (vercel#4…

…1270)

Fixes vercel#39262

The solution is to call `stream.push(null)` to trigger the `end` event
which allows `getRawBody` to run completely.

<!--
Thanks for opening a PR! Your contribution is much appreciated.
To make sure your PR is handled as smoothly as possible we request that
you follow the checklist sections below.
Choose the right checklist for the change that you're making:
-->

## Bug

- [x] Related issues linked using `fixes #number`
- [x] Integration tests added
- [ ] Errors have a helpful link attached, see `contributing.md`

## Feature

- [ ] Implements an existing feature request or RFC. Make sure the
feature request has been accepted for implementation before opening a
PR.
- [ ] Related issues linked using `fixes #number`
- [ ] Integration tests added
- [ ] Documentation added
- [ ] Telemetry added. In case of a feature if it's used or not.
- [ ] Errors have a helpful link attached, see `contributing.md`

## Documentation / Examples

- [x] Make sure the linting passes by running `pnpm lint`
- [x] The "examples guidelines" are followed from [our contributing
doc](https://github.com/vercel/next.js/blob/canary/contributing/examples/adding-examples.md)

Co-authored-by: JJ Kasper <[email protected]>

packages/next/server/body-streams.ts

@@ -76,8 +76,14 @@ export function getClonableBody<T extends IncomingMessage>(
       const input = buffered ?? readable
       const p1 = new PassThrough()
       const p2 = new PassThrough()
-      input.pipe(p1)
-      input.pipe(p2)
+      input.on('data', (chunk) => {
+        p1.push(chunk)
+        p2.push(chunk)
+      })
+      input.on('end', () => {
+        p1.push(null)
+        p2.push(null)
+      })
       buffered = p2
       return p1
     },

test/e2e/middleware-fetches-with-body/index.test.ts

@@ -0,0 +1,265 @@
+import { createNext } from 'e2e-utils'
+import { NextInstance } from 'test/lib/next-modes/base'
+import { fetchViaHTTP } from 'next-test-utils'
+
+describe('Middleware fetches with body', () => {
+  let next: NextInstance
+
+  beforeAll(async () => {
+    next = await createNext({
+      files: {
+        'pages/api/default.js': `
+          export default (req, res) => res.json({ body: req.body })
+        `,
+        'pages/api/size_limit_5kb.js': `
+          export const config = { api: { bodyParser: { sizeLimit: '5kb' } } }
+          export default (req, res) => res.json({ body: req.body })
+        `,
+        'pages/api/size_limit_5mb.js': `
+          export const config = { api: { bodyParser: { sizeLimit: '5mb' } } }
+          export default (req, res) => res.json({ body: req.body })
+        `,
+        'pages/api/body_parser_false.js': `
+          export const config = { api: { bodyParser: false } }
+
+          async function buffer(readable) {
+            const chunks = []
+            for await (const chunk of readable) {
+              chunks.push(typeof chunk === 'string' ? Buffer.from(chunk) : chunk)
+            }
+            return Buffer.concat(chunks)
+          }
+
+          export default async (req, res) => {
+            const buf = await buffer(req)
+            const rawBody = buf.toString('utf8');
+
+            res.json({ rawBody, body: req.body })
+          }
+        `,
+        'middleware.js': `
+          import { NextResponse } from 'next/server';
+
+          export default async (req) => NextResponse.next();
+        `,
+      },
+      dependencies: {},
+    })
+  })
+  afterAll(() => next.destroy())
+
+  describe('with default bodyParser sizeLimit (1mb)', () => {
+    it('should return 413 for body greater than 1mb', async () => {
+      const bodySize = 1024 * 1024 + 1
+      const body = 'r'.repeat(bodySize)
+
+      const res = await fetchViaHTTP(
+        next.url,
+        '/api/default',
+        {},
+        {
+          body,
+          method: 'POST',
+        }
+      )
+
+      expect(res.status).toBe(413)
+      expect(res.statusText).toBe('Body exceeded 1mb limit')
+    })
+
+    it('should be able to send and return body size equal to 1mb', async () => {
+      const bodySize = 1024 * 1024
+      const body = 'B1C2D3E4F5G6H7I8J9K0LaMbNcOdPeQf'.repeat(bodySize / 32)
+
+      const res = await fetchViaHTTP(
+        next.url,
+        '/api/default',
+        {},
+        {
+          body,
+          method: 'POST',
+        }
+      )
+      const data = await res.json()
+
+      expect(res.status).toBe(200)
+      expect(data.body.length).toBe(bodySize)
+      expect(data.body.split('B1C2D3E4F5G6H7I8J9K0LaMbNcOdPeQf').length).toBe(
+        bodySize / 32 + 1
+      )
+    })
+
+    it('should be able to send and return body greater than default highWaterMark (16KiB)', async () => {
+      const bodySize = 16 * 1024 + 1
+      const body =
+        'CD1E2F3G4H5I6J7K8L9M0NaObPcQdReS'.repeat(bodySize / 32) + 'C'
+
+      const res = await fetchViaHTTP(
+        next.url,
+        '/api/default',
+        {},
+        {
+          body,
+          method: 'POST',
+        }
+      )
+      const data = await res.json()
+
+      expect(res.status).toBe(200)
+      expect(data.body.length).toBe(bodySize)
+      expect(data.body.split('CD1E2F3G4H5I6J7K8L9M0NaObPcQdReS').length).toBe(
+        512 + 1
+      )
+    })
+  })
+
+  describe('with custom bodyParser sizeLimit (5kb)', () => {
+    it('should return 413 for body greater than 5kb', async () => {
+      const bodySize = 5 * 1024 + 1
+      const body = 's'.repeat(bodySize)
+
+      const res = await fetchViaHTTP(
+        next.url,
+        '/api/size_limit_5kb',
+        {},
+        {
+          body,
+          method: 'POST',
+        }
+      )
+
+      expect(res.status).toBe(413)
+      expect(res.statusText).toBe('Body exceeded 5kb limit')
+    })
+
+    it('should be able to send and return body size equal to 5kb', async () => {
+      const bodySize = 5120
+      const body = 'DEF1G2H3I4J5K6L7M8N9O0PaQbRcSdTe'.repeat(bodySize / 32)
+
+      const res = await fetchViaHTTP(
+        next.url,
+        '/api/size_limit_5kb',
+        {},
+        {
+          body,
+          method: 'POST',
+        }
+      )
+      const data = await res.json()
+
+      expect(res.status).toBe(200)
+      expect(data.body.length).toBe(bodySize)
+      expect(data.body.split('DEF1G2H3I4J5K6L7M8N9O0PaQbRcSdTe').length).toBe(
+        bodySize / 32 + 1
+      )
+    })
+  })
+
+  describe('with custom bodyParser sizeLimit (5mb)', () => {
+    it('should return 413 for body equal to 10mb', async () => {
+      const bodySize = 10 * 1024 * 1024
+      const body = 't'.repeat(bodySize)
+
+      const res = await fetchViaHTTP(
+        next.url,
+        '/api/size_limit_5mb',
+        {},
+        {
+          body,
+          method: 'POST',
+        }
+      )
+
+      expect(res.status).toBe(413)
+      expect(res.statusText).toBe('Body exceeded 5mb limit')
+    })
+
+    it('should return 413 for body greater than 5mb', async () => {
+      const bodySize = 5 * 1024 * 1024 + 1
+      const body = 'u'.repeat(bodySize)
+
+      const res = await fetchViaHTTP(
+        next.url,
+        '/api/size_limit_5mb',
+        {},
+        {
+          body,
+          method: 'POST',
+        }
+      )
+
+      expect(res.status).toBe(413)
+      expect(res.statusText).toBe('Body exceeded 5mb limit')
+    })
+
+    it('should be able to send and return body size equal to 5mb', async () => {
+      const bodySize = 5 * 1024 * 1024
+      const body = 'FGHI1J2K3L4M5N6O7P8Q9R0SaTbUcVdW'.repeat(bodySize / 32)
+
+      const res = await fetchViaHTTP(
+        next.url,
+        '/api/size_limit_5mb',
+        {},
+        {
+          body,
+          method: 'POST',
+        }
+      )
+      const data = await res.json()
+
+      expect(res.status).toBe(200)
+      expect(data.body.length).toBe(bodySize)
+      expect(data.body.split('FGHI1J2K3L4M5N6O7P8Q9R0SaTbUcVdW').length).toBe(
+        bodySize / 32 + 1
+      )
+    })
+  })
+
+  describe('with bodyParser = false', () => {
+    it('should be able to send and return with body size equal to 16KiB', async () => {
+      const bodySize = 16 * 1024
+      const body = 'HIJK1L2M3N4O5P6Q7R8S9T0UaVbWcXdY'.repeat(bodySize / 32)
+
+      const res = await fetchViaHTTP(
+        next.url,
+        '/api/body_parser_false',
+        {},
+        {
+          body,
+          method: 'POST',
+        }
+      )
+      const data = await res.json()
+
+      expect(res.status).toBe(200)
+      expect(data.body).toBeUndefined()
+      expect(data.rawBody.length).toBe(bodySize)
+      expect(
+        data.rawBody.split('HIJK1L2M3N4O5P6Q7R8S9T0UaVbWcXdY').length
+      ).toBe(bodySize / 32 + 1)
+    })
+
+    it('should be able to send and return with body greater than 16KiB', async () => {
+      const bodySize = 1024 * 1024
+      const body = 'JKLM1N2O3P4Q5R6S7T8U9V0WaXbYcZdA'.repeat(bodySize / 32)
+
+      const res = await fetchViaHTTP(
+        next.url,
+        '/api/body_parser_false',
+        {},
+        {
+          body,
+          method: 'POST',
+        }
+      )
+      const data = await res.json()
+
+      expect(res.status).toBe(200)
+      expect(data.body).toBeUndefined()
+      expect(data.rawBody.length).toBe(bodySize)
+      expect(
+        data.rawBody.split('JKLM1N2O3P4Q5R6S7T8U9V0WaXbYcZdA').length
+      ).toBe(bodySize / 32 + 1)
+    })
+  })
+})

test/production/reading-request-body-in-middleware/index.test.ts

@@ -101,6 +101,32 @@ describe('reading request body in middleware', () => {
     expect(response.headers.has('data')).toBe(false)
   })
 
+  it('passes the body greater than 64KiB to the api endpoint', async () => {
+    const response = await fetchViaHTTP(
+      next.url,
+      '/api/hi',
+      {
+        next: '1',
+      },
+      {
+        method: 'POST',
+        headers: {
+          'content-type': 'application/json',
+        },
+        body: JSON.stringify({
+          foo: 'bar'.repeat(22 * 1024),
+        }),
+      }
+    )
+    const data = await response.json()
+    expect(response.status).toEqual(200)
+    expect(data.foo.length).toBe(22 * 1024 * 3)
+    expect(data.foo.split('bar').length).toBe(22 * 1024 + 1)
+    expect(data.api).toBeTrue()
+    expect(response.headers.get('x-from-root-middleware')).toEqual('1')
+    expect(response.headers.has('data')).toBe(false)
+  })
+
   it('passes the body to the api endpoint when no body is consumed on middleware', async () => {
     const response = await fetchViaHTTP(
       next.url,
@@ -127,4 +153,31 @@ describe('reading request body in middleware', () => {
     expect(response.headers.get('x-from-root-middleware')).toEqual('1')
     expect(response.headers.has('data')).toBe(false)
   })
+
+  it('passes the body greater than 64KiB to the api endpoint when no body is consumed on middleware', async () => {
+    const response = await fetchViaHTTP(
+      next.url,
+      '/api/hi',
+      {
+        next: '1',
+        no_reading: '1',
+      },
+      {
+        method: 'POST',
+        headers: {
+          'content-type': 'application/json',
+        },
+        body: JSON.stringify({
+          foo: 'bar'.repeat(22 * 1024),
+        }),
+      }
+    )
+    const data = await response.json()
+    expect(response.status).toEqual(200)
+    expect(data.foo.length).toBe(22 * 1024 * 3)
+    expect(data.foo.split('bar').length).toBe(22 * 1024 + 1)
+    expect(data.api).toBeTrue()
+    expect(response.headers.get('x-from-root-middleware')).toEqual('1')
+    expect(response.headers.has('data')).toBe(false)
+  })
 })