Skip to content

Commit

Permalink
fix(recorder): recompress fetch body on record (#2810)
Browse files Browse the repository at this point in the history
  • Loading branch information
@mikicho
mikicho authored Nov 25, 2024
1 parent a143911 commit be35f23
Show file tree
Hide file tree
Showing 2 changed files with 211 additions and 105 deletions.
236 changes: 131 additions & 105 deletions lib/recorder.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,21 +6,18 @@ const { inspect } = require('util')

const common = require('./common')
const { restoreOverriddenClientRequest } = require('./intercept')
const { BatchInterceptor } = require('@mswjs/interceptors')
const { EventEmitter } = require('stream')
const { gzipSync, brotliCompressSync, deflateSync } = require('zlib')
const {
default: nodeInterceptors,
} = require('@mswjs/interceptors/presets/node')
const { EventEmitter } = require('stream')

const SEPARATOR = '\n<<<<<<-- cut here -->>>>>>\n'
let recordingInProgress = false
let outputs = []

// TODO: Consider use one BatchInterceptor (and not one for intercept and one for record)
const interceptor = new BatchInterceptor({
name: 'nock-interceptor',
interceptors: nodeInterceptors,
})
// TODO: don't reuse the nodeInterceptors, create new ones.
const clientRequestInterceptor = nodeInterceptors[0]
const fetchRequestInterceptor = nodeInterceptors[2]

function getScope(options) {
const { proto, host, port } = common.normalizeRequestOptions(options)
Expand Down Expand Up @@ -226,109 +223,137 @@ function record(recOptions) {
restoreOverriddenClientRequest()

// We override the requests so that we can save information on them before executing.
interceptor.apply()
interceptor.on(
'request',
async function ({ request: mswRequest, requestId }) {
const request = mswRequest.clone()
const { options } = common.normalizeClientRequestArgs(request.url)
options.method = request.method
const proto = options.protocol.slice(0, -1)

// Node 0.11 https.request calls http.request -- don't want to record things
// twice.
/* istanbul ignore if */
if (options._recording) {
return
}
options._recording = true

const req = new EventEmitter()
req.on('response', function () {
debug(thisRecordingId, 'intercepting', proto, 'request to record')

// Intercept "res.once('end', ...)"-like event
interceptor.once(
'response',
async function ({ response: mswResponse }) {
const response = mswResponse.clone()
debug(thisRecordingId, proto, 'intercepted request ended')

let reqheaders
// Ignore request headers completely unless it was explicitly enabled by the user (see README)
if (enableReqHeadersRecording) {
// We never record user-agent headers as they are worse than useless -
// they actually make testing more difficult without providing any benefit (see README)
reqheaders = Object.fromEntries(request.headers.entries())
common.deleteHeadersField(reqheaders, 'user-agent')
}
clientRequestInterceptor.apply()
fetchRequestInterceptor.apply()
clientRequestInterceptor.on('request', async function ({ request }) {
await recordRequest(request)
})
fetchRequestInterceptor.on('request', async function ({ request }) {
await recordRequest(request)
})

const headers = Object.fromEntries(response.headers.entries())
const res = {
statusCode: response.status,
headers,
rawHeaders: headers,
}
async function recordRequest(mswRequest) {
const request = mswRequest.clone()
const { options } = common.normalizeClientRequestArgs(request.url)
options.method = request.method
const proto = options.protocol.slice(0, -1)

// Node 0.11 https.request calls http.request -- don't want to record things
// twice.
/* istanbul ignore if */
if (options._recording) {
return
}
options._recording = true

const generateFn = outputObjects
? generateRequestAndResponseObject
: generateRequestAndResponse
let out = generateFn({
req: options,
bodyChunks: [Buffer.from(await request.arrayBuffer())],
options,
res,
dataChunks: [Buffer.from(await response.arrayBuffer())],
reqheaders,
})

debug('out:', out)

// Check that the request was made during the current recording.
// If it hasn't then skip it. There is no other simple way to handle
// this as it depends on the timing of requests and responses. Throwing
// will make some recordings/unit tests fail randomly depending on how
// fast/slow the response arrived.
// If you are seeing this error then you need to make sure that all
// the requests made during a single recording session finish before
// ending the same recording session.
if (thisRecordingId !== currentRecordingId) {
debug('skipping recording of an out-of-order request', out)
return
}
const req = new EventEmitter()
req.on('response', function () {
debug(thisRecordingId, 'intercepting', proto, 'request to record')

outputs.push(out)

if (!dontPrint) {
if (useSeparator) {
if (typeof out !== 'string') {
out = JSON.stringify(out, null, 2)
}
logging(SEPARATOR + out + SEPARATOR)
} else {
logging(out)
}
}
},
)
clientRequestInterceptor.once('response', async function ({ response }) {
await recordResponse(response)
})
fetchRequestInterceptor.once('response', async function ({ response }) {
// fetch decompresses the body automatically, so we need to recompress it
const codings =
response.headers
.get('content-encoding')
?.toLowerCase()
.split(',')
.map(c => c.trim()) || []

let body = await response.arrayBuffer()
for (const coding of codings) {
if (coding === 'gzip') {
body = gzipSync(body)
} else if (coding === 'deflate') {
body = deflateSync(body)
} else if (coding === 'br') {
body = brotliCompressSync(body)
}
}

debug('finished setting up intercepting')
await recordResponse(new Response(body, response))
})

// We override both the http and the https modules; when we are
// serializing the request, we need to know which was called.
// By stuffing the state, we can make sure that nock records
// the intended protocol.
if (proto === 'https') {
options.proto = 'https'
// Intercept "res.once('end', ...)"-like event
async function recordResponse(mswResponse) {
const response = mswResponse.clone()
debug(thisRecordingId, proto, 'intercepted request ended')

let reqheaders
// Ignore request headers completely unless it was explicitly enabled by the user (see README)
if (enableReqHeadersRecording) {
// We never record user-agent headers as they are worse than useless -
// they actually make testing more difficult without providing any benefit (see README)
reqheaders = Object.fromEntries(request.headers.entries())
common.deleteHeadersField(reqheaders, 'user-agent')
}
})

// This is a massive change, we are trying to change minimum code, so we emit end event here
// because mswjs take care for these events
// TODO: refactor the recorder, we no longer need all the listeners and can just record the request we get from MSW
req.emit('response')
},
)
const headers = Object.fromEntries(response.headers.entries())
const res = {
statusCode: response.status,
headers,
rawHeaders: headers,
}

const generateFn = outputObjects
? generateRequestAndResponseObject
: generateRequestAndResponse
let out = generateFn({
req: options,
bodyChunks: [Buffer.from(await request.arrayBuffer())],
options,
res,
dataChunks: [Buffer.from(await response.arrayBuffer())],
reqheaders,
})

debug('out:', out)

// Check that the request was made during the current recording.
// If it hasn't then skip it. There is no other simple way to handle
// this as it depends on the timing of requests and responses. Throwing
// will make some recordings/unit tests fail randomly depending on how
// fast/slow the response arrived.
// If you are seeing this error then you need to make sure that all
// the requests made during a single recording session finish before
// ending the same recording session.
if (thisRecordingId !== currentRecordingId) {
debug('skipping recording of an out-of-order request', out)
return
}

outputs.push(out)

if (!dontPrint) {
if (useSeparator) {
if (typeof out !== 'string') {
out = JSON.stringify(out, null, 2)
}
logging(SEPARATOR + out + SEPARATOR)
} else {
logging(out)
}
}
}

debug('finished setting up intercepting')

// We override both the http and the https modules; when we are
// serializing the request, we need to know which was called.
// By stuffing the state, we can make sure that nock records
// the intended protocol.
if (proto === 'https') {
options.proto = 'https'
}
})

// This is a massive change, we are trying to change minimum code, so we emit end event here
// because mswjs take care for these events
// TODO: refactor the recorder, we no longer need all the listeners and can just record the request we get from MSW
req.emit('response')
}
}

// Restore *all* the overridden http/https modules' properties.
Expand All @@ -338,7 +363,8 @@ function restore() {
'restoring all the overridden http/https properties',
)

interceptor.dispose()
clientRequestInterceptor.dispose()
fetchRequestInterceptor.dispose()
restoreOverriddenClientRequest()
recordingInProgress = false
}
Expand Down
80 changes: 80 additions & 0 deletions tests/test_fetch.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -509,4 +509,84 @@ describe('Native Fetch', () => {
expect(body).to.be.empty()
})
})

describe('recording', () => {
it('records and replays gzipped nocks correctly', async () => {
const exampleText = '<html><body>example</body></html>'

const { origin } = await startHttpServer((request, response) => {
// TODO: flip the order of the encoding, this is a bug in fetch
// const body = zlib.brotliCompressSync(zlib.gzipSync(exampleText))
const body = zlib.gzipSync(zlib.brotliCompressSync(exampleText))

response.writeHead(200, { 'content-encoding': 'gzip, br' })
response.end(body)
})

nock.restore()
nock.recorder.clear()
expect(nock.recorder.play()).to.be.empty()

nock.recorder.rec({
dont_print: true,
output_objects: true,
})

const response1 = await fetch(origin)
expect(await response1.text()).to.equal(exampleText)
expect(response1.headers.get('content-encoding')).to.equal('gzip, br')

nock.restore()
const recorded = nock.recorder.play()
nock.recorder.clear()
nock.activate()

expect(recorded).to.have.lengthOf(1)
const nocks = nock.define(recorded)

const response2 = await fetch(origin)
expect(await response2.text()).to.equal(exampleText)
expect(response1.headers.get('content-encoding')).to.equal('gzip, br')

nocks.forEach(nock => nock.done())
})

it('records and replays deflated nocks correctly', async () => {
const exampleText = '<html><body>example</body></html>'

const { origin } = await startHttpServer((request, response) => {
const body = zlib.deflateSync(exampleText)

response.writeHead(200, { 'content-encoding': 'deflate' })
response.end(body)
})

nock.restore()
nock.recorder.clear()
expect(nock.recorder.play()).to.be.empty()

nock.recorder.rec({
dont_print: true,
output_objects: true,
})

const response1 = await fetch(origin)
expect(await response1.text()).to.equal(exampleText)
expect(response1.headers.get('content-encoding')).to.equal('deflate')

nock.restore()
const recorded = nock.recorder.play()
nock.recorder.clear()
nock.activate()

expect(recorded).to.have.lengthOf(1)
const nocks = nock.define(recorded)

const response2 = await fetch(origin)
expect(await response2.text()).to.equal(exampleText)
expect(response1.headers.get('content-encoding')).to.equal('deflate')

nocks.forEach(nock => nock.done())
})
})
})

0 comments on commit be35f23

Please sign in to comment.