From ee589e7eb85a93d7d7622f0e5592eacbd5091837 Mon Sep 17 00:00:00 2001 From: vigneshshanmugam Date: Thu, 4 Feb 2021 11:29:29 -0800 Subject: [PATCH 1/4] feat: move fields to ECS format --- src/common_types.ts | 2 +- src/reporters/json.ts | 163 ++++++++++++++++++++++++++++++++++-------- 2 files changed, 135 insertions(+), 30 deletions(-) diff --git a/src/common_types.ts b/src/common_types.ts index 4c9b7d07..51592df9 100644 --- a/src/common_types.ts +++ b/src/common_types.ts @@ -46,7 +46,7 @@ export type NetworkInfo = { method: string; type: string; request: Protocol.Network.Request; - response: Protocol.Network.Response; + response: Protocol.Network.Response | null; isNavigationRequest: boolean; requestSentTime: number; loadEndTime: number; diff --git a/src/reporters/json.ts b/src/reporters/json.ts index dbaa54fa..82ede33d 100644 --- a/src/reporters/json.ts +++ b/src/reporters/json.ts @@ -27,18 +27,117 @@ import BaseReporter from './base'; import { formatError, getTimestamp } from '../helpers'; import { Journey, Step } from '../dsl'; import snakeCaseKeys from 'snakecase-keys'; +import { NetworkInfo } from '../common_types'; /* eslint-disable @typescript-eslint/no-var-requires */ -const programVersion = require('../../package.json').version; +const { version, name } = require('../../package.json'); + +type OutputType = + | 'journey/register' + | 'journey/start' + | 'step/screenshot' + | 'step/end' + | 'journey/network_info' + | 'journey/filmstrips' + | 'journey/browserconsole' + | 'journey/end'; + +type OutputFields = { + type: OutputType; + journey: Journey; + timestamp?: number; + url?: string; + step?: Partial; + error?: Error; + root_fields?: Record; + payload?: Record; + blob?: string; +}; + +function getMetadata() { + return { + process: { + pid: process.pid, + ppid: process.ppid, + title: process.title, + args: process.argv, + }, + os: { + platform: process.platform, + }, + package: { + name, + version, + }, + }; +} + +function formatHttpVersion(protocol: string) { + if (protocol === 'h2') { + return 2; + } else if (protocol === 'http/1.1') { + return 1.1; + } else if (protocol === 'http/1.0') { + return 1.0; + } + return; +} + +function formatECSFields(network: NetworkInfo) { + const { request, response, url } = network; + const postdata = request.postData || ''; + const tls = response?.securityDetails; + + return { + // URL and USER AGENT would be parsed and mapped by heartbeat + url, + user_agent: request.headers['user_agent'], + http: { + version: formatHttpVersion(response?.protocol), + request: { + body: { + bytes: postdata.length, + content: postdata, + }, + method: request.method, + referrer: request.headers['referer'], + }, + response: response + ? { + body: { + bytes: response.encodedDataLength, + }, + mime_type: response.mimeType, + status_code: response.status, + } + : undefined, + }, + tls: tls + ? { + cipher: tls.cipher, + client: { + issuer: tls.issuer, + not_after: new Date(tls.validTo).toISOString(), + not_before: new Date(tls.validFrom).toISOString(), + }, + } + : undefined, + }; +} export default class JSONReporter extends BaseReporter { _registerListeners() { this.runner.on('journey:register', ({ journey }) => { - this.writeJSON('journey/register', journey, {}); + this.writeJSON({ + type: 'journey/register', + journey, + }); }); this.runner.on('journey:start', ({ journey, timestamp, params }) => { - this.writeJSON('journey/start', journey, { + this.writeJSON({ + type: 'journey/start', + journey, timestamp, payload: { params, source: journey.callback.toString }, }); @@ -58,12 +157,16 @@ export default class JSONReporter extends BaseReporter { metrics, }) => { if (screenshot) { - this.writeJSON('step/screenshot', journey, { + this.writeJSON({ + type: 'step/screenshot', + journey, step, blob: screenshot, }); } - this.writeJSON('step/end', journey, { + this.writeJSON({ + type: 'step/end', + journey, step, url, error, @@ -93,8 +196,11 @@ export default class JSONReporter extends BaseReporter { }) => { if (networkinfo) { networkinfo.forEach(ni => { - this.writeJSON('journey/network_info', journey, { + this.writeJSON({ + type: 'journey/network_info', + journey, timestamp: ni.timestamp, + root_fields: formatECSFields(ni), step: ni.step, payload: snakeCaseKeys(ni), }); @@ -103,7 +209,9 @@ export default class JSONReporter extends BaseReporter { if (filmstrips) { // Write each filmstrip separately so that we don't get documents that are too large filmstrips.forEach((strip, index) => { - this.writeJSON('journey/filmstrips', journey, { + this.writeJSON({ + type: 'journey/filmstrips', + journey, payload: { index, ...{ @@ -117,14 +225,18 @@ export default class JSONReporter extends BaseReporter { } if (browserconsole) { browserconsole.forEach(({ timestamp, text, type, step }) => { - this.writeJSON('journey/browserconsole', journey, { + this.writeJSON({ + type: 'journey/browserconsole', + journey, timestamp, step, payload: { text, type }, }); }); } - this.writeJSON('journey/end', journey, { + this.writeJSON({ + type: 'journey/end', + journey, error, payload: { start, @@ -141,25 +253,17 @@ export default class JSONReporter extends BaseReporter { // it before passing it into this function! // The payload field is an un-indexed field with no ES mapping, so users can put arbitary structured // stuff in there - writeJSON( - type: string, - journey: Journey, - { - timestamp, - step, - error, - payload, - blob, - url, - }: { - timestamp?: number; - url?: string; - step?: Partial; - error?: Error; - payload?: { [key: string]: any }; - blob?: string; - } - ) { + writeJSON({ + journey, + type, + timestamp, + step, + root_fields, + error, + payload, + blob, + url, + }: OutputFields) { this.write({ type, '@timestamp': timestamp || getTimestamp(), @@ -173,11 +277,12 @@ export default class JSONReporter extends BaseReporter { index: step.index, } : undefined, + root_fields: { ...(root_fields || {}), ...getMetadata() }, payload, blob, error: formatError(error), url, - package_version: programVersion, + package_version: version, }); } } From f8559f8b741259bb72ae7f939082d0d4a57182be Mon Sep 17 00:00:00 2001 From: vigneshshanmugam Date: Thu, 4 Feb 2021 17:01:41 -0800 Subject: [PATCH 2/4] chore: add tests --- __tests__/fixtures/networkinfo.ts | 200 ++++++++++++++++++ .../reporters/__snapshots__/json.test.ts.snap | 100 ++++++++- __tests__/reporters/json.test.ts | 27 ++- src/common_types.ts | 2 +- src/core/runner.ts | 2 +- src/plugins/network.ts | 30 ++- src/reporters/json.ts | 14 +- 7 files changed, 345 insertions(+), 30 deletions(-) create mode 100644 __tests__/fixtures/networkinfo.ts diff --git a/__tests__/fixtures/networkinfo.ts b/__tests__/fixtures/networkinfo.ts new file mode 100644 index 00000000..f52f8037 --- /dev/null +++ b/__tests__/fixtures/networkinfo.ts @@ -0,0 +1,200 @@ +/** + * MIT License + * + * Copyright (c) 2020-present, Elastic NV + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ + +export const NETWORK_INFO = [ + { + step: { + name: 'go to app', + index: 1, + }, + timestamp: 1612482095137858.2, + url: 'https://vigneshh.in/', + request: { + url: 'https://vigneshh.in/', + method: 'GET', + headers: { + 'Upgrade-Insecure-Requests': '1', + 'User-Agent': + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/90.0.4392.0 Safari/537.36', + ':method': 'GET', + ':path': '/', + 'user-agent': + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/90.0.4392.0 Safari/537.36', + 'accept-encoding': 'gzip, deflate, br', + }, + mixedContentType: 'none', + initialPriority: 'VeryHigh', + referrerPolicy: 'strict-origin-when-cross-origin', + }, + type: 'Document', + method: 'GET', + requestSentTime: 2355505.10179, + isNavigationRequest: true, + status: 200, + loadEndTime: 2355505.448326, + responseReceivedTime: 2355505.4514, + response: { + url: 'https://vigneshh.in/', + status: 200, + statusText: '', + mimeType: 'text/html', + connectionReused: false, + connectionId: 12, + remoteIPAddress: '[2606:4700:3035::ac43:83e0]', + remotePort: 443, + fromDiskCache: false, + fromServiceWorker: false, + fromPrefetchCache: false, + encodedDataLength: 3329, + responseTime: 1612482095476.922, + protocol: 'h2', + securityState: 'secure', + securityDetails: { + protocol: 'TLS 1.3', + keyExchange: '', + keyExchangeGroup: 'X25519', + cipher: 'AES_128_GCM', + certificateId: 0, + subjectName: 'sni.cloudflaressl.com', + sanList: [Array], + issuer: 'Cloudflare Inc ECC CA-3', + validFrom: 1595980800, + validTo: 1627560000, + signedCertificateTimestampList: [], + certificateTransparencyCompliance: 'unknown', + }, + }, + timings: { + blocked: 2.080999780446291, + queueing: 2.145999576896429, + proxy: -1, + dns: 45.81400007009506, + ssl: 61.003000009804964, + connect: 78.5130001604557, + send: 0.5449997261166573, + wait: 212.02100021764636, + receive: 3.9220000617206097, + total: 346.5359997935593, + }, + }, + { + step: { + name: 'go to app', + index: 1, + }, + timestamp: 1612482095517516, + url: 'https://vigneshh.in/static/main.js', + request: { + url: 'https://vigneshh.in/static/main.js', + method: 'GET', + headers: { + Referer: 'https://vigneshh.in/', + 'User-Agent': + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/90.0.4392.0 Safari/537.36', + }, + mixedContentType: 'none', + initialPriority: 'Low', + referrerPolicy: 'strict-origin-when-cross-origin', + }, + type: 'Script', + method: 'GET', + requestSentTime: 2355505.472502, + isNavigationRequest: false, + status: 200, + loadEndTime: 2355505.589043, + responseReceivedTime: 2355505.603987, + response: { + url: 'https://vigneshh.in/static/main.js', + status: 200, + statusText: '', + headers: {}, + mimeType: 'application/javascript', + connectionReused: true, + connectionId: 12, + remoteIPAddress: '[2606:4700:3035::ac43:83e0]', + remotePort: 443, + fromDiskCache: false, + fromServiceWorker: false, + fromPrefetchCache: false, + encodedDataLength: 425, + responseTime: 1612482095622.864, + protocol: 'h2', + securityState: 'secure', + securityDetails: { + protocol: 'TLS 1.3', + keyExchange: '', + keyExchangeGroup: 'X25519', + cipher: 'AES_128_GCM', + certificateId: 0, + subjectName: 'sni.cloudflaressl.com', + sanList: [Array], + issuer: 'Cloudflare Inc ECC CA-3', + validFrom: 1595980800, + validTo: 1627560000, + signedCertificateTimestampList: [], + certificateTransparencyCompliance: 'unknown', + }, + }, + timings: { + blocked: 1.058999914675951, + queueing: 46.335999853909016, + proxy: -1, + dns: -1, + ssl: -1, + connect: -1, + send: 0.2100002020597458, + wait: 68.73499974608421, + receive: 0.2009999006986618, + total: 116.54099961742759, + }, + }, + { + timestamp: 1612482095713278.2, + url: 'https://www.google-analytics.com/', + request: { + url: 'https://www.google-analytics.com/', + method: 'POST', + headers: { + Referer: 'https://vigneshh.in/', + 'User-Agent': + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/90.0.4392.0 Safari/537.36', + 'Content-Type': 'text/plain', + }, + hasPostData: true, + mixedContentType: 'none', + initialPriority: 'High', + referrerPolicy: 'strict-origin-when-cross-origin', + }, + type: 'XHR', + method: 'POST', + requestSentTime: 2355505.677688, + isNavigationRequest: false, + status: 0, + loadEndTime: -1, + responseReceivedTime: -1, + response: null, + timings: null, + }, +]; diff --git a/__tests__/reporters/__snapshots__/json.test.ts.snap b/__tests__/reporters/__snapshots__/json.test.ts.snap index 8599d406..d5e0082a 100644 --- a/__tests__/reporters/__snapshots__/json.test.ts.snap +++ b/__tests__/reporters/__snapshots__/json.test.ts.snap @@ -1,12 +1,98 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP +exports[`json reporter formats network fields in ECS format 1`] = ` +Object { + "http": Object { + "request": Object { + "body": Object { + "bytes": 0, + "content": "", + }, + "method": "GET", + "referrer": undefined, + }, + "response": Object { + "body": Object { + "bytes": 3329, + }, + "mime_type": "text/html", + "status_code": 200, + }, + "version": 2, + }, + "tls": Object { + "cipher": "AES_128_GCM", + "client": Object { + "issuer": "Cloudflare Inc ECC CA-3", + "not_after": "1970-01-19T20:06:00.000Z", + "not_before": "1970-01-19T11:19:40.800Z", + }, + }, + "url": "https://vigneshh.in/", + "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/90.0.4392.0 Safari/537.36", +} +`; + +exports[`json reporter formats network fields in ECS format 2`] = ` +Object { + "http": Object { + "request": Object { + "body": Object { + "bytes": 0, + "content": "", + }, + "method": "GET", + "referrer": "https://vigneshh.in/", + }, + "response": Object { + "body": Object { + "bytes": 425, + }, + "mime_type": "application/javascript", + "status_code": 200, + }, + "version": 2, + }, + "tls": Object { + "cipher": "AES_128_GCM", + "client": Object { + "issuer": "Cloudflare Inc ECC CA-3", + "not_after": "1970-01-19T20:06:00.000Z", + "not_before": "1970-01-19T11:19:40.800Z", + }, + }, + "url": "https://vigneshh.in/static/main.js", + "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/90.0.4392.0 Safari/537.36", +} +`; + +exports[`json reporter formats network fields in ECS format 3`] = ` +Object { + "http": Object { + "request": Object { + "body": Object { + "bytes": 0, + "content": "", + }, + "method": "POST", + "referrer": "https://vigneshh.in/", + }, + "response": undefined, + "version": undefined, + }, + "tls": undefined, + "url": "https://www.google-analytics.com/", + "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/90.0.4392.0 Safari/537.36", +} +`; + exports[`json reporter writes each step as NDJSON to the FD 1`] = ` -"{\\"type\\":\\"journey/register\\",\\"@timestamp\\":1600300800000000,\\"journey\\":{\\"name\\":\\"j1\\",\\"id\\":\\"j1\\"},\\"package_version\\":\\"0.0.1\\"} -{\\"type\\":\\"journey/start\\",\\"@timestamp\\":1600300800000000,\\"journey\\":{\\"name\\":\\"j1\\",\\"id\\":\\"j1\\"},\\"payload\\":{\\"params\\":{}},\\"package_version\\":\\"0.0.1\\"} -{\\"type\\":\\"step/screenshot\\",\\"@timestamp\\":1600300800000000,\\"journey\\":{\\"name\\":\\"j1\\",\\"id\\":\\"j1\\"},\\"step\\":{\\"name\\":\\"s1\\",\\"index\\":1},\\"blob\\":\\"dummy\\",\\"package_version\\":\\"0.0.1\\"} -{\\"type\\":\\"step/end\\",\\"@timestamp\\":1600300800000000,\\"journey\\":{\\"name\\":\\"j1\\",\\"id\\":\\"j1\\"},\\"step\\":{\\"name\\":\\"s1\\",\\"index\\":1},\\"payload\\":{\\"source\\":\\"async () => { }\\",\\"start\\":0,\\"end\\":10,\\"url\\":\\"dummy\\",\\"status\\":\\"succeeded\\"},\\"url\\":\\"dummy\\",\\"package_version\\":\\"0.0.1\\"} -{\\"type\\":\\"journey/network_info\\",\\"@timestamp\\":1600300800000000,\\"journey\\":{\\"name\\":\\"j1\\",\\"id\\":\\"j1\\"},\\"payload\\":{\\"request\\":{},\\"response\\":{},\\"is_navigation_request\\":true},\\"package_version\\":\\"0.0.1\\"} -{\\"type\\":\\"journey/filmstrips\\",\\"@timestamp\\":1600300800000000,\\"journey\\":{\\"name\\":\\"j1\\",\\"id\\":\\"j1\\"},\\"payload\\":{\\"index\\":0,\\"startTime\\":392583.998697,\\"ts\\":392583998697},\\"blob\\":\\"dummy\\",\\"package_version\\":\\"0.0.1\\"} -{\\"type\\":\\"journey/end\\",\\"@timestamp\\":1600300800000000,\\"journey\\":{\\"name\\":\\"j1\\",\\"id\\":\\"j1\\"},\\"payload\\":{\\"start\\":0,\\"end\\":11,\\"status\\":\\"succeeded\\"},\\"package_version\\":\\"0.0.1\\"} +"{\\"type\\":\\"journey/register\\",\\"@timestamp\\":1600300800000000,\\"journey\\":{\\"name\\":\\"j1\\",\\"id\\":\\"j1\\"},\\"root_fields\\":{\\"process\\":{\\"pid\\":110,\\"ppid\\":110,\\"title\\":\\"node\\",\\"args\\":[\\"npx\\",\\"@elastic/synthetics\\",\\"suites\\"]},\\"os\\":{\\"platform\\":\\"darwin\\"},\\"package\\":{\\"name\\":\\"@elastic/synthetics\\",\\"version\\":\\"0.0.1\\"}},\\"package_version\\":\\"0.0.1\\"} +{\\"type\\":\\"journey/start\\",\\"@timestamp\\":1600300800000000,\\"journey\\":{\\"name\\":\\"j1\\",\\"id\\":\\"j1\\"},\\"root_fields\\":{\\"process\\":{\\"pid\\":110,\\"ppid\\":110,\\"title\\":\\"node\\",\\"args\\":[\\"npx\\",\\"@elastic/synthetics\\",\\"suites\\"]},\\"os\\":{\\"platform\\":\\"darwin\\"},\\"package\\":{\\"name\\":\\"@elastic/synthetics\\",\\"version\\":\\"0.0.1\\"}},\\"payload\\":{\\"params\\":{}},\\"package_version\\":\\"0.0.1\\"} +{\\"type\\":\\"step/screenshot\\",\\"@timestamp\\":1600300800000000,\\"journey\\":{\\"name\\":\\"j1\\",\\"id\\":\\"j1\\"},\\"step\\":{\\"name\\":\\"s1\\",\\"index\\":1},\\"root_fields\\":{\\"process\\":{\\"pid\\":110,\\"ppid\\":110,\\"title\\":\\"node\\",\\"args\\":[\\"npx\\",\\"@elastic/synthetics\\",\\"suites\\"]},\\"os\\":{\\"platform\\":\\"darwin\\"},\\"package\\":{\\"name\\":\\"@elastic/synthetics\\",\\"version\\":\\"0.0.1\\"}},\\"blob\\":\\"dummy\\",\\"package_version\\":\\"0.0.1\\"} +{\\"type\\":\\"step/end\\",\\"@timestamp\\":1600300800000000,\\"journey\\":{\\"name\\":\\"j1\\",\\"id\\":\\"j1\\"},\\"step\\":{\\"name\\":\\"s1\\",\\"index\\":1},\\"root_fields\\":{\\"process\\":{\\"pid\\":110,\\"ppid\\":110,\\"title\\":\\"node\\",\\"args\\":[\\"npx\\",\\"@elastic/synthetics\\",\\"suites\\"]},\\"os\\":{\\"platform\\":\\"darwin\\"},\\"package\\":{\\"name\\":\\"@elastic/synthetics\\",\\"version\\":\\"0.0.1\\"}},\\"payload\\":{\\"source\\":\\"async () => { }\\",\\"start\\":0,\\"end\\":10,\\"url\\":\\"dummy\\",\\"status\\":\\"succeeded\\"},\\"url\\":\\"dummy\\",\\"package_version\\":\\"0.0.1\\"} +{\\"type\\":\\"journey/network_info\\",\\"@timestamp\\":1600300800000000,\\"journey\\":{\\"name\\":\\"j1\\",\\"id\\":\\"j1\\"},\\"root_fields\\":{\\"http\\":{\\"request\\":{\\"body\\":{\\"bytes\\":0,\\"content\\":\\"\\"}}},\\"process\\":{\\"pid\\":110,\\"ppid\\":110,\\"title\\":\\"node\\",\\"args\\":[\\"npx\\",\\"@elastic/synthetics\\",\\"suites\\"]},\\"os\\":{\\"platform\\":\\"darwin\\"},\\"package\\":{\\"name\\":\\"@elastic/synthetics\\",\\"version\\":\\"0.0.1\\"}},\\"payload\\":{\\"request\\":{},\\"is_navigation_request\\":true},\\"package_version\\":\\"0.0.1\\"} +{\\"type\\":\\"journey/filmstrips\\",\\"@timestamp\\":1600300800000000,\\"journey\\":{\\"name\\":\\"j1\\",\\"id\\":\\"j1\\"},\\"root_fields\\":{\\"process\\":{\\"pid\\":110,\\"ppid\\":110,\\"title\\":\\"node\\",\\"args\\":[\\"npx\\",\\"@elastic/synthetics\\",\\"suites\\"]},\\"os\\":{\\"platform\\":\\"darwin\\"},\\"package\\":{\\"name\\":\\"@elastic/synthetics\\",\\"version\\":\\"0.0.1\\"}},\\"payload\\":{\\"index\\":0,\\"startTime\\":392583.998697,\\"ts\\":392583998697},\\"blob\\":\\"dummy\\",\\"package_version\\":\\"0.0.1\\"} +{\\"type\\":\\"journey/end\\",\\"@timestamp\\":1600300800000000,\\"journey\\":{\\"name\\":\\"j1\\",\\"id\\":\\"j1\\"},\\"root_fields\\":{\\"process\\":{\\"pid\\":110,\\"ppid\\":110,\\"title\\":\\"node\\",\\"args\\":[\\"npx\\",\\"@elastic/synthetics\\",\\"suites\\"]},\\"os\\":{\\"platform\\":\\"darwin\\"},\\"package\\":{\\"name\\":\\"@elastic/synthetics\\",\\"version\\":\\"0.0.1\\"}},\\"payload\\":{\\"start\\":0,\\"end\\":11,\\"status\\":\\"succeeded\\"},\\"package_version\\":\\"0.0.1\\"} " `; diff --git a/__tests__/reporters/json.test.ts b/__tests__/reporters/json.test.ts index 95760d6f..58bebd23 100644 --- a/__tests__/reporters/json.test.ts +++ b/__tests__/reporters/json.test.ts @@ -25,9 +25,10 @@ import fs from 'fs'; import { step, journey } from '../../src/core'; -import JSONReporter from '../../src/reporters/json'; +import JSONReporter, { formatNetworkFields } from '../../src/reporters/json'; import * as helpers from '../../src/helpers'; import Runner from '../../src/core/runner'; +import { NETWORK_INFO } from '../fixtures/networkinfo'; /** * Mock package version to avoid breaking JSON payload @@ -35,7 +36,7 @@ import Runner from '../../src/core/runner'; */ jest.mock( '../../package.json', - jest.fn(() => ({ version: '0.0.1' })) + jest.fn(() => ({ version: '0.0.1', name: '@elastic/synthetics' })) ); describe('json reporter', () => { @@ -81,6 +82,16 @@ describe('json reporter', () => { }; it('writes each step as NDJSON to the FD', async () => { + const originalProcess = global.process; + global.process = { + ...originalProcess, + pid: 110, + ppid: 110, + title: 'node', + platform: 'darwin', + argv: ['npx', '@elastic/synthetics', 'suites'], + }; + runner.emit('journey:register', { journey: j1, }); @@ -100,7 +111,6 @@ describe('json reporter', () => { }); runner.emit('journey:end', { journey: j1, - params: {}, status: 'succeeded', start: 0, end: 11, @@ -114,16 +124,22 @@ describe('json reporter', () => { networkinfo: [ { request: {}, - response: {}, + response: undefined, isNavigationRequest: true, } as any, ], }); runner.emit('end', 'done'); - + global.process = originalProcess; expect((await readAndCloseStream()).toString()).toMatchSnapshot(); }); + it('formats network fields in ECS format', async () => { + for (const network of NETWORK_INFO) { + expect(formatNetworkFields(network as any)).toMatchSnapshot(); + } + }); + it('writes step errors to the top level', async () => { const myErr = new Error('myError'); @@ -151,7 +167,6 @@ describe('json reporter', () => { journey: j1, start: 0, end: 1, - params: {}, status: 'failed', error: myErr, }); diff --git a/src/common_types.ts b/src/common_types.ts index 51592df9..551d3ab6 100644 --- a/src/common_types.ts +++ b/src/common_types.ts @@ -46,7 +46,7 @@ export type NetworkInfo = { method: string; type: string; request: Protocol.Network.Request; - response: Protocol.Network.Response | null; + response?: Protocol.Network.Response; isNavigationRequest: boolean; requestSentTime: number; loadEndTime: number; diff --git a/src/core/runner.ts b/src/core/runner.ts index e06be0fa..675ced4a 100644 --- a/src/core/runner.ts +++ b/src/core/runner.ts @@ -59,7 +59,7 @@ export type RunOptions = { }; type BaseContext = { - params: RunParamaters; + params?: RunParamaters; start: number; end?: number; }; diff --git a/src/plugins/network.ts b/src/plugins/network.ts index e2d5671f..49045bb0 100644 --- a/src/plugins/network.ts +++ b/src/plugins/network.ts @@ -42,6 +42,10 @@ export class NetworkManager { 'Network.requestWillBeSent', this._onRequestWillBeSent.bind(this) ); + client.on( + 'Network.requestWillBeSentExtraInfo', + this._onRequestWillBeSentExtraInfo.bind(this) + ); client.on('Network.responseReceived', this._onResponseReceived.bind(this)); client.on('Network.loadingFinished', this._onLoadingFinished.bind(this)); client.on('Network.loadingFailed', this._onLoadingFailed.bind(this)); @@ -107,6 +111,23 @@ export class NetworkManager { }); } + _onRequestWillBeSentExtraInfo( + event: Protocol.Network.requestWillBeSentExtraInfoPayload + ) { + const { requestId, headers } = event; + const record = this.waterfallMap.get(requestId); + if (!record) { + return; + } + /** + * Enhance request headers with additional information + */ + record.request.headers = { + ...record.request.headers, + ...headers, + }; + } + _onResponseReceived(event: Protocol.Network.responseReceivedPayload) { const { requestId, response, timestamp } = event; const record = this.waterfallMap.get(requestId); @@ -118,15 +139,6 @@ export class NetworkManager { response, responseReceivedTime: timestamp, }); - /** - * Enhance request headers with additional information - */ - if (response.requestHeaders) { - record.request.headers = { - ...record.request.headers, - ...response.requestHeaders, - }; - } } _onLoadingFinished(event: Protocol.Network.loadingFinishedPayload) { diff --git a/src/reporters/json.ts b/src/reporters/json.ts index 82ede33d..be1f6bca 100644 --- a/src/reporters/json.ts +++ b/src/reporters/json.ts @@ -72,18 +72,20 @@ function getMetadata() { }; } -function formatHttpVersion(protocol: string) { +function formatHTTPVersion(protocol: string) { if (protocol === 'h2') { return 2; } else if (protocol === 'http/1.1') { return 1.1; } else if (protocol === 'http/1.0') { return 1.0; + } else if (protocol && protocol.startsWith('h3')) { + return 3; } return; } -function formatECSFields(network: NetworkInfo) { +export function formatNetworkFields(network: NetworkInfo) { const { request, response, url } = network; const postdata = request.postData || ''; const tls = response?.securityDetails; @@ -91,16 +93,16 @@ function formatECSFields(network: NetworkInfo) { return { // URL and USER AGENT would be parsed and mapped by heartbeat url, - user_agent: request.headers['user_agent'], + user_agent: request.headers?.['User-Agent'], http: { - version: formatHttpVersion(response?.protocol), + version: formatHTTPVersion(response?.protocol), request: { body: { bytes: postdata.length, content: postdata, }, method: request.method, - referrer: request.headers['referer'], + referrer: request.headers?.Referer, }, response: response ? { @@ -200,7 +202,7 @@ export default class JSONReporter extends BaseReporter { type: 'journey/network_info', journey, timestamp: ni.timestamp, - root_fields: formatECSFields(ni), + root_fields: formatNetworkFields(ni), step: ni.step, payload: snakeCaseKeys(ni), }); From 0f1267a39255b44e8470a86ad3e7a223b3370ccd Mon Sep 17 00:00:00 2001 From: vigneshshanmugam Date: Mon, 8 Feb 2021 12:02:55 -0800 Subject: [PATCH 3/4] chore: address review comments --- .../reporters/__snapshots__/json.test.ts.snap | 142 +++++++++++++++--- __tests__/reporters/json.test.ts | 10 +- src/reporters/json.ts | 104 ++++++++----- 3 files changed, 193 insertions(+), 63 deletions(-) diff --git a/__tests__/reporters/__snapshots__/json.test.ts.snap b/__tests__/reporters/__snapshots__/json.test.ts.snap index d5e0082a..a2bc686c 100644 --- a/__tests__/reporters/__snapshots__/json.test.ts.snap +++ b/__tests__/reporters/__snapshots__/json.test.ts.snap @@ -8,25 +8,75 @@ Object { "bytes": 0, "content": "", }, + "headers": Object { + "accept_encoding": "gzip, deflate, br", + "method": "GET", + "path": "/", + "upgrade_insecure_requests": "1", + "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/90.0.4392.0 Safari/537.36", + }, + "initial_priority": "VeryHigh", "method": "GET", + "mixed_content_type": "none", "referrer": undefined, + "referrer_policy": "strict-origin-when-cross-origin", + "url": "https://vigneshh.in/", }, "response": Object { "body": Object { "bytes": 3329, }, + "connection_id": 12, + "connection_reused": false, + "encoded_data_length": 3329, + "from_disk_cache": false, + "from_prefetch_cache": false, + "from_service_worker": false, "mime_type": "text/html", + "protocol": "h2", + "remote_i_p_address": "[2606:4700:3035::ac43:83e0]", + "remote_port": 443, + "response_time": 1612482095476.922, + "security_details": Object { + "certificate_id": 0, + "certificate_transparency_compliance": "unknown", + "cipher": "AES_128_GCM", + "issuer": "Cloudflare Inc ECC CA-3", + "key_exchange": "", + "key_exchange_group": "X25519", + "protocol": "TLS 1.3", + "san_list": Array [ + [Function], + ], + "signed_certificate_timestamp_list": Array [], + "subject_name": "sni.cloudflaressl.com", + "valid_from": 1595980800, + "valid_to": 1627560000, + }, + "security_state": "secure", + "status": 200, "status_code": 200, + "status_text": "", + "url": "https://vigneshh.in/", }, "version": 2, }, "tls": Object { - "cipher": "AES_128_GCM", - "client": Object { - "issuer": "Cloudflare Inc ECC CA-3", - "not_after": "1970-01-19T20:06:00.000Z", - "not_before": "1970-01-19T11:19:40.800Z", + "cipher": "AES_128_GCM_X25519", + "server": Object { + "x509": Object { + "issuer": Object { + "common_name": "Cloudflare Inc ECC CA-3", + }, + "not_after": "2021-07-29T12:00:00.000Z", + "not_before": "2020-07-29T00:00:00.000Z", + "subject": Object { + "common_name": "sni.cloudflaressl.com", + }, + }, }, + "version": "1.3", + "version_protocol": "tls", }, "url": "https://vigneshh.in/", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/90.0.4392.0 Safari/537.36", @@ -41,25 +91,73 @@ Object { "bytes": 0, "content": "", }, + "headers": Object { + "referer": "https://vigneshh.in/", + "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/90.0.4392.0 Safari/537.36", + }, + "initial_priority": "Low", "method": "GET", + "mixed_content_type": "none", "referrer": "https://vigneshh.in/", + "referrer_policy": "strict-origin-when-cross-origin", + "url": "https://vigneshh.in/static/main.js", }, "response": Object { "body": Object { "bytes": 425, }, + "connection_id": 12, + "connection_reused": true, + "encoded_data_length": 425, + "from_disk_cache": false, + "from_prefetch_cache": false, + "from_service_worker": false, + "headers": Object {}, "mime_type": "application/javascript", + "protocol": "h2", + "remote_i_p_address": "[2606:4700:3035::ac43:83e0]", + "remote_port": 443, + "response_time": 1612482095622.864, + "security_details": Object { + "certificate_id": 0, + "certificate_transparency_compliance": "unknown", + "cipher": "AES_128_GCM", + "issuer": "Cloudflare Inc ECC CA-3", + "key_exchange": "", + "key_exchange_group": "X25519", + "protocol": "TLS 1.3", + "san_list": Array [ + [Function], + ], + "signed_certificate_timestamp_list": Array [], + "subject_name": "sni.cloudflaressl.com", + "valid_from": 1595980800, + "valid_to": 1627560000, + }, + "security_state": "secure", + "status": 200, "status_code": 200, + "status_text": "", + "url": "https://vigneshh.in/static/main.js", }, "version": 2, }, "tls": Object { - "cipher": "AES_128_GCM", - "client": Object { - "issuer": "Cloudflare Inc ECC CA-3", - "not_after": "1970-01-19T20:06:00.000Z", - "not_before": "1970-01-19T11:19:40.800Z", + "cipher": "AES_128_GCM_X25519", + "server": Object { + "x509": Object { + "issuer": Object { + "common_name": "Cloudflare Inc ECC CA-3", + }, + "not_after": "2021-07-29T12:00:00.000Z", + "not_before": "2020-07-29T00:00:00.000Z", + "subject": Object { + "common_name": "sni.cloudflaressl.com", + }, + }, }, + "version": "1.3", + "version_protocol": "tls", }, "url": "https://vigneshh.in/static/main.js", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/90.0.4392.0 Safari/537.36", @@ -74,8 +172,18 @@ Object { "bytes": 0, "content": "", }, + "has_post_data": true, + "headers": Object { + "content_type": "text/plain", + "referer": "https://vigneshh.in/", + "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/90.0.4392.0 Safari/537.36", + }, + "initial_priority": "High", "method": "POST", + "mixed_content_type": "none", "referrer": "https://vigneshh.in/", + "referrer_policy": "strict-origin-when-cross-origin", + "url": "https://www.google-analytics.com/", }, "response": undefined, "version": undefined, @@ -87,12 +195,12 @@ Object { `; exports[`json reporter writes each step as NDJSON to the FD 1`] = ` -"{\\"type\\":\\"journey/register\\",\\"@timestamp\\":1600300800000000,\\"journey\\":{\\"name\\":\\"j1\\",\\"id\\":\\"j1\\"},\\"root_fields\\":{\\"process\\":{\\"pid\\":110,\\"ppid\\":110,\\"title\\":\\"node\\",\\"args\\":[\\"npx\\",\\"@elastic/synthetics\\",\\"suites\\"]},\\"os\\":{\\"platform\\":\\"darwin\\"},\\"package\\":{\\"name\\":\\"@elastic/synthetics\\",\\"version\\":\\"0.0.1\\"}},\\"package_version\\":\\"0.0.1\\"} -{\\"type\\":\\"journey/start\\",\\"@timestamp\\":1600300800000000,\\"journey\\":{\\"name\\":\\"j1\\",\\"id\\":\\"j1\\"},\\"root_fields\\":{\\"process\\":{\\"pid\\":110,\\"ppid\\":110,\\"title\\":\\"node\\",\\"args\\":[\\"npx\\",\\"@elastic/synthetics\\",\\"suites\\"]},\\"os\\":{\\"platform\\":\\"darwin\\"},\\"package\\":{\\"name\\":\\"@elastic/synthetics\\",\\"version\\":\\"0.0.1\\"}},\\"payload\\":{\\"params\\":{}},\\"package_version\\":\\"0.0.1\\"} -{\\"type\\":\\"step/screenshot\\",\\"@timestamp\\":1600300800000000,\\"journey\\":{\\"name\\":\\"j1\\",\\"id\\":\\"j1\\"},\\"step\\":{\\"name\\":\\"s1\\",\\"index\\":1},\\"root_fields\\":{\\"process\\":{\\"pid\\":110,\\"ppid\\":110,\\"title\\":\\"node\\",\\"args\\":[\\"npx\\",\\"@elastic/synthetics\\",\\"suites\\"]},\\"os\\":{\\"platform\\":\\"darwin\\"},\\"package\\":{\\"name\\":\\"@elastic/synthetics\\",\\"version\\":\\"0.0.1\\"}},\\"blob\\":\\"dummy\\",\\"package_version\\":\\"0.0.1\\"} -{\\"type\\":\\"step/end\\",\\"@timestamp\\":1600300800000000,\\"journey\\":{\\"name\\":\\"j1\\",\\"id\\":\\"j1\\"},\\"step\\":{\\"name\\":\\"s1\\",\\"index\\":1},\\"root_fields\\":{\\"process\\":{\\"pid\\":110,\\"ppid\\":110,\\"title\\":\\"node\\",\\"args\\":[\\"npx\\",\\"@elastic/synthetics\\",\\"suites\\"]},\\"os\\":{\\"platform\\":\\"darwin\\"},\\"package\\":{\\"name\\":\\"@elastic/synthetics\\",\\"version\\":\\"0.0.1\\"}},\\"payload\\":{\\"source\\":\\"async () => { }\\",\\"start\\":0,\\"end\\":10,\\"url\\":\\"dummy\\",\\"status\\":\\"succeeded\\"},\\"url\\":\\"dummy\\",\\"package_version\\":\\"0.0.1\\"} -{\\"type\\":\\"journey/network_info\\",\\"@timestamp\\":1600300800000000,\\"journey\\":{\\"name\\":\\"j1\\",\\"id\\":\\"j1\\"},\\"root_fields\\":{\\"http\\":{\\"request\\":{\\"body\\":{\\"bytes\\":0,\\"content\\":\\"\\"}}},\\"process\\":{\\"pid\\":110,\\"ppid\\":110,\\"title\\":\\"node\\",\\"args\\":[\\"npx\\",\\"@elastic/synthetics\\",\\"suites\\"]},\\"os\\":{\\"platform\\":\\"darwin\\"},\\"package\\":{\\"name\\":\\"@elastic/synthetics\\",\\"version\\":\\"0.0.1\\"}},\\"payload\\":{\\"request\\":{},\\"is_navigation_request\\":true},\\"package_version\\":\\"0.0.1\\"} -{\\"type\\":\\"journey/filmstrips\\",\\"@timestamp\\":1600300800000000,\\"journey\\":{\\"name\\":\\"j1\\",\\"id\\":\\"j1\\"},\\"root_fields\\":{\\"process\\":{\\"pid\\":110,\\"ppid\\":110,\\"title\\":\\"node\\",\\"args\\":[\\"npx\\",\\"@elastic/synthetics\\",\\"suites\\"]},\\"os\\":{\\"platform\\":\\"darwin\\"},\\"package\\":{\\"name\\":\\"@elastic/synthetics\\",\\"version\\":\\"0.0.1\\"}},\\"payload\\":{\\"index\\":0,\\"startTime\\":392583.998697,\\"ts\\":392583998697},\\"blob\\":\\"dummy\\",\\"package_version\\":\\"0.0.1\\"} -{\\"type\\":\\"journey/end\\",\\"@timestamp\\":1600300800000000,\\"journey\\":{\\"name\\":\\"j1\\",\\"id\\":\\"j1\\"},\\"root_fields\\":{\\"process\\":{\\"pid\\":110,\\"ppid\\":110,\\"title\\":\\"node\\",\\"args\\":[\\"npx\\",\\"@elastic/synthetics\\",\\"suites\\"]},\\"os\\":{\\"platform\\":\\"darwin\\"},\\"package\\":{\\"name\\":\\"@elastic/synthetics\\",\\"version\\":\\"0.0.1\\"}},\\"payload\\":{\\"start\\":0,\\"end\\":11,\\"status\\":\\"succeeded\\"},\\"package_version\\":\\"0.0.1\\"} +"{\\"type\\":\\"journey/register\\",\\"@timestamp\\":1600300800000000,\\"journey\\":{\\"name\\":\\"j1\\",\\"id\\":\\"j1\\"},\\"root_fields\\":{\\"os\\":{\\"platform\\":\\"darwin\\"},\\"package\\":{\\"name\\":\\"@elastic/synthetics\\",\\"version\\":\\"0.0.1\\"}},\\"package_version\\":\\"0.0.1\\"} +{\\"type\\":\\"journey/start\\",\\"@timestamp\\":1600300800000000,\\"journey\\":{\\"name\\":\\"j1\\",\\"id\\":\\"j1\\"},\\"root_fields\\":{\\"os\\":{\\"platform\\":\\"darwin\\"},\\"package\\":{\\"name\\":\\"@elastic/synthetics\\",\\"version\\":\\"0.0.1\\"}},\\"payload\\":{\\"params\\":{}},\\"package_version\\":\\"0.0.1\\"} +{\\"type\\":\\"step/screenshot\\",\\"@timestamp\\":1600300800000000,\\"journey\\":{\\"name\\":\\"j1\\",\\"id\\":\\"j1\\"},\\"step\\":{\\"name\\":\\"s1\\",\\"index\\":1},\\"root_fields\\":{\\"os\\":{\\"platform\\":\\"darwin\\"},\\"package\\":{\\"name\\":\\"@elastic/synthetics\\",\\"version\\":\\"0.0.1\\"}},\\"blob\\":\\"dummy\\",\\"package_version\\":\\"0.0.1\\"} +{\\"type\\":\\"step/end\\",\\"@timestamp\\":1600300800000000,\\"journey\\":{\\"name\\":\\"j1\\",\\"id\\":\\"j1\\"},\\"step\\":{\\"name\\":\\"s1\\",\\"index\\":1},\\"root_fields\\":{\\"os\\":{\\"platform\\":\\"darwin\\"},\\"package\\":{\\"name\\":\\"@elastic/synthetics\\",\\"version\\":\\"0.0.1\\"}},\\"payload\\":{\\"source\\":\\"async () => { }\\",\\"start\\":0,\\"end\\":10,\\"url\\":\\"dummy\\",\\"status\\":\\"succeeded\\"},\\"url\\":\\"dummy\\",\\"package_version\\":\\"0.0.1\\"} +{\\"type\\":\\"journey/network_info\\",\\"@timestamp\\":1600300800000000,\\"journey\\":{\\"name\\":\\"j1\\",\\"id\\":\\"j1\\"},\\"root_fields\\":{\\"http\\":{\\"request\\":{\\"body\\":{\\"bytes\\":0,\\"content\\":\\"\\"}}},\\"os\\":{\\"platform\\":\\"darwin\\"},\\"package\\":{\\"name\\":\\"@elastic/synthetics\\",\\"version\\":\\"0.0.1\\"}},\\"payload\\":{\\"request\\":{},\\"is_navigation_request\\":true},\\"package_version\\":\\"0.0.1\\"} +{\\"type\\":\\"journey/filmstrips\\",\\"@timestamp\\":1600300800000000,\\"journey\\":{\\"name\\":\\"j1\\",\\"id\\":\\"j1\\"},\\"root_fields\\":{\\"os\\":{\\"platform\\":\\"darwin\\"},\\"package\\":{\\"name\\":\\"@elastic/synthetics\\",\\"version\\":\\"0.0.1\\"}},\\"payload\\":{\\"index\\":0,\\"startTime\\":392583.998697,\\"ts\\":392583998697},\\"blob\\":\\"dummy\\",\\"package_version\\":\\"0.0.1\\"} +{\\"type\\":\\"journey/end\\",\\"@timestamp\\":1600300800000000,\\"journey\\":{\\"name\\":\\"j1\\",\\"id\\":\\"j1\\"},\\"root_fields\\":{\\"os\\":{\\"platform\\":\\"darwin\\"},\\"package\\":{\\"name\\":\\"@elastic/synthetics\\",\\"version\\":\\"0.0.1\\"}},\\"payload\\":{\\"start\\":0,\\"end\\":11,\\"status\\":\\"succeeded\\"},\\"package_version\\":\\"0.0.1\\"} " `; diff --git a/__tests__/reporters/json.test.ts b/__tests__/reporters/json.test.ts index 58bebd23..1964e6b9 100644 --- a/__tests__/reporters/json.test.ts +++ b/__tests__/reporters/json.test.ts @@ -24,6 +24,7 @@ */ import fs from 'fs'; +import snakeCaseKeys from 'snakecase-keys'; import { step, journey } from '../../src/core'; import JSONReporter, { formatNetworkFields } from '../../src/reporters/json'; import * as helpers from '../../src/helpers'; @@ -82,14 +83,11 @@ describe('json reporter', () => { }; it('writes each step as NDJSON to the FD', async () => { + // Mocking the process in node environment const originalProcess = global.process; global.process = { ...originalProcess, - pid: 110, - ppid: 110, - title: 'node', platform: 'darwin', - argv: ['npx', '@elastic/synthetics', 'suites'], }; runner.emit('journey:register', { @@ -136,7 +134,9 @@ describe('json reporter', () => { it('formats network fields in ECS format', async () => { for (const network of NETWORK_INFO) { - expect(formatNetworkFields(network as any)).toMatchSnapshot(); + expect( + snakeCaseKeys(formatNetworkFields(network as any)) + ).toMatchSnapshot(); } }); diff --git a/src/reporters/json.ts b/src/reporters/json.ts index be1f6bca..af691a80 100644 --- a/src/reporters/json.ts +++ b/src/reporters/json.ts @@ -28,6 +28,7 @@ import { formatError, getTimestamp } from '../helpers'; import { Journey, Step } from '../dsl'; import snakeCaseKeys from 'snakecase-keys'; import { NetworkInfo } from '../common_types'; +import { Protocol } from 'playwright-chromium/types/protocol'; /* eslint-disable @typescript-eslint/no-var-requires */ const { version, name } = require('../../package.json'); @@ -56,12 +57,6 @@ type OutputFields = { function getMetadata() { return { - process: { - pid: process.pid, - ppid: process.ppid, - title: process.title, - args: process.argv, - }, os: { platform: process.platform, }, @@ -72,58 +67,85 @@ function getMetadata() { }; } -function formatHTTPVersion(protocol: string) { +function formatVersion(protocol: string | undefined) { + if (!protocol) { + return; + } if (protocol === 'h2') { return 2; } else if (protocol === 'http/1.1') { return 1.1; } else if (protocol === 'http/1.0') { return 1.0; - } else if (protocol && protocol.startsWith('h3')) { + } else if (protocol.startsWith('h3')) { return 3; } - return; +} + +function formatRequest(request: Protocol.Network.Request) { + const postData = request.postData ? request.postData : ''; + return { + ...request, + body: { + bytes: postData.length, + content: postData, + }, + referrer: request.headers?.Referer, + }; +} + +function formatResponse(response: Protocol.Network.Response) { + if (!response) { + return; + } + return { + ...response, + body: { + bytes: response.encodedDataLength, + }, + status_code: response.status, + }; +} + +function formatTLS(tls: Protocol.Network.SecurityDetails) { + if (!tls) { + return; + } + const cipher = `${tls.keyExchange ? tls.keyExchange + '_' : ''}${ + tls.cipher + }_${tls.keyExchangeGroup}`; + const [name, version] = tls.protocol.toLowerCase().split(' '); + return { + cipher, + server: { + x509: { + issuer: { + common_name: tls.issuer, + }, + subject: { + common_name: tls.subjectName, + }, + not_after: new Date(tls.validTo * 1000).toISOString(), + not_before: new Date(tls.validFrom * 1000).toISOString(), + }, + }, + version_protocol: name, + version: version, + }; } export function formatNetworkFields(network: NetworkInfo) { const { request, response, url } = network; - const postdata = request.postData || ''; - const tls = response?.securityDetails; - return { // URL and USER AGENT would be parsed and mapped by heartbeat url, user_agent: request.headers?.['User-Agent'], http: { - version: formatHTTPVersion(response?.protocol), - request: { - body: { - bytes: postdata.length, - content: postdata, - }, - method: request.method, - referrer: request.headers?.Referer, - }, - response: response - ? { - body: { - bytes: response.encodedDataLength, - }, - mime_type: response.mimeType, - status_code: response.status, - } - : undefined, + version: formatVersion(response?.protocol), + request: formatRequest(request), + response: formatResponse(response), }, - tls: tls - ? { - cipher: tls.cipher, - client: { - issuer: tls.issuer, - not_after: new Date(tls.validTo).toISOString(), - not_before: new Date(tls.validFrom).toISOString(), - }, - } - : undefined, + tls: formatTLS(response?.securityDetails), }; } @@ -202,7 +224,7 @@ export default class JSONReporter extends BaseReporter { type: 'journey/network_info', journey, timestamp: ni.timestamp, - root_fields: formatNetworkFields(ni), + root_fields: snakeCaseKeys(formatNetworkFields(ni)), step: ni.step, payload: snakeCaseKeys(ni), }); From 2d7a8e657209fa4e70587230a787be546484101f Mon Sep 17 00:00:00 2001 From: vigneshshanmugam Date: Tue, 9 Feb 2021 17:18:08 -0800 Subject: [PATCH 4/4] chore: add user agent field --- __tests__/fixtures/networkinfo.ts | 3 +++ .../reporters/__snapshots__/json.test.ts.snap | 20 +++++++++++++++---- __tests__/reporters/json.test.ts | 1 + src/common_types.ts | 6 ++++++ src/plugins/network.ts | 7 ++++++- src/reporters/json.ts | 10 +++++++--- 6 files changed, 39 insertions(+), 8 deletions(-) diff --git a/__tests__/fixtures/networkinfo.ts b/__tests__/fixtures/networkinfo.ts index f52f8037..3aae73ea 100644 --- a/__tests__/fixtures/networkinfo.ts +++ b/__tests__/fixtures/networkinfo.ts @@ -25,6 +25,7 @@ export const NETWORK_INFO = [ { + browser: { name: 'HeadlessChrome', version: '90.0.4392.0' }, step: { name: 'go to app', index: 1, @@ -100,6 +101,7 @@ export const NETWORK_INFO = [ }, }, { + browser: { name: 'HeadlessChrome', version: '90.0.4392.0' }, step: { name: 'go to app', index: 1, @@ -171,6 +173,7 @@ export const NETWORK_INFO = [ }, }, { + browser: { name: 'HeadlessChrome', version: '90.0.4392.0' }, timestamp: 1612482095713278.2, url: 'https://www.google-analytics.com/', request: { diff --git a/__tests__/reporters/__snapshots__/json.test.ts.snap b/__tests__/reporters/__snapshots__/json.test.ts.snap index a2bc686c..377be5b6 100644 --- a/__tests__/reporters/__snapshots__/json.test.ts.snap +++ b/__tests__/reporters/__snapshots__/json.test.ts.snap @@ -79,7 +79,11 @@ Object { "version_protocol": "tls", }, "url": "https://vigneshh.in/", - "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/90.0.4392.0 Safari/537.36", + "user_agent": Object { + "name": "HeadlessChrome", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/90.0.4392.0 Safari/537.36", + "version": "90.0.4392.0", + }, } `; @@ -160,7 +164,11 @@ Object { "version_protocol": "tls", }, "url": "https://vigneshh.in/static/main.js", - "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/90.0.4392.0 Safari/537.36", + "user_agent": Object { + "name": "HeadlessChrome", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/90.0.4392.0 Safari/537.36", + "version": "90.0.4392.0", + }, } `; @@ -190,7 +198,11 @@ Object { }, "tls": undefined, "url": "https://www.google-analytics.com/", - "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/90.0.4392.0 Safari/537.36", + "user_agent": Object { + "name": "HeadlessChrome", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/90.0.4392.0 Safari/537.36", + "version": "90.0.4392.0", + }, } `; @@ -199,7 +211,7 @@ exports[`json reporter writes each step as NDJSON to the FD 1`] = ` {\\"type\\":\\"journey/start\\",\\"@timestamp\\":1600300800000000,\\"journey\\":{\\"name\\":\\"j1\\",\\"id\\":\\"j1\\"},\\"root_fields\\":{\\"os\\":{\\"platform\\":\\"darwin\\"},\\"package\\":{\\"name\\":\\"@elastic/synthetics\\",\\"version\\":\\"0.0.1\\"}},\\"payload\\":{\\"params\\":{}},\\"package_version\\":\\"0.0.1\\"} {\\"type\\":\\"step/screenshot\\",\\"@timestamp\\":1600300800000000,\\"journey\\":{\\"name\\":\\"j1\\",\\"id\\":\\"j1\\"},\\"step\\":{\\"name\\":\\"s1\\",\\"index\\":1},\\"root_fields\\":{\\"os\\":{\\"platform\\":\\"darwin\\"},\\"package\\":{\\"name\\":\\"@elastic/synthetics\\",\\"version\\":\\"0.0.1\\"}},\\"blob\\":\\"dummy\\",\\"package_version\\":\\"0.0.1\\"} {\\"type\\":\\"step/end\\",\\"@timestamp\\":1600300800000000,\\"journey\\":{\\"name\\":\\"j1\\",\\"id\\":\\"j1\\"},\\"step\\":{\\"name\\":\\"s1\\",\\"index\\":1},\\"root_fields\\":{\\"os\\":{\\"platform\\":\\"darwin\\"},\\"package\\":{\\"name\\":\\"@elastic/synthetics\\",\\"version\\":\\"0.0.1\\"}},\\"payload\\":{\\"source\\":\\"async () => { }\\",\\"start\\":0,\\"end\\":10,\\"url\\":\\"dummy\\",\\"status\\":\\"succeeded\\"},\\"url\\":\\"dummy\\",\\"package_version\\":\\"0.0.1\\"} -{\\"type\\":\\"journey/network_info\\",\\"@timestamp\\":1600300800000000,\\"journey\\":{\\"name\\":\\"j1\\",\\"id\\":\\"j1\\"},\\"root_fields\\":{\\"http\\":{\\"request\\":{\\"body\\":{\\"bytes\\":0,\\"content\\":\\"\\"}}},\\"os\\":{\\"platform\\":\\"darwin\\"},\\"package\\":{\\"name\\":\\"@elastic/synthetics\\",\\"version\\":\\"0.0.1\\"}},\\"payload\\":{\\"request\\":{},\\"is_navigation_request\\":true},\\"package_version\\":\\"0.0.1\\"} +{\\"type\\":\\"journey/network_info\\",\\"@timestamp\\":1600300800000000,\\"journey\\":{\\"name\\":\\"j1\\",\\"id\\":\\"j1\\"},\\"root_fields\\":{\\"user_agent\\":{},\\"http\\":{\\"request\\":{\\"body\\":{\\"bytes\\":0,\\"content\\":\\"\\"}}},\\"os\\":{\\"platform\\":\\"darwin\\"},\\"package\\":{\\"name\\":\\"@elastic/synthetics\\",\\"version\\":\\"0.0.1\\"}},\\"payload\\":{\\"request\\":{},\\"is_navigation_request\\":true,\\"browser\\":{}},\\"package_version\\":\\"0.0.1\\"} {\\"type\\":\\"journey/filmstrips\\",\\"@timestamp\\":1600300800000000,\\"journey\\":{\\"name\\":\\"j1\\",\\"id\\":\\"j1\\"},\\"root_fields\\":{\\"os\\":{\\"platform\\":\\"darwin\\"},\\"package\\":{\\"name\\":\\"@elastic/synthetics\\",\\"version\\":\\"0.0.1\\"}},\\"payload\\":{\\"index\\":0,\\"startTime\\":392583.998697,\\"ts\\":392583998697},\\"blob\\":\\"dummy\\",\\"package_version\\":\\"0.0.1\\"} {\\"type\\":\\"journey/end\\",\\"@timestamp\\":1600300800000000,\\"journey\\":{\\"name\\":\\"j1\\",\\"id\\":\\"j1\\"},\\"root_fields\\":{\\"os\\":{\\"platform\\":\\"darwin\\"},\\"package\\":{\\"name\\":\\"@elastic/synthetics\\",\\"version\\":\\"0.0.1\\"}},\\"payload\\":{\\"start\\":0,\\"end\\":11,\\"status\\":\\"succeeded\\"},\\"package_version\\":\\"0.0.1\\"} " diff --git a/__tests__/reporters/json.test.ts b/__tests__/reporters/json.test.ts index 1964e6b9..b0dea979 100644 --- a/__tests__/reporters/json.test.ts +++ b/__tests__/reporters/json.test.ts @@ -124,6 +124,7 @@ describe('json reporter', () => { request: {}, response: undefined, isNavigationRequest: true, + browser: {}, } as any, ], }); diff --git a/src/common_types.ts b/src/common_types.ts index 551d3ab6..6f960c78 100644 --- a/src/common_types.ts +++ b/src/common_types.ts @@ -41,8 +41,14 @@ export type DefaultPluginOutput = { timestamp: number; }; +export type BrowserInfo = { + name: string; + version: string; +}; + export type NetworkInfo = { url: string; + browser: BrowserInfo; method: string; type: string; request: Protocol.Network.Request; diff --git a/src/plugins/network.ts b/src/plugins/network.ts index 49045bb0..0d8d6942 100644 --- a/src/plugins/network.ts +++ b/src/plugins/network.ts @@ -25,15 +25,19 @@ import { CDPSession } from 'playwright-chromium'; import { Protocol } from 'playwright-chromium/types/protocol'; -import { NetworkInfo } from '../common_types'; +import { NetworkInfo, BrowserInfo } from '../common_types'; import { Step } from '../dsl'; import { getTimestamp } from '../helpers'; export class NetworkManager { + private _browser: BrowserInfo; _currentStep: Partial = null; waterfallMap = new Map(); async start(client: CDPSession) { + const { product } = await client.send('Browser.getVersion'); + const [name, version] = product.split('/'); + this._browser = { name, version }; await client.send('Network.enable'); /** * Listen for all network events @@ -95,6 +99,7 @@ export class NetworkManager { } this.waterfallMap.set(requestId, { + browser: this._browser, step: this._currentStep, timestamp: getTimestamp(), url, diff --git a/src/reporters/json.ts b/src/reporters/json.ts index af691a80..edf3a842 100644 --- a/src/reporters/json.ts +++ b/src/reporters/json.ts @@ -135,11 +135,15 @@ function formatTLS(tls: Protocol.Network.SecurityDetails) { } export function formatNetworkFields(network: NetworkInfo) { - const { request, response, url } = network; + const { request, response, url, browser } = network; return { - // URL and USER AGENT would be parsed and mapped by heartbeat + // URL would be parsed and mapped by heartbeat url, - user_agent: request.headers?.['User-Agent'], + user_agent: { + name: browser.name, + version: browser.version, + original: request.headers?.['User-Agent'], + }, http: { version: formatVersion(response?.protocol), request: formatRequest(request),