From 2f9016e7634da91820222bb4249f4d087a9e1701 Mon Sep 17 00:00:00 2001 From: Brian Hackett Date: Mon, 28 Feb 2022 08:10:32 -0800 Subject: [PATCH] Show jest tests which ran in the recording as console messages (#5496) --- .../client/webconsole/actions/messages.js | 2 + src/protocol/analysisManager.ts | 10 + src/protocol/execution-point-utils.ts | 4 + src/protocol/find-tests.ts | 445 ++++++++++++++++++ src/protocol/logpoint.ts | 2 +- src/ui/actions/session.ts | 3 + 6 files changed, 465 insertions(+), 1 deletion(-) create mode 100644 src/protocol/find-tests.ts diff --git a/src/devtools/client/webconsole/actions/messages.js b/src/devtools/client/webconsole/actions/messages.js index 49e4ead1de5..a4272827b3a 100644 --- a/src/devtools/client/webconsole/actions/messages.js +++ b/src/devtools/client/webconsole/actions/messages.js @@ -13,6 +13,7 @@ const { const { IdGenerator } = require("devtools/client/webconsole/utils/id-generator"); const { ThreadFront } = require("protocol/thread"); const { LogpointHandlers } = require("protocol/logpoint"); +const { TestMessageHandlers } = require("protocol/find-tests"); const { MESSAGES_ADD, @@ -36,6 +37,7 @@ export function setupMessages(store) { LogpointHandlers.onResult = (logGroupId, point, time, location, pause, values) => store.dispatch(onLogpointResult(logGroupId, point, time, location, pause, values)); LogpointHandlers.clearLogpoint = logGroupId => store.dispatch(messagesClearLogpoint(logGroupId)); + TestMessageHandlers.onTestMessage = msg => store.dispatch(onConsoleMessage(msg)); ThreadFront.findConsoleMessages( (_, msg) => store.dispatch(onConsoleMessage(msg)), diff --git a/src/protocol/analysisManager.ts b/src/protocol/analysisManager.ts index 615216bec5d..cc2921fcaf6 100644 --- a/src/protocol/analysisManager.ts +++ b/src/protocol/analysisManager.ts @@ -7,6 +7,7 @@ import { AnalysisId, analysisPoints, analysisResult, + ExecutionPoint, PointDescription, SessionId, } from "@recordreplay/protocol"; @@ -22,6 +23,7 @@ export interface AnalysisParams { eventHandlerEntryPoints?: EventHandlerEntryPoint[]; exceptionPoints?: boolean; randomPoints?: number; + points?: ExecutionPoint[]; sessionId: SessionId; } @@ -98,6 +100,14 @@ class AnalysisManager { ); } + if (params.points) { + await sendMessage( + "Analysis.addPoints", + { analysisId, points: params.points }, + params.sessionId + ); + } + this.handlers.set(analysisId, handler); await Promise.all([ !handler.onAnalysisResult || diff --git a/src/protocol/execution-point-utils.ts b/src/protocol/execution-point-utils.ts index 7bbbd5b45ca..517f3eda44a 100644 --- a/src/protocol/execution-point-utils.ts +++ b/src/protocol/execution-point-utils.ts @@ -8,3 +8,7 @@ export function pointEquals(p1: ExecutionPoint, p2: ExecutionPoint) { export function pointPrecedes(p1: ExecutionPoint, p2: ExecutionPoint) { return compareNumericStrings(p1, p2) < 0; } + +export function comparePoints(p1: ExecutionPoint, p2: ExecutionPoint) { + return compareNumericStrings(p1, p2); +} diff --git a/src/protocol/find-tests.ts b/src/protocol/find-tests.ts new file mode 100644 index 00000000000..51c7972d43b --- /dev/null +++ b/src/protocol/find-tests.ts @@ -0,0 +1,445 @@ +// Perform some analysis to find and describe automated tests that were recorded. + +import { Pause, ThreadFront, ValueFront } from "./thread"; +import analysisManager, { AnalysisHandler, AnalysisParams } from "./analysisManager"; +import { Helpers } from "./logpoint"; +import { assert } from "protocol/utils"; +import { client } from "./socket"; +import { comparePoints, pointPrecedes } from "./execution-point-utils"; +import { + AnalysisEntry, + ExecutionPoint, + Message, + Location, + PointDescription, +} from "@recordreplay/protocol"; + +// Information about a jest test which ran in the recording. +interface JestTestInfo { + // Hierarchical names for this test, from child to parent. + names: string[]; + + // Start point of the test. + startPoint: PointDescription; + + // If the test failed, the place where it failed at. + errorPoint?: PointDescription; + + // If the test failed, description of the failure. + errorText?: string; + + // If the test failed, whether errorPoint is an exception site. + errorException?: boolean; +} + +// Mapper returning analysis results for jest test callback invocations +// which are associated with a test instead of a hook, returning the name +// of the test and the point where the callback is invoked. +const JestTestMapper = ` +${Helpers} +const { point, time, pauseId } = input; +const { frameId, functionName } = getTopFrame(); +const { result: isHookResult } = sendCommand("Pause.evaluateInFrame", { + frameId, + expression: "isHook", +}); +if (isHookResult.returned && isHookResult.returned.value) { + return []; +} +const names = []; +const nameExpressions = [ + "testOrHook.name", + "testOrHook.parent.name", + "testOrHook.parent.parent.name", + "testOrHook.parent.parent.parent.name", + "testOrHook.parent.parent.parent.parent.name", + "testOrHook.parent.parent.parent.parent.parent.name", +]; +for (const expression of nameExpressions) { + const { result: nameResult } = sendCommand("Pause.evaluateInFrame", { + frameId, + expression, + }); + if (nameResult.returned && nameResult.returned.value) { + names.push(nameResult.returned.value); + } else { + break; + } +} +return [{ + key: point, + value: { names }, +}]; +`; + +// Mapper extracting the "name" and "message" properties from a thrown exception +// for use in test failure messages. +const JestExceptionMapper = ` +${Helpers} +const { point, time, pauseId } = input; +const { frameId, location } = getTopFrame(); +const { exception, data: exceptionData } = sendCommand("Pause.getExceptionValue"); +addPauseData(exceptionData); +return [{ + key: point, + value: { time, pauseId, location, exception, data: finalData }, +}]; +`; + +// Manages the state associated with any jest tests within the recording. +class JestTestState { + sessionId: string; + + // Locations where the inner callback passed to callAsyncCircusFn is invoked, + // starting the test or hook. + invokeCallbackLocations: Location[]; + + // Location of the catch block which indicates a test failure. + catchBlockLocation: Location; + + // Any tests we found. + tests: JestTestInfo[] = []; + + constructor(invokeCallbackLocations: Location[], catchBlockLocation: Location) { + assert(ThreadFront.sessionId); + this.sessionId = ThreadFront.sessionId; + + this.invokeCallbackLocations = invokeCallbackLocations; + this.catchBlockLocation = catchBlockLocation; + } + + async loadTests() { + const params: AnalysisParams = { + sessionId: this.sessionId, + mapper: JestTestMapper, + effectful: true, + locations: this.invokeCallbackLocations.map(location => ({ location })), + }; + + const analysisResults: AnalysisEntry[] = []; + + const handler: AnalysisHandler = {}; + handler.onAnalysisResult = results => analysisResults.push(...results); + + await analysisManager.runAnalysis(params, handler); + + await Promise.all( + analysisResults.map(async ({ key: callPoint, value: { names } }) => { + const { target } = await client.Debugger.findStepInTarget( + { point: callPoint }, + this.sessionId + ); + if (target.frame) { + this.tests.push({ names, startPoint: target }); + } + }) + ); + + this.tests.sort((a, b) => comparePoints(a.startPoint.point, b.startPoint.point)); + } + + async loadFailures() { + const failurePoints = await this.getFailurePoints(); + if (!failurePoints.length) { + return; + } + + const exceptionPoints = await this.getExceptionPoints(); + + // Exceptions which are associated with a test failure. + const failureExceptionPoints: PointDescription[] = []; + + await Promise.all( + failurePoints.map(async point => { + // Associate this failure with the most recent test. + const test = this.mostRecentTest(point.point); + if (!test || test.errorPoint) { + return; + } + + const exceptionPoint = this.mostRecentPoint(point.point, exceptionPoints); + if (exceptionPoint && pointPrecedes(test.startPoint.point, exceptionPoint.point)) { + test.errorPoint = exceptionPoint; + test.errorException = true; + failureExceptionPoints.push(exceptionPoint); + } else { + test.errorPoint = point; + test.errorException = false; + } + }) + ); + + if (!failureExceptionPoints.length) { + return; + } + + const params: AnalysisParams = { + sessionId: this.sessionId, + mapper: JestExceptionMapper, + effectful: true, + points: failureExceptionPoints.map(p => p.point), + }; + + const analysisResults: AnalysisEntry[] = []; + + const handler: AnalysisHandler = {}; + handler.onAnalysisResult = results => analysisResults.push(...results); + + await analysisManager.runAnalysis(params, handler); + + await Promise.all( + this.tests.map(async test => { + if (!test.errorException) { + return; + } + assert(test.errorPoint); + + const result = analysisResults.find(r => r.key == test.errorPoint?.point); + if (!result) { + test.errorText = "unknown exception"; + return; + } + + const { time, pauseId, location, exception, data } = result.value; + + const pause = new Pause(ThreadFront.sessionId!); + pause.addData(data); + pause.instantiate(pauseId, test.errorPoint.point, time, /* hasFrames */ true); + const exceptionValue = new ValueFront(pause, exception); + + const exceptionContents = exceptionValue.previewValueMap(); + const exceptionProperty = exceptionContents.message || exceptionContents.name; + if (exceptionProperty && exceptionProperty.isString()) { + test.errorText = String(exceptionProperty.primitive()); + } else { + test.errorText = "unknown exception"; + } + }) + ); + } + + // Get the points where test failures occurred. + private async getFailurePoints(): Promise { + const params: AnalysisParams = { + sessionId: this.sessionId, + mapper: "", + effectful: true, + locations: [{ location: this.catchBlockLocation }], + }; + + const handler: AnalysisHandler = {}; + const failurePoints: PointDescription[] = []; + + handler.onAnalysisPoints = points => failurePoints.push(...points); + await analysisManager.runAnalysis(params, handler); + + return failurePoints; + } + + // Get a sorted array of the points where exceptions were thrown. + private async getExceptionPoints(): Promise { + const params: AnalysisParams = { + sessionId: this.sessionId, + mapper: "", + effectful: true, + exceptionPoints: true, + }; + + const handler: AnalysisHandler = {}; + const exceptionPoints: PointDescription[] = []; + + handler.onAnalysisPoints = points => exceptionPoints.push(...points); + await analysisManager.runAnalysis(params, handler); + + exceptionPoints.sort((a, b) => comparePoints(a.point, b.point)); + + return exceptionPoints; + } + + private mostRecentTest(point: ExecutionPoint): JestTestInfo | null { + for (let i = 0; i < this.tests.length; i++) { + if (pointPrecedes(point, this.tests[i].startPoint.point)) { + return i ? this.tests[i - 1] : null; + } + } + return null; + } + + private mostRecentPoint( + point: ExecutionPoint, + pointArray: PointDescription[] + ): PointDescription | null { + for (let i = 0; i < pointArray.length; i++) { + if (pointPrecedes(point, pointArray[i].point)) { + return i ? pointArray[i - 1] : null; + } + } + return null; + } +} + +// Look for a source containing the specified pattern. +async function findMatchingSourceId(pattern: string): Promise { + await ThreadFront.ensureAllSources(); + for (const [sourceId, source] of ThreadFront.sources.entries()) { + if (source.url?.includes(pattern)) { + return sourceId; + } + } + return null; +} + +// Get the location of the first breakpoint on the specified line. +// Note: This requires a linear scan of the lines containing breakpoints +// and will be inefficient if used on large sources. +async function getBreakpointLocationOnLine( + sourceId: string, + targetLine: number +): Promise { + const breakpointPositions = await ThreadFront.getBreakpointPositionsCompressed(sourceId); + for (const { line, columns } of breakpointPositions) { + if (line == targetLine && columns.length) { + return { sourceId, line, column: columns[0] }; + } + } + return null; +} + +// Look for places in the recording used to run jest tests. +async function setupJestTests(): Promise { + // Look for a source containing the callAsyncCircusFn function which is used to + // run tests using recent versions of Jest. + const circusUtilsSourceId = await findMatchingSourceId("jest-circus/build/utils.js"); + if (!circusUtilsSourceId) { + return null; + } + + const { contents } = await ThreadFront.getSourceContents(circusUtilsSourceId); + const lines = contents.split("\n"); + + // Whether we've seen the start of the callAsyncCircusFn function. + let foundCallAsyncCircusFn = false; + + const invokeCallbackLocations: Location[] = []; + let catchBlockLocation: Location | undefined; + + // Whether we are inside the callAsyncCircusFn catch block. + let insideCatchBlock = false; + + for (let i = 0; i < lines.length; i++) { + const lineContents = lines[i]; + const line = i + 1; + + if (lineContents.includes("const callAsyncCircusFn = ")) { + foundCallAsyncCircusFn = true; + } + + if (!foundCallAsyncCircusFn) { + continue; + } + + // Lines invoking the callback start with "returnedValue = ..." + // except when setting the initial undefined value of returnedValue. + if ( + lineContents.includes("returnedValue = ") && + !lineContents.includes("returnedValue = undefined") + ) { + const location = await getBreakpointLocationOnLine(circusUtilsSourceId, line); + if (location) { + invokeCallbackLocations.push(location); + } + } + + if (lineContents.includes(".catch(error => {")) { + insideCatchBlock = true; + } + + // We should be able to break at this line in the catch block. + if (insideCatchBlock && lineContents.includes("completed = true")) { + const location = await getBreakpointLocationOnLine(circusUtilsSourceId, line); + if (location) { + catchBlockLocation = location; + } + break; + } + } + + // There should be three places where the inner callback can be invoked. + if (invokeCallbackLocations.length != 3) { + return null; + } + + // We should have found a catch block location to break at. + if (!catchBlockLocation) { + return null; + } + + return new JestTestState(invokeCallbackLocations, catchBlockLocation); +} + +function removeTerminalColors(str: string): string { + return str.replace(/\x1B[[(?);]{0,2}(;?\d)*./g, ""); +} + +async function findJestTests() { + const state = await setupJestTests(); + if (!state) { + return; + } + + await state.loadTests(); + + await Promise.all( + state.tests.map(async ({ names, startPoint }) => { + let name = names[0] || ""; + for (let i = 1; i < names.length; i++) { + if (names[i] != "ROOT_DESCRIBE_BLOCK") { + name = names[i] + " \u25b6 " + name; + } + } + const pause = ThreadFront.ensurePause(startPoint.point, startPoint.time); + const pauseId = await pause.pauseIdWaiter.promise; + TestMessageHandlers.onTestMessage?.({ + source: "ConsoleAPI", + level: "info", + text: `JestTest ${name}`, + point: startPoint, + pauseId, + data: {}, + }); + }) + ); + + await state.loadFailures(); + + await Promise.all( + state.tests.map(async ({ errorPoint, errorText }) => { + if (!errorPoint) { + return; + } + if (!errorText) { + errorText = "unknown test failure"; + } + errorText = removeTerminalColors(errorText); + const pause = ThreadFront.ensurePause(errorPoint.point, errorPoint.time); + const pauseId = await pause.pauseIdWaiter.promise; + TestMessageHandlers.onTestMessage?.({ + source: "ConsoleAPI", + level: "error", + text: `JestFailure ${errorText}`, + point: errorPoint, + pauseId, + data: {}, + }); + }) + ); +} + +// Look for automated tests associated with a recent version of jest. +export async function findAutomatedTests() { + await findJestTests(); +} + +export const TestMessageHandlers: { + onTestMessage?: (msg: Message) => void; +} = {}; diff --git a/src/protocol/logpoint.ts b/src/protocol/logpoint.ts index 1faef9a2322..8a7832287ff 100644 --- a/src/protocol/logpoint.ts +++ b/src/protocol/logpoint.ts @@ -126,7 +126,7 @@ function saveAnalysisError(locations: Location[], condition: string) { } // Define some logpoint helpers to manage pause data. -const Helpers = ` +export const Helpers = ` const finalData = { frames: [], scopes: [], objects: [] }; function addPauseData({ frames, scopes, objects }) { finalData.frames.push(...(frames || [])); diff --git a/src/ui/actions/session.ts b/src/ui/actions/session.ts index b22a078c007..7f5b3778212 100644 --- a/src/ui/actions/session.ts +++ b/src/ui/actions/session.ts @@ -7,6 +7,7 @@ import { UIThunkAction } from "ui/actions"; import * as actions from "ui/actions/app"; import * as selectors from "ui/reducers/app"; import { ThreadFront } from "protocol/thread"; +import { findAutomatedTests } from "protocol/find-tests"; import { assert, waitForTime } from "protocol/utils"; import { validateUUID } from "ui/utils/helpers"; import { prefs } from "ui/utils/prefs"; @@ -145,6 +146,8 @@ export function createSession(recordingId: string): UIThunkAction { const recordingTarget = await ThreadFront.recordingTargetWaiter.promise; dispatch(actions.setRecordingTarget(recordingTarget)); + findAutomatedTests(); + // We don't want to show the non-dev version of the app for node replays. if (recordingTarget === "node") { dispatch(setViewMode("dev"));