Skip to content

Commit

Permalink
Improve test file detection for GitHub annotations (#885)
Browse files Browse the repository at this point in the history
  • Loading branch information
@72636c
72636c authored May 30, 2022
1 parent fdfd677 commit 8923dfc
Show file tree
Hide file tree
Showing 3 changed files with 113 additions and 57 deletions.
5 changes: 5 additions & 0 deletions .changeset/big-buses-yell.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'skuba': patch
---

test: Improve file detection for GitHub annotations
139 changes: 89 additions & 50 deletions src/cli/test/reporters/github/annotations.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,30 +4,34 @@ import { createAnnotations } from './annotations';

jest.spyOn(process, 'cwd').mockReturnValue('/workdir/skuba');

it('should create annotations from Jest test results', () => {
const testResult = {
leaks: false,
numFailingTests: 1,
numPassingTests: 0,
numPendingTests: 0,
numTodoTests: 0,
openHandles: [],
perfStats: {
end: 1636247736632,
runtime: 2646,
slow: false,
start: 1636247733986,
},
skipped: false,
snapshot: {
added: 0,
fileDeleted: false,
matched: 0,
unchecked: 0,
uncheckedKeys: [],
unmatched: 0,
updated: 0,
},
const COMMON_TEST_RESULT_FIELDS = {
leaks: false,
numFailingTests: 1,
numPassingTests: 0,
numPendingTests: 0,
numTodoTests: 0,
openHandles: [],
perfStats: {
end: 1636247736632,
runtime: 2646,
slow: false,
start: 1636247733986,
},
skipped: false,
snapshot: {
added: 0,
fileDeleted: false,
matched: 0,
unchecked: 0,
uncheckedKeys: [],
unmatched: 0,
updated: 0,
},
};

it('should create annotation from Jest test failure', () => {
const testResult: TestResult = {
...COMMON_TEST_RESULT_FIELDS,
testFilePath: '/workdir/skuba/src/test.test.ts',
testResults: [
{
Expand Down Expand Up @@ -58,7 +62,7 @@ it('should create annotations from Jest test results', () => {
],
failureMessage:
"\u001b[1m\u001b[31m \u001b[1m● \u001b[22m\u001b[1mshould output a\u001b[39m\u001b[22m\n\n \u001b[2mexpect(\u001b[22m\u001b[31mreceived\u001b[39m\u001b[2m).\u001b[22mtoBe\u001b[2m(\u001b[22m\u001b[32mexpected\u001b[39m\u001b[2m) // Object.is equality\u001b[22m\n\n Expected: \u001b[32m\"a\"\u001b[39m\n Received: \u001b[31m\"b\"\u001b[39m\n\u001b[2m\u001b[22m\n\u001b[2m \u001b[0m \u001b[90m 1 |\u001b[39m it(\u001b[32m'should output a'\u001b[39m\u001b[33m,\u001b[39m () \u001b[33m=>\u001b[39m {\u001b[0m\u001b[22m\n\u001b[2m \u001b[0m\u001b[31m\u001b[1m>\u001b[22m\u001b[2m\u001b[39m\u001b[90m 2 |\u001b[39m expect(\u001b[32m'b'\u001b[39m)\u001b[33m.\u001b[39mtoBe(\u001b[32m'a'\u001b[39m)\u001b[33m;\u001b[39m\u001b[0m\u001b[22m\n\u001b[2m \u001b[0m \u001b[90m |\u001b[39m \u001b[31m\u001b[1m^\u001b[22m\u001b[2m\u001b[39m\u001b[0m\u001b[22m\n\u001b[2m \u001b[0m \u001b[90m 3 |\u001b[39m })\u001b[33m;\u001b[39m\u001b[0m\u001b[22m\n\u001b[2m \u001b[0m \u001b[90m 4 |\u001b[39m\u001b[0m\u001b[22m\n\u001b[2m\u001b[22m\n\u001b[2m \u001b[2mat Object.<anonymous> (\u001b[22m\u001b[2m\u001b[0m\u001b[36msrc/test.test.ts\u001b[39m\u001b[0m\u001b[2m:2:15)\u001b[22m\u001b[2m\u001b[22m\n",
} as TestResult;
};

const annotations = createAnnotations([testResult]);
expect(annotations).toMatchInlineSnapshot(`
Expand Down Expand Up @@ -90,7 +94,62 @@ it('should create annotations from Jest test results', () => {
`);
});

it('should create annotations from Jest exec errors', () => {
it('should create annotation from Jest timeout', () => {
const testResult: TestResult = {
...COMMON_TEST_RESULT_FIELDS,
testFilePath: '/workdir/skuba/src/test.test.ts',
testResults: [
{
ancestorTitles: ['someFunction'],
duration: 5002,
failureDetails: [
{
message:
'thrown: "Exceeded timeout of 5000 ms for a test.\nUse jest.setTimeout(newTimeout) to increase the timeout value, if this is a long-running test."',
},
],
failureMessages: [
'Error: thrown: "Exceeded timeout of 5000 ms for a test.\nUse jest.setTimeout(newTimeout) to increase the timeout value, if this is a long-running test."\n at /workdir/skuba/src/test.test.ts:55:6\n at _dispatchDescribe (/workdir/skuba/node_modules/jest-circus/build/index.js:98:26)\n at describe (/workdir/skuba/node_modules/jest-circus/build/index.js:60:5)\n at Object.<anonymous> (/workdir/skuba/src/test.test.ts:21:1)\n at Runtime._execModule (/workdir/skuba/node_modules/jest-runtime/build/index.js:1646:24)\n at Runtime._loadModule (/workdir/skuba/node_modules/jest-runtime/build/index.js:1185:12)\n at Runtime.requireModule (/workdir/skuba/node_modules/jest-runtime/build/index.js:1009:12)\n at jestAdapter (/workdir/skuba/node_modules/jest-circus/build/legacy-code-todo-rewrite/jestAdapter.js:79:13)\n at runTestInternal (/workdir/skuba/node_modules/jest-runner/build/runTest.js:389:16)\n at runTest (/workdir/skuba/node_modules/jest-runner/build/runTest.js:475:34)',
],
fullName: 'someFunction should output a',
invocations: 1,
location: null,
numPassingAsserts: 0,
status: 'failed',
title: 'should output a',
},
],
};

const annotations = createAnnotations([testResult]);
expect(annotations).toMatchInlineSnapshot(`
Array [
Object {
"annotation_level": "failure",
"end_column": 6,
"end_line": 55,
"message": "Error: thrown: \\"Exceeded timeout of 5000 ms for a test.
Use jest.setTimeout(newTimeout) to increase the timeout value, if this is a long-running test.\\"
at /workdir/skuba/src/test.test.ts:55:6
at _dispatchDescribe (/workdir/skuba/node_modules/jest-circus/build/index.js:98:26)
at describe (/workdir/skuba/node_modules/jest-circus/build/index.js:60:5)
at Object.<anonymous> (/workdir/skuba/src/test.test.ts:21:1)
at Runtime._execModule (/workdir/skuba/node_modules/jest-runtime/build/index.js:1646:24)
at Runtime._loadModule (/workdir/skuba/node_modules/jest-runtime/build/index.js:1185:12)
at Runtime.requireModule (/workdir/skuba/node_modules/jest-runtime/build/index.js:1009:12)
at jestAdapter (/workdir/skuba/node_modules/jest-circus/build/legacy-code-todo-rewrite/jestAdapter.js:79:13)
at runTestInternal (/workdir/skuba/node_modules/jest-runner/build/runTest.js:389:16)
at runTest (/workdir/skuba/node_modules/jest-runner/build/runTest.js:475:34)",
"path": "src/test.test.ts",
"start_column": 6,
"start_line": 55,
"title": "Jest",
},
]
`);
});

it('should create annotation from Jest exec error', () => {
const error = {
message:
"\x1B[96msrc/test.ts\x1B[0m:\x1B[93m1\x1B[0m:\x1B[93m1\x1B[0m - \x1B[91merror\x1B[0m\x1B[90m TS6133: \x1B[0m'a' is declared but its value is never read.\n" +
Expand All @@ -102,35 +161,15 @@ it('should create annotations from Jest exec errors', () => {
"\x1B[7m1\x1B[0m import { a } from 'b';\n" +
'\x1B[7m \x1B[0m \x1B[91m ~~~\x1B[0m',
} as SerializableError;
const testResult = {

const testResult: TestResult = {
...COMMON_TEST_RESULT_FIELDS,
failureMessage:
" \u001b[1m● \u001b[22mTest suite failed to run\n\n \u001b[96msrc/test.ts\u001b[0m:\u001b[93m1\u001b[0m:\u001b[93m1\u001b[0m - \u001b[91merror\u001b[0m\u001b[90m TS6133: \u001b[0m'a' is declared but its value is never read.\n\n \u001b[7m1\u001b[0m import { a } from 'b';\n \u001b[7m \u001b[0m \u001b[91m~~~~~~~~~~~~~~~~~~~~~~\u001b[0m\n \u001b[96msrc/test.ts\u001b[0m:\u001b[93m1\u001b[0m:\u001b[93m19\u001b[0m - \u001b[91merror\u001b[0m\u001b[90m TS2307: \u001b[0mCannot find module 'b' or its corresponding type declarations.\n\n \u001b[7m1\u001b[0m import { a } from 'b';\n \u001b[7m \u001b[0m \u001b[91m ~~~\u001b[0m\n",
leaks: false,
numFailingTests: 0,
numPassingTests: 0,
numPendingTests: 0,
numTodoTests: 0,
openHandles: [],
perfStats: {
end: 0,
runtime: 0,
slow: false,
start: 0,
},
skipped: false,
snapshot: {
added: 0,
fileDeleted: false,
matched: 0,
unchecked: 0,
uncheckedKeys: [],
unmatched: 0,
updated: 0,
},
testExecError: error,
testFilePath: '/workdir/skuba/src/test.test.ts',
testResults: [],
} as TestResult;
};

const annotations = createAnnotations([testResult]);
expect(annotations).toMatchInlineSnapshot(`
Expand Down
26 changes: 19 additions & 7 deletions src/cli/test/reporters/github/annotations.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,25 @@ import type * as GitHub from '../../../../api/github';
* ...
* ```
*
* or:
*
* ```console
* Error: expect(received).toBe(expected) // Object.is equality
*
* Expected: "a"
* Received: "b"
* at /workdir/skuba/src/test.test.ts:2:15
* at Promise.then.completed (/workdir/skuba/node_modules/jest-circus/build/utils.js:390:28)
* ...
* ```
*
* This pattern will produce the following matches:
*
* 1. /workdir/skuba/src/test.test.ts
* 2. 2
* 2. 15
*/
const JEST_LOCATION_REGEX = /\((.+?):(\d+):(\d+)\)/;
const JEST_LOCATION_REGEX = /\n +at (.+\()?(.+?):(\d+):(\d+)/;

export const createAnnotations = (
testResults: TestResult[],
Expand All @@ -49,14 +61,14 @@ export const createAnnotations = (
return testResult.testResults.flatMap((assertionResult) =>
assertionResult.failureMessages.flatMap((failureMessage) => {
const match = JEST_LOCATION_REGEX.exec(failureMessage);
if (match?.length === 4) {
if (match?.length === 5) {
return {
annotation_level: 'failure',
path: path.relative(cwd, match[1]),
start_line: Number(match[2]),
end_line: Number(match[2]),
start_column: Number(match[3]),
end_column: Number(match[3]),
path: path.relative(cwd, match[2]),
start_line: Number(match[3]),
end_line: Number(match[3]),
start_column: Number(match[4]),
end_column: Number(match[4]),
message: stripAnsi(failureMessage),
title: 'Jest',
};
Expand Down

0 comments on commit 8923dfc

Please sign in to comment.