From d3702e903144b3ecc86e0a08893b2c7fc004a8ce Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Thu, 15 Jul 2021 22:14:13 -0700 Subject: [PATCH] feat(test-runner): implement basic allure reporter --- src/test/expect.ts | 10 ++- src/test/golden.ts | 6 +- src/test/reporters/allure.ts | 122 +++++++++++++++++++++++++++++++++++ src/test/runner.ts | 2 + 4 files changed, 138 insertions(+), 2 deletions(-) create mode 100644 src/test/reporters/allure.ts diff --git a/src/test/expect.ts b/src/test/expect.ts index f73c9f4b9ad44..478564c45a777 100644 --- a/src/test/expect.ts +++ b/src/test/expect.ts @@ -38,7 +38,7 @@ function toMatchSnapshot(this: ReturnType, received: Buffer options.threshold = projectThreshold; const withNegateComparison = this.isNot; - const { pass, message } = compare( + const { pass, message, expectedPath, actualPath, diffPath, mimeType } = compare( received, options.name, testInfo.snapshotPath, @@ -47,6 +47,14 @@ function toMatchSnapshot(this: ReturnType, received: Buffer withNegateComparison, options ); + if (expectedPath) { + testInfo.data['toMatchSnapshot'] = { + expectedPath, + actualPath, + diffPath, + mimeType + }; + } return { pass, message: () => message }; } diff --git a/src/test/golden.ts b/src/test/golden.ts index 460a29aa75142..ea29ea5bfa01d 100644 --- a/src/test/golden.ts +++ b/src/test/golden.ts @@ -88,7 +88,7 @@ export function compare( updateSnapshots: UpdateSnapshots, withNegateComparison: boolean, options?: { threshold?: number } -): { pass: boolean; message?: string; } { +): { pass: boolean; message?: string; expectedPath?: string, actualPath?: string, diffPath?: string, mimeType?: string } { const snapshotFile = snapshotPath(name); if (!fs.existsSync(snapshotFile)) { @@ -179,6 +179,10 @@ export function compare( return { pass: false, message: output.join('\n'), + expectedPath, + actualPath, + diffPath, + mimeType }; } diff --git a/src/test/reporters/allure.ts b/src/test/reporters/allure.ts new file mode 100644 index 0000000000000..8b22d5d67128a --- /dev/null +++ b/src/test/reporters/allure.ts @@ -0,0 +1,122 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { AllureGroup, AllureRuntime, LabelName, Status } from 'allure-js-commons'; +import path from 'path'; +import { FullConfig, Suite, Test, TestStatus } from '../reporter'; +import { stripAscii } from './base'; +import EmptyReporter from './empty'; + +const startTimeSymbol = Symbol('startTime'); + +class AllureReporter extends EmptyReporter { + config!: FullConfig; + suite!: Suite; + + onBegin(config: FullConfig, suite: Suite) { + this.config = config; + this.suite = suite; + } + + onTestBegin(test: Test) { + (test.results[test.results.length - 1] as any)[startTimeSymbol] = Date.now(); + } + + onTimeout() { + this.onEnd(); + } + + async onEnd() { + const resultsDir = path.join(process.cwd(), 'allure-results'); + const runtime = new AllureRuntime({resultsDir}); + const processSuite = (suite: Suite, parent: AllureGroup | AllureRuntime, groupNamePath: string[]) => { + const groupName = 'Root'; + if (suite.file) { + // TODO: use suite's project + const project = this.config.projects[0]; + const groupName = suite.title || path.relative(project.testDir, suite.file); + groupNamePath = [...groupNamePath, groupName]; + } + const group = parent.startGroup(groupName); + for (const test of suite.tests) { + for (const result of test.results) { + const startTime = (result as any)[startTimeSymbol]; + const endTime = startTime + result.duration; + + const allureTest = group.startTest(test.fullTitle(), startTime); + const [parentSuite, suite, ...subSuites] = groupNamePath; + if (parentSuite) + allureTest.addLabel(LabelName.PARENT_SUITE, parentSuite); + if (suite) + allureTest.addLabel(LabelName.SUITE, suite); + if (subSuites.length > 0) + allureTest.addLabel(LabelName.SUB_SUITE, subSuites.join(' > ')); + + allureTest.historyId = test.fullTitle(); + allureTest.fullName = test.fullTitle(); + allureTest.status = statusToAllureStats(result.status!); + + if (result.error) { + const message = result.error.message && stripAscii(result.error.message); + let trace = result.error.stack && stripAscii(result.error.stack); + if (trace && message && trace.startsWith(message)) + trace = trace.substr(message.length); + allureTest.statusDetails = { + message, + trace, + }; + } + + if (test.projectName) + allureTest.addParameter('project', test.projectName); + + if (result.data['toMatchSnapshot']) { + const { expectedPath, actualPath, diffPath, mimeType } = result.data['toMatchSnapshot']; + allureTest.addLabel('testType', 'screenshotDiff'); + allureTest.addAttachment('expected', mimeType, expectedPath); + allureTest.addAttachment('actual', mimeType, actualPath); + allureTest.addAttachment('diff', mimeType, diffPath); + } + + for (const stdout of result.stdout) + allureTest.addAttachment('stdout', 'text/plain', runtime.writeAttachment(stdout, 'text/plain')); + for (const stderr of result.stderr) + allureTest.addAttachment('stderr', 'text/plain', runtime.writeAttachment(stderr, 'text/plain')); + allureTest.endTest(endTime); + } + } + for (const child of suite.suites) + processSuite(child, group, groupNamePath); + group.endGroup(); + }; + processSuite(this.suite, runtime, []); + } +} + +function statusToAllureStats(status: TestStatus): Status { + switch (status) { + case 'failed': + return Status.FAILED; + case 'passed': + return Status.PASSED; + case 'skipped': + return Status.SKIPPED; + case 'timedOut': + return Status.BROKEN; + } +} + +export default AllureReporter; diff --git a/src/test/runner.ts b/src/test/runner.ts index d661bf100d69b..44592149623e6 100644 --- a/src/test/runner.ts +++ b/src/test/runner.ts @@ -26,6 +26,7 @@ import { Test, Suite } from './test'; import { Loader } from './loader'; import { Reporter } from './reporter'; import { Multiplexer } from './reporters/multiplexer'; +import AllureReporter from './reporters/allure'; import DotReporter from './reporters/dot'; import LineReporter from './reporters/line'; import ListReporter from './reporters/list'; @@ -65,6 +66,7 @@ export class Runner { private async _createReporter() { const reporters: Reporter[] = []; const defaultReporters = { + allure: AllureReporter, dot: DotReporter, line: LineReporter, list: ListReporter,