diff --git a/.eslintignore b/.eslintignore index 602d431..fe1556f 100644 --- a/.eslintignore +++ b/.eslintignore @@ -5,4 +5,5 @@ docs/ .vscode/ dist/ differencify_report/ -screenshots/ \ No newline at end of file +screenshots/ +coverage/ diff --git a/src/Reporter.js b/src/Reporter.js index d63b88f..82a6b7b 100644 --- a/src/Reporter.js +++ b/src/Reporter.js @@ -18,34 +18,56 @@ const getReport = (key, results) => { } }; -class Reporter { +const getFiles = dir => + fs + .readdirSync(dir) + .filter(file => fs.lstatSync(path.join(dir, file)).isFile()); + +const hasImage = (files, file) => (files.includes(file) ? file : null); - constructor() { +class Reporter { + constructor(options = {}) { + this.options = options; this.results = []; } - addResult(outcome, fileName, message, diff) { + addResult({ outcome, testName, result }) { this.results.push({ outcome, - fileName: path.basename(fileName), - message, - diff: diff ? path.basename(diff) : null, + testName, + result, }); } getResults() { - return this.results; + const images = getFiles(this.options.testReportPath); + return this.results.map(result => + // eslint-disable-next-line prefer-object-spread/prefer-object-spread + Object.assign( + { + referenceFileName: hasImage(images, `${result.testName}.png`), + diffFileName: hasImage( + images, + `${result.testName}_differencified.png`, + ), + }, + result, + ), + ); } generate(types, testReportPath) { + const results = this.getResults(); Object.keys(types).forEach((type) => { const filepath = path.join(testReportPath, types[type]); try { - const template = getReport(type, this.getResults()); + const template = getReport(type, results); saveReport(filepath, template); logger.log(`Generated ${type} report at ${filepath}`); } catch (err) { - logger.error(`Unable to generate ${type} report at ${filepath}: ${err}`); + logger.error( + `Unable to generate ${type} report at ${filepath}: ${err}`, + ); } }); return true; diff --git a/src/Reporter.test.js b/src/Reporter.test.js index bb6e842..180dbe7 100644 --- a/src/Reporter.test.js +++ b/src/Reporter.test.js @@ -1,9 +1,11 @@ import fs from 'fs'; import Reporter, { getHtmlReport, getJsonReport } from './Reporter'; import logger from './logger'; +import { globalConfig } from './defaultConfig'; jest.mock('fs', () => ({ writeFileSync: jest.fn(), + readdirSync: jest.fn(() => []), })); jest.mock('./logger', () => ({ @@ -13,21 +15,18 @@ jest.mock('./logger', () => ({ const results = [ { outcome: true, - fileName: 'image1.png', - message: 'no mismatch found', - diff: null, + testName: 'default', + result: 'no mismatch found', }, { outcome: true, - fileName: 'image2.png', - message: 'no mismatch found', - diff: null, + testName: 'default2', + result: 'no mismatch found', }, { outcome: false, - fileName: 'image2.png', - message: 'mismatch found!', - diff: 'image2_diff.png', + testName: 'default3', + result: 'mismatch found!', }, ]; @@ -35,19 +34,19 @@ describe('Generate report index', () => { let reporter; beforeEach(() => { - reporter = new Reporter(); + reporter = new Reporter(globalConfig); results.forEach(result => - reporter.addResult( - result.outcome, - result.fileName, - result.message, - result.diff, - ), + reporter.addResult({ + outcome: result.outcome, + testName: result.testName, + result: result.result, + }), ); }); afterEach(() => { fs.writeFileSync.mockClear(); + fs.readdirSync.mockClear(); logger.log.mockClear(); }); @@ -58,9 +57,10 @@ describe('Generate report index', () => { }, './example/path', ); + expect(fs.readdirSync).toHaveBeenCalledWith(globalConfig.testReportPath); expect(fs.writeFileSync).toHaveBeenCalledWith( 'example/path/index.html', - getHtmlReport(results), + getHtmlReport(reporter.getResults()), ); }); @@ -71,9 +71,10 @@ describe('Generate report index', () => { }, './example/path', ); + expect(fs.readdirSync).toHaveBeenCalledWith(globalConfig.testReportPath); expect(fs.writeFileSync).toHaveBeenCalledWith( 'example/path/report.json', - getJsonReport(results), + getJsonReport(reporter.getResults()), ); }); diff --git a/src/chromyRunner.js b/src/chromyRunner.js index 4b5ae0c..967d78e 100644 --- a/src/chromyRunner.js +++ b/src/chromyRunner.js @@ -67,6 +67,11 @@ const run = async (chromy, options, test, reporter) => { const result = await compareImage(options, test.name, reporter); prefixedLogger.log(result); } catch (error) { + reporter.addResult({ + outcome: false, + testName: test.name, + result: error.message || '', + }); prefixedLogger.error(error); return false; } diff --git a/src/chromyRunner.test.js b/src/chromyRunner.test.js index ac56cda..583a176 100644 --- a/src/chromyRunner.test.js +++ b/src/chromyRunner.test.js @@ -6,6 +6,7 @@ import { globalConfig, testConfig, configTypes } from './defaultConfig'; import actions from './actions'; import functionToString from './helpers/functionToString'; import freezeImage from './freezeImage'; +import Reporter from './Reporter'; jest.mock('chromy', () => () => ({ @@ -29,6 +30,10 @@ jest.mock('./compareImage', () => jest.fn(arg => jest.mock('./helpers/functionToString'); +jest.mock('./Reporter', () => () => ({ + addResult: jest.fn(), +})); + let loggerCalls = []; logger.prefix = () => logger; logger.log = (...args) => { @@ -40,9 +45,15 @@ fs.writeFileSync = (...args) => { writeFileSyncCalls.push(...args); }; +let mockReporter; + const chromy = new Chromy(); describe('ChromyRunner', () => { + beforeEach(() => { + mockReporter = new Reporter(); + }); + afterEach(() => { loggerCalls = []; writeFileSyncCalls = []; @@ -55,7 +66,7 @@ describe('ChromyRunner', () => { }); it('run update', async () => { testConfig.type = configTypes.update; - const result = await run(chromy, globalConfig, testConfig); + const result = await run(chromy, globalConfig, testConfig, mockReporter); expect(result).toEqual(true); expect(chromy.goto).toHaveBeenCalledWith('www.example.com'); expect(chromy.screenshotDocument).toHaveBeenCalledTimes(1); @@ -66,7 +77,7 @@ describe('ChromyRunner', () => { }); it('run test', async () => { testConfig.type = configTypes.test; - const result = await run(chromy, globalConfig, testConfig); + const result = await run(chromy, globalConfig, testConfig, mockReporter); expect(result).toEqual(true); expect(chromy.goto).toHaveBeenCalledWith('www.example.com'); expect(chromy.screenshotDocument).toHaveBeenCalledTimes(1); @@ -75,11 +86,24 @@ describe('ChromyRunner', () => { expect(loggerCalls[2]).toEqual('screenshot saved in -> ./differencify_report/default.png'); expect(writeFileSyncCalls).toEqual(['./differencify_report/default.png', 'png file']); }); + it('run test fail', async () => { + // eslint-disable-next-line prefer-object-spread/prefer-object-spread + const failTestConfig = Object.assign({}, testConfig); + failTestConfig.steps = [{ + name: 'test', + }]; + // eslint-disable-next-line prefer-object-spread/prefer-object-spread + const failGlobalConfig = Object.assign({}, globalConfig); + failGlobalConfig.screenshots = null; + const result = await run(chromy, failGlobalConfig, failTestConfig, mockReporter); + expect(result).toEqual(false); + expect(mockReporter.addResult).toHaveBeenCalledWith({ outcome: false, result: '', testName: 'default' }); + }); describe('Chromy runner', () => { it('Step runner: test action', async () => { testConfig.type = configTypes.test; testConfig.steps.push({ name: actions.test, value: globalConfig.testReportPath }); - const result = await run(chromy, globalConfig, testConfig); + const result = await run(chromy, globalConfig, testConfig, mockReporter); testConfig.steps.pop({ name: actions.test, value: globalConfig.testReportPath }); expect(result).toEqual(true); expect(chromy.goto).toHaveBeenCalledWith('www.example.com'); @@ -91,7 +115,7 @@ describe('ChromyRunner', () => { }); it('Step runner: update action', async () => { testConfig.type = configTypes.update; - const result = await run(chromy, globalConfig, testConfig); + const result = await run(chromy, globalConfig, testConfig, mockReporter); expect(result).toEqual(true); expect(chromy.goto).toHaveBeenCalledWith('www.example.com'); expect(chromy.screenshotDocument).toHaveBeenCalledTimes(1); @@ -115,7 +139,7 @@ describe('ChromyRunner', () => { ], }; newConfig.type = configTypes.test; - const result = await run(chromy, globalConfig, newConfig); + const result = await run(chromy, globalConfig, newConfig, mockReporter); expect(result).toEqual(true); expect(chromy.goto).toHaveBeenCalledWith('www.example.com'); expect(chromy.screenshot).toHaveBeenCalledTimes(1); @@ -137,7 +161,7 @@ describe('ChromyRunner', () => { ], }; newConfig.type = configTypes.test; - const result = await run(chromy, globalConfig, newConfig); + const result = await run(chromy, globalConfig, newConfig, mockReporter); expect(result).toEqual(true); expect(chromy.goto).toHaveBeenCalledWith('www.example.com'); expect(chromy.screenshotDocument).toHaveBeenCalledTimes(1); @@ -159,7 +183,7 @@ describe('ChromyRunner', () => { ], }; newConfig.type = configTypes.test; - const result = await run(chromy, globalConfig, newConfig); + const result = await run(chromy, globalConfig, newConfig, mockReporter); expect(result).toEqual(true); expect(chromy.goto).toHaveBeenCalledWith('www.example.com'); expect(chromy.screenshotSelector).toHaveBeenCalledTimes(1); @@ -183,7 +207,7 @@ describe('ChromyRunner', () => { ], }; newConfig.type = configTypes.test; - const result = await run(chromy, globalConfig, newConfig); + const result = await run(chromy, globalConfig, newConfig, mockReporter); expect(result).toEqual(true); expect(chromy.goto).toHaveBeenCalledWith('www.example.com'); expect(chromy.wait).toHaveBeenCalledWith(10); @@ -203,7 +227,7 @@ describe('ChromyRunner', () => { ], }; newConfig.type = configTypes.test; - const result = await run(chromy, globalConfig, newConfig); + const result = await run(chromy, globalConfig, newConfig, mockReporter); expect(result).toEqual(true); expect(chromy.goto).toHaveBeenCalledWith('www.example.com'); expect(chromy.wait).toHaveBeenCalledWith('selector name'); @@ -223,7 +247,7 @@ describe('ChromyRunner', () => { ], }; newConfig.type = configTypes.test; - const result = await run(chromy, globalConfig, newConfig); + const result = await run(chromy, globalConfig, newConfig, mockReporter); expect(result).toEqual(true); expect(chromy.goto).toHaveBeenCalledWith('www.example.com'); expect(chromy.wait).toHaveBeenCalledTimes(1); @@ -243,7 +267,7 @@ describe('ChromyRunner', () => { ], }; newConfig.type = configTypes.test; - const result = await run(chromy, globalConfig, newConfig); + const result = await run(chromy, globalConfig, newConfig, mockReporter); expect(result).toEqual(false); expect(chromy.goto).toHaveBeenCalledWith('www.example.com'); expect(chromy.wait).toHaveBeenCalledTimes(0); @@ -263,7 +287,7 @@ describe('ChromyRunner', () => { { name: 'execute', value: () => {} }, ], }; - const result = await run(chromy, globalConfig, newConfig); + const result = await run(chromy, globalConfig, newConfig, mockReporter); expect(result).toEqual(true); expect(chromy.evaluate).toHaveBeenCalledTimes(1); expect(loggerCalls[0]).toEqual('waiting for to execute function in browser'); @@ -279,7 +303,7 @@ describe('ChromyRunner', () => { { name: 'execute', value: 123 }, ], }; - const result = await run(chromy, globalConfig, newConfig); + const result = await run(chromy, globalConfig, newConfig, mockReporter); expect(result).toEqual(false); expect(chromy.evaluate).toHaveBeenCalledTimes(0); expect(loggerCalls[0]).toEqual('failed to detect execute function'); @@ -299,7 +323,7 @@ describe('ChromyRunner', () => { { name: 'freezeImage', value: 'selector' }, ], }; - const result = await run(chromy, globalConfig, newConfig); + const result = await run(chromy, globalConfig, newConfig, mockReporter); expect(result).toEqual(true); expect(chromy.evaluate).toHaveBeenCalledWith('return string function'); expect(functionToString).toHaveBeenCalledWith(freezeImage, 'selector'); @@ -318,7 +342,7 @@ describe('ChromyRunner', () => { { name: 'freezeImage', value: 'selector' }, ], }; - const result = await run(chromy, globalConfig, newConfig); + const result = await run(chromy, globalConfig, newConfig, mockReporter); expect(result).toEqual(false); expect(chromy.evaluate).toHaveBeenCalledWith('return string function'); expect(functionToString).toHaveBeenCalledWith(freezeImage, 'selector'); diff --git a/src/compareImage.js b/src/compareImage.js index 869a6c7..3a564d2 100644 --- a/src/compareImage.js +++ b/src/compareImage.js @@ -26,31 +26,30 @@ const compareImage = async (options, testName, reporter) => { const diff = Jimp.diff(referenceImage, testImage, options.mismatchThreshold); if (distance < options.mismatchThreshold && diff.percent < options.mismatchThreshold) { const result = 'no mismatch found ✅'; - reporter.addResult(true, testFile, result); + reporter.addResult({ + outcome: true, + testName, + result, + }); return result; } - const result = `mismatch found❗ - Result: - distance: ${distance} - diff: ${diff.percent} - misMatchThreshold: ${options.mismatchThreshold} - `; - if (options.saveDifferencifiedImage) { try { const diffPath = `${options.testReportPath}/${testName}_differencified.png`; diff.image.write(diffPath); prefixedLogger.log(`saved the diff image to disk at ${diffPath}`); - reporter.addResult(false, testFile, result, diffPath); } catch (err) { throw new Error(`failed to save the diff image ${err}`); } - } else { - reporter.addResult(false, testFile, result); } - throw new Error(result); + throw new Error(`mismatch found❗ + Result: + distance: ${distance} + diff: ${diff.percent} + misMatchThreshold: ${options.mismatchThreshold} + `); }; export default compareImage; diff --git a/src/compareImage.test.js b/src/compareImage.test.js index 651d713..d1bf04c 100644 --- a/src/compareImage.test.js +++ b/src/compareImage.test.js @@ -67,6 +67,15 @@ describe('Compare Image', () => { expect(result).toEqual('no mismatch found ✅'); }); + it('add a result to reporter if difference below threshold', async () => { + await compareImage(mockConfig, 'test', mockReporter); + expect(mockReporter.addResult).toHaveBeenCalledWith({ + outcome: true, + result: 'no mismatch found ✅', + testName: 'test', + }); + }); + it('returns mismatch found❗ if only difference above threshold', async () => { expect.assertions(1); Jimp.diff.mockReturnValue({ percent: 0.02 }); diff --git a/src/index.js b/src/index.js index 7cd5ca2..3ad2cb2 100644 --- a/src/index.js +++ b/src/index.js @@ -29,10 +29,10 @@ const getFreePort = async () => { }; export default class Differencify { - constructor(conf, reporter = new Reporter()) { + constructor(conf, reporter) { this.configuration = sanitiseGlobalConfiguration(conf); this.chromeInstances = {}; - this.reporter = reporter; + this.reporter = reporter || new Reporter(this.configuration); if (this.configuration.debug === true) { logger.enable(); } diff --git a/src/index.test.js b/src/index.test.js index 237e5ac..8d21891 100644 --- a/src/index.test.js +++ b/src/index.test.js @@ -1,7 +1,6 @@ import fs from 'fs'; import Chromy from 'chromy'; import getPort from 'get-port'; -import Reporter from './Reporter'; import Differencify from './index'; import logger from './logger'; import run from './chromyRunner'; @@ -30,9 +29,7 @@ jest.mock('./logger', () => ({ enable: jest.fn(), })); -jest.mock('./Reporter'); - -Reporter.mockImplementation(() => ({ +jest.mock('./Reporter', () => () => ({ generate: jest.fn(), })); diff --git a/src/reportTypes/htmlReport.js b/src/reportTypes/htmlReport.js index 21556b3..d414cdb 100644 --- a/src/reportTypes/htmlReport.js +++ b/src/reportTypes/htmlReport.js @@ -3,15 +3,26 @@ export default files => Differencify report @@ -20,27 +31,34 @@ export default files => - + + + - ${files.map(file => ` + + - `).join('')} diff --git a/src/reportTypes/htmlReport.test.js b/src/reportTypes/htmlReport.test.js index 063de41..50ff06b 100644 --- a/src/reportTypes/htmlReport.test.js +++ b/src/reportTypes/htmlReport.test.js @@ -4,15 +4,13 @@ import getHtmlReport from './htmlReport'; const results = [ { outcome: true, - fileName: 'image1.png', - message: 'no mismatch found', - diff: null, + testName: 'default', + result: 'no mismatch found', }, { outcome: false, - fileName: 'image2.png', - message: 'mismatch found!', - diff: 'image2_diff.png', + testName: 'default2', + result: 'mismatch found!', }, ]; @@ -22,10 +20,10 @@ describe('HTML report', () => { const $ = cheerio.load(report); expect($('tbody tr').length).toBe(2); expect($('tbody tr:first-child td:first-child').text().trim()).toBe( - results[0].fileName, + results[0].testName, ); expect($('tbody tr:nth-child(2) td:first-child').text().trim()).toBe( - results[1].fileName, + results[1].testName, ); }); }); diff --git a/src/reportTypes/jsonReport.test.js b/src/reportTypes/jsonReport.test.js index f568c28..26bd50d 100644 --- a/src/reportTypes/jsonReport.test.js +++ b/src/reportTypes/jsonReport.test.js @@ -3,9 +3,8 @@ import getJsonReport from './jsonReport'; const results = [ { outcome: true, - fileName: 'image1.png', - message: 'no mismatch found', - diff: null, + testName: 'default', + result: 'no mismatch found', }, ];
FilenameTest nameReferenceDifference OutcomeMessage
- - ${file.fileName} - + ${file.testName} + + ${file.referenceFileName ? ` + + + + ` : ''} + + ${(file.diffFileName) ? ` + + + + ` : ''} ${file.outcome ? 'Pass' : 'Fail'} - ${file.message} - ${file.diff ? ` - - View diff - ` : '' - } + ${file.result}