diff --git a/test/e2e/check-coverage.js b/test/e2e/check-coverage.js index e69d345f715bdf..da0498d719187b 100644 --- a/test/e2e/check-coverage.js +++ b/test/e2e/check-coverage.js @@ -1,53 +1,55 @@ -import fs from 'fs'; +import chalk from 'chalk'; +import * as fs from 'fs/promises'; -// examples -const E = fs.readdirSync( './examples' ) - .filter( s => s.slice( - 5 ) === '.html' ) - .map( s => s.slice( 0, s.length - 5 ) ) - .filter( f => f !== 'index' ); +console.red = msg => console.log( chalk.red( msg ) ); +console.green = msg => console.log( chalk.green( msg ) ); -// screenshots -const S = fs.readdirSync( './examples/screenshots' ) - .filter( s => s.slice( - 4 ) === '.jpg' ) - .map( s => s.slice( 0, s.length - 4 ) ); +main(); -// files.js -const F = []; +async function main() { -const files = JSON.parse( fs.readFileSync( './examples/files.json' ) ); + // examples + const E = ( await fs.readdir( 'examples' ) ) + .filter( s => s.endsWith( '.html' ) ) + .map( s => s.slice( 0, s.indexOf( '.' ) ) ) + .filter( f => f !== 'index' ); -for ( const key in files ) { + // screenshots + const S = ( await fs.readdir( 'examples/screenshots' ) ) + .filter( s => s.indexOf( '.' ) !== -1 ) + .map( s => s.slice( 0, s.indexOf( '.' ) ) ); - const section = files[ key ]; - for ( let i = 0, len = section.length; i < len; i ++ ) { + // files.js + const F = []; - F.push( section[ i ] ); + const files = JSON.parse( await fs.readFile( 'examples/files.json' ) ); - } + for ( const section of Object.values( files ) ) { -} + F.push( ...section ); -const subES = E.filter( x => ! S.includes( x ) ); -const subSE = S.filter( x => ! E.includes( x ) ); -const subEF = E.filter( x => ! F.includes( x ) ); -const subFE = F.filter( x => ! E.includes( x ) ); + } -console.green = ( msg ) => console.log( `\x1b[32m${ msg }\x1b[37m` ); -console.red = ( msg ) => console.log( `\x1b[31m${ msg }\x1b[37m` ); + const subES = E.filter( x => ! S.includes( x ) ); + const subSE = S.filter( x => ! E.includes( x ) ); + const subEF = E.filter( x => ! F.includes( x ) ); + const subFE = F.filter( x => ! E.includes( x ) ); -if ( subES.length + subSE.length + subEF.length + subFE.length === 0 ) { + if ( subES.length + subSE.length + subEF.length + subFE.length === 0 ) { - console.green( 'TEST PASSED! All examples is covered with screenshots and descriptions in files.json!' ); + console.green( 'TEST PASSED! All examples is covered with screenshots and descriptions in files.json!' ); -} else { + } else { - if ( subES.length > 0 ) console.red( 'Make screenshot for example(s): ' + subES.join( ' ' ) ); - if ( subSE.length > 0 ) console.red( 'Remove unnecessary screenshot(s): ' + subSE.join( ' ' ) ); - if ( subEF.length > 0 ) console.red( 'Add description in files.json for example(s): ' + subEF.join( ' ' ) ); - if ( subFE.length > 0 ) console.red( 'Remove description in files.json for example(s): ' + subFE.join( ' ' ) ); + if ( subES.length > 0 ) console.red( 'Make screenshot for example(s): ' + subES.join( ' ' ) ); + if ( subSE.length > 0 ) console.red( 'Remove unnecessary screenshot(s): ' + subSE.join( ' ' ) ); + if ( subEF.length > 0 ) console.red( 'Add description in files.json for example(s): ' + subEF.join( ' ' ) ); + if ( subFE.length > 0 ) console.red( 'Remove description in files.json for example(s): ' + subFE.join( ' ' ) ); - console.red( 'TEST FAILED!' ); + console.red( 'TEST FAILED!' ); - process.exit( 1 ); + process.exit( 1 ); + + } } diff --git a/test/e2e/clean-page.js b/test/e2e/clean-page.js index c82fff07d3d3bc..5fa59a778bba86 100644 --- a/test/e2e/clean-page.js +++ b/test/e2e/clean-page.js @@ -1,42 +1,26 @@ - ( function () { - - /* Remove start screen ( or press some button ) */ + /* Remove start screen (or press some button ) */ const button = document.getElementById( 'startButton' ); - - if ( button ) { - - button.click(); - - } - + if ( button ) button.click(); /* Remove gui and fonts */ const style = document.createElement( 'style' ); style.type = 'text/css'; - style.innerHTML = `body { font size: 0 !important; } - #info, button, input, body > div.lil-gui, body > div.lbl { display: none !important; }`; - - const head = document.getElementsByTagName( 'head' ); - - if ( head.length > 0 ) { - - head[ 0 ].appendChild( style ); - - } + style.innerHTML = '#info, button, input, body > div.lil-gui, body > div.lbl { display: none !important; }'; - /* Remove stats.js */ + document.querySelector( 'head' ).appendChild( style ); - const canvas = document.getElementsByTagName( 'canvas' ); + /* Remove Stats.js */ - for ( let i = 0; i < canvas.length; ++ i ) { + for ( const element of document.querySelectorAll( 'div' ) ) { - if ( canvas[ i ].height === 48 ) { + if ( getComputedStyle( element ).zIndex === '10000' ) { - canvas[ i ].style.display = 'none'; + element.remove(); + break; } diff --git a/test/e2e/deterministic-injection.js b/test/e2e/deterministic-injection.js index 112132e890c317..a0e9e99840877b 100644 --- a/test/e2e/deterministic-injection.js +++ b/test/e2e/deterministic-injection.js @@ -1,4 +1,3 @@ - ( function () { /* Deterministic random */ @@ -13,7 +12,6 @@ }; - /* Deterministic timer */ window.performance._now = performance.now; @@ -24,7 +22,6 @@ window.Date.prototype.getTime = now; window.performance.now = now; - /* Deterministic RAF */ const RAF = window.requestAnimationFrame; @@ -62,7 +59,6 @@ }; - /* Semi-determitistic video */ const play = HTMLVideoElement.prototype.play; diff --git a/test/e2e/puppeteer.js b/test/e2e/puppeteer.js index 554f1eecc76496..42c538694f3662 100644 --- a/test/e2e/puppeteer.js +++ b/test/e2e/puppeteer.js @@ -1,357 +1,400 @@ +import chalk from 'chalk'; import puppeteer from 'puppeteer'; -import handler from 'serve-handler'; -import http from 'http'; +import express from 'express'; +import path from 'path'; import pixelmatch from 'pixelmatch'; import jimp from 'jimp'; -import fs from 'fs'; +import * as fs from 'fs/promises'; + +/* CONFIG VARIABLES START */ + +const idleTime = 3; // 3 seconds - for how long there should be no network requests +const parseTime = 2; // 2 seconds per megabyte + +const exceptionList = [ + + // video tag not deterministic enough + 'css3d_youtube', + 'webgl_video_kinect', + 'webgl_video_panorama_equirectangular', + + 'webaudio_visualizer', // audio can't be analyzed without proper audio hook + + 'webgl_effects_ascii', // blink renders text differently in every platform + + 'webxr_ar_lighting', // webxr + + 'webgl_worker_offscreencanvas', // in a worker, not robust + + // TODO: most of these can be fixed just by increasing idleTime and parseTime + 'webgl_lensflares', + 'webgl_lines_sphere', + 'webgl_loader_imagebitmap', + 'webgl_loader_texture_lottie', + 'webgl_loader_texture_pvrtc', + 'webgl_morphtargets_face', + 'webgl_nodes_materials_standard', + 'webgl_postprocessing_crossfade', + 'webgl_raymarching_reflect', + 'webgl_renderer_pathtracer', + 'webgl_shadowmap_progressive', + 'webgl_test_memory2', + 'webgl_tiled_forward' + +]; + +/* CONFIG VARIABLES END */ const port = 1234; const pixelThreshold = 0.1; // threshold error in one pixel -const maxFailedPixels = 0.05; // total failed pixels +const maxFailedPixels = 0.05; // at most 5% failed pixels -const networkTimeout = 600; -const networkTax = 2000; // additional timeout for resources size -const pageSizeMinTax = 1.0; // in mb, when networkTax = 0 -const pageSizeMaxTax = 5.0; // in mb, when networkTax = networkTax -const renderTimeout = 1200; -const maxAttemptId = 3; // progresseve attempts -const progressFunc = n => 1 + n; +const networkTimeout = 30; // 30 seconds, set to 0 to disable +const renderTimeout = 1.5; // 1.5 seconds, set to 0 to disable + +const numAttempts = 3; // perform 3 progressive attempts before failing + +const numCIJobs = 8; // GitHub Actions run the script in 8 threads const width = 400; const height = 250; const viewScale = 2; const jpgQuality = 95; -const exceptionList = [ +console.red = msg => console.log( chalk.red( msg ) ); +console.yellow = msg => console.log( chalk.yellow( msg ) ); +console.green = msg => console.log( chalk.green( msg ) ); - 'index', - 'css3d_youtube', // video tag not deterministic enough - 'webaudio_visualizer', // audio can't be analyzed without proper audio hook - 'webgl_effects_ascii', // blink renders text differently in every platform - 'webgl_loader_imagebitmap', // takes too long to load? - 'webgl_loader_texture_lottie', // not sure why this fails - 'webgl_loader_texture_pvrtc', // not supported in CI, useless - 'webgl_morphtargets_face', // To investigate... - 'webgl_nodes_materials_standard', // puppeteer does not support import maps yet - 'webgl_postprocessing_crossfade', // fails for some misterious reason - 'webgl_raymarching_reflect', // exception for Github Actions - 'webgl_renderer_pathtracer', // slow to render - 'webgl_test_memory2', // gives fatal error in puppeteer - 'webgl_tiled_forward', // exception for Github Actions - 'webgl_video_kinect', // video tag not deterministic enough - 'webgl_video_panorama_equirectangular', // video tag not deterministic enough? - 'webgl_worker_offscreencanvas', // in a worker, not robust - // webxr - 'webxr_ar_lighting' -]; +let browser; -console.green = ( msg ) => console.log( `\x1b[32m${ msg }\x1b[37m` ); -console.red = ( msg ) => console.log( `\x1b[31m${ msg }\x1b[37m` ); -console.null = () => {}; +/* Launch server */ +const app = express(); +app.use( express.static( path.resolve() ) ); +const server = app.listen( port, main ); -/* Launch server */ +process.on( 'SIGINT', () => close() ); -const server = http.createServer( ( req, resp ) => handler( req, resp ) ); -server.listen( port, async () => await pup ); -server.on( 'SIGINT', () => process.exit( 1 ) ); +async function main() { + /* Find files */ -/* Launch browser */ + const isMakeScreenshot = process.argv[ 2 ] === '--make'; -const pup = puppeteer.launch( { - headless: ! process.env.VISIBLE, - args: [ - '--use-gl=swiftshader', - '--no-sandbox', - '--enable-surface-synchronization', + const exactList = process.argv.slice( isMakeScreenshot ? 3 : 2 ) + .map( f => f.replace( '.html', '' ) ); - '--enable-unsafe-webgpu', - '--enable-features=Vulkan', - '--use-angle=swiftshader', - '--use-vulkan=swiftshader', - '--use-webgpu-adapter=swiftshader' - ] -} ).then( async browser => { + const isExactList = exactList.length !== 0; + let files = ( await fs.readdir( 'examples' ) ) + .filter( s => s.slice( - 5 ) === '.html' && s !== 'index.html' ) + .map( s => s.slice( 0, s.length - 5 ) ) + .filter( f => isExactList ? exactList.includes( f ) : ! exceptionList.includes( f ) ); - /* Prepare page */ + if ( isExactList ) { - const page = ( await browser.pages() )[ 0 ]; - await page.setViewport( { width: width * viewScale, height: height * viewScale } ); + for ( const file of exactList ) { - const cleanPage = fs.readFileSync( 'test/e2e/clean-page.js', 'utf8' ); - const injection = fs.readFileSync( 'test/e2e/deterministic-injection.js', 'utf8' ); - await page.evaluateOnNewDocument( injection ); + if ( ! files.includes( file ) ) { - const threeJsBuild = fs.readFileSync( 'build/three.module.js', 'utf8' ) - .replace( /Math\.random\(\) \* 0xffffffff/g, 'Math._random() * 0xffffffff' ); - await page.setRequestInterception( true ); + console.log( `Warning! Unrecognised example name: ${ file }` ); - page.on( 'console', msg => ( msg.text().slice( 0, 8 ) === 'Warning.' ) ? console.null( msg.text() ) : {} ); - page.on( 'request', async ( request ) => { + } - if ( request.url() === 'http://localhost:1234/build/three.module.js' ) { + } - await request.respond( { - status: 200, - contentType: 'application/javascript; charset=utf-8', - body: threeJsBuild - } ); + } - } else { + /* CI parallelism */ - await request.continue(); + if ( 'CI' in process.env ) { - } + const CI = parseInt( process.env.CI ); + files = files.slice( + Math.floor( CI * files.length / numCIJobs ), + Math.floor( ( CI + 1 ) * files.length / numCIJobs ) + ); + + } + + /* Launch browser */ + + const flags = [ '--hide-scrollbars', '--enable-unsafe-webgpu' ]; + flags.push( '--enable-features=Vulkan', '--use-gl=swiftshader', '--use-angle=swiftshader', '--use-vulkan=swiftshader', '--use-webgpu-adapter=swiftshader' ); + // if ( process.platform === 'linux' ) flags.push( '--enable-features=Vulkan,UseSkiaRenderer', '--use-vulkan=native', '--disable-vulkan-surface', '--disable-features=VaapiVideoDecoder', '--ignore-gpu-blocklist', '--use-angle=vulkan' ); + + const viewport = { width: width * viewScale, height: height * viewScale }; + + browser = await puppeteer.launch( { + headless: ! process.env.VISIBLE, + args: flags, + defaultViewport: viewport, + handleSIGINT: false } ); - page.on( 'response', async ( response ) => { - try { + // this line is intended to stop the script if the browser (in headful mode) is closed by user (while debugging) + // browser.on( 'targetdestroyed', target => ( target.type() === 'other' ) ? close() : null ); + // for some reason it randomly stops the script after about ~30 screenshots processed - await response.buffer().then( buffer => pageSize += buffer.length ); + /* Prepare injections */ - } catch ( e ) { + const cleanPage = await fs.readFile( 'test/e2e/clean-page.js', 'utf8' ); + const injection = await fs.readFile( 'test/e2e/deterministic-injection.js', 'utf8' ); + const build = ( await fs.readFile( 'build/three.module.js', 'utf8' ) ).replace( /Math\.random\(\) \* 0xffffffff/g, 'Math._random() * 0xffffffff' ); - console.null( `Warning. Wrong request. \n${ e }` ); + /* Prepare page */ - } + const page = ( await browser.pages() )[ 0 ]; + await preparePage( page, injection, build ); - } ); + /* Loop for each file */ + const failedScreenshots = []; - /* Find files */ + for ( const file of files ) await makeAttempt( page, failedScreenshots, cleanPage, isMakeScreenshot, file ); - const isMakeScreenshot = process.argv[ 2 ] == '--make'; - const isExactList = process.argv.length > ( 2 + isMakeScreenshot ); + /* Finish */ - const exactList = process.argv.slice( isMakeScreenshot ? 3 : 2 ) - .map( f => f.replace( '.html', '' ) ); + const list = failedScreenshots.join( ' ' ); - const files = fs.readdirSync( './examples' ) - .filter( s => s.slice( - 5 ) === '.html' ) - .map( s => s.slice( 0, s.length - 5 ) ) - .filter( f => isExactList ? exactList.includes( f ) : ! exceptionList.includes( f ) ); + if ( isMakeScreenshot && failedScreenshots.length ) { + console.red( 'List of failed screenshots: ' + list ); + console.red( `If you are sure that everything is correct, try to run "npm run make-screenshot ${ list }". If this does not help, try increasing idleTime and parseTime variables in /test/e2e/puppeteer.js file. If this also does not help, add remaining screenshots to the exception list.` ); + console.red( `${ failedScreenshots.length } from ${ files.length } screenshots have not generated succesfully.` ); - /* Loop for each file, with CI parallelism */ + } else if ( isMakeScreenshot && ! failedScreenshots.length ) { - let pageSize, file, attemptProgress; - const failedScreenshots = []; + console.green( `${ files.length } screenshots succesfully generated.` ); - let beginId = 0; - let endId = files.length; + } else if ( failedScreenshots.length ) { - if ( 'CI' in process.env ) { + console.red( 'List of failed screenshots: ' + list ); + console.red( `If you are sure that everything is correct, try to run "npm run make-screenshot ${ list }". If this does not help, try increasing idleTime and parseTime variables in /test/e2e/puppeteer.js file. If this also does not help, add remaining screenshots to the exception list.` ); + console.red( `TEST FAILED! ${ failedScreenshots.length } from ${ files.length } screenshots have not rendered correctly.` ); - const jobs = 8; + } else { - beginId = Math.floor( parseInt( process.env.CI.slice( 0, 1 ) ) * files.length / jobs ); - endId = Math.floor( ( parseInt( process.env.CI.slice( - 1 ) ) + 1 ) * files.length / jobs ); + console.green( `TEST PASSED! ${ files.length } screenshots rendered correctly.` ); } - for ( let id = beginId; id < endId; ++ id ) { - - /* At least 3 attempts before fail */ + setTimeout( close, 300, failedScreenshots.length ); - let attemptId = isMakeScreenshot ? 1.5 : 0; +} - while ( attemptId < maxAttemptId ) { +async function preparePage( page, injection, build ) { - /* Load target page */ + /* let page.pageSize */ - file = files[ id ]; - attemptProgress = progressFunc( attemptId ); - pageSize = 0; + await page.evaluateOnNewDocument( injection ); + await page.setRequestInterception( true ); - try { + page.on( 'response', async ( response ) => { - await page.goto( `http://localhost:${ port }/examples/${ file }.html`, { - waitUntil: 'networkidle2', - timeout: networkTimeout * attemptProgress - } ); + try { - } catch { + if ( response.status === 200 ) { - console.null( 'Warning. Network timeout exceeded...' ); + await response.buffer().then( buffer => page.pageSize += buffer.length ); } - try { + } catch {} - /* Render page */ + } ); - await page.evaluate( cleanPage ); + page.on( 'request', async ( request ) => { - await page.evaluate( async ( pageSize, pageSizeMinTax, pageSizeMaxTax, networkTax, renderTimeout, attemptProgress ) => { + if ( request.url() === `http://localhost:${ port }/build/three.module.js` ) { + await request.respond( { + status: 200, + contentType: 'application/javascript; charset=utf-8', + body: build + } ); - /* Resource timeout */ + } else { - const resourcesSize = Math.min( 1, ( pageSize / 1024 / 1024 - pageSizeMinTax ) / pageSizeMaxTax ); - await new Promise( resolve => setTimeout( resolve, networkTax * resourcesSize * attemptProgress ) ); + await request.continue(); + } - /* Resolve render promise */ + } ); - window._renderStarted = true; +} - await new Promise( function ( resolve ) { +async function makeAttempt( page, failedScreenshots, cleanPage, isMakeScreenshot, file, attemptID = 0 ) { - performance._now = performance._now || performance.now; + const timeoutCoefficient = attemptID + 1; - const renderStart = performance._now(); + try { - const waitingLoop = setInterval( function () { + page.pageSize = 0; - const renderEcceded = ( performance._now() - renderStart > renderTimeout * attemptProgress ); - if ( window._renderFinished || renderEcceded ) { + /* Load target page */ - if ( renderEcceded ) { + try { - console.log( 'Warning. Render timeout exceeded...' ); + await page.goto( `http://localhost:${ port }/examples/${ file }.html`, { + waitUntil: 'networkidle0', + timeout: networkTimeout * timeoutCoefficient * 1000 + } ); - } + } catch ( e ) { - clearInterval( waitingLoop ); - resolve(); + throw new Error( `Error happened while loading file ${ file }: ${ e }` ); - } + } - }, 0 ); + try { - } ); + /* Render page */ - }, pageSize, pageSizeMinTax, pageSizeMaxTax, networkTax, renderTimeout, attemptProgress ); + await page.evaluate( cleanPage ); - } catch ( e ) { + await page.waitForNetworkIdle( { + timeout: networkTimeout * timeoutCoefficient * 1000, + idleTime: idleTime * timeoutCoefficient * 1000 + } ); - if ( ++ attemptId === maxAttemptId ) { + await page.evaluate( async ( renderTimeout, parseTime ) => { - console.red( `Something completely wrong. 'Network timeout' is small for your machine. file: ${ file } \n${ e }` ); - failedScreenshots.push( file ); - continue; + await new Promise( resolve => setTimeout( resolve, parseTime ) ); - } else { + /* Resolve render promise */ - console.log( 'Another attempt..' ); - await new Promise( resolve => setTimeout( resolve, networkTimeout * attemptProgress ) ); + window._renderStarted = true; - } + await new Promise( function ( resolve, reject ) { - } + const renderStart = performance._now(); + const waitingLoop = setInterval( function () { - if ( isMakeScreenshot ) { + const renderTimeoutExceeded = ( renderTimeout > 0 ) && ( performance._now() - renderStart > 1000 * renderTimeout ); + if ( renderTimeoutExceeded ) { - /* Make screenshots */ + clearInterval( waitingLoop ); + reject( 'Render timeout exceeded' ); - attemptId = maxAttemptId; - ( await jimp.read( await page.screenshot() ) ) - .scale( 1 / viewScale ).quality( jpgQuality ) - .write( `./examples/screenshots/${ file }.jpg` ); + } else if ( window._renderFinished ) { - console.green( `file: ${ file } generated` ); + clearInterval( waitingLoop ); + resolve(); + } - } else if ( fs.existsSync( `./examples/screenshots/${ file }.jpg` ) ) { + }, 10 ); + } ); - /* Diff screenshots */ + }, renderTimeout * timeoutCoefficient, page.pageSize / 1024 / 1024 * parseTime * 1000 * timeoutCoefficient ); - const actual = ( await jimp.read( await page.screenshot() ) ).scale( 1 / viewScale ).quality( jpgQuality ).bitmap; - const expected = ( await jimp.read( fs.readFileSync( `./examples/screenshots/${ file }.jpg` ) ) ).bitmap; - const diff = actual; + } catch ( e ) { - let numFailedPixels; + if ( e.message.includes( 'Render timeout exceeded' ) ) { // This can mean that the example doesn't use requestAnimationFrame loop - try { + console.yellow( `Render timeout exceeded in file ${ file }` ); - numFailedPixels = pixelmatch( expected.data, actual.data, diff.data, actual.width, actual.height, { - threshold: pixelThreshold, - alpha: 0.2, - diffMask: process.env.FORCE_COLOR === '0', - diffColor: process.env.FORCE_COLOR === '0' ? [ 255, 255, 255 ] : [ 255, 0, 0 ] - } ); + } else { - } catch { + throw new Error( `Error happened while rendering file ${ file }: ${ e }` ); - attemptId = maxAttemptId; - console.red( `Something completely wrong. Image sizes does not match in file: ${ file }` ); - failedScreenshots.push( file ); - continue; + } - } + } - numFailedPixels /= actual.width * actual.height; + const screenshot = ( await jimp.read( await page.screenshot() ) ).scale( 1 / viewScale ).quality( jpgQuality ); - /* Print results */ - const percFailedPixels = 100 * numFailedPixels; - if ( numFailedPixels < maxFailedPixels ) { + if ( isMakeScreenshot ) { - attemptId = maxAttemptId; - console.green( `diff: ${ percFailedPixels.toFixed( 1 ) }%, file: ${ file }` ); + /* Make screenshots */ - } else { + await screenshot.writeAsync( `examples/screenshots/${ file }.jpg` ); - if ( ++ attemptId === maxAttemptId ) { + console.green( `Screenshot generated for file ${ file }` ); - console.red( `ERROR! Diff wrong in ${ percFailedPixels.toFixed( 1 ) }% of pixels in file: ${ file }` ); - failedScreenshots.push( file ); - continue; + } else { - } else { + /* Diff screenshots */ - console.log( 'Another attempt...' ); + let expected; - } + try { - } + expected = await jimp.read( `examples/screenshots/${ file }.jpg` ); - } else { + } catch { - attemptId = maxAttemptId; - console.log( `Warning! Screenshot not exists: ${ file }` ); - continue; + throw new Error( `Screenshot does not exist: ${ file }` ); } - } + const actual = screenshot.bitmap; + const diff = screenshot.clone(); - } + let numFailedPixels; + try { - /* Finish */ + numFailedPixels = pixelmatch( expected.bitmap.data, actual.data, diff.bitmap.data, actual.width, actual.height, { + threshold: pixelThreshold, + alpha: 0.2, + diffMask: process.env.FORCE_COLOR === '0', + diffColor: process.env.FORCE_COLOR === '0' ? [ 255, 255, 255 ] : [ 255, 0, 0 ] + } ); - if ( failedScreenshots.length ) { + } catch { - if ( failedScreenshots.length > 1 ) { + throw new Error( `Image sizes does not match in file: ${ file }` ); - console.red( 'List of failed screenshots: ' + failedScreenshots.join( ' ' ) ); + } - } else { + numFailedPixels /= actual.width * actual.height; + + /* Print results */ + + const percFailedPixels = 100 * numFailedPixels; + + if ( numFailedPixels < maxFailedPixels ) { - console.red( `If you sure that all is right, try to run \`npm run make-screenshot ${ failedScreenshots[ 0 ] }\`` ); + console.green( `Diff ${ percFailedPixels.toFixed( 1 ) }% in file: ${ file }` ); + + } else { + + throw new Error( `Diff wrong in ${ percFailedPixels.toFixed( 1 ) }% of pixels in file: ${ file }` ); + + } } - console.red( `TEST FAILED! ${ failedScreenshots.length } from ${ endId - beginId } screenshots not pass.` ); + } catch ( e ) { + + if ( attemptID === numAttempts - 1 ) { - } else if ( ! isMakeScreenshot ) { + console.red( e ); + failedScreenshots.push( file ); + + } else { - console.green( `TEST PASSED! ${ endId - beginId } screenshots correctly rendered.` ); + console.yellow( `${ e }, another attempt...` ); + await makeAttempt( page, failedScreenshots, cleanPage, isMakeScreenshot, file, attemptID + 1 ); + + } } - setTimeout( () => { +} + +function close( exitCode = 1 ) { - server.close(); - browser.close(); - process.exit( failedScreenshots.length ); + console.log( 'Closing...' ); - }, 300 ); + if ( browser !== undefined ) browser.close(); + server.close(); + process.exit( exitCode ); -} ); +}