From 98bcdfe7960525e221265601d67af9c1215b3a7e Mon Sep 17 00:00:00 2001 From: Patty RoDee Date: Fri, 30 Mar 2018 15:08:32 -0700 Subject: [PATCH] feat: remote hosted screenshot testing (#12) --- .eslintrc.yml | 2 + .travis.yml | 6 +- package-lock.json | 42 +++- package.json | 7 +- test/screenshot/golden.json | 4 + test/screenshot/screenshot.js | 197 +++++++++++++++--- .../temporary-package/foo.html.golden.png | Bin 4099 -> 0 bytes .../temporary-package/index.html.golden.png | Bin 4628 -> 0 bytes test/screenshot/upload-screenshots.js | 59 ------ test/screenshot/webpack-bundles.js | 35 ++-- 10 files changed, 237 insertions(+), 115 deletions(-) create mode 100644 test/screenshot/golden.json delete mode 100644 test/screenshot/temporary-package/foo.html.golden.png delete mode 100644 test/screenshot/temporary-package/index.html.golden.png delete mode 100644 test/screenshot/upload-screenshots.js diff --git a/.eslintrc.yml b/.eslintrc.yml index 36a7e1783..0843b05ea 100644 --- a/.eslintrc.yml +++ b/.eslintrc.yml @@ -9,3 +9,5 @@ rules: require-jsdoc: off valid-jsdoc: off switch-colon-spacing: 0 + max-len: [error, 120] + indent: [error, 2, {"SwitchCase":1}] diff --git a/.travis.yml b/.travis.yml index 6ccab7698..1acb5dc54 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,10 @@ git: dist: trusty sudo: required +branches: + only: + - master + matrix: include: - node_js: 8 @@ -24,5 +28,3 @@ matrix: - ./test/screenshot/start.sh - sleep 10s - npm run test:image-diff - after_script: - - COMMIT_HASH=$(git rev-parse --short HEAD) npm run upload:screenshots diff --git a/package-lock.json b/package-lock.json index 46a4e353c..7c1ea9bb3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2515,6 +2515,39 @@ "sha.js": "2.4.10" } }, + "cross-env": { + "version": "5.1.4", + "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-5.1.4.tgz", + "integrity": "sha512-Mx8mw6JWhfpYoEk7PGvHxJMLQwQHORAs8+2bX+C1lGQ4h3GkDb1zbzC2Nw85YH9ZQMlO0BHZxMacgrfPmMFxbg==", + "dev": true, + "requires": { + "cross-spawn": "5.1.0", + "is-windows": "1.0.2" + }, + "dependencies": { + "cross-spawn": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-5.1.0.tgz", + "integrity": "sha1-6L0O/uWPz/b4+UUQoKVUu/ojVEk=", + "dev": true, + "requires": { + "lru-cache": "4.1.2", + "shebang-command": "1.2.0", + "which": "1.3.0" + } + }, + "lru-cache": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.2.tgz", + "integrity": "sha512-wgeVXhrDwAWnIF/yZARsFnMBtdFXOg1b8RIrhilp+0iDYN4mdQcNZElDZ0e4B64BhaxeQ5zN7PMyvu7we1kPeQ==", + "dev": true, + "requires": { + "pseudomap": "1.0.2", + "yallist": "2.1.2" + } + } + } + }, "cross-spawn": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-3.0.1.tgz", @@ -4174,15 +4207,6 @@ "null-check": "1.0.0" } }, - "fs-readfile-promise": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/fs-readfile-promise/-/fs-readfile-promise-3.0.1.tgz", - "integrity": "sha512-LsSxMeaJdYH27XrW7Dmq0Gx63mioULCRel63B5VeELYLavi1wF5s0XfsIdKDFdCL9hsfQ2qBvXJszQtQJ9h17A==", - "dev": true, - "requires": { - "graceful-fs": "4.1.11" - } - }, "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", diff --git a/package.json b/package.json index 69040a129..1195e9dd2 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "scripts": { "start": "webpack-dev-server --config test/screenshot/webpack.config.js --content-base test/screenshot", "stop": "./test/screenshot/stop.sh", - "capture": "./node_modules/.bin/mocha --compilers js:babel-core/register --ui tdd test/screenshot/capture-suite.js", + "capture": "cross-env MDC_COMMIT_HASH=$(git rev-parse --short HEAD) MDC_BRANCH_NAME=$(git rev-parse --abbrev-ref HEAD) mocha --compilers js:babel-core/register --ui tdd --timeout 30000 test/screenshot/capture-suite.js", "commitmsg": "validate-commit-msg", "fix": "eslint --fix packages test", "lint": "eslint packages test", @@ -15,7 +15,7 @@ "test:watch": "karma start karma.local.js --auto-watch", "test:unit": "karma start karma.local.js --single-run", "test:unit-ci": "karma start karma.ci.js --single-run", - "test:image-diff": "./node_modules/.bin/mocha --compilers js:babel-core/register --ui tdd --timeout 30000 test/screenshot/diff-suite.js", + "test:image-diff": "cross-env MDC_COMMIT_HASH=$(git rev-parse --short HEAD) MDC_BRANCH_NAME=$(git rev-parse --abbrev-ref HEAD) mocha --compilers js:babel-core/register --ui tdd --timeout 30000 test/screenshot/diff-suite.js", "upload:screenshots": "node ./test/screenshot/upload-screenshots.js" }, "config": { @@ -36,13 +36,12 @@ "babel-preset-react": "^6.24.1", "capture-chrome": "^2.0.0", "chai": "^4.1.2", + "cross-env": "^5.1.4", "css-loader": "^0.28.10", "eslint": "^3.19.0", "eslint-config-google": "^0.9.1", "eslint-plugin-react": "^7.7.0", "extract-text-webpack-plugin": "^3.0.2", - "fs-readfile-promise": "^3.0.1", - "glob": "^7.1.2", "husky": "^0.14.3", "istanbul": "^0.4.5", "istanbul-instrumenter-loader": "^3.0.0", diff --git a/test/screenshot/golden.json b/test/screenshot/golden.json new file mode 100644 index 000000000..7ec5ec069 --- /dev/null +++ b/test/screenshot/golden.json @@ -0,0 +1,4 @@ +{ + "temporary-package/index.html": "57de91e42bd22a845074bae80f71eca3902ab7c6719f129960bc9018c79db95a", + "temporary-package/foo.html": "91f0795700eaba345092dd7507e046daa0d33e1136bad7992011ab454ddb1faf" +} diff --git a/test/screenshot/screenshot.js b/test/screenshot/screenshot.js index 6c1de7986..341053569 100644 --- a/test/screenshot/screenshot.js +++ b/test/screenshot/screenshot.js @@ -1,60 +1,209 @@ +import {Readable} from 'stream'; +import {createHash} from 'crypto'; +import {readFile, writeFile} from 'fs'; +import {promisify} from 'util'; import puppeteer from 'puppeteer'; import compareImages from 'resemblejs/compareImages'; -import {promisify} from 'util'; -import {readFile, writeFile} from 'fs'; import {assert} from 'chai'; +import Storage from '@google-cloud/storage'; import comparisonOptions from './screenshot-comparison-options'; const readFilePromise = promisify(readFile); const writeFilePromise = promisify(writeFile); +const serviceAccountKey = process.env.MDC_GCLOUD_SERVICE_ACCOUNT_KEY; +const branchName = process.env.MDC_BRANCH_NAME; +const commitHash = process.env.MDC_COMMIT_HASH; +const goldenFilePath = './test/screenshot/golden.json'; +const bucketName = 'screenshot-uploads'; +const defaultMetadata = { + commit: commitHash, + branch: branchName, +}; + +const storage = new Storage({ + credentials: JSON.parse(serviceAccountKey), +}); + +const bucket = storage.bucket(bucketName); + export default class Screenshot { + /** + * @param {string} urlPath The URL path to test + */ constructor(urlPath) { + /** @private {string} */ this.urlPath_ = urlPath; - this.imagePath_ = `${urlPath}.golden.png`; - this.snapshotImagePath_ = `${urlPath}.snapshot.png`; - this.diffPath_ = `${urlPath}.diff.png`; // TODO allow clients to specify capture-chrome options, like viewport size } + /** + * Captures a screenshot of the test URL and marks it as the new golden image + */ capture() { test(this.urlPath_, async () => { - const url = `http://localhost:8080/${this.urlPath_}`; - const imagePath = `./test/screenshot/${this.imagePath_}`; - await this.createScreenshotTask_(url, imagePath); + const golden = await this.takeScreenshot_(); + const goldenHash = this.generateImageHash_(golden); + const goldenPath = this.getImagePath_(goldenHash, 'golden'); + await Promise.all([ + this.saveImage_(goldenPath, golden), + this.saveGoldenHash_(goldenHash), + ]); return; }); } + /** + * Diffs a screenshot of the test URL with the existing golden screenshot + */ diff() { test(this.urlPath_, async () => { - const url = `http://localhost:8080/${this.urlPath_}`; - const imagePath = `./test/screenshot/${this.imagePath_}`; - const snapshotImagePath = `./test/screenshot/${this.snapshotImagePath_}`; - const diffPath = `./test/screenshot/${this.diffPath_}`; - - const [newScreenshot, oldScreenshot] = await Promise.all([ - this.createScreenshotTask_(url, snapshotImagePath), - readFilePromise(imagePath), + // Get the golden file path from the golden hash + const goldenHash = await this.getGoldenHash_(); + const goldenPath = this.getImagePath_(goldenHash, 'golden'); + + // Take a snapshot and download the golden iamge + const [snapshot, golden] = await Promise.all([ + this.takeScreenshot_(), + this.readImage_(goldenPath), ]); - const data = await compareImages(newScreenshot, oldScreenshot, - comparisonOptions); + // Compare the images + const data = await compareImages(snapshot, golden, comparisonOptions); + const diff = data.getBuffer(); + + // Use the same hash for the snapshot path and diff path so it's easy can associate the two + const snapshotHash = this.generateImageHash_(snapshot); + const snapshotPath = this.getImagePath_(snapshotHash, 'snapshot'); + const diffPath = this.getImagePath_(snapshotHash, 'diff'); + const metadata = {golden: goldenHash}; + + // Save the snapshot and the diff + await Promise.all([ + this.saveImage_(snapshotPath, snapshot, metadata), + this.saveImage_(diffPath, diff, metadata), + ]); + + return assert.isBelow(Number(data.misMatchPercentage), 0.01); + }); + } + + /** + * Generates a unique hash from an image's contents + * @param {!Buffer} imageBuffer The image buffer to hash + * @return {string} + * @private + */ + generateImageHash_(imageBuffer) { + return createHash('sha256').update(imageBuffer).digest('hex'); + } + + /** + * Returns the golden hash + * @return {string|undefined} + * @private + */ + async getGoldenHash_() { + const goldenFile = await readFilePromise(goldenFilePath); + const goldenJSON = JSON.parse(goldenFile); + return goldenJSON[this.urlPath_]; + } + + /** + * Returns the correct image path + * @param {string} imageHash The image hash + * @param {string} imageType The image type + * @return {string|undefined} + * @private + */ + getImagePath_(imageHash, imageType) { + if (imageType === 'golden') { + return `${this.urlPath_}/${imageHash}.golden.png`; + } + + if (['snapshot', 'diff'].includes(imageType)) { + return `${this.urlPath_}/${commitHash}/${imageHash}.${imageType}.png`; + } + } + + /** + * Downloads an image from Google Cloud Storage + * @param {string} gcsFilePath The file path on Google Cloud Storage + * @return {Buffer} + * @private + */ + async readImage_(gcsFilePath) { + const data = await bucket.file(gcsFilePath).download(); + return data[0]; + } - await writeFilePromise(diffPath, data.getBuffer()); + /** + * Saves the golden hash + * @param {string} goldenHash The hash of the golden image + * @private + */ + async saveGoldenHash_(goldenHash) { + const goldenFile = await readFilePromise(goldenFilePath); + const goldenJSON = JSON.parse(goldenFile); + goldenJSON[this.urlPath_] = goldenHash; + const goldenContent = JSON.stringify(goldenJSON, null, ' '); + await writeFilePromise(goldenFilePath, `${goldenContent}\r\n`); + } - assert.isBelow(Number(data.misMatchPercentage), 0.01); + /** + * Saves the given image to Google Cloud Storage with optional metadata + * @param {string} imagePath The path to the image + * @param {!Buffer} imageBuffer The image buffer + * @param {!Object=} customMetadata Optional custom metadata + * @private + */ + async saveImage_(imagePath, imageBuffer, customMetadata={}) { + const metadata = Object.assign({}, defaultMetadata, customMetadata); + const file = bucket.file(imagePath); + + // Check if file exists and exit if it does + const [exists] = await file.exists(); + if (exists) { + console.log('✔︎ No changes to', imagePath); return; + } + + // Create a new stream from the image buffer + let stream = new Readable(); + stream.push(imageBuffer); + stream.push(null); + + // The promise is resolved or rejected inside the stream event callbacks + return new Promise((resolve, reject) => { + stream.pipe(file.createWriteStream()) + .on('error', (err) => { + reject(err); + }).on('finish', async () => { + try { + // Make the image public and set it's metadata + await file.makePublic(); + await file.setMetadata({metadata}); + console.log('✔︎ Uploaded', imagePath); + resolve(); + } catch (err) { + reject(err); + } + }); }); } - async createScreenshotTask_(url, path) { + /** + * Takes a screenshot of the URL + * @return {!Buffer} + * @private + */ + async takeScreenshot_() { const browser = await puppeteer.launch(); const page = await browser.newPage(); - await page.goto(url); - const image = await page.screenshot({path}); + await page.goto(`http://localhost:8080/${this.urlPath_}`); + const imageBuffer = await page.screenshot(); await browser.close(); - return image; + return imageBuffer; } } diff --git a/test/screenshot/temporary-package/foo.html.golden.png b/test/screenshot/temporary-package/foo.html.golden.png deleted file mode 100644 index d324981c21217251e0edf48d074a0cae328d4f1d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4099 zcmeHKYfMvD96u-s!61IZM5LGuBn*T|la>m!s4R@I6|*fo#KL&BCd#wC8fd#=DnT}h zuhADYhGptxbl89tXp15bp%2B)reGC88^{P)^34-5smp-mnA^b7W> zqs=(qX3ZBrU*g_k$umC<4_uXpgf_)QT=fda#N4|gCti=dQy9SwxyfR2rygV}$S1<4 z_#HOVrC*XIX-c%eu2qeQEH$nHr&ir#-;oJz7b06H^>A6wbf+*3-tAPM!vGxrBoTn` zegGZ-#O(6~VElt%XGztZ)oLAd?93P)W}RwQQinYC4(5;KZ&H+g0PHf!sUDBTVqA^3 zO3jZg&nJ^}7?z?q{ z_S_qQvKI_&BHp^(p%GD3a(TQ_j;5NmH7YR`A(IWO8WGjB{3 z3>Jk#K{G0_l;YyzfaZ}^os5aMEZrvLg#$2@t6e`Zu561#zV}A4Ih@NvNp(rz{Ix6+ zsV$gkFF%BekBye@E*!&E6z0^H)wtL zh-x+cg)5sIs^56(&0L#+0?PO&-+SPy-u(gjQKU}midg?sHl+I_c9xwSOs|}!A#LJy zsSQGpgi@}yunBw&P+o$30`kt5WV-<~KJTL^gJ;MPQEXQv!aL3U((W)B|0gcy8QAU&E+st=yW=&UZVl@Kx(=x^;`>R;*Nr4byJc-yL|*-RAR6V@f>*|(LizZB7JJ7Op*BIlru zZ?a2C$3)%Tr{FZ6cA!SJtCYobgO+l1x%%ux^w)LI# z8fOiAyS5KMyWAxv<6U+mu6}nT-t_Vu8ZD|buUU2%TaezF5~#x6yLT-c!~V#_`|Vr= z8nYl8YY7&dYdqx0YlcyFqIyl(K^f8*%wX#)a2cekKV?BiD7RvCbo2_eCOsqCGI9`r z4eU;%rwY%O31=>=RfNA!A`otk2$&AD!Enq_jj8+A?}2@6Y@`Ae(rX-+u?49 iyB#0-qxgS5F6{nUzk6TgHLwf-UMZgule*4++weDJ!NvOk diff --git a/test/screenshot/temporary-package/index.html.golden.png b/test/screenshot/temporary-package/index.html.golden.png deleted file mode 100644 index 6d1cdb959b34429dc72fb70f8c998c2098b71793..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4628 zcmeHK?Nd`%9zLR7T)WD$-Im!RFSE9#v)K;HRo((bx3K7T0JrS`0tv-Hq5=lUVhl;R zE*+#!l|ZY7D-cFI6{`?Mgg^oW1|+y%as?N{i;zO(Mu-IDl7xgLchlXO{j`6;e#n>e z;W;19bI$MgJkRf3I-VH6d)K?W006s>9gY1IfbD?*c%<(9g$ucUuB-q6@0nw3f^~*a<+Dw~N~0KVNVGzuV>kK=%IxJANCuT53Vx zudxe)XK4^<}(fvPHBx03`_*(Cz;Fx#|c=X&Y}4Ks5}xnYw%%X`r%W97`W_6uecG z89;60Xz|IWm_yv z`rm*!@D_TyX{38q1ktRIQH1Tk`NXj@rQqRwEIq4WG!9A?S zYdW!VrHeGn5J;Ws?K+;4dTw=YTE7EaP_&U#%NLs@k;JuoXx>aHv0j4-9R{d|9!gwo zLhWXm!Dr3V%ZnQxGXH!vD~5G%KH>@ODLrQ#pThIy22+<_ZMS03Dj!2Bj8^Hv=hNmD z! zi@}g4gS+OCSkdCeY!<%$M6+=}thX>>@%o7)MU7j{cS9gZYgMC5O`IXs(1FVElSDWK ziG?p%!o4U@sgPdI8&nO&K!#KR`ub-V<6h2m%(h3V+s9>`%yuV67K^+&F-+dPLpFB_{q3@ zf5-3%_DGcVajv(mBIe;On#gn)5|{k*@4O!)uL|b4cnhb&I#$DqA&~at_HRC!$S=mI zoEBa9Rk~v0-D=nJ`Ca7D?%dV6w?ln%Ar?zbgFo`u_6VXgxg%)@VcU|74i^a+j^($L zIp+tiPncRbBi(D|2EjWwwu8PKejRY^W_fTk^~lOxH1V`T-ED4%I3>^Uh98W>>2o1O zB@gX37U%wDB9{RX4{AobNrz0Sg(R&KEP~C=)2oq+=)8~iE8CwXrt{Y2n+r%qR5txU zgn1GPhWT+%ll?>+Y1|^yxB;%N7n$YDMK7w;X)a%X?&sGElk<)`M?zSOBF&GB(q~}O zS)8s_$jM}f(_(n1+czb_c*pl_E&u74gRYwAJ<9U>wafE9j>!Aj`+)JQf197j4giLC zx;6s$9saK)x8n}hv7J^#{ZXviM(mfvh=d { - const fileName = path.resolve(fname); - const bucketFileName = fname.replace(DIR, COMMIT_HASH); - const file = bucket.file(bucketFileName); - - fs.createReadStream(fileName) - .pipe(file.createWriteStream()) - .on('error', (err) => { - console.error(err); - }).on('finish', () => { - file.makePublic().then(() => { - console.log('✔︎ Uploaded', fileName); - }).catch((err) => { - console.error(err); - }); - }); -}); diff --git a/test/screenshot/webpack-bundles.js b/test/screenshot/webpack-bundles.js index 588f2a923..1f12d37bb 100644 --- a/test/screenshot/webpack-bundles.js +++ b/test/screenshot/webpack-bundles.js @@ -18,27 +18,28 @@ module.exports.bundle = function(testPath, outputPath) { test: /\.scss$/, use: ExtractTextPlugin.extract({ use: [ - { - loader: 'css-loader', - }, - { - loader: 'sass-loader', - options: { - importer: function(url, prev) { - if (url.indexOf('@material') === 0) { - let filePath = url.split('@material')[1]; - let nodeModulePath = `./node_modules/@material/${filePath}`; + { + loader: 'css-loader', + }, + { + loader: 'sass-loader', + options: { + importer: function(url, prev) { + if (url.indexOf('@material') === 0) { + let filePath = url.split('@material')[1]; + let nodeModulePath = `./node_modules/@material/${filePath}`; + return { + file: require('path').resolve(nodeModulePath), + }; + } return { - file: require('path').resolve(nodeModulePath), + file: url, }; - } - return { - file: url, - }; + }, }, }, - }, - ]}), + ], + }), }], }, plugins: [