Add screenshot-comparison utility, to protect against visual regressi…

…ons during CSS refactors.

- Remove test/output and added test/screenshots (requires a Jenkins change).
- Add test/screenshots/baseline images. These document the expected state of the UI.
- Add dependency on image-diff package.
- Add utilities/compareScreenshots.js, which can be run via 'npm run compareScreenshots'.

.gitignore

@@ -12,7 +12,9 @@ target
 .idea
 *.iml
 *.log
-/test/output
+/test/screenshots/diff
+/test/screenshots/failure
+/test/screenshots/session
 /esvm
 .htpasswd
 .eslintcache

package.json

@@ -59,7 +59,8 @@
     "makelogs": "makelogs",
     "mocha": "mocha",
     "mocha:debug": "mocha --debug-brk",
-    "sterilize": "grunt sterilize"
+    "sterilize": "grunt sterilize",
+    "compareScreenshots": "node utilities/compareScreenshots"
   },
   "repository": {
     "type": "git",
@@ -171,6 +172,7 @@
     "gruntify-eslint": "1.0.1",
     "html-entities": "1.1.3",
     "husky": "0.8.1",
+    "image-diff": "1.6.0",
     "intern": "3.0.1",
     "istanbul-instrumenter-loader": "0.1.3",
     "karma": "0.13.9",

test/support/pages/common.js

@@ -1,11 +1,16 @@
 import { config, defaultTryTimeout, defaultFindTimeout, remote, shieldPage } from '../';
+import fs from 'fs';
+import mkdirp from 'mkdirp';
+import { promisify } from 'bluebird';
+
+const mkdirpAsync = promisify(mkdirp);
+const writeFileAsync = promisify(fs.writeFile);
 
 export default (function () {
   var Promise = require('bluebird');
   var moment = require('moment');
   var testSubjSelector = require('@spalger/test-subj-selector');
   var getUrl = require('../../utils/get_url');
-  var fs = require('fs');
   var _ = require('lodash');
   var parse = require('url').parse;
   var format = require('url').format;
@@ -244,34 +249,32 @@ export default (function () {
       .then(function () { self.debug('... sleep(' + sleepMilliseconds + ') end'); });
     },
 
-    handleError: function (testObj) {
-      var self = this;
-      var testName = (testObj.parent) ? [testObj.parent.name, testObj.name].join('_') : testObj.name;
-
-      return function (reason) {
-        var now = Date.now();
-        var filename = ['failure', now, testName].join('_') + '.png';
+    handleError(testObj) {
+      const testName = (testObj.parent) ? [testObj.parent.name, testObj.name].join('_') : testObj.name;
+      return reason => {
+        const now = Date.now();
+        const fileName = `failure_${now}_${testName}.png`;
 
-        return self.saveScreenshot(filename)
+        return this.saveScreenshot(fileName, true)
         .finally(function () {
           throw reason;
         });
       };
     },
 
-    saveScreenshot: function saveScreenshot(filename) {
-      var self = this;
-      var outDir = path.resolve('test', 'output');
-
-      return self.remote.takeScreenshot()
-      .then(function writeScreenshot(data) {
-        var filepath = path.resolve(outDir, filename);
-        self.debug('Taking screenshot "' + filepath + '"');
-        fs.writeFileSync(filepath, data);
-      })
-      .catch(function (err) {
-        self.log('SCREENSHOT FAILED: ' + err);
-      });
+    async saveScreenshot(fileName, isFailure = false) {
+      try {
+        const directoryName = isFailure ? 'failure' : 'session';
+        const directoryPath = path.resolve(`test/screenshots/${directoryName}`);
+        const filePath = path.resolve(directoryPath, fileName);
+        this.debug(`Taking screenshot "${filePath}"`);
+
+        const screenshot = await this.remote.takeScreenshot();
+        await mkdirpAsync(directoryPath);
+        await writeFileAsync(filePath, screenshot);
+      } catch (err) {
+        this.log(`SCREENSHOT FAILED: ${err}`);
+      }
     },
 
     findTestSubject: function findTestSubject(selector) {

utilities/compareScreenshots.js

@@ -0,0 +1,43 @@
+
+const fs = require('fs');
+const path = require('path');
+const imageDiff = require('image-diff');
+const mkdirp = require('mkdirp');
+
+function compareScreenshots() {
+  const SCREENSHOTS_DIR = 'test/screenshots';
+  const BASELINE_SCREENSHOTS_DIR = path.resolve(SCREENSHOTS_DIR, 'baseline');
+  const DIFF_SCREENSHOTS_DIR = path.resolve(SCREENSHOTS_DIR, 'diff');
+  const SESSION_SCREENSHOTS_DIR = path.resolve(SCREENSHOTS_DIR, 'session');
+
+  // We don't need to create the baseline dir because it's committed.
+  mkdirp.sync(DIFF_SCREENSHOTS_DIR);
+  mkdirp.sync(SESSION_SCREENSHOTS_DIR);
+
+  fs.readdir(SESSION_SCREENSHOTS_DIR, (readDirError, files) => {
+    const screenshots = files.filter(file => file.indexOf('.png') !== -1);
+
+    screenshots.forEach(screenshot => {
+      const sessionImagePath = path.resolve(SESSION_SCREENSHOTS_DIR, screenshot);
+      const baselineImagePath = path.resolve(BASELINE_SCREENSHOTS_DIR, screenshot);
+      const diffImagePath = path.resolve(DIFF_SCREENSHOTS_DIR, screenshot);
+
+      imageDiff.getFullResult({
+        actualImage: sessionImagePath,
+        expectedImage: baselineImagePath,
+        diffImage: diffImagePath,
+        shadow: true,
+      }, (comparisonError, result) => {
+        if (comparisonError) {
+          throw comparisonError;
+        }
+
+        const change = result.percentage;
+        const changePercentage = (change * 100).toFixed(2);
+        console.log(`${screenshot} has changed by ${changePercentage}%`);
+      });
+    });
+  });
+}
+
+compareScreenshots();