feat: Integrated Visual Regression with ember-backstop (#94)

* Integrating Visual Regression

* Removed lerna since flag

* made change to .gitignore

* Backstop ref changes

* Made all computed properties computed

* Checking an artifact workflow

* Changed artifact workflow with path

* Changed backstop config

* Backstop Debug

* Backstop Debug Window Eliminated

* Enabled CI Reporting

* Reduced async limit

* Trial with max threshold

* Trial check for screenshots

* Full threshold

* New inline banner

* no tests for banners

* Run 2 VR tests

* Removed workflow downloads

* Experiments I with async

* Experiments II with async

* Experiments III with async

* Experiments IV with async

* Experiments V with async

* Experiments VI

* Experiments VII

* Parallel process

* Normal threshold limits

* fix(tests): concurrency

* fix(lerna): removed --no-bail

* New ref images

* fix: button ref images

* changed threshold

* Update continuous-integration-workflow.yml

* Hopefully final ref images

* Cleanup for final review

* Made required changes

Co-authored-by: Shibu Lijack <[email protected]>

.github/workflows/continuous-integration-workflow.yml

@@ -8,7 +8,7 @@ on:
 jobs:
   build:
     name: Build
-    runs-on: ubuntu-latest
+    runs-on: macos-latest
     steps:
     - uses: actions/checkout@v2
 

.gitignore

@@ -28,3 +28,7 @@ tmp
 /bower.json.ember-try
 /package.json.ember-try
 .DS_Store
+
+#backstopjs 
+html_report
+bitmaps_test

package.json

@@ -8,7 +8,7 @@
   ],
   "scripts": {
     "start": "yarn workspace nucleus start",
-    "test": "lerna run test --parallel --no-private",
+    "test": "lerna run test --no-private --stream --concurrency 1",
     "build": "lerna run build --parallel --no-private",
     "lint": "lerna run lint:hbs",
     "lerna:version": "lerna version prerelease --preid beta",
@@ -21,7 +21,7 @@
     "ember-cli": "~3.11.0",
     "ember-source-channel-url": "^2.0.0",
     "ember-try": "^1.1.0",
-    "lerna": "^3.18.4"
+    "lerna": "^3.20.2"
   },
   "engines": {
     "node": "^10.13.0",

packages/banner/ember-backstop/backstop.js

@@ -0,0 +1,38 @@
+const path = require('path');
+
+module.exports = {
+  id: `ember-backstop test`,
+  viewports: [
+    {
+      label: 'webview',
+      width: 1440,
+      height: 900,
+    },
+  ],
+  onBeforeScript: `puppet/onBefore.js`,
+  onReadyScript: `puppet/onReady.js`,
+  scenarios: [
+    {
+      label: '{testName}',
+      cookiePath: 'backstop_data/engine_scripts/cookies.json',
+      url: '{origin}/backstop/dview/{testId}/{scenarioId}',
+      delay: 500,
+    },
+  ],
+  paths: {
+    bitmaps_reference: 'backstop_data/bitmaps_reference',
+    bitmaps_test: 'backstop_data/bitmaps_test',
+    engine_scripts: 'backstop_data/engine_scripts',
+    html_report: 'backstop_data/html_report',
+    ci_report: 'backstop_data/ci_report',
+  },
+  report: [],
+  engine: 'puppet',
+  engineOptions: {
+    args: ['--no-sandbox'],
+  },
+  asyncCaptureLimit: 10,
+  asyncCompareLimit: 50,
+  debug: false,
+  debugWindow: false,
+};

packages/banner/ember-backstop/backstop_data/engine_scripts/puppet/clickAndHoverHelper.js

@@ -0,0 +1,41 @@
+/* eslint-env browser, node */
+
+module.exports = async (page, scenario) => {
+  const hoverSelector = scenario.hoverSelectors || scenario.hoverSelector;
+  const clickSelector = scenario.clickSelectors || scenario.clickSelector;
+  const keyPressSelector = scenario.keyPressSelectors || scenario.keyPressSelector;
+  const scrollToSelector = scenario.scrollToSelector;
+  const postInteractionWait = scenario.postInteractionWait; // selector [str] | ms [int]
+
+  if (keyPressSelector) {
+    for (const keyPressSelectorItem of [].concat(keyPressSelector)) {
+      await page.waitFor(keyPressSelectorItem.selector);
+      await page.type(keyPressSelectorItem.selector, keyPressSelectorItem.keyPress);
+    }
+  }
+
+  if (hoverSelector) {
+    for (const hoverSelectorIndex of [].concat(hoverSelector)) {
+      await page.waitFor(hoverSelectorIndex);
+      await page.hover(hoverSelectorIndex);
+    }
+  }
+
+  if (clickSelector) {
+    for (const clickSelectorIndex of [].concat(clickSelector)) {
+      await page.waitFor(clickSelectorIndex);
+      await page.click(clickSelectorIndex);
+    }
+  }
+
+  if (postInteractionWait) {
+    await page.waitFor(postInteractionWait);
+  }
+
+  if (scrollToSelector) {
+    await page.waitFor(scrollToSelector);
+    await page.evaluate(scrollToSelector => {
+      document.querySelector(scrollToSelector).scrollIntoView();
+    }, scrollToSelector);
+  }
+};

packages/banner/ember-backstop/backstop_data/engine_scripts/puppet/onReady.js

@@ -0,0 +1,11 @@
+/* eslint-env browser, node */
+
+const debug = require('debug')('BackstopJS');
+
+module.exports = async (page, scenario, vp) => {
+  debug('SCENARIO > ' + scenario.label);
+  await require('./overrideCSS')(page, scenario);
+  await require('./clickAndHoverHelper')(page, scenario);
+
+  // add more ready handlers here...
+};

packages/banner/ember-backstop/backstop_data/engine_scripts/puppet/overrideCSS.js

@@ -0,0 +1,13 @@
+/* eslint-env browser, node */
+
+module.exports = function(page, scenario) {
+  // inject arbitrary css to override styles
+  page.evaluate(() => {
+    const BACKSTOP_TEST_CSS_OVERRIDE = `#ember-testing {width: 100% !important; height: 100% !important; -webkit-transform: scale(1) !important; transform: scale(1) !important;}`;
+    let style = document.createElement('style');
+    style.type = 'text/css';
+    let styleNode = document.createTextNode(BACKSTOP_TEST_CSS_OVERRIDE);
+    style.appendChild(styleNode);
+    document.head.appendChild(style);
+  });
+};

packages/banner/ember-cli-build.js

@@ -5,6 +5,7 @@ const EmberAddon = require('ember-cli/lib/broccoli/ember-addon');
 module.exports = function(defaults) {
   let app = new EmberAddon(defaults,
     {
+      hinting: false,
       stylelint: {
         linterConfig:{
           syntax: 'scss'

packages/banner/package.json

@@ -18,7 +18,8 @@
     "lint:js": "eslint .",
     "start": "ember serve -p 4003",
     "deploy": "ember deploy production",
-    "test": "COVERAGE=TRUE ember test --test-port=1512",
+    "test": "ember backstop-remote & COVERAGE=TRUE ember test --test-port=1512",
+    "posttest": "ember backstop-stop",
     "test:dev": "ember test --server -launch=false"
   },
   "dependencies": {
@@ -41,6 +42,7 @@
     "broccoli-funnel": "^2.0.2",
     "broccoli-merge-trees": "^3.0.2",
     "ember-a11y-testing": "^1.0.0",
+    "ember-backstop": "^1.3.4",
     "ember-cli": "~3.13.1",
     "ember-cli-autoprefixer": "^0.8.1",
     "ember-cli-code-coverage": "^1.0.0-beta.8",

packages/banner/testem.js

@@ -20,5 +20,11 @@ module.exports = {
         '--window-size=1440,900'
       ].filter(Boolean)
     }
+  },
+  proxies: {
+    '/backstop': {
+      target: 'http://localhost:1512',
+      secure: false
+    }
   }
 }

packages/banner/tests/integration/components/nucleus-banner-test.js

@@ -7,6 +7,7 @@ import hbs from 'htmlbars-inline-precompile';
 import a11yAudit from 'ember-a11y-testing/test-support/audit';
 import setupBanner from '../../helpers/setup-banner';
 import { ITEMS } from '../../constants/nucleus-banner';
+import backstop from 'ember-backstop/test-support/backstop';
 
 let StubMapsService = Service.extend({
   items: ITEMS
@@ -50,4 +51,41 @@ module('Integration | Component | nucleus-banner', function(hooks) {
       assert.ok(true, 'no a11y errors found!');
     });
   });
+
+  test('normal banner passes visual regression tests', async function(assert){ 
+    var bannerObject = [{title:'This is a banner', type:'success'}]
+    const nucleusBanner = Service.extend({
+      items: bannerObject,
+    });
+    this.owner.register('service:nucleusBanner', nucleusBanner);
+
+    await render(hbs`
+    {{nucleus-banner}}
+    `);
+    await backstop(assert, {scenario: {misMatchThreshold: 0.1}});
+  });
+
+  test('banner with link passes visual regression tests', async function(assert){ 
+    let closeAction = this.spy();
+    let displayedItems = [{ title: 'This is another banner',
+    type: 'danger',
+    isDismissible: true,
+    content: {
+      linkAction: closeAction,
+      linkText: 'Click here'
+    },
+    }, {
+      title:"This is another banner",
+      type: 'danger',
+      isDismissible: true
+    }]
+    const nucleusBanner = Service.extend({
+      items: displayedItems,
+    });
+    this.owner.register('service:nucleusBanner', nucleusBanner);
+    await render(hbs`
+    {{nucleus-banner}}
+    `);
+    await backstop(assert,{scenario: {misMatchThreshold: 0.1}});
+  });
 });

packages/button/ember-backstop/backstop.js

@@ -0,0 +1,38 @@
+const path = require('path');
+
+module.exports = {
+  id: `ember-backstop test`,
+  viewports: [
+    {
+      label: 'webview',
+      width: 1440,
+      height: 900,
+    },
+  ],
+  onBeforeScript: `puppet/onBefore.js`,
+  onReadyScript: `puppet/onReady.js`,
+  scenarios: [
+    {
+      label: '{testName}',
+      cookiePath: 'backstop_data/engine_scripts/cookies.json',
+      url: '{origin}/backstop/dview/{testId}/{scenarioId}',
+      delay: 500,
+    },
+  ],
+  paths: {
+    bitmaps_reference: 'backstop_data/bitmaps_reference',
+    bitmaps_test: 'backstop_data/bitmaps_test',
+    engine_scripts: 'backstop_data/engine_scripts',
+    html_report: 'backstop_data/html_report',
+    ci_report: 'backstop_data/ci_report',
+  },
+  report: [],
+  engine: 'puppet',
+  engineOptions: {
+    args: ['--no-sandbox'],
+  },
+  asyncCaptureLimit: 10,
+  asyncCompareLimit: 50,
+  debug: false,
+  debugWindow: false,
+};

packages/button/ember-backstop/backstop_data/engine_scripts/puppet/clickAndHoverHelper.js

@@ -0,0 +1,41 @@
+/* eslint-env browser, node */
+
+module.exports = async (page, scenario) => {
+  const hoverSelector = scenario.hoverSelectors || scenario.hoverSelector;
+  const clickSelector = scenario.clickSelectors || scenario.clickSelector;
+  const keyPressSelector = scenario.keyPressSelectors || scenario.keyPressSelector;
+  const scrollToSelector = scenario.scrollToSelector;
+  const postInteractionWait = scenario.postInteractionWait; // selector [str] | ms [int]
+
+  if (keyPressSelector) {
+    for (const keyPressSelectorItem of [].concat(keyPressSelector)) {
+      await page.waitFor(keyPressSelectorItem.selector);
+      await page.type(keyPressSelectorItem.selector, keyPressSelectorItem.keyPress);
+    }
+  }
+
+  if (hoverSelector) {
+    for (const hoverSelectorIndex of [].concat(hoverSelector)) {
+      await page.waitFor(hoverSelectorIndex);
+      await page.hover(hoverSelectorIndex);
+    }
+  }
+
+  if (clickSelector) {
+    for (const clickSelectorIndex of [].concat(clickSelector)) {
+      await page.waitFor(clickSelectorIndex);
+      await page.click(clickSelectorIndex);
+    }
+  }
+
+  if (postInteractionWait) {
+    await page.waitFor(postInteractionWait);
+  }
+
+  if (scrollToSelector) {
+    await page.waitFor(scrollToSelector);
+    await page.evaluate(scrollToSelector => {
+      document.querySelector(scrollToSelector).scrollIntoView();
+    }, scrollToSelector);
+  }
+};

packages/button/ember-backstop/backstop_data/engine_scripts/puppet/onReady.js

@@ -0,0 +1,11 @@
+/* eslint-env browser, node */
+
+const debug = require('debug')('BackstopJS');
+
+module.exports = async (page, scenario, vp) => {
+  debug('SCENARIO > ' + scenario.label);
+  await require('./overrideCSS')(page, scenario);
+  await require('./clickAndHoverHelper')(page, scenario);
+
+  // add more ready handlers here...
+};

packages/button/ember-backstop/backstop_data/engine_scripts/puppet/overrideCSS.js

@@ -0,0 +1,13 @@
+/* eslint-env browser, node */
+
+module.exports = function(page, scenario) {
+  // inject arbitrary css to override styles
+  page.evaluate(() => {
+    const BACKSTOP_TEST_CSS_OVERRIDE = `#ember-testing {width: 100% !important; height: 100% !important; -webkit-transform: scale(1) !important; transform: scale(1) !important;}`;
+    let style = document.createElement('style');
+    style.type = 'text/css';
+    let styleNode = document.createTextNode(BACKSTOP_TEST_CSS_OVERRIDE);
+    style.appendChild(styleNode);
+    document.head.appendChild(style);
+  });
+};

packages/button/ember-cli-build.js

@@ -5,6 +5,7 @@ const EmberAddon = require('ember-cli/lib/broccoli/ember-addon');
 module.exports = function(defaults) {
   let app = new EmberAddon(defaults,
     {
+      hinting: false,
       stylelint: {
         linterConfig:{
           syntax: 'scss'

packages/button/package.json

@@ -18,7 +18,8 @@
     "lint:js": "eslint .",
     "start": "ember serve -p 4003",
     "deploy": "ember deploy production",
-    "test": "COVERAGE=TRUE ember test --test-port=1509",
+    "test": "ember backstop-remote & COVERAGE=TRUE ember test --test-port=1509",
+    "posttest": "ember backstop-stop",
     "test:dev": "COVERAGE=TRUE ember test --server -launch=false"
   },
   "dependencies": {
@@ -39,6 +40,7 @@
     "broccoli-funnel": "^2.0.2",
     "broccoli-merge-trees": "^3.0.2",
     "ember-a11y-testing": "^1.0.0",
+    "ember-backstop": "^1.3.4",
     "ember-cli": "~3.13.1",
     "ember-cli-autoprefixer": "^0.8.1",
     "ember-cli-code-coverage": "^1.0.0-beta.8",

packages/button/testem.js

@@ -20,5 +20,11 @@ module.exports = {
         '--window-size=1440,900'
       ].filter(Boolean)
     }
+  },
+  proxies: {
+    '/backstop': {
+      target: 'http://localhost:1509',
+      secure: false
+    }
   }
 }