Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add mechanism for dashboard snapshots #15463

Merged
merged 13 commits into from
Dec 13, 2017
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -263,6 +263,7 @@
"istanbul-instrumenter-loader": "3.0.0",
"jest": "21.2.1",
"jest-cli": "21.2.1",
"jimp": "0.2.28",
"jsdom": "9.9.1",
"karma": "1.7.0",
"karma-chrome-launcher": "2.1.1",
Expand Down
4 changes: 3 additions & 1 deletion src/functional_test_runner/cli.js
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ cmd
.option('--verbose', 'Log everything', false)
.option('--quiet', 'Only log errors', false)
.option('--silent', 'Log nothing', false)
.option('--updateBaselines', 'Replace baseline screenshots with whatever is generated from the test', false)
.option('--debug', 'Run in debug mode', false)
.parse(process.argv);

Expand All @@ -35,7 +36,8 @@ const functionalTestRunner = createFunctionalTestRunner({
mochaOpts: {
bail: cmd.bail,
grep: cmd.grep,
}
},
updateBaselines: cmd.updateBaselines
}
});

Expand Down
2 changes: 2 additions & 0 deletions src/functional_test_runner/lib/config/schema.js
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,8 @@ export const schema = Joi.object().keys({
ui: Joi.string().default('bdd'),
}).default(),

updateBaselines: Joi.boolean().default(false),

junit: Joi.object().keys({
enabled: Joi.boolean().default(!!process.env.CI),
reportName: Joi.string(),
Expand Down
3 changes: 2 additions & 1 deletion src/functional_test_runner/lib/mocha/load_test_files.js
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import { decorateMochaUi } from './decorate_mocha_ui';
* @param {String} path
* @return {undefined} - mutates mocha, no return value
*/
export const loadTestFiles = (mocha, log, lifecycle, providers, paths) => {
export const loadTestFiles = (mocha, log, lifecycle, providers, paths, updateBaselines) => {
const innerLoadTestFile = (path) => {
if (typeof path !== 'string' || !isAbsolute(path)) {
throw new TypeError('loadTestFile() only accepts absolute paths');
Expand Down Expand Up @@ -46,6 +46,7 @@ export const loadTestFiles = (mocha, log, lifecycle, providers, paths) => {
getService: providers.getService,
getPageObject: providers.getPageObject,
getPageObjects: providers.getPageObjects,
updateBaselines,
});

if (returnVal && typeof returnVal.then === 'function') {
Expand Down
2 changes: 1 addition & 1 deletion src/functional_test_runner/lib/mocha/setup_mocha.js
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,6 @@ export async function setupMocha(lifecycle, log, config, providers) {
await lifecycle.trigger('beforeEachTest');
});

loadTestFiles(mocha, log, lifecycle, providers, config.get('testFiles'));
loadTestFiles(mocha, log, lifecycle, providers, config.get('testFiles'), config.get('updateBaselines'));
return mocha;
}
68 changes: 68 additions & 0 deletions test/functional/apps/dashboard/_dashboard_snapshots.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
import expect from 'expect.js';

import { AREA_CHART_VIS_NAME } from '../../page_objects/dashboard_page';


export default function ({ getService, getPageObjects, updateBaselines }) {
const dashboardVisualizations = getService('dashboardVisualizations');
const PageObjects = getPageObjects(['dashboard', 'header', 'visualize']);
const screenshot = getService('screenshots');
const remote = getService('remote');

describe('dashboard snapshots', function describeIndexTests() {
before(async function () {
await PageObjects.dashboard.initTests();
await PageObjects.dashboard.preserveCrossAppState();
await remote.setWindowSize(1000, 500);
});

after(async function () {
// avoids any 'Object with id x not found' errors when switching tests.
await PageObjects.header.clickVisualize();
await PageObjects.visualize.gotoLandingPage();
await PageObjects.header.clickDashboard();
await PageObjects.dashboard.gotoDashboardLandingPage();
});

// This one won't work because of https://github.com/elastic/kibana/issues/15501. See if we can get it to work
// once TSVB has timezone support.
it.skip('compare TSVB snapshot', async () => {
await PageObjects.dashboard.gotoDashboardLandingPage();
await PageObjects.dashboard.clickNewDashboard();
await PageObjects.dashboard.setTimepickerInDataRange();
await dashboardVisualizations.createAndAddTSVBVisualization('TSVB');
await PageObjects.dashboard.saveDashboard('tsvb');
await PageObjects.header.clickToastOK();

await PageObjects.dashboard.clickFullScreenMode();
await PageObjects.dashboard.toggleExpandPanel();

await PageObjects.dashboard.waitForRenderCounter(2);
const percentSimilar = await screenshot.compareAgainstBaseline('tsvb_dashboard', updateBaselines);

await PageObjects.dashboard.clickExitFullScreenLogoButton();

expect(percentSimilar).to.be(0);
});

it('compare area chart snapshot', async () => {
await PageObjects.dashboard.gotoDashboardLandingPage();
await PageObjects.dashboard.clickNewDashboard();
await PageObjects.dashboard.setTimepickerInDataRange();
await PageObjects.dashboard.addVisualizations([AREA_CHART_VIS_NAME]);
await PageObjects.dashboard.saveDashboard('area');
await PageObjects.header.clickToastOK();

await PageObjects.dashboard.clickFullScreenMode();
await PageObjects.dashboard.toggleExpandPanel();

await PageObjects.dashboard.waitForRenderCounter(6);
const percentSimilar = await screenshot.compareAgainstBaseline('area_chart', updateBaselines);

await PageObjects.dashboard.clickExitFullScreenLogoButton();

// Testing some OS/browser differnces were shown to cause .009 percent difference.
expect(percentSimilar).to.be.lessThan(0.05);
});
});
}
1 change: 1 addition & 0 deletions test/functional/apps/dashboard/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ export default function ({ getService, loadTestFile }) {
before(() => remote.setWindowSize(1200, 900));
loadTestFile(require.resolve('./_bwc_shared_urls'));
loadTestFile(require.resolve('./_dashboard_queries'));
loadTestFile(require.resolve('./_dashboard_snapshots'));
loadTestFile(require.resolve('./_dashboard_grid'));
loadTestFile(require.resolve('./_panel_controls'));
loadTestFile(require.resolve('./_view_edit'));
Expand Down
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
37 changes: 37 additions & 0 deletions test/functional/services/lib/compare_pngs.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
import Jimp from 'jimp';

export async function comparePngs(actualPath, expectedPath, diffPath, log) {
log.debug(`comparePngs: ${actualPath} vs ${expectedPath}`);
const actual = (await Jimp.read(actualPath)).clone();
const expected = (await Jimp.read(expectedPath)).clone();

if (actual.bitmap.width !== expected.bitmap.width || actual.bitmap.height !== expected.bitmap.height) {
console.log('expected height ' + expected.bitmap.height + ' and width ' + expected.bitmap.width);
console.log('actual height ' + actual.bitmap.height + ' and width ' + actual.bitmap.width);

const width = Math.min(actual.bitmap.width, expected.bitmap.width);
const height = Math.min(actual.bitmap.height, expected.bitmap.height);
actual.resize(width, height);//, Jimp.HORIZONTAL_ALIGN_LEFT | Jimp.VERTICAL_ALIGN_TOP);
expected.resize(width, height);//, Jimp.HORIZONTAL_ALIGN_LEFT | Jimp.VERTICAL_ALIGN_TOP);
}

actual.quality(60);
expected.quality(60);

log.debug(`calculating diff pixels...`);
// Note that this threshold value only affects color comparison from pixel to pixel. It won't have
// any affect when comparing neighboring pixels - so slight shifts, font variations, or "blurry-ness"
// will still show up as diffs, but upping this will not help that. Instead we keep the threshold low, and expect
// some the diffCount to be lower than our own threshold value.
const THRESHOLD = .1;
const { image, percent } = Jimp.diff(actual, expected, THRESHOLD);
log.debug(`percentSimilar: ${percent}`);
if (percent > 0) {
image.write(diffPath);

// For debugging purposes it'll help to see the resized images and how they compare.
actual.write(actualPath.substring(0, actualPath.length - 4) + '-resized.png');
expected.write(expectedPath.substring(0, expectedPath.length - 4) + '-resized.png');
}
return percent;
}
35 changes: 32 additions & 3 deletions test/functional/services/screenshots.js
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
import { resolve, dirname } from 'path';
import { writeFile } from 'fs';

import { fromNode as fcb } from 'bluebird';
import { writeFile, readFileSync } from 'fs';
import { fromNode as fcb, promisify } from 'bluebird';
import mkdirp from 'mkdirp';
import del from 'del';
import { comparePngs } from './lib/compare_pngs';

const mkdirAsync = promisify(mkdirp);
const writeFileAsync = promisify(writeFile);

export async function ScreenshotsProvider({ getService }) {
const log = getService('log');
Expand All @@ -13,9 +16,35 @@ export async function ScreenshotsProvider({ getService }) {

const SESSION_DIRECTORY = resolve(config.get('screenshots.directory'), 'session');
const FAILURE_DIRECTORY = resolve(config.get('screenshots.directory'), 'failure');
const BASELINE_DIRECTORY = resolve(config.get('screenshots.directory'), 'baseline');
await del([SESSION_DIRECTORY, FAILURE_DIRECTORY]);

class Screenshots {

/**
*
* @param name {string} name of the file to use for comparison
* @param updateBaselines {boolean} optional, pass true to update the baseline snapshot.
* @return {Promise.<number>} Percentage difference between the baseline and the current snapshot.
*/
async compareAgainstBaseline(name, updateBaselines) {
log.debug('compareAgainstBaseline');
const sessionPath = resolve(SESSION_DIRECTORY, `${name}.png`);
await this._take(sessionPath);

const baselinePath = resolve(BASELINE_DIRECTORY, `${name}.png`);
const failurePath = resolve(FAILURE_DIRECTORY, `${name}.png`);

if (updateBaselines) {
log.debug('Updating baseline snapshot');
await writeFileAsync(baselinePath, readFileSync(sessionPath));
return 0;
} else {
await mkdirAsync(FAILURE_DIRECTORY);
return await comparePngs(sessionPath, baselinePath, failurePath, log);
}
}

async take(name) {
return await this._take(resolve(SESSION_DIRECTORY, `${name}.png`));
}
Expand Down