From 692e2a5693bfe2746bcff36b33dfec223b7ac280 Mon Sep 17 00:00:00 2001 From: Spencer Date: Thu, 5 Mar 2020 18:13:04 -0700 Subject: [PATCH] [7.6] Fix visual baseline job (#59348) (#59471) * Establish Percy baselines * move Jenkinsfile changed back to `.ci` directory * rename xpack workers Co-authored-by: Elastic Machine Co-authored-by: Elastic Machine --- .ci/Jenkinsfile_visual_baseline | 14 +-- test/functional/services/elastic_chart.ts | 6 +- test/scripts/jenkins_visual_regression.sh | 14 ++- .../jenkins_xpack_visual_regression.sh | 18 +++- .../services/visual_testing/visual_testing.ts | 7 ++ ...isualization.js => chart_visualization.ts} | 91 +++++++------------ .../tests/discover/{index.js => index.ts} | 8 +- .../{config.js => config.ts} | 11 +-- .../ftr_provider_context.d.ts | 12 +++ x-pack/test/visual_regression/page_objects.ts | 9 ++ x-pack/test/visual_regression/services.ts | 13 +++ .../tests/{login_page.js => login_page.ts} | 4 +- 12 files changed, 119 insertions(+), 88 deletions(-) rename test/visual_regression/tests/discover/{chart_visualization.js => chart_visualization.ts} (55%) rename test/visual_regression/tests/discover/{index.js => index.ts} (86%) rename x-pack/test/visual_regression/{config.js => config.ts} (69%) create mode 100644 x-pack/test/visual_regression/ftr_provider_context.d.ts create mode 100644 x-pack/test/visual_regression/page_objects.ts create mode 100644 x-pack/test/visual_regression/services.ts rename x-pack/test/visual_regression/tests/{login_page.js => login_page.ts} (91%) diff --git a/.ci/Jenkinsfile_visual_baseline b/.ci/Jenkinsfile_visual_baseline index 4a1e0f7d74e07..5c13ccccd9c6f 100644 --- a/.ci/Jenkinsfile_visual_baseline +++ b/.ci/Jenkinsfile_visual_baseline @@ -6,13 +6,15 @@ kibanaLibrary.load() kibanaPipeline(timeoutMinutes: 120) { catchError { parallel([ - workers.base(name: 'oss-visualRegression', label: 'linux && immutable') { - kibanaPipeline.buildOss() - kibanaPipeline.functionalTestProcess('oss-visualRegression', './test/scripts/jenkins_visual_regression.sh') + 'oss-visualRegression': { + workers.ci(name: 'oss-visualRegression', label: 'linux && immutable', ramDisk: false) { + kibanaPipeline.functionalTestProcess('oss-visualRegression', './test/scripts/jenkins_visual_regression.sh')(1) + } }, - workers.base(name: 'xpack-visualRegression', label: 'linux && immutable') { - kibanaPipeline.buildXpack() - kibanaPipeline.functionalTestProcess('xpack-visualRegression', './test/scripts/jenkins_xpack_visual_regression.sh') + 'xpack-visualRegression': { + workers.ci(name: 'xpack-visualRegression', label: 'linux && immutable', ramDisk: false) { + kibanaPipeline.functionalTestProcess('xpack-visualRegression', './test/scripts/jenkins_xpack_visual_regression.sh')(1) + } }, ]) } diff --git a/test/functional/services/elastic_chart.ts b/test/functional/services/elastic_chart.ts index 45ad157fc5c02..1c3071ac01587 100644 --- a/test/functional/services/elastic_chart.ts +++ b/test/functional/services/elastic_chart.ts @@ -51,11 +51,11 @@ export function ElasticChartProvider({ getService }: FtrProviderContext) { return Number(renderingCount); } - public async waitForRenderingCount(dataTestSubj: string, previousCount = 1) { - await retry.waitFor(`rendering count to be equal to [${previousCount + 1}]`, async () => { + public async waitForRenderingCount(dataTestSubj: string, minimumCount: number) { + await retry.waitFor(`rendering count to be equal to [${minimumCount}]`, async () => { const currentRenderingCount = await this.getVisualizationRenderingCount(dataTestSubj); log.debug(`-- currentRenderingCount=${currentRenderingCount}`); - return currentRenderingCount === previousCount + 1; + return currentRenderingCount >= minimumCount; }); } } diff --git a/test/scripts/jenkins_visual_regression.sh b/test/scripts/jenkins_visual_regression.sh index dda966dea98d0..4fdd197147eac 100755 --- a/test/scripts/jenkins_visual_regression.sh +++ b/test/scripts/jenkins_visual_regression.sh @@ -1,10 +1,18 @@ #!/usr/bin/env bash -source test/scripts/jenkins_test_setup_xpack.sh +source src/dev/ci_setup/setup_env.sh source "$KIBANA_DIR/src/dev/ci_setup/setup_percy.sh" -checks-reporter-with-killswitch "Kibana visual regression tests" \ - yarn run percy exec -t 500 \ +echo " -> building and extracting OSS Kibana distributable for use in functional tests" +node scripts/build --debug --oss +linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')" +installDir="$PARENT_DIR/install/kibana" +mkdir -p "$installDir" +tar -xzf "$linuxBuild" -C "$installDir" --strip=1 + +echo " -> running visual regression tests from kibana directory" +checks-reporter-with-killswitch "X-Pack visual regression tests" \ + yarn percy exec -t 500 \ node scripts/functional_tests \ --debug --bail \ --kibana-install-dir "$installDir" \ diff --git a/test/scripts/jenkins_xpack_visual_regression.sh b/test/scripts/jenkins_xpack_visual_regression.sh index 6e3d4dd7c249b..73e92da3bad63 100755 --- a/test/scripts/jenkins_xpack_visual_regression.sh +++ b/test/scripts/jenkins_xpack_visual_regression.sh @@ -1,11 +1,21 @@ #!/usr/bin/env bash -source test/scripts/jenkins_test_setup_xpack.sh +source src/dev/ci_setup/setup_env.sh source "$KIBANA_DIR/src/dev/ci_setup/setup_percy.sh" +echo " -> building and extracting default Kibana distributable" +cd "$KIBANA_DIR" +node scripts/build --debug --no-oss +linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')" +installDir="$PARENT_DIR/install/kibana" +mkdir -p "$installDir" +tar -xzf "$linuxBuild" -C "$installDir" --strip=1 + +echo " -> running visual regression tests from x-pack directory" +cd "$XPACK_DIR" checks-reporter-with-killswitch "X-Pack visual regression tests" \ - yarn run percy exec -t 500 \ + yarn percy exec -t 500 \ node scripts/functional_tests \ --debug --bail \ - --kibana-install-dir "$KIBANA_INSTALL_DIR" \ - --config test/visual_regression/config.js; + --kibana-install-dir "$installDir" \ + --config test/visual_regression/config.ts; diff --git a/test/visual_regression/services/visual_testing/visual_testing.ts b/test/visual_regression/services/visual_testing/visual_testing.ts index 4ad97f8d98717..0882beecf7f5c 100644 --- a/test/visual_regression/services/visual_testing/visual_testing.ts +++ b/test/visual_regression/services/visual_testing/visual_testing.ts @@ -71,6 +71,13 @@ export async function VisualTestingProvider({ getService }: FtrProviderContext) return new (class VisualTesting { public async snapshot(options: SnapshotOptions = {}) { + if (process.env.DISABLE_VISUAL_TESTING) { + log.warning( + 'Capturing of percy snapshots disabled, would normally capture a snapshot here!' + ); + return; + } + log.debug('Capturing percy snapshot'); if (!currentTest) { diff --git a/test/visual_regression/tests/discover/chart_visualization.js b/test/visual_regression/tests/discover/chart_visualization.ts similarity index 55% rename from test/visual_regression/tests/discover/chart_visualization.js rename to test/visual_regression/tests/discover/chart_visualization.ts index 10ac559b9f982..49c3057a27cb0 100644 --- a/test/visual_regression/tests/discover/chart_visualization.js +++ b/test/visual_regression/tests/discover/chart_visualization.ts @@ -19,8 +19,9 @@ import expect from '@kbn/expect'; -export default function({ getService, getPageObjects }) { - const log = getService('log'); +import { FtrProviderContext } from '../../ftr_provider_context'; + +export default function({ getService, getPageObjects }: FtrProviderContext) { const retry = getService('retry'); const esArchiver = getService('esArchiver'); const browser = getService('browser'); @@ -34,58 +35,56 @@ export default function({ getService, getPageObjects }) { describe('discover', function describeIndexTests() { before(async function() { - log.debug('load kibana index with default index pattern'); await esArchiver.load('discover'); // and load a set of makelogs data await esArchiver.loadIfNeeded('logstash_functional'); await kibanaServer.uiSettings.replace(defaultSettings); - log.debug('discover'); await PageObjects.common.navigateToApp('discover'); await PageObjects.timePicker.setDefaultAbsoluteRange(); }); + after(function unloadMakelogs() { + return esArchiver.unload('logstash_functional'); + }); + + async function refreshDiscover() { + await browser.refresh(); + await PageObjects.header.awaitKibanaChrome(); + await PageObjects.header.awaitGlobalLoadingIndicatorHidden(); + await PageObjects.discover.waitUntilSearchingHasFinished(); + await PageObjects.discover.waitForChartLoadingComplete(1); + } + + async function takeSnapshot() { + await refreshDiscover(); + await visualTesting.snapshot({ + show: ['discoverChart'], + }); + } + describe('query', function() { this.tags(['skipFirefox']); - let renderCounter = 0; it('should show bars in the correct time zone', async function() { await PageObjects.header.awaitGlobalLoadingIndicatorHidden(); await PageObjects.discover.waitUntilSearchingHasFinished(); - await PageObjects.discover.waitForChartLoadingComplete(++renderCounter); - await visualTesting.snapshot({ - show: ['discoverChart'], - }); + await takeSnapshot(); }); it('should show correct data for chart interval Hourly', async function() { - await PageObjects.header.awaitGlobalLoadingIndicatorHidden(); - await PageObjects.discover.waitUntilSearchingHasFinished(); await PageObjects.discover.setChartInterval('Hourly'); - await PageObjects.discover.waitForChartLoadingComplete(++renderCounter); - await visualTesting.snapshot({ - show: ['discoverChart'], - }); + await takeSnapshot(); }); it('should show correct data for chart interval Daily', async function() { - await PageObjects.header.awaitGlobalLoadingIndicatorHidden(); - await PageObjects.discover.waitUntilSearchingHasFinished(); await PageObjects.discover.setChartInterval('Daily'); - await PageObjects.discover.waitForChartLoadingComplete(++renderCounter); - await visualTesting.snapshot({ - show: ['discoverChart'], - }); + await takeSnapshot(); }); it('should show correct data for chart interval Weekly', async function() { - await PageObjects.header.awaitGlobalLoadingIndicatorHidden(); - await PageObjects.discover.waitUntilSearchingHasFinished(); await PageObjects.discover.setChartInterval('Weekly'); - await PageObjects.discover.waitForChartLoadingComplete(++renderCounter); - await visualTesting.snapshot({ - show: ['discoverChart'], - }); + await takeSnapshot(); }); it('browser back button should show previous interval Daily', async function() { @@ -94,57 +93,31 @@ export default function({ getService, getPageObjects }) { const actualInterval = await PageObjects.discover.getChartInterval(); expect(actualInterval).to.be('Daily'); }); - await PageObjects.header.awaitGlobalLoadingIndicatorHidden(); - await PageObjects.discover.waitUntilSearchingHasFinished(); - await PageObjects.discover.waitForChartLoadingComplete(++renderCounter); - await visualTesting.snapshot({ - show: ['discoverChart'], - }); + await takeSnapshot(); }); it('should show correct data for chart interval Monthly', async function() { - await PageObjects.header.awaitGlobalLoadingIndicatorHidden(); - await PageObjects.discover.waitUntilSearchingHasFinished(); await PageObjects.discover.setChartInterval('Monthly'); - await PageObjects.discover.waitForChartLoadingComplete(++renderCounter); - await visualTesting.snapshot({ - show: ['discoverChart'], - }); + await takeSnapshot(); }); it('should show correct data for chart interval Yearly', async function() { - await PageObjects.header.awaitGlobalLoadingIndicatorHidden(); - await PageObjects.discover.waitUntilSearchingHasFinished(); await PageObjects.discover.setChartInterval('Yearly'); - await PageObjects.discover.waitForChartLoadingComplete(++renderCounter); - await visualTesting.snapshot({ - show: ['discoverChart'], - }); + await takeSnapshot(); }); it('should show correct data for chart interval Auto', async function() { - await PageObjects.header.awaitGlobalLoadingIndicatorHidden(); - await PageObjects.discover.waitUntilSearchingHasFinished(); await PageObjects.discover.setChartInterval('Auto'); - await PageObjects.discover.waitForChartLoadingComplete(++renderCounter); - await visualTesting.snapshot({ - show: ['discoverChart'], - }); + await takeSnapshot(); }); }); describe('time zone switch', () => { it('should show bars in the correct time zone after switching', async function() { await kibanaServer.uiSettings.replace({ 'dateFormat:tz': 'America/Phoenix' }); - await browser.refresh(); - await PageObjects.header.awaitKibanaChrome(); + await refreshDiscover(); await PageObjects.timePicker.setDefaultAbsoluteRange(); - await PageObjects.header.awaitGlobalLoadingIndicatorHidden(); - await PageObjects.discover.waitUntilSearchingHasFinished(); - await PageObjects.discover.waitForChartLoadingComplete(1); - await visualTesting.snapshot({ - show: ['discoverChart'], - }); + await takeSnapshot(); }); }); }); diff --git a/test/visual_regression/tests/discover/index.js b/test/visual_regression/tests/discover/index.ts similarity index 86% rename from test/visual_regression/tests/discover/index.js rename to test/visual_regression/tests/discover/index.ts index f98aac52aa4cb..d036327ae7475 100644 --- a/test/visual_regression/tests/discover/index.js +++ b/test/visual_regression/tests/discover/index.ts @@ -18,12 +18,12 @@ */ import { DEFAULT_OPTIONS } from '../../services/visual_testing/visual_testing'; +import { FtrProviderContext } from '../../ftr_provider_context'; // Width must be the same as visual_testing or canvas image widths will get skewed const [SCREEN_WIDTH] = DEFAULT_OPTIONS.widths || []; -export default function({ getService, loadTestFile }) { - const esArchiver = getService('esArchiver'); +export default function({ getService, loadTestFile }: FtrProviderContext) { const browser = getService('browser'); describe('discover app', function() { @@ -33,10 +33,6 @@ export default function({ getService, loadTestFile }) { return browser.setWindowSize(SCREEN_WIDTH, 1000); }); - after(function unloadMakelogs() { - return esArchiver.unload('logstash_functional'); - }); - loadTestFile(require.resolve('./chart_visualization')); }); } diff --git a/x-pack/test/visual_regression/config.js b/x-pack/test/visual_regression/config.ts similarity index 69% rename from x-pack/test/visual_regression/config.js rename to x-pack/test/visual_regression/config.ts index aff6aaaf4114a..dce17348f75e6 100644 --- a/x-pack/test/visual_regression/config.js +++ b/x-pack/test/visual_regression/config.ts @@ -4,9 +4,11 @@ * you may not use this file except in compliance with the Elastic License. */ -import { services as ossVisualRegressionServices } from '../../../test/visual_regression/services'; +import { FtrConfigProviderContext } from '@kbn/test/types/ftr'; -export default async function({ readConfigFile }) { +import { services } from './services'; + +export default async function({ readConfigFile }: FtrConfigProviderContext) { const functionalConfig = await readConfigFile(require.resolve('../functional/config')); return { @@ -19,10 +21,7 @@ export default async function({ readConfigFile }) { require.resolve('./tests/infra'), ], - services: { - ...functionalConfig.get('services'), - visualTesting: ossVisualRegressionServices.visualTesting, - }, + services, junit: { reportName: 'X-Pack Visual Regression Tests', diff --git a/x-pack/test/visual_regression/ftr_provider_context.d.ts b/x-pack/test/visual_regression/ftr_provider_context.d.ts new file mode 100644 index 0000000000000..bb257cdcbfe1b --- /dev/null +++ b/x-pack/test/visual_regression/ftr_provider_context.d.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { GenericFtrProviderContext } from '@kbn/test/types/ftr'; + +import { pageObjects } from './page_objects'; +import { services } from './services'; + +export type FtrProviderContext = GenericFtrProviderContext; diff --git a/x-pack/test/visual_regression/page_objects.ts b/x-pack/test/visual_regression/page_objects.ts new file mode 100644 index 0000000000000..ea3e49d0ccc5e --- /dev/null +++ b/x-pack/test/visual_regression/page_objects.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { pageObjects } from '../functional/page_objects'; + +export { pageObjects }; diff --git a/x-pack/test/visual_regression/services.ts b/x-pack/test/visual_regression/services.ts new file mode 100644 index 0000000000000..447c16281b838 --- /dev/null +++ b/x-pack/test/visual_regression/services.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { services as ossVisualRegressionServices } from '../../../test/visual_regression/services'; +import { services as functionalServices } from '../functional/services'; + +export const services = { + ...functionalServices, + visualTesting: ossVisualRegressionServices.visualTesting, +}; diff --git a/x-pack/test/visual_regression/tests/login_page.js b/x-pack/test/visual_regression/tests/login_page.ts similarity index 91% rename from x-pack/test/visual_regression/tests/login_page.js rename to x-pack/test/visual_regression/tests/login_page.ts index b290b8f819589..ce90669a6bfe1 100644 --- a/x-pack/test/visual_regression/tests/login_page.js +++ b/x-pack/test/visual_regression/tests/login_page.ts @@ -4,7 +4,9 @@ * you may not use this file except in compliance with the Elastic License. */ -export default function({ getService, getPageObjects }) { +import { FtrProviderContext } from '../ftr_provider_context'; + +export default function({ getService, getPageObjects }: FtrProviderContext) { const esArchiver = getService('esArchiver'); const visualTesting = getService('visualTesting'); const testSubjects = getService('testSubjects');