From 0a7462dc4acb79bc28873b4ce82a510aa624397c Mon Sep 17 00:00:00 2001 From: Lee Drengenberg Date: Thu, 1 Oct 2020 15:35:36 -0500 Subject: [PATCH] move apps lower in tree, add metricbeat dashboard screenshot test (#79001) --- .../services/common/failure_debugging.ts | 2 +- .../functional/services/common/screenshots.ts | 2 +- test/functional/services/common/snapshots.ts | 2 +- .../{test/functional => }/apps/ccs/ccs.js | 0 .../{test/functional => }/apps/ccs/index.js | 0 .../functional => }/apps/filebeat/filebeat.js | 0 .../functional => }/apps/filebeat/index.js | 0 .../apps/heartbeat/_heartbeat.js | 0 .../functional => }/apps/heartbeat/index.js | 0 .../apps/management/_index_pattern_create.js | 0 .../functional => }/apps/management/index.js | 0 .../apps/metricbeat/_metricbeat.js | 0 .../apps/metricbeat/_metricbeat_dashboard.js | 59 +++++++++++++++++++ .../functional => }/apps/metricbeat/index.js | 1 + .../apps/monitoring/_monitoring.js | 0 .../apps/monitoring/_monitoring_metricbeat.js | 0 .../functional => }/apps/monitoring/index.js | 0 .../apps/packetbeat/_packetbeat.js | 0 .../functional => }/apps/packetbeat/index.js | 0 .../functional => }/apps/reporting/index.js | 0 .../apps/reporting/reporting_watcher.js | 0 .../apps/reporting/reporting_watcher_png.js | 0 .../functional => }/apps/reporting/util.js | 0 .../apps/sample_data/e_commerce.js | 0 .../functional => }/apps/sample_data/index.js | 0 .../apps/telemetry/_telemetry.js | 0 .../functional => }/apps/telemetry/index.js | 0 .../apps/winlogbeat/_winlogbeat.js | 0 .../functional => }/apps/winlogbeat/index.js | 0 ...onfig.stack_functional_integration_base.js | 17 ++++-- .../configs/tests_list.js | 6 +- 31 files changed, 81 insertions(+), 8 deletions(-) rename x-pack/test/stack_functional_integration/{test/functional => }/apps/ccs/ccs.js (100%) rename x-pack/test/stack_functional_integration/{test/functional => }/apps/ccs/index.js (100%) rename x-pack/test/stack_functional_integration/{test/functional => }/apps/filebeat/filebeat.js (100%) rename x-pack/test/stack_functional_integration/{test/functional => }/apps/filebeat/index.js (100%) rename x-pack/test/stack_functional_integration/{test/functional => }/apps/heartbeat/_heartbeat.js (100%) rename x-pack/test/stack_functional_integration/{test/functional => }/apps/heartbeat/index.js (100%) rename x-pack/test/stack_functional_integration/{test/functional => }/apps/management/_index_pattern_create.js (100%) rename x-pack/test/stack_functional_integration/{test/functional => }/apps/management/index.js (100%) rename x-pack/test/stack_functional_integration/{test/functional => }/apps/metricbeat/_metricbeat.js (100%) create mode 100644 x-pack/test/stack_functional_integration/apps/metricbeat/_metricbeat_dashboard.js rename x-pack/test/stack_functional_integration/{test/functional => }/apps/metricbeat/index.js (86%) rename x-pack/test/stack_functional_integration/{test/functional => }/apps/monitoring/_monitoring.js (100%) rename x-pack/test/stack_functional_integration/{test/functional => }/apps/monitoring/_monitoring_metricbeat.js (100%) rename x-pack/test/stack_functional_integration/{test/functional => }/apps/monitoring/index.js (100%) rename x-pack/test/stack_functional_integration/{test/functional => }/apps/packetbeat/_packetbeat.js (100%) rename x-pack/test/stack_functional_integration/{test/functional => }/apps/packetbeat/index.js (100%) rename x-pack/test/stack_functional_integration/{test/functional => }/apps/reporting/index.js (100%) rename x-pack/test/stack_functional_integration/{test/functional => }/apps/reporting/reporting_watcher.js (100%) rename x-pack/test/stack_functional_integration/{test/functional => }/apps/reporting/reporting_watcher_png.js (100%) rename x-pack/test/stack_functional_integration/{test/functional => }/apps/reporting/util.js (100%) rename x-pack/test/stack_functional_integration/{test/functional => }/apps/sample_data/e_commerce.js (100%) rename x-pack/test/stack_functional_integration/{test/functional => }/apps/sample_data/index.js (100%) rename x-pack/test/stack_functional_integration/{test/functional => }/apps/telemetry/_telemetry.js (100%) rename x-pack/test/stack_functional_integration/{test/functional => }/apps/telemetry/index.js (100%) rename x-pack/test/stack_functional_integration/{test/functional => }/apps/winlogbeat/_winlogbeat.js (100%) rename x-pack/test/stack_functional_integration/{test/functional => }/apps/winlogbeat/index.js (100%) diff --git a/test/functional/services/common/failure_debugging.ts b/test/functional/services/common/failure_debugging.ts index aa67c455e0100..8b0e095b71ff8 100644 --- a/test/functional/services/common/failure_debugging.ts +++ b/test/functional/services/common/failure_debugging.ts @@ -38,7 +38,7 @@ export async function FailureDebuggingProvider({ getService }: FtrProviderContex const log = getService('log'); const browser = getService('browser'); - if (process.env.CI !== 'true') { + if (process.env.CI !== 'true' && !process.env.stack_functional_integration) { await del(config.get('failureDebugging.htmlDirectory')); } diff --git a/test/functional/services/common/screenshots.ts b/test/functional/services/common/screenshots.ts index daa55240f3eb7..5bce0d4cf6c87 100644 --- a/test/functional/services/common/screenshots.ts +++ b/test/functional/services/common/screenshots.ts @@ -40,7 +40,7 @@ export async function ScreenshotsProvider({ getService }: FtrProviderContext) { const FAILURE_DIRECTORY = resolve(config.get('screenshots.directory'), 'failure'); const BASELINE_DIRECTORY = resolve(config.get('screenshots.directory'), 'baseline'); - if (process.env.CI !== 'true') { + if (process.env.CI !== 'true' && !process.env.stack_functional_integration) { await del([SESSION_DIRECTORY, FAILURE_DIRECTORY]); } diff --git a/test/functional/services/common/snapshots.ts b/test/functional/services/common/snapshots.ts index 2e0b360e594e5..03eadff82e31f 100644 --- a/test/functional/services/common/snapshots.ts +++ b/test/functional/services/common/snapshots.ts @@ -35,7 +35,7 @@ export async function SnapshotsProvider({ getService }: FtrProviderContext) { const SESSION_DIRECTORY = resolve(config.get('snapshots.directory'), 'session'); const BASELINE_DIRECTORY = resolve(config.get('snapshots.directory'), 'baseline'); - if (process.env.CI !== 'true') { + if (process.env.CI !== 'true' && !process.env.stack_functional_integration) { await del([SESSION_DIRECTORY]); } diff --git a/x-pack/test/stack_functional_integration/test/functional/apps/ccs/ccs.js b/x-pack/test/stack_functional_integration/apps/ccs/ccs.js similarity index 100% rename from x-pack/test/stack_functional_integration/test/functional/apps/ccs/ccs.js rename to x-pack/test/stack_functional_integration/apps/ccs/ccs.js diff --git a/x-pack/test/stack_functional_integration/test/functional/apps/ccs/index.js b/x-pack/test/stack_functional_integration/apps/ccs/index.js similarity index 100% rename from x-pack/test/stack_functional_integration/test/functional/apps/ccs/index.js rename to x-pack/test/stack_functional_integration/apps/ccs/index.js diff --git a/x-pack/test/stack_functional_integration/test/functional/apps/filebeat/filebeat.js b/x-pack/test/stack_functional_integration/apps/filebeat/filebeat.js similarity index 100% rename from x-pack/test/stack_functional_integration/test/functional/apps/filebeat/filebeat.js rename to x-pack/test/stack_functional_integration/apps/filebeat/filebeat.js diff --git a/x-pack/test/stack_functional_integration/test/functional/apps/filebeat/index.js b/x-pack/test/stack_functional_integration/apps/filebeat/index.js similarity index 100% rename from x-pack/test/stack_functional_integration/test/functional/apps/filebeat/index.js rename to x-pack/test/stack_functional_integration/apps/filebeat/index.js diff --git a/x-pack/test/stack_functional_integration/test/functional/apps/heartbeat/_heartbeat.js b/x-pack/test/stack_functional_integration/apps/heartbeat/_heartbeat.js similarity index 100% rename from x-pack/test/stack_functional_integration/test/functional/apps/heartbeat/_heartbeat.js rename to x-pack/test/stack_functional_integration/apps/heartbeat/_heartbeat.js diff --git a/x-pack/test/stack_functional_integration/test/functional/apps/heartbeat/index.js b/x-pack/test/stack_functional_integration/apps/heartbeat/index.js similarity index 100% rename from x-pack/test/stack_functional_integration/test/functional/apps/heartbeat/index.js rename to x-pack/test/stack_functional_integration/apps/heartbeat/index.js diff --git a/x-pack/test/stack_functional_integration/test/functional/apps/management/_index_pattern_create.js b/x-pack/test/stack_functional_integration/apps/management/_index_pattern_create.js similarity index 100% rename from x-pack/test/stack_functional_integration/test/functional/apps/management/_index_pattern_create.js rename to x-pack/test/stack_functional_integration/apps/management/_index_pattern_create.js diff --git a/x-pack/test/stack_functional_integration/test/functional/apps/management/index.js b/x-pack/test/stack_functional_integration/apps/management/index.js similarity index 100% rename from x-pack/test/stack_functional_integration/test/functional/apps/management/index.js rename to x-pack/test/stack_functional_integration/apps/management/index.js diff --git a/x-pack/test/stack_functional_integration/test/functional/apps/metricbeat/_metricbeat.js b/x-pack/test/stack_functional_integration/apps/metricbeat/_metricbeat.js similarity index 100% rename from x-pack/test/stack_functional_integration/test/functional/apps/metricbeat/_metricbeat.js rename to x-pack/test/stack_functional_integration/apps/metricbeat/_metricbeat.js diff --git a/x-pack/test/stack_functional_integration/apps/metricbeat/_metricbeat_dashboard.js b/x-pack/test/stack_functional_integration/apps/metricbeat/_metricbeat_dashboard.js new file mode 100644 index 0000000000000..42f707fb77854 --- /dev/null +++ b/x-pack/test/stack_functional_integration/apps/metricbeat/_metricbeat_dashboard.js @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; +import { REPO_ROOT } from '@kbn/dev-utils'; + +export default function ({ getService, getPageObjects, updateBaselines }) { + const screenshot = getService('screenshots'); + const browser = getService('browser'); + const esArchiver = getService('esArchiver'); + const PageObjects = getPageObjects(['common', 'dashboard', 'timePicker']); + + describe('check metricbeat Dashboard', function () { + before(async function () { + await esArchiver.load(`${REPO_ROOT}/../integration-test/test/es_archives/metricbeat`); + + // this navigateToActualURL takes the place of navigating to the dashboard landing page, + // filtering on the dashboard name, selecting it, setting the timepicker, and going to full screen + await PageObjects.common.navigateToActualUrl( + 'dashboard', + 'view/Metricbeat-system-overview-ecs?_g=(filters:!(),refreshInterval:(pause:!t,value:0),' + + 'time:(from:%272020-09-29T19:02:37.902Z%27,to:%272020-09-29T19:06:43.218Z%27))&_a=' + + '(description:%27Overview%20of%20system%20metrics%27,filters:!(),fullScreenMode:!t,' + + 'options:(darkTheme:!f),query:(language:kuery,query:%27%27),timeRestore:!f,' + + 'title:%27%5BMetricbeat%20System%5D%20Overview%20ECS%27,viewMode:view)', + { + ensureCurrentUrl: false, + shouldLoginIfPrompted: true, + } + ); + // await PageObjects.common.navigateToApp('dashboard', { insertTimestamp: false }); + // await PageObjects.dashboard.loadSavedDashboard('[Metricbeat System] Overview ECS'); + // await PageObjects.timePicker.setAbsoluteRange( + // 'Sep 29, 2020 @ 14:02:37.902', + // 'Sep 29, 2020 @ 14:06:43.218' + // ); + // await PageObjects.dashboard.clickFullScreenMode(); + + await PageObjects.common.sleep(2000); + await PageObjects.dashboard.waitForRenderComplete(); + await browser.setScreenshotSize(1000, 1000); + }); + + it('[Metricbeat System] Overview ECS should match snapshot', async function () { + try { + const percentDifference = await screenshot.compareAgainstBaseline( + 'metricbeat_dashboard', + updateBaselines + ); + expect(percentDifference).to.be.lessThan(0.01); + } finally { + await PageObjects.dashboard.clickExitFullScreenLogoButton(); + } + }); + }); +} diff --git a/x-pack/test/stack_functional_integration/test/functional/apps/metricbeat/index.js b/x-pack/test/stack_functional_integration/apps/metricbeat/index.js similarity index 86% rename from x-pack/test/stack_functional_integration/test/functional/apps/metricbeat/index.js rename to x-pack/test/stack_functional_integration/apps/metricbeat/index.js index d45d6c835a315..148762c6a8b77 100644 --- a/x-pack/test/stack_functional_integration/test/functional/apps/metricbeat/index.js +++ b/x-pack/test/stack_functional_integration/apps/metricbeat/index.js @@ -7,5 +7,6 @@ export default function ({ loadTestFile }) { describe('metricbeat app', function () { loadTestFile(require.resolve('./_metricbeat')); + loadTestFile(require.resolve('./_metricbeat_dashboard')); }); } diff --git a/x-pack/test/stack_functional_integration/test/functional/apps/monitoring/_monitoring.js b/x-pack/test/stack_functional_integration/apps/monitoring/_monitoring.js similarity index 100% rename from x-pack/test/stack_functional_integration/test/functional/apps/monitoring/_monitoring.js rename to x-pack/test/stack_functional_integration/apps/monitoring/_monitoring.js diff --git a/x-pack/test/stack_functional_integration/test/functional/apps/monitoring/_monitoring_metricbeat.js b/x-pack/test/stack_functional_integration/apps/monitoring/_monitoring_metricbeat.js similarity index 100% rename from x-pack/test/stack_functional_integration/test/functional/apps/monitoring/_monitoring_metricbeat.js rename to x-pack/test/stack_functional_integration/apps/monitoring/_monitoring_metricbeat.js diff --git a/x-pack/test/stack_functional_integration/test/functional/apps/monitoring/index.js b/x-pack/test/stack_functional_integration/apps/monitoring/index.js similarity index 100% rename from x-pack/test/stack_functional_integration/test/functional/apps/monitoring/index.js rename to x-pack/test/stack_functional_integration/apps/monitoring/index.js diff --git a/x-pack/test/stack_functional_integration/test/functional/apps/packetbeat/_packetbeat.js b/x-pack/test/stack_functional_integration/apps/packetbeat/_packetbeat.js similarity index 100% rename from x-pack/test/stack_functional_integration/test/functional/apps/packetbeat/_packetbeat.js rename to x-pack/test/stack_functional_integration/apps/packetbeat/_packetbeat.js diff --git a/x-pack/test/stack_functional_integration/test/functional/apps/packetbeat/index.js b/x-pack/test/stack_functional_integration/apps/packetbeat/index.js similarity index 100% rename from x-pack/test/stack_functional_integration/test/functional/apps/packetbeat/index.js rename to x-pack/test/stack_functional_integration/apps/packetbeat/index.js diff --git a/x-pack/test/stack_functional_integration/test/functional/apps/reporting/index.js b/x-pack/test/stack_functional_integration/apps/reporting/index.js similarity index 100% rename from x-pack/test/stack_functional_integration/test/functional/apps/reporting/index.js rename to x-pack/test/stack_functional_integration/apps/reporting/index.js diff --git a/x-pack/test/stack_functional_integration/test/functional/apps/reporting/reporting_watcher.js b/x-pack/test/stack_functional_integration/apps/reporting/reporting_watcher.js similarity index 100% rename from x-pack/test/stack_functional_integration/test/functional/apps/reporting/reporting_watcher.js rename to x-pack/test/stack_functional_integration/apps/reporting/reporting_watcher.js diff --git a/x-pack/test/stack_functional_integration/test/functional/apps/reporting/reporting_watcher_png.js b/x-pack/test/stack_functional_integration/apps/reporting/reporting_watcher_png.js similarity index 100% rename from x-pack/test/stack_functional_integration/test/functional/apps/reporting/reporting_watcher_png.js rename to x-pack/test/stack_functional_integration/apps/reporting/reporting_watcher_png.js diff --git a/x-pack/test/stack_functional_integration/test/functional/apps/reporting/util.js b/x-pack/test/stack_functional_integration/apps/reporting/util.js similarity index 100% rename from x-pack/test/stack_functional_integration/test/functional/apps/reporting/util.js rename to x-pack/test/stack_functional_integration/apps/reporting/util.js diff --git a/x-pack/test/stack_functional_integration/test/functional/apps/sample_data/e_commerce.js b/x-pack/test/stack_functional_integration/apps/sample_data/e_commerce.js similarity index 100% rename from x-pack/test/stack_functional_integration/test/functional/apps/sample_data/e_commerce.js rename to x-pack/test/stack_functional_integration/apps/sample_data/e_commerce.js diff --git a/x-pack/test/stack_functional_integration/test/functional/apps/sample_data/index.js b/x-pack/test/stack_functional_integration/apps/sample_data/index.js similarity index 100% rename from x-pack/test/stack_functional_integration/test/functional/apps/sample_data/index.js rename to x-pack/test/stack_functional_integration/apps/sample_data/index.js diff --git a/x-pack/test/stack_functional_integration/test/functional/apps/telemetry/_telemetry.js b/x-pack/test/stack_functional_integration/apps/telemetry/_telemetry.js similarity index 100% rename from x-pack/test/stack_functional_integration/test/functional/apps/telemetry/_telemetry.js rename to x-pack/test/stack_functional_integration/apps/telemetry/_telemetry.js diff --git a/x-pack/test/stack_functional_integration/test/functional/apps/telemetry/index.js b/x-pack/test/stack_functional_integration/apps/telemetry/index.js similarity index 100% rename from x-pack/test/stack_functional_integration/test/functional/apps/telemetry/index.js rename to x-pack/test/stack_functional_integration/apps/telemetry/index.js diff --git a/x-pack/test/stack_functional_integration/test/functional/apps/winlogbeat/_winlogbeat.js b/x-pack/test/stack_functional_integration/apps/winlogbeat/_winlogbeat.js similarity index 100% rename from x-pack/test/stack_functional_integration/test/functional/apps/winlogbeat/_winlogbeat.js rename to x-pack/test/stack_functional_integration/apps/winlogbeat/_winlogbeat.js diff --git a/x-pack/test/stack_functional_integration/test/functional/apps/winlogbeat/index.js b/x-pack/test/stack_functional_integration/apps/winlogbeat/index.js similarity index 100% rename from x-pack/test/stack_functional_integration/test/functional/apps/winlogbeat/index.js rename to x-pack/test/stack_functional_integration/apps/winlogbeat/index.js diff --git a/x-pack/test/stack_functional_integration/configs/config.stack_functional_integration_base.js b/x-pack/test/stack_functional_integration/configs/config.stack_functional_integration_base.js index 96d338a04b01b..a838b129242a1 100644 --- a/x-pack/test/stack_functional_integration/configs/config.stack_functional_integration_base.js +++ b/x-pack/test/stack_functional_integration/configs/config.stack_functional_integration_base.js @@ -6,12 +6,12 @@ import { resolve } from 'path'; import buildState from './build_state'; -import { ToolingLog } from '@kbn/dev-utils'; +import { ToolingLog, REPO_ROOT } from '@kbn/dev-utils'; import chalk from 'chalk'; import { esTestConfig, kbnTestConfig } from '@kbn/test'; const reportName = 'Stack Functional Integration Tests'; -const testsFolder = '../test/functional/apps'; +const testsFolder = '../apps'; const log = new ToolingLog({ level: 'info', writeTo: process.stdout, @@ -19,13 +19,14 @@ const log = new ToolingLog({ log.info(`WORKSPACE in config file ${process.env.WORKSPACE}`); const stateFilePath = process.env.WORKSPACE ? `${process.env.WORKSPACE}/qa/envvars.sh` - : '../../../../../integration-test/qa/envvars.sh'; + : `${REPO_ROOT}/../integration-test/qa/envvars.sh`; const prepend = (testFile) => require.resolve(`${testsFolder}/${testFile}`); export default async ({ readConfigFile }) => { const defaultConfigs = await readConfigFile(require.resolve('../../functional/config')); const { tests, ...provisionedConfigs } = buildState(resolve(__dirname, stateFilePath)); + process.env.stack_functional_integration = true; const servers = { kibana: kbnTestConfig.getUrlParts(), @@ -43,6 +44,14 @@ export default async ({ readConfigFile }) => { // If we need to do things like disable animations, we can do it in configure_start_kibana.sh, in the provisioner...which lives in the integration-test private repo uiSettings: {}, security: { disableTestUser: true }, + // choose where screenshots should be saved + screenshots: { + directory: resolve(`${REPO_ROOT}/../integration-test`, 'test/screenshots'), + }, + // choose where esArchiver should load archives from + esArchiver: { + directory: resolve(`${REPO_ROOT}/../integration-test`, 'test/es_archives'), + }, }; return settings; }; @@ -55,7 +64,7 @@ function truncate(testPath) { return dropKibanaPath(testPath); } function highLight(testPath) { - const dropTestsPath = splitRight(/^.+test[\\/]functional[\\/]apps[\\/](.*)[\\/]/gm); + const dropTestsPath = splitRight(/^.+apps[\\/](.*)[\\/]/gm); const cleaned = dropTestsPath(testPath); const colored = chalk.greenBright.bold(cleaned); return testPath.replace(cleaned, colored); diff --git a/x-pack/test/stack_functional_integration/configs/tests_list.js b/x-pack/test/stack_functional_integration/configs/tests_list.js index 0d91a078b73fd..44b622a8bc9c5 100644 --- a/x-pack/test/stack_functional_integration/configs/tests_list.js +++ b/x-pack/test/stack_functional_integration/configs/tests_list.js @@ -20,7 +20,11 @@ export default (envObj) => { } if (envObj.BEATS.includes('metricbeat')) { - xs.push('metricbeat'); + xs.push('metricbeat/_metricbeat'); + if (envObj.XPACK === 'YES') { + // the esArchive and dashboard png are specific to the default distribution (with XPACK) + xs.push('metricbeat/_metricbeat_dashboard'); + } } if (envObj.BEATS.includes('filebeat')) { xs.push('filebeat');