From 5f1da5ce7630403aeb1f930df17896a7fef739f1 Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Thu, 14 Mar 2024 14:30:52 +0000 Subject: [PATCH] =?UTF-8?q?[Entity=20Analytics]=20Hopefully=20fix=20flaky?= =?UTF-8?q?=20tests=20=F0=9F=A4=9E=20=20(#177421)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Summary Closes https://github.com/elastic/kibana/issues/174860 Closes https://github.com/elastic/kibana/issues/174859 Hopefully fix some flaky tests for good. Flaky test run 1 (50 ESS): https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/5268 ✅ Flaky test run 2 (100 ESS 100 Serverless) https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/5278 ✅ Flaky test run 3 100 again https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/5436 ✅ (ran as a lot of time has passed) --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../risk_information/index.test.tsx | 70 ++++++++----------- .../components/risk_information/index.tsx | 27 +------ .../host_details/risk_tab.cy.ts | 26 ++----- .../hosts/host_risk_tab.cy.ts | 19 ++--- .../cypress/tasks/host_risk.ts | 15 +++- 5 files changed, 58 insertions(+), 99 deletions(-) diff --git a/x-pack/plugins/security_solution/public/entity_analytics/components/risk_information/index.test.tsx b/x-pack/plugins/security_solution/public/entity_analytics/components/risk_information/index.test.tsx index 722b14140105..d24ad253b551 100644 --- a/x-pack/plugins/security_solution/public/entity_analytics/components/risk_information/index.test.tsx +++ b/x-pack/plugins/security_solution/public/entity_analytics/components/risk_information/index.test.tsx @@ -7,59 +7,51 @@ import { render, fireEvent, within } from '@testing-library/react'; import React from 'react'; -import { RiskInformationButtonEmpty, RiskInformationButtonIcon } from '.'; +import { RiskInformationButtonEmpty } from '.'; import { TestProviders } from '../../../common/mock'; import { RiskScoreEntity } from '../../../../common/search_strategy'; describe.each([RiskScoreEntity.host, RiskScoreEntity.user])( 'Risk Information entityType: %s', (riskEntity) => { - describe.each(['RiskInformationButtonIcon', 'RiskInformationButtonEmpty'])( - `%s component`, - (componentType) => { - const Component = - componentType === 'RiskInformationButtonIcon' - ? RiskInformationButtonIcon - : RiskInformationButtonEmpty; + describe('RiskInformationButtonEmpty', () => { + it('renders', () => { + const { queryByTestId } = render(); - it('renders', () => { - const { queryByTestId } = render(); + expect(queryByTestId('open-risk-information-flyout-trigger')).toBeInTheDocument(); + }); - expect(queryByTestId('open-risk-information-flyout-trigger')).toBeInTheDocument(); - }); + it('opens and displays a risk levels table with 5 rows', () => { + const NUMBER_OF_ROWS = 1 + 5; // 1 header row + 5 severity rows + const { getByTestId, queryByTestId } = render( + + + + ); - it('opens and displays a risk levels table with 5 rows', () => { - const NUMBER_OF_ROWS = 1 + 5; // 1 header row + 5 severity rows - const { getByTestId, queryByTestId } = render( - - - - ); + fireEvent.click(getByTestId('open-risk-information-flyout-trigger')); - fireEvent.click(getByTestId('open-risk-information-flyout-trigger')); + const riskLevelsTable = queryByTestId('risk-level-information-table'); - const riskLevelsTable = queryByTestId('risk-level-information-table'); + expect(riskLevelsTable).toBeInTheDocument(); + expect(within(riskLevelsTable!).queryAllByRole('row')).toHaveLength(NUMBER_OF_ROWS); + }); - expect(riskLevelsTable).toBeInTheDocument(); - expect(within(riskLevelsTable!).queryAllByRole('row')).toHaveLength(NUMBER_OF_ROWS); - }); + it('opens and displays an asset criticality tiers table with four (4) rows', () => { + const NUMBER_OF_ROWS = 1 + 4; // 1 header row + 4 tier rows + const { getByTestId, queryByTestId } = render( + + + + ); - it('opens and displays an asset criticality tiers table with four (4) rows', () => { - const NUMBER_OF_ROWS = 1 + 4; // 1 header row + 4 tier rows - const { getByTestId, queryByTestId } = render( - - - - ); + fireEvent.click(getByTestId('open-risk-information-flyout-trigger')); - fireEvent.click(getByTestId('open-risk-information-flyout-trigger')); + const criticalityTiersTable = queryByTestId('criticality-level-information-table'); - const criticalityTiersTable = queryByTestId('criticality-level-information-table'); - - expect(criticalityTiersTable).toBeInTheDocument(); - expect(within(criticalityTiersTable!).queryAllByRole('row')).toHaveLength(NUMBER_OF_ROWS); - }); - } - ); + expect(criticalityTiersTable).toBeInTheDocument(); + expect(within(criticalityTiersTable!).queryAllByRole('row')).toHaveLength(NUMBER_OF_ROWS); + }); + }); } ); diff --git a/x-pack/plugins/security_solution/public/entity_analytics/components/risk_information/index.tsx b/x-pack/plugins/security_solution/public/entity_analytics/components/risk_information/index.tsx index 3324762bf840..e0880bf56321 100644 --- a/x-pack/plugins/security_solution/public/entity_analytics/components/risk_information/index.tsx +++ b/x-pack/plugins/security_solution/public/entity_analytics/components/risk_information/index.tsx @@ -11,7 +11,6 @@ import { EuiBasicTable, EuiButton, EuiButtonEmpty, - EuiButtonIcon, EuiFlexGroup, EuiFlexItem, EuiFlyout, @@ -32,7 +31,8 @@ import { css } from '@emotion/react'; import * as i18n from './translations'; import { useOnOpenCloseHandler } from '../../../helper_hooks'; import { RiskScoreLevel } from '../severity/common'; -import { RiskScoreEntity, RiskSeverity } from '../../../../common/search_strategy'; +import type { RiskScoreEntity } from '../../../../common/search_strategy'; +import { RiskSeverity } from '../../../../common/search_strategy'; import { CriticalityLevels, CriticalityModifiers, @@ -105,29 +105,6 @@ const getCriticalityLevelTableColumns = (): Array< export const HOST_RISK_INFO_BUTTON_CLASS = 'HostRiskInformation__button'; export const USER_RISK_INFO_BUTTON_CLASS = 'UserRiskInformation__button'; -export const RiskInformationButtonIcon = ({ riskEntity }: { riskEntity: RiskScoreEntity }) => { - const [isFlyoutVisible, handleOnOpen, handleOnClose] = useOnOpenCloseHandler(); - - return ( - <> - - {isFlyoutVisible && } - - ); -}; - export const RiskInformationButtonEmpty = ({ riskEntity }: { riskEntity: RiskScoreEntity }) => { const [isFlyoutVisible, handleOnOpen, handleOnClose] = useOnOpenCloseHandler(); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/entity_analytics/host_details/risk_tab.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/entity_analytics/host_details/risk_tab.cy.ts index 86303a84455a..3b81ccfdd36d 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/entity_analytics/host_details/risk_tab.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/entity_analytics/host_details/risk_tab.cy.ts @@ -8,7 +8,6 @@ import { login } from '../../../tasks/login'; import { visitHostDetailsPage } from '../../../tasks/navigation'; -import { waitForTableToLoad } from '../../../tasks/common'; import { TABLE_CELL, TABLE_ROWS } from '../../../screens/alerts_details'; import { deleteRiskEngineConfiguration } from '../../../tasks/api_calls/risk_engine'; import { openRiskInformationFlyout, mockRiskEngineEnabled } from '../../../tasks/entity_analytics'; @@ -18,26 +17,20 @@ import { navigateToHostRiskDetailTab } from '../../../tasks/host_risk'; import { deleteAlertsAndRules } from '../../../tasks/api_calls/common'; describe('risk tab', { tags: ['@ess', '@serverless'] }, () => { - // FLAKY: https://github.com/elastic/kibana/issues/169033 - // FLAKY: https://github.com/elastic/kibana/issues/169034 - describe.skip('with legacy risk score', () => { - before(() => { - cy.task('esArchiverLoad', { archiveName: 'risk_hosts' }); - }); - + describe('with legacy risk score', () => { beforeEach(() => { + cy.task('esArchiverLoad', { archiveName: 'risk_hosts' }); login(); deleteRiskEngineConfiguration(); }); - after(() => { + afterEach(() => { cy.task('esArchiverUnload', 'risk_hosts'); }); it('renders risk tab', () => { visitHostDetailsPage('siem-kibana'); navigateToHostRiskDetailTab(); - waitForTableToLoad(); cy.get('[data-test-subj="topRiskScoreContributors"]') .find(TABLE_ROWS) @@ -49,7 +42,6 @@ describe('risk tab', { tags: ['@ess', '@serverless'] }, () => { it('shows risk information overlay when button is clicked', () => { visitHostDetailsPage('siem-kibana'); navigateToHostRiskDetailTab(); - waitForTableToLoad(); openRiskInformationFlyout(); @@ -57,19 +49,15 @@ describe('risk tab', { tags: ['@ess', '@serverless'] }, () => { }); }); - // FLAKY: https://github.com/elastic/kibana/issues/177964 - describe.skip('with new risk score', () => { - before(() => { + describe('with new risk score', () => { + beforeEach(() => { cy.task('esArchiverLoad', { archiveName: 'risk_scores_new_complete_data' }); cy.task('esArchiverLoad', { archiveName: 'query_alert', useCreate: true, docsOnly: true }); - }); - - beforeEach(() => { mockRiskEngineEnabled(); login(); }); - after(() => { + afterEach(() => { cy.task('esArchiverUnload', 'risk_scores_new_complete_data'); deleteAlertsAndRules(); // esArchiverUnload doesn't work properly when using with `useCreate` and `docsOnly` flags deleteRiskEngineConfiguration(); @@ -78,7 +66,6 @@ describe('risk tab', { tags: ['@ess', '@serverless'] }, () => { it('renders risk tab', () => { visitHostDetailsPage('Host-fwarau82er'); navigateToHostRiskDetailTab(); - waitForTableToLoad(); cy.get(ALERTS_COUNT).should('have.text', '1 alert'); cy.get(ALERT_GRID_CELL).contains('Endpoint Security'); @@ -87,7 +74,6 @@ describe('risk tab', { tags: ['@ess', '@serverless'] }, () => { it('shows risk information overlay when button is clicked', () => { visitHostDetailsPage('siem-kibana'); navigateToHostRiskDetailTab(); - waitForTableToLoad(); openRiskInformationFlyout(); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/entity_analytics/hosts/host_risk_tab.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/entity_analytics/hosts/host_risk_tab.cy.ts index 88a5b55efe9e..2e6fa22be430 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/entity_analytics/hosts/host_risk_tab.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/entity_analytics/hosts/host_risk_tab.cy.ts @@ -23,8 +23,7 @@ import { kqlSearch } from '../../../tasks/security_header'; import { mockRiskEngineEnabled } from '../../../tasks/entity_analytics'; describe('risk tab', { tags: ['@ess', '@serverless'] }, () => { - // FLAKY: https://github.com/elastic/kibana/issues/174859 - describe.skip('with legacy risk score', () => { + describe('with legacy risk score', () => { before(() => { cy.task('esArchiverLoad', { archiveName: 'risk_hosts' }); }); @@ -36,10 +35,6 @@ describe('risk tab', { tags: ['@ess', '@serverless'] }, () => { // this fix wait until we fave host in all host table, and then we go to risk tab cy.contains('siem-kibana'); - // Sometimes it doesn't navigate to the risk tab an causes flakiness - // Curiously the "renders the table" test doesn't fail - // https://github.com/elastic/kibana/issues/174860 - // https://github.com/elastic/kibana/issues/174859 navigateToHostRiskDetailTab(); }); @@ -55,8 +50,7 @@ describe('risk tab', { tags: ['@ess', '@serverless'] }, () => { cy.get(HOST_BY_RISK_TABLE_CELL).eq(7).should('have.text', 'Low'); }); - // Flaky - it.skip('filters the table', () => { + it('filters the table', () => { openRiskTableFilterAndSelectTheCriticalOption(); cy.get(HOST_BY_RISK_TABLE_CELL).eq(3).should('not.have.text', 'siem-kibana'); @@ -64,8 +58,7 @@ describe('risk tab', { tags: ['@ess', '@serverless'] }, () => { removeCriticalFilterAndCloseRiskTableFilter(); }); - // Flaky - it.skip('should be able to change items count per page', () => { + it('should be able to change items count per page', () => { selectFiveItemsPerPageOption(); cy.get(HOST_BY_RISK_TABLE_HOSTNAME_CELL).should('have.length', 5); @@ -104,8 +97,7 @@ describe('risk tab', { tags: ['@ess', '@serverless'] }, () => { cy.get(HOST_BY_RISK_TABLE_CELL).eq(7).should('have.text', 'Critical'); }); - // Flaky - it.skip('filters the table', () => { + it('filters the table', () => { openRiskTableFilterAndSelectTheCriticalOption(); cy.get(HOST_BY_RISK_TABLE_CELL).eq(3).should('not.have.text', 'siem-kibana'); @@ -113,8 +105,7 @@ describe('risk tab', { tags: ['@ess', '@serverless'] }, () => { removeCriticalFilterAndCloseRiskTableFilter(); }); - // Flaky - it.skip('should be able to change items count per page', () => { + it('should be able to change items count per page', () => { selectFiveItemsPerPageOption(); cy.get(HOST_BY_RISK_TABLE_HOSTNAME_CELL).should('have.length', 5); diff --git a/x-pack/test/security_solution_cypress/cypress/tasks/host_risk.ts b/x-pack/test/security_solution_cypress/cypress/tasks/host_risk.ts index ebf95a1d6aec..cda088972466 100644 --- a/x-pack/test/security_solution_cypress/cypress/tasks/host_risk.ts +++ b/x-pack/test/security_solution_cypress/cypress/tasks/host_risk.ts @@ -15,9 +15,22 @@ import { RISK_DETAILS_NAV, } from '../screens/hosts/host_risk'; -export const navigateToHostRiskDetailTab = () => { +export const navigateToHostRiskDetailTab = (attempts = 3) => { + // this was causing flakiness + // on very rare occasions the tab is not navigated to, so we now retry + if (attempts === 0) { + throw new Error('navigateToHostRiskDetailTab failed after 3 attempts'); + } + cy.get(LOADING_SPINNER).should('not.exist'); cy.get(RISK_DETAILS_NAV).click(); cy.get(LOADING_SPINNER).should('not.exist'); + cy.get(RISK_DETAILS_NAV).then(($tab) => { + if ($tab.hasClass('euiTab-isSelected')) { + return; + } else if (attempts > 0) { + navigateToHostRiskDetailTab(attempts - 1); + } + }); }; export const openRiskTableFilterAndSelectTheCriticalOption = () => {