[Security Solutions] Fixes flake with cypress tests (elastic#97329)

## Summary Fixes some recent flakeyness with Cypress tests * Adds cypress.pipe() on button clicks around the area of flakes * Adds an alerting threshold to the utilities so we can wait for when an exact number of alerts are available on a page * Changes the alerts to not run again with 10 seconds, because if a test takes longer than 10 seconds, the rule can run a second time which can invalidate some of the text when running checks when timeline or other components update on their button clicks. ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios
kibanamachine · Apr 16, 2021 · 721f4b5 · 721f4b5
1 parent 1cbdb26
commit 721f4b5
Show file tree

Hide file tree

Showing 11 changed files with 38 additions and 21 deletions.
diff --git a/x-pack/plugins/security_solution/cypress/integration/detection_alerts/attach_to_case.spec.ts b/x-pack/plugins/security_solution/cypress/integration/detection_alerts/attach_to_case.spec.ts
@@ -32,7 +32,7 @@ describe('Alerts timeline', () => {
     waitForAlertsIndexToBeCreated();
     createCustomRuleActivated(newRule);
     refreshPage();
-    waitForAlertsToPopulate();
+    waitForAlertsToPopulate(500);
 
     // Then we login as read-only user to test.
     login(ROLES.reader);

diff --git a/x-pack/plugins/security_solution/cypress/integration/detection_alerts/closing.spec.ts b/x-pack/plugins/security_solution/cypress/integration/detection_alerts/closing.spec.ts
@@ -39,9 +39,9 @@ describe('Closing alerts', () => {
     loginAndWaitForPage(DETECTIONS_URL);
     waitForAlertsPanelToBeLoaded();
     waitForAlertsIndexToBeCreated();
-    createCustomRuleActivated(newRule);
+    createCustomRuleActivated(newRule, '1', '100m', 100);
     refreshPage();
-    waitForAlertsToPopulate();
+    waitForAlertsToPopulate(100);
     deleteCustomRule();
   });
 

diff --git a/x-pack/plugins/security_solution/cypress/integration/detection_alerts/in_progress.spec.ts b/x-pack/plugins/security_solution/cypress/integration/detection_alerts/in_progress.spec.ts
@@ -38,7 +38,7 @@ describe('Marking alerts as in-progress', () => {
     waitForAlertsIndexToBeCreated();
     createCustomRuleActivated(newRule);
     refreshPage();
-    waitForAlertsToPopulate();
+    waitForAlertsToPopulate(500);
   });
 
   it('Mark one alert in progress when more than one open alerts are selected', () => {

diff --git a/...ns/security_solution/cypress/integration/detection_alerts/investigate_in_timeline.spec.ts b/...ns/security_solution/cypress/integration/detection_alerts/investigate_in_timeline.spec.ts
@@ -29,7 +29,7 @@ describe('Alerts timeline', () => {
     waitForAlertsIndexToBeCreated();
     createCustomRuleActivated(newRule);
     refreshPage();
-    waitForAlertsToPopulate();
+    waitForAlertsToPopulate(500);
   });
 
   it('Investigate alert in default timeline', () => {

diff --git a/x-pack/plugins/security_solution/cypress/integration/detection_alerts/opening.spec.ts b/x-pack/plugins/security_solution/cypress/integration/detection_alerts/opening.spec.ts
@@ -39,7 +39,7 @@ describe('Opening alerts', () => {
     waitForAlertsIndexToBeCreated();
     createCustomRuleActivated(newRule);
     refreshPage();
-    waitForAlertsToPopulate();
+    waitForAlertsToPopulate(500);
     selectNumberOfAlerts(5);
 
     cy.get(SELECTED_ALERTS).should('have.text', `Selected 5 alerts`);

diff --git a/x-pack/plugins/security_solution/cypress/integration/exceptions/from_alert.spec.ts b/x-pack/plugins/security_solution/cypress/integration/exceptions/from_alert.spec.ts
@@ -43,7 +43,7 @@ describe('From alert', () => {
     cleanKibana();
     loginAndWaitForPageWithoutDateRange(DETECTIONS_URL);
     waitForAlertsIndexToBeCreated();
-    createCustomRule(newRule);
+    createCustomRule(newRule, 'rule_testing', '10s');
     goToManageAlertsDetectionRules();
     goToRuleDetails();
 

diff --git a/x-pack/plugins/security_solution/cypress/integration/exceptions/from_rule.spec.ts b/x-pack/plugins/security_solution/cypress/integration/exceptions/from_rule.spec.ts
@@ -41,7 +41,7 @@ describe('From rule', () => {
     cleanKibana();
     loginAndWaitForPageWithoutDateRange(DETECTIONS_URL);
     waitForAlertsIndexToBeCreated();
-    createCustomRule(newRule);
+    createCustomRule(newRule, 'rule_testing', '10s');
     goToManageAlertsDetectionRules();
     goToRuleDetails();
 

diff --git a/x-pack/plugins/security_solution/cypress/objects/rule.ts b/x-pack/plugins/security_solution/cypress/objects/rule.ts
@@ -185,7 +185,7 @@ export const existingRule: CustomRule = {
   name: 'Rule 1',
   description: 'Description for Rule 1',
   index: ['auditbeat-*'],
-  interval: '10s',
+  interval: '100m',
   severity: 'High',
   riskScore: '19',
   tags: ['rule1'],
@@ -332,5 +332,5 @@ export const editedRule = {
 export const expectedExportedRule = (ruleResponse: Cypress.Response) => {
   const jsonrule = ruleResponse.body;
 
-  return `{"id":"${jsonrule.id}","updated_at":"${jsonrule.updated_at}","updated_by":"elastic","created_at":"${jsonrule.created_at}","created_by":"elastic","name":"${jsonrule.name}","tags":[],"interval":"10s","enabled":false,"description":"${jsonrule.description}","risk_score":${jsonrule.risk_score},"severity":"${jsonrule.severity}","output_index":".siem-signals-default","author":[],"false_positives":[],"from":"now-17520h","rule_id":"rule_testing","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[],"immutable":false,"type":"query","language":"kuery","index":["exceptions-*"],"query":"${jsonrule.query}","throttle":"no_actions","actions":[]}\n{"exported_count":1,"missing_rules":[],"missing_rules_count":0}\n`;
+  return `{"id":"${jsonrule.id}","updated_at":"${jsonrule.updated_at}","updated_by":"elastic","created_at":"${jsonrule.created_at}","created_by":"elastic","name":"${jsonrule.name}","tags":[],"interval":"100m","enabled":false,"description":"${jsonrule.description}","risk_score":${jsonrule.risk_score},"severity":"${jsonrule.severity}","output_index":".siem-signals-default","author":[],"false_positives":[],"from":"now-17520h","rule_id":"rule_testing","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[],"immutable":false,"type":"query","language":"kuery","index":["exceptions-*"],"query":"${jsonrule.query}","throttle":"no_actions","actions":[]}\n{"exported_count":1,"missing_rules":[],"missing_rules_count":0}\n`;
 };
diff --git a/x-pack/plugins/security_solution/cypress/tasks/alerts.ts b/x-pack/plugins/security_solution/cypress/tasks/alerts.ts
@@ -35,13 +35,25 @@ export const addExceptionFromFirstAlert = () => {
 };
 
 export const closeFirstAlert = () => {
-  cy.get(TIMELINE_CONTEXT_MENU_BTN).first().click({ force: true });
-  cy.get(CLOSE_ALERT_BTN).click();
+  cy.get(TIMELINE_CONTEXT_MENU_BTN)
+    .first()
+    .pipe(($el) => $el.trigger('click'))
+    .should('be.visible');
+
+  cy.get(CLOSE_ALERT_BTN)
+    .pipe(($el) => $el.trigger('click'))
+    .should('not.be.visible');
 };
 
 export const closeAlerts = () => {
-  cy.get(TAKE_ACTION_POPOVER_BTN).click({ force: true });
-  cy.get(CLOSE_SELECTED_ALERTS_BTN).click();
+  cy.get(TAKE_ACTION_POPOVER_BTN)
+    .first()
+    .pipe(($el) => $el.trigger('click'))
+    .should('be.visible');
+
+  cy.get(CLOSE_SELECTED_ALERTS_BTN)
+    .pipe(($el) => $el.trigger('click'))
+    .should('not.be.visible');
 };
 
 export const expandFirstAlert = () => {

diff --git a/x-pack/plugins/security_solution/cypress/tasks/api_calls/rules.ts b/x-pack/plugins/security_solution/cypress/tasks/api_calls/rules.ts
@@ -7,15 +7,15 @@
 
 import { CustomRule, ThreatIndicatorRule } from '../../objects/rule';
 
-export const createCustomRule = (rule: CustomRule, ruleId = 'rule_testing') =>
+export const createCustomRule = (rule: CustomRule, ruleId = 'rule_testing', interval = '100m') =>
   cy.request({
     method: 'POST',
     url: 'api/detection_engine/rules',
     body: {
       rule_id: ruleId,
       risk_score: parseInt(rule.riskScore, 10),
       description: rule.description,
-      interval: '10s',
+      interval,
       name: rule.name,
       severity: rule.severity.toLocaleLowerCase(),
       type: 'query',
@@ -67,15 +67,20 @@ export const createCustomIndicatorRule = (rule: ThreatIndicatorRule, ruleId = 'r
     failOnStatusCode: false,
   });
 
-export const createCustomRuleActivated = (rule: CustomRule, ruleId = '1') =>
+export const createCustomRuleActivated = (
+  rule: CustomRule,
+  ruleId = '1',
+  interval = '100m',
+  maxSignals = 500
+) =>
   cy.request({
     method: 'POST',
     url: 'api/detection_engine/rules',
     body: {
       rule_id: ruleId,
       risk_score: parseInt(rule.riskScore, 10),
       description: rule.description,
-      interval: '10s',
+      interval,
       name: rule.name,
       severity: rule.severity.toLocaleLowerCase(),
       type: 'query',
@@ -85,7 +90,7 @@ export const createCustomRuleActivated = (rule: CustomRule, ruleId = '1') =>
       language: 'kuery',
       enabled: true,
       tags: ['rule1'],
-      max_signals: 500,
+      max_signals: maxSignals,
     },
     headers: { 'kbn-xsrf': 'cypress-creds' },
     failOnStatusCode: false,

diff --git a/x-pack/plugins/security_solution/cypress/tasks/create_new_rule.ts b/x-pack/plugins/security_solution/cypress/tasks/create_new_rule.ts
@@ -479,7 +479,7 @@ export const selectThresholdRuleType = () => {
   cy.get(THRESHOLD_TYPE).click({ force: true });
 };
 
-export const waitForAlertsToPopulate = async () => {
+export const waitForAlertsToPopulate = async (alertCountThreshold = 1) => {
   cy.waitUntil(
     () => {
       refreshPage();
@@ -488,7 +488,7 @@ export const waitForAlertsToPopulate = async () => {
         .invoke('text')
         .then((countText) => {
           const alertCount = parseInt(countText, 10) || 0;
-          return alertCount > 0;
+          return alertCount >= alertCountThreshold;
         });
     },
     { interval: 500, timeout: 12000 }