Run tests in parallel in many detection_engine_api_integration data_types suites

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts

@@ -125,7 +125,9 @@ export default ({ getService }: FtrProviderContext) => {
         await waitForRuleSuccessOrStatus(supertest, log, id);
         await waitForSignalsToBePresent(supertest, log, 1, [id]);
         const signalsOpen = await getSignalsByIds(supertest, log, [id]);
-        expect(signalsOpen.hits.hits[0]._source![ALERT_RULE_RULE_ID]).eql(getSimpleRule().rule_id);
+        expect(signalsOpen.hits.hits[0]._source![ALERT_RULE_RULE_ID]).eql(
+          getSimpleRule(rule.rule_id).rule_id
+        );
       });
 
       it('should query and get back expected signal structure using a basic KQL query', async () => {

x-pack/test/detection_engine_api_integration/utils/create_container_with_entries.ts

@@ -15,6 +15,7 @@ import { createExceptionList } from './create_exception_list';
 import { createExceptionListItem } from './create_exception_list_item';
 import { waitFor } from './wait_for';
 
+let listIdCounter = 0;
 /**
  * Convenience testing function where you can pass in just the endpoint entries and you will
  * get a container created with the entries.
@@ -31,11 +32,13 @@ export const createContainerWithEntries = async (
   if (entries.length === 0) {
     return [];
   }
+
+  const listId = `some-list-id-${++listIdCounter}`;
   // Create the rule exception list container
   // eslint-disable-next-line @typescript-eslint/naming-convention
   const { id, list_id, namespace_type, type } = await createExceptionList(supertest, log, {
     description: 'some description',
-    list_id: 'some-list-id',
+    list_id: listId,
     name: 'some name',
     type: 'detection',
   });
@@ -45,7 +48,7 @@ export const createContainerWithEntries = async (
     entries.map((entry) => {
       const exceptionListItem: CreateExceptionListItemSchema = {
         description: 'some description',
-        list_id: 'some-list-id',
+        list_id: listId,
         name: 'some name',
         type: 'simple',
         entries: entry,

x-pack/test/detection_engine_api_integration/utils/create_parallel_test_runner.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/**
+ * Creates a test runner that can be used to run multiple async test cases
+ * in parallel. Makes it easy/convenient to convert `it('', ...)` to
+ * `runner.test('', ...)`
+ */
+export const createParallelTestRunner = () => {
+  const r = {
+    tests: {} as Record<string, () => Promise<any>>,
+    test: (key: string, test: () => Promise<any>) => {
+      r.tests[key] = test;
+    },
+    run: async () => {
+      const promises = Object.keys(r.tests).map(async (key) => {
+        try {
+          await r.tests[key]();
+        } catch (ex: any) {
+          throw new Error(`during ${key}:\n${ex.toString()}`);
+        }
+      });
+      await Promise.all(promises);
+    },
+  };
+
+  return r;
+};

x-pack/test/detection_engine_api_integration/utils/get_rule_for_signal_testing.ts

@@ -14,19 +14,23 @@ import type { QueryCreateSchema } from '@kbn/security-solution-plugin/common/det
  * @param ruleId The optional ruleId which is rule-1 by default.
  * @param enabled Enables the rule on creation or not. Defaulted to true.
  */
+let id = 0;
+
 export const getRuleForSignalTesting = (
   index: string[],
   ruleId = 'rule-1',
   enabled = true
-): QueryCreateSchema => ({
-  name: 'Signal Testing Query',
-  description: 'Tests a simple query',
-  enabled,
-  risk_score: 1,
-  rule_id: ruleId,
-  severity: 'high',
-  index,
-  type: 'query',
-  query: '*:*',
-  from: '1900-01-01T00:00:00.000Z',
-});
+): QueryCreateSchema => {
+  return {
+    name: 'Signal Testing Query',
+    description: 'Tests a simple query',
+    enabled,
+    risk_score: 1,
+    rule_id: ruleId ? ruleId : `rule-${++id}`,
+    severity: 'high',
+    index,
+    type: 'query',
+    query: '*:*',
+    from: '1900-01-01T00:00:00.000Z',
+  };
+};

x-pack/test/detection_engine_api_integration/utils/index.ts

@@ -14,6 +14,7 @@ export * from './create_exception_list';
 export * from './create_exception_list_item';
 export * from './create_legacy_rule_action';
 export * from './create_new_action';
+export * from './create_parallel_test_runner';
 export * from './create_rule';
 export * from './create_rule_with_auth';
 export * from './create_rule_with_exception_entries';

x-pack/test/lists_api_integration/utils.ts

@@ -25,6 +25,8 @@ import { ToolingLog } from '@kbn/tooling-log';
 import { getImportListItemAsBuffer } from '@kbn/lists-plugin/common/schemas/request/import_list_item_schema.mock';
 import { countDownTest } from '../detection_engine_api_integration/utils';
 
+let fileCounter = 0;
+
 /**
  * Creates the lists and lists items index for use inside of beforeEach blocks of tests
  * This will retry 50 times before giving up and hopefully still not interfere with other tests
@@ -219,12 +221,15 @@ export const deleteAllExceptionsByType = async (
         .set('kbn-xsrf', 'true')
         .send();
       const ids: string[] = body.data.map((exception: ExceptionList) => exception.id);
-      for await (const id of ids) {
-        await supertest
-          .delete(`${EXCEPTION_LIST_URL}?id=${id}&namespace_type=${type}`)
-          .set('kbn-xsrf', 'true')
-          .send();
-      }
+      await Promise.all(
+        ids.map((id) =>
+          supertest
+            .delete(`${EXCEPTION_LIST_URL}?id=${id}&namespace_type=${type}`)
+            .set('kbn-xsrf', 'true')
+            .send()
+        )
+      );
+
       const { body: finalCheck } = await supertest
         .get(`${EXCEPTION_LIST_URL}/_find?namespace_type=${type}`)
         .set('kbn-xsrf', 'true')
@@ -254,13 +259,14 @@ export const importFile = async (
   log: ToolingLog,
   type: Type,
   contents: string[],
-  fileName: string,
+  fileName?: string,
   testValues?: string[]
-): Promise<void> => {
+): Promise<string> => {
+  const finalFilename = fileName ? fileName : `file-${fileCounter++}.txt`;
   const response = await supertest
     .post(`${LIST_ITEM_URL}/_import?type=${type}`)
     .set('kbn-xsrf', 'true')
-    .attach('file', getImportListItemAsBuffer(contents), fileName)
+    .attach('file', getImportListItemAsBuffer(contents), finalFilename)
     .expect('Content-Type', 'application/json; charset=utf-8');
 
   if (response.status !== 200) {
@@ -274,7 +280,8 @@ export const importFile = async (
   // although we have pushed the list and its items, it is async so we
   // have to wait for the contents before continuing
   const testValuesOrContents = testValues ?? contents;
-  await waitForListItems(supertest, log, testValuesOrContents, fileName);
+  await waitForListItems(supertest, log, testValuesOrContents, finalFilename);
+  return finalFilename;
 };
 
 /**
@@ -291,12 +298,13 @@ export const importTextFile = async (
   log: ToolingLog,
   type: Type,
   contents: string[],
-  fileName: string
-): Promise<void> => {
+  fileName: string = ''
+): Promise<string> => {
+  const finalFilename = fileName ? fileName : `file-${fileCounter++}.txt`;
   const response = await supertest
     .post(`${LIST_ITEM_URL}/_import?type=${type}`)
     .set('kbn-xsrf', 'true')
-    .attach('file', getImportListItemAsBuffer(contents), fileName)
+    .attach('file', getImportListItemAsBuffer(contents), finalFilename)
     .expect('Content-Type', 'application/json; charset=utf-8');
 
   if (response.status !== 200) {
@@ -309,7 +317,8 @@ export const importTextFile = async (
 
   // although we have pushed the list and its items, it is async so we
   // have to wait for the contents before continuing
-  await waitForTextListItems(supertest, log, contents, fileName);
+  await waitForTextListItems(supertest, log, contents, finalFilename);
+  return finalFilename;
 };
 
 /**