chore(NA): create new x-pack cigroups and rebalancing them all (#88366)…

… (#88834) * chore(NA): create new x-pack cigroups and rebalancing them all * chore(NA): better cigroups balancing * chore(NA): push rollup tests back into ciGroup1 * chore(NA): move some functional ml tests from cigroup3 into cigroup13 * chore(NA): move some more tests into ciGroup13 * chore(NA): use a single top level describe at x-pack/test/functional/apps/ml * chore(NA): move settings into ciGroup13 * temporary test for es snapshots env * Revert "temporary test for es snapshots env" This reverts commit 789ebe7. * docs(NA): add missing documentation on the function tests describe split Co-authored-by: Kibana Machine <[email protected]> # Conflicts: # vars/kibanaCoverage.groovy
elastic · Jan 20, 2021 · c92d962 · c92d962
1 parent fa61113
commit c92d962
Show file tree

Hide file tree

Showing 33 changed files with 164 additions and 102 deletions.
diff --git a/.ci/es-snapshots/Jenkinsfile_verify_es b/.ci/es-snapshots/Jenkinsfile_verify_es
@@ -56,6 +56,8 @@ kibanaPipeline(timeoutMinutes: 150) {
             'xpack-ciGroup9': kibanaPipeline.xpackCiGroupProcess(9),
             'xpack-ciGroup10': kibanaPipeline.xpackCiGroupProcess(10),
             'xpack-ciGroup11': kibanaPipeline.xpackCiGroupProcess(11),
+            'xpack-ciGroup12': kibanaPipeline.xpackCiGroupProcess(12),
+            'xpack-ciGroup13': kibanaPipeline.xpackCiGroupProcess(13),
           ]),
         ])
       }

diff --git a/.ci/jobs.yml b/.ci/jobs.yml
@@ -32,6 +32,8 @@ JOB:
   - x-pack-ciGroup9
   - x-pack-ciGroup10
   - x-pack-ciGroup11
+  - x-pack-ciGroup12
+  - x-pack-ciGroup13
   - x-pack-accessibility
   - x-pack-visualRegression
 

diff --git a/test/functional/apps/visualize/index.ts b/test/functional/apps/visualize/index.ts
@@ -55,7 +55,7 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) {
     });
 
     describe('', function () {
-      this.tags('ciGroup11');
+      this.tags('ciGroup4');
 
       loadTestFile(require.resolve('./_line_chart'));
       loadTestFile(require.resolve('./_pie_chart'));

diff --git a/test/scripts/jenkins_xpack_build_kibana.sh b/test/scripts/jenkins_xpack_build_kibana.sh
@@ -23,7 +23,9 @@ node scripts/functional_tests --assert-none-excluded \
   --include-tag ciGroup8 \
   --include-tag ciGroup9 \
   --include-tag ciGroup10 \
-  --include-tag ciGroup11
+  --include-tag ciGroup11 \
+  --include-tag ciGroup12 \
+  --include-tag ciGroup13
 
 echo " -> building and extracting default Kibana distributable for use in functional tests"
 cd "$KIBANA_DIR"

diff --git a/vars/tasks.groovy b/vars/tasks.groovy
@@ -94,7 +94,7 @@ def functionalXpack(Map params = [:]) {
     kibanaPipeline.buildXpack(10)
 
     if (config.ciGroups) {
-      def ciGroups = 1..11
+      def ciGroups = 1..13
       tasks(ciGroups.collect { kibanaPipeline.xpackCiGroupProcess(it) })
     }
 

diff --git a/x-pack/test/alerting_api_integration/basic/tests/index.ts b/x-pack/test/alerting_api_integration/basic/tests/index.ts
@@ -12,7 +12,7 @@ export default function alertingApiIntegrationTests({
   getService,
 }: FtrProviderContext) {
   describe('alerting api integration basic license', function () {
-    this.tags('ciGroup3');
+    this.tags('ciGroup13');
 
     loadTestFile(require.resolve('./actions'));
     loadTestFile(require.resolve('./alerts'));

diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/index.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/index.ts
@@ -10,7 +10,7 @@ import { Spaces } from '../scenarios';
 // eslint-disable-next-line import/no-default-export
 export default function alertingApiIntegrationTests({ loadTestFile }: FtrProviderContext) {
   describe('alerting api integration spaces only', function () {
-    this.tags('ciGroup9');
+    this.tags('ciGroup12');
 
     loadTestFile(require.resolve('./actions'));
     loadTestFile(require.resolve('./alerting'));

diff --git a/x-pack/test/api_integration_basic/apis/index.ts b/x-pack/test/api_integration_basic/apis/index.ts
@@ -8,7 +8,7 @@ import { FtrProviderContext } from '../ftr_provider_context';
 
 export default function ({ loadTestFile }: FtrProviderContext) {
   describe('apis', function () {
-    this.tags('ciGroup2');
+    this.tags('ciGroup11');
 
     loadTestFile(require.resolve('./ml'));
     loadTestFile(require.resolve('./transform'));

diff --git a/..._engine_api_integration/security_and_spaces/tests/exception_operators_data_types/index.ts b/..._engine_api_integration/security_and_spaces/tests/exception_operators_data_types/index.ts
@@ -9,18 +9,30 @@ import { FtrProviderContext } from '../../../common/ftr_provider_context';
 // eslint-disable-next-line import/no-default-export
 export default ({ loadTestFile }: FtrProviderContext): void => {
   describe('Detection exceptions data types and operators', function () {
-    this.tags('ciGroup11');
+    describe('', function () {
+      this.tags('ciGroup11');
 
-    loadTestFile(require.resolve('./date'));
-    loadTestFile(require.resolve('./double'));
-    loadTestFile(require.resolve('./float'));
-    loadTestFile(require.resolve('./integer'));
-    loadTestFile(require.resolve('./ip'));
-    loadTestFile(require.resolve('./ip_array'));
-    loadTestFile(require.resolve('./keyword'));
-    loadTestFile(require.resolve('./keyword_array'));
-    loadTestFile(require.resolve('./long'));
-    loadTestFile(require.resolve('./text'));
-    loadTestFile(require.resolve('./text_array'));
+      loadTestFile(require.resolve('./date'));
+      loadTestFile(require.resolve('./double'));
+      loadTestFile(require.resolve('./float'));
+      loadTestFile(require.resolve('./integer'));
+    });
+
+    describe('', function () {
+      this.tags('ciGroup12');
+
+      loadTestFile(require.resolve('./ip'));
+      loadTestFile(require.resolve('./ip_array'));
+      loadTestFile(require.resolve('./keyword'));
+      loadTestFile(require.resolve('./keyword_array'));
+      loadTestFile(require.resolve('./long'));
+    });
+
+    describe('', function () {
+      this.tags('ciGroup13');
+
+      loadTestFile(require.resolve('./text'));
+      loadTestFile(require.resolve('./text_array'));
+    });
   });
 };
diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts
@@ -9,33 +9,40 @@ import { FtrProviderContext } from '../../common/ftr_provider_context';
 // eslint-disable-next-line import/no-default-export
 export default ({ loadTestFile }: FtrProviderContext): void => {
   describe('detection engine api security and spaces enabled', function () {
-    this.tags('ciGroup11');
+    describe('', function () {
+      this.tags('ciGroup11');
 
-    loadTestFile(require.resolve('./add_actions'));
-    loadTestFile(require.resolve('./add_prepackaged_rules'));
-    loadTestFile(require.resolve('./create_rules'));
-    loadTestFile(require.resolve('./create_rules_bulk'));
-    loadTestFile(require.resolve('./create_threat_matching'));
-    loadTestFile(require.resolve('./create_exceptions'));
-    loadTestFile(require.resolve('./delete_rules'));
-    loadTestFile(require.resolve('./delete_rules_bulk'));
-    loadTestFile(require.resolve('./exception_operators_data_types/index'));
-    loadTestFile(require.resolve('./export_rules'));
-    loadTestFile(require.resolve('./find_rules'));
-    loadTestFile(require.resolve('./find_statuses'));
-    loadTestFile(require.resolve('./generating_signals'));
-    loadTestFile(require.resolve('./get_prepackaged_rules_status'));
-    loadTestFile(require.resolve('./import_rules'));
-    loadTestFile(require.resolve('./read_rules'));
-    loadTestFile(require.resolve('./update_rules'));
-    loadTestFile(require.resolve('./update_rules_bulk'));
-    loadTestFile(require.resolve('./patch_rules_bulk'));
-    loadTestFile(require.resolve('./patch_rules'));
-    loadTestFile(require.resolve('./query_signals'));
-    loadTestFile(require.resolve('./open_close_signals'));
-    loadTestFile(require.resolve('./get_signals_migration_status'));
-    loadTestFile(require.resolve('./create_signals_migrations'));
-    loadTestFile(require.resolve('./finalize_signals_migrations'));
-    loadTestFile(require.resolve('./delete_signals_migrations'));
+      loadTestFile(require.resolve('./add_actions'));
+      loadTestFile(require.resolve('./add_prepackaged_rules'));
+      loadTestFile(require.resolve('./create_rules'));
+      loadTestFile(require.resolve('./create_rules_bulk'));
+      loadTestFile(require.resolve('./create_threat_matching'));
+      loadTestFile(require.resolve('./create_exceptions'));
+      loadTestFile(require.resolve('./delete_rules'));
+      loadTestFile(require.resolve('./delete_rules_bulk'));
+      loadTestFile(require.resolve('./export_rules'));
+      loadTestFile(require.resolve('./find_rules'));
+      loadTestFile(require.resolve('./find_statuses'));
+      loadTestFile(require.resolve('./generating_signals'));
+      loadTestFile(require.resolve('./get_prepackaged_rules_status'));
+      loadTestFile(require.resolve('./import_rules'));
+      loadTestFile(require.resolve('./read_rules'));
+      loadTestFile(require.resolve('./update_rules'));
+      loadTestFile(require.resolve('./update_rules_bulk'));
+      loadTestFile(require.resolve('./patch_rules_bulk'));
+      loadTestFile(require.resolve('./patch_rules'));
+      loadTestFile(require.resolve('./query_signals'));
+      loadTestFile(require.resolve('./open_close_signals'));
+      loadTestFile(require.resolve('./get_signals_migration_status'));
+      loadTestFile(require.resolve('./create_signals_migrations'));
+      loadTestFile(require.resolve('./finalize_signals_migrations'));
+      loadTestFile(require.resolve('./delete_signals_migrations'));
+    });
+
+    // That split here enable us on using a different ciGroup to run the tests
+    // listed on ./exception_operators_data_types/index
+    describe('', function () {
+      loadTestFile(require.resolve('./exception_operators_data_types/index'));
+    });
   });
 };
diff --git a/x-pack/test/encrypted_saved_objects_api_integration/tests/index.ts b/x-pack/test/encrypted_saved_objects_api_integration/tests/index.ts
@@ -8,7 +8,7 @@ import { FtrProviderContext } from '../ftr_provider_context';
 
 export default function ({ loadTestFile }: FtrProviderContext) {
   describe('encryptedSavedObjects', function encryptedSavedObjectsSuite() {
-    this.tags('ciGroup2');
+    this.tags('ciGroup13');
     loadTestFile(require.resolve('./encrypted_saved_objects_api'));
   });
 }
diff --git a/x-pack/test/functional/apps/api_keys/feature_controls/index.ts b/x-pack/test/functional/apps/api_keys/feature_controls/index.ts
@@ -8,7 +8,7 @@ import { FtrProviderContext } from '../../../ftr_provider_context';
 
 export default function ({ loadTestFile }: FtrProviderContext) {
   describe('feature controls', function () {
-    this.tags(['ciGroup2']);
+    this.tags(['ciGroup8']);
 
     loadTestFile(require.resolve('./api_keys_security'));
   });

diff --git a/x-pack/test/functional/apps/cross_cluster_replication/feature_controls/index.ts b/x-pack/test/functional/apps/cross_cluster_replication/feature_controls/index.ts
@@ -8,7 +8,7 @@ import { FtrProviderContext } from '../../../ftr_provider_context';
 
 export default function ({ loadTestFile }: FtrProviderContext) {
   describe('feature controls', function () {
-    this.tags(['ciGroup2']);
+    this.tags(['ciGroup8']);
 
     loadTestFile(require.resolve('./ccr_security'));
   });

diff --git a/x-pack/test/functional/apps/dev_tools/index.ts b/x-pack/test/functional/apps/dev_tools/index.ts
@@ -7,7 +7,7 @@ import { FtrProviderContext } from '../../ftr_provider_context';
 
 export default function ({ loadTestFile }: FtrProviderContext) {
   describe('console', function () {
-    this.tags('ciGroup10');
+    this.tags('ciGroup13');
 
     loadTestFile(require.resolve('./feature_controls'));
     loadTestFile(require.resolve('./searchprofiler_editor'));

diff --git a/x-pack/test/functional/apps/graph/index.ts b/x-pack/test/functional/apps/graph/index.ts
@@ -8,7 +8,7 @@ import { FtrProviderContext } from '../../ftr_provider_context';
 
 export default function ({ loadTestFile }: FtrProviderContext) {
   describe('graph app', function () {
-    this.tags('ciGroup1');
+    this.tags('ciGroup12');
 
     loadTestFile(require.resolve('./feature_controls'));
     loadTestFile(require.resolve('./graph'));

diff --git a/x-pack/test/functional/apps/grok_debugger/index.js b/x-pack/test/functional/apps/grok_debugger/index.js
@@ -6,7 +6,7 @@
 
 export default function ({ loadTestFile }) {
   describe('logstash', function () {
-    this.tags('ciGroup2');
+    this.tags('ciGroup13');
 
     loadTestFile(require.resolve('./grok_debugger'));
   });

diff --git a/x-pack/test/functional/apps/index_management/index.ts b/x-pack/test/functional/apps/index_management/index.ts
@@ -8,7 +8,7 @@ import { FtrProviderContext } from '../../ftr_provider_context';
 
 export default ({ loadTestFile }: FtrProviderContext) => {
   describe('Index Management app', function () {
-    this.tags('ciGroup3');
+    this.tags('ciGroup13');
     loadTestFile(require.resolve('./feature_controls'));
     loadTestFile(require.resolve('./home_page'));
   });

diff --git a/x-pack/test/functional/apps/ingest_pipelines/index.ts b/x-pack/test/functional/apps/ingest_pipelines/index.ts
@@ -8,7 +8,7 @@ import { FtrProviderContext } from '../../ftr_provider_context';
 
 export default ({ loadTestFile }: FtrProviderContext) => {
   describe('Ingest pipelines app', function () {
-    this.tags('ciGroup3');
+    this.tags('ciGroup13');
     loadTestFile(require.resolve('./feature_controls'));
     loadTestFile(require.resolve('./ingest_pipelines'));
   });

diff --git a/x-pack/test/functional/apps/ml/index.ts b/x-pack/test/functional/apps/ml/index.ts
@@ -10,44 +10,81 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) {
   const ml = getService('ml');
 
   describe('machine learning', function () {
-    this.tags('ciGroup3');
+    describe('', function () {
+      this.tags('ciGroup3');
 
-    before(async () => {
-      await ml.securityCommon.createMlRoles();
-      await ml.securityCommon.createMlUsers();
-    });
+      before(async () => {
+        await ml.securityCommon.createMlRoles();
+        await ml.securityCommon.createMlUsers();
+      });
+
+      after(async () => {
+        await ml.securityCommon.cleanMlUsers();
+        await ml.securityCommon.cleanMlRoles();
+        await ml.testResources.deleteSavedSearches();
+        await ml.testResources.deleteDashboards();
+        await ml.testResources.deleteIndexPatternByTitle('ft_farequote');
+        await ml.testResources.deleteIndexPatternByTitle('ft_ecommerce');
+        await ml.testResources.deleteIndexPatternByTitle('ft_categorization');
+        await ml.testResources.deleteIndexPatternByTitle('ft_event_rate_gen_trend_nanos');
+        await ml.testResources.deleteIndexPatternByTitle('ft_bank_marketing');
+        await ml.testResources.deleteIndexPatternByTitle('ft_ihp_outlier');
+        await ml.testResources.deleteIndexPatternByTitle('ft_egs_regression');
+        await ml.testResources.deleteIndexPatternByTitle('ft_module_sample_ecommerce');
+        await esArchiver.unload('ml/farequote');
+        await esArchiver.unload('ml/ecommerce');
+        await esArchiver.unload('ml/categorization');
+        await esArchiver.unload('ml/event_rate_nanos');
+        await esArchiver.unload('ml/bm_classification');
+        await esArchiver.unload('ml/ihp_outlier');
+        await esArchiver.unload('ml/egs_regression');
+        await esArchiver.unload('ml/module_sample_ecommerce');
+        await ml.testResources.resetKibanaTimeZone();
+        await ml.securityUI.logout();
+      });
 
-    after(async () => {
-      await ml.securityCommon.cleanMlUsers();
-      await ml.securityCommon.cleanMlRoles();
-      await ml.testResources.deleteSavedSearches();
-      await ml.testResources.deleteDashboards();
-      await ml.testResources.deleteIndexPatternByTitle('ft_farequote');
-      await ml.testResources.deleteIndexPatternByTitle('ft_ecommerce');
-      await ml.testResources.deleteIndexPatternByTitle('ft_categorization');
-      await ml.testResources.deleteIndexPatternByTitle('ft_event_rate_gen_trend_nanos');
-      await ml.testResources.deleteIndexPatternByTitle('ft_bank_marketing');
-      await ml.testResources.deleteIndexPatternByTitle('ft_ihp_outlier');
-      await ml.testResources.deleteIndexPatternByTitle('ft_egs_regression');
-      await ml.testResources.deleteIndexPatternByTitle('ft_module_sample_ecommerce');
-      await esArchiver.unload('ml/farequote');
-      await esArchiver.unload('ml/ecommerce');
-      await esArchiver.unload('ml/categorization');
-      await esArchiver.unload('ml/event_rate_nanos');
-      await esArchiver.unload('ml/bm_classification');
-      await esArchiver.unload('ml/ihp_outlier');
-      await esArchiver.unload('ml/egs_regression');
-      await esArchiver.unload('ml/module_sample_ecommerce');
-      await ml.testResources.resetKibanaTimeZone();
-      await ml.securityUI.logout();
+      loadTestFile(require.resolve('./permissions'));
+      loadTestFile(require.resolve('./pages'));
+      loadTestFile(require.resolve('./anomaly_detection'));
+      loadTestFile(require.resolve('./data_visualizer'));
+      loadTestFile(require.resolve('./data_frame_analytics'));
     });
 
-    loadTestFile(require.resolve('./feature_controls'));
-    loadTestFile(require.resolve('./permissions'));
-    loadTestFile(require.resolve('./pages'));
-    loadTestFile(require.resolve('./anomaly_detection'));
-    loadTestFile(require.resolve('./data_visualizer'));
-    loadTestFile(require.resolve('./data_frame_analytics'));
-    loadTestFile(require.resolve('./settings'));
+    describe('', function () {
+      this.tags('ciGroup13');
+
+      before(async () => {
+        await ml.securityCommon.createMlRoles();
+        await ml.securityCommon.createMlUsers();
+      });
+
+      after(async () => {
+        await ml.securityCommon.cleanMlUsers();
+        await ml.securityCommon.cleanMlRoles();
+        await ml.testResources.deleteSavedSearches();
+        await ml.testResources.deleteDashboards();
+        await ml.testResources.deleteIndexPatternByTitle('ft_farequote');
+        await ml.testResources.deleteIndexPatternByTitle('ft_ecommerce');
+        await ml.testResources.deleteIndexPatternByTitle('ft_categorization');
+        await ml.testResources.deleteIndexPatternByTitle('ft_event_rate_gen_trend_nanos');
+        await ml.testResources.deleteIndexPatternByTitle('ft_bank_marketing');
+        await ml.testResources.deleteIndexPatternByTitle('ft_ihp_outlier');
+        await ml.testResources.deleteIndexPatternByTitle('ft_egs_regression');
+        await ml.testResources.deleteIndexPatternByTitle('ft_module_sample_ecommerce');
+        await esArchiver.unload('ml/farequote');
+        await esArchiver.unload('ml/ecommerce');
+        await esArchiver.unload('ml/categorization');
+        await esArchiver.unload('ml/event_rate_nanos');
+        await esArchiver.unload('ml/bm_classification');
+        await esArchiver.unload('ml/ihp_outlier');
+        await esArchiver.unload('ml/egs_regression');
+        await esArchiver.unload('ml/module_sample_ecommerce');
+        await ml.testResources.resetKibanaTimeZone();
+        await ml.securityUI.logout();
+      });
+
+      loadTestFile(require.resolve('./feature_controls'));
+      loadTestFile(require.resolve('./settings'));
+    });
   });
 }
diff --git a/x-pack/test/functional_cors/tests/index.ts b/x-pack/test/functional_cors/tests/index.ts
@@ -8,7 +8,7 @@ import { FtrProviderContext } from '../ftr_provider_context';
 
 export default function ({ loadTestFile }: FtrProviderContext) {
   describe('Kibana cors', function () {
-    this.tags('ciGroup2');
+    this.tags('ciGroup12');
     loadTestFile(require.resolve('./cors'));
   });
 }
diff --git a/x-pack/test/licensing_plugin/public/index.ts b/x-pack/test/licensing_plugin/public/index.ts
@@ -9,7 +9,7 @@ import { FtrProviderContext } from '../services';
 // eslint-disable-next-line import/no-default-export
 export default function ({ loadTestFile }: FtrProviderContext) {
   describe('Licensing plugin public client', function () {
-    this.tags('ciGroup2');
+    this.tags('ciGroup5');
     loadTestFile(require.resolve('./feature_usage'));
     // MUST BE LAST! CHANGES LICENSE TYPE!
     loadTestFile(require.resolve('./updates'));