[ftr] automatically determine config run order

.buildkite/ftr_configs.yml

@@ -0,0 +1,236 @@
+disabled:
+  # TODO: Enable once RBAC timeline search strategy test updated
+  - x-pack/test/timeline/security_and_spaces/config_basic.ts
+
+  # Base config files, only necessary to inform config finding script
+  - test/functional/config.base.js
+  - x-pack/test/functional/config.base.js
+  - x-pack/test/detection_engine_api_integration/security_and_spaces/config.base.ts
+  - x-pack/test/functional_enterprise_search/base_config.ts
+  - test/server_integration/config.base.js
+
+  # QA suites that are run out-of-band
+  - x-pack/test/stack_functional_integration/configs/config.stack_functional_integration_base.js
+  - x-pack/test/upgrade/config.ts
+  - test/functional/config.edge.js
+  - x-pack/test/functional/config.edge.js
+
+  # Cypress configs, for now these are still run manually
+  - x-pack/test/fleet_cypress/cli_config.ts
+  - x-pack/test/fleet_cypress/config.ts
+  - x-pack/test/fleet_cypress/visual_config.ts
+  - x-pack/test/functional_enterprise_search/cypress.config.ts
+  - x-pack/test/osquery_cypress/cli_config.ts
+  - x-pack/test/osquery_cypress/config.ts
+  - x-pack/test/osquery_cypress/visual_config.ts
+  - x-pack/test/security_solution_cypress/cases_cli_config.ts
+  - x-pack/test/security_solution_cypress/ccs_config.ts
+  - x-pack/test/security_solution_cypress/cli_config.ts
+  - x-pack/test/security_solution_cypress/config.firefox.ts
+  - x-pack/test/security_solution_cypress/config.ts
+  - x-pack/test/security_solution_cypress/response_ops_cli_config.ts
+  - x-pack/test/security_solution_cypress/upgrade_config.ts
+  - x-pack/test/security_solution_cypress/visual_config.ts
+  - x-pack/test/functional_enterprise_search/with_host_configured.config.ts
+  - x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts
+  - x-pack/plugins/apm/ftr_e2e/ftr_config.ts
+  - x-pack/plugins/synthetics/e2e/config.ts
+
+  # Configs that exist but weren't running in CI when this file was introduced
+  - test/visual_regression/config.ts
+  - x-pack/test/visual_regression/config.ts
+  - x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/config.ts
+  - x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/telemetry/config.ts
+  - x-pack/test/alerting_api_integration/spaces_only_legacy/config.ts
+  - x-pack/test/banners_functional/config.ts
+  - x-pack/test/cloud_integration/config.ts
+  - x-pack/test/performance/config.playwright.ts
+  - x-pack/test/load/config.ts
+  - x-pack/test/plugin_api_perf/config.js
+  - x-pack/test/screenshot_creation/config.ts
+
+enabled:
+  - test/accessibility/config.ts
+  - test/analytics/config.ts
+  - test/api_integration/config.js
+  - test/examples/config.js
+  - test/functional/apps/bundles/config.ts
+  - test/functional/apps/console/config.ts
+  - test/functional/apps/context/config.ts
+  - test/functional/apps/dashboard_elements/config.ts
+  - test/functional/apps/dashboard/group1/config.ts
+  - test/functional/apps/dashboard/group2/config.ts
+  - test/functional/apps/dashboard/group3/config.ts
+  - test/functional/apps/dashboard/group4/config.ts
+  - test/functional/apps/dashboard/group5/config.ts
+  - test/functional/apps/discover/config.ts
+  - test/functional/apps/getting_started/config.ts
+  - test/functional/apps/home/config.ts
+  - test/functional/apps/management/config.ts
+  - test/functional/apps/saved_objects_management/config.ts
+  - test/functional/apps/status_page/config.ts
+  - test/functional/apps/visualize/group1/config.ts
+  - test/functional/apps/visualize/group2/config.ts
+  - test/functional/apps/visualize/group3/config.ts
+  - test/functional/apps/visualize/group4/config.ts
+  - test/functional/apps/visualize/group5/config.ts
+  - test/functional/apps/visualize/group6/config.ts
+  - test/functional/apps/visualize/replaced_vislib_chart_types/config.ts
+  - test/functional/config.ccs.ts
+  - test/functional/config.firefox.js
+  - test/interactive_setup_api_integration/enrollment_flow.config.ts
+  - test/interactive_setup_api_integration/manual_configuration_flow_without_tls.config.ts
+  - test/interactive_setup_api_integration/manual_configuration_flow.config.ts
+  - test/interactive_setup_functional/enrollment_token.config.ts
+  - test/interactive_setup_functional/manual_configuration_without_security.config.ts
+  - test/interactive_setup_functional/manual_configuration_without_tls.config.ts
+  - test/interactive_setup_functional/manual_configuration.config.ts
+  - test/interpreter_functional/config.ts
+  - test/new_visualize_flow/config.ts
+  - test/plugin_functional/config.ts
+  - test/server_integration/http/platform/config.status.ts
+  - test/server_integration/http/platform/config.ts
+  - test/server_integration/http/ssl_redirect/config.js
+  - test/server_integration/http/ssl_with_p12_intermediate/config.js
+  - test/server_integration/http/ssl_with_p12/config.js
+  - test/server_integration/http/ssl/config.js
+  - test/ui_capabilities/newsfeed_err/config.ts
+  - x-pack/test/accessibility/config.ts
+  - x-pack/test/alerting_api_integration/basic/config.ts
+  - x-pack/test/alerting_api_integration/security_and_spaces/group1/config.ts
+  - x-pack/test/alerting_api_integration/security_and_spaces/group2/config.ts
+  - x-pack/test/alerting_api_integration/spaces_only/config.ts
+  - x-pack/test/api_integration_basic/config.ts
+  - x-pack/test/api_integration/config_security_basic.ts
+  - x-pack/test/api_integration/config_security_trial.ts
+  - x-pack/test/api_integration/config.ts
+  - x-pack/test/apm_api_integration/basic/config.ts
+  - x-pack/test/apm_api_integration/rules/config.ts
+  - x-pack/test/apm_api_integration/trial/config.ts
+  - x-pack/test/cases_api_integration/security_and_spaces/config_basic.ts
+  - x-pack/test/cases_api_integration/security_and_spaces/config_trial.ts
+  - x-pack/test/cases_api_integration/spaces_only/config.ts
+  - x-pack/test/detection_engine_api_integration/basic/config.ts
+  - x-pack/test/detection_engine_api_integration/security_and_spaces/group1/config.ts
+  - x-pack/test/detection_engine_api_integration/security_and_spaces/group2/config.ts
+  - x-pack/test/detection_engine_api_integration/security_and_spaces/group3/config.ts
+  - x-pack/test/detection_engine_api_integration/security_and_spaces/group4/config.ts
+  - x-pack/test/detection_engine_api_integration/security_and_spaces/group5/config.ts
+  - x-pack/test/detection_engine_api_integration/security_and_spaces/group6/config.ts
+  - x-pack/test/detection_engine_api_integration/security_and_spaces/group7/config.ts
+  - x-pack/test/detection_engine_api_integration/security_and_spaces/group8/config.ts
+  - x-pack/test/detection_engine_api_integration/security_and_spaces/group9/config.ts
+  - x-pack/test/encrypted_saved_objects_api_integration/config.ts
+  - x-pack/test/endpoint_api_integration_no_ingest/config.ts
+  - x-pack/test/examples/config.ts
+  - x-pack/test/fleet_api_integration/config.ts
+  - x-pack/test/fleet_functional/config.ts
+  - x-pack/test/functional_basic/config.ts
+  - x-pack/test/functional_cors/config.ts
+  - x-pack/test/functional_embedded/config.ts
+  - x-pack/test/functional_enterprise_search/without_host_configured.config.ts
+  - x-pack/test/functional_execution_context/config.ts
+  - x-pack/test/functional_synthetics/config.js
+  - x-pack/test/functional_with_es_ssl/config.ts
+  - x-pack/test/functional/apps/advanced_settings/config.ts
+  - x-pack/test/functional/apps/api_keys/config.ts
+  - x-pack/test/functional/apps/apm/config.ts
+  - x-pack/test/functional/apps/canvas/config.ts
+  - x-pack/test/functional/apps/cross_cluster_replication/config.ts
+  - x-pack/test/functional/apps/dashboard/group1/config.ts
+  - x-pack/test/functional/apps/dashboard/group2/config.ts
+  - x-pack/test/functional/apps/data_views/config.ts
+  - x-pack/test/functional/apps/dev_tools/config.ts
+  - x-pack/test/functional/apps/discover/config.ts
+  - x-pack/test/functional/apps/graph/config.ts
+  - x-pack/test/functional/apps/grok_debugger/config.ts
+  - x-pack/test/functional/apps/home/config.ts
+  - x-pack/test/functional/apps/index_lifecycle_management/config.ts
+  - x-pack/test/functional/apps/index_management/config.ts
+  - x-pack/test/functional/apps/infra/config.ts
+  - x-pack/test/functional/apps/ingest_pipelines/config.ts
+  - x-pack/test/functional/apps/lens/group1/config.ts
+  - x-pack/test/functional/apps/lens/group2/config.ts
+  - x-pack/test/functional/apps/lens/group3/config.ts
+  - x-pack/test/functional/apps/license_management/config.ts
+  - x-pack/test/functional/apps/logstash/config.ts
+  - x-pack/test/functional/apps/management/config.ts
+  - x-pack/test/functional/apps/maps/group1/config.ts
+  - x-pack/test/functional/apps/maps/group2/config.ts
+  - x-pack/test/functional/apps/maps/group3/config.ts
+  - x-pack/test/functional/apps/maps/group4/config.ts
+  - x-pack/test/functional/apps/ml/data_visualizer/config.ts
+  - x-pack/test/functional/apps/ml/group1/config.ts
+  - x-pack/test/functional/apps/ml/group2/config.ts
+  - x-pack/test/functional/apps/ml/group3/config.ts
+  - x-pack/test/functional/apps/monitoring/config.ts
+  - x-pack/test/functional/apps/remote_clusters/config.ts
+  - x-pack/test/functional/apps/reporting_management/config.ts
+  - x-pack/test/functional/apps/rollup_job/config.ts
+  - x-pack/test/functional/apps/saved_objects_management/config.ts
+  - x-pack/test/functional/apps/security/config.ts
+  - x-pack/test/functional/apps/snapshot_restore/config.ts
+  - x-pack/test/functional/apps/spaces/config.ts
+  - x-pack/test/functional/apps/status_page/config.ts
+  - x-pack/test/functional/apps/transform/config.ts
+  - x-pack/test/functional/apps/upgrade_assistant/config.ts
+  - x-pack/test/functional/apps/uptime/config.ts
+  - x-pack/test/functional/apps/visualize/config.ts
+  - x-pack/test/functional/apps/watcher/config.ts
+  - x-pack/test/functional/config_security_basic.ts
+  - x-pack/test/functional/config.ccs.ts
+  - x-pack/test/functional/config.firefox.js
+  - x-pack/test/licensing_plugin/config.public.ts
+  - x-pack/test/licensing_plugin/config.ts
+  - x-pack/test/lists_api_integration/security_and_spaces/config.ts
+  - x-pack/test/observability_api_integration/basic/config.ts
+  - x-pack/test/observability_api_integration/trial/config.ts
+  - x-pack/test/observability_functional/with_rac_write.config.ts
+  - x-pack/test/plugin_api_integration/config.ts
+  - x-pack/test/plugin_functional/config.ts
+  - x-pack/test/reporting_api_integration/reporting_and_security.config.ts
+  - x-pack/test/reporting_api_integration/reporting_without_security.config.ts
+  - x-pack/test/reporting_functional/reporting_and_deprecated_security.config.ts
+  - x-pack/test/reporting_functional/reporting_and_security.config.ts
+  - x-pack/test/reporting_functional/reporting_without_security.config.ts
+  - x-pack/test/rule_registry/security_and_spaces/config_basic.ts
+  - x-pack/test/rule_registry/security_and_spaces/config_trial.ts
+  - x-pack/test/rule_registry/spaces_only/config_basic.ts
+  - x-pack/test/rule_registry/spaces_only/config_trial.ts
+  - x-pack/test/saved_object_api_integration/security_and_spaces/config_basic.ts
+  - x-pack/test/saved_object_api_integration/security_and_spaces/config_trial.ts
+  - x-pack/test/saved_object_api_integration/spaces_only/config.ts
+  - x-pack/test/saved_object_tagging/api_integration/security_and_spaces/config.ts
+  - x-pack/test/saved_object_tagging/api_integration/tagging_api/config.ts
+  - x-pack/test/saved_object_tagging/functional/config.ts
+  - x-pack/test/saved_objects_field_count/config.ts
+  - x-pack/test/search_sessions_integration/config.ts
+  - x-pack/test/security_api_integration/anonymous_es_anonymous.config.ts
+  - x-pack/test/security_api_integration/anonymous.config.ts
+  - x-pack/test/security_api_integration/audit.config.ts
+  - x-pack/test/security_api_integration/http_bearer.config.ts
+  - x-pack/test/security_api_integration/http_no_auth_providers.config.ts
+  - x-pack/test/security_api_integration/kerberos_anonymous_access.config.ts
+  - x-pack/test/security_api_integration/kerberos.config.ts
+  - x-pack/test/security_api_integration/login_selector.config.ts
+  - x-pack/test/security_api_integration/oidc_implicit_flow.config.ts
+  - x-pack/test/security_api_integration/oidc.config.ts
+  - x-pack/test/security_api_integration/pki.config.ts
+  - x-pack/test/security_api_integration/saml.config.ts
+  - x-pack/test/security_api_integration/session_idle.config.ts
+  - x-pack/test/security_api_integration/session_invalidate.config.ts
+  - x-pack/test/security_api_integration/session_lifespan.config.ts
+  - x-pack/test/security_api_integration/token.config.ts
+  - x-pack/test/security_functional/login_selector.config.ts
+  - x-pack/test/security_functional/oidc.config.ts
+  - x-pack/test/security_functional/saml.config.ts
+  - x-pack/test/security_solution_endpoint_api_int/config.ts
+  - x-pack/test/security_solution_endpoint/config.ts
+  - x-pack/test/spaces_api_integration/security_and_spaces/config_basic.ts
+  - x-pack/test/spaces_api_integration/security_and_spaces/config_trial.ts
+  - x-pack/test/spaces_api_integration/spaces_only/config.ts
+  - x-pack/test/timeline/security_and_spaces/config_trial.ts
+  - x-pack/test/ui_capabilities/security_and_spaces/config.ts
+  - x-pack/test/ui_capabilities/spaces_only/config.ts
+  - x-pack/test/upgrade_assistant_integration/config.js
+  - x-pack/test/usage_collection/config.ts

.buildkite/package.json

@@ -3,6 +3,6 @@
   "version": "1.0.0",
   "private": true,
   "dependencies": {
-    "kibana-buildkite-library": "git+https://[email protected]/elastic/kibana-buildkite-library#0f95579ace8100de9f1ad4cc16976b9ec6d5841e"
+    "kibana-buildkite-library": "git+https://[email protected]/elastic/kibana-buildkite-library#f5381bea52e0a71f50a6919cb6357ff3262cf2d6"
   }
 }

.buildkite/pipelines/es_snapshots/verify.yml

@@ -27,50 +27,17 @@ steps:
     if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
     timeout_in_minutes: 60
 
-  - command: .buildkite/scripts/steps/functional/xpack_cigroup.sh
-    label: 'Default CI Group'
-    parallelism: 31
-    agents:
-      queue: n2-4
-    depends_on: build
-    timeout_in_minutes: 150
-    key: default-cigroup
-    retry:
-      automatic:
-        - exit_status: '*'
-          limit: 1
-
-  - command: .buildkite/scripts/steps/functional/oss_cigroup.sh
-    label: 'OSS CI Group'
-    parallelism: 12
-    agents:
-      queue: ci-group-4d
-    depends_on: build
-    timeout_in_minutes: 120
-    key: oss-cigroup
-    retry:
-      automatic:
-        - exit_status: '*'
-          limit: 1
-
-  - command: .buildkite/scripts/steps/test/pick_jest_config_run_order.sh
-    label: 'Pick Jest Config Run Order'
+  - command: .buildkite/scripts/steps/test/pick_test_group_run_order.sh
+    label: 'Pick Test Group Run Order'
     agents:
       queue: kibana-default
     env:
-      FILTER_JEST_CONFIG_TYPE: integration
+      LIMIT_CONFIG_TYPE: integration,functional
     retry:
       automatic:
         - exit_status: '*'
           limit: 1
 
-  - command: .buildkite/scripts/steps/test/api_integration.sh
-    label: 'API Integration Tests'
-    agents:
-      queue: n2-2
-    timeout_in_minutes: 120
-    key: api-integration
-
   - command: .buildkite/scripts/steps/es_snapshots/trigger_promote.sh
     label: Trigger promotion
     timeout_in_minutes: 10

.buildkite/pipelines/flaky_tests/groups.json

@@ -1,46 +1,20 @@
 {
   "groups": [
     {
-      "key": "oss/cigroup",
-      "name": "OSS CI Group",
-      "ciGroups": 12
-    },
-    {
-      "key": "oss/firefox",
-      "name": "OSS Firefox"
-    },
-    {
-      "key": "oss/accessibility",
-      "name": "OSS Accessibility"
-    },
-    {
-      "key": "xpack/cypress/security_solution",
+      "key": "cypress/security_solution",
       "name": "Security Solution - Cypress"
     },
     {
-      "key": "xpack/cypress/osquery_cypress",
+      "key": "cypress/osquery_cypress",
       "name": "Osquery - Cypress"
     },
     {
-      "key": "xpack/cypress/fleet_cypress",
+      "key": "cypress/fleet_cypress",
       "name": "Fleet - Cypress"
     },
     {
-      "key": "xpack/cypress/apm_cypress",
+      "key": "cypress/apm_cypress",
       "name": "APM - Cypress"
-    },
-    {
-      "key": "xpack/cigroup",
-      "name": "Default CI Group",
-      "ciGroups": 30
-    },
-    {
-      "key": "xpack/firefox",
-      "name": "Default Firefox"
-    },
-    {
-      "key": "xpack/accessibility",
-      "name": "Default Accessibility"
     }
   ]
 }