Merge branch 'main' into api/serverlessQA

.buildkite/pipeline-resource-definitions/kibana-es-serverless-snapshots.yml

@@ -22,7 +22,7 @@ spec:
         SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
         ES_SERVERLESS_IMAGE: latest
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
-      allow_rebuilds: false
+      allow_rebuilds: true
       branch_configuration: main
       default_branch: main
       repository: elastic/kibana
@@ -35,7 +35,7 @@ spec:
         trigger_mode: none
         build_tags: false
         prefix_pull_request_fork_branch_names: false
-        skip_pull_request_builds_for_existing_commits: true
+        skip_pull_request_builds_for_existing_commits: false
       teams:
         everyone:
           access_level: BUILD_AND_READ

.buildkite/pipeline-resource-definitions/kibana-es-snapshots.yml

@@ -21,7 +21,7 @@ spec:
       env:
         SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
-      allow_rebuilds: false
+      allow_rebuilds: true
       branch_configuration: main 8.13 7.17
       default_branch: main
       repository: elastic/kibana
@@ -34,7 +34,7 @@ spec:
         trigger_mode: none
         build_tags: false
         prefix_pull_request_fork_branch_names: false
-        skip_pull_request_builds_for_existing_commits: true
+        skip_pull_request_builds_for_existing_commits: false
       teams:
         everyone:
           access_level: BUILD_AND_READ
@@ -81,7 +81,7 @@ spec:
       env:
         SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
-      allow_rebuilds: false
+      allow_rebuilds: true
       branch_configuration: main 8.13 7.17
       default_branch: main
       repository: elastic/kibana
@@ -94,7 +94,7 @@ spec:
         trigger_mode: none
         build_tags: false
         prefix_pull_request_fork_branch_names: false
-        skip_pull_request_builds_for_existing_commits: true
+        skip_pull_request_builds_for_existing_commits: false
       teams:
         everyone:
           access_level: BUILD_AND_READ
@@ -128,7 +128,7 @@ spec:
       env:
         SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
-      allow_rebuilds: false
+      allow_rebuilds: true
       branch_configuration: main 8.13 7.17
       default_branch: main
       repository: elastic/kibana
@@ -141,7 +141,7 @@ spec:
         trigger_mode: none
         build_tags: false
         prefix_pull_request_fork_branch_names: false
-        skip_pull_request_builds_for_existing_commits: true
+        skip_pull_request_builds_for_existing_commits: false
       teams:
         everyone:
           access_level: BUILD_AND_READ

.buildkite/pipeline-resource-definitions/kibana-flaky.yml

@@ -0,0 +1,41 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-flaky-test-suite-runner
+  description: ':warning: Trigger a new build here: https://ci-stats.kibana.dev/trigger_flaky_test_runner :warning:'
+  links:
+    - url: 'https://buildkite.com/elastic/kibana-flaky-test-suite-runner'
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  owner: 'group:kibana-operations'
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / flaky-test-suite-runner
+      description: ':warning: Trigger a new build here: https://ci-stats.kibana.dev/trigger_flaky_test_runner :warning:'
+    spec:
+      allow_rebuilds: true
+      default_branch: refs/pull/INSERT_PR_NUMBER/head
+      repository: elastic/kibana
+      pipeline_file: .buildkite/pipelines/flaky_tests/pipeline.sh
+      skip_intermediate_builds: false
+      provider_settings:
+        build_branches: true
+        build_pull_requests: false
+        publish_commit_status: false
+        trigger_mode: none
+        prefix_pull_request_fork_branch_names: false
+        skip_pull_request_builds_for_existing_commits: false
+      teams:
+        everyone:
+          access_level: BUILD_AND_READ
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        appex-qa:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-tech-leads:
+          access_level: MANAGE_BUILD_AND_READ

.buildkite/pipeline-resource-definitions/kibana-on-merge-unsupported-ftrs.yml

@@ -0,0 +1,46 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-on-merge-unsupported-ftrs
+  description: Runs unsupported ftr tests for each commit of Kibana
+  links:
+    - url: 'https://buildkite.com/elastic/kibana-on-merge-unsupported-ftrs'
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  owner: 'group:kibana-operations'
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / on merge unsupported ftrs
+      description: Runs unsupported ftr tests for each commit of Kibana
+    spec:
+      env:
+        SLACK_NOTIFICATIONS_CHANNEL: '#kibana-unsupported-ftrs-alerts'
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+      allow_rebuilds: true
+      branch_configuration: main 8.13 7.17
+      default_branch: main
+      repository: elastic/kibana
+      pipeline_file: .buildkite/pipelines/on_merge_unsupported_ftrs.yml
+      skip_intermediate_builds: false
+      provider_settings:
+        build_branches: false
+        build_pull_requests: false
+        publish_commit_status: false
+        trigger_mode: none
+        build_tags: false
+        prefix_pull_request_fork_branch_names: false
+        skip_pull_request_builds_for_existing_commits: false
+      teams:
+        everyone:
+          access_level: BUILD_AND_READ
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        appex-qa:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-tech-leads:
+          access_level: MANAGE_BUILD_AND_READ

.buildkite/pipeline-resource-definitions/locations.yml

@@ -1,3 +1,4 @@
+# * This file is auto-updated by running fix-location-collection.ts * #
 apiVersion: backstage.io/v1alpha1
 kind: Location
 metadata:
@@ -10,7 +11,9 @@ spec:
     - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-serverless-snapshots.yml'
     - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-snapshots.yml'
     - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-esql-grammar-sync.yml'
+    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-flaky.yml'
     - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-migration-staging.yml'
+    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-on-merge-unsupported-ftrs.yml'
     - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-purge-cloud-deployments.yml'
     - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-release.yml'
     - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-ess/security-solution-ess.yml'

.buildkite/pipelines/flaky_tests/pipeline.sh

@@ -5,4 +5,4 @@ set -euo pipefail
 UUID="$(cat /proc/sys/kernel/random/uuid)"
 export UUID
 
-ts-node .buildkite/pipelines/flaky_tests/pipeline.ts | buildkite-agent pipeline upload
+ts-node .buildkite/pipelines/flaky_tests/pipeline.ts

.buildkite/pipelines/flaky_tests/pipeline.ts

@@ -31,6 +31,34 @@ if (Number.isNaN(concurrency)) {
 const BASE_JOBS = 1;
 const MAX_JOBS = 500;
 
+// TODO: remove this after https://github.com/elastic/kibana-operations/issues/15 is finalized
+/** This function bridges the agent targeting between gobld and kibana-buildkite agent targeting */
+const getAgentRule = (queueName: string = 'n2-4-spot') => {
+  if (
+    process.env.BUILDKITE_AGENT_META_DATA_QUEUE === 'gobld' ||
+    process.env.BUILDKITE_AGENT_META_DATA_PROVIDER === 'k8s'
+  ) {
+    const [kind, cores, addition] = queueName.split('-');
+    const additionalProps =
+      {
+        spot: { preemptible: true },
+        virt: { localSsdInterface: 'nvme', enableNestedVirtualization: true, localSsds: 1 },
+      }[addition] || {};
+
+    return {
+      provider: 'gcp',
+      image: 'family/kibana-ubuntu-2004',
+      imageProject: 'elastic-images-qa',
+      machineType: `${kind}-standard-${cores}`,
+      ...additionalProps,
+    };
+  } else {
+    return {
+      queue: queueName,
+    };
+  }
+};
+
 function getTestSuitesFromJson(json: string) {
   const fail = (errorMsg: string) => {
     console.error('+++ Invalid test config provided');
@@ -49,8 +77,10 @@ function getTestSuitesFromJson(json: string) {
     fail(`JSON test config must be an array`);
   }
 
-  /** @type {Array<{ type: 'group', key: string; count: number } | { type: 'ftrConfig', ftrConfig: string; count: number }>} */
-  const testSuites = [];
+  const testSuites: Array<
+    | { type: 'group'; key: string; count: number }
+    | { type: 'ftrConfig'; ftrConfig: string; count: number }
+  > = [];
   for (const item of parsed) {
     if (typeof item !== 'object' || item === null) {
       fail(`testSuites must be objects`);
@@ -73,6 +103,7 @@ function getTestSuitesFromJson(json: string) {
       }
 
       testSuites.push({
+        type: 'ftrConfig',
         ftrConfig,
         count,
       });
@@ -84,6 +115,7 @@ function getTestSuitesFromJson(json: string) {
       fail(`testSuite.key must be a string`);
     }
     testSuites.push({
+      type: 'group',
       key,
       count,
     });
@@ -117,7 +149,7 @@ const pipeline = {
 steps.push({
   command: '.buildkite/scripts/steps/build_kibana.sh',
   label: 'Build Kibana Distribution and Plugins',
-  agents: { queue: 'c2-8' },
+  agents: getAgentRule('c2-8'),
   key: 'build',
   if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''",
 });
@@ -127,7 +159,7 @@ for (const testSuite of testSuites) {
     continue;
   }
 
-  if (testSuite.ftrConfig) {
+  if (testSuite.type === 'ftrConfig') {
     steps.push({
       command: `.buildkite/scripts/steps/test/ftr_configs.sh`,
       env: {
@@ -138,9 +170,7 @@ for (const testSuite of testSuites) {
       concurrency,
       concurrency_group: process.env.UUID,
       concurrency_method: 'eager',
-      agents: {
-        queue: 'n2-4-spot-2',
-      },
+      agents: getAgentRule('n2-4-spot'),
       depends_on: 'build',
       timeout_in_minutes: 150,
       cancel_on_build_failing: true,
@@ -164,7 +194,7 @@ for (const testSuite of testSuites) {
       steps.push({
         command: `.buildkite/scripts/steps/functional/${suiteName}.sh`,
         label: group.name,
-        agents: { queue: agentQueue },
+        agents: getAgentRule(agentQueue),
         depends_on: 'build',
         timeout_in_minutes: 150,
         parallelism: testSuite.count,

.buildkite/pipelines/on_merge_unsupported_ftrs.yml

@@ -3,7 +3,10 @@ steps:
     label: Pre-Build
     timeout_in_minutes: 10
     agents:
-      queue: kibana-default
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-2
     retry:
       automatic:
         - exit_status: '*'
@@ -12,7 +15,11 @@ steps:
   - command: .buildkite/scripts/steps/build_kibana.sh
     label: Build Kibana Distribution and Plugins
     agents:
-      queue: n2-16-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-16
+      preemptible: true
     key: build
     if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
     timeout_in_minutes: 60
@@ -24,7 +31,11 @@ steps:
   - command: .buildkite/scripts/steps/functional/apm_cypress.sh
     label: 'APM Cypress Tests'
     agents:
-      queue: n2-4-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -37,7 +48,11 @@ steps:
   - command: .buildkite/scripts/steps/functional/profiling_cypress.sh
     label: 'Profiling Cypress Tests'
     agents:
-      queue: n2-4-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -50,7 +65,11 @@ steps:
   - command: .buildkite/scripts/steps/functional/synthetics_plugin.sh
     label: 'Synthetics @elastic/synthetics Tests'
     agents:
-      queue: n2-4-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
     depends_on: build
     timeout_in_minutes: 120
     artifact_paths:
@@ -65,7 +84,13 @@ steps:
   - command: .buildkite/scripts/steps/functional/defend_workflows.sh
     label: 'Defend Workflows Cypress Tests'
     agents:
-      queue: n2-4-virt
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      enableNestedVirtualization: true
+      localSsdInterface: nvme
+      localSsds: 1
+      machineType: n2-standard-4
     depends_on: build
     timeout_in_minutes: 60
     parallelism: 20
@@ -77,7 +102,13 @@ steps:
   - command: .buildkite/scripts/steps/functional/defend_workflows_serverless.sh
     label: 'Defend Workflows Cypress Tests on Serverless'
     agents:
-      queue: n2-4-virt
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      enableNestedVirtualization: true
+      localSsdInterface: nvme
+      localSsds: 1
+      machineType: n2-standard-4
     depends_on: build
     timeout_in_minutes: 60
     parallelism: 14

.buildkite/pipelines/upload_pipeline.yml

@@ -2,4 +2,10 @@
 
 steps:
   - label: Upload tested pipeline
-    command: buildkite-agent pipeline upload ${TESTED_PIPELINE_PATH:-.buildkite/pipelines/pipeline_to_test.yml}
+    command: |
+      if [[ $TESTED_PIPELINE_PATH == *.yml ]]; then
+        echo "Uploading pipeline $TESTED_PIPELINE_PATH"
+        buildkite-agent pipeline upload $TESTED_PIPELINE_PATH
+      else
+        $TESTED_PIPELINE_PATH | buildkite-agent pipeline upload
+      fi