[ci / FIPS] Dynamic agent selection. Add FIPS agents (elastic#183777)

## Summary - Closes elastic/kibana-operations#100 - Utilizes FIPS agent from elastic/ci-agent-images#686 - Adds dynamic agent selection during PR pipeline upload - FIPS agents can be used with `FTR_ENABLE_FIPS_AGENT` env variable or `ci:enable-fips-agent` label - Removes agent image config from individual steps in favor of image config for the whole pipeline. - Steps can still override this config by adding `image`, `imageProject` etc - Adds a conditional assertion to `Check` CI step which validates that FIPS is working properly ### Testing - [Pipeline run using FIPS agents](https://buildkite.com/elastic/kibana-pull-request/builds/215332) - Failures are expected and this possibly ran with flaky tests
bhapas · Jun 24, 2024 · fc82d29 · fc82d29
1 parent 1e5835d
commit fc82d29
Show file tree

Hide file tree

Showing 40 changed files with 108 additions and 179 deletions.
diff --git a/.buildkite/pipeline-utils/agent_images.ts b/.buildkite/pipeline-utils/agent_images.ts
@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { dump } from 'js-yaml';
+import { BuildkiteClient, BuildkiteCommandStep } from './buildkite';
+
+type AgentImageConfig = BuildkiteCommandStep['agents'];
+
+const DEFAULT_AGENT_IMAGE_CONFIG: AgentImageConfig = {
+  provider: 'gcp',
+  image: 'family/kibana-ubuntu-2004',
+  imageProject: 'elastic-images-prod',
+};
+
+const FIPS_AGENT_IMAGE_CONFIG: AgentImageConfig = {
+  provider: 'gcp',
+  image: 'family/kibana-fips-ubuntu-2004',
+  imageProject: 'elastic-images-qa',
+};
+
+const GITHUB_PR_LABELS = process.env.GITHUB_PR_LABELS ?? '';
+const FTR_ENABLE_FIPS_AGENT = process.env.FTR_ENABLE_FIPS_AGENT?.toLowerCase() === 'true';
+
+// Narrow the return type with overloads
+function getAgentImageConfig(): AgentImageConfig;
+function getAgentImageConfig(options: { returnYaml: true }): string;
+function getAgentImageConfig({ returnYaml = false } = {}): string | AgentImageConfig {
+  const bk = new BuildkiteClient();
+  let config: AgentImageConfig;
+
+  if (FTR_ENABLE_FIPS_AGENT || GITHUB_PR_LABELS.includes('ci:enable-fips-agent')) {
+    config = FIPS_AGENT_IMAGE_CONFIG;
+
+    bk.setAnnotation(
+      'agent image config',
+      'info',
+      '#### FIPS Agents Enabled<br />\nFIPS mode can produce new test failures. If you did not intend this remove ```KBN_ENABLE_FIPS``` environment variable and/or the ```ci:enable-fips-agent``` Github label.'
+    );
+  } else {
+    config = DEFAULT_AGENT_IMAGE_CONFIG;
+  }
+
+  if (returnYaml) {
+    return dump({ agents: config });
+  }
+
+  return config;
+}
+
+export { getAgentImageConfig };
diff --git a/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts b/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts
@@ -16,6 +16,7 @@ import { BuildkiteClient, BuildkiteStep } from '../buildkite';
 import { CiStatsClient, TestGroupRunOrderResponse } from './client';
 
 import DISABLED_JEST_CONFIGS from '../../disabled_jest_configs.json';
+import { getAgentImageConfig } from '#pipeline-utils';
 
 type RunGroup = TestGroupRunOrderResponse['types'][0];
 
@@ -25,9 +26,7 @@ const getAgentRule = (queueName: string = 'n2-4-spot') => {
   if (process.env?.BUILDKITE_AGENT_META_DATA_QUEUE === 'gobld') {
     const [kind, cores, spot] = queueName.split('-');
     return {
-      provider: 'gcp',
-      image: 'family/kibana-ubuntu-2004',
-      imageProject: 'elastic-images-prod',
+      ...getAgentImageConfig(),
       machineType: `${kind}-standard-${cores}`,
       preemptible: spot === 'spot',
     };

diff --git a/.buildkite/pipeline-utils/index.ts b/.buildkite/pipeline-utils/index.ts
@@ -6,6 +6,7 @@
  * Side Public License, v 1.
  */
 
+export * from './agent_images';
 export * from './buildkite';
 export * as CiStats from './ci-stats';
 export * from './github';

diff --git a/.buildkite/pipelines/pull_request/apm_cypress.yml b/.buildkite/pipelines/pull_request/apm_cypress.yml
@@ -2,9 +2,6 @@ steps:
   - command: .buildkite/scripts/steps/functional/apm_cypress.sh
     label: 'APM Cypress Tests'
     agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
       machineType: n2-standard-4
       preemptible: true
     depends_on:

diff --git a/.buildkite/pipelines/pull_request/base.yml b/.buildkite/pipelines/pull_request/base.yml
@@ -3,19 +3,13 @@ steps:
     label: Pre-Build
     timeout_in_minutes: 10
     agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
       machineType: n2-standard-2
 
   - wait
 
   - command: .buildkite/scripts/steps/build_kibana.sh
     label: Build Kibana Distribution and Plugins
     agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
       machineType: n2-standard-16
       preemptible: true
     key: build
@@ -29,9 +23,6 @@ steps:
   - command: .buildkite/scripts/steps/quick_checks.sh
     label: 'Quick Checks'
     agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
       machineType: n2-standard-2
       preemptible: true
     key: quick_checks
@@ -46,9 +37,6 @@ steps:
   - command: .buildkite/scripts/steps/ci_stats_ready.sh
     label: Mark CI Stats as ready
     agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
       machineType: n2-standard-2
     timeout_in_minutes: 10
     depends_on:
@@ -62,9 +50,6 @@ steps:
   - command: .buildkite/scripts/steps/test/pick_test_group_run_order.sh
     label: 'Pick Test Group Run Order'
     agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
       machineType: n2-standard-2
     timeout_in_minutes: 10
     env:
@@ -79,9 +64,6 @@ steps:
   - command: .buildkite/scripts/steps/lint.sh
     label: 'Linting'
     agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
       machineType: n2-standard-8
       preemptible: true
     key: linting
@@ -94,9 +76,6 @@ steps:
   - command: .buildkite/scripts/steps/check_types.sh
     label: 'Check Types'
     agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
       machineType: n2-standard-4
       preemptible: true
     key: check_types
@@ -109,9 +88,6 @@ steps:
   - command: .buildkite/scripts/steps/lint_with_types.sh
     label: 'Linting (with types)'
     agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
       machineType: n2-standard-16
       preemptible: true
     key: linting_with_types
@@ -125,9 +101,6 @@ steps:
     label: 'Checks'
     key: checks
     agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
       machineType: n2-standard-2
       preemptible: true
     timeout_in_minutes: 60
@@ -139,9 +112,6 @@ steps:
   - command: .buildkite/scripts/steps/api_docs/build_api_docs.sh
     label: 'Build API Docs'
     agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
       machineType: n2-standard-4
       preemptible: true
     key: build_api_docs

diff --git a/.buildkite/pipelines/pull_request/build_project.yml b/.buildkite/pipelines/pull_request/build_project.yml
@@ -2,9 +2,6 @@ steps:
   - command: .buildkite/scripts/steps/artifacts/docker_image.sh
     label: 'Build Project Image'
     agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
       machineType: n2-standard-16
       preemptible: true
     timeout_in_minutes: 60

diff --git a/.buildkite/pipelines/pull_request/check_next_docs.yml b/.buildkite/pipelines/pull_request/check_next_docs.yml
@@ -2,9 +2,6 @@ steps:
   - command: .buildkite/scripts/steps/next_docs/build_and_validate_docs.sh
     label: 'Build and Validate Next Docs'
     agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
       machineType: n2-standard-4
       preemptible: true
     timeout_in_minutes: 30

diff --git a/.buildkite/pipelines/pull_request/deploy_cloud.yml b/.buildkite/pipelines/pull_request/deploy_cloud.yml
@@ -2,9 +2,6 @@ steps:
   - command: .buildkite/scripts/steps/cloud/build_and_deploy.sh
     label: 'Build and Deploy to Cloud'
     agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
       machineType: n2-standard-2
       preemptible: true
     depends_on:

diff --git a/.buildkite/pipelines/pull_request/deploy_project.yml b/.buildkite/pipelines/pull_request/deploy_project.yml
@@ -3,9 +3,6 @@ steps:
     label: 'Build Project Image'
     key: build_project_image
     agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
       machineType: n2-standard-16
       preemptible: true
     timeout_in_minutes: 60
@@ -16,9 +13,6 @@ steps:
   - command: .buildkite/scripts/steps/serverless/deploy.sh
     label: 'Deploy Project'
     agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
       machineType: n2-standard-4
       preemptible: true
     timeout_in_minutes: 10

diff --git a/.buildkite/pipelines/pull_request/exploratory_view_plugin.yml b/.buildkite/pipelines/pull_request/exploratory_view_plugin.yml
@@ -2,9 +2,6 @@ steps:
   - command: .buildkite/scripts/steps/functional/exploratory_view_plugin.sh
     label: 'Exploratory View @elastic/synthetics Tests'
     agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
       machineType: n2-standard-4
       preemptible: true
     depends_on:

diff --git a/.buildkite/pipelines/pull_request/fips.yml b/.buildkite/pipelines/pull_request/fips.yml
@@ -2,9 +2,6 @@ steps:
   - command: .buildkite/scripts/steps/fips/build.sh
     label: 'Build FIPS Image'
     agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
       machineType: n2-standard-2
       preemptible: true
     depends_on:

diff --git a/.buildkite/pipelines/pull_request/fleet_cypress.yml b/.buildkite/pipelines/pull_request/fleet_cypress.yml
@@ -2,9 +2,6 @@ steps:
   - command: .buildkite/scripts/steps/functional/fleet_cypress.sh
     label: 'Fleet Cypress Tests'
     agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
       machineType: n2-standard-4
       preemptible: true
     depends_on:

diff --git a/.buildkite/pipelines/pull_request/kbn_handlebars.yml b/.buildkite/pipelines/pull_request/kbn_handlebars.yml
@@ -2,9 +2,6 @@ steps:
   - command: .buildkite/scripts/steps/test/kbn_handlebars.sh
     label: 'Check @kbn/handlebars for upstream differences'
     agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
       machineType: n2-standard-2
       preemptible: true
     depends_on:

diff --git a/.buildkite/pipelines/pull_request/observability_onboarding_cypress.yml b/.buildkite/pipelines/pull_request/observability_onboarding_cypress.yml
@@ -2,9 +2,6 @@ steps:
   - command: .buildkite/scripts/steps/functional/observability_onboarding_cypress.sh
     label: 'Observability onboarding Cypress Tests'
     agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
       machineType: n2-standard-4
       preemptible: true
     depends_on:

diff --git a/.buildkite/pipelines/pull_request/post_build.yml b/.buildkite/pipelines/pull_request/post_build.yml
@@ -5,7 +5,4 @@ steps:
   - command: .buildkite/scripts/lifecycle/post_build.sh
     label: Post-Build
     agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
       machineType: n2-standard-2
diff --git a/.buildkite/pipelines/pull_request/profiling_cypress.yml b/.buildkite/pipelines/pull_request/profiling_cypress.yml
@@ -2,9 +2,6 @@ steps:
   - command: .buildkite/scripts/steps/functional/profiling_cypress.sh
     label: 'Profiling Cypress Tests'
     agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
       machineType: n2-standard-4
       preemptible: true
     depends_on:

diff --git a/.buildkite/pipelines/pull_request/response_ops.yml b/.buildkite/pipelines/pull_request/response_ops.yml
@@ -2,9 +2,6 @@ steps:
   - command: .buildkite/scripts/steps/functional/response_ops.sh
     label: 'Rules, Alerts and Exceptions ResponseOps Cypress Tests on Security Solution'
     agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
       machineType: n2-standard-4
       preemptible: true
     depends_on:

diff --git a/.buildkite/pipelines/pull_request/response_ops_cases.yml b/.buildkite/pipelines/pull_request/response_ops_cases.yml
@@ -2,9 +2,6 @@ steps:
   - command: .buildkite/scripts/steps/functional/response_ops_cases.sh
     label: 'Cases Cypress Tests on Security Solution'
     agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
       machineType: n2-standard-4
       preemptible: true
     depends_on:

diff --git a/.buildkite/pipelines/pull_request/security_solution/ai_assistant.yml b/.buildkite/pipelines/pull_request/security_solution/ai_assistant.yml
@@ -2,9 +2,6 @@ steps:
   - command: .buildkite/scripts/steps/functional/security_serverless_ai_assistant.sh
     label: 'Serverless AI Assistant - Security Solution Cypress Tests'
     agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
       machineType: n2-standard-4
       preemptible: true
     depends_on:
@@ -20,9 +17,6 @@ steps:
   - command: .buildkite/scripts/steps/functional/security_solution_ai_assistant.sh
     label: 'AI Assistant - Security Solution Cypress Tests'
     agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
       machineType: n2-standard-4
       preemptible: true
     depends_on:

diff --git a/.buildkite/pipelines/pull_request/security_solution/cypress_burn.yml b/.buildkite/pipelines/pull_request/security_solution/cypress_burn.yml
@@ -2,9 +2,6 @@ steps:
   - command: .buildkite/scripts/steps/functional/defend_workflows_burn.sh
     label: '[Soft fail] Defend Workflows Cypress Tests, burning changed specs'
     agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
       enableNestedVirtualization: true
       localSsds: 1
       localSsdInterface: nvme
@@ -21,9 +18,6 @@ steps:
   - command: .buildkite/scripts/steps/functional/defend_workflows_serverless_burn.sh
     label: '[Soft fail] Defend Workflows Cypress Tests on Serverless, burning changed specs'
     agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
       enableNestedVirtualization: true
       localSsds: 1
       localSsdInterface: nvme
@@ -40,9 +34,6 @@ steps:
   - command: .buildkite/scripts/steps/functional/security_solution_burn.sh
     label: '[Soft fail] Security Solution Cypress tests, burning changed specs'
     agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
       machineType: n2-standard-4
       preemptible: true
     depends_on:
@@ -57,9 +48,6 @@ steps:
   - command: .buildkite/scripts/steps/functional/osquery_cypress_burn.sh
     label: '[Soft fail] Osquery Cypress Tests, burning changed specs'
     agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
       machineType: n2-standard-4
       preemptible: true
     depends_on: