[ci] Add FIPS Vagrant box and nightly testing pipeline (#176980)

## Summary

- Closes elastic/kibana-operations#26
- Adds a Vagrant box and corresponding Ansible playbook to create a test
environment for FIPS
- Adds a daily pipeline to run a subset of FTR tests in FIPS mode

### Known Issues
1. The compilation of OpenSSL in FIPS mode is breaking some of the OS
libraries and functionality (`sudo` / `dnf` likely more). Possibly due
to custom OpenSSL installation using different locations than the OS
version.
2. ES is having trouble starting, likely due to issue 1 ([Log
link](https://buildkite.com/elastic/kibana-pull-request/builds/205420#018f0c58-3dc3-41c5-a1a5-9d9a9e14aacc/265-552)).
Disabling ML is a temp workaround added in
803945c, but we likely need it enabled
in the future anyways, so best to find a proper fix. Tracking at
elastic/kibana-operations#96

### Reviewers
You can view a run of the new pipeline during testing
[here](https://buildkite.com/elastic/kibana-migration-pipeline-staging/builds/84).

---------

Co-authored-by: Tiago Costa <[email protected]>

.buildkite/pipeline-resource-definitions/kibana-fips-daily.yml

@@ -0,0 +1,38 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-fips-daily
+  description: Run Kibana FIPS smoke tests
+  links:
+    - title: Pipeline link
+      url: https://buildkite.com/elastic/kibana-fips
+spec:
+  type: buildkite-pipeline
+  owner: group:kibana-operations
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / fips
+      description: Run Kibana FIPS smoke tests
+    spec:
+      env:
+        SLACK_NOTIFICATIONS_CHANNEL: "#kibana-operations-alerts"
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "true"
+      repository: elastic/kibana
+      branch_configuration: main
+      default_branch: main
+      pipeline_file: ".buildkite/pipelines/fips.yml"
+      provider_settings:
+        trigger_mode: none
+      schedules:
+        daily:
+          branch: main
+          cronline: 0 9 * * * America/New_York
+      teams:
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: READ_ONLY

.buildkite/pipeline-resource-definitions/locations.yml

@@ -12,6 +12,7 @@ spec:
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-serverless-snapshots.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-snapshots.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-esql-grammar-sync.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-fips-daily.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-flaky.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-fleet-packages-daily.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-migration-staging.yml

.buildkite/pipelines/fips.yml

@@ -0,0 +1,32 @@
+env:
+  DISABLE_CI_STATS_SHIPPING: "true"
+steps:
+  - command: .buildkite/scripts/steps/build_kibana.sh
+    label: Build Kibana Distribution and Plugins
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-16
+      preemptible: true
+    key: build
+    if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
+    timeout_in_minutes: 60
+    retry:
+      automatic:
+        - exit_status: "-1"
+          limit: 3
+
+  - wait
+
+  - command: TEST_PACKAGE=fips .buildkite/scripts/steps/package_testing/test.sh
+    label: "Smoke testing for FIPS"
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      enableNestedVirtualization: true
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-4
+    timeout_in_minutes: 600

.buildkite/scripts/steps/fips/smoke_test.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+
+if [ -z "$KIBANA_BUILD_LOCATION" ]; then
+  export KIBANA_BUILD_LOCATION="/usr/share/kibana"
+fi
+
+# a FTR failure will result in the script returning an exit code of 10
+exitCode=0
+
+configs=(
+  "x-pack/test/reporting_functional/reporting_and_security.config.ts"
+  "x-pack/test/saved_object_api_integration/security_and_spaces/config_trial.ts"
+  "x-pack/test/alerting_api_integration/security_and_spaces/group1/config.ts"
+  "x-pack/test/alerting_api_integration/security_and_spaces/group2/config.ts"
+  "x-pack/test/alerting_api_integration/security_and_spaces/group3/config.ts"
+  "x-pack/test/alerting_api_integration/security_and_spaces/group4/config.ts"
+  "x-pack/test/functional/apps/saved_objects_management/config.ts"
+  "x-pack/test/functional/apps/user_profiles/config.ts"
+  "x-pack/test/functional/apps/security/config.ts"
+)
+
+cd /home/vagrant/kibana
+
+for config in "${configs[@]}"; do
+  set +e
+  node /home/vagrant/kibana/scripts/functional_tests \
+    --bail \
+    --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
+    --config="$config"
+  lastCode=$?
+  set -e
+
+  if [ $lastCode -ne 0 ]; then
+    exitCode=10
+    echo "FTR exited with code $lastCode"
+    echo "^^^ +++"
+
+    if [[ "$failedConfigs" ]]; then
+      failedConfigs="${failedConfigs}"$'\n'"$config"
+    else
+      failedConfigs="$config"
+    fi
+  fi
+done
+
+if [[ "$failedConfigs" ]]; then
+  echo "$failedConfigs" >/home/vagrant/ftr_failed_configs
+fi
+
+echo "--- FIPS smoke test complete"
+
+exit $exitCode

.buildkite/scripts/steps/package_testing/test.sh

@@ -21,17 +21,25 @@ elif [[ "$TEST_PACKAGE" == "rpm" ]]; then
 elif [[ "$TEST_PACKAGE" == "docker" ]]; then
   download_artifact "kibana-$KIBANA_PKG_VERSION*-docker-image.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
   KIBANA_IP_ADDRESS="192.168.56.7"
+elif [[ "$TEST_PACKAGE" == "fips" ]]; then
+  download_artifact kibana-default.tar.gz . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+  download_artifact kibana-default-plugins.tar.gz . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
 fi
 cd ..
 
 export VAGRANT_CWD=$PWD/test/package
-vagrant up "$TEST_PACKAGE" --no-provision
 
-node scripts/es snapshot \
-  -E network.bind_host=127.0.0.1,192.168.56.1 \
-  -E discovery.type=single-node \
-  --license=trial &
-while ! timeout 1 bash -c "echo > /dev/tcp/localhost/9200"; do sleep 30; done
+if [[ "$TEST_PACKAGE" == "fips" ]]; then
+  vagrant up "$TEST_PACKAGE"
+else
+  vagrant up "$TEST_PACKAGE" --no-provision
+
+  node scripts/es snapshot \
+    -E network.bind_host=127.0.0.1,192.168.56.1 \
+    -E discovery.type=single-node \
+    --license=trial &
+  while ! timeout 1 bash -c "echo > /dev/tcp/localhost/9200"; do sleep 30; done
+fi
 
 function echoKibanaLogs {
   if [[ "$TEST_PACKAGE" == "deb" ]] || [[ "$TEST_PACKAGE" == "rpm" ]]; then
@@ -47,13 +55,25 @@ function echoKibanaLogs {
 }
 trap "echoKibanaLogs" EXIT
 
-vagrant provision "$TEST_PACKAGE"
+if [[ "$TEST_PACKAGE" == "fips" ]]; then
+  set +e
+  vagrant ssh $TEST_PACKAGE -t -c "/home/vagrant/kibana/.buildkite/scripts/steps/fips/smoke_test.sh"
+  vagrant ssh $TEST_PACKAGE -t -c "cat /home/vagrant/ftr_failed_configs 2>/dev/null" >ftr_failed_configs
+  set -e
+
+  if [ -s ftr_failed_configs ]; then
+    buildkite-agent meta-data set "ftr-failed-configs" <./ftr_failed_configs
+  fi
+else
+  vagrant provision "$TEST_PACKAGE"
+
+  export TEST_BROWSER_HEADLESS=1
+  export TEST_KIBANA_URL="http://elastic:changeme@$KIBANA_IP_ADDRESS:5601"
+  export TEST_ES_URL="http://elastic:[email protected]:9200"
 
-export TEST_BROWSER_HEADLESS=1
-export TEST_KIBANA_URL="http://elastic:changeme@$KIBANA_IP_ADDRESS:5601"
-export TEST_ES_URL="http://elastic:[email protected]:9200"
+  echo "--- FTR - Reporting"
 
-cd x-pack
+  cd x-pack
 
-echo "--- FTR - Reporting"
-node scripts/functional_test_runner.js --config test/functional/apps/visualize/config.ts --include-tag=smoke --quiet
+  node scripts/functional_test_runner.js --config test/functional/apps/visualize/config.ts --include-tag=smoke --quiet
+fi

packages/kbn-test/src/es/test_es_cluster.ts

@@ -182,6 +182,7 @@ export function createTestEsCluster<
   } = options;
 
   const clusterName = `${CI_PARALLEL_PROCESS_PREFIX}${customClusterName}`;
+  const isFIPSMode = process.env.FTR_FIPS_MODE === '1';
 
   const defaultEsArgs = [
     `cluster.name=${clusterName}`,
@@ -192,7 +193,12 @@ export function createTestEsCluster<
       : ['discovery.type=single-node']),
   ];
 
-  const esArgs = assignArgs(defaultEsArgs, customEsArgs);
+  const esArgs = assignArgs(
+    defaultEsArgs,
+    // ML has issues running in FIPS mode due to custom OpenSSL
+    // Remove after https://github.com/elastic/kibana-operations/issues/96
+    isFIPSMode ? [...customEsArgs, 'xpack.ml.enabled=false'] : customEsArgs
+  );
 
   const config = {
     version: esTestConfig.getVersion(),

test/package/Vagrantfile

@@ -36,4 +36,23 @@ Vagrant.configure("2") do |config|
     end
     docker.vm.network "private_network", ip: "192.168.56.7"
   end
+
+  config.vm.define "fips" do |fips|
+    fips.vm.synced_folder '../../', '/home/vagrant/kibana', SharedFoldersEnableSymlinksCreate: false
+    fips.vm.provider :virtualbox do |vb|
+      vb.memory = 4096
+      vb.cpus = 2
+    end
+    fips.vm.box = 'generic/rhel9'
+    fips.vm.provision "shell", inline: <<-SHELL
+      echo "export OPENSSL_MODULES=/usr/local/lib64/ossl-modules" >> /etc/profile.d/kibana-fips-env.sh
+      echo "export TEST_BROWSER_HEADLESS=1" >> /etc/profile.d/kibana-fips-env.sh
+      echo "export ES_TMPDIR=/home/vagrant/kibana/.es/tmp" >> /etc/profile.d/kibana-fips-env.sh
+      # Remove after https://github.com/elastic/kibana-operations/issues/96
+      echo "export FTR_FIPS_MODE=1" >> /etc/profile.d/kibana-fips-env.sh
+    SHELL
+    fips.vm.provision "ansible" do |ansible|
+      ansible.playbook = "fips.yml"
+    end
+  end
 end

test/package/fips.yml

@@ -0,0 +1,12 @@
+- name: test kibana fips docker package
+  hosts: fips
+  vars:
+    kibana_dist_path: "/usr/share/kibana"
+    kibana_src_path: "/home/vagrant/kibana"
+    nvm_ver: "0.39.7"
+    openssl_sha: "sha256:6c13d2bf38fdf31eac3ce2a347073673f5d63263398f1f69d0df4a41253e4b3e"
+    openssl_ver: "3.0.8"
+  roles:
+    - upgrade_yum_packages
+    - install_kibana_fips
+    - assert_fips_enabled

test/package/roles/assert_fips_enabled/tasks/main.yml

@@ -0,0 +1,11 @@
+- name: register kibana node getFips
+  command: "{{ kibana_dist_path }}/node/bin/node --enable-fips --openssl-config={{ kibana_dist_path }}/config/nodejs.cnf -p 'crypto.getFips()'"
+  register: kibana_node_fips
+
+- debug:
+    msg: "{{ kibana_node_fips }}"
+
+- name: assert FIPS enabled
+  assert:
+    that:
+      - kibana_node_fips.stdout == "1"