Merge branch 'main' into eui/v93.2.0

.buildkite/ftr_configs.yml

@@ -33,9 +33,9 @@ disabled:
   - x-pack/test/defend_workflows_cypress/cli_config.ts
   - x-pack/test/defend_workflows_cypress/config.ts
   - x-pack/test/defend_workflows_cypress/serverless_config.ts
-  - x-pack/plugins/observability_onboarding/e2e/ftr_config_open.ts
-  - x-pack/plugins/observability_onboarding/e2e/ftr_config_runner.ts
-  - x-pack/plugins/observability_onboarding/e2e/ftr_config.ts
+  - x-pack/plugins/observability_solution/observability_onboarding/e2e/ftr_config_open.ts
+  - x-pack/plugins/observability_solution/observability_onboarding/e2e/ftr_config_runner.ts
+  - x-pack/plugins/observability_solution/observability_onboarding/e2e/ftr_config.ts
   - x-pack/test/osquery_cypress/cli_config.ts
   - x-pack/test/osquery_cypress/serverless_cli_config.ts
   - x-pack/test/osquery_cypress/config.ts
@@ -47,22 +47,22 @@ disabled:
   - x-pack/test/functional_enterprise_search/visual_config.ts
   - x-pack/test/functional_enterprise_search/cli_config.ts
   - x-pack/test_serverless/functional/test_suites/security/cypress/security_config.ts
-  - x-pack/plugins/apm/ftr_e2e/ftr_config_open.ts
-  - x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts
-  - x-pack/plugins/apm/ftr_e2e/ftr_config.ts
+  - x-pack/plugins/observability_solution/apm/ftr_e2e/ftr_config_open.ts
+  - x-pack/plugins/observability_solution/apm/ftr_e2e/ftr_config_run.ts
+  - x-pack/plugins/observability_solution/apm/ftr_e2e/ftr_config.ts
   - x-pack/test_serverless/functional/test_suites/observability/cypress/config_headless.ts
   - x-pack/test_serverless/functional/test_suites/observability/cypress/config_runner.ts
   - x-pack/test/security_solution_cypress/serverless_config.ts
-  - x-pack/plugins/profiling/e2e/ftr_config_open.ts
-  - x-pack/plugins/profiling/e2e/ftr_config_runner.ts
-  - x-pack/plugins/profiling/e2e/ftr_config.ts
+  - x-pack/plugins/observability_solution/profiling/e2e/ftr_config_open.ts
+  - x-pack/plugins/observability_solution/profiling/e2e/ftr_config_runner.ts
+  - x-pack/plugins/observability_solution/profiling/e2e/ftr_config.ts
 
   # Elastic Synthetics configs
-  - x-pack/plugins/synthetics/e2e/config.ts
-  - x-pack/plugins/uptime/e2e/uptime/synthetics_run.ts
-  - x-pack/plugins/synthetics/e2e/synthetics/synthetics_run.ts
-  - x-pack/plugins/ux/e2e/synthetics_run.ts
-  - x-pack/plugins/exploratory_view/e2e/synthetics_run.ts
+  - x-pack/plugins/observability_solution/uptime/e2e/uptime/synthetics_run.ts
+  - x-pack/plugins/observability_solution/synthetics/e2e/config.ts
+  - x-pack/plugins/observability_solution/synthetics/e2e/synthetics/synthetics_run.ts
+  - x-pack/plugins/observability_solution/exploratory_view/e2e/synthetics_run.ts
+  - x-pack/plugins/observability_solution/ux/e2e/synthetics_run.ts
 
   # Configs that exist but weren't running in CI when this file was introduced
   - x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/config.ts

.buildkite/pipeline-utils/buildkite/client.ts

@@ -29,7 +29,7 @@ export interface BuildkiteGroup {
   steps: BuildkiteStep[];
 }
 
-export type BuildkiteStep = BuildkiteCommandStep | BuildkiteInputStep;
+export type BuildkiteStep = BuildkiteCommandStep | BuildkiteInputStep | BuildkiteTriggerStep;
 
 export interface BuildkiteCommandStep {
   command: string;
@@ -94,6 +94,25 @@ export interface BuildkiteInputStep {
   env?: { [key: string]: string };
 }
 
+export interface BuildkiteTriggerStep {
+  trigger: string;
+  label?: string;
+  build?: {
+    message?: string; // The message for the build. Supports emoji.
+    commit?: string; // The commit hash for the build.
+    branch?: string; // The branch for the build.
+    meta_data?: string; // A map of meta-data for the build.
+    env?: Record<string, string>; // A map of environment variables for the build.
+  };
+  async?: boolean;
+  branches?: string;
+  if?: string;
+  allow_dependency_failure?: boolean;
+  soft_fail?: boolean;
+  depends_on?: string | string[];
+  skip?: string;
+}
+
 export interface BuildkiteTriggerBuildParams {
   commit: string;
   branch: string;

.buildkite/pipelines/on_merge_unsupported_ftrs.yml

@@ -54,7 +54,7 @@ steps:
     depends_on: build
     timeout_in_minutes: 120
     artifact_paths:
-      - 'x-pack/plugins/synthetics/e2e/.journeys/**/*'
+      - 'x-pack/plugins/observability_solution/synthetics/e2e/.journeys/**/*'
     retry:
       automatic:
         - exit_status: '-1'
@@ -68,7 +68,7 @@ steps:
       queue: n2-4-virt
     depends_on: build
     timeout_in_minutes: 60
-    parallelism: 10
+    parallelism: 16
     retry:
       automatic:
         - exit_status: '*'
@@ -80,7 +80,7 @@ steps:
       queue: n2-4-virt
     depends_on: build
     timeout_in_minutes: 60
-    parallelism: 6
+    parallelism: 10
     retry:
       automatic:
         - exit_status: '*'

.buildkite/pipelines/pull_request/exploratory_view_plugin.yml

@@ -8,7 +8,7 @@ steps:
       - quick_checks
     timeout_in_minutes: 60
     artifact_paths:
-      - 'x-pack/plugins/exploratory_view/e2e/.journeys/**/*'
+      - 'x-pack/plugins/observability_solution/exploratory_view/e2e/.journeys/**/*'
     retry:
       automatic:
         - exit_status: '-1'

.buildkite/pipelines/pull_request/synthetics_plugin.yml

@@ -8,7 +8,7 @@ steps:
       - quick_checks
     timeout_in_minutes: 60
     artifact_paths:
-      - 'x-pack/plugins/synthetics/e2e/.journeys/**/*'
+      - 'x-pack/plugins/observability_solution/synthetics/e2e/.journeys/**/*'
     retry:
       automatic:
         - exit_status: '-1'

.buildkite/pipelines/pull_request/uptime_plugin.yml

@@ -8,7 +8,7 @@ steps:
       - quick_checks
     timeout_in_minutes: 60
     artifact_paths:
-      - 'x-pack/plugins/synthetics/e2e/.journeys/**/*'
+      - 'x-pack/plugins/observability_solution/synthetics/e2e/.journeys/**/*'
     retry:
       automatic:
         - exit_status: '-1'

.buildkite/pipelines/pull_request/ux_plugin_e2e.yml

@@ -8,7 +8,7 @@ steps:
       - quick_checks
     timeout_in_minutes: 60
     artifact_paths:
-      - 'x-pack/plugins/ux/e2e/.journeys/**/*'
+      - 'x-pack/plugins/observability_solution/ux/e2e/.journeys/**/*'
     retry:
       automatic:
         - exit_status: '-1'

.buildkite/pipelines/serverless_deployment/create_deployment_tag.yml

@@ -33,5 +33,9 @@ steps:
       - ts-node .buildkite/scripts/serverless/create_deploy_tag/release_wizard_messaging.ts --state create_deploy_tag
       - bash .buildkite/scripts/serverless/create_deploy_tag/create_deploy_tag.sh
       - ts-node .buildkite/scripts/serverless/create_deploy_tag/release_wizard_messaging.ts --state tag_created
+      - ts-node .buildkite/scripts/serverless/create_deploy_tag/generate_gpctl_trigger.ts
+      - ts-node .buildkite/scripts/serverless/create_deploy_tag/release_wizard_messaging.ts --state trigger_gpctl
     env:
       DRY_RUN: $DRY_RUN
+
+  - wait: ~

.buildkite/scripts/lifecycle/pre_command.sh

@@ -151,6 +151,9 @@ export GCS_SA_CDN_EMAIL
 GCS_SA_CDN_BUCKET="$(vault_get gcs-sa-cdn-prod bucket)"
 export GCS_SA_CDN_BUCKET
 
+GCS_SA_CDN_URL="$(vault_get gcs-sa-cdn-prod cdn)"
+export GCS_SA_CDN_URL
+
 # Setup Failed Test Reporter Elasticsearch credentials
 {
   TEST_FAILURES_ES_CLOUD_ID=$(vault_get failed_tests_reporter_es cloud_id)

.buildkite/scripts/serverless/create_deploy_tag/create_deploy_tag.sh

@@ -33,4 +33,4 @@ else
   echo "Skipping tag push to GitHub due to DRY_RUN=$DRY_RUN"
 fi
 
-echo "Created deploy tag: $DEPLOY_TAG - your QA release should start @ https://buildkite.com/elastic/kibana-serverless-release/builds?branch=$DEPLOY_TAG"
+echo "Created deploy tag: $DEPLOY_TAG"

.buildkite/scripts/serverless/create_deploy_tag/generate_gpctl_trigger.ts

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { buildkite } from './shared';
+import { getSelectedCommitHash } from './info_sections/commit_info';
+import { BuildkiteTriggerStep } from '#pipeline-utils';
+
+const IS_DRY_RUN = process.env.DRY_RUN?.match(/(1|true)/i);
+const REMOTE_SERVICE_CONFIG = `https://raw.githubusercontent.com/elastic/serverless-gitops/main/gen/gpctl/kibana/config.yaml`;
+
+async function main() {
+  const selectedSha = getSelectedCommitHash();
+  uploadTriggerStep(selectedSha);
+}
+
+function uploadTriggerStep(commitSha: string) {
+  const triggerStep: BuildkiteTriggerStep = {
+    label: ':releasethekaken: Trigger GPCTL / Release Kibana',
+    trigger: 'gpctl-promote',
+    async: true,
+    build: {
+      message: 'Triggered by Kibana serverless release pipeline',
+      env: {
+        SERVICE_COMMIT_HASH: commitSha.slice(0, 12),
+        REMOTE_SERVICE_CONFIG,
+      },
+    },
+  };
+
+  if (IS_DRY_RUN) {
+    console.log('Dry run: skipping upload of GPCTL trigger step. Step definition:', triggerStep);
+  } else {
+    buildkite.uploadSteps([triggerStep]);
+  }
+}
+
+main()
+  .then(() => {
+    console.log('GPCTL Trigger step uploaded.');
+  })
+  .catch((error) => {
+    console.error(error);
+    process.exit(1);
+  });

.buildkite/scripts/serverless/create_deploy_tag/release_wizard_messaging.ts

@@ -34,9 +34,9 @@ type StateNames =
   | 'wait_for_confirmation'
   | 'create_deploy_tag'
   | 'tag_created'
+  | 'trigger_gpctl'
   | 'end'
-  | 'error_generic'
-  | string;
+  | 'error_generic';
 
 interface StateShape {
   name: string;
@@ -117,7 +117,20 @@ const states: Record<StateNames, StateShape> = {
     name: 'Release tag created',
     description: 'The initial step release is completed, follow up jobs will be triggered soon.',
     instruction: `<h3>Deploy tag successfully created!</h3>`,
-    post: async () => {
+    instructionStyle: 'success',
+    display: true,
+  },
+  trigger_gpctl: {
+    name: 'Triggering GPCTL deployment',
+    description: 'Triggering the GPCTL deployment for the release - sit back and relax.',
+    instruction: `GPCTL deployment triggered, follow the trigger step for more info.`,
+    instructionStyle: 'info',
+    display: true,
+  },
+  end: {
+    name: 'End of the release process',
+    description: 'The release process has ended.',
+    pre: async () => {
       // The deployTag here is only for communication, if it's missing, it's not a big deal, but it's an error
       const deployTag =
         buildkite.getMetadata(DEPLOY_TAG_META_KEY) ||
@@ -128,8 +141,7 @@ const states: Record<StateNames, StateShape> = {
       buildkite.setAnnotation(
         WIZARD_CTX_INSTRUCTION,
         'success',
-        `<h3>Deploy tag successfully created!</h3><br/>
-Your deployment will appear <a href='https://buildkite.com/elastic/kibana-serverless-release/builds?branch=${deployTag}'>here on buildkite.</a>`
+        `<h3>Release successfully initiated!</h3>`
       );
 
       if (!selectedCommit) {
@@ -153,12 +165,6 @@ Your deployment will appear <a href='https://buildkite.com/elastic/kibana-server
         deployTag,
       });
     },
-    instructionStyle: 'success',
-    display: true,
-  },
-  end: {
-    name: 'End of the release process',
-    description: 'The release process has ended.',
     display: false,
   },
   error_generic: {
@@ -179,15 +185,15 @@ export async function main(args: string[]) {
   if (!args.includes('--state')) {
     throw new Error('Missing --state argument');
   }
-  const targetState = args.slice(args.indexOf('--state') + 1)[0];
+  const targetState = args.slice(args.indexOf('--state') + 1)[0] as StateNames;
 
   let data: any;
   if (args.includes('--data')) {
     data = args.slice(args.indexOf('--data') + 1)[0];
   }
 
   const resultingTargetState = await transition(targetState, data);
-  if (resultingTargetState === 'tag_created') {
+  if (resultingTargetState === 'trigger_gpctl') {
     return await transition('end');
   } else {
     return resultingTargetState;
@@ -196,7 +202,7 @@ export async function main(args: string[]) {
 
 export async function transition(targetStateName: StateNames, data?: any) {
   // use the buildkite agent to find what state we are in:
-  const currentStateName = buildkite.getMetadata('release_state') || 'start';
+  const currentStateName = (buildkite.getMetadata('release_state') || 'start') as StateNames;
   const stateData = JSON.parse(buildkite.getMetadata('state_data') || '{}');
 
   if (!currentStateName) {
@@ -243,10 +249,10 @@ function updateWizardState(stateData: Record<string, 'ok' | 'nok' | 'pending' |
     : `<h3>:kibana: Kibana Serverless deployment wizard :mage:</h3>`;
 
   const wizardSteps = Object.keys(states)
-    .filter((stateName) => states[stateName].display)
-    .filter((stateName) => !(IS_AUTOMATED_RUN && states[stateName].skipWhenAutomated))
+    .filter((stateName) => states[stateName as StateNames].display)
+    .filter((stateName) => !(IS_AUTOMATED_RUN && states[stateName as StateNames].skipWhenAutomated))
     .map((stateName) => {
-      const stateInfo = states[stateName];
+      const stateInfo = states[stateName as StateNames];
       const stateStatus = stateData[stateName];
       const stateEmoji = {
         ok: ':white_check_mark:',
@@ -271,7 +277,7 @@ ${wizardSteps.join('\n')}
 }
 
 function updateWizardInstruction(targetState: string, stateData: any) {
-  const { instructionStyle, instruction } = states[targetState];
+  const { instructionStyle, instruction } = states[targetState as StateNames];
 
   if (IS_AUTOMATED_RUN) {
     buildkite.setAnnotation(

.buildkite/scripts/steps/artifacts/docker_image.sh

@@ -98,14 +98,28 @@ tar -xf "kibana-$BASE_VERSION-cdn-assets.tar.gz" -C "$CDN_ASSETS_FOLDER" --strip
 gsutil -m cp -r "$CDN_ASSETS_FOLDER/*" "gs://$GCS_SA_CDN_BUCKET/$GIT_ABBREV_COMMIT"
 gcloud auth revoke "$GCS_SA_CDN_EMAIL"
 
+echo "--- Validate CDN assets"
+(
+  shopt -s globstar
+  THREADS=$(grep -c ^processor /proc/cpuinfo)
+  i=0
+  cd $CDN_ASSETS_FOLDER
+  for CDN_ASSET in **/*; do
+    ((i=(i+1)%THREADS)) || wait
+    if [[ -f "$CDN_ASSET" ]]; then
+      echo -n "Testing $CDN_ASSET..."
+      curl -I --write-out '%{http_code}\n' --fail --silent --output /dev/null "$GCS_SA_CDN_URL/$CDN_ASSET"
+    fi
+  done
+)
+
 echo "--- Upload archives"
 buildkite-agent artifact upload "kibana-$BASE_VERSION-linux-x86_64.tar.gz"
 buildkite-agent artifact upload "kibana-$BASE_VERSION-linux-aarch64.tar.gz"
 buildkite-agent artifact upload "kibana-$BASE_VERSION-docker-image.tar.gz"
 buildkite-agent artifact upload "kibana-$BASE_VERSION-docker-image-aarch64.tar.gz"
 buildkite-agent artifact upload "kibana-$BASE_VERSION-cdn-assets.tar.gz"
 buildkite-agent artifact upload "dependencies-$GIT_ABBREV_COMMIT.csv"
-cd -
 
 # This part is related with updating the configuration of kibana-controller,
 # so that new stack instances contain the latest and greatest image of kibana,

.buildkite/scripts/steps/functional/exploratory_view_plugin.sh

@@ -13,4 +13,4 @@ echo "--- Exploratory View plugin @elastic/synthetics Tests"
 
 cd "$XPACK_DIR"
 
-node plugins/exploratory_view/scripts/e2e.js --kibana-install-dir "$KIBANA_BUILD_LOCATION" ${GREP:+--grep \"${GREP}\"}
+node plugins/observability_solution/exploratory_view/scripts/e2e.js --kibana-install-dir "$KIBANA_BUILD_LOCATION" ${GREP:+--grep \"${GREP}\"}

.buildkite/scripts/steps/functional/observability_onboarding_cypress.sh

@@ -13,5 +13,5 @@ echo "--- Observability onboarding Cypress Tests"
 
 cd "$XPACK_DIR"
 
-node plugins/observability_onboarding/scripts/test/e2e.js \
+node plugins/observability_solution/observability_onboarding/scripts/test/e2e.js \
   --kibana-install-dir "$KIBANA_BUILD_LOCATION" \

.buildkite/scripts/steps/functional/profiling_cypress.sh

@@ -13,5 +13,5 @@ echo "--- Profiling Cypress Tests"
 
 cd "$XPACK_DIR"
 
-NODE_OPTIONS=--openssl-legacy-provider node plugins/profiling/scripts/test/e2e.js \
+NODE_OPTIONS=--openssl-legacy-provider node plugins/observability_solution/profiling/scripts/test/e2e.js \
   --kibana-install-dir "$KIBANA_BUILD_LOCATION" \

.buildkite/scripts/steps/functional/synthetics_plugin.sh

@@ -13,4 +13,4 @@ echo "--- Synthetics plugin @elastic/synthetics Tests"
 
 cd "$XPACK_DIR"
 
-node plugins/synthetics/scripts/e2e.js --kibana-install-dir "$KIBANA_BUILD_LOCATION" ${GREP:+--grep \"${GREP}\"}
+node plugins/observability_solution/synthetics/scripts/e2e.js --kibana-install-dir "$KIBANA_BUILD_LOCATION" ${GREP:+--grep \"${GREP}\"}