Merge remote-tracking branch 'upstream/main' into ml-dv-stats-for-non…

…-aggregatable

.buildkite/ftr_configs.yml

@@ -191,6 +191,7 @@ enabled:
   - x-pack/test/api_integration/apis/es/config.ts
   - x-pack/test/api_integration/apis/features/config.ts
   - x-pack/test/api_integration/apis/file_upload/config.ts
+  - x-pack/test/api_integration/apis/grok_debugger/config.ts
   - x-pack/test/api_integration/apis/kibana/config.ts
   - x-pack/test/api_integration/apis/lists/config.ts
   - x-pack/test/api_integration/apis/logs_ui/config.ts

.buildkite/pipeline-utils/buildkite/client.ts

@@ -7,24 +7,31 @@
  */
 
 import axios, { AxiosInstance } from 'axios';
-import { execSync } from 'child_process';
+import { execSync, ExecSyncOptions } from 'child_process';
 import { dump } from 'js-yaml';
 import { parseLinkHeader } from './parse_link_header';
 import { Artifact } from './types/artifact';
 import { Build, BuildStatus } from './types/build';
 import { Job, JobState } from './types/job';
 
+type ExecType =
+  | ((command: string, execOpts: ExecSyncOptions) => Buffer | null)
+  | ((command: string, execOpts: ExecSyncOptions) => string | null);
+
 export interface BuildkiteClientConfig {
   baseUrl?: string;
   token?: string;
+  exec?: ExecType;
 }
 
 export interface BuildkiteGroup {
   group: string;
   steps: BuildkiteStep[];
 }
 
-export interface BuildkiteStep {
+export type BuildkiteStep = BuildkiteCommandStep | BuildkiteInputStep;
+
+export interface BuildkiteCommandStep {
   command: string;
   label: string;
   parallelism?: number;
@@ -43,6 +50,50 @@ export interface BuildkiteStep {
   env?: { [key: string]: string };
 }
 
+interface BuildkiteInputTextField {
+  text: string;
+  key: string;
+  hint?: string;
+  required?: boolean;
+  default?: string;
+}
+
+interface BuildkiteInputSelectField {
+  select: string;
+  key: string;
+  hint?: string;
+  required?: boolean;
+  default?: string;
+  multiple?: boolean;
+  options: Array<{
+    label: string;
+    value: string;
+  }>;
+}
+
+export interface BuildkiteInputStep {
+  input: string;
+  prompt?: string;
+  fields: Array<BuildkiteInputTextField | BuildkiteInputSelectField>;
+  if?: string;
+  allow_dependency_failure?: boolean;
+  branches?: string;
+  parallelism?: number;
+  agents?: {
+    queue: string;
+  };
+  timeout_in_minutes?: number;
+  key?: string;
+  depends_on?: string | string[];
+  retry?: {
+    automatic: Array<{
+      exit_status: string;
+      limit: number;
+    }>;
+  };
+  env?: { [key: string]: string };
+}
+
 export interface BuildkiteTriggerBuildParams {
   commit: string;
   branch: string;
@@ -61,6 +112,7 @@ export interface BuildkiteTriggerBuildParams {
 
 export class BuildkiteClient {
   http: AxiosInstance;
+  exec: ExecType;
 
   constructor(config: BuildkiteClientConfig = {}) {
     const BUILDKITE_BASE_URL =
@@ -78,6 +130,8 @@ export class BuildkiteClient {
       },
     });
 
+    this.exec = config.exec ?? execSync;
+
     // this.agentHttp = axios.create({
     //   baseURL: BUILDKITE_AGENT_BASE_URL,
     //   headers: {
@@ -97,6 +151,32 @@ export class BuildkiteClient {
     return resp.data as Build;
   };
 
+  getBuildsAfterDate = async (
+    pipelineSlug: string,
+    date: string,
+    numberOfBuilds: number
+  ): Promise<Build[]> => {
+    const response = await this.http.get(
+      `v2/organizations/elastic/pipelines/${pipelineSlug}/builds?created_from=${date}&per_page=${numberOfBuilds}`
+    );
+    return response.data as Build[];
+  };
+
+  getBuildForCommit = async (pipelineSlug: string, commit: string): Promise<Build | null> => {
+    if (commit.length !== 40) {
+      throw new Error(`Invalid commit hash: ${commit}, this endpoint works with full SHAs only`);
+    }
+
+    const response = await this.http.get(
+      `v2/organizations/elastic/pipelines/${pipelineSlug}/builds?commit=${commit}`
+    );
+    const builds = response.data as Build[];
+    if (builds.length === 0) {
+      return null;
+    }
+    return builds[0];
+  };
+
   getCurrentBuild = (includeRetriedJobs = false) => {
     if (!process.env.BUILDKITE_PIPELINE_SLUG || !process.env.BUILDKITE_BUILD_NUMBER) {
       throw new Error(
@@ -235,31 +315,52 @@ export class BuildkiteClient {
   };
 
   setMetadata = (key: string, value: string) => {
-    execSync(`buildkite-agent meta-data set '${key}'`, {
+    this.exec(`buildkite-agent meta-data set '${key}'`, {
       input: value,
       stdio: ['pipe', 'inherit', 'inherit'],
     });
   };
 
+  getMetadata(key: string, defaultValue: string | null = null): string | null {
+    try {
+      const stdout = this.exec(`buildkite-agent meta-data get '${key}'`, {
+        stdio: ['pipe'],
+      });
+      return stdout?.toString().trim() || defaultValue;
+    } catch (e) {
+      if (e.message.includes('404 Not Found')) {
+        return defaultValue;
+      } else {
+        throw e;
+      }
+    }
+  }
+
   setAnnotation = (
     context: string,
     style: 'info' | 'success' | 'warning' | 'error',
-    value: string
+    value: string,
+    append: boolean = false
   ) => {
-    execSync(`buildkite-agent annotate --context '${context}' --style '${style}'`, {
-      input: value,
-      stdio: ['pipe', 'inherit', 'inherit'],
-    });
+    this.exec(
+      `buildkite-agent annotate --context '${context}' --style '${style}' ${
+        append ? '--append' : ''
+      }`,
+      {
+        input: value,
+        stdio: ['pipe', 'inherit', 'inherit'],
+      }
+    );
   };
 
   uploadArtifacts = (pattern: string) => {
-    execSync(`buildkite-agent artifact upload '${pattern}'`, {
+    this.exec(`buildkite-agent artifact upload '${pattern}'`, {
       stdio: ['ignore', 'inherit', 'inherit'],
     });
   };
 
   uploadSteps = (steps: Array<BuildkiteStep | BuildkiteGroup>) => {
-    execSync(`buildkite-agent pipeline upload`, {
+    this.exec(`buildkite-agent pipeline upload`, {
       input: dump({ steps }),
       stdio: ['pipe', 'inherit', 'inherit'],
     });

.buildkite/pipeline-utils/github/github.ts

@@ -91,3 +91,7 @@ export const doAnyChangesMatch = async (
 
   return anyFilesMatchRequired;
 };
+
+export function getGithubClient() {
+  return github;
+}

.buildkite/pipeline-utils/index.ts

@@ -10,3 +10,4 @@ export * from './buildkite';
 export * as CiStats from './ci-stats';
 export * from './github';
 export * as TestFailures from './test-failures';
+export * from './utils';

.buildkite/pipeline-utils/utils.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { execSync } from 'child_process';
+
+const getKibanaDir = (() => {
+  let kibanaDir: string | undefined;
+  return () => {
+    if (!kibanaDir) {
+      kibanaDir = execSync('git rev-parse --show-toplevel', { encoding: 'utf-8' })
+        .toString()
+        .trim();
+    }
+
+    return kibanaDir;
+  };
+})();
+
+export { getKibanaDir };

.buildkite/pipelines/flaky_tests/groups.json

@@ -23,6 +23,23 @@
     {
       "key": "cypress/security_serverless_explore",
       "name": "[Serverless] Security Solution Explore - Cypress"
+    },   
+    {
+      "key": "cypress/security_solution_rule_management",
+      "name": "Security Solution Rule Management - Cypress"
+    },
+    {
+      "key": "cypress/security_serverless_rule_management",
+      "name": "[Serverless] Security Solution Rule Management - Cypress"
+    },
+
+    {
+      "key": "cypress/security_solution_rule_management_prebuilt_rules",
+      "name": "Security Solution Rule Management - Prebuilt Rules - Cypress"
+    },
+    {
+      "key": "cypress/security_serverless_rule_management_prebuilt_rules",
+      "name": "[Serverless] Security Solution Rule Management - Prebuilt Rules - Cypress"
     },
     {
       "key": "cypress/defend_workflows",

.buildkite/pipelines/serverless_deployment/create_deployment_tag.yml

@@ -0,0 +1,37 @@
+## Creates deploy@<timestamp> tag on Kibana
+
+agents:
+ queue: kibana-default
+
+steps:
+  - label: "List potential commits"
+    commands:
+      - ts-node .buildkite/scripts/serverless/create_deploy_tag/release_wizard_messaging.ts --state initialize
+      - ts-node .buildkite/scripts/serverless/create_deploy_tag/release_wizard_messaging.ts --state collect_commits
+      - ts-node .buildkite/scripts/serverless/create_deploy_tag/list_commit_candidates.ts 50
+      - ts-node .buildkite/scripts/serverless/create_deploy_tag/release_wizard_messaging.ts --state wait_for_selection
+    key: select_commit
+    env:
+      AUTO_SELECT_COMMIT: $AUTO_SELECT_COMMIT
+
+  - wait: ~
+
+  - label: "Collect commit info"
+    commands:
+      - ts-node .buildkite/scripts/serverless/create_deploy_tag/release_wizard_messaging.ts --state collect_commit_info
+      - bash .buildkite/scripts/serverless/create_deploy_tag/collect_commit_info.sh
+      - ts-node .buildkite/scripts/serverless/create_deploy_tag/release_wizard_messaging.ts --state wait_for_confirmation
+    key: collect_data
+    depends_on: select_commit
+    env:
+      AUTO_SELECT_COMMIT: $AUTO_SELECT_COMMIT
+
+  - wait: ~
+
+  - label: ":ship: Create Deploy Tag"
+    commands:
+      - ts-node .buildkite/scripts/serverless/create_deploy_tag/release_wizard_messaging.ts --state create_deploy_tag
+      - bash .buildkite/scripts/serverless/create_deploy_tag/create_deploy_tag.sh
+      - ts-node .buildkite/scripts/serverless/create_deploy_tag/release_wizard_messaging.ts --state tag_created
+    env:
+      DRY_RUN: $DRY_RUN

.buildkite/scripts/lifecycle/pre_command.sh

@@ -139,6 +139,9 @@ export SYNTHETICS_REMOTE_KIBANA_PASSWORD
 SYNTHETICS_REMOTE_KIBANA_URL=${SYNTHETICS_REMOTE_KIBANA_URL-"$(retry 5 5 vault read -field=url secret/kibana-issues/dev/kibana-ci-synthetics-remote-credentials)"}
 export SYNTHETICS_REMOTE_KIBANA_URL
 
+DEPLOY_TAGGER_SLACK_WEBHOOK_URL=${DEPLOY_TAGGER_SLACK_WEBHOOK_URL:-"$(retry 5 5 vault read -field=DEPLOY_TAGGER_SLACK_WEBHOOK_URL secret/kibana-issues/dev/kibana-serverless-release-tools)"}
+export DEPLOY_TAGGER_SLACK_WEBHOOK_URL
+
 # Setup Failed Test Reporter Elasticsearch credentials
 {
   TEST_FAILURES_ES_CLOUD_ID=$(retry 5 5 vault read -field=cloud_id secret/kibana-issues/dev/failed_tests_reporter_es)

.buildkite/scripts/serverless/create_deploy_tag/collect_commit_info.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# SO migration comparison lives in the Kibana dev app code, needs bootstrapping
+.buildkite/scripts/bootstrap.sh
+
+echo "--- Collecting commit info"
+ts-node .buildkite/scripts/serverless/create_deploy_tag/collect_commit_info.ts
+
+if [[ "$AUTO_SELECT_COMMIT" == "true" || "$AUTO_SELECT_COMMIT" == "1" ]]; then
+  echo "--- Auto promoting to RC, skipping confirmation"
+else
+  echo "--- Uploading confirmation step"
+  cat << EOF | buildkite-agent pipeline upload
+  steps:
+    - block: "Confirm deployment"
+      prompt: "Are you sure you want to deploy to production? (dry run: ${DRY_RUN:-false})"
+      depends_on: collect_data
+EOF
+fi