Merge branch 'edit-mode' of github.com:angorayc/kibana into edit-mode

.buildkite/pipelines/pipeline.kibana-serverless-release.yaml

@@ -10,3 +10,6 @@ steps:
         SERVICE: kibana-controller
         NAMESPACE: kibana-ci
         IMAGE_NAME: kibana-serverless
+
+notify:
+  - slack: "#kibana-mission-control"

.buildkite/pipelines/pull_request/base.yml

@@ -140,7 +140,7 @@ steps:
       queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 40
-    parallelism: 12
+    parallelism: 2
     soft_fail: true
     retry:
       automatic:
@@ -166,7 +166,7 @@ steps:
       queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 40
-    parallelism: 6
+    parallelism: 2
     soft_fail: true
     retry:
       automatic:

.buildkite/pipelines/serverless.yml

@@ -105,32 +105,33 @@ steps:
       queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 40
-    parallelism: 12
+    parallelism: 2
     retry:
       automatic:
         - exit_status: '*'
           limit: 1
 
-  - command: .buildkite/scripts/steps/functional/security_serverless_explore.sh
-    label: 'Serverless Explore - Security Solution Cypress Tests'
+  - command: .buildkite/scripts/steps/functional/security_serverless_investigations.sh
+    label: 'Serverless Security Investigations Cypress Tests'
     agents:
       queue: n2-4-spot
     depends_on: build
-    timeout_in_minutes: 60
-    parallelism: 4
+    timeout_in_minutes: 120
+    parallelism: 2
     retry:
       automatic:
         - exit_status: '*'
           limit: 1
 
-  - command: .buildkite/scripts/steps/functional/security_serverless_investigations.sh
-    label: 'Serverless Investigations - Security Solution Cypress Tests'
+  - command: .buildkite/scripts/steps/functional/security_serverless_explore.sh
+    label: 'Serverless Security Explore Cypress Tests'
     agents:
       queue: n2-4-spot
     depends_on: build
-    timeout_in_minutes: 120
-    parallelism: 6
+    timeout_in_minutes: 60
+    parallelism: 2
     retry:
       automatic:
         - exit_status: '*'
           limit: 1
+

.buildkite/scripts/lifecycle/post_command.sh

@@ -31,8 +31,10 @@ if [[ "$IS_TEST_EXECUTION_STEP" == "true" ]]; then
   buildkite-agent artifact upload '.es/**/*.hprof'
   buildkite-agent artifact upload 'data/es_debug_*.tar.gz'
 
-  echo "--- Run Failed Test Reporter"
-  node scripts/report_failed_tests --build-url="${BUILDKITE_BUILD_URL}#${BUILDKITE_JOB_ID}" 'target/junit/**/*.xml'
+  if [[ $BUILDKITE_COMMAND_EXIT_STATUS -ne 0 ]]; then
+    echo "--- Run Failed Test Reporter"
+    node scripts/report_failed_tests --build-url="${BUILDKITE_BUILD_URL}#${BUILDKITE_JOB_ID}" 'target/junit/**/*.xml'
+  fi
 
   if [[ -d 'target/test_failures' ]]; then
     buildkite-agent artifact upload 'target/test_failures/**/*'

config/serverless.yml

@@ -119,6 +119,7 @@ xpack.alerting.rules.run.ruleTypeOverrides:
 xpack.alerting.rules.minimumScheduleInterval.enforce: true
 xpack.alerting.rules.maxScheduledPerMinute: 400
 xpack.actions.run.maxAttempts: 10
+xpack.actions.queued.max: 10000
 
 # Disables ESQL in advanced settings (hides it from the UI)
 uiSettings:

docs/settings/alert-action-settings.asciidoc

@@ -227,6 +227,9 @@ xpack.actions.run:
           maxAttempts: 5
 --
 
+`xpack.actions.queued.max` {ess-icon}::
+Specifies the maximum number of actions that can be queued. Default: 1000000
+
 [float]
 [[preconfigured-connector-settings]]
 === Preconfigured connector settings

docs/user/alerting/create-and-manage-rules.asciidoc

@@ -1,4 +1,3 @@
-[role="xpack"]
 [[create-and-manage-rules]]
 == Create and manage rules
 :frontmatter-description: Set up alerting in the {kib} {stack-manage-app} app and manage your rules.
@@ -52,10 +51,11 @@ For more details, refer to the https://registry.terraform.io/providers/elastic/e
 
 Depending on the {kib} app and context, you might be prompted to choose the type of rule to create. Some apps will preselect the type of rule for you.
 
-Each rule type provides its own way of defining the conditions to detect, but an expression formed by a series of clauses is a common pattern. For example, in a metric threshold rule, the `WHEN` clause enables you to select an aggregation operation to apply to a numeric field.
+Each rule type provides its own way of defining the conditions to detect, but an expression formed by a series of clauses is a common pattern.
+For example, in an {es} query rule, you specify an index, a query, and a threshold, which uses a metric aggregation operation (`count`, `average`, `max`, `min`, or `sum`):
 
 [role="screenshot"]
-image::images/rule-flyout-rule-conditions.png[UI for defining rule conditions on a metric threshold rule,500]
+image::images/es-query-rule-conditions.png[UI for defining rule conditions in an {es} query rule,500]
 // NOTE: This is an autogenerated screenshot. Do not edit it directly.
 
 All rules must have a check interval, which defines how often to evaluate the rule conditions. Checks are queued; they run as close to the defined value as capacity allows.
@@ -71,26 +71,27 @@ conditions are met and when they are no longer met.
 
 Each action uses a connector, which provides connection information for a {kib} service or third party integration, depending on where you want to send the notifications. If no connectors exist, click **Add connector** to create one.
 
-After you select a connector, set the action frequency. If the rule type supports alert summaries, you can choose to create a summary of alerts on each check interval or on a custom interval. For example, if you create a metrics threshold rule, you can send email notifications that summarize the new, ongoing, and recovered alerts each hour:
+After you select a connector, set the action frequency. If the rule type supports alert summaries, you can choose to create a summary of alerts on each check interval or on a custom interval.
+For example, if you create an {es} query rule, you can send notifications that summarize the new, ongoing, and recovered alerts on a custom interval:
 
 [role="screenshot"]
-image::images/action-alert-summary.png[UI for defining rule conditions on a metric threshold rule,500]
+image::images/es-query-rule-action-summary.png[UI for defining alert summary action in an {es} query rule,500]
 // NOTE: This is an autogenerated screenshot. Do not edit it directly.
 
 [NOTE]
 ====
-* The rules that support alert summaries, such as this metric threshold rule, enable you to further refine when actions run by adding time frame and query filters.
+* Some rules that support alert summaries, such as metric threshold rules, enable you to further refine when actions run by adding time frame and query filters.
 * If you choose a custom action interval, it cannot be shorter than the rule's check interval.
 ====
 
 Alternatively, you can set the action frequency such that the action runs for each alert.
 If the rule type does not support alert summaries, this is your only available option.
 You must choose when the action runs (for example, at each check interval, only when the alert status changes, or at a custom action interval).
 You must also choose an action group, which affects whether the action runs. Each rule type has a specific set of valid action groups.
-For example, you can set *Run when* to `Alert`, `Warning`, `No data`, or `Recovered` for the metric threshold rule:
+For example, you can set *Run when* to `Query matched` or `Recovered` for the {es} query rule:
 
 [role="screenshot"]
-image::images/rule-flyout-action-details.png[UI for defining an email action,500]
+image::images/es-query-rule-recovery-action.png[UI for defining a recovery action,500]
 // NOTE: This is an autogenerated screenshot. Do not edit it directly.
 
 Each connector enables different action properties. For example, an email connector enables you to set the recipients, the subject, and a message body in markdown format. For more information about connectors, refer to <<action-types>>.
@@ -123,7 +124,7 @@ You can pass rule values to an action at the time a condition is detected.
 To view the list of variables available for your rule, click the "add rule variable" button:
 
 [role="screenshot"]
-image::images/rule-flyout-action-variables.png[Passing rule values to an action,500]
+image::images/es-query-rule-action-variables.png[Passing rule values to an action,500]
 // NOTE: This is an autogenerated screenshot. Do not edit it directly.
 
 For more information about common action variables, refer to <<rule-action-variables>>.

docs/user/alerting/rule-types/es-query.asciidoc

@@ -1,7 +1,7 @@
 [[rule-type-es-query]]
 == {es} query
 
-:frontmatter-description: An {es} query rule generates alerts when your query meets a threshold.
+:frontmatter-description: Create an {es} query rule, which generates alerts when your query meets a threshold.
 :frontmatter-tags-products: [kibana,alerting]
 :frontmatter-tags-content-type: [overview]
 :frontmatter-tags-user-goals: [analyze]
@@ -10,18 +10,15 @@ The {es} query rule type runs a user-configured query, compares the number of
 matches to a configured threshold, and schedules actions to run when the 
 threshold condition is met.
 
-
 [float]
 === Create the rule
 
-Fill in the name and optional tags, then select 
-*{es} query*. {es} query rule can be defined using KQL/Lucene or Query DSL.
+In *{stack-manage-app}* > *{rules-ui}*, click *Create rule*, fill in the name and optional tags, then select *{es} query*.
+An {es} query rule can be defined using KQL/Lucene or Query DSL.
 
 [float]
 === Define the conditions
 
-Define properties to detect the condition.
-
 [role="screenshot"]
 image::user/alerting/images/rule-types-es-query-conditions.png[Define the condition to detect]
 // NOTE: This is an autogenerated screenshot. Do not edit it directly.
@@ -46,13 +43,48 @@ Exclude matches from previous run:: Turn on to avoid alert duplication by
 excluding documents that have already been detected by the previous rule run. This
 option is not available when a grouping field is specified.
 
+[float]
+=== Add actions
+
+You can optionally send notifications when the rule conditions are met and when they are no longer met.
+In particular, this rule type supports:
+
+* alert summaries
+* actions that run when the query is matched
+* recovery actions that run when the rule conditions are no longer met
+
+For each action, you must choose a connector, which provides connection information for a {kib} service or third party integration. For more information about all the supported connectors, go to <<action-types>>.
+
+After you select a connector, you must set the action frequency.
+You can choose to create a summary of alerts on each check interval or on a custom interval.
+For example, send email notifications that summarize the new, ongoing, and recovered alerts at a custom interval:
+
+[role="screenshot"]
+image::images/es-query-rule-action-summary.png[UI for defining alert summary action in an {es} query rule]
+// NOTE: This is an autogenerated screenshot. Do not edit it directly.
+
+Alternatively, you can set the action frequency such that actions run for each alert.
+Choose how often the action runs (at each check interval, only when the alert status changes, or at a custom action interval).
+You must also choose an action group, which indicates whether the action runs when the query is matched or when the alert is recovered.
+For example:
+
+[role="screenshot"]
+image::images/es-query-rule-action-query-matched.png[UI for defining a recovery action]
+// NOTE: This is an autogenerated screenshot. Do not edit it directly.
+
 [float]
 === Add action variables
 
-<<defining-rules-actions-details, Add an action>> to run when the rule condition 
-is met. The following variables are specific to the {es} query rule. You can 
-also specify 
-<<defining-rules-actions-variables, variables common to all rules>>.
+You can pass rule values to an action to provide contextual details.
+To view the list of variables available for each action, click the "add rule variable" button.
+For example:
+
+[role="screenshot"]
+image::images/es-query-rule-action-variables.png[Passing rule values to an action]
+// NOTE: This is an autogenerated screenshot. Do not edit it directly.
+
+The following variables are specific to the {es} query rule.
+You can also specify <<defining-rules-actions-variables,variables common to all rules>>.
 
 `context.title`:: A preconstructed title for the rule. Example: 
 `rule term match alert query matched`.
@@ -76,17 +108,26 @@ Example: `2022-02-03T20:29:27.732Z`.
 
 `context.hits`:: The most recent documents that matched the query. Using the 
 https://mustache.github.io/[Mustache] template array syntax, you can iterate 
-over these hits to get values from the ES documents into your actions.
-+
-[role="screenshot"]
-image::images/rule-types-es-query-example-action-variable.png[Iterate over hits using Mustache template syntax]
+over these hits to get values from the {es} documents into your actions.
+For example, the message in an email connector action might contain:
 +
+--
+[source,sh]
+--------------------------------------------------
+Elasticsearch query rule '{{rule.name}}' is active:
+
+{{#context.hits}}
+Document with {{_id}} and hostname {{_source.host.name}} has
+{{_source.system.memory.actual.free}} bytes of memory free
+{{/context.hits}}
+--------------------------------------------------
+
 The documents returned by `context.hits` include the {ref}/mapping-source-field.html[`_source`] field.
 If the {es} query search API's {ref}/search-fields.html#search-fields-param[`fields`] parameter is used, documents will also return the `fields` field, 
-which can be used to access any runtime fields defined by the {ref}/runtime-search-request.html[`runtime_mappings`] parameter as the following example shows:
-+
---
-[source]
+which can be used to access any runtime fields defined by the {ref}/runtime-search-request.html[`runtime_mappings`] parameter. 
+For example:
+
+[source,sh]
 --------------------------------------------------
 {{#context.hits}}
 timestamp: {{_source.@timestamp}}
@@ -95,13 +136,12 @@ day of the week: {{fields.day_of_week}} <1>
 --------------------------------------------------
 // NOTCONSOLE
 <1> The `fields` parameter here is used to access the `day_of_week` runtime field.
---
-+
+
 As the {ref}/search-fields.html#search-fields-response[`fields`] response always returns an array of values for each field, 
-the https://mustache.github.io/[Mustache] template array syntax is used to iterate over these values in your actions as the following example shows:
-+
---
-[source]
+the https://mustache.github.io/[Mustache] template array syntax is used to iterate over these values in your actions.
+For example:
+
+[source,sh]
 --------------------------------------------------
 {{#context.hits}}
 Labels:

packages/kbn-config-schema/index.ts

@@ -140,6 +140,22 @@ function recordOf<K extends string, V>(
   return new RecordOfType(keyType, valueType, options);
 }
 
+function oneOf<A, B, C, D, E, F, G, H, I, J, K>(
+  types: [
+    Type<A>,
+    Type<B>,
+    Type<C>,
+    Type<D>,
+    Type<E>,
+    Type<F>,
+    Type<G>,
+    Type<H>,
+    Type<I>,
+    Type<J>,
+    Type<K>
+  ],
+  options?: TypeOptions<A | B | C | D | E | F | G | H | I | J | K>
+): Type<A | B | C | D | E | F | G | H | I | J | K>;
 function oneOf<A, B, C, D, E, F, G, H, I, J>(
   types: [Type<A>, Type<B>, Type<C>, Type<D>, Type<E>, Type<F>, Type<G>, Type<H>, Type<I>, Type<J>],
   options?: TypeOptions<A | B | C | D | E | F | G | H | I | J>

packages/kbn-es/src/utils/docker.test.ts

@@ -476,15 +476,18 @@ describe('runServerlessCluster()', () => {
         [baseEsPath]: {},
       });
       execa.mockImplementation(() => Promise.resolve({ stdout: '' }));
-      const info = jest.fn();
-      jest.requireMock('@elastic/elasticsearch').Client.mockImplementation(() => ({ info }));
+      const health = jest.fn();
+      jest
+        .requireMock('@elastic/elasticsearch')
+        .Client.mockImplementation(() => ({ cluster: { health } }));
 
-      info.mockImplementationOnce(() => Promise.reject()); // first call fails
-      info.mockImplementationOnce(() => Promise.resolve()); // then succeeds
+      health.mockImplementationOnce(() => Promise.reject()); // first call fails
+      health.mockImplementationOnce(() => Promise.resolve({ status: 'red' })); // second call return wrong status
+      health.mockImplementationOnce(() => Promise.resolve({ status: 'green' })); // then succeeds
 
       await runServerlessCluster(log, { basePath: baseEsPath, waitForReady: true });
-      expect(info).toHaveBeenCalledTimes(2);
-    });
+      expect(health).toHaveBeenCalledTimes(3);
+    }, 10000);
   });
 });
 

packages/kbn-es/src/utils/docker.ts

@@ -35,6 +35,7 @@ import {
   ELASTIC_SERVERLESS_SUPERUSER_PASSWORD,
 } from './ess_file_realm';
 import { SYSTEM_INDICES_SUPERUSER } from './native_realm';
+import { waitUntilClusterReady } from './wait_until_cluster_ready';
 
 interface BaseOptions {
   tag?: string;
@@ -560,25 +561,6 @@ function getESClient(clientOptions: ClientOptions): Client {
   });
 }
 
-const delay = (ms: number) => new Promise((res) => setTimeout(res, ms));
-async function waitUntilClusterReady(
-  clientOptions: ClientOptions,
-  timeoutMs = 60 * 1000
-): Promise<void> {
-  const started = Date.now();
-  const client = getESClient(clientOptions);
-
-  while (started + timeoutMs > Date.now()) {
-    try {
-      await client.info();
-      break;
-    } catch (e) {
-      await delay(1000);
-      /* trap to continue */
-    }
-  }
-}
-
 /**
  * Runs an ES Serverless Cluster through Docker
  */
@@ -636,7 +618,7 @@ export async function runServerlessCluster(log: ToolingLog, options: ServerlessO
       portCmd[1].lastIndexOf(':')
     )}`;
 
-    await waitUntilClusterReady({
+    const client = getESClient({
       node: esNodeUrl,
       ...(options.ssl
         ? {
@@ -654,6 +636,7 @@ export async function runServerlessCluster(log: ToolingLog, options: ServerlessO
           }
         : {}),
     });
+    await waitUntilClusterReady({ client, log });
     log.success('ES is ready');
   }