Merge branch 'main' into de_8_14/new_terms_suppression

.buildkite/pipeline-resource-definitions/_template/template.yml

@@ -0,0 +1,72 @@
+###
+# For more information on authoring pipeline definitions,
+# follow the guides at https://docs.elastic.dev/ci/getting-started-with-buildkite-at-elastic
+###
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  # This will be the URL slug in Backstage UI as:
+  #  https://backstage.elastic.dev/catalog/default/resource/bk-kibana-your-pipeline-name
+  # bk-pipeline-<pipeline-name-slugified>
+  name: bk-kibana-your-pipeline-name
+  # This will be displayed in the Backstage UI
+  description: '<Describe your pipeline here>'
+  links:
+      # These are relevant links to your pipeline that will be listed in the Backstage UI
+      # The URL slug here is the .spec.implementation.metadata.name field slugified
+    - url: 'https://buildkite.com/elastic/kibana-your-pipeline-name'
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  system: buildkite
+  # The owner team's github group name in the format 'group:<github-group-name>'
+  owner: 'group:github-group-name'
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      # <context / pipeline name> - this will be displayed in the Buildkite UI as title
+      #  and this will be slugified to form the URL in the Backstage UI
+      name: kibana / your pipeline name
+      # This will appear as description on the Buildkite UI
+      description: '<Describe your pipeline here>'
+    spec:
+      # Environment variables that will be set for the pipeline
+      env:
+        # Slack channel to send notifications to, if ELASTIC_SLACK_NOTIFICATIONS_ENABLED = 'true'
+        SLACK_NOTIFICATIONS_CHANNEL: '#team-slack-channel-name'
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+
+      allow_rebuilds: false
+      branch_configuration: main
+      default_branch: main
+      repository: elastic/kibana
+      # Point to a pipeline implementation, detailing the pipeline steps to run
+      pipeline_file: .buildkite/pipelines/your-pipeline-name.yml
+      skip_intermediate_builds: false
+      provider_settings:
+        prefix_pull_request_fork_branch_names: false
+        skip_pull_request_builds_for_existing_commits: true
+        trigger_mode: none
+      # Teams and their access levels to the pipeline,
+      # please keep [kibana-operations, appex-qa, kibana-tech-leads] as MANAGE_BUILD_AND_READ
+      #         and [everyone] as BUILD_AND_READ
+      teams:
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        appex-qa:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-tech-leads:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: BUILD_AND_READ
+      # Scheduled runs for the pipeline
+      schedules:
+        Daily 6 am UTC:
+          cronline: 0 5 * * *
+          message: Daily 6 am UTC
+          branch: main
+          # Optionally, set schedule-specific env-vars here
+          env:
+            SCHEDULED: 'true'

.buildkite/pipeline-resource-definitions/fix-location-collection.ts

@@ -11,7 +11,7 @@ import jsYaml from 'js-yaml';
 import path from 'path';
 import { execSync } from 'child_process';
 
-const EXCLUDE_LIST = ['locations.yml'];
+const EXCLUDE_LIST = ['locations.yml', '_template/template.yml'];
 const REPO_FILES_BASE = 'https://github.com/elastic/kibana/blob/main';
 
 type BackstageLocationResource = object & {

catalog-info.yaml

@@ -220,3 +220,6 @@ spec:
           access_level: BUILD_AND_READ
         everyone:
           access_level: READ_ONLY
+
+# Please avoid creating new kibana pipelines in this file to avoid bloating.
+# Instead, create a new file in the pipeline-resource-definitions directory, and wire it in through the locations.yml file.

docs/CHANGELOG.asciidoc

@@ -10,6 +10,7 @@
 
 Review important information about the {kib} 8.x releases.
 
+* <<release-notes-8.13.2>>
 * <<release-notes-8.13.1>>
 * <<release-notes-8.13.0>>
 * <<release-notes-8.12.2>>
@@ -62,6 +63,35 @@ Review important information about the {kib} 8.x releases.
 * <<release-notes-8.0.0-alpha1>>
 
 --
+[[release-notes-8.13.2]]
+== {kib} 8.13.2
+
+The 8.13.2 release includes the following bug fixes.
+
+[float]
+[[fixes-v8.13.2]]
+=== Bug Fixes
+Canvas::
+* Fixes text settings to be honored in Canvas markdown elements ({kibana-pull}179948[#179948]).
+Discover::
+* Fixes keyboard navigation for search input on the document viewer flyout ({kibana-pull}180022[#180022]).
+Elastic Security::
+For the Elastic Security 8.13.2 release information, refer to {security-guide}/release-notes.html[_Elastic Security Solution Release Notes_].
+Fleet::
+* Fixes having to wait ten minutes after agent upgrade if agent cleared watching state ({kibana-pull}179917[#179917]).
+Fixes using the latest available version in K8's manifest instead of the latest compatible version ({kibana-pull}179662[#179662]).
+* Fixes a step in add agent instructions where a query to get all agents was unnecessary ({kibana-pull}179603[#179603]).
+Machine Learning::
+* Single Metric Viewer embeddable: Ensures the detector index is passed to chart correctly ({kibana-pull}179761[#179761]).
+* AIOps: Fixes text field candidate selection for log rate analysis ({kibana-pull}179699[#179699]).
+Management::
+* Fixes the Response tab loading time to be faster ({kibana-pull}180035[#180035]).
+Maps::
+* Fixes APM data view ID ({kibana-pull}179257[#179257]).
+Monitoring::
+* Fixes a runtime error by adding a default value for source and target ({kibana-pull}180043[#180043]).
+Operations::
+* Fixes an issue with {kib} looking for a configuration file outside of the {kib} home directory, potentially preventing startup due to insufficient permissions ({kibana-pull}179847[#179847]).
 
 [[release-notes-8.13.1]]
 == {kib} 8.13.1

packages/core/saved-objects/core-saved-objects-base-server-internal/src/constants.ts

@@ -159,7 +159,7 @@ export const HASH_TO_VERSION_MAP = {
   'endpoint:user-artifact-manifest|7502b5c5bc923abe8aa5ccfd636e8c3d': '10.0.0',
   'enterprise_search_telemetry|3d1b76c39bfb2cc8296b024d73854724': '10.0.0',
   'epm-packages-assets|44621b2f6052ef966da47b7c3a00f33b': '10.0.0',
-  'epm-packages|c1e2020399dbebba2448096ca007c668': '10.1.0',
+  'epm-packages|8ce219acd0f6f3529237d52193866afb': '10.2.0',
   'event_loop_delays_daily|5df7e292ddd5028e07c1482e130e6654': '10.0.0',
   'event-annotation-group|df07b1a361c32daf4e6842c1d5521dbe': '10.0.0',
   'exception-list-agnostic|8a1defe5981db16792cb9a772e84bb9a': '10.0.0',

packages/kbn-apm-synthtrace/src/lib/apm/client/apm_synthtrace_kibana_client.ts

@@ -20,10 +20,18 @@ export class ApmSynthtraceKibanaClient {
     this.target = options.target;
   }
 
+  getFleetApmPackagePath(packageVersion?: string): string {
+    let path = `${this.target}/api/fleet/epm/packages/apm`;
+    if (packageVersion) {
+      path = `${path}/${packageVersion}`;
+    }
+    return path;
+  }
+
   async fetchLatestApmPackageVersion() {
     this.logger.debug(`Fetching latest APM package version`);
-    const fleetPackageApiUrl = `${this.target}/api/fleet/epm/packages/apm?prerelease=true`;
-    const response = await fetch(fleetPackageApiUrl, {
+    const url = `${this.getFleetApmPackagePath()}?prerelease=true`;
+    const response = await fetch(url, {
       method: 'GET',
       headers: kibanaHeaders(),
     });
@@ -41,10 +49,13 @@ export class ApmSynthtraceKibanaClient {
     return latestVersion as string;
   }
 
-  async installApmPackage(packageVersion: string) {
+  async installApmPackage(packageVersion?: string) {
     this.logger.debug(`Installing APM package ${packageVersion}`);
+    if (!packageVersion) {
+      packageVersion = await this.fetchLatestApmPackageVersion();
+    }
 
-    const url = `${this.target}/api/fleet/epm/packages/apm/${packageVersion}`;
+    const url = this.getFleetApmPackagePath(packageVersion);
     const response = await pRetry(
       async () => {
         const res = await fetch(url, {
@@ -53,10 +64,14 @@ export class ApmSynthtraceKibanaClient {
           body: '{"force":true}',
         });
 
-        if (res.status >= 400) {
+        if (!res.ok) {
           const errorJson = await res.json();
+          const errorMessage =
+            typeof errorJson.message === 'string'
+              ? errorJson.message
+              : 'An error occurred during APM package installation.';
           throw new Error(
-            `APM package installation returned ${res.status} status code\nError: ${errorJson}`
+            `APM package installation returned ${res.status} status code\nError: ${errorMessage}`
           );
         }
         return res;
@@ -75,10 +90,59 @@ export class ApmSynthtraceKibanaClient {
 
     if (!responseJson.items) {
       throw new Error(
-        `Failed to install APM package version ${packageVersion}, received HTTP ${response.status} and message: ${responseJson.message} for url ${url}`
+        `No installed assets received for APM package version ${packageVersion}, received HTTP ${response.status} for url ${url}`
       );
     }
 
     this.logger.info(`Installed APM package ${packageVersion}`);
+    return { version: packageVersion };
+  }
+
+  async uninstallApmPackage() {
+    this.logger.debug('Uninstalling APM package');
+    const latestApmPackageVersion = await this.fetchLatestApmPackageVersion();
+
+    const url = this.getFleetApmPackagePath(latestApmPackageVersion);
+    const response = await pRetry(
+      async () => {
+        const res = await fetch(url, {
+          method: 'DELETE',
+          headers: kibanaHeaders(),
+          body: '{"force":true}',
+        });
+
+        if (!res.ok) {
+          const errorJson = await res.json();
+          const errorMessage =
+            typeof errorJson.message === 'string'
+              ? errorJson.message
+              : 'An error occurred during APM package uninstallation.';
+          throw new Error(
+            `APM package uninstallation returned ${res.status} status code\nError: ${errorMessage}`
+          );
+        }
+        return res;
+      },
+      {
+        retries: 5,
+        onFailedAttempt: (error) => {
+          this.logger.debug(
+            `APM package version ${latestApmPackageVersion} uninstallation failure. ${
+              error.retriesLeft >= 1 ? 'Retrying' : 'Aborting'
+            }`
+          );
+        },
+      }
+    );
+
+    const responseJson = await response.json();
+
+    if (!responseJson.items) {
+      throw new Error(
+        `No uninstalled assets received for APM package version ${latestApmPackageVersion}, received HTTP ${response.status} for url ${url}`
+      );
+    }
+
+    this.logger.info(`Uninstalled APM package ${latestApmPackageVersion}`);
   }
 }

packages/kbn-check-mappings-update-cli/current_fields.json

@@ -290,6 +290,7 @@
     "installed_kibana_space_id",
     "internal",
     "keep_policies_up_to_date",
+    "latest_executed_state",
     "latest_install_failed_attempts",
     "name",
     "package_assets",

packages/kbn-check-mappings-update-cli/current_mappings.json

@@ -1003,6 +1003,10 @@
         "index": false,
         "type": "boolean"
       },
+      "latest_executed_state": {
+        "enabled": false,
+        "type": "object"
+      },
       "latest_install_failed_attempts": {
         "enabled": false,
         "type": "object"

packages/kbn-esql-ast/src/antlr_error_listener.ts

@@ -25,8 +25,8 @@ export class ESQLErrorListener extends ErrorListener<any> {
     const textMessage = `SyntaxError: ${message}`;
 
     const tokenPosition = getPosition(offendingSymbol);
-    const startColumn = tokenPosition?.min + 1 || column;
-    const endColumn = tokenPosition?.max + 1 || column + 1;
+    const startColumn = offendingSymbol && tokenPosition ? tokenPosition.min + 1 : column + 1;
+    const endColumn = offendingSymbol && tokenPosition ? tokenPosition.max + 1 : column + 2;
 
     this.errors.push({
       startLineNumber: line,

packages/kbn-esql-validation-autocomplete/README.md

@@ -116,6 +116,7 @@ const {title, edits} = await getActions(
   queryString,
   errors,
   getAstAndSyntaxErrors,
+  undefined,
   myCallbacks
 );
 
@@ -124,6 +125,33 @@ const {title, edits} = await getActions(
 console.log({ title, edits });
 ```
 
+Like with validation also `getActions` can 'relax' its internal checks when no callbacks, either all or specific ones, are passed.
+
+```js
+import { getAstAndSyntaxErrors } from '@kbn/esql-ast';
+import { validateQuery, getActions } from '@kbn/esql-validation-autocomplete';
+
+const queryString = "from index2 | keep unquoted-field"
+
+const myCallbacks = {
+  getSources: async () => [{name: 'index', hidden: false}],
+  ...
+};
+const { errors, warnings } = await validateQuery(queryString, getAstAndSyntaxErrors, undefined, myCallbacks);
+
+const {title, edits} = await getActions(
+  queryString,
+  errors,
+  getAstAndSyntaxErrors,
+  { relaxOnMissingCallbacks: true },
+  myCallbacks
+);
+
+console.log(edits[0].text); // => `unquoted-field`
+```
+
+**Note**: this behaviour is still experimental, and applied for few error types, like the unquoted fields case.
+
 ### getAstContext
 
 This is an important function in order to build more features on top of the existing one.