Merge branch 'master' into ml-apm-toggle-button

.buildkite/scripts/common/persist_bazel_cache.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+source .buildkite/scripts/common/util.sh
+
+KIBANA_BUILDBUDDY_CI_API_KEY=$(retry 5 5 vault read -field=value secret/kibana-issues/dev/kibana-buildbuddy-ci-api-key)
+export KIBANA_BUILDBUDDY_CI_API_KEY
+
+# overwrites the file checkout .bazelrc file with the one intended for CI env
+cp "$KIBANA_DIR/src/dev/ci_setup/.bazelrc-ci" "$KIBANA_DIR/.bazelrc"
+
+###
+### append auth token to buildbuddy into "$KIBANA_DIR/.bazelrc";
+###
+echo "# Appended by .buildkite/scripts/persist_bazel_cache.sh" >> "$KIBANA_DIR/.bazelrc"
+echo "build --remote_header=x-buildbuddy-api-key=$KIBANA_BUILDBUDDY_CI_API_KEY" >> "$KIBANA_DIR/.bazelrc"

.buildkite/scripts/steps/on_merge_build_and_metrics.sh

@@ -3,7 +3,7 @@
 set -euo pipefail
 
 # Write Bazel cache for Linux
-.buildkite/scripts/common/setup_bazel.sh
+.buildkite/scripts/common/persist_bazel_cache.sh
 
 .buildkite/scripts/bootstrap.sh
 .buildkite/scripts/build_kibana.sh

.github/CODEOWNERS

@@ -63,7 +63,7 @@
 /packages/kbn-interpreter/ @elastic/kibana-app-services
 /src/plugins/bfetch/ @elastic/kibana-app-services
 /src/plugins/data/ @elastic/kibana-app-services
-/src/plugins/data-views/ @elastic/kibana-app-services
+/src/plugins/data_views/ @elastic/kibana-app-services
 /src/plugins/embeddable/ @elastic/kibana-app-services
 /src/plugins/expressions/ @elastic/kibana-app-services
 /src/plugins/field_formats/ @elastic/kibana-app-services
@@ -254,7 +254,6 @@
 /src/plugins/kibana_overview/ @elastic/kibana-core
 /x-pack/plugins/global_search_bar/ @elastic/kibana-core
 #CC# /src/core/server/csp/ @elastic/kibana-core
-#CC# /src/plugins/xpack_legacy/ @elastic/kibana-core
 #CC# /src/plugins/saved_objects/ @elastic/kibana-core
 #CC# /x-pack/plugins/cloud/ @elastic/kibana-core
 #CC# /x-pack/plugins/features/ @elastic/kibana-core

api_docs/plugin_directory.mdx

@@ -153,7 +153,6 @@ warning: This document is auto-generated and is meant to be viewed inside our ex
 | <DocLink id="kibVisualizationsPluginApi" text="visualizations"/> | [Vis Editors](https://github.com/orgs/elastic/teams/kibana-vis-editors) | Contains the shared architecture among all the legacy visualizations, e.g. the visualization type registry or the visualization embeddable. | 304 | 13 | 286 | 16 |
 | <DocLink id="kibVisualizePluginApi" text="visualize"/> | [Vis Editors](https://github.com/orgs/elastic/teams/kibana-vis-editors) | Contains the visualize application which includes the listing page and the app frame, which will load the visualization's editor. | 24 | 0 | 23 | 1 |
 | watcher | [Stack Management](https://github.com/orgs/elastic/teams/kibana-stack-management) | - | 0 | 0 | 0 | 0 |
-| xpackLegacy | [Kibana Core](https://github.com/orgs/elastic/teams/kibana-core) | - | 0 | 0 | 0 | 0 |
 
 ## Package Directory
 

dev_docs/key_concepts/kibana_platform_plugin_intro.mdx

@@ -33,13 +33,28 @@ At a super high-level, Kibana is composed of **plugins**, **core**, and **Kibana
 
 <DocAccordion buttonContent="(Internal only) FAQ: Should I put my code in a plugin or a package?" color="warning">
 <DocCallOut color="warning">
-If it's stateful, it has to go in a plugin, but packages are often a good choices for stateless utilities. Stateless code exported publicly from a plugin will increase the page load bundle size of _every single page_, even if none of those plugin's services are actually needed. With packages, however, only code that is needed for the current page is downloaded.
+When the [Bazel migration](https://github.com/elastic/kibana/blob/master/legacy_rfcs/text/0015_bazel.md) is complete, all code, including plugins, will be a package. With that, packages won't be required to be in the `packages/` directory and can be located somewhere that makes more sense structurally.
 
-The downside however is that the packages folder is far away from the plugins folder so having a part of your code in a plugin and the rest in a package may make it hard to find, leading to duplication.
+In the meantime, the following can be used to determine whether it makes sense to add code to a package inside the `packages` folder, or a plugin inside `src/plugins` or `x-pack/plugins`. 
 
-The Operations team hopes to resolve this conundrum by supporting co-located packages and plugins and automatically putting all stateless code inside a package. You can track this work by following [this issue](https://github.com/elastic/kibana/issues/112886).
 
-Until then, consider whether it makes sense to logically separate the code, and consider the size of the exports, when determining whether you should put stateless public exports in a package or a plugin.
+**If the code is stateful, it has to be exposed from a plugin's <DocLink id="kibPlatformIntro" section="lifecycle-methods" text="lifecycle methods"/>. Do not statically export stateful code.**
+
+Benefits to packages:
+
+1. <b>_Potentially_ reduced page load time</b>. All code that is statically exported from plugins will be downloaded on _every single page load_, even if that code isn't needed. With packages, only code that is imported is downloaded, which can be minimized by using async imports.
+2. <b>Puts the consumer is in charge of how and when to async import</b>. If a consumer async imports code exported from a plugin, it makes no difference, because of the above point. It's already been downloaded. However, simply moving code into a package is _not_ a guaranteed performance improvement. It does give the consumer the power to make smart performance choices, however. If they require code from multiple packages, the consumer can async import from multiple packages at the same time. Read more in our <DocLink id="kibDevPerformance" text="performance docs"/>.
+
+Downsides to packages:
+
+1. <b>It's not <DocLink id="kibDevPrinciples" text="organized by domain"/></b>. The packages folder is far away from the plugins folder. Having your stateless code in a plugin and the rest in a package may make it hard to find, leading to duplication. The Operations team hopes to fix this by supporting packages and plugins existing in the same folder. You can track this work by following [this issue](https://github.com/elastic/kibana/issues/112886).
+
+2. <b>Development overhead</b>. Developers have to run `yarn kbn watch` to have changes rebuilt automatically. [Phase II](https://github.com/elastic/kibana/blob/master/legacy_rfcs/text/0015_bazel.md#phase-ii---docs-developer-experience) of the Bazel migration work will bring the development experience on par with plugin development. This work can be tracked [here](https://github.com/elastic/kibana/issues/104519). 
+
+3. <b>Development performance</b>. Rebuild time is typically longer than it would be for the same code in a plugin. The reasons are captured in [this issue](https://github.com/elastic/kibana/issues/107648). The ops team is actively working to reduce this performance increase.
+
+
+As you can see, the answer to "Should I put my code in a plugin or a package" is 'It Depends'. If you are still having a hard time determining what the best path location is, reach out to the Kibana Operations Team (#kibana-operations) for help.
 
 </DocCallOut>
 </DocAccordion>

docs/api/logstash-configuration-management/create-logstash.asciidoc

@@ -27,7 +27,16 @@ experimental[] Create a centrally-managed Logstash pipeline, or update an existi
   (Required, string) The pipeline definition.
 
 `settings`::
-  (Optional, object) The pipeline settings. Supported settings, represented as object keys, are `pipeline.workers`, `pipeline.batch.size`, `pipeline.batch.delay`, `queue.type`, `queue.max_bytes`, and `queue.checkpoint.writes`.
+  (Optional, object) The pipeline settings. Supported settings, represented as object keys, include the following:
+
+  * `pipeline.workers`
+  * `pipeline.batch.size`
+  * `pipeline.batch.delay`
+  * `pipeline.ecs_compatibility`
+  * `pipeline.ordered`
+  * `queue.type`
+  * `queue.max_bytes`
+  * `queue.checkpoint.writes`
 
 [[logstash-configuration-management-api-create-codes]]
 ==== Response code

docs/apm/api.asciidoc

@@ -484,8 +484,7 @@ An example is below.
 [[api-create-apm-index-pattern]]
 ==== Customize the APM index pattern
 
-As an alternative to updating <<apm-settings-in-kibana,`apm_oss.indexPattern`>> in your `kibana.yml` configuration file,
-you can use Kibana's <<saved-objects-api-update,update object API>> to update the default APM index pattern on the fly.
+Use Kibana's <<saved-objects-api-update,update object API>> to update the default APM index pattern on the fly.
 
 The following example sets the default APM app index pattern to `some-other-pattern-*`:
 

docs/apm/troubleshooting.asciidoc

@@ -76,7 +76,7 @@ If you change the default, you must also configure the `setup.template.name` and
 See {apm-server-ref}/configuration-template.html[Load the Elasticsearch index template].
 If the Elasticsearch index template has already been successfully loaded to the index,
 you can customize the indices that the APM app uses to display data.
-Navigate to *APM* > *Settings* > *Indices*, and change all `apm_oss.*Pattern` values to
+Navigate to *APM* > *Settings* > *Indices*, and change all `xpack.apm.indices.*` values to
 include the new index pattern. For example: `customIndexName-*`.
 
 [float]

docs/developer/plugin-list.asciidoc

@@ -617,10 +617,6 @@ in their infrastructure.
 |This plugins adopts some conventions in addition to or in place of conventions in Kibana (at the time of the plugin's creation):
 
 
-|{kib-repo}blob/{branch}/x-pack/plugins/xpack_legacy/README.md[xpackLegacy]
-|Contains HTTP endpoints and UiSettings that are slated for removal.
-
-
 |===
 
 include::{kibana-root}/src/plugins/dashboard/README.asciidoc[leveloffset=+1]

docs/development/core/public/kibana-plugin-core-public.doclinksstart.links.md

@@ -240,6 +240,7 @@ readonly links: {
             upgradeElasticAgent: string;
             upgradeElasticAgent712lower: string;
             learnMoreBlog: string;
+            apiKeysLearnMore: string;
         }>;
         readonly ecs: {
             readonly guide: string;

docs/index.asciidoc

@@ -13,12 +13,8 @@ include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[]
 :es-docker-image: {es-docker-repo}:{version}
 :blob:            {kib-repo}blob/{branch}/
 :security-ref:    https://www.elastic.co/community/security/
-:Data-Sources:    Data Views
-:Data-source:     Data view
 :data-source:     data view
-:Data-sources:    Data views
 :data-sources:    data views
-:A-data-source:   A data view
 :a-data-source:   a data view
 
 include::{docs-root}/shared/attributes.asciidoc[]

docs/settings/apm-settings.asciidoc

@@ -87,22 +87,22 @@ Changing these settings may disable features of the APM App.
 | `xpack.apm.agent.migrations.enabled` {ess-icon}
   | Set to `false` to disable cloud APM migrations. Defaults to `true`.
 
-| `xpack.apm.errorIndices` {ess-icon}
-  | Matcher for all {apm-server-ref}/error-indices.html[error indices]. Defaults to `apm-*`.
+| `xpack.apm.indices.error` {ess-icon}
+  | Matcher for all {apm-server-ref}/error-indices.html[error indices]. Defaults to `logs-apm*,apm-*`.
 
-| `xpack.apm.onboardingIndices` {ess-icon}
+| `xpack.apm.indices.onboarding` {ess-icon}
   | Matcher for all onboarding indices. Defaults to `apm-*`.
 
-| `xpack.apm.spanIndices` {ess-icon}
-  | Matcher for all {apm-server-ref}/span-indices.html[span indices]. Defaults to `apm-*`.
+| `xpack.apm.indices.span` {ess-icon}
+  | Matcher for all {apm-server-ref}/span-indices.html[span indices]. Defaults to `traces-apm*,apm-*`.
 
-| `xpack.apm.transactionIndices` {ess-icon}
-  | Matcher for all {apm-server-ref}/transaction-indices.html[transaction indices]. Defaults to `apm-*`.
+| `xpack.apm.indices.transaction` {ess-icon}
+  | Matcher for all {apm-server-ref}/transaction-indices.html[transaction indices]. Defaults to `traces-apm*,apm-*`.
 
-| `xpack.apm.metricsIndices` {ess-icon}
-  | Matcher for all {apm-server-ref}/metricset-indices.html[metrics indices]. Defaults to `apm-*`.
+| `xpack.apm.indices.metric` {ess-icon}
+  | Matcher for all {apm-server-ref}/metricset-indices.html[metrics indices]. Defaults to `metrics-apm*,apm-*`.
 
-| `xpack.apm.sourcemapIndices` {ess-icon}
+| `xpack.apm.indices.sourcemap` {ess-icon}
   | Matcher for all {apm-server-ref}/sourcemap-indices.html[source map indices]. Defaults to `apm-*`.
 
 |===

docs/settings/spaces-settings.asciidoc

@@ -5,20 +5,23 @@
 <titleabbrev>Spaces settings</titleabbrev>
 ++++
 
-By default, Spaces is enabled in Kibana, and you can secure Spaces using
-roles when Security is enabled.
-
-[float]
-[[spaces-settings]]
-==== Spaces settings
+By default, spaces is enabled in {kib}. To secure spaces, <<security-settings-kb,enable security>>.
 
 `xpack.spaces.enabled`::
-deprecated:[7.16.0,"In 8.0 and later, this setting will no longer be supported."]
-Set to `true` (default) to enable Spaces in {kib}.
-This setting is deprecated. Starting in 8.0, it will not be possible to disable this plugin.
+deprecated:[7.16.0,"In 8.0 and later, this setting will no longer be supported and it will not be possible to disable this plugin."]
+To enable spaces, set to `true`. 
+The default is `true`.
 
 `xpack.spaces.maxSpaces`::
-The maximum amount of Spaces that can be used with this instance of {kib}. Some operations
-in {kib} return all spaces using a single `_search` from {es}, so this must be
-set lower than the `index.max_result_window` in {es}.
-Defaults to `1000`.
+The maximum number of spaces that you can use with the {kib} instance. Some {kib} operations
+return all spaces using a single `_search` from {es}, so you must
+configure this setting lower than the `index.max_result_window` in {es}.
+The default is `1000`.
+
+`monitoring.cluster_alerts-allowedSpaces` {ess-icon}:: 
+Specifies the spaces where cluster alerts are automatically generated. 
+You must specify all spaces where you want to generate alerts, including the default space. 
+When the default space is unspecified, {kib} is unable to generate an alert for the default space.
+{es} clusters that run on {es} services are all containers. To send monitoring data 
+from your self-managed {es} installation to {es} services, set to `false`. 
+The default is `true`.

docs/user/reporting/index.asciidoc

@@ -13,9 +13,9 @@
 
 You access the options from the *Share* menu in the toolbar. The sharing options include the following:
 
-* *PDF Reports* — Generate and download a PDF file of a dashboard, visualization, or *Canvas* workpad.
+* *PDF Reports* — Generate and download a PDF file of a dashboard, visualization, or *Canvas* workpad. PDF reports are a link:https://www.elastic.co/subscriptions[subscription feature].
 
-* *PNG Reports* — Generate and download a PNG file of a dashboard or visualization.
+* *PNG Reports* — Generate and download a PNG file of a dashboard or visualization. PNG reports are a link:https://www.elastic.co/subscriptions[subscription feature].
 
 * *CSV Reports* — Generate and download a CSV file of a *Discover* saved search.
 

package.json

@@ -101,7 +101,7 @@
     "@elastic/charts": "37.0.0",
     "@elastic/datemath": "link:bazel-bin/packages/elastic-datemath",
     "@elastic/elasticsearch": "npm:@elastic/elasticsearch-canary@^8.0.0-canary.21",
-    "@elastic/ems-client": "7.15.0",
+    "@elastic/ems-client": "7.16.0",
     "@elastic/eui": "39.0.0",
     "@elastic/filesaver": "1.1.2",
     "@elastic/maki": "6.3.0",

packages/elastic-apm-generator/src/index.ts

@@ -12,3 +12,4 @@ export { getTransactionMetrics } from './lib/utils/get_transaction_metrics';
 export { getSpanDestinationMetrics } from './lib/utils/get_span_destination_metrics';
 export { getObserverDefaults } from './lib/defaults/get_observer_defaults';
 export { toElasticsearchOutput } from './lib/output/to_elasticsearch_output';
+export { getBreakdownMetrics } from './lib/utils/get_breakdown_metrics';

packages/elastic-apm-generator/src/lib/base_span.ts

@@ -8,10 +8,12 @@
 
 import { Fields } from './entity';
 import { Serializable } from './serializable';
+import { Span } from './span';
+import { Transaction } from './transaction';
 import { generateTraceId } from './utils/generate_id';
 
 export class BaseSpan extends Serializable {
-  private _children: BaseSpan[] = [];
+  private readonly _children: BaseSpan[] = [];
 
   constructor(fields: Fields) {
     super({
@@ -22,20 +24,29 @@ export class BaseSpan extends Serializable {
     });
   }
 
-  traceId(traceId: string) {
-    this.fields['trace.id'] = traceId;
+  parent(span: BaseSpan) {
+    this.fields['trace.id'] = span.fields['trace.id'];
+    this.fields['parent.id'] = span.isSpan()
+      ? span.fields['span.id']
+      : span.fields['transaction.id'];
+
+    if (this.isSpan()) {
+      this.fields['transaction.id'] = span.fields['transaction.id'];
+    }
     this._children.forEach((child) => {
-      child.fields['trace.id'] = traceId;
+      child.parent(this);
     });
+
     return this;
   }
 
   children(...children: BaseSpan[]) {
-    this._children.push(...children);
     children.forEach((child) => {
-      child.traceId(this.fields['trace.id']!);
+      child.parent(this);
     });
 
+    this._children.push(...children);
+
     return this;
   }
 
@@ -52,4 +63,12 @@ export class BaseSpan extends Serializable {
   serialize(): Fields[] {
     return [this.fields, ...this._children.flatMap((child) => child.serialize())];
   }
+
+  isSpan(): this is Span {
+    return this.fields['processor.event'] === 'span';
+  }
+
+  isTransaction(): this is Transaction {
+    return this.fields['processor.event'] === 'transaction';
+  }
 }

packages/elastic-apm-generator/src/lib/entity.ts

@@ -10,9 +10,11 @@ export type Fields = Partial<{
   '@timestamp': number;
   'agent.name': string;
   'agent.version': string;
+  'container.id': string;
   'ecs.version': string;
   'event.outcome': string;
   'event.ingested': number;
+  'host.name': string;
   'metricset.name': string;
   'observer.version': string;
   'observer.version_major': number;
@@ -42,6 +44,8 @@ export type Fields = Partial<{
   'span.destination.service.type': string;
   'span.destination.service.response_time.sum.us': number;
   'span.destination.service.response_time.count': number;
+  'span.self_time.count': number;
+  'span.self_time.sum.us': number;
 }>;
 
 export class Entity {

packages/elastic-apm-generator/src/lib/output/to_elasticsearch_output.ts

@@ -14,10 +14,12 @@ export function toElasticsearchOutput(events: Fields[], versionOverride?: string
   return events.map((event) => {
     const values = {
       ...event,
+      ...getObserverDefaults(),
       '@timestamp': new Date(event['@timestamp']!).toISOString(),
       'timestamp.us': event['@timestamp']! * 1000,
       'ecs.version': '1.4',
-      ...getObserverDefaults(),
+      'service.node.name':
+        event['service.node.name'] || event['container.id'] || event['host.name'],
     };
 
     const document = {};

packages/elastic-apm-generator/src/lib/service.ts

@@ -14,6 +14,7 @@ export class Service extends Entity {
     return new Instance({
       ...this.fields,
       ['service.node.name']: instanceName,
+      'container.id': instanceName,
     });
   }
 }

packages/elastic-apm-generator/src/lib/span.ts

@@ -19,18 +19,6 @@ export class Span extends BaseSpan {
     });
   }
 
-  children(...children: BaseSpan[]) {
-    super.children(...children);
-
-    children.forEach((child) =>
-      child.defaults({
-        'parent.id': this.fields['span.id'],
-      })
-    );
-
-    return this;
-  }
-
   duration(duration: number) {
     this.fields['span.duration.us'] = duration * 1000;
     return this;