Merge branch 'feature/exploratory-view-chart-creation-context' of htt…

…ps://github.com/dominiqueclarke/kibana into feature/exploratory-view-chart-creation-context

.buildkite/pipelines/performance/nightly.yml

@@ -0,0 +1,35 @@
+steps:
+  - block: ":gear: Performance Tests Configuration"
+    prompt: "Fill out the details for performance test"
+    fields:
+      - text: ":arrows_counterclockwise: Iterations"
+        key: "performance-test-iteration-count"
+        hint: "How many times you want to run tests? "
+        required: true
+    if: build.env('ITERATION_COUNT_ENV') == null
+
+  - label: ":male-mechanic::skin-tone-2: Pre-Build"
+    command: .buildkite/scripts/lifecycle/pre_build.sh
+
+  - wait
+
+  - label: ":factory_worker: Build Kibana Distribution and Plugins"
+    command: .buildkite/scripts/steps/build_kibana.sh
+    agents:
+      queue: c2-16
+    key: build
+
+  - label: ":muscle: Performance Tests"
+    command: .buildkite/scripts/steps/functional/performance.sh
+    agents:
+      queue: ci-group-6
+    depends_on: build
+    concurrency: 50
+    concurrency_group: 'performance-test-group'
+
+  - wait: ~
+    continue_on_failure: true
+
+  - label: ":male_superhero::skin-tone-2: Post-Build"
+    command: .buildkite/scripts/lifecycle/post_build.sh
+

.buildkite/scripts/steps/functional/performance.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -uo pipefail
+
+if [ -z "${ITERATION_COUNT_ENV+x}" ]; then
+  ITERATION_COUNT="$(buildkite-agent meta-data get performance-test-iteration-count)"
+else
+  ITERATION_COUNT=$ITERATION_COUNT_ENV
+fi
+
+tput setab 2; tput setaf 0; echo "Performance test will be run at ${BUILDKITE_BRANCH} ${ITERATION_COUNT} times"
+
+cat << EOF | buildkite-agent pipeline upload
+steps:
+  - command: .buildkite/scripts/steps/functional/performance_sub.sh
+    parallelism: "$ITERATION_COUNT"
+EOF
+
+
+

.buildkite/scripts/steps/functional/performance_sub.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+.buildkite/scripts/bootstrap.sh
+.buildkite/scripts/download_build_artifacts.sh
+
+cd "$XPACK_DIR"
+
+echo --- Run Performance Tests
+checks-reporter-with-killswitch "Run Performance Tests" \
+  node scripts/functional_tests \
+    --debug --bail \
+    --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
+    --config test/performance/config.ts;

docs/dev-tools/console/console.asciidoc

@@ -129,3 +129,12 @@ image::dev-tools/console/images/console-settings.png["Console Settings", width=6
 
 For a list of available keyboard
 shortcuts, click *Help*.
+
+[float]
+[[console-settings]]
+=== Disable Console
+
+If you don’t want to use *Console*, you can disable it by setting `console.ui.enabled`
+to `false` in your `kibana.yml` configuration file. Changing this setting
+causes the server to regenerate assets on the next startup,
+which might cause a delay before pages start being served.

docs/getting-started/quick-start-guide.asciidoc

@@ -138,7 +138,7 @@ image::images/dashboard_sampleDataAddFilter_7.15.0.png[The [eCommerce] Revenue D
 [[quick-start-whats-next]]
 == What's next?
 
-*Add your own data.* Ready to add your own data? Go to {fleet-guide}/fleet-quick-start.html[Quick start: Get logs and metrics into the Elastic Stack] to learn how to ingest your data, or go to <<connect-to-elasticsearch,Add data to {kib}>> and learn about all the other ways you can add data.
+*Add your own data.* Ready to add your own data? Go to {observability-guide}/ingest-logs-metrics-uptime.html[Ingest logs, metrics, and uptime data with {agent}], or go to <<connect-to-elasticsearch,Add data to {kib}>> and learn about all the other ways you can add data.
 
 *Explore your own data in Discover.* Ready to learn more about exploring your data in *Discover*? Go to <<discover, Discover>>.
 

docs/management/connectors/action-types/pagerduty.asciidoc

@@ -68,7 +68,7 @@ PagerDuty actions have the following properties.
 
 Severity::      The perceived severity of on the affected system. This can be one of `Critical`, `Error`, `Warning` or `Info`(default).
 Event action::  One of `Trigger` (default), `Resolve`, or `Acknowledge`. See https://v2.developer.pagerduty.com/docs/events-api-v2#event-action[event action] for more details.
-Dedup Key::     All actions sharing this key will be associated with the same PagerDuty alert. This value is used to correlate trigger and resolution. This value is *optional*, and if not set, defaults to `<alert ID>:<alert instance ID>`. The maximum length is *255* characters. See https://v2.developer.pagerduty.com/docs/events-api-v2#alert-de-duplication[alert deduplication] for details. 
+Dedup Key::     All actions sharing this key will be associated with the same PagerDuty alert. This value is used to correlate trigger and resolution. This value is *optional*, and if not set, defaults to `<rule ID>:<alert ID>`. The maximum length is *255* characters. See https://v2.developer.pagerduty.com/docs/events-api-v2#alert-de-duplication[alert deduplication] for details. 
 Timestamp::     An *optional* https://v2.developer.pagerduty.com/v2/docs/types#datetime[ISO-8601 format date-time], indicating the time the event was detected or generated.
 Component::     An *optional* value indicating the component of the source machine that is responsible for the event, for example `mysql` or `eth0`.
 Group::         An *optional* value indicating the logical grouping of components of a service, for example `app-stack`.

docs/management/connectors/action-types/servicenow-sir.asciidoc

@@ -72,13 +72,11 @@ image::management/connectors/images/servicenow-sir-params-test.png[ServiceNow Se
 ServiceNow SecOps actions have the following configuration properties.
 
 Short description::    A short description for the incident, used for searching the contents of the knowledge base.
-Source Ips::           A list of source IPs related to the incident. The IPs will be added as observables to the security incident.
-Destination Ips::      A list of destination IPs related to the incident. The IPs will be added as observables to the security incident.
-Malware URLs::         A list of malware URLs related to the incident. The URLs will be added as observables to the security incident.
-Malware Hashes::       A list of malware hashes related to the incident. The hashes  will be added as observables to the security incident.
 Priority::             The priority of the incident.
 Category::             The category of the incident.
 Subcategory::          The subcategory of the incident.
+Correlation ID::       All actions sharing this ID will be associated with the same ServiceNow security incident. If an incident exists in ServiceNow with the same correlation ID the security incident will be updated. Default value: `<rule ID>:<alert instance ID>`.
+Correlation Display::  A descriptive label of the alert for correlation purposes in ServiceNow.
 Description::          The details about the incident.
 Additional comments::  Additional information for the client, such as how to troubleshoot the issue.
 

docs/management/connectors/action-types/servicenow.asciidoc

@@ -76,6 +76,8 @@ Severity::             The severity of the incident.
 Impact::               The effect an incident has on business. Can be measured by the number of affected users or by how critical it is to the business in question.
 Category::             The category of the incident.
 Subcategory::          The category of the incident.
+Correlation ID::       All actions sharing this ID will be associated with the same ServiceNow incident. If an incident exists in ServiceNow with the same correlation ID the incident will be updated. Default value: `<rule ID>:<alert instance ID>`.
+Correlation Display::  A descriptive label of the alert for correlation purposes in ServiceNow.
 Short description::    A short description for the incident, used for searching the contents of the knowledge base.
 Description::          The details about the incident.
 Additional comments::  Additional information for the client, such as how to troubleshoot the issue.

docs/osquery/osquery.asciidoc

@@ -365,7 +365,7 @@ The following is an example of an **error response** for an undefined action que
 == System requirements
 
 * {fleet-guide}/fleet-overview.html[Fleet] is enabled on your cluster, and
-one or more {fleet-guide}/elastic-agent-installation-configuration.html[Elastic Agents] is enrolled.
+one or more {fleet-guide}/elastic-agent-installation.html[Elastic Agents] is enrolled.
 * The https://docs.elastic.co/en/integrations/osquery_manager[*Osquery Manager*] integration
 has been added and configured
 for an agent policy through Fleet.

docs/settings/spaces-settings.asciidoc

@@ -7,11 +7,6 @@
 
 By default, spaces is enabled in {kib}. To secure spaces, <<security-settings-kb,enable security>>.
 
-`xpack.spaces.enabled`::
-deprecated:[7.16.0,"In 8.0 and later, this setting will no longer be supported and it will not be possible to disable this plugin."]
-To enable spaces, set to `true`. 
-The default is `true`.
-
 `xpack.spaces.maxSpaces`::
 The maximum number of spaces that you can use with the {kib} instance. Some {kib} operations
 return all spaces using a single `_search` from {es}, so you must

docs/setup/connect-to-elasticsearch.asciidoc

@@ -47,7 +47,7 @@ so you can quickly get insights into your data, and {fleet} mode offers several
 image::images/addData_fleet_7.15.0.png[Add data using Fleet]
 
 To get started, refer to
-{fleet-guide}/fleet-quick-start.html[Quick start: Get logs and metrics into the Elastic Stack].
+{observability-guide}/ingest-logs-metrics-uptime.html[Ingest logs, metrics, and uptime data with {agent}].
 
 [discrete]
 [[upload-data-kibana]]

docs/setup/settings.asciidoc

@@ -20,6 +20,11 @@ configuration using `${MY_ENV_VAR}` syntax.
 [cols="2*<"]
 |===
 
+| `console.ui.enabled:`
+Toggling this causes the server to regenerate assets on the next startup,
+which may cause a delay before pages start being served.
+Set to `false` to disable Console. *Default: `true`*
+
 | `csp.rules:`
  | deprecated:[7.14.0,"In 8.0 and later, this setting will no longer be supported."]
 A https://w3c.github.io/webappsec-csp/[Content Security Policy] template
@@ -275,7 +280,7 @@ that the {kib} server uses to perform maintenance on the {kib} index at startup.
 is an alternative to `elasticsearch.username` and `elasticsearch.password`.
 
 | `enterpriseSearch.host`
-  | The URL of your Enterprise Search instance
+  | The http(s) URL of your Enterprise Search instance. For example, in a local self-managed setup, set this to `http://localhost:3002`. Authentication between Kibana and the Enterprise Search host URL, such as via OAuth, is not supported. You can also {enterprise-search-ref}/configure-ssl-tls.html#configure-ssl-tls-in-kibana[configure Kibana to trust your Enterprise Search TLS certificate authority].
 
 | `interpreter.enableInVisualize`
   | Enables use of interpreter in Visualize. *Default: `true`*
@@ -681,6 +686,10 @@ out through *Advanced Settings*. *Default: `true`*
  | Set this value to true to allow Vega to use any URL to access external data
 sources and images. When false, Vega can only get data from {es}. *Default: `false`*
 
+| `xpack.ccr.ui.enabled`
+Set this value to false to disable the Cross-Cluster Replication UI.
+*Default: `true`*
+
 |[[settings-explore-data-in-context]] `xpack.discoverEnhanced.actions.`
 `exploreDataInContextMenu.enabled`
  | Enables the *Explore underlying data* option that allows you to open *Discover* from a dashboard panel and view the panel data. *Default: `false`*
@@ -689,6 +698,31 @@ sources and images. When false, Vega can only get data from {es}. *Default: `fal
 `exploreDataInChart.enabled`
  | Enables you to view the underlying documents in a data series from a dashboard panel. *Default: `false`*
 
+| `xpack.ilm.ui.enabled`
+Set this value to false to disable the Index Lifecycle Policies UI.
+*Default: `true`*
+
+| `xpack.index_management.ui.enabled`
+Set this value to false to disable the Index Management UI.
+*Default: `true`*
+
+| `xpack.license_management.ui.enabled`
+Set this value to false to disable the License Management UI.
+*Default: `true`*
+
+| `xpack.remote_clusters.ui.enabled`
+Set this value to false to disable the Remote Clusters UI.
+*Default: `true`*
+
+| `xpack.rollup.ui.enabled:`
+Set this value to false to disable the Rollup Jobs UI. *Default: true*
+
+| `xpack.snapshot_restore.ui.enabled:`
+Set this value to false to disable the Snapshot and Restore UI. *Default: true*
+
+| `xpack.upgrade_assistant.ui.enabled:`
+Set this value to false to disable the Upgrade Assistant UI. *Default: true*
+
 | `i18n.locale` {ess-icon}
  | Set this value to change the {kib} interface language.
 Valid locales are: `en`, `zh-CN`, `ja-JP`. *Default: `en`*

docs/spaces/index.asciidoc

@@ -109,11 +109,6 @@ image::images/spaces-configure-landing-page.png["Configure space-level landing p
 
 [float]
 [[spaces-delete-started]]
-=== Disable and version updates
-
-Spaces are automatically enabled in {kib}. If you don't want use this feature,
-you can disable it. For more information, refer to <<spaces-settings-kb,Spaces settings in {kib}>>.
-
-When you upgrade {kib}, the default space contains all of your existing saved objects.
-
+=== Disabling spaces
 
+Starting in {kib} 8.0, the Spaces feature cannot be disabled.

packages/elastic-apm-generator/README.md

@@ -0,0 +1,93 @@
+# @elastic/apm-generator
+
+`@elastic/apm-generator` is an experimental tool to generate synthetic APM data. It is intended to be used for development and testing of the Elastic APM app in Kibana. 
+
+At a high-level, the module works by modeling APM events/metricsets with [a fluent API](https://en.wikipedia.org/wiki/Fluent_interface). The models can then be serialized and converted to Elasticsearch documents. In the future we might support APM Server as an output as well.
+
+## Usage
+
+This section assumes that you've installed Kibana's dependencies by running `yarn kbn bootstrap` in the repository's root folder.
+
+This library can currently be used in two ways:
+
+- Imported as a Node.js module, for instance to be used in Kibana's functional test suite.
+- With a command line interface, to index data based on some example scenarios.
+
+### Using the Node.js module
+
+#### Concepts
+
+- `Service`: a logical grouping for a monitored service. A `Service` object contains fields like `service.name`, `service.environment` and `agent.name`.
+- `Instance`: a single instance of a monitored service. E.g., the workload for a monitored service might be spread across multiple containers. An `Instance` object contains fields like `service.node.name` and `container.id`.
+- `Timerange`: an object that will return an array of timestamps based on an interval and a rate. These timestamps can be used to generate events/metricsets.
+- `Transaction`, `Span`, `APMError` and `Metricset`: events/metricsets that occur on an instance. For more background, see the [explanation of the APM data model](https://www.elastic.co/guide/en/apm/get-started/7.15/apm-data-model.html)
+
+
+#### Example
+
+```ts
+import { service, timerange, toElasticsearchOutput } from '@elastic/apm-generator';
+
+const instance = service('synth-go', 'production', 'go')
+  .instance('instance-a');
+
+const from = new Date('2021-01-01T12:00:00.000Z').getTime();
+const to = new Date('2021-01-01T12:00:00.000Z').getTime() - 1;
+
+const traceEvents = timerange(from, to)
+  .interval('1m')
+  .rate(10)
+  .flatMap(timestamp => instance.transaction('GET /api/product/list')
+    .timestamp(timestamp)
+    .duration(1000)
+    .success()
+    .children(
+      instance.span('GET apm-*/_search', 'db', 'elasticsearch')
+        .timestamp(timestamp + 50)
+        .duration(900)
+        .destination('elasticsearch')
+        .success()
+    ).serialize()
+  );
+
+const metricsets = timerange(from, to)
+  .interval('30s')
+  .rate(1)
+  .flatMap(timestamp => instance.appMetrics({
+    'system.memory.actual.free': 800,
+    'system.memory.total': 1000,
+    'system.cpu.total.norm.pct': 0.6,
+    'system.process.cpu.total.norm.pct': 0.7,
+  }).timestamp(timestamp)
+    .serialize()
+  );
+
+const esEvents = toElasticsearchOutput(traceEvents.concat(metricsets));
+```
+
+#### Generating metricsets
+
+`@elastic/apm-generator` can also automatically generate transaction metrics, span destination metrics and transaction breakdown metrics based on the generated trace events. If we expand on the previous example:
+
+```ts
+import { getTransactionMetrics, getSpanDestinationMetrics, getBreakdownMetrics } from '@elastic/apm-generator';
+
+const esEvents = toElasticsearchOutput([
+  ...traceEvents,
+  ...getTransactionMetrics(traceEvents),
+  ...getSpanDestinationMetrics(traceEvents),
+  ...getBreakdownMetrics(traceEvents)
+]);
+```
+
+### CLI
+
+Via the CLI, you can upload examples. The supported examples are listed in `src/lib/es.ts`. A `--target` option that specifies the Elasticsearch URL should be defined when running the `example` command. Here's an example:
+
+`$ node packages/elastic-apm-generator/src/scripts/es.js example simple-trace --target=http://admin:changeme@localhost:9200`
+
+The following options are supported:
+- `to`: the end of the time range, in ISO format. By default, the current time will be used.
+- `from`: the start of the time range, in ISO format. By default, `to` minus 15 minutes will be used.
+- `apm-server-version`: the version used in the index names bootstrapped by APM Server, e.g. `7.16.0`. __If these indices do not exist, the script will exit with an error. It will not bootstrap the indices itself.__
+

packages/kbn-securitysolution-list-constants/src/index.ts

@@ -73,6 +73,6 @@ export const ENDPOINT_EVENT_FILTERS_LIST_DESCRIPTION = 'Endpoint Security Event
 
 export const ENDPOINT_HOST_ISOLATION_EXCEPTIONS_LIST_ID = 'endpoint_host_isolation_exceptions';
 export const ENDPOINT_HOST_ISOLATION_EXCEPTIONS_LIST_NAME =
-  'Endpoint Security Host Isolation Exceptions List';
+  'Endpoint Security Host isolation exceptions List';
 export const ENDPOINT_HOST_ISOLATION_EXCEPTIONS_LIST_DESCRIPTION =
-  'Endpoint Security Host Isolation Exceptions List';
+  'Endpoint Security Host isolation exceptions List';

src/core/public/chrome/ui/header/collapsible_nav.tsx

@@ -362,7 +362,7 @@ export function CollapsibleNav({
               iconType="plusInCircleFilled"
             >
               {i18n.translate('core.ui.primaryNav.addData', {
-                defaultMessage: 'Add data',
+                defaultMessage: 'Add integrations',
               })}
             </EuiButton>
           </EuiCollapsibleNavGroup>

src/core/public/doc_links/doc_links_service.ts

@@ -327,6 +327,7 @@ export class DocLinksService {
           preconfiguredConnectors: `${KIBANA_DOCS}pre-configured-connectors.html`,
           preconfiguredAlertHistoryConnector: `${KIBANA_DOCS}index-action-type.html#preconfigured-connector-alert-history`,
           serviceNowAction: `${KIBANA_DOCS}servicenow-action-type.html#configuring-servicenow`,
+          serviceNowSIRAction: `${KIBANA_DOCS}servicenow-sir-action-type.html`,
           setupPrerequisites: `${KIBANA_DOCS}alerting-setup.html#alerting-prerequisites`,
           slackAction: `${KIBANA_DOCS}slack-action-type.html#configuring-slack`,
           teamsAction: `${KIBANA_DOCS}teams-action-type.html#configuring-teams`,