Merge remote-tracking branch 'upstream/master' into 115468-threshold-…

…kql-cacheclear

.backportrc.json

@@ -2,7 +2,7 @@
   "upstream": "elastic/kibana",
   "targetBranchChoices": [
     { "name": "master", "checked": true },
-    { "name": "7.x", "checked": true },
+    "7.16",
     "7.15",
     "7.14",
     "7.13",
@@ -33,7 +33,6 @@
   "targetPRLabels": ["backport"],
   "branchLabelMapping": {
     "^v8.0.0$": "master",
-    "^v7.16.0$": "7.x",
     "^v(\\d+).(\\d+).\\d+$": "$1.$2"
   },
   "autoMerge": true,

.buildkite/pipelines/flaky_tests/pipeline.js

@@ -17,12 +17,6 @@ const inputs = [
     default: 0,
     required: true,
   },
-  {
-    key: 'ftsr-concurrency',
-    text: 'Max concurrency per step',
-    default: 20,
-    required: true,
-  },
 ];
 
 for (let i = 1; i <= OSS_CI_GROUPS; i++) {
@@ -36,7 +30,7 @@ for (let i = 1; i <= XPACK_CI_GROUPS; i++) {
 const pipeline = {
   steps: [
     {
-      input: 'Number of Runs',
+      input: 'Number of Runs - Click Me',
       fields: inputs,
     },
     {

.buildkite/pipelines/flaky_tests/runner.js

@@ -9,8 +9,10 @@ const overrideCount = parseInt(
   execSync(`buildkite-agent meta-data get 'ftsr-override-count'`).toString().trim()
 );
 
-const concurrency =
-  parseInt(execSync(`buildkite-agent meta-data get 'ftsr-concurrency'`).toString().trim()) || 20;
+const concurrency = 25;
+const initialJobs = 3;
+
+let totalJobs = initialJobs;
 
 const testSuites = [];
 for (const key of keys) {
@@ -21,12 +23,25 @@ for (const key of keys) {
   const value =
     overrideCount || execSync(`buildkite-agent meta-data get '${key}'`).toString().trim();
 
+  const count = value === '' ? defaultCount : parseInt(value);
+  totalJobs += count;
+
   testSuites.push({
     key: key.replace('ftsr-suite/', ''),
-    count: value === '' ? defaultCount : parseInt(value),
+    count: count,
   });
 }
 
+if (totalJobs > 500) {
+  console.error('+++ Too many tests');
+  console.error(
+    `Buildkite builds can only contain 500 steps in total. Found ${totalJobs} in total. Make sure your test runs are less than ${
+      500 - initialJobs
+    }`
+  );
+  process.exit(1);
+}
+
 const steps = [];
 const pipeline = {
   env: {
@@ -46,7 +61,7 @@ steps.push({
 for (const testSuite of testSuites) {
   const TEST_SUITE = testSuite.key;
   const RUN_COUNT = testSuite.count;
-  const UUID = TEST_SUITE + process.env.UUID;
+  const UUID = process.env.UUID;
 
   const JOB_PARTS = TEST_SUITE.split('/');
   const IS_XPACK = JOB_PARTS[0] === 'xpack';
@@ -65,6 +80,7 @@ for (const testSuite of testSuites) {
       parallelism: RUN_COUNT,
       concurrency: concurrency,
       concurrency_group: UUID,
+      concurrency_method: 'eager',
     });
   } else {
     steps.push({
@@ -75,6 +91,7 @@ for (const testSuite of testSuites) {
       parallelism: RUN_COUNT,
       concurrency: concurrency,
       concurrency_group: UUID,
+      concurrency_method: 'eager',
     });
   }
 }

.buildkite/pull_requests.json

@@ -15,7 +15,8 @@
       "build_on_comment": true,
       "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))",
       "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))",
-      "labels": ["buildkite-ci"]
+      "skip_ci_labels": ["skip-ci", "jenkins-ci"],
+      "skip_target_branches": ["6.8"]
     }
   ]
 }

.buildkite/scripts/pipelines/pull_request/pipeline.js

@@ -66,12 +66,12 @@ const uploadPipeline = (pipelineContent) => {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/security_solution.yml'));
     }
 
-    // if (
-    //   (await doAnyChangesMatch([/^x-pack\/plugins\/apm/])) ||
-    //   process.env.GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
-    // ) {
-    //   pipeline.push(getPipeline('.buildkite/pipelines/pull_request/apm_cypress.yml'));
-    // }
+    if (
+      (await doAnyChangesMatch([/^x-pack\/plugins\/apm/])) ||
+      process.env.GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
+    ) {
+      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/apm_cypress.yml'));
+    }
 
     if (await doAnyChangesMatch([/^x-pack\/plugins\/uptime/])) {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/uptime.yml'));

.buildkite/scripts/steps/functional/apm_cypress.sh

@@ -2,7 +2,10 @@
 
 set -euo pipefail
 
-source .buildkite/scripts/steps/functional/common.sh
+source .buildkite/scripts/common/util.sh
+
+.buildkite/scripts/bootstrap.sh
+.buildkite/scripts/download_build_artifacts.sh
 
 export JOB=kibana-apm-cypress
 
@@ -11,4 +14,5 @@ echo "--- APM Cypress Tests"
 cd "$XPACK_DIR"
 
 checks-reporter-with-killswitch "APM Cypress Tests" \
-  node plugins/apm/scripts/test/e2e.js
+  node plugins/apm/scripts/test/e2e.js \
+  --kibana-install-dir "$KIBANA_BUILD_LOCATION"

.ci/packer_cache.sh

@@ -8,4 +8,4 @@ if [[ "$(which docker)" != "" && "$(command uname -m)" != "aarch64" ]]; then
 fi
 
 ./.ci/packer_cache_for_branch.sh master
-./.ci/packer_cache_for_branch.sh 7.x
+./.ci/packer_cache_for_branch.sh 7.16

docs/apm/agent-configuration.asciidoc

@@ -23,16 +23,6 @@ However, if APM Server is slow to respond, is offline, reports an error, etc.,
 APM agents will use local defaults until they're able to update the configuration.
 For this reason, it is still essential to set custom default configurations locally in each of your agents.
 
-[float]
-==== APM Server setup
-
-This feature requires {apm-server-ref}/setup-kibana-endpoint.html[Kibana endpoint configuration] in APM Server.
-In addition, if an APM agent is using {apm-server-ref}/configuration-anonymous.html[anonymous authentication] to communicate with the APM Server,
-the agent's service name must be included in the `apm-server.auth.anonymous.allow_service` list.
-
-APM Server acts as a proxy between the agents and Kibana.
-Kibana communicates any changed settings to APM Server so that your agents only need to poll APM Server to determine which settings have changed.
-
 [float]
 ==== Supported configurations
 

docs/apm/api.asciidoc

@@ -563,9 +563,7 @@ More information on Kibana's API is available in <<api,REST API>>.
 === RUM source map API
 
 IMPORTANT: This endpoint is only compatible with the
-{apm-server-ref}/apm-integration.html[APM integration for Elastic Agent].
-Users with a standalone APM Server should instead use the APM Server
-{apm-server-ref}/sourcemap-api.html[source map upload API].
+{apm-guide-ref}/index.html[APM integration for Elastic Agent].
 
 A source map allows minified files to be mapped back to original source code --
 allowing you to maintain the speed advantage of minified code,

docs/apm/apm-app-users.asciidoc

@@ -56,8 +56,8 @@ To create an APM reader user:
 include::./tab-widgets/apm-app-reader/widget.asciidoc[]
 --
 +
-TIP: Using the {apm-server-ref-v}/apm-integration.html[APM integration for Elastic Agent]?
-Add the privileges under the **Data streams** tab.
+TIP: Using the deprecated APM Server binaries?
+Add the privileges under the **Classic APM indices** tab above.
 
 . Assign the `read-apm` role created in the previous step, and the following built-in roles to
 any APM reader users:
@@ -84,8 +84,8 @@ In some instances, you may wish to restrict certain Kibana apps that a user has
 include::./tab-widgets/apm-app-reader/widget.asciidoc[]
 --
 +
-TIP: Using the {apm-server-ref-v}/apm-integration.html[APM integration for Elastic Agent]?
-Add the privileges under the **Data streams** tab.
+TIP: Using the deprecated APM Server binaries?
+Add the privileges under the **Classic APM indices** tab above.
 
 . Assign feature privileges to any Kibana feature that the user needs access to.
 Here are two examples:
@@ -184,8 +184,8 @@ Central configuration users need to be able to view, create, update, and delete
 include::./tab-widgets/central-config-users/widget.asciidoc[]
 --
 +
-TIP: Using the {apm-server-ref-v}/apm-integration.html[APM integration for Elastic Agent]?
-Add the privileges under the **Data streams** tab.
+TIP: Using the deprecated APM Server binaries?
+Add the privileges under the **Classic APM indices** tab above.
 
 . Assign the `central-config-manager` role created in the previous step,
 and the following Kibana feature privileges to anyone who needs to manage central configurations:
@@ -211,8 +211,8 @@ but not create, update, or delete them.
 include::./tab-widgets/central-config-users/widget.asciidoc[]
 --
 +
-TIP: Using the {apm-server-ref-v}/apm-integration.html[APM integration for Elastic Agent]?
-Add the privileges under the **Data streams** tab.
+TIP: Using the deprecated APM Server binaries?
+Add the privileges under the **Classic APM indices** tab above.
 
 . Assign the `central-config-reader` role created in the previous step,
 and the following Kibana feature privileges to anyone who needs to read central configurations:

docs/apm/errors.asciidoc

@@ -2,7 +2,7 @@
 [[errors]]
 === Errors
 
-TIP: {apm-overview-ref-v}/errors.html[Errors] are groups of exceptions with a similar exception or log message.
+TIP: {apm-guide-ref}/data-model-errors.html[Errors] are groups of exceptions with a similar exception or log message.
 
 The *Errors* overview provides a high-level view of the exceptions that APM agents catch,
 or that users manually report with APM agent APIs.

docs/apm/getting-started.asciidoc

@@ -41,7 +41,7 @@ Notice something awry? Select a service or trace and dive deeper with:
 * <<metrics>>
 
 TIP: Want to learn more about the Elastic APM ecosystem?
-See the {apm-get-started-ref}/overview.html[APM Overview].
+See the {apm-guide-ref}/apm-overview.html[APM Overview].
 
 include::services.asciidoc[]
 

docs/apm/service-maps.asciidoc

@@ -41,7 +41,7 @@ We currently surface two types of service maps:
 === How do service maps work?
 
 Service maps rely on distributed traces to draw connections between services.
-As {apm-overview-ref-v}/distributed-tracing.html[distributed tracing] is enabled out-of-the-box for supported technologies, so are service maps.
+As {apm-guide-ref}/apm-distributed-tracing.html[distributed tracing] is enabled out-of-the-box for supported technologies, so are service maps.
 However, if a service isn't instrumented,
 or a `traceparent` header isn't being propagated to it,
 distributed tracing will not work, and the connection will not be drawn on the map.

docs/apm/spans.asciidoc

@@ -16,7 +16,7 @@ You also get a stack trace, which shows the SQL query in your code.
 Finally, APM knows which files are your code and which are just modules or libraries that you've installed.
 These library frames will be minimized by default in order to show you the most relevant stack trace.
 
-TIP: A {apm-overview-ref-v}/transaction-spans.html[span] is the duration of a single event.
+TIP: A {apm-guide-ref}/data-model-spans.html[span] is the duration of a single event.
 Spans are automatically captured by APM agents, and you can also define custom spans.
 Each span has a type and is defined by a different color in the timeline/waterfall visualization.
 

docs/apm/tab-widgets/apm-app-reader/widget.asciidoc

@@ -2,37 +2,37 @@
 <div class="tabs" data-tab-group="apm-app-reader">
   <div role="tablist" aria-label="APM app reader">
     <button role="tab"
-            aria-selected="true"
-            aria-controls="classic-indices-tab"
-            id="classic-indices">
-      Classic APM indices
+          aria-selected="true"
+          aria-controls="data-streams-tab"
+          id="data-streams"
+          tabindex="-1">
+      Data streams
     </button>
     <button role="tab"
             aria-selected="false"
-            aria-controls="data-streams-tab"
-            id="data-streams"
-            tabindex="-1">
-      Data streams
+            aria-controls="classic-indices-tab"
+            id="classic-indices">
+      Classic APM indices
     </button>
   </div>
   <div tabindex="0"
        role="tabpanel"
-       id="classic-indices-tab"
-       aria-labelledby="classic-indices">
+       id="data-streams-tab"
+       aria-labelledby="data-streams"
+       hidden="">
 ++++
 
-include::content.asciidoc[tag=classic-indices]
+include::content.asciidoc[tag=data-streams]
 
 ++++
   </div>
   <div tabindex="0"
        role="tabpanel"
-       id="data-streams-tab"
-       aria-labelledby="data-streams"
-       hidden="">
+       id="classic-indices-tab"
+       aria-labelledby="classic-indices">
 ++++
 
-include::content.asciidoc[tag=data-streams]
+include::content.asciidoc[tag=classic-indices]
 
 ++++
   </div>

docs/apm/tab-widgets/central-config-users/widget.asciidoc

@@ -2,37 +2,37 @@
 <div class="tabs" data-tab-group="central-config-manager">
   <div role="tablist" aria-label="Central config manager">
     <button role="tab"
-            aria-selected="true"
-            aria-controls="classic-indices-tab"
-            id="classic-indices">
-      Classic APM indices
+          aria-selected="true"
+          aria-controls="data-streams-tab"
+          id="data-streams"
+          tabindex="-1">
+      Data streams
     </button>
     <button role="tab"
             aria-selected="false"
-            aria-controls="data-streams-tab"
-            id="data-streams"
-            tabindex="-1">
-      Data streams
+            aria-controls="classic-indices-tab"
+            id="classic-indices">
+      Classic APM indices
     </button>
   </div>
   <div tabindex="0"
        role="tabpanel"
-       id="classic-indices-tab"
-       aria-labelledby="classic-indices">
+       id="data-streams-tab"
+       aria-labelledby="data-streams"
+       hidden="">
 ++++
 
-include::content.asciidoc[tag=classic-indices]
+include::content.asciidoc[tag=data-streams]
 
 ++++
   </div>
   <div tabindex="0"
        role="tabpanel"
-       id="data-streams-tab"
-       aria-labelledby="data-streams"
-       hidden="">
+       id="classic-indices-tab"
+       aria-labelledby="classic-indices">
 ++++
 
-include::content.asciidoc[tag=data-streams]
+include::content.asciidoc[tag=classic-indices]
 
 ++++
   </div>

docs/apm/transactions.asciidoc

@@ -2,7 +2,7 @@
 [[transactions]]
 === Transactions
 
-TIP: A {apm-overview-ref-v}/transactions.html[transaction] describes an event captured by an Elastic APM agent instrumenting a service.
+TIP: A {apm-guide-ref}/data-model-transactions.html[transaction] describes an event captured by an Elastic APM agent instrumenting a service.
 APM agents automatically collect performance metrics on HTTP requests, database queries, and much more.
 
 [role="screenshot"]