Merge branch 'main' into task/update_list_api_summary_endpoint_to_use…

…_filter
elastic · Jan 26, 2022 · 7e2cdb8 · 7e2cdb8
2 parents bdc0c97 + 4d3a792
commit 7e2cdb8
Show file tree

Hide file tree

Showing 236 changed files with 4,055 additions and 1,544 deletions.
diff --git a/.buildkite/pipelines/pull_request/base.yml b/.buildkite/pipelines/pull_request/base.yml
@@ -166,3 +166,10 @@ steps:
       queue: c2-4
     key: storybooks
     timeout_in_minutes: 60
+
+  - command: .buildkite/scripts/steps/webpack_bundle_analyzer/build_and_upload.sh
+    label: 'Build Webpack Bundle Analyzer reports'
+    agents:
+      queue: n2-2
+    key: webpack_bundle_analyzer
+    timeout_in_minutes: 60
diff --git a/.buildkite/pull_requests.json b/.buildkite/pull_requests.json
@@ -16,7 +16,7 @@
       "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))",
       "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))",
       "skip_ci_labels": ["skip-ci", "jenkins-ci"],
-      "skip_target_branches": ["6.8"]
+      "skip_target_branches": ["6.8", "7.11", "7.12"]
     }
   ]
 }
diff --git a/.buildkite/scripts/common/env.sh b/.buildkite/scripts/common/env.sh
@@ -98,12 +98,3 @@ fi
 
 export BUILD_TS_REFS_DISABLE=true
 export DISABLE_BOOTSTRAP_VALIDATION=true
-
-export TEST_KIBANA_HOST=localhost
-export TEST_KIBANA_PORT=6101
-export TEST_KIBANA_URL="http://elastic:changeme@localhost:6101"
-export TEST_ES_URL="http://elastic:changeme@localhost:6102"
-export TEST_ES_TRANSPORT_PORT=6301-6309
-export TEST_CORS_SERVER_PORT=6106
-export ALERTING_PROXY_PORT=6105
-export TEST_PROXY_SERVER_PORT=6107
diff --git a/.buildkite/scripts/steps/webpack_bundle_analyzer/build_and_upload.sh b/.buildkite/scripts/steps/webpack_bundle_analyzer/build_and_upload.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+.buildkite/scripts/bootstrap.sh
+
+node scripts/build_kibana_platform_plugins.js --dist --profile
+
+mkdir -p built_assets/webpack_bundle_analyzer
+find . -path "*target/public/*" -name "stats.json" | while read line; do
+  PLUGIN=$(echo $line | xargs dirname | xargs dirname | xargs dirname | xargs basename)
+  ./node_modules/.bin/webpack-bundle-analyzer $line --report "built_assets/webpack_bundle_analyzer/$PLUGIN.html" --mode static --no-open
+done
+
+node .buildkite/scripts/steps/webpack_bundle_analyzer/upload.js
diff --git a/.buildkite/scripts/steps/webpack_bundle_analyzer/upload.js b/.buildkite/scripts/steps/webpack_bundle_analyzer/upload.js
@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+const execSync = require('child_process').execSync;
+const fs = require('fs');
+const path = require('path');
+
+const GITHUB_CONTEXT = 'Build and Publish Webpack bundle analyzer reports';
+
+const WEBPACK_REPORTS =
+  process.env.BUILDKITE_PULL_REQUEST && process.env.BUILDKITE_PULL_REQUEST !== 'false'
+    ? `pr-${process.env.BUILDKITE_PULL_REQUEST}`
+    : process.env.BUILDKITE_BRANCH.replace('/', '__');
+const WEBPACK_REPORTS_BUCKET = 'ci-artifacts.kibana.dev/webpack_bundle_analyzer';
+const WEBPACK_REPORTS_BUCKET_URL = `https://${WEBPACK_REPORTS_BUCKET}/${WEBPACK_REPORTS}`;
+const WEBPACK_REPORTS_BASE_URL = `${WEBPACK_REPORTS_BUCKET_URL}/${process.env.BUILDKITE_COMMIT}`;
+
+const exec = (...args) => execSync(args.join(' '), { stdio: 'inherit' });
+
+const ghStatus = (state, description) =>
+  exec(
+    `gh api "repos/elastic/kibana/statuses/${process.env.BUILDKITE_COMMIT}"`,
+    `-f state=${state}`,
+    `-f target_url="${process.env.BUILDKITE_BUILD_URL}"`,
+    `-f context="${GITHUB_CONTEXT}"`,
+    `-f description="${description}"`,
+    `--silent`
+  );
+
+const upload = () => {
+  const originalDirectory = process.cwd();
+  process.chdir(path.join('.', 'built_assets', 'webpack_bundle_analyzer'));
+  try {
+    const reports = execSync(`ls -1`).toString().trim().split('\n');
+    const listHtml = reports
+      .map((report) => `<li><a href="${WEBPACK_REPORTS_BASE_URL}/${report}">${report}</a></li>`)
+      .join('\n');
+
+    const html = `
+      <html>
+        <body>
+          <h1>Webpack Bundle Analyzer</h1>
+          <ul>
+            ${listHtml}
+          </ul>
+        </body>
+      </html>
+    `;
+
+    fs.writeFileSync('index.html', html);
+    console.log('--- Uploading Webpack Bundle Analyzer reports');
+    exec(`
+      gsutil -q -m cp -r -z html '*' 'gs://${WEBPACK_REPORTS_BUCKET}/${WEBPACK_REPORTS}/${process.env.BUILDKITE_COMMIT}/'
+      gsutil -h "Cache-Control:no-cache, max-age=0, no-transform" cp -z html 'index.html' 'gs://${WEBPACK_REPORTS_BUCKET}/${WEBPACK_REPORTS}/latest/'
+    `);
+
+    if (process.env.BUILDKITE_PULL_REQUEST && process.env.BUILDKITE_PULL_REQUEST !== 'false') {
+      exec(
+        `buildkite-agent meta-data set pr_comment:webpack_bundle_reports:head '* [Webpack Bundle Analyzer](${WEBPACK_REPORTS_BASE_URL})'`
+      );
+    }
+  } finally {
+    process.chdir(originalDirectory);
+  }
+};
+
+try {
+  ghStatus('pending', 'Building Webpack Bundle Analyzer reports');
+  upload();
+  ghStatus('success', 'Webpack bundle analyzer reports built');
+} catch (error) {
+  ghStatus('error', 'Building Webpack Bundle Analyzer reports failed');
+  throw error;
+}
diff --git a/docs/canvas/canvas-tutorial.asciidoc b/docs/canvas/canvas-tutorial.asciidoc
@@ -26,10 +26,14 @@ To customize your workpad to look the way you want, add your own images.
 +
 The default Elastic logo image appears on the page.
 
-. To add your own image, click the Elastic logo, then drag the image file to the *Select or drag and drop an image* field.
+. Add your own image.
+
+.. Click the Elastic logo.
+
+.. Drag the image file to the *Select or drag and drop an image* field.
 +
 [role="screenshot"]
-image::images/canvas-image-element.png[Image showing how to add the image element]
+image::images/canvas_tutorialCustomImage_7.17.0.png[The Analytics logo added to the workpad]
 
 [float]
 === Customize your data with metrics
@@ -42,9 +46,12 @@ By default, the element is connected to the demo data, which enables you to expe
 
 . To connect the element to your own data source, make sure that the element is selected, click *Data > Demo data > Elasticsearch SQL*.
 
-.. In the *Query* field, enter the following:
+.. In the *Query* field, enter:
 +
-`SELECT sum(taxless_total_price) AS sum_total_price FROM "kibana_sample_data_ecommerce"`
+[source,text]
+--
+SELECT sum(taxless_total_price) AS sum_total_price FROM "kibana_sample_data_ecommerce"
+--
 
 .. Click *Save*.
 +
@@ -54,7 +61,7 @@ The query selects the total price field and sets it to the sum_total_price field
 
 .. Click *Display*
 
-.. From the *Value* drop-down lists, make sure that *Unique* is selected, then select *sum_total_price*.
+.. From the *Value* dropdowns, make sure *Unique* is selected, then select *sum_total_price*.
 
 .. Change the *Label* to `Total sales`.
 
@@ -69,7 +76,7 @@ You're now looking at the raw data syntax that Canvas uses to display the elemen
 .. Click *Run*.
 
 [role="screenshot"]
-image::images/canvas-metric-element.png[Image showing changes to the Canvas workpad]
+image::images/canvas_tutorialCustomMetric_7.17.0.png[The total sales metric added to the workpad using Elasticsearch SQL]
 
 [float]
 === Show off your data with charts
@@ -82,7 +89,10 @@ To show what your data can do, add charts, graphs, progress monitors, and more t
 
 .. To obtain the taxless total price by date, enter the following in the *Query* field:
 +
-`SELECT order_date, taxless_total_price FROM "kibana_sample_data_ecommerce" ORDER BY order_date`
+[source,text]
+--
+SELECT order_date, taxless_total_price FROM "kibana_sample_data_ecommerce" ORDER BY order_date
+--
 
 .. Click *Save*.
 
@@ -95,7 +105,7 @@ To show what your data can do, add charts, graphs, progress monitors, and more t
 .. From the *Y-axis* drop-down lists, select *Value*, then select *taxless_total_price*.
 
 [role="screenshot"]
-image::images/canvas-chart-element.png[Image showing Canvas workpad with sample data graph]
+image::images/canvas_tutorialCustomChart_7.17.0.png[Custom line chart added to the workpad using Elasticsearch SQL]
 
 [float]
 === Show how your data changes over time
@@ -109,7 +119,7 @@ To focus your data on a specific time range, add the time filter.
 . To use the date time field from the sample data, enter `order_date` in the *Column* field, then click *Set*.
 
 [role="screenshot"]
-image::images/canvas-timefilter-element.png[Image showing Canvas workpad with filtered sample data graph]
+image::images/canvas_tutorialCustomTimeFilter_7.17.0.png[Custom time filter added to the workpad]
 
 To see how the data changes, set the time filter to *Last 7 days*. As you change the time filter options, the elements automatically update.
 

diff --git a/docs/canvas/images/canvas-add-pages.gif b/docs/canvas/images/canvas-add-pages.gif
diff --git a/docs/canvas/images/canvas-chart-element.png b/docs/canvas/images/canvas-chart-element.png
diff --git a/docs/canvas/images/canvas-image-element.png b/docs/canvas/images/canvas-image-element.png
diff --git a/docs/canvas/images/canvas-metric-element.png b/docs/canvas/images/canvas-metric-element.png
diff --git a/docs/canvas/images/canvas-timefilter-element.png b/docs/canvas/images/canvas-timefilter-element.png
diff --git a/docs/canvas/images/canvas_logWebTrafficWorkpadTemplate_7.17.0.png b/docs/canvas/images/canvas_logWebTrafficWorkpadTemplate_7.17.0.png
diff --git a/docs/canvas/images/canvas_tutorialCustomChart_7.17.0.png b/docs/canvas/images/canvas_tutorialCustomChart_7.17.0.png
diff --git a/docs/canvas/images/canvas_tutorialCustomImage_7.17.0.png b/docs/canvas/images/canvas_tutorialCustomImage_7.17.0.png
diff --git a/docs/canvas/images/canvas_tutorialCustomMetric_7.17.0.png b/docs/canvas/images/canvas_tutorialCustomMetric_7.17.0.png
diff --git a/docs/canvas/images/canvas_tutorialCustomTimeFilter_7.17.0.png b/docs/canvas/images/canvas_tutorialCustomTimeFilter_7.17.0.png
diff --git a/docs/setup/images/canvas_tutorialCustomTimeFilter_7.17.0.png b/docs/setup/images/canvas_tutorialCustomTimeFilter_7.17.0.png
diff --git a/docs/user/canvas.asciidoc b/docs/user/canvas.asciidoc
@@ -19,14 +19,7 @@ With *Canvas*, you can:
 
 * Focus the data you want to display with filters.
 
-++++
-<iframe width="560" height="315" 
-src="https://www.youtube.com/embed/ZqvF_5-1xjQ" 
-frameborder="0" 
-allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" 
-allowfullscreen></iframe>
-</br>
-++++
+image::images/canvas_logWebTrafficWorkpadTemplate_7.17.0.png[Logs Web Traffic workpad template]
 
 [float]
 [[create-workpads]]
@@ -138,15 +131,13 @@ To access your saved elements, click *Add element > My elements*.
 
 Add a panel that you saved in *Visualize Library* to your workpad. 
 
-. Click *Add element > Add from {kib}*.
-
-. Select the panel you want to add.
+. Click *Add from library*, then select the panel you want to add.
 
 . To use the customization options, open the panel menu, then select one of the following options:
 
 * *Edit map* &mdash; Opens <<maps,Maps>> so that you can edit the panel.
 
-* *Edit visualization* &mdash; Opens the visualization editor so that you can edit the panel.
+* *Edit Visualization* &mdash; Opens the visualization editor so that you can edit the panel.
 
 * *Edit panel title* &mdash; Allows you to change the panel title.
 
@@ -164,7 +155,7 @@ To personalize your workpad, add your own logos and graphics.
 
 . On the *Manage workpad assets* window, drag and drop your images.
 
-. To add the image to the workpad, click the *Create image element* icon.
+. To add the image to the workpad, click *Create image element*.
 
 [float]
 [[add-more-pages]]
@@ -175,9 +166,6 @@ Organize and separate your ideas by adding more pages.
 . Click *Page 1*, then click *+*.
 
 . On the *Page* editor panel, select the page transition from the *Transition* dropdown.
-+
-[role="screenshot"]
-image::images/canvas-add-pages.gif[Add pages]
 
 [float]
 [[workpad-share-options]]

diff --git a/docs/user/security/audit-logging.asciidoc b/docs/user/security/audit-logging.asciidoc
@@ -24,8 +24,9 @@ an <<audit-logging-settings, appender>> to write the audit log to a location of
 
 Refer to the table of events that can be logged for auditing purposes. 
 
-Each event is broken down into <<field-event-category, category>>, <<field-event-type, type>>, <<field-event-action, action>> and <<field-event-outcome, outcome>> fields
-to make it easy to filter, query and aggregate the resulting logs. 
+Each event is broken down into <<field-event-category, category>>, <<field-event-type, type>>, <<field-event-action, action>> and
+<<field-event-outcome, outcome>> fields to make it easy to filter, query and aggregate the resulting logs. The <<field-trace-id, trace.id>>
+field can be used to correlate multiple events that originate from the same request.
 
 Refer to <<xpack-security-ecs-audit-schema>> for a table of fields that get logged with audit event. 
 
@@ -423,7 +424,57 @@ Example: `https`
 | *Field*
 | *Description*
 
-| `trace.id`
+| [[field-trace-id]] `trace.id`
 | Unique identifier allowing events of the same transaction from {kib} and {es} to be be correlated.
 
 |======
+
+[[xpack-security-ecs-audit-correlation]]
+==== Correlating audit events
+
+Audit events can be correlated in two ways:
+
+1. Multiple {kib} audit events that resulted from the same request can be correlated together.
+2. If {ref}/enable-audit-logging.html[{es} audit logging] is enabled, {kib} audit events from one request can be correlated with backend
+   calls that create {es} audit events.
+
+NOTE: The examples below are simplified, many fields have been omitted and values have been shortened for clarity.
+
+===== Example 1: correlating multiple {kib} audit events
+
+When "thom" creates a new alerting rule, five audit events are written:
+
+[source,json]
+-------------
+{"event":{"action":"http_request","category":["web"],"outcome":"unknown"},"http":{"request":{"method":"post"}},"url":{"domain":"localhost","path":"/api/alerting/rule","port":5601,"scheme":"https"},"user":{"name":"thom","roles":["superuser"]},"kibana":{"space_id":"default","session_id":"3dHCZRB..."},"@timestamp":"2022-01-25T13:05:34.449-05:00","message":"User is requesting [/api/alerting/rule] endpoint","trace":{"id":"e300e06..."}}
+{"event":{"action":"space_get","category":["database"],"type":["access"],"outcome":"success"},"kibana":{"space_id":"default","session_id":"3dHCZRB...","saved_object":{"type":"space","id":"default"}},"user":{"name":"thom","roles":["superuser"]},"@timestamp":"2022-01-25T13:05:34.454-05:00","message":"User has accessed space [id=default]","trace":{"id":"e300e06..."}}
+{"event":{"action":"connector_get","category":["database"],"type":["access"],"outcome":"success"},"kibana":{"space_id":"default","session_id":"3dHCZRB...","saved_object":{"type":"action","id":"5e3b1ae..."}},"user":{"name":"thom","roles":["superuser"]},"@timestamp":"2022-01-25T13:05:34.948-05:00","message":"User has accessed connector [id=5e3b1ae...]","trace":{"id":"e300e06..."}}
+{"event":{"action":"connector_get","category":["database"],"type":["access"],"outcome":"success"},"kibana":{"space_id":"default","session_id":"3dHCZRB...","saved_object":{"type":"action","id":"5e3b1ae..."}},"user":{"name":"thom","roles":["superuser"]},"@timestamp":"2022-01-25T13:05:34.956-05:00","message":"User has accessed connector [id=5e3b1ae...]","trace":{"id":"e300e06..."}}
+{"event":{"action":"rule_create","category":["database"],"type":["creation"],"outcome":"unknown"},"kibana":{"space_id":"default","session_id":"3dHCZRB...","saved_object":{"type":"alert","id":"64517c3..."}},"user":{"name":"thom","roles":["superuser"]},"@timestamp":"2022-01-25T13:05:34.956-05:00","message":"User is creating rule [id=64517c3...]","trace":{"id":"e300e06..."}}
+-------------
+
+All of these audit events can be correlated together by the same `trace.id` value `"e300e06..."`. The first event is the HTTP API call, the
+next audit events are checks to validate the space and the connectors, and the last audit event is the actual rule creation.
+
+===== Example 2: correlating a {kib} audit event with {es} audit events
+
+When "thom" logs in, a "user_login" {kib} audit event is written:
+
+[source,json]
+-------------
+{"event":{"action":"user_login","category":["authentication"],"outcome":"success"},"user":{"name":"thom","roles":["superuser"]},"@timestamp":"2022-01-25T09:40:39.267-05:00","message":"User [thom] has logged in using basic provider [name=basic]","trace":{"id":"818cbf3..."}}
+-------------
+
+The `trace.id` value `"818cbf3..."` in the {kib} audit event can be correlated with the `opaque_id` value in these six {es} audit events:
+
+[source,json]
+-------------
+{"type":"audit", "timestamp":"2022-01-25T09:40:38,604-0500", "event.action":"access_granted", "user.name":"thom", "user.roles":["superuser"], "request.id":"YCx8wxs...", "action":"cluster:admin/xpack/security/user/authenticate", "request.name":"AuthenticateRequest", "opaque_id":"818cbf3..."}
+{"type":"audit", "timestamp":"2022-01-25T09:40:38,613-0500", "event.action":"access_granted", "user.name":"kibana_system", "user.roles":["kibana_system"], "request.id":"Ksx73Ad...", "action":"indices:data/write/index", "request.name":"IndexRequest", "indices":[".kibana_security_session_1"], "opaque_id":"818cbf3..."}
+{"type":"audit", "timestamp":"2022-01-25T09:40:38,613-0500", "event.action":"access_granted", "user.name":"kibana_system", "user.roles":["kibana_system"], "request.id":"Ksx73Ad...", "action":"indices:data/write/bulk", "request.name":"BulkRequest", "opaque_id":"818cbf3..."}
+{"type":"audit", "timestamp":"2022-01-25T09:40:38,613-0500", "event.action":"access_granted", "user.name":"kibana_system", "user.roles":["kibana_system"], "request.id":"Ksx73Ad...", "action":"indices:data/write/bulk[s]", "request.name":"BulkShardRequest", "indices":[".kibana_security_session_1"], "opaque_id":"818cbf3..."}
+{"type":"audit", "timestamp":"2022-01-25T09:40:38,613-0500", "event.action":"access_granted", "user.name":"kibana_system", "user.roles":["kibana_system"], "request.id":"Ksx73Ad...", "action":"indices:data/write/index:op_type/create", "request.name":"BulkItemRequest", "indices":[".kibana_security_session_1"], "opaque_id":"818cbf3..."}
+{"type":"audit", "timestamp":"2022-01-25T09:40:38,613-0500", "event.action":"access_granted", "user.name":"kibana_system", "user.roles":["kibana_system"], "request.id":"Ksx73Ad...", "action":"indices:data/write/bulk[s][p]", "request.name":"BulkShardRequest", "indices":[".kibana_security_session_1"], "opaque_id":"818cbf3..."}
+-------------
+
+The {es} audit events show that "thom" authenticated, then subsequently "kibana_system" created a session for that user.
diff --git a/package.json b/package.json
@@ -722,7 +722,7 @@
     "callsites": "^3.1.0",
     "chai": "3.5.0",
     "chance": "1.0.18",
-    "chromedriver": "^97.0.0",
+    "chromedriver": "^97.0.2",
     "clean-webpack-plugin": "^3.0.0",
     "cmd-shim": "^2.1.0",
     "compression-webpack-plugin": "^4.0.0",
@@ -881,6 +881,7 @@
     "wait-on": "^5.2.1",
     "watchpack": "^1.6.0",
     "webpack": "^4.41.5",
+    "webpack-bundle-analyzer": "^4.5.0",
     "webpack-cli": "^3.3.12",
     "webpack-dev-server": "^3.11.0",
     "webpack-merge": "^4.2.2",