Merge branch 'master' into bump-decompress-package-to-4.2.1

.ci/Jenkinsfile_coverage

@@ -11,8 +11,11 @@ kibanaPipeline(timeoutMinutes: 240) {
       'CODE_COVERAGE=1', // Enables coverage.  Needed for multiple ci scripts, such as remote.ts, test/scripts/*.sh, schema.js, etc.
     ]) {
       workers.base(name: 'coverage-worker', size: 'l', ramDisk: false, bootstrapped: false) {
-        kibanaCoverage.runTests()
-        handleIngestion(TIME_STAMP)
+        catchError {
+          kibanaCoverage.runTests()
+          handleIngestion(TIME_STAMP)
+        }
+        handleFail()
       }
     }
     kibanaPipeline.sendMail()
@@ -29,4 +32,13 @@ def handleIngestion(timestamp) {
   kibanaCoverage.uploadCoverageStaticSite(timestamp)
 }
 
+def handleFail() {
+  def buildStatus = buildUtils.getBuildStatus()
+  if(params.NOTIFY_ON_FAILURE && buildStatus != 'SUCCESS' && buildStatus != 'ABORTED') {
+    slackNotifications.sendFailedBuild(
+      channel: '#kibana-qa',
+      username: 'Kibana QA'
+    )
+  }
+}
 

Jenkinsfile

@@ -41,6 +41,7 @@ kibanaPipeline(timeoutMinutes: 155, checkPrChanges: true) {
             'xpack-ciGroup9': kibanaPipeline.xpackCiGroupProcess(9),
             'xpack-ciGroup10': kibanaPipeline.xpackCiGroupProcess(10),
             'xpack-accessibility': kibanaPipeline.functionalTestProcess('xpack-accessibility', './test/scripts/jenkins_xpack_accessibility.sh'),
+            'xpack-pageLoadMetrics': kibanaPipeline.functionalTestProcess('xpack-pageLoadMetrics', './test/scripts/jenkins_xpack_page_load_metrics.sh'),
             'xpack-securitySolutionCypress': { processNumber ->
               whenChanged(['x-pack/plugins/security_solution/', 'x-pack/test/security_solution_cypress/']) {
                 kibanaPipeline.functionalTestProcess('xpack-securitySolutionCypress', './test/scripts/jenkins_security_solution_cypress.sh')(processNumber)

docs/management/managing-remote-clusters.asciidoc

@@ -31,6 +31,10 @@ to reproduce indices in the remote cluster on a local cluster.
 [role="screenshot"]
 image::images/add_remote_cluster.png[][UI for adding a remote cluster]
 
+To create an index pattern to search across clusters,
+use the same syntax that you’d use in a raw cross-cluster search request in {es}: <cluster-names>:<pattern>.
+See <<management-cross-cluster-search, Creating an index pattern>> for examples.
+
 [float]
 [[manage-remote-clusters]]
 === Manage remote clusters

docs/setup/connect-to-elasticsearch.asciidoc

@@ -1,44 +1,105 @@
 [[connect-to-elasticsearch]]
-== Connect Kibana with Elasticsearch
+== Adding data
 
-Before you can start using Kibana, you need to tell it which Elasticsearch indices you want to explore.
-The first time you access Kibana, you are prompted to define an _index pattern_ that matches the name of
-one or more of your indices. That's it. That's all you need to configure to start using Kibana. You can
-add index patterns at any time from the <<settings-create-pattern,Management tab>>.
+To start working with your data in {kib}, you can:
 
-TIP: By default, Kibana connects to the Elasticsearch instance running on `localhost`. To connect to a
-different Elasticsearch instance, modify the Elasticsearch URL in the `kibana.yml` configuration file and
-restart Kibana. For information about using Kibana with your production nodes, see <<production>>.
+* Upload a CSV, JSON, or log file with the File Data Visualizer.
 
-To configure the Elasticsearch indices you want to access with Kibana:
+* Upload geospatial data with the GeoJSON Upload feature.
 
-. Point your browser at port 5601 to access the Kibana UI. For example, `localhost:5601` or
-`http://YOURDOMAIN.com:5601`.
-+
-image:images/Start-Page.png[Kibana start page]
-+
-. Specify an index pattern that matches the name of one or more of your Elasticsearch indices. The pattern
-can include an asterisk (*) to matches zero or more characters in an index's name. When filling out your
-index pattern, any matched indices will be displayed.
-. Click *Next Step* to select the index field that contains the timestamp you want to use to perform time-based
-comparisons. Kibana reads the index mapping to list all of the fields that contain a timestamp. If your
-index doesn't have time-based data, choose *I don't want to use the Time Filter* option.
-+
-. Click *Create index pattern* to add the index pattern. This first pattern is automatically configured as the default.
-When you have more than one index pattern, you can designate which one to use as the default by clicking
-on the star icon above the index pattern title from *Management > Index Patterns*.
+* Index logs, metrics, events, or application data by setting up a Beats module.
+
+* Connect {kib} with existing {es} indices.
+
+If you're not ready to use your own data, you can add a <<get-data-in, sample data set>>
+to see all that you can do in {kib}.
+
+[float]
+[[upload-data-kibana]]
+=== Upload a CSV, JSON, or log file
+
+To visualize data in a CSV, JSON, or log file, you can
+upload it using the File Data Visualizer.  On the home page,
+click *Import a CSV, NDSON, or log file*, and then drag your file into the
+File Data Visualizer.
+
+You can upload a file up to 100 MB. This value is configurable up to 1 GB in
+<<kibana-ml-settings, Advanced Settings>>.
+
+[role="screenshot"]
+image::images/add-data-fv.png[File Data Visualizer]
+
+The File Data Visualizer uses the {ref}/ml-find-file-structure.html[find_file_structure API] to analyze
+the uploaded file and to suggest ingest pipelines and mappings for your data.
+
+NOTE: This feature is not intended for use as part of a
+repeated production process, but rather for the initial exploration of your data.
+
+[float]
+[[upload-geoipdata-kibana]]
+=== Upload geospatial data
+
+To visualize geospatial data in a point or shape file, you can upload it using the <<geojson-upload, GeoJSON Upload>>
+feature in *Elastic Maps*, and then use that data as a layer in a map.
+The data is also available for use in the broader Kibana ecosystem, for example,
+in visualizations and Canvas workpads.
+With GeoJSON Upload, you can upload a file up to 50 MB.
+
+[role="screenshot"]
+image::images/fu_gs_select_source_file_upload.png[]
 
-All done! Kibana is now connected to your Elasticsearch data. Kibana displays a read-only list of fields
-configured for the matching index.
 
 [float]
-[[explore]]
-=== Start Exploring your Data!
-You're ready to dive in to your data:
+[[add-data-tutorial-kibana]]
+===  Index metrics, log, security, and application data
 
-* Search and browse your data interactively from the <<discover, Discover>> page.
-* Chart and map your data from the <<visualize, Visualize>> page.
-* Create and view custom dashboards from the <<dashboard, Dashboard>> page.
+The built-in data tutorials can help you quickly get up and running with
+metrics data, log analytics, security events, and application data.
+These tutorials walk you through installing and configuring a
+Beats data shipper to periodically collect and send data to {es}.
+You can then use the pre-built dashboards to explore and analyze the data.
 
-For a step-by-step introduction to these core Kibana concepts, see the <<getting-started,
-Getting Started>> tutorial.
+You access the tutorials from the home page.
+If a tutorial doesn’t exist for your data, go to the {beats-ref}/beats-reference.html[Beats overview]
+to learn about other data shippers in the Beats family.
+
+[role="screenshot"]
+image::images/add-data-tutorials.png[Add Data tutorials]
+
+
+[float]
+[[connect-to-es]]
+=== Connect with {es} indices
+
+To visualize data in existing {es} indices, you must
+create an index pattern that matches the names of the indices that you want to explore.
+When you add data with the File Data Visualizer, GeoJSON Upload feature,
+or built-in tutorial, an index pattern is created for you.
+
+. Go to *Stack Management*, and then click *Index Patterns*.
+
+. Click *Create index pattern*.
+
+. Specify an index pattern that matches the name of one or more of your Elasticsearch indices.
++
+For example, an index pattern can point to your Apache data from yesterday,
+`filebeat-apache-4-3-2022`, or any index that matches the pattern, `filebeat-*`.
+Using a wildcard is the more popular approach.
+
+
+. Click *Next Step*, and then select the index field that contains the timestamp you want to use to perform time-based
+comparisons.
++
+Kibana reads the index mapping and lists all fields that contain a timestamp. If your
+index doesn't have time-based data, choose *I don't want to use the Time Filter*.
++
+You must select a time field to use global time filters on your dashboards.
+
+. Click *Create index pattern*.
++
+{kib} is now configured to access your {es} indices.
+You’ll see a list of fields configured for the matching index.
+You can designate your index pattern as the default by clicking the star icon on this page.
++
+When searching in *Discover* and creating visualizations, you choose a pattern
+from the index pattern menu to specify the {es} indices that contain the data you want to explore.

package.json

@@ -429,7 +429,7 @@
     "eslint-plugin-prefer-object-spread": "^1.2.1",
     "eslint-plugin-prettier": "^3.1.3",
     "eslint-plugin-react": "^7.17.0",
-    "eslint-plugin-react-hooks": "^2.3.0",
+    "eslint-plugin-react-hooks": "^4.0.4",
     "eslint-plugin-react-perf": "^3.2.3",
     "exit-hook": "^2.2.0",
     "faker": "1.1.0",

packages/kbn-test/package.json

@@ -14,6 +14,7 @@
     "@kbn/babel-preset": "1.0.0",
     "@kbn/dev-utils": "1.0.0",
     "@types/parse-link-header": "^1.0.0",
+    "@types/puppeteer": "^3.0.0",
     "@types/strip-ansi": "^5.2.1",
     "@types/xml2js": "^0.4.5",
     "diff": "^4.0.1"
@@ -25,6 +26,7 @@
     "getopts": "^2.2.4",
     "glob": "^7.1.2",
     "parse-link-header": "^1.0.1",
+    "puppeteer": "^3.3.0",
     "strip-ansi": "^5.2.0",
     "rxjs": "^6.5.3",
     "tar-fs": "^1.16.3",

packages/kbn-test/src/index.ts

@@ -58,3 +58,5 @@ export { runFailedTestsReporterCli } from './failed_tests_reporter';
 export { makeJunitReportPath } from './junit_report_path';
 
 export { CI_PARALLEL_PROCESS_PREFIX } from './ci_parallel_process_prefix';
+
+export * from './page_load_metrics';

packages/kbn-test/src/page_load_metrics/capture_page_load_metrics.ts

@@ -0,0 +1,81 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { ToolingLog } from '@kbn/dev-utils';
+import { NavigationOptions, createUrl, navigateToApps } from './navigation';
+
+export async function capturePageLoadMetrics(log: ToolingLog, options: NavigationOptions) {
+  const responsesByPageView = await navigateToApps(log, options);
+
+  const assetSizeMeasurements = new Map<string, number[]>();
+
+  const numberOfPagesVisited = responsesByPageView.size;
+
+  for (const [, frameResponses] of responsesByPageView) {
+    for (const [, { url, dataLength }] of frameResponses) {
+      if (url.length === 0) {
+        throw new Error('navigateToApps(); failed to identify the url of the request');
+      }
+      if (assetSizeMeasurements.has(url)) {
+        assetSizeMeasurements.set(url, [dataLength].concat(assetSizeMeasurements.get(url) || []));
+      } else {
+        assetSizeMeasurements.set(url, [dataLength]);
+      }
+    }
+  }
+
+  return Array.from(assetSizeMeasurements.entries())
+    .map(([url, measurements]) => {
+      const baseUrl = createUrl('/', options.appConfig.url);
+      const relativeUrl = url
+        // remove the baseUrl (expect the trailing slash) to make url relative
+        .replace(baseUrl.slice(0, -1), '')
+        // strip the build number from asset urls
+        .replace(/^\/\d+\//, '/');
+      return [relativeUrl, measurements] as const;
+    })
+    .filter(([url, measurements]) => {
+      if (measurements.length !== numberOfPagesVisited) {
+        // ignore urls seen only on some pages
+        return false;
+      }
+
+      if (url.startsWith('data:')) {
+        // ignore data urls since they are already counted by other assets
+        return false;
+      }
+
+      if (url.startsWith('/api/') || url.startsWith('/internal/')) {
+        // ignore api requests since they don't have deterministic sizes
+        return false;
+      }
+
+      const allMetricsAreEqual = measurements.every((x, i) =>
+        i === 0 ? true : x === measurements[i - 1]
+      );
+      if (!allMetricsAreEqual) {
+        throw new Error(`measurements for url [${url}] are not equal [${measurements.join(',')}]`);
+      }
+
+      return true;
+    })
+    .map(([url, measurements]) => {
+      return { group: 'page load asset size', id: url, value: measurements[0] };
+    });
+}

packages/kbn-test/src/page_load_metrics/cli.ts

@@ -0,0 +1,90 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import Url from 'url';
+
+import { run, createFlagError } from '@kbn/dev-utils';
+import { resolve, basename } from 'path';
+import { capturePageLoadMetrics } from './capture_page_load_metrics';
+
+const defaultScreenshotsDir = resolve(__dirname, 'screenshots');
+
+export function runPageLoadMetricsCli() {
+  run(
+    async ({ flags, log }) => {
+      const kibanaUrl = flags['kibana-url'];
+      if (!kibanaUrl || typeof kibanaUrl !== 'string') {
+        throw createFlagError('Expect --kibana-url to be a string');
+      }
+
+      const parsedUrl = Url.parse(kibanaUrl);
+
+      const [username, password] = parsedUrl.auth
+        ? parsedUrl.auth.split(':')
+        : [flags.username, flags.password];
+
+      if (typeof username !== 'string' || typeof password !== 'string') {
+        throw createFlagError(
+          'Mising username and/or password, either specify in --kibana-url or pass --username and --password'
+        );
+      }
+
+      const headless = !flags.head;
+
+      const screenshotsDir = flags.screenshotsDir || defaultScreenshotsDir;
+
+      if (typeof screenshotsDir !== 'string' || screenshotsDir === basename(screenshotsDir)) {
+        throw createFlagError('Expect screenshotsDir to be valid path string');
+      }
+
+      const metrics = await capturePageLoadMetrics(log, {
+        headless,
+        appConfig: {
+          url: kibanaUrl,
+          username,
+          password,
+        },
+        screenshotsDir,
+      });
+      for (const metric of metrics) {
+        log.info(`${metric.id}: ${metric.value}`);
+      }
+    },
+    {
+      description: `Loads several pages with Puppeteer to capture the size of assets`,
+      flags: {
+        string: ['kibana-url', 'username', 'password', 'screenshotsDir'],
+        boolean: ['head'],
+        default: {
+          username: 'elastic',
+          password: 'changeme',
+          debug: true,
+          screenshotsDir: defaultScreenshotsDir,
+        },
+        help: `
+          --kibana-url       Url for Kibana we should connect to, can include login info
+          --head             Run puppeteer with graphical user interface
+          --username         Set username, defaults to 'elastic'
+          --password         Set password, defaults to 'changeme'
+          --screenshotsDir   Set screenshots directory, defaults to '${defaultScreenshotsDir}'
+        `,
+      },
+    }
+  );
+}