Merge branch '7.x' into ua/7.x/stats-component

docs/development/core/public/kibana-plugin-core-public.doclinksstart.links.md

@@ -190,6 +190,9 @@ readonly links: {
         }>;
         readonly observability: Readonly<{
             guide: string;
+            infrastructureThreshold: string;
+            logsThreshold: string;
+            metricsThreshold: string;
             monitorStatus: string;
             monitorUptime: string;
             tlsCertificate: string;

docs/management/action-types.asciidoc

@@ -135,5 +135,14 @@ image::images/connectors-with-missing-secrets.png[Connectors with missing secret
 For out-of-the-box and standardized connectors, you can <<preconfigured-connector-example, preconfigure connectors>>
 before {kib} starts.
 
+[float]
+[[montoring-connectors]]
+=== Monitoring connectors
+
+The <<task-manager-health-monitoring,Task Manager health API>> helps you understand the performance of all tasks in your environment.
+However, if connectors fail to execute, they will report as successful to Task Manager. The failure stats will not
+accurately depict the performance of connectors.
+
+For more information on connector successes and failures, refer to the <<event-log-index,Event log index>>.
 
 include::connectors/index.asciidoc[]

docs/user/production-considerations/task-manager-health-monitoring.asciidoc

@@ -111,6 +111,7 @@ a| Runtime
 
 | This section tracks excution performance of Task Manager, tracking task _drift_, worker _load_, and execution stats broken down by type, including duration and execution results.
 
+
 a| Capacity Estimation
 
 | This section provides a rough estimate about the sufficiency of its capacity. As the name suggests, these are estimates based on historical data and should not be used as predictions. Use these estimations when following the Task Manager <<task-manager-scaling-guidance>>.
@@ -123,6 +124,14 @@ The root `status` indicates the `status` of the system overall.
 
 The Runtime `status` indicates whether task executions have exceeded any of the <<task-manager-configuring-health-monitoring,configured health thresholds>>. An `OK` status means none of the threshold have been exceeded. A `Warning` status means that at least one warning threshold has been exceeded. An `Error` status means that at least one error threshold has been exceeded.
 
+[IMPORTANT]
+==============================================
+Some tasks (such as <<action-types,connectors>>) will incorrectly report their status as successful even if the task failed.
+The runtime and workload block will return data about success and failures and will not take this into consideration.
+
+To get a better sense of action failures, please refer to the <<event-log-index,Event log index>> for more accurate context into failures and successes.
+==============================================
+
 The Capacity Estimation `status` indicates the sufficiency of the observed capacity. An `OK` status means capacity is sufficient. A `Warning` status means that capacity is sufficient for the scheduled recurring tasks, but non-recurring tasks often cause the cluster to exceed capacity. An `Error` status means that there is insufficient capacity across all types of tasks.
 
 By monitoring the `status` of the system overall, and the `status` of specific task types of interest, you can evaluate the health of the {kib} Task Management system.

package.json

@@ -97,7 +97,7 @@
     "@elastic/datemath": "link:bazel-bin/packages/elastic-datemath",
     "@elastic/elasticsearch": "npm:@elastic/elasticsearch-canary@^7.16.0-canary.2",
     "@elastic/ems-client": "7.15.0",
-    "@elastic/eui": "37.3.1",
+    "@elastic/eui": "37.6.0",
     "@elastic/filesaver": "1.1.2",
     "@elastic/good": "^9.0.1-kibana3",
     "@elastic/maki": "6.3.0",
@@ -388,7 +388,7 @@
     "suricata-sid-db": "^1.0.2",
     "symbol-observable": "^1.2.0",
     "tabbable": "1.1.3",
-    "tar": "4.4.13",
+    "tar": "^6.1.11",
     "tinycolor2": "1.4.1",
     "tinygradient": "0.4.3",
     "topojson-client": "3.1.0",
@@ -400,6 +400,7 @@
     "typescript-fsa-reducers": "^1.2.2",
     "unified": "^9.2.1",
     "use-resize-observer": "^6.0.0",
+    "usng.js": "^0.4.5",
     "utility-types": "^3.10.0",
     "uuid": "3.3.2",
     "vega": "^5.19.1",
@@ -621,7 +622,7 @@
     "@types/styled-components": "^5.1.0",
     "@types/supertest": "^2.0.5",
     "@types/tapable": "^1.0.6",
-    "@types/tar": "^4.0.3",
+    "@types/tar": "^4.0.5",
     "@types/tar-fs": "^1.16.1",
     "@types/tempy": "^0.2.0",
     "@types/testing-library__jest-dom": "^5.9.5",

packages/kbn-es/src/cluster.js

@@ -278,7 +278,8 @@ exports.Cluster = class Cluster {
     // especially because we currently run many instances of ES on the same machine during CI
     // inital and max must be the same, so we only need to check the max
     if (!esJavaOpts.includes('Xmx')) {
-      esJavaOpts += ' -Xms1g -Xmx1g';
+      // 1536m === 1.5g
+      esJavaOpts += ' -Xms1536m -Xmx1536m';
     }
 
     this._log.debug('ES_JAVA_OPTS: %s', esJavaOpts.trim());

packages/kbn-es/src/integration_tests/cluster.test.js

@@ -368,7 +368,7 @@ describe('#run()', () => {
     const cluster = new Cluster({ log });
     await cluster.run();
 
-    expect(execa.mock.calls[0][2].env.ES_JAVA_OPTS).toEqual('-Xms1g -Xmx1g');
+    expect(execa.mock.calls[0][2].env.ES_JAVA_OPTS).toMatchInlineSnapshot(`"-Xms1536m -Xmx1536m"`);
   });
 
   it('allows Java heap to be overwritten', async () => {
@@ -378,7 +378,7 @@ describe('#run()', () => {
     const cluster = new Cluster({ log });
     await cluster.run();
 
-    expect(execa.mock.calls[0][2].env.ES_JAVA_OPTS).toEqual('-Xms5g -Xmx5g');
+    expect(execa.mock.calls[0][2].env.ES_JAVA_OPTS).toMatchInlineSnapshot(`"-Xms5g -Xmx5g"`);
   });
 });
 

packages/kbn-utils/src/path/index.ts

@@ -15,15 +15,15 @@ const isString = (v: any): v is string => typeof v === 'string';
 
 const CONFIG_PATHS = [
   process.env.KBN_PATH_CONF && join(process.env.KBN_PATH_CONF, 'kibana.yml'),
-  process.env.KIBANA_PATH_CONF && join(process.env.KIBANA_PATH_CONF, 'kibana.yml'),
+  process.env.KIBANA_PATH_CONF && join(process.env.KIBANA_PATH_CONF, 'kibana.yml'), // deprecated
   process.env.CONFIG_PATH, // deprecated
   join(REPO_ROOT, 'config/kibana.yml'),
   '/etc/kibana/kibana.yml',
 ].filter(isString);
 
 const CONFIG_DIRECTORIES = [
   process.env.KBN_PATH_CONF,
-  process.env.KIBANA_PATH_CONF,
+  process.env.KIBANA_PATH_CONF, // deprecated
   join(REPO_ROOT, 'config'),
   '/etc/kibana',
 ].filter(isString);

src/core/public/doc_links/doc_links_service.ts

@@ -284,6 +284,9 @@ export class DocLinksService {
         },
         observability: {
           guide: `${ELASTIC_WEBSITE_URL}guide/en/observability/${DOC_LINK_VERSION}/index.html`,
+          infrastructureThreshold: `{ELASTIC_WEBSITE_URL}guide/en/observability/${DOC_LINK_VERSION}/infrastructure-threshold-alert.html`,
+          logsThreshold: `{ELASTIC_WEBSITE_URL}guide/en/observability/${DOC_LINK_VERSION}/logs-threshold-alert.html`,
+          metricsThreshold: `{ELASTIC_WEBSITE_URL}guide/en/observability/${DOC_LINK_VERSION}/metrics-threshold-alert.html`,
           monitorStatus: `${ELASTIC_WEBSITE_URL}guide/en/observability/${DOC_LINK_VERSION}/monitor-status-alert.html`,
           monitorUptime: `${ELASTIC_WEBSITE_URL}guide/en/observability/${DOC_LINK_VERSION}/monitor-uptime.html`,
           tlsCertificate: `${ELASTIC_WEBSITE_URL}guide/en/observability/${DOC_LINK_VERSION}/tls-certificate-alert.html`,
@@ -647,6 +650,9 @@ export interface DocLinksStart {
     }>;
     readonly observability: Readonly<{
       guide: string;
+      infrastructureThreshold: string;
+      logsThreshold: string;
+      metricsThreshold: string;
       monitorStatus: string;
       monitorUptime: string;
       tlsCertificate: string;

src/core/public/public.api.md

@@ -656,6 +656,9 @@ export interface DocLinksStart {
         }>;
         readonly observability: Readonly<{
             guide: string;
+            infrastructureThreshold: string;
+            logsThreshold: string;
+            metricsThreshold: string;
             monitorStatus: string;
             monitorUptime: string;
             tlsCertificate: string;

src/core/server/config/deprecation/core_deprecations.test.ts

@@ -36,6 +36,24 @@ describe('core deprecations', () => {
     });
   });
 
+  describe('kibanaPathConf', () => {
+    it('logs a warning if KIBANA_PATH_CONF environ variable is set', () => {
+      process.env.KIBANA_PATH_CONF = 'somepath';
+      const { messages } = applyCoreDeprecations();
+      expect(messages).toMatchInlineSnapshot(`
+        Array [
+          "Environment variable \\"KIBANA_PATH_CONF\\" is deprecated. It has been replaced with \\"KBN_PATH_CONF\\" pointing to a config folder",
+        ]
+      `);
+    });
+
+    it('does not log a warning if KIBANA_PATH_CONF environ variable is unset', () => {
+      delete process.env.KIBANA_PATH_CONF;
+      const { messages } = applyCoreDeprecations();
+      expect(messages).toHaveLength(0);
+    });
+  });
+
   describe('dataPath', () => {
     it('logs a warning if DATA_PATH environ variable is set', () => {
       process.env.DATA_PATH = 'somepath';

src/core/server/config/deprecation/core_deprecations.ts

@@ -8,6 +8,19 @@
 
 import { ConfigDeprecationProvider, ConfigDeprecation } from '@kbn/config';
 
+const kibanaPathConf: ConfigDeprecation = (settings, fromPath, addDeprecation) => {
+  if (process.env?.KIBANA_PATH_CONF) {
+    addDeprecation({
+      message: `Environment variable "KIBANA_PATH_CONF" is deprecated. It has been replaced with "KBN_PATH_CONF" pointing to a config folder`,
+      correctiveActions: {
+        manualSteps: [
+          'Use "KBN_PATH_CONF" instead of "KIBANA_PATH_CONF" to point to a config folder.',
+        ],
+      },
+    });
+  }
+};
+
 const configPathDeprecation: ConfigDeprecation = (settings, fromPath, addDeprecation) => {
   if (process.env?.CONFIG_PATH) {
     addDeprecation({
@@ -442,6 +455,7 @@ export const coreDeprecationProvider: ConfigDeprecationProvider = ({
   unusedFromRoot('elasticsearch.startupTimeout'),
   rewriteCorsSettings,
   configPathDeprecation,
+  kibanaPathConf,
   dataPathDeprecation,
   rewriteBasePathDeprecation,
   cspRulesDeprecation,

src/dev/license_checker/config.ts

@@ -75,7 +75,7 @@ export const LICENSE_OVERRIDES = {
   '@mapbox/[email protected]': ['MIT'], // license in readme https://github.com/tmcw/jsonlint
   '[email protected]': ['(GPL-2.0 OR MIT)'], // GPL-2.0* https://github.com/taozhi8833998/node-sql-parser
   '@elastic/[email protected]': ['Elastic License 2.0'],
-  '@elastic/eui@37.3.1': ['SSPL-1.0 OR Elastic License 2.0'],
+  '@elastic/eui@37.6.0': ['SSPL-1.0 OR Elastic License 2.0'],
 
   // TODO can be removed if the https://github.com/jindw/xmldom/issues/239 is released
   '[email protected]': ['MIT'],