diff --git a/docs/development/core/server/kibana-plugin-core-server.md b/docs/development/core/server/kibana-plugin-core-server.md index 89330d2a86f76..dfffdffb08a08 100644 --- a/docs/development/core/server/kibana-plugin-core-server.md +++ b/docs/development/core/server/kibana-plugin-core-server.md @@ -123,7 +123,7 @@ The plugin integrates with the core system via lifecycle events: `setup` | [LoggerFactory](./kibana-plugin-core-server.loggerfactory.md) | The single purpose of LoggerFactory interface is to define a way to retrieve a context-based logger instance. | | [LoggingServiceSetup](./kibana-plugin-core-server.loggingservicesetup.md) | Provides APIs to plugins for customizing the plugin's logger. | | [LogMeta](./kibana-plugin-core-server.logmeta.md) | Contextual metadata | -| [MetricsServiceSetup](./kibana-plugin-core-server.metricsservicesetup.md) | | +| [MetricsServiceSetup](./kibana-plugin-core-server.metricsservicesetup.md) | APIs to retrieves metrics gathered and exposed by the core platform. | | [NodesVersionCompatibility](./kibana-plugin-core-server.nodesversioncompatibility.md) | | | [OnPostAuthToolkit](./kibana-plugin-core-server.onpostauthtoolkit.md) | A tool set defining an outcome of OnPostAuth interceptor for incoming request. | | [OnPreAuthToolkit](./kibana-plugin-core-server.onpreauthtoolkit.md) | A tool set defining an outcome of OnPreAuth interceptor for incoming request. | diff --git a/docs/development/core/server/kibana-plugin-core-server.metricsservicesetup.collectioninterval.md b/docs/development/core/server/kibana-plugin-core-server.metricsservicesetup.collectioninterval.md new file mode 100644 index 0000000000000..6f05526b66c83 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.metricsservicesetup.collectioninterval.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [MetricsServiceSetup](./kibana-plugin-core-server.metricsservicesetup.md) > [collectionInterval](./kibana-plugin-core-server.metricsservicesetup.collectioninterval.md) + +## MetricsServiceSetup.collectionInterval property + +Interval metrics are collected in milliseconds + +Signature: + +```typescript +readonly collectionInterval: number; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.metricsservicesetup.getopsmetrics_.md b/docs/development/core/server/kibana-plugin-core-server.metricsservicesetup.getopsmetrics_.md new file mode 100644 index 0000000000000..61107fbf20ad9 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.metricsservicesetup.getopsmetrics_.md @@ -0,0 +1,24 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [MetricsServiceSetup](./kibana-plugin-core-server.metricsservicesetup.md) > [getOpsMetrics$](./kibana-plugin-core-server.metricsservicesetup.getopsmetrics_.md) + +## MetricsServiceSetup.getOpsMetrics$ property + +Retrieve an observable emitting the [OpsMetrics](./kibana-plugin-core-server.opsmetrics.md) gathered. The observable will emit an initial value during core's `start` phase, and a new value every fixed interval of time, based on the `opts.interval` configuration property. + +Signature: + +```typescript +getOpsMetrics$: () => Observable; +``` + +## Example + + +```ts +core.metrics.getOpsMetrics$().subscribe(metrics => { + // do something with the metrics +}) + +``` + diff --git a/docs/development/core/server/kibana-plugin-core-server.metricsservicesetup.md b/docs/development/core/server/kibana-plugin-core-server.metricsservicesetup.md index 0bec919797b6f..5fcb1417dea0e 100644 --- a/docs/development/core/server/kibana-plugin-core-server.metricsservicesetup.md +++ b/docs/development/core/server/kibana-plugin-core-server.metricsservicesetup.md @@ -4,8 +4,18 @@ ## MetricsServiceSetup interface +APIs to retrieves metrics gathered and exposed by the core platform. + Signature: ```typescript export interface MetricsServiceSetup ``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [collectionInterval](./kibana-plugin-core-server.metricsservicesetup.collectioninterval.md) | number | Interval metrics are collected in milliseconds | +| [getOpsMetrics$](./kibana-plugin-core-server.metricsservicesetup.getopsmetrics_.md) | () => Observable<OpsMetrics> | Retrieve an observable emitting the [OpsMetrics](./kibana-plugin-core-server.opsmetrics.md) gathered. The observable will emit an initial value during core's start phase, and a new value every fixed interval of time, based on the opts.interval configuration property. | + diff --git a/docs/development/core/server/kibana-plugin-core-server.opsmetrics.collected_at.md b/docs/development/core/server/kibana-plugin-core-server.opsmetrics.collected_at.md new file mode 100644 index 0000000000000..25125569b7b38 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.opsmetrics.collected_at.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [OpsMetrics](./kibana-plugin-core-server.opsmetrics.md) > [collected\_at](./kibana-plugin-core-server.opsmetrics.collected_at.md) + +## OpsMetrics.collected\_at property + +Time metrics were recorded at. + +Signature: + +```typescript +collected_at: Date; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.opsmetrics.md b/docs/development/core/server/kibana-plugin-core-server.opsmetrics.md index d2d4782385c06..9803c0fbd53cc 100644 --- a/docs/development/core/server/kibana-plugin-core-server.opsmetrics.md +++ b/docs/development/core/server/kibana-plugin-core-server.opsmetrics.md @@ -16,6 +16,7 @@ export interface OpsMetrics | Property | Type | Description | | --- | --- | --- | +| [collected\_at](./kibana-plugin-core-server.opsmetrics.collected_at.md) | Date | Time metrics were recorded at. | | [concurrent\_connections](./kibana-plugin-core-server.opsmetrics.concurrent_connections.md) | OpsServerMetrics['concurrent_connections'] | number of current concurrent connections to the server | | [os](./kibana-plugin-core-server.opsmetrics.os.md) | OpsOsMetrics | OS related metrics | | [process](./kibana-plugin-core-server.opsmetrics.process.md) | OpsProcessMetrics | Process related metrics | diff --git a/docs/development/core/server/kibana-plugin-core-server.opsosmetrics.cpu.md b/docs/development/core/server/kibana-plugin-core-server.opsosmetrics.cpu.md new file mode 100644 index 0000000000000..095c45266a251 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.opsosmetrics.cpu.md @@ -0,0 +1,22 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [OpsOsMetrics](./kibana-plugin-core-server.opsosmetrics.md) > [cpu](./kibana-plugin-core-server.opsosmetrics.cpu.md) + +## OpsOsMetrics.cpu property + +cpu cgroup metrics, undefined when not running in a cgroup + +Signature: + +```typescript +cpu?: { + control_group: string; + cfs_period_micros: number; + cfs_quota_micros: number; + stat: { + number_of_elapsed_periods: number; + number_of_times_throttled: number; + time_throttled_nanos: number; + }; + }; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.opsosmetrics.cpuacct.md b/docs/development/core/server/kibana-plugin-core-server.opsosmetrics.cpuacct.md new file mode 100644 index 0000000000000..140646a0d1a35 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.opsosmetrics.cpuacct.md @@ -0,0 +1,16 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [OpsOsMetrics](./kibana-plugin-core-server.opsosmetrics.md) > [cpuacct](./kibana-plugin-core-server.opsosmetrics.cpuacct.md) + +## OpsOsMetrics.cpuacct property + +cpu accounting metrics, undefined when not running in a cgroup + +Signature: + +```typescript +cpuacct?: { + control_group: string; + usage_nanos: number; + }; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.opsosmetrics.md b/docs/development/core/server/kibana-plugin-core-server.opsosmetrics.md index 5fedb76a9c8d7..8938608531139 100644 --- a/docs/development/core/server/kibana-plugin-core-server.opsosmetrics.md +++ b/docs/development/core/server/kibana-plugin-core-server.opsosmetrics.md @@ -16,6 +16,8 @@ export interface OpsOsMetrics | Property | Type | Description | | --- | --- | --- | +| [cpu](./kibana-plugin-core-server.opsosmetrics.cpu.md) | {
control_group: string;
cfs_period_micros: number;
cfs_quota_micros: number;
stat: {
number_of_elapsed_periods: number;
number_of_times_throttled: number;
time_throttled_nanos: number;
};
} | cpu cgroup metrics, undefined when not running in a cgroup | +| [cpuacct](./kibana-plugin-core-server.opsosmetrics.cpuacct.md) | {
control_group: string;
usage_nanos: number;
} | cpu accounting metrics, undefined when not running in a cgroup | | [distro](./kibana-plugin-core-server.opsosmetrics.distro.md) | string | The os distrib. Only present for linux platforms | | [distroRelease](./kibana-plugin-core-server.opsosmetrics.distrorelease.md) | string | The os distrib release, prefixed by the os distrib. Only present for linux platforms | | [load](./kibana-plugin-core-server.opsosmetrics.load.md) | {
'1m': number;
'5m': number;
'15m': number;
} | cpu load metrics | diff --git a/docs/development/core/server/kibana-plugin-core-server.statusservicesetup.dependencies_.md b/docs/development/core/server/kibana-plugin-core-server.statusservicesetup.dependencies_.md new file mode 100644 index 0000000000000..7475f0e3a4c1c --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.statusservicesetup.dependencies_.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [StatusServiceSetup](./kibana-plugin-core-server.statusservicesetup.md) > [dependencies$](./kibana-plugin-core-server.statusservicesetup.dependencies_.md) + +## StatusServiceSetup.dependencies$ property + +Current status for all plugins this plugin depends on. Each key of the `Record` is a plugin id. + +Signature: + +```typescript +dependencies$: Observable>; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.statusservicesetup.derivedstatus_.md b/docs/development/core/server/kibana-plugin-core-server.statusservicesetup.derivedstatus_.md new file mode 100644 index 0000000000000..6c65e44270a06 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.statusservicesetup.derivedstatus_.md @@ -0,0 +1,20 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [StatusServiceSetup](./kibana-plugin-core-server.statusservicesetup.md) > [derivedStatus$](./kibana-plugin-core-server.statusservicesetup.derivedstatus_.md) + +## StatusServiceSetup.derivedStatus$ property + +The status of this plugin as derived from its dependencies. + +Signature: + +```typescript +derivedStatus$: Observable; +``` + +## Remarks + +By default, plugins inherit this derived status from their dependencies. Calling overrides this default status. + +This may emit multliple times for a single status change event as propagates through the dependency tree + diff --git a/docs/development/core/server/kibana-plugin-core-server.statusservicesetup.md b/docs/development/core/server/kibana-plugin-core-server.statusservicesetup.md index 3d3b73ccda25f..ba0645be4d26c 100644 --- a/docs/development/core/server/kibana-plugin-core-server.statusservicesetup.md +++ b/docs/development/core/server/kibana-plugin-core-server.statusservicesetup.md @@ -12,10 +12,73 @@ API for accessing status of Core and this plugin's dependencies as well as for c export interface StatusServiceSetup ``` +## Remarks + +By default, a plugin inherits it's current status from the most severe status level of any Core services and any plugins that it depends on. This default status is available on the API. + +Plugins may customize their status calculation by calling the API with an Observable. Within this Observable, a plugin may choose to only depend on the status of some of its dependencies, to ignore severe status levels of particular Core services they are not concerned with, or to make its status dependent on other external services. + +## Example 1 + +Customize a plugin's status to only depend on the status of SavedObjects: + +```ts +core.status.set( + core.status.core$.pipe( +. map((coreStatus) => { + return coreStatus.savedObjects; + }) ; + ); +); + +``` + +## Example 2 + +Customize a plugin's status to include an external service: + +```ts +const externalStatus$ = interval(1000).pipe( + switchMap(async () => { + const resp = await fetch(`https://myexternaldep.com/_healthz`); + const body = await resp.json(); + if (body.ok) { + return of({ level: ServiceStatusLevels.available, summary: 'External Service is up'}); + } else { + return of({ level: ServiceStatusLevels.available, summary: 'External Service is unavailable'}); + } + }), + catchError((error) => { + of({ level: ServiceStatusLevels.unavailable, summary: `External Service is down`, meta: { error }}) + }) +); + +core.status.set( + combineLatest([core.status.derivedStatus$, externalStatus$]).pipe( + map(([derivedStatus, externalStatus]) => { + if (externalStatus.level > derivedStatus) { + return externalStatus; + } else { + return derivedStatus; + } + }) + ) +); + +``` + ## Properties | Property | Type | Description | | --- | --- | --- | | [core$](./kibana-plugin-core-server.statusservicesetup.core_.md) | Observable<CoreStatus> | Current status for all Core services. | +| [dependencies$](./kibana-plugin-core-server.statusservicesetup.dependencies_.md) | Observable<Record<string, ServiceStatus>> | Current status for all plugins this plugin depends on. Each key of the Record is a plugin id. | +| [derivedStatus$](./kibana-plugin-core-server.statusservicesetup.derivedstatus_.md) | Observable<ServiceStatus> | The status of this plugin as derived from its dependencies. | | [overall$](./kibana-plugin-core-server.statusservicesetup.overall_.md) | Observable<ServiceStatus> | Overall system status for all of Kibana. | +## Methods + +| Method | Description | +| --- | --- | +| [set(status$)](./kibana-plugin-core-server.statusservicesetup.set.md) | Allows a plugin to specify a custom status dependent on its own criteria. Completely overrides the default inherited status. | + diff --git a/docs/development/core/server/kibana-plugin-core-server.statusservicesetup.set.md b/docs/development/core/server/kibana-plugin-core-server.statusservicesetup.set.md new file mode 100644 index 0000000000000..143cd397c40ae --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.statusservicesetup.set.md @@ -0,0 +1,28 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [StatusServiceSetup](./kibana-plugin-core-server.statusservicesetup.md) > [set](./kibana-plugin-core-server.statusservicesetup.set.md) + +## StatusServiceSetup.set() method + +Allows a plugin to specify a custom status dependent on its own criteria. Completely overrides the default inherited status. + +Signature: + +```typescript +set(status$: Observable): void; +``` + +## Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| status$ | Observable<ServiceStatus> | | + +Returns: + +`void` + +## Remarks + +See the [StatusServiceSetup.derivedStatus$](./kibana-plugin-core-server.statusservicesetup.derivedstatus_.md) API for leveraging the default status calculation that is provided by Core. + diff --git a/docs/setup/settings.asciidoc b/docs/setup/settings.asciidoc index 4a931aabd3646..f03022e9e9f00 100644 --- a/docs/setup/settings.asciidoc +++ b/docs/setup/settings.asciidoc @@ -20,12 +20,12 @@ which may cause a delay before pages start being served. Set to `false` to disable Console. *Default: `true`* | `cpu.cgroup.path.override:` - | Override for cgroup cpu path when mounted in a -manner that is inconsistent with `/proc/self/cgroup`. + | *deprecated* This setting has been renamed to `ops.cGroupOverrides.cpuPath` +and the old name will no longer be supported as of 8.0. | `cpuacct.cgroup.path.override:` - | Override for cgroup cpuacct path when mounted -in a manner that is inconsistent with `/proc/self/cgroup`. + | *deprecated* This setting has been renamed to `ops.cGroupOverrides.cpuAcctPath` +and the old name will no longer be supported as of 8.0. | `csp.rules:` | A https://w3c.github.io/webappsec-csp/[content-security-policy] template @@ -438,6 +438,14 @@ not saved in {es}. *Default: `data`* | Set the interval in milliseconds to sample system and process performance metrics. The minimum value is 100. *Default: `5000`* +| `ops.cGroupOverrides.cpuPath:` + | Override for cgroup cpu path when mounted in a +manner that is inconsistent with `/proc/self/cgroup`. + +| `ops.cGroupOverrides.cpuAcctPath:` + | Override for cgroup cpuacct path when mounted +in a manner that is inconsistent with `/proc/self/cgroup`. + | `server.basePath:` | Enables you to specify a path to mount {kib} at if you are running behind a proxy. Use the `server.rewriteBasePath` setting to tell {kib} diff --git a/docs/user/reporting/reporting-troubleshooting.asciidoc b/docs/user/reporting/reporting-troubleshooting.asciidoc index dc4ffdfebdae9..82f0aa7ca0f19 100644 --- a/docs/user/reporting/reporting-troubleshooting.asciidoc +++ b/docs/user/reporting/reporting-troubleshooting.asciidoc @@ -7,6 +7,7 @@ Having trouble? Here are solutions to common problems you might encounter while using Reporting. +* <> * <> * <> * <> @@ -15,6 +16,11 @@ Having trouble? Here are solutions to common problems you might encounter while * <> * <> +[float] +[[reporting-diagnostics]] +=== Reporting Diagnostics +Reporting comes with a built-in utility to try to automatically find common issues. When Kibana is running, navigate to the Report Listing page, and click the "Run reporting diagnostics..." button. This will open up a diagnostic tool that checks various parts of the Kibana deployment to come up with any relevant recommendations. + [float] [[reporting-troubleshooting-system-dependencies]] === System dependencies diff --git a/rfcs/text/0010_service_status.md b/rfcs/text/0010_service_status.md index ded594930a367..76195c4f1ab89 100644 --- a/rfcs/text/0010_service_status.md +++ b/rfcs/text/0010_service_status.md @@ -137,7 +137,7 @@ interface StatusSetup { * Current status for all dependencies of the current plugin. * Each key of the `Record` is a plugin id. */ - plugins$: Observable>; + dependencies$: Observable>; /** * The status of this plugin as derived from its dependencies. diff --git a/src/legacy/utils/binder.ts b/src/cli/cluster/binder.ts similarity index 100% rename from src/legacy/utils/binder.ts rename to src/cli/cluster/binder.ts diff --git a/src/legacy/utils/binder_for.ts b/src/cli/cluster/binder_for.ts similarity index 100% rename from src/legacy/utils/binder_for.ts rename to src/cli/cluster/binder_for.ts diff --git a/src/cli/cluster/worker.ts b/src/cli/cluster/worker.ts index 097a549187429..c8a8a067d30bf 100644 --- a/src/cli/cluster/worker.ts +++ b/src/cli/cluster/worker.ts @@ -21,7 +21,7 @@ import _ from 'lodash'; import cluster from 'cluster'; import { EventEmitter } from 'events'; -import { BinderFor } from '../../legacy/utils/binder_for'; +import { BinderFor } from './binder_for'; import { fromRoot } from '../../core/server/utils'; const cliPath = fromRoot('src/cli'); diff --git a/src/cli_keystore/add.js b/src/cli_keystore/add.js index 462259ec942dd..232392f34c63b 100644 --- a/src/cli_keystore/add.js +++ b/src/cli_keystore/add.js @@ -18,7 +18,7 @@ */ import { Logger } from '../cli_plugin/lib/logger'; -import { confirm, question } from '../legacy/server/utils'; +import { confirm, question } from './utils'; import { createPromiseFromStreams, createConcatStream } from '../core/server/utils'; /** diff --git a/src/cli_keystore/add.test.js b/src/cli_keystore/add.test.js index b5d5009667eb4..f1adee8879bc2 100644 --- a/src/cli_keystore/add.test.js +++ b/src/cli_keystore/add.test.js @@ -42,7 +42,7 @@ import { PassThrough } from 'stream'; import { Keystore } from '../legacy/server/keystore'; import { add } from './add'; import { Logger } from '../cli_plugin/lib/logger'; -import * as prompt from '../legacy/server/utils/prompt'; +import * as prompt from './utils/prompt'; describe('Kibana keystore', () => { describe('add', () => { diff --git a/src/cli_keystore/create.js b/src/cli_keystore/create.js index 8be1eb36882f1..55fe2c151dec0 100644 --- a/src/cli_keystore/create.js +++ b/src/cli_keystore/create.js @@ -18,7 +18,7 @@ */ import { Logger } from '../cli_plugin/lib/logger'; -import { confirm } from '../legacy/server/utils'; +import { confirm } from './utils'; export async function create(keystore, command, options) { const logger = new Logger(options); diff --git a/src/cli_keystore/create.test.js b/src/cli_keystore/create.test.js index f48b3775ddfff..cb85475eab1cb 100644 --- a/src/cli_keystore/create.test.js +++ b/src/cli_keystore/create.test.js @@ -41,7 +41,7 @@ import sinon from 'sinon'; import { Keystore } from '../legacy/server/keystore'; import { create } from './create'; import { Logger } from '../cli_plugin/lib/logger'; -import * as prompt from '../legacy/server/utils/prompt'; +import * as prompt from './utils/prompt'; describe('Kibana keystore', () => { describe('create', () => { diff --git a/src/legacy/server/utils/index.js b/src/cli_keystore/utils/index.js similarity index 100% rename from src/legacy/server/utils/index.js rename to src/cli_keystore/utils/index.js diff --git a/src/legacy/server/utils/prompt.js b/src/cli_keystore/utils/prompt.js similarity index 100% rename from src/legacy/server/utils/prompt.js rename to src/cli_keystore/utils/prompt.js diff --git a/src/legacy/server/utils/prompt.test.js b/src/cli_keystore/utils/prompt.test.js similarity index 100% rename from src/legacy/server/utils/prompt.test.js rename to src/cli_keystore/utils/prompt.test.js diff --git a/src/core/public/core_app/status/lib/load_status.test.ts b/src/core/public/core_app/status/lib/load_status.test.ts index 3a444a4448467..5a9f982e106a7 100644 --- a/src/core/public/core_app/status/lib/load_status.test.ts +++ b/src/core/public/core_app/status/lib/load_status.test.ts @@ -57,6 +57,7 @@ const mockedResponse: StatusResponse = { ], }, metrics: { + collected_at: new Date('2020-01-01 01:00:00'), collection_interval_in_millis: 1000, os: { platform: 'darwin' as const, diff --git a/src/core/server/config/deprecation/core_deprecations.ts b/src/core/server/config/deprecation/core_deprecations.ts index e4e881ab24372..2b8b8e383da24 100644 --- a/src/core/server/config/deprecation/core_deprecations.ts +++ b/src/core/server/config/deprecation/core_deprecations.ts @@ -113,7 +113,7 @@ const mapManifestServiceUrlDeprecation: ConfigDeprecation = (settings, fromPath, return settings; }; -export const coreDeprecationProvider: ConfigDeprecationProvider = ({ unusedFromRoot }) => [ +export const coreDeprecationProvider: ConfigDeprecationProvider = ({ rename, unusedFromRoot }) => [ unusedFromRoot('savedObjects.indexCheckTimeout'), unusedFromRoot('server.xsrf.token'), unusedFromRoot('maps.manifestServiceUrl'), @@ -136,6 +136,8 @@ export const coreDeprecationProvider: ConfigDeprecationProvider = ({ unusedFromR unusedFromRoot('optimize.workers'), unusedFromRoot('optimize.profile'), unusedFromRoot('optimize.validateSyntaxOfNodeModules'), + rename('cpu.cgroup.path.override', 'ops.cGroupOverrides.cpuPath'), + rename('cpuacct.cgroup.path.override', 'ops.cGroupOverrides.cpuAcctPath'), configPathDeprecation, dataPathDeprecation, rewriteBasePathDeprecation, diff --git a/src/core/server/legacy/legacy_service.ts b/src/core/server/legacy/legacy_service.ts index b95644590b4e9..6e6d5cfc24340 100644 --- a/src/core/server/legacy/legacy_service.ts +++ b/src/core/server/legacy/legacy_service.ts @@ -264,6 +264,7 @@ export class LegacyService implements CoreService { getTypeRegistry: startDeps.core.savedObjects.getTypeRegistry, }, metrics: { + collectionInterval: startDeps.core.metrics.collectionInterval, getOpsMetrics$: startDeps.core.metrics.getOpsMetrics$, }, uiSettings: { asScopedToClient: startDeps.core.uiSettings.asScopedToClient }, @@ -310,6 +311,17 @@ export class LegacyService implements CoreService { status: { core$: setupDeps.core.status.core$, overall$: setupDeps.core.status.overall$, + set: () => { + throw new Error(`core.status.set is unsupported in legacy`); + }, + // @ts-expect-error + get dependencies$() { + throw new Error(`core.status.dependencies$ is unsupported in legacy`); + }, + // @ts-expect-error + get derivedStatus$() { + throw new Error(`core.status.derivedStatus$ is unsupported in legacy`); + }, }, uiSettings: { register: setupDeps.core.uiSettings.register, diff --git a/src/core/server/metrics/collectors/cgroup.test.ts b/src/core/server/metrics/collectors/cgroup.test.ts new file mode 100644 index 0000000000000..39f917b9f0ba1 --- /dev/null +++ b/src/core/server/metrics/collectors/cgroup.test.ts @@ -0,0 +1,115 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import mockFs from 'mock-fs'; +import { OsCgroupMetricsCollector } from './cgroup'; + +describe('OsCgroupMetricsCollector', () => { + afterEach(() => mockFs.restore()); + + it('returns empty object when no cgroup file present', async () => { + mockFs({ + '/proc/self': { + /** empty directory */ + }, + }); + + const collector = new OsCgroupMetricsCollector({}); + expect(await collector.collect()).toEqual({}); + }); + + it('collects default cgroup data', async () => { + mockFs({ + '/proc/self/cgroup': ` +123:memory:/groupname +123:cpu:/groupname +123:cpuacct:/groupname + `, + '/sys/fs/cgroup/cpuacct/groupname/cpuacct.usage': '111', + '/sys/fs/cgroup/cpu/groupname/cpu.cfs_period_us': '222', + '/sys/fs/cgroup/cpu/groupname/cpu.cfs_quota_us': '333', + '/sys/fs/cgroup/cpu/groupname/cpu.stat': ` +nr_periods 444 +nr_throttled 555 +throttled_time 666 + `, + }); + + const collector = new OsCgroupMetricsCollector({}); + expect(await collector.collect()).toMatchInlineSnapshot(` + Object { + "cpu": Object { + "cfs_period_micros": 222, + "cfs_quota_micros": 333, + "control_group": "/groupname", + "stat": Object { + "number_of_elapsed_periods": 444, + "number_of_times_throttled": 555, + "time_throttled_nanos": 666, + }, + }, + "cpuacct": Object { + "control_group": "/groupname", + "usage_nanos": 111, + }, + } + `); + }); + + it('collects override cgroup data', async () => { + mockFs({ + '/proc/self/cgroup': ` +123:memory:/groupname +123:cpu:/groupname +123:cpuacct:/groupname + `, + '/sys/fs/cgroup/cpuacct/xxcustomcpuacctxx/cpuacct.usage': '111', + '/sys/fs/cgroup/cpu/xxcustomcpuxx/cpu.cfs_period_us': '222', + '/sys/fs/cgroup/cpu/xxcustomcpuxx/cpu.cfs_quota_us': '333', + '/sys/fs/cgroup/cpu/xxcustomcpuxx/cpu.stat': ` +nr_periods 444 +nr_throttled 555 +throttled_time 666 + `, + }); + + const collector = new OsCgroupMetricsCollector({ + cpuAcctPath: 'xxcustomcpuacctxx', + cpuPath: 'xxcustomcpuxx', + }); + expect(await collector.collect()).toMatchInlineSnapshot(` + Object { + "cpu": Object { + "cfs_period_micros": 222, + "cfs_quota_micros": 333, + "control_group": "xxcustomcpuxx", + "stat": Object { + "number_of_elapsed_periods": 444, + "number_of_times_throttled": 555, + "time_throttled_nanos": 666, + }, + }, + "cpuacct": Object { + "control_group": "xxcustomcpuacctxx", + "usage_nanos": 111, + }, + } + `); + }); +}); diff --git a/src/core/server/metrics/collectors/cgroup.ts b/src/core/server/metrics/collectors/cgroup.ts new file mode 100644 index 0000000000000..867ea44dff1ae --- /dev/null +++ b/src/core/server/metrics/collectors/cgroup.ts @@ -0,0 +1,194 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import fs from 'fs'; +import { join as joinPath } from 'path'; +import { MetricsCollector, OpsOsMetrics } from './types'; + +type OsCgroupMetrics = Pick; + +interface OsCgroupMetricsCollectorOptions { + cpuPath?: string; + cpuAcctPath?: string; +} + +export class OsCgroupMetricsCollector implements MetricsCollector { + /** Used to prevent unnecessary file reads on systems not using cgroups. */ + private noCgroupPresent = false; + private cpuPath?: string; + private cpuAcctPath?: string; + + constructor(private readonly options: OsCgroupMetricsCollectorOptions) {} + + public async collect(): Promise { + try { + await this.initializePaths(); + if (this.noCgroupPresent || !this.cpuAcctPath || !this.cpuPath) { + return {}; + } + + const [cpuAcctUsage, cpuFsPeriod, cpuFsQuota, cpuStat] = await Promise.all([ + readCPUAcctUsage(this.cpuAcctPath), + readCPUFsPeriod(this.cpuPath), + readCPUFsQuota(this.cpuPath), + readCPUStat(this.cpuPath), + ]); + + return { + cpuacct: { + control_group: this.cpuAcctPath, + usage_nanos: cpuAcctUsage, + }, + + cpu: { + control_group: this.cpuPath, + cfs_period_micros: cpuFsPeriod, + cfs_quota_micros: cpuFsQuota, + stat: cpuStat, + }, + }; + } catch (err) { + if (err.code === 'ENOENT') { + this.noCgroupPresent = true; + return {}; + } else { + throw err; + } + } + } + + public reset() {} + + private async initializePaths() { + // Perform this setup lazily on the first collect call and then memoize the results. + // Makes the assumption this data doesn't change while the process is running. + if (this.cpuPath && this.cpuAcctPath) { + return; + } + + // Only read the file if both options are undefined. + if (!this.options.cpuPath || !this.options.cpuAcctPath) { + const cgroups = await readControlGroups(); + this.cpuPath = this.options.cpuPath || cgroups[GROUP_CPU]; + this.cpuAcctPath = this.options.cpuAcctPath || cgroups[GROUP_CPUACCT]; + } else { + this.cpuPath = this.options.cpuPath; + this.cpuAcctPath = this.options.cpuAcctPath; + } + + // prevents undefined cgroup paths + if (!this.cpuPath || !this.cpuAcctPath) { + this.noCgroupPresent = true; + } + } +} + +const CONTROL_GROUP_RE = new RegExp('\\d+:([^:]+):(/.*)'); +const CONTROLLER_SEPARATOR_RE = ','; + +const PROC_SELF_CGROUP_FILE = '/proc/self/cgroup'; +const PROC_CGROUP_CPU_DIR = '/sys/fs/cgroup/cpu'; +const PROC_CGROUP_CPUACCT_DIR = '/sys/fs/cgroup/cpuacct'; + +const GROUP_CPUACCT = 'cpuacct'; +const CPUACCT_USAGE_FILE = 'cpuacct.usage'; + +const GROUP_CPU = 'cpu'; +const CPU_FS_PERIOD_US_FILE = 'cpu.cfs_period_us'; +const CPU_FS_QUOTA_US_FILE = 'cpu.cfs_quota_us'; +const CPU_STATS_FILE = 'cpu.stat'; + +async function readControlGroups() { + const data = await fs.promises.readFile(PROC_SELF_CGROUP_FILE); + + return data + .toString() + .split(/\n/) + .reduce((acc, line) => { + const matches = line.match(CONTROL_GROUP_RE); + + if (matches !== null) { + const controllers = matches[1].split(CONTROLLER_SEPARATOR_RE); + controllers.forEach((controller) => { + acc[controller] = matches[2]; + }); + } + + return acc; + }, {} as Record); +} + +async function fileContentsToInteger(path: string) { + const data = await fs.promises.readFile(path); + return parseInt(data.toString(), 10); +} + +function readCPUAcctUsage(controlGroup: string) { + return fileContentsToInteger(joinPath(PROC_CGROUP_CPUACCT_DIR, controlGroup, CPUACCT_USAGE_FILE)); +} + +function readCPUFsPeriod(controlGroup: string) { + return fileContentsToInteger(joinPath(PROC_CGROUP_CPU_DIR, controlGroup, CPU_FS_PERIOD_US_FILE)); +} + +function readCPUFsQuota(controlGroup: string) { + return fileContentsToInteger(joinPath(PROC_CGROUP_CPU_DIR, controlGroup, CPU_FS_QUOTA_US_FILE)); +} + +async function readCPUStat(controlGroup: string) { + const stat = { + number_of_elapsed_periods: -1, + number_of_times_throttled: -1, + time_throttled_nanos: -1, + }; + + try { + const data = await fs.promises.readFile( + joinPath(PROC_CGROUP_CPU_DIR, controlGroup, CPU_STATS_FILE) + ); + return data + .toString() + .split(/\n/) + .reduce((acc, line) => { + const fields = line.split(/\s+/); + + switch (fields[0]) { + case 'nr_periods': + acc.number_of_elapsed_periods = parseInt(fields[1], 10); + break; + + case 'nr_throttled': + acc.number_of_times_throttled = parseInt(fields[1], 10); + break; + + case 'throttled_time': + acc.time_throttled_nanos = parseInt(fields[1], 10); + break; + } + + return acc; + }, stat); + } catch (err) { + if (err.code === 'ENOENT') { + return stat; + } + + throw err; + } +} diff --git a/src/core/server/metrics/collectors/collector.mock.ts b/src/core/server/metrics/collectors/collector.mock.ts new file mode 100644 index 0000000000000..2a942e1fafe63 --- /dev/null +++ b/src/core/server/metrics/collectors/collector.mock.ts @@ -0,0 +1,33 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { MetricsCollector } from './types'; + +const createCollector = (collectReturnValue: any = {}): jest.Mocked> => { + const collector: jest.Mocked> = { + collect: jest.fn().mockResolvedValue(collectReturnValue), + reset: jest.fn(), + }; + + return collector; +}; + +export const metricsCollectorMock = { + create: createCollector, +}; diff --git a/src/core/server/metrics/collectors/index.ts b/src/core/server/metrics/collectors/index.ts index f58ab02e63881..4540cb79be74b 100644 --- a/src/core/server/metrics/collectors/index.ts +++ b/src/core/server/metrics/collectors/index.ts @@ -18,6 +18,6 @@ */ export { OpsProcessMetrics, OpsOsMetrics, OpsServerMetrics, MetricsCollector } from './types'; -export { OsMetricsCollector } from './os'; +export { OsMetricsCollector, OpsMetricsCollectorOptions } from './os'; export { ProcessMetricsCollector } from './process'; export { ServerMetricsCollector } from './server'; diff --git a/src/legacy/utils/path_contains.js b/src/core/server/metrics/collectors/os.test.mocks.ts similarity index 78% rename from src/legacy/utils/path_contains.js rename to src/core/server/metrics/collectors/os.test.mocks.ts index 60d05c1099554..ee02b8c802151 100644 --- a/src/legacy/utils/path_contains.js +++ b/src/core/server/metrics/collectors/os.test.mocks.ts @@ -17,8 +17,9 @@ * under the License. */ -import { relative } from 'path'; +import { metricsCollectorMock } from './collector.mock'; -export default function pathContains(root, child) { - return relative(child, root).slice(0, 2) !== '..'; -} +export const cgroupCollectorMock = metricsCollectorMock.create(); +jest.doMock('./cgroup', () => ({ + OsCgroupMetricsCollector: jest.fn(() => cgroupCollectorMock), +})); diff --git a/src/core/server/metrics/collectors/os.test.ts b/src/core/server/metrics/collectors/os.test.ts index 7d5a6da90b7d6..5e52cecb76be3 100644 --- a/src/core/server/metrics/collectors/os.test.ts +++ b/src/core/server/metrics/collectors/os.test.ts @@ -20,6 +20,7 @@ jest.mock('getos', () => (cb: Function) => cb(null, { dist: 'distrib', release: 'release' })); import os from 'os'; +import { cgroupCollectorMock } from './os.test.mocks'; import { OsMetricsCollector } from './os'; describe('OsMetricsCollector', () => { @@ -27,6 +28,8 @@ describe('OsMetricsCollector', () => { beforeEach(() => { collector = new OsMetricsCollector(); + cgroupCollectorMock.collect.mockReset(); + cgroupCollectorMock.reset.mockReset(); }); afterEach(() => { @@ -96,4 +99,9 @@ describe('OsMetricsCollector', () => { '15m': fifteenMinLoad, }); }); + + it('calls the cgroup sub-collector', async () => { + await collector.collect(); + expect(cgroupCollectorMock.collect).toHaveBeenCalled(); + }); }); diff --git a/src/core/server/metrics/collectors/os.ts b/src/core/server/metrics/collectors/os.ts index 59bef9d8ddd2b..eae49278405a9 100644 --- a/src/core/server/metrics/collectors/os.ts +++ b/src/core/server/metrics/collectors/os.ts @@ -21,10 +21,22 @@ import os from 'os'; import getosAsync, { LinuxOs } from 'getos'; import { promisify } from 'util'; import { OpsOsMetrics, MetricsCollector } from './types'; +import { OsCgroupMetricsCollector } from './cgroup'; const getos = promisify(getosAsync); +export interface OpsMetricsCollectorOptions { + cpuPath?: string; + cpuAcctPath?: string; +} + export class OsMetricsCollector implements MetricsCollector { + private readonly cgroupCollector: OsCgroupMetricsCollector; + + constructor(options: OpsMetricsCollectorOptions = {}) { + this.cgroupCollector = new OsCgroupMetricsCollector(options); + } + public async collect(): Promise { const platform = os.platform(); const load = os.loadavg(); @@ -43,20 +55,30 @@ export class OsMetricsCollector implements MetricsCollector { used_in_bytes: os.totalmem() - os.freemem(), }, uptime_in_millis: os.uptime() * 1000, + ...(await this.getDistroStats(platform)), + ...(await this.cgroupCollector.collect()), }; + return metrics; + } + + public reset() {} + + private async getDistroStats( + platform: string + ): Promise> { if (platform === 'linux') { try { const distro = (await getos()) as LinuxOs; - metrics.distro = distro.dist; - metrics.distroRelease = `${distro.dist}-${distro.release}`; + return { + distro: distro.dist, + distroRelease: `${distro.dist}-${distro.release}`, + }; } catch (e) { // ignore errors } } - return metrics; + return {}; } - - public reset() {} } diff --git a/src/core/server/metrics/collectors/types.ts b/src/core/server/metrics/collectors/types.ts index 73e8975a6b362..77ea13a1f0787 100644 --- a/src/core/server/metrics/collectors/types.ts +++ b/src/core/server/metrics/collectors/types.ts @@ -85,6 +85,33 @@ export interface OpsOsMetrics { }; /** the OS uptime */ uptime_in_millis: number; + + /** cpu accounting metrics, undefined when not running in a cgroup */ + cpuacct?: { + /** name of this process's cgroup */ + control_group: string; + /** cpu time used by this process's cgroup */ + usage_nanos: number; + }; + + /** cpu cgroup metrics, undefined when not running in a cgroup */ + cpu?: { + /** name of this process's cgroup */ + control_group: string; + /** the length of the cfs period */ + cfs_period_micros: number; + /** total available run-time within a cfs period */ + cfs_quota_micros: number; + /** current stats on the cfs periods */ + stat: { + /** number of cfs periods that elapsed */ + number_of_elapsed_periods: number; + /** number of times the cgroup has been throttled */ + number_of_times_throttled: number; + /** total amount of time the cgroup has been throttled for */ + time_throttled_nanos: number; + }; + }; } /** diff --git a/src/core/server/metrics/metrics_service.mock.ts b/src/core/server/metrics/metrics_service.mock.ts index 769f6ee2a549a..2af653004a479 100644 --- a/src/core/server/metrics/metrics_service.mock.ts +++ b/src/core/server/metrics/metrics_service.mock.ts @@ -21,20 +21,18 @@ import { MetricsService } from './metrics_service'; import { InternalMetricsServiceSetup, InternalMetricsServiceStart, + MetricsServiceSetup, MetricsServiceStart, } from './types'; const createInternalSetupContractMock = () => { - const setupContract: jest.Mocked = {}; - return setupContract; -}; - -const createStartContractMock = () => { - const startContract: jest.Mocked = { + const setupContract: jest.Mocked = { + collectionInterval: 30000, getOpsMetrics$: jest.fn(), }; - startContract.getOpsMetrics$.mockReturnValue( + setupContract.getOpsMetrics$.mockReturnValue( new BehaviorSubject({ + collected_at: new Date('2020-01-01 01:00:00'), process: { memory: { heap: { total_in_bytes: 1, used_in_bytes: 1, size_limit: 1 }, @@ -56,11 +54,21 @@ const createStartContractMock = () => { concurrent_connections: 1, }) ); + return setupContract; +}; + +const createSetupContractMock = () => { + const startContract: jest.Mocked = createInternalSetupContractMock(); return startContract; }; const createInternalStartContractMock = () => { - const startContract: jest.Mocked = createStartContractMock(); + const startContract: jest.Mocked = createInternalSetupContractMock(); + return startContract; +}; + +const createStartContractMock = () => { + const startContract: jest.Mocked = createInternalSetupContractMock(); return startContract; }; @@ -77,7 +85,7 @@ const createMock = () => { export const metricsServiceMock = { create: createMock, - createSetupContract: createStartContractMock, + createSetupContract: createSetupContractMock, createStartContract: createStartContractMock, createInternalSetupContract: createInternalSetupContractMock, createInternalStartContract: createInternalStartContractMock, diff --git a/src/core/server/metrics/metrics_service.ts b/src/core/server/metrics/metrics_service.ts index f28fb21aaac0d..d4696b3aa9aaf 100644 --- a/src/core/server/metrics/metrics_service.ts +++ b/src/core/server/metrics/metrics_service.ts @@ -17,7 +17,7 @@ * under the License. */ -import { Subject } from 'rxjs'; +import { ReplaySubject } from 'rxjs'; import { first } from 'rxjs/operators'; import { CoreService } from '../../types'; import { CoreContext } from '../core_context'; @@ -37,26 +37,21 @@ export class MetricsService private readonly logger: Logger; private metricsCollector?: OpsMetricsCollector; private collectInterval?: NodeJS.Timeout; - private metrics$ = new Subject(); + private metrics$ = new ReplaySubject(); + private service?: InternalMetricsServiceSetup; constructor(private readonly coreContext: CoreContext) { this.logger = coreContext.logger.get('metrics'); } public async setup({ http }: MetricsServiceSetupDeps): Promise { - this.metricsCollector = new OpsMetricsCollector(http.server); - return {}; - } - - public async start(): Promise { - if (!this.metricsCollector) { - throw new Error('#setup() needs to be run first'); - } const config = await this.coreContext.configService .atPath(opsConfig.path) .pipe(first()) .toPromise(); + this.metricsCollector = new OpsMetricsCollector(http.server, config.cGroupOverrides); + await this.refreshMetrics(); this.collectInterval = setInterval(() => { @@ -65,9 +60,20 @@ export class MetricsService const metricsObservable = this.metrics$.asObservable(); - return { + this.service = { + collectionInterval: config.interval.asMilliseconds(), getOpsMetrics$: () => metricsObservable, }; + + return this.service; + } + + public async start(): Promise { + if (!this.service) { + throw new Error('#setup() needs to be run first'); + } + + return this.service; } private async refreshMetrics() { diff --git a/src/core/server/metrics/ops_config.ts b/src/core/server/metrics/ops_config.ts index bd6ae5cc5474d..5f3f67e931c38 100644 --- a/src/core/server/metrics/ops_config.ts +++ b/src/core/server/metrics/ops_config.ts @@ -23,6 +23,10 @@ export const opsConfig = { path: 'ops', schema: schema.object({ interval: schema.duration({ defaultValue: '5s' }), + cGroupOverrides: schema.object({ + cpuPath: schema.maybe(schema.string()), + cpuAcctPath: schema.maybe(schema.string()), + }), }), }; diff --git a/src/core/server/metrics/ops_metrics_collector.test.ts b/src/core/server/metrics/ops_metrics_collector.test.ts index 9e76895b14578..7aa3f7cd3baf0 100644 --- a/src/core/server/metrics/ops_metrics_collector.test.ts +++ b/src/core/server/metrics/ops_metrics_collector.test.ts @@ -30,7 +30,7 @@ describe('OpsMetricsCollector', () => { beforeEach(() => { const hapiServer = httpServiceMock.createInternalSetupContract().server; - collector = new OpsMetricsCollector(hapiServer); + collector = new OpsMetricsCollector(hapiServer, {}); mockOsCollector.collect.mockResolvedValue('osMetrics'); }); @@ -51,6 +51,7 @@ describe('OpsMetricsCollector', () => { expect(mockServerCollector.collect).toHaveBeenCalledTimes(1); expect(metrics).toEqual({ + collected_at: expect.any(Date), process: 'processMetrics', os: 'osMetrics', requests: 'serverRequestsMetrics', diff --git a/src/core/server/metrics/ops_metrics_collector.ts b/src/core/server/metrics/ops_metrics_collector.ts index 525515dba1457..af74caa6cb386 100644 --- a/src/core/server/metrics/ops_metrics_collector.ts +++ b/src/core/server/metrics/ops_metrics_collector.ts @@ -21,6 +21,7 @@ import { Server as HapiServer } from 'hapi'; import { ProcessMetricsCollector, OsMetricsCollector, + OpsMetricsCollectorOptions, ServerMetricsCollector, MetricsCollector, } from './collectors'; @@ -31,9 +32,9 @@ export class OpsMetricsCollector implements MetricsCollector { private readonly osCollector: OsMetricsCollector; private readonly serverCollector: ServerMetricsCollector; - constructor(server: HapiServer) { + constructor(server: HapiServer, opsOptions: OpsMetricsCollectorOptions) { this.processCollector = new ProcessMetricsCollector(); - this.osCollector = new OsMetricsCollector(); + this.osCollector = new OsMetricsCollector(opsOptions); this.serverCollector = new ServerMetricsCollector(server); } @@ -44,6 +45,7 @@ export class OpsMetricsCollector implements MetricsCollector { this.serverCollector.collect(), ]); return { + collected_at: new Date(), process, os, ...server, diff --git a/src/core/server/metrics/types.ts b/src/core/server/metrics/types.ts index cbf0acacd6bab..c177b3ed25115 100644 --- a/src/core/server/metrics/types.ts +++ b/src/core/server/metrics/types.ts @@ -20,14 +20,15 @@ import { Observable } from 'rxjs'; import { OpsProcessMetrics, OpsOsMetrics, OpsServerMetrics } from './collectors'; -// eslint-disable-next-line @typescript-eslint/no-empty-interface -export interface MetricsServiceSetup {} /** * APIs to retrieves metrics gathered and exposed by the core platform. * * @public */ -export interface MetricsServiceStart { +export interface MetricsServiceSetup { + /** Interval metrics are collected in milliseconds */ + readonly collectionInterval: number; + /** * Retrieve an observable emitting the {@link OpsMetrics} gathered. * The observable will emit an initial value during core's `start` phase, and a new value every fixed interval of time, @@ -42,6 +43,12 @@ export interface MetricsServiceStart { */ getOpsMetrics$: () => Observable; } +/** + * {@inheritdoc MetricsServiceSetup} + * + * @public + */ +export type MetricsServiceStart = MetricsServiceSetup; export type InternalMetricsServiceSetup = MetricsServiceSetup; export type InternalMetricsServiceStart = MetricsServiceStart; @@ -53,6 +60,8 @@ export type InternalMetricsServiceStart = MetricsServiceStart; * @public */ export interface OpsMetrics { + /** Time metrics were recorded at. */ + collected_at: Date; /** Process related metrics */ process: OpsProcessMetrics; /** OS related metrics */ diff --git a/src/core/server/plugins/plugin_context.ts b/src/core/server/plugins/plugin_context.ts index fa2659ca130a0..af0b0e19b3227 100644 --- a/src/core/server/plugins/plugin_context.ts +++ b/src/core/server/plugins/plugin_context.ts @@ -185,6 +185,9 @@ export function createPluginSetupContext( status: { core$: deps.status.core$, overall$: deps.status.overall$, + set: deps.status.plugins.set.bind(null, plugin.name), + dependencies$: deps.status.plugins.getDependenciesStatus$(plugin.name), + derivedStatus$: deps.status.plugins.getDerivedStatus$(plugin.name), }, uiSettings: { register: deps.uiSettings.register, @@ -233,6 +236,7 @@ export function createPluginStartContext( getTypeRegistry: deps.savedObjects.getTypeRegistry, }, metrics: { + collectionInterval: deps.metrics.collectionInterval, getOpsMetrics$: deps.metrics.getOpsMetrics$, }, uiSettings: { diff --git a/src/core/server/plugins/plugins_system.test.ts b/src/core/server/plugins/plugins_system.test.ts index 7af77491df1ab..71ac31db13f92 100644 --- a/src/core/server/plugins/plugins_system.test.ts +++ b/src/core/server/plugins/plugins_system.test.ts @@ -100,15 +100,27 @@ test('getPluginDependencies returns dependency tree of symbols', () => { pluginsSystem.addPlugin(createPlugin('no-dep')); expect(pluginsSystem.getPluginDependencies()).toMatchInlineSnapshot(` - Map { - Symbol(plugin-a) => Array [ - Symbol(no-dep), - ], - Symbol(plugin-b) => Array [ - Symbol(plugin-a), - Symbol(no-dep), - ], - Symbol(no-dep) => Array [], + Object { + "asNames": Map { + "plugin-a" => Array [ + "no-dep", + ], + "plugin-b" => Array [ + "plugin-a", + "no-dep", + ], + "no-dep" => Array [], + }, + "asOpaqueIds": Map { + Symbol(plugin-a) => Array [ + Symbol(no-dep), + ], + Symbol(plugin-b) => Array [ + Symbol(plugin-a), + Symbol(no-dep), + ], + Symbol(no-dep) => Array [], + }, } `); }); diff --git a/src/core/server/plugins/plugins_system.ts b/src/core/server/plugins/plugins_system.ts index f5c1b35d678a3..b2acd9a6fd04b 100644 --- a/src/core/server/plugins/plugins_system.ts +++ b/src/core/server/plugins/plugins_system.ts @@ -20,10 +20,11 @@ import { CoreContext } from '../core_context'; import { Logger } from '../logging'; import { PluginWrapper } from './plugin'; -import { DiscoveredPlugin, PluginName, PluginOpaqueId } from './types'; +import { DiscoveredPlugin, PluginName } from './types'; import { createPluginSetupContext, createPluginStartContext } from './plugin_context'; import { PluginsServiceSetupDeps, PluginsServiceStartDeps } from './plugins_service'; import { withTimeout } from '../../utils'; +import { PluginDependencies } from '.'; const Sec = 1000; /** @internal */ @@ -45,9 +46,19 @@ export class PluginsSystem { * @returns a ReadonlyMap of each plugin and an Array of its available dependencies * @internal */ - public getPluginDependencies(): ReadonlyMap { - // Return dependency map of opaque ids - return new Map( + public getPluginDependencies(): PluginDependencies { + const asNames = new Map( + [...this.plugins].map(([name, plugin]) => [ + plugin.name, + [ + ...new Set([ + ...plugin.requiredPlugins, + ...plugin.optionalPlugins.filter((optPlugin) => this.plugins.has(optPlugin)), + ]), + ].map((depId) => this.plugins.get(depId)!.name), + ]) + ); + const asOpaqueIds = new Map( [...this.plugins].map(([name, plugin]) => [ plugin.opaqueId, [ @@ -58,6 +69,8 @@ export class PluginsSystem { ].map((depId) => this.plugins.get(depId)!.opaqueId), ]) ); + + return { asNames, asOpaqueIds }; } public async setupPlugins(deps: PluginsServiceSetupDeps) { diff --git a/src/core/server/plugins/types.ts b/src/core/server/plugins/types.ts index eb2a9ca3daf5f..517261b5bc9bb 100644 --- a/src/core/server/plugins/types.ts +++ b/src/core/server/plugins/types.ts @@ -93,6 +93,12 @@ export type PluginName = string; /** @public */ export type PluginOpaqueId = symbol; +/** @internal */ +export interface PluginDependencies { + asNames: ReadonlyMap; + asOpaqueIds: ReadonlyMap; +} + /** * Describes the set of required and optional properties plugin can define in its * mandatory JSON manifest file. diff --git a/src/core/server/saved_objects/migrations/core/index_migrator.test.ts b/src/core/server/saved_objects/migrations/core/index_migrator.test.ts index df89137a1d798..13f771c16bc67 100644 --- a/src/core/server/saved_objects/migrations/core/index_migrator.test.ts +++ b/src/core/server/saved_objects/migrations/core/index_migrator.test.ts @@ -369,6 +369,30 @@ describe('IndexMigrator', () => { ], }); }); + + test('rejects when the migration function throws an error', async () => { + const { client } = testOpts; + const migrateDoc = jest.fn((doc: SavedObjectUnsanitizedDoc) => { + throw new Error('error migrating document'); + }); + + testOpts.documentMigrator = { + migrationVersion: { foo: '1.2.3' }, + migrate: migrateDoc, + }; + + withIndex(client, { + numOutOfDate: 1, + docs: [ + [{ _id: 'foo:1', _source: { type: 'foo', foo: { name: 'Bar' } } }], + [{ _id: 'foo:2', _source: { type: 'foo', foo: { name: 'Baz' } } }], + ], + }); + + await expect(new IndexMigrator(testOpts).migrate()).rejects.toThrowErrorMatchingInlineSnapshot( + `"error migrating document"` + ); + }); }); function withIndex( diff --git a/src/core/server/saved_objects/migrations/core/migrate_raw_docs.test.ts b/src/core/server/saved_objects/migrations/core/migrate_raw_docs.test.ts index 4c9d2e870a7bb..83dc042d2b96b 100644 --- a/src/core/server/saved_objects/migrations/core/migrate_raw_docs.test.ts +++ b/src/core/server/saved_objects/migrations/core/migrate_raw_docs.test.ts @@ -90,4 +90,18 @@ describe('migrateRawDocs', () => { expect(logger.error).toBeCalledTimes(1); }); + + test('rejects when the transform function throws an error', async () => { + const transform = jest.fn((doc: any) => { + throw new Error('error during transform'); + }); + await expect( + migrateRawDocs( + new SavedObjectsSerializer(new SavedObjectTypeRegistry()), + transform, + [{ _id: 'a:b', _source: { type: 'a', a: { name: 'AAA' } } }], + createSavedObjectsMigrationLoggerMock() + ) + ).rejects.toThrowErrorMatchingInlineSnapshot(`"error during transform"`); + }); }); diff --git a/src/core/server/saved_objects/migrations/core/migrate_raw_docs.ts b/src/core/server/saved_objects/migrations/core/migrate_raw_docs.ts index 2bdf59d25dc74..5a5048d8ad88f 100644 --- a/src/core/server/saved_objects/migrations/core/migrate_raw_docs.ts +++ b/src/core/server/saved_objects/migrations/core/migrate_raw_docs.ts @@ -78,10 +78,14 @@ function transformNonBlocking( ): (doc: SavedObjectUnsanitizedDoc) => Promise { // promises aren't enough to unblock the event loop return (doc: SavedObjectUnsanitizedDoc) => - new Promise((resolve) => { + new Promise((resolve, reject) => { // set immediate is though setImmediate(() => { - resolve(transform(doc)); + try { + resolve(transform(doc)); + } catch (e) { + reject(e); + } }); }); } diff --git a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts index b9f24a75c01d2..18a385c6994b8 100644 --- a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts +++ b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts @@ -120,9 +120,17 @@ export class KibanaMigrator { Array<{ status: string }> > { if (this.migrationResult === undefined || rerun) { - this.status$.next({ status: 'running' }); + // Reruns are only used by CI / EsArchiver. Publishing status updates on reruns results in slowing down CI + // unnecessarily, so we skip it in this case. + if (!rerun) { + this.status$.next({ status: 'running' }); + } + this.migrationResult = this.runMigrationsInternal().then((result) => { - this.status$.next({ status: 'completed', result }); + // Similar to above, don't publish status updates when rerunning in CI. + if (!rerun) { + this.status$.next({ status: 'completed', result }); + } return result; }); } diff --git a/src/core/server/server.api.md b/src/core/server/server.api.md index 37023a0a8ef67..aef1bda9ccf4e 100644 --- a/src/core/server/server.api.md +++ b/src/core/server/server.api.md @@ -1531,10 +1531,10 @@ export interface LogRecord { timestamp: Date; } -// Warning: (ae-missing-release-tag) "MetricsServiceSetup" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export interface MetricsServiceSetup { + readonly collectionInterval: number; + getOpsMetrics$: () => Observable; } // @public @deprecated (undocumented) @@ -1621,6 +1621,7 @@ export interface OnPreRoutingToolkit { // @public export interface OpsMetrics { + collected_at: Date; concurrent_connections: OpsServerMetrics['concurrent_connections']; os: OpsOsMetrics; process: OpsProcessMetrics; @@ -1630,6 +1631,20 @@ export interface OpsMetrics { // @public export interface OpsOsMetrics { + cpu?: { + control_group: string; + cfs_period_micros: number; + cfs_quota_micros: number; + stat: { + number_of_elapsed_periods: number; + number_of_times_throttled: number; + time_throttled_nanos: number; + }; + }; + cpuacct?: { + control_group: string; + usage_nanos: number; + }; distro?: string; distroRelease?: string; load: { @@ -2763,10 +2778,17 @@ export type SharedGlobalConfig = RecursiveReadonly<{ // @public export type StartServicesAccessor = () => Promise<[CoreStart, TPluginsStart, TStart]>; +// Warning: (ae-unresolved-link) The @link reference could not be resolved: The package "kibana" does not have an export "ServiceStatusSetup" +// Warning: (ae-unresolved-link) The @link reference could not be resolved: The package "kibana" does not have an export "ServiceStatusSetup" +// // @public export interface StatusServiceSetup { core$: Observable; + dependencies$: Observable>; + // Warning: (ae-unresolved-link) The @link reference could not be resolved: The package "kibana" does not have an export "StatusSetup" + derivedStatus$: Observable; overall$: Observable; + set(status$: Observable): void; } // @public @@ -2855,8 +2877,8 @@ export const validBodyOutput: readonly ["data", "stream"]; // // src/core/server/http/router/response.ts:316:3 - (ae-forgotten-export) The symbol "KibanaResponse" needs to be exported by the entry point index.d.ts // src/core/server/legacy/types.ts:135:16 - (ae-forgotten-export) The symbol "LegacyPluginSpec" needs to be exported by the entry point index.d.ts -// src/core/server/plugins/types.ts:266:3 - (ae-forgotten-export) The symbol "KibanaConfigType" needs to be exported by the entry point index.d.ts -// src/core/server/plugins/types.ts:266:3 - (ae-forgotten-export) The symbol "SharedGlobalConfigKeys" needs to be exported by the entry point index.d.ts -// src/core/server/plugins/types.ts:268:3 - (ae-forgotten-export) The symbol "PathConfigType" needs to be exported by the entry point index.d.ts +// src/core/server/plugins/types.ts:272:3 - (ae-forgotten-export) The symbol "KibanaConfigType" needs to be exported by the entry point index.d.ts +// src/core/server/plugins/types.ts:272:3 - (ae-forgotten-export) The symbol "SharedGlobalConfigKeys" needs to be exported by the entry point index.d.ts +// src/core/server/plugins/types.ts:274:3 - (ae-forgotten-export) The symbol "PathConfigType" needs to be exported by the entry point index.d.ts ``` diff --git a/src/core/server/server.test.ts b/src/core/server/server.test.ts index 417f66a2988c2..8bf16d9130ef5 100644 --- a/src/core/server/server.test.ts +++ b/src/core/server/server.test.ts @@ -49,7 +49,7 @@ const rawConfigService = rawConfigServiceMock.create({}); beforeEach(() => { mockConfigService.atPath.mockReturnValue(new BehaviorSubject({ autoListen: true })); mockPluginsService.discover.mockResolvedValue({ - pluginTree: new Map(), + pluginTree: { asOpaqueIds: new Map(), asNames: new Map() }, uiPlugins: { internal: new Map(), public: new Map(), browserConfigs: new Map() }, }); }); @@ -98,7 +98,7 @@ test('injects legacy dependency to context#setup()', async () => { [pluginB, [pluginA]], ]); mockPluginsService.discover.mockResolvedValue({ - pluginTree: pluginDependencies, + pluginTree: { asOpaqueIds: pluginDependencies, asNames: new Map() }, uiPlugins: { internal: new Map(), public: new Map(), browserConfigs: new Map() }, }); diff --git a/src/core/server/server.ts b/src/core/server/server.ts index 278dd72d72bb1..a02b0f51b559f 100644 --- a/src/core/server/server.ts +++ b/src/core/server/server.ts @@ -121,10 +121,13 @@ export class Server { const contextServiceSetup = this.context.setup({ // We inject a fake "legacy plugin" with dependencies on every plugin so that legacy plugins: - // 1) Can access context from any NP plugin + // 1) Can access context from any KP plugin // 2) Can register context providers that will only be available to other legacy plugins and will not leak into // New Platform plugins. - pluginDependencies: new Map([...pluginTree, [this.legacy.legacyId, [...pluginTree.keys()]]]), + pluginDependencies: new Map([ + ...pluginTree.asOpaqueIds, + [this.legacy.legacyId, [...pluginTree.asOpaqueIds.keys()]], + ]), }); const auditTrailSetup = this.auditTrail.setup(); @@ -153,6 +156,7 @@ export class Server { const statusSetup = await this.status.setup({ elasticsearch: elasticsearchServiceSetup, + pluginDependencies: pluginTree.asNames, savedObjects: savedObjectsSetup, }); diff --git a/src/core/server/status/get_summary_status.test.ts b/src/core/server/status/get_summary_status.test.ts index 7516e82ee784d..d97083162b502 100644 --- a/src/core/server/status/get_summary_status.test.ts +++ b/src/core/server/status/get_summary_status.test.ts @@ -94,6 +94,38 @@ describe('getSummaryStatus', () => { describe('summary', () => { describe('when a single service is at highest level', () => { it('returns all information about that single service', () => { + expect( + getSummaryStatus( + Object.entries({ + s1: degraded, + s2: { + level: ServiceStatusLevels.unavailable, + summary: 'Lorem ipsum', + meta: { + custom: { data: 'here' }, + }, + }, + }) + ) + ).toEqual({ + level: ServiceStatusLevels.unavailable, + summary: '[s2]: Lorem ipsum', + detail: 'See the status page for more information', + meta: { + affectedServices: { + s2: { + level: ServiceStatusLevels.unavailable, + summary: 'Lorem ipsum', + meta: { + custom: { data: 'here' }, + }, + }, + }, + }, + }); + }); + + it('allows the single service to override the detail and documentationUrl fields', () => { expect( getSummaryStatus( Object.entries({ @@ -115,7 +147,17 @@ describe('getSummaryStatus', () => { detail: 'Vivamus pulvinar sem ac luctus ultrices.', documentationUrl: 'http://helpmenow.com/problem1', meta: { - custom: { data: 'here' }, + affectedServices: { + s2: { + level: ServiceStatusLevels.unavailable, + summary: 'Lorem ipsum', + detail: 'Vivamus pulvinar sem ac luctus ultrices.', + documentationUrl: 'http://helpmenow.com/problem1', + meta: { + custom: { data: 'here' }, + }, + }, + }, }, }); }); diff --git a/src/core/server/status/get_summary_status.ts b/src/core/server/status/get_summary_status.ts index 748a54f0bf8bb..8d97cdbd9b15b 100644 --- a/src/core/server/status/get_summary_status.ts +++ b/src/core/server/status/get_summary_status.ts @@ -23,62 +23,60 @@ import { ServiceStatus, ServiceStatusLevels, ServiceStatusLevel } from './types' * Returns a single {@link ServiceStatus} that summarizes the most severe status level from a group of statuses. * @param statuses */ -export const getSummaryStatus = (statuses: Array<[string, ServiceStatus]>): ServiceStatus => { - const grouped = groupByLevel(statuses); - const highestSeverityLevel = getHighestSeverityLevel(grouped.keys()); - const highestSeverityGroup = grouped.get(highestSeverityLevel)!; +export const getSummaryStatus = ( + statuses: Array<[string, ServiceStatus]>, + { allAvailableSummary = `All services are available` }: { allAvailableSummary?: string } = {} +): ServiceStatus => { + const { highestLevel, highestStatuses } = highestLevelSummary(statuses); - if (highestSeverityLevel === ServiceStatusLevels.available) { + if (highestLevel === ServiceStatusLevels.available) { return { level: ServiceStatusLevels.available, - summary: `All services are available`, + summary: allAvailableSummary, }; - } else if (highestSeverityGroup.size === 1) { - const [serviceName, status] = [...highestSeverityGroup.entries()][0]; + } else if (highestStatuses.length === 1) { + const [serviceName, status] = highestStatuses[0]! as [string, ServiceStatus]; return { ...status, summary: `[${serviceName}]: ${status.summary!}`, + // TODO: include URL to status page + detail: status.detail ?? `See the status page for more information`, + meta: { + affectedServices: { [serviceName]: status }, + }, }; } else { return { - level: highestSeverityLevel, - summary: `[${highestSeverityGroup.size}] services are ${highestSeverityLevel.toString()}`, + level: highestLevel, + summary: `[${highestStatuses.length}] services are ${highestLevel.toString()}`, // TODO: include URL to status page detail: `See the status page for more information`, meta: { - affectedServices: Object.fromEntries([...highestSeverityGroup]), + affectedServices: Object.fromEntries(highestStatuses), }, }; } }; -const groupByLevel = ( - statuses: Array<[string, ServiceStatus]> -): Map> => { - const byLevel = new Map>(); +type StatusPair = [string, ServiceStatus]; - for (const [serviceName, status] of statuses) { - let levelMap = byLevel.get(status.level); - if (!levelMap) { - levelMap = new Map(); - byLevel.set(status.level, levelMap); - } +const highestLevelSummary = ( + statuses: StatusPair[] +): { highestLevel: ServiceStatusLevel; highestStatuses: StatusPair[] } => { + let highestLevel: ServiceStatusLevel = ServiceStatusLevels.available; + let highestStatuses: StatusPair[] = []; - levelMap.set(serviceName, status); + for (const pair of statuses) { + if (pair[1].level === highestLevel) { + highestStatuses.push(pair); + } else if (pair[1].level > highestLevel) { + highestLevel = pair[1].level; + highestStatuses = [pair]; + } } - return byLevel; -}; - -const getHighestSeverityLevel = (levels: Iterable): ServiceStatusLevel => { - const sorted = [...levels].sort((a, b) => { - if (a < b) { - return -1; - } else if (a > b) { - return 1; - } else { - return 0; - } - }); - return sorted[sorted.length - 1] ?? ServiceStatusLevels.available; + return { + highestLevel, + highestStatuses, + }; }; diff --git a/src/core/server/status/plugins_status.test.ts b/src/core/server/status/plugins_status.test.ts new file mode 100644 index 0000000000000..a75dc8c283698 --- /dev/null +++ b/src/core/server/status/plugins_status.test.ts @@ -0,0 +1,338 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { PluginName } from '../plugins'; +import { PluginsStatusService } from './plugins_status'; +import { of, Observable, BehaviorSubject } from 'rxjs'; +import { ServiceStatusLevels, CoreStatus, ServiceStatus } from './types'; +import { first } from 'rxjs/operators'; +import { ServiceStatusLevelSnapshotSerializer } from './test_utils'; + +expect.addSnapshotSerializer(ServiceStatusLevelSnapshotSerializer); + +describe('PluginStatusService', () => { + const coreAllAvailable$: Observable = of({ + elasticsearch: { level: ServiceStatusLevels.available, summary: 'elasticsearch avail' }, + savedObjects: { level: ServiceStatusLevels.available, summary: 'savedObjects avail' }, + }); + const coreOneDegraded$: Observable = of({ + elasticsearch: { level: ServiceStatusLevels.available, summary: 'elasticsearch avail' }, + savedObjects: { level: ServiceStatusLevels.degraded, summary: 'savedObjects degraded' }, + }); + const coreOneCriticalOneDegraded$: Observable = of({ + elasticsearch: { level: ServiceStatusLevels.critical, summary: 'elasticsearch critical' }, + savedObjects: { level: ServiceStatusLevels.degraded, summary: 'savedObjects degraded' }, + }); + const pluginDependencies: Map = new Map([ + ['a', []], + ['b', ['a']], + ['c', ['a', 'b']], + ]); + + describe('getDerivedStatus$', () => { + it(`defaults to core's most severe status`, async () => { + const serviceAvailable = new PluginsStatusService({ + core$: coreAllAvailable$, + pluginDependencies, + }); + expect(await serviceAvailable.getDerivedStatus$('a').pipe(first()).toPromise()).toEqual({ + level: ServiceStatusLevels.available, + summary: 'All dependencies are available', + }); + + const serviceDegraded = new PluginsStatusService({ + core$: coreOneDegraded$, + pluginDependencies, + }); + expect(await serviceDegraded.getDerivedStatus$('a').pipe(first()).toPromise()).toEqual({ + level: ServiceStatusLevels.degraded, + summary: '[savedObjects]: savedObjects degraded', + detail: 'See the status page for more information', + meta: expect.any(Object), + }); + + const serviceCritical = new PluginsStatusService({ + core$: coreOneCriticalOneDegraded$, + pluginDependencies, + }); + expect(await serviceCritical.getDerivedStatus$('a').pipe(first()).toPromise()).toEqual({ + level: ServiceStatusLevels.critical, + summary: '[elasticsearch]: elasticsearch critical', + detail: 'See the status page for more information', + meta: expect.any(Object), + }); + }); + + it(`provides a summary status when core and dependencies are at same severity level`, async () => { + const service = new PluginsStatusService({ core$: coreOneDegraded$, pluginDependencies }); + service.set('a', of({ level: ServiceStatusLevels.degraded, summary: 'a is degraded' })); + expect(await service.getDerivedStatus$('b').pipe(first()).toPromise()).toEqual({ + level: ServiceStatusLevels.degraded, + summary: '[2] services are degraded', + detail: 'See the status page for more information', + meta: expect.any(Object), + }); + }); + + it(`allows dependencies status to take precedence over lower severity core statuses`, async () => { + const service = new PluginsStatusService({ core$: coreOneDegraded$, pluginDependencies }); + service.set('a', of({ level: ServiceStatusLevels.unavailable, summary: 'a is not working' })); + expect(await service.getDerivedStatus$('b').pipe(first()).toPromise()).toEqual({ + level: ServiceStatusLevels.unavailable, + summary: '[a]: a is not working', + detail: 'See the status page for more information', + meta: expect.any(Object), + }); + }); + + it(`allows core status to take precedence over lower severity dependencies statuses`, async () => { + const service = new PluginsStatusService({ + core$: coreOneCriticalOneDegraded$, + pluginDependencies, + }); + service.set('a', of({ level: ServiceStatusLevels.unavailable, summary: 'a is not working' })); + expect(await service.getDerivedStatus$('b').pipe(first()).toPromise()).toEqual({ + level: ServiceStatusLevels.critical, + summary: '[elasticsearch]: elasticsearch critical', + detail: 'See the status page for more information', + meta: expect.any(Object), + }); + }); + + it(`allows a severe dependency status to take precedence over a less severe dependency status`, async () => { + const service = new PluginsStatusService({ core$: coreOneDegraded$, pluginDependencies }); + service.set('a', of({ level: ServiceStatusLevels.degraded, summary: 'a is degraded' })); + service.set('b', of({ level: ServiceStatusLevels.unavailable, summary: 'b is not working' })); + expect(await service.getDerivedStatus$('c').pipe(first()).toPromise()).toEqual({ + level: ServiceStatusLevels.unavailable, + summary: '[b]: b is not working', + detail: 'See the status page for more information', + meta: expect.any(Object), + }); + }); + }); + + describe('getAll$', () => { + it('defaults to empty record if no plugins', async () => { + const service = new PluginsStatusService({ + core$: coreAllAvailable$, + pluginDependencies: new Map(), + }); + expect(await service.getAll$().pipe(first()).toPromise()).toEqual({}); + }); + + it('defaults to core status when no plugin statuses are set', async () => { + const serviceAvailable = new PluginsStatusService({ + core$: coreAllAvailable$, + pluginDependencies, + }); + expect(await serviceAvailable.getAll$().pipe(first()).toPromise()).toEqual({ + a: { level: ServiceStatusLevels.available, summary: 'All dependencies are available' }, + b: { level: ServiceStatusLevels.available, summary: 'All dependencies are available' }, + c: { level: ServiceStatusLevels.available, summary: 'All dependencies are available' }, + }); + + const serviceDegraded = new PluginsStatusService({ + core$: coreOneDegraded$, + pluginDependencies, + }); + expect(await serviceDegraded.getAll$().pipe(first()).toPromise()).toEqual({ + a: { + level: ServiceStatusLevels.degraded, + summary: '[savedObjects]: savedObjects degraded', + detail: 'See the status page for more information', + meta: expect.any(Object), + }, + b: { + level: ServiceStatusLevels.degraded, + summary: '[2] services are degraded', + detail: 'See the status page for more information', + meta: expect.any(Object), + }, + c: { + level: ServiceStatusLevels.degraded, + summary: '[3] services are degraded', + detail: 'See the status page for more information', + meta: expect.any(Object), + }, + }); + + const serviceCritical = new PluginsStatusService({ + core$: coreOneCriticalOneDegraded$, + pluginDependencies, + }); + expect(await serviceCritical.getAll$().pipe(first()).toPromise()).toEqual({ + a: { + level: ServiceStatusLevels.critical, + summary: '[elasticsearch]: elasticsearch critical', + detail: 'See the status page for more information', + meta: expect.any(Object), + }, + b: { + level: ServiceStatusLevels.critical, + summary: '[2] services are critical', + detail: 'See the status page for more information', + meta: expect.any(Object), + }, + c: { + level: ServiceStatusLevels.critical, + summary: '[3] services are critical', + detail: 'See the status page for more information', + meta: expect.any(Object), + }, + }); + }); + + it('uses the manually set status level if plugin specifies one', async () => { + const service = new PluginsStatusService({ core$: coreOneDegraded$, pluginDependencies }); + service.set('a', of({ level: ServiceStatusLevels.available, summary: 'a status' })); + + expect(await service.getAll$().pipe(first()).toPromise()).toEqual({ + a: { level: ServiceStatusLevels.available, summary: 'a status' }, // a is available depsite savedObjects being degraded + b: { + level: ServiceStatusLevels.degraded, + summary: '[savedObjects]: savedObjects degraded', + detail: 'See the status page for more information', + meta: expect.any(Object), + }, + c: { + level: ServiceStatusLevels.degraded, + summary: '[2] services are degraded', + detail: 'See the status page for more information', + meta: expect.any(Object), + }, + }); + }); + + it('updates when a new plugin status observable is set', async () => { + const service = new PluginsStatusService({ + core$: coreAllAvailable$, + pluginDependencies: new Map([['a', []]]), + }); + const statusUpdates: Array> = []; + const subscription = service + .getAll$() + .subscribe((pluginStatuses) => statusUpdates.push(pluginStatuses)); + + service.set('a', of({ level: ServiceStatusLevels.degraded, summary: 'a degraded' })); + service.set('a', of({ level: ServiceStatusLevels.unavailable, summary: 'a unavailable' })); + service.set('a', of({ level: ServiceStatusLevels.available, summary: 'a available' })); + subscription.unsubscribe(); + + expect(statusUpdates).toEqual([ + { a: { level: ServiceStatusLevels.available, summary: 'All dependencies are available' } }, + { a: { level: ServiceStatusLevels.degraded, summary: 'a degraded' } }, + { a: { level: ServiceStatusLevels.unavailable, summary: 'a unavailable' } }, + { a: { level: ServiceStatusLevels.available, summary: 'a available' } }, + ]); + }); + }); + + describe('getDependenciesStatus$', () => { + it('only includes dependencies of specified plugin', async () => { + const service = new PluginsStatusService({ + core$: coreAllAvailable$, + pluginDependencies, + }); + expect(await service.getDependenciesStatus$('a').pipe(first()).toPromise()).toEqual({}); + expect(await service.getDependenciesStatus$('b').pipe(first()).toPromise()).toEqual({ + a: { level: ServiceStatusLevels.available, summary: 'All dependencies are available' }, + }); + expect(await service.getDependenciesStatus$('c').pipe(first()).toPromise()).toEqual({ + a: { level: ServiceStatusLevels.available, summary: 'All dependencies are available' }, + b: { level: ServiceStatusLevels.available, summary: 'All dependencies are available' }, + }); + }); + + it('uses the manually set status level if plugin specifies one', async () => { + const service = new PluginsStatusService({ core$: coreOneDegraded$, pluginDependencies }); + service.set('a', of({ level: ServiceStatusLevels.available, summary: 'a status' })); + + expect(await service.getDependenciesStatus$('c').pipe(first()).toPromise()).toEqual({ + a: { level: ServiceStatusLevels.available, summary: 'a status' }, // a is available depsite savedObjects being degraded + b: { + level: ServiceStatusLevels.degraded, + summary: '[savedObjects]: savedObjects degraded', + detail: 'See the status page for more information', + meta: expect.any(Object), + }, + }); + }); + + it('throws error if unknown plugin passed', () => { + const service = new PluginsStatusService({ core$: coreAllAvailable$, pluginDependencies }); + expect(() => { + service.getDependenciesStatus$('dont-exist'); + }).toThrowError(); + }); + + it('debounces events in quick succession', async () => { + const service = new PluginsStatusService({ + core$: coreAllAvailable$, + pluginDependencies, + }); + const available: ServiceStatus = { + level: ServiceStatusLevels.available, + summary: 'a available', + }; + const degraded: ServiceStatus = { + level: ServiceStatusLevels.degraded, + summary: 'a degraded', + }; + const pluginA$ = new BehaviorSubject(available); + service.set('a', pluginA$); + + const statusUpdates: Array> = []; + const subscription = service + .getDependenciesStatus$('b') + .subscribe((status) => statusUpdates.push(status)); + const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + + pluginA$.next(degraded); + pluginA$.next(available); + pluginA$.next(degraded); + pluginA$.next(available); + pluginA$.next(degraded); + pluginA$.next(available); + pluginA$.next(degraded); + // Waiting for the debounce timeout should cut a new update + await delay(500); + pluginA$.next(available); + await delay(500); + subscription.unsubscribe(); + + expect(statusUpdates).toMatchInlineSnapshot(` + Array [ + Object { + "a": Object { + "level": degraded, + "summary": "a degraded", + }, + }, + Object { + "a": Object { + "level": available, + "summary": "a available", + }, + }, + ] + `); + }); + }); +}); diff --git a/src/core/server/status/plugins_status.ts b/src/core/server/status/plugins_status.ts new file mode 100644 index 0000000000000..113d59b327c11 --- /dev/null +++ b/src/core/server/status/plugins_status.ts @@ -0,0 +1,98 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { BehaviorSubject, Observable, combineLatest, of } from 'rxjs'; +import { map, distinctUntilChanged, switchMap, debounceTime } from 'rxjs/operators'; +import { isDeepStrictEqual } from 'util'; + +import { PluginName } from '../plugins'; +import { ServiceStatus, CoreStatus } from './types'; +import { getSummaryStatus } from './get_summary_status'; + +interface Deps { + core$: Observable; + pluginDependencies: ReadonlyMap; +} + +export class PluginsStatusService { + private readonly pluginStatuses = new Map>(); + private readonly update$ = new BehaviorSubject(true); + constructor(private readonly deps: Deps) {} + + public set(plugin: PluginName, status$: Observable) { + this.pluginStatuses.set(plugin, status$); + this.update$.next(true); // trigger all existing Observables to update from the new source Observable + } + + public getAll$(): Observable> { + return this.getPluginStatuses$([...this.deps.pluginDependencies.keys()]); + } + + public getDependenciesStatus$(plugin: PluginName): Observable> { + const dependencies = this.deps.pluginDependencies.get(plugin); + if (!dependencies) { + throw new Error(`Unknown plugin: ${plugin}`); + } + + return this.getPluginStatuses$(dependencies).pipe( + // Prevent many emissions at once from dependency status resolution from making this too noisy + debounceTime(500) + ); + } + + public getDerivedStatus$(plugin: PluginName): Observable { + return combineLatest([this.deps.core$, this.getDependenciesStatus$(plugin)]).pipe( + map(([coreStatus, pluginStatuses]) => { + return getSummaryStatus( + [...Object.entries(coreStatus), ...Object.entries(pluginStatuses)], + { + allAvailableSummary: `All dependencies are available`, + } + ); + }) + ); + } + + private getPluginStatuses$(plugins: PluginName[]): Observable> { + if (plugins.length === 0) { + return of({}); + } + + return this.update$.pipe( + switchMap(() => { + const pluginStatuses = plugins + .map( + (depName) => + [depName, this.pluginStatuses.get(depName) ?? this.getDerivedStatus$(depName)] as [ + PluginName, + Observable + ] + ) + .map(([pName, status$]) => + status$.pipe(map((status) => [pName, status] as [PluginName, ServiceStatus])) + ); + + return combineLatest(pluginStatuses).pipe( + map((statuses) => Object.fromEntries(statuses)), + distinctUntilChanged(isDeepStrictEqual) + ); + }) + ); + } +} diff --git a/src/core/server/status/status_service.mock.ts b/src/core/server/status/status_service.mock.ts index 47ef8659b4079..42b3eecdca310 100644 --- a/src/core/server/status/status_service.mock.ts +++ b/src/core/server/status/status_service.mock.ts @@ -40,6 +40,9 @@ const createSetupContractMock = () => { const setupContract: jest.Mocked = { core$: new BehaviorSubject(availableCoreStatus), overall$: new BehaviorSubject(available), + set: jest.fn(), + dependencies$: new BehaviorSubject({}), + derivedStatus$: new BehaviorSubject(available), }; return setupContract; @@ -50,6 +53,11 @@ const createInternalSetupContractMock = () => { core$: new BehaviorSubject(availableCoreStatus), overall$: new BehaviorSubject(available), isStatusPageAnonymous: jest.fn().mockReturnValue(false), + plugins: { + set: jest.fn(), + getDependenciesStatus$: jest.fn(), + getDerivedStatus$: jest.fn(), + }, }; return setupContract; diff --git a/src/core/server/status/status_service.test.ts b/src/core/server/status/status_service.test.ts index 863fe34e8ecea..dcb1e0a559f5d 100644 --- a/src/core/server/status/status_service.test.ts +++ b/src/core/server/status/status_service.test.ts @@ -34,6 +34,7 @@ describe('StatusService', () => { service = new StatusService(mockCoreContext.create()); }); + const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); const available: ServiceStatus = { level: ServiceStatusLevels.available, summary: 'Available', @@ -53,6 +54,7 @@ describe('StatusService', () => { savedObjects: { status$: of(degraded), }, + pluginDependencies: new Map(), }); expect(await setup.core$.pipe(first()).toPromise()).toEqual({ elasticsearch: available, @@ -68,6 +70,7 @@ describe('StatusService', () => { savedObjects: { status$: of(degraded), }, + pluginDependencies: new Map(), }); const subResult1 = await setup.core$.pipe(first()).toPromise(); const subResult2 = await setup.core$.pipe(first()).toPromise(); @@ -96,6 +99,7 @@ describe('StatusService', () => { savedObjects: { status$: savedObjects$, }, + pluginDependencies: new Map(), }); const statusUpdates: CoreStatus[] = []; @@ -158,6 +162,7 @@ describe('StatusService', () => { savedObjects: { status$: of(degraded), }, + pluginDependencies: new Map(), }); expect(await setup.overall$.pipe(first()).toPromise()).toMatchObject({ level: ServiceStatusLevels.degraded, @@ -173,6 +178,7 @@ describe('StatusService', () => { savedObjects: { status$: of(degraded), }, + pluginDependencies: new Map(), }); const subResult1 = await setup.overall$.pipe(first()).toPromise(); const subResult2 = await setup.overall$.pipe(first()).toPromise(); @@ -201,26 +207,95 @@ describe('StatusService', () => { savedObjects: { status$: savedObjects$, }, + pluginDependencies: new Map(), }); const statusUpdates: ServiceStatus[] = []; const subscription = setup.overall$.subscribe((status) => statusUpdates.push(status)); + // Wait for timers to ensure that duplicate events are still filtered out regardless of debouncing. elasticsearch$.next(available); + await delay(500); elasticsearch$.next(available); + await delay(500); elasticsearch$.next({ level: ServiceStatusLevels.available, summary: `Wow another summary`, }); + await delay(500); savedObjects$.next(degraded); + await delay(500); savedObjects$.next(available); + await delay(500); savedObjects$.next(available); + await delay(500); subscription.unsubscribe(); expect(statusUpdates).toMatchInlineSnapshot(` Array [ Object { + "detail": "See the status page for more information", "level": degraded, + "meta": Object { + "affectedServices": Object { + "savedObjects": Object { + "level": degraded, + "summary": "This is degraded!", + }, + }, + }, + "summary": "[savedObjects]: This is degraded!", + }, + Object { + "level": available, + "summary": "All services are available", + }, + ] + `); + }); + + it('debounces events in quick succession', async () => { + const savedObjects$ = new BehaviorSubject(available); + const setup = await service.setup({ + elasticsearch: { + status$: new BehaviorSubject(available), + }, + savedObjects: { + status$: savedObjects$, + }, + pluginDependencies: new Map(), + }); + + const statusUpdates: ServiceStatus[] = []; + const subscription = setup.overall$.subscribe((status) => statusUpdates.push(status)); + + // All of these should debounced into a single `available` status + savedObjects$.next(degraded); + savedObjects$.next(available); + savedObjects$.next(degraded); + savedObjects$.next(available); + savedObjects$.next(degraded); + savedObjects$.next(available); + savedObjects$.next(degraded); + // Waiting for the debounce timeout should cut a new update + await delay(500); + savedObjects$.next(available); + await delay(500); + subscription.unsubscribe(); + + expect(statusUpdates).toMatchInlineSnapshot(` + Array [ + Object { + "detail": "See the status page for more information", + "level": degraded, + "meta": Object { + "affectedServices": Object { + "savedObjects": Object { + "level": degraded, + "summary": "This is degraded!", + }, + }, + }, "summary": "[savedObjects]: This is degraded!", }, Object { diff --git a/src/core/server/status/status_service.ts b/src/core/server/status/status_service.ts index aea335e64babf..8fe65eddb61d3 100644 --- a/src/core/server/status/status_service.ts +++ b/src/core/server/status/status_service.ts @@ -18,7 +18,7 @@ */ import { Observable, combineLatest } from 'rxjs'; -import { map, distinctUntilChanged, shareReplay, take } from 'rxjs/operators'; +import { map, distinctUntilChanged, shareReplay, take, debounceTime } from 'rxjs/operators'; import { isDeepStrictEqual } from 'util'; import { CoreService } from '../../types'; @@ -26,13 +26,16 @@ import { CoreContext } from '../core_context'; import { Logger } from '../logging'; import { InternalElasticsearchServiceSetup } from '../elasticsearch'; import { InternalSavedObjectsServiceSetup } from '../saved_objects'; +import { PluginName } from '../plugins'; import { config, StatusConfigType } from './status_config'; import { ServiceStatus, CoreStatus, InternalStatusServiceSetup } from './types'; import { getSummaryStatus } from './get_summary_status'; +import { PluginsStatusService } from './plugins_status'; interface SetupDeps { elasticsearch: Pick; + pluginDependencies: ReadonlyMap; savedObjects: Pick; } @@ -40,26 +43,44 @@ export class StatusService implements CoreService { private readonly logger: Logger; private readonly config$: Observable; + private pluginsStatus?: PluginsStatusService; + constructor(coreContext: CoreContext) { this.logger = coreContext.logger.get('status'); this.config$ = coreContext.configService.atPath(config.path); } - public async setup(core: SetupDeps) { + public async setup({ elasticsearch, pluginDependencies, savedObjects }: SetupDeps) { const statusConfig = await this.config$.pipe(take(1)).toPromise(); - const core$ = this.setupCoreStatus(core); - const overall$: Observable = core$.pipe( - map((coreStatus) => { - const summary = getSummaryStatus(Object.entries(coreStatus)); + const core$ = this.setupCoreStatus({ elasticsearch, savedObjects }); + this.pluginsStatus = new PluginsStatusService({ core$, pluginDependencies }); + + const overall$: Observable = combineLatest( + core$, + this.pluginsStatus.getAll$() + ).pipe( + // Prevent many emissions at once from dependency status resolution from making this too noisy + debounceTime(500), + map(([coreStatus, pluginsStatus]) => { + const summary = getSummaryStatus([ + ...Object.entries(coreStatus), + ...Object.entries(pluginsStatus), + ]); this.logger.debug(`Recalculated overall status`, { status: summary }); return summary; }), - distinctUntilChanged(isDeepStrictEqual) + distinctUntilChanged(isDeepStrictEqual), + shareReplay(1) ); return { core$, overall$, + plugins: { + set: this.pluginsStatus.set.bind(this.pluginsStatus), + getDependenciesStatus$: this.pluginsStatus.getDependenciesStatus$.bind(this.pluginsStatus), + getDerivedStatus$: this.pluginsStatus.getDerivedStatus$.bind(this.pluginsStatus), + }, isStatusPageAnonymous: () => statusConfig.allowAnonymous, }; } @@ -68,7 +89,10 @@ export class StatusService implements CoreService { public stop() {} - private setupCoreStatus({ elasticsearch, savedObjects }: SetupDeps): Observable { + private setupCoreStatus({ + elasticsearch, + savedObjects, + }: Pick): Observable { return combineLatest([elasticsearch.status$, savedObjects.status$]).pipe( map(([elasticsearchStatus, savedObjectsStatus]) => ({ elasticsearch: elasticsearchStatus, diff --git a/src/core/server/status/types.ts b/src/core/server/status/types.ts index 2ecf11deb2960..f884b80316fa8 100644 --- a/src/core/server/status/types.ts +++ b/src/core/server/status/types.ts @@ -19,6 +19,7 @@ import { Observable } from 'rxjs'; import { deepFreeze } from '../../utils'; +import { PluginName } from '../plugins'; /** * The current status of a service at a point in time. @@ -116,6 +117,60 @@ export interface CoreStatus { /** * API for accessing status of Core and this plugin's dependencies as well as for customizing this plugin's status. + * + * @remarks + * By default, a plugin inherits it's current status from the most severe status level of any Core services and any + * plugins that it depends on. This default status is available on the + * {@link ServiceStatusSetup.derivedStatus$ | core.status.derviedStatus$} API. + * + * Plugins may customize their status calculation by calling the {@link ServiceStatusSetup.set | core.status.set} API + * with an Observable. Within this Observable, a plugin may choose to only depend on the status of some of its + * dependencies, to ignore severe status levels of particular Core services they are not concerned with, or to make its + * status dependent on other external services. + * + * @example + * Customize a plugin's status to only depend on the status of SavedObjects: + * ```ts + * core.status.set( + * core.status.core$.pipe( + * . map((coreStatus) => { + * return coreStatus.savedObjects; + * }) ; + * ); + * ); + * ``` + * + * @example + * Customize a plugin's status to include an external service: + * ```ts + * const externalStatus$ = interval(1000).pipe( + * switchMap(async () => { + * const resp = await fetch(`https://myexternaldep.com/_healthz`); + * const body = await resp.json(); + * if (body.ok) { + * return of({ level: ServiceStatusLevels.available, summary: 'External Service is up'}); + * } else { + * return of({ level: ServiceStatusLevels.available, summary: 'External Service is unavailable'}); + * } + * }), + * catchError((error) => { + * of({ level: ServiceStatusLevels.unavailable, summary: `External Service is down`, meta: { error }}) + * }) + * ); + * + * core.status.set( + * combineLatest([core.status.derivedStatus$, externalStatus$]).pipe( + * map(([derivedStatus, externalStatus]) => { + * if (externalStatus.level > derivedStatus) { + * return externalStatus; + * } else { + * return derivedStatus; + * } + * }) + * ) + * ); + * ``` + * * @public */ export interface StatusServiceSetup { @@ -134,9 +189,43 @@ export interface StatusServiceSetup { * only depend on the statuses of {@link StatusServiceSetup.core$ | Core} or their dependencies. */ overall$: Observable; + + /** + * Allows a plugin to specify a custom status dependent on its own criteria. + * Completely overrides the default inherited status. + * + * @remarks + * See the {@link StatusServiceSetup.derivedStatus$} API for leveraging the default status + * calculation that is provided by Core. + */ + set(status$: Observable): void; + + /** + * Current status for all plugins this plugin depends on. + * Each key of the `Record` is a plugin id. + */ + dependencies$: Observable>; + + /** + * The status of this plugin as derived from its dependencies. + * + * @remarks + * By default, plugins inherit this derived status from their dependencies. + * Calling {@link StatusSetup.set} overrides this default status. + * + * This may emit multliple times for a single status change event as propagates + * through the dependency tree + */ + derivedStatus$: Observable; } /** @internal */ -export interface InternalStatusServiceSetup extends StatusServiceSetup { +export interface InternalStatusServiceSetup extends Pick { isStatusPageAnonymous: () => boolean; + // Namespaced under `plugins` key to improve clarity that these are APIs for plugins specifically. + plugins: { + set(plugin: PluginName, status$: Observable): void; + getDependenciesStatus$(plugin: PluginName): Observable>; + getDerivedStatus$(plugin: PluginName): Observable; + }; } diff --git a/src/legacy/utils/index.js b/src/legacy/utils/index.js index 529b1ddfd8a4d..e2e2331b3aea6 100644 --- a/src/legacy/utils/index.js +++ b/src/legacy/utils/index.js @@ -17,8 +17,6 @@ * under the License. */ -export { BinderBase } from './binder'; -export { BinderFor } from './binder_for'; export { deepCloneWithBuffers } from './deep_clone_with_buffers'; export { unset } from './unset'; export { IS_KIBANA_DISTRIBUTABLE } from './artifact_type'; diff --git a/src/plugins/dashboard/public/application/actions/open_replace_panel_flyout.tsx b/src/plugins/dashboard/public/application/actions/open_replace_panel_flyout.tsx index c676ca052d687..54a294fd2f4ac 100644 --- a/src/plugins/dashboard/public/application/actions/open_replace_panel_flyout.tsx +++ b/src/plugins/dashboard/public/application/actions/open_replace_panel_flyout.tsx @@ -60,7 +60,8 @@ export async function openReplacePanelFlyout(options: { /> ), { - 'data-test-subj': 'replacePanelFlyout', + 'data-test-subj': 'dashboardReplacePanel', + ownFocus: true, } ); } diff --git a/src/plugins/dashboard/public/application/actions/replace_panel_flyout.tsx b/src/plugins/dashboard/public/application/actions/replace_panel_flyout.tsx index 0000f63c48c2d..4e228bc1a7a06 100644 --- a/src/plugins/dashboard/public/application/actions/replace_panel_flyout.tsx +++ b/src/plugins/dashboard/public/application/actions/replace_panel_flyout.tsx @@ -19,16 +19,15 @@ import { i18n } from '@kbn/i18n'; import React from 'react'; -import _ from 'lodash'; -import { EuiFlyout, EuiFlyoutBody, EuiFlyoutHeader, EuiTitle } from '@elastic/eui'; +import { EuiFlyoutBody, EuiFlyoutHeader, EuiTitle } from '@elastic/eui'; import { NotificationsStart, Toast } from 'src/core/public'; import { DashboardPanelState } from '../embeddable'; import { - IContainer, - IEmbeddable, EmbeddableInput, EmbeddableOutput, EmbeddableStart, + IContainer, + IEmbeddable, SavedObjectEmbeddableInput, } from '../../embeddable_plugin'; @@ -122,7 +121,7 @@ export class ReplacePanelFlyout extends React.Component { const panelToReplace = 'Replace panel ' + this.props.panelToRemove.getTitle() + ' with:'; return ( - + <>

@@ -131,7 +130,7 @@ export class ReplacePanelFlyout extends React.Component { {savedObjectsFinder} - + ); } } diff --git a/src/plugins/kibana_usage_collection/server/collectors/application_usage/schema.ts b/src/plugins/kibana_usage_collection/server/collectors/application_usage/schema.ts index 6efe872553583..2e79cdaa7fc6b 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/application_usage/schema.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/application_usage/schema.ts @@ -66,6 +66,7 @@ export const applicationUsageSchema = { csm: commonSchema, canvas: commonSchema, dashboard_mode: commonSchema, // It's a forward app so we'll likely never report it + enterpriseSearch: commonSchema, appSearch: commonSchema, workplaceSearch: commonSchema, graph: commonSchema, diff --git a/src/plugins/kibana_usage_collection/server/collectors/ops_stats/index.test.ts b/src/plugins/kibana_usage_collection/server/collectors/ops_stats/index.test.ts index 359d3a396665d..a527d4d03c6fc 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/ops_stats/index.test.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/ops_stats/index.test.ts @@ -39,6 +39,7 @@ describe('telemetry_ops_stats', () => { const callCluster = jest.fn(); const metric: OpsMetrics = { + collected_at: new Date('2020-01-01 01:00:00'), process: { memory: { heap: { diff --git a/src/plugins/kibana_usage_collection/server/collectors/ops_stats/ops_stats_collector.ts b/src/plugins/kibana_usage_collection/server/collectors/ops_stats/ops_stats_collector.ts index 6e8b71d675f7b..d3be601540582 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/ops_stats/ops_stats_collector.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/ops_stats/ops_stats_collector.ts @@ -18,13 +18,13 @@ */ import { Observable } from 'rxjs'; -import { cloneDeep } from 'lodash'; +import { cloneDeep, omit } from 'lodash'; import moment from 'moment'; import { OpsMetrics } from 'kibana/server'; import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; import { KIBANA_STATS_TYPE } from '../../../common/constants'; -interface OpsStatsMetrics extends Omit { +interface OpsStatsMetrics extends Omit { timestamp: string; response_times: { average: number; @@ -52,9 +52,9 @@ export function getOpsStatsCollector( // @ts-expect-error delete metrics.requests.statusCodes; lastMetrics = { - ...metrics, + ...omit(metrics, ['collected_at']), response_times: responseTimes, - timestamp: moment.utc().toISOString(), + timestamp: moment.utc(metrics.collected_at).toISOString(), }; }); diff --git a/src/plugins/saved_objects/public/save_modal/saved_object_save_modal.tsx b/src/plugins/saved_objects/public/save_modal/saved_object_save_modal.tsx index 3b9efbee22ba6..9cdef8b9392bb 100644 --- a/src/plugins/saved_objects/public/save_modal/saved_object_save_modal.tsx +++ b/src/plugins/saved_objects/public/save_modal/saved_object_save_modal.tsx @@ -294,7 +294,7 @@ export class SavedObjectSaveModal extends React.Component id="savedObjects.saveModal.duplicateTitleDescription" defaultMessage="Saving '{title}' creates a duplicate title." values={{ - title: this.props.title, + title: this.state.title, }} />

diff --git a/src/plugins/telemetry/schema/oss_plugins.json b/src/plugins/telemetry/schema/oss_plugins.json index acd575badbe5b..5bce03a292760 100644 --- a/src/plugins/telemetry/schema/oss_plugins.json +++ b/src/plugins/telemetry/schema/oss_plugins.json @@ -414,6 +414,34 @@ } } }, + "enterpriseSearch": { + "properties": { + "clicks_total": { + "type": "long" + }, + "clicks_7_days": { + "type": "long" + }, + "clicks_30_days": { + "type": "long" + }, + "clicks_90_days": { + "type": "long" + }, + "minutes_on_screen_total": { + "type": "float" + }, + "minutes_on_screen_7_days": { + "type": "float" + }, + "minutes_on_screen_30_days": { + "type": "float" + }, + "minutes_on_screen_90_days": { + "type": "float" + } + } + }, "appSearch": { "properties": { "clicks_total": { diff --git a/src/plugins/ui_actions/public/tests/test_samples/hello_world_action.tsx b/src/plugins/ui_actions/public/tests/test_samples/hello_world_action.tsx index 8fff231a867bf..a4cfe172dd109 100644 --- a/src/plugins/ui_actions/public/tests/test_samples/hello_world_action.tsx +++ b/src/plugins/ui_actions/public/tests/test_samples/hello_world_action.tsx @@ -18,7 +18,7 @@ */ import React from 'react'; -import { EuiFlyout, EuiFlexGroup, EuiFlexItem, EuiBadge } from '@elastic/eui'; +import { EuiFlexGroup, EuiFlexItem, EuiBadge, EuiFlyoutBody } from '@elastic/eui'; import { CoreStart } from 'src/core/public'; import { createAction, ActionByType } from '../../actions'; import { toMountPoint, reactToUiComponent } from '../../../../kibana_react/public'; @@ -49,14 +49,11 @@ export function createHelloWorldAction( getIconType: () => 'lock', MenuItem: UiMenuItem, execute: async () => { - const flyoutSession = overlays.openFlyout( - toMountPoint( - flyoutSession && flyoutSession.close()}> - Hello World, I am a hello world action! - - ), + overlays.openFlyout( + toMountPoint(Hello World, I am a hello world action!), { 'data-test-subj': 'helloWorldAction', + ownFocus: true, } ); }, diff --git a/test/functional/apps/management/_create_index_pattern_wizard.js b/test/functional/apps/management/_create_index_pattern_wizard.js index 9760527371408..8b11a02099f61 100644 --- a/test/functional/apps/management/_create_index_pattern_wizard.js +++ b/test/functional/apps/management/_create_index_pattern_wizard.js @@ -66,6 +66,18 @@ export default function ({ getService, getPageObjects }) { await PageObjects.settings.createIndexPattern('alias1', false); }); + + after(async () => { + await es.transport.request({ + path: '/_aliases', + method: 'POST', + body: { actions: [{ remove: { index: 'blogs', alias: 'alias1' } }] }, + }); + await es.transport.request({ + path: '/blogs', + method: 'DELETE', + }); + }); }); }); } diff --git a/x-pack/package.json b/x-pack/package.json index 899eca1095923..3a074ba1f1d7d 100644 --- a/x-pack/package.json +++ b/x-pack/package.json @@ -195,7 +195,7 @@ "jsdom": "13.1.0", "jsondiffpatch": "0.4.1", "jsts": "^1.6.2", - "kea": "2.2.0-rc.4", + "kea": "^2.2.0", "loader-utils": "^1.2.3", "lz-string": "^1.4.4", "madge": "3.4.4", diff --git a/x-pack/plugins/enterprise_search/common/__mocks__/initial_app_data.ts b/x-pack/plugins/enterprise_search/common/__mocks__/initial_app_data.ts index 2d31be65dd30e..4533383ebd80e 100644 --- a/x-pack/plugins/enterprise_search/common/__mocks__/initial_app_data.ts +++ b/x-pack/plugins/enterprise_search/common/__mocks__/initial_app_data.ts @@ -7,9 +7,20 @@ export const DEFAULT_INITIAL_APP_DATA = { readOnlyMode: false, ilmEnabled: true, + isFederatedAuth: false, configuredLimits: { - maxDocumentByteSize: 102400, - maxEnginesPerMetaEngine: 15, + appSearch: { + engine: { + maxDocumentByteSize: 102400, + maxEnginesPerMetaEngine: 15, + }, + }, + workplaceSearch: { + customApiSource: { + maxDocumentByteSize: 102400, + totalFields: 64, + }, + }, }, appSearch: { accountId: 'some-id-string', @@ -29,17 +40,16 @@ export const DEFAULT_INITIAL_APP_DATA = { }, }, workplaceSearch: { - canCreateInvitations: true, - isFederatedAuth: false, organization: { name: 'ACME Donuts', defaultOrgName: 'My Organization', }, - fpAccount: { + account: { id: 'some-id-string', groups: ['Default', 'Cats'], isAdmin: true, canCreatePersonalSources: true, + canCreateInvitations: true, isCurated: false, viewedOnboardingPage: true, }, diff --git a/x-pack/plugins/enterprise_search/common/constants.ts b/x-pack/plugins/enterprise_search/common/constants.ts index 05d27d7337a6e..c6ca0d532ce07 100644 --- a/x-pack/plugins/enterprise_search/common/constants.ts +++ b/x-pack/plugins/enterprise_search/common/constants.ts @@ -11,7 +11,24 @@ export const ENTERPRISE_SEARCH_PLUGIN = { NAME: i18n.translate('xpack.enterpriseSearch.productName', { defaultMessage: 'Enterprise Search', }), - URL: '/app/enterprise_search', + NAV_TITLE: i18n.translate('xpack.enterpriseSearch.navTitle', { + defaultMessage: 'Overview', + }), + SUBTITLE: i18n.translate('xpack.enterpriseSearch.featureCatalogue.subtitle', { + defaultMessage: 'Search everything', + }), + DESCRIPTIONS: [ + i18n.translate('xpack.enterpriseSearch.featureCatalogueDescription1', { + defaultMessage: 'Build a powerful search experience.', + }), + i18n.translate('xpack.enterpriseSearch.featureCatalogueDescription2', { + defaultMessage: 'Connect your users to relevant data.', + }), + i18n.translate('xpack.enterpriseSearch.featureCatalogueDescription3', { + defaultMessage: 'Unify your team content.', + }), + ], + URL: '/app/enterprise_search/overview', }; export const APP_SEARCH_PLUGIN = { @@ -23,6 +40,10 @@ export const APP_SEARCH_PLUGIN = { defaultMessage: 'Leverage dashboards, analytics, and APIs for advanced application search made simple.', }), + CARD_DESCRIPTION: i18n.translate('xpack.enterpriseSearch.appSearch.productCardDescription', { + defaultMessage: + 'Elastic App Search provides user-friendly tools to design and deploy a powerful search to your websites or web/mobile applications.', + }), URL: '/app/enterprise_search/app_search', SUPPORT_URL: 'https://discuss.elastic.co/c/enterprise-search/app-search/', }; @@ -36,12 +57,22 @@ export const WORKPLACE_SEARCH_PLUGIN = { defaultMessage: 'Search all documents, files, and sources available across your virtual workplace.', }), + CARD_DESCRIPTION: i18n.translate( + 'xpack.enterpriseSearch.workplaceSearch.productCardDescription', + { + defaultMessage: + "Unify all your team's content in one place, with instant connectivity to popular productivity and collaboration tools.", + } + ), URL: '/app/enterprise_search/workplace_search', SUPPORT_URL: 'https://discuss.elastic.co/c/enterprise-search/workplace-search/', }; export const LICENSED_SUPPORT_URL = 'https://support.elastic.co'; -export const JSON_HEADER = { 'Content-Type': 'application/json' }; // This needs specific casing or Chrome throws a 415 error +export const JSON_HEADER = { + 'Content-Type': 'application/json', // This needs specific casing or Chrome throws a 415 error + Accept: 'application/json', // Required for Enterprise Search APIs +}; export const ENGINES_PAGE_SIZE = 10; diff --git a/x-pack/plugins/enterprise_search/common/types/app_search.ts b/x-pack/plugins/enterprise_search/common/types/app_search.ts index 5d6ec079e66e0..72259ecd2343d 100644 --- a/x-pack/plugins/enterprise_search/common/types/app_search.ts +++ b/x-pack/plugins/enterprise_search/common/types/app_search.ts @@ -23,3 +23,10 @@ export interface IRole { availableRoleTypes: string[]; }; } + +export interface IConfiguredLimits { + engine: { + maxDocumentByteSize: number; + maxEnginesPerMetaEngine: number; + }; +} diff --git a/x-pack/plugins/enterprise_search/common/types/index.ts b/x-pack/plugins/enterprise_search/common/types/index.ts index 008afb234a376..d5774adc0d516 100644 --- a/x-pack/plugins/enterprise_search/common/types/index.ts +++ b/x-pack/plugins/enterprise_search/common/types/index.ts @@ -4,18 +4,29 @@ * you may not use this file except in compliance with the Elastic License. */ -import { IAccount as IAppSearchAccount } from './app_search'; -import { IWorkplaceSearchInitialData } from './workplace_search'; +import { + IAccount as IAppSearchAccount, + IConfiguredLimits as IAppSearchConfiguredLimits, +} from './app_search'; +import { + IWorkplaceSearchInitialData, + IConfiguredLimits as IWorkplaceSearchConfiguredLimits, +} from './workplace_search'; export interface IInitialAppData { readOnlyMode?: boolean; ilmEnabled?: boolean; + isFederatedAuth?: boolean; configuredLimits?: IConfiguredLimits; + access?: { + hasAppSearchAccess: boolean; + hasWorkplaceSearchAccess: boolean; + }; appSearch?: IAppSearchAccount; workplaceSearch?: IWorkplaceSearchInitialData; } export interface IConfiguredLimits { - maxDocumentByteSize: number; - maxEnginesPerMetaEngine: number; + appSearch: IAppSearchConfiguredLimits; + workplaceSearch: IWorkplaceSearchConfiguredLimits; } diff --git a/x-pack/plugins/enterprise_search/common/types/workplace_search.ts b/x-pack/plugins/enterprise_search/common/types/workplace_search.ts index bc4e39b0788d9..6c82206706b32 100644 --- a/x-pack/plugins/enterprise_search/common/types/workplace_search.ts +++ b/x-pack/plugins/enterprise_search/common/types/workplace_search.ts @@ -10,6 +10,7 @@ export interface IAccount { isAdmin: boolean; isCurated: boolean; canCreatePersonalSources: boolean; + canCreateInvitations?: boolean; viewedOnboardingPage: boolean; } @@ -19,8 +20,13 @@ export interface IOrganization { } export interface IWorkplaceSearchInitialData { - canCreateInvitations: boolean; - isFederatedAuth: boolean; organization: IOrganization; - fpAccount: IAccount; + account: IAccount; +} + +export interface IConfiguredLimits { + customApiSource: { + maxDocumentByteSize: number; + totalFields: number; + }; } diff --git a/x-pack/plugins/enterprise_search/public/applications/__mocks__/react_router_history.mock.ts b/x-pack/plugins/enterprise_search/public/applications/__mocks__/react_router_history.mock.ts index 779eb1a043e8c..842dcefd3aef8 100644 --- a/x-pack/plugins/enterprise_search/public/applications/__mocks__/react_router_history.mock.ts +++ b/x-pack/plugins/enterprise_search/public/applications/__mocks__/react_router_history.mock.ts @@ -9,7 +9,7 @@ * Jest to accept its use within a jest.mock() */ export const mockHistory = { - createHref: jest.fn(({ pathname }) => `/enterprise_search${pathname}`), + createHref: jest.fn(({ pathname }) => `/app/enterprise_search${pathname}`), push: jest.fn(), location: { pathname: '/current-path', diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search/assets/app_search.png b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/assets/app_search.png new file mode 100644 index 0000000000000..6cf0639167e2f Binary files /dev/null and b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/assets/app_search.png differ diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search/assets/bg_enterprise_search.png b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/assets/bg_enterprise_search.png new file mode 100644 index 0000000000000..1b5e1e489fd96 Binary files /dev/null and b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/assets/bg_enterprise_search.png differ diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search/assets/workplace_search.png b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/assets/workplace_search.png new file mode 100644 index 0000000000000..984662b65cb5d Binary files /dev/null and b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/assets/workplace_search.png differ diff --git a/x-pack/plugins/infra/server/lib/snapshot/constants.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/components/product_card/index.ts similarity index 65% rename from x-pack/plugins/infra/server/lib/snapshot/constants.ts rename to x-pack/plugins/enterprise_search/public/applications/enterprise_search/components/product_card/index.ts index 0420878dbcf50..df85a10f7e9de 100644 --- a/x-pack/plugins/infra/server/lib/snapshot/constants.ts +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/components/product_card/index.ts @@ -4,6 +4,4 @@ * you may not use this file except in compliance with the Elastic License. */ -// TODO: Make SNAPSHOT_COMPOSITE_REQUEST_SIZE configurable from kibana.yml - -export const SNAPSHOT_COMPOSITE_REQUEST_SIZE = 75; +export { ProductCard } from './product_card'; diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search/components/product_card/product_card.scss b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/components/product_card/product_card.scss new file mode 100644 index 0000000000000..d6b6bd3442590 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/components/product_card/product_card.scss @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +.productCard { + margin: $euiSizeS; + + &__imageContainer { + max-height: 115px; + overflow: hidden; + background-color: #0076cc; + + @include euiBreakpoint('s', 'm', 'l', 'xl') { + max-height: none; + } + } + + &__image { + width: 100%; + height: auto; + } + + .euiCard__content { + max-width: 350px; + margin-top: $euiSizeL; + + @include euiBreakpoint('s', 'm', 'l', 'xl') { + margin-top: $euiSizeXL; + } + } + + .euiCard__title { + margin-bottom: $euiSizeM; + font-weight: $euiFontWeightBold; + + @include euiBreakpoint('s', 'm', 'l', 'xl') { + margin-bottom: $euiSizeL; + font-size: $euiSizeL; + } + } + + .euiCard__description { + font-weight: $euiFontWeightMedium; + color: $euiColorMediumShade; + margin-bottom: $euiSize; + } + + .euiCard__footer { + margin-bottom: $euiSizeS; + + @include euiBreakpoint('s', 'm', 'l', 'xl') { + margin-bottom: $euiSizeM; + font-size: $euiSizeL; + } + } +} diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search/components/product_card/product_card.test.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/components/product_card/product_card.test.tsx new file mode 100644 index 0000000000000..a76b654ccddd0 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/components/product_card/product_card.test.tsx @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { shallow } from 'enzyme'; + +import { EuiCard } from '@elastic/eui'; +import { EuiButton } from '../../../shared/react_router_helpers'; +import { APP_SEARCH_PLUGIN, WORKPLACE_SEARCH_PLUGIN } from '../../../../../common/constants'; + +jest.mock('../../../shared/telemetry', () => ({ + sendTelemetry: jest.fn(), +})); +import { sendTelemetry } from '../../../shared/telemetry'; + +import { ProductCard } from './'; + +describe('ProductCard', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('renders an App Search card', () => { + const wrapper = shallow(); + const card = wrapper.find(EuiCard).dive().shallow(); + + expect(card.find('h2').text()).toEqual('Elastic App Search'); + expect(card.find('.productCard__image').prop('src')).toEqual('as.jpg'); + + const button = card.find(EuiButton); + expect(button.prop('to')).toEqual('/app/enterprise_search/app_search'); + expect(button.prop('data-test-subj')).toEqual('LaunchAppSearchButton'); + + button.simulate('click'); + expect(sendTelemetry).toHaveBeenCalledWith(expect.objectContaining({ metric: 'app_search' })); + }); + + it('renders a Workplace Search card', () => { + const wrapper = shallow(); + const card = wrapper.find(EuiCard).dive().shallow(); + + expect(card.find('h2').text()).toEqual('Elastic Workplace Search'); + expect(card.find('.productCard__image').prop('src')).toEqual('ws.jpg'); + + const button = card.find(EuiButton); + expect(button.prop('to')).toEqual('/app/enterprise_search/workplace_search'); + expect(button.prop('data-test-subj')).toEqual('LaunchWorkplaceSearchButton'); + + button.simulate('click'); + expect(sendTelemetry).toHaveBeenCalledWith( + expect.objectContaining({ metric: 'workplace_search' }) + ); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search/components/product_card/product_card.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/components/product_card/product_card.tsx new file mode 100644 index 0000000000000..334ca126cabb9 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/components/product_card/product_card.tsx @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { useContext } from 'react'; +import upperFirst from 'lodash/upperFirst'; +import snakeCase from 'lodash/snakeCase'; +import { i18n } from '@kbn/i18n'; +import { EuiCard, EuiTextColor } from '@elastic/eui'; + +import { EuiButton } from '../../../shared/react_router_helpers'; +import { sendTelemetry } from '../../../shared/telemetry'; +import { KibanaContext, IKibanaContext } from '../../../index'; + +import './product_card.scss'; + +interface IProductCard { + // Expects product plugin constants (@see common/constants.ts) + product: { + ID: string; + NAME: string; + CARD_DESCRIPTION: string; + URL: string; + }; + image: string; +} + +export const ProductCard: React.FC = ({ product, image }) => { + const { http } = useContext(KibanaContext) as IKibanaContext; + + return ( + + + + } + paddingSize="l" + description={{product.CARD_DESCRIPTION}} + footer={ + + sendTelemetry({ + http, + product: 'enterprise_search', + action: 'clicked', + metric: snakeCase(product.ID), + }) + } + data-test-subj={`Launch${upperFirst(product.ID)}Button`} + > + {i18n.translate('xpack.enterpriseSearch.overview.productCard.button', { + defaultMessage: `Launch {productName}`, + values: { productName: product.NAME }, + })} + + } + /> + ); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search/index.scss b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/index.scss new file mode 100644 index 0000000000000..d937943352317 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/index.scss @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +.enterpriseSearchOverview { + padding-top: 78px; + background-image: url('./assets/bg_enterprise_search.png'); + background-repeat: no-repeat; + background-size: 670px; + background-position: center -27px; + + @include euiBreakpoint('m', 'l', 'xl') { + padding-top: 158px; + background-size: 1160px; + background-position: center -48px; + } + + &__header { + text-align: center; + margin: auto; + } + + &__heading { + @include euiBreakpoint('xs', 's') { + font-size: $euiFontSizeXL; + line-height: map-get(map-get($euiTitles, 'm'), 'line-height'); + } + } + + &__subheading { + color: $euiColorMediumShade; + font-size: $euiFontSize; + + @include euiBreakpoint('m', 'l', 'xl') { + font-size: $euiFontSizeL; + margin-bottom: $euiSizeL; + } + } + + // EUI override + .euiTitle + .euiTitle { + margin-top: 0; + + @include euiBreakpoint('m', 'l', 'xl') { + margin-top: $euiSizeS; + } + } + + .enterpriseSearchOverview__card { + flex-basis: 50%; + } +} diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search/index.test.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/index.test.tsx new file mode 100644 index 0000000000000..cd2a22a45bbb4 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/index.test.tsx @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { shallow } from 'enzyme'; + +import { EuiPage } from '@elastic/eui'; + +import { EnterpriseSearch } from './'; +import { ProductCard } from './components/product_card'; + +describe('EnterpriseSearch', () => { + it('renders the overview page and product cards', () => { + const wrapper = shallow( + + ); + + expect(wrapper.find(EuiPage).hasClass('enterpriseSearchOverview')).toBe(true); + expect(wrapper.find(ProductCard)).toHaveLength(2); + }); + + describe('access checks', () => { + it('does not render the App Search card if the user does not have access to AS', () => { + const wrapper = shallow( + + ); + + expect(wrapper.find(ProductCard)).toHaveLength(1); + expect(wrapper.find(ProductCard).prop('product').ID).toEqual('workplaceSearch'); + }); + + it('does not render the Workplace Search card if the user does not have access to WS', () => { + const wrapper = shallow( + + ); + + expect(wrapper.find(ProductCard)).toHaveLength(1); + expect(wrapper.find(ProductCard).prop('product').ID).toEqual('appSearch'); + }); + + it('does not render any cards if the user does not have access', () => { + const wrapper = shallow(); + + expect(wrapper.find(ProductCard)).toHaveLength(0); + }); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search/index.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/index.tsx new file mode 100644 index 0000000000000..373f595a6a9ea --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/index.tsx @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { + EuiPage, + EuiPageBody, + EuiPageHeader, + EuiPageHeaderSection, + EuiPageContentBody, + EuiFlexGroup, + EuiFlexItem, + EuiSpacer, + EuiTitle, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; + +import { IInitialAppData } from '../../../common/types'; +import { APP_SEARCH_PLUGIN, WORKPLACE_SEARCH_PLUGIN } from '../../../common/constants'; + +import { SetEnterpriseSearchChrome as SetPageChrome } from '../shared/kibana_chrome'; +import { SendEnterpriseSearchTelemetry as SendTelemetry } from '../shared/telemetry'; + +import { ProductCard } from './components/product_card'; + +import AppSearchImage from './assets/app_search.png'; +import WorkplaceSearchImage from './assets/workplace_search.png'; +import './index.scss'; + +export const EnterpriseSearch: React.FC = ({ access = {} }) => { + const { hasAppSearchAccess, hasWorkplaceSearchAccess } = access; + + return ( + + + + + + + + +

+ {i18n.translate('xpack.enterpriseSearch.overview.heading', { + defaultMessage: 'Welcome to Elastic Enterprise Search', + })} +

+ + +

+ {i18n.translate('xpack.enterpriseSearch.overview.subheading', { + defaultMessage: 'Select a product to get started', + })} +

+ + + + + + {hasAppSearchAccess && ( + + + + )} + {hasWorkplaceSearchAccess && ( + + + + )} + + + + + + ); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/kibana_chrome/generate_breadcrumbs.test.ts b/x-pack/plugins/enterprise_search/public/applications/shared/kibana_chrome/generate_breadcrumbs.test.ts index 9e86b239432a7..3c8b3a7218862 100644 --- a/x-pack/plugins/enterprise_search/public/applications/shared/kibana_chrome/generate_breadcrumbs.test.ts +++ b/x-pack/plugins/enterprise_search/public/applications/shared/kibana_chrome/generate_breadcrumbs.test.ts @@ -37,27 +37,37 @@ describe('useBreadcrumbs', () => { expect(breadcrumb).toEqual([ { text: 'Hello', - href: '/enterprise_search/hello', + href: '/app/enterprise_search/hello', onClick: expect.any(Function), }, { text: 'World', - href: '/enterprise_search/world', + href: '/app/enterprise_search/world', onClick: expect.any(Function), }, ]); }); it('prevents default navigation and uses React Router history on click', () => { - const breadcrumb = useBreadcrumbs([{ text: '', path: '/' }])[0] as any; + const breadcrumb = useBreadcrumbs([{ text: '', path: '/test' }])[0] as any; const event = { preventDefault: jest.fn() }; breadcrumb.onClick(event); - expect(mockKibanaContext.navigateToUrl).toHaveBeenCalled(); + expect(mockKibanaContext.navigateToUrl).toHaveBeenCalledWith('/app/enterprise_search/test'); expect(mockHistory.createHref).toHaveBeenCalled(); expect(event.preventDefault).toHaveBeenCalled(); }); + it('does not call createHref if shouldNotCreateHref is passed', () => { + const breadcrumb = useBreadcrumbs([ + { text: '', path: '/test', shouldNotCreateHref: true }, + ])[0] as any; + breadcrumb.onClick({ preventDefault: () => null }); + + expect(mockKibanaContext.navigateToUrl).toHaveBeenCalledWith('/test'); + expect(mockHistory.createHref).not.toHaveBeenCalled(); + }); + it('does not prevent default browser behavior on new tab/window clicks', () => { const breadcrumb = useBreadcrumbs([{ text: '', path: '/' }])[0] as any; @@ -95,15 +105,17 @@ describe('useEnterpriseSearchBreadcrumbs', () => { expect(useEnterpriseSearchBreadcrumbs(breadcrumbs)).toEqual([ { text: 'Enterprise Search', + href: '/app/enterprise_search/overview', + onClick: expect.any(Function), }, { text: 'Page 1', - href: '/enterprise_search/page1', + href: '/app/enterprise_search/page1', onClick: expect.any(Function), }, { text: 'Page 2', - href: '/enterprise_search/page2', + href: '/app/enterprise_search/page2', onClick: expect.any(Function), }, ]); @@ -113,6 +125,8 @@ describe('useEnterpriseSearchBreadcrumbs', () => { expect(useEnterpriseSearchBreadcrumbs()).toEqual([ { text: 'Enterprise Search', + href: '/app/enterprise_search/overview', + onClick: expect.any(Function), }, ]); }); @@ -122,7 +136,7 @@ describe('useAppSearchBreadcrumbs', () => { beforeEach(() => { jest.clearAllMocks(); mockHistory.createHref.mockImplementation( - ({ pathname }: any) => `/enterprise_search/app_search${pathname}` + ({ pathname }: any) => `/app/enterprise_search/app_search${pathname}` ); }); @@ -141,20 +155,22 @@ describe('useAppSearchBreadcrumbs', () => { expect(useAppSearchBreadcrumbs(breadcrumbs)).toEqual([ { text: 'Enterprise Search', + href: '/app/enterprise_search/overview', + onClick: expect.any(Function), }, { text: 'App Search', - href: '/enterprise_search/app_search/', + href: '/app/enterprise_search/app_search/', onClick: expect.any(Function), }, { text: 'Page 1', - href: '/enterprise_search/app_search/page1', + href: '/app/enterprise_search/app_search/page1', onClick: expect.any(Function), }, { text: 'Page 2', - href: '/enterprise_search/app_search/page2', + href: '/app/enterprise_search/app_search/page2', onClick: expect.any(Function), }, ]); @@ -164,10 +180,12 @@ describe('useAppSearchBreadcrumbs', () => { expect(useAppSearchBreadcrumbs()).toEqual([ { text: 'Enterprise Search', + href: '/app/enterprise_search/overview', + onClick: expect.any(Function), }, { text: 'App Search', - href: '/enterprise_search/app_search/', + href: '/app/enterprise_search/app_search/', onClick: expect.any(Function), }, ]); @@ -178,7 +196,7 @@ describe('useWorkplaceSearchBreadcrumbs', () => { beforeEach(() => { jest.clearAllMocks(); mockHistory.createHref.mockImplementation( - ({ pathname }: any) => `/enterprise_search/workplace_search${pathname}` + ({ pathname }: any) => `/app/enterprise_search/workplace_search${pathname}` ); }); @@ -197,20 +215,22 @@ describe('useWorkplaceSearchBreadcrumbs', () => { expect(useWorkplaceSearchBreadcrumbs(breadcrumbs)).toEqual([ { text: 'Enterprise Search', + href: '/app/enterprise_search/overview', + onClick: expect.any(Function), }, { text: 'Workplace Search', - href: '/enterprise_search/workplace_search/', + href: '/app/enterprise_search/workplace_search/', onClick: expect.any(Function), }, { text: 'Page 1', - href: '/enterprise_search/workplace_search/page1', + href: '/app/enterprise_search/workplace_search/page1', onClick: expect.any(Function), }, { text: 'Page 2', - href: '/enterprise_search/workplace_search/page2', + href: '/app/enterprise_search/workplace_search/page2', onClick: expect.any(Function), }, ]); @@ -220,10 +240,12 @@ describe('useWorkplaceSearchBreadcrumbs', () => { expect(useWorkplaceSearchBreadcrumbs()).toEqual([ { text: 'Enterprise Search', + href: '/app/enterprise_search/overview', + onClick: expect.any(Function), }, { text: 'Workplace Search', - href: '/enterprise_search/workplace_search/', + href: '/app/enterprise_search/workplace_search/', onClick: expect.any(Function), }, ]); diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/kibana_chrome/generate_breadcrumbs.ts b/x-pack/plugins/enterprise_search/public/applications/shared/kibana_chrome/generate_breadcrumbs.ts index 6eab936719d01..19714608e73e9 100644 --- a/x-pack/plugins/enterprise_search/public/applications/shared/kibana_chrome/generate_breadcrumbs.ts +++ b/x-pack/plugins/enterprise_search/public/applications/shared/kibana_chrome/generate_breadcrumbs.ts @@ -26,6 +26,9 @@ import { letBrowserHandleEvent } from '../react_router_helpers'; interface IBreadcrumb { text: string; path?: string; + // Used to navigate outside of the React Router basename, + // i.e. if we need to go from App Search to Enterprise Search + shouldNotCreateHref?: boolean; } export type TBreadcrumbs = IBreadcrumb[]; @@ -33,11 +36,11 @@ export const useBreadcrumbs = (breadcrumbs: TBreadcrumbs) => { const history = useHistory(); const { navigateToUrl } = useContext(KibanaContext) as IKibanaContext; - return breadcrumbs.map(({ text, path }) => { + return breadcrumbs.map(({ text, path, shouldNotCreateHref }) => { const breadcrumb = { text } as EuiBreadcrumb; if (path) { - const href = history.createHref({ pathname: path }) as string; + const href = shouldNotCreateHref ? path : (history.createHref({ pathname: path }) as string); breadcrumb.href = href; breadcrumb.onClick = (event) => { @@ -56,7 +59,14 @@ export const useBreadcrumbs = (breadcrumbs: TBreadcrumbs) => { */ export const useEnterpriseSearchBreadcrumbs = (breadcrumbs: TBreadcrumbs = []) => - useBreadcrumbs([{ text: ENTERPRISE_SEARCH_PLUGIN.NAME }, ...breadcrumbs]); + useBreadcrumbs([ + { + text: ENTERPRISE_SEARCH_PLUGIN.NAME, + path: ENTERPRISE_SEARCH_PLUGIN.URL, + shouldNotCreateHref: true, + }, + ...breadcrumbs, + ]); export const useAppSearchBreadcrumbs = (breadcrumbs: TBreadcrumbs = []) => useEnterpriseSearchBreadcrumbs([{ text: APP_SEARCH_PLUGIN.NAME, path: '/' }, ...breadcrumbs]); diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/kibana_chrome/generate_title.ts b/x-pack/plugins/enterprise_search/public/applications/shared/kibana_chrome/generate_title.ts index 706baefc00cc2..de5f72de79192 100644 --- a/x-pack/plugins/enterprise_search/public/applications/shared/kibana_chrome/generate_title.ts +++ b/x-pack/plugins/enterprise_search/public/applications/shared/kibana_chrome/generate_title.ts @@ -20,7 +20,7 @@ export type TTitle = string[]; /** * Given an array of page titles, return a final formatted document title * @param pages - e.g., ['Curations', 'some Engine', 'App Search'] - * @returns - e.g., 'Curations | some Engine | App Search' + * @returns - e.g., 'Curations - some Engine - App Search' */ export const generateTitle = (pages: TTitle) => pages.join(' - '); diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/kibana_chrome/index.ts b/x-pack/plugins/enterprise_search/public/applications/shared/kibana_chrome/index.ts index 4468d11ba94c9..02013a03c3395 100644 --- a/x-pack/plugins/enterprise_search/public/applications/shared/kibana_chrome/index.ts +++ b/x-pack/plugins/enterprise_search/public/applications/shared/kibana_chrome/index.ts @@ -4,4 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ -export { SetAppSearchChrome, SetWorkplaceSearchChrome } from './set_chrome'; +export { + SetEnterpriseSearchChrome, + SetAppSearchChrome, + SetWorkplaceSearchChrome, +} from './set_chrome'; diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/kibana_chrome/set_chrome.test.tsx b/x-pack/plugins/enterprise_search/public/applications/shared/kibana_chrome/set_chrome.test.tsx index bda816c9a5554..61a066bb92216 100644 --- a/x-pack/plugins/enterprise_search/public/applications/shared/kibana_chrome/set_chrome.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/shared/kibana_chrome/set_chrome.test.tsx @@ -12,18 +12,24 @@ import React from 'react'; import { mockKibanaContext, mountWithKibanaContext } from '../../__mocks__'; jest.mock('./generate_breadcrumbs', () => ({ + useEnterpriseSearchBreadcrumbs: jest.fn(() => (crumbs: any) => crumbs), useAppSearchBreadcrumbs: jest.fn(() => (crumbs: any) => crumbs), useWorkplaceSearchBreadcrumbs: jest.fn(() => (crumbs: any) => crumbs), })); -import { useAppSearchBreadcrumbs, useWorkplaceSearchBreadcrumbs } from './generate_breadcrumbs'; +import { + useEnterpriseSearchBreadcrumbs, + useAppSearchBreadcrumbs, + useWorkplaceSearchBreadcrumbs, +} from './generate_breadcrumbs'; jest.mock('./generate_title', () => ({ + enterpriseSearchTitle: jest.fn((title: any) => title), appSearchTitle: jest.fn((title: any) => title), workplaceSearchTitle: jest.fn((title: any) => title), })); -import { appSearchTitle, workplaceSearchTitle } from './generate_title'; +import { enterpriseSearchTitle, appSearchTitle, workplaceSearchTitle } from './generate_title'; -import { SetAppSearchChrome, SetWorkplaceSearchChrome } from './'; +import { SetEnterpriseSearchChrome, SetAppSearchChrome, SetWorkplaceSearchChrome } from './'; describe('Set Kibana Chrome helpers', () => { beforeEach(() => { @@ -35,6 +41,27 @@ describe('Set Kibana Chrome helpers', () => { expect(mockKibanaContext.setDocTitle).toHaveBeenCalled(); }); + describe('SetEnterpriseSearchChrome', () => { + it('sets breadcrumbs and document title', () => { + mountWithKibanaContext(); + + expect(enterpriseSearchTitle).toHaveBeenCalledWith(['Hello World']); + expect(useEnterpriseSearchBreadcrumbs).toHaveBeenCalledWith([ + { + text: 'Hello World', + path: '/current-path', + }, + ]); + }); + + it('sets empty breadcrumbs and document title when isRoot is true', () => { + mountWithKibanaContext(); + + expect(enterpriseSearchTitle).toHaveBeenCalledWith([]); + expect(useEnterpriseSearchBreadcrumbs).toHaveBeenCalledWith([]); + }); + }); + describe('SetAppSearchChrome', () => { it('sets breadcrumbs and document title', () => { mountWithKibanaContext(); diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/kibana_chrome/set_chrome.tsx b/x-pack/plugins/enterprise_search/public/applications/shared/kibana_chrome/set_chrome.tsx index 43db93c1583d1..5e8d972e1a135 100644 --- a/x-pack/plugins/enterprise_search/public/applications/shared/kibana_chrome/set_chrome.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/shared/kibana_chrome/set_chrome.tsx @@ -10,11 +10,17 @@ import { EuiBreadcrumb } from '@elastic/eui'; import { KibanaContext, IKibanaContext } from '../../index'; import { + useEnterpriseSearchBreadcrumbs, useAppSearchBreadcrumbs, useWorkplaceSearchBreadcrumbs, TBreadcrumbs, } from './generate_breadcrumbs'; -import { appSearchTitle, workplaceSearchTitle, TTitle } from './generate_title'; +import { + enterpriseSearchTitle, + appSearchTitle, + workplaceSearchTitle, + TTitle, +} from './generate_title'; /** * Helpers for setting Kibana chrome (breadcrumbs, doc titles) on React view mount @@ -33,6 +39,24 @@ interface IRootBreadcrumbsProps { } type TBreadcrumbsProps = IBreadcrumbsProps | IRootBreadcrumbsProps; +export const SetEnterpriseSearchChrome: React.FC = ({ text, isRoot }) => { + const history = useHistory(); + const { setBreadcrumbs, setDocTitle } = useContext(KibanaContext) as IKibanaContext; + + const title = isRoot ? [] : [text]; + const docTitle = enterpriseSearchTitle(title as TTitle | []); + + const crumb = isRoot ? [] : [{ text, path: history.location.pathname }]; + const breadcrumbs = useEnterpriseSearchBreadcrumbs(crumb as TBreadcrumbs | []); + + useEffect(() => { + setBreadcrumbs(breadcrumbs); + setDocTitle(docTitle); + }, []); + + return null; +}; + export const SetAppSearchChrome: React.FC = ({ text, isRoot }) => { const history = useHistory(); const { setBreadcrumbs, setDocTitle } = useContext(KibanaContext) as IKibanaContext; diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/react_router_helpers/eui_link.test.tsx b/x-pack/plugins/enterprise_search/public/applications/shared/react_router_helpers/eui_link.test.tsx index 063118f94cd19..0c7bac99085dd 100644 --- a/x-pack/plugins/enterprise_search/public/applications/shared/react_router_helpers/eui_link.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/shared/react_router_helpers/eui_link.test.tsx @@ -45,10 +45,18 @@ describe('EUI & React Router Component Helpers', () => { const link = wrapper.find(EuiLink); expect(link.prop('onClick')).toBeInstanceOf(Function); - expect(link.prop('href')).toEqual('/enterprise_search/foo/bar'); + expect(link.prop('href')).toEqual('/app/enterprise_search/foo/bar'); expect(mockHistory.createHref).toHaveBeenCalled(); }); + it('renders with the correct non-basenamed href when shouldNotCreateHref is passed', () => { + const wrapper = mount(); + const link = wrapper.find(EuiLink); + + expect(link.prop('href')).toEqual('/foo/bar'); + expect(mockHistory.createHref).not.toHaveBeenCalled(); + }); + describe('onClick', () => { it('prevents default navigation and uses React Router history', () => { const wrapper = mount(); diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/react_router_helpers/eui_link.tsx b/x-pack/plugins/enterprise_search/public/applications/shared/react_router_helpers/eui_link.tsx index 7221a61d0997b..e3b46632ddf9e 100644 --- a/x-pack/plugins/enterprise_search/public/applications/shared/react_router_helpers/eui_link.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/shared/react_router_helpers/eui_link.tsx @@ -21,14 +21,22 @@ import { letBrowserHandleEvent } from './link_events'; interface IEuiReactRouterProps { to: string; onClick?(): void; + // Used to navigate outside of the React Router plugin basename but still within Kibana, + // e.g. if we need to go from Enterprise Search to App Search + shouldNotCreateHref?: boolean; } -export const EuiReactRouterHelper: React.FC = ({ to, onClick, children }) => { +export const EuiReactRouterHelper: React.FC = ({ + to, + onClick, + shouldNotCreateHref, + children, +}) => { const history = useHistory(); const { navigateToUrl } = useContext(KibanaContext) as IKibanaContext; // Generate the correct link href (with basename etc. accounted for) - const href = history.createHref({ pathname: to }); + const href = shouldNotCreateHref ? to : history.createHref({ pathname: to }); const reactRouterLinkClick = (event: React.MouseEvent) => { if (onClick) onClick(); // Run any passed click events (e.g. telemetry) @@ -51,9 +59,10 @@ type TEuiReactRouterButtonProps = EuiButtonProps & IEuiReactRouterProps; export const EuiReactRouterLink: React.FC = ({ to, onClick, + shouldNotCreateHref, ...rest }) => ( - + ); @@ -61,9 +70,10 @@ export const EuiReactRouterLink: React.FC = ({ export const EuiReactRouterButton: React.FC = ({ to, onClick, + shouldNotCreateHref, ...rest }) => ( - + ); diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/telemetry/index.ts b/x-pack/plugins/enterprise_search/public/applications/shared/telemetry/index.ts index eadf7fa805590..a8b9636c3ff3e 100644 --- a/x-pack/plugins/enterprise_search/public/applications/shared/telemetry/index.ts +++ b/x-pack/plugins/enterprise_search/public/applications/shared/telemetry/index.ts @@ -5,5 +5,8 @@ */ export { sendTelemetry } from './send_telemetry'; -export { SendAppSearchTelemetry } from './send_telemetry'; -export { SendWorkplaceSearchTelemetry } from './send_telemetry'; +export { + SendEnterpriseSearchTelemetry, + SendAppSearchTelemetry, + SendWorkplaceSearchTelemetry, +} from './send_telemetry'; diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/telemetry/send_telemetry.test.tsx b/x-pack/plugins/enterprise_search/public/applications/shared/telemetry/send_telemetry.test.tsx index 3c873dbc25e37..8f7cf090e2d57 100644 --- a/x-pack/plugins/enterprise_search/public/applications/shared/telemetry/send_telemetry.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/shared/telemetry/send_telemetry.test.tsx @@ -10,7 +10,12 @@ import { httpServiceMock } from 'src/core/public/mocks'; import { JSON_HEADER as headers } from '../../../../common/constants'; import { mountWithKibanaContext } from '../../__mocks__'; -import { sendTelemetry, SendAppSearchTelemetry, SendWorkplaceSearchTelemetry } from './'; +import { + sendTelemetry, + SendEnterpriseSearchTelemetry, + SendAppSearchTelemetry, + SendWorkplaceSearchTelemetry, +} from './'; describe('Shared Telemetry Helpers', () => { const httpMock = httpServiceMock.createSetupContract(); @@ -44,6 +49,17 @@ describe('Shared Telemetry Helpers', () => { }); describe('React component helpers', () => { + it('SendEnterpriseSearchTelemetry component', () => { + mountWithKibanaContext(, { + http: httpMock, + }); + + expect(httpMock.put).toHaveBeenCalledWith('/api/enterprise_search/telemetry', { + headers, + body: '{"product":"enterprise_search","action":"viewed","metric":"page"}', + }); + }); + it('SendAppSearchTelemetry component', () => { mountWithKibanaContext(, { http: httpMock, @@ -56,13 +72,13 @@ describe('Shared Telemetry Helpers', () => { }); it('SendWorkplaceSearchTelemetry component', () => { - mountWithKibanaContext(, { + mountWithKibanaContext(, { http: httpMock, }); expect(httpMock.put).toHaveBeenCalledWith('/api/enterprise_search/telemetry', { headers, - body: '{"product":"workplace_search","action":"viewed","metric":"page"}', + body: '{"product":"workplace_search","action":"error","metric":"not_found"}', }); }); }); diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/telemetry/send_telemetry.tsx b/x-pack/plugins/enterprise_search/public/applications/shared/telemetry/send_telemetry.tsx index 715d61b31512c..4df1428221de6 100644 --- a/x-pack/plugins/enterprise_search/public/applications/shared/telemetry/send_telemetry.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/shared/telemetry/send_telemetry.tsx @@ -35,9 +35,21 @@ export const sendTelemetry = async ({ http, product, action, metric }: ISendTele /** * React component helpers - useful for on-page-load/views - * TODO: SendEnterpriseSearchTelemetry */ +export const SendEnterpriseSearchTelemetry: React.FC = ({ + action, + metric, +}) => { + const { http } = useContext(KibanaContext) as IKibanaContext; + + useEffect(() => { + sendTelemetry({ http, action, metric, product: 'enterprise_search' }); + }, [action, metric, http]); + + return null; +}; + export const SendAppSearchTelemetry: React.FC = ({ action, metric }) => { const { http } = useContext(KibanaContext) as IKibanaContext; diff --git a/x-pack/plugins/enterprise_search/public/plugin.ts b/x-pack/plugins/enterprise_search/public/plugin.ts index 83598a0dc971d..b735db7c49520 100644 --- a/x-pack/plugins/enterprise_search/public/plugin.ts +++ b/x-pack/plugins/enterprise_search/public/plugin.ts @@ -12,7 +12,6 @@ import { AppMountParameters, HttpSetup, } from 'src/core/public'; -import { i18n } from '@kbn/i18n'; import { FeatureCatalogueCategory, HomePublicPluginSetup, @@ -52,6 +51,25 @@ export class EnterpriseSearchPlugin implements Plugin { } public setup(core: CoreSetup, plugins: PluginsSetup) { + core.application.register({ + id: ENTERPRISE_SEARCH_PLUGIN.ID, + title: ENTERPRISE_SEARCH_PLUGIN.NAV_TITLE, + appRoute: ENTERPRISE_SEARCH_PLUGIN.URL, + category: DEFAULT_APP_CATEGORIES.enterpriseSearch, + mount: async (params: AppMountParameters) => { + const [coreStart] = await core.getStartServices(); + const { chrome } = coreStart; + chrome.docTitle.change(ENTERPRISE_SEARCH_PLUGIN.NAME); + + await this.getInitialData(coreStart.http); + + const { renderApp } = await import('./applications'); + const { EnterpriseSearch } = await import('./applications/enterprise_search'); + + return renderApp(EnterpriseSearch, params, coreStart, plugins, this.config, this.data); + }, + }); + core.application.register({ id: APP_SEARCH_PLUGIN.ID, title: APP_SEARCH_PLUGIN.NAME, @@ -94,22 +112,10 @@ export class EnterpriseSearchPlugin implements Plugin { plugins.home.featureCatalogue.registerSolution({ id: ENTERPRISE_SEARCH_PLUGIN.ID, title: ENTERPRISE_SEARCH_PLUGIN.NAME, - subtitle: i18n.translate('xpack.enterpriseSearch.featureCatalogue.subtitle', { - defaultMessage: 'Search everything', - }), + subtitle: ENTERPRISE_SEARCH_PLUGIN.SUBTITLE, icon: 'logoEnterpriseSearch', - descriptions: [ - i18n.translate('xpack.enterpriseSearch.featureCatalogueDescription1', { - defaultMessage: 'Build a powerful search experience.', - }), - i18n.translate('xpack.enterpriseSearch.featureCatalogueDescription2', { - defaultMessage: 'Connect your users to relevant data.', - }), - i18n.translate('xpack.enterpriseSearch.featureCatalogueDescription3', { - defaultMessage: 'Unify your team content.', - }), - ], - path: APP_SEARCH_PLUGIN.URL, // TODO: Change this to enterprise search overview page once available + descriptions: ENTERPRISE_SEARCH_PLUGIN.DESCRIPTIONS, + path: ENTERPRISE_SEARCH_PLUGIN.URL, }); plugins.home.featureCatalogue.register({ diff --git a/x-pack/plugins/enterprise_search/server/collectors/enterprise_search/telemetry.test.ts b/x-pack/plugins/enterprise_search/server/collectors/enterprise_search/telemetry.test.ts new file mode 100644 index 0000000000000..c3e2aff6551c9 --- /dev/null +++ b/x-pack/plugins/enterprise_search/server/collectors/enterprise_search/telemetry.test.ts @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { mockLogger } from '../../__mocks__'; + +import { registerTelemetryUsageCollector } from './telemetry'; + +describe('Enterprise Search Telemetry Usage Collector', () => { + const makeUsageCollectorStub = jest.fn(); + const registerStub = jest.fn(); + const usageCollectionMock = { + makeUsageCollector: makeUsageCollectorStub, + registerCollector: registerStub, + } as any; + + const savedObjectsRepoStub = { + get: () => ({ + attributes: { + 'ui_viewed.overview': 10, + 'ui_clicked.app_search': 2, + 'ui_clicked.workplace_search': 3, + }, + }), + incrementCounter: jest.fn(), + }; + const savedObjectsMock = { + createInternalRepository: jest.fn(() => savedObjectsRepoStub), + } as any; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + describe('registerTelemetryUsageCollector', () => { + it('should make and register the usage collector', () => { + registerTelemetryUsageCollector(usageCollectionMock, savedObjectsMock, mockLogger); + + expect(registerStub).toHaveBeenCalledTimes(1); + expect(makeUsageCollectorStub).toHaveBeenCalledTimes(1); + expect(makeUsageCollectorStub.mock.calls[0][0].type).toBe('enterprise_search'); + expect(makeUsageCollectorStub.mock.calls[0][0].isReady()).toBe(true); + }); + }); + + describe('fetchTelemetryMetrics', () => { + it('should return existing saved objects data', async () => { + registerTelemetryUsageCollector(usageCollectionMock, savedObjectsMock, mockLogger); + const savedObjectsCounts = await makeUsageCollectorStub.mock.calls[0][0].fetch(); + + expect(savedObjectsCounts).toEqual({ + ui_viewed: { + overview: 10, + }, + ui_clicked: { + app_search: 2, + workplace_search: 3, + }, + }); + }); + + it('should return a default telemetry object if no saved data exists', async () => { + const emptySavedObjectsMock = { + createInternalRepository: () => ({ + get: () => ({ attributes: null }), + }), + } as any; + + registerTelemetryUsageCollector(usageCollectionMock, emptySavedObjectsMock, mockLogger); + const savedObjectsCounts = await makeUsageCollectorStub.mock.calls[0][0].fetch(); + + expect(savedObjectsCounts).toEqual({ + ui_viewed: { + overview: 0, + }, + ui_clicked: { + app_search: 0, + workplace_search: 0, + }, + }); + }); + }); +}); diff --git a/x-pack/plugins/enterprise_search/server/collectors/enterprise_search/telemetry.ts b/x-pack/plugins/enterprise_search/server/collectors/enterprise_search/telemetry.ts new file mode 100644 index 0000000000000..a124a185b9a34 --- /dev/null +++ b/x-pack/plugins/enterprise_search/server/collectors/enterprise_search/telemetry.ts @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { get } from 'lodash'; +import { SavedObjectsServiceStart, Logger } from 'src/core/server'; +import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; + +import { getSavedObjectAttributesFromRepo } from '../lib/telemetry'; + +interface ITelemetry { + ui_viewed: { + overview: number; + }; + ui_clicked: { + app_search: number; + workplace_search: number; + }; +} + +export const ES_TELEMETRY_NAME = 'enterprise_search_telemetry'; + +/** + * Register the telemetry collector + */ + +export const registerTelemetryUsageCollector = ( + usageCollection: UsageCollectionSetup, + savedObjects: SavedObjectsServiceStart, + log: Logger +) => { + const telemetryUsageCollector = usageCollection.makeUsageCollector({ + type: 'enterprise_search', + fetch: async () => fetchTelemetryMetrics(savedObjects, log), + isReady: () => true, + schema: { + ui_viewed: { + overview: { type: 'long' }, + }, + ui_clicked: { + app_search: { type: 'long' }, + workplace_search: { type: 'long' }, + }, + }, + }); + usageCollection.registerCollector(telemetryUsageCollector); +}; + +/** + * Fetch the aggregated telemetry metrics from our saved objects + */ + +const fetchTelemetryMetrics = async (savedObjects: SavedObjectsServiceStart, log: Logger) => { + const savedObjectsRepository = savedObjects.createInternalRepository(); + const savedObjectAttributes = await getSavedObjectAttributesFromRepo( + ES_TELEMETRY_NAME, + savedObjectsRepository, + log + ); + + const defaultTelemetrySavedObject: ITelemetry = { + ui_viewed: { + overview: 0, + }, + ui_clicked: { + app_search: 0, + workplace_search: 0, + }, + }; + + // If we don't have an existing/saved telemetry object, return the default + if (!savedObjectAttributes) { + return defaultTelemetrySavedObject; + } + + return { + ui_viewed: { + overview: get(savedObjectAttributes, 'ui_viewed.overview', 0), + }, + ui_clicked: { + app_search: get(savedObjectAttributes, 'ui_clicked.app_search', 0), + workplace_search: get(savedObjectAttributes, 'ui_clicked.workplace_search', 0), + }, + } as ITelemetry; +}; diff --git a/x-pack/plugins/enterprise_search/server/collectors/lib/telemetry.test.ts b/x-pack/plugins/enterprise_search/server/collectors/lib/telemetry.test.ts index aae162c23ccb4..6cf0be9fd1f31 100644 --- a/x-pack/plugins/enterprise_search/server/collectors/lib/telemetry.test.ts +++ b/x-pack/plugins/enterprise_search/server/collectors/lib/telemetry.test.ts @@ -15,7 +15,7 @@ import { SavedObjectsErrorHelpers } from '../../../../../../src/core/server'; import { getSavedObjectAttributesFromRepo, incrementUICounter } from './telemetry'; -describe('App Search Telemetry Usage Collector', () => { +describe('Telemetry helpers', () => { beforeEach(() => { jest.clearAllMocks(); }); diff --git a/x-pack/plugins/enterprise_search/server/lib/enterprise_search_config_api.test.ts b/x-pack/plugins/enterprise_search/server/lib/enterprise_search_config_api.test.ts index 323f79e63bc6f..8e3ae2cfbeb86 100644 --- a/x-pack/plugins/enterprise_search/server/lib/enterprise_search_config_api.test.ts +++ b/x-pack/plugins/enterprise_search/server/lib/enterprise_search_config_api.test.ts @@ -38,51 +38,63 @@ describe('callEnterpriseSearchConfigAPI', () => { external_url: 'http://some.vanity.url/', read_only_mode: false, ilm_enabled: true, + is_federated_auth: false, configured_limits: { - max_document_byte_size: 102400, - max_engines_per_meta_engine: 15, + app_search: { + engine: { + document_size_in_bytes: 102400, + source_engines_per_meta_engine: 15, + }, + }, + workplace_search: { + custom_api_source: { + document_size_in_bytes: 102400, + total_fields: 64, + }, + }, + }, + }, + current_user: { + name: 'someuser', + access: { + app_search: true, + workplace_search: false, }, app_search: { - account_id: 'some-id-string', - onboarding_complete: true, + account: { + id: 'some-id-string', + onboarding_complete: true, + }, + role: { + id: 'account_id:somestring|user_oid:somestring', + role_type: 'owner', + ability: { + access_all_engines: true, + destroy: ['session'], + manage: ['account_credentials', 'account_engines'], // etc + edit: ['LocoMoco::Account'], // etc + view: ['Engine'], // etc + credential_types: ['admin', 'private', 'search'], + available_role_types: ['owner', 'admin'], + }, + }, }, workplace_search: { - can_create_invitations: true, - is_federated_auth: false, organization: { name: 'ACME Donuts', default_org_name: 'My Organization', }, - fp_account: { + account: { id: 'some-id-string', groups: ['Default', 'Cats'], is_admin: true, can_create_personal_sources: true, + can_create_invitations: true, is_curated: false, viewed_onboarding_page: true, }, }, }, - current_user: { - name: 'someuser', - access: { - app_search: true, - workplace_search: false, - }, - app_search_role: { - id: 'account_id:somestring|user_oid:somestring', - role_type: 'owner', - ability: { - access_all_engines: true, - destroy: ['session'], - manage: ['account_credentials', 'account_engines'], // etc - edit: ['LocoMoco::Account'], // etc - view: ['Engine'], // etc - credential_types: ['admin', 'private', 'search'], - available_role_types: ['owner', 'admin'], - }, - }, - }, }; beforeEach(() => { @@ -91,7 +103,7 @@ describe('callEnterpriseSearchConfigAPI', () => { it('calls the config API endpoint', async () => { fetchMock.mockImplementationOnce((url: string) => { - expect(url).toEqual('http://localhost:3002/api/ent/v1/internal/client_config'); + expect(url).toEqual('http://localhost:3002/api/ent/v2/internal/client_config'); return Promise.resolve(new Response(JSON.stringify(mockResponse))); }); @@ -116,9 +128,20 @@ describe('callEnterpriseSearchConfigAPI', () => { publicUrl: undefined, readOnlyMode: false, ilmEnabled: false, + isFederatedAuth: false, configuredLimits: { - maxDocumentByteSize: undefined, - maxEnginesPerMetaEngine: undefined, + appSearch: { + engine: { + maxDocumentByteSize: undefined, + maxEnginesPerMetaEngine: undefined, + }, + }, + workplaceSearch: { + customApiSource: { + maxDocumentByteSize: undefined, + totalFields: undefined, + }, + }, }, appSearch: { accountId: undefined, @@ -138,17 +161,16 @@ describe('callEnterpriseSearchConfigAPI', () => { }, }, workplaceSearch: { - canCreateInvitations: false, - isFederatedAuth: false, organization: { name: undefined, defaultOrgName: undefined, }, - fpAccount: { + account: { id: undefined, groups: [], isAdmin: false, canCreatePersonalSources: false, + canCreateInvitations: false, isCurated: false, viewedOnboardingPage: false, }, diff --git a/x-pack/plugins/enterprise_search/server/lib/enterprise_search_config_api.ts b/x-pack/plugins/enterprise_search/server/lib/enterprise_search_config_api.ts index c9cbec15169d9..10a75e59cb249 100644 --- a/x-pack/plugins/enterprise_search/server/lib/enterprise_search_config_api.ts +++ b/x-pack/plugins/enterprise_search/server/lib/enterprise_search_config_api.ts @@ -29,7 +29,7 @@ interface IReturn extends IInitialAppData { * useful various settings (e.g. product access, external URL) * needed by the Kibana plugin at the setup stage */ -const ENDPOINT = '/api/ent/v1/internal/client_config'; +const ENDPOINT = '/api/ent/v2/internal/client_config'; export const callEnterpriseSearchConfigAPI = async ({ config, @@ -67,44 +67,60 @@ export const callEnterpriseSearchConfigAPI = async ({ publicUrl: stripTrailingSlash(data?.settings?.external_url), readOnlyMode: !!data?.settings?.read_only_mode, ilmEnabled: !!data?.settings?.ilm_enabled, + isFederatedAuth: !!data?.settings?.is_federated_auth, // i.e., not standard auth configuredLimits: { - maxDocumentByteSize: data?.settings?.configured_limits?.max_document_byte_size, - maxEnginesPerMetaEngine: data?.settings?.configured_limits?.max_engines_per_meta_engine, + appSearch: { + engine: { + maxDocumentByteSize: + data?.settings?.configured_limits?.app_search?.engine?.document_size_in_bytes, + maxEnginesPerMetaEngine: + data?.settings?.configured_limits?.app_search?.engine?.source_engines_per_meta_engine, + }, + }, + workplaceSearch: { + customApiSource: { + maxDocumentByteSize: + data?.settings?.configured_limits?.workplace_search?.custom_api_source + ?.document_size_in_bytes, + totalFields: + data?.settings?.configured_limits?.workplace_search?.custom_api_source?.total_fields, + }, + }, }, appSearch: { - accountId: data?.settings?.app_search?.account_id, - onBoardingComplete: !!data?.settings?.app_search?.onboarding_complete, + accountId: data?.current_user?.app_search?.account?.id, + onBoardingComplete: !!data?.current_user?.app_search?.account?.onboarding_complete, role: { - id: data?.current_user?.app_search_role?.id, - roleType: data?.current_user?.app_search_role?.role_type, + id: data?.current_user?.app_search?.role?.id, + roleType: data?.current_user?.app_search?.role?.role_type, ability: { - accessAllEngines: !!data?.current_user?.app_search_role?.ability?.access_all_engines, - destroy: data?.current_user?.app_search_role?.ability?.destroy || [], - manage: data?.current_user?.app_search_role?.ability?.manage || [], - edit: data?.current_user?.app_search_role?.ability?.edit || [], - view: data?.current_user?.app_search_role?.ability?.view || [], - credentialTypes: data?.current_user?.app_search_role?.ability?.credential_types || [], + accessAllEngines: !!data?.current_user?.app_search?.role?.ability?.access_all_engines, + destroy: data?.current_user?.app_search?.role?.ability?.destroy || [], + manage: data?.current_user?.app_search?.role?.ability?.manage || [], + edit: data?.current_user?.app_search?.role?.ability?.edit || [], + view: data?.current_user?.app_search?.role?.ability?.view || [], + credentialTypes: data?.current_user?.app_search?.role?.ability?.credential_types || [], availableRoleTypes: - data?.current_user?.app_search_role?.ability?.available_role_types || [], + data?.current_user?.app_search?.role?.ability?.available_role_types || [], }, }, }, workplaceSearch: { - canCreateInvitations: !!data?.settings?.workplace_search?.can_create_invitations, - isFederatedAuth: !!data?.settings?.workplace_search?.is_federated_auth, organization: { - name: data?.settings?.workplace_search?.organization?.name, - defaultOrgName: data?.settings?.workplace_search?.organization?.default_org_name, + name: data?.current_user?.workplace_search?.organization?.name, + defaultOrgName: data?.current_user?.workplace_search?.organization?.default_org_name, }, - fpAccount: { - id: data?.settings?.workplace_search?.fp_account.id, - groups: data?.settings?.workplace_search?.fp_account.groups || [], - isAdmin: !!data?.settings?.workplace_search?.fp_account?.is_admin, - canCreatePersonalSources: !!data?.settings?.workplace_search?.fp_account + account: { + id: data?.current_user?.workplace_search?.account?.id, + groups: data?.current_user?.workplace_search?.account?.groups || [], + isAdmin: !!data?.current_user?.workplace_search?.account?.is_admin, + canCreatePersonalSources: !!data?.current_user?.workplace_search?.account ?.can_create_personal_sources, - isCurated: !!data?.settings?.workplace_search?.fp_account.is_curated, - viewedOnboardingPage: !!data?.settings?.workplace_search?.fp_account - .viewed_onboarding_page, + canCreateInvitations: !!data?.current_user?.workplace_search?.account + ?.can_create_invitations, + isCurated: !!data?.current_user?.workplace_search?.account?.is_curated, + viewedOnboardingPage: !!data?.current_user?.workplace_search?.account + ?.viewed_onboarding_page, }, }, }; diff --git a/x-pack/plugins/enterprise_search/server/lib/enterprise_search_request_handler.test.ts b/x-pack/plugins/enterprise_search/server/lib/enterprise_search_request_handler.test.ts index 3f3f182433144..34f83ef3a3fd2 100644 --- a/x-pack/plugins/enterprise_search/server/lib/enterprise_search_request_handler.test.ts +++ b/x-pack/plugins/enterprise_search/server/lib/enterprise_search_request_handler.test.ts @@ -5,6 +5,7 @@ */ import { mockConfig, mockLogger } from '../__mocks__'; +import { JSON_HEADER } from '../../common/constants'; import { EnterpriseSearchRequestHandler } from './enterprise_search_request_handler'; @@ -150,18 +151,26 @@ describe('EnterpriseSearchRequestHandler', () => { ); }); - it('returns an error when user authentication to Enterprise Search fails', async () => { - EnterpriseSearchAPI.mockReturn({}, { url: 'http://localhost:3002/login' }); - const requestHandler = enterpriseSearchRequestHandler.createRequest({ - path: '/api/unauthenticated', + describe('user authentication errors', () => { + afterEach(async () => { + const requestHandler = enterpriseSearchRequestHandler.createRequest({ + path: '/api/unauthenticated', + }); + await makeAPICall(requestHandler); + + EnterpriseSearchAPI.shouldHaveBeenCalledWith('http://localhost:3002/api/unauthenticated'); + expect(responseMock.customError).toHaveBeenCalledWith({ + body: 'Error connecting to Enterprise Search: Cannot authenticate Enterprise Search user', + statusCode: 502, + }); }); - await makeAPICall(requestHandler); - EnterpriseSearchAPI.shouldHaveBeenCalledWith('http://localhost:3002/api/unauthenticated'); + it('errors when redirected to /login', async () => { + EnterpriseSearchAPI.mockReturn({}, { url: 'http://localhost:3002/login' }); + }); - expect(responseMock.customError).toHaveBeenCalledWith({ - body: 'Error connecting to Enterprise Search: Cannot authenticate Enterprise Search user', - statusCode: 502, + it('errors when redirected to /ent/select', async () => { + EnterpriseSearchAPI.mockReturn({}, { url: 'http://localhost:3002/ent/select' }); }); }); }); @@ -185,7 +194,7 @@ const makeAPICall = (handler: Function, params = {}) => { const EnterpriseSearchAPI = { shouldHaveBeenCalledWith(expectedUrl: string, expectedParams = {}) { expect(fetchMock).toHaveBeenCalledWith(expectedUrl, { - headers: { Authorization: 'Basic 123' }, + headers: { Authorization: 'Basic 123', ...JSON_HEADER }, method: 'GET', body: undefined, ...expectedParams, diff --git a/x-pack/plugins/enterprise_search/server/lib/enterprise_search_request_handler.ts b/x-pack/plugins/enterprise_search/server/lib/enterprise_search_request_handler.ts index 8f31bd9063d4a..18f10c590847c 100644 --- a/x-pack/plugins/enterprise_search/server/lib/enterprise_search_request_handler.ts +++ b/x-pack/plugins/enterprise_search/server/lib/enterprise_search_request_handler.ts @@ -14,6 +14,7 @@ import { Logger, } from 'src/core/server'; import { ConfigType } from '../index'; +import { JSON_HEADER } from '../../common/constants'; interface IConstructorDependencies { config: ConfigType; @@ -25,7 +26,7 @@ interface IRequestParams { hasValidData?: (body?: ResponseBody) => boolean; } export interface IEnterpriseSearchRequestHandler { - createRequest(requestParams?: object): RequestHandler, unknown>; + createRequest(requestParams?: object): RequestHandler; } /** @@ -52,12 +53,12 @@ export class EnterpriseSearchRequestHandler { }: IRequestParams) { return async ( _context: RequestHandlerContext, - request: KibanaRequest, unknown>, + request: KibanaRequest, response: KibanaResponseFactory ) => { try { // Set up API URL - const queryParams = { ...request.query, ...params }; + const queryParams = { ...(request.query as object), ...params }; const queryString = !this.isEmptyObj(queryParams) ? `?${querystring.stringify(queryParams)}` : ''; @@ -65,7 +66,7 @@ export class EnterpriseSearchRequestHandler { // Set up API options const { method } = request.route; - const headers = { Authorization: request.headers.authorization as string }; + const headers = { Authorization: request.headers.authorization as string, ...JSON_HEADER }; const body = !this.isEmptyObj(request.body as object) ? JSON.stringify(request.body) : undefined; @@ -73,7 +74,7 @@ export class EnterpriseSearchRequestHandler { // Call the Enterprise Search API and pass back response to the front-end const apiResponse = await fetch(url, { method, headers, body }); - if (apiResponse.url.endsWith('/login')) { + if (apiResponse.url.endsWith('/login') || apiResponse.url.endsWith('/ent/select')) { throw new Error('Cannot authenticate Enterprise Search user'); } diff --git a/x-pack/plugins/enterprise_search/server/plugin.ts b/x-pack/plugins/enterprise_search/server/plugin.ts index 617210a544262..729a03d24065e 100644 --- a/x-pack/plugins/enterprise_search/server/plugin.ts +++ b/x-pack/plugins/enterprise_search/server/plugin.ts @@ -31,8 +31,10 @@ import { IEnterpriseSearchRequestHandler, } from './lib/enterprise_search_request_handler'; -import { registerConfigDataRoute } from './routes/enterprise_search/config_data'; +import { enterpriseSearchTelemetryType } from './saved_objects/enterprise_search/telemetry'; +import { registerTelemetryUsageCollector as registerESTelemetryUsageCollector } from './collectors/enterprise_search/telemetry'; import { registerTelemetryRoute } from './routes/enterprise_search/telemetry'; +import { registerConfigDataRoute } from './routes/enterprise_search/config_data'; import { appSearchTelemetryType } from './saved_objects/app_search/telemetry'; import { registerTelemetryUsageCollector as registerASTelemetryUsageCollector } from './collectors/app_search/telemetry'; @@ -81,8 +83,12 @@ export class EnterpriseSearchPlugin implements Plugin { name: ENTERPRISE_SEARCH_PLUGIN.NAME, order: 0, icon: 'logoEnterpriseSearch', - navLinkId: APP_SEARCH_PLUGIN.ID, // TODO - remove this once functional tests no longer rely on navLinkId - app: ['kibana', APP_SEARCH_PLUGIN.ID, WORKPLACE_SEARCH_PLUGIN.ID], + app: [ + 'kibana', + ENTERPRISE_SEARCH_PLUGIN.ID, + APP_SEARCH_PLUGIN.ID, + WORKPLACE_SEARCH_PLUGIN.ID, + ], catalogue: [ENTERPRISE_SEARCH_PLUGIN.ID, APP_SEARCH_PLUGIN.ID, WORKPLACE_SEARCH_PLUGIN.ID], privileges: null, }); @@ -94,14 +100,16 @@ export class EnterpriseSearchPlugin implements Plugin { const dependencies = { config, security, request, log }; const { hasAppSearchAccess, hasWorkplaceSearchAccess } = await checkAccess(dependencies); + const showEnterpriseSearchOverview = hasAppSearchAccess || hasWorkplaceSearchAccess; return { navLinks: { + enterpriseSearch: showEnterpriseSearchOverview, appSearch: hasAppSearchAccess, workplaceSearch: hasWorkplaceSearchAccess, }, catalogue: { - enterpriseSearch: hasAppSearchAccess || hasWorkplaceSearchAccess, + enterpriseSearch: showEnterpriseSearchOverview, appSearch: hasAppSearchAccess, workplaceSearch: hasWorkplaceSearchAccess, }, @@ -123,6 +131,7 @@ export class EnterpriseSearchPlugin implements Plugin { /** * Bootstrap the routes, saved objects, and collector for telemetry */ + savedObjects.registerType(enterpriseSearchTelemetryType); savedObjects.registerType(appSearchTelemetryType); savedObjects.registerType(workplaceSearchTelemetryType); let savedObjectsStarted: SavedObjectsServiceStart; @@ -131,6 +140,7 @@ export class EnterpriseSearchPlugin implements Plugin { savedObjectsStarted = coreStart.savedObjects; if (usageCollection) { + registerESTelemetryUsageCollector(usageCollection, savedObjectsStarted, this.logger); registerASTelemetryUsageCollector(usageCollection, savedObjectsStarted, this.logger); registerWSTelemetryUsageCollector(usageCollection, savedObjectsStarted, this.logger); } diff --git a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/telemetry.ts b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/telemetry.ts index 7ed1d7b17753c..bfc07c8b64ef5 100644 --- a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/telemetry.ts +++ b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/telemetry.ts @@ -9,12 +9,13 @@ import { schema } from '@kbn/config-schema'; import { IRouteDependencies } from '../../plugin'; import { incrementUICounter } from '../../collectors/lib/telemetry'; +import { ES_TELEMETRY_NAME } from '../../collectors/enterprise_search/telemetry'; import { AS_TELEMETRY_NAME } from '../../collectors/app_search/telemetry'; import { WS_TELEMETRY_NAME } from '../../collectors/workplace_search/telemetry'; const productToTelemetryMap = { + enterprise_search: ES_TELEMETRY_NAME, app_search: AS_TELEMETRY_NAME, workplace_search: WS_TELEMETRY_NAME, - enterprise_search: 'TODO', }; export function registerTelemetryRoute({ diff --git a/x-pack/plugins/enterprise_search/server/saved_objects/enterprise_search/telemetry.ts b/x-pack/plugins/enterprise_search/server/saved_objects/enterprise_search/telemetry.ts new file mode 100644 index 0000000000000..54044e67939da --- /dev/null +++ b/x-pack/plugins/enterprise_search/server/saved_objects/enterprise_search/telemetry.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +/* istanbul ignore file */ + +import { SavedObjectsType } from 'src/core/server'; +import { ES_TELEMETRY_NAME } from '../../collectors/enterprise_search/telemetry'; + +export const enterpriseSearchTelemetryType: SavedObjectsType = { + name: ES_TELEMETRY_NAME, + hidden: false, + namespaceType: 'agnostic', + mappings: { + dynamic: false, + properties: {}, + }, +}; diff --git a/x-pack/plugins/features/server/__snapshots__/oss_features.test.ts.snap b/x-pack/plugins/features/server/__snapshots__/oss_features.test.ts.snap index e4014cf49778c..63a59d59d6d07 100644 --- a/x-pack/plugins/features/server/__snapshots__/oss_features.test.ts.snap +++ b/x-pack/plugins/features/server/__snapshots__/oss_features.test.ts.snap @@ -111,6 +111,7 @@ Array [ "visualization", "timelion-sheet", "canvas-workpad", + "lens", "map", "dashboard", "query", diff --git a/x-pack/plugins/features/server/oss_features.ts b/x-pack/plugins/features/server/oss_features.ts index e37c7491de5dc..4122c590e74b1 100644 --- a/x-pack/plugins/features/server/oss_features.ts +++ b/x-pack/plugins/features/server/oss_features.ts @@ -172,6 +172,7 @@ export const buildOSSFeatures = ({ savedObjectTypes, includeTimelion }: BuildOSS 'visualization', 'timelion-sheet', 'canvas-workpad', + 'lens', 'map', 'dashboard', 'query', diff --git a/x-pack/plugins/infra/common/http_api/index.ts b/x-pack/plugins/infra/common/http_api/index.ts index 818009417fb1c..4c729d11ba8c1 100644 --- a/x-pack/plugins/infra/common/http_api/index.ts +++ b/x-pack/plugins/infra/common/http_api/index.ts @@ -10,3 +10,4 @@ export * from './log_entries'; export * from './metrics_explorer'; export * from './metrics_api'; export * from './log_alerts'; +export * from './snapshot_api'; diff --git a/x-pack/plugins/infra/common/http_api/metrics_api.ts b/x-pack/plugins/infra/common/http_api/metrics_api.ts index 7436566f039ca..41657fdce2153 100644 --- a/x-pack/plugins/infra/common/http_api/metrics_api.ts +++ b/x-pack/plugins/infra/common/http_api/metrics_api.ts @@ -33,7 +33,6 @@ export const MetricsAPIRequestRT = rt.intersection([ afterKey: rt.union([rt.null, afterKeyObjectRT]), limit: rt.union([rt.number, rt.null, rt.undefined]), filters: rt.array(rt.object), - forceInterval: rt.boolean, dropLastBucket: rt.boolean, alignDataToEnd: rt.boolean, }), @@ -59,7 +58,10 @@ export const MetricsAPIRowRT = rt.intersection([ rt.type({ timestamp: rt.number, }), - rt.record(rt.string, rt.union([rt.string, rt.number, rt.null, rt.undefined])), + rt.record( + rt.string, + rt.union([rt.string, rt.number, rt.null, rt.undefined, rt.array(rt.object)]) + ), ]); export const MetricsAPISeriesRT = rt.intersection([ diff --git a/x-pack/plugins/infra/common/http_api/metrics_explorer.ts b/x-pack/plugins/infra/common/http_api/metrics_explorer.ts index c5776e0b0ced1..460b2bf9d802e 100644 --- a/x-pack/plugins/infra/common/http_api/metrics_explorer.ts +++ b/x-pack/plugins/infra/common/http_api/metrics_explorer.ts @@ -89,7 +89,10 @@ export const metricsExplorerRowRT = rt.intersection([ rt.type({ timestamp: rt.number, }), - rt.record(rt.string, rt.union([rt.string, rt.number, rt.null, rt.undefined])), + rt.record( + rt.string, + rt.union([rt.string, rt.number, rt.null, rt.undefined, rt.array(rt.object)]) + ), ]); export const metricsExplorerSeriesRT = rt.intersection([ diff --git a/x-pack/plugins/infra/common/http_api/snapshot_api.ts b/x-pack/plugins/infra/common/http_api/snapshot_api.ts index 11cb57238f917..e1b8dfa4770ba 100644 --- a/x-pack/plugins/infra/common/http_api/snapshot_api.ts +++ b/x-pack/plugins/infra/common/http_api/snapshot_api.ts @@ -6,7 +6,7 @@ import * as rt from 'io-ts'; import { SnapshotMetricTypeRT, ItemTypeRT } from '../inventory_models/types'; -import { metricsExplorerSeriesRT } from './metrics_explorer'; +import { MetricsAPISeriesRT } from './metrics_api'; export const SnapshotNodePathRT = rt.intersection([ rt.type({ @@ -22,7 +22,7 @@ const SnapshotNodeMetricOptionalRT = rt.partial({ value: rt.union([rt.number, rt.null]), avg: rt.union([rt.number, rt.null]), max: rt.union([rt.number, rt.null]), - timeseries: metricsExplorerSeriesRT, + timeseries: MetricsAPISeriesRT, }); const SnapshotNodeMetricRequiredRT = rt.type({ @@ -36,6 +36,7 @@ export const SnapshotNodeMetricRT = rt.intersection([ export const SnapshotNodeRT = rt.type({ metrics: rt.array(SnapshotNodeMetricRT), path: rt.array(SnapshotNodePathRT), + name: rt.string, }); export const SnapshotNodeResponseRT = rt.type({ diff --git a/x-pack/plugins/infra/common/inventory_models/types.ts b/x-pack/plugins/infra/common/inventory_models/types.ts index 570220bbc7aa5..851646ef1fa12 100644 --- a/x-pack/plugins/infra/common/inventory_models/types.ts +++ b/x-pack/plugins/infra/common/inventory_models/types.ts @@ -281,6 +281,10 @@ export const ESSumBucketAggRT = rt.type({ }), }); +export const ESTopHitsAggRT = rt.type({ + top_hits: rt.object, +}); + interface SnapshotTermsWithAggregation { terms: { field: string }; aggregations: MetricsUIAggregation; @@ -304,6 +308,7 @@ export const ESAggregationRT = rt.union([ ESSumBucketAggRT, ESTermsWithAggregationRT, ESCaridnalityAggRT, + ESTopHitsAggRT, ]); export const MetricsUIAggregationRT = rt.record(rt.string, ESAggregationRT); diff --git a/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/waffle/conditional_tooltip.test.tsx b/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/waffle/conditional_tooltip.test.tsx index d2c30a4f38ee9..e01ca3ab6e844 100644 --- a/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/waffle/conditional_tooltip.test.tsx +++ b/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/waffle/conditional_tooltip.test.tsx @@ -88,6 +88,7 @@ describe('ConditionalToolTip', () => { mockedUseSnapshot.mockReturnValue({ nodes: [ { + name: 'host-01', path: [{ label: 'host-01', value: 'host-01', ip: '192.168.1.10' }], metrics: [ { name: 'cpu', value: 0.1, avg: 0.4, max: 0.7 }, diff --git a/x-pack/plugins/infra/public/pages/metrics/inventory_view/lib/calculate_bounds_from_nodes.test.ts b/x-pack/plugins/infra/public/pages/metrics/inventory_view/lib/calculate_bounds_from_nodes.test.ts index fbb6aa933219a..49f4b56532936 100644 --- a/x-pack/plugins/infra/public/pages/metrics/inventory_view/lib/calculate_bounds_from_nodes.test.ts +++ b/x-pack/plugins/infra/public/pages/metrics/inventory_view/lib/calculate_bounds_from_nodes.test.ts @@ -7,6 +7,7 @@ import { calculateBoundsFromNodes } from './calculate_bounds_from_nodes'; import { SnapshotNode } from '../../../../../common/http_api/snapshot_api'; const nodes: SnapshotNode[] = [ { + name: 'host-01', path: [{ value: 'host-01', label: 'host-01' }], metrics: [ { @@ -18,6 +19,7 @@ const nodes: SnapshotNode[] = [ ], }, { + name: 'host-02', path: [{ value: 'host-02', label: 'host-02' }], metrics: [ { diff --git a/x-pack/plugins/infra/public/pages/metrics/inventory_view/lib/sort_nodes.test.ts b/x-pack/plugins/infra/public/pages/metrics/inventory_view/lib/sort_nodes.test.ts index 2a9f8b911c124..f7d9f029f00df 100644 --- a/x-pack/plugins/infra/public/pages/metrics/inventory_view/lib/sort_nodes.test.ts +++ b/x-pack/plugins/infra/public/pages/metrics/inventory_view/lib/sort_nodes.test.ts @@ -9,6 +9,7 @@ import { SnapshotNode } from '../../../../../common/http_api/snapshot_api'; const nodes: SnapshotNode[] = [ { + name: 'host-01', path: [{ value: 'host-01', label: 'host-01' }], metrics: [ { @@ -20,6 +21,7 @@ const nodes: SnapshotNode[] = [ ], }, { + name: 'host-02', path: [{ value: 'host-02', label: 'host-02' }], metrics: [ { diff --git a/x-pack/plugins/infra/server/lib/alerting/inventory_metric_threshold/evaluate_condition.ts b/x-pack/plugins/infra/server/lib/alerting/inventory_metric_threshold/evaluate_condition.ts index 2f3593a11f664..d6592719d0723 100644 --- a/x-pack/plugins/infra/server/lib/alerting/inventory_metric_threshold/evaluate_condition.ts +++ b/x-pack/plugins/infra/server/lib/alerting/inventory_metric_threshold/evaluate_condition.ts @@ -16,12 +16,11 @@ import { } from '../../adapters/framework/adapter_types'; import { Comparator, InventoryMetricConditions } from './types'; import { AlertServices } from '../../../../../alerts/server'; -import { InfraSnapshot } from '../../snapshot'; -import { parseFilterQuery } from '../../../utils/serialized_query'; import { InventoryItemType, SnapshotMetricType } from '../../../../common/inventory_models/types'; -import { InfraTimerangeInput } from '../../../../common/http_api/snapshot_api'; -import { InfraSourceConfiguration } from '../../sources'; +import { InfraTimerangeInput, SnapshotRequest } from '../../../../common/http_api/snapshot_api'; +import { InfraSource } from '../../sources'; import { UNGROUPED_FACTORY_KEY } from '../common/utils'; +import { getNodes } from '../../../routes/snapshot/lib/get_nodes'; type ConditionResult = InventoryMetricConditions & { shouldFire: boolean[]; @@ -33,7 +32,7 @@ type ConditionResult = InventoryMetricConditions & { export const evaluateCondition = async ( condition: InventoryMetricConditions, nodeType: InventoryItemType, - sourceConfiguration: InfraSourceConfiguration, + source: InfraSource, callCluster: AlertServices['callCluster'], filterQuery?: string, lookbackSize?: number @@ -55,7 +54,7 @@ export const evaluateCondition = async ( nodeType, metric, timerange, - sourceConfiguration, + source, filterQuery, customMetric ); @@ -94,12 +93,11 @@ const getData = async ( nodeType: InventoryItemType, metric: SnapshotMetricType, timerange: InfraTimerangeInput, - sourceConfiguration: InfraSourceConfiguration, + source: InfraSource, filterQuery?: string, customMetric?: SnapshotCustomMetricInput ) => { - const snapshot = new InfraSnapshot(); - const esClient = ( + const client = ( options: CallWithRequestParams ): Promise> => callCluster('search', options); @@ -107,17 +105,17 @@ const getData = async ( metric === 'custom' ? (customMetric as SnapshotCustomMetricInput) : { type: metric }, ]; - const options = { - filterQuery: parseFilterQuery(filterQuery), + const snapshotRequest: SnapshotRequest = { + filterQuery, nodeType, groupBy: [], - sourceConfiguration, + sourceId: 'default', metrics, timerange, includeTimeseries: Boolean(timerange.lookbackSize), }; try { - const { nodes } = await snapshot.getNodes(esClient, options); + const { nodes } = await getNodes(client, snapshotRequest, source); if (!nodes.length) return { [UNGROUPED_FACTORY_KEY]: null }; // No Data state diff --git a/x-pack/plugins/infra/server/lib/alerting/inventory_metric_threshold/inventory_metric_threshold_executor.ts b/x-pack/plugins/infra/server/lib/alerting/inventory_metric_threshold/inventory_metric_threshold_executor.ts index bdac9dcd1dee8..99904f15b4606 100644 --- a/x-pack/plugins/infra/server/lib/alerting/inventory_metric_threshold/inventory_metric_threshold_executor.ts +++ b/x-pack/plugins/infra/server/lib/alerting/inventory_metric_threshold/inventory_metric_threshold_executor.ts @@ -50,9 +50,7 @@ export const createInventoryMetricThresholdExecutor = (libs: InfraBackendLibs) = ); const results = await Promise.all( - criteria.map((c) => - evaluateCondition(c, nodeType, source.configuration, services.callCluster, filterQuery) - ) + criteria.map((c) => evaluateCondition(c, nodeType, source, services.callCluster, filterQuery)) ); const inventoryItems = Object.keys(first(results)!); diff --git a/x-pack/plugins/infra/server/lib/alerting/inventory_metric_threshold/preview_inventory_metric_threshold_alert.ts b/x-pack/plugins/infra/server/lib/alerting/inventory_metric_threshold/preview_inventory_metric_threshold_alert.ts index 755c395818f5a..2ab015b6b37a2 100644 --- a/x-pack/plugins/infra/server/lib/alerting/inventory_metric_threshold/preview_inventory_metric_threshold_alert.ts +++ b/x-pack/plugins/infra/server/lib/alerting/inventory_metric_threshold/preview_inventory_metric_threshold_alert.ts @@ -26,7 +26,7 @@ interface InventoryMetricThresholdParams { interface PreviewInventoryMetricThresholdAlertParams { callCluster: ILegacyScopedClusterClient['callAsCurrentUser']; params: InventoryMetricThresholdParams; - config: InfraSource['configuration']; + source: InfraSource; lookback: Unit; alertInterval: string; } @@ -34,7 +34,7 @@ interface PreviewInventoryMetricThresholdAlertParams { export const previewInventoryMetricThresholdAlert = async ({ callCluster, params, - config, + source, lookback, alertInterval, }: PreviewInventoryMetricThresholdAlertParams) => { @@ -55,7 +55,7 @@ export const previewInventoryMetricThresholdAlert = async ({ try { const results = await Promise.all( criteria.map((c) => - evaluateCondition(c, nodeType, config, callCluster, filterQuery, lookbackSize) + evaluateCondition(c, nodeType, source, callCluster, filterQuery, lookbackSize) ) ); diff --git a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.ts b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.ts index 078ca46d42e60..8696081043ff7 100644 --- a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.ts +++ b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.ts @@ -8,8 +8,8 @@ import { networkTraffic } from '../../../../../common/inventory_models/shared/me import { MetricExpressionParams, Aggregators } from '../types'; import { getIntervalInSeconds } from '../../../../utils/get_interval_in_seconds'; import { roundTimestamp } from '../../../../utils/round_timestamp'; -import { getDateHistogramOffset } from '../../../snapshot/query_helpers'; import { createPercentileAggregation } from './create_percentile_aggregation'; +import { calculateDateHistogramOffset } from '../../../metrics/lib/calculate_date_histogram_offset'; const MINIMUM_BUCKETS = 5; @@ -46,7 +46,7 @@ export const getElasticsearchMetricQuery = ( timeUnit ); - const offset = getDateHistogramOffset(from, interval); + const offset = calculateDateHistogramOffset({ from, to, interval, field: timefield }); const aggregations = aggType === Aggregators.COUNT diff --git a/x-pack/plugins/infra/server/lib/infra_types.ts b/x-pack/plugins/infra/server/lib/infra_types.ts index 9896ad6ac1cd1..084ece52302b0 100644 --- a/x-pack/plugins/infra/server/lib/infra_types.ts +++ b/x-pack/plugins/infra/server/lib/infra_types.ts @@ -8,7 +8,6 @@ import { InfraSourceConfiguration } from '../../common/graphql/types'; import { InfraFieldsDomain } from './domains/fields_domain'; import { InfraLogEntriesDomain } from './domains/log_entries_domain'; import { InfraMetricsDomain } from './domains/metrics_domain'; -import { InfraSnapshot } from './snapshot'; import { InfraSources } from './sources'; import { InfraSourceStatus } from './source_status'; import { InfraConfig } from '../plugin'; @@ -30,7 +29,6 @@ export interface InfraDomainLibs { export interface InfraBackendLibs extends InfraDomainLibs { configuration: InfraConfig; framework: KibanaFramework; - snapshot: InfraSnapshot; sources: InfraSources; sourceStatus: InfraSourceStatus; } diff --git a/x-pack/plugins/infra/server/lib/metrics/lib/__snapshots__/create_aggregations.test.ts.snap b/x-pack/plugins/infra/server/lib/metrics/lib/__snapshots__/create_aggregations.test.ts.snap index d2d90914eced5..2cbbc623aed38 100644 --- a/x-pack/plugins/infra/server/lib/metrics/lib/__snapshots__/create_aggregations.test.ts.snap +++ b/x-pack/plugins/infra/server/lib/metrics/lib/__snapshots__/create_aggregations.test.ts.snap @@ -53,7 +53,6 @@ Object { "groupBy0": Object { "terms": Object { "field": "host.name", - "order": "asc", }, }, }, diff --git a/x-pack/plugins/infra/server/lib/metrics/lib/convert_histogram_buckets_to_timeseries.ts b/x-pack/plugins/infra/server/lib/metrics/lib/convert_histogram_buckets_to_timeseries.ts index 95e6ece215133..90e584368e9ad 100644 --- a/x-pack/plugins/infra/server/lib/metrics/lib/convert_histogram_buckets_to_timeseries.ts +++ b/x-pack/plugins/infra/server/lib/metrics/lib/convert_histogram_buckets_to_timeseries.ts @@ -5,6 +5,7 @@ */ import { get, values, first } from 'lodash'; +import * as rt from 'io-ts'; import { MetricsAPIRequest, MetricsAPISeries, @@ -13,15 +14,20 @@ import { } from '../../../../common/http_api/metrics_api'; import { HistogramBucket, - MetricValueType, BasicMetricValueRT, NormalizedMetricValueRT, PercentilesTypeRT, PercentilesKeyedTypeRT, + TopHitsTypeRT, + MetricValueTypeRT, } from '../types'; + const BASE_COLUMNS = [{ name: 'timestamp', type: 'date' }] as MetricsAPIColumn[]; -const getValue = (valueObject: string | number | MetricValueType) => { +const ValueObjectTypeRT = rt.union([rt.string, rt.number, MetricValueTypeRT]); +type ValueObjectType = rt.TypeOf; + +const getValue = (valueObject: ValueObjectType) => { if (NormalizedMetricValueRT.is(valueObject)) { return valueObject.normalized_value || valueObject.value; } @@ -50,6 +56,10 @@ const getValue = (valueObject: string | number | MetricValueType) => { return valueObject.value; } + if (TopHitsTypeRT.is(valueObject)) { + return valueObject.hits.hits.map((hit) => hit._source); + } + return null; }; @@ -61,8 +71,8 @@ const convertBucketsToRows = ( const ids = options.metrics.map((metric) => metric.id); const metrics = ids.reduce((acc, id) => { const valueObject = get(bucket, [id]); - return { ...acc, [id]: getValue(valueObject) }; - }, {} as Record); + return { ...acc, [id]: ValueObjectTypeRT.is(valueObject) ? getValue(valueObject) : null }; + }, {} as Record); return { timestamp: bucket.key as number, ...metrics }; }); }; diff --git a/x-pack/plugins/infra/server/lib/metrics/lib/create_aggregations.ts b/x-pack/plugins/infra/server/lib/metrics/lib/create_aggregations.ts index 991e5febfc634..63fdbb3d2b30f 100644 --- a/x-pack/plugins/infra/server/lib/metrics/lib/create_aggregations.ts +++ b/x-pack/plugins/infra/server/lib/metrics/lib/create_aggregations.ts @@ -33,7 +33,7 @@ export const createAggregations = (options: MetricsAPIRequest) => { composite: { size: limit, sources: options.groupBy.map((field, index) => ({ - [`groupBy${index}`]: { terms: { field, order: 'asc' } }, + [`groupBy${index}`]: { terms: { field } }, })), }, aggs: histogramAggregation, diff --git a/x-pack/plugins/infra/server/lib/metrics/types.ts b/x-pack/plugins/infra/server/lib/metrics/types.ts index d1866470e0cf9..8746614b559d6 100644 --- a/x-pack/plugins/infra/server/lib/metrics/types.ts +++ b/x-pack/plugins/infra/server/lib/metrics/types.ts @@ -25,17 +25,51 @@ export const PercentilesKeyedTypeRT = rt.type({ values: rt.array(rt.type({ key: rt.string, value: NumberOrNullRT })), }); +export const TopHitsTypeRT = rt.type({ + hits: rt.type({ + total: rt.type({ + value: rt.number, + relation: rt.string, + }), + hits: rt.array( + rt.intersection([ + rt.type({ + _index: rt.string, + _id: rt.string, + _score: NumberOrNullRT, + _source: rt.object, + }), + rt.partial({ + sort: rt.array(rt.union([rt.string, rt.number])), + max_score: NumberOrNullRT, + }), + ]) + ), + }), +}); + export const MetricValueTypeRT = rt.union([ BasicMetricValueRT, NormalizedMetricValueRT, PercentilesTypeRT, PercentilesKeyedTypeRT, + TopHitsTypeRT, ]); export type MetricValueType = rt.TypeOf; +export const TermsWithMetrics = rt.intersection([ + rt.type({ + buckets: rt.array(rt.record(rt.string, rt.union([rt.number, rt.string, MetricValueTypeRT]))), + }), + rt.partial({ + sum_other_doc_count: rt.number, + doc_count_error_upper_bound: rt.number, + }), +]); + export const HistogramBucketRT = rt.record( rt.string, - rt.union([rt.number, rt.string, MetricValueTypeRT]) + rt.union([rt.number, rt.string, MetricValueTypeRT, TermsWithMetrics]) ); export const HistogramResponseRT = rt.type({ diff --git a/x-pack/plugins/infra/server/lib/snapshot/query_helpers.ts b/x-pack/plugins/infra/server/lib/snapshot/query_helpers.ts deleted file mode 100644 index ca63043ba868e..0000000000000 --- a/x-pack/plugins/infra/server/lib/snapshot/query_helpers.ts +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { i18n } from '@kbn/i18n'; -import { findInventoryModel, findInventoryFields } from '../../../common/inventory_models/index'; -import { InfraSnapshotRequestOptions } from './types'; -import { getIntervalInSeconds } from '../../utils/get_interval_in_seconds'; -import { - MetricsUIAggregation, - MetricsUIAggregationRT, - InventoryItemType, -} from '../../../common/inventory_models/types'; -import { - SnapshotMetricInput, - SnapshotCustomMetricInputRT, -} from '../../../common/http_api/snapshot_api'; -import { networkTraffic } from '../../../common/inventory_models/shared/metrics/snapshot/network_traffic'; - -interface GroupBySource { - [id: string]: { - terms: { - field: string | null | undefined; - missing_bucket?: boolean; - }; - }; -} - -export const getFieldByNodeType = (options: InfraSnapshotRequestOptions) => { - const inventoryFields = findInventoryFields(options.nodeType, options.sourceConfiguration.fields); - return inventoryFields.id; -}; - -export const getGroupedNodesSources = (options: InfraSnapshotRequestOptions) => { - const fields = findInventoryFields(options.nodeType, options.sourceConfiguration.fields); - const sources: GroupBySource[] = options.groupBy.map((gb) => { - return { [`${gb.field}`]: { terms: { field: gb.field } } }; - }); - sources.push({ - id: { - terms: { field: fields.id }, - }, - }); - sources.push({ - name: { terms: { field: fields.name, missing_bucket: true } }, - }); - return sources; -}; - -export const getMetricsSources = (options: InfraSnapshotRequestOptions) => { - const fields = findInventoryFields(options.nodeType, options.sourceConfiguration.fields); - return [{ id: { terms: { field: fields.id } } }]; -}; - -export const metricToAggregation = ( - nodeType: InventoryItemType, - metric: SnapshotMetricInput, - index: number -) => { - const inventoryModel = findInventoryModel(nodeType); - if (SnapshotCustomMetricInputRT.is(metric)) { - if (metric.aggregation === 'rate') { - return networkTraffic(`custom_${index}`, metric.field); - } - return { - [`custom_${index}`]: { - [metric.aggregation]: { - field: metric.field, - }, - }, - }; - } - return inventoryModel.metrics.snapshot?.[metric.type]; -}; - -export const getMetricsAggregations = ( - options: InfraSnapshotRequestOptions -): MetricsUIAggregation => { - const { metrics } = options; - return metrics.reduce((aggs, metric, index) => { - const aggregation = metricToAggregation(options.nodeType, metric, index); - if (!MetricsUIAggregationRT.is(aggregation)) { - throw new Error( - i18n.translate('xpack.infra.snapshot.missingSnapshotMetricError', { - defaultMessage: 'The aggregation for {metric} for {nodeType} is not available.', - values: { - nodeType: options.nodeType, - metric: metric.type, - }, - }) - ); - } - return { ...aggs, ...aggregation }; - }, {}); -}; - -export const getDateHistogramOffset = (from: number, interval: string): string => { - const fromInSeconds = Math.floor(from / 1000); - const bucketSizeInSeconds = getIntervalInSeconds(interval); - - // negative offset to align buckets with full intervals (e.g. minutes) - const offset = (fromInSeconds % bucketSizeInSeconds) - bucketSizeInSeconds; - return `${offset}s`; -}; diff --git a/x-pack/plugins/infra/server/lib/snapshot/response_helpers.test.ts b/x-pack/plugins/infra/server/lib/snapshot/response_helpers.test.ts deleted file mode 100644 index 74840afc157d2..0000000000000 --- a/x-pack/plugins/infra/server/lib/snapshot/response_helpers.test.ts +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { - isIPv4, - getIPFromBucket, - InfraSnapshotNodeGroupByBucket, - getMetricValueFromBucket, - InfraSnapshotMetricsBucket, -} from './response_helpers'; - -describe('InfraOps ResponseHelpers', () => { - describe('isIPv4', () => { - it('should return true for IPv4', () => { - expect(isIPv4('192.168.2.4')).toBe(true); - }); - it('should return false for anything else', () => { - expect(isIPv4('0:0:0:0:0:0:0:1')).toBe(false); - }); - }); - - describe('getIPFromBucket', () => { - it('should return IPv4 address', () => { - const bucket: InfraSnapshotNodeGroupByBucket = { - key: { - id: 'example-01', - name: 'example-01', - }, - ip: { - hits: { - total: { value: 1 }, - hits: [ - { - _index: 'metricbeat-2019-01-01', - _type: '_doc', - _id: '29392939', - _score: null, - sort: [], - _source: { - host: { - ip: ['2001:db8:85a3::8a2e:370:7334', '192.168.1.4'], - }, - }, - }, - ], - }, - }, - }; - expect(getIPFromBucket('host', bucket)).toBe('192.168.1.4'); - }); - it('should NOT return ipv6 address', () => { - const bucket: InfraSnapshotNodeGroupByBucket = { - key: { - id: 'example-01', - name: 'example-01', - }, - ip: { - hits: { - total: { value: 1 }, - hits: [ - { - _index: 'metricbeat-2019-01-01', - _type: '_doc', - _id: '29392939', - _score: null, - sort: [], - _source: { - host: { - ip: ['2001:db8:85a3::8a2e:370:7334'], - }, - }, - }, - ], - }, - }, - }; - expect(getIPFromBucket('host', bucket)).toBe(null); - }); - }); - - describe('getMetricValueFromBucket', () => { - it('should return the value of a bucket with data', () => { - expect(getMetricValueFromBucket('custom', testBucket, 1)).toBe(0.5); - }); - it('should return the normalized value of a bucket with data', () => { - expect(getMetricValueFromBucket('cpu', testNormalizedBucket, 1)).toBe(50); - }); - it('should return null for a bucket with no data', () => { - expect(getMetricValueFromBucket('custom', testEmptyBucket, 1)).toBe(null); - }); - }); -}); - -// Hack to get around TypeScript -const buckets = [ - { - key: 'a', - doc_count: 1, - custom_1: { - value: 0.5, - }, - }, - { - key: 'b', - doc_count: 1, - cpu: { - value: 0.5, - normalized_value: 50, - }, - }, - { - key: 'c', - doc_count: 0, - }, -] as InfraSnapshotMetricsBucket[]; -const [testBucket, testNormalizedBucket, testEmptyBucket] = buckets; diff --git a/x-pack/plugins/infra/server/lib/snapshot/response_helpers.ts b/x-pack/plugins/infra/server/lib/snapshot/response_helpers.ts deleted file mode 100644 index 2652e362b7eff..0000000000000 --- a/x-pack/plugins/infra/server/lib/snapshot/response_helpers.ts +++ /dev/null @@ -1,208 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { isNumber, last, max, sum, get } from 'lodash'; -import moment from 'moment'; - -import { MetricsExplorerSeries } from '../../../common/http_api/metrics_explorer'; -import { getIntervalInSeconds } from '../../utils/get_interval_in_seconds'; -import { InfraSnapshotRequestOptions } from './types'; -import { findInventoryModel } from '../../../common/inventory_models'; -import { InventoryItemType, SnapshotMetricType } from '../../../common/inventory_models/types'; -import { SnapshotNodeMetric, SnapshotNodePath } from '../../../common/http_api/snapshot_api'; - -export interface InfraSnapshotNodeMetricsBucket { - key: { id: string }; - histogram: { - buckets: InfraSnapshotMetricsBucket[]; - }; -} - -// Jumping through TypeScript hoops here: -// We need an interface that has the known members 'key' and 'doc_count' and also -// an unknown number of members with unknown names but known format, containing the -// metrics. -// This union type is the only way I found to express this that TypeScript accepts. -export interface InfraSnapshotBucketWithKey { - key: string | number; - doc_count: number; -} - -export interface InfraSnapshotBucketWithValues { - [name: string]: { value: number; normalized_value?: number }; -} - -export type InfraSnapshotMetricsBucket = InfraSnapshotBucketWithKey & InfraSnapshotBucketWithValues; - -interface InfraSnapshotIpHit { - _index: string; - _type: string; - _id: string; - _score: number | null; - _source: { - host: { - ip: string[] | string; - }; - }; - sort: number[]; -} - -export interface InfraSnapshotNodeGroupByBucket { - key: { - id: string; - name: string; - [groupByField: string]: string; - }; - ip: { - hits: { - total: { value: number }; - hits: InfraSnapshotIpHit[]; - }; - }; -} - -export const isIPv4 = (subject: string) => /^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$/.test(subject); - -export const getIPFromBucket = ( - nodeType: InventoryItemType, - bucket: InfraSnapshotNodeGroupByBucket -): string | null => { - const inventoryModel = findInventoryModel(nodeType); - if (!inventoryModel.fields.ip) { - return null; - } - const ip = get(bucket, `ip.hits.hits[0]._source.${inventoryModel.fields.ip}`, null) as - | string[] - | null; - if (Array.isArray(ip)) { - return ip.find(isIPv4) || null; - } else if (typeof ip === 'string') { - return ip; - } - - return null; -}; - -export const getNodePath = ( - groupBucket: InfraSnapshotNodeGroupByBucket, - options: InfraSnapshotRequestOptions -): SnapshotNodePath[] => { - const node = groupBucket.key; - const path = options.groupBy.map((gb) => { - return { value: node[`${gb.field}`], label: node[`${gb.field}`] } as SnapshotNodePath; - }); - const ip = getIPFromBucket(options.nodeType, groupBucket); - path.push({ value: node.id, label: node.name || node.id, ip }); - return path; -}; - -interface NodeMetricsForLookup { - [nodeId: string]: InfraSnapshotMetricsBucket[]; -} - -export const getNodeMetricsForLookup = ( - metrics: InfraSnapshotNodeMetricsBucket[] -): NodeMetricsForLookup => { - return metrics.reduce((acc: NodeMetricsForLookup, metric) => { - acc[`${metric.key.id}`] = metric.histogram.buckets; - return acc; - }, {}); -}; - -// In the returned object, -// value contains the value from the last bucket spanning a full interval -// max and avg are calculated from all buckets returned for the timerange -export const getNodeMetrics = ( - nodeBuckets: InfraSnapshotMetricsBucket[], - options: InfraSnapshotRequestOptions -): SnapshotNodeMetric[] => { - if (!nodeBuckets) { - return options.metrics.map((metric) => ({ - name: metric.type, - value: null, - max: null, - avg: null, - })); - } - const lastBucket = findLastFullBucket(nodeBuckets, options); - if (!lastBucket) return []; - return options.metrics.map((metric, index) => { - const metricResult: SnapshotNodeMetric = { - name: metric.type, - value: getMetricValueFromBucket(metric.type, lastBucket, index), - max: calculateMax(nodeBuckets, metric.type, index), - avg: calculateAvg(nodeBuckets, metric.type, index), - }; - if (options.includeTimeseries) { - metricResult.timeseries = getTimeseriesData(nodeBuckets, metric.type, index); - } - return metricResult; - }); -}; - -const findLastFullBucket = ( - buckets: InfraSnapshotMetricsBucket[], - options: InfraSnapshotRequestOptions -) => { - const to = moment.utc(options.timerange.to); - const bucketSize = getIntervalInSeconds(options.timerange.interval); - return buckets.reduce((current, item) => { - const itemKey = isNumber(item.key) ? item.key : parseInt(item.key, 10); - const date = moment.utc(itemKey + bucketSize * 1000); - if (!date.isAfter(to) && item.doc_count > 0) { - return item; - } - return current; - }, last(buckets)); -}; - -export const getMetricValueFromBucket = ( - type: SnapshotMetricType, - bucket: InfraSnapshotMetricsBucket, - index: number -) => { - const key = type === 'custom' ? `custom_${index}` : type; - const metric = bucket[key]; - const value = metric && (metric.normalized_value || metric.value); - return isFinite(value) ? value : null; -}; - -function calculateMax( - buckets: InfraSnapshotMetricsBucket[], - type: SnapshotMetricType, - index: number -) { - return max(buckets.map((bucket) => getMetricValueFromBucket(type, bucket, index))) || 0; -} - -function calculateAvg( - buckets: InfraSnapshotMetricsBucket[], - type: SnapshotMetricType, - index: number -) { - return ( - sum(buckets.map((bucket) => getMetricValueFromBucket(type, bucket, index))) / buckets.length || - 0 - ); -} - -function getTimeseriesData( - buckets: InfraSnapshotMetricsBucket[], - type: SnapshotMetricType, - index: number -): MetricsExplorerSeries { - return { - id: type, - columns: [ - { name: 'timestamp', type: 'date' }, - { name: 'metric_0', type: 'number' }, - ], - rows: buckets.map((bucket) => ({ - timestamp: bucket.key as number, - metric_0: getMetricValueFromBucket(type, bucket, index), - })), - }; -} diff --git a/x-pack/plugins/infra/server/lib/snapshot/snapshot.ts b/x-pack/plugins/infra/server/lib/snapshot/snapshot.ts deleted file mode 100644 index 33d8e738a717e..0000000000000 --- a/x-pack/plugins/infra/server/lib/snapshot/snapshot.ts +++ /dev/null @@ -1,238 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -import { InfraDatabaseSearchResponse, CallWithRequestParams } from '../adapters/framework'; - -import { JsonObject } from '../../../common/typed_json'; -import { SNAPSHOT_COMPOSITE_REQUEST_SIZE } from './constants'; -import { - getGroupedNodesSources, - getMetricsAggregations, - getMetricsSources, - getDateHistogramOffset, -} from './query_helpers'; -import { - getNodeMetrics, - getNodeMetricsForLookup, - getNodePath, - InfraSnapshotNodeGroupByBucket, - InfraSnapshotNodeMetricsBucket, -} from './response_helpers'; -import { getAllCompositeData } from '../../utils/get_all_composite_data'; -import { createAfterKeyHandler } from '../../utils/create_afterkey_handler'; -import { findInventoryModel } from '../../../common/inventory_models'; -import { InfraSnapshotRequestOptions } from './types'; -import { createTimeRangeWithInterval } from './create_timerange_with_interval'; -import { SnapshotNode } from '../../../common/http_api/snapshot_api'; - -type NamedSnapshotNode = SnapshotNode & { name: string }; - -export type ESSearchClient = ( - options: CallWithRequestParams -) => Promise>; -export class InfraSnapshot { - public async getNodes( - client: ESSearchClient, - options: InfraSnapshotRequestOptions - ): Promise<{ nodes: NamedSnapshotNode[]; interval: string }> { - // Both requestGroupedNodes and requestNodeMetrics may send several requests to elasticsearch - // in order to page through the results of their respective composite aggregations. - // Both chains of requests are supposed to run in parallel, and their results be merged - // when they have both been completed. - const timeRangeWithIntervalApplied = await createTimeRangeWithInterval(client, options); - const optionsWithTimerange = { ...options, timerange: timeRangeWithIntervalApplied }; - - const groupedNodesPromise = requestGroupedNodes(client, optionsWithTimerange); - const nodeMetricsPromise = requestNodeMetrics(client, optionsWithTimerange); - const [groupedNodeBuckets, nodeMetricBuckets] = await Promise.all([ - groupedNodesPromise, - nodeMetricsPromise, - ]); - return { - nodes: mergeNodeBuckets(groupedNodeBuckets, nodeMetricBuckets, options), - interval: timeRangeWithIntervalApplied.interval, - }; - } -} - -const bucketSelector = ( - response: InfraDatabaseSearchResponse<{}, InfraSnapshotAggregationResponse> -) => (response.aggregations && response.aggregations.nodes.buckets) || []; - -const handleAfterKey = createAfterKeyHandler( - 'body.aggregations.nodes.composite.after', - (input) => input?.aggregations?.nodes?.after_key -); - -const callClusterFactory = (search: ESSearchClient) => (opts: any) => - search<{}, InfraSnapshotAggregationResponse>(opts); - -const requestGroupedNodes = async ( - client: ESSearchClient, - options: InfraSnapshotRequestOptions -): Promise => { - const inventoryModel = findInventoryModel(options.nodeType); - const query = { - allowNoIndices: true, - index: `${options.sourceConfiguration.logAlias},${options.sourceConfiguration.metricAlias}`, - ignoreUnavailable: true, - body: { - query: { - bool: { - filter: buildFilters(options), - }, - }, - size: 0, - aggregations: { - nodes: { - composite: { - size: options.overrideCompositeSize || SNAPSHOT_COMPOSITE_REQUEST_SIZE, - sources: getGroupedNodesSources(options), - }, - aggs: { - ip: { - top_hits: { - sort: [{ [options.sourceConfiguration.fields.timestamp]: { order: 'desc' } }], - _source: { - includes: inventoryModel.fields.ip ? [inventoryModel.fields.ip] : [], - }, - size: 1, - }, - }, - }, - }, - }, - }, - }; - return getAllCompositeData( - callClusterFactory(client), - query, - bucketSelector, - handleAfterKey - ); -}; - -const calculateIndexPatterBasedOnMetrics = (options: InfraSnapshotRequestOptions) => { - const { metrics } = options; - if (metrics.every((m) => m.type === 'logRate')) { - return options.sourceConfiguration.logAlias; - } - if (metrics.some((m) => m.type === 'logRate')) { - return `${options.sourceConfiguration.logAlias},${options.sourceConfiguration.metricAlias}`; - } - return options.sourceConfiguration.metricAlias; -}; - -const requestNodeMetrics = async ( - client: ESSearchClient, - options: InfraSnapshotRequestOptions -): Promise => { - const index = calculateIndexPatterBasedOnMetrics(options); - const query = { - allowNoIndices: true, - index, - ignoreUnavailable: true, - body: { - query: { - bool: { - filter: buildFilters(options, false), - }, - }, - size: 0, - aggregations: { - nodes: { - composite: { - size: options.overrideCompositeSize || SNAPSHOT_COMPOSITE_REQUEST_SIZE, - sources: getMetricsSources(options), - }, - aggregations: { - histogram: { - date_histogram: { - field: options.sourceConfiguration.fields.timestamp, - interval: options.timerange.interval || '1m', - offset: getDateHistogramOffset(options.timerange.from, options.timerange.interval), - extended_bounds: { - min: options.timerange.from, - max: options.timerange.to, - }, - }, - aggregations: getMetricsAggregations(options), - }, - }, - }, - }, - }, - }; - return getAllCompositeData( - callClusterFactory(client), - query, - bucketSelector, - handleAfterKey - ); -}; - -// buckets can be InfraSnapshotNodeGroupByBucket[] or InfraSnapshotNodeMetricsBucket[] -// but typing this in a way that makes TypeScript happy is unreadable (if possible at all) -interface InfraSnapshotAggregationResponse { - nodes: { - buckets: any[]; - after_key: { [id: string]: string }; - }; -} - -const mergeNodeBuckets = ( - nodeGroupByBuckets: InfraSnapshotNodeGroupByBucket[], - nodeMetricsBuckets: InfraSnapshotNodeMetricsBucket[], - options: InfraSnapshotRequestOptions -): NamedSnapshotNode[] => { - const nodeMetricsForLookup = getNodeMetricsForLookup(nodeMetricsBuckets); - - return nodeGroupByBuckets.map((node) => { - return { - name: node.key.name || node.key.id, // For type safety; name can be derived from getNodePath but not in a TS-friendly way - path: getNodePath(node, options), - metrics: getNodeMetrics(nodeMetricsForLookup[node.key.id], options), - }; - }); -}; - -const createQueryFilterClauses = (filterQuery: JsonObject | undefined) => - filterQuery ? [filterQuery] : []; - -const buildFilters = (options: InfraSnapshotRequestOptions, withQuery = true) => { - let filters: any = [ - { - range: { - [options.sourceConfiguration.fields.timestamp]: { - gte: options.timerange.from, - lte: options.timerange.to, - format: 'epoch_millis', - }, - }, - }, - ]; - - if (withQuery) { - filters = [...createQueryFilterClauses(options.filterQuery), ...filters]; - } - - if (options.accountId) { - filters.push({ - term: { - 'cloud.account.id': options.accountId, - }, - }); - } - - if (options.region) { - filters.push({ - term: { - 'cloud.region': options.region, - }, - }); - } - - return filters; -}; diff --git a/x-pack/plugins/infra/server/lib/snapshot/types.ts b/x-pack/plugins/infra/server/lib/snapshot/types.ts deleted file mode 100644 index 7e17cb91c6a59..0000000000000 --- a/x-pack/plugins/infra/server/lib/snapshot/types.ts +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { JsonObject } from '../../../common/typed_json'; -import { InfraSourceConfiguration } from '../../../common/graphql/types'; -import { SnapshotRequest } from '../../../common/http_api/snapshot_api'; - -export interface InfraSnapshotRequestOptions - extends Omit { - sourceConfiguration: InfraSourceConfiguration; - filterQuery: JsonObject | undefined; -} diff --git a/x-pack/plugins/infra/server/lib/sources/has_data.ts b/x-pack/plugins/infra/server/lib/sources/has_data.ts index 79b1375059dcb..53297640e541d 100644 --- a/x-pack/plugins/infra/server/lib/sources/has_data.ts +++ b/x-pack/plugins/infra/server/lib/sources/has_data.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { ESSearchClient } from '../snapshot'; +import { ESSearchClient } from '../metrics/types'; export const hasData = async (index: string, client: ESSearchClient) => { const params = { diff --git a/x-pack/plugins/infra/server/plugin.ts b/x-pack/plugins/infra/server/plugin.ts index 51f91d7189db7..90b73b9a7585a 100644 --- a/x-pack/plugins/infra/server/plugin.ts +++ b/x-pack/plugins/infra/server/plugin.ts @@ -19,7 +19,6 @@ import { InfraElasticsearchSourceStatusAdapter } from './lib/adapters/source_sta import { InfraFieldsDomain } from './lib/domains/fields_domain'; import { InfraLogEntriesDomain } from './lib/domains/log_entries_domain'; import { InfraMetricsDomain } from './lib/domains/metrics_domain'; -import { InfraSnapshot } from './lib/snapshot'; import { InfraSourceStatus } from './lib/source_status'; import { InfraSources } from './lib/sources'; import { InfraServerPluginDeps } from './lib/adapters/framework'; @@ -105,7 +104,6 @@ export class InfraServerPlugin { sources, } ); - const snapshot = new InfraSnapshot(); // register saved object types core.savedObjects.registerType(infraSourceConfigurationSavedObjectType); @@ -129,7 +127,6 @@ export class InfraServerPlugin { this.libs = { configuration: this.config, framework, - snapshot, sources, sourceStatus, ...domainLibs, diff --git a/x-pack/plugins/infra/server/routes/alerting/preview.ts b/x-pack/plugins/infra/server/routes/alerting/preview.ts index 5594323d706de..40d09dadfe050 100644 --- a/x-pack/plugins/infra/server/routes/alerting/preview.ts +++ b/x-pack/plugins/infra/server/routes/alerting/preview.ts @@ -82,7 +82,7 @@ export const initAlertPreviewRoute = ({ framework, sources }: InfraBackendLibs) callCluster, params: { criteria, filterQuery, nodeType }, lookback, - config: source.configuration, + source, alertInterval, }); diff --git a/x-pack/plugins/infra/server/routes/metrics_explorer/lib/find_interval_for_metrics.ts b/x-pack/plugins/infra/server/routes/metrics_explorer/lib/find_interval_for_metrics.ts index 876bbb4199441..8ab0f4a44c85d 100644 --- a/x-pack/plugins/infra/server/routes/metrics_explorer/lib/find_interval_for_metrics.ts +++ b/x-pack/plugins/infra/server/routes/metrics_explorer/lib/find_interval_for_metrics.ts @@ -7,9 +7,9 @@ import { uniq } from 'lodash'; import LRU from 'lru-cache'; import { MetricsExplorerRequestBody } from '../../../../common/http_api'; -import { ESSearchClient } from '../../../lib/snapshot'; import { getDatasetForField } from './get_dataset_for_field'; import { calculateMetricInterval } from '../../../utils/calculate_metric_interval'; +import { ESSearchClient } from '../../../lib/metrics/types'; const cache = new LRU({ max: 100, diff --git a/x-pack/plugins/infra/server/routes/metrics_explorer/lib/get_dataset_for_field.ts b/x-pack/plugins/infra/server/routes/metrics_explorer/lib/get_dataset_for_field.ts index 94e91d32b14bb..85bb5b106c87c 100644 --- a/x-pack/plugins/infra/server/routes/metrics_explorer/lib/get_dataset_for_field.ts +++ b/x-pack/plugins/infra/server/routes/metrics_explorer/lib/get_dataset_for_field.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { ESSearchClient } from '../../../lib/snapshot'; +import { ESSearchClient } from '../../../lib/metrics/types'; interface EventDatasetHit { _source: { diff --git a/x-pack/plugins/infra/server/routes/snapshot/index.ts b/x-pack/plugins/infra/server/routes/snapshot/index.ts index 00bc1e74ea871..3f09ae89bc97e 100644 --- a/x-pack/plugins/infra/server/routes/snapshot/index.ts +++ b/x-pack/plugins/infra/server/routes/snapshot/index.ts @@ -10,10 +10,10 @@ import { fold } from 'fp-ts/lib/Either'; import { identity } from 'fp-ts/lib/function'; import { InfraBackendLibs } from '../../lib/infra_types'; import { UsageCollector } from '../../usage/usage_collector'; -import { parseFilterQuery } from '../../utils/serialized_query'; import { SnapshotRequestRT, SnapshotNodeResponseRT } from '../../../common/http_api/snapshot_api'; import { throwErrors } from '../../../common/runtime_types'; import { createSearchClient } from '../../lib/create_search_client'; +import { getNodes } from './lib/get_nodes'; const escapeHatch = schema.object({}, { unknowns: 'allow' }); @@ -30,43 +30,22 @@ export const initSnapshotRoute = (libs: InfraBackendLibs) => { }, async (requestContext, request, response) => { try { - const { - filterQuery, - nodeType, - groupBy, - sourceId, - metrics, - timerange, - accountId, - region, - includeTimeseries, - overrideCompositeSize, - } = pipe( + const snapshotRequest = pipe( SnapshotRequestRT.decode(request.body), fold(throwErrors(Boom.badRequest), identity) ); + const source = await libs.sources.getSourceConfiguration( requestContext.core.savedObjects.client, - sourceId + snapshotRequest.sourceId ); - UsageCollector.countNode(nodeType); - const options = { - filterQuery: parseFilterQuery(filterQuery), - accountId, - region, - nodeType, - groupBy, - sourceConfiguration: source.configuration, - metrics, - timerange, - includeTimeseries, - overrideCompositeSize, - }; + UsageCollector.countNode(snapshotRequest.nodeType); const client = createSearchClient(requestContext, framework); - const nodesWithInterval = await libs.snapshot.getNodes(client, options); + const snapshotResponse = await getNodes(client, snapshotRequest, source); + return response.ok({ - body: SnapshotNodeResponseRT.encode(nodesWithInterval), + body: SnapshotNodeResponseRT.encode(snapshotResponse), }); } catch (error) { return response.internalError({ diff --git a/x-pack/plugins/infra/server/routes/snapshot/lib/apply_metadata_to_last_path.ts b/x-pack/plugins/infra/server/routes/snapshot/lib/apply_metadata_to_last_path.ts new file mode 100644 index 0000000000000..f41d76bbc156f --- /dev/null +++ b/x-pack/plugins/infra/server/routes/snapshot/lib/apply_metadata_to_last_path.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { get, last, first, isArray } from 'lodash'; +import { findInventoryFields } from '../../../../common/inventory_models'; +import { + SnapshotRequest, + SnapshotNodePath, + SnapshotNode, + MetricsAPISeries, + MetricsAPIRow, +} from '../../../../common/http_api'; +import { META_KEY } from './constants'; +import { InfraSource } from '../../../lib/sources'; + +export const isIPv4 = (subject: string) => /^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$/.test(subject); + +type RowWithMetadata = MetricsAPIRow & { + [META_KEY]: object[]; +}; + +export const applyMetadataToLastPath = ( + series: MetricsAPISeries, + node: SnapshotNode, + snapshotRequest: SnapshotRequest, + source: InfraSource +): SnapshotNodePath[] => { + // First we need to find a row with metadata + const rowWithMeta = series.rows.find( + (row) => (row[META_KEY] && isArray(row[META_KEY]) && (row[META_KEY] as object[]).length) || 0 + ) as RowWithMetadata | undefined; + + if (rowWithMeta) { + // We need just the first doc, there should only be one + const firstMetaDoc = first(rowWithMeta[META_KEY]); + // We also need the last path to add the metadata to + const lastPath = last(node.path); + if (firstMetaDoc && lastPath) { + // We will need the inventory fields so we can use the field paths to get + // the values from the metadata document + const inventoryFields = findInventoryFields( + snapshotRequest.nodeType, + source.configuration.fields + ); + // Set the label as the name and fallback to the id OR path.value + lastPath.label = get(firstMetaDoc, inventoryFields.name, lastPath.value); + // If the inventory fields contain an ip address, we need to try and set that + // on the path object. IP addersses are typically stored as multiple fields. We will + // use the first IPV4 address we find. + if (inventoryFields.ip) { + const ipAddresses = get(firstMetaDoc, inventoryFields.ip) as string[]; + if (Array.isArray(ipAddresses)) { + lastPath.ip = ipAddresses.find(isIPv4) || null; + } else if (typeof ipAddresses === 'string') { + lastPath.ip = ipAddresses; + } + } + return [...node.path.slice(0, node.path.length - 1), lastPath]; + } + } + return node.path; +}; diff --git a/x-pack/plugins/infra/server/routes/snapshot/lib/calculate_index_pattern_based_on_metrics.ts b/x-pack/plugins/infra/server/routes/snapshot/lib/calculate_index_pattern_based_on_metrics.ts new file mode 100644 index 0000000000000..4218aecfe74a8 --- /dev/null +++ b/x-pack/plugins/infra/server/routes/snapshot/lib/calculate_index_pattern_based_on_metrics.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SnapshotRequest } from '../../../../common/http_api'; +import { InfraSource } from '../../../lib/sources'; + +export const calculateIndexPatterBasedOnMetrics = ( + options: SnapshotRequest, + source: InfraSource +) => { + const { metrics } = options; + if (metrics.every((m) => m.type === 'logRate')) { + return source.configuration.logAlias; + } + if (metrics.some((m) => m.type === 'logRate')) { + return `${source.configuration.logAlias},${source.configuration.metricAlias}`; + } + return source.configuration.metricAlias; +}; diff --git a/x-pack/plugins/infra/server/lib/snapshot/index.ts b/x-pack/plugins/infra/server/routes/snapshot/lib/constants.ts similarity index 85% rename from x-pack/plugins/infra/server/lib/snapshot/index.ts rename to x-pack/plugins/infra/server/routes/snapshot/lib/constants.ts index 8db54da803648..563c720224435 100644 --- a/x-pack/plugins/infra/server/lib/snapshot/index.ts +++ b/x-pack/plugins/infra/server/routes/snapshot/lib/constants.ts @@ -4,4 +4,4 @@ * you may not use this file except in compliance with the Elastic License. */ -export * from './snapshot'; +export const META_KEY = '__metadata__'; diff --git a/x-pack/plugins/infra/server/routes/snapshot/lib/copy_missing_metrics.ts b/x-pack/plugins/infra/server/routes/snapshot/lib/copy_missing_metrics.ts new file mode 100644 index 0000000000000..36397862e4153 --- /dev/null +++ b/x-pack/plugins/infra/server/routes/snapshot/lib/copy_missing_metrics.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { memoize, last, first } from 'lodash'; +import { SnapshotNode, SnapshotNodeResponse } from '../../../../common/http_api'; + +const createMissingMetricFinder = (nodes: SnapshotNode[]) => + memoize((id: string) => { + const nodeWithMetrics = nodes.find((node) => { + const lastPath = last(node.path); + const metric = first(node.metrics); + return lastPath && metric && lastPath.value === id && metric.value !== null; + }); + if (nodeWithMetrics) { + return nodeWithMetrics.metrics; + } + }); + +/** + * This function will look for nodes with missing data and try to find a node to copy the data from. + * This functionality exists to suppor the use case where the user requests a group by on "Service type". + * Since that grouping naturally excludeds every metric (except the metric for the service.type), we still + * want to display the node with a value. A good example is viewing hosts by CPU Usage and grouping by service + * Without this every service but `system` would be null. + */ +export const copyMissingMetrics = (response: SnapshotNodeResponse) => { + const { nodes } = response; + const find = createMissingMetricFinder(nodes); + const newNodes = nodes.map((node) => { + const lastPath = last(node.path); + const metric = first(node.metrics); + const allRowsNull = metric?.timeseries?.rows.every((r) => r.metric_0 == null) ?? true; + if (lastPath && metric && metric.value === null && allRowsNull) { + const newMetrics = find(lastPath.value); + if (newMetrics) { + return { ...node, metrics: newMetrics }; + } + } + return node; + }); + return { ...response, nodes: newNodes }; +}; diff --git a/x-pack/plugins/infra/server/lib/snapshot/create_timerange_with_interval.ts b/x-pack/plugins/infra/server/routes/snapshot/lib/create_timerange_with_interval.ts similarity index 80% rename from x-pack/plugins/infra/server/lib/snapshot/create_timerange_with_interval.ts rename to x-pack/plugins/infra/server/routes/snapshot/lib/create_timerange_with_interval.ts index 719ffdb8fa7c4..827e0901c1c01 100644 --- a/x-pack/plugins/infra/server/lib/snapshot/create_timerange_with_interval.ts +++ b/x-pack/plugins/infra/server/routes/snapshot/lib/create_timerange_with_interval.ts @@ -5,14 +5,16 @@ */ import { uniq } from 'lodash'; -import { InfraSnapshotRequestOptions } from './types'; -import { getMetricsAggregations } from './query_helpers'; -import { calculateMetricInterval } from '../../utils/calculate_metric_interval'; -import { MetricsUIAggregation, ESBasicMetricAggRT } from '../../../common/inventory_models/types'; -import { getDatasetForField } from '../../routes/metrics_explorer/lib/get_dataset_for_field'; -import { InfraTimerangeInput } from '../../../common/http_api/snapshot_api'; -import { ESSearchClient } from '.'; -import { getIntervalInSeconds } from '../../utils/get_interval_in_seconds'; +import { InfraTimerangeInput } from '../../../../common/http_api'; +import { ESSearchClient } from '../../../lib/metrics/types'; +import { getIntervalInSeconds } from '../../../utils/get_interval_in_seconds'; +import { calculateMetricInterval } from '../../../utils/calculate_metric_interval'; +import { getMetricsAggregations, InfraSnapshotRequestOptions } from './get_metrics_aggregations'; +import { + MetricsUIAggregation, + ESBasicMetricAggRT, +} from '../../../../common/inventory_models/types'; +import { getDatasetForField } from '../../metrics_explorer/lib/get_dataset_for_field'; const createInterval = async (client: ESSearchClient, options: InfraSnapshotRequestOptions) => { const { timerange } = options; diff --git a/x-pack/plugins/infra/server/routes/snapshot/lib/get_metrics_aggregations.ts b/x-pack/plugins/infra/server/routes/snapshot/lib/get_metrics_aggregations.ts new file mode 100644 index 0000000000000..2421469eb1bdd --- /dev/null +++ b/x-pack/plugins/infra/server/routes/snapshot/lib/get_metrics_aggregations.ts @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; +import { JsonObject } from '../../../../common/typed_json'; +import { + InventoryItemType, + MetricsUIAggregation, + MetricsUIAggregationRT, +} from '../../../../common/inventory_models/types'; +import { + SnapshotMetricInput, + SnapshotCustomMetricInputRT, + SnapshotRequest, +} from '../../../../common/http_api'; +import { findInventoryModel } from '../../../../common/inventory_models'; +import { networkTraffic } from '../../../../common/inventory_models/shared/metrics/snapshot/network_traffic'; +import { InfraSourceConfiguration } from '../../../lib/sources'; + +export interface InfraSnapshotRequestOptions + extends Omit { + sourceConfiguration: InfraSourceConfiguration; + filterQuery: JsonObject | undefined; +} + +export const metricToAggregation = ( + nodeType: InventoryItemType, + metric: SnapshotMetricInput, + index: number +) => { + const inventoryModel = findInventoryModel(nodeType); + if (SnapshotCustomMetricInputRT.is(metric)) { + if (metric.aggregation === 'rate') { + return networkTraffic(`custom_${index}`, metric.field); + } + return { + [`custom_${index}`]: { + [metric.aggregation]: { + field: metric.field, + }, + }, + }; + } + return inventoryModel.metrics.snapshot?.[metric.type]; +}; + +export const getMetricsAggregations = ( + options: InfraSnapshotRequestOptions +): MetricsUIAggregation => { + const { metrics } = options; + return metrics.reduce((aggs, metric, index) => { + const aggregation = metricToAggregation(options.nodeType, metric, index); + if (!MetricsUIAggregationRT.is(aggregation)) { + throw new Error( + i18n.translate('xpack.infra.snapshot.missingSnapshotMetricError', { + defaultMessage: 'The aggregation for {metric} for {nodeType} is not available.', + values: { + nodeType: options.nodeType, + metric: metric.type, + }, + }) + ); + } + return { ...aggs, ...aggregation }; + }, {}); +}; diff --git a/x-pack/plugins/infra/server/routes/snapshot/lib/get_nodes.ts b/x-pack/plugins/infra/server/routes/snapshot/lib/get_nodes.ts new file mode 100644 index 0000000000000..9332d5aee1f52 --- /dev/null +++ b/x-pack/plugins/infra/server/routes/snapshot/lib/get_nodes.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SnapshotRequest } from '../../../../common/http_api'; +import { ESSearchClient } from '../../../lib/metrics/types'; +import { InfraSource } from '../../../lib/sources'; +import { transformRequestToMetricsAPIRequest } from './transform_request_to_metrics_api_request'; +import { queryAllData } from './query_all_data'; +import { transformMetricsApiResponseToSnapshotResponse } from './trasform_metrics_ui_response'; +import { copyMissingMetrics } from './copy_missing_metrics'; + +export const getNodes = async ( + client: ESSearchClient, + snapshotRequest: SnapshotRequest, + source: InfraSource +) => { + const metricsApiRequest = await transformRequestToMetricsAPIRequest( + client, + source, + snapshotRequest + ); + const metricsApiResponse = await queryAllData(client, metricsApiRequest); + return copyMissingMetrics( + transformMetricsApiResponseToSnapshotResponse( + metricsApiRequest, + snapshotRequest, + source, + metricsApiResponse + ) + ); +}; diff --git a/x-pack/plugins/infra/server/routes/snapshot/lib/query_all_data.ts b/x-pack/plugins/infra/server/routes/snapshot/lib/query_all_data.ts new file mode 100644 index 0000000000000..a9d2352cf55b7 --- /dev/null +++ b/x-pack/plugins/infra/server/routes/snapshot/lib/query_all_data.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { MetricsAPIRequest, MetricsAPIResponse } from '../../../../common/http_api'; +import { ESSearchClient } from '../../../lib/metrics/types'; +import { query } from '../../../lib/metrics'; + +const handleResponse = ( + client: ESSearchClient, + options: MetricsAPIRequest, + previousResponse?: MetricsAPIResponse +) => async (resp: MetricsAPIResponse): Promise => { + const combinedResponse = previousResponse + ? { + ...previousResponse, + series: [...previousResponse.series, ...resp.series], + info: resp.info, + } + : resp; + if (resp.info.afterKey) { + return query(client, { ...options, afterKey: resp.info.afterKey }).then( + handleResponse(client, options, combinedResponse) + ); + } + return combinedResponse; +}; + +export const queryAllData = (client: ESSearchClient, options: MetricsAPIRequest) => { + return query(client, options).then(handleResponse(client, options)); +}; diff --git a/x-pack/plugins/infra/server/routes/snapshot/lib/transform_request_to_metrics_api_request.ts b/x-pack/plugins/infra/server/routes/snapshot/lib/transform_request_to_metrics_api_request.ts new file mode 100644 index 0000000000000..700f4ef39bb66 --- /dev/null +++ b/x-pack/plugins/infra/server/routes/snapshot/lib/transform_request_to_metrics_api_request.ts @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { findInventoryFields } from '../../../../common/inventory_models'; +import { MetricsAPIRequest, SnapshotRequest } from '../../../../common/http_api'; +import { ESSearchClient } from '../../../lib/metrics/types'; +import { InfraSource } from '../../../lib/sources'; +import { createTimeRangeWithInterval } from './create_timerange_with_interval'; +import { parseFilterQuery } from '../../../utils/serialized_query'; +import { transformSnapshotMetricsToMetricsAPIMetrics } from './transform_snapshot_metrics_to_metrics_api_metrics'; +import { calculateIndexPatterBasedOnMetrics } from './calculate_index_pattern_based_on_metrics'; +import { META_KEY } from './constants'; + +export const transformRequestToMetricsAPIRequest = async ( + client: ESSearchClient, + source: InfraSource, + snapshotRequest: SnapshotRequest +): Promise => { + const timeRangeWithIntervalApplied = await createTimeRangeWithInterval(client, { + ...snapshotRequest, + filterQuery: parseFilterQuery(snapshotRequest.filterQuery), + sourceConfiguration: source.configuration, + }); + + const metricsApiRequest: MetricsAPIRequest = { + indexPattern: calculateIndexPatterBasedOnMetrics(snapshotRequest, source), + timerange: { + field: source.configuration.fields.timestamp, + from: timeRangeWithIntervalApplied.from, + to: timeRangeWithIntervalApplied.to, + interval: timeRangeWithIntervalApplied.interval, + }, + metrics: transformSnapshotMetricsToMetricsAPIMetrics(snapshotRequest), + limit: snapshotRequest.overrideCompositeSize ? snapshotRequest.overrideCompositeSize : 10, + alignDataToEnd: true, + }; + + const filters = []; + const parsedFilters = parseFilterQuery(snapshotRequest.filterQuery); + if (parsedFilters) { + filters.push(parsedFilters); + } + + if (snapshotRequest.accountId) { + filters.push({ term: { 'cloud.account.id': snapshotRequest.accountId } }); + } + + if (snapshotRequest.region) { + filters.push({ term: { 'cloud.region': snapshotRequest.region } }); + } + + const inventoryFields = findInventoryFields( + snapshotRequest.nodeType, + source.configuration.fields + ); + const groupBy = snapshotRequest.groupBy.map((g) => g.field).filter(Boolean) as string[]; + metricsApiRequest.groupBy = [...groupBy, inventoryFields.id]; + + const metaAggregation = { + id: META_KEY, + aggregations: { + [META_KEY]: { + top_hits: { + size: 1, + _source: [inventoryFields.name], + sort: [{ [source.configuration.fields.timestamp]: 'desc' }], + }, + }, + }, + }; + if (inventoryFields.ip) { + metaAggregation.aggregations[META_KEY].top_hits._source.push(inventoryFields.ip); + } + metricsApiRequest.metrics.push(metaAggregation); + + if (filters.length) { + metricsApiRequest.filters = filters; + } + + return metricsApiRequest; +}; diff --git a/x-pack/plugins/infra/server/routes/snapshot/lib/transform_snapshot_metrics_to_metrics_api_metrics.ts b/x-pack/plugins/infra/server/routes/snapshot/lib/transform_snapshot_metrics_to_metrics_api_metrics.ts new file mode 100644 index 0000000000000..6f7c88eda5d7a --- /dev/null +++ b/x-pack/plugins/infra/server/routes/snapshot/lib/transform_snapshot_metrics_to_metrics_api_metrics.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { networkTraffic } from '../../../../common/inventory_models/shared/metrics/snapshot/network_traffic'; +import { findInventoryModel } from '../../../../common/inventory_models'; +import { + MetricsAPIMetric, + SnapshotRequest, + SnapshotCustomMetricInputRT, +} from '../../../../common/http_api'; + +export const transformSnapshotMetricsToMetricsAPIMetrics = ( + snapshotRequest: SnapshotRequest +): MetricsAPIMetric[] => { + return snapshotRequest.metrics.map((metric, index) => { + const inventoryModel = findInventoryModel(snapshotRequest.nodeType); + if (SnapshotCustomMetricInputRT.is(metric)) { + const customId = `custom_${index}`; + if (metric.aggregation === 'rate') { + return { id: customId, aggregations: networkTraffic(customId, metric.field) }; + } + return { + id: customId, + aggregations: { + [customId]: { + [metric.aggregation]: { + field: metric.field, + }, + }, + }, + }; + } + return { id: metric.type, aggregations: inventoryModel.metrics.snapshot?.[metric.type] }; + }); +}; diff --git a/x-pack/plugins/infra/server/routes/snapshot/lib/trasform_metrics_ui_response.ts b/x-pack/plugins/infra/server/routes/snapshot/lib/trasform_metrics_ui_response.ts new file mode 100644 index 0000000000000..309598d71c361 --- /dev/null +++ b/x-pack/plugins/infra/server/routes/snapshot/lib/trasform_metrics_ui_response.ts @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { get, max, sum, last, isNumber } from 'lodash'; +import { SnapshotMetricType } from '../../../../common/inventory_models/types'; +import { + MetricsAPIResponse, + SnapshotNodeResponse, + MetricsAPIRequest, + MetricsExplorerColumnType, + MetricsAPIRow, + SnapshotRequest, + SnapshotNodePath, + SnapshotNodeMetric, +} from '../../../../common/http_api'; +import { META_KEY } from './constants'; +import { InfraSource } from '../../../lib/sources'; +import { applyMetadataToLastPath } from './apply_metadata_to_last_path'; + +const getMetricValue = (row: MetricsAPIRow) => { + if (!isNumber(row.metric_0)) return null; + const value = row.metric_0; + return isFinite(value) ? value : null; +}; + +const calculateMax = (rows: MetricsAPIRow[]) => { + return max(rows.map(getMetricValue)) || 0; +}; + +const calculateAvg = (rows: MetricsAPIRow[]): number => { + return sum(rows.map(getMetricValue)) / rows.length || 0; +}; + +const getLastValue = (rows: MetricsAPIRow[]) => { + const row = last(rows); + if (!row) return null; + return getMetricValue(row); +}; + +export const transformMetricsApiResponseToSnapshotResponse = ( + options: MetricsAPIRequest, + snapshotRequest: SnapshotRequest, + source: InfraSource, + metricsApiResponse: MetricsAPIResponse +): SnapshotNodeResponse => { + const nodes = metricsApiResponse.series.map((series) => { + const node = { + metrics: options.metrics + .filter((m) => m.id !== META_KEY) + .map((metric) => { + const name = metric.id as SnapshotMetricType; + const timeseries = { + id: name, + columns: [ + { name: 'timestamp', type: 'date' as MetricsExplorerColumnType }, + { name: 'metric_0', type: 'number' as MetricsExplorerColumnType }, + ], + rows: series.rows.map((row) => { + return { timestamp: row.timestamp, metric_0: get(row, metric.id, null) }; + }), + }; + const maxValue = calculateMax(timeseries.rows); + const avg = calculateAvg(timeseries.rows); + const value = getLastValue(timeseries.rows); + const nodeMetric: SnapshotNodeMetric = { name, max: maxValue, value, avg }; + if (snapshotRequest.includeTimeseries) { + nodeMetric.timeseries = timeseries; + } + return nodeMetric; + }), + path: + series.keys?.map((key) => { + return { value: key, label: key } as SnapshotNodePath; + }) ?? [], + name: '', + }; + + const path = applyMetadataToLastPath(series, node, snapshotRequest, source); + const lastPath = last(path); + const name = (lastPath && lastPath.label) || 'N/A'; + return { ...node, path, name }; + }); + return { nodes, interval: `${metricsApiResponse.info.interval}s` }; +}; diff --git a/x-pack/plugins/infra/server/utils/calculate_metric_interval.ts b/x-pack/plugins/infra/server/utils/calculate_metric_interval.ts index a3d674b324ae8..6d16e045d26d5 100644 --- a/x-pack/plugins/infra/server/utils/calculate_metric_interval.ts +++ b/x-pack/plugins/infra/server/utils/calculate_metric_interval.ts @@ -8,7 +8,7 @@ import { findInventoryModel } from '../../common/inventory_models'; // import { KibanaFramework } from '../lib/adapters/framework/kibana_framework_adapter'; import { InventoryItemType } from '../../common/inventory_models/types'; -import { ESSearchClient } from '../lib/snapshot'; +import { ESSearchClient } from '../lib/metrics/types'; interface Options { indexPattern: string; diff --git a/x-pack/plugins/ingest_manager/common/openapi/spec_oas3.json b/x-pack/plugins/ingest_manager/common/openapi/spec_oas3.json index d75a914e080d7..b7856e6d57402 100644 --- a/x-pack/plugins/ingest_manager/common/openapi/spec_oas3.json +++ b/x-pack/plugins/ingest_manager/common/openapi/spec_oas3.json @@ -1425,11 +1425,13 @@ }, "icons": [ { - "src": "/package/coredns-1.0.1/img/icon.png", + "path": "/package/coredns-1.0.1/img/icon.png", + "src": "/img/icon.png", "size": "1800x1800" }, { - "src": "/package/coredns-1.0.1/img/icon.svg", + "path": "/package/coredns-1.0.1/img/icon.svg", + "src": "/img/icon.svg", "size": "255x144", "type": "image/svg+xml" } @@ -1704,7 +1706,8 @@ }, "icons": [ { - "src": "/package/endpoint/0.3.0/img/logo-endpoint-64-color.svg", + "path": "/package/endpoint/0.3.0/img/logo-endpoint-64-color.svg", + "src": "/img/logo-endpoint-64-color.svg", "size": "16x16", "type": "image/svg+xml" } @@ -2001,7 +2004,8 @@ "download": "/epr/aws/aws-0.0.3.tar.gz", "icons": [ { - "src": "/package/aws/0.0.3/img/logo_aws.svg", + "path": "/package/aws/0.0.3/img/logo_aws.svg", + "src": "/img/logo_aws.svg", "title": "logo aws", "size": "32x32", "type": "image/svg+xml" @@ -2019,7 +2023,8 @@ "download": "/epr/endpoint/endpoint-0.1.0.tar.gz", "icons": [ { - "src": "/package/endpoint/0.1.0/img/logo-endpoint-64-color.svg", + "path": "/package/endpoint/0.1.0/img/logo-endpoint-64-color.svg", + "src": "/img/logo-endpoint-64-color.svg", "size": "16x16", "type": "image/svg+xml" } @@ -2087,7 +2092,8 @@ "download": "/epr/log/log-0.9.0.tar.gz", "icons": [ { - "src": "/package/log/0.9.0/img/icon.svg", + "path": "/package/log/0.9.0/img/icon.svg", + "src": "/img/icon.svg", "type": "image/svg+xml" } ], @@ -2103,7 +2109,8 @@ "download": "/epr/longdocs/longdocs-1.0.4.tar.gz", "icons": [ { - "src": "/package/longdocs/1.0.4/img/icon.svg", + "path": "/package/longdocs/1.0.4/img/icon.svg", + "src": "/img/icon.svg", "type": "image/svg+xml" } ], @@ -2119,7 +2126,8 @@ "download": "/epr/metricsonly/metricsonly-2.0.1.tar.gz", "icons": [ { - "src": "/package/metricsonly/2.0.1/img/icon.svg", + "path": "/package/metricsonly/2.0.1/img/icon.svg", + "src": "/img/icon.svg", "type": "image/svg+xml" } ], @@ -2135,7 +2143,8 @@ "download": "/epr/multiversion/multiversion-1.1.0.tar.gz", "icons": [ { - "src": "/package/multiversion/1.1.0/img/icon.svg", + "path": "/package/multiversion/1.1.0/img/icon.svg", + "src": "/img/icon.svg", "type": "image/svg+xml" } ], @@ -2151,7 +2160,8 @@ "download": "/epr/mysql/mysql-0.1.0.tar.gz", "icons": [ { - "src": "/package/mysql/0.1.0/img/logo_mysql.svg", + "path": "/package/mysql/0.1.0/img/logo_mysql.svg", + "src": "/img/logo_mysql.svg", "title": "logo mysql", "size": "32x32", "type": "image/svg+xml" @@ -2169,7 +2179,8 @@ "download": "/epr/nginx/nginx-0.1.0.tar.gz", "icons": [ { - "src": "/package/nginx/0.1.0/img/logo_nginx.svg", + "path": "/package/nginx/0.1.0/img/logo_nginx.svg", + "src": "/img/logo_nginx.svg", "title": "logo nginx", "size": "32x32", "type": "image/svg+xml" @@ -2187,7 +2198,8 @@ "download": "/epr/redis/redis-0.1.0.tar.gz", "icons": [ { - "src": "/package/redis/0.1.0/img/logo_redis.svg", + "path": "/package/redis/0.1.0/img/logo_redis.svg", + "src": "/img/logo_redis.svg", "title": "logo redis", "size": "32x32", "type": "image/svg+xml" @@ -2205,7 +2217,8 @@ "download": "/epr/reference/reference-1.0.0.tar.gz", "icons": [ { - "src": "/package/reference/1.0.0/img/icon.svg", + "path": "/package/reference/1.0.0/img/icon.svg", + "src": "/img/icon.svg", "size": "32x32", "type": "image/svg+xml" } @@ -2222,7 +2235,8 @@ "download": "/epr/system/system-0.1.0.tar.gz", "icons": [ { - "src": "/package/system/0.1.0/img/system.svg", + "path": "/package/system/0.1.0/img/system.svg", + "src": "/img/system.svg", "title": "system", "size": "1000x1000", "type": "image/svg+xml" @@ -3913,11 +3927,20 @@ "src": { "type": "string" }, + "path": { + "type": "string" + }, "title": { "type": "string" + }, + "size": { + "type": "string" + }, + "type": { + "type": "string" } }, - "required": ["src"] + "required": ["src", "path"] } }, "icons": { diff --git a/x-pack/plugins/ingest_manager/common/types/models/epm.ts b/x-pack/plugins/ingest_manager/common/types/models/epm.ts index f083400997870..8bc5d9f7210b2 100644 --- a/x-pack/plugins/ingest_manager/common/types/models/epm.ts +++ b/x-pack/plugins/ingest_manager/common/types/models/epm.ts @@ -74,10 +74,8 @@ export interface RegistryPackage { } interface RegistryImage { - // https://github.com/elastic/package-registry/blob/master/util/package.go#L74 - // says src is potentially missing but I couldn't find any examples - // it seems like src should be required. How can you have an image with no reference to the content? src: string; + path: string; title?: string; size?: string; type?: string; diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_package_icon_type.ts b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_package_icon_type.ts index e5a7191372e9c..690ffdf46f704 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_package_icon_type.ts +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_package_icon_type.ts @@ -42,7 +42,7 @@ export const usePackageIconType = ({ const svgIcons = (paramIcons || iconList)?.filter( (iconDef) => iconDef.type === 'image/svg+xml' ); - const localIconSrc = Array.isArray(svgIcons) && svgIcons[0]?.src; + const localIconSrc = Array.isArray(svgIcons) && (svgIcons[0].path || svgIcons[0].src); if (localIconSrc) { CACHED_ICONS.set(pkgKey, toImage(localIconSrc)); setIconType(CACHED_ICONS.get(pkgKey) || ''); diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/screens/detail/screenshots.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/screens/detail/screenshots.tsx index d8388a71556d6..6326e9072be8e 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/screens/detail/screenshots.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/screens/detail/screenshots.tsx @@ -75,7 +75,7 @@ export function Screenshots(props: ScreenshotProps) { set image to same width. Will need to update if size changes. */} {'" "'}, + }} + /> + ), validations: [ { validator: emptyField( @@ -52,9 +58,15 @@ const fieldsConfig: FieldsConfig = { label: i18n.translate('xpack.ingestPipelines.pipelineEditor.kvForm.valueSplitFieldLabel', { defaultMessage: 'Value split', }), - helpText: i18n.translate('xpack.ingestPipelines.pipelineEditor.kvForm.valueSplitHelpText', { - defaultMessage: 'Regex pattern for splitting the key from the value within a key-value pair.', - }), + helpText: ( + {'"="'}, + }} + /> + ), validations: [ { validator: emptyField( @@ -75,8 +87,7 @@ const fieldsConfig: FieldsConfig = { defaultMessage: 'Include keys', }), helpText: i18n.translate('xpack.ingestPipelines.pipelineEditor.kvForm.includeKeysHelpText', { - defaultMessage: - 'List of keys to filter and insert into document. Defaults to including all keys.', + defaultMessage: 'List of extracted keys to include in the output. Defaults to all keys.', }), }, @@ -88,7 +99,7 @@ const fieldsConfig: FieldsConfig = { defaultMessage: 'Exclude keys', }), helpText: i18n.translate('xpack.ingestPipelines.pipelineEditor.kvForm.excludeKeysHelpText', { - defaultMessage: 'List of keys to exclude from document.', + defaultMessage: 'List of extracted keys to exclude from the output.', }), }, @@ -99,7 +110,7 @@ const fieldsConfig: FieldsConfig = { defaultMessage: 'Prefix', }), helpText: i18n.translate('xpack.ingestPipelines.pipelineEditor.kvForm.prefixHelpText', { - defaultMessage: 'Prefix to be added to extracted keys.', + defaultMessage: 'Prefix to add to extracted keys.', }), }, @@ -136,7 +147,7 @@ const fieldsConfig: FieldsConfig = { helpText: ( {'()'}, angle: <>, @@ -154,7 +165,7 @@ export const Kv: FunctionComponent = () => { <> @@ -166,8 +177,7 @@ export const Kv: FunctionComponent = () => { helpText={i18n.translate( 'xpack.ingestPipelines.pipelineEditor.kvForm.targetFieldHelpText', { - defaultMessage: - 'Field to insert the extracted keys into. Defaults to the root of the document.', + defaultMessage: 'Output field for the extracted fields. Defaults to the document root.', } )} /> diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/manage_processor_form/processors/lowercase.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/manage_processor_form/processors/lowercase.tsx index 9db313a05007f..0d8170338ea10 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/manage_processor_form/processors/lowercase.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/manage_processor_form/processors/lowercase.tsx @@ -6,8 +6,6 @@ import React, { FunctionComponent } from 'react'; import { i18n } from '@kbn/i18n'; -import { FormattedMessage } from '@kbn/i18n/react'; -import { EuiCode } from '@elastic/eui'; import { FieldNameField } from './common_fields/field_name_field'; import { TargetField } from './common_fields/target_field'; @@ -23,17 +21,7 @@ export const Lowercase: FunctionComponent = () => { )} /> - {'field'}, - }} - /> - } - /> + diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/manage_processor_form/processors/pipeline.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/manage_processor_form/processors/pipeline.tsx index c785cf935833d..57843e2411359 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/manage_processor_form/processors/pipeline.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/manage_processor_form/processors/pipeline.tsx @@ -27,7 +27,7 @@ const fieldsConfig: FieldsConfig = { helpText: i18n.translate( 'xpack.ingestPipelines.pipelineEditor.pipelineForm.pipelineNameFieldHelpText', { - defaultMessage: 'Name of the pipeline to execute.', + defaultMessage: 'Name of the ingest pipeline to run.', } ), validations: [ diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/manage_processor_form/processors/remove.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/manage_processor_form/processors/remove.tsx index 3e90ce2b76f7b..3ba1cdb0c802d 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/manage_processor_form/processors/remove.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/manage_processor_form/processors/remove.tsx @@ -29,7 +29,7 @@ const fieldsConfig: FieldsConfig = { defaultMessage: 'Fields', }), helpText: i18n.translate('xpack.ingestPipelines.pipelineEditor.removeForm.fieldNameHelpText', { - defaultMessage: 'Fields to be removed.', + defaultMessage: 'Fields to remove.', }), validations: [ { diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/manage_processor_form/processors/rename.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/manage_processor_form/processors/rename.tsx index 8b796d9664586..099e2bd2c80fb 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/manage_processor_form/processors/rename.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/manage_processor_form/processors/rename.tsx @@ -21,7 +21,7 @@ export const Rename: FunctionComponent = () => { @@ -31,7 +31,7 @@ export const Rename: FunctionComponent = () => { })} helpText={i18n.translate( 'xpack.ingestPipelines.pipelineEditor.renameForm.targetFieldHelpText', - { defaultMessage: 'Name of the new field.' } + { defaultMessage: 'New field name. This field cannot already exist.' } )} validations={[ { diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/manage_processor_form/processors/script.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/manage_processor_form/processors/script.tsx index ae0bbbb490ae9..de28f66766603 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/manage_processor_form/processors/script.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/manage_processor_form/processors/script.tsx @@ -32,7 +32,7 @@ const fieldsConfig: FieldsConfig = { helpText: i18n.translate( 'xpack.ingestPipelines.pipelineEditor.scriptForm.storedScriptIDFieldHelpText', { - defaultMessage: 'Stored script reference.', + defaultMessage: 'ID of the stored script to run.', } ), validations: [ @@ -55,7 +55,7 @@ const fieldsConfig: FieldsConfig = { helpText: i18n.translate( 'xpack.ingestPipelines.pipelineEditor.scriptForm.sourceFieldHelpText', { - defaultMessage: 'Script to be executed.', + defaultMessage: 'Inline script to run.', } ), validations: [ @@ -98,7 +98,7 @@ const fieldsConfig: FieldsConfig = { helpText: i18n.translate( 'xpack.ingestPipelines.pipelineEditor.scriptForm.paramsFieldHelpText', { - defaultMessage: 'Script parameters.', + defaultMessage: 'Named parameters passed to the script as variables.', } ), validations: [ @@ -128,7 +128,7 @@ export const Script: FormFieldsComponent = ({ initialFieldValues }) => { setShowId((v) => !v)} diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/manage_processor_form/processors/set.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/manage_processor_form/processors/set.tsx index c282be35e5071..04ea0c44c3513 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/manage_processor_form/processors/set.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/manage_processor_form/processors/set.tsx @@ -32,13 +32,13 @@ const fieldsConfig: FieldsConfig = { defaultMessage: 'Value', }), helpText: i18n.translate('xpack.ingestPipelines.pipelineEditor.setForm.valueFieldHelpText', { - defaultMessage: 'Value to be set for the field', + defaultMessage: 'Value for the field.', }), validations: [ { validator: emptyField( i18n.translate('xpack.ingestPipelines.pipelineEditor.setForm.valueRequiredError', { - defaultMessage: 'A value is required', + defaultMessage: 'A value is required.', }) ), }, @@ -53,9 +53,15 @@ const fieldsConfig: FieldsConfig = { label: i18n.translate('xpack.ingestPipelines.pipelineEditor.setForm.overrideFieldLabel', { defaultMessage: 'Override', }), - helpText: i18n.translate('xpack.ingestPipelines.pipelineEditor.setForm.overrideFieldHelpText', { - defaultMessage: 'If disabled, fields containing non-null values will not be updated.', - }), + helpText: ( + {'null'}, + }} + /> + ), }, ignore_empty_value: { type: FIELD_TYPES.TOGGLE, @@ -71,7 +77,8 @@ const fieldsConfig: FieldsConfig = { helpText: ( {'value'}, nullValue: {'null'}, @@ -89,7 +96,7 @@ export const SetProcessor: FunctionComponent = () => { <> diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/manage_processor_form/processors/set_security_user.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/manage_processor_form/processors/set_security_user.tsx index 78128b3d54c75..46bfe8c97ebea 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/manage_processor_form/processors/set_security_user.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/manage_processor_form/processors/set_security_user.tsx @@ -44,7 +44,7 @@ const fieldsConfig: FieldsConfig = { helpText: ( [{helpTextValues}], }} @@ -60,7 +60,7 @@ export const SetSecurityUser: FunctionComponent = () => { helpText={i18n.translate( 'xpack.ingestPipelines.pipelineEditor.setSecurityUserForm.fieldNameField', { - defaultMessage: 'Field to store the user information', + defaultMessage: 'Output field.', } )} /> diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/manage_processor_form/processors/sort.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/manage_processor_form/processors/sort.tsx index cdd0ff888accf..c8c0562011fd6 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/manage_processor_form/processors/sort.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/manage_processor_form/processors/sort.tsx @@ -24,7 +24,8 @@ const fieldsConfig: FieldsConfig = { defaultMessage: 'Order', }), helpText: i18n.translate('xpack.ingestPipelines.pipelineEditor.sortForm.orderFieldHelpText', { - defaultMessage: 'Sort order to use', + defaultMessage: + 'Sort order. Arrays containing a mix of strings and numbers are sorted lexicographically.', }), }, }; @@ -35,7 +36,7 @@ export const Sort: FunctionComponent = () => { diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/manage_processor_form/processors/split.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/manage_processor_form/processors/split.tsx index b48ce74110b39..fa178aaddd314 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/manage_processor_form/processors/split.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/manage_processor_form/processors/split.tsx @@ -33,7 +33,7 @@ const fieldsConfig: FieldsConfig = { helpText: i18n.translate( 'xpack.ingestPipelines.pipelineEditor.splitForm.separatorFieldHelpText', { - defaultMessage: 'Regex to match a separator', + defaultMessage: 'Regex pattern used to delimit the field value.', } ), validations: [ @@ -60,7 +60,7 @@ const fieldsConfig: FieldsConfig = { ), helpText: i18n.translate( 'xpack.ingestPipelines.pipelineEditor.splitForm.preserveTrailingFieldHelpText', - { defaultMessage: 'If enabled, preserve any trailing space.' } + { defaultMessage: 'Preserve any trailing whitespace in the split field values.' } ), }, }; @@ -71,7 +71,7 @@ export const Split: FunctionComponent = () => { diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/shared/map_processor_type_to_form.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/shared/map_processor_type_to_form.tsx index 59ec64944a3c9..9de371f8d0024 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/shared/map_processor_type_to_form.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/shared/map_processor_type_to_form.tsx @@ -107,7 +107,7 @@ export const mapProcessorTypeToDescriptor: MapProcessorTypeToDescriptor = { defaultMessage: 'CSV', }), description: i18n.translate('xpack.ingestPipelines.processors.description.csv', { - defaultMessage: 'Extracts fields values from CSV data.', + defaultMessage: 'Extracts field values from CSV data.', }), }, date: { @@ -306,7 +306,10 @@ export const mapProcessorTypeToDescriptor: MapProcessorTypeToDescriptor = { FieldsComponent: Kv, docLinkPath: '/kv-processor.html', label: i18n.translate('xpack.ingestPipelines.processors.label.kv', { - defaultMessage: 'KV', + defaultMessage: 'Key-value (KV)', + }), + description: i18n.translate('xpack.ingestPipelines.processors.description.kv', { + defaultMessage: 'Extracts fields from a string containing key-value pairs.', }), }, lowercase: { @@ -315,6 +318,9 @@ export const mapProcessorTypeToDescriptor: MapProcessorTypeToDescriptor = { label: i18n.translate('xpack.ingestPipelines.processors.label.lowercase', { defaultMessage: 'Lowercase', }), + description: i18n.translate('xpack.ingestPipelines.processors.description.lowercase', { + defaultMessage: 'Converts a string to lowercase.', + }), }, pipeline: { FieldsComponent: Pipeline, @@ -322,6 +328,9 @@ export const mapProcessorTypeToDescriptor: MapProcessorTypeToDescriptor = { label: i18n.translate('xpack.ingestPipelines.processors.label.pipeline', { defaultMessage: 'Pipeline', }), + description: i18n.translate('xpack.ingestPipelines.processors.description.pipeline', { + defaultMessage: 'Runs another ingest node pipeline.', + }), }, remove: { FieldsComponent: Remove, @@ -329,6 +338,9 @@ export const mapProcessorTypeToDescriptor: MapProcessorTypeToDescriptor = { label: i18n.translate('xpack.ingestPipelines.processors.label.remove', { defaultMessage: 'Remove', }), + description: i18n.translate('xpack.ingestPipelines.processors.description.remove', { + defaultMessage: 'Removes one or more fields.', + }), }, rename: { FieldsComponent: Rename, @@ -336,6 +348,9 @@ export const mapProcessorTypeToDescriptor: MapProcessorTypeToDescriptor = { label: i18n.translate('xpack.ingestPipelines.processors.label.rename', { defaultMessage: 'Rename', }), + description: i18n.translate('xpack.ingestPipelines.processors.description.rename', { + defaultMessage: 'Renames an existing field.', + }), }, script: { FieldsComponent: Script, @@ -343,6 +358,9 @@ export const mapProcessorTypeToDescriptor: MapProcessorTypeToDescriptor = { label: i18n.translate('xpack.ingestPipelines.processors.label.script', { defaultMessage: 'Script', }), + description: i18n.translate('xpack.ingestPipelines.processors.description.script', { + defaultMessage: 'Runs a script on incoming documents.', + }), }, set: { FieldsComponent: SetProcessor, @@ -350,6 +368,9 @@ export const mapProcessorTypeToDescriptor: MapProcessorTypeToDescriptor = { label: i18n.translate('xpack.ingestPipelines.processors.label.set', { defaultMessage: 'Set', }), + description: i18n.translate('xpack.ingestPipelines.processors.description.set', { + defaultMessage: 'Sets the value of a field.', + }), }, set_security_user: { FieldsComponent: SetSecurityUser, @@ -357,12 +378,9 @@ export const mapProcessorTypeToDescriptor: MapProcessorTypeToDescriptor = { label: i18n.translate('xpack.ingestPipelines.processors.label.setSecurityUser', { defaultMessage: 'Set security user', }), - }, - split: { - FieldsComponent: Split, - docLinkPath: '/split-processor.html', - label: i18n.translate('xpack.ingestPipelines.processors.label.split', { - defaultMessage: 'Split', + description: i18n.translate('xpack.ingestPipelines.processors.description.setSecurityUser', { + defaultMessage: + 'Adds details about the current user, such user name and email address, to incoming documents. Requires an authenticated user for the indexing request.', }), }, sort: { @@ -371,6 +389,19 @@ export const mapProcessorTypeToDescriptor: MapProcessorTypeToDescriptor = { label: i18n.translate('xpack.ingestPipelines.processors.label.sort', { defaultMessage: 'Sort', }), + description: i18n.translate('xpack.ingestPipelines.processors.description.sort', { + defaultMessage: "Sorts a field's array elements.", + }), + }, + split: { + FieldsComponent: Split, + docLinkPath: '/split-processor.html', + label: i18n.translate('xpack.ingestPipelines.processors.label.split', { + defaultMessage: 'Split', + }), + description: i18n.translate('xpack.ingestPipelines.processors.description.split', { + defaultMessage: 'Splits a field value into an array.', + }), }, trim: { FieldsComponent: undefined, // TODO: Implement diff --git a/x-pack/plugins/maps/public/classes/joins/inner_join.js b/x-pack/plugins/maps/public/classes/joins/inner_join.js index 76afe2430b818..75bf59d9d6404 100644 --- a/x-pack/plugins/maps/public/classes/joins/inner_join.js +++ b/x-pack/plugins/maps/public/classes/joins/inner_join.js @@ -94,8 +94,8 @@ export class InnerJoin { return this._descriptor; } - async filterAndFormatPropertiesForTooltip(properties) { - return await this._rightSource.filterAndFormatPropertiesToHtml(properties); + async getTooltipProperties(properties) { + return await this._rightSource.getTooltipProperties(properties); } getIndexPatternIds() { diff --git a/x-pack/plugins/maps/public/classes/layers/vector_layer/vector_layer.js b/x-pack/plugins/maps/public/classes/layers/vector_layer/vector_layer.js index c49d0044e6ad6..27c344b713a60 100644 --- a/x-pack/plugins/maps/public/classes/layers/vector_layer/vector_layer.js +++ b/x-pack/plugins/maps/public/classes/layers/vector_layer/vector_layer.js @@ -949,13 +949,11 @@ export class VectorLayer extends AbstractLayer { async getPropertiesForTooltip(properties) { const vectorSource = this.getSource(); - let allProperties = await vectorSource.filterAndFormatPropertiesToHtml(properties); + let allProperties = await vectorSource.getTooltipProperties(properties); this._addJoinsToSourceTooltips(allProperties); for (let i = 0; i < this.getJoins().length; i++) { - const propsFromJoin = await this.getJoins()[i].filterAndFormatPropertiesForTooltip( - properties - ); + const propsFromJoin = await this.getJoins()[i].getTooltipProperties(properties); allProperties = [...allProperties, ...propsFromJoin]; } return allProperties; diff --git a/x-pack/plugins/maps/public/classes/sources/ems_file_source/ems_file_source.test.tsx b/x-pack/plugins/maps/public/classes/sources/ems_file_source/ems_file_source.test.tsx index c5d6ced76b5c0..674ee832daab9 100644 --- a/x-pack/plugins/maps/public/classes/sources/ems_file_source/ems_file_source.test.tsx +++ b/x-pack/plugins/maps/public/classes/sources/ems_file_source/ems_file_source.test.tsx @@ -17,10 +17,10 @@ function makeEMSFileSource(tooltipProperties: string[]) { } describe('EMS file source', () => { - describe('filterAndFormatPropertiesToHtml', () => { + describe('getTooltipProperties', () => { it('should create tooltip-properties with human readable label', async () => { const mockEMSFileSource = makeEMSFileSource(['iso2']); - const out = await mockEMSFileSource.filterAndFormatPropertiesToHtml({ + const out = await mockEMSFileSource.getTooltipProperties({ iso2: 'US', }); @@ -33,7 +33,7 @@ describe('EMS file source', () => { it('should order tooltip-properties', async () => { const tooltipProperties = ['iso3', 'iso2', 'name']; const mockEMSFileSource = makeEMSFileSource(tooltipProperties); - const out = await mockEMSFileSource.filterAndFormatPropertiesToHtml({ + const out = await mockEMSFileSource.getTooltipProperties({ name: 'United States', iso3: 'USA', iso2: 'US', diff --git a/x-pack/plugins/maps/public/classes/sources/ems_file_source/ems_file_source.tsx b/x-pack/plugins/maps/public/classes/sources/ems_file_source/ems_file_source.tsx index f55a7434d1217..5f73a9e23431b 100644 --- a/x-pack/plugins/maps/public/classes/sources/ems_file_source/ems_file_source.tsx +++ b/x-pack/plugins/maps/public/classes/sources/ems_file_source/ems_file_source.tsx @@ -23,7 +23,6 @@ import { ITooltipProperty } from '../../tooltips/tooltip_property'; export interface IEmsFileSource extends IVectorSource { getEmsFieldLabel(emsFieldName: string): Promise; - createField({ fieldName }: { fieldName: string }): IField; } export const sourceTitle = i18n.translate('xpack.maps.source.emsFileTitle', { @@ -168,7 +167,7 @@ export class EMSFileSource extends AbstractVectorSource implements IEmsFileSourc return this._tooltipFields.length > 0; } - async filterAndFormatPropertiesToHtml(properties: unknown): Promise { + async getTooltipProperties(properties: unknown): Promise { const promises = this._tooltipFields.map((field) => { // @ts-ignore const value = properties[field.getName()]; diff --git a/x-pack/plugins/maps/public/classes/sources/es_agg_source/es_agg_source.d.ts b/x-pack/plugins/maps/public/classes/sources/es_agg_source/es_agg_source.d.ts deleted file mode 100644 index eb50cd7528c8b..0000000000000 --- a/x-pack/plugins/maps/public/classes/sources/es_agg_source/es_agg_source.d.ts +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { IESSource } from '../es_source'; -import { AbstractESSource } from '../es_source'; -import { AGG_TYPE } from '../../../../common/constants'; -import { IESAggField } from '../../fields/es_agg_field'; -import { AbstractESAggSourceDescriptor } from '../../../../common/descriptor_types'; - -export interface IESAggSource extends IESSource { - getAggKey(aggType: AGG_TYPE, fieldName: string): string; - getAggLabel(aggType: AGG_TYPE, fieldName: string): string; - getMetricFields(): IESAggField[]; - hasMatchingMetricField(fieldName: string): boolean; - getMetricFieldForName(fieldName: string): IESAggField | null; -} - -export class AbstractESAggSource extends AbstractESSource implements IESAggSource { - constructor(sourceDescriptor: AbstractESAggSourceDescriptor, inspectorAdapters: object); - - getAggKey(aggType: AGG_TYPE, fieldName: string): string; - getAggLabel(aggType: AGG_TYPE, fieldName: string): string; - getMetricFields(): IESAggField[]; - hasMatchingMetricField(fieldName: string): boolean; - getMetricFieldForName(fieldName: string): IESAggField | null; -} diff --git a/x-pack/plugins/maps/public/classes/sources/es_agg_source/es_agg_source.js b/x-pack/plugins/maps/public/classes/sources/es_agg_source/es_agg_source.ts similarity index 56% rename from x-pack/plugins/maps/public/classes/sources/es_agg_source/es_agg_source.js rename to x-pack/plugins/maps/public/classes/sources/es_agg_source/es_agg_source.ts index e20c509ccd4a2..a9c886617d3af 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_agg_source/es_agg_source.js +++ b/x-pack/plugins/maps/public/classes/sources/es_agg_source/es_agg_source.ts @@ -5,19 +5,38 @@ */ import { i18n } from '@kbn/i18n'; +import { Adapters } from 'src/plugins/inspector/public'; +import { GeoJsonProperties } from 'geojson'; +import { IESSource } from '../es_source'; import { AbstractESSource } from '../es_source'; import { esAggFieldsFactory } from '../../fields/es_agg_field'; import { AGG_TYPE, COUNT_PROP_LABEL, FIELD_ORIGIN } from '../../../../common/constants'; +import { IESAggField } from '../../fields/es_agg_field'; import { getSourceAggKey } from '../../../../common/get_agg_key'; +import { AbstractESAggSourceDescriptor, AggDescriptor } from '../../../../common/descriptor_types'; +import { IndexPattern } from '../../../../../../../src/plugins/data/public'; +import { IField } from '../../fields/field'; +import { ITooltipProperty } from '../../tooltips/tooltip_property'; export const DEFAULT_METRIC = { type: AGG_TYPE.COUNT }; +export interface IESAggSource extends IESSource { + getAggKey(aggType: AGG_TYPE, fieldName: string): string; + getAggLabel(aggType: AGG_TYPE, fieldName: string): string; + getMetricFields(): IESAggField[]; + hasMatchingMetricField(fieldName: string): boolean; + getMetricFieldForName(fieldName: string): IESAggField | null; + getValueAggsDsl(indexPattern: IndexPattern): { [key: string]: unknown }; +} + export class AbstractESAggSource extends AbstractESSource { - constructor(descriptor, inspectorAdapters) { + private readonly _metricFields: IESAggField[]; + + constructor(descriptor: AbstractESAggSourceDescriptor, inspectorAdapters: Adapters) { super(descriptor, inspectorAdapters); this._metricFields = []; - if (this._descriptor.metrics) { - this._descriptor.metrics.forEach((aggDescriptor) => { + if (descriptor.metrics) { + descriptor.metrics.forEach((aggDescriptor: AggDescriptor) => { this._metricFields.push( ...esAggFieldsFactory(aggDescriptor, this, this.getOriginForField()) ); @@ -25,30 +44,31 @@ export class AbstractESAggSource extends AbstractESSource { } } - getFieldByName(name) { - return this.getMetricFieldForName(name); + getFieldByName(fieldName: string) { + return this.getMetricFieldForName(fieldName); } - createField() { + createField({ fieldName }: { fieldName: string }): IField { throw new Error('Cannot create a new field from just a fieldname for an es_agg_source.'); } - hasMatchingMetricField(fieldName) { + hasMatchingMetricField(fieldName: string): boolean { const matchingField = this.getMetricFieldForName(fieldName); return !!matchingField; } - getMetricFieldForName(fieldName) { - return this.getMetricFields().find((metricField) => { + getMetricFieldForName(fieldName: string): IESAggField | null { + const targetMetricField = this.getMetricFields().find((metricField: IESAggField) => { return metricField.getName() === fieldName; }); + return targetMetricField ? targetMetricField : null; } getOriginForField() { return FIELD_ORIGIN.SOURCE; } - getMetricFields() { + getMetricFields(): IESAggField[] { const metrics = this._metricFields.filter((esAggField) => esAggField.isValid()); // Handle case where metrics is empty because older saved object state is empty array or there are no valid aggs. return metrics.length === 0 @@ -56,14 +76,14 @@ export class AbstractESAggSource extends AbstractESSource { : metrics; } - getAggKey(aggType, fieldName) { + getAggKey(aggType: AGG_TYPE, fieldName: string): string { return getSourceAggKey({ aggType, aggFieldName: fieldName, }); } - getAggLabel(aggType, fieldName) { + getAggLabel(aggType: AGG_TYPE, fieldName: string): string { switch (aggType) { case AGG_TYPE.COUNT: return COUNT_PROP_LABEL; @@ -81,8 +101,8 @@ export class AbstractESAggSource extends AbstractESSource { return this.getMetricFields(); } - getValueAggsDsl(indexPattern) { - const valueAggsDsl = {}; + getValueAggsDsl(indexPattern: IndexPattern) { + const valueAggsDsl: { [key: string]: unknown } = {}; this.getMetricFields().forEach((esAggMetric) => { const aggDsl = esAggMetric.getValueAggDsl(indexPattern); if (aggDsl) { @@ -92,9 +112,9 @@ export class AbstractESAggSource extends AbstractESSource { return valueAggsDsl; } - async filterAndFormatPropertiesToHtmlForMetricFields(properties) { - const metricFields = this.getMetricFields(); - const tooltipPropertiesPromises = []; + async getTooltipProperties(properties: GeoJsonProperties) { + const metricFields = await this.getFields(); + const promises: Array> = []; metricFields.forEach((metricField) => { let value; for (const key in properties) { @@ -105,9 +125,9 @@ export class AbstractESAggSource extends AbstractESSource { } const tooltipPromise = metricField.createTooltipProperty(value); - tooltipPropertiesPromises.push(tooltipPromise); + promises.push(tooltipPromise); }); - return await Promise.all(tooltipPropertiesPromises); + return await Promise.all(promises); } } diff --git a/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/es_geo_grid_source.d.ts b/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/es_geo_grid_source.d.ts index 51ee15e7ea5af..2ce4353fca13c 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/es_geo_grid_source.d.ts +++ b/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/es_geo_grid_source.d.ts @@ -7,6 +7,7 @@ import { AbstractESAggSource } from '../es_agg_source'; import { ESGeoGridSourceDescriptor } from '../../../../common/descriptor_types'; import { GRID_RESOLUTION } from '../../../../common/constants'; +import { IField } from '../../fields/field'; export class ESGeoGridSource extends AbstractESAggSource { static createDescriptor({ @@ -21,4 +22,5 @@ export class ESGeoGridSource extends AbstractESAggSource { getFieldNames(): string[]; getGridResolution(): GRID_RESOLUTION; getGeoGridPrecision(zoom: number): number; + createField({ fieldName }: { fieldName: string }): IField; } diff --git a/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/es_geo_grid_source.js b/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/es_geo_grid_source.js index a6322ff3ba784..aa167cb577672 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/es_geo_grid_source.js +++ b/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/es_geo_grid_source.js @@ -321,10 +321,6 @@ export class ESGeoGridSource extends AbstractESAggSource { return true; } - async filterAndFormatPropertiesToHtml(properties) { - return await this.filterAndFormatPropertiesToHtmlForMetricFields(properties); - } - async getSupportedShapeTypes() { if (this._descriptor.requestType === RENDER_AS.GRID) { return [VECTOR_SHAPE_TYPE.POLYGON]; diff --git a/x-pack/plugins/maps/public/classes/sources/es_pew_pew_source/es_pew_pew_source.js b/x-pack/plugins/maps/public/classes/sources/es_pew_pew_source/es_pew_pew_source.js index 92b0c717f6724..9ec54335d4e78 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_pew_pew_source/es_pew_pew_source.js +++ b/x-pack/plugins/maps/public/classes/sources/es_pew_pew_source/es_pew_pew_source.js @@ -223,10 +223,6 @@ export class ESPewPewSource extends AbstractESAggSource { canFormatFeatureProperties() { return true; } - - async filterAndFormatPropertiesToHtml(properties) { - return await this.filterAndFormatPropertiesToHtmlForMetricFields(properties); - } } registerSource({ diff --git a/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.js b/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.js index 7ac2738eaeb51..df83bd1cf5e60 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.js +++ b/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.js @@ -438,7 +438,7 @@ export class ESSearchSource extends AbstractESSource { return properties; } - async filterAndFormatPropertiesToHtml(properties) { + async getTooltipProperties(properties) { const indexPattern = await this.getIndexPattern(); const propertyValues = await this._loadTooltipProperties( properties._id, diff --git a/x-pack/plugins/maps/public/classes/sources/es_term_source/es_term_source.js b/x-pack/plugins/maps/public/classes/sources/es_term_source/es_term_source.js index 8cc8dd5c4a080..b4ad256c1598a 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_term_source/es_term_source.js +++ b/x-pack/plugins/maps/public/classes/sources/es_term_source/es_term_source.js @@ -129,10 +129,6 @@ export class ESTermSource extends AbstractESAggSource { return `es_table ${this.getIndexPatternId()}`; } - async filterAndFormatPropertiesToHtml(properties) { - return await this.filterAndFormatPropertiesToHtmlForMetricFields(properties); - } - getFieldNames() { return this.getMetricFields().map((esAggMetricField) => esAggMetricField.getName()); } diff --git a/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_vector_source.test.tsx b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_vector_source.test.tsx index 4e9e1e9cd7680..48f7b30261f38 100644 --- a/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_vector_source.test.tsx +++ b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_vector_source.test.tsx @@ -45,7 +45,7 @@ describe('canFormatFeatureProperties', () => { }); }); -describe('filterAndFormatPropertiesToHtml', () => { +describe('getTooltipProperties', () => { const descriptorWithFields = { ...descriptor, fields: [ @@ -67,7 +67,7 @@ describe('filterAndFormatPropertiesToHtml', () => { it('should get tooltipproperties', async () => { const source = new MVTSingleLayerVectorSource(descriptorWithFields); - const tooltipProperties = await source.filterAndFormatPropertiesToHtml({ + const tooltipProperties = await source.getTooltipProperties({ foo: 'bar', fooz: 123, }); diff --git a/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_vector_source.tsx b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_vector_source.tsx index 52dc89a6bba58..3e515613b3fd0 100644 --- a/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_vector_source.tsx +++ b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_vector_source.tsx @@ -192,7 +192,7 @@ export class MVTSingleLayerVectorSource return false; } - async filterAndFormatPropertiesToHtml( + async getTooltipProperties( properties: GeoJsonProperties, featureId?: string | number ): Promise { diff --git a/x-pack/plugins/maps/public/classes/sources/vector_source/vector_source.d.ts b/x-pack/plugins/maps/public/classes/sources/vector_source/vector_source.d.ts index fd9c179275444..a481e273bc33e 100644 --- a/x-pack/plugins/maps/public/classes/sources/vector_source/vector_source.d.ts +++ b/x-pack/plugins/maps/public/classes/sources/vector_source/vector_source.d.ts @@ -36,7 +36,7 @@ export type BoundsFilters = { }; export interface IVectorSource extends ISource { - filterAndFormatPropertiesToHtml(properties: GeoJsonProperties): Promise; + getTooltipProperties(properties: GeoJsonProperties): Promise; getBoundsForFilters( boundsFilters: BoundsFilters, registerCancelCallback: (requestToken: symbol, callback: () => void) => void @@ -58,7 +58,7 @@ export interface IVectorSource extends ISource { } export class AbstractVectorSource extends AbstractSource implements IVectorSource { - filterAndFormatPropertiesToHtml(properties: GeoJsonProperties): Promise; + getTooltipProperties(properties: GeoJsonProperties): Promise; getBoundsForFilters( boundsFilters: BoundsFilters, registerCancelCallback: (requestToken: symbol, callback: () => void) => void diff --git a/x-pack/plugins/maps/public/classes/sources/vector_source/vector_source.js b/x-pack/plugins/maps/public/classes/sources/vector_source/vector_source.js index 98ed89a6ff0ad..9569b8626aabf 100644 --- a/x-pack/plugins/maps/public/classes/sources/vector_source/vector_source.js +++ b/x-pack/plugins/maps/public/classes/sources/vector_source/vector_source.js @@ -106,7 +106,7 @@ export class AbstractVectorSource extends AbstractSource { } // Allow source to filter and format feature properties before displaying to user - async filterAndFormatPropertiesToHtml(properties) { + async getTooltipProperties(properties) { const tooltipProperties = []; for (const key in properties) { if (key.startsWith('__kbn')) { diff --git a/x-pack/plugins/maps/public/components/_index.scss b/x-pack/plugins/maps/public/components/_index.scss index 76ce9f1bc79e3..726573ce4307d 100644 --- a/x-pack/plugins/maps/public/components/_index.scss +++ b/x-pack/plugins/maps/public/components/_index.scss @@ -1,4 +1,4 @@ @import 'action_select'; -@import 'metric_editors'; +@import 'metrics_editor/metric_editors'; @import './geometry_filter'; @import 'tooltip_selector/tooltip_selector'; diff --git a/x-pack/plugins/maps/public/components/__snapshots__/metrics_editor.test.js.snap b/x-pack/plugins/maps/public/components/metrics_editor/__snapshots__/metrics_editor.test.tsx.snap similarity index 92% rename from x-pack/plugins/maps/public/components/__snapshots__/metrics_editor.test.js.snap rename to x-pack/plugins/maps/public/components/metrics_editor/__snapshots__/metrics_editor.test.tsx.snap index 0d4f1f99e464c..bd58ded41e7f5 100644 --- a/x-pack/plugins/maps/public/components/__snapshots__/metrics_editor.test.js.snap +++ b/x-pack/plugins/maps/public/components/metrics_editor/__snapshots__/metrics_editor.test.tsx.snap @@ -16,8 +16,9 @@ exports[`should add default count metric when metrics is empty array 1`] = ` "type": "count", } } - metricsFilter={[Function]} onChange={[Function]} + onRemove={[Function]} + showRemoveButton={false} /> @@ -59,8 +60,9 @@ exports[`should render metrics editor 1`] = ` "type": "sum", } } - metricsFilter={[Function]} onChange={[Function]} + onRemove={[Function]} + showRemoveButton={false} /> diff --git a/x-pack/plugins/maps/public/components/_metric_editors.scss b/x-pack/plugins/maps/public/components/metrics_editor/_metric_editors.scss similarity index 100% rename from x-pack/plugins/maps/public/components/_metric_editors.scss rename to x-pack/plugins/maps/public/components/metrics_editor/_metric_editors.scss diff --git a/x-pack/plugins/maps/public/components/metrics_editor/index.ts b/x-pack/plugins/maps/public/components/metrics_editor/index.ts new file mode 100644 index 0000000000000..3c105c2d798ff --- /dev/null +++ b/x-pack/plugins/maps/public/components/metrics_editor/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { MetricsEditor } from './metrics_editor'; diff --git a/x-pack/plugins/maps/public/components/metric_editor.js b/x-pack/plugins/maps/public/components/metrics_editor/metric_editor.tsx similarity index 59% rename from x-pack/plugins/maps/public/components/metric_editor.js rename to x-pack/plugins/maps/public/components/metrics_editor/metric_editor.tsx index 96b52d84653b2..543d144efdcc7 100644 --- a/x-pack/plugins/maps/public/components/metric_editor.js +++ b/x-pack/plugins/maps/public/components/metrics_editor/metric_editor.tsx @@ -4,18 +4,20 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { Fragment } from 'react'; -import PropTypes from 'prop-types'; +import React, { ChangeEvent, Fragment } from 'react'; import { i18n } from '@kbn/i18n'; -import { EuiFieldText, EuiFormRow } from '@elastic/eui'; +import { EuiButtonEmpty, EuiComboBoxOptionOption, EuiFieldText, EuiFormRow } from '@elastic/eui'; -import { MetricSelect, METRIC_AGGREGATION_VALUES } from './metric_select'; -import { SingleFieldSelect } from './single_field_select'; -import { AGG_TYPE } from '../../common/constants'; -import { getTermsFields } from '../index_pattern_util'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { MetricSelect } from './metric_select'; +import { SingleFieldSelect } from '../single_field_select'; +import { AggDescriptor } from '../../../common/descriptor_types'; +import { AGG_TYPE } from '../../../common/constants'; +import { getTermsFields } from '../../index_pattern_util'; +import { IFieldType } from '../../../../../../src/plugins/data/public'; -function filterFieldsForAgg(fields, aggType) { +function filterFieldsForAgg(fields: IFieldType[], aggType: AGG_TYPE) { if (!fields) { return []; } @@ -34,8 +36,27 @@ function filterFieldsForAgg(fields, aggType) { }); } -export function MetricEditor({ fields, metricsFilter, metric, onChange, removeButton }) { - const onAggChange = (metricAggregationType) => { +interface Props { + metric: AggDescriptor; + fields: IFieldType[]; + onChange: (metric: AggDescriptor) => void; + onRemove: () => void; + metricsFilter?: (metricOption: EuiComboBoxOptionOption) => boolean; + showRemoveButton: boolean; +} + +export function MetricEditor({ + fields, + metricsFilter, + metric, + onChange, + showRemoveButton, + onRemove, +}: Props) { + const onAggChange = (metricAggregationType?: AGG_TYPE) => { + if (!metricAggregationType) { + return; + } const newMetricProps = { ...metric, type: metricAggregationType, @@ -54,13 +75,16 @@ export function MetricEditor({ fields, metricsFilter, metric, onChange, removeBu onChange(newMetricProps); }; - const onFieldChange = (fieldName) => { + const onFieldChange = (fieldName?: string) => { + if (!fieldName) { + return; + } onChange({ ...metric, field: fieldName, }); }; - const onLabelChange = (e) => { + const onLabelChange = (e: ChangeEvent) => { onChange({ ...metric, label: e.target.value, @@ -80,7 +104,7 @@ export function MetricEditor({ fields, metricsFilter, metric, onChange, removeBu placeholder={i18n.translate('xpack.maps.metricsEditor.selectFieldPlaceholder', { defaultMessage: 'Select field', })} - value={metric.field} + value={metric.field ? metric.field : null} onChange={onFieldChange} fields={filterFieldsForAgg(fields, metric.type)} isClearable={false} @@ -108,6 +132,28 @@ export function MetricEditor({ fields, metricsFilter, metric, onChange, removeBu ); } + let removeButton; + if (showRemoveButton) { + removeButton = ( +
+ + + +
+ ); + } + return ( ); } - -MetricEditor.propTypes = { - metric: PropTypes.shape({ - type: PropTypes.oneOf(METRIC_AGGREGATION_VALUES), - field: PropTypes.string, - label: PropTypes.string, - }), - fields: PropTypes.array, - onChange: PropTypes.func.isRequired, - metricsFilter: PropTypes.func, -}; diff --git a/x-pack/plugins/maps/public/components/metric_select.js b/x-pack/plugins/maps/public/components/metrics_editor/metric_select.tsx similarity index 80% rename from x-pack/plugins/maps/public/components/metric_select.js rename to x-pack/plugins/maps/public/components/metrics_editor/metric_select.tsx index 2ebfcf99dece6..197c5466fe0fd 100644 --- a/x-pack/plugins/maps/public/components/metric_select.js +++ b/x-pack/plugins/maps/public/components/metrics_editor/metric_select.tsx @@ -5,10 +5,9 @@ */ import React from 'react'; -import PropTypes from 'prop-types'; import { i18n } from '@kbn/i18n'; -import { EuiComboBox } from '@elastic/eui'; -import { AGG_TYPE } from '../../common/constants'; +import { EuiComboBox, EuiComboBoxOptionOption, EuiComboBoxProps } from '@elastic/eui'; +import { AGG_TYPE } from '../../../common/constants'; const AGG_OPTIONS = [ { @@ -55,17 +54,19 @@ const AGG_OPTIONS = [ }, ]; -export const METRIC_AGGREGATION_VALUES = AGG_OPTIONS.map(({ value }) => { - return value; -}); +type Props = Omit, 'onChange'> & { + value: AGG_TYPE; + onChange: (aggType: AGG_TYPE) => void; + metricsFilter?: (metricOption: EuiComboBoxOptionOption) => boolean; +}; -export function MetricSelect({ value, onChange, metricsFilter, ...rest }) { - function onAggChange(selectedOptions) { +export function MetricSelect({ value, onChange, metricsFilter, ...rest }: Props) { + function onAggChange(selectedOptions: Array>) { if (selectedOptions.length === 0) { return; } - const aggType = selectedOptions[0].value; + const aggType = selectedOptions[0].value!; onChange(aggType); } @@ -87,9 +88,3 @@ export function MetricSelect({ value, onChange, metricsFilter, ...rest }) { /> ); } - -MetricSelect.propTypes = { - metricsFilter: PropTypes.func, - value: PropTypes.oneOf(METRIC_AGGREGATION_VALUES), - onChange: PropTypes.func.isRequired, -}; diff --git a/x-pack/plugins/maps/public/components/metrics_editor.test.js b/x-pack/plugins/maps/public/components/metrics_editor/metrics_editor.test.tsx similarity index 84% rename from x-pack/plugins/maps/public/components/metrics_editor.test.js rename to x-pack/plugins/maps/public/components/metrics_editor/metrics_editor.test.tsx index bcbeef29875ee..7ce7fbce2b066 100644 --- a/x-pack/plugins/maps/public/components/metrics_editor.test.js +++ b/x-pack/plugins/maps/public/components/metrics_editor/metrics_editor.test.tsx @@ -7,7 +7,7 @@ import React from 'react'; import { shallow } from 'enzyme'; import { MetricsEditor } from './metrics_editor'; -import { AGG_TYPE } from '../../common/constants'; +import { AGG_TYPE } from '../../../common/constants'; const defaultProps = { metrics: [ @@ -19,15 +19,14 @@ const defaultProps = { fields: [], onChange: () => {}, allowMultipleMetrics: true, - metricsFilter: () => {}, }; -test('should render metrics editor', async () => { +test('should render metrics editor', () => { const component = shallow(); expect(component).toMatchSnapshot(); }); -test('should add default count metric when metrics is empty array', async () => { +test('should add default count metric when metrics is empty array', () => { const component = shallow(); expect(component).toMatchSnapshot(); }); diff --git a/x-pack/plugins/maps/public/components/metrics_editor.js b/x-pack/plugins/maps/public/components/metrics_editor/metrics_editor.tsx similarity index 54% rename from x-pack/plugins/maps/public/components/metrics_editor.js rename to x-pack/plugins/maps/public/components/metrics_editor/metrics_editor.tsx index 7d4d7bf3ec7ab..dae1f51469281 100644 --- a/x-pack/plugins/maps/public/components/metrics_editor.js +++ b/x-pack/plugins/maps/public/components/metrics_editor/metrics_editor.tsx @@ -5,48 +5,42 @@ */ import React, { Fragment } from 'react'; -import PropTypes from 'prop-types'; -import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; -import { EuiButtonEmpty, EuiSpacer, EuiTextAlign } from '@elastic/eui'; +import { EuiButtonEmpty, EuiComboBoxOptionOption, EuiSpacer, EuiTextAlign } from '@elastic/eui'; import { MetricEditor } from './metric_editor'; -import { DEFAULT_METRIC } from '../classes/sources/es_agg_source'; +import { DEFAULT_METRIC } from '../../classes/sources/es_agg_source'; +import { IFieldType } from '../../../../../../src/plugins/data/public'; +import { AggDescriptor } from '../../../common/descriptor_types'; +import { AGG_TYPE } from '../../../common/constants'; -export function MetricsEditor({ fields, metrics, onChange, allowMultipleMetrics, metricsFilter }) { +interface Props { + allowMultipleMetrics: boolean; + metrics: AggDescriptor[]; + fields: IFieldType[]; + onChange: (metrics: AggDescriptor[]) => void; + metricsFilter?: (metricOption: EuiComboBoxOptionOption) => boolean; +} + +export function MetricsEditor({ + fields, + metrics = [DEFAULT_METRIC], + onChange, + allowMultipleMetrics = true, + metricsFilter, +}: Props) { function renderMetrics() { // There was a bug in 7.8 that initialized metrics to []. // This check is needed to handle any saved objects created before the bug was patched. const nonEmptyMetrics = metrics.length === 0 ? [DEFAULT_METRIC] : metrics; return nonEmptyMetrics.map((metric, index) => { - const onMetricChange = (metric) => { - onChange([...metrics.slice(0, index), metric, ...metrics.slice(index + 1)]); + const onMetricChange = (updatedMetric: AggDescriptor) => { + onChange([...metrics.slice(0, index), updatedMetric, ...metrics.slice(index + 1)]); }; const onRemove = () => { onChange([...metrics.slice(0, index), ...metrics.slice(index + 1)]); }; - let removeButton; - if (index > 0) { - removeButton = ( -
- - - -
- ); - } return (
0} + onRemove={onRemove} />
); @@ -62,7 +57,7 @@ export function MetricsEditor({ fields, metrics, onChange, allowMultipleMetrics, } function addMetric() { - onChange([...metrics, {}]); + onChange([...metrics, { type: AGG_TYPE.AVG }]); } function renderAddMetricButton() { @@ -71,7 +66,7 @@ export function MetricsEditor({ fields, metrics, onChange, allowMultipleMetrics, } return ( - <> + @@ -81,7 +76,7 @@ export function MetricsEditor({ fields, metrics, onChange, allowMultipleMetrics, /> - + ); } @@ -93,16 +88,3 @@ export function MetricsEditor({ fields, metrics, onChange, allowMultipleMetrics, ); } - -MetricsEditor.propTypes = { - metrics: PropTypes.array, - fields: PropTypes.array, - onChange: PropTypes.func.isRequired, - allowMultipleMetrics: PropTypes.bool, - metricsFilter: PropTypes.func, -}; - -MetricsEditor.defaultProps = { - metrics: [DEFAULT_METRIC], - allowMultipleMetrics: true, -}; diff --git a/x-pack/plugins/maps/public/connected_components/layer_panel/join_editor/resources/metrics_expression.test.js b/x-pack/plugins/maps/public/connected_components/layer_panel/join_editor/resources/metrics_expression.test.js index 3cd8a3c42879a..e0e1556ecde06 100644 --- a/x-pack/plugins/maps/public/connected_components/layer_panel/join_editor/resources/metrics_expression.test.js +++ b/x-pack/plugins/maps/public/connected_components/layer_panel/join_editor/resources/metrics_expression.test.js @@ -4,12 +4,6 @@ * you may not use this file except in compliance with the Elastic License. */ -jest.mock('../../../../components/metric_editor', () => ({ - MetricsEditor: () => { - return
mockMetricsEditor
; - }, -})); - import React from 'react'; import { shallow } from 'enzyme'; import { MetricsExpression } from './metrics_expression'; diff --git a/x-pack/plugins/ml/common/constants/data_frame_analytics.ts b/x-pack/plugins/ml/common/constants/data_frame_analytics.ts new file mode 100644 index 0000000000000..830537cbadbc8 --- /dev/null +++ b/x-pack/plugins/ml/common/constants/data_frame_analytics.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export const DEFAULT_RESULTS_FIELD = 'ml'; diff --git a/x-pack/plugins/ml/common/types/data_frame_analytics.ts b/x-pack/plugins/ml/common/types/data_frame_analytics.ts index f0aac75047585..60d2ca63dda59 100644 --- a/x-pack/plugins/ml/common/types/data_frame_analytics.ts +++ b/x-pack/plugins/ml/common/types/data_frame_analytics.ts @@ -79,3 +79,9 @@ export interface DataFrameAnalyticsConfig { version: string; allow_lazy_start?: boolean; } + +export enum ANALYSIS_CONFIG_TYPE { + OUTLIER_DETECTION = 'outlier_detection', + REGRESSION = 'regression', + CLASSIFICATION = 'classification', +} diff --git a/x-pack/plugins/ml/common/types/feature_importance.ts b/x-pack/plugins/ml/common/types/feature_importance.ts new file mode 100644 index 0000000000000..d2ab9f6c58608 --- /dev/null +++ b/x-pack/plugins/ml/common/types/feature_importance.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export interface ClassFeatureImportance { + class_name: string | boolean; + importance: number; +} +export interface FeatureImportance { + feature_name: string; + importance?: number; + classes?: ClassFeatureImportance[]; +} + +export interface TopClass { + class_name: string; + class_probability: number; + class_score: number; +} + +export type TopClasses = TopClass[]; diff --git a/x-pack/plugins/ml/common/types/file_datavisualizer.ts b/x-pack/plugins/ml/common/types/file_datavisualizer.ts index c997a4e24f868..a8b775c8d5f60 100644 --- a/x-pack/plugins/ml/common/types/file_datavisualizer.ts +++ b/x-pack/plugins/ml/common/types/file_datavisualizer.ts @@ -84,7 +84,12 @@ export interface Settings { } export interface Mappings { - [key: string]: any; + _meta?: { + created_by: string; + }; + properties: { + [key: string]: any; + }; } export interface IngestPipelineWrapper { diff --git a/x-pack/plugins/ml/common/util/analytics_utils.ts b/x-pack/plugins/ml/common/util/analytics_utils.ts new file mode 100644 index 0000000000000..d725984a47d66 --- /dev/null +++ b/x-pack/plugins/ml/common/util/analytics_utils.ts @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + AnalysisConfig, + ClassificationAnalysis, + OutlierAnalysis, + RegressionAnalysis, + ANALYSIS_CONFIG_TYPE, +} from '../types/data_frame_analytics'; + +export const isOutlierAnalysis = (arg: any): arg is OutlierAnalysis => { + const keys = Object.keys(arg); + return keys.length === 1 && keys[0] === ANALYSIS_CONFIG_TYPE.OUTLIER_DETECTION; +}; + +export const isRegressionAnalysis = (arg: any): arg is RegressionAnalysis => { + const keys = Object.keys(arg); + return keys.length === 1 && keys[0] === ANALYSIS_CONFIG_TYPE.REGRESSION; +}; + +export const isClassificationAnalysis = (arg: any): arg is ClassificationAnalysis => { + const keys = Object.keys(arg); + return keys.length === 1 && keys[0] === ANALYSIS_CONFIG_TYPE.CLASSIFICATION; +}; + +export const getDependentVar = ( + analysis: AnalysisConfig +): + | RegressionAnalysis['regression']['dependent_variable'] + | ClassificationAnalysis['classification']['dependent_variable'] => { + let depVar = ''; + + if (isRegressionAnalysis(analysis)) { + depVar = analysis.regression.dependent_variable; + } + + if (isClassificationAnalysis(analysis)) { + depVar = analysis.classification.dependent_variable; + } + return depVar; +}; + +export const getPredictionFieldName = ( + analysis: AnalysisConfig +): + | RegressionAnalysis['regression']['prediction_field_name'] + | ClassificationAnalysis['classification']['prediction_field_name'] => { + // If undefined will be defaulted to dependent_variable when config is created + let predictionFieldName; + if (isRegressionAnalysis(analysis) && analysis.regression.prediction_field_name !== undefined) { + predictionFieldName = analysis.regression.prediction_field_name; + } else if ( + isClassificationAnalysis(analysis) && + analysis.classification.prediction_field_name !== undefined + ) { + predictionFieldName = analysis.classification.prediction_field_name; + } + return predictionFieldName; +}; + +export const getDefaultPredictionFieldName = (analysis: AnalysisConfig) => { + return `${getDependentVar(analysis)}_prediction`; +}; +export const getPredictedFieldName = ( + resultsField: string, + analysis: AnalysisConfig, + forSort?: boolean +) => { + // default is 'ml' + const predictionFieldName = getPredictionFieldName(analysis); + const predictedField = `${resultsField}.${ + predictionFieldName ? predictionFieldName : getDefaultPredictionFieldName(analysis) + }`; + return predictedField; +}; diff --git a/x-pack/plugins/ml/public/application/components/data_grid/common.ts b/x-pack/plugins/ml/public/application/components/data_grid/common.ts index 1f0fcb63f019d..f252729cc20cd 100644 --- a/x-pack/plugins/ml/public/application/components/data_grid/common.ts +++ b/x-pack/plugins/ml/public/application/components/data_grid/common.ts @@ -119,13 +119,14 @@ export const getDataGridSchemasFromFieldTypes = (fieldTypes: FieldTypes, results schema = 'numeric'; } - if ( - field.includes(`${resultsField}.${FEATURE_IMPORTANCE}`) || - field.includes(`${resultsField}.${TOP_CLASSES}`) - ) { + if (field.includes(`${resultsField}.${TOP_CLASSES}`)) { schema = 'json'; } + if (field.includes(`${resultsField}.${FEATURE_IMPORTANCE}`)) { + schema = 'featureImportance'; + } + return { id: field, schema, isSortable }; }); }; @@ -250,10 +251,6 @@ export const useRenderCellValue = ( return cellValue ? 'true' : 'false'; } - if (typeof cellValue === 'object' && cellValue !== null) { - return JSON.stringify(cellValue); - } - return cellValue; }; }, [indexPattern?.fields, pagination.pageIndex, pagination.pageSize, tableItems]); diff --git a/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx b/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx index d4be2eab13d26..22815fe593d57 100644 --- a/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx +++ b/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx @@ -5,8 +5,7 @@ */ import { isEqual } from 'lodash'; -import React, { memo, useEffect, FC } from 'react'; - +import React, { memo, useEffect, FC, useMemo } from 'react'; import { i18n } from '@kbn/i18n'; import { @@ -24,13 +23,16 @@ import { } from '@elastic/eui'; import { CoreSetup } from 'src/core/public'; - import { DEFAULT_SAMPLER_SHARD_SIZE } from '../../../../common/constants/field_histograms'; -import { INDEX_STATUS } from '../../data_frame_analytics/common'; +import { ANALYSIS_CONFIG_TYPE, INDEX_STATUS } from '../../data_frame_analytics/common'; import { euiDataGridStyle, euiDataGridToolbarSettings } from './common'; import { UseIndexDataReturnType } from './types'; +import { DecisionPathPopover } from './feature_importance/decision_path_popover'; +import { TopClasses } from '../../../../common/types/feature_importance'; +import { DEFAULT_RESULTS_FIELD } from '../../../../common/constants/data_frame_analytics'; + // TODO Fix row hovering + bar highlighting // import { hoveredRow$ } from './column_chart'; @@ -41,6 +43,9 @@ export const DataGridTitle: FC<{ title: string }> = ({ title }) => ( ); interface PropsWithoutHeader extends UseIndexDataReturnType { + baseline?: number; + analysisType?: ANALYSIS_CONFIG_TYPE; + resultsField?: string; dataTestSubj: string; toastNotifications: CoreSetup['notifications']['toasts']; } @@ -60,6 +65,7 @@ type Props = PropsWithHeader | PropsWithoutHeader; export const DataGrid: FC = memo( (props) => { const { + baseline, chartsVisible, chartsButtonVisible, columnsWithCharts, @@ -80,8 +86,10 @@ export const DataGrid: FC = memo( toastNotifications, toggleChartVisibility, visibleColumns, + predictionFieldName, + resultsField, + analysisType, } = props; - // TODO Fix row hovering + bar highlighting // const getRowProps = (item: any) => { // return { @@ -90,6 +98,45 @@ export const DataGrid: FC = memo( // }; // }; + const popOverContent = useMemo(() => { + return analysisType === ANALYSIS_CONFIG_TYPE.REGRESSION || + analysisType === ANALYSIS_CONFIG_TYPE.CLASSIFICATION + ? { + featureImportance: ({ children }: { cellContentsElement: any; children: any }) => { + const rowIndex = children?.props?.visibleRowIndex; + const row = data[rowIndex]; + if (!row) return
; + // if resultsField for some reason is not available then use ml + const mlResultsField = resultsField ?? DEFAULT_RESULTS_FIELD; + const parsedFIArray = row[mlResultsField].feature_importance; + let predictedValue: string | number | undefined; + let topClasses: TopClasses = []; + if ( + predictionFieldName !== undefined && + row && + row[mlResultsField][predictionFieldName] !== undefined + ) { + predictedValue = row[mlResultsField][predictionFieldName]; + topClasses = row[mlResultsField].top_classes; + } + + return ( + + ); + }, + } + : undefined; + }, [baseline, data]); + useEffect(() => { if (invalidSortingColumnns.length > 0) { invalidSortingColumnns.forEach((columnId) => { @@ -225,6 +272,7 @@ export const DataGrid: FC = memo( } : {}), }} + popoverContents={popOverContent} pagination={{ ...pagination, pageSizeOptions: [5, 10, 25], diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx new file mode 100644 index 0000000000000..b546ac1db57dd --- /dev/null +++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx @@ -0,0 +1,166 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + AnnotationDomainTypes, + Axis, + AxisStyle, + Chart, + LineAnnotation, + LineAnnotationStyle, + LineAnnotationDatum, + LineSeries, + PartialTheme, + Position, + RecursivePartial, + ScaleType, + Settings, +} from '@elastic/charts'; +import { EuiIcon } from '@elastic/eui'; + +import React, { useCallback, useMemo } from 'react'; +import { i18n } from '@kbn/i18n'; +import euiVars from '@elastic/eui/dist/eui_theme_light.json'; +import { DecisionPathPlotData } from './use_classification_path_data'; + +const { euiColorFullShade, euiColorMediumShade } = euiVars; +const axisColor = euiColorMediumShade; + +const baselineStyle: LineAnnotationStyle = { + line: { + strokeWidth: 1, + stroke: euiColorFullShade, + opacity: 0.75, + }, + details: { + fontFamily: 'Arial', + fontSize: 10, + fontStyle: 'bold', + fill: euiColorMediumShade, + padding: 0, + }, +}; + +const axes: RecursivePartial = { + axisLine: { + stroke: axisColor, + }, + tickLabel: { + fontSize: 10, + fill: axisColor, + }, + tickLine: { + stroke: axisColor, + }, + gridLine: { + horizontal: { + dash: [1, 2], + }, + vertical: { + strokeWidth: 0, + }, + }, +}; +const theme: PartialTheme = { + axes, +}; + +interface DecisionPathChartProps { + decisionPathData: DecisionPathPlotData; + predictionFieldName?: string; + baseline?: number; + minDomain: number | undefined; + maxDomain: number | undefined; +} + +const DECISION_PATH_MARGIN = 125; +const DECISION_PATH_ROW_HEIGHT = 10; +const NUM_PRECISION = 3; +const AnnotationBaselineMarker = ; + +export const DecisionPathChart = ({ + decisionPathData, + predictionFieldName, + minDomain, + maxDomain, + baseline, +}: DecisionPathChartProps) => { + // adjust the height so it's compact for items with more features + const baselineData: LineAnnotationDatum[] = useMemo( + () => [ + { + dataValue: baseline, + header: baseline ? baseline.toPrecision(NUM_PRECISION) : '', + details: i18n.translate( + 'xpack.ml.dataframe.analytics.explorationResults.decisionPathBaselineText', + { + defaultMessage: + 'baseline (average of predictions for all data points in the training data set)', + } + ), + }, + ], + [baseline] + ); + // guarantee up to num_precision significant digits + // without having it in scientific notation + const tickFormatter = useCallback((d) => Number(d.toPrecision(NUM_PRECISION)).toString(), []); + + return ( + + + {baseline && ( + + )} + + + + + + ); +}; diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_classification.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_classification.tsx new file mode 100644 index 0000000000000..bd001fa81a582 --- /dev/null +++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_classification.tsx @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FC, useMemo, useState } from 'react'; +import { i18n } from '@kbn/i18n'; +import { EuiHealth, EuiSpacer, EuiSuperSelect, EuiTitle } from '@elastic/eui'; +import d3 from 'd3'; +import { + isDecisionPathData, + useDecisionPathData, + getStringBasedClassName, +} from './use_classification_path_data'; +import { FeatureImportance, TopClasses } from '../../../../../common/types/feature_importance'; +import { DecisionPathChart } from './decision_path_chart'; +import { MissingDecisionPathCallout } from './missing_decision_path_callout'; + +interface ClassificationDecisionPathProps { + predictedValue: string | boolean; + predictionFieldName?: string; + featureImportance: FeatureImportance[]; + topClasses: TopClasses; +} + +export const ClassificationDecisionPath: FC = ({ + featureImportance, + predictedValue, + topClasses, + predictionFieldName, +}) => { + const [currentClass, setCurrentClass] = useState( + getStringBasedClassName(topClasses[0].class_name) + ); + const { decisionPathData } = useDecisionPathData({ + featureImportance, + predictedValue: currentClass, + }); + const options = useMemo(() => { + const predictionValueStr = getStringBasedClassName(predictedValue); + + return Array.isArray(topClasses) + ? topClasses.map((c) => { + const className = getStringBasedClassName(c.class_name); + return { + value: className, + inputDisplay: + className === predictionValueStr ? ( + + {className} + + ) : ( + className + ), + }; + }) + : undefined; + }, [topClasses, predictedValue]); + + const domain = useMemo(() => { + let maxDomain; + let minDomain; + // if decisionPathData has calculated cumulative path + if (decisionPathData && isDecisionPathData(decisionPathData)) { + const [min, max] = d3.extent(decisionPathData, (d: [string, number, number]) => d[2]); + const buffer = Math.abs(max - min) * 0.1; + maxDomain = max + buffer; + minDomain = min - buffer; + } + return { maxDomain, minDomain }; + }, [decisionPathData]); + + if (!decisionPathData) return ; + + return ( + <> + + + + {i18n.translate( + 'xpack.ml.dataframe.analytics.explorationResults.classificationDecisionPathClassNameTitle', + { + defaultMessage: 'Class name', + } + )} + + + {options !== undefined && ( + + )} + + + ); +}; diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_json_viewer.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_json_viewer.tsx new file mode 100644 index 0000000000000..343324b27f9b5 --- /dev/null +++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_json_viewer.tsx @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FC } from 'react'; +import { EuiCodeBlock } from '@elastic/eui'; +import { FeatureImportance } from '../../../../../common/types/feature_importance'; + +interface DecisionPathJSONViewerProps { + featureImportance: FeatureImportance[]; +} +export const DecisionPathJSONViewer: FC = ({ featureImportance }) => { + return {JSON.stringify(featureImportance)}; +}; diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx new file mode 100644 index 0000000000000..263337f93e9a8 --- /dev/null +++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx @@ -0,0 +1,134 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FC, useState } from 'react'; +import { EuiLink, EuiTab, EuiTabs, EuiText } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { RegressionDecisionPath } from './decision_path_regression'; +import { DecisionPathJSONViewer } from './decision_path_json_viewer'; +import { FeatureImportance, TopClasses } from '../../../../../common/types/feature_importance'; +import { ANALYSIS_CONFIG_TYPE } from '../../../data_frame_analytics/common'; +import { ClassificationDecisionPath } from './decision_path_classification'; +import { useMlKibana } from '../../../contexts/kibana'; + +interface DecisionPathPopoverProps { + featureImportance: FeatureImportance[]; + analysisType: ANALYSIS_CONFIG_TYPE; + predictionFieldName?: string; + baseline?: number; + predictedValue?: number | string | undefined; + topClasses?: TopClasses; +} + +enum DECISION_PATH_TABS { + CHART = 'decision_path_chart', + JSON = 'decision_path_json', +} + +export interface ExtendedFeatureImportance extends FeatureImportance { + absImportance?: number; +} + +export const DecisionPathPopover: FC = ({ + baseline, + featureImportance, + predictedValue, + topClasses, + analysisType, + predictionFieldName, +}) => { + const [selectedTabId, setSelectedTabId] = useState(DECISION_PATH_TABS.CHART); + const { + services: { docLinks }, + } = useMlKibana(); + const { ELASTIC_WEBSITE_URL, DOC_LINK_VERSION } = docLinks; + + if (featureImportance.length < 2) { + return ; + } + + const tabs = [ + { + id: DECISION_PATH_TABS.CHART, + name: ( + + ), + }, + { + id: DECISION_PATH_TABS.JSON, + name: ( + + ), + }, + ]; + + return ( + <> +
+ + {tabs.map((tab) => ( + setSelectedTabId(tab.id)} + key={tab.id} + > + {tab.name} + + ))} + +
+ {selectedTabId === DECISION_PATH_TABS.CHART && ( + <> + + + + + ), + }} + /> + + {analysisType === ANALYSIS_CONFIG_TYPE.CLASSIFICATION && ( + + )} + {analysisType === ANALYSIS_CONFIG_TYPE.REGRESSION && ( + + )} + + )} + {selectedTabId === DECISION_PATH_TABS.JSON && ( + + )} + + ); +}; diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_regression.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_regression.tsx new file mode 100644 index 0000000000000..345269a944f02 --- /dev/null +++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_regression.tsx @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FC, useMemo } from 'react'; +import { EuiCallOut } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n/react'; +import d3 from 'd3'; +import { FeatureImportance, TopClasses } from '../../../../../common/types/feature_importance'; +import { useDecisionPathData, isDecisionPathData } from './use_classification_path_data'; +import { DecisionPathChart } from './decision_path_chart'; +import { MissingDecisionPathCallout } from './missing_decision_path_callout'; + +interface RegressionDecisionPathProps { + predictionFieldName?: string; + baseline?: number; + predictedValue?: number | undefined; + featureImportance: FeatureImportance[]; + topClasses?: TopClasses; +} + +export const RegressionDecisionPath: FC = ({ + baseline, + featureImportance, + predictedValue, + predictionFieldName, +}) => { + const { decisionPathData } = useDecisionPathData({ + baseline, + featureImportance, + predictedValue, + }); + const domain = useMemo(() => { + let maxDomain; + let minDomain; + // if decisionPathData has calculated cumulative path + if (decisionPathData && isDecisionPathData(decisionPathData)) { + const [min, max] = d3.extent(decisionPathData, (d: [string, number, number]) => d[2]); + maxDomain = max; + minDomain = min; + const buffer = Math.abs(maxDomain - minDomain) * 0.1; + maxDomain = + (typeof baseline === 'number' ? Math.max(maxDomain, baseline) : maxDomain) + buffer; + minDomain = + (typeof baseline === 'number' ? Math.min(minDomain, baseline) : minDomain) - buffer; + } + return { maxDomain, minDomain }; + }, [decisionPathData, baseline]); + + if (!decisionPathData) return ; + + return ( + <> + {baseline === undefined && ( + + } + color="warning" + iconType="alert" + /> + )} + + + ); +}; diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/missing_decision_path_callout.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/missing_decision_path_callout.tsx new file mode 100644 index 0000000000000..66eb2047b1314 --- /dev/null +++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/missing_decision_path_callout.tsx @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { EuiCallOut } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n/react'; + +export const MissingDecisionPathCallout = () => { + return ( + + + + ); +}; diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/use_classification_path_data.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/use_classification_path_data.tsx new file mode 100644 index 0000000000000..90216c4a58ffc --- /dev/null +++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/use_classification_path_data.tsx @@ -0,0 +1,173 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { useMemo } from 'react'; +import { i18n } from '@kbn/i18n'; +import { FeatureImportance, TopClasses } from '../../../../../common/types/feature_importance'; +import { ExtendedFeatureImportance } from './decision_path_popover'; + +export type DecisionPathPlotData = Array<[string, number, number]>; + +interface UseDecisionPathDataParams { + featureImportance: FeatureImportance[]; + baseline?: number; + predictedValue?: string | number | undefined; + topClasses?: TopClasses; +} + +interface RegressionDecisionPathProps { + baseline?: number; + predictedValue?: number | undefined; + featureImportance: FeatureImportance[]; + topClasses?: TopClasses; +} +const FEATURE_NAME = 'feature_name'; +const FEATURE_IMPORTANCE = 'importance'; + +export const isDecisionPathData = (decisionPathData: any): boolean => { + return ( + Array.isArray(decisionPathData) && + decisionPathData.length > 0 && + decisionPathData[0].length === 3 + ); +}; + +// cast to 'True' | 'False' | value to match Eui display +export const getStringBasedClassName = (v: string | boolean | undefined | number): string => { + if (v === undefined) { + return ''; + } + if (typeof v === 'boolean') { + return v ? 'True' : 'False'; + } + if (typeof v === 'number') { + return v.toString(); + } + return v; +}; + +export const useDecisionPathData = ({ + baseline, + featureImportance, + predictedValue, +}: UseDecisionPathDataParams): { decisionPathData: DecisionPathPlotData | undefined } => { + const decisionPathData = useMemo(() => { + return baseline + ? buildRegressionDecisionPathData({ + baseline, + featureImportance, + predictedValue: predictedValue as number | undefined, + }) + : buildClassificationDecisionPathData({ + featureImportance, + currentClass: predictedValue as string | undefined, + }); + }, [baseline, featureImportance, predictedValue]); + + return { decisionPathData }; +}; + +export const buildDecisionPathData = (featureImportance: ExtendedFeatureImportance[]) => { + const finalResult: DecisionPathPlotData = featureImportance + // sort so absolute importance so it goes from bottom (baseline) to top + .sort( + (a: ExtendedFeatureImportance, b: ExtendedFeatureImportance) => + b.absImportance! - a.absImportance! + ) + .map((d) => [d[FEATURE_NAME] as string, d[FEATURE_IMPORTANCE] as number, NaN]); + + // start at the baseline and end at predicted value + // for regression, cumulativeSum should add up to baseline + let cumulativeSum = 0; + for (let i = featureImportance.length - 1; i >= 0; i--) { + cumulativeSum += finalResult[i][1]; + finalResult[i][2] = cumulativeSum; + } + return finalResult; +}; +export const buildRegressionDecisionPathData = ({ + baseline, + featureImportance, + predictedValue, +}: RegressionDecisionPathProps): DecisionPathPlotData | undefined => { + let mappedFeatureImportance: ExtendedFeatureImportance[] = featureImportance; + mappedFeatureImportance = mappedFeatureImportance.map((d) => ({ + ...d, + absImportance: Math.abs(d[FEATURE_IMPORTANCE] as number), + })); + + if (baseline && predictedValue !== undefined && Number.isFinite(predictedValue)) { + // get the adjusted importance needed for when # of fields included in c++ analysis != max allowed + // if num fields included = num features allowed exactly, adjustedImportance should be 0 + const adjustedImportance = + predictedValue - + mappedFeatureImportance.reduce( + (accumulator, currentValue) => accumulator + currentValue.importance!, + 0 + ) - + baseline; + + mappedFeatureImportance.push({ + [FEATURE_NAME]: i18n.translate( + 'xpack.ml.dataframe.analytics.decisionPathFeatureBaselineTitle', + { + defaultMessage: 'baseline', + } + ), + [FEATURE_IMPORTANCE]: baseline, + absImportance: -1, + }); + + // if the difference is small enough then no need to plot the residual feature importance + if (Math.abs(adjustedImportance) > 1e-5) { + mappedFeatureImportance.push({ + [FEATURE_NAME]: i18n.translate( + 'xpack.ml.dataframe.analytics.decisionPathFeatureOtherTitle', + { + defaultMessage: 'other', + } + ), + [FEATURE_IMPORTANCE]: adjustedImportance, + absImportance: 0, // arbitrary importance so this will be of higher importance than baseline + }); + } + } + const filteredFeatureImportance = mappedFeatureImportance.filter( + (f) => f !== undefined + ) as ExtendedFeatureImportance[]; + + return buildDecisionPathData(filteredFeatureImportance); +}; + +export const buildClassificationDecisionPathData = ({ + featureImportance, + currentClass, +}: { + featureImportance: FeatureImportance[]; + currentClass: string | undefined; +}): DecisionPathPlotData | undefined => { + if (currentClass === undefined) return []; + const mappedFeatureImportance: Array< + ExtendedFeatureImportance | undefined + > = featureImportance.map((feature) => { + const classFeatureImportance = Array.isArray(feature.classes) + ? feature.classes.find((c) => getStringBasedClassName(c.class_name) === currentClass) + : feature; + if (classFeatureImportance && typeof classFeatureImportance[FEATURE_IMPORTANCE] === 'number') { + return { + [FEATURE_NAME]: feature[FEATURE_NAME], + [FEATURE_IMPORTANCE]: classFeatureImportance[FEATURE_IMPORTANCE], + absImportance: Math.abs(classFeatureImportance[FEATURE_IMPORTANCE] as number), + }; + } + return undefined; + }); + const filteredFeatureImportance = mappedFeatureImportance.filter( + (f) => f !== undefined + ) as ExtendedFeatureImportance[]; + + return buildDecisionPathData(filteredFeatureImportance); +}; diff --git a/x-pack/plugins/ml/public/application/components/data_grid/types.ts b/x-pack/plugins/ml/public/application/components/data_grid/types.ts index 756f74c8f9302..f9ee8c37fabf7 100644 --- a/x-pack/plugins/ml/public/application/components/data_grid/types.ts +++ b/x-pack/plugins/ml/public/application/components/data_grid/types.ts @@ -74,6 +74,9 @@ export interface UseIndexDataReturnType | 'tableItems' | 'toggleChartVisibility' | 'visibleColumns' + | 'baseline' + | 'predictionFieldName' + | 'resultsField' > { renderCellValue: RenderCellValue; } @@ -105,4 +108,7 @@ export interface UseDataGridReturnType { tableItems: DataGridItem[]; toggleChartVisibility: () => void; visibleColumns: ColumnId[]; + baseline?: number; + predictionFieldName?: string; + resultsField?: string; } diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/common/analytics.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/common/analytics.ts index 8ad861e616b7a..97098ea9e75c6 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/common/analytics.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/common/analytics.ts @@ -15,18 +15,19 @@ import { SavedSearchQuery } from '../../contexts/ml'; import { AnalysisConfig, ClassificationAnalysis, - OutlierAnalysis, RegressionAnalysis, + ANALYSIS_CONFIG_TYPE, } from '../../../../common/types/data_frame_analytics'; - +import { + isOutlierAnalysis, + isRegressionAnalysis, + isClassificationAnalysis, + getPredictionFieldName, + getDependentVar, + getPredictedFieldName, +} from '../../../../common/util/analytics_utils'; export type IndexPattern = string; -export enum ANALYSIS_CONFIG_TYPE { - OUTLIER_DETECTION = 'outlier_detection', - REGRESSION = 'regression', - CLASSIFICATION = 'classification', -} - export enum ANALYSIS_ADVANCED_FIELDS { ETA = 'eta', FEATURE_BAG_FRACTION = 'feature_bag_fraction', @@ -156,23 +157,6 @@ export const getAnalysisType = (analysis: AnalysisConfig): string => { return 'unknown'; }; -export const getDependentVar = ( - analysis: AnalysisConfig -): - | RegressionAnalysis['regression']['dependent_variable'] - | ClassificationAnalysis['classification']['dependent_variable'] => { - let depVar = ''; - - if (isRegressionAnalysis(analysis)) { - depVar = analysis.regression.dependent_variable; - } - - if (isClassificationAnalysis(analysis)) { - depVar = analysis.classification.dependent_variable; - } - return depVar; -}; - export const getTrainingPercent = ( analysis: AnalysisConfig ): @@ -190,24 +174,6 @@ export const getTrainingPercent = ( return trainingPercent; }; -export const getPredictionFieldName = ( - analysis: AnalysisConfig -): - | RegressionAnalysis['regression']['prediction_field_name'] - | ClassificationAnalysis['classification']['prediction_field_name'] => { - // If undefined will be defaulted to dependent_variable when config is created - let predictionFieldName; - if (isRegressionAnalysis(analysis) && analysis.regression.prediction_field_name !== undefined) { - predictionFieldName = analysis.regression.prediction_field_name; - } else if ( - isClassificationAnalysis(analysis) && - analysis.classification.prediction_field_name !== undefined - ) { - predictionFieldName = analysis.classification.prediction_field_name; - } - return predictionFieldName; -}; - export const getNumTopClasses = ( analysis: AnalysisConfig ): ClassificationAnalysis['classification']['num_top_classes'] => { @@ -238,35 +204,6 @@ export const getNumTopFeatureImportanceValues = ( return numTopFeatureImportanceValues; }; -export const getPredictedFieldName = ( - resultsField: string, - analysis: AnalysisConfig, - forSort?: boolean -) => { - // default is 'ml' - const predictionFieldName = getPredictionFieldName(analysis); - const defaultPredictionField = `${getDependentVar(analysis)}_prediction`; - const predictedField = `${resultsField}.${ - predictionFieldName ? predictionFieldName : defaultPredictionField - }`; - return predictedField; -}; - -export const isOutlierAnalysis = (arg: any): arg is OutlierAnalysis => { - const keys = Object.keys(arg); - return keys.length === 1 && keys[0] === ANALYSIS_CONFIG_TYPE.OUTLIER_DETECTION; -}; - -export const isRegressionAnalysis = (arg: any): arg is RegressionAnalysis => { - const keys = Object.keys(arg); - return keys.length === 1 && keys[0] === ANALYSIS_CONFIG_TYPE.REGRESSION; -}; - -export const isClassificationAnalysis = (arg: any): arg is ClassificationAnalysis => { - const keys = Object.keys(arg); - return keys.length === 1 && keys[0] === ANALYSIS_CONFIG_TYPE.CLASSIFICATION; -}; - export const isResultsSearchBoolQuery = (arg: any): arg is ResultsSearchBoolQuery => { if (arg === undefined) return false; const keys = Object.keys(arg); @@ -607,3 +544,13 @@ export const loadDocsCount = async ({ }; } }; + +export { + isOutlierAnalysis, + isRegressionAnalysis, + isClassificationAnalysis, + getPredictionFieldName, + ANALYSIS_CONFIG_TYPE, + getDependentVar, + getPredictedFieldName, +}; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/common/constants.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/common/constants.ts index 2f14dfdfdfca3..c2295a92af89c 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/common/constants.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/common/constants.ts @@ -3,8 +3,6 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ - -export const DEFAULT_RESULTS_FIELD = 'ml'; export const FEATURE_IMPORTANCE = 'feature_importance'; export const FEATURE_INFLUENCE = 'feature_influence'; export const TOP_CLASSES = 'top_classes'; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/common/fields.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/common/fields.ts index 847aefefbc6c8..f9c9bf26a9d16 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/common/fields.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/common/fields.ts @@ -4,17 +4,16 @@ * you may not use this file except in compliance with the Elastic License. */ +import { getNumTopClasses, getNumTopFeatureImportanceValues } from './analytics'; +import { Field } from '../../../../common/types/fields'; import { - getNumTopClasses, - getNumTopFeatureImportanceValues, getPredictedFieldName, getDependentVar, getPredictionFieldName, isClassificationAnalysis, isOutlierAnalysis, isRegressionAnalysis, -} from './analytics'; -import { Field } from '../../../../common/types/fields'; +} from '../../../../common/util/analytics_utils'; import { ES_FIELD_TYPES, KBN_FIELD_TYPES } from '../../../../../../../src/plugins/data/public'; import { newJobCapsService } from '../../services/new_job_capabilities_service'; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/classification_exploration/classification_exploration.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/classification_exploration/classification_exploration.tsx index ccac9a697210b..2e3a5d89367ce 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/classification_exploration/classification_exploration.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/classification_exploration/classification_exploration.tsx @@ -9,7 +9,6 @@ import React, { FC } from 'react'; import { i18n } from '@kbn/i18n'; import { ExplorationPageWrapper } from '../exploration_page_wrapper'; - import { EvaluatePanel } from './evaluate_panel'; interface Props { diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_page_wrapper/exploration_page_wrapper.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_page_wrapper/exploration_page_wrapper.tsx index 34ff36c59fa6c..84b44ef0d349f 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_page_wrapper/exploration_page_wrapper.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_page_wrapper/exploration_page_wrapper.tsx @@ -51,7 +51,6 @@ export const ExplorationPageWrapper: FC = ({ jobId, title, EvaluatePanel /> ); } - return ( <> {isLoadingJobConfig === true && jobConfig === undefined && } diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/exploration_results_table.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/exploration_results_table.tsx index 8395a11bd6fda..eea579ef1d064 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/exploration_results_table.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/exploration_results_table.tsx @@ -28,6 +28,8 @@ import { INDEX_STATUS, SEARCH_SIZE, defaultSearchQuery, + getAnalysisType, + ANALYSIS_CONFIG_TYPE, } from '../../../../common'; import { getTaskStateBadge } from '../../../analytics_management/components/analytics_list/use_columns'; import { DATA_FRAME_TASK_STATE } from '../../../analytics_management/components/analytics_list/common'; @@ -36,6 +38,7 @@ import { ExplorationQueryBar } from '../exploration_query_bar'; import { IndexPatternPrompt } from '../index_pattern_prompt'; import { useExplorationResults } from './use_exploration_results'; +import { useMlKibana } from '../../../../../contexts/kibana'; const showingDocs = i18n.translate( 'xpack.ml.dataframe.analytics.explorationResults.documentsShownHelpText', @@ -70,18 +73,27 @@ export const ExplorationResultsTable: FC = React.memo( setEvaluateSearchQuery, title, }) => { + const { + services: { + mlServices: { mlApiServices }, + }, + } = useMlKibana(); const [searchQuery, setSearchQuery] = useState(defaultSearchQuery); useEffect(() => { setEvaluateSearchQuery(searchQuery); }, [JSON.stringify(searchQuery)]); + const analysisType = getAnalysisType(jobConfig.analysis); + const classificationData = useExplorationResults( indexPattern, jobConfig, searchQuery, - getToastNotifications() + getToastNotifications(), + mlApiServices ); + const docFieldsCount = classificationData.columnsWithCharts.length; const { columnsWithCharts, @@ -94,7 +106,6 @@ export const ExplorationResultsTable: FC = React.memo( if (jobConfig === undefined || classificationData === undefined) { return null; } - // if it's a searchBar syntax error leave the table visible so they can try again if (status === INDEX_STATUS.ERROR && !errorMessage.includes('failed to create query')) { return ( @@ -184,6 +195,7 @@ export const ExplorationResultsTable: FC = React.memo( {...classificationData} dataTestSubj="mlExplorationDataGrid" toastNotifications={getToastNotifications()} + analysisType={(analysisType as unknown) as ANALYSIS_CONFIG_TYPE} /> diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts index 8d53214d23d47..a56345017258e 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts @@ -4,12 +4,14 @@ * you may not use this file except in compliance with the Elastic License. */ -import { useEffect, useMemo } from 'react'; +import { useCallback, useEffect, useMemo, useState } from 'react'; import { EuiDataGridColumn } from '@elastic/eui'; import { CoreSetup } from 'src/core/public'; +import { i18n } from '@kbn/i18n'; +import { MlApiServices } from '../../../../../services/ml_api_service'; import { IndexPattern } from '../../../../../../../../../../src/plugins/data/public'; import { DataLoader } from '../../../../../datavisualizer/index_based/data_loader'; @@ -23,21 +25,26 @@ import { UseIndexDataReturnType, } from '../../../../../components/data_grid'; import { SavedSearchQuery } from '../../../../../contexts/ml'; - import { getIndexData, getIndexFields, DataFrameAnalyticsConfig } from '../../../../common'; import { - DEFAULT_RESULTS_FIELD, - FEATURE_IMPORTANCE, - TOP_CLASSES, -} from '../../../../common/constants'; + getPredictionFieldName, + getDefaultPredictionFieldName, +} from '../../../../../../../common/util/analytics_utils'; +import { FEATURE_IMPORTANCE, TOP_CLASSES } from '../../../../common/constants'; +import { DEFAULT_RESULTS_FIELD } from '../../../../../../../common/constants/data_frame_analytics'; import { sortExplorationResultsFields, ML__ID_COPY } from '../../../../common/fields'; +import { isRegressionAnalysis } from '../../../../common/analytics'; +import { extractErrorMessage } from '../../../../../../../common/util/errors'; export const useExplorationResults = ( indexPattern: IndexPattern | undefined, jobConfig: DataFrameAnalyticsConfig | undefined, searchQuery: SavedSearchQuery, - toastNotifications: CoreSetup['notifications']['toasts'] + toastNotifications: CoreSetup['notifications']['toasts'], + mlApiServices: MlApiServices ): UseIndexDataReturnType => { + const [baseline, setBaseLine] = useState(); + const needsDestIndexFields = indexPattern !== undefined && indexPattern.title === jobConfig?.source.index[0]; @@ -52,7 +59,6 @@ export const useExplorationResults = ( ) ); } - const dataGrid = useDataGrid( columns, 25, @@ -107,16 +113,60 @@ export const useExplorationResults = ( jobConfig?.dest.index, JSON.stringify([searchQuery, dataGrid.visibleColumns]), ]); + const predictionFieldName = useMemo(() => { + if (jobConfig) { + return ( + getPredictionFieldName(jobConfig.analysis) ?? + getDefaultPredictionFieldName(jobConfig.analysis) + ); + } + return undefined; + }, [jobConfig]); + + const getAnalyticsBaseline = useCallback(async () => { + try { + if ( + jobConfig !== undefined && + jobConfig.analysis !== undefined && + isRegressionAnalysis(jobConfig.analysis) + ) { + const result = await mlApiServices.dataFrameAnalytics.getAnalyticsBaseline(jobConfig.id); + if (result?.baseline) { + setBaseLine(result.baseline); + } + } + } catch (e) { + const error = extractErrorMessage(e); + + toastNotifications.addDanger({ + title: i18n.translate( + 'xpack.ml.dataframe.analytics.explorationResults.baselineErrorMessageToast', + { + defaultMessage: 'An error occurred getting feature importance baseline', + } + ), + text: error, + }); + } + }, [mlApiServices, jobConfig]); + + useEffect(() => { + getAnalyticsBaseline(); + }, [jobConfig]); + const resultsField = jobConfig?.dest.results_field ?? DEFAULT_RESULTS_FIELD; const renderCellValue = useRenderCellValue( indexPattern, dataGrid.pagination, dataGrid.tableItems, - jobConfig?.dest.results_field ?? DEFAULT_RESULTS_FIELD + resultsField ); return { ...dataGrid, renderCellValue, + baseline, + predictionFieldName, + resultsField, }; }; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/use_outlier_data.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/use_outlier_data.ts index 24649ae5f1e71..151e5ea4e6feb 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/use_outlier_data.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/use_outlier_data.ts @@ -29,7 +29,8 @@ import { SavedSearchQuery } from '../../../../../contexts/ml'; import { getToastNotifications } from '../../../../../util/dependency_cache'; import { getIndexData, getIndexFields, DataFrameAnalyticsConfig } from '../../../../common'; -import { DEFAULT_RESULTS_FIELD, FEATURE_INFLUENCE } from '../../../../common/constants'; +import { FEATURE_INFLUENCE } from '../../../../common/constants'; +import { DEFAULT_RESULTS_FIELD } from '../../../../../../../common/constants/data_frame_analytics'; import { sortExplorationResultsFields, ML__ID_COPY } from '../../../../common/fields'; import { getFeatureCount, getOutlierScoreFieldName } from './common'; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/action_clone/clone_action_name.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/action_clone/clone_action_name.tsx index 60c699ba0d370..ce24892c9de45 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/action_clone/clone_action_name.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/action_clone/clone_action_name.tsx @@ -12,7 +12,7 @@ import { IIndexPattern } from 'src/plugins/data/common'; import { DeepReadonly } from '../../../../../../../common/types/common'; import { DataFrameAnalyticsConfig, isOutlierAnalysis } from '../../../../common'; import { isClassificationAnalysis, isRegressionAnalysis } from '../../../../common/analytics'; -import { DEFAULT_RESULTS_FIELD } from '../../../../common/constants'; +import { DEFAULT_RESULTS_FIELD } from '../../../../../../../common/constants/data_frame_analytics'; import { useMlKibana, useNavigateToPath } from '../../../../../contexts/kibana'; import { DEFAULT_NUM_TOP_FEATURE_IMPORTANCE_VALUES } from '../../hooks/use_create_analytics_form'; import { State } from '../../hooks/use_create_analytics_form/state'; diff --git a/x-pack/plugins/ml/public/application/services/ml_api_service/data_frame_analytics.ts b/x-pack/plugins/ml/public/application/services/ml_api_service/data_frame_analytics.ts index 7de39d91047ef..434200d0383f5 100644 --- a/x-pack/plugins/ml/public/application/services/ml_api_service/data_frame_analytics.ts +++ b/x-pack/plugins/ml/public/application/services/ml_api_service/data_frame_analytics.ts @@ -135,4 +135,10 @@ export const dataFrameAnalytics = { method: 'GET', }); }, + getAnalyticsBaseline(analyticsId: string) { + return http({ + path: `${basePath()}/data_frame/analytics/${analyticsId}/baseline`, + method: 'POST', + }); + }, }; diff --git a/x-pack/plugins/ml/server/models/data_frame_analytics/feature_importance.ts b/x-pack/plugins/ml/server/models/data_frame_analytics/feature_importance.ts new file mode 100644 index 0000000000000..94f54a5654873 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_frame_analytics/feature_importance.ts @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IScopedClusterClient } from 'kibana/server'; +import { + getDefaultPredictionFieldName, + getPredictionFieldName, + isRegressionAnalysis, +} from '../../../common/util/analytics_utils'; +import { DEFAULT_RESULTS_FIELD } from '../../../common/constants/data_frame_analytics'; +// Obtains data for the data frame analytics feature importance functionalities +// such as baseline, decision paths, or importance summary. +export function analyticsFeatureImportanceProvider({ + asInternalUser, + asCurrentUser, +}: IScopedClusterClient) { + async function getRegressionAnalyticsBaseline(analyticsId: string): Promise { + const { body } = await asInternalUser.ml.getDataFrameAnalytics({ + id: analyticsId, + }); + const jobConfig = body.data_frame_analytics[0]; + if (!isRegressionAnalysis) return undefined; + const destinationIndex = jobConfig.dest.index; + const predictionFieldName = getPredictionFieldName(jobConfig.analysis); + const mlResultsField = jobConfig.dest?.results_field ?? DEFAULT_RESULTS_FIELD; + const predictedField = `${mlResultsField}.${ + predictionFieldName ? predictionFieldName : getDefaultPredictionFieldName(jobConfig.analysis) + }`; + const isTrainingField = `${mlResultsField}.is_training`; + + const params = { + index: destinationIndex, + size: 0, + body: { + query: { + bool: { + filter: [ + { + term: { + [isTrainingField]: true, + }, + }, + ], + }, + }, + aggs: { + featureImportanceBaseline: { + avg: { + field: predictedField, + }, + }, + }, + }, + }; + let baseline; + const { body: aggregationResult } = await asCurrentUser.search(params); + if (aggregationResult) { + baseline = aggregationResult.aggregations.featureImportanceBaseline.value; + } + return baseline; + } + + return { + getRegressionAnalyticsBaseline, + }; +} diff --git a/x-pack/plugins/ml/server/models/file_data_visualizer/import_data.ts b/x-pack/plugins/ml/server/models/file_data_visualizer/import_data.ts index 6108454c08aa7..26dba7c2f00c1 100644 --- a/x-pack/plugins/ml/server/models/file_data_visualizer/import_data.ts +++ b/x-pack/plugins/ml/server/models/file_data_visualizer/import_data.ts @@ -94,7 +94,7 @@ export function importDataProvider({ asCurrentUser }: IScopedClusterClient) { _meta: { created_by: INDEX_META_DATA_CREATED_BY, }, - properties: mappings, + properties: mappings.properties, }, }; diff --git a/x-pack/plugins/ml/server/routes/data_frame_analytics.ts b/x-pack/plugins/ml/server/routes/data_frame_analytics.ts index dea4803e8275e..7606420eacefc 100644 --- a/x-pack/plugins/ml/server/routes/data_frame_analytics.ts +++ b/x-pack/plugins/ml/server/routes/data_frame_analytics.ts @@ -20,6 +20,7 @@ import { import { IndexPatternHandler } from '../models/data_frame_analytics/index_patterns'; import { DeleteDataFrameAnalyticsWithIndexStatus } from '../../common/types/data_frame_analytics'; import { getAuthorizationHeader } from '../lib/request_authorization'; +import { analyticsFeatureImportanceProvider } from '../models/data_frame_analytics/feature_importance'; function getIndexPatternId(context: RequestHandlerContext, patternName: string) { const iph = new IndexPatternHandler(context.core.savedObjects.client); @@ -545,4 +546,38 @@ export function dataFrameAnalyticsRoutes({ router, mlLicense }: RouteInitializat } }) ); + + /** + * @apiGroup DataFrameAnalytics + * + * @api {get} /api/ml/data_frame/analytics/baseline Get analytics's feature importance baseline + * @apiName GetDataFrameAnalyticsBaseline + * @apiDescription Returns the baseline for data frame analytics job. + * + * @apiSchema (params) analyticsIdSchema + */ + router.post( + { + path: '/api/ml/data_frame/analytics/{analyticsId}/baseline', + validate: { + params: analyticsIdSchema, + }, + options: { + tags: ['access:ml:canGetDataFrameAnalytics'], + }, + }, + mlLicense.fullLicenseAPIGuard(async ({ client, request, response }) => { + try { + const { analyticsId } = request.params; + const { getRegressionAnalyticsBaseline } = analyticsFeatureImportanceProvider(client); + const baseline = await getRegressionAnalyticsBaseline(analyticsId); + + return response.ok({ + body: { baseline }, + }); + } catch (e) { + return response.customError(wrapError(e)); + } + }) + ); } diff --git a/x-pack/plugins/reporting/common/constants.ts b/x-pack/plugins/reporting/common/constants.ts index c461c2de4e2ad..e5bca43cef562 100644 --- a/x-pack/plugins/reporting/common/constants.ts +++ b/x-pack/plugins/reporting/common/constants.ts @@ -16,6 +16,7 @@ export const API_BASE_URL_V1 = '/api/reporting/v1'; // export const API_BASE_GENERATE_V1 = `${API_BASE_URL_V1}/generate`; export const API_LIST_URL = '/api/reporting/jobs'; export const API_GENERATE_IMMEDIATE = `${API_BASE_URL_V1}/generate/immediate/csv/saved-object`; +export const API_DIAGNOSE_URL = `${API_BASE_URL}/diagnose`; export const CONTENT_TYPE_CSV = 'text/csv'; export const CSV_REPORTING_ACTION = 'downloadCsvReport'; diff --git a/x-pack/plugins/reporting/public/components/report_diagnostic.tsx b/x-pack/plugins/reporting/public/components/report_diagnostic.tsx new file mode 100644 index 0000000000000..b5b055207ddbb --- /dev/null +++ b/x-pack/plugins/reporting/public/components/report_diagnostic.tsx @@ -0,0 +1,281 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; +import React, { useState, Fragment } from 'react'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { + EuiButton, + EuiButtonEmpty, + EuiCallOut, + EuiCodeBlock, + EuiFlyout, + EuiFlyoutBody, + EuiFlyoutHeader, + EuiSpacer, + EuiSteps, + EuiText, + EuiTitle, +} from '@elastic/eui'; +import { ReportingAPIClient, DiagnoseResponse } from '../lib/reporting_api_client'; + +interface Props { + apiClient: ReportingAPIClient; +} + +type ResultStatus = 'danger' | 'incomplete' | 'complete'; + +enum statuses { + configStatus = 'configStatus', + chromeStatus = 'chromeStatus', + screenshotStatus = 'screenshotStatus', +} + +interface State { + isFlyoutVisible: boolean; + configStatus: ResultStatus; + chromeStatus: ResultStatus; + screenshotStatus: ResultStatus; + help: string[]; + logs: string; + isBusy: boolean; + success: boolean; +} + +const initialState: State = { + [statuses.configStatus]: 'incomplete', + [statuses.chromeStatus]: 'incomplete', + [statuses.screenshotStatus]: 'incomplete', + isFlyoutVisible: false, + help: [], + logs: '', + isBusy: false, + success: true, +}; + +export const ReportDiagnostic = ({ apiClient }: Props) => { + const [state, setStateBase] = useState(initialState); + const setState = (s: Partial) => + setStateBase({ + ...state, + ...s, + }); + const { + configStatus, + isBusy, + screenshotStatus, + chromeStatus, + isFlyoutVisible, + help, + logs, + success, + } = state; + + const closeFlyout = () => setState({ ...initialState, isFlyoutVisible: false }); + const showFlyout = () => setState({ isFlyoutVisible: true }); + const apiWrapper = (apiMethod: () => Promise, statusProp: statuses) => () => { + setState({ isBusy: true, [statusProp]: 'incomplete' }); + apiMethod() + .then((response) => { + setState({ + isBusy: false, + help: response.help, + logs: response.logs, + success: response.success, + [statusProp]: response.success ? 'complete' : 'danger', + }); + }) + .catch((error) => { + setState({ + isBusy: false, + help: [ + i18n.translate('xpack.reporting.listing.diagnosticApiCallFailure', { + defaultMessage: `There was a problem running the diagnostic: {error}`, + values: { error }, + }), + ], + logs: `${error.message}`, + success: false, + [statusProp]: 'danger', + }); + }); + }; + + const steps = [ + { + title: i18n.translate('xpack.reporting.listing.diagnosticConfigTitle', { + defaultMessage: 'Verify Kibana Configuration', + }), + children: ( + + + + + + + + ), + status: !success && configStatus !== 'complete' ? 'danger' : configStatus, + }, + ]; + + if (configStatus === 'complete') { + steps.push({ + title: i18n.translate('xpack.reporting.listing.diagnosticBrowserTitle', { + defaultMessage: 'Check Browser', + }), + children: ( + + + + + + + + ), + status: !success && chromeStatus !== 'complete' ? 'danger' : chromeStatus, + }); + } + + if (chromeStatus === 'complete') { + steps.push({ + title: i18n.translate('xpack.reporting.listing.diagnosticScreenshotTitle', { + defaultMessage: 'Check Screen Capture Capabilities', + }), + children: ( + + + + + + + + ), + status: !success && screenshotStatus !== 'complete' ? 'danger' : screenshotStatus, + }); + } + + if (screenshotStatus === 'complete') { + steps.push({ + title: i18n.translate('xpack.reporting.listing.diagnosticSuccessTitle', { + defaultMessage: 'All set!', + }), + children: ( + + + + ), + status: !success ? 'danger' : screenshotStatus, + }); + } + + if (!success) { + steps.push({ + title: i18n.translate('xpack.reporting.listing.diagnosticFailureTitle', { + defaultMessage: "Whoops! Looks like something isn't working properly.", + }), + children: ( + + {help.length ? ( + + +

{help.join('\n')}

+ + + ) : null} + {logs.length ? ( + + + + + {logs} + + ) : null} + + ), + status: 'danger', + }); + } + + let flyout; + if (isFlyoutVisible) { + flyout = ( + + + +

+ +

+ + + + + + + + + + + ); + } + return ( +
+ {flyout} + + + +
+ ); +}; diff --git a/x-pack/plugins/reporting/public/components/report_listing.tsx b/x-pack/plugins/reporting/public/components/report_listing.tsx index afcae93a8db16..65db13f22788b 100644 --- a/x-pack/plugins/reporting/public/components/report_listing.tsx +++ b/x-pack/plugins/reporting/public/components/report_listing.tsx @@ -6,6 +6,8 @@ import { EuiBasicTable, + EuiFlexItem, + EuiFlexGroup, EuiPageContent, EuiSpacer, EuiText, @@ -31,6 +33,7 @@ import { ReportErrorButton, ReportInfoButton, } from './buttons'; +import { ReportDiagnostic } from './report_diagnostic'; export interface Job { id: string; @@ -134,23 +137,38 @@ class ReportListingUi extends Component { public render() { return ( - - -

- -

- - -

- -

- - - {this.renderTable()} - +
+ + + + +

+ +

+ + +

+ +

+ + + + + {this.renderTable()} + + + + + + + +
); } diff --git a/x-pack/plugins/reporting/public/lib/reporting_api_client.ts b/x-pack/plugins/reporting/public/lib/reporting_api_client.ts index 54bdc99532320..2f813bd811c6c 100644 --- a/x-pack/plugins/reporting/public/lib/reporting_api_client.ts +++ b/x-pack/plugins/reporting/public/lib/reporting_api_client.ts @@ -8,7 +8,12 @@ import { stringify } from 'query-string'; import rison from 'rison-node'; import { HttpSetup } from 'src/core/public'; import { JobId, SourceJob } from '../../common/types'; -import { API_BASE_GENERATE, API_LIST_URL, REPORTING_MANAGEMENT_HOME } from '../../constants'; +import { + API_BASE_URL, + API_BASE_GENERATE, + API_LIST_URL, + REPORTING_MANAGEMENT_HOME, +} from '../../constants'; import { add } from './job_completion_notifications'; export interface JobQueueEntry { @@ -59,6 +64,12 @@ interface JobParams { [paramName: string]: any; } +export interface DiagnoseResponse { + help: string[]; + success: boolean; + logs: string; +} + export class ReportingAPIClient { private http: HttpSetup; @@ -157,4 +168,28 @@ export class ReportingAPIClient { * provides the raw server basePath to allow it to be stripped out from relativeUrls in job params */ public getServerBasePath = () => this.http.basePath.serverBasePath; + + /* + * Diagnostic-related API calls + */ + public verifyConfig = (): Promise => + this.http.post(`${API_BASE_URL}/diagnose/config`, { + asSystemRequest: true, + }); + + /* + * Diagnostic-related API calls + */ + public verifyBrowser = (): Promise => + this.http.post(`${API_BASE_URL}/diagnose/browser`, { + asSystemRequest: true, + }); + + /* + * Diagnostic-related API calls + */ + public verifyScreenCapture = (): Promise => + this.http.post(`${API_BASE_URL}/diagnose/screenshot`, { + asSystemRequest: true, + }); } diff --git a/x-pack/plugins/reporting/server/browsers/chromium/driver_factory/index.ts b/x-pack/plugins/reporting/server/browsers/chromium/driver_factory/index.ts index 809bfb57dd4fa..88be86d1ecc30 100644 --- a/x-pack/plugins/reporting/server/browsers/chromium/driver_factory/index.ts +++ b/x-pack/plugins/reporting/server/browsers/chromium/driver_factory/index.ts @@ -59,28 +59,6 @@ export class HeadlessChromiumDriverFactory { type = BROWSER_TYPE; - test(logger: LevelLogger) { - const chromiumArgs = args({ - userDataDir: this.userDataDir, - viewport: { width: 800, height: 600 }, - disableSandbox: this.browserConfig.disableSandbox, - proxy: this.browserConfig.proxy, - }); - - return puppeteerLaunch({ - userDataDir: this.userDataDir, - executablePath: this.binaryPath, - ignoreHTTPSErrors: true, - args: chromiumArgs, - } as LaunchOptions).catch((error: Error) => { - logger.error( - `The Reporting plugin encountered issues launching Chromium in a self-test. You may have trouble generating reports.` - ); - logger.error(error); - return null; - }); - } - /* * Return an observable to objects which will drive screenshot capture for a page */ diff --git a/x-pack/plugins/reporting/server/browsers/chromium/driver_factory/start_logs.ts b/x-pack/plugins/reporting/server/browsers/chromium/driver_factory/start_logs.ts new file mode 100644 index 0000000000000..8eafbd8e0ddbe --- /dev/null +++ b/x-pack/plugins/reporting/server/browsers/chromium/driver_factory/start_logs.ts @@ -0,0 +1,133 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; +import { spawn } from 'child_process'; +import del from 'del'; +import { mkdtempSync } from 'fs'; +import { uniq } from 'lodash'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { createInterface } from 'readline'; +import { fromEvent, timer, merge, of } from 'rxjs'; +import { takeUntil, map, reduce, tap, catchError } from 'rxjs/operators'; +import { ReportingCore } from '../../..'; +import { LevelLogger } from '../../../lib'; +import { getBinaryPath } from '../../install'; +import { args } from './args'; + +const browserLaunchTimeToWait = 5 * 1000; + +// Default args used by pptr +// https://github.com/puppeteer/puppeteer/blob/13ea347/src/node/Launcher.ts#L168 +const defaultArgs = [ + '--disable-background-networking', + '--enable-features=NetworkService,NetworkServiceInProcess', + '--disable-background-timer-throttling', + '--disable-backgrounding-occluded-windows', + '--disable-breakpad', + '--disable-client-side-phishing-detection', + '--disable-component-extensions-with-background-pages', + '--disable-default-apps', + '--disable-dev-shm-usage', + '--disable-extensions', + '--disable-features=TranslateUI', + '--disable-hang-monitor', + '--disable-ipc-flooding-protection', + '--disable-popup-blocking', + '--disable-prompt-on-repost', + '--disable-renderer-backgrounding', + '--disable-sync', + '--force-color-profile=srgb', + '--metrics-recording-only', + '--no-first-run', + '--enable-automation', + '--password-store=basic', + '--use-mock-keychain', + '--remote-debugging-port=0', + '--headless', +]; + +export const browserStartLogs = ( + core: ReportingCore, + logger: LevelLogger, + overrideFlags: string[] = [] +) => { + const config = core.getConfig(); + const proxy = config.get('capture', 'browser', 'chromium', 'proxy'); + const disableSandbox = config.get('capture', 'browser', 'chromium', 'disableSandbox'); + const userDataDir = mkdtempSync(join(tmpdir(), 'chromium-')); + const binaryPath = getBinaryPath(); + const kbnArgs = args({ + userDataDir, + viewport: { width: 800, height: 600 }, + disableSandbox, + proxy, + }); + const finalArgs = uniq([...defaultArgs, ...kbnArgs, ...overrideFlags]); + + // On non-windows platforms, `detached: true` makes child process a + // leader of a new process group, making it possible to kill child + // process tree with `.kill(-pid)` command. @see + // https://nodejs.org/api/child_process.html#child_process_options_detached + const browserProcess = spawn(binaryPath, finalArgs, { + detached: process.platform !== 'win32', + }); + + const rl = createInterface({ input: browserProcess.stderr }); + + const exit$ = fromEvent(browserProcess, 'exit').pipe( + map((code) => { + logger.error(`Browser exited abnormally, received code: ${code}`); + return i18n.translate('xpack.reporting.diagnostic.browserCrashed', { + defaultMessage: `Browser exited abnormally during startup`, + }); + }) + ); + + const error$ = fromEvent(browserProcess, 'error').pipe( + map(() => { + logger.error(`Browser process threw an error on startup`); + return i18n.translate('xpack.reporting.diagnostic.browserErrored', { + defaultMessage: `Browser process threw an error on startup`, + }); + }) + ); + + const browserProcessLogger = logger.clone(['chromium-stderr']); + const log$ = fromEvent(rl, 'line').pipe( + tap((message: unknown) => { + if (typeof message === 'string') { + browserProcessLogger.info(message); + } + }) + ); + + // Collect all events (exit, error and on log-lines), but let chromium keep spitting out + // logs as sometimes it's "bind" successfully for remote connections, but later emit + // a log indicative of an issue (for example, no default font found). + return merge(exit$, error$, log$).pipe( + takeUntil(timer(browserLaunchTimeToWait)), + reduce((acc, curr) => `${acc}${curr}\n`, ''), + tap(() => { + if (browserProcess && browserProcess.pid && !browserProcess.killed) { + browserProcess.kill('SIGKILL'); + logger.info(`Successfully sent 'SIGKILL' to browser process (PID: ${browserProcess.pid})`); + } + browserProcess.removeAllListeners(); + rl.removeAllListeners(); + rl.close(); + del(userDataDir, { force: true }).catch((error) => { + logger.error(`Error deleting user data directory at [${userDataDir}]!`); + logger.error(error); + }); + }), + catchError((error) => { + logger.error(error); + return of(error); + }) + ); +}; diff --git a/x-pack/plugins/reporting/server/browsers/install.ts b/x-pack/plugins/reporting/server/browsers/install.ts index 9eddbe5ef0498..35cc5b6d8b7c2 100644 --- a/x-pack/plugins/reporting/server/browsers/install.ts +++ b/x-pack/plugins/reporting/server/browsers/install.ts @@ -4,24 +4,43 @@ * you may not use this file except in compliance with the Elastic License. */ +import del from 'del'; import os from 'os'; import path from 'path'; -import del from 'del'; - import * as Rx from 'rxjs'; import { LevelLogger } from '../lib'; +import { paths } from './chromium/paths'; import { ensureBrowserDownloaded } from './download'; // @ts-ignore import { md5 } from './download/checksum'; // @ts-ignore import { extract } from './extract'; -import { paths } from './chromium/paths'; interface Package { platforms: string[]; architecture: string; } +/** + * Small helper util to resolve where chromium is installed + */ +export const getBinaryPath = ( + chromiumPath: string = path.resolve(__dirname, '../../chromium'), + platform: string = process.platform, + architecture: string = os.arch() +) => { + const pkg = paths.packages.find((p: Package) => { + return p.platforms.includes(platform) && p.architecture === architecture; + }); + + if (!pkg) { + // TODO: validate this + throw new Error(`Unsupported platform: ${platform}-${architecture}`); + } + + return path.join(chromiumPath, pkg.binaryRelativePath); +}; + /** * "install" a browser by type into installs path by extracting the downloaded * archive. If there is an error extracting the archive an `ExtractError` is thrown @@ -43,7 +62,7 @@ export function installBrowser( throw new Error(`Unsupported platform: ${platform}-${architecture}`); } - const binaryPath = path.join(chromiumPath, pkg.binaryRelativePath); + const binaryPath = getBinaryPath(chromiumPath, platform, architecture); const binaryChecksum = await md5(binaryPath).catch(() => ''); if (binaryChecksum !== pkg.binaryChecksum) { diff --git a/x-pack/plugins/reporting/server/export_types/png/lib/generate_png.ts b/x-pack/plugins/reporting/server/export_types/png/lib/generate_png.ts index c3d5b2cc60051..096d0bd428214 100644 --- a/x-pack/plugins/reporting/server/export_types/png/lib/generate_png.ts +++ b/x-pack/plugins/reporting/server/export_types/png/lib/generate_png.ts @@ -28,7 +28,7 @@ export async function generatePngObservableFactory(reporting: ReportingCore) { if (!layoutParams || !layoutParams.dimensions) { throw new Error(`LayoutParams.Dimensions is undefined.`); } - const layout = new PreserveLayout(layoutParams.dimensions); + const layout = new PreserveLayout(layoutParams.dimensions, layoutParams.selectors); if (apmLayout) apmLayout.end(); const apmScreenshots = apmTrans?.startSpan('screenshots_pipeline', 'setup'); diff --git a/x-pack/plugins/reporting/server/lib/index.ts b/x-pack/plugins/reporting/server/lib/index.ts index f3a09cffbb104..9e5a3ca76126d 100644 --- a/x-pack/plugins/reporting/server/lib/index.ts +++ b/x-pack/plugins/reporting/server/lib/index.ts @@ -13,4 +13,3 @@ export { LevelLogger } from './level_logger'; export { statuses } from './statuses'; export { ReportingStore } from './store'; export { startTrace } from './trace'; -export { runValidations } from './validate'; diff --git a/x-pack/plugins/reporting/server/lib/layouts/index.ts b/x-pack/plugins/reporting/server/lib/layouts/index.ts index d46f088475222..507b7614072ea 100644 --- a/x-pack/plugins/reporting/server/lib/layouts/index.ts +++ b/x-pack/plugins/reporting/server/lib/layouts/index.ts @@ -54,6 +54,7 @@ export interface Size { export interface LayoutParams { id: string; dimensions: Size; + selectors?: LayoutSelectorDictionary; } interface LayoutSelectors { diff --git a/x-pack/plugins/reporting/server/lib/layouts/preserve_layout.ts b/x-pack/plugins/reporting/server/lib/layouts/preserve_layout.ts index 9041055ddce2d..e8d182dac0b1d 100644 --- a/x-pack/plugins/reporting/server/lib/layouts/preserve_layout.ts +++ b/x-pack/plugins/reporting/server/lib/layouts/preserve_layout.ts @@ -25,12 +25,16 @@ export class PreserveLayout extends Layout { private readonly scaledHeight: number; private readonly scaledWidth: number; - constructor(size: Size) { + constructor(size: Size, layoutSelectors?: LayoutSelectorDictionary) { super(LayoutTypes.PRESERVE_LAYOUT); this.height = size.height; this.width = size.width; this.scaledHeight = size.height * ZOOM; this.scaledWidth = size.width * ZOOM; + + if (layoutSelectors) { + this.selectors = layoutSelectors; + } } public getCssOverridesPath() { diff --git a/x-pack/plugins/reporting/server/lib/store/store.test.ts b/x-pack/plugins/reporting/server/lib/store/store.test.ts index e6c4eb7346460..b87466ca289cf 100644 --- a/x-pack/plugins/reporting/server/lib/store/store.test.ts +++ b/x-pack/plugins/reporting/server/lib/store/store.test.ts @@ -7,8 +7,7 @@ import sinon from 'sinon'; import { ElasticsearchServiceSetup } from 'src/core/server'; import { ReportingConfig, ReportingCore } from '../..'; -import { createMockReportingCore } from '../../test_helpers'; -import { createMockLevelLogger } from '../../test_helpers/create_mock_levellogger'; +import { createMockReportingCore, createMockLevelLogger } from '../../test_helpers'; import { Report } from './report'; import { ReportingStore } from './store'; diff --git a/x-pack/plugins/reporting/server/lib/validate/index.ts b/x-pack/plugins/reporting/server/lib/validate/index.ts deleted file mode 100644 index d20df6b7315be..0000000000000 --- a/x-pack/plugins/reporting/server/lib/validate/index.ts +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { i18n } from '@kbn/i18n'; -import { ElasticsearchServiceSetup } from 'kibana/server'; -import { ReportingConfig } from '../../'; -import { HeadlessChromiumDriverFactory } from '../../browsers/chromium/driver_factory'; -import { LevelLogger } from '../'; -import { validateBrowser } from './validate_browser'; -import { validateMaxContentLength } from './validate_max_content_length'; - -export async function runValidations( - config: ReportingConfig, - elasticsearch: ElasticsearchServiceSetup, - browserFactory: HeadlessChromiumDriverFactory, - parentLogger: LevelLogger -) { - const logger = parentLogger.clone(['validations']); - try { - await Promise.all([ - validateBrowser(browserFactory, logger), - validateMaxContentLength(config, elasticsearch, logger), - ]); - logger.debug( - i18n.translate('xpack.reporting.selfCheck.ok', { - defaultMessage: `Reporting plugin self-check ok!`, - }) - ); - } catch (err) { - logger.error(err); - logger.warning( - i18n.translate('xpack.reporting.selfCheck.warning', { - defaultMessage: `Reporting plugin self-check generated a warning: {err}`, - values: { - err, - }, - }) - ); - } -} diff --git a/x-pack/plugins/reporting/server/lib/validate/validate_browser.ts b/x-pack/plugins/reporting/server/lib/validate/validate_browser.ts deleted file mode 100644 index d29aa522dad90..0000000000000 --- a/x-pack/plugins/reporting/server/lib/validate/validate_browser.ts +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { Browser } from 'puppeteer'; -import { BROWSER_TYPE } from '../../../common/constants'; -import { HeadlessChromiumDriverFactory } from '../../browsers/chromium/driver_factory'; -import { LevelLogger } from '../'; - -/* - * Validate the Reporting headless browser can launch, and that it can connect - * to the locally running Kibana instance. - */ -export const validateBrowser = async ( - browserFactory: HeadlessChromiumDriverFactory, - logger: LevelLogger -) => { - if (browserFactory.type === BROWSER_TYPE) { - return browserFactory.test(logger).then((browser: Browser | null) => { - if (browser && browser.close) { - browser.close(); - } else { - throw new Error('Could not close browser client handle!'); - } - }); - } -}; diff --git a/x-pack/plugins/reporting/server/lib/validate/validate_max_content_length.test.js b/x-pack/plugins/reporting/server/lib/validate/validate_max_content_length.test.js deleted file mode 100644 index f358021560cff..0000000000000 --- a/x-pack/plugins/reporting/server/lib/validate/validate_max_content_length.test.js +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import sinon from 'sinon'; -import { validateMaxContentLength } from './validate_max_content_length'; - -const FIVE_HUNDRED_MEGABYTES = 524288000; -const ONE_HUNDRED_MEGABYTES = 104857600; - -describe('Reporting: Validate Max Content Length', () => { - const elasticsearch = { - legacy: { - client: { - callAsInternalUser: () => ({ - defaults: { - http: { - max_content_length: '100mb', - }, - }, - }), - }, - }, - }; - - const logger = { - warning: sinon.spy(), - }; - - beforeEach(() => { - logger.warning.resetHistory(); - }); - - it('should log warning messages when reporting has a higher max-size than elasticsearch', async () => { - const config = { get: sinon.stub().returns(FIVE_HUNDRED_MEGABYTES) }; - const elasticsearch = { - legacy: { - client: { - callAsInternalUser: () => ({ - defaults: { - http: { - max_content_length: '100mb', - }, - }, - }), - }, - }, - }; - - await validateMaxContentLength(config, elasticsearch, logger); - - sinon.assert.calledWithMatch( - logger.warning, - `xpack.reporting.csv.maxSizeBytes (524288000) is higher` - ); - sinon.assert.calledWithMatch( - logger.warning, - `than ElasticSearch's http.max_content_length (104857600)` - ); - sinon.assert.calledWithMatch( - logger.warning, - 'Please set http.max_content_length in ElasticSearch to match' - ); - sinon.assert.calledWithMatch( - logger.warning, - 'or lower your xpack.reporting.csv.maxSizeBytes in Kibana' - ); - }); - - it('should do nothing when reporting has the same max-size as elasticsearch', async () => { - const config = { get: sinon.stub().returns(ONE_HUNDRED_MEGABYTES) }; - - expect( - async () => await validateMaxContentLength(config, elasticsearch, logger.warning) - ).not.toThrow(); - sinon.assert.notCalled(logger.warning); - }); -}); diff --git a/x-pack/plugins/reporting/server/lib/validate/validate_max_content_length.ts b/x-pack/plugins/reporting/server/lib/validate/validate_max_content_length.ts deleted file mode 100644 index c38c6e5297854..0000000000000 --- a/x-pack/plugins/reporting/server/lib/validate/validate_max_content_length.ts +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import numeral from '@elastic/numeral'; -import { ElasticsearchServiceSetup } from 'kibana/server'; -import { defaults, get } from 'lodash'; -import { ReportingConfig } from '../../'; -import { LevelLogger } from '../'; - -const KIBANA_MAX_SIZE_BYTES_PATH = 'csv.maxSizeBytes'; -const ES_MAX_SIZE_BYTES_PATH = 'http.max_content_length'; - -export async function validateMaxContentLength( - config: ReportingConfig, - elasticsearch: ElasticsearchServiceSetup, - logger: LevelLogger -) { - const { callAsInternalUser } = elasticsearch.legacy.client; - - const elasticClusterSettingsResponse = await callAsInternalUser('cluster.getSettings', { - includeDefaults: true, - }); - const { persistent, transient, defaults: defaultSettings } = elasticClusterSettingsResponse; - const elasticClusterSettings = defaults({}, persistent, transient, defaultSettings); - - const elasticSearchMaxContent = get(elasticClusterSettings, 'http.max_content_length', '100mb'); - const elasticSearchMaxContentBytes = numeral().unformat(elasticSearchMaxContent.toUpperCase()); - const kibanaMaxContentBytes = config.get('csv', 'maxSizeBytes'); - - if (kibanaMaxContentBytes > elasticSearchMaxContentBytes) { - // TODO this should simply throw an error and let the handler conver it to a warning mesasge. See validateServerHost. - logger.warning( - `xpack.reporting.${KIBANA_MAX_SIZE_BYTES_PATH} (${kibanaMaxContentBytes}) is higher than ElasticSearch's ${ES_MAX_SIZE_BYTES_PATH} (${elasticSearchMaxContentBytes}). ` + - `Please set ${ES_MAX_SIZE_BYTES_PATH} in ElasticSearch to match, or lower your xpack.reporting.${KIBANA_MAX_SIZE_BYTES_PATH} in Kibana to avoid this warning.` - ); - } -} diff --git a/x-pack/plugins/reporting/server/plugin.ts b/x-pack/plugins/reporting/server/plugin.ts index 8c0e352aa06c5..af1ccfd592b96 100644 --- a/x-pack/plugins/reporting/server/plugin.ts +++ b/x-pack/plugins/reporting/server/plugin.ts @@ -11,7 +11,7 @@ import { PLUGIN_ID, UI_SETTINGS_CUSTOM_PDF_LOGO } from '../common/constants'; import { ReportingCore } from './'; import { initializeBrowserDriverFactory } from './browsers'; import { buildConfig, ReportingConfigType } from './config'; -import { createQueueFactory, LevelLogger, ReportingStore, runValidations } from './lib'; +import { createQueueFactory, LevelLogger, ReportingStore } from './lib'; import { registerRoutes } from './routes'; import { setFieldFormats } from './services'; import { ReportingSetup, ReportingSetupDeps, ReportingStart, ReportingStartDeps } from './types'; @@ -105,7 +105,6 @@ export class ReportingPlugin setFieldFormats(plugins.data.fieldFormats); const { logger, reportingCore } = this; - const { elasticsearch } = reportingCore.getPluginSetupDeps(); // async background start (async () => { @@ -124,9 +123,6 @@ export class ReportingPlugin store, }); - // run self-check validations - runValidations(config, elasticsearch, browserDriverFactory, this.logger); - this.logger.debug('Start complete'); })().catch((e) => { this.logger.error(`Error in Reporting start, reporting may not function properly`); diff --git a/x-pack/plugins/reporting/server/routes/diagnostic/browser.test.ts b/x-pack/plugins/reporting/server/routes/diagnostic/browser.test.ts new file mode 100644 index 0000000000000..f92fbfc7013cf --- /dev/null +++ b/x-pack/plugins/reporting/server/routes/diagnostic/browser.test.ts @@ -0,0 +1,250 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { UnwrapPromise } from '@kbn/utility-types'; +import { spawn } from 'child_process'; +import { createInterface } from 'readline'; +import { setupServer } from 'src/core/server/test_utils'; +import supertest from 'supertest'; +import { ReportingCore } from '../..'; +import { createMockLevelLogger, createMockReportingCore } from '../../test_helpers'; +import { registerDiagnoseBrowser } from './browser'; + +jest.mock('child_process'); +jest.mock('readline'); + +type SetupServerReturn = UnwrapPromise>; + +const devtoolMessage = 'DevTools listening on (ws://localhost:4000)'; +const fontNotFoundMessage = 'Could not find the default font'; + +describe('POST /diagnose/browser', () => { + jest.setTimeout(6000); + const reportingSymbol = Symbol('reporting'); + const mockLogger = createMockLevelLogger(); + + let server: SetupServerReturn['server']; + let httpSetup: SetupServerReturn['httpSetup']; + let core: ReportingCore; + const mockedSpawn: any = spawn; + const mockedCreateInterface: any = createInterface; + + const config = { + get: jest.fn().mockImplementation(() => ({})), + kbnConfig: { get: jest.fn() }, + }; + + beforeEach(async () => { + ({ server, httpSetup } = await setupServer(reportingSymbol)); + httpSetup.registerRouteHandlerContext(reportingSymbol, 'reporting', () => ({})); + + const mockSetupDeps = ({ + elasticsearch: { + legacy: { client: { callAsInternalUser: jest.fn() } }, + }, + router: httpSetup.createRouter(''), + } as unknown) as any; + + core = await createMockReportingCore(config, mockSetupDeps); + + mockedSpawn.mockImplementation(() => ({ + removeAllListeners: jest.fn(), + kill: jest.fn(), + pid: 123, + stderr: 'stderr', + addEventListener: jest.fn(), + removeEventListener: jest.fn(), + })); + + mockedCreateInterface.mockImplementation(() => ({ + addEventListener: jest.fn(), + removeEventListener: jest.fn(), + removeAllListeners: jest.fn(), + close: jest.fn(), + })); + }); + + afterEach(async () => { + await server.stop(); + }); + + it('returns a 200 when successful', async () => { + registerDiagnoseBrowser(core, mockLogger); + + await server.start(); + + mockedCreateInterface.mockImplementation(() => ({ + addEventListener: (e: string, cb: any) => setTimeout(() => cb(devtoolMessage), 0), + removeEventListener: jest.fn(), + removeAllListeners: jest.fn(), + close: jest.fn(), + })); + + return supertest(httpSetup.server.listener) + .post('/api/reporting/diagnose/browser') + .expect(200) + .then(({ body }) => { + expect(body.success).toEqual(true); + expect(body.help).toEqual([]); + }); + }); + + it('returns logs when browser crashes + helpful links', async () => { + const logs = `Could not find the default font`; + registerDiagnoseBrowser(core, mockLogger); + + await server.start(); + + mockedCreateInterface.mockImplementation(() => ({ + addEventListener: (e: string, cb: any) => setTimeout(() => cb(logs), 0), + removeEventListener: jest.fn(), + removeAllListeners: jest.fn(), + close: jest.fn(), + })); + + mockedSpawn.mockImplementation(() => ({ + removeAllListeners: jest.fn(), + kill: jest.fn(), + addEventListener: jest.fn(), + removeEventListener: jest.fn(), + })); + + return supertest(httpSetup.server.listener) + .post('/api/reporting/diagnose/browser') + .expect(200) + .then(({ body }) => { + expect(body).toMatchInlineSnapshot(` + Object { + "help": Array [ + "The browser couldn't locate a default font. Please see https://www.elastic.co/guide/en/kibana/current/reporting-troubleshooting.html#reporting-troubleshooting-system-dependencies to fix this issue.", + ], + "logs": "Could not find the default font + ", + "success": false, + } + `); + }); + }); + + it('logs a message when the browser starts, but then has problems later', async () => { + registerDiagnoseBrowser(core, mockLogger); + + await server.start(); + + mockedCreateInterface.mockImplementation(() => ({ + addEventListener: (e: string, cb: any) => { + setTimeout(() => cb(devtoolMessage), 0); + setTimeout(() => cb(fontNotFoundMessage), 0); + }, + removeEventListener: jest.fn(), + removeAllListeners: jest.fn(), + close: jest.fn(), + })); + + mockedSpawn.mockImplementation(() => ({ + removeAllListeners: jest.fn(), + kill: jest.fn(), + addEventListener: jest.fn(), + removeEventListener: jest.fn(), + })); + + return supertest(httpSetup.server.listener) + .post('/api/reporting/diagnose/browser') + .expect(200) + .then(({ body }) => { + expect(body).toMatchInlineSnapshot(` + Object { + "help": Array [ + "The browser couldn't locate a default font. Please see https://www.elastic.co/guide/en/kibana/current/reporting-troubleshooting.html#reporting-troubleshooting-system-dependencies to fix this issue.", + ], + "logs": "DevTools listening on (ws://localhost:4000) + Could not find the default font + ", + "success": false, + } + `); + }); + }); + + it('logs a message when the browser starts, but then crashes', async () => { + registerDiagnoseBrowser(core, mockLogger); + + await server.start(); + + mockedCreateInterface.mockImplementation(() => ({ + addEventListener: (e: string, cb: any) => { + setTimeout(() => cb(fontNotFoundMessage), 0); + }, + removeEventListener: jest.fn(), + removeAllListeners: jest.fn(), + close: jest.fn(), + })); + + mockedSpawn.mockImplementation(() => ({ + removeAllListeners: jest.fn(), + kill: jest.fn(), + addEventListener: (e: string, cb: any) => { + if (e === 'exit') { + setTimeout(() => cb(), 5); + } + }, + removeEventListener: jest.fn(), + })); + + return supertest(httpSetup.server.listener) + .post('/api/reporting/diagnose/browser') + .expect(200) + .then(({ body }) => { + expect(body).toMatchInlineSnapshot(` + Object { + "help": Array [ + "The browser couldn't locate a default font. Please see https://www.elastic.co/guide/en/kibana/current/reporting-troubleshooting.html#reporting-troubleshooting-system-dependencies to fix this issue.", + ], + "logs": "Could not find the default font + Browser exited abnormally during startup + ", + "success": false, + } + `); + }); + }); + + it('cleans up process and subscribers', async () => { + registerDiagnoseBrowser(core, mockLogger); + + await server.start(); + const killMock = jest.fn(); + const spawnListenersMock = jest.fn(); + const createInterfaceListenersMock = jest.fn(); + const createInterfaceCloseMock = jest.fn(); + + mockedSpawn.mockImplementation(() => ({ + removeAllListeners: spawnListenersMock, + kill: killMock, + pid: 123, + stderr: 'stderr', + addEventListener: jest.fn(), + removeEventListener: jest.fn(), + })); + + mockedCreateInterface.mockImplementation(() => ({ + addEventListener: (e: string, cb: any) => setTimeout(() => cb(devtoolMessage), 0), + removeEventListener: jest.fn(), + removeAllListeners: createInterfaceListenersMock, + close: createInterfaceCloseMock, + })); + + return supertest(httpSetup.server.listener) + .post('/api/reporting/diagnose/browser') + .expect(200) + .then(() => { + expect(killMock.mock.calls.length).toBe(1); + expect(spawnListenersMock.mock.calls.length).toBe(1); + expect(createInterfaceListenersMock.mock.calls.length).toBe(1); + expect(createInterfaceCloseMock.mock.calls.length).toBe(1); + }); + }); +}); diff --git a/x-pack/plugins/reporting/server/routes/diagnostic/browser.ts b/x-pack/plugins/reporting/server/routes/diagnostic/browser.ts new file mode 100644 index 0000000000000..24b85220defb4 --- /dev/null +++ b/x-pack/plugins/reporting/server/routes/diagnostic/browser.ts @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; +import { ReportingCore } from '../..'; +import { API_DIAGNOSE_URL } from '../../../common/constants'; +import { browserStartLogs } from '../../browsers/chromium/driver_factory/start_logs'; +import { LevelLogger as Logger } from '../../lib'; +import { DiagnosticResponse } from '../../types'; +import { authorizedUserPreRoutingFactory } from '../lib/authorized_user_pre_routing'; + +const logsToHelpMap = { + 'error while loading shared libraries': i18n.translate( + 'xpack.reporting.diagnostic.browserMissingDependency', + { + defaultMessage: `The browser couldn't start properly due to missing system dependencies. Please see {url}`, + values: { + url: + 'https://www.elastic.co/guide/en/kibana/current/reporting-troubleshooting.html#reporting-troubleshooting-system-dependencies', + }, + } + ), + + 'Could not find the default font': i18n.translate( + 'xpack.reporting.diagnostic.browserMissingFonts', + { + defaultMessage: `The browser couldn't locate a default font. Please see {url} to fix this issue.`, + values: { + url: + 'https://www.elastic.co/guide/en/kibana/current/reporting-troubleshooting.html#reporting-troubleshooting-system-dependencies', + }, + } + ), + + 'No usable sandbox': i18n.translate('xpack.reporting.diagnostic.noUsableSandbox', { + defaultMessage: `Unable to use Chromium sandbox. This can be disabled at your own risk with 'xpack.reporting.capture.browser.chromium.disableSandbox'. Please see {url}`, + values: { + url: + 'https://www.elastic.co/guide/en/kibana/current/reporting-troubleshooting.html#reporting-troubleshooting-sandbox-dependency', + }, + }), +}; + +export const registerDiagnoseBrowser = (reporting: ReportingCore, logger: Logger) => { + const { router } = reporting.getPluginSetupDeps(); + const userHandler = authorizedUserPreRoutingFactory(reporting); + + router.post( + { + path: `${API_DIAGNOSE_URL}/browser`, + validate: {}, + }, + userHandler(async (user, context, req, res) => { + const logs = await browserStartLogs(reporting, logger).toPromise(); + const knownIssues = Object.keys(logsToHelpMap) as Array; + + const boundSuccessfully = logs.includes(`DevTools listening on`); + const help = knownIssues.reduce((helpTexts: string[], knownIssue) => { + const helpText = logsToHelpMap[knownIssue]; + if (logs.includes(knownIssue)) { + helpTexts.push(helpText); + } + return helpTexts; + }, []); + + const response: DiagnosticResponse = { + success: boundSuccessfully && !help.length, + help, + logs, + }; + + return res.ok({ body: response }); + }) + ); +}; diff --git a/x-pack/plugins/reporting/server/routes/diagnostic/config.test.ts b/x-pack/plugins/reporting/server/routes/diagnostic/config.test.ts new file mode 100644 index 0000000000000..624397246656d --- /dev/null +++ b/x-pack/plugins/reporting/server/routes/diagnostic/config.test.ts @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { UnwrapPromise } from '@kbn/utility-types'; +import { setupServer } from 'src/core/server/test_utils'; +import supertest from 'supertest'; +import { ReportingCore } from '../..'; +import { createMockReportingCore, createMockLevelLogger } from '../../test_helpers'; +import { registerDiagnoseConfig } from './config'; + +type SetupServerReturn = UnwrapPromise>; + +describe('POST /diagnose/config', () => { + const reportingSymbol = Symbol('reporting'); + let server: SetupServerReturn['server']; + let httpSetup: SetupServerReturn['httpSetup']; + let core: ReportingCore; + let mockSetupDeps: any; + let config: any; + + const mockLogger = createMockLevelLogger(); + + beforeEach(async () => { + ({ server, httpSetup } = await setupServer(reportingSymbol)); + httpSetup.registerRouteHandlerContext(reportingSymbol, 'reporting', () => ({})); + + mockSetupDeps = ({ + elasticsearch: { + legacy: { client: { callAsInternalUser: jest.fn() } }, + }, + router: httpSetup.createRouter(''), + } as unknown) as any; + + config = { + get: jest.fn(), + kbnConfig: { get: jest.fn() }, + }; + + core = await createMockReportingCore(config, mockSetupDeps); + }); + + afterEach(async () => { + await server.stop(); + }); + + it('returns a 200 by default when configured properly', async () => { + mockSetupDeps.elasticsearch.legacy.client.callAsInternalUser.mockImplementation(() => + Promise.resolve({ + defaults: { + http: { + max_content_length: '100mb', + }, + }, + }) + ); + registerDiagnoseConfig(core, mockLogger); + + await server.start(); + + await supertest(httpSetup.server.listener) + .post('/api/reporting/diagnose/config') + .expect(200) + .then(({ body }) => { + expect(body).toMatchInlineSnapshot(` + Object { + "help": Array [], + "logs": "", + "success": true, + } + `); + }); + }); + + it('returns a 200 with help text when not configured properly', async () => { + config.get.mockImplementation(() => 10485760); + mockSetupDeps.elasticsearch.legacy.client.callAsInternalUser.mockImplementation(() => + Promise.resolve({ + defaults: { + http: { + max_content_length: '5mb', + }, + }, + }) + ); + registerDiagnoseConfig(core, mockLogger); + + await server.start(); + + await supertest(httpSetup.server.listener) + .post('/api/reporting/diagnose/config') + .expect(200) + .then(({ body }) => { + expect(body).toMatchInlineSnapshot(` + Object { + "help": Array [ + "xpack.reporting.csv.maxSizeBytes (10485760) is higher than ElasticSearch's http.max_content_length (5242880). Please set http.max_content_length in ElasticSearch to match, or lower your xpack.reporting.csv.maxSizeBytes in Kibana.", + ], + "logs": "xpack.reporting.csv.maxSizeBytes (10485760) is higher than ElasticSearch's http.max_content_length (5242880). Please set http.max_content_length in ElasticSearch to match, or lower your xpack.reporting.csv.maxSizeBytes in Kibana.", + "success": false, + } + `); + }); + }); +}); diff --git a/x-pack/plugins/reporting/server/routes/diagnostic/config.ts b/x-pack/plugins/reporting/server/routes/diagnostic/config.ts new file mode 100644 index 0000000000000..198ba63e2614d --- /dev/null +++ b/x-pack/plugins/reporting/server/routes/diagnostic/config.ts @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; +import numeral from '@elastic/numeral'; +import { defaults, get } from 'lodash'; +import { ReportingCore } from '../..'; +import { API_DIAGNOSE_URL } from '../../../common/constants'; +import { LevelLogger as Logger } from '../../lib'; +import { DiagnosticResponse } from '../../types'; +import { authorizedUserPreRoutingFactory } from '../lib/authorized_user_pre_routing'; + +const KIBANA_MAX_SIZE_BYTES_PATH = 'csv.maxSizeBytes'; +const ES_MAX_SIZE_BYTES_PATH = 'http.max_content_length'; + +export const registerDiagnoseConfig = (reporting: ReportingCore, logger: Logger) => { + const setupDeps = reporting.getPluginSetupDeps(); + const userHandler = authorizedUserPreRoutingFactory(reporting); + const { router, elasticsearch } = setupDeps; + + router.post( + { + path: `${API_DIAGNOSE_URL}/config`, + validate: {}, + }, + userHandler(async (user, context, req, res) => { + const warnings = []; + const { callAsInternalUser } = elasticsearch.legacy.client; + const config = reporting.getConfig(); + + const elasticClusterSettingsResponse = await callAsInternalUser('cluster.getSettings', { + includeDefaults: true, + }); + const { persistent, transient, defaults: defaultSettings } = elasticClusterSettingsResponse; + const elasticClusterSettings = defaults({}, persistent, transient, defaultSettings); + + const elasticSearchMaxContent = get( + elasticClusterSettings, + 'http.max_content_length', + '100mb' + ); + const elasticSearchMaxContentBytes = numeral().unformat( + elasticSearchMaxContent.toUpperCase() + ); + const kibanaMaxContentBytes = config.get('csv', 'maxSizeBytes'); + + if (kibanaMaxContentBytes > elasticSearchMaxContentBytes) { + const maxContentSizeWarning = i18n.translate( + 'xpack.reporting.diagnostic.configSizeMismatch', + { + defaultMessage: + `xpack.reporting.{KIBANA_MAX_SIZE_BYTES_PATH} ({kibanaMaxContentBytes}) is higher than ElasticSearch's {ES_MAX_SIZE_BYTES_PATH} ({elasticSearchMaxContentBytes}). ` + + `Please set {ES_MAX_SIZE_BYTES_PATH} in ElasticSearch to match, or lower your xpack.reporting.{KIBANA_MAX_SIZE_BYTES_PATH} in Kibana.`, + values: { + kibanaMaxContentBytes, + elasticSearchMaxContentBytes, + KIBANA_MAX_SIZE_BYTES_PATH, + ES_MAX_SIZE_BYTES_PATH, + }, + } + ); + warnings.push(maxContentSizeWarning); + } + + if (warnings.length) { + warnings.forEach((warn) => logger.warn(warn)); + } + + const body: DiagnosticResponse = { + help: warnings, + success: !warnings.length, + logs: warnings.join('\n'), + }; + + return res.ok({ body }); + }) + ); +}; diff --git a/x-pack/plugins/reporting/server/routes/diagnostic/index.ts b/x-pack/plugins/reporting/server/routes/diagnostic/index.ts new file mode 100644 index 0000000000000..895dee32614f1 --- /dev/null +++ b/x-pack/plugins/reporting/server/routes/diagnostic/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { registerDiagnoseBrowser } from './browser'; +import { registerDiagnoseConfig } from './config'; +import { registerDiagnoseScreenshot } from './screenshot'; +import { LevelLogger as Logger } from '../../lib'; +import { ReportingCore } from '../../core'; + +export const registerDiagnosticRoutes = (reporting: ReportingCore, logger: Logger) => { + registerDiagnoseBrowser(reporting, logger); + registerDiagnoseConfig(reporting, logger); + registerDiagnoseScreenshot(reporting, logger); +}; diff --git a/x-pack/plugins/reporting/server/routes/diagnostic/screenshot.test.ts b/x-pack/plugins/reporting/server/routes/diagnostic/screenshot.test.ts new file mode 100644 index 0000000000000..ec4ab0446ae5f --- /dev/null +++ b/x-pack/plugins/reporting/server/routes/diagnostic/screenshot.test.ts @@ -0,0 +1,112 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { UnwrapPromise } from '@kbn/utility-types'; +import { setupServer } from 'src/core/server/test_utils'; +import supertest from 'supertest'; +import { ReportingCore } from '../..'; +import { createMockReportingCore, createMockLevelLogger } from '../../test_helpers'; +import { registerDiagnoseScreenshot } from './screenshot'; + +jest.mock('../../export_types/png/lib/generate_png'); + +import { generatePngObservableFactory } from '../../export_types/png/lib/generate_png'; + +type SetupServerReturn = UnwrapPromise>; + +describe('POST /diagnose/screenshot', () => { + const reportingSymbol = Symbol('reporting'); + let server: SetupServerReturn['server']; + let httpSetup: SetupServerReturn['httpSetup']; + let core: ReportingCore; + + const setScreenshotResponse = (resp: object | Error) => { + const generateMock = Promise.resolve(() => ({ + pipe: () => ({ + toPromise: () => (resp instanceof Error ? Promise.reject(resp) : Promise.resolve(resp)), + }), + })); + (generatePngObservableFactory as any).mockResolvedValue(generateMock); + }; + + const config = { + get: jest.fn(), + kbnConfig: { get: jest.fn() }, + }; + const mockLogger = createMockLevelLogger(); + + beforeEach(async () => { + ({ server, httpSetup } = await setupServer(reportingSymbol)); + httpSetup.registerRouteHandlerContext(reportingSymbol, 'reporting', () => ({})); + + const mockSetupDeps = ({ + elasticsearch: { + legacy: { client: { callAsInternalUser: jest.fn() } }, + }, + router: httpSetup.createRouter(''), + } as unknown) as any; + + core = await createMockReportingCore(config, mockSetupDeps); + }); + + afterEach(async () => { + await server.stop(); + }); + + it('returns a 200 by default', async () => { + registerDiagnoseScreenshot(core, mockLogger); + setScreenshotResponse({ warnings: [] }); + await server.start(); + + await supertest(httpSetup.server.listener) + .post('/api/reporting/diagnose/screenshot') + .expect(200) + .then(({ body }) => { + expect(body).toMatchInlineSnapshot(` + Object { + "help": Array [], + "logs": "", + "success": true, + } + `); + }); + }); + + it('returns a 200 when it fails and sets success to false', async () => { + registerDiagnoseScreenshot(core, mockLogger); + setScreenshotResponse({ warnings: [`Timeout waiting for .dank to load`] }); + await server.start(); + + await supertest(httpSetup.server.listener) + .post('/api/reporting/diagnose/screenshot') + .expect(200) + .then(({ body }) => { + expect(body).toMatchInlineSnapshot(` + Object { + "help": Array [], + "logs": Array [ + "Timeout waiting for .dank to load", + ], + "success": false, + } + `); + }); + }); + + it('catches errors and returns a well formed response', async () => { + registerDiagnoseScreenshot(core, mockLogger); + setScreenshotResponse(new Error('Failure to start chromium!')); + await server.start(); + + await supertest(httpSetup.server.listener) + .post('/api/reporting/diagnose/screenshot') + .expect(200) + .then(({ body }) => { + expect(body.help).toContain(`We couldn't screenshot your Kibana install.`); + expect(body.logs).toContain(`Failure to start chromium!`); + }); + }); +}); diff --git a/x-pack/plugins/reporting/server/routes/diagnostic/screenshot.ts b/x-pack/plugins/reporting/server/routes/diagnostic/screenshot.ts new file mode 100644 index 0000000000000..7e07779b5fd37 --- /dev/null +++ b/x-pack/plugins/reporting/server/routes/diagnostic/screenshot.ts @@ -0,0 +1,116 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; +import { ReportingCore } from '../..'; +import { API_DIAGNOSE_URL } from '../../../common/constants'; +import { omitBlacklistedHeaders } from '../../export_types/common'; +import { getAbsoluteUrlFactory } from '../../export_types/common/get_absolute_url'; +import { generatePngObservableFactory } from '../../export_types/png/lib/generate_png'; +import { LevelLogger as Logger } from '../../lib'; +import { DiagnosticResponse } from '../../types'; +import { authorizedUserPreRoutingFactory } from '../lib/authorized_user_pre_routing'; + +export const registerDiagnoseScreenshot = (reporting: ReportingCore, logger: Logger) => { + const setupDeps = reporting.getPluginSetupDeps(); + const userHandler = authorizedUserPreRoutingFactory(reporting); + const { router } = setupDeps; + + router.post( + { + path: `${API_DIAGNOSE_URL}/screenshot`, + validate: {}, + }, + userHandler(async (user, context, req, res) => { + const generatePngObservable = await generatePngObservableFactory(reporting); + const config = reporting.getConfig(); + const decryptedHeaders = req.headers as Record; + const [basePath, protocol, hostname, port] = [ + config.kbnConfig.get('server', 'basePath'), + config.get('kibanaServer', 'protocol'), + config.get('kibanaServer', 'hostname'), + config.get('kibanaServer', 'port'), + ] as string[]; + + const getAbsoluteUrl = getAbsoluteUrlFactory({ + defaultBasePath: basePath, + protocol, + hostname, + port, + }); + + const hashUrl = getAbsoluteUrl({ + basePath, + path: '/', + hash: '', + search: '', + }); + + // Hack the layout to make the base/login page work + const layout = { + id: 'png', + dimensions: { + width: 1440, + height: 2024, + }, + selectors: { + screenshot: '.application', + renderComplete: '.application', + itemsCountAttribute: 'data-test-subj="kibanaChrome"', + timefilterDurationAttribute: 'data-test-subj="kibanaChrome"', + }, + }; + + const headers = { + headers: omitBlacklistedHeaders({ + job: null, + decryptedHeaders, + }), + conditions: { + hostname, + port: +port, + basePath, + protocol, + }, + }; + + return generatePngObservable(logger, hashUrl, 'America/Los_Angeles', headers, layout) + .pipe() + .toPromise() + .then((screenshot) => { + if (screenshot.warnings.length) { + return res.ok({ + body: { + success: false, + help: [], + logs: screenshot.warnings, + }, + }); + } + return res.ok({ + body: { + success: true, + help: [], + logs: '', + } as DiagnosticResponse, + }); + }) + .catch((error) => + res.ok({ + body: { + success: false, + help: [ + i18n.translate('xpack.reporting.diagnostic.screenshotFailureMessage', { + defaultMessage: `We couldn't screenshot your Kibana install.`, + }), + ], + logs: error.message, + } as DiagnosticResponse, + }) + ); + }) + ); +}; diff --git a/x-pack/plugins/reporting/server/routes/generation.test.ts b/x-pack/plugins/reporting/server/routes/generation.test.ts index 0db0073149e57..dd905223a81d5 100644 --- a/x-pack/plugins/reporting/server/routes/generation.test.ts +++ b/x-pack/plugins/reporting/server/routes/generation.test.ts @@ -11,8 +11,7 @@ import { setupServer } from 'src/core/server/test_utils'; import supertest from 'supertest'; import { ReportingCore } from '..'; import { ExportTypesRegistry } from '../lib/export_types_registry'; -import { createMockReportingCore } from '../test_helpers'; -import { createMockLevelLogger } from '../test_helpers/create_mock_levellogger'; +import { createMockReportingCore, createMockLevelLogger } from '../test_helpers'; import { registerJobGenerationRoutes } from './generation'; type SetupServerReturn = UnwrapPromise>; diff --git a/x-pack/plugins/reporting/server/routes/index.ts b/x-pack/plugins/reporting/server/routes/index.ts index 005d82086665c..11ad4cc9d4eb8 100644 --- a/x-pack/plugins/reporting/server/routes/index.ts +++ b/x-pack/plugins/reporting/server/routes/index.ts @@ -8,8 +8,10 @@ import { LevelLogger as Logger } from '../lib'; import { registerJobGenerationRoutes } from './generation'; import { registerJobInfoRoutes } from './jobs'; import { ReportingCore } from '../core'; +import { registerDiagnosticRoutes } from './diagnostic'; export function registerRoutes(reporting: ReportingCore, logger: Logger) { registerJobGenerationRoutes(reporting, logger); registerJobInfoRoutes(reporting); + registerDiagnosticRoutes(reporting, logger); } diff --git a/x-pack/plugins/reporting/server/test_helpers/create_mock_reportingplugin.ts b/x-pack/plugins/reporting/server/test_helpers/create_mock_reportingplugin.ts index c508ee6974ca0..d1ebb4d59e631 100644 --- a/x-pack/plugins/reporting/server/test_helpers/create_mock_reportingplugin.ts +++ b/x-pack/plugins/reporting/server/test_helpers/create_mock_reportingplugin.ts @@ -8,7 +8,6 @@ jest.mock('../routes'); jest.mock('../usage'); jest.mock('../browsers'); jest.mock('../lib/create_queue'); -jest.mock('../lib/validate'); import * as Rx from 'rxjs'; import { ReportingConfig, ReportingCore } from '../'; diff --git a/x-pack/plugins/reporting/server/test_helpers/index.ts b/x-pack/plugins/reporting/server/test_helpers/index.ts index b37b447dc05a9..2d5ef9fdd768d 100644 --- a/x-pack/plugins/reporting/server/test_helpers/index.ts +++ b/x-pack/plugins/reporting/server/test_helpers/index.ts @@ -8,3 +8,4 @@ export { createMockServer } from './create_mock_server'; export { createMockReportingCore, createMockConfigSchema } from './create_mock_reportingplugin'; export { createMockBrowserDriverFactory } from './create_mock_browserdriverfactory'; export { createMockLayoutInstance } from './create_mock_layoutinstance'; +export { createMockLevelLogger } from './create_mock_levellogger'; diff --git a/x-pack/plugins/reporting/server/types.ts b/x-pack/plugins/reporting/server/types.ts index 10519842d9dec..bb2d5368cd181 100644 --- a/x-pack/plugins/reporting/server/types.ts +++ b/x-pack/plugins/reporting/server/types.ts @@ -160,3 +160,9 @@ export interface ExportTypeDefinition< runTaskFnFactory: RunTaskFnFactory; validLicenses: string[]; } + +export interface DiagnosticResponse { + help: string[]; + success: boolean; + logs: string; +} diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts index 0955f196df176..e1ff34463d215 100644 --- a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts @@ -1132,7 +1132,8 @@ export class EndpointDocGenerator { path: '/package/endpoint/0.5.0', icons: [ { - src: '/package/endpoint/0.5.0/img/logo-endpoint-64-color.svg', + path: '/package/endpoint/0.5.0/img/logo-endpoint-64-color.svg', + src: '/img/logo-endpoint-64-color.svg', size: '16x16', type: 'image/svg+xml', }, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json index 7d80a319e9e52..cfce019910071 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json @@ -22,11 +22,33 @@ } } }, + "parents": { + "properties": { + "rule": { + "type": "keyword" + }, + "index": { + "type": "keyword" + }, + "id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "depth": { + "type": "long" + } + } + }, "ancestors": { "properties": { "rule": { "type": "keyword" }, + "index": { + "type": "keyword" + }, "id": { "type": "keyword" }, @@ -299,6 +321,9 @@ }, "threshold_count": { "type": "float" + }, + "depth": { + "type": "integer" } } } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/README.md b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/README.md index 2310ba979da20..7cf7d11e4c1f8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/README.md +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/README.md @@ -22,7 +22,7 @@ which will write a single signal document into the signals index by searching fo signal_on_signal_depth_1.json ``` -which has this key part of its query: `"query": "signal.parent.depth: 1 and _id: *"` which will only create signals +which has this key part of its query: `"query": "signal.depth: 1 and _id: *"` which will only create signals from all signals that point directly to an event (signal -> event). Then a second rule called @@ -34,7 +34,7 @@ signal_on_signal_depth_2.json which will only create signals from all signals that point directly to another signal (signal -> signal) with this query ```json -"query": "signal.parent.depth: 2 and _id: *" +"query": "signal.depth: 2 and _id: *" ``` ## Setup @@ -90,38 +90,43 @@ And then you can query against that: GET .siem-signals-default/_search ``` -Check your parent section of the signal and you will see something like this: +Check your `signal` section of the signal and you will see something like this: ```json -"parent" : { - "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", - "id" : "o8G7vm8BvLT8jmu5B1-M", - "type" : "event", - "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 -}, -"ancestors" : [ +"parents" : [ { - "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 } -] +], +"ancestors" : [ + { + "id" : "o8G7vm8BvLT8jmu5B1-M", + "type" : "event", + "index" : "filebeat-8.0.0-2019.12.18-000001", + "depth" : 0 + }, +], +"depth": 1, +"rule": { + "id": "74e0dd0c-4609-416f-b65e-90f8b2564612" +} ``` -The parent and ancestors structure is defined as: +The parents structure is defined as: ``` -rule -> The id of the rule. You can view the rule by ./get_rule_by_rule_id.sh ded57b36-9c4e-4ee4-805d-be4e92033e41 +rule -> The id of the rule, if the parent was generated by a rule. You can view the rule by ./get_rule_by_rule_id.sh ded57b36-9c4e-4ee4-805d-be4e92033e41 id -> The original _id of the document type -> The type of the document, it will be either event or signal index -> The original location of the index -depth -> The depth of this signal. It will be at least 1 to indicate it is a signal generated from a event. Otherwise 2 or more to indicate a signal on signal and what depth we are at -ancestors -> An array tracking all of the parents of this particular signal. As depth increases this will too. +depth -> The depth of the parent event/signal. It will be 0 if the parent is an event, or 1+ if the parent is another signal. ``` +The ancestors structure has the same fields as parents, but is an array of all ancestors (parents, grandparents, etc) of the signal. + This is indicating that you have a single parent of an event from the signal (signal -> event) and this document has a single ancestor of that event. Each 30 seconds that goes it will use de-duplication technique to ensure that this signal is not re-inserted. If after each 30 seconds you DO SEE multiple signals then the bug is a de-duplication bug and a critical bug. If you ever see a duplicate rule in the @@ -138,55 +143,64 @@ running in the system which are generating signals on top of signals. After 30 s documents in the signals index. The first signal is our original (signal -> event) document with a rule id: ```json -"parent" : { - "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", - "id" : "o8G7vm8BvLT8jmu5B1-M", - "type" : "event", - "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 -}, +"parents" : [ + { + "id" : "o8G7vm8BvLT8jmu5B1-M", + "type" : "event", + "index" : "filebeat-8.0.0-2019.12.18-000001", + "depth" : 0 + } +], "ancestors" : [ { - "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 } -] +], +"depth": 1, +"rule": { + "id": "74e0dd0c-4609-416f-b65e-90f8b2564612" +} ``` and the second document is a signal on top of a signal like so: ```json -"parent" : { - "rule" : "1d3b3735-66ef-4e53-b7f5-4340026cc40c", - "id" : "4cc69c1cbecdd2ace4075fd1d8a5c28e7d46e4bf31aecc8d2da39252c50c96b4", - "type" : "signal", - "index" : ".siem-signals-default-000001", - "depth" : 2 -}, -"ancestors" : [ +"parents" : [ { "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", + "id" : "4cc69c1cbecdd2ace4075fd1d8a5c28e7d46e4bf31aecc8d2da39252c50c96b4", + "type" : "signal", + "index" : ".siem-signals-default-000001", + "depth" : 1 + } +] +"ancestors" : [ + { "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 }, { - "rule" : "1d3b3735-66ef-4e53-b7f5-4340026cc40c", + "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", "id" : "4cc69c1cbecdd2ace4075fd1d8a5c28e7d46e4bf31aecc8d2da39252c50c96b4", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 2 + "depth" : 1 } -] +], +"depth": 2, +"rule": { + "id": "1d3b3735-66ef-4e53-b7f5-4340026cc40c" +} ``` Notice that the depth indicates it is at level 2 and its parent is that of a signal. Also notice that the ancestors is an array of size 2 indicating that this signal terminates at an event. Each and every signal ancestors array should terminate at an event and should ONLY contain 1 -event and NEVER 2 or more events. After 30+ seconds you should NOT see any new documents being created and you should be stable +event and NEVER 2 or more events for KQL query based rules. EQL query based rules that use sequences may have multiple parents at the same level. After 30+ seconds you should NOT see any new documents being created and you should be stable at 2. Otherwise we have AND/OR a de-duplication issue, signal on signal issue. Now, post this same rule a second time as a second instance which is going to run against these two documents. @@ -212,79 +226,93 @@ The expected behavior is that eventually you will get 3 total documents but not The original event rule 74e0dd0c-4609-416f-b65e-90f8b2564612 (event -> signal) ```json -"parent" : { - "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", - "id" : "o8G7vm8BvLT8jmu5B1-M", - "type" : "event", - "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 -}, +"parents" : [ + { + "id" : "o8G7vm8BvLT8jmu5B1-M", + "type" : "event", + "index" : "filebeat-8.0.0-2019.12.18-000001", + "depth" : 0 + } +], "ancestors" : [ { - "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 } -] +], +"depth": 1, +"rule": { + "id": "74e0dd0c-4609-416f-b65e-90f8b2564612" +} ``` The first signal to signal rule 1d3b3735-66ef-4e53-b7f5-4340026cc40c (signal -> event) ```json -"parent" : { - "rule" : "1d3b3735-66ef-4e53-b7f5-4340026cc40c", - "id" : "4cc69c1cbecdd2ace4075fd1d8a5c28e7d46e4bf31aecc8d2da39252c50c96b4", - "type" : "signal", - "index" : ".siem-signals-default-000001", - "depth" : 2 -}, -"ancestors" : [ +"parents" : [ { "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", + "id" : "4cc69c1cbecdd2ace4075fd1d8a5c28e7d46e4bf31aecc8d2da39252c50c96b4", + "type" : "signal", + "index" : ".siem-signals-default-000001", + "depth" : 1 + } +] +"ancestors" : [ + { "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 }, { - "rule" : "1d3b3735-66ef-4e53-b7f5-4340026cc40c", + "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", "id" : "4cc69c1cbecdd2ace4075fd1d8a5c28e7d46e4bf31aecc8d2da39252c50c96b4", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 2 + "depth" : 1 } -] +], +"depth": 2, +"rule": { + "id": "1d3b3735-66ef-4e53-b7f5-4340026cc40c" +} ``` Then our second signal to signal rule c93ddb57-e7e9-4973-9886-72ddefb4d22e (signal -> event) which finds the same thing as the first signal to signal ```json -"parent" : { - "rule" : "c93ddb57-e7e9-4973-9886-72ddefb4d22e", - "id" : "4cc69c1cbecdd2ace4075fd1d8a5c28e7d46e4bf31aecc8d2da39252c50c96b4", - "type" : "signal", - "index" : ".siem-signals-default-000001", - "depth" : 2 -}, -"ancestors" : [ +"parents" : [ { "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", + "id" : "4cc69c1cbecdd2ace4075fd1d8a5c28e7d46e4bf31aecc8d2da39252c50c96b4", + "type" : "signal", + "index" : ".siem-signals-default-000001", + "depth" : 1 + } +], +"ancestors" : [ + { "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 }, { - "rule" : "c93ddb57-e7e9-4973-9886-72ddefb4d22e", + "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", "id" : "4cc69c1cbecdd2ace4075fd1d8a5c28e7d46e4bf31aecc8d2da39252c50c96b4", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 2 + "depth" : 1 } -] +], +"depth": 2, +"rule": { + "id": "c93ddb57-e7e9-4973-9886-72ddefb4d22e" +} ``` We should be able to post this depth level as many times as we want and get only 1 new document each time. If we decide though to @@ -298,69 +326,79 @@ The expectation is that a document for each of the previous depth 1 documents wo depth 1 rules running then the signals at depth 2 will produce two new ones and those two will look like so: ```json -"parent" : { - "rule" : "a1f7b520-5bfd-451d-af59-428f60753fee", - "id" : "365236ce5e77770508152403b4e16613f407ae4b1a135a450dcfec427f2a3231", - "type" : "signal", - "index" : ".siem-signals-default-000001", - "depth" : 3 -}, +"parents" : [ + { + "rule" : "1d3b3735-66ef-4e53-b7f5-4340026cc40c", + "id" : "365236ce5e77770508152403b4e16613f407ae4b1a135a450dcfec427f2a3231", + "type" : "signal", + "index" : ".siem-signals-default-000001", + "depth" : 2 + } +], "ancestors" : [ { - "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 }, { - "rule" : "1d3b3735-66ef-4e53-b7f5-4340026cc40c", + "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", "id" : "4cc69c1cbecdd2ace4075fd1d8a5c28e7d46e4bf31aecc8d2da39252c50c96b4", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 2 + "depth" : 1 }, { - "rule" : "a1f7b520-5bfd-451d-af59-428f60753fee", + "rule" : "1d3b3735-66ef-4e53-b7f5-4340026cc40c", "id" : "365236ce5e77770508152403b4e16613f407ae4b1a135a450dcfec427f2a3231", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 3 + "depth" : 2 } -] +], +"depth": 3, +"rule": { + "id": "a1f7b520-5bfd-451d-af59-428f60753fee" +} ``` ```json -"parent" : { - "rule" : "a1f7b520-5bfd-451d-af59-428f60753fee", - "id" : "e8b1f1adb40fd642fa524dea89ef94232e67b05e99fb0b2683f1e47e90b759fb", - "type" : "signal", - "index" : ".siem-signals-default-000001", - "depth" : 3 -}, +"parents" : [ + { + "rule" : "c93ddb57-e7e9-4973-9886-72ddefb4d22e", + "id" : "e8b1f1adb40fd642fa524dea89ef94232e67b05e99fb0b2683f1e47e90b759fb", + "type" : "signal", + "index" : ".siem-signals-default-000001", + "depth" : 2 + } +], "ancestors" : [ { - "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 }, { - "rule" : "c93ddb57-e7e9-4973-9886-72ddefb4d22e", + "rule" : "74e0dd0c-4609-416f-b65e-90f8b2564612", "id" : "4cc69c1cbecdd2ace4075fd1d8a5c28e7d46e4bf31aecc8d2da39252c50c96b4", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 2 + "depth" : 1 }, { - "rule" : "a1f7b520-5bfd-451d-af59-428f60753fee", + "rule" : "c93ddb57-e7e9-4973-9886-72ddefb4d22e", "id" : "e8b1f1adb40fd642fa524dea89ef94232e67b05e99fb0b2683f1e47e90b759fb", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 3 + "depth" : 2 } -] +], +"depth": 3, +"rule": { + "id": "a1f7b520-5bfd-451d-af59-428f60753fee" +} ``` The total number of documents should be 5 at this point. If you were to post this same rule a second time to get a second instance diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/query_single_id.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/query_single_id.json index dc05c656d7cf1..305aa34992623 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/query_single_id.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/query_single_id.json @@ -7,6 +7,6 @@ "from": "now-1d", "interval": "30s", "to": "now", - "query": "_id: o8G7vm8BvLT8jmu5B1-M", + "query": "event.id: 08cde4aa-d249-4e6b-8300-06f3d56c7fe7", "enabled": true } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/signal_on_signal_depth_1.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/signal_on_signal_depth_1.json index fb13413a02791..c9132ddb0a590 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/signal_on_signal_depth_1.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/signal_on_signal_depth_1.json @@ -7,7 +7,7 @@ "from": "now-1d", "interval": "30s", "to": "now", - "query": "signal.parent.depth: 1 and _id: *", + "query": "signal.depth: 1 and _id: *", "enabled": true, - "index": ".siem-signals-default" + "index": [".siem-signals-default"] } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/signal_on_signal_depth_2.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/signal_on_signal_depth_2.json index c1b7594653ec7..d1a2749792686 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/signal_on_signal_depth_2.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/depth_test/signal_on_signal_depth_2.json @@ -7,7 +7,7 @@ "from": "now-1d", "interval": "30s", "to": "now", - "query": "signal.parent.depth: 2 and _id: *", + "query": "signal.depth: 2 and _id: *", "enabled": true, - "index": ".siem-signals-default" + "index": [".siem-signals-default"] } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/halting_test/README.md b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/halting_test/README.md index b1a83f5317776..01b21bf762e44 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/halting_test/README.md +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/test_cases/signals_on_signals/halting_test/README.md @@ -69,38 +69,43 @@ And then you can query against that: GET .siem-signals-default/_search ``` -Check your parent section of the signal and you will see something like this: +Check your `signal` section of the signal and you will see something like this: ```json -"parent" : { - "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", - "id" : "o8G7vm8BvLT8jmu5B1-M", - "type" : "event", - "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 -}, -"ancestors" : [ +"parents" : [ { - "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 } -] +], +"ancestors" : [ + { + "id" : "o8G7vm8BvLT8jmu5B1-M", + "type" : "event", + "index" : "filebeat-8.0.0-2019.12.18-000001", + "depth" : 0 + }, +], +"depth": 1, +"rule": { + "id": "ded57b36-9c4e-4ee4-805d-be4e92033e41" +} ``` -The parent and ancestors structure is defined as: +The parents structure is defined as: ``` -rule -> The id of the rule. You can view the rule by ./get_rule_by_rule_id.sh ded57b36-9c4e-4ee4-805d-be4e92033e41 +rule -> The id of the rule, if the parent was generated by a rule. You can view the rule by ./get_rule_by_rule_id.sh ded57b36-9c4e-4ee4-805d-be4e92033e41 id -> The original _id of the document type -> The type of the document, it will be either event or signal index -> The original location of the index -depth -> The depth of this signal. It will be at least 1 to indicate it is a signal generated from a event. Otherwise 2 or more to indicate a signal on signal and what depth we are at -ancestors -> An array tracking all of the parents of this particular signal. As depth increases this will too. +depth -> The depth of the parent event/signal. It will be 0 if the parent is an event, or 1+ if the parent is another signal. ``` +The ancestors structure has the same fields as parents, but is an array of all ancestors (parents, grandparents, etc) of the signal. + This is indicating that you have a single parent of an event from the signal (signal -> event) and this document has a single ancestor of that event. Each 30 seconds that goes it will use de-duplication technique to ensure that this signal is not re-inserted. If after each 30 seconds you DO SEE multiple signals then the bug is a de-duplication bug and a critical bug. If you ever see a duplicate rule in the @@ -119,22 +124,26 @@ documents in the signals index. The first signal is our original (signal -> even (signal -> event) ```json -"parent" : { - "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", - "id" : "o8G7vm8BvLT8jmu5B1-M", - "type" : "event", - "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 -}, -"ancestors" : [ +"parents" : [ { - "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 } -] +], +"ancestors" : [ + { + "id" : "o8G7vm8BvLT8jmu5B1-M", + "type" : "event", + "index" : "filebeat-8.0.0-2019.12.18-000001", + "depth" : 0 + }, +], +"depth": 1, +"rule": { + "id": "ded57b36-9c4e-4ee4-805d-be4e92033e41" +} ``` and the second document is a signal on top of a signal like so: @@ -143,28 +152,31 @@ and the second document is a signal on top of a signal like so: ```json "parent" : { - "rule" : "161fa5b8-0b96-4985-b066-0d99b2bcb904", + "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "9d8710925adbf1a9c469621805407e74334dd08ca2c2ea414840fe971a571938", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 2 + "depth" : 1 }, "ancestors" : [ { - "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 }, { - "rule" : "161fa5b8-0b96-4985-b066-0d99b2bcb904", + "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "9d8710925adbf1a9c469621805407e74334dd08ca2c2ea414840fe971a571938", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 2 + "depth" : 1 } -] +], +"depth": 2, +"rule": { + "id": "161fa5b8-0b96-4985-b066-0d99b2bcb904" +} ``` Notice that the depth indicates it is at level 2 and its parent is that of a signal. Also notice that the ancestors is an array of size 2 @@ -195,50 +207,57 @@ The expected behavior is that eventually you will get 5 total documents but not The original event rule ded57b36-9c4e-4ee4-805d-be4e92033e41 (event -> signal) ```json -"parent" : { - "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", - "id" : "o8G7vm8BvLT8jmu5B1-M", - "type" : "event", - "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 -}, -"ancestors" : [ +"parents" : [ { - "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 } -] +], +"ancestors" : [ + { + "id" : "o8G7vm8BvLT8jmu5B1-M", + "type" : "event", + "index" : "filebeat-8.0.0-2019.12.18-000001", + "depth" : 0 + }, +], +"depth": 1, +"rule": { + "id": "ded57b36-9c4e-4ee4-805d-be4e92033e41" +} ``` The first signal to signal rule 161fa5b8-0b96-4985-b066-0d99b2bcb904 (signal -> event) ```json "parent" : { - "rule" : "161fa5b8-0b96-4985-b066-0d99b2bcb904", + "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "9d8710925adbf1a9c469621805407e74334dd08ca2c2ea414840fe971a571938", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 2 + "depth" : 1 }, "ancestors" : [ { - "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 }, { - "rule" : "161fa5b8-0b96-4985-b066-0d99b2bcb904", + "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "9d8710925adbf1a9c469621805407e74334dd08ca2c2ea414840fe971a571938", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 2 + "depth" : 1 } -] +], +"depth": 2, +"rule": { + "id": "161fa5b8-0b96-4985-b066-0d99b2bcb904" +} ``` Then our second signal to signal rule f2b70c4a-4d8f-4db5-9ed7-d3ab0630e406 (signal -> event) which finds the same thing as the first @@ -246,28 +265,31 @@ signal to signal ```json "parent" : { - "rule" : "f2b70c4a-4d8f-4db5-9ed7-d3ab0630e406", + "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "9d8710925adbf1a9c469621805407e74334dd08ca2c2ea414840fe971a571938", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 2 + "depth" : 1 }, "ancestors" : [ { - "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 }, { - "rule" : "f2b70c4a-4d8f-4db5-9ed7-d3ab0630e406", + "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "9d8710925adbf1a9c469621805407e74334dd08ca2c2ea414840fe971a571938", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 2 + "depth" : 1 } -] +], +"depth": 2, +"rule": { + "id": "f2b70c4a-4d8f-4db5-9ed7-d3ab0630e406" +} ``` But then f2b70c4a-4d8f-4db5-9ed7-d3ab0630e406 also finds the first signal to signal rule from 161fa5b8-0b96-4985-b066-0d99b2bcb904 @@ -275,35 +297,38 @@ and writes that document out with a depth of 3. (signal -> signal -> event) ```json "parent" : { - "rule" : "f2b70c4a-4d8f-4db5-9ed7-d3ab0630e406", + "rule" : "161fa5b8-0b96-4985-b066-0d99b2bcb904", "id" : "c627e5e2576f1b10952c6c57249947e89b6153b763a59fb9e391d0b56be8e7fe", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 3 + "depth" : 2 }, "ancestors" : [ { - "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 }, { - "rule" : "161fa5b8-0b96-4985-b066-0d99b2bcb904", + "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "9d8710925adbf1a9c469621805407e74334dd08ca2c2ea414840fe971a571938", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 2 + "depth" : 1 }, { - "rule" : "f2b70c4a-4d8f-4db5-9ed7-d3ab0630e406", + "rule" : "161fa5b8-0b96-4985-b066-0d99b2bcb904", "id" : "c627e5e2576f1b10952c6c57249947e89b6153b763a59fb9e391d0b56be8e7fe", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 3 + "depth" : 2 } -] +], +"depth": 3, +"rule": { + "id": "f2b70c4a-4d8f-4db5-9ed7-d3ab0630e406" +} ``` Since it wrote that document, the first signal to signal 161fa5b8-0b96-4985-b066-0d99b2bcb904 writes out it found this newly created signal @@ -311,35 +336,38 @@ Since it wrote that document, the first signal to signal 161fa5b8-0b96-4985-b066 ```json "parent" : { - "rule" : "161fa5b8-0b96-4985-b066-0d99b2bcb904", + "rule" : "f2b70c4a-4d8f-4db5-9ed7-d3ab0630e406", "id" : "efbe514e8d806a5ef3da7658cfa73961e25befefc84f622e963b45dcac798868", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 3 + "depth" : 2 }, "ancestors" : [ { - "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "o8G7vm8BvLT8jmu5B1-M", "type" : "event", "index" : "filebeat-8.0.0-2019.12.18-000001", - "depth" : 1 + "depth" : 0 }, { - "rule" : "f2b70c4a-4d8f-4db5-9ed7-d3ab0630e406", + "rule" : "ded57b36-9c4e-4ee4-805d-be4e92033e41", "id" : "9d8710925adbf1a9c469621805407e74334dd08ca2c2ea414840fe971a571938", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 2 + "depth" : 1 }, { - "rule" : "161fa5b8-0b96-4985-b066-0d99b2bcb904", + "rule" : "f2b70c4a-4d8f-4db5-9ed7-d3ab0630e406", "id" : "efbe514e8d806a5ef3da7658cfa73961e25befefc84f622e963b45dcac798868", "type" : "signal", "index" : ".siem-signals-default-000001", - "depth" : 3 + "depth" : 2 } -] +], +"depth": 3, +"rule": { + "id": "161fa5b8-0b96-4985-b066-0d99b2bcb904" +} ``` You will be "halted" at this point as the signal ancestry and de-duplication ensures that we do not report twice on signals and that we do not diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts index 95ec753c21fd8..9d3eb29be08dd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -149,21 +149,23 @@ export const sampleDocWithAncestors = (): SignalSearchResponse => { delete sampleDoc._source.source; sampleDoc._source.signal = { parent: { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, ancestors: [ { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, ], + rule: { + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + }, + depth: 1, }; return { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts index ee83c826371bc..967dc5331e46b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts @@ -48,19 +48,25 @@ describe('buildBulkBody', () => { }, signal: { parent: { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: sampleIdGuid, type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, + parents: [ + { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], ancestors: [ { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: sampleIdGuid, type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, ], original_time: '2020-04-20T21:27:45+0000', @@ -102,6 +108,7 @@ describe('buildBulkBody', () => { updated_at: fakeSignalSourceHit.signal.rule?.updated_at, exceptions_list: getListArrayMock(), }, + depth: 1, }, }; expect(fakeSignalSourceHit).toEqual(expected); @@ -151,19 +158,25 @@ describe('buildBulkBody', () => { module: 'system', }, parent: { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: sampleIdGuid, type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, + parents: [ + { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], ancestors: [ { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: sampleIdGuid, type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, ], original_time: '2020-04-20T21:27:45+0000', @@ -205,6 +218,7 @@ describe('buildBulkBody', () => { threat: [], exceptions_list: getListArrayMock(), }, + depth: 1, }, }; expect(fakeSignalSourceHit).toEqual(expected); @@ -252,19 +266,25 @@ describe('buildBulkBody', () => { module: 'system', }, parent: { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: sampleIdGuid, type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, + parents: [ + { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], ancestors: [ { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: sampleIdGuid, type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, ], original_time: '2020-04-20T21:27:45+0000', @@ -306,6 +326,7 @@ describe('buildBulkBody', () => { throttle: 'no_actions', exceptions_list: getListArrayMock(), }, + depth: 1, }, }; expect(fakeSignalSourceHit).toEqual(expected); @@ -346,19 +367,25 @@ describe('buildBulkBody', () => { kind: 'event', }, parent: { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: sampleIdGuid, type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, + parents: [ + { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], ancestors: [ { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: sampleIdGuid, type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, ], original_time: '2020-04-20T21:27:45+0000', @@ -400,6 +427,7 @@ describe('buildBulkBody', () => { throttle: 'no_actions', exceptions_list: getListArrayMock(), }, + depth: 1, }, }; expect(fakeSignalSourceHit).toEqual(expected); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts index 218750ac30a2a..7be97e46f91f2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts @@ -4,9 +4,9 @@ * you may not use this file except in compliance with the Elastic License. */ -import { SignalSourceHit, SignalHit } from './types'; +import { SignalSourceHit, SignalHit, Signal } from './types'; import { buildRule } from './build_rule'; -import { buildSignal } from './build_signal'; +import { additionalSignalFields, buildSignal } from './build_signal'; import { buildEventTypeSignal } from './build_event_type_signal'; import { RuleAlertAction } from '../../../../common/detection_engine/types'; import { RuleTypeParams } from '../types'; @@ -58,7 +58,10 @@ export const buildBulkBody = ({ tags, throttle, }); - const signal = buildSignal(doc, rule); + const signal: Signal = { + ...buildSignal([doc], rule), + ...additionalSignalFields(doc), + }; const event = buildEventTypeSignal(doc); const signalHit: SignalHit = { ...doc._source, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts index 7257e5952ff05..ba815a0b62f0d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts @@ -4,10 +4,12 @@ * you may not use this file except in compliance with the Elastic License. */ -import { buildRule } from './build_rule'; +import { buildRule, removeInternalTagsFromRule } from './build_rule'; import { sampleDocNoSortId, sampleRuleAlertParams, sampleRuleGuid } from './__mocks__/es_results'; import { RulesSchema } from '../../../../common/detection_engine/schemas/response/rules_schema'; import { getListArrayMock } from '../../../../common/detection_engine/schemas/types/lists.mock'; +import { INTERNAL_RULE_ID_KEY, INTERNAL_IMMUTABLE_KEY } from '../../../../common/constants'; +import { getPartialRulesSchemaMock } from '../../../../common/detection_engine/schemas/response/rules_schema.mocks'; describe('buildRule', () => { beforeEach(() => { @@ -208,4 +210,102 @@ describe('buildRule', () => { }; expect(rule).toEqual(expected); }); + + test('it builds a rule and removes internal tags', () => { + const ruleParams = sampleRuleAlertParams(); + const rule = buildRule({ + actions: [], + doc: sampleDocNoSortId(), + ruleParams, + name: 'some-name', + id: sampleRuleGuid, + enabled: false, + createdAt: '2020-01-28T15:58:34.810Z', + updatedAt: '2020-01-28T15:59:14.004Z', + createdBy: 'elastic', + updatedBy: 'elastic', + interval: 'some interval', + tags: [ + 'some fake tag 1', + 'some fake tag 2', + `${INTERNAL_RULE_ID_KEY}:rule-1`, + `${INTERNAL_IMMUTABLE_KEY}:true`, + ], + throttle: 'no_actions', + }); + const expected: Partial = { + actions: [], + author: ['Elastic'], + building_block_type: 'default', + created_by: 'elastic', + description: 'Detecting root and admin users', + enabled: false, + false_positives: [], + from: 'now-6m', + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: 'some interval', + language: 'kuery', + license: 'Elastic License', + max_signals: 10000, + name: 'some-name', + output_index: '.siem-signals', + query: 'user.name: root or user.name: admin', + references: ['http://google.com'], + risk_score: 50, + risk_score_mapping: [], + rule_id: 'rule-1', + severity: 'high', + severity_mapping: [], + tags: ['some fake tag 1', 'some fake tag 2'], + threat: [], + to: 'now', + type: 'query', + note: '', + updated_by: 'elastic', + updated_at: rule.updated_at, + created_at: rule.created_at, + throttle: 'no_actions', + exceptions_list: getListArrayMock(), + version: 1, + }; + expect(rule).toEqual(expected); + }); + + test('it removes internal tags from a typical rule', () => { + const rule = getPartialRulesSchemaMock(); + rule.tags = [ + 'some fake tag 1', + 'some fake tag 2', + `${INTERNAL_RULE_ID_KEY}:rule-1`, + `${INTERNAL_IMMUTABLE_KEY}:true`, + ]; + const noInternals = removeInternalTagsFromRule(rule); + expect(noInternals).toEqual(getPartialRulesSchemaMock()); + }); + + test('it works with an empty array', () => { + const rule = getPartialRulesSchemaMock(); + rule.tags = []; + const noInternals = removeInternalTagsFromRule(rule); + const expected = getPartialRulesSchemaMock(); + expected.tags = []; + expect(noInternals).toEqual(expected); + }); + + test('it works if tags does not exist', () => { + const rule = getPartialRulesSchemaMock(); + delete rule.tags; + const noInternals = removeInternalTagsFromRule(rule); + const expected = getPartialRulesSchemaMock(); + delete expected.tags; + expect(noInternals).toEqual(expected); + }); + + test('it works if tags contains normal values and no internal values', () => { + const rule = getPartialRulesSchemaMock(); + const noInternals = removeInternalTagsFromRule(rule); + expect(noInternals).toEqual(rule); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts index e02a0154d63c9..aacf9b8be31b4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts @@ -12,6 +12,7 @@ import { buildRiskScoreFromMapping } from './mappings/build_risk_score_from_mapp import { SignalSourceHit } from './types'; import { buildSeverityFromMapping } from './mappings/build_severity_from_mapping'; import { buildRuleNameFromMapping } from './mappings/build_rule_name_from_mapping'; +import { INTERNAL_IDENTIFIER } from '../../../../common/constants'; interface BuildRuleParams { ruleParams: RuleTypeParams; @@ -64,7 +65,7 @@ export const buildRule = ({ const meta = { ...ruleParams.meta, ...riskScoreMeta, ...severityMeta, ...ruleNameMeta }; - return pickBy((value: unknown) => value != null, { + const rule = pickBy((value: unknown) => value != null, { id, rule_id: ruleParams.ruleId ?? '(unknown rule_id)', actions, @@ -111,4 +112,17 @@ export const buildRule = ({ anomaly_threshold: ruleParams.anomalyThreshold, threshold: ruleParams.threshold, }); + return removeInternalTagsFromRule(rule); +}; + +export const removeInternalTagsFromRule = (rule: Partial): Partial => { + if (rule.tags == null) { + return rule; + } else { + const ruleWithoutInternalTags: Partial = { + ...rule, + tags: rule.tags.filter((tag) => !tag.startsWith(INTERNAL_IDENTIFIER)), + }; + return ruleWithoutInternalTags; + } }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts index 6aebf8815659a..d684807a09126 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts @@ -5,14 +5,8 @@ */ import { sampleDocNoSortId } from './__mocks__/es_results'; -import { - buildSignal, - buildAncestor, - buildAncestorsSignal, - removeInternalTagsFromRule, -} from './build_signal'; +import { buildSignal, buildParent, buildAncestors, additionalSignalFields } from './build_signal'; import { Signal, Ancestor } from './types'; -import { INTERNAL_RULE_ID_KEY, INTERNAL_IMMUTABLE_KEY } from '../../../../common/constants'; import { getPartialRulesSchemaMock } from '../../../../common/detection_engine/schemas/response/rules_schema.mocks'; describe('buildSignal', () => { @@ -24,22 +18,31 @@ describe('buildSignal', () => { const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); delete doc._source.event; const rule = getPartialRulesSchemaMock(); - const signal = buildSignal(doc, rule); + const signal = { + ...buildSignal([doc], rule), + ...additionalSignalFields(doc), + }; const expected: Signal = { parent: { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, + parents: [ + { + id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], ancestors: [ { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, ], original_time: '2020-04-20T21:27:45+0000', @@ -71,6 +74,7 @@ describe('buildSignal', () => { updated_at: signal.rule.updated_at, created_at: signal.rule.created_at, }, + depth: 1, }; expect(signal).toEqual(expected); }); @@ -84,94 +88,31 @@ describe('buildSignal', () => { module: 'system', }; const rule = getPartialRulesSchemaMock(); - const signal = buildSignal(doc, rule); + const signal = { + ...buildSignal([doc], rule), + ...additionalSignalFields(doc), + }; const expected: Signal = { parent: { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, - ancestors: [ + parents: [ { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, ], - original_time: '2020-04-20T21:27:45+0000', - original_event: { - action: 'socket_opened', - dataset: 'socket', - kind: 'event', - module: 'system', - }, - status: 'open', - rule: { - created_by: 'elastic', - description: 'Detecting root and admin users', - enabled: true, - false_positives: [], - from: 'now-6m', - id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', - immutable: false, - index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], - interval: '5m', - risk_score: 50, - rule_id: 'rule-1', - language: 'kuery', - max_signals: 100, - name: 'Detect Root/Admin Users', - output_index: '.siem-signals', - query: 'user.name: root or user.name: admin', - references: ['http://www.example.com', 'https://ww.example.com'], - severity: 'high', - updated_by: 'elastic', - tags: ['some fake tag 1', 'some fake tag 2'], - to: 'now', - type: 'query', - note: '', - updated_at: signal.rule.updated_at, - created_at: signal.rule.created_at, - }, - }; - expect(signal).toEqual(expected); - }); - - test('it builds a signal as expected with original_event if is present and without internal tags in them', () => { - const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); - doc._source.event = { - action: 'socket_opened', - dataset: 'socket', - kind: 'event', - module: 'system', - }; - const rule = getPartialRulesSchemaMock(); - rule.tags = [ - 'some fake tag 1', - 'some fake tag 2', - `${INTERNAL_RULE_ID_KEY}:rule-1`, - `${INTERNAL_IMMUTABLE_KEY}:true`, - ]; - const signal = buildSignal(doc, rule); - const expected: Signal = { - parent: { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', - id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', - type: 'event', - index: 'myFakeSignalIndex', - depth: 1, - }, ancestors: [ { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, ], original_time: '2020-04-20T21:27:45+0000', @@ -209,6 +150,7 @@ describe('buildSignal', () => { updated_at: signal.rule.updated_at, created_at: signal.rule.created_at, }, + depth: 1, }; expect(signal).toEqual(expected); }); @@ -221,14 +163,12 @@ describe('buildSignal', () => { kind: 'event', module: 'system', }; - const rule = getPartialRulesSchemaMock(); - const signal = buildAncestor(doc, rule); + const signal = buildParent(doc); const expected: Ancestor = { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }; expect(signal).toEqual(expected); }); @@ -242,76 +182,34 @@ describe('buildSignal', () => { module: 'system', }; doc._source.signal = { - parent: { - rule: '98c0bf9e-4d38-46f4-9a6a-8a820426256b', - id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', - type: 'event', - index: 'myFakeSignalIndex', - depth: 1, - }, - ancestors: [ + parents: [ { - rule: '98c0bf9e-4d38-46f4-9a6a-8a820426256b', id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, ], - }; - const rule = getPartialRulesSchemaMock(); - const signal = buildAncestor(doc, rule); - const expected: Ancestor = { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', - id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', - type: 'signal', - index: 'myFakeSignalIndex', - depth: 2, - }; - expect(signal).toEqual(expected); - }); - - test('it builds a ancestor correctly if the parent does exist without internal tags in them', () => { - const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); - doc._source.event = { - action: 'socket_opened', - dataset: 'socket', - kind: 'event', - module: 'system', - }; - doc._source.signal = { - parent: { - rule: '98c0bf9e-4d38-46f4-9a6a-8a820426256b', - id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', - type: 'event', - index: 'myFakeSignalIndex', - depth: 1, - }, ancestors: [ { - rule: '98c0bf9e-4d38-46f4-9a6a-8a820426256b', id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, ], + depth: 1, + rule: { + id: '98c0bf9e-4d38-46f4-9a6a-8a820426256b', + }, }; - const rule = getPartialRulesSchemaMock(); - rule.tags = [ - 'some fake tag 1', - 'some fake tag 2', - `${INTERNAL_RULE_ID_KEY}:rule-1`, - `${INTERNAL_IMMUTABLE_KEY}:true`, - ]; - - const signal = buildAncestor(doc, rule); + const signal = buildParent(doc); const expected: Ancestor = { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + rule: '98c0bf9e-4d38-46f4-9a6a-8a820426256b', id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', type: 'signal', index: 'myFakeSignalIndex', - depth: 2, + depth: 1, }; expect(signal).toEqual(expected); }); @@ -324,15 +222,13 @@ describe('buildSignal', () => { kind: 'event', module: 'system', }; - const rule = getPartialRulesSchemaMock(); - const signal = buildAncestorsSignal(doc, rule); + const signal = buildAncestors(doc); const expected: Ancestor[] = [ { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, ]; expect(signal).toEqual(expected); @@ -347,77 +243,43 @@ describe('buildSignal', () => { module: 'system', }; doc._source.signal = { - parent: { - rule: '98c0bf9e-4d38-46f4-9a6a-8a820426256b', - id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', - type: 'event', - index: 'myFakeSignalIndex', - depth: 1, - }, + parents: [ + { + id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], ancestors: [ { - rule: '98c0bf9e-4d38-46f4-9a6a-8a820426256b', id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, ], + rule: { + id: '98c0bf9e-4d38-46f4-9a6a-8a820426256b', + }, + depth: 1, }; - const rule = getPartialRulesSchemaMock(); - const signal = buildAncestorsSignal(doc, rule); + const signal = buildAncestors(doc); const expected: Ancestor[] = [ { - rule: '98c0bf9e-4d38-46f4-9a6a-8a820426256b', id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', type: 'event', index: 'myFakeSignalIndex', - depth: 1, + depth: 0, }, { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + rule: '98c0bf9e-4d38-46f4-9a6a-8a820426256b', id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', type: 'signal', index: 'myFakeSignalIndex', - depth: 2, + depth: 1, }, ]; expect(signal).toEqual(expected); }); - - test('it removes internal tags from a typical rule', () => { - const rule = getPartialRulesSchemaMock(); - rule.tags = [ - 'some fake tag 1', - 'some fake tag 2', - `${INTERNAL_RULE_ID_KEY}:rule-1`, - `${INTERNAL_IMMUTABLE_KEY}:true`, - ]; - const noInternals = removeInternalTagsFromRule(rule); - expect(noInternals).toEqual(getPartialRulesSchemaMock()); - }); - - test('it works with an empty array', () => { - const rule = getPartialRulesSchemaMock(); - rule.tags = []; - const noInternals = removeInternalTagsFromRule(rule); - const expected = getPartialRulesSchemaMock(); - expected.tags = []; - expect(noInternals).toEqual(expected); - }); - - test('it works if tags does not exist', () => { - const rule = getPartialRulesSchemaMock(); - delete rule.tags; - const noInternals = removeInternalTagsFromRule(rule); - const expected = getPartialRulesSchemaMock(); - delete expected.tags; - expect(noInternals).toEqual(expected); - }); - - test('it works if tags contains normal values and no internal values', () => { - const rule = getPartialRulesSchemaMock(); - const noInternals = removeInternalTagsFromRule(rule); - expect(noInternals).toEqual(rule); - }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts index e7098c015c165..78818779dd661 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts @@ -5,35 +5,41 @@ */ import { RulesSchema } from '../../../../common/detection_engine/schemas/response/rules_schema'; -import { INTERNAL_IDENTIFIER } from '../../../../common/constants'; import { SignalSourceHit, Signal, Ancestor } from './types'; -export const buildAncestor = (doc: SignalSourceHit, rule: Partial): Ancestor => { - const existingSignal = doc._source.signal?.parent; - if (existingSignal != null) { +/** + * Takes a parent signal or event document and extracts the information needed for the corresponding entry in the child + * signal's `signal.parents` array. + * @param doc The parent signal or event + */ +export const buildParent = (doc: SignalSourceHit): Ancestor => { + if (doc._source.signal != null) { return { - rule: rule.id != null ? rule.id : '', + rule: doc._source.signal.rule.id, id: doc._id, type: 'signal', index: doc._index, - depth: existingSignal.depth + 1, + // We first look for signal.depth and use that if it exists. If it doesn't exist, this should be a pre-7.10 signal + // and should have signal.parent.depth instead. signal.parent.depth in this case is treated as equivalent to signal.depth. + depth: doc._source.signal.depth ?? doc._source.signal.parent?.depth ?? 1, }; } else { return { - rule: rule.id != null ? rule.id : '', id: doc._id, type: 'event', index: doc._index, - depth: 1, + depth: 0, }; } }; -export const buildAncestorsSignal = ( - doc: SignalSourceHit, - rule: Partial -): Signal['ancestors'] => { - const newAncestor = buildAncestor(doc, rule); +/** + * Takes a parent signal or event document with N ancestors and adds the parent document to the ancestry array, + * creating an array of N+1 ancestors. + * @param doc The parent signal/event for which to extend the ancestry. + */ +export const buildAncestors = (doc: SignalSourceHit): Ancestor[] => { + const newAncestor = buildParent(doc); const existingAncestors = doc._source.signal?.ancestors; if (existingAncestors != null) { return [...existingAncestors, newAncestor]; @@ -42,35 +48,33 @@ export const buildAncestorsSignal = ( } }; -export const buildSignal = (doc: SignalSourceHit, rule: Partial): Signal => { - const ruleWithoutInternalTags = removeInternalTagsFromRule(rule); - const parent = buildAncestor(doc, rule); - const ancestors = buildAncestorsSignal(doc, rule); - let signal: Signal = { - parent, +/** + * Builds the `signal.*` fields that are common across all signals. + * @param docs The parent signals/events of the new signal to be built. + * @param rule The rule that is generating the new signal. + */ +export const buildSignal = (docs: SignalSourceHit[], rule: Partial): Signal => { + const parents = docs.map(buildParent); + const depth = parents.reduce((acc, parent) => Math.max(parent.depth, acc), 0) + 1; + const ancestors = docs.reduce((acc: Ancestor[], doc) => acc.concat(buildAncestors(doc)), []); + return { + parents, ancestors, - original_time: doc._source['@timestamp'], status: 'open', - rule: ruleWithoutInternalTags, + rule, + depth, }; - if (doc._source.event != null) { - signal = { ...signal, original_event: doc._source.event }; - } - if (doc._source.threshold_count != null) { - signal = { ...signal, threshold_count: doc._source.threshold_count }; - delete doc._source.threshold_count; - } - return signal; }; -export const removeInternalTagsFromRule = (rule: Partial): Partial => { - if (rule.tags == null) { - return rule; - } else { - const ruleWithoutInternalTags: Partial = { - ...rule, - tags: rule.tags.filter((tag) => !tag.startsWith(INTERNAL_IDENTIFIER)), - }; - return ruleWithoutInternalTags; - } +/** + * Creates signal fields that are only available in the special case where a signal has only 1 parent signal/event. + * @param doc The parent signal/event of the new signal to be built. + */ +export const additionalSignalFields = (doc: SignalSourceHit) => { + return { + parent: buildParent(doc), + original_time: doc._source['@timestamp'], + original_event: doc._source.event ?? undefined, + threshold_count: doc._source.threshold_count ?? undefined, + }; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index da17d4a1f123a..7ee157beec789 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -120,7 +120,6 @@ export const signalRulesAlertType = ({ enabled, schedule: { interval }, throttle, - params: ruleParams, } = savedObject.attributes; const updatedAt = savedObject.updated_at ?? ''; const refresh = actions.length ? 'wait_for' : false; @@ -343,7 +342,7 @@ export const signalRulesAlertType = ({ if (result.success) { if (actions.length) { const notificationRuleParams: NotificationRuleTypeParams = { - ...ruleParams, + ...params, name, id: savedObject.id, }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.test.ts index 8b9fb0574efe9..41c825ea4d978 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.test.ts @@ -291,37 +291,7 @@ describe('singleBulkCreate', () => { test('filter duplicate rules will return nothing filtered when the two rule ids do not match with each other', () => { const filtered = filterDuplicateRules('some id', sampleDocWithAncestors()); - expect(filtered).toEqual([ - { - _index: 'myFakeSignalIndex', - _type: 'doc', - _score: 100, - _version: 1, - _id: 'e1e08ddc-5e37-49ff-a258-5393aa44435a', - _source: { - someKey: 'someValue', - '@timestamp': '2020-04-20T21:27:45+0000', - signal: { - parent: { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', - id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', - type: 'event', - index: 'myFakeSignalIndex', - depth: 1, - }, - ancestors: [ - { - rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', - id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', - type: 'event', - index: 'myFakeSignalIndex', - depth: 1, - }, - ], - }, - }, - }, - ]); + expect(filtered).toEqual(sampleDocWithAncestors().hits.hits); }); test('filters duplicate rules will return empty array when the two rule ids match each other', () => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts index 74709f31563ee..be71c67615a4c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts @@ -51,7 +51,10 @@ export const filterDuplicateRules = ( if (doc._source.signal == null) { return true; } else { - return !doc._source.signal.ancestors.some((ancestor) => ancestor.rule === ruleId); + return !( + doc._source.signal.ancestors.some((ancestor) => ancestor.rule === ruleId) || + doc._source.signal.rule.id === ruleId + ); } }); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index aecdbe10695d2..700a8fb5022d7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -44,8 +44,16 @@ export interface SignalSource { [key: string]: SearchTypes; '@timestamp': string; signal?: { - parent: Ancestor; + // parent is deprecated: new signals should populate parents instead + // both are optional until all signals with parent are gone and we can safely remove it + parent?: Ancestor; + parents?: Ancestor[]; ancestors: Ancestor[]; + rule: { + id: string; + }; + // signal.depth doesn't exist on pre-7.10 signals + depth?: number; }; } @@ -113,7 +121,7 @@ export type SignalRuleAlertTypeDefinition = Omit & { }; export interface Ancestor { - rule: string; + rule?: string; id: string; type: string; index: string; @@ -122,12 +130,15 @@ export interface Ancestor { export interface Signal { rule: Partial; - parent: Ancestor; + // DEPRECATED: use parents instead of parent + parent?: Ancestor; + parents: Ancestor[]; ancestors: Ancestor[]; - original_time: string; + original_time?: string; original_event?: SearchTypes; status: Status; threshold_count?: SearchTypes; + depth: number; } export interface SignalHit { diff --git a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json index 2435d8a9aaf04..a7330d3ebd552 100644 --- a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json +++ b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @@ -51,6 +51,27 @@ } } }, + "enterprise_search": { + "properties": { + "ui_viewed": { + "properties": { + "overview": { + "type": "long" + } + } + }, + "ui_clicked": { + "properties": { + "app_search": { + "type": "long" + }, + "workplace_search": { + "type": "long" + } + } + } + } + }, "workplace_search": { "properties": { "ui_viewed": { diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index f753e0ec87064..54c92d323fcff 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -14079,8 +14079,6 @@ "xpack.reporting.screencapture.waitingForRenderComplete": "レンダリングの完了を待っています", "xpack.reporting.screencapture.waitingForRenderedElements": "レンダリングされた {itemsCount} 個の要素が DOM に入るのを待っています", "xpack.reporting.screenCapturePanelContent.optimizeForPrintingLabel": "印刷用に最適化", - "xpack.reporting.selfCheck.ok": "レポートプラグイン自己チェックOK!", - "xpack.reporting.selfCheck.warning": "レポートプラグイン自己チェックで警告が発生しました: {err}", "xpack.reporting.serverConfig.autoSet.sandboxDisabled": "Chromiumサンドボックスは保護が強化されていますが、{osName} OSではサポートされていません。自動的に'{configKey}: true'を設定しています。", "xpack.reporting.serverConfig.autoSet.sandboxEnabled": "Chromiumサンドボックスは保護が強化され、{osName} OSでサポートされています。自動的にChromiumサンドボックスを有効にしています。", "xpack.reporting.serverConfig.invalidServerHostname": "Kibana構成で「server.host:\"0\"」が見つかりました。これはReportingと互換性がありません。レポートが動作するように、「{configKey}:0.0.0.0」が自動的に構成になります。設定を「server.host:0.0.0.0」に変更するか、kibana.ymlに「{configKey}:0.0.0.0'」を追加して、このメッセージが表示されないようにすることができます。", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 8841db0be8d95..df721cb624662 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -14088,8 +14088,6 @@ "xpack.reporting.screencapture.waitingForRenderComplete": "正在等候渲染完成", "xpack.reporting.screencapture.waitingForRenderedElements": "正在等候 {itemsCount} 个已渲染元素进入 DOM", "xpack.reporting.screenCapturePanelContent.optimizeForPrintingLabel": "打印优化", - "xpack.reporting.selfCheck.ok": "Reporting 插件自检正常！", - "xpack.reporting.selfCheck.warning": "Reporting 插件自检生成警告：{err}", "xpack.reporting.serverConfig.autoSet.sandboxDisabled": "Chromium 沙盒提供附加保护层，但不受 {osName} OS 支持。自动设置“{configKey}: true”。", "xpack.reporting.serverConfig.autoSet.sandboxEnabled": "Chromium 沙盒提供附加保护层，受 {osName} OS 支持。自动启用 Chromium 沙盒。", "xpack.reporting.serverConfig.invalidServerHostname": "在 Kibana 配置中找到“server.host:\"0\"”。其不与 Reporting 兼容。要使 Reporting 运行，“{configKey}:0.0.0.0”将自动添加到配置中。可以将该设置更改为“server.host:0.0.0.0”或在 kibana.yml 中添加“{configKey}:0.0.0.0”，以阻止此消息。", diff --git a/x-pack/test/api_integration/apis/metrics_ui/snapshot.ts b/x-pack/test/api_integration/apis/metrics_ui/snapshot.ts index bb0934b73a4c7..7339c142fb028 100644 --- a/x-pack/test/api_integration/apis/metrics_ui/snapshot.ts +++ b/x-pack/test/api_integration/apis/metrics_ui/snapshot.ts @@ -67,7 +67,6 @@ export default function ({ getService }: FtrProviderContext) { 'value', '242fddb9d376bbf0e38025d81764847ee5ec0308adfa095918fd3266f9d06c6a' ); - expect(first(firstNode.path)).to.have.property('label', 'docker-autodiscovery_nginx_1'); expect(firstNode).to.have.property('metrics'); expect(firstNode.metrics).to.eql([ { @@ -136,7 +135,7 @@ export default function ({ getService }: FtrProviderContext) { expect(snapshot).to.have.property('nodes'); if (snapshot) { const { nodes } = snapshot; - expect(nodes.length).to.equal(136); + expect(nodes.length).to.equal(135); const firstNode = first(nodes) as any; expect(firstNode).to.have.property('path'); expect(firstNode.path.length).to.equal(1); @@ -295,7 +294,7 @@ export default function ({ getService }: FtrProviderContext) { expect(firstNode).to.have.property('metrics'); expect(firstNode.metrics).to.eql([ { - name: 'custom', + name: 'custom_0', value: 0.0016, max: 0.0018333333333333333, avg: 0.0013666666666666669, diff --git a/x-pack/test/api_integration/apis/security/basic_login.js b/x-pack/test/api_integration/apis/security/basic_login.js index 4b39b1bf32d5b..43ef8e6b81eac 100644 --- a/x-pack/test/api_integration/apis/security/basic_login.js +++ b/x-pack/test/api_integration/apis/security/basic_login.js @@ -148,11 +148,8 @@ export default function ({ getService }) { ]); expect(apiResponse.body.username).to.be(validUsername); expect(apiResponse.body.authentication_provider).to.eql('__http__'); - expect(apiResponse.body.authentication_realm).to.eql({ - name: 'reserved', - type: 'reserved', - }); expect(apiResponse.body.authentication_type).to.be('realm'); + // Do not assert on the `authentication_realm`, as the value differes for on-prem vs cloud }); describe('with session cookie', () => { @@ -197,11 +194,8 @@ export default function ({ getService }) { ]); expect(apiResponse.body.username).to.be(validUsername); expect(apiResponse.body.authentication_provider).to.eql('basic'); - expect(apiResponse.body.authentication_realm).to.eql({ - name: 'reserved', - type: 'reserved', - }); expect(apiResponse.body.authentication_type).to.be('realm'); + // Do not assert on the `authentication_realm`, as the value differes for on-prem vs cloud }); it('should extend cookie on every successful non-system API call', async () => { diff --git a/x-pack/test/functional/apps/maps/index.js b/x-pack/test/functional/apps/maps/index.js index ef8b4ad4c0f19..03b75601ec2a8 100644 --- a/x-pack/test/functional/apps/maps/index.js +++ b/x-pack/test/functional/apps/maps/index.js @@ -28,7 +28,7 @@ export default function ({ loadTestFile, getService }) { }); describe('', function () { - this.tags('ciGroup7'); + this.tags('ciGroup9'); loadTestFile(require.resolve('./documents_source')); loadTestFile(require.resolve('./blended_vector_layer')); loadTestFile(require.resolve('./vector_styling')); diff --git a/x-pack/test/functional/services/ml/data_frame_analytics_creation.ts b/x-pack/test/functional/services/ml/data_frame_analytics_creation.ts index ffa1d9fd46c75..e01e065867ac7 100644 --- a/x-pack/test/functional/services/ml/data_frame_analytics_creation.ts +++ b/x-pack/test/functional/services/ml/data_frame_analytics_creation.ts @@ -10,25 +10,9 @@ import { FtrProviderContext } from '../../ftr_provider_context'; import { MlCommonUI } from './common_ui'; import { MlApi } from './api'; import { - ClassificationAnalysis, - RegressionAnalysis, -} from '../../../../plugins/ml/common/types/data_frame_analytics'; - -enum ANALYSIS_CONFIG_TYPE { - OUTLIER_DETECTION = 'outlier_detection', - REGRESSION = 'regression', - CLASSIFICATION = 'classification', -} - -const isRegressionAnalysis = (arg: any): arg is RegressionAnalysis => { - const keys = Object.keys(arg); - return keys.length === 1 && keys[0] === ANALYSIS_CONFIG_TYPE.REGRESSION; -}; - -const isClassificationAnalysis = (arg: any): arg is ClassificationAnalysis => { - const keys = Object.keys(arg); - return keys.length === 1 && keys[0] === ANALYSIS_CONFIG_TYPE.CLASSIFICATION; -}; + isRegressionAnalysis, + isClassificationAnalysis, +} from '../../../../plugins/ml/common/util/analytics_utils'; export function MachineLearningDataFrameAnalyticsCreationProvider( { getService }: FtrProviderContext, diff --git a/x-pack/test/kerberos_api_integration/apis/security/kerberos_login.ts b/x-pack/test/kerberos_api_integration/apis/security/kerberos_login.ts index 1f4428e198539..459dc4739897c 100644 --- a/x-pack/test/kerberos_api_integration/apis/security/kerberos_login.ts +++ b/x-pack/test/kerberos_api_integration/apis/security/kerberos_login.ts @@ -79,9 +79,9 @@ export default function ({ getService }: FtrProviderContext) { .expect(200); expect(user.username).to.eql(username); - expect(user.authentication_realm).to.eql({ name: 'reserved', type: 'reserved' }); expect(user.authentication_provider).to.eql('basic'); expect(user.authentication_type).to.eql('realm'); + // Do not assert on the `authentication_realm`, as the value differes for on-prem vs cloud }); describe('initiating SPNEGO', () => { diff --git a/x-pack/test/login_selector_api_integration/apis/login_selector.ts b/x-pack/test/login_selector_api_integration/apis/login_selector.ts index 7eb1f07d67506..44582355cf890 100644 --- a/x-pack/test/login_selector_api_integration/apis/login_selector.ts +++ b/x-pack/test/login_selector_api_integration/apis/login_selector.ts @@ -36,7 +36,7 @@ export default function ({ getService }: FtrProviderContext) { sessionCookie: Cookie, username: string, providerName: string, - authenticationRealm: { name: string; type: string }, + authenticationRealm: { name: string; type: string } | null, authenticationType: string ) { expect(sessionCookie.key).to.be('sid'); @@ -67,7 +67,9 @@ export default function ({ getService }: FtrProviderContext) { expect(apiResponse.body.username).to.be(username); expect(apiResponse.body.authentication_provider).to.be(providerName); - expect(apiResponse.body.authentication_realm).to.eql(authenticationRealm); + if (authenticationRealm) { + expect(apiResponse.body.authentication_realm).to.eql(authenticationRealm); + } expect(apiResponse.body.authentication_type).to.be(authenticationType); } @@ -228,16 +230,9 @@ export default function ({ getService }: FtrProviderContext) { const basicSessionCookie = request.cookie( basicAuthenticationResponse.headers['set-cookie'][0] )!; - await checkSessionCookie( - basicSessionCookie, - 'elastic', - 'basic1', - { - name: 'reserved', - type: 'reserved', - }, - 'realm' - ); + // Skip auth provider check since this comes from the reserved realm, + // which is not available when running on ESS + await checkSessionCookie(basicSessionCookie, 'elastic', 'basic1', null, 'realm'); const authenticationResponse = await supertest .post('/api/security/saml/callback') diff --git a/x-pack/test/oidc_api_integration/apis/authorization_code_flow/oidc_auth.ts b/x-pack/test/oidc_api_integration/apis/authorization_code_flow/oidc_auth.ts index 0a230ac84d991..c2335cf04504f 100644 --- a/x-pack/test/oidc_api_integration/apis/authorization_code_flow/oidc_auth.ts +++ b/x-pack/test/oidc_api_integration/apis/authorization_code_flow/oidc_auth.ts @@ -43,9 +43,9 @@ export default function ({ getService }: FtrProviderContext) { .expect(200); expect(user.username).to.eql(username); - expect(user.authentication_realm).to.eql({ name: 'reserved', type: 'reserved' }); expect(user.authentication_provider).to.eql('basic'); expect(user.authentication_type).to.be('realm'); + // Do not assert on the `authentication_realm`, as the value differes for on-prem vs cloud }); describe('initiating handshake', () => { diff --git a/x-pack/test/pki_api_integration/apis/security/pki_auth.ts b/x-pack/test/pki_api_integration/apis/security/pki_auth.ts index 2f6b088ab7190..0559e9e96fe3f 100644 --- a/x-pack/test/pki_api_integration/apis/security/pki_auth.ts +++ b/x-pack/test/pki_api_integration/apis/security/pki_auth.ts @@ -93,8 +93,8 @@ export default function ({ getService }: FtrProviderContext) { .expect(200); expect(user.username).to.eql(username); - expect(user.authentication_realm).to.eql({ name: 'reserved', type: 'reserved' }); expect(user.authentication_provider).to.eql('basic'); + // Do not assert on the `authentication_realm`, as the value differes for on-prem vs cloud }); it('should properly set cookie and authenticate user', async () => { diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/spaces.ts b/x-pack/test/reporting_api_integration/reporting_and_security/spaces.ts index 6a68bd530cf63..9eafd0c318383 100644 --- a/x-pack/test/reporting_api_integration/reporting_and_security/spaces.ts +++ b/x-pack/test/reporting_api_integration/reporting_and_security/spaces.ts @@ -27,8 +27,7 @@ export default function ({ getService }: FtrProviderContext) { ); }; - // FLAKY: https://github.com/elastic/kibana/issues/76551 - describe.skip('Exports from Non-default Space', () => { + describe('Exports from Non-default Space', () => { before(async () => { await esArchiver.load('reporting/ecommerce'); await esArchiver.load('reporting/ecommerce_kibana_spaces'); // dashboard in non default space @@ -54,7 +53,8 @@ export default function ({ getService }: FtrProviderContext) { expect(reportCompleted).to.match(/^"order_date",/); }); - it('should complete a job of PNG export of a dashboard in non-default space', async () => { + // FLAKY: https://github.com/elastic/kibana/issues/76551 + it.skip('should complete a job of PNG export of a dashboard in non-default space', async () => { const downloadPath = await reportingAPI.postJob( `/s/non_default_space/api/reporting/generate/png?jobParams=%28browserTimezone%3AUTC%2Clayout%3A%28dimensions%3A%28height%3A512%2Cwidth%3A2402%29%2Cid%3Apng%29%2CobjectType%3Adashboard%2CrelativeUrl%3A%27%2Fs%2Fnon_default_space%2Fapp%2Fdashboards%23%2Fview%2F3c9ee360-e7ee-11ea-a730-d58e9ea7581b%3F_g%3D%28filters%3A%21%21%28%29%2CrefreshInterval%3A%28pause%3A%21%21t%2Cvalue%3A0%29%2Ctime%3A%28from%3A%21%272019-06-10T03%3A17%3A28.800Z%21%27%2Cto%3A%21%272019-07-14T19%3A25%3A06.385Z%21%27%29%29%26_a%3D%28description%3A%21%27%21%27%2Cfilters%3A%21%21%28%29%2CfullScreenMode%3A%21%21f%2Coptions%3A%28hidePanelTitles%3A%21%21f%2CuseMargins%3A%21%21t%29%2Cquery%3A%28language%3Akuery%2Cquery%3A%21%27%21%27%29%2CtimeRestore%3A%21%21t%2Ctitle%3A%21%27Ecom%2520Dashboard%2520Non%2520Default%2520Space%21%27%2CviewMode%3Aview%29%27%2Ctitle%3A%27Ecom%20Dashboard%20Non%20Default%20Space%27%29` ); @@ -64,7 +64,8 @@ export default function ({ getService }: FtrProviderContext) { expect(reportCompleted).to.not.be(null); }); - it('should complete a job of PDF export of a dashboard in non-default space', async () => { + // FLAKY: https://github.com/elastic/kibana/issues/76551 + it.skip('should complete a job of PDF export of a dashboard in non-default space', async () => { const downloadPath = await reportingAPI.postJob( `/s/non_default_space/api/reporting/generate/printablePdf?jobParams=%28browserTimezone%3AUTC%2Clayout%3A%28dimensions%3A%28height%3A512%2Cwidth%3A2402%29%2Cid%3Apreserve_layout%29%2CobjectType%3Adashboard%2CrelativeUrls%3A%21%28%27%2Fs%2Fnon_default_space%2Fapp%2Fdashboards%23%2Fview%2F3c9ee360-e7ee-11ea-a730-d58e9ea7581b%3F_g%3D%28filters%3A%21%21%28%29%2CrefreshInterval%3A%28pause%3A%21%21t%2Cvalue%3A0%29%2Ctime%3A%28from%3A%21%272019-06-10T03%3A17%3A28.800Z%21%27%2Cto%3A%21%272019-07-14T19%3A25%3A06.385Z%21%27%29%29%26_a%3D%28description%3A%21%27%21%27%2Cfilters%3A%21%21%28%29%2CfullScreenMode%3A%21%21f%2Coptions%3A%28hidePanelTitles%3A%21%21f%2CuseMargins%3A%21%21t%29%2Cquery%3A%28language%3Akuery%2Cquery%3A%21%27%21%27%29%2CtimeRestore%3A%21%21t%2Ctitle%3A%21%27Ecom%2520Dashboard%2520Non%2520Default%2520Space%21%27%2CviewMode%3Aview%29%27%29%2Ctitle%3A%27Ecom%20Dashboard%20Non%20Default%20Space%27%29` ); diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/usage.ts b/x-pack/test/reporting_api_integration/reporting_and_security/usage.ts index 49db8696c1134..aaf4dd3926411 100644 --- a/x-pack/test/reporting_api_integration/reporting_and_security/usage.ts +++ b/x-pack/test/reporting_api_integration/reporting_and_security/usage.ts @@ -21,8 +21,7 @@ export default function ({ getService }: FtrProviderContext) { const reportingAPI = getService('reportingAPI'); const usageAPI = getService('usageAPI'); - // FAILING: https://github.com/elastic/kibana/issues/76581 - describe.skip('Usage', () => { + describe('Usage', () => { before(async () => { await esArchiver.load(OSS_KIBANA_ARCHIVE_PATH); await esArchiver.load(OSS_DATA_ARCHIVE_PATH); @@ -116,7 +115,8 @@ export default function ({ getService }: FtrProviderContext) { }); }); - describe('from new jobs posted', () => { + // FAILING: https://github.com/elastic/kibana/issues/76581 + describe.skip('from new jobs posted', () => { it('should handle csv', async () => { await reportingAPI.expectAllJobsToFinishSuccessfully( await Promise.all([ diff --git a/x-pack/test/saml_api_integration/apis/security/saml_login.ts b/x-pack/test/saml_api_integration/apis/security/saml_login.ts index 501e1e5f2c203..2da7c92cd07b6 100644 --- a/x-pack/test/saml_api_integration/apis/security/saml_login.ts +++ b/x-pack/test/saml_api_integration/apis/security/saml_login.ts @@ -93,9 +93,9 @@ export default function ({ getService }: FtrProviderContext) { .expect(200); expect(user.username).to.eql(username); - expect(user.authentication_realm).to.eql({ name: 'reserved', type: 'reserved' }); expect(user.authentication_provider).to.eql('basic'); expect(user.authentication_type).to.be('realm'); + // Do not assert on the `authentication_realm`, as the value differes for on-prem vs cloud }); describe('initiating handshake', () => { diff --git a/x-pack/test/saved_object_api_integration/security_and_spaces/apis/index.ts b/x-pack/test/saved_object_api_integration/security_and_spaces/apis/index.ts index 81ffc5eea9220..ed501b235a457 100644 --- a/x-pack/test/saved_object_api_integration/security_and_spaces/apis/index.ts +++ b/x-pack/test/saved_object_api_integration/security_and_spaces/apis/index.ts @@ -12,7 +12,7 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { const supertest = getService('supertest'); describe('saved objects security and spaces enabled', function () { - this.tags('ciGroup5'); + this.tags('ciGroup8'); before(async () => { await createUsersAndRoles(es, supertest); diff --git a/x-pack/test/ui_capabilities/security_only/tests/nav_links.ts b/x-pack/test/ui_capabilities/security_only/tests/nav_links.ts index d7a0dfa1cf80a..091bbccd6f87a 100644 --- a/x-pack/test/ui_capabilities/security_only/tests/nav_links.ts +++ b/x-pack/test/ui_capabilities/security_only/tests/nav_links.ts @@ -49,7 +49,13 @@ export default function navLinksTests({ getService }: FtrProviderContext) { expect(uiCapabilities.success).to.be(true); expect(uiCapabilities.value).to.have.property('navLinks'); expect(uiCapabilities.value!.navLinks).to.eql( - navLinksBuilder.except('ml', 'monitoring', 'appSearch', 'workplaceSearch') + navLinksBuilder.except( + 'ml', + 'monitoring', + 'enterpriseSearch', + 'appSearch', + 'workplaceSearch' + ) ); break; case 'foo_all': diff --git a/yarn.lock b/yarn.lock index ddb83b3cf1532..105c5e3cba5ae 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12288,9 +12288,9 @@ eventemitter2@~0.4.13: integrity sha1-j2G3XN4BKy6esoTUVFWDtWQ7Yas= eventemitter3@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-4.0.0.tgz#d65176163887ee59f386d64c82610b696a4a74eb" - integrity sha512-qerSRB0p+UDEssxTtm6EDKcE7W4OaoisfIMl4CngyEhjpYglocpNg6UEqCvemdGhosAsg4sO2dXJOdyBifPGCg== + version "4.0.7" + resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-4.0.7.tgz#2de9b68f6528d5644ef5c59526a1b4a07306169f" + integrity sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw== events@^1.0.2: version "1.1.1" @@ -18194,10 +18194,10 @@ kdbush@^3.0.0: resolved "https://registry.yarnpkg.com/kdbush/-/kdbush-3.0.0.tgz#f8484794d47004cc2d85ed3a79353dbe0abc2bf0" integrity sha512-hRkd6/XW4HTsA9vjVpY9tuXJYLSlelnkTmVFu4M9/7MIYQtFcHpbugAU7UbOfjOiVSVYl2fqgBuJ32JUmRo5Ew== -kea@2.2.0-rc.4: - version "2.2.0-rc.4" - resolved "https://registry.yarnpkg.com/kea/-/kea-2.2.0-rc.4.tgz#cc0376950530a6751f73387c4b25a39efa1faa77" - integrity sha512-pYuwaCiJkBvHZShi8kqhk8dC4DjeELdK51Lw7Pn0tNdJgZJDF6COhsUiF/yrh9d7woNYDxKfuxH+QWZFfo8PkA== +kea@^2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/kea/-/kea-2.2.0.tgz#1ba4a174a53880cca8002a67cf62b19b30d09702" + integrity sha512-IzgTC6SC89wTLfiBMPlduG4r4YanxONYK4werz8RMZxPvcYw4XEEK8xQJguwVYtLCEGm4x5YiLCubGqGfRcbEw== kew@~0.1.7: version "0.1.7"