diff --git a/docs/user/alerting/alerting-setup.asciidoc b/docs/user/alerting/alerting-setup.asciidoc index 40ecdedad2cf4..7f184a295ac51 100644 --- a/docs/user/alerting/alerting-setup.asciidoc +++ b/docs/user/alerting/alerting-setup.asciidoc @@ -58,26 +58,14 @@ feature. To change rule settings, you must have `all` privileges for the such as flapping detection. For more information on configuring roles that provide access to features, go to <>. -For details about the prerequisites for each API, refer to <>. +Each rule also has a rule visibility value (or `consumer` in the APIs), which affects the {kib} feature privileges that are required to access it. +To view or edit a rule that has a `Stack Rules` rule visibility, for example, you must have the appropriate *Management > {stack-rules-feature}* feature privileges. -[float] -[[alerting-restricting-actions]] -==== Restrict actions - -For security reasons you may wish to limit the extent to which {kib} can connect -to external services. <> allows you to disable certain -<> and allowlist the hostnames that {kib} can connect with. - -[float] -[[alerting-spaces]] -=== Space isolation - -Rules and connectors are isolated to the {kib} space in which they were created. -A rule or connector created in one space will not be visible in another. +For details about the prerequisites required to run each API, refer to <>. [float] [[alerting-authorization]] -=== Authorization +==== API keys Rules are authorized using an API key. Its credentials are used to run all background tasks associated with the rule, including condition checks like {es} queries and triggered actions. @@ -100,11 +88,25 @@ In both cases, the API key is subsequently associated with the rule and used whe [IMPORTANT] ============================================== -If a rule requires certain privileges, such as index privileges, to run and a user without those privileges updates the rule, the rule will no longer function. +If a rule requires certain privileges, such as index privileges, to run and a user without those privileges updates the rule, the rule will no longer function. Conversely, if a user with greater or administrator privileges modifies the rule, it will begin running with increased privileges. The same behavior occurs when you change the API key in the header of your API calls. ============================================== +[float] +[[alerting-restricting-actions]] +==== Restrict actions + +For security reasons you may wish to limit the extent to which {kib} can connect to external services. +You can use <> to disable certain <> and allowlist the hostnames that {kib} can connect with. + +[float] +[[alerting-spaces]] +=== Space isolation + +Rules and connectors are isolated to the {kib} space in which they were created. +A rule or connector created in one space will not be visible in another. + [float] [[alerting-ccs-setup]] === {ccs-cap} diff --git a/docs/user/alerting/images/rule-types-es-query-conditions.png b/docs/user/alerting/images/rule-types-es-query-conditions.png index cbe18e7bf0bdb..47eed98caf5e6 100644 Binary files a/docs/user/alerting/images/rule-types-es-query-conditions.png and b/docs/user/alerting/images/rule-types-es-query-conditions.png differ diff --git a/docs/user/alerting/images/rule-types-es-query-invalid.png b/docs/user/alerting/images/rule-types-es-query-invalid.png deleted file mode 100644 index 0d36e33741484..0000000000000 Binary files a/docs/user/alerting/images/rule-types-es-query-invalid.png and /dev/null differ diff --git a/docs/user/alerting/images/rule-types-es-query-valid.png b/docs/user/alerting/images/rule-types-es-query-valid.png index 938662f49a063..6c63f777dfdc3 100644 Binary files a/docs/user/alerting/images/rule-types-es-query-valid.png and b/docs/user/alerting/images/rule-types-es-query-valid.png differ diff --git a/docs/user/alerting/rule-types/es-query.asciidoc b/docs/user/alerting/rule-types/es-query.asciidoc index 2f5e53b7b342d..6d7c24bcacf07 100644 --- a/docs/user/alerting/rule-types/es-query.asciidoc +++ b/docs/user/alerting/rule-types/es-query.asciidoc @@ -42,6 +42,26 @@ Exclude matches from previous run:: Turn on to avoid alert duplication by excluding documents that have already been detected by the previous rule run. This option is not available when a grouping field is specified. +You can optionally change the check interval, which defines how often to evaluate the rule conditions. + +You must select a scope value, which affects the <> that are required to access the rule. +For example when it's set to `Stack Rules`, you must have the appropriate *Management > {stack-rules-feature}* feature privileges to view or edit the rule. + +[float] +=== Test your query + +Use the *Test query* feature to verify that your query is valid. + +Valid queries are run against the selected indices using the configured time window. +The number of documents that match the query is displayed. +For example: + +[role="screenshot"] +image::user/alerting/images/rule-types-es-query-valid.png[Test {es} query returns number of matches when valid] +// NOTE: This is an autogenerated screenshot. Do not edit it directly. + +An error message is shown if the query is invalid. + [float] === Add actions @@ -155,24 +175,6 @@ Labels: // NOTCONSOLE -- -[float] -=== Test your query - -Use the *Test query* feature to verify that your query DSL is valid. - -* Valid queries are run against the configured *index* using the configured -*time window*. The number of documents that match the query is displayed. -+ -[role="screenshot"] -image::user/alerting/images/rule-types-es-query-valid.png[Test {es} query returns number of matches when valid] -// NOTE: This is an autogenerated screenshot. Do not edit it directly. - -* An error message is shown if the query is invalid. -+ -[role="screenshot"] -image::user/alerting/images/rule-types-es-query-invalid.png[Test {es} query shows error when invalid] -// NOTE: This is an autogenerated screenshot. Do not edit it directly. - [float] === Handling multiple matches of the same document diff --git a/docs/user/production-considerations/alerting-production-considerations.asciidoc b/docs/user/production-considerations/alerting-production-considerations.asciidoc index 59c8a4bfa6d15..90e687a8f39a4 100644 --- a/docs/user/production-considerations/alerting-production-considerations.asciidoc +++ b/docs/user/production-considerations/alerting-production-considerations.asciidoc @@ -72,6 +72,11 @@ There are several scenarios where running alerting rules and actions can start t Running large numbers of rules at very short intervals can quickly clog up Task Manager throughput, leading to higher schedule drift. Use `xpack.alerting.rules.minimumScheduleInterval.value` to set a minimum schedule interval for rules. The default (and recommended) value for this configuration is `1m`. Use `xpack.alerting.rules.minimumScheduleInterval.enforce` to specify whether to strictly enforce this minimum. While the default value for this setting is `false` to maintain backwards compatibility with existing rules, set this to `true` to prevent new and updated rules from running at an interval below the minimum. +Another related setting is `xpack.alerting.rules.maxScheduledPerMinute`, which limits the number of rules that can run per minute. +For example if it's set to `400`, you can have 400 rules with one minute check intervals or 2,000 rules with 5 minute check intervals. +You cannot create or edit a rule if its check interval would cause this setting to be exceeded. +To stay within this limit, delete or disable some rules or update the check intervals so that your rules run less frequently. + [float] ==== Rules that run for a long time @@ -106,4 +111,4 @@ xpack.alerting.rules.run: connectorTypeOverrides: - id: '.email' max: 200 --- \ No newline at end of file +-- diff --git a/packages/kbn-management/cards_navigation/src/cards_navigation.tsx b/packages/kbn-management/cards_navigation/src/cards_navigation.tsx index 303742dd25fdf..b7ab58a4777db 100644 --- a/packages/kbn-management/cards_navigation/src/cards_navigation.tsx +++ b/packages/kbn-management/cards_navigation/src/cards_navigation.tsx @@ -184,7 +184,7 @@ export const CardsNavigation = ({ } + icon={} titleSize="xs" title={app.title} description={app.description} diff --git a/x-pack/plugins/aiops/common/api/log_categorization/schema.ts b/x-pack/plugins/aiops/common/api/log_categorization/schema.ts index 09b8c75bf0b48..6445e32252695 100644 --- a/x-pack/plugins/aiops/common/api/log_categorization/schema.ts +++ b/x-pack/plugins/aiops/common/api/log_categorization/schema.ts @@ -65,3 +65,6 @@ export const categorizationFieldValidationSchema = schema.object({ indicesOptions: indicesOptionsSchema, includeExamples: schema.boolean(), }); +export type CategorizationFieldValidationSchema = TypeOf< + typeof categorizationFieldValidationSchema +>; diff --git a/x-pack/plugins/aiops/server/plugin.ts b/x-pack/plugins/aiops/server/plugin.ts index 5cb23cb6ce57d..22ab62ecf52b3 100755 --- a/x-pack/plugins/aiops/server/plugin.ts +++ b/x-pack/plugins/aiops/server/plugin.ts @@ -19,9 +19,8 @@ import { AiopsPluginSetupDeps, AiopsPluginStartDeps, } from './types'; - -import { defineLogRateAnalysisRoute } from './routes'; -import { defineLogCategorizationRoutes } from './routes/log_categorization'; +import { defineRoute as defineLogRateAnalysisRoute } from './routes/log_rate_analysis/define_route'; +import { defineRoute as defineCategorizationFieldValidationRoute } from './routes/categorization_field_validation/define_route'; import { registerCasesPersistableState } from './register_cases'; export class AiopsPlugin @@ -59,7 +58,7 @@ export class AiopsPlugin // Register server side APIs core.getStartServices().then(([coreStart, depsStart]) => { defineLogRateAnalysisRoute(router, aiopsLicense, this.logger, coreStart, this.usageCounter); - defineLogCategorizationRoutes(router, aiopsLicense, this.usageCounter); + defineCategorizationFieldValidationRoute(router, aiopsLicense, this.usageCounter); }); return {}; diff --git a/x-pack/plugins/aiops/server/routes/categorization_field_validation/define_route.ts b/x-pack/plugins/aiops/server/routes/categorization_field_validation/define_route.ts new file mode 100644 index 0000000000000..e2daf77392cfc --- /dev/null +++ b/x-pack/plugins/aiops/server/routes/categorization_field_validation/define_route.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IRouter } from '@kbn/core/server'; +import type { DataRequestHandlerContext } from '@kbn/data-plugin/server'; +import type { UsageCounter } from '@kbn/usage-collection-plugin/server'; +import { categorizationFieldValidationSchema } from '../../../common/api/log_categorization/schema'; +import { AIOPS_API_ENDPOINT } from '../../../common/api'; +import type { AiopsLicense } from '../../types'; +import { routeHandlerFactory } from './route_handler_factory'; + +export const defineRoute = ( + router: IRouter, + license: AiopsLicense, + usageCounter?: UsageCounter +) => { + router.versioned + .post({ + path: AIOPS_API_ENDPOINT.CATEGORIZATION_FIELD_VALIDATION, + access: 'internal', + }) + .addVersion( + { + version: '1', + validate: { + request: { + body: categorizationFieldValidationSchema, + }, + }, + }, + routeHandlerFactory(license, usageCounter) + ); +}; diff --git a/x-pack/plugins/aiops/server/routes/categorization_field_validation/route_handler_factory.ts b/x-pack/plugins/aiops/server/routes/categorization_field_validation/route_handler_factory.ts new file mode 100644 index 0000000000000..3d203595813d4 --- /dev/null +++ b/x-pack/plugins/aiops/server/routes/categorization_field_validation/route_handler_factory.ts @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + KibanaRequest, + RequestHandlerContext, + RequestHandler, + KibanaResponseFactory, +} from '@kbn/core/server'; +import { categorizationExamplesProvider } from '@kbn/ml-category-validator'; +import type { UsageCounter } from '@kbn/usage-collection-plugin/server'; +import { wrapError } from '../error_wrapper'; +import { trackAIOpsRouteUsage } from '../../lib/track_route_usage'; +import { AIOPS_TELEMETRY_ID } from '../../../common/constants'; +import { AIOPS_API_ENDPOINT } from '../../../common/api'; +import type { AiopsLicense } from '../../types'; +import type { CategorizationFieldValidationSchema } from '../../../common/api/log_categorization/schema'; + +export const routeHandlerFactory: ( + license: AiopsLicense, + usageCounter?: UsageCounter +) => RequestHandler = + (license, usageCounter) => + async ( + context: RequestHandlerContext, + request: KibanaRequest, + response: KibanaResponseFactory + ) => { + const { headers } = request; + trackAIOpsRouteUsage( + `POST ${AIOPS_API_ENDPOINT.CATEGORIZATION_FIELD_VALIDATION}`, + headers[AIOPS_TELEMETRY_ID.AIOPS_ANALYSIS_RUN_ORIGIN], + usageCounter + ); + + if (!license.isActivePlatinumLicense) { + return response.forbidden(); + } + try { + const { + elasticsearch: { client }, + } = await context.core; + + const { + indexPatternTitle, + timeField, + query, + size, + field, + start, + end, + analyzer, + runtimeMappings, + indicesOptions, + includeExamples, + } = request.body; + + const { validateCategoryExamples } = categorizationExamplesProvider(client); + const resp = await validateCategoryExamples( + indexPatternTitle, + query, + size, + field, + timeField, + start, + end, + analyzer ?? {}, + runtimeMappings, + indicesOptions, + includeExamples + ); + + return response.ok({ + body: resp, + }); + } catch (e) { + return response.customError(wrapError(e)); + } + }; diff --git a/x-pack/plugins/aiops/server/routes/index.ts b/x-pack/plugins/aiops/server/routes/index.ts deleted file mode 100755 index ff9e54f1fcc2d..0000000000000 --- a/x-pack/plugins/aiops/server/routes/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export { defineLogRateAnalysisRoute } from './log_rate_analysis'; diff --git a/x-pack/plugins/aiops/server/routes/log_categorization.ts b/x-pack/plugins/aiops/server/routes/log_categorization.ts deleted file mode 100644 index dd437f68617ad..0000000000000 --- a/x-pack/plugins/aiops/server/routes/log_categorization.ts +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { IRouter } from '@kbn/core/server'; -import type { DataRequestHandlerContext } from '@kbn/data-plugin/server'; -import { categorizationExamplesProvider } from '@kbn/ml-category-validator'; -import type { UsageCounter } from '@kbn/usage-collection-plugin/server'; -import { categorizationFieldValidationSchema } from '../../common/api/log_categorization/schema'; -import { AIOPS_API_ENDPOINT } from '../../common/api'; -import type { AiopsLicense } from '../types'; -import { wrapError } from './error_wrapper'; -import { trackAIOpsRouteUsage } from '../lib/track_route_usage'; -import { AIOPS_TELEMETRY_ID } from '../../common/constants'; - -export const defineLogCategorizationRoutes = ( - router: IRouter, - license: AiopsLicense, - usageCounter?: UsageCounter -) => { - router.versioned - .post({ - path: AIOPS_API_ENDPOINT.CATEGORIZATION_FIELD_VALIDATION, - access: 'internal', - }) - .addVersion( - { - version: '1', - validate: { - request: { - body: categorizationFieldValidationSchema, - }, - }, - }, - async (context, request, response) => { - const { headers } = request; - trackAIOpsRouteUsage( - `POST ${AIOPS_API_ENDPOINT.CATEGORIZATION_FIELD_VALIDATION}`, - headers[AIOPS_TELEMETRY_ID.AIOPS_ANALYSIS_RUN_ORIGIN], - usageCounter - ); - - if (!license.isActivePlatinumLicense) { - return response.forbidden(); - } - try { - const { - elasticsearch: { client }, - } = await context.core; - - const { - indexPatternTitle, - timeField, - query, - size, - field, - start, - end, - analyzer, - runtimeMappings, - indicesOptions, - includeExamples, - } = request.body; - - const { validateCategoryExamples } = categorizationExamplesProvider(client); - const resp = await validateCategoryExamples( - indexPatternTitle, - query, - size, - field, - timeField, - start, - end, - analyzer ?? {}, - runtimeMappings, - indicesOptions, - includeExamples - ); - - return response.ok({ - body: resp, - }); - } catch (e) { - return response.customError(wrapError(e)); - } - } - ); -}; diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis.ts deleted file mode 100644 index 7576faa22ec27..0000000000000 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis.ts +++ /dev/null @@ -1,867 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { queue } from 'async'; - -import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; - -import { i18n } from '@kbn/i18n'; -import type { CoreStart, IRouter } from '@kbn/core/server'; -import { KBN_FIELD_TYPES } from '@kbn/field-types'; -import type { Logger } from '@kbn/logging'; -import type { DataRequestHandlerContext } from '@kbn/data-plugin/server'; -import { streamFactory } from '@kbn/ml-response-stream/server'; -import type { - SignificantTerm, - SignificantTermGroup, - NumericChartData, - NumericHistogramField, -} from '@kbn/ml-agg-utils'; -import { SIGNIFICANT_TERM_TYPE } from '@kbn/ml-agg-utils'; -import { fetchHistogramsForFields } from '@kbn/ml-agg-utils'; -import { createExecutionContext } from '@kbn/ml-route-utils'; -import type { UsageCounter } from '@kbn/usage-collection-plugin/server'; - -import { RANDOM_SAMPLER_SEED, AIOPS_TELEMETRY_ID } from '../../common/constants'; -import { - addSignificantTermsAction, - addSignificantTermsGroupAction, - addSignificantTermsGroupHistogramAction, - addSignificantTermsHistogramAction, - aiopsLogRateAnalysisSchema, - addErrorAction, - pingAction, - resetAllAction, - resetErrorsAction, - resetGroupsAction, - updateLoadingStateAction, - AiopsLogRateAnalysisApiAction, -} from '../../common/api/log_rate_analysis'; -import { getCategoryQuery } from '../../common/api/log_categorization/get_category_query'; -import { AIOPS_API_ENDPOINT } from '../../common/api'; - -import { PLUGIN_ID } from '../../common'; - -import { isRequestAbortedError } from '../lib/is_request_aborted_error'; -import type { AiopsLicense } from '../types'; - -import { fetchSignificantCategories } from './queries/fetch_significant_categories'; -import { fetchSignificantTermPValues } from './queries/fetch_significant_term_p_values'; -import { fetchIndexInfo } from './queries/fetch_index_info'; -import { fetchFrequentItemSets } from './queries/fetch_frequent_item_sets'; -import { fetchTerms2CategoriesCounts } from './queries/fetch_terms_2_categories_counts'; -import { getHistogramQuery } from './queries/get_histogram_query'; -import { getGroupFilter } from './queries/get_group_filter'; -import { getSignificantTermGroups } from './queries/get_significant_term_groups'; -import { trackAIOpsRouteUsage } from '../lib/track_route_usage'; - -// 10s ping frequency to keep the stream alive. -const PING_FREQUENCY = 10000; - -// Overall progress is a float from 0 to 1. -const LOADED_FIELD_CANDIDATES = 0.2; -const PROGRESS_STEP_P_VALUES = 0.5; -const PROGRESS_STEP_GROUPING = 0.1; -const PROGRESS_STEP_HISTOGRAMS = 0.1; -const PROGRESS_STEP_HISTOGRAMS_GROUPS = 0.1; - -export const defineLogRateAnalysisRoute = ( - router: IRouter, - license: AiopsLicense, - logger: Logger, - coreStart: CoreStart, - usageCounter?: UsageCounter -) => { - router.versioned - .post({ - path: AIOPS_API_ENDPOINT.LOG_RATE_ANALYSIS, - - access: 'internal', - }) - .addVersion( - { - version: '1', - validate: { - request: { - body: aiopsLogRateAnalysisSchema, - }, - }, - }, - async (context, request, response) => { - const { headers } = request; - - trackAIOpsRouteUsage( - `POST ${AIOPS_API_ENDPOINT.LOG_RATE_ANALYSIS}`, - headers[AIOPS_TELEMETRY_ID.AIOPS_ANALYSIS_RUN_ORIGIN], - usageCounter - ); - - if (!license.isActivePlatinumLicense) { - return response.forbidden(); - } - - const client = (await context.core).elasticsearch.client.asCurrentUser; - const executionContext = createExecutionContext(coreStart, PLUGIN_ID, request.route.path); - - return await coreStart.executionContext.withContext(executionContext, () => { - let logMessageCounter = 1; - - function logDebugMessage(msg: string) { - logger.debug(`Log Rate Analysis #${logMessageCounter}: ${msg}`); - logMessageCounter++; - } - - logDebugMessage('Starting analysis.'); - - const groupingEnabled = !!request.body.grouping; - const sampleProbability = request.body.sampleProbability ?? 1; - - const controller = new AbortController(); - const abortSignal = controller.signal; - - let isRunning = false; - let loaded = 0; - let shouldStop = false; - request.events.aborted$.subscribe(() => { - logDebugMessage('aborted$ subscription trigger.'); - shouldStop = true; - controller.abort(); - }); - request.events.completed$.subscribe(() => { - logDebugMessage('completed$ subscription trigger.'); - shouldStop = true; - controller.abort(); - }); - - const { - end: streamEnd, - push, - responseWithHeaders, - } = streamFactory( - request.headers, - logger, - request.body.compressResponse, - request.body.flushFix - ); - - function pushPingWithTimeout() { - setTimeout(() => { - if (isRunning) { - logDebugMessage('Ping message.'); - push(pingAction()); - pushPingWithTimeout(); - } - }, PING_FREQUENCY); - } - - function end() { - if (isRunning) { - isRunning = false; - logDebugMessage('Ending analysis.'); - streamEnd(); - } else { - logDebugMessage('end() was called again with isRunning already being false.'); - } - } - - function endWithUpdatedLoadingState() { - push( - updateLoadingStateAction({ - ccsWarning: false, - loaded: 1, - loadingState: i18n.translate( - 'xpack.aiops.logRateAnalysis.loadingState.doneMessage', - { - defaultMessage: 'Done.', - } - ), - }) - ); - - end(); - } - - function pushError(m: string) { - logDebugMessage('Push error.'); - push(addErrorAction(m)); - } - - async function runAnalysis() { - try { - isRunning = true; - - if (!request.body.overrides) { - logDebugMessage('Full Reset.'); - push(resetAllAction()); - } else { - logDebugMessage('Reset Errors.'); - push(resetErrorsAction()); - } - - if (request.body.overrides?.regroupOnly) { - logDebugMessage('Reset Groups.'); - push(resetGroupsAction()); - } - - if (request.body.overrides?.loaded) { - logDebugMessage(`Set 'loaded' override to '${request.body.overrides?.loaded}'.`); - loaded = request.body.overrides?.loaded; - } - - pushPingWithTimeout(); - - // Step 1: Index Info: Field candidates, total doc count, sample probability - - const fieldCandidates: string[] = []; - let fieldCandidatesCount = fieldCandidates.length; - - const textFieldCandidates: string[] = []; - - let totalDocCount = 0; - - if (!request.body.overrides?.remainingFieldCandidates) { - logDebugMessage('Fetch index information.'); - push( - updateLoadingStateAction({ - ccsWarning: false, - loaded, - loadingState: i18n.translate( - 'xpack.aiops.logRateAnalysis.loadingState.loadingIndexInformation', - { - defaultMessage: 'Loading index information.', - } - ), - }) - ); - - try { - const indexInfo = await fetchIndexInfo( - client, - request.body, - ['message', 'error.message'], - abortSignal - ); - - fieldCandidates.push(...indexInfo.fieldCandidates); - fieldCandidatesCount = fieldCandidates.length; - textFieldCandidates.push(...indexInfo.textFieldCandidates); - totalDocCount = indexInfo.totalDocCount; - } catch (e) { - if (!isRequestAbortedError(e)) { - logger.error(`Failed to fetch index information, got: \n${e.toString()}`); - pushError(`Failed to fetch index information.`); - } - end(); - return; - } - - logDebugMessage(`Total document count: ${totalDocCount}`); - logDebugMessage(`Sample probability: ${sampleProbability}`); - - loaded += LOADED_FIELD_CANDIDATES; - - pushPingWithTimeout(); - - push( - updateLoadingStateAction({ - ccsWarning: false, - loaded, - loadingState: i18n.translate( - 'xpack.aiops.logRateAnalysis.loadingState.identifiedFieldCandidates', - { - defaultMessage: - 'Identified {fieldCandidatesCount, plural, one {# field candidate} other {# field candidates}}.', - values: { - fieldCandidatesCount, - }, - } - ), - }) - ); - - if (fieldCandidatesCount === 0) { - endWithUpdatedLoadingState(); - } else if (shouldStop) { - logDebugMessage('shouldStop after fetching field candidates.'); - end(); - return; - } - } - - // Step 2: Significant Categories and Terms - - // This will store the combined count of detected significant log patterns and keywords - let fieldValuePairsCount = 0; - - const significantCategories: SignificantTerm[] = request.body.overrides - ?.significantTerms - ? request.body.overrides?.significantTerms.filter( - (d) => d.type === SIGNIFICANT_TERM_TYPE.LOG_PATTERN - ) - : []; - - // Get significant categories of text fields - if (textFieldCandidates.length > 0) { - significantCategories.push( - ...(await fetchSignificantCategories( - client, - request.body, - textFieldCandidates, - logger, - sampleProbability, - pushError, - abortSignal - )) - ); - - if (significantCategories.length > 0) { - push(addSignificantTermsAction(significantCategories)); - } - } - - const significantTerms: SignificantTerm[] = request.body.overrides?.significantTerms - ? request.body.overrides?.significantTerms.filter( - (d) => d.type === SIGNIFICANT_TERM_TYPE.KEYWORD - ) - : []; - - const fieldsToSample = new Set(); - - // Don't use more than 10 here otherwise Kibana will emit an error - // regarding a limit of abort signal listeners of more than 10. - const MAX_CONCURRENT_QUERIES = 10; - - let remainingFieldCandidates: string[]; - let loadingStepSizePValues = PROGRESS_STEP_P_VALUES; - - if (request.body.overrides?.remainingFieldCandidates) { - fieldCandidates.push(...request.body.overrides?.remainingFieldCandidates); - remainingFieldCandidates = request.body.overrides?.remainingFieldCandidates; - fieldCandidatesCount = fieldCandidates.length; - loadingStepSizePValues = - LOADED_FIELD_CANDIDATES + - PROGRESS_STEP_P_VALUES - - (request.body.overrides?.loaded ?? PROGRESS_STEP_P_VALUES); - } else { - remainingFieldCandidates = fieldCandidates; - } - - logDebugMessage('Fetch p-values.'); - - const pValuesQueue = queue(async function (fieldCandidate: string) { - loaded += (1 / fieldCandidatesCount) * loadingStepSizePValues; - - let pValues: Awaited>; - - try { - pValues = await fetchSignificantTermPValues( - client, - request.body, - [fieldCandidate], - logger, - sampleProbability, - pushError, - abortSignal - ); - } catch (e) { - if (!isRequestAbortedError(e)) { - logger.error( - `Failed to fetch p-values for '${fieldCandidate}', got: \n${e.toString()}` - ); - pushError(`Failed to fetch p-values for '${fieldCandidate}'.`); - } - return; - } - - remainingFieldCandidates = remainingFieldCandidates.filter( - (d) => d !== fieldCandidate - ); - - if (pValues.length > 0) { - pValues.forEach((d) => { - fieldsToSample.add(d.fieldName); - }); - significantTerms.push(...pValues); - - push(addSignificantTermsAction(pValues)); - } - - push( - updateLoadingStateAction({ - ccsWarning: false, - loaded, - loadingState: i18n.translate( - 'xpack.aiops.logRateAnalysis.loadingState.identifiedFieldValuePairs', - { - defaultMessage: - 'Identified {fieldValuePairsCount, plural, one {# significant field/value pair} other {# significant field/value pairs}}.', - values: { - fieldValuePairsCount, - }, - } - ), - remainingFieldCandidates, - }) - ); - }, MAX_CONCURRENT_QUERIES); - - pValuesQueue.push(fieldCandidates, (err) => { - if (err) { - logger.error(`Failed to fetch p-values.', got: \n${err.toString()}`); - pushError(`Failed to fetch p-values.`); - pValuesQueue.kill(); - end(); - } else if (shouldStop) { - logDebugMessage('shouldStop fetching p-values.'); - pValuesQueue.kill(); - end(); - } - }); - await pValuesQueue.drain(); - - fieldValuePairsCount = significantCategories.length + significantTerms.length; - - if (fieldValuePairsCount === 0) { - logDebugMessage('Stopping analysis, did not find significant terms.'); - endWithUpdatedLoadingState(); - return; - } - - const histogramFields: [NumericHistogramField] = [ - { fieldName: request.body.timeFieldName, type: KBN_FIELD_TYPES.DATE }, - ]; - - logDebugMessage('Fetch overall histogram.'); - - let overallTimeSeries: NumericChartData | undefined; - - const overallHistogramQuery = getHistogramQuery(request.body); - - try { - overallTimeSeries = ( - (await fetchHistogramsForFields( - client, - request.body.index, - overallHistogramQuery, - // fields - histogramFields, - // samplerShardSize - -1, - undefined, - abortSignal, - sampleProbability, - RANDOM_SAMPLER_SEED - )) as [NumericChartData] - )[0]; - } catch (e) { - if (!isRequestAbortedError(e)) { - logger.error( - `Failed to fetch the overall histogram data, got: \n${e.toString()}` - ); - pushError(`Failed to fetch overall histogram data.`); - } - // Still continue the analysis even if loading the overall histogram fails. - } - - function pushHistogramDataLoadingState() { - push( - updateLoadingStateAction({ - ccsWarning: false, - loaded, - loadingState: i18n.translate( - 'xpack.aiops.logRateAnalysis.loadingState.loadingHistogramData', - { - defaultMessage: 'Loading histogram data.', - } - ), - }) - ); - } - - if (shouldStop) { - logDebugMessage('shouldStop after fetching overall histogram.'); - end(); - return; - } - - if (groupingEnabled) { - logDebugMessage('Group results.'); - - push( - updateLoadingStateAction({ - ccsWarning: false, - loaded, - loadingState: i18n.translate( - 'xpack.aiops.logRateAnalysis.loadingState.groupingResults', - { - defaultMessage: 'Transforming significant field/value pairs into groups.', - } - ), - groupsMissing: true, - }) - ); - - try { - const { fields, itemSets } = await fetchFrequentItemSets( - client, - request.body.index, - JSON.parse(request.body.searchQuery) as estypes.QueryDslQueryContainer, - significantTerms, - request.body.timeFieldName, - request.body.deviationMin, - request.body.deviationMax, - logger, - sampleProbability, - pushError, - abortSignal - ); - - if (significantCategories.length > 0 && significantTerms.length > 0) { - const { - fields: significantCategoriesFields, - itemSets: significantCategoriesItemSets, - } = await fetchTerms2CategoriesCounts( - client, - request.body, - JSON.parse(request.body.searchQuery) as estypes.QueryDslQueryContainer, - significantTerms, - itemSets, - significantCategories, - request.body.deviationMin, - request.body.deviationMax, - logger, - pushError, - abortSignal - ); - - fields.push(...significantCategoriesFields); - itemSets.push(...significantCategoriesItemSets); - } - - if (shouldStop) { - logDebugMessage('shouldStop after fetching frequent_item_sets.'); - end(); - return; - } - - if (fields.length > 0 && itemSets.length > 0) { - const significantTermGroups = getSignificantTermGroups( - itemSets, - [...significantTerms, ...significantCategories], - fields - ); - - // We'll find out if there's at least one group with at least two items, - // only then will we return the groups to the clients and make the grouping option available. - const maxItems = Math.max(...significantTermGroups.map((g) => g.group.length)); - - if (maxItems > 1) { - push(addSignificantTermsGroupAction(significantTermGroups)); - } - - loaded += PROGRESS_STEP_GROUPING; - - pushHistogramDataLoadingState(); - - if (shouldStop) { - logDebugMessage('shouldStop after grouping.'); - end(); - return; - } - - logDebugMessage(`Fetch ${significantTermGroups.length} group histograms.`); - - const groupHistogramQueue = queue(async function (cpg: SignificantTermGroup) { - if (shouldStop) { - logDebugMessage('shouldStop abort fetching group histograms.'); - groupHistogramQueue.kill(); - end(); - return; - } - - if (overallTimeSeries !== undefined) { - const histogramQuery = getHistogramQuery(request.body, getGroupFilter(cpg)); - - let cpgTimeSeries: NumericChartData; - try { - cpgTimeSeries = ( - (await fetchHistogramsForFields( - client, - request.body.index, - histogramQuery, - // fields - [ - { - fieldName: request.body.timeFieldName, - type: KBN_FIELD_TYPES.DATE, - interval: overallTimeSeries.interval, - min: overallTimeSeries.stats[0], - max: overallTimeSeries.stats[1], - }, - ], - // samplerShardSize - -1, - undefined, - abortSignal, - sampleProbability, - RANDOM_SAMPLER_SEED - )) as [NumericChartData] - )[0]; - } catch (e) { - if (!isRequestAbortedError(e)) { - logger.error( - `Failed to fetch the histogram data for group #${ - cpg.id - }, got: \n${e.toString()}` - ); - pushError(`Failed to fetch the histogram data for group #${cpg.id}.`); - } - return; - } - const histogram = - overallTimeSeries.data.map((o) => { - const current = cpgTimeSeries.data.find( - (d1) => d1.key_as_string === o.key_as_string - ) ?? { - doc_count: 0, - }; - return { - key: o.key, - key_as_string: o.key_as_string ?? '', - doc_count_significant_term: current.doc_count, - doc_count_overall: Math.max(0, o.doc_count - current.doc_count), - }; - }) ?? []; - - push( - addSignificantTermsGroupHistogramAction([ - { - id: cpg.id, - histogram, - }, - ]) - ); - } - }, MAX_CONCURRENT_QUERIES); - - groupHistogramQueue.push(significantTermGroups); - await groupHistogramQueue.drain(); - } - } catch (e) { - if (!isRequestAbortedError(e)) { - logger.error( - `Failed to transform field/value pairs into groups, got: \n${e.toString()}` - ); - pushError(`Failed to transform field/value pairs into groups.`); - } - } - } - - loaded += PROGRESS_STEP_HISTOGRAMS_GROUPS; - - logDebugMessage(`Fetch ${significantTerms.length} field/value histograms.`); - - // time series filtered by fields - if ( - significantTerms.length > 0 && - overallTimeSeries !== undefined && - !request.body.overrides?.regroupOnly - ) { - const fieldValueHistogramQueue = queue(async function (cp: SignificantTerm) { - if (shouldStop) { - logDebugMessage('shouldStop abort fetching field/value histograms.'); - fieldValueHistogramQueue.kill(); - end(); - return; - } - - if (overallTimeSeries !== undefined) { - const histogramQuery = getHistogramQuery(request.body, [ - { - term: { [cp.fieldName]: cp.fieldValue }, - }, - ]); - - let cpTimeSeries: NumericChartData; - - try { - cpTimeSeries = ( - (await fetchHistogramsForFields( - client, - request.body.index, - histogramQuery, - // fields - [ - { - fieldName: request.body.timeFieldName, - type: KBN_FIELD_TYPES.DATE, - interval: overallTimeSeries.interval, - min: overallTimeSeries.stats[0], - max: overallTimeSeries.stats[1], - }, - ], - // samplerShardSize - -1, - undefined, - abortSignal, - sampleProbability, - RANDOM_SAMPLER_SEED - )) as [NumericChartData] - )[0]; - } catch (e) { - logger.error( - `Failed to fetch the histogram data for field/value pair "${cp.fieldName}:${ - cp.fieldValue - }", got: \n${e.toString()}` - ); - pushError( - `Failed to fetch the histogram data for field/value pair "${cp.fieldName}:${cp.fieldValue}".` - ); - return; - } - - const histogram = - overallTimeSeries.data.map((o) => { - const current = cpTimeSeries.data.find( - (d1) => d1.key_as_string === o.key_as_string - ) ?? { - doc_count: 0, - }; - return { - key: o.key, - key_as_string: o.key_as_string ?? '', - doc_count_significant_term: current.doc_count, - doc_count_overall: Math.max(0, o.doc_count - current.doc_count), - }; - }) ?? []; - - const { fieldName, fieldValue } = cp; - - loaded += (1 / fieldValuePairsCount) * PROGRESS_STEP_HISTOGRAMS; - pushHistogramDataLoadingState(); - push( - addSignificantTermsHistogramAction([ - { - fieldName, - fieldValue, - histogram, - }, - ]) - ); - } - }, MAX_CONCURRENT_QUERIES); - - fieldValueHistogramQueue.push(significantTerms); - await fieldValueHistogramQueue.drain(); - } - - // histograms for text field patterns - if ( - overallTimeSeries !== undefined && - significantCategories.length > 0 && - !request.body.overrides?.regroupOnly - ) { - const significantCategoriesHistogramQueries = significantCategories.map((d) => { - const histogramQuery = getHistogramQuery(request.body); - const categoryQuery = getCategoryQuery(d.fieldName, [ - { key: `${d.key}`, count: d.doc_count, examples: [] }, - ]); - if (Array.isArray(histogramQuery.bool?.filter)) { - histogramQuery.bool?.filter?.push(categoryQuery); - } - return histogramQuery; - }); - - for (const [i, histogramQuery] of significantCategoriesHistogramQueries.entries()) { - const cp = significantCategories[i]; - let catTimeSeries: NumericChartData; - - try { - catTimeSeries = ( - (await fetchHistogramsForFields( - client, - request.body.index, - histogramQuery, - // fields - [ - { - fieldName: request.body.timeFieldName, - type: KBN_FIELD_TYPES.DATE, - interval: overallTimeSeries.interval, - min: overallTimeSeries.stats[0], - max: overallTimeSeries.stats[1], - }, - ], - // samplerShardSize - -1, - undefined, - abortSignal, - sampleProbability, - RANDOM_SAMPLER_SEED - )) as [NumericChartData] - )[0]; - } catch (e) { - logger.error( - `Failed to fetch the histogram data for field/value pair "${cp.fieldName}:${ - cp.fieldValue - }", got: \n${e.toString()}` - ); - pushError( - `Failed to fetch the histogram data for field/value pair "${cp.fieldName}:${cp.fieldValue}".` - ); - return; - } - - const histogram = - overallTimeSeries.data.map((o) => { - const current = catTimeSeries.data.find( - (d1) => d1.key_as_string === o.key_as_string - ) ?? { - doc_count: 0, - }; - return { - key: o.key, - key_as_string: o.key_as_string ?? '', - doc_count_significant_term: current.doc_count, - doc_count_overall: Math.max(0, o.doc_count - current.doc_count), - }; - }) ?? []; - - const { fieldName, fieldValue } = cp; - - loaded += (1 / fieldValuePairsCount) * PROGRESS_STEP_HISTOGRAMS; - pushHistogramDataLoadingState(); - push( - addSignificantTermsHistogramAction([ - { - fieldName, - fieldValue, - histogram, - }, - ]) - ); - } - } - - endWithUpdatedLoadingState(); - } catch (e) { - if (!isRequestAbortedError(e)) { - logger.error(`Log Rate Analysis failed to finish, got: \n${e.toString()}`); - pushError(`Log Rate Analysis failed to finish.`); - } - end(); - } - } - - // Do not call this using `await` so it will run asynchronously while we return the stream already. - runAnalysis(); - - return response.ok(responseWithHeaders); - }); - } - ); -}; diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/define_route.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/define_route.ts new file mode 100644 index 0000000000000..3be8ef8dcdd2b --- /dev/null +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/define_route.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { CoreStart, IRouter } from '@kbn/core/server'; +import type { Logger } from '@kbn/logging'; +import type { DataRequestHandlerContext } from '@kbn/data-plugin/server'; +import type { UsageCounter } from '@kbn/usage-collection-plugin/server'; + +import { aiopsLogRateAnalysisSchema } from '../../../common/api/log_rate_analysis'; +import { AIOPS_API_ENDPOINT } from '../../../common/api'; + +import type { AiopsLicense } from '../../types'; + +import { routeHandlerFactory } from './route_handler_factory'; + +export const defineRoute = ( + router: IRouter, + license: AiopsLicense, + logger: Logger, + coreStart: CoreStart, + usageCounter?: UsageCounter +) => { + router.versioned + .post({ + path: AIOPS_API_ENDPOINT.LOG_RATE_ANALYSIS, + + access: 'internal', + }) + .addVersion( + { + version: '1', + validate: { + request: { + body: aiopsLogRateAnalysisSchema, + }, + }, + }, + routeHandlerFactory(license, logger, coreStart, usageCounter) + ); +}; diff --git a/x-pack/plugins/aiops/server/routes/queries/duplicate_identifier.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/duplicate_identifier.ts similarity index 100% rename from x-pack/plugins/aiops/server/routes/queries/duplicate_identifier.ts rename to x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/duplicate_identifier.ts diff --git a/x-pack/plugins/aiops/server/routes/queries/fetch_categories.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_categories.ts similarity index 90% rename from x-pack/plugins/aiops/server/routes/queries/fetch_categories.ts rename to x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_categories.ts index b58e438e3882a..bb49622ad999c 100644 --- a/x-pack/plugins/aiops/server/routes/queries/fetch_categories.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_categories.ts @@ -15,16 +15,16 @@ import { type RandomSamplerWrapper, } from '@kbn/ml-random-sampler-utils'; -import { RANDOM_SAMPLER_SEED } from '../../../common/constants'; -import type { AiopsLogRateAnalysisSchema } from '../../../common/api/log_rate_analysis'; -import { createCategoryRequest } from '../../../common/api/log_categorization/create_category_request'; +import { RANDOM_SAMPLER_SEED } from '../../../../common/constants'; +import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis'; +import { createCategoryRequest } from '../../../../common/api/log_categorization/create_category_request'; import type { Category, CategoriesAgg, SparkLinesPerCategory, -} from '../../../common/api/log_categorization/types'; +} from '../../../../common/api/log_categorization/types'; -import { isRequestAbortedError } from '../../lib/is_request_aborted_error'; +import { isRequestAbortedError } from '../../../lib/is_request_aborted_error'; import { getQueryWithParams } from './get_query_with_params'; diff --git a/x-pack/plugins/aiops/server/routes/queries/fetch_category_counts.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_category_counts.ts similarity index 90% rename from x-pack/plugins/aiops/server/routes/queries/fetch_category_counts.ts rename to x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_category_counts.ts index f27d2190a8ca6..fc7c14fb8f2ee 100644 --- a/x-pack/plugins/aiops/server/routes/queries/fetch_category_counts.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_category_counts.ts @@ -12,11 +12,11 @@ import { ElasticsearchClient } from '@kbn/core/server'; import type { Logger } from '@kbn/logging'; import { isPopulatedObject } from '@kbn/ml-is-populated-object'; -import type { AiopsLogRateAnalysisSchema } from '../../../common/api/log_rate_analysis'; -import { getCategoryQuery } from '../../../common/api/log_categorization/get_category_query'; -import type { Category } from '../../../common/api/log_categorization/types'; +import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis'; +import { getCategoryQuery } from '../../../../common/api/log_categorization/get_category_query'; +import type { Category } from '../../../../common/api/log_categorization/types'; -import { isRequestAbortedError } from '../../lib/is_request_aborted_error'; +import { isRequestAbortedError } from '../../../lib/is_request_aborted_error'; import { getQueryWithParams } from './get_query_with_params'; import type { FetchCategoriesResponse } from './fetch_categories'; diff --git a/x-pack/plugins/aiops/server/routes/queries/fetch_frequent_item_sets.test.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_frequent_item_sets.test.ts similarity index 93% rename from x-pack/plugins/aiops/server/routes/queries/fetch_frequent_item_sets.test.ts rename to x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_frequent_item_sets.test.ts index e2dbeb781b999..d262ecc026481 100644 --- a/x-pack/plugins/aiops/server/routes/queries/fetch_frequent_item_sets.test.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_frequent_item_sets.test.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { significantTerms } from '../../../common/__mocks__/artificial_logs/significant_terms'; +import { significantTerms } from '../../../../common/__mocks__/artificial_logs/significant_terms'; import { getShouldClauses, getFrequentItemSetsAggFields } from './fetch_frequent_item_sets'; diff --git a/x-pack/plugins/aiops/server/routes/queries/fetch_frequent_item_sets.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_frequent_item_sets.ts similarity index 98% rename from x-pack/plugins/aiops/server/routes/queries/fetch_frequent_item_sets.ts rename to x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_frequent_item_sets.ts index d73c3742e8e66..c74030efaea7f 100644 --- a/x-pack/plugins/aiops/server/routes/queries/fetch_frequent_item_sets.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_frequent_item_sets.ts @@ -15,12 +15,12 @@ import type { Logger } from '@kbn/logging'; import { type SignificantTerm } from '@kbn/ml-agg-utils'; import { createRandomSamplerWrapper } from '@kbn/ml-random-sampler-utils'; -import { RANDOM_SAMPLER_SEED, LOG_RATE_ANALYSIS_SETTINGS } from '../../../common/constants'; +import { RANDOM_SAMPLER_SEED, LOG_RATE_ANALYSIS_SETTINGS } from '../../../../common/constants'; import type { SignificantTermDuplicateGroup, ItemSet, FetchFrequentItemSetsResponse, -} from '../../../common/types'; +} from '../../../../common/types'; interface FrequentItemSetsAggregation extends estypes.AggregationsSamplerAggregation { fi: { diff --git a/x-pack/plugins/aiops/server/routes/queries/fetch_index_info.test.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_index_info.test.ts similarity index 97% rename from x-pack/plugins/aiops/server/routes/queries/fetch_index_info.test.ts rename to x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_index_info.test.ts index 3cbdf3d25420b..790989df9833c 100644 --- a/x-pack/plugins/aiops/server/routes/queries/fetch_index_info.test.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_index_info.test.ts @@ -9,7 +9,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { ElasticsearchClient } from '@kbn/core/server'; -import type { AiopsLogRateAnalysisSchema } from '../../../common/api/log_rate_analysis'; +import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis'; import { fetchIndexInfo, getRandomDocsRequest } from './fetch_index_info'; diff --git a/x-pack/plugins/aiops/server/routes/queries/fetch_index_info.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_index_info.ts similarity index 97% rename from x-pack/plugins/aiops/server/routes/queries/fetch_index_info.ts rename to x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_index_info.ts index 08c510405e32c..cfb9426a739ad 100644 --- a/x-pack/plugins/aiops/server/routes/queries/fetch_index_info.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_index_info.ts @@ -10,7 +10,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { ES_FIELD_TYPES } from '@kbn/field-types'; import type { ElasticsearchClient } from '@kbn/core/server'; -import type { AiopsLogRateAnalysisSchema } from '../../../common/api/log_rate_analysis'; +import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis'; import { getQueryWithParams } from './get_query_with_params'; import { getRequestBase } from './get_request_base'; diff --git a/x-pack/plugins/aiops/server/routes/queries/fetch_significant_categories.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_significant_categories.ts similarity index 95% rename from x-pack/plugins/aiops/server/routes/queries/fetch_significant_categories.ts rename to x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_significant_categories.ts index d8bd92f04e6a6..c9e54be509426 100644 --- a/x-pack/plugins/aiops/server/routes/queries/fetch_significant_categories.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_significant_categories.ts @@ -12,9 +12,9 @@ import type { Logger } from '@kbn/logging'; import { criticalTableLookup, type Histogram } from '@kbn/ml-chi2test'; import { type SignificantTerm, SIGNIFICANT_TERM_TYPE } from '@kbn/ml-agg-utils'; -import type { Category } from '../../../common/api/log_categorization/types'; -import type { AiopsLogRateAnalysisSchema } from '../../../common/api/log_rate_analysis'; -import { LOG_RATE_ANALYSIS_SETTINGS } from '../../../common/constants'; +import type { Category } from '../../../../common/api/log_categorization/types'; +import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis'; +import { LOG_RATE_ANALYSIS_SETTINGS } from '../../../../common/constants'; import { fetchCategories } from './fetch_categories'; import { fetchCategoryCounts } from './fetch_category_counts'; diff --git a/x-pack/plugins/aiops/server/routes/queries/fetch_significant_term_p_values.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_significant_term_p_values.ts similarity index 96% rename from x-pack/plugins/aiops/server/routes/queries/fetch_significant_term_p_values.ts rename to x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_significant_term_p_values.ts index ec1500092168f..00bcd918f48d9 100644 --- a/x-pack/plugins/aiops/server/routes/queries/fetch_significant_term_p_values.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_significant_term_p_values.ts @@ -15,10 +15,10 @@ import { type RandomSamplerWrapper, } from '@kbn/ml-random-sampler-utils'; -import { LOG_RATE_ANALYSIS_SETTINGS, RANDOM_SAMPLER_SEED } from '../../../common/constants'; -import type { AiopsLogRateAnalysisSchema } from '../../../common/api/log_rate_analysis'; +import { LOG_RATE_ANALYSIS_SETTINGS, RANDOM_SAMPLER_SEED } from '../../../../common/constants'; +import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis'; -import { isRequestAbortedError } from '../../lib/is_request_aborted_error'; +import { isRequestAbortedError } from '../../../lib/is_request_aborted_error'; import { getNormalizedScore } from './get_normalized_score'; import { getQueryWithParams } from './get_query_with_params'; diff --git a/x-pack/plugins/aiops/server/routes/queries/fetch_terms_2_categories_counts.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_terms_2_categories_counts.ts similarity index 92% rename from x-pack/plugins/aiops/server/routes/queries/fetch_terms_2_categories_counts.ts rename to x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_terms_2_categories_counts.ts index a88090d3ab059..f1629f212ae99 100644 --- a/x-pack/plugins/aiops/server/routes/queries/fetch_terms_2_categories_counts.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_terms_2_categories_counts.ts @@ -14,13 +14,13 @@ import type { Logger } from '@kbn/logging'; import type { FieldValuePair, SignificantTerm } from '@kbn/ml-agg-utils'; import { isPopulatedObject } from '@kbn/ml-is-populated-object'; -import type { AiopsLogRateAnalysisSchema } from '../../../common/api/log_rate_analysis'; -import type { FetchFrequentItemSetsResponse, ItemSet } from '../../../common/types'; -import { getCategoryQuery } from '../../../common/api/log_categorization/get_category_query'; -import type { Category } from '../../../common/api/log_categorization/types'; -import { LOG_RATE_ANALYSIS_SETTINGS } from '../../../common/constants'; +import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis'; +import type { FetchFrequentItemSetsResponse, ItemSet } from '../../../../common/types'; +import { getCategoryQuery } from '../../../../common/api/log_categorization/get_category_query'; +import type { Category } from '../../../../common/api/log_categorization/types'; +import { LOG_RATE_ANALYSIS_SETTINGS } from '../../../../common/constants'; -import { isRequestAbortedError } from '../../lib/is_request_aborted_error'; +import { isRequestAbortedError } from '../../../lib/is_request_aborted_error'; import { getQueryWithParams } from './get_query_with_params'; diff --git a/x-pack/plugins/aiops/server/routes/queries/get_field_value_pair_counts.test.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_field_value_pair_counts.test.ts similarity index 78% rename from x-pack/plugins/aiops/server/routes/queries/get_field_value_pair_counts.test.ts rename to x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_field_value_pair_counts.test.ts index a762c04f14810..fb04844c5bd3d 100644 --- a/x-pack/plugins/aiops/server/routes/queries/get_field_value_pair_counts.test.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_field_value_pair_counts.test.ts @@ -5,10 +5,10 @@ * 2.0. */ -import { significantTermGroups } from '../../../common/__mocks__/farequote/significant_term_groups'; -import { fields } from '../../../common/__mocks__/artificial_logs/fields'; -import { filteredFrequentItemSets } from '../../../common/__mocks__/artificial_logs/filtered_frequent_item_sets'; -import { significantTerms } from '../../../common/__mocks__/artificial_logs/significant_terms'; +import { significantTermGroups } from '../../../../common/__mocks__/farequote/significant_term_groups'; +import { fields } from '../../../../common/__mocks__/artificial_logs/fields'; +import { filteredFrequentItemSets } from '../../../../common/__mocks__/artificial_logs/filtered_frequent_item_sets'; +import { significantTerms } from '../../../../common/__mocks__/artificial_logs/significant_terms'; import { getFieldValuePairCounts } from './get_field_value_pair_counts'; import { getSimpleHierarchicalTree } from './get_simple_hierarchical_tree'; diff --git a/x-pack/plugins/aiops/server/routes/queries/get_field_value_pair_counts.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_field_value_pair_counts.ts similarity index 91% rename from x-pack/plugins/aiops/server/routes/queries/get_field_value_pair_counts.ts rename to x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_field_value_pair_counts.ts index 7637bf27919ab..429306139d402 100644 --- a/x-pack/plugins/aiops/server/routes/queries/get_field_value_pair_counts.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_field_value_pair_counts.ts @@ -7,7 +7,7 @@ import type { SignificantTermGroup } from '@kbn/ml-agg-utils'; -import type { FieldValuePairCounts } from '../../../common/types'; +import type { FieldValuePairCounts } from '../../../../common/types'; /** * Get a nested record of field/value pairs with counts diff --git a/x-pack/plugins/aiops/server/routes/queries/get_filters.test.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_filters.test.ts similarity index 100% rename from x-pack/plugins/aiops/server/routes/queries/get_filters.test.ts rename to x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_filters.test.ts diff --git a/x-pack/plugins/aiops/server/routes/queries/get_filters.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_filters.ts similarity index 90% rename from x-pack/plugins/aiops/server/routes/queries/get_filters.ts rename to x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_filters.ts index a9d61755ec43b..5e58e138dac6b 100644 --- a/x-pack/plugins/aiops/server/routes/queries/get_filters.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_filters.ts @@ -9,7 +9,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { ESFilter } from '@kbn/es-types'; -import type { AiopsLogRateAnalysisSchema } from '../../../common/api/log_rate_analysis'; +import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis'; export function rangeQuery( start?: number, diff --git a/x-pack/plugins/aiops/server/routes/queries/get_group_filter.test.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_group_filter.test.ts similarity index 84% rename from x-pack/plugins/aiops/server/routes/queries/get_group_filter.test.ts rename to x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_group_filter.test.ts index c27916a67eb63..e4d30a4438c6e 100644 --- a/x-pack/plugins/aiops/server/routes/queries/get_group_filter.test.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_group_filter.test.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { finalSignificantTermGroups } from '../../../common/__mocks__/artificial_logs/final_significant_term_groups'; +import { finalSignificantTermGroups } from '../../../../common/__mocks__/artificial_logs/final_significant_term_groups'; import { getGroupFilter } from './get_group_filter'; diff --git a/x-pack/plugins/aiops/server/routes/queries/get_group_filter.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_group_filter.ts similarity index 95% rename from x-pack/plugins/aiops/server/routes/queries/get_group_filter.ts rename to x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_group_filter.ts index 86fd60b9fe8b0..d968ce90ec91b 100644 --- a/x-pack/plugins/aiops/server/routes/queries/get_group_filter.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_group_filter.ts @@ -9,7 +9,7 @@ import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { type SignificantTermGroup, SIGNIFICANT_TERM_TYPE } from '@kbn/ml-agg-utils'; -import { getCategoryQuery } from '../../../common/api/log_categorization/get_category_query'; +import { getCategoryQuery } from '../../../../common/api/log_categorization/get_category_query'; // Transforms a list of significant terms from a group in a query filter. // Uses a `term` filter for single field value combinations. diff --git a/x-pack/plugins/aiops/server/routes/queries/get_groups_with_readded_duplicates.test.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_groups_with_readded_duplicates.test.ts similarity index 89% rename from x-pack/plugins/aiops/server/routes/queries/get_groups_with_readded_duplicates.test.ts rename to x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_groups_with_readded_duplicates.test.ts index c0a2da80a080b..3c7325cdb49eb 100644 --- a/x-pack/plugins/aiops/server/routes/queries/get_groups_with_readded_duplicates.test.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_groups_with_readded_duplicates.test.ts @@ -5,8 +5,8 @@ * 2.0. */ -import { significantTermGroups } from '../../../common/__mocks__/artificial_logs/significant_term_groups'; -import { significantTerms } from '../../../common/__mocks__/artificial_logs/significant_terms'; +import { significantTermGroups } from '../../../../common/__mocks__/artificial_logs/significant_term_groups'; +import { significantTerms } from '../../../../common/__mocks__/artificial_logs/significant_terms'; import { duplicateIdentifier } from './duplicate_identifier'; import { getGroupsWithReaddedDuplicates } from './get_groups_with_readded_duplicates'; diff --git a/x-pack/plugins/aiops/server/routes/queries/get_groups_with_readded_duplicates.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_groups_with_readded_duplicates.ts similarity index 94% rename from x-pack/plugins/aiops/server/routes/queries/get_groups_with_readded_duplicates.ts rename to x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_groups_with_readded_duplicates.ts index e6afb5e52ab53..6defb9d886662 100644 --- a/x-pack/plugins/aiops/server/routes/queries/get_groups_with_readded_duplicates.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_groups_with_readded_duplicates.ts @@ -9,7 +9,7 @@ import { uniqWith, isEqual } from 'lodash'; import type { SignificantTermGroup } from '@kbn/ml-agg-utils'; -import type { SignificantTermDuplicateGroup } from '../../../common/types'; +import type { SignificantTermDuplicateGroup } from '../../../../common/types'; export function getGroupsWithReaddedDuplicates( groups: SignificantTermGroup[], diff --git a/x-pack/plugins/aiops/server/routes/queries/get_histogram_query.test.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_histogram_query.test.ts similarity index 100% rename from x-pack/plugins/aiops/server/routes/queries/get_histogram_query.test.ts rename to x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_histogram_query.test.ts diff --git a/x-pack/plugins/aiops/server/routes/queries/get_histogram_query.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_histogram_query.ts similarity index 92% rename from x-pack/plugins/aiops/server/routes/queries/get_histogram_query.ts rename to x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_histogram_query.ts index ad99a967894f6..b45d7d026638e 100644 --- a/x-pack/plugins/aiops/server/routes/queries/get_histogram_query.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_histogram_query.ts @@ -7,7 +7,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import type { AiopsLogRateAnalysisSchema } from '../../../common/api/log_rate_analysis'; +import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis'; import { getQueryWithParams } from './get_query_with_params'; diff --git a/x-pack/plugins/aiops/server/routes/queries/get_marked_duplicates.test.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_marked_duplicates.test.ts similarity index 90% rename from x-pack/plugins/aiops/server/routes/queries/get_marked_duplicates.test.ts rename to x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_marked_duplicates.test.ts index 694767a17b55d..9c0d86a392e4d 100644 --- a/x-pack/plugins/aiops/server/routes/queries/get_marked_duplicates.test.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_marked_duplicates.test.ts @@ -5,10 +5,10 @@ * 2.0. */ -import { significantTermGroups } from '../../../common/__mocks__/farequote/significant_term_groups'; -import { fields } from '../../../common/__mocks__/artificial_logs/fields'; -import { filteredFrequentItemSets } from '../../../common/__mocks__/artificial_logs/filtered_frequent_item_sets'; -import { significantTerms } from '../../../common/__mocks__/artificial_logs/significant_terms'; +import { significantTermGroups } from '../../../../common/__mocks__/farequote/significant_term_groups'; +import { fields } from '../../../../common/__mocks__/artificial_logs/fields'; +import { filteredFrequentItemSets } from '../../../../common/__mocks__/artificial_logs/filtered_frequent_item_sets'; +import { significantTerms } from '../../../../common/__mocks__/artificial_logs/significant_terms'; import { getFieldValuePairCounts } from './get_field_value_pair_counts'; import { getMarkedDuplicates } from './get_marked_duplicates'; diff --git a/x-pack/plugins/aiops/server/routes/queries/get_marked_duplicates.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_marked_duplicates.ts similarity index 91% rename from x-pack/plugins/aiops/server/routes/queries/get_marked_duplicates.ts rename to x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_marked_duplicates.ts index 202aa4f016326..7708d0634bda3 100644 --- a/x-pack/plugins/aiops/server/routes/queries/get_marked_duplicates.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_marked_duplicates.ts @@ -7,7 +7,7 @@ import type { SignificantTermGroup } from '@kbn/ml-agg-utils'; -import type { FieldValuePairCounts } from '../../../common/types'; +import type { FieldValuePairCounts } from '../../../../common/types'; /** * Analyse duplicate field/value pairs in groups. diff --git a/x-pack/plugins/aiops/server/routes/queries/get_missing_significant_terms.test.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_missing_significant_terms.test.ts similarity index 91% rename from x-pack/plugins/aiops/server/routes/queries/get_missing_significant_terms.test.ts rename to x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_missing_significant_terms.test.ts index e721143ad150c..412b7013b64d6 100644 --- a/x-pack/plugins/aiops/server/routes/queries/get_missing_significant_terms.test.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_missing_significant_terms.test.ts @@ -5,8 +5,8 @@ * 2.0. */ -import { significantTermGroups } from '../../../common/__mocks__/artificial_logs/significant_term_groups'; -import { significantTerms } from '../../../common/__mocks__/artificial_logs/significant_terms'; +import { significantTermGroups } from '../../../../common/__mocks__/artificial_logs/significant_term_groups'; +import { significantTerms } from '../../../../common/__mocks__/artificial_logs/significant_terms'; import { duplicateIdentifier } from './duplicate_identifier'; import { getGroupsWithReaddedDuplicates } from './get_groups_with_readded_duplicates'; diff --git a/x-pack/plugins/aiops/server/routes/queries/get_missing_significant_terms.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_missing_significant_terms.ts similarity index 100% rename from x-pack/plugins/aiops/server/routes/queries/get_missing_significant_terms.ts rename to x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_missing_significant_terms.ts diff --git a/x-pack/plugins/aiops/server/routes/queries/get_normalized_score.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_normalized_score.ts similarity index 100% rename from x-pack/plugins/aiops/server/routes/queries/get_normalized_score.ts rename to x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_normalized_score.ts diff --git a/x-pack/plugins/aiops/server/routes/queries/get_query_with_params.test.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_query_with_params.test.ts similarity index 100% rename from x-pack/plugins/aiops/server/routes/queries/get_query_with_params.test.ts rename to x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_query_with_params.test.ts diff --git a/x-pack/plugins/aiops/server/routes/queries/get_query_with_params.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_query_with_params.ts similarity index 92% rename from x-pack/plugins/aiops/server/routes/queries/get_query_with_params.ts rename to x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_query_with_params.ts index c9d15d6b89232..2d68c666b78ea 100644 --- a/x-pack/plugins/aiops/server/routes/queries/get_query_with_params.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_query_with_params.ts @@ -9,7 +9,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { FieldValuePair } from '@kbn/ml-agg-utils'; -import type { AiopsLogRateAnalysisSchema } from '../../../common/api/log_rate_analysis'; +import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis'; import { getFilters } from './get_filters'; diff --git a/x-pack/plugins/aiops/server/routes/queries/get_request_base.test.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_request_base.test.ts similarity index 100% rename from x-pack/plugins/aiops/server/routes/queries/get_request_base.test.ts rename to x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_request_base.test.ts diff --git a/x-pack/plugins/aiops/server/routes/queries/get_request_base.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_request_base.ts similarity index 82% rename from x-pack/plugins/aiops/server/routes/queries/get_request_base.ts rename to x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_request_base.ts index a6449249573f6..2410be74ea6b0 100644 --- a/x-pack/plugins/aiops/server/routes/queries/get_request_base.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_request_base.ts @@ -5,7 +5,7 @@ * 2.0. */ -import type { AiopsLogRateAnalysisSchema } from '../../../common/api/log_rate_analysis'; +import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis'; export const getRequestBase = ({ index, includeFrozen }: AiopsLogRateAnalysisSchema) => ({ index, diff --git a/x-pack/plugins/aiops/server/routes/queries/get_significant_term_groups.test.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_significant_term_groups.test.ts similarity index 64% rename from x-pack/plugins/aiops/server/routes/queries/get_significant_term_groups.test.ts rename to x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_significant_term_groups.test.ts index f15be7b8ed61b..05a682bc4ea34 100644 --- a/x-pack/plugins/aiops/server/routes/queries/get_significant_term_groups.test.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_significant_term_groups.test.ts @@ -7,10 +7,10 @@ import { orderBy } from 'lodash'; -import { fields } from '../../../common/__mocks__/artificial_logs/fields'; -import { frequentItemSets } from '../../../common/__mocks__/artificial_logs/frequent_item_sets'; -import { significantTerms } from '../../../common/__mocks__/artificial_logs/significant_terms'; -import { finalSignificantTermGroups } from '../../../common/__mocks__/artificial_logs/final_significant_term_groups'; +import { fields } from '../../../../common/__mocks__/artificial_logs/fields'; +import { frequentItemSets } from '../../../../common/__mocks__/artificial_logs/frequent_item_sets'; +import { significantTerms } from '../../../../common/__mocks__/artificial_logs/significant_terms'; +import { finalSignificantTermGroups } from '../../../../common/__mocks__/artificial_logs/final_significant_term_groups'; import { getSignificantTermGroups } from './get_significant_term_groups'; diff --git a/x-pack/plugins/aiops/server/routes/queries/get_significant_term_groups.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_significant_term_groups.ts similarity index 98% rename from x-pack/plugins/aiops/server/routes/queries/get_significant_term_groups.ts rename to x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_significant_term_groups.ts index 33337603bd04e..1b498b0d76595 100644 --- a/x-pack/plugins/aiops/server/routes/queries/get_significant_term_groups.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_significant_term_groups.ts @@ -15,7 +15,7 @@ import { getSimpleHierarchicalTree } from './get_simple_hierarchical_tree'; import { getSimpleHierarchicalTreeLeaves } from './get_simple_hierarchical_tree_leaves'; import { getMissingSignificantTerms } from './get_missing_significant_terms'; import { transformSignificantTermToGroup } from './transform_significant_term_to_group'; -import type { ItemSet } from '../../../common/types'; +import type { ItemSet } from '../../../../common/types'; export function getSignificantTermGroups( itemsets: ItemSet[], diff --git a/x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree.test.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_simple_hierarchical_tree.test.ts similarity index 93% rename from x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree.test.ts rename to x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_simple_hierarchical_tree.test.ts index 1713e677c2b14..32f73d6ca2387 100644 --- a/x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree.test.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_simple_hierarchical_tree.test.ts @@ -5,9 +5,9 @@ * 2.0. */ -import { fields } from '../../../common/__mocks__/artificial_logs/fields'; -import { filteredFrequentItemSets } from '../../../common/__mocks__/artificial_logs/filtered_frequent_item_sets'; -import { significantTerms } from '../../../common/__mocks__/artificial_logs/significant_terms'; +import { fields } from '../../../../common/__mocks__/artificial_logs/fields'; +import { filteredFrequentItemSets } from '../../../../common/__mocks__/artificial_logs/filtered_frequent_item_sets'; +import { significantTerms } from '../../../../common/__mocks__/artificial_logs/significant_terms'; import { getSimpleHierarchicalTree } from './get_simple_hierarchical_tree'; diff --git a/x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_simple_hierarchical_tree.ts similarity index 99% rename from x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree.ts rename to x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_simple_hierarchical_tree.ts index fc445fd88f1a6..2ffcc184fa71e 100644 --- a/x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_simple_hierarchical_tree.ts @@ -7,7 +7,7 @@ import type { SignificantTerm } from '@kbn/ml-agg-utils'; -import type { ItemSet, SimpleHierarchicalTreeNode } from '../../../common/types'; +import type { ItemSet, SimpleHierarchicalTreeNode } from '../../../../common/types'; import { getValueCounts } from './get_value_counts'; import { getValuesDescending } from './get_values_descending'; diff --git a/x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree_leaves.test.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_simple_hierarchical_tree_leaves.test.ts similarity index 87% rename from x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree_leaves.test.ts rename to x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_simple_hierarchical_tree_leaves.test.ts index 5ca23395c9815..6cb4f4b3c6532 100644 --- a/x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree_leaves.test.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_simple_hierarchical_tree_leaves.test.ts @@ -5,9 +5,9 @@ * 2.0. */ -import { fields } from '../../../common/__mocks__/artificial_logs/fields'; -import { filteredFrequentItemSets } from '../../../common/__mocks__/artificial_logs/filtered_frequent_item_sets'; -import { significantTerms } from '../../../common/__mocks__/artificial_logs/significant_terms'; +import { fields } from '../../../../common/__mocks__/artificial_logs/fields'; +import { filteredFrequentItemSets } from '../../../../common/__mocks__/artificial_logs/filtered_frequent_item_sets'; +import { significantTerms } from '../../../../common/__mocks__/artificial_logs/significant_terms'; import { getSimpleHierarchicalTree } from './get_simple_hierarchical_tree'; import { getSimpleHierarchicalTreeLeaves } from './get_simple_hierarchical_tree_leaves'; diff --git a/x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree_leaves.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_simple_hierarchical_tree_leaves.ts similarity index 95% rename from x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree_leaves.ts rename to x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_simple_hierarchical_tree_leaves.ts index f51d88c6ac3a5..bd183239eeadf 100644 --- a/x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree_leaves.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_simple_hierarchical_tree_leaves.ts @@ -9,7 +9,7 @@ import { orderBy } from 'lodash'; import type { SignificantTermGroup } from '@kbn/ml-agg-utils'; import { stringHash } from '@kbn/ml-string-hash'; -import type { SimpleHierarchicalTreeNode } from '../../../common/types'; +import type { SimpleHierarchicalTreeNode } from '../../../../common/types'; /** * Get leaves from hierarchical tree. diff --git a/x-pack/plugins/aiops/server/routes/queries/get_value_counts.test.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_value_counts.test.ts similarity index 88% rename from x-pack/plugins/aiops/server/routes/queries/get_value_counts.test.ts rename to x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_value_counts.test.ts index 515031c0e2af0..9187ed7c0eaab 100644 --- a/x-pack/plugins/aiops/server/routes/queries/get_value_counts.test.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_value_counts.test.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { frequentItemSets } from '../../../common/__mocks__/artificial_logs/frequent_item_sets'; +import { frequentItemSets } from '../../../../common/__mocks__/artificial_logs/frequent_item_sets'; import { getValueCounts } from './get_value_counts'; describe('getValueCounts', () => { diff --git a/x-pack/plugins/aiops/server/routes/queries/get_value_counts.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_value_counts.ts similarity index 90% rename from x-pack/plugins/aiops/server/routes/queries/get_value_counts.ts rename to x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_value_counts.ts index 42f022db5dccf..5c54412e46a5b 100644 --- a/x-pack/plugins/aiops/server/routes/queries/get_value_counts.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_value_counts.ts @@ -5,7 +5,7 @@ * 2.0. */ -import type { ItemSet } from '../../../common/types'; +import type { ItemSet } from '../../../../common/types'; export function getValueCounts(df: ItemSet[], field: string) { return df.reduce>((p, c) => { diff --git a/x-pack/plugins/aiops/server/routes/queries/get_values_descending.test.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_values_descending.test.ts similarity index 88% rename from x-pack/plugins/aiops/server/routes/queries/get_values_descending.test.ts rename to x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_values_descending.test.ts index d8beffed5c8ce..cc5fc2d39bee3 100644 --- a/x-pack/plugins/aiops/server/routes/queries/get_values_descending.test.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_values_descending.test.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { frequentItemSets } from '../../../common/__mocks__/artificial_logs/frequent_item_sets'; +import { frequentItemSets } from '../../../../common/__mocks__/artificial_logs/frequent_item_sets'; import { getValuesDescending } from './get_values_descending'; describe('getValuesDescending', () => { diff --git a/x-pack/plugins/aiops/server/routes/queries/get_values_descending.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_values_descending.ts similarity index 90% rename from x-pack/plugins/aiops/server/routes/queries/get_values_descending.ts rename to x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_values_descending.ts index bad62b3056ace..9e6fd92178e82 100644 --- a/x-pack/plugins/aiops/server/routes/queries/get_values_descending.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_values_descending.ts @@ -5,7 +5,7 @@ * 2.0. */ -import type { ItemSet } from '../../../common/types'; +import type { ItemSet } from '../../../../common/types'; import { getValueCounts } from './get_value_counts'; diff --git a/x-pack/plugins/aiops/server/routes/queries/transform_significant_term_to_group.test.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/transform_significant_term_to_group.test.ts similarity index 90% rename from x-pack/plugins/aiops/server/routes/queries/transform_significant_term_to_group.test.ts rename to x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/transform_significant_term_to_group.test.ts index e22d6fcec784a..860baa3c800a2 100644 --- a/x-pack/plugins/aiops/server/routes/queries/transform_significant_term_to_group.test.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/transform_significant_term_to_group.test.ts @@ -5,8 +5,8 @@ * 2.0. */ -import { significantTermGroups } from '../../../common/__mocks__/artificial_logs/significant_term_groups'; -import { significantTerms } from '../../../common/__mocks__/artificial_logs/significant_terms'; +import { significantTermGroups } from '../../../../common/__mocks__/artificial_logs/significant_term_groups'; +import { significantTerms } from '../../../../common/__mocks__/artificial_logs/significant_terms'; import { duplicateIdentifier } from './duplicate_identifier'; import { getGroupsWithReaddedDuplicates } from './get_groups_with_readded_duplicates'; diff --git a/x-pack/plugins/aiops/server/routes/queries/transform_significant_term_to_group.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/transform_significant_term_to_group.ts similarity index 95% rename from x-pack/plugins/aiops/server/routes/queries/transform_significant_term_to_group.ts rename to x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/transform_significant_term_to_group.ts index 54ae0839c5c19..9f95a5c0fa2db 100644 --- a/x-pack/plugins/aiops/server/routes/queries/transform_significant_term_to_group.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/transform_significant_term_to_group.ts @@ -8,7 +8,7 @@ import { stringHash } from '@kbn/ml-string-hash'; import type { SignificantTerm, SignificantTermGroup } from '@kbn/ml-agg-utils'; -import type { SignificantTermDuplicateGroup } from '../../../common/types'; +import type { SignificantTermDuplicateGroup } from '../../../../common/types'; export function transformSignificantTermToGroup( significantTerm: SignificantTerm, diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/route_handler_factory.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/route_handler_factory.ts new file mode 100644 index 0000000000000..b522d6d6ee818 --- /dev/null +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/route_handler_factory.ts @@ -0,0 +1,851 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { queue } from 'async'; + +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; + +import type { + CoreStart, + KibanaRequest, + RequestHandlerContext, + RequestHandler, + KibanaResponseFactory, +} from '@kbn/core/server'; +import type { Logger } from '@kbn/logging'; +import { i18n } from '@kbn/i18n'; +import { KBN_FIELD_TYPES } from '@kbn/field-types'; +import { streamFactory } from '@kbn/ml-response-stream/server'; +import type { + SignificantTerm, + SignificantTermGroup, + NumericChartData, + NumericHistogramField, +} from '@kbn/ml-agg-utils'; +import { SIGNIFICANT_TERM_TYPE } from '@kbn/ml-agg-utils'; +import { fetchHistogramsForFields } from '@kbn/ml-agg-utils'; +import { createExecutionContext } from '@kbn/ml-route-utils'; +import type { UsageCounter } from '@kbn/usage-collection-plugin/server'; + +import { RANDOM_SAMPLER_SEED, AIOPS_TELEMETRY_ID } from '../../../common/constants'; +import { + addSignificantTermsAction, + addSignificantTermsGroupAction, + addSignificantTermsGroupHistogramAction, + addSignificantTermsHistogramAction, + addErrorAction, + pingAction, + resetAllAction, + resetErrorsAction, + resetGroupsAction, + updateLoadingStateAction, + AiopsLogRateAnalysisApiAction, + type AiopsLogRateAnalysisSchema, +} from '../../../common/api/log_rate_analysis'; +import { getCategoryQuery } from '../../../common/api/log_categorization/get_category_query'; +import { AIOPS_API_ENDPOINT } from '../../../common/api'; + +import { PLUGIN_ID } from '../../../common'; + +import { isRequestAbortedError } from '../../lib/is_request_aborted_error'; +import type { AiopsLicense } from '../../types'; + +import { fetchSignificantCategories } from './queries/fetch_significant_categories'; +import { fetchSignificantTermPValues } from './queries/fetch_significant_term_p_values'; +import { fetchIndexInfo } from './queries/fetch_index_info'; +import { fetchFrequentItemSets } from './queries/fetch_frequent_item_sets'; +import { fetchTerms2CategoriesCounts } from './queries/fetch_terms_2_categories_counts'; +import { getHistogramQuery } from './queries/get_histogram_query'; +import { getGroupFilter } from './queries/get_group_filter'; +import { getSignificantTermGroups } from './queries/get_significant_term_groups'; +import { trackAIOpsRouteUsage } from '../../lib/track_route_usage'; + +// 10s ping frequency to keep the stream alive. +const PING_FREQUENCY = 10000; + +// Overall progress is a float from 0 to 1. +const LOADED_FIELD_CANDIDATES = 0.2; +const PROGRESS_STEP_P_VALUES = 0.5; +const PROGRESS_STEP_GROUPING = 0.1; +const PROGRESS_STEP_HISTOGRAMS = 0.1; +const PROGRESS_STEP_HISTOGRAMS_GROUPS = 0.1; + +export const routeHandlerFactory: ( + license: AiopsLicense, + logger: Logger, + coreStart: CoreStart, + usageCounter?: UsageCounter +) => RequestHandler = + (license, logger, coreStart, usageCounter) => + async ( + context: RequestHandlerContext, + request: KibanaRequest, + response: KibanaResponseFactory + ) => { + const { headers } = request; + + trackAIOpsRouteUsage( + `POST ${AIOPS_API_ENDPOINT.LOG_RATE_ANALYSIS}`, + headers[AIOPS_TELEMETRY_ID.AIOPS_ANALYSIS_RUN_ORIGIN], + usageCounter + ); + + if (!license.isActivePlatinumLicense) { + return response.forbidden(); + } + + const client = (await context.core).elasticsearch.client.asCurrentUser; + const executionContext = createExecutionContext(coreStart, PLUGIN_ID, request.route.path); + + return await coreStart.executionContext.withContext(executionContext, () => { + let logMessageCounter = 1; + + function logDebugMessage(msg: string) { + logger.debug(`Log Rate Analysis #${logMessageCounter}: ${msg}`); + logMessageCounter++; + } + + logDebugMessage('Starting analysis.'); + + const groupingEnabled = !!request.body.grouping; + const sampleProbability = request.body.sampleProbability ?? 1; + + const controller = new AbortController(); + const abortSignal = controller.signal; + + let isRunning = false; + let loaded = 0; + let shouldStop = false; + request.events.aborted$.subscribe(() => { + logDebugMessage('aborted$ subscription trigger.'); + shouldStop = true; + controller.abort(); + }); + request.events.completed$.subscribe(() => { + logDebugMessage('completed$ subscription trigger.'); + shouldStop = true; + controller.abort(); + }); + + const { + end: streamEnd, + push, + responseWithHeaders, + } = streamFactory( + request.headers, + logger, + request.body.compressResponse, + request.body.flushFix + ); + + function pushPingWithTimeout() { + setTimeout(() => { + if (isRunning) { + logDebugMessage('Ping message.'); + push(pingAction()); + pushPingWithTimeout(); + } + }, PING_FREQUENCY); + } + + function end() { + if (isRunning) { + isRunning = false; + logDebugMessage('Ending analysis.'); + streamEnd(); + } else { + logDebugMessage('end() was called again with isRunning already being false.'); + } + } + + function endWithUpdatedLoadingState() { + push( + updateLoadingStateAction({ + ccsWarning: false, + loaded: 1, + loadingState: i18n.translate('xpack.aiops.logRateAnalysis.loadingState.doneMessage', { + defaultMessage: 'Done.', + }), + }) + ); + + end(); + } + + function pushError(m: string) { + logDebugMessage('Push error.'); + push(addErrorAction(m)); + } + + async function runAnalysis() { + try { + isRunning = true; + + if (!request.body.overrides) { + logDebugMessage('Full Reset.'); + push(resetAllAction()); + } else { + logDebugMessage('Reset Errors.'); + push(resetErrorsAction()); + } + + if (request.body.overrides?.regroupOnly) { + logDebugMessage('Reset Groups.'); + push(resetGroupsAction()); + } + + if (request.body.overrides?.loaded) { + logDebugMessage(`Set 'loaded' override to '${request.body.overrides?.loaded}'.`); + loaded = request.body.overrides?.loaded; + } + + pushPingWithTimeout(); + + // Step 1: Index Info: Field candidates, total doc count, sample probability + + const fieldCandidates: string[] = []; + let fieldCandidatesCount = fieldCandidates.length; + + const textFieldCandidates: string[] = []; + + let totalDocCount = 0; + + if (!request.body.overrides?.remainingFieldCandidates) { + logDebugMessage('Fetch index information.'); + push( + updateLoadingStateAction({ + ccsWarning: false, + loaded, + loadingState: i18n.translate( + 'xpack.aiops.logRateAnalysis.loadingState.loadingIndexInformation', + { + defaultMessage: 'Loading index information.', + } + ), + }) + ); + + try { + const indexInfo = await fetchIndexInfo( + client, + request.body, + ['message', 'error.message'], + abortSignal + ); + + fieldCandidates.push(...indexInfo.fieldCandidates); + fieldCandidatesCount = fieldCandidates.length; + textFieldCandidates.push(...indexInfo.textFieldCandidates); + totalDocCount = indexInfo.totalDocCount; + } catch (e) { + if (!isRequestAbortedError(e)) { + logger.error(`Failed to fetch index information, got: \n${e.toString()}`); + pushError(`Failed to fetch index information.`); + } + end(); + return; + } + + logDebugMessage(`Total document count: ${totalDocCount}`); + logDebugMessage(`Sample probability: ${sampleProbability}`); + + loaded += LOADED_FIELD_CANDIDATES; + + pushPingWithTimeout(); + + push( + updateLoadingStateAction({ + ccsWarning: false, + loaded, + loadingState: i18n.translate( + 'xpack.aiops.logRateAnalysis.loadingState.identifiedFieldCandidates', + { + defaultMessage: + 'Identified {fieldCandidatesCount, plural, one {# field candidate} other {# field candidates}}.', + values: { + fieldCandidatesCount, + }, + } + ), + }) + ); + + if (fieldCandidatesCount === 0) { + endWithUpdatedLoadingState(); + } else if (shouldStop) { + logDebugMessage('shouldStop after fetching field candidates.'); + end(); + return; + } + } + + // Step 2: Significant Categories and Terms + + // This will store the combined count of detected significant log patterns and keywords + let fieldValuePairsCount = 0; + + const significantCategories: SignificantTerm[] = request.body.overrides?.significantTerms + ? request.body.overrides?.significantTerms.filter( + (d) => d.type === SIGNIFICANT_TERM_TYPE.LOG_PATTERN + ) + : []; + + // Get significant categories of text fields + if (textFieldCandidates.length > 0) { + significantCategories.push( + ...(await fetchSignificantCategories( + client, + request.body, + textFieldCandidates, + logger, + sampleProbability, + pushError, + abortSignal + )) + ); + + if (significantCategories.length > 0) { + push(addSignificantTermsAction(significantCategories)); + } + } + + const significantTerms: SignificantTerm[] = request.body.overrides?.significantTerms + ? request.body.overrides?.significantTerms.filter( + (d) => d.type === SIGNIFICANT_TERM_TYPE.KEYWORD + ) + : []; + + const fieldsToSample = new Set(); + + // Don't use more than 10 here otherwise Kibana will emit an error + // regarding a limit of abort signal listeners of more than 10. + const MAX_CONCURRENT_QUERIES = 10; + + let remainingFieldCandidates: string[]; + let loadingStepSizePValues = PROGRESS_STEP_P_VALUES; + + if (request.body.overrides?.remainingFieldCandidates) { + fieldCandidates.push(...request.body.overrides?.remainingFieldCandidates); + remainingFieldCandidates = request.body.overrides?.remainingFieldCandidates; + fieldCandidatesCount = fieldCandidates.length; + loadingStepSizePValues = + LOADED_FIELD_CANDIDATES + + PROGRESS_STEP_P_VALUES - + (request.body.overrides?.loaded ?? PROGRESS_STEP_P_VALUES); + } else { + remainingFieldCandidates = fieldCandidates; + } + + logDebugMessage('Fetch p-values.'); + + const pValuesQueue = queue(async function (fieldCandidate: string) { + loaded += (1 / fieldCandidatesCount) * loadingStepSizePValues; + + let pValues: Awaited>; + + try { + pValues = await fetchSignificantTermPValues( + client, + request.body, + [fieldCandidate], + logger, + sampleProbability, + pushError, + abortSignal + ); + } catch (e) { + if (!isRequestAbortedError(e)) { + logger.error( + `Failed to fetch p-values for '${fieldCandidate}', got: \n${e.toString()}` + ); + pushError(`Failed to fetch p-values for '${fieldCandidate}'.`); + } + return; + } + + remainingFieldCandidates = remainingFieldCandidates.filter((d) => d !== fieldCandidate); + + if (pValues.length > 0) { + pValues.forEach((d) => { + fieldsToSample.add(d.fieldName); + }); + significantTerms.push(...pValues); + + push(addSignificantTermsAction(pValues)); + } + + push( + updateLoadingStateAction({ + ccsWarning: false, + loaded, + loadingState: i18n.translate( + 'xpack.aiops.logRateAnalysis.loadingState.identifiedFieldValuePairs', + { + defaultMessage: + 'Identified {fieldValuePairsCount, plural, one {# significant field/value pair} other {# significant field/value pairs}}.', + values: { + fieldValuePairsCount, + }, + } + ), + remainingFieldCandidates, + }) + ); + }, MAX_CONCURRENT_QUERIES); + + pValuesQueue.push(fieldCandidates, (err) => { + if (err) { + logger.error(`Failed to fetch p-values.', got: \n${err.toString()}`); + pushError(`Failed to fetch p-values.`); + pValuesQueue.kill(); + end(); + } else if (shouldStop) { + logDebugMessage('shouldStop fetching p-values.'); + pValuesQueue.kill(); + end(); + } + }); + await pValuesQueue.drain(); + + fieldValuePairsCount = significantCategories.length + significantTerms.length; + + if (fieldValuePairsCount === 0) { + logDebugMessage('Stopping analysis, did not find significant terms.'); + endWithUpdatedLoadingState(); + return; + } + + const histogramFields: [NumericHistogramField] = [ + { fieldName: request.body.timeFieldName, type: KBN_FIELD_TYPES.DATE }, + ]; + + logDebugMessage('Fetch overall histogram.'); + + let overallTimeSeries: NumericChartData | undefined; + + const overallHistogramQuery = getHistogramQuery(request.body); + + try { + overallTimeSeries = ( + (await fetchHistogramsForFields( + client, + request.body.index, + overallHistogramQuery, + // fields + histogramFields, + // samplerShardSize + -1, + undefined, + abortSignal, + sampleProbability, + RANDOM_SAMPLER_SEED + )) as [NumericChartData] + )[0]; + } catch (e) { + if (!isRequestAbortedError(e)) { + logger.error(`Failed to fetch the overall histogram data, got: \n${e.toString()}`); + pushError(`Failed to fetch overall histogram data.`); + } + // Still continue the analysis even if loading the overall histogram fails. + } + + function pushHistogramDataLoadingState() { + push( + updateLoadingStateAction({ + ccsWarning: false, + loaded, + loadingState: i18n.translate( + 'xpack.aiops.logRateAnalysis.loadingState.loadingHistogramData', + { + defaultMessage: 'Loading histogram data.', + } + ), + }) + ); + } + + if (shouldStop) { + logDebugMessage('shouldStop after fetching overall histogram.'); + end(); + return; + } + + if (groupingEnabled) { + logDebugMessage('Group results.'); + + push( + updateLoadingStateAction({ + ccsWarning: false, + loaded, + loadingState: i18n.translate( + 'xpack.aiops.logRateAnalysis.loadingState.groupingResults', + { + defaultMessage: 'Transforming significant field/value pairs into groups.', + } + ), + groupsMissing: true, + }) + ); + + try { + const { fields, itemSets } = await fetchFrequentItemSets( + client, + request.body.index, + JSON.parse(request.body.searchQuery) as estypes.QueryDslQueryContainer, + significantTerms, + request.body.timeFieldName, + request.body.deviationMin, + request.body.deviationMax, + logger, + sampleProbability, + pushError, + abortSignal + ); + + if (significantCategories.length > 0 && significantTerms.length > 0) { + const { + fields: significantCategoriesFields, + itemSets: significantCategoriesItemSets, + } = await fetchTerms2CategoriesCounts( + client, + request.body, + JSON.parse(request.body.searchQuery) as estypes.QueryDslQueryContainer, + significantTerms, + itemSets, + significantCategories, + request.body.deviationMin, + request.body.deviationMax, + logger, + pushError, + abortSignal + ); + + fields.push(...significantCategoriesFields); + itemSets.push(...significantCategoriesItemSets); + } + + if (shouldStop) { + logDebugMessage('shouldStop after fetching frequent_item_sets.'); + end(); + return; + } + + if (fields.length > 0 && itemSets.length > 0) { + const significantTermGroups = getSignificantTermGroups( + itemSets, + [...significantTerms, ...significantCategories], + fields + ); + + // We'll find out if there's at least one group with at least two items, + // only then will we return the groups to the clients and make the grouping option available. + const maxItems = Math.max(...significantTermGroups.map((g) => g.group.length)); + + if (maxItems > 1) { + push(addSignificantTermsGroupAction(significantTermGroups)); + } + + loaded += PROGRESS_STEP_GROUPING; + + pushHistogramDataLoadingState(); + + if (shouldStop) { + logDebugMessage('shouldStop after grouping.'); + end(); + return; + } + + logDebugMessage(`Fetch ${significantTermGroups.length} group histograms.`); + + const groupHistogramQueue = queue(async function (cpg: SignificantTermGroup) { + if (shouldStop) { + logDebugMessage('shouldStop abort fetching group histograms.'); + groupHistogramQueue.kill(); + end(); + return; + } + + if (overallTimeSeries !== undefined) { + const histogramQuery = getHistogramQuery(request.body, getGroupFilter(cpg)); + + let cpgTimeSeries: NumericChartData; + try { + cpgTimeSeries = ( + (await fetchHistogramsForFields( + client, + request.body.index, + histogramQuery, + // fields + [ + { + fieldName: request.body.timeFieldName, + type: KBN_FIELD_TYPES.DATE, + interval: overallTimeSeries.interval, + min: overallTimeSeries.stats[0], + max: overallTimeSeries.stats[1], + }, + ], + // samplerShardSize + -1, + undefined, + abortSignal, + sampleProbability, + RANDOM_SAMPLER_SEED + )) as [NumericChartData] + )[0]; + } catch (e) { + if (!isRequestAbortedError(e)) { + logger.error( + `Failed to fetch the histogram data for group #${ + cpg.id + }, got: \n${e.toString()}` + ); + pushError(`Failed to fetch the histogram data for group #${cpg.id}.`); + } + return; + } + const histogram = + overallTimeSeries.data.map((o) => { + const current = cpgTimeSeries.data.find( + (d1) => d1.key_as_string === o.key_as_string + ) ?? { + doc_count: 0, + }; + return { + key: o.key, + key_as_string: o.key_as_string ?? '', + doc_count_significant_term: current.doc_count, + doc_count_overall: Math.max(0, o.doc_count - current.doc_count), + }; + }) ?? []; + + push( + addSignificantTermsGroupHistogramAction([ + { + id: cpg.id, + histogram, + }, + ]) + ); + } + }, MAX_CONCURRENT_QUERIES); + + groupHistogramQueue.push(significantTermGroups); + await groupHistogramQueue.drain(); + } + } catch (e) { + if (!isRequestAbortedError(e)) { + logger.error( + `Failed to transform field/value pairs into groups, got: \n${e.toString()}` + ); + pushError(`Failed to transform field/value pairs into groups.`); + } + } + } + + loaded += PROGRESS_STEP_HISTOGRAMS_GROUPS; + + logDebugMessage(`Fetch ${significantTerms.length} field/value histograms.`); + + // time series filtered by fields + if ( + significantTerms.length > 0 && + overallTimeSeries !== undefined && + !request.body.overrides?.regroupOnly + ) { + const fieldValueHistogramQueue = queue(async function (cp: SignificantTerm) { + if (shouldStop) { + logDebugMessage('shouldStop abort fetching field/value histograms.'); + fieldValueHistogramQueue.kill(); + end(); + return; + } + + if (overallTimeSeries !== undefined) { + const histogramQuery = getHistogramQuery(request.body, [ + { + term: { [cp.fieldName]: cp.fieldValue }, + }, + ]); + + let cpTimeSeries: NumericChartData; + + try { + cpTimeSeries = ( + (await fetchHistogramsForFields( + client, + request.body.index, + histogramQuery, + // fields + [ + { + fieldName: request.body.timeFieldName, + type: KBN_FIELD_TYPES.DATE, + interval: overallTimeSeries.interval, + min: overallTimeSeries.stats[0], + max: overallTimeSeries.stats[1], + }, + ], + // samplerShardSize + -1, + undefined, + abortSignal, + sampleProbability, + RANDOM_SAMPLER_SEED + )) as [NumericChartData] + )[0]; + } catch (e) { + logger.error( + `Failed to fetch the histogram data for field/value pair "${cp.fieldName}:${ + cp.fieldValue + }", got: \n${e.toString()}` + ); + pushError( + `Failed to fetch the histogram data for field/value pair "${cp.fieldName}:${cp.fieldValue}".` + ); + return; + } + + const histogram = + overallTimeSeries.data.map((o) => { + const current = cpTimeSeries.data.find( + (d1) => d1.key_as_string === o.key_as_string + ) ?? { + doc_count: 0, + }; + return { + key: o.key, + key_as_string: o.key_as_string ?? '', + doc_count_significant_term: current.doc_count, + doc_count_overall: Math.max(0, o.doc_count - current.doc_count), + }; + }) ?? []; + + const { fieldName, fieldValue } = cp; + + loaded += (1 / fieldValuePairsCount) * PROGRESS_STEP_HISTOGRAMS; + pushHistogramDataLoadingState(); + push( + addSignificantTermsHistogramAction([ + { + fieldName, + fieldValue, + histogram, + }, + ]) + ); + } + }, MAX_CONCURRENT_QUERIES); + + fieldValueHistogramQueue.push(significantTerms); + await fieldValueHistogramQueue.drain(); + } + + // histograms for text field patterns + if ( + overallTimeSeries !== undefined && + significantCategories.length > 0 && + !request.body.overrides?.regroupOnly + ) { + const significantCategoriesHistogramQueries = significantCategories.map((d) => { + const histogramQuery = getHistogramQuery(request.body); + const categoryQuery = getCategoryQuery(d.fieldName, [ + { key: `${d.key}`, count: d.doc_count, examples: [] }, + ]); + if (Array.isArray(histogramQuery.bool?.filter)) { + histogramQuery.bool?.filter?.push(categoryQuery); + } + return histogramQuery; + }); + + for (const [i, histogramQuery] of significantCategoriesHistogramQueries.entries()) { + const cp = significantCategories[i]; + let catTimeSeries: NumericChartData; + + try { + catTimeSeries = ( + (await fetchHistogramsForFields( + client, + request.body.index, + histogramQuery, + // fields + [ + { + fieldName: request.body.timeFieldName, + type: KBN_FIELD_TYPES.DATE, + interval: overallTimeSeries.interval, + min: overallTimeSeries.stats[0], + max: overallTimeSeries.stats[1], + }, + ], + // samplerShardSize + -1, + undefined, + abortSignal, + sampleProbability, + RANDOM_SAMPLER_SEED + )) as [NumericChartData] + )[0]; + } catch (e) { + logger.error( + `Failed to fetch the histogram data for field/value pair "${cp.fieldName}:${ + cp.fieldValue + }", got: \n${e.toString()}` + ); + pushError( + `Failed to fetch the histogram data for field/value pair "${cp.fieldName}:${cp.fieldValue}".` + ); + return; + } + + const histogram = + overallTimeSeries.data.map((o) => { + const current = catTimeSeries.data.find( + (d1) => d1.key_as_string === o.key_as_string + ) ?? { + doc_count: 0, + }; + return { + key: o.key, + key_as_string: o.key_as_string ?? '', + doc_count_significant_term: current.doc_count, + doc_count_overall: Math.max(0, o.doc_count - current.doc_count), + }; + }) ?? []; + + const { fieldName, fieldValue } = cp; + + loaded += (1 / fieldValuePairsCount) * PROGRESS_STEP_HISTOGRAMS; + pushHistogramDataLoadingState(); + push( + addSignificantTermsHistogramAction([ + { + fieldName, + fieldValue, + histogram, + }, + ]) + ); + } + } + + endWithUpdatedLoadingState(); + } catch (e) { + if (!isRequestAbortedError(e)) { + logger.error(`Log Rate Analysis failed to finish, got: \n${e.toString()}`); + pushError(`Log Rate Analysis failed to finish.`); + } + end(); + } + } + + // Do not call this using `await` so it will run asynchronously while we return the stream already. + runAnalysis(); + + return response.ok(responseWithHeaders); + }); + }; diff --git a/x-pack/plugins/infra/public/components/asset_details/components/expandable_content.tsx b/x-pack/plugins/infra/public/components/asset_details/components/expandable_content.tsx index 0cd5e1a53013a..b5ef833c13c66 100644 --- a/x-pack/plugins/infra/public/components/asset_details/components/expandable_content.tsx +++ b/x-pack/plugins/infra/public/components/asset_details/components/expandable_content.tsx @@ -11,9 +11,10 @@ import { i18n } from '@kbn/i18n'; import React from 'react'; import { FormattedMessage } from '@kbn/i18n-react'; import useToggle from 'react-use/lib/useToggle'; +import type { Field } from '../tabs/metadata/utils'; interface ExpandableContentProps { - values: string | string[] | undefined; + values?: Field['value']; } export const ExpandableContent = (props: ExpandableContentProps) => { const { values } = props; diff --git a/x-pack/plugins/infra/public/components/asset_details/tabs/metadata/add_metadata_filter_button.tsx b/x-pack/plugins/infra/public/components/asset_details/tabs/metadata/add_metadata_filter_button.tsx index 49d912fab3f50..6e92714ad28b0 100644 --- a/x-pack/plugins/infra/public/components/asset_details/tabs/metadata/add_metadata_filter_button.tsx +++ b/x-pack/plugins/infra/public/components/asset_details/tabs/metadata/add_metadata_filter_button.tsx @@ -12,12 +12,10 @@ import { useMetricsDataViewContext } from '../../../../pages/metrics/hosts/hooks import { useKibanaContextForPlugin } from '../../../../hooks/use_kibana'; import { buildMetadataFilter } from './build_metadata_filter'; import { useUnifiedSearchContext } from '../../../../pages/metrics/hosts/hooks/use_unified_search'; +import type { Field } from './utils'; interface AddMetadataFilterButtonProps { - item: { - name: string; - value: string | string[] | undefined; - }; + item: Field; } const filterAddedToastTitle = i18n.translate('xpack.infra.metadataEmbeddable.filterAdded', { diff --git a/x-pack/plugins/infra/public/components/asset_details/tabs/metadata/utils.test.ts b/x-pack/plugins/infra/public/components/asset_details/tabs/metadata/utils.test.ts new file mode 100644 index 0000000000000..5c78fca3f48db --- /dev/null +++ b/x-pack/plugins/infra/public/components/asset_details/tabs/metadata/utils.test.ts @@ -0,0 +1,372 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { InfraMetadata } from '../../../../../common/http_api'; +import { getAllFields } from './utils'; + +describe('#getAllFields', () => { + const host = { + architecture: 'x86_64', + containerized: false, + hostname: 'host1', + ip: [ + '10.10.10.10', + '10.10.10.10', + '100.100.100.1', + 'fe10::1c10:10ff:fe10:f10b', + 'fe10::1c8b:10ff:fe5b:7faa', + 'fe10::20c2:36ff:feed:f47f', + ], + mac: ['01-33-22-33-71-83', '1E-15-33-68-F8-5B', '1E-8B-55-5B-7F-AA'], + name: 'host1', + os: { + codename: 'focal', + family: 'debian', + kernel: '5.15.109+', + name: 'Ubuntu', + platform: 'ubuntu', + version: '20.04.6 LTS (Focal Fossa)', + }, + }; + const agent = { + ephemeral_id: '2a1f1122-09f8-8d3b-b000-e5dcea22a1b2', + id: 'agent1', + name: 'host1', + type: 'metricbeat', + version: '8.11.0', + }; + + const cloud = { + account: { + id: 'elastic-observability', + }, + availability_zone: 'us-central1-c', + instance: { + id: '1111111111111111111', + name: 'host1', + }, + machine: { + type: 'e2-standard-4', + }, + project: { + id: 'elastic-observability', + }, + provider: 'gcp', + region: 'us-central1', + }; + + it('should return empty array if no metadata info', async () => { + const result: InfraMetadata = { + id: 'host1', + name: 'host1', + features: [ + { + name: 'system.core', + source: 'metrics', + }, + ], + }; + expect(getAllFields(result)).toHaveLength(0); + }); + + it('should return empty array if no field value is provided', async () => { + const result: InfraMetadata = { + id: 'host1', + name: 'host1', + features: [ + { + name: 'system.core', + source: 'metrics', + }, + ], + info: { + host: { + name: undefined, + }, + }, + }; + expect(getAllFields(result)).toHaveLength(0); + }); + + it('should map metadata with nested properties', async () => { + const result: InfraMetadata = { + id: 'host1', + name: 'host1', + features: [ + { + name: 'system.core', + source: 'metrics', + }, + ], + info: { + host: { + os: { + name: 'Ubuntu', + }, + }, + }, + }; + expect(getAllFields(result)).toStrictEqual([{ name: 'host.os.name', value: 'Ubuntu' }]); + }); + + it('should map metadata with partial host, agent, could data', async () => { + const result: InfraMetadata = { + id: 'host1', + name: 'host1', + features: [ + { + name: 'system.core', + source: 'metrics', + }, + ], + info: { + host: { + name: 'host2', + os: { + name: 'Ubuntu', + }, + mac: ['01-33-22-33-71-83', '1E-15-33-68-F8-5B', '1E-8B-55-5B-7F-AA'], + }, + cloud: { + instance: { + id: '1111111111111111111', + name: 'host1', + }, + }, + agent: { + id: 'agent2', + }, + }, + }; + expect(getAllFields(result)).toStrictEqual([ + { + name: 'host.name', + value: 'host2', + }, + { name: 'host.os.name', value: 'Ubuntu' }, + { + name: 'host.mac', + value: ['01-33-22-33-71-83', '1E-15-33-68-F8-5B', '1E-8B-55-5B-7F-AA'], + }, + { + name: 'agent.id', + value: 'agent2', + }, + { + name: 'cloud.instance.id', + value: '1111111111111111111', + }, + { + name: 'cloud.instance.name', + value: 'host1', + }, + ]); + }); + + it('should map metadata with only host data', async () => { + const result: InfraMetadata = { + id: 'host1', + name: 'host1', + features: [ + { + name: 'system.core', + source: 'metrics', + }, + ], + info: { + host, + }, + }; + expect(getAllFields(result)).toStrictEqual([ + { name: 'host.architecture', value: 'x86_64' }, + { name: 'host.containerized', value: 'false' }, + { name: 'host.hostname', value: 'host1' }, + { + name: 'host.ip', + value: [ + '10.10.10.10', + '10.10.10.10', + '100.100.100.1', + 'fe10::1c10:10ff:fe10:f10b', + 'fe10::1c8b:10ff:fe5b:7faa', + 'fe10::20c2:36ff:feed:f47f', + ], + }, + { + name: 'host.mac', + value: ['01-33-22-33-71-83', '1E-15-33-68-F8-5B', '1E-8B-55-5B-7F-AA'], + }, + { name: 'host.name', value: 'host1' }, + { name: 'host.os.codename', value: 'focal' }, + { name: 'host.os.family', value: 'debian' }, + { name: 'host.os.kernel', value: '5.15.109+' }, + { name: 'host.os.name', value: 'Ubuntu' }, + { + name: 'host.os.platform', + value: 'ubuntu', + }, + { + name: 'host.os.version', + value: '20.04.6 LTS (Focal Fossa)', + }, + ]); + }); + + it('should map metadata with host and cloud data', async () => { + const result: InfraMetadata = { + id: 'host1', + name: 'host1', + features: [ + { + name: 'system.core', + source: 'metrics', + }, + ], + info: { + host, + cloud, + }, + }; + + expect(getAllFields(result)).toStrictEqual([ + { name: 'host.architecture', value: 'x86_64' }, + { name: 'host.containerized', value: 'false' }, + { name: 'host.hostname', value: 'host1' }, + { + name: 'host.ip', + value: [ + '10.10.10.10', + '10.10.10.10', + '100.100.100.1', + 'fe10::1c10:10ff:fe10:f10b', + 'fe10::1c8b:10ff:fe5b:7faa', + 'fe10::20c2:36ff:feed:f47f', + ], + }, + { + name: 'host.mac', + value: ['01-33-22-33-71-83', '1E-15-33-68-F8-5B', '1E-8B-55-5B-7F-AA'], + }, + { name: 'host.name', value: 'host1' }, + { name: 'host.os.codename', value: 'focal' }, + { name: 'host.os.family', value: 'debian' }, + { name: 'host.os.kernel', value: '5.15.109+' }, + { name: 'host.os.name', value: 'Ubuntu' }, + { + name: 'host.os.platform', + value: 'ubuntu', + }, + { + name: 'host.os.version', + value: '20.04.6 LTS (Focal Fossa)', + }, + { + name: 'cloud.account.id', + value: 'elastic-observability', + }, + { + name: 'cloud.availability_zone', + value: 'us-central1-c', + }, + { + name: 'cloud.instance.id', + value: '1111111111111111111', + }, + { + name: 'cloud.instance.name', + value: 'host1', + }, + { + name: 'cloud.machine.type', + value: 'e2-standard-4', + }, + { + name: 'cloud.project.id', + value: 'elastic-observability', + }, + { + name: 'cloud.provider', + value: 'gcp', + }, + { + name: 'cloud.region', + value: 'us-central1', + }, + ]); + }); + + it('should map metadata with host and agent data', async () => { + const result: InfraMetadata = { + id: 'host1', + name: 'host1', + features: [ + { + name: 'system.core', + source: 'metrics', + }, + ], + info: { + host, + agent, + }, + }; + + expect(getAllFields(result)).toStrictEqual([ + { name: 'host.architecture', value: 'x86_64' }, + { name: 'host.containerized', value: 'false' }, + { name: 'host.hostname', value: 'host1' }, + { + name: 'host.ip', + value: [ + '10.10.10.10', + '10.10.10.10', + '100.100.100.1', + 'fe10::1c10:10ff:fe10:f10b', + 'fe10::1c8b:10ff:fe5b:7faa', + 'fe10::20c2:36ff:feed:f47f', + ], + }, + { + name: 'host.mac', + value: ['01-33-22-33-71-83', '1E-15-33-68-F8-5B', '1E-8B-55-5B-7F-AA'], + }, + { name: 'host.name', value: 'host1' }, + { name: 'host.os.codename', value: 'focal' }, + { name: 'host.os.family', value: 'debian' }, + { name: 'host.os.kernel', value: '5.15.109+' }, + { name: 'host.os.name', value: 'Ubuntu' }, + { + name: 'host.os.platform', + value: 'ubuntu', + }, + { + name: 'host.os.version', + value: '20.04.6 LTS (Focal Fossa)', + }, + { + name: 'agent.ephemeral_id', + value: '2a1f1122-09f8-8d3b-b000-e5dcea22a1b2', + }, + { + name: 'agent.id', + value: 'agent1', + }, + { + name: 'agent.name', + value: 'host1', + }, + { + name: 'agent.type', + value: 'metricbeat', + }, + { + name: 'agent.version', + value: '8.11.0', + }, + ]); + }); +}); diff --git a/x-pack/plugins/infra/public/components/asset_details/tabs/metadata/utils.ts b/x-pack/plugins/infra/public/components/asset_details/tabs/metadata/utils.ts index e52242c74d4f6..e41a09b7534e6 100644 --- a/x-pack/plugins/infra/public/components/asset_details/tabs/metadata/utils.ts +++ b/x-pack/plugins/infra/public/components/asset_details/tabs/metadata/utils.ts @@ -9,108 +9,56 @@ import type { InfraMetadata } from '../../../../../common/http_api'; export interface Field { name: string; - value: string | string[] | undefined; + value?: string | string[]; +} +interface FieldsByCategory { + [key: string]: string | boolean | string[] | { [key: string]: string }; } export const getAllFields = (metadata: InfraMetadata | null) => { if (!metadata?.info) return []; - return prune([ - { - name: 'host.architecture', - value: metadata.info.host?.architecture, - }, - { - name: 'host.hostname', - value: metadata.info.host?.name, - }, - { - name: 'host.id', - value: metadata.info.host?.id, - }, - { - name: 'host.ip', - value: metadata.info.host?.ip, - }, - { - name: 'host.mac', - value: metadata.info.host?.mac, - }, - { - name: 'host.name', - value: metadata.info.host?.name, - }, - { - name: 'host.os.build', - value: metadata.info.host?.os?.build, - }, - { - name: 'host.os.family', - value: metadata.info.host?.os?.family, - }, - { - name: 'host.os.name', - value: metadata.info.host?.os?.name, - }, - { - name: 'host.os.kernel', - value: metadata.info.host?.os?.kernel, - }, - { - name: 'host.os.platform', - value: metadata.info.host?.os?.platform, - }, - { - name: 'host.os.version', - value: metadata.info.host?.os?.version, - }, - { - name: 'cloud.account.id', - value: metadata.info.cloud?.account?.id, - }, - { - name: 'cloud.account.name', - value: metadata.info.cloud?.account?.name, - }, - { - name: 'cloud.availability_zone', - value: metadata.info.cloud?.availability_zone, - }, - { - name: 'cloud.instance.id', - value: metadata.info.cloud?.instance?.id, - }, - { - name: 'cloud.instance.name', - value: metadata.info.cloud?.instance?.name, - }, - { - name: 'cloud.machine.type', - value: metadata.info.cloud?.machine?.type, - }, - { - name: 'cloud.provider', - value: metadata.info.cloud?.provider, - }, - { - name: 'cloud.region', - value: metadata.info.cloud?.region, - }, - { - name: 'agent.id', - value: metadata.info.agent?.id, - }, - { - name: 'agent.version', - value: metadata.info.agent?.version, - }, - { - name: 'agent.policy', - value: metadata.info.agent?.policy, - }, - ]); + + const mapNestedProperties = (category: 'cloud' | 'host' | 'agent', property: string) => { + const fieldsByCategory: FieldsByCategory = metadata?.info?.[`${category}`] ?? {}; + if (fieldsByCategory.hasOwnProperty(property)) { + const value = fieldsByCategory[property]; + + if (typeof value === 'boolean') { + return { + name: `${category}.${property}`, + value: String(value), + }; + } + + if (typeof value === 'string' || Array.isArray(value)) { + return { + name: `${category}.${property}`, + value, + }; + } else { + return Object.entries(value ?? {}).map(([prop, subProp]) => ({ + name: `${category}.${property}.${prop}`, + value: subProp, + })); + } + } + return []; + }; + + const agent = Object.keys(metadata.info.agent ?? {}).flatMap((prop) => + mapNestedProperties('agent', prop) + ); + const cloud = Object.keys(metadata.info.cloud ?? {}).flatMap((prop) => + mapNestedProperties('cloud', prop) + ); + const host = Object.keys(metadata?.info?.host ?? {}).flatMap((prop) => + mapNestedProperties('host', prop) + ); + + return prune([...host, ...agent, ...cloud]); }; -const prune = (fields: Field[]) => fields.filter((f) => !!f.value); +const prune = (fields: Field[]) => fields.filter((f) => !!f?.value); export const getRowsWithPins = (rows: Field[], pinnedItems: Array) => { if (pinnedItems.length > 0) { diff --git a/x-pack/plugins/licensing/common/license_update.ts b/x-pack/plugins/licensing/common/license_update.ts index b344d8ce2d16a..a35b7aa6e6785 100644 --- a/x-pack/plugins/licensing/common/license_update.ts +++ b/x-pack/plugins/licensing/common/license_update.ts @@ -17,6 +17,7 @@ import { takeUntil, finalize, startWith, + throttleTime, } from 'rxjs/operators'; import { hasLicenseInfoChanged } from './has_license_info_changed'; import type { ILicense } from './types'; @@ -29,11 +30,15 @@ export function createLicenseUpdate( ) { const manuallyRefresh$ = new Subject(); - const fetched$ = merge(triggerRefresh$, manuallyRefresh$).pipe( - takeUntil(stop$), - exhaustMap(fetcher), - share() - ); + const fetched$ = merge( + triggerRefresh$, + manuallyRefresh$.pipe( + throttleTime(1000, undefined, { + leading: true, + trailing: true, + }) + ) + ).pipe(takeUntil(stop$), exhaustMap(fetcher), share()); // provide a first, empty license, so that we can compare in the filter below const startWithArgs = initialValues ? [undefined, initialValues] : [undefined]; diff --git a/x-pack/plugins/licensing/server/license_fetcher.test.ts b/x-pack/plugins/licensing/server/license_fetcher.test.ts new file mode 100644 index 0000000000000..efd9b001fa0ff --- /dev/null +++ b/x-pack/plugins/licensing/server/license_fetcher.test.ts @@ -0,0 +1,172 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { getLicenseFetcher } from './license_fetcher'; +import { loggerMock, type MockedLogger } from '@kbn/logging-mocks'; +import { elasticsearchServiceMock } from '@kbn/core/server/mocks'; + +type EsLicense = estypes.XpackInfoMinimalLicenseInformation; + +const delay = (ms: number) => new Promise((res) => setTimeout(res, ms)); + +function buildRawLicense(options: Partial = {}): EsLicense { + return { + uid: 'uid-000000001234', + status: 'active', + type: 'basic', + mode: 'basic', + expiry_date_in_millis: 1000, + ...options, + }; +} + +describe('LicenseFetcher', () => { + let logger: MockedLogger; + let clusterClient: ReturnType; + + beforeEach(() => { + logger = loggerMock.create(); + clusterClient = elasticsearchServiceMock.createClusterClient(); + }); + + it('returns the license for successful calls', async () => { + clusterClient.asInternalUser.xpack.info.mockResponse({ + license: buildRawLicense({ + uid: 'license-1', + }), + features: {}, + } as any); + + const fetcher = getLicenseFetcher({ + logger, + clusterClient, + cacheDurationMs: 50_000, + }); + + const license = await fetcher(); + expect(license.uid).toEqual('license-1'); + }); + + it('returns the latest license for successful calls', async () => { + clusterClient.asInternalUser.xpack.info + .mockResponseOnce({ + license: buildRawLicense({ + uid: 'license-1', + }), + features: {}, + } as any) + .mockResponseOnce({ + license: buildRawLicense({ + uid: 'license-2', + }), + features: {}, + } as any); + + const fetcher = getLicenseFetcher({ + logger, + clusterClient, + cacheDurationMs: 50_000, + }); + + let license = await fetcher(); + expect(license.uid).toEqual('license-1'); + + license = await fetcher(); + expect(license.uid).toEqual('license-2'); + }); + + it('returns an error license in case of error', async () => { + clusterClient.asInternalUser.xpack.info.mockResponseImplementation(() => { + throw new Error('woups'); + }); + + const fetcher = getLicenseFetcher({ + logger, + clusterClient, + cacheDurationMs: 50_000, + }); + + const license = await fetcher(); + expect(license.error).toEqual('woups'); + }); + + it('returns a license successfully fetched after an error', async () => { + clusterClient.asInternalUser.xpack.info + .mockResponseImplementationOnce(() => { + throw new Error('woups'); + }) + .mockResponseOnce({ + license: buildRawLicense({ + uid: 'license-1', + }), + features: {}, + } as any); + + const fetcher = getLicenseFetcher({ + logger, + clusterClient, + cacheDurationMs: 50_000, + }); + + let license = await fetcher(); + expect(license.error).toEqual('woups'); + license = await fetcher(); + expect(license.uid).toEqual('license-1'); + }); + + it('returns the latest fetched license after an error within the cache duration period', async () => { + clusterClient.asInternalUser.xpack.info + .mockResponseOnce({ + license: buildRawLicense({ + uid: 'license-1', + }), + features: {}, + } as any) + .mockResponseImplementationOnce(() => { + throw new Error('woups'); + }); + + const fetcher = getLicenseFetcher({ + logger, + clusterClient, + cacheDurationMs: 50_000, + }); + + let license = await fetcher(); + expect(license.uid).toEqual('license-1'); + license = await fetcher(); + expect(license.uid).toEqual('license-1'); + }); + + it('returns an error license after an error exceeding the cache duration period', async () => { + clusterClient.asInternalUser.xpack.info + .mockResponseOnce({ + license: buildRawLicense({ + uid: 'license-1', + }), + features: {}, + } as any) + .mockResponseImplementationOnce(() => { + throw new Error('woups'); + }); + + const fetcher = getLicenseFetcher({ + logger, + clusterClient, + cacheDurationMs: 1, + }); + + let license = await fetcher(); + expect(license.uid).toEqual('license-1'); + + await delay(50); + + license = await fetcher(); + expect(license.error).toEqual('woups'); + }); +}); diff --git a/x-pack/plugins/licensing/server/license_fetcher.ts b/x-pack/plugins/licensing/server/license_fetcher.ts new file mode 100644 index 0000000000000..43d9c204bbf66 --- /dev/null +++ b/x-pack/plugins/licensing/server/license_fetcher.ts @@ -0,0 +1,133 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { createHash } from 'crypto'; +import stringify from 'json-stable-stringify'; +import type { MaybePromise } from '@kbn/utility-types'; +import { isPromise } from '@kbn/std'; +import type { IClusterClient, Logger } from '@kbn/core/server'; +import type { + ILicense, + PublicLicense, + PublicFeatures, + LicenseType, + LicenseStatus, +} from '../common/types'; +import { License } from '../common/license'; +import type { ElasticsearchError, LicenseFetcher } from './types'; + +export const getLicenseFetcher = ({ + clusterClient, + logger, + cacheDurationMs, +}: { + clusterClient: MaybePromise; + logger: Logger; + cacheDurationMs: number; +}): LicenseFetcher => { + let currentLicense: ILicense | undefined; + let lastSuccessfulFetchTime: number | undefined; + + return async () => { + const client = isPromise(clusterClient) ? await clusterClient : clusterClient; + try { + const response = await client.asInternalUser.xpack.info(); + const normalizedLicense = + response.license && response.license.type !== 'missing' + ? normalizeServerLicense(response.license) + : undefined; + const normalizedFeatures = response.features + ? normalizeFeatures(response.features) + : undefined; + + const signature = sign({ + license: normalizedLicense, + features: normalizedFeatures, + error: '', + }); + + currentLicense = new License({ + license: normalizedLicense, + features: normalizedFeatures, + signature, + }); + lastSuccessfulFetchTime = Date.now(); + + return currentLicense; + } catch (error) { + logger.warn( + `License information could not be obtained from Elasticsearch due to ${error} error` + ); + + if (lastSuccessfulFetchTime && lastSuccessfulFetchTime + cacheDurationMs > Date.now()) { + return currentLicense!; + } else { + const errorMessage = getErrorMessage(error); + const signature = sign({ error: errorMessage }); + + return new License({ + error: getErrorMessage(error), + signature, + }); + } + } + }; +}; + +function normalizeServerLicense( + license: estypes.XpackInfoMinimalLicenseInformation +): PublicLicense { + return { + uid: license.uid, + type: license.type as LicenseType, + mode: license.mode as LicenseType, + expiryDateInMillis: + typeof license.expiry_date_in_millis === 'string' + ? parseInt(license.expiry_date_in_millis, 10) + : license.expiry_date_in_millis, + status: license.status as LicenseStatus, + }; +} + +function normalizeFeatures(rawFeatures: estypes.XpackInfoFeatures) { + const features: PublicFeatures = {}; + for (const [name, feature] of Object.entries(rawFeatures)) { + features[name] = { + isAvailable: feature.available, + isEnabled: feature.enabled, + }; + } + return features; +} + +function sign({ + license, + features, + error, +}: { + license?: PublicLicense; + features?: PublicFeatures; + error?: string; +}) { + return createHash('sha256') + .update( + stringify({ + license, + features, + error, + }) + ) + .digest('hex'); +} + +function getErrorMessage(error: ElasticsearchError): string { + if (error.status === 400) { + return 'X-Pack plugin is not installed on the Elasticsearch cluster.'; + } + return error.message; +} diff --git a/x-pack/plugins/licensing/server/licensing_config.ts b/x-pack/plugins/licensing/server/licensing_config.ts index 459c69b650dbb..66899602e04cb 100644 --- a/x-pack/plugins/licensing/server/licensing_config.ts +++ b/x-pack/plugins/licensing/server/licensing_config.ts @@ -10,12 +10,18 @@ import { PluginConfigDescriptor } from '@kbn/core/server'; const configSchema = schema.object({ api_polling_frequency: schema.duration({ defaultValue: '30s' }), + license_cache_duration: schema.duration({ + defaultValue: '300s', + validate: (value) => { + if (value.asMinutes() > 15) { + return 'license cache duration must be shorter than 15 minutes'; + } + }, + }), }); export type LicenseConfigType = TypeOf; export const config: PluginConfigDescriptor = { - schema: schema.object({ - api_polling_frequency: schema.duration({ defaultValue: '30s' }), - }), + schema: configSchema, }; diff --git a/x-pack/plugins/licensing/server/plugin.test.ts b/x-pack/plugins/licensing/server/plugin.test.ts index b087b6f3f03fa..129dc6aee66da 100644 --- a/x-pack/plugins/licensing/server/plugin.test.ts +++ b/x-pack/plugins/licensing/server/plugin.test.ts @@ -56,22 +56,23 @@ describe('licensing plugin', () => { return client; }; - describe('#start', () => { - describe('#license$', () => { - let plugin: LicensingPlugin; - let pluginInitContextMock: ReturnType; + let plugin: LicensingPlugin; + let pluginInitContextMock: ReturnType; - beforeEach(() => { - pluginInitContextMock = coreMock.createPluginInitializerContext({ - api_polling_frequency: moment.duration(100), - }); - plugin = new LicensingPlugin(pluginInitContextMock); - }); + beforeEach(() => { + pluginInitContextMock = coreMock.createPluginInitializerContext({ + api_polling_frequency: moment.duration(100), + license_cache_duration: moment.duration(1000), + }); + plugin = new LicensingPlugin(pluginInitContextMock); + }); - afterEach(async () => { - await plugin.stop(); - }); + afterEach(async () => { + await plugin?.stop(); + }); + describe('#start', () => { + describe('#license$', () => { it('returns license', async () => { const esClient = createEsClient({ license: buildRawLicense(), @@ -79,8 +80,8 @@ describe('licensing plugin', () => { }); const coreSetup = createCoreSetupWith(esClient); - await plugin.setup(coreSetup); - const { license$ } = await plugin.start(); + plugin.setup(coreSetup); + const { license$ } = plugin.start(); const license = await firstValueFrom(license$); expect(license.isAvailable).toBe(true); }); @@ -92,8 +93,8 @@ describe('licensing plugin', () => { }); const coreSetup = createCoreSetupWith(esClient); - await plugin.setup(coreSetup); - const { license$ } = await plugin.start(); + plugin.setup(coreSetup); + const { license$ } = plugin.start(); await firstValueFrom(license$); expect(esClient.asInternalUser.xpack.info).toHaveBeenCalledTimes(1); @@ -111,8 +112,8 @@ describe('licensing plugin', () => { }); const coreSetup = createCoreSetupWith(esClient); - await plugin.setup(coreSetup); - const { license$ } = await plugin.start(); + plugin.setup(coreSetup); + const { license$ } = plugin.start(); const [first, second, third] = await firstValueFrom(license$.pipe(take(3), toArray())); expect(first.type).toBe('basic'); @@ -125,8 +126,8 @@ describe('licensing plugin', () => { esClient.asInternalUser.xpack.info.mockRejectedValue(new Error('test')); const coreSetup = createCoreSetupWith(esClient); - await plugin.setup(coreSetup); - const { license$ } = await plugin.start(); + plugin.setup(coreSetup); + const { license$ } = plugin.start(); const license = await firstValueFrom(license$); expect(license.isAvailable).toBe(false); @@ -140,8 +141,8 @@ describe('licensing plugin', () => { esClient.asInternalUser.xpack.info.mockRejectedValue(error); const coreSetup = createCoreSetupWith(esClient); - await plugin.setup(coreSetup); - const { license$ } = await plugin.start(); + plugin.setup(coreSetup); + const { license$ } = plugin.start(); const license = await firstValueFrom(license$); expect(license.isAvailable).toBe(false); @@ -169,8 +170,8 @@ describe('licensing plugin', () => { }); const coreSetup = createCoreSetupWith(esClient); - await plugin.setup(coreSetup); - const { license$ } = await plugin.start(); + plugin.setup(coreSetup); + const { license$ } = plugin.start(); const [first, second, third] = await firstValueFrom(license$.pipe(take(3), toArray())); @@ -186,8 +187,8 @@ describe('licensing plugin', () => { }); const coreSetup = createCoreSetupWith(esClient); - await plugin.setup(coreSetup); - await plugin.start(); + plugin.setup(coreSetup); + plugin.start(); await flushPromises(); @@ -201,8 +202,8 @@ describe('licensing plugin', () => { }); const coreSetup = createCoreSetupWith(esClient); - await plugin.setup(coreSetup); - await plugin.start(); + plugin.setup(coreSetup); + plugin.start(); await flushPromises(); @@ -229,8 +230,8 @@ describe('licensing plugin', () => { }); const coreSetup = createCoreSetupWith(esClient); - await plugin.setup(coreSetup); - const { license$ } = await plugin.start(); + plugin.setup(coreSetup); + const { license$ } = plugin.start(); const [first, second, third] = await firstValueFrom(license$.pipe(take(3), toArray())); expect(first.signature === third.signature).toBe(true); @@ -239,16 +240,12 @@ describe('licensing plugin', () => { }); describe('#refresh', () => { - let plugin: LicensingPlugin; - afterEach(async () => { - await plugin.stop(); - }); - it('forces refresh immediately', async () => { plugin = new LicensingPlugin( coreMock.createPluginInitializerContext({ // disable polling mechanism api_polling_frequency: moment.duration(50000), + license_cache_duration: moment.duration(1000), }) ); const esClient = createEsClient({ @@ -257,31 +254,26 @@ describe('licensing plugin', () => { }); const coreSetup = createCoreSetupWith(esClient); - await plugin.setup(coreSetup); - const { refresh, license$ } = await plugin.start(); + plugin.setup(coreSetup); + const { refresh, license$ } = plugin.start(); expect(esClient.asInternalUser.xpack.info).toHaveBeenCalledTimes(0); - await license$.pipe(take(1)).toPromise(); + await firstValueFrom(license$); expect(esClient.asInternalUser.xpack.info).toHaveBeenCalledTimes(1); - refresh(); + await refresh(); await flushPromises(); expect(esClient.asInternalUser.xpack.info).toHaveBeenCalledTimes(2); }); }); describe('#createLicensePoller', () => { - let plugin: LicensingPlugin; - - afterEach(async () => { - await plugin.stop(); - }); - it(`creates a poller fetching license from passed 'clusterClient' every 'api_polling_frequency' ms`, async () => { plugin = new LicensingPlugin( coreMock.createPluginInitializerContext({ api_polling_frequency: moment.duration(50000), + license_cache_duration: moment.duration(1000), }) ); @@ -290,8 +282,8 @@ describe('licensing plugin', () => { features: {}, }); const coreSetup = createCoreSetupWith(esClient); - await plugin.setup(coreSetup); - const { createLicensePoller, license$ } = await plugin.start(); + plugin.setup(coreSetup); + const { createLicensePoller, license$ } = plugin.start(); const customClient = createEsClient({ license: buildRawLicense({ type: 'gold' }), @@ -313,19 +305,13 @@ describe('licensing plugin', () => { expect(customLicense.isAvailable).toBe(true); expect(customLicense.type).toBe('gold'); - expect(await license$.pipe(take(1)).toPromise()).not.toBe(customLicense); + expect(await firstValueFrom(license$)).not.toBe(customLicense); }); it('creates a poller with a manual refresh control', async () => { - plugin = new LicensingPlugin( - coreMock.createPluginInitializerContext({ - api_polling_frequency: moment.duration(100), - }) - ); - const coreSetup = coreMock.createSetup(); - await plugin.setup(coreSetup); - const { createLicensePoller } = await plugin.start(); + plugin.setup(coreSetup); + const { createLicensePoller } = plugin.start(); const customClient = createEsClient({ license: buildRawLicense({ type: 'gold' }), @@ -344,24 +330,10 @@ describe('licensing plugin', () => { }); describe('extends core contexts', () => { - let plugin: LicensingPlugin; - - beforeEach(() => { - plugin = new LicensingPlugin( - coreMock.createPluginInitializerContext({ - api_polling_frequency: moment.duration(100), - }) - ); - }); - - afterEach(async () => { - await plugin.stop(); - }); - it('provides a licensing context to http routes', async () => { const coreSetup = coreMock.createSetup(); - await plugin.setup(coreSetup); + plugin.setup(coreSetup); expect(coreSetup.http.registerRouteHandlerContext.mock.calls).toMatchInlineSnapshot(` Array [ @@ -375,22 +347,10 @@ describe('licensing plugin', () => { }); describe('registers on pre-response interceptor', () => { - let plugin: LicensingPlugin; - - beforeEach(() => { - plugin = new LicensingPlugin( - coreMock.createPluginInitializerContext({ api_polling_frequency: moment.duration(100) }) - ); - }); - - afterEach(async () => { - await plugin.stop(); - }); - it('once', async () => { const coreSetup = coreMock.createSetup(); - await plugin.setup(coreSetup); + plugin.setup(coreSetup); expect(coreSetup.http.registerOnPreResponse).toHaveBeenCalledTimes(1); }); @@ -399,14 +359,9 @@ describe('licensing plugin', () => { describe('#stop', () => { it('stops polling', async () => { - const plugin = new LicensingPlugin( - coreMock.createPluginInitializerContext({ - api_polling_frequency: moment.duration(100), - }) - ); const coreSetup = coreMock.createSetup(); - await plugin.setup(coreSetup); - const { license$ } = await plugin.start(); + plugin.setup(coreSetup); + const { license$ } = plugin.start(); let completed = false; license$.subscribe({ complete: () => (completed = true) }); diff --git a/x-pack/plugins/licensing/server/plugin.ts b/x-pack/plugins/licensing/server/plugin.ts index 0d21cd689bf46..b3ac583e7c81e 100644 --- a/x-pack/plugins/licensing/server/plugin.ts +++ b/x-pack/plugins/licensing/server/plugin.ts @@ -8,12 +8,7 @@ import type { Observable, Subject, Subscription } from 'rxjs'; import { ReplaySubject, timer } from 'rxjs'; import moment from 'moment'; -import { createHash } from 'crypto'; -import stringify from 'json-stable-stringify'; - -import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { MaybePromise } from '@kbn/utility-types'; -import { isPromise } from '@kbn/std'; import type { CoreSetup, Logger, @@ -21,73 +16,17 @@ import type { PluginInitializerContext, IClusterClient, } from '@kbn/core/server'; - import { registerAnalyticsContextProvider } from '../common/register_analytics_context_provider'; -import type { - ILicense, - PublicLicense, - PublicFeatures, - LicenseType, - LicenseStatus, -} from '../common/types'; +import type { ILicense } from '../common/types'; import type { LicensingPluginSetup, LicensingPluginStart } from './types'; -import { License } from '../common/license'; import { createLicenseUpdate } from '../common/license_update'; - -import type { ElasticsearchError } from './types'; import { registerRoutes } from './routes'; import { FeatureUsageService } from './services'; - import type { LicenseConfigType } from './licensing_config'; import { createRouteHandlerContext } from './licensing_route_handler_context'; import { createOnPreResponseHandler } from './on_pre_response_handler'; import { getPluginStatus$ } from './plugin_status'; - -function normalizeServerLicense( - license: estypes.XpackInfoMinimalLicenseInformation -): PublicLicense { - return { - uid: license.uid, - type: license.type as LicenseType, - mode: license.mode as LicenseType, - expiryDateInMillis: - typeof license.expiry_date_in_millis === 'string' - ? parseInt(license.expiry_date_in_millis, 10) - : license.expiry_date_in_millis, - status: license.status as LicenseStatus, - }; -} - -function normalizeFeatures(rawFeatures: estypes.XpackInfoFeatures) { - const features: PublicFeatures = {}; - for (const [name, feature] of Object.entries(rawFeatures)) { - features[name] = { - isAvailable: feature.available, - isEnabled: feature.enabled, - }; - } - return features; -} - -function sign({ - license, - features, - error, -}: { - license?: PublicLicense; - features?: PublicFeatures; - error?: string; -}) { - return createHash('sha256') - .update( - stringify({ - license, - features, - error, - }) - ) - .digest('hex'); -} +import { getLicenseFetcher } from './license_fetcher'; /** * @public @@ -153,9 +92,16 @@ export class LicensingPlugin implements Plugin - this.fetchLicense(clusterClient) + const { license$, refreshManually } = createLicenseUpdate( + intervalRefresh$, + this.stop$, + licenseFetcher ); this.loggingSubscription = license$.subscribe((license) => @@ -178,50 +124,6 @@ export class LicensingPlugin implements Plugin): Promise => { - const client = isPromise(clusterClient) ? await clusterClient : clusterClient; - try { - const response = await client.asInternalUser.xpack.info(); - const normalizedLicense = - response.license && response.license.type !== 'missing' - ? normalizeServerLicense(response.license) - : undefined; - const normalizedFeatures = response.features - ? normalizeFeatures(response.features) - : undefined; - - const signature = sign({ - license: normalizedLicense, - features: normalizedFeatures, - error: '', - }); - - return new License({ - license: normalizedLicense, - features: normalizedFeatures, - signature, - }); - } catch (error) { - this.logger.warn( - `License information could not be obtained from Elasticsearch due to ${error} error` - ); - const errorMessage = this.getErrorMessage(error); - const signature = sign({ error: errorMessage }); - - return new License({ - error: this.getErrorMessage(error), - signature, - }); - } - }; - - private getErrorMessage(error: ElasticsearchError): string { - if (error.status === 400) { - return 'X-Pack plugin is not installed on the Elasticsearch cluster.'; - } - return error.message; - } - public start() { if (!this.refresh || !this.license$) { throw new Error('Setup has not been completed'); diff --git a/x-pack/plugins/licensing/server/types.ts b/x-pack/plugins/licensing/server/types.ts index 83b39cb663715..fcccdecb66c00 100644 --- a/x-pack/plugins/licensing/server/types.ts +++ b/x-pack/plugins/licensing/server/types.ts @@ -14,6 +14,8 @@ export interface ElasticsearchError extends Error { status?: number; } +export type LicenseFetcher = () => Promise; + /** * Result from remote request fetching raw feature set. * @internal diff --git a/x-pack/plugins/licensing/tsconfig.json b/x-pack/plugins/licensing/tsconfig.json index 323f77b3b0ebc..1deb735f99466 100644 --- a/x-pack/plugins/licensing/tsconfig.json +++ b/x-pack/plugins/licensing/tsconfig.json @@ -15,7 +15,8 @@ "@kbn/i18n", "@kbn/analytics-client", "@kbn/subscription-tracking", - "@kbn/core-analytics-browser" + "@kbn/core-analytics-browser", + "@kbn/logging-mocks" ], "exclude": ["target/**/*"] } diff --git a/x-pack/plugins/security_solution/public/common/icons/entity_analytics.tsx b/x-pack/plugins/security_solution/public/common/icons/entity_analytics.tsx index a9e4123fe4d41..e282f07c4dcdb 100644 --- a/x-pack/plugins/security_solution/public/common/icons/entity_analytics.tsx +++ b/x-pack/plugins/security_solution/public/common/icons/entity_analytics.tsx @@ -20,12 +20,13 @@ export const IconEntityAnalytics: React.FC> = ({ ...prop fillRule="evenodd" clipRule="evenodd" d="M25.332 7C25.332 8.10457 26.2275 9 27.332 9C28.4366 9 29.332 8.10457 29.332 7C29.332 5.89543 28.4366 5 27.332 5C26.2275 5 25.332 5.89543 25.332 7ZM23.332 7C23.332 7.37644 23.384 7.74073 23.4812 8.08609L17.6976 11.1707C15.9888 8.65367 13.1035 7 9.83203 7C4.58533 7 0.332031 11.2533 0.332031 16.5C0.332031 21.7467 4.58533 26 9.83203 26C12.6903 26 15.2537 24.7377 16.9952 22.7403L23.387 26.3356C23.3508 26.5517 23.332 26.7737 23.332 27C23.332 29.2091 25.1229 31 27.332 31C29.5412 31 31.332 29.2091 31.332 27C31.332 24.7909 29.5412 23 27.332 23C26.0677 23 24.9404 23.5866 24.2074 24.5024L18.1491 21.0946C18.672 20.15 19.0387 19.1068 19.2143 18H24.4581C24.9021 19.7252 26.4682 21 28.332 21C30.5412 21 32.332 19.2091 32.332 17C32.332 14.7909 30.5412 13 28.332 13C26.4682 13 24.9021 14.2748 24.458 16H19.3191C19.2631 14.9207 19.027 13.8891 18.6403 12.9346L24.49 9.81475C25.2149 10.5466 26.2205 11 27.332 11C29.5412 11 31.332 9.20914 31.332 7C31.332 4.79086 29.5412 3 27.332 3C25.1229 3 23.332 4.79086 23.332 7ZM28.332 19C27.2275 19 26.332 18.1046 26.332 17C26.332 15.8954 27.2275 15 28.332 15C29.4366 15 30.332 15.8954 30.332 17C30.332 18.1046 29.4366 19 28.332 19ZM25.332 27C25.332 28.1046 26.2275 29 27.332 29C28.4366 29 29.332 28.1046 29.332 27C29.332 25.8954 28.4366 25 27.332 25C26.2275 25 25.332 25.8954 25.332 27ZM9.83203 24C5.68989 24 2.33203 20.6421 2.33203 16.5C2.33203 12.3579 5.6899 9 9.83203 9C13.9742 9 17.332 12.3579 17.332 16.5C17.332 20.6421 13.9742 24 9.83203 24Z" - className="euiIcon__fillSecondary" + fill="#343741" /> diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index ba03ddd298d5f..d1e0682fac22e 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -39373,4 +39373,4 @@ "xpack.painlessLab.walkthroughButtonLabel": "Présentation", "xpack.serverlessObservability.nav.getStarted": "Démarrer" } -} \ No newline at end of file +} diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index b2a00bc71bd43..582fe51373f9a 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -39364,4 +39364,4 @@ "xpack.painlessLab.walkthroughButtonLabel": "実地検証", "xpack.serverlessObservability.nav.getStarted": "使ってみる" } -} \ No newline at end of file +} diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 5c97128b8c8f1..264c37f71ad83 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -39358,4 +39358,4 @@ "xpack.painlessLab.walkthroughButtonLabel": "指导", "xpack.serverlessObservability.nav.getStarted": "开始使用" } -} \ No newline at end of file +} diff --git a/x-pack/plugins/uptime/server/legacy_uptime/lib/alerts/status_check.test.ts b/x-pack/plugins/uptime/server/legacy_uptime/lib/alerts/status_check.test.ts index 8755427253369..9f35a9c6f6c73 100644 --- a/x-pack/plugins/uptime/server/legacy_uptime/lib/alerts/status_check.test.ts +++ b/x-pack/plugins/uptime/server/legacy_uptime/lib/alerts/status_check.test.ts @@ -129,7 +129,7 @@ const mockStatusAlertDocument = ( return { fields: { ...mockCommonAlertDocumentFields(monitor.monitorInfo), - [ALERT_REASON]: `Monitor "First" from ${monitor.monitorInfo.observer?.geo?.name} failed ${count} times in the last ${interval}. Alert when > ${numTimes}. Checked at ${checkedAt}.`, + [ALERT_REASON]: `Monitor "First" from ${monitor.monitorInfo.observer?.geo?.name} failed ${count} times in the last ${interval}. Alert when >= ${numTimes}. Checked at ${checkedAt}.`, }, id: getInstanceId( monitorInfo, @@ -293,8 +293,8 @@ describe('status check alert', () => { "monitorUrl": "localhost:8080", "observerHostname": undefined, "observerLocation": "harrisburg", - "reason": "Monitor \\"First\\" from harrisburg failed 234 times in the last 15 mins. Alert when > 5. Checked at July 6, 2020 9:14 PM.", - "statusMessage": "failed 234 times in the last 15 mins. Alert when > 5.", + "reason": "Monitor \\"First\\" from harrisburg failed 234 times in the last 15 mins. Alert when >= 5. Checked at July 6, 2020 9:14 PM.", + "statusMessage": "failed 234 times in the last 15 mins. Alert when >= 5.", }, ] `); @@ -312,8 +312,8 @@ describe('status check alert', () => { "monitorUrl": "localhost:8080", "observerHostname": undefined, "observerLocation": "harrisburg", - "reason": "Monitor \\"First\\" from harrisburg failed 234 times in the last 15 mins. Alert when > 5. Checked at July 6, 2020 9:14 PM.", - "statusMessage": "failed 234 times in the last 15 mins. Alert when > 5.", + "reason": "Monitor \\"First\\" from harrisburg failed 234 times in the last 15 mins. Alert when >= 5. Checked at July 6, 2020 9:14 PM.", + "statusMessage": "failed 234 times in the last 15 mins. Alert when >= 5.", "viewInAppUrl": "http://localhost:5601/hfe/app/uptime/monitor/Zmlyc3Q=?dateRangeEnd=now&dateRangeStart=2022-03-17T13%3A13%3A33.755Z&filters=%5B%5B%22observer.geo.name%22%2C%5B%22harrisburg%22%5D%5D%5D", }, ] @@ -375,8 +375,8 @@ describe('status check alert', () => { "monitorUrl": "localhost:8080", "observerHostname": undefined, "observerLocation": "harrisburg", - "reason": "Monitor \\"First\\" from harrisburg failed 234 times in the last 15m. Alert when > 5. Checked at July 6, 2020 9:14 PM.", - "statusMessage": "failed 234 times in the last 15m. Alert when > 5.", + "reason": "Monitor \\"First\\" from harrisburg failed 234 times in the last 15m. Alert when >= 5. Checked at July 6, 2020 9:14 PM.", + "statusMessage": "failed 234 times in the last 15m. Alert when >= 5.", }, ] `); @@ -394,8 +394,8 @@ describe('status check alert', () => { "monitorUrl": "localhost:8080", "observerHostname": undefined, "observerLocation": "harrisburg", - "reason": "Monitor \\"First\\" from harrisburg failed 234 times in the last 15m. Alert when > 5. Checked at July 6, 2020 9:14 PM.", - "statusMessage": "failed 234 times in the last 15m. Alert when > 5.", + "reason": "Monitor \\"First\\" from harrisburg failed 234 times in the last 15m. Alert when >= 5. Checked at July 6, 2020 9:14 PM.", + "statusMessage": "failed 234 times in the last 15m. Alert when >= 5.", "viewInAppUrl": "http://localhost:5601/hfe/app/uptime/monitor/Zmlyc3Q=?dateRangeEnd=now&dateRangeStart=2022-03-17T13%3A13%3A33.755Z&filters=%5B%5B%22observer.geo.name%22%2C%5B%22harrisburg%22%5D%5D%5D", }, ] @@ -448,8 +448,8 @@ describe('status check alert', () => { "monitorUrl": "localhost:8080", "observerHostname": undefined, "observerLocation": "harrisburg", - "reason": "Monitor \\"First\\" from harrisburg failed 234 times in the last 14h. Alert when > 4. Checked at July 6, 2020 9:14 PM.", - "statusMessage": "failed 234 times in the last 14h. Alert when > 4.", + "reason": "Monitor \\"First\\" from harrisburg failed 234 times in the last 14h. Alert when >= 4. Checked at July 6, 2020 9:14 PM.", + "statusMessage": "failed 234 times in the last 14h. Alert when >= 4.", }, ] `); @@ -665,8 +665,8 @@ describe('status check alert', () => { "monitorUrl": "localhost:8080", "observerHostname": undefined, "observerLocation": "harrisburg", - "reason": "Monitor \\"First\\" from harrisburg failed 234 times in the last 15 mins. Alert when > 3. Checked at July 6, 2020 9:14 PM.", - "statusMessage": "failed 234 times in the last 15 mins. Alert when > 3.", + "reason": "Monitor \\"First\\" from harrisburg failed 234 times in the last 15 mins. Alert when >= 3. Checked at July 6, 2020 9:14 PM.", + "statusMessage": "failed 234 times in the last 15 mins. Alert when >= 3.", }, ] `); @@ -1476,7 +1476,7 @@ describe('status check alert', () => { numTimes: 10, interval: '30 days', }) - ).toMatchInlineSnapshot(`"failed 235 times in the last 30 days. Alert when > 10."`); + ).toMatchInlineSnapshot(`"failed 235 times in the last 30 days. Alert when >= 10."`); }); it('creates message for availability item', () => { @@ -1539,7 +1539,7 @@ describe('status check alert', () => { } ) ).toMatchInlineSnapshot( - `"failed 235 times in the last 30 days. Alert when > 10. The 5 mins availability is 58.04%. Alert when < 90%."` + `"failed 235 times in the last 30 days. Alert when >= 10. The 5 mins availability is 58.04%. Alert when < 90%."` ); }); }); diff --git a/x-pack/plugins/uptime/server/legacy_uptime/lib/alerts/translations.ts b/x-pack/plugins/uptime/server/legacy_uptime/lib/alerts/translations.ts index 4a0990384426e..e48b59e627688 100644 --- a/x-pack/plugins/uptime/server/legacy_uptime/lib/alerts/translations.ts +++ b/x-pack/plugins/uptime/server/legacy_uptime/lib/alerts/translations.ts @@ -340,7 +340,7 @@ export const durationAnomalyTranslations = { export const statusCheckTranslations = { downMonitorsLabel: (count: number, interval: string, numTimes: number) => i18n.translate('xpack.uptime.alerts.monitorStatus.actionVariables.down', { - defaultMessage: `failed {count} times in the last {interval}. Alert when > {numTimes}.`, + defaultMessage: `failed {count} times in the last {interval}. Alert when >= {numTimes}.`, values: { count, interval, diff --git a/x-pack/plugins/watcher/public/application/sections/watch_list_page/watch_list_page.tsx b/x-pack/plugins/watcher/public/application/sections/watch_list_page/watch_list_page.tsx index 76a8328fb532a..37d73d0c8a25a 100644 --- a/x-pack/plugins/watcher/public/application/sections/watch_list_page/watch_list_page.tsx +++ b/x-pack/plugins/watcher/public/application/sections/watch_list_page/watch_list_page.tsx @@ -47,6 +47,80 @@ import { goToCreateThresholdAlert, goToCreateAdvancedWatch } from '../../lib/nav import { useAppContext } from '../../app_context'; import { PageError as GenericPageError } from '../../shared_imports'; +/* + * EuiMemoryTable relies on referential equality of a column's name field when sorting by that column. + * Therefore, we want the JSX elements preserved through renders. + */ +const stateColumnHeader = ( + + + {i18n.translate('xpack.watcher.sections.watchList.watchTable.stateHeader', { + defaultMessage: 'State', + })}{' '} + + + +); + +const conditionLastMetHeader = ( + + + {i18n.translate('xpack.watcher.sections.watchList.watchTable.lastFiredHeader', { + defaultMessage: 'Condition last met', + })}{' '} + + + +); + +const lastCheckedHeader = ( + + + {i18n.translate('xpack.watcher.sections.watchList.watchTable.lastTriggeredHeader', { + defaultMessage: 'Last checked', + })}{' '} + + + +); + +const commentHeader = ( + + + {i18n.translate('xpack.watcher.sections.watchList.watchTable.commentHeader', { + defaultMessage: 'Comment', + })}{' '} + + + +); + export const WatchListPage = () => { // hooks const { @@ -273,46 +347,14 @@ export const WatchListPage = () => { }, { field: 'watchStatus.state', - name: ( - - - {i18n.translate('xpack.watcher.sections.watchList.watchTable.stateHeader', { - defaultMessage: 'State', - })}{' '} - - - - ), + name: stateColumnHeader, sortable: true, width: '130px', render: (state: string) => , }, { field: 'watchStatus.lastMetCondition', - name: ( - - - {i18n.translate('xpack.watcher.sections.watchList.watchTable.lastFiredHeader', { - defaultMessage: 'Condition last met', - })}{' '} - - - - ), + name: conditionLastMetHeader, sortable: true, truncateText: true, width: '160px', @@ -322,23 +364,7 @@ export const WatchListPage = () => { }, { field: 'watchStatus.lastChecked', - name: ( - - - {i18n.translate('xpack.watcher.sections.watchList.watchTable.lastTriggeredHeader', { - defaultMessage: 'Last checked', - })}{' '} - - - - ), + name: lastCheckedHeader, sortable: true, truncateText: true, width: '160px', @@ -348,24 +374,7 @@ export const WatchListPage = () => { }, { field: 'watchStatus.comment', - name: ( - - - {i18n.translate('xpack.watcher.sections.watchList.watchTable.commentHeader', { - defaultMessage: 'Comment', - })}{' '} - - - - ), + name: commentHeader, sortable: true, truncateText: true, }, diff --git a/x-pack/test/functional/apps/transform/creation/runtime_mappings_saved_search/creation_runtime_mappings.ts b/x-pack/test/functional/apps/transform/creation/runtime_mappings_saved_search/creation_runtime_mappings.ts index ae6ac9adab2f5..3f0adc5783893 100644 --- a/x-pack/test/functional/apps/transform/creation/runtime_mappings_saved_search/creation_runtime_mappings.ts +++ b/x-pack/test/functional/apps/transform/creation/runtime_mappings_saved_search/creation_runtime_mappings.ts @@ -34,8 +34,7 @@ export default function ({ getService }: FtrProviderContext) { }, }; - // Failing: See https://github.com/elastic/kibana/issues/166395 - describe.skip('creation with runtime mappings', function () { + describe('creation with runtime mappings', function () { before(async () => { await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/farequote'); await transform.testResources.createIndexPatternIfNeeded('ft_farequote', '@timestamp'); diff --git a/x-pack/test/screenshot_creation/apps/response_ops_docs/stack_alerting/es_query_rule.ts b/x-pack/test/screenshot_creation/apps/response_ops_docs/stack_alerting/es_query_rule.ts index e1a0bf73dd975..ef41d15fcd6e3 100644 --- a/x-pack/test/screenshot_creation/apps/response_ops_docs/stack_alerting/es_query_rule.ts +++ b/x-pack/test/screenshot_creation/apps/response_ops_docs/stack_alerting/es_query_rule.ts @@ -31,26 +31,12 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }, }, }); - const invalidQueryJson = JSON.stringify({ - query: { - bool: { - filter: [ - { - error_clause: { - 'host.keyword': 'www.elastic.co', - }, - }, - ], - }, - }, - }); describe('elasticsearch query rule', function () { it('create rule screenshot', async () => { await pageObjects.common.navigateToApp('triggersActions'); await pageObjects.header.waitUntilLoadingHasFinished(); await rules.common.clickCreateAlertButton(); - await testSubjects.scrollIntoView('ruleNameInput'); await testSubjects.setValue('ruleNameInput', ruleName); await testSubjects.click(`.es-query-SelectOption`); await testSubjects.click('queryFormType_esQuery'); @@ -60,11 +46,13 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await testSubjects.click('thresholdAlertTimeFieldSelect'); await testSubjects.setValue('thresholdAlertTimeFieldSelect', '@timestamp'); await testSubjects.click('closePopover'); + await comboBox.set('ruleFormConsumerSelect', 'Stack Rules'); + await testSubjects.scrollIntoView('ruleNameInput'); await commonScreenshots.takeScreenshot( 'rule-types-es-query-conditions', screenshotDirectories, 1400, - 1500 + 1900 ); // Test a valid query await testSubjects.setValue('queryJsonEditor', '', { @@ -87,23 +75,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { 1400, 1500 ); - // Test an invalid query - await testSubjects.setValue('queryJsonEditor', '', { - clearWithKeyboard: true, - }); - await queryJsonEditor.clearValue(); - await testSubjects.setValue('queryJsonEditor', invalidQueryJson, { - clearWithKeyboard: true, - }); - await testSubjects.click('testQuery'); - await testSubjects.scrollIntoView('ruleNameInput'); - await pageObjects.header.waitUntilLoadingHasFinished(); - await commonScreenshots.takeScreenshot( - 'rule-types-es-query-invalid', - screenshotDirectories, - 1400, - 1500 - ); // Create an email connector action await testSubjects.click('.email-alerting-ActionTypeSelectOption'); await testSubjects.scrollIntoView('addAlertActionButton'); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/entity_analytics/enrichments.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/entity_analytics/enrichments.cy.ts index 99cbf31012d75..c93b83b448e3a 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/entity_analytics/enrichments.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/entity_analytics/enrichments.cy.ts @@ -36,7 +36,8 @@ import { enableRiskEngine } from '../../tasks/entity_analytics'; const CURRENT_HOST_RISK_LEVEL = 'Current host risk level'; const ORIGINAL_HOST_RISK_LEVEL = 'Original host risk level'; -describe('Enrichment', { tags: ['@ess', '@serverless'] }, () => { +// FLAKY: https://github.com/elastic/kibana/issues/169154 +describe.skip('Enrichment', { tags: ['@ess', '@serverless'] }, () => { before(() => { cleanKibana(); cy.task('esArchiverUnload', 'risk_scores_new'); @@ -49,8 +50,7 @@ describe('Enrichment', { tags: ['@ess', '@serverless'] }, () => { }); describe('Custom query rule', () => { - // FLAKY: https://github.com/elastic/kibana/issues/169154 - describe.skip('from legacy risk scores', () => { + describe('from legacy risk scores', () => { beforeEach(() => { disableExpandableFlyout(); cy.task('esArchiverLoad', { archiveName: 'risk_hosts' });