diff --git a/src/dev/storybook/aliases.ts b/src/dev/storybook/aliases.ts index 9d9f5616b5a33..d31a408e98c67 100644 --- a/src/dev/storybook/aliases.ts +++ b/src/dev/storybook/aliases.ts @@ -23,7 +23,7 @@ export const storybookAliases = { codeeditor: 'src/plugins/kibana_react/public/code_editor/scripts/storybook.ts', dashboard_enhanced: 'x-pack/plugins/dashboard_enhanced/scripts/storybook.js', embeddable: 'src/plugins/embeddable/scripts/storybook.js', - infra: 'x-pack/legacy/plugins/infra/scripts/storybook.js', + infra: 'x-pack/plugins/infra/scripts/storybook.js', security_solution: 'x-pack/plugins/security_solution/scripts/storybook.js', ui_actions_enhanced: 'x-pack/plugins/ui_actions_enhanced/scripts/storybook.js', observability: 'x-pack/plugins/observability/scripts/storybook.js', diff --git a/x-pack/plugins/infra/common/http_api/log_analysis/results/index.ts b/x-pack/plugins/infra/common/http_api/log_analysis/results/index.ts index a01042616a872..e696477253823 100644 --- a/x-pack/plugins/infra/common/http_api/log_analysis/results/index.ts +++ b/x-pack/plugins/infra/common/http_api/log_analysis/results/index.ts @@ -6,6 +6,7 @@ export * from './log_entry_categories'; export * from './log_entry_category_datasets'; +export * from './log_entry_category_datasets_stats'; export * from './log_entry_category_examples'; export * from './log_entry_rate'; export * from './log_entry_examples'; diff --git a/x-pack/plugins/infra/common/http_api/log_analysis/results/log_entry_category_datasets_stats.ts b/x-pack/plugins/infra/common/http_api/log_analysis/results/log_entry_category_datasets_stats.ts new file mode 100644 index 0000000000000..4511678242f1c --- /dev/null +++ b/x-pack/plugins/infra/common/http_api/log_analysis/results/log_entry_category_datasets_stats.ts @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as rt from 'io-ts'; + +import { timeRangeRT, routeTimingMetadataRT } from '../../shared'; + +export const LOG_ANALYSIS_GET_LATEST_LOG_ENTRY_CATEGORY_DATASETS_STATS_PATH = + '/api/infra/log_analysis/results/latest_log_entry_category_datasets_stats'; + +const categorizerStatusRT = rt.keyof({ + ok: null, + warn: null, +}); + +export type CategorizerStatus = rt.TypeOf; + +/** + * request + */ + +export const getLatestLogEntryCategoryDatasetsStatsRequestPayloadRT = rt.type({ + data: rt.type({ + // the ids of the categorization jobs + jobIds: rt.array(rt.string), + // the time range to fetch the category datasets stats for + timeRange: timeRangeRT, + // the categorizer statuses to include stats for, empty means all + includeCategorizerStatuses: rt.array(categorizerStatusRT), + }), +}); + +export type GetLatestLogEntryCategoryDatasetsStatsRequestPayload = rt.TypeOf< + typeof getLatestLogEntryCategoryDatasetsStatsRequestPayloadRT +>; + +/** + * response + */ + +const logEntryCategoriesDatasetStatsRT = rt.type({ + categorization_status: categorizerStatusRT, + categorized_doc_count: rt.number, + dataset: rt.string, + dead_category_count: rt.number, + failed_category_count: rt.number, + frequent_category_count: rt.number, + job_id: rt.string, + log_time: rt.number, + rare_category_count: rt.number, + total_category_count: rt.number, +}); + +export type LogEntryCategoriesDatasetStats = rt.TypeOf; + +export const getLatestLogEntryCategoryDatasetsStatsSuccessResponsePayloadRT = rt.intersection([ + rt.type({ + data: rt.type({ + datasetStats: rt.array(logEntryCategoriesDatasetStatsRT), + }), + }), + rt.partial({ + timing: routeTimingMetadataRT, + }), +]); + +export type GetLatestLogEntryCategoryDatasetsStatsSuccessResponsePayload = rt.TypeOf< + typeof getLatestLogEntryCategoryDatasetsStatsSuccessResponsePayloadRT +>; diff --git a/x-pack/plugins/infra/common/log_analysis/index.ts b/x-pack/plugins/infra/common/log_analysis/index.ts index 22137e63ab7e7..0b4fa374a5da9 100644 --- a/x-pack/plugins/infra/common/log_analysis/index.ts +++ b/x-pack/plugins/infra/common/log_analysis/index.ts @@ -5,6 +5,7 @@ */ export * from './log_analysis'; +export * from './log_analysis_quality'; export * from './log_analysis_results'; export * from './log_entry_rate_analysis'; export * from './log_entry_categories_analysis'; diff --git a/x-pack/plugins/infra/common/log_analysis/log_analysis_quality.ts b/x-pack/plugins/infra/common/log_analysis/log_analysis_quality.ts new file mode 100644 index 0000000000000..7ffa6c172886b --- /dev/null +++ b/x-pack/plugins/infra/common/log_analysis/log_analysis_quality.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +interface ManyCategoriesWarningReason { + type: 'manyCategories'; + categoriesDocumentRatio: number; +} +interface ManyDeadCategoriesWarningReason { + type: 'manyDeadCategories'; + deadCategoriesRatio: number; +} +interface ManyRareCategoriesWarningReason { + type: 'manyRareCategories'; + rareCategoriesRatio: number; +} +interface NoFrequentCategoriesWarningReason { + type: 'noFrequentCategories'; +} +interface SingleCategoryWarningReason { + type: 'singleCategory'; +} + +export type CategoryQualityWarningReason = + | ManyCategoriesWarningReason + | ManyDeadCategoriesWarningReason + | ManyRareCategoriesWarningReason + | NoFrequentCategoriesWarningReason + | SingleCategoryWarningReason; + +export type CategoryQualityWarningReasonType = CategoryQualityWarningReason['type']; + +export interface CategoryQualityWarning { + type: 'categoryQualityWarning'; + jobId: string; + dataset: string; + reasons: CategoryQualityWarningReason[]; +} + +export type QualityWarning = CategoryQualityWarning; diff --git a/x-pack/plugins/infra/public/components/logging/log_analysis_job_status/job_configuration_outdated_callout.tsx b/x-pack/plugins/infra/public/components/logging/log_analysis_job_status/job_configuration_outdated_callout.tsx index 0489bd7d9929a..5b2ce862f7a81 100644 --- a/x-pack/plugins/infra/public/components/logging/log_analysis_job_status/job_configuration_outdated_callout.tsx +++ b/x-pack/plugins/infra/public/components/logging/log_analysis_job_status/job_configuration_outdated_callout.tsx @@ -31,6 +31,7 @@ export const JobConfigurationOutdatedCallout: React.FC<{ values={{ moduleName, }} + tagName="p" /> ); diff --git a/x-pack/plugins/infra/public/components/logging/log_analysis_job_status/job_definition_outdated_callout.tsx b/x-pack/plugins/infra/public/components/logging/log_analysis_job_status/job_definition_outdated_callout.tsx index df9de49ea0445..b9e68b25482b6 100644 --- a/x-pack/plugins/infra/public/components/logging/log_analysis_job_status/job_definition_outdated_callout.tsx +++ b/x-pack/plugins/infra/public/components/logging/log_analysis_job_status/job_definition_outdated_callout.tsx @@ -31,6 +31,7 @@ export const JobDefinitionOutdatedCallout: React.FC<{ values={{ moduleName, }} + tagName="p" /> ); diff --git a/x-pack/plugins/infra/public/components/logging/log_analysis_job_status/notices_section.tsx b/x-pack/plugins/infra/public/components/logging/log_analysis_job_status/notices_section.tsx index 2535058322cba..3785d0e8d9423 100644 --- a/x-pack/plugins/infra/public/components/logging/log_analysis_job_status/notices_section.tsx +++ b/x-pack/plugins/infra/public/components/logging/log_analysis_job_status/notices_section.tsx @@ -5,7 +5,7 @@ */ import React from 'react'; -import { QualityWarning } from '../../../containers/logs/log_analysis/log_analysis_module_types'; +import { QualityWarning } from '../../../../common/log_analysis'; import { LogAnalysisJobProblemIndicator } from './log_analysis_job_problem_indicator'; import { CategoryQualityWarnings } from './quality_warning_notices'; @@ -41,6 +41,10 @@ export const CategoryJobNoticesSection: React.FC<{ onRecreateMlJobForReconfiguration={onRecreateMlJobForReconfiguration} onRecreateMlJobForUpdate={onRecreateMlJobForUpdate} /> - + ); diff --git a/x-pack/plugins/infra/public/components/logging/log_analysis_job_status/quality_warning_notices.stories.tsx b/x-pack/plugins/infra/public/components/logging/log_analysis_job_status/quality_warning_notices.stories.tsx new file mode 100644 index 0000000000000..7caf75417091a --- /dev/null +++ b/x-pack/plugins/infra/public/components/logging/log_analysis_job_status/quality_warning_notices.stories.tsx @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { action } from '@storybook/addon-actions'; +import { storiesOf } from '@storybook/react'; +import React from 'react'; +import { EuiThemeProvider } from '../../../../../observability/public'; +import { QualityWarning } from '../../../../common/log_analysis'; +import { CategoryQualityWarnings } from './quality_warning_notices'; + +storiesOf('infra/logAnalysis/CategoryQualityWarnings', module) + .addDecorator((renderStory) => {renderStory()}) + .add('Partitioned warnings', () => { + return ( + + ); + }) + .add('Unpartitioned warnings', () => { + return ( + + ); + }); + +const partitionedQualityWarnings: QualityWarning[] = [ + { + type: 'categoryQualityWarning', + jobId: 'theMlJobId', + dataset: 'first.dataset', + reasons: [ + { type: 'singleCategory' }, + { type: 'manyRareCategories', rareCategoriesRatio: 0.95 }, + { type: 'manyCategories', categoriesDocumentRatio: 0.7 }, + ], + }, + { + type: 'categoryQualityWarning', + jobId: 'theMlJobId', + dataset: 'second.dataset', + reasons: [ + { type: 'noFrequentCategories' }, + { type: 'manyDeadCategories', deadCategoriesRatio: 0.7 }, + ], + }, +]; + +const unpartitionedQualityWarnings: QualityWarning[] = [ + { + type: 'categoryQualityWarning', + jobId: 'theMlJobId', + dataset: '', + reasons: [ + { type: 'singleCategory' }, + { type: 'manyRareCategories', rareCategoriesRatio: 0.95 }, + { type: 'manyCategories', categoriesDocumentRatio: 0.7 }, + ], + }, +]; diff --git a/x-pack/plugins/infra/public/components/logging/log_analysis_job_status/quality_warning_notices.tsx b/x-pack/plugins/infra/public/components/logging/log_analysis_job_status/quality_warning_notices.tsx index 0d93ead5a82c6..928c9738c4761 100644 --- a/x-pack/plugins/infra/public/components/logging/log_analysis_job_status/quality_warning_notices.tsx +++ b/x-pack/plugins/infra/public/components/logging/log_analysis_job_status/quality_warning_notices.tsx @@ -4,43 +4,89 @@ * you may not use this file except in compliance with the Elastic License. */ -import { EuiCallOut } from '@elastic/eui'; +import { + EuiAccordion, + EuiDescriptionList, + EuiDescriptionListDescription, + EuiDescriptionListTitle, + EuiSpacer, + htmlIdGenerator, +} from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; -import React from 'react'; -import type { +import groupBy from 'lodash/groupBy'; +import React, { Fragment, useState } from 'react'; +import { euiStyled } from '../../../../../observability/public'; +import { + CategoryQualityWarning, CategoryQualityWarningReason, - QualityWarning, -} from '../../../containers/logs/log_analysis/log_analysis_module_types'; + getFriendlyNameForPartitionId, +} from '../../../../common/log_analysis'; +import { RecreateJobCallout } from './recreate_job_callout'; -export const CategoryQualityWarnings: React.FC<{ qualityWarnings: QualityWarning[] }> = ({ - qualityWarnings, -}) => ( - <> - {qualityWarnings.map((qualityWarning, qualityWarningIndex) => ( - -

+export const CategoryQualityWarnings: React.FC<{ + hasSetupCapabilities: boolean; + onRecreateMlJob: () => void; + qualityWarnings: CategoryQualityWarning[]; +}> = ({ hasSetupCapabilities, onRecreateMlJob, qualityWarnings }) => { + const [detailAccordionId] = useState(htmlIdGenerator()()); + + const categoryQualityWarningsByJob = groupBy(qualityWarnings, 'jobId'); + + return ( + <> + {Object.entries(categoryQualityWarningsByJob).map(([jobId, qualityWarningsForJob]) => ( + -

-
    - {qualityWarning.reasons.map((reason, reasonIndex) => ( -
  • - -
    • - ))} -
- - ))} - -); + + } + paddingSize="m" + > + + {qualityWarningsForJob.flatMap((qualityWarning) => ( + + + {getFriendlyNameForPartitionId(qualityWarning.dataset)} + + {qualityWarning.reasons.map((reason) => ( + + + + ))} + + ))} + + + + + ))} + + ); +}; + +const QualityWarningReasonDescription = euiStyled(EuiDescriptionListDescription)` + display: list-item; + list-style-type: disc; + margin-left: ${(props) => props.theme.eui.paddingSizes.m}; +`; const categoryQualityWarningCalloutTitle = i18n.translate( 'xpack.infra.logs.logEntryCategories.categoryQUalityWarningCalloutTitle', @@ -49,7 +95,7 @@ const categoryQualityWarningCalloutTitle = i18n.translate( } ); -const CategoryQualityWarningReasonDescription: React.FC<{ +export const CategoryQualityWarningReasonDescription: React.FC<{ reason: CategoryQualityWarningReason; }> = ({ reason }) => { switch (reason.type) { @@ -57,7 +103,7 @@ const CategoryQualityWarningReasonDescription: React.FC<{ return ( ); case 'manyRareCategories': diff --git a/x-pack/plugins/infra/public/components/logging/log_analysis_job_status/recreate_job_callout.tsx b/x-pack/plugins/infra/public/components/logging/log_analysis_job_status/recreate_job_callout.tsx index cdf030a849fa1..2a0337bd99767 100644 --- a/x-pack/plugins/infra/public/components/logging/log_analysis_job_status/recreate_job_callout.tsx +++ b/x-pack/plugins/infra/public/components/logging/log_analysis_job_status/recreate_job_callout.tsx @@ -14,7 +14,7 @@ export const RecreateJobCallout: React.FC<{ title?: React.ReactNode; }> = ({ children, hasSetupCapabilities, onRecreateMlJob, title }) => ( -

{children}

+ {children} void; + previousQualityWarnings?: QualityWarning[]; validationErrors?: ValidationIndicesError[]; }> = ({ disabled = false, indices, isValidating, onChangeSelectedIndices, + previousQualityWarnings = [], validationErrors = [], }) => { const changeIsIndexSelected = useCallback( @@ -81,6 +84,7 @@ export const AnalysisSetupIndicesForm: React.FunctionComponent<{ key={index.name} onChangeIsSelected={changeIsIndexSelected} onChangeDatasetFilter={changeDatasetFilter} + previousQualityWarnings={previousQualityWarnings} /> ))} diff --git a/x-pack/plugins/infra/public/components/logging/log_analysis_setup/initial_configuration_step/index_setup_dataset_filter.tsx b/x-pack/plugins/infra/public/components/logging/log_analysis_setup/initial_configuration_step/index_setup_dataset_filter.tsx index d3ed8aeaf6155..481cc6071864c 100644 --- a/x-pack/plugins/infra/public/components/logging/log_analysis_setup/initial_configuration_step/index_setup_dataset_filter.tsx +++ b/x-pack/plugins/infra/public/components/logging/log_analysis_setup/initial_configuration_step/index_setup_dataset_filter.tsx @@ -7,6 +7,7 @@ import { EuiFilterButton, EuiFilterGroup, + EuiIconTip, EuiPopover, EuiPopoverTitle, EuiSelectable, @@ -14,11 +15,15 @@ import { } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; import React, { useCallback, useMemo } from 'react'; -import { DatasetFilter } from '../../../../../common/log_analysis'; +import { DatasetFilter, QualityWarning } from '../../../../../common/log_analysis'; import { useVisibilityState } from '../../../../utils/use_visibility_state'; +import { CategoryQualityWarningReasonDescription } from '../../log_analysis_job_status/quality_warning_notices'; export const IndexSetupDatasetFilter: React.FC<{ - availableDatasets: string[]; + availableDatasets: Array<{ + dataset: string; + warnings: QualityWarning[]; + }>; datasetFilter: DatasetFilter; isDisabled?: boolean; onChangeDatasetFilter: (datasetFilter: DatasetFilter) => void; @@ -40,12 +45,13 @@ export const IndexSetupDatasetFilter: React.FC<{ [onChangeDatasetFilter] ); - const selectableOptions: EuiSelectableOption[] = useMemo( + const selectableOptions = useMemo( () => - availableDatasets.map((datasetName) => ({ - label: datasetName, + availableDatasets.map(({ dataset, warnings }) => ({ + label: dataset, + append: warnings.length > 0 ? : null, checked: - datasetFilter.type === 'includeSome' && datasetFilter.datasets.includes(datasetName) + datasetFilter.type === 'includeSome' && datasetFilter.datasets.includes(dataset) ? 'on' : undefined, })), @@ -86,3 +92,15 @@ export const IndexSetupDatasetFilter: React.FC<{ ); }; + +const DatasetWarningMarker: React.FC<{ warnings: QualityWarning[] }> = ({ warnings }) => { + const warningDescriptions = warnings.flatMap((warning) => + warning.type === 'categoryQualityWarning' + ? warning.reasons.map((reason) => ( + + )) + : [] + ); + + return ; +}; diff --git a/x-pack/plugins/infra/public/components/logging/log_analysis_setup/initial_configuration_step/index_setup_row.tsx b/x-pack/plugins/infra/public/components/logging/log_analysis_setup/initial_configuration_step/index_setup_row.tsx index 92774dbd6838b..b101b9b0cab0c 100644 --- a/x-pack/plugins/infra/public/components/logging/log_analysis_setup/initial_configuration_step/index_setup_row.tsx +++ b/x-pack/plugins/infra/public/components/logging/log_analysis_setup/initial_configuration_step/index_setup_row.tsx @@ -4,10 +4,10 @@ * you may not use this file except in compliance with the Elastic License. */ -import { EuiCheckbox, EuiCode, EuiFlexGroup, EuiFlexItem, EuiIcon, EuiToolTip } from '@elastic/eui'; +import { EuiCheckbox, EuiCode, EuiFlexGroup, EuiFlexItem, EuiIconTip } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; -import React, { useCallback } from 'react'; -import { DatasetFilter } from '../../../../../common/log_analysis'; +import React, { useCallback, useMemo } from 'react'; +import { DatasetFilter, QualityWarning } from '../../../../../common/log_analysis'; import { IndexSetupDatasetFilter } from './index_setup_dataset_filter'; import { AvailableIndex, ValidationUIError } from './validation'; @@ -16,7 +16,14 @@ export const IndexSetupRow: React.FC<{ isDisabled: boolean; onChangeDatasetFilter: (indexName: string, datasetFilter: DatasetFilter) => void; onChangeIsSelected: (indexName: string, isSelected: boolean) => void; -}> = ({ index, isDisabled, onChangeDatasetFilter, onChangeIsSelected }) => { + previousQualityWarnings: QualityWarning[]; +}> = ({ + index, + isDisabled, + onChangeDatasetFilter, + onChangeIsSelected, + previousQualityWarnings, +}) => { const changeIsSelected = useCallback( (event: React.ChangeEvent) => { onChangeIsSelected(index.name, event.currentTarget.checked); @@ -29,6 +36,29 @@ export const IndexSetupRow: React.FC<{ [index.name, onChangeDatasetFilter] ); + const datasets = useMemo( + () => + index.validity === 'valid' + ? index.availableDatasets.map((availableDataset) => ({ + dataset: availableDataset, + warnings: previousQualityWarnings.filter(({ dataset }) => dataset === availableDataset), + })) + : [], + [index, previousQualityWarnings] + ); + + const datasetIndependentQualityWarnings = useMemo( + () => previousQualityWarnings.filter(({ dataset }) => dataset === ''), + [previousQualityWarnings] + ); + + const hasWarnings = useMemo( + () => + datasetIndependentQualityWarnings.length > 0 || + datasets.some(({ warnings }) => warnings.length > 0), + [datasetIndependentQualityWarnings, datasets] + ); + const isSelected = index.validity === 'valid' && index.isSelected; return ( @@ -37,7 +67,23 @@ export const IndexSetupRow: React.FC<{ {index.name}} + label={ + <> + {index.name}{' '} + {index.validity === 'valid' && hasWarnings ? ( + + } + type="alert" + color="warning" + /> + ) : null} + + } onChange={changeIsSelected} checked={isSelected} disabled={isDisabled || index.validity === 'invalid'} @@ -45,12 +91,10 @@ export const IndexSetupRow: React.FC<{ {index.validity === 'invalid' ? ( - - - + ) : index.validity === 'valid' ? ( ( + +
{renderStory()}
+ + )) + .add('Reconfiguration with partitioned warnings', () => { + return ( + + ); + }) + .add('Reconfiguration with unpartitioned warnings', () => { + return ( + + ); + }); + +const storyActions = actions('setStartTime', 'setEndTime', 'setValidatedIndices'); diff --git a/x-pack/plugins/infra/public/components/logging/log_analysis_setup/initial_configuration_step/initial_configuration_step.tsx b/x-pack/plugins/infra/public/components/logging/log_analysis_setup/initial_configuration_step/initial_configuration_step.tsx index d4c3c727bd34e..1ea972335d8fc 100644 --- a/x-pack/plugins/infra/public/components/logging/log_analysis_setup/initial_configuration_step/initial_configuration_step.tsx +++ b/x-pack/plugins/infra/public/components/logging/log_analysis_setup/initial_configuration_step/initial_configuration_step.tsx @@ -9,7 +9,7 @@ import { EuiContainedStepProps } from '@elastic/eui/src/components/steps/steps'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; import React, { useMemo } from 'react'; -import { SetupStatus } from '../../../../../common/log_analysis'; +import { QualityWarning, SetupStatus } from '../../../../../common/log_analysis'; import { AnalysisSetupIndicesForm } from './analysis_setup_indices_form'; import { AnalysisSetupTimerangeForm } from './analysis_setup_timerange_form'; import { @@ -31,6 +31,7 @@ interface InitialConfigurationStepProps { setupStatus: SetupStatus; setValidatedIndices: (selectedIndices: AvailableIndex[]) => void; validationErrors?: ValidationUIError[]; + previousQualityWarnings?: QualityWarning[]; } export const createInitialConfigurationStep = ( @@ -50,6 +51,7 @@ export const InitialConfigurationStep: React.FunctionComponent { const disabled = useMemo(() => !editableFormStatus.includes(setupStatus.type), [setupStatus]); @@ -75,6 +77,7 @@ export const InitialConfigurationStep: React.FunctionComponent diff --git a/x-pack/plugins/infra/public/components/logging/log_analysis_setup/setup_flyout/log_entry_categories_setup_view.tsx b/x-pack/plugins/infra/public/components/logging/log_analysis_setup/setup_flyout/log_entry_categories_setup_view.tsx index 2bc5b08a1016a..e7961a11a4d52 100644 --- a/x-pack/plugins/infra/public/components/logging/log_analysis_setup/setup_flyout/log_entry_categories_setup_view.tsx +++ b/x-pack/plugins/infra/public/components/logging/log_analysis_setup/setup_flyout/log_entry_categories_setup_view.tsx @@ -6,6 +6,7 @@ import { EuiSpacer, EuiSteps, EuiText, EuiTitle } from '@elastic/eui'; import React, { useCallback, useMemo } from 'react'; +import { useMount } from 'react-use'; import { useLogEntryCategoriesSetup } from '../../../../containers/logs/log_analysis/modules/log_entry_categories'; import { createInitialConfigurationStep } from '../initial_configuration_step'; import { createProcessStep } from '../process_step'; @@ -14,8 +15,10 @@ export const LogEntryCategoriesSetupView: React.FC<{ onClose: () => void; }> = ({ onClose }) => { const { + categoryQualityWarnings, cleanUpAndSetUp, endTime, + fetchJobStatus, isValidating, lastSetupErrorMessages, moduleDescriptor, @@ -30,6 +33,10 @@ export const LogEntryCategoriesSetupView: React.FC<{ viewResults, } = useLogEntryCategoriesSetup(); + useMount(() => { + fetchJobStatus(); + }); + const viewResultsAndClose = useCallback(() => { viewResults(); onClose(); @@ -47,6 +54,7 @@ export const LogEntryCategoriesSetupView: React.FC<{ setupStatus, setValidatedIndices, validationErrors, + previousQualityWarnings: categoryQualityWarnings, }), createProcessStep({ cleanUpAndSetUp, @@ -58,6 +66,7 @@ export const LogEntryCategoriesSetupView: React.FC<{ }), ], [ + categoryQualityWarnings, cleanUpAndSetUp, endTime, isValidating, diff --git a/x-pack/plugins/infra/public/components/logging/log_analysis_setup/setup_flyout/setup_flyout.tsx b/x-pack/plugins/infra/public/components/logging/log_analysis_setup/setup_flyout/setup_flyout.tsx index 8e00254431438..407c851f2de95 100644 --- a/x-pack/plugins/infra/public/components/logging/log_analysis_setup/setup_flyout/setup_flyout.tsx +++ b/x-pack/plugins/infra/public/components/logging/log_analysis_setup/setup_flyout/setup_flyout.tsx @@ -15,14 +15,16 @@ import { } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; import React from 'react'; -import { LogEntryRateSetupView } from './log_entry_rate_setup_view'; import { LogEntryCategoriesSetupView } from './log_entry_categories_setup_view'; +import { LogEntryRateSetupView } from './log_entry_rate_setup_view'; import { LogAnalysisModuleList } from './module_list'; -import { useLogAnalysisSetupFlyoutStateContext } from './setup_flyout_state'; +import { ModuleId, moduleIds, useLogAnalysisSetupFlyoutStateContext } from './setup_flyout_state'; const FLYOUT_HEADING_ID = 'logAnalysisSetupFlyoutHeading'; -export const LogAnalysisSetupFlyout: React.FC = () => { +export const LogAnalysisSetupFlyout: React.FC<{ + allowedModules?: ModuleId[]; +}> = ({ allowedModules = moduleIds }) => { const { closeFlyout, flyoutView, @@ -49,32 +51,58 @@ export const LogAnalysisSetupFlyout: React.FC = () => { {flyoutView.view === 'moduleList' ? ( - ) : flyoutView.view === 'moduleSetup' && flyoutView.module === 'logs_ui_analysis' ? ( - - - - ) : flyoutView.view === 'moduleSetup' && flyoutView.module === 'logs_ui_categories' ? ( - - - + ) : flyoutView.view === 'moduleSetup' && allowedModules.includes(flyoutView.module) ? ( + 1 ? showModuleList : undefined} + /> ) : null} ); }; +const ModuleSetupView: React.FC<{ + moduleId: ModuleId; + onClose: () => void; + onViewModuleList?: () => void; +}> = ({ moduleId, onClose, onViewModuleList }) => { + switch (moduleId) { + case 'logs_ui_analysis': + return ( + + + + ); + case 'logs_ui_categories': + return ( + + + + ); + } +}; + const LogAnalysisSetupFlyoutSubPage: React.FC<{ - onViewModuleList: () => void; + onViewModuleList?: () => void; }> = ({ children, onViewModuleList }) => ( - - - - - + {onViewModuleList ? ( + + + + + + ) : null} {children} ); diff --git a/x-pack/plugins/infra/public/components/logging/log_analysis_setup/setup_flyout/setup_flyout_state.ts b/x-pack/plugins/infra/public/components/logging/log_analysis_setup/setup_flyout/setup_flyout_state.ts index 7a64584df4303..5f131daf952bf 100644 --- a/x-pack/plugins/infra/public/components/logging/log_analysis_setup/setup_flyout/setup_flyout_state.ts +++ b/x-pack/plugins/infra/public/components/logging/log_analysis_setup/setup_flyout/setup_flyout_state.ts @@ -9,6 +9,8 @@ import { useState, useCallback } from 'react'; export type ModuleId = 'logs_ui_analysis' | 'logs_ui_categories'; +export const moduleIds = ['logs_ui_analysis', 'logs_ui_categories'] as const; + type FlyoutView = | { view: 'hidden' } | { view: 'moduleList' } diff --git a/x-pack/plugins/infra/public/containers/logs/log_analysis/api/get_latest_categories_datasets_stats.ts b/x-pack/plugins/infra/public/containers/logs/log_analysis/api/get_latest_categories_datasets_stats.ts new file mode 100644 index 0000000000000..c095c7000f031 --- /dev/null +++ b/x-pack/plugins/infra/public/containers/logs/log_analysis/api/get_latest_categories_datasets_stats.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { HttpHandler } from 'src/core/public'; +import { + CategorizerStatus, + getLatestLogEntryCategoryDatasetsStatsRequestPayloadRT, + getLatestLogEntryCategoryDatasetsStatsSuccessResponsePayloadRT, + LogEntryCategoriesDatasetStats, + LOG_ANALYSIS_GET_LATEST_LOG_ENTRY_CATEGORY_DATASETS_STATS_PATH, +} from '../../../../../common/http_api'; +import { decodeOrThrow } from '../../../../../common/runtime_types'; + +export { LogEntryCategoriesDatasetStats }; + +export const callGetLatestCategoriesDatasetsStatsAPI = async ( + { + jobIds, + startTime, + endTime, + includeCategorizerStatuses, + }: { + jobIds: string[]; + startTime: number; + endTime: number; + includeCategorizerStatuses: CategorizerStatus[]; + }, + fetch: HttpHandler +) => { + const response = await fetch(LOG_ANALYSIS_GET_LATEST_LOG_ENTRY_CATEGORY_DATASETS_STATS_PATH, { + method: 'POST', + body: JSON.stringify( + getLatestLogEntryCategoryDatasetsStatsRequestPayloadRT.encode({ + data: { + jobIds, + timeRange: { startTime, endTime }, + includeCategorizerStatuses, + }, + }) + ), + }); + + return decodeOrThrow(getLatestLogEntryCategoryDatasetsStatsSuccessResponsePayloadRT)(response); +}; diff --git a/x-pack/plugins/infra/public/containers/logs/log_analysis/api/ml_get_jobs_summary_api.ts b/x-pack/plugins/infra/public/containers/logs/log_analysis/api/ml_get_jobs_summary_api.ts index dbd75a646b532..7441c0ab7d34c 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_analysis/api/ml_get_jobs_summary_api.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_analysis/api/ml_get_jobs_summary_api.ts @@ -54,6 +54,17 @@ const jobStateRT = rt.keyof({ opening: null, }); +const jobAnalysisConfigRT = rt.partial({ + per_partition_categorization: rt.intersection([ + rt.type({ + enabled: rt.boolean, + }), + rt.partial({ + stop_on_warn: rt.boolean, + }), + ]), +}); + const jobCategorizationStatusRT = rt.keyof({ ok: null, warn: null, @@ -64,6 +75,7 @@ const jobModelSizeStatsRT = rt.type({ categorized_doc_count: rt.number, dead_category_count: rt.number, frequent_category_count: rt.number, + log_time: rt.number, rare_category_count: rt.number, total_category_count: rt.number, }); @@ -79,6 +91,8 @@ export const jobSummaryRT = rt.intersection([ datafeedIndices: rt.array(rt.string), datafeedState: datafeedStateRT, fullJob: rt.partial({ + analysis_config: jobAnalysisConfigRT, + create_time: rt.number, custom_settings: jobCustomSettingsRT, finished_time: rt.number, model_size_stats: jobModelSizeStatsRT, diff --git a/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_types.ts b/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_types.ts index 4930c8b478a9c..ba355ad195b11 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_types.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_types.ts @@ -50,43 +50,3 @@ export interface ModuleSourceConfiguration { spaceId: string; timestampField: string; } - -interface ManyCategoriesWarningReason { - type: 'manyCategories'; - categoriesDocumentRatio: number; -} - -interface ManyDeadCategoriesWarningReason { - type: 'manyDeadCategories'; - deadCategoriesRatio: number; -} - -interface ManyRareCategoriesWarningReason { - type: 'manyRareCategories'; - rareCategoriesRatio: number; -} - -interface NoFrequentCategoriesWarningReason { - type: 'noFrequentCategories'; -} - -interface SingleCategoryWarningReason { - type: 'singleCategory'; -} - -export type CategoryQualityWarningReason = - | ManyCategoriesWarningReason - | ManyDeadCategoriesWarningReason - | ManyRareCategoriesWarningReason - | NoFrequentCategoriesWarningReason - | SingleCategoryWarningReason; - -export type CategoryQualityWarningReasonType = CategoryQualityWarningReason['type']; - -export interface CategoryQualityWarning { - type: 'categoryQualityWarning'; - jobId: string; - reasons: CategoryQualityWarningReason[]; -} - -export type QualityWarning = CategoryQualityWarning; diff --git a/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_categories/use_log_entry_categories_quality.ts b/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_categories/use_log_entry_categories_quality.ts index 346281fa94e1b..6bad94ec49f87 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_categories/use_log_entry_categories_quality.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_categories/use_log_entry_categories_quality.ts @@ -4,43 +4,124 @@ * you may not use this file except in compliance with the Elastic License. */ -import { useMemo } from 'react'; +import { useMemo, useState } from 'react'; +import { useDeepCompareEffect } from 'react-use'; import { - JobModelSizeStats, - JobSummary, - QualityWarning, CategoryQualityWarningReason, -} from '../../log_analysis_module_types'; + QualityWarning, +} from '../../../../../../common/log_analysis'; +import { useKibanaContextForPlugin } from '../../../../../hooks/use_kibana'; +import { useTrackedPromise } from '../../../../../utils/use_tracked_promise'; +import { + callGetLatestCategoriesDatasetsStatsAPI, + LogEntryCategoriesDatasetStats, +} from '../../api/get_latest_categories_datasets_stats'; +import { JobModelSizeStats, JobSummary } from '../../log_analysis_module_types'; export const useLogEntryCategoriesQuality = ({ jobSummaries }: { jobSummaries: JobSummary[] }) => { + const { + services: { + http: { fetch }, + }, + } = useKibanaContextForPlugin(); + + const [lastestWarnedDatasetsStats, setLatestWarnedDatasetsStats] = useState< + LogEntryCategoriesDatasetStats[] + >([]); + + const jobSummariesWithCategoryWarnings = useMemo( + () => jobSummaries.filter(isJobWithCategoryWarnings), + [jobSummaries] + ); + + const jobSummariesWithPartitionedCategoryWarnings = useMemo( + () => jobSummariesWithCategoryWarnings.filter(isJobWithPartitionedCategories), + [jobSummariesWithCategoryWarnings] + ); + + const [fetchLatestWarnedDatasetsStatsRequest, fetchLatestWarnedDatasetsStats] = useTrackedPromise( + { + cancelPreviousOn: 'creation', + createPromise: ( + statsIntervals: Array<{ jobId: string; startTime: number; endTime: number }> + ) => + Promise.all( + statsIntervals.map(({ jobId, startTime, endTime }) => + callGetLatestCategoriesDatasetsStatsAPI( + { jobIds: [jobId], startTime, endTime, includeCategorizerStatuses: ['warn'] }, + fetch + ) + ) + ), + onResolve: (results) => { + setLatestWarnedDatasetsStats(results.flatMap(({ data: { datasetStats } }) => datasetStats)); + }, + }, + [] + ); + + useDeepCompareEffect(() => { + fetchLatestWarnedDatasetsStats( + jobSummariesWithPartitionedCategoryWarnings.map((jobSummary) => ({ + jobId: jobSummary.id, + startTime: jobSummary.fullJob?.create_time ?? 0, + endTime: jobSummary.fullJob?.model_size_stats?.log_time ?? Date.now(), + })) + ); + }, [jobSummariesWithPartitionedCategoryWarnings]); + const categoryQualityWarnings: QualityWarning[] = useMemo( - () => - jobSummaries - .filter( - (jobSummary) => jobSummary.fullJob?.model_size_stats?.categorization_status === 'warn' - ) + () => [ + ...jobSummariesWithCategoryWarnings + .filter((jobSummary) => !isJobWithPartitionedCategories(jobSummary)) .map((jobSummary) => ({ - type: 'categoryQualityWarning', + type: 'categoryQualityWarning' as const, jobId: jobSummary.id, + dataset: '', reasons: jobSummary.fullJob?.model_size_stats ? getCategoryQualityWarningReasons(jobSummary.fullJob.model_size_stats) : [], })), - [jobSummaries] + ...lastestWarnedDatasetsStats.map((datasetStats) => ({ + type: 'categoryQualityWarning' as const, + jobId: datasetStats.job_id, + dataset: datasetStats.dataset, + reasons: getCategoryQualityWarningReasons(datasetStats), + })), + ], + [jobSummariesWithCategoryWarnings, lastestWarnedDatasetsStats] ); return { categoryQualityWarnings, + lastLatestWarnedDatasetsStatsRequestErrors: + fetchLatestWarnedDatasetsStatsRequest.state === 'rejected' + ? fetchLatestWarnedDatasetsStatsRequest.value + : null, + isLoadingCategoryQualityWarnings: fetchLatestWarnedDatasetsStatsRequest.state === 'pending', }; }; +const isJobWithCategoryWarnings = (jobSummary: JobSummary) => + jobSummary.fullJob?.model_size_stats?.categorization_status === 'warn'; + +const isJobWithPartitionedCategories = (jobSummary: JobSummary) => + jobSummary.fullJob?.analysis_config?.per_partition_categorization ?? false; + const getCategoryQualityWarningReasons = ({ categorized_doc_count: categorizedDocCount, dead_category_count: deadCategoryCount, frequent_category_count: frequentCategoryCount, rare_category_count: rareCategoryCount, total_category_count: totalCategoryCount, -}: JobModelSizeStats): CategoryQualityWarningReason[] => { +}: Pick< + JobModelSizeStats, + | 'categorized_doc_count' + | 'dead_category_count' + | 'frequent_category_count' + | 'rare_category_count' + | 'total_category_count' +>): CategoryQualityWarningReason[] => { const rareCategoriesRatio = rareCategoryCount / totalCategoryCount; const categoriesDocumentRatio = totalCategoryCount / categorizedDocCount; const deadCategoriesRatio = deadCategoryCount / totalCategoryCount; diff --git a/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_categories/use_log_entry_categories_setup.tsx b/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_categories/use_log_entry_categories_setup.tsx index 399c30cf47e71..269b64c6f4076 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_categories/use_log_entry_categories_setup.tsx +++ b/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_categories/use_log_entry_categories_setup.tsx @@ -9,7 +9,9 @@ import { useLogEntryCategoriesModuleContext } from './use_log_entry_categories_m export const useLogEntryCategoriesSetup = () => { const { + categoryQualityWarnings, cleanUpAndSetUpModule, + fetchJobStatus, lastSetupErrorMessages, moduleDescriptor, setUpModule, @@ -37,8 +39,10 @@ export const useLogEntryCategoriesSetup = () => { }); return { + categoryQualityWarnings, cleanUpAndSetUp, endTime, + fetchJobStatus, isValidating, lastSetupErrorMessages, moduleDescriptor, diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/page_content.tsx b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/page_content.tsx index 2880b1b794443..b5765942e9f10 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/page_content.tsx +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/page_content.tsx @@ -5,7 +5,7 @@ */ import { i18n } from '@kbn/i18n'; -import React, { useCallback, useEffect, useState } from 'react'; +import React, { useCallback, useEffect } from 'react'; import { isJobStatusWithResults } from '../../../../common/log_analysis'; import { LoadingPage } from '../../../components/loading_page'; import { @@ -14,6 +14,10 @@ import { MissingSetupPrivilegesPrompt, SubscriptionSplashContent, } from '../../../components/logging/log_analysis_setup'; +import { + LogAnalysisSetupFlyout, + useLogAnalysisSetupFlyoutStateContext, +} from '../../../components/logging/log_analysis_setup/setup_flyout'; import { SourceErrorPage } from '../../../components/source_error_page'; import { SourceLoadingPage } from '../../../components/source_loading_page'; import { useLogAnalysisCapabilitiesContext } from '../../../containers/logs/log_analysis'; @@ -21,7 +25,6 @@ import { useLogEntryCategoriesModuleContext } from '../../../containers/logs/log import { useLogSourceContext } from '../../../containers/logs/log_source'; import { LogEntryCategoriesResultsContent } from './page_results_content'; import { LogEntryCategoriesSetupContent } from './page_setup_content'; -import { LogEntryCategoriesSetupFlyout } from './setup_flyout'; export const LogEntryCategoriesPageContent = () => { const { @@ -40,9 +43,10 @@ export const LogEntryCategoriesPageContent = () => { const { fetchJobStatus, setupStatus, jobStatus } = useLogEntryCategoriesModuleContext(); - const [isFlyoutOpen, setIsFlyoutOpen] = useState(false); - const openFlyout = useCallback(() => setIsFlyoutOpen(true), []); - const closeFlyout = useCallback(() => setIsFlyoutOpen(false), []); + const { showModuleSetup } = useLogAnalysisSetupFlyoutStateContext(); + const showCategoriesModuleSetup = useCallback(() => showModuleSetup('logs_ui_categories'), [ + showModuleSetup, + ]); useEffect(() => { if (hasLogAnalysisReadCapabilities) { @@ -71,8 +75,8 @@ export const LogEntryCategoriesPageContent = () => { } else if (isJobStatusWithResults(jobStatus['log-entry-categories-count'])) { return ( <> - - + + ); } else if (!hasLogAnalysisSetupCapabilities) { @@ -80,9 +84,11 @@ export const LogEntryCategoriesPageContent = () => { } else { return ( <> - - + + ); } }; + +const allowedSetupModules = ['logs_ui_categories' as const]; diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/page_providers.tsx b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/page_providers.tsx index 723d833799e29..7d2f1d5418bc5 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/page_providers.tsx +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/page_providers.tsx @@ -5,6 +5,7 @@ */ import React from 'react'; +import { LogAnalysisSetupFlyoutStateProvider } from '../../../components/logging/log_analysis_setup/setup_flyout'; import { LogEntryCategoriesModuleProvider } from '../../../containers/logs/log_analysis/modules/log_entry_categories'; import { useLogSourceContext } from '../../../containers/logs/log_source'; import { useActiveKibanaSpace } from '../../../hooks/use_kibana_space'; @@ -27,7 +28,7 @@ export const LogEntryCategoriesPageProviders: React.FunctionComponent = ({ child spaceId={space.id} timestampField={sourceConfiguration.configuration.fields.timestamp} > - {children} + {children} ); }; diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/setup_flyout.tsx b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/setup_flyout.tsx deleted file mode 100644 index a038765de2bf3..0000000000000 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/setup_flyout.tsx +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { - EuiFlyout, - EuiFlyoutBody, - EuiFlyoutHeader, - EuiSpacer, - EuiSteps, - EuiText, - EuiTitle, -} from '@elastic/eui'; -import { FormattedMessage } from '@kbn/i18n/react'; -import React, { useCallback, useMemo } from 'react'; -import { - createInitialConfigurationStep, - createProcessStep, -} from '../../../components/logging/log_analysis_setup'; -import { useLogEntryCategoriesSetup } from '../../../containers/logs/log_analysis/modules/log_entry_categories'; - -interface LogEntryCategoriesSetupFlyoutProps { - isOpen: boolean; - onClose: () => void; -} - -export const LogEntryCategoriesSetupFlyout: React.FC = ({ - isOpen, - onClose, -}) => { - const { - cleanUpAndSetUp, - endTime, - isValidating, - lastSetupErrorMessages, - setEndTime, - setStartTime, - setValidatedIndices, - setUp, - setupStatus, - startTime, - validatedIndices, - validationErrors, - viewResults, - } = useLogEntryCategoriesSetup(); - - const viewResultsAndClose = useCallback(() => { - viewResults(); - onClose(); - }, [viewResults, onClose]); - - const steps = useMemo( - () => [ - createInitialConfigurationStep({ - setStartTime, - setEndTime, - startTime, - endTime, - isValidating, - validatedIndices, - setupStatus, - setValidatedIndices, - validationErrors, - }), - createProcessStep({ - cleanUpAndSetUp, - errorMessages: lastSetupErrorMessages, - isConfigurationValid: validationErrors.length <= 0 && !isValidating, - setUp, - setupStatus, - viewResults: viewResultsAndClose, - }), - ], - [ - cleanUpAndSetUp, - endTime, - isValidating, - lastSetupErrorMessages, - setEndTime, - setStartTime, - setUp, - setValidatedIndices, - setupStatus, - startTime, - validatedIndices, - validationErrors, - viewResultsAndClose, - ] - ); - - if (!isOpen) { - return null; - } - return ( - - - -

- -

- - - - -

- -

- - - - - - - - - ); -}; diff --git a/x-pack/plugins/infra/server/infra_server.ts b/x-pack/plugins/infra/server/infra_server.ts index 206fffdd2e188..1d89b7be43296 100644 --- a/x-pack/plugins/infra/server/infra_server.ts +++ b/x-pack/plugins/infra/server/infra_server.ts @@ -13,6 +13,7 @@ import { InfraBackendLibs } from './lib/infra_types'; import { initGetLogEntryCategoriesRoute, initGetLogEntryCategoryDatasetsRoute, + initGetLogEntryCategoryDatasetsStatsRoute, initGetLogEntryCategoryExamplesRoute, initGetLogEntryRateRoute, initGetLogEntryExamplesRoute, @@ -54,6 +55,7 @@ export const initInfraServer = (libs: InfraBackendLibs) => { initIpToHostName(libs); initGetLogEntryCategoriesRoute(libs); initGetLogEntryCategoryDatasetsRoute(libs); + initGetLogEntryCategoryDatasetsStatsRoute(libs); initGetLogEntryCategoryExamplesRoute(libs); initGetLogEntryRateRoute(libs); initGetLogEntryAnomaliesRoute(libs); diff --git a/x-pack/plugins/infra/server/lib/log_analysis/common.ts b/x-pack/plugins/infra/server/lib/log_analysis/common.ts index 4d2be94c7cd62..7e4a714a47d1f 100644 --- a/x-pack/plugins/infra/server/lib/log_analysis/common.ts +++ b/x-pack/plugins/infra/server/lib/log_analysis/common.ts @@ -36,7 +36,7 @@ export async function fetchMlJob(mlAnomalyDetectors: MlAnomalyDetectors, jobId: }; } -const COMPOSITE_AGGREGATION_BATCH_SIZE = 1000; +export const COMPOSITE_AGGREGATION_BATCH_SIZE = 1000; // Finds datasets related to ML job ids export async function getLogEntryDatasets( diff --git a/x-pack/plugins/infra/server/lib/log_analysis/index.ts b/x-pack/plugins/infra/server/lib/log_analysis/index.ts index c9a176be0a28f..bb571a8edf39b 100644 --- a/x-pack/plugins/infra/server/lib/log_analysis/index.ts +++ b/x-pack/plugins/infra/server/lib/log_analysis/index.ts @@ -6,5 +6,6 @@ export * from './errors'; export * from './log_entry_categories_analysis'; +export * from './log_entry_categories_datasets_stats'; export * from './log_entry_rate_analysis'; export * from './log_entry_anomalies'; diff --git a/x-pack/plugins/infra/server/lib/log_analysis/log_entry_categories_datasets_stats.ts b/x-pack/plugins/infra/server/lib/log_analysis/log_entry_categories_datasets_stats.ts new file mode 100644 index 0000000000000..ec5f3c88dff2a --- /dev/null +++ b/x-pack/plugins/infra/server/lib/log_analysis/log_entry_categories_datasets_stats.ts @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { startTracingSpan } from '../../../common/performance_tracing'; +import { decodeOrThrow } from '../../../common/runtime_types'; +import type { MlAnomalyDetectors, MlSystem } from '../../types'; +import { COMPOSITE_AGGREGATION_BATCH_SIZE } from './common'; +import { + CompositeDatasetKey, + createLatestLogEntryCategoriesDatasetsStatsQuery, + latestLogEntryCategoriesDatasetsStatsResponseRT, + LogEntryCategoryDatasetStatsBucket, +} from './queries/latest_log_entry_categories_datasets_stats'; + +export async function getLatestLogEntriesCategoriesDatasetsStats( + context: { + infra: { + mlAnomalyDetectors: MlAnomalyDetectors; + mlSystem: MlSystem; + }; + }, + jobIds: string[], + startTime: number, + endTime: number, + includeCategorizerStatuses: Array<'ok' | 'warn'> = [] +) { + const finalizeLogEntryCategoriesDatasetsStats = startTracingSpan('get categories datasets stats'); + + let latestLogEntryCategoriesDatasetsStatsBuckets: LogEntryCategoryDatasetStatsBucket[] = []; + let afterLatestBatchKey: CompositeDatasetKey | undefined; + + while (true) { + const latestLogEntryCategoriesDatasetsStatsResponse = await context.infra.mlSystem.mlAnomalySearch( + createLatestLogEntryCategoriesDatasetsStatsQuery( + jobIds, + startTime, + endTime, + COMPOSITE_AGGREGATION_BATCH_SIZE, + afterLatestBatchKey + ) + ); + + const { after_key: afterKey, buckets: latestBatchBuckets = [] } = + decodeOrThrow(latestLogEntryCategoriesDatasetsStatsResponseRT)( + latestLogEntryCategoriesDatasetsStatsResponse + ).aggregations?.dataset_composite_terms ?? {}; + + const latestIncludedBatchBuckets = + includeCategorizerStatuses.length > 0 + ? latestBatchBuckets.filter((bucket) => + bucket.categorizer_stats_top_hits.hits.hits.some((hit) => + includeCategorizerStatuses.includes(hit._source.categorization_status) + ) + ) + : latestBatchBuckets; + + latestLogEntryCategoriesDatasetsStatsBuckets = [ + ...latestLogEntryCategoriesDatasetsStatsBuckets, + ...latestIncludedBatchBuckets, + ]; + + afterLatestBatchKey = afterKey; + if (afterKey == null || latestBatchBuckets.length < COMPOSITE_AGGREGATION_BATCH_SIZE) { + break; + } + } + + const logEntryCategoriesDatasetsStatsSpan = finalizeLogEntryCategoriesDatasetsStats(); + + return { + data: latestLogEntryCategoriesDatasetsStatsBuckets.map((bucket) => { + const latestHitSource = bucket.categorizer_stats_top_hits.hits.hits[0]._source; + + return { + categorization_status: latestHitSource.categorization_status, + categorized_doc_count: latestHitSource.categorized_doc_count, + dataset: bucket.key.dataset ?? '', + dead_category_count: latestHitSource.dead_category_count, + failed_category_count: latestHitSource.failed_category_count, + frequent_category_count: latestHitSource.frequent_category_count, + job_id: latestHitSource.job_id, + log_time: latestHitSource.log_time, + rare_category_count: latestHitSource.rare_category_count, + total_category_count: latestHitSource.total_category_count, + }; + }), + timing: { + spans: [logEntryCategoriesDatasetsStatsSpan], + }, + }; +} diff --git a/x-pack/plugins/infra/server/lib/log_analysis/queries/common.ts b/x-pack/plugins/infra/server/lib/log_analysis/queries/common.ts index 63e39ef022392..bb1a1969e99eb 100644 --- a/x-pack/plugins/infra/server/lib/log_analysis/queries/common.ts +++ b/x-pack/plugins/infra/server/lib/log_analysis/queries/common.ts @@ -40,7 +40,20 @@ export const createTimeRangeFilters = (startTime: number, endTime: number) => [ }, ]; -export const createResultTypeFilters = (resultTypes: Array<'model_plot' | 'record'>) => [ +export const createLogTimeRangeFilters = (startTime: number, endTime: number) => [ + { + range: { + log_time: { + gte: startTime, + lte: endTime, + }, + }, + }, +]; + +export const createResultTypeFilters = ( + resultTypes: Array<'categorizer_stats' | 'model_plot' | 'record'> +) => [ { terms: { result_type: resultTypes, diff --git a/x-pack/plugins/infra/server/lib/log_analysis/queries/latest_log_entry_categories_datasets_stats.ts b/x-pack/plugins/infra/server/lib/log_analysis/queries/latest_log_entry_categories_datasets_stats.ts new file mode 100644 index 0000000000000..b9224e8125a48 --- /dev/null +++ b/x-pack/plugins/infra/server/lib/log_analysis/queries/latest_log_entry_categories_datasets_stats.ts @@ -0,0 +1,133 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as rt from 'io-ts'; +import { commonSearchSuccessResponseFieldsRT } from '../../../utils/elasticsearch_runtime_types'; +import { + createJobIdsFilters, + createResultTypeFilters, + defaultRequestParameters, + createLogTimeRangeFilters, +} from './common'; + +export const createLatestLogEntryCategoriesDatasetsStatsQuery = ( + logEntryCategoriesJobIds: string[], + startTime: number, + endTime: number, + size: number, + afterKey?: CompositeDatasetKey +) => ({ + ...defaultRequestParameters, + body: { + query: { + bool: { + filter: [ + ...createJobIdsFilters(logEntryCategoriesJobIds), + ...createResultTypeFilters(['categorizer_stats']), + ...createLogTimeRangeFilters(startTime, endTime), + ], + }, + }, + aggregations: { + dataset_composite_terms: { + composite: { + after: afterKey, + size, + sources: [ + { + dataset: { + terms: { + field: 'partition_field_value', + missing_bucket: true, + }, + }, + }, + ], + }, + aggs: { + categorizer_stats_top_hits: { + top_hits: { + size: 1, + sort: [ + { + log_time: 'desc', + }, + ], + _source: [ + 'categorization_status', + 'categorized_doc_count', + 'dead_category_count', + 'failed_category_count', + 'frequent_category_count', + 'job_id', + 'log_time', + 'rare_category_count', + 'total_category_count', + ], + }, + }, + }, + }, + }, + }, + size: 0, +}); + +export const logEntryCategoryStatusRT = rt.keyof({ + ok: null, + warn: null, +}); + +export const logEntryCategorizerStatsHitRT = rt.type({ + _source: rt.type({ + categorization_status: logEntryCategoryStatusRT, + categorized_doc_count: rt.number, + dead_category_count: rt.number, + failed_category_count: rt.number, + frequent_category_count: rt.number, + job_id: rt.string, + log_time: rt.number, + rare_category_count: rt.number, + total_category_count: rt.number, + }), +}); + +export type LogEntryCategorizerStatsHit = rt.TypeOf; + +const compositeDatasetKeyRT = rt.type({ + dataset: rt.union([rt.string, rt.null]), +}); + +export type CompositeDatasetKey = rt.TypeOf; + +const logEntryCategoryDatasetStatsBucketRT = rt.type({ + key: compositeDatasetKeyRT, + categorizer_stats_top_hits: rt.type({ + hits: rt.type({ + hits: rt.array(logEntryCategorizerStatsHitRT), + }), + }), +}); + +export type LogEntryCategoryDatasetStatsBucket = rt.TypeOf< + typeof logEntryCategoryDatasetStatsBucketRT +>; + +export const latestLogEntryCategoriesDatasetsStatsResponseRT = rt.intersection([ + commonSearchSuccessResponseFieldsRT, + rt.partial({ + aggregations: rt.type({ + dataset_composite_terms: rt.type({ + after_key: compositeDatasetKeyRT, + buckets: rt.array(logEntryCategoryDatasetStatsBucketRT), + }), + }), + }), +]); + +export type LatestLogEntryCategoriesDatasetsStatsResponse = rt.TypeOf< + typeof latestLogEntryCategoriesDatasetsStatsResponseRT +>; diff --git a/x-pack/plugins/infra/server/routes/log_analysis/results/index.ts b/x-pack/plugins/infra/server/routes/log_analysis/results/index.ts index a01042616a872..e696477253823 100644 --- a/x-pack/plugins/infra/server/routes/log_analysis/results/index.ts +++ b/x-pack/plugins/infra/server/routes/log_analysis/results/index.ts @@ -6,6 +6,7 @@ export * from './log_entry_categories'; export * from './log_entry_category_datasets'; +export * from './log_entry_category_datasets_stats'; export * from './log_entry_category_examples'; export * from './log_entry_rate'; export * from './log_entry_examples'; diff --git a/x-pack/plugins/infra/server/routes/log_analysis/results/log_entry_category_datasets_stats.ts b/x-pack/plugins/infra/server/routes/log_analysis/results/log_entry_category_datasets_stats.ts new file mode 100644 index 0000000000000..8414fc2062ae9 --- /dev/null +++ b/x-pack/plugins/infra/server/routes/log_analysis/results/log_entry_category_datasets_stats.ts @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import Boom from 'boom'; +import { + getLatestLogEntryCategoryDatasetsStatsRequestPayloadRT, + getLatestLogEntryCategoryDatasetsStatsSuccessResponsePayloadRT, + LOG_ANALYSIS_GET_LATEST_LOG_ENTRY_CATEGORY_DATASETS_STATS_PATH, +} from '../../../../common/http_api/log_analysis'; +import { createValidationFunction } from '../../../../common/runtime_types'; +import type { InfraBackendLibs } from '../../../lib/infra_types'; +import { getLatestLogEntriesCategoriesDatasetsStats } from '../../../lib/log_analysis'; +import { isMlPrivilegesError } from '../../../lib/log_analysis/errors'; +import { assertHasInfraMlPlugins } from '../../../utils/request_context'; + +export const initGetLogEntryCategoryDatasetsStatsRoute = ({ framework }: InfraBackendLibs) => { + framework.registerRoute( + { + method: 'post', + path: LOG_ANALYSIS_GET_LATEST_LOG_ENTRY_CATEGORY_DATASETS_STATS_PATH, + validate: { + body: createValidationFunction(getLatestLogEntryCategoryDatasetsStatsRequestPayloadRT), + }, + }, + framework.router.handleLegacyErrors(async (requestContext, request, response) => { + const { + data: { + jobIds, + timeRange: { startTime, endTime }, + includeCategorizerStatuses, + }, + } = request.body; + + try { + assertHasInfraMlPlugins(requestContext); + + const { data: datasetStats, timing } = await getLatestLogEntriesCategoriesDatasetsStats( + requestContext, + jobIds, + startTime, + endTime, + includeCategorizerStatuses + ); + + return response.ok({ + body: getLatestLogEntryCategoryDatasetsStatsSuccessResponsePayloadRT.encode({ + data: { + datasetStats, + }, + timing, + }), + }); + } catch (error) { + if (Boom.isBoom(error)) { + throw error; + } + + if (isMlPrivilegesError(error)) { + return response.customError({ + statusCode: 403, + body: { + message: error.message, + }, + }); + } + + return response.customError({ + statusCode: error.statusCode ?? 500, + body: { + message: error.message ?? 'An unexpected error occurred', + }, + }); + } + }) + ); +}; diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/logs_ui_categories/ml/log_entry_categories_count.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/logs_ui_categories/ml/log_entry_categories_count.json index b4fb242f16522..40c47352371d4 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/logs_ui_categories/ml/log_entry_categories_count.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/logs_ui_categories/ml/log_entry_categories_count.json @@ -14,7 +14,11 @@ "use_null": true } ], - "influencers": ["event.dataset", "mlcategory"] + "influencers": ["event.dataset", "mlcategory"], + "per_partition_categorization": { + "enabled": true, + "stop_on_warn": false + } }, "analysis_limits": { "model_memory_limit": "100mb", @@ -29,6 +33,6 @@ }, "custom_settings": { "created_by": "ml-module-logs-ui-categories", - "job_revision": 0 + "job_revision": 1 } } diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index ed66d56d552a5..7c34f7d5c84f7 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -8487,9 +8487,6 @@ "xpack.infra.logs.search.searchInLogsAriaLabel": "検索", "xpack.infra.logs.search.searchInLogsPlaceholder": "検索", "xpack.infra.logs.searchResultTooltip": "{bucketCount, plural, one {# 件のハイライトされたエントリー} other {# 件のハイライトされたエントリー}}", - "xpack.infra.logs.setupFlyout.logCategoriesDescription": "機械学習を使用して、ログメッセージを自動的に分類します。", - "xpack.infra.logs.setupFlyout.logCategoriesTitle": "ログカテゴリー", - "xpack.infra.logs.setupFlyout.setupFlyoutTitle": "機械学習を使用した異常検知", "xpack.infra.logs.showingEntriesFromTimestamp": "{timestamp} 以降のエントリーを表示中", "xpack.infra.logs.showingEntriesUntilTimestamp": "{timestamp} までのエントリーを表示中", "xpack.infra.logs.startStreamingButtonLabel": "ライブストリーム", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 103ff4ab146a4..c6345ac6955c4 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -8492,9 +8492,6 @@ "xpack.infra.logs.search.searchInLogsAriaLabel": "搜索", "xpack.infra.logs.search.searchInLogsPlaceholder": "搜索", "xpack.infra.logs.searchResultTooltip": "{bucketCount, plural, one {# 个高亮条目} other {# 个高亮条目}}", - "xpack.infra.logs.setupFlyout.logCategoriesDescription": "使用 Machine Learning 自动归类日志消息。", - "xpack.infra.logs.setupFlyout.logCategoriesTitle": "日志类别", - "xpack.infra.logs.setupFlyout.setupFlyoutTitle": "通过 Machine Learning 检测异常", "xpack.infra.logs.showingEntriesFromTimestamp": "正在显示自 {timestamp} 起的条目", "xpack.infra.logs.showingEntriesUntilTimestamp": "正在显示截止于 {timestamp} 的条目", "xpack.infra.logs.startStreamingButtonLabel": "实时流式传输",