diff --git a/x-pack/packages/ml/agg_utils/src/types.ts b/x-pack/packages/ml/agg_utils/src/types.ts index 026daf861058f..d8e76239d0e09 100644 --- a/x-pack/packages/ml/agg_utils/src/types.ts +++ b/x-pack/packages/ml/agg_utils/src/types.ts @@ -149,17 +149,10 @@ export interface SignificantTerm extends FieldValuePair { unique?: boolean; } -/** - * Represents a data item in a significant term histogram. - * @interface - */ -export interface SignificantTermHistogramItem { +interface SignificantTermHistogramItemBase { /** The document count for this item in the overall context. */ doc_count_overall: number; - /** The document count for this item in the significant term context. */ - doc_count_significant_term: number; - /** The numeric key associated with this item. */ key: number; @@ -167,6 +160,26 @@ export interface SignificantTermHistogramItem { key_as_string: string; } +/** + * @deprecated since version 2 of internal log rate analysis REST API endpoint + */ +interface SignificantTermHistogramItemV1 extends SignificantTermHistogramItemBase { + /** The document count for this item in the significant term context. */ + doc_count_significant_term: number; +} + +interface SignificantTermHistogramItemV2 extends SignificantTermHistogramItemBase { + /** The document count for this histogram item in the significant item context. */ + doc_count_significant_item: number; +} + +/** + * Represents a data item in a significant term histogram. + */ +export type SignificantTermHistogramItem = + | SignificantTermHistogramItemV1 + | SignificantTermHistogramItemV2; + /** * Represents histogram data for a field/value pair. * @interface diff --git a/x-pack/plugins/aiops/common/api/index.ts b/x-pack/plugins/aiops/common/api/index.ts index 42f1cfa512900..cd1852299f265 100644 --- a/x-pack/plugins/aiops/common/api/index.ts +++ b/x-pack/plugins/aiops/common/api/index.ts @@ -5,14 +5,6 @@ * 2.0. */ -import type { HttpSetup } from '@kbn/core/public'; - -import type { - AiopsLogRateAnalysisSchema, - AiopsLogRateAnalysisApiAction, -} from './log_rate_analysis'; -import { streamReducer } from './stream_reducer'; - export const AIOPS_API_ENDPOINT = { LOG_RATE_ANALYSIS: '/internal/aiops/log_rate_analysis', CATEGORIZATION_FIELD_VALIDATION: '/internal/aiops/categorization_field_validation', @@ -20,12 +12,3 @@ export const AIOPS_API_ENDPOINT = { type AiopsApiEndpointKeys = keyof typeof AIOPS_API_ENDPOINT; export type AiopsApiEndpoint = typeof AIOPS_API_ENDPOINT[AiopsApiEndpointKeys]; - -export interface AiopsApiLogRateAnalysis { - http: HttpSetup; - endpoint: AiopsApiEndpoint; - apiVersion: string; - reducer: typeof streamReducer; - body: AiopsLogRateAnalysisSchema; - actions: AiopsLogRateAnalysisApiAction; -} diff --git a/x-pack/plugins/aiops/common/api/log_rate_analysis/actions.ts b/x-pack/plugins/aiops/common/api/log_rate_analysis/actions.ts index b0ab5ae260955..4075ddffd27f1 100644 --- a/x-pack/plugins/aiops/common/api/log_rate_analysis/actions.ts +++ b/x-pack/plugins/aiops/common/api/log_rate_analysis/actions.ts @@ -12,10 +12,24 @@ import type { SignificantTermGroupHistogram, } from '@kbn/ml-agg-utils'; +import type { AiopsLogRateAnalysisApiVersion as ApiVersion } from './schema'; + export const API_ACTION_NAME = { + /** @since API v2 */ + ADD_SIGNIFICANT_ITEMS: 'add_significant_items', + /** @since API v2 */ + ADD_SIGNIFICANT_ITEMS_HISTOGRAM: 'add_significant_items_histogram', + /** @since API v2 */ + ADD_SIGNIFICANT_ITEMS_GROUP: 'add_significant_items_group', + /** @since API v2 */ + ADD_SIGNIFICANT_ITEMS_GROUP_HISTOGRAM: 'add_significant_items_group_histogram', + /** @deprecated since API v2 */ ADD_SIGNIFICANT_TERMS: 'add_significant_terms', + /** @deprecated since API v2 */ ADD_SIGNIFICANT_TERMS_HISTOGRAM: 'add_significant_terms_histogram', + /** @deprecated since API v2 */ ADD_SIGNIFICANT_TERMS_GROUP: 'add_significant_terms_group', + /** @deprecated since API v2 */ ADD_SIGNIFICANT_TERMS_GROUP_HISTOGRAM: 'add_significant_terms_group_histogram', ADD_ERROR: 'add_error', PING: 'ping', @@ -26,60 +40,108 @@ export const API_ACTION_NAME = { } as const; export type ApiActionName = typeof API_ACTION_NAME[keyof typeof API_ACTION_NAME]; -interface ApiActionAddSignificantTerms { - type: typeof API_ACTION_NAME.ADD_SIGNIFICANT_TERMS; +interface ApiActionAddSignificantTerms { + type: T extends '1' + ? typeof API_ACTION_NAME.ADD_SIGNIFICANT_TERMS + : T extends '2' + ? typeof API_ACTION_NAME.ADD_SIGNIFICANT_ITEMS + : never; payload: SignificantTerm[]; } -export function addSignificantTermsAction( - payload: ApiActionAddSignificantTerms['payload'] -): ApiActionAddSignificantTerms { +export function addSignificantTermsAction( + payload: ApiActionAddSignificantTerms['payload'], + version: T +): ApiActionAddSignificantTerms { + if (version === '1') { + return { + type: API_ACTION_NAME.ADD_SIGNIFICANT_TERMS, + payload, + } as ApiActionAddSignificantTerms; + } + return { - type: API_ACTION_NAME.ADD_SIGNIFICANT_TERMS, + type: API_ACTION_NAME.ADD_SIGNIFICANT_ITEMS, payload, - }; + } as ApiActionAddSignificantTerms; } -interface ApiActionAddSignificantTermsHistogram { - type: typeof API_ACTION_NAME.ADD_SIGNIFICANT_TERMS_HISTOGRAM; +interface ApiActionAddSignificantTermsHistogram { + type: T extends '1' + ? typeof API_ACTION_NAME.ADD_SIGNIFICANT_TERMS_HISTOGRAM + : T extends '2' + ? typeof API_ACTION_NAME.ADD_SIGNIFICANT_ITEMS_HISTOGRAM + : never; payload: SignificantTermHistogram[]; } -export function addSignificantTermsHistogramAction( - payload: ApiActionAddSignificantTermsHistogram['payload'] -): ApiActionAddSignificantTermsHistogram { +export function addSignificantTermsHistogramAction( + payload: ApiActionAddSignificantTermsHistogram['payload'], + version: T +): ApiActionAddSignificantTermsHistogram { + if (version === '1') { + return { + type: API_ACTION_NAME.ADD_SIGNIFICANT_TERMS_HISTOGRAM, + payload, + } as ApiActionAddSignificantTermsHistogram; + } + return { - type: API_ACTION_NAME.ADD_SIGNIFICANT_TERMS_HISTOGRAM, + type: API_ACTION_NAME.ADD_SIGNIFICANT_ITEMS_HISTOGRAM, payload, - }; + } as ApiActionAddSignificantTermsHistogram; } -interface ApiActionAddSignificantTermsGroup { - type: typeof API_ACTION_NAME.ADD_SIGNIFICANT_TERMS_GROUP; +interface ApiActionAddSignificantTermsGroup { + type: T extends '1' + ? typeof API_ACTION_NAME.ADD_SIGNIFICANT_TERMS_GROUP + : T extends '2' + ? typeof API_ACTION_NAME.ADD_SIGNIFICANT_ITEMS_GROUP + : never; payload: SignificantTermGroup[]; } -export function addSignificantTermsGroupAction( - payload: ApiActionAddSignificantTermsGroup['payload'] -) { +export function addSignificantTermsGroupAction( + payload: ApiActionAddSignificantTermsGroup['payload'], + version: T +): ApiActionAddSignificantTermsGroup { + if (version === '1') { + return { + type: API_ACTION_NAME.ADD_SIGNIFICANT_TERMS_GROUP, + payload, + } as ApiActionAddSignificantTermsGroup; + } + return { - type: API_ACTION_NAME.ADD_SIGNIFICANT_TERMS_GROUP, + type: API_ACTION_NAME.ADD_SIGNIFICANT_ITEMS_GROUP, payload, - }; + } as ApiActionAddSignificantTermsGroup; } -interface ApiActionAddSignificantTermsGroupHistogram { - type: typeof API_ACTION_NAME.ADD_SIGNIFICANT_TERMS_GROUP_HISTOGRAM; +interface ApiActionAddSignificantTermsGroupHistogram { + type: T extends '1' + ? typeof API_ACTION_NAME.ADD_SIGNIFICANT_TERMS_GROUP_HISTOGRAM + : T extends '2' + ? typeof API_ACTION_NAME.ADD_SIGNIFICANT_ITEMS_GROUP_HISTOGRAM + : never; payload: SignificantTermGroupHistogram[]; } -export function addSignificantTermsGroupHistogramAction( - payload: ApiActionAddSignificantTermsGroupHistogram['payload'] -): ApiActionAddSignificantTermsGroupHistogram { +export function addSignificantTermsGroupHistogramAction( + payload: ApiActionAddSignificantTermsGroupHistogram['payload'], + version: T +): ApiActionAddSignificantTermsGroupHistogram { + if (version === '1') { + return { + type: API_ACTION_NAME.ADD_SIGNIFICANT_TERMS_GROUP_HISTOGRAM, + payload, + } as ApiActionAddSignificantTermsGroupHistogram; + } + return { - type: API_ACTION_NAME.ADD_SIGNIFICANT_TERMS_GROUP_HISTOGRAM, + type: API_ACTION_NAME.ADD_SIGNIFICANT_ITEMS_GROUP_HISTOGRAM, payload, - }; + } as ApiActionAddSignificantTermsGroupHistogram; } interface ApiActionAddError { @@ -148,11 +210,11 @@ export function updateLoadingStateAction( }; } -export type AiopsLogRateAnalysisApiAction = - | ApiActionAddSignificantTerms - | ApiActionAddSignificantTermsGroup - | ApiActionAddSignificantTermsHistogram - | ApiActionAddSignificantTermsGroupHistogram +export type AiopsLogRateAnalysisApiAction = + | ApiActionAddSignificantTerms + | ApiActionAddSignificantTermsGroup + | ApiActionAddSignificantTermsHistogram + | ApiActionAddSignificantTermsGroupHistogram | ApiActionAddError | ApiActionPing | ApiActionResetAll diff --git a/x-pack/plugins/aiops/common/api/log_rate_analysis/index.ts b/x-pack/plugins/aiops/common/api/log_rate_analysis/index.ts deleted file mode 100644 index 4687bb8872840..0000000000000 --- a/x-pack/plugins/aiops/common/api/log_rate_analysis/index.ts +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export { - addSignificantTermsAction, - addSignificantTermsGroupAction, - addSignificantTermsGroupHistogramAction, - addSignificantTermsHistogramAction, - addErrorAction, - pingAction, - resetAllAction, - resetErrorsAction, - resetGroupsAction, - updateLoadingStateAction, - API_ACTION_NAME, -} from './actions'; -export type { AiopsLogRateAnalysisApiAction } from './actions'; - -export { aiopsLogRateAnalysisSchema } from './schema'; -export type { AiopsLogRateAnalysisSchema } from './schema'; diff --git a/x-pack/plugins/aiops/common/api/log_rate_analysis/schema.ts b/x-pack/plugins/aiops/common/api/log_rate_analysis/schema.ts index 499b61c5ba5ca..a7fbcfd303b55 100644 --- a/x-pack/plugins/aiops/common/api/log_rate_analysis/schema.ts +++ b/x-pack/plugins/aiops/common/api/log_rate_analysis/schema.ts @@ -5,37 +5,17 @@ * 2.0. */ -import { schema, TypeOf } from '@kbn/config-schema'; +import type { AiopsLogRateAnalysisSchemaV1 } from './schema_v1'; +import type { AiopsLogRateAnalysisSchemaV2 } from './schema_v2'; -export const aiopsLogRateAnalysisSchema = schema.object({ - start: schema.number(), - end: schema.number(), - searchQuery: schema.string(), - timeFieldName: schema.string(), - includeFrozen: schema.maybe(schema.boolean()), - grouping: schema.maybe(schema.boolean()), - /** Analysis selection time ranges */ - baselineMin: schema.number(), - baselineMax: schema.number(), - deviationMin: schema.number(), - deviationMax: schema.number(), - /** The index to query for log rate analysis */ - index: schema.string(), - /** Settings to override headers derived compression and flush fix */ - compressResponse: schema.maybe(schema.boolean()), - flushFix: schema.maybe(schema.boolean()), - /** Overrides to skip steps of the analysis with existing data */ - overrides: schema.maybe( - schema.object({ - loaded: schema.maybe(schema.number()), - remainingFieldCandidates: schema.maybe(schema.arrayOf(schema.string())), - // TODO Improve schema - significantTerms: schema.maybe(schema.arrayOf(schema.any())), - regroupOnly: schema.maybe(schema.boolean()), - }) - ), - /** Probability used for the random sampler aggregations */ - sampleProbability: schema.maybe(schema.number()), -}); +export type AiopsLogRateAnalysisApiVersion = '1' | '2'; -export type AiopsLogRateAnalysisSchema = TypeOf; +const LATEST_API_VERSION: AiopsLogRateAnalysisApiVersion = '2'; + +export type AiopsLogRateAnalysisSchema< + T extends AiopsLogRateAnalysisApiVersion = typeof LATEST_API_VERSION +> = T extends '1' + ? AiopsLogRateAnalysisSchemaV1 + : T extends '2' + ? AiopsLogRateAnalysisSchemaV2 + : never; diff --git a/x-pack/plugins/aiops/common/api/log_rate_analysis/schema_v1.ts b/x-pack/plugins/aiops/common/api/log_rate_analysis/schema_v1.ts new file mode 100644 index 0000000000000..e994ecc33dafe --- /dev/null +++ b/x-pack/plugins/aiops/common/api/log_rate_analysis/schema_v1.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema, TypeOf } from '@kbn/config-schema'; + +export const aiopsLogRateAnalysisSchemaV1 = schema.object({ + start: schema.number(), + end: schema.number(), + searchQuery: schema.string(), + timeFieldName: schema.string(), + includeFrozen: schema.maybe(schema.boolean()), + grouping: schema.maybe(schema.boolean()), + /** Analysis selection time ranges */ + baselineMin: schema.number(), + baselineMax: schema.number(), + deviationMin: schema.number(), + deviationMax: schema.number(), + /** The index to query for log rate analysis */ + index: schema.string(), + /** Settings to override headers derived compression and flush fix */ + compressResponse: schema.maybe(schema.boolean()), + flushFix: schema.maybe(schema.boolean()), + /** Overrides to skip steps of the analysis with existing data */ + overrides: schema.maybe( + schema.object({ + loaded: schema.maybe(schema.number()), + remainingFieldCandidates: schema.maybe(schema.arrayOf(schema.string())), + // TODO Improve schema + significantTerms: schema.maybe(schema.arrayOf(schema.any())), + regroupOnly: schema.maybe(schema.boolean()), + }) + ), + /** Probability used for the random sampler aggregations */ + sampleProbability: schema.maybe(schema.number()), +}); + +export type AiopsLogRateAnalysisSchemaV1 = TypeOf; diff --git a/x-pack/plugins/aiops/common/api/log_rate_analysis/schema_v2.ts b/x-pack/plugins/aiops/common/api/log_rate_analysis/schema_v2.ts new file mode 100644 index 0000000000000..2438c04971e91 --- /dev/null +++ b/x-pack/plugins/aiops/common/api/log_rate_analysis/schema_v2.ts @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema, TypeOf } from '@kbn/config-schema'; + +const significantItem = schema.object({ + key: schema.string(), + type: schema.oneOf([schema.literal('keyword'), schema.literal('log_pattern')]), + fieldName: schema.string(), + fieldValue: schema.oneOf([schema.string(), schema.number()]), + doc_count: schema.number(), + bg_count: schema.number(), + total_doc_count: schema.number(), + total_bg_count: schema.number(), + score: schema.number(), + pValue: schema.nullable(schema.number()), + normalizedScore: schema.number(), + histogram: schema.maybe( + schema.arrayOf( + schema.object({ + doc_count_overall: schema.number(), + doc_count_significant_item: schema.number(), + key: schema.number(), + key_as_string: schema.string(), + }) + ) + ), + unique: schema.maybe(schema.boolean()), +}); + +export const aiopsLogRateAnalysisSchemaV2 = schema.object({ + start: schema.number(), + end: schema.number(), + searchQuery: schema.string(), + timeFieldName: schema.string(), + includeFrozen: schema.maybe(schema.boolean()), + grouping: schema.maybe(schema.boolean()), + /** Analysis selection time ranges */ + baselineMin: schema.number(), + baselineMax: schema.number(), + deviationMin: schema.number(), + deviationMax: schema.number(), + /** The index to query for log rate analysis */ + index: schema.string(), + /** Settings to override headers derived compression and flush fix */ + compressResponse: schema.maybe(schema.boolean()), + flushFix: schema.maybe(schema.boolean()), + /** Overrides to skip steps of the analysis with existing data */ + overrides: schema.maybe( + schema.object({ + loaded: schema.maybe(schema.number()), + remainingFieldCandidates: schema.maybe(schema.arrayOf(schema.string())), + significantItems: schema.maybe(schema.arrayOf(significantItem)), + regroupOnly: schema.maybe(schema.boolean()), + }) + ), + /** Probability used for the random sampler aggregations */ + sampleProbability: schema.maybe(schema.number()), +}); + +export type AiopsLogRateAnalysisSchemaV2 = TypeOf; +export type AiopsLogRateAnalysisSchemaSignificantItem = TypeOf; diff --git a/x-pack/plugins/aiops/common/api/stream_reducer.test.ts b/x-pack/plugins/aiops/common/api/stream_reducer.test.ts index d779ccab356b3..7802d61e1781e 100644 --- a/x-pack/plugins/aiops/common/api/stream_reducer.test.ts +++ b/x-pack/plugins/aiops/common/api/stream_reducer.test.ts @@ -14,7 +14,7 @@ import { resetAllAction, resetGroupsAction, updateLoadingStateAction, -} from './log_rate_analysis'; +} from './log_rate_analysis/actions'; import { initialState, streamReducer } from './stream_reducer'; describe('streamReducer', () => { @@ -28,56 +28,59 @@ describe('streamReducer', () => { ccsWarning: true, loaded: 50, loadingState: 'Loaded 50%', - significantTerms: [], - significantTermsGroups: [], + significantItems: [], + significantItemsGroups: [], errors: [], }); }); - it('adds significant term, then resets all state again', () => { + it('adds significant item, then resets all state again', () => { const state1 = streamReducer( initialState, - addSignificantTermsAction([ - { - key: 'the-field-name:the-field-value', - type: 'keyword', - fieldName: 'the-field-name', - fieldValue: 'the-field-value', - doc_count: 10, - bg_count: 100, - total_doc_count: 1000, - total_bg_count: 10000, - score: 0.1, - pValue: 0.01, - normalizedScore: 0.123, - }, - ]) + addSignificantTermsAction( + [ + { + key: 'the-field-name:the-field-value', + type: 'keyword', + fieldName: 'the-field-name', + fieldValue: 'the-field-value', + doc_count: 10, + bg_count: 100, + total_doc_count: 1000, + total_bg_count: 10000, + score: 0.1, + pValue: 0.01, + normalizedScore: 0.123, + }, + ], + '2' + ) ); - expect(state1.significantTerms).toHaveLength(1); + expect(state1.significantItems).toHaveLength(1); const state2 = streamReducer(state1, resetAllAction()); - expect(state2.significantTerms).toHaveLength(0); + expect(state2.significantItems).toHaveLength(0); }); - it('adds significant terms and groups, then resets groups only', () => { - const state1 = streamReducer(initialState, addSignificantTermsAction(significantTerms)); + it('adds significant items and groups, then resets groups only', () => { + const state1 = streamReducer(initialState, addSignificantTermsAction(significantTerms, '2')); - expect(state1.significantTerms).toHaveLength(4); - expect(state1.significantTermsGroups).toHaveLength(0); + expect(state1.significantItems).toHaveLength(4); + expect(state1.significantItemsGroups).toHaveLength(0); const state2 = streamReducer( state1, - addSignificantTermsGroupAction(finalSignificantTermGroups) + addSignificantTermsGroupAction(finalSignificantTermGroups, '2') ); - expect(state2.significantTerms).toHaveLength(4); - expect(state2.significantTermsGroups).toHaveLength(4); + expect(state2.significantItems).toHaveLength(4); + expect(state2.significantItemsGroups).toHaveLength(4); const state3 = streamReducer(state2, resetGroupsAction()); - expect(state3.significantTerms).toHaveLength(4); - expect(state3.significantTermsGroups).toHaveLength(0); + expect(state3.significantItems).toHaveLength(4); + expect(state3.significantItemsGroups).toHaveLength(0); }); }); diff --git a/x-pack/plugins/aiops/common/api/stream_reducer.ts b/x-pack/plugins/aiops/common/api/stream_reducer.ts index 278c3843a2811..ba9e8014665c8 100644 --- a/x-pack/plugins/aiops/common/api/stream_reducer.ts +++ b/x-pack/plugins/aiops/common/api/stream_reducer.ts @@ -7,12 +7,12 @@ import type { SignificantTerm, SignificantTermGroup } from '@kbn/ml-agg-utils'; -import { API_ACTION_NAME, AiopsLogRateAnalysisApiAction } from './log_rate_analysis'; +import { API_ACTION_NAME, AiopsLogRateAnalysisApiAction } from './log_rate_analysis/actions'; interface StreamState { ccsWarning: boolean; - significantTerms: SignificantTerm[]; - significantTermsGroups: SignificantTermGroup[]; + significantItems: SignificantTerm[]; + significantItemsGroups: SignificantTermGroup[]; errors: string[]; loaded: number; loadingState: string; @@ -22,8 +22,8 @@ interface StreamState { export const initialState: StreamState = { ccsWarning: false, - significantTerms: [], - significantTermsGroups: [], + significantItems: [], + significantItemsGroups: [], errors: [], loaded: 0, loadingState: '', @@ -31,17 +31,17 @@ export const initialState: StreamState = { export function streamReducer( state: StreamState, - action: AiopsLogRateAnalysisApiAction | AiopsLogRateAnalysisApiAction[] + action: AiopsLogRateAnalysisApiAction<'2'> | Array> ): StreamState { if (Array.isArray(action)) { return action.reduce(streamReducer, state); } switch (action.type) { - case API_ACTION_NAME.ADD_SIGNIFICANT_TERMS: - return { ...state, significantTerms: [...state.significantTerms, ...action.payload] }; - case API_ACTION_NAME.ADD_SIGNIFICANT_TERMS_HISTOGRAM: - const significantTerms = state.significantTerms.map((cp) => { + case API_ACTION_NAME.ADD_SIGNIFICANT_ITEMS: + return { ...state, significantItems: [...state.significantItems, ...action.payload] }; + case API_ACTION_NAME.ADD_SIGNIFICANT_ITEMS_HISTOGRAM: + const significantItems = state.significantItems.map((cp) => { const cpHistogram = action.payload.find( (h) => h.fieldName === cp.fieldName && h.fieldValue === cp.fieldValue ); @@ -50,24 +50,24 @@ export function streamReducer( } return cp; }); - return { ...state, significantTerms }; - case API_ACTION_NAME.ADD_SIGNIFICANT_TERMS_GROUP: - return { ...state, significantTermsGroups: action.payload }; - case API_ACTION_NAME.ADD_SIGNIFICANT_TERMS_GROUP_HISTOGRAM: - const significantTermsGroups = state.significantTermsGroups.map((cpg) => { + return { ...state, significantItems }; + case API_ACTION_NAME.ADD_SIGNIFICANT_ITEMS_GROUP: + return { ...state, significantItemsGroups: action.payload }; + case API_ACTION_NAME.ADD_SIGNIFICANT_ITEMS_GROUP_HISTOGRAM: + const significantItemsGroups = state.significantItemsGroups.map((cpg) => { const cpHistogram = action.payload.find((h) => h.id === cpg.id); if (cpHistogram) { cpg.histogram = cpHistogram.histogram; } return cpg; }); - return { ...state, significantTermsGroups }; + return { ...state, significantItemsGroups }; case API_ACTION_NAME.ADD_ERROR: return { ...state, errors: [...state.errors, action.payload] }; case API_ACTION_NAME.RESET_ERRORS: return { ...state, errors: [] }; case API_ACTION_NAME.RESET_GROUPS: - return { ...state, significantTermsGroups: [] }; + return { ...state, significantItemsGroups: [] }; case API_ACTION_NAME.RESET_ALL: return initialState; case API_ACTION_NAME.UPDATE_LOADING_STATE: diff --git a/x-pack/plugins/aiops/public/components/log_categorization/category_table/category_table.tsx b/x-pack/plugins/aiops/public/components/log_categorization/category_table/category_table.tsx index 283bb71e35a76..3796af67f8cd2 100644 --- a/x-pack/plugins/aiops/public/components/log_categorization/category_table/category_table.tsx +++ b/x-pack/plugins/aiops/public/components/log_categorization/category_table/category_table.tsx @@ -142,7 +142,7 @@ export const CategoryTable: FC = ({ return { doc_count_overall: adjustedDocCount, - doc_count_significant_term: newTerm, + doc_count_significant_item: newTerm, key: catKey, key_as_string: `${catKey}`, }; diff --git a/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_results.tsx b/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_results.tsx index 97d7201f0140d..3a236085cd6cf 100644 --- a/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_results.tsx +++ b/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_results.tsx @@ -35,7 +35,8 @@ import type { SignificantTerm, SignificantTermGroup } from '@kbn/ml-agg-utils'; import { useAiopsAppContext } from '../../hooks/use_aiops_app_context'; import { initialState, streamReducer } from '../../../common/api/stream_reducer'; -import type { AiopsApiLogRateAnalysis } from '../../../common/api'; +import type { AiopsLogRateAnalysisSchema } from '../../../common/api/log_rate_analysis/schema'; +import type { AiopsLogRateAnalysisSchemaSignificantItem } from '../../../common/api/log_rate_analysis/schema_v2'; import { AIOPS_TELEMETRY_ID } from '../../../common/constants'; import { getGroupTableItems, @@ -145,9 +146,9 @@ export const LogRateAnalysisResults: FC = ({ const [groupResults, setGroupResults] = useState(false); const [groupSkipFields, setGroupSkipFields] = useState([]); const [uniqueFieldNames, setUniqueFieldNames] = useState([]); - const [overrides, setOverrides] = useState< - AiopsApiLogRateAnalysis['body']['overrides'] | undefined - >(undefined); + const [overrides, setOverrides] = useState( + undefined + ); const [shouldStart, setShouldStart] = useState(false); const [toggleIdSelected, setToggleIdSelected] = useState(resultsGroupedOffId); @@ -164,7 +165,9 @@ export const LogRateAnalysisResults: FC = ({ setOverrides({ loaded: 0, remainingFieldCandidates: [], - significantTerms: data.significantTerms.filter((d) => !skippedFields.includes(d.fieldName)), + significantItems: data.significantItems.filter( + (d) => !skippedFields.includes(d.fieldName) + ) as AiopsLogRateAnalysisSchemaSignificantItem[], regroupOnly: true, }); startHandler(true, false); @@ -176,10 +179,10 @@ export const LogRateAnalysisResults: FC = ({ data, isRunning, errors: streamErrors, - } = useFetchStream( + } = useFetchStream, typeof streamReducer>( http, '/internal/aiops/log_rate_analysis', - '1', + '2', { start: earliest, end: latest, @@ -206,10 +209,10 @@ export const LogRateAnalysisResults: FC = ({ { [AIOPS_TELEMETRY_ID.AIOPS_ANALYSIS_RUN_ORIGIN]: embeddingOrigin } ); - const { significantTerms } = data; + const { significantItems } = data; useEffect( - () => setUniqueFieldNames(uniq(significantTerms.map((d) => d.fieldName)).sort()), - [significantTerms] + () => setUniqueFieldNames(uniq(significantItems.map((d) => d.fieldName)).sort()), + [significantItems] ); useEffect(() => { @@ -221,14 +224,18 @@ export const LogRateAnalysisResults: FC = ({ ((Array.isArray(remainingFieldCandidates) && remainingFieldCandidates.length > 0) || groupsMissing) ) { - setOverrides({ loaded, remainingFieldCandidates, significantTerms: data.significantTerms }); + setOverrides({ + loaded, + remainingFieldCandidates, + significantItems: data.significantItems as AiopsLogRateAnalysisSchemaSignificantItem[], + }); } else { setOverrides(undefined); if (onAnalysisCompleted) { onAnalysisCompleted({ analysisType, - significantTerms: data.significantTerms, - significantTermsGroups: data.significantTermsGroups, + significantTerms: data.significantItems, + significantTermsGroups: data.significantItemsGroups, }); } } @@ -277,8 +284,8 @@ export const LogRateAnalysisResults: FC = ({ }, []); const groupTableItems = useMemo( - () => getGroupTableItems(data.significantTermsGroups), - [data.significantTermsGroups] + () => getGroupTableItems(data.significantItemsGroups), + [data.significantItemsGroups] ); const shouldRerunAnalysis = useMemo( @@ -288,7 +295,7 @@ export const LogRateAnalysisResults: FC = ({ [currentAnalysisWindowParameters, windowParameters] ); - const showLogRateAnalysisResultsTable = data?.significantTerms.length > 0; + const showLogRateAnalysisResultsTable = data?.significantItems.length > 0; const groupItemCount = groupTableItems.reduce((p, c) => { return p + c.groupItemsSortedByUniqueness.length; }, 0); @@ -475,7 +482,7 @@ export const LogRateAnalysisResults: FC = ({ > {showLogRateAnalysisResultsTable && groupResults ? ( = ({ ) : null} {showLogRateAnalysisResultsTable && !groupResults ? ( = ({ xScaleType={ScaleType.Time} yScaleType={ScaleType.Linear} xAccessor={'key'} - yAccessors={['doc_count_significant_term']} + yAccessors={['doc_count_significant_item']} data={chartData} stackAccessors={[0]} color={barHighlightColor} diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/define_route.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/define_route.ts index 3be8ef8dcdd2b..6f9343a50ae62 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/define_route.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/define_route.ts @@ -10,7 +10,8 @@ import type { Logger } from '@kbn/logging'; import type { DataRequestHandlerContext } from '@kbn/data-plugin/server'; import type { UsageCounter } from '@kbn/usage-collection-plugin/server'; -import { aiopsLogRateAnalysisSchema } from '../../../common/api/log_rate_analysis'; +import { aiopsLogRateAnalysisSchemaV1 } from '../../../common/api/log_rate_analysis/schema_v1'; +import { aiopsLogRateAnalysisSchemaV2 } from '../../../common/api/log_rate_analysis/schema_v2'; import { AIOPS_API_ENDPOINT } from '../../../common/api'; import type { AiopsLicense } from '../../types'; @@ -27,7 +28,6 @@ export const defineRoute = ( router.versioned .post({ path: AIOPS_API_ENDPOINT.LOG_RATE_ANALYSIS, - access: 'internal', }) .addVersion( @@ -35,10 +35,21 @@ export const defineRoute = ( version: '1', validate: { request: { - body: aiopsLogRateAnalysisSchema, + body: aiopsLogRateAnalysisSchemaV1, + }, + }, + }, + routeHandlerFactory('1', license, logger, coreStart, usageCounter) + ) + .addVersion( + { + version: '2', + validate: { + request: { + body: aiopsLogRateAnalysisSchemaV2, }, }, }, - routeHandlerFactory(license, logger, coreStart, usageCounter) + routeHandlerFactory('2', license, logger, coreStart, usageCounter) ); }; diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_categories.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_categories.ts index bb49622ad999c..e70255ababffd 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_categories.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_categories.ts @@ -16,7 +16,7 @@ import { } from '@kbn/ml-random-sampler-utils'; import { RANDOM_SAMPLER_SEED } from '../../../../common/constants'; -import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis'; +import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis/schema'; import { createCategoryRequest } from '../../../../common/api/log_categorization/create_category_request'; import type { Category, diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_category_counts.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_category_counts.ts index fc7c14fb8f2ee..608734ae0e19a 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_category_counts.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_category_counts.ts @@ -12,7 +12,7 @@ import { ElasticsearchClient } from '@kbn/core/server'; import type { Logger } from '@kbn/logging'; import { isPopulatedObject } from '@kbn/ml-is-populated-object'; -import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis'; +import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis/schema'; import { getCategoryQuery } from '../../../../common/api/log_categorization/get_category_query'; import type { Category } from '../../../../common/api/log_categorization/types'; diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_index_info.test.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_index_info.test.ts index 790989df9833c..db14c30caebdc 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_index_info.test.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_index_info.test.ts @@ -9,7 +9,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { ElasticsearchClient } from '@kbn/core/server'; -import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis'; +import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis/schema'; import { fetchIndexInfo, getRandomDocsRequest } from './fetch_index_info'; diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_index_info.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_index_info.ts index cfb9426a739ad..595e212159437 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_index_info.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_index_info.ts @@ -10,7 +10,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { ES_FIELD_TYPES } from '@kbn/field-types'; import type { ElasticsearchClient } from '@kbn/core/server'; -import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis'; +import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis/schema'; import { getQueryWithParams } from './get_query_with_params'; import { getRequestBase } from './get_request_base'; diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_significant_categories.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_significant_categories.ts index c9e54be509426..4cf106a369464 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_significant_categories.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_significant_categories.ts @@ -13,7 +13,7 @@ import { criticalTableLookup, type Histogram } from '@kbn/ml-chi2test'; import { type SignificantTerm, SIGNIFICANT_TERM_TYPE } from '@kbn/ml-agg-utils'; import type { Category } from '../../../../common/api/log_categorization/types'; -import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis'; +import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis/schema'; import { LOG_RATE_ANALYSIS_SETTINGS } from '../../../../common/constants'; import { fetchCategories } from './fetch_categories'; diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_significant_term_p_values.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_significant_term_p_values.ts index 00bcd918f48d9..10b488940ce2c 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_significant_term_p_values.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_significant_term_p_values.ts @@ -16,7 +16,7 @@ import { } from '@kbn/ml-random-sampler-utils'; import { LOG_RATE_ANALYSIS_SETTINGS, RANDOM_SAMPLER_SEED } from '../../../../common/constants'; -import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis'; +import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis/schema'; import { isRequestAbortedError } from '../../../lib/is_request_aborted_error'; diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_terms_2_categories_counts.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_terms_2_categories_counts.ts index f1629f212ae99..14be571331fd0 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_terms_2_categories_counts.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_terms_2_categories_counts.ts @@ -14,7 +14,7 @@ import type { Logger } from '@kbn/logging'; import type { FieldValuePair, SignificantTerm } from '@kbn/ml-agg-utils'; import { isPopulatedObject } from '@kbn/ml-is-populated-object'; -import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis'; +import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis/schema'; import type { FetchFrequentItemSetsResponse, ItemSet } from '../../../../common/types'; import { getCategoryQuery } from '../../../../common/api/log_categorization/get_category_query'; import type { Category } from '../../../../common/api/log_categorization/types'; diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_filters.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_filters.ts index 5e58e138dac6b..e910d2e997441 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_filters.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_filters.ts @@ -9,7 +9,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { ESFilter } from '@kbn/es-types'; -import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis'; +import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis/schema'; export function rangeQuery( start?: number, diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_histogram_query.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_histogram_query.ts index b45d7d026638e..0f3510df73dcf 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_histogram_query.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_histogram_query.ts @@ -7,7 +7,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis'; +import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis/schema'; import { getQueryWithParams } from './get_query_with_params'; diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_query_with_params.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_query_with_params.ts index 2d68c666b78ea..b7e2e2619316a 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_query_with_params.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_query_with_params.ts @@ -9,7 +9,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { FieldValuePair } from '@kbn/ml-agg-utils'; -import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis'; +import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis/schema'; import { getFilters } from './get_filters'; @@ -18,7 +18,7 @@ export const getTermsQuery = ({ fieldName, fieldValue }: FieldValuePair) => { }; interface QueryParams { - params: AiopsLogRateAnalysisSchema; + params: AiopsLogRateAnalysisSchema<'2'>; termFilters?: FieldValuePair[]; filter?: estypes.QueryDslQueryContainer; } diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_request_base.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_request_base.ts index 2410be74ea6b0..ddc1a2b3a0a34 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_request_base.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_request_base.ts @@ -5,7 +5,7 @@ * 2.0. */ -import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis'; +import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis/schema'; export const getRequestBase = ({ index, includeFrozen }: AiopsLogRateAnalysisSchema) => ({ index, diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/route_handler_factory.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/route_handler_factory.ts index b522d6d6ee818..b92458cc1e3de 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/route_handler_factory.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/route_handler_factory.ts @@ -23,6 +23,7 @@ import { streamFactory } from '@kbn/ml-response-stream/server'; import type { SignificantTerm, SignificantTermGroup, + SignificantTermHistogramItem, NumericChartData, NumericHistogramField, } from '@kbn/ml-agg-utils'; @@ -44,8 +45,11 @@ import { resetGroupsAction, updateLoadingStateAction, AiopsLogRateAnalysisApiAction, - type AiopsLogRateAnalysisSchema, -} from '../../../common/api/log_rate_analysis'; +} from '../../../common/api/log_rate_analysis/actions'; +import type { + AiopsLogRateAnalysisSchema, + AiopsLogRateAnalysisApiVersion as ApiVersion, +} from '../../../common/api/log_rate_analysis/schema'; import { getCategoryQuery } from '../../../common/api/log_categorization/get_category_query'; import { AIOPS_API_ENDPOINT } from '../../../common/api'; @@ -74,16 +78,16 @@ const PROGRESS_STEP_GROUPING = 0.1; const PROGRESS_STEP_HISTOGRAMS = 0.1; const PROGRESS_STEP_HISTOGRAMS_GROUPS = 0.1; -export const routeHandlerFactory: ( +export function routeHandlerFactory( + version: T, license: AiopsLicense, logger: Logger, coreStart: CoreStart, usageCounter?: UsageCounter -) => RequestHandler = - (license, logger, coreStart, usageCounter) => - async ( +): RequestHandler> { + return async ( context: RequestHandlerContext, - request: KibanaRequest, + request: KibanaRequest>, response: KibanaResponseFactory ) => { const { headers } = request; @@ -135,7 +139,7 @@ export const routeHandlerFactory: ( end: streamEnd, push, responseWithHeaders, - } = streamFactory( + } = streamFactory>( request.headers, logger, request.body.compressResponse, @@ -288,11 +292,27 @@ export const routeHandlerFactory: ( // This will store the combined count of detected significant log patterns and keywords let fieldValuePairsCount = 0; - const significantCategories: SignificantTerm[] = request.body.overrides?.significantTerms - ? request.body.overrides?.significantTerms.filter( + const significantCategories: SignificantTerm[] = []; + + if (version === '1') { + significantCategories.push( + ...(( + request.body as AiopsLogRateAnalysisSchema<'1'> + ).overrides?.significantTerms?.filter( (d) => d.type === SIGNIFICANT_TERM_TYPE.LOG_PATTERN - ) - : []; + ) ?? []) + ); + } + + if (version === '2') { + significantCategories.push( + ...(( + request.body as AiopsLogRateAnalysisSchema<'2'> + ).overrides?.significantItems?.filter( + (d) => d.type === SIGNIFICANT_TERM_TYPE.LOG_PATTERN + ) ?? []) + ); + } // Get significant categories of text fields if (textFieldCandidates.length > 0) { @@ -309,15 +329,31 @@ export const routeHandlerFactory: ( ); if (significantCategories.length > 0) { - push(addSignificantTermsAction(significantCategories)); + push(addSignificantTermsAction(significantCategories, version)); } } - const significantTerms: SignificantTerm[] = request.body.overrides?.significantTerms - ? request.body.overrides?.significantTerms.filter( + const significantTerms: SignificantTerm[] = []; + + if (version === '1') { + significantTerms.push( + ...(( + request.body as AiopsLogRateAnalysisSchema<'1'> + ).overrides?.significantTerms?.filter( (d) => d.type === SIGNIFICANT_TERM_TYPE.KEYWORD - ) - : []; + ) ?? []) + ); + } + + if (version === '2') { + significantTerms.push( + ...(( + request.body as AiopsLogRateAnalysisSchema<'2'> + ).overrides?.significantItems?.filter( + (d) => d.type === SIGNIFICANT_TERM_TYPE.KEYWORD + ) ?? []) + ); + } const fieldsToSample = new Set(); @@ -375,7 +411,7 @@ export const routeHandlerFactory: ( }); significantTerms.push(...pValues); - push(addSignificantTermsAction(pValues)); + push(addSignificantTermsAction(pValues, version)); } push( @@ -546,7 +582,7 @@ export const routeHandlerFactory: ( const maxItems = Math.max(...significantTermGroups.map((g) => g.group.length)); if (maxItems > 1) { - push(addSignificantTermsGroupAction(significantTermGroups)); + push(addSignificantTermsGroupAction(significantTermGroups, version)); } loaded += PROGRESS_STEP_GROUPING; @@ -608,28 +644,41 @@ export const routeHandlerFactory: ( } return; } - const histogram = + const histogram: SignificantTermHistogramItem[] = overallTimeSeries.data.map((o) => { const current = cpgTimeSeries.data.find( (d1) => d1.key_as_string === o.key_as_string ) ?? { doc_count: 0, }; + + if (version === '1') { + return { + key: o.key, + key_as_string: o.key_as_string ?? '', + doc_count_significant_term: current.doc_count, + doc_count_overall: Math.max(0, o.doc_count - current.doc_count), + }; + } + return { key: o.key, key_as_string: o.key_as_string ?? '', - doc_count_significant_term: current.doc_count, + doc_count_significant_item: current.doc_count, doc_count_overall: Math.max(0, o.doc_count - current.doc_count), }; }) ?? []; push( - addSignificantTermsGroupHistogramAction([ - { - id: cpg.id, - histogram, - }, - ]) + addSignificantTermsGroupHistogramAction( + [ + { + id: cpg.id, + histogram, + }, + ], + version + ) ); } }, MAX_CONCURRENT_QUERIES); @@ -710,17 +759,26 @@ export const routeHandlerFactory: ( return; } - const histogram = + const histogram: SignificantTermHistogramItem[] = overallTimeSeries.data.map((o) => { const current = cpTimeSeries.data.find( (d1) => d1.key_as_string === o.key_as_string ) ?? { doc_count: 0, }; + if (version === '1') { + return { + key: o.key, + key_as_string: o.key_as_string ?? '', + doc_count_significant_term: current.doc_count, + doc_count_overall: Math.max(0, o.doc_count - current.doc_count), + }; + } + return { key: o.key, key_as_string: o.key_as_string ?? '', - doc_count_significant_term: current.doc_count, + doc_count_significant_item: current.doc_count, doc_count_overall: Math.max(0, o.doc_count - current.doc_count), }; }) ?? []; @@ -730,13 +788,16 @@ export const routeHandlerFactory: ( loaded += (1 / fieldValuePairsCount) * PROGRESS_STEP_HISTOGRAMS; pushHistogramDataLoadingState(); push( - addSignificantTermsHistogramAction([ - { - fieldName, - fieldValue, - histogram, - }, - ]) + addSignificantTermsHistogramAction( + [ + { + fieldName, + fieldValue, + histogram, + }, + ], + version + ) ); } }, MAX_CONCURRENT_QUERIES); @@ -802,17 +863,27 @@ export const routeHandlerFactory: ( return; } - const histogram = + const histogram: SignificantTermHistogramItem[] = overallTimeSeries.data.map((o) => { const current = catTimeSeries.data.find( (d1) => d1.key_as_string === o.key_as_string ) ?? { doc_count: 0, }; + + if (version === '1') { + return { + key: o.key, + key_as_string: o.key_as_string ?? '', + doc_count_significant_term: current.doc_count, + doc_count_overall: Math.max(0, o.doc_count - current.doc_count), + }; + } + return { key: o.key, key_as_string: o.key_as_string ?? '', - doc_count_significant_term: current.doc_count, + doc_count_significant_item: current.doc_count, doc_count_overall: Math.max(0, o.doc_count - current.doc_count), }; }) ?? []; @@ -822,13 +893,16 @@ export const routeHandlerFactory: ( loaded += (1 / fieldValuePairsCount) * PROGRESS_STEP_HISTOGRAMS; pushHistogramDataLoadingState(); push( - addSignificantTermsHistogramAction([ - { - fieldName, - fieldValue, - histogram, - }, - ]) + addSignificantTermsHistogramAction( + [ + { + fieldName, + fieldValue, + histogram, + }, + ], + version + ) ); } } @@ -849,3 +923,4 @@ export const routeHandlerFactory: ( return response.ok(responseWithHeaders); }); }; +} diff --git a/x-pack/test/api_integration/apis/aiops/log_rate_analysis_full_analysis.ts b/x-pack/test/api_integration/apis/aiops/log_rate_analysis_full_analysis.ts index c9fe22a472f4f..4c3d787402864 100644 --- a/x-pack/test/api_integration/apis/aiops/log_rate_analysis_full_analysis.ts +++ b/x-pack/test/api_integration/apis/aiops/log_rate_analysis_full_analysis.ts @@ -10,13 +10,19 @@ import fetch from 'node-fetch'; import { format as formatUrl } from 'url'; import expect from '@kbn/expect'; -import type { AiopsApiLogRateAnalysis } from '@kbn/aiops-plugin/common/api'; +import type { AiopsLogRateAnalysisSchema } from '@kbn/aiops-plugin/common/api/log_rate_analysis/schema'; import { ELASTIC_HTTP_VERSION_HEADER } from '@kbn/core-http-common'; import type { FtrProviderContext } from '../../ftr_provider_context'; import { parseStream } from './parse_stream'; -import { logRateAnalysisTestData } from './test_data'; +import { getLogRateAnalysisTestData, API_VERSIONS } from './test_data'; +import { + getAddSignificationItemsActions, + getHistogramActions, + getGroupActions, + getGroupHistogramActions, +} from './test_helpers'; export default ({ getService }: FtrProviderContext) => { const aiops = getService('aiops'); @@ -26,202 +32,202 @@ export default ({ getService }: FtrProviderContext) => { const esArchiver = getService('esArchiver'); describe('POST /internal/aiops/log_rate_analysis - full analysis', () => { - logRateAnalysisTestData.forEach((testData) => { - describe(`with ${testData.testName}`, () => { - before(async () => { - if (testData.esArchive) { - await esArchiver.loadIfNeeded(testData.esArchive); - } else if (testData.dataGenerator) { - await aiops.logRateAnalysisDataGenerator.generateData(testData.dataGenerator); - } - }); - - after(async () => { - if (testData.esArchive) { - await esArchiver.unload(testData.esArchive); - } else if (testData.dataGenerator) { - await aiops.logRateAnalysisDataGenerator.removeGeneratedData(testData.dataGenerator); - } - }); - - async function assertAnalysisResult(data: any[]) { - expect(data.length).to.eql( - testData.expected.actionsLength, - `Expected 'actionsLength' to be ${testData.expected.actionsLength}, got ${data.length}.` - ); - data.forEach((d) => { - expect(typeof d.type).to.be('string'); + API_VERSIONS.forEach((apiVersion) => { + getLogRateAnalysisTestData().forEach((testData) => { + describe(`with v${apiVersion} - ${testData.testName}`, () => { + before(async () => { + if (testData.esArchive) { + await esArchiver.loadIfNeeded(testData.esArchive); + } else if (testData.dataGenerator) { + await aiops.logRateAnalysisDataGenerator.generateData(testData.dataGenerator); + } }); - const addSignificantTermsActions = data.filter( - (d) => d.type === testData.expected.significantTermFilter - ); - expect(addSignificantTermsActions.length).to.greaterThan(0); - - const significantTerms = orderBy( - addSignificantTermsActions.flatMap((d) => d.payload), - ['doc_count'], - ['desc'] - ); - - expect(significantTerms).to.eql( - testData.expected.significantTerms, - 'Significant terms do not match expected values.' - ); - - const histogramActions = data.filter((d) => d.type === testData.expected.histogramFilter); - const histograms = histogramActions.flatMap((d) => d.payload); - // for each significant term we should get a histogram - expect(histogramActions.length).to.be(significantTerms.length); - // each histogram should have a length of 20 items. - histograms.forEach((h, index) => { - expect(h.histogram.length).to.be(20); + after(async () => { + if (testData.esArchive) { + await esArchiver.unload(testData.esArchive); + } else if (testData.dataGenerator) { + await aiops.logRateAnalysisDataGenerator.removeGeneratedData(testData.dataGenerator); + } }); - const groupActions = data.filter((d) => d.type === testData.expected.groupFilter); - const groups = groupActions.flatMap((d) => d.payload); - - expect(orderBy(groups, ['docCount'], ['desc'])).to.eql( - orderBy(testData.expected.groups, ['docCount'], ['desc']), - 'Grouping result does not match expected values.' - ); - - const groupHistogramActions = data.filter( - (d) => d.type === testData.expected.groupHistogramFilter - ); - const groupHistograms = groupHistogramActions.flatMap((d) => d.payload); - // for each significant terms group we should get a histogram - expect(groupHistograms.length).to.be(groups.length); - // each histogram should have a length of 20 items. - groupHistograms.forEach((h, index) => { - expect(h.histogram.length).to.be(20); - }); - } - - async function requestWithoutStreaming(body: AiopsApiLogRateAnalysis['body']) { - const resp = await supertest - .post(`/internal/aiops/log_rate_analysis`) - .set('kbn-xsrf', 'kibana') - .set(ELASTIC_HTTP_VERSION_HEADER, '1') - .send(body) - .expect(200); - - // compression is on by default so if the request body is undefined - // the response header should include "gzip" and otherwise be "undefined" - if (body.compressResponse === undefined) { - expect(resp.header['content-encoding']).to.be('gzip'); - } else if (body.compressResponse === false) { - expect(resp.header['content-encoding']).to.be(undefined); + async function assertAnalysisResult(data: any[]) { + expect(data.length).to.eql( + testData.expected.actionsLength, + `Expected 'actionsLength' to be ${testData.expected.actionsLength}, got ${data.length}.` + ); + data.forEach((d) => { + expect(typeof d.type).to.be('string'); + }); + + const addSignificantItemsActions = getAddSignificationItemsActions(data, apiVersion); + expect(addSignificantItemsActions.length).to.greaterThan(0); + + const significantItems = orderBy( + addSignificantItemsActions.flatMap((d) => d.payload), + ['doc_count'], + ['desc'] + ); + + expect(significantItems).to.eql( + testData.expected.significantTerms, + 'Significant terms do not match expected values.' + ); + + const histogramActions = getHistogramActions(data, apiVersion); + const histograms = histogramActions.flatMap((d) => d.payload); + // for each significant term we should get a histogram + expect(histogramActions.length).to.be(significantItems.length); + // each histogram should have a length of 20 items. + histograms.forEach((h, index) => { + expect(h.histogram.length).to.be(20); + }); + + const groupActions = getGroupActions(data, apiVersion); + const groups = groupActions.flatMap((d) => d.payload); + + expect(orderBy(groups, ['docCount'], ['desc'])).to.eql( + orderBy(testData.expected.groups, ['docCount'], ['desc']), + 'Grouping result does not match expected values.' + ); + + const groupHistogramActions = getGroupHistogramActions(data, apiVersion); + const groupHistograms = groupHistogramActions.flatMap((d) => d.payload); + // for each significant terms group we should get a histogram + expect(groupHistograms.length).to.be(groups.length); + // each histogram should have a length of 20 items. + groupHistograms.forEach((h, index) => { + expect(h.histogram.length).to.be(20); + }); } - expect(Buffer.isBuffer(resp.body)).to.be(true); + async function requestWithoutStreaming( + body: AiopsLogRateAnalysisSchema + ) { + const resp = await supertest + .post(`/internal/aiops/log_rate_analysis`) + .set('kbn-xsrf', 'kibana') + .set(ELASTIC_HTTP_VERSION_HEADER, apiVersion) + .send(body) + .expect(200); + + // compression is on by default so if the request body is undefined + // the response header should include "gzip" and otherwise be "undefined" + if (body.compressResponse === undefined) { + expect(resp.header['content-encoding']).to.be('gzip'); + } else if (body.compressResponse === false) { + expect(resp.header['content-encoding']).to.be(undefined); + } - const chunks: string[] = resp.body.toString().split('\n'); + expect(Buffer.isBuffer(resp.body)).to.be(true); - expect(chunks.length).to.eql( - testData.expected.chunksLength, - `Expected 'chunksLength' to be ${testData.expected.chunksLength}, got ${chunks.length}.` - ); + const chunks: string[] = resp.body.toString().split('\n'); - const lastChunk = chunks.pop(); - expect(lastChunk).to.be(''); + expect(chunks.length).to.eql( + testData.expected.chunksLength, + `Expected 'chunksLength' to be ${testData.expected.chunksLength}, got ${chunks.length}.` + ); - let data: any[] = []; + const lastChunk = chunks.pop(); + expect(lastChunk).to.be(''); - expect(() => { - data = chunks.map((c) => JSON.parse(c)); - }).not.to.throwError(); + let data: any[] = []; - await assertAnalysisResult(data); - } + expect(() => { + data = chunks.map((c) => JSON.parse(c)); + }).not.to.throwError(); - it('should return full data without streaming with compression with flushFix', async () => { - await requestWithoutStreaming(testData.requestBody); - }); + await assertAnalysisResult(data); + } - it('should return full data without streaming with compression without flushFix', async () => { - await requestWithoutStreaming({ ...testData.requestBody, flushFix: false }); - }); + it('should return full data without streaming with compression with flushFix', async () => { + await requestWithoutStreaming(testData.requestBody); + }); - it('should return full data without streaming without compression with flushFix', async () => { - await requestWithoutStreaming({ ...testData.requestBody, compressResponse: false }); - }); + it('should return full data without streaming with compression without flushFix', async () => { + await requestWithoutStreaming({ ...testData.requestBody, flushFix: false }); + }); - it('should return full data without streaming without compression without flushFix', async () => { - await requestWithoutStreaming({ - ...testData.requestBody, - compressResponse: false, - flushFix: false, + it('should return full data without streaming without compression with flushFix', async () => { + await requestWithoutStreaming({ ...testData.requestBody, compressResponse: false }); }); - }); - async function requestWithStreaming(body: AiopsApiLogRateAnalysis['body']) { - const resp = await fetch(`${kibanaServerUrl}/internal/aiops/log_rate_analysis`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - [ELASTIC_HTTP_VERSION_HEADER]: '1', - 'kbn-xsrf': 'stream', - }, - body: JSON.stringify(body), + it('should return full data without streaming without compression without flushFix', async () => { + await requestWithoutStreaming({ + ...testData.requestBody, + compressResponse: false, + flushFix: false, + }); }); - // compression is on by default so if the request body is undefined - // the response header should include "gzip" and otherwise be "null" - if (body.compressResponse === undefined) { - expect(resp.headers.get('content-encoding')).to.be('gzip'); - } else if (body.compressResponse === false) { - expect(resp.headers.get('content-encoding')).to.be(null); - } + async function requestWithStreaming(body: AiopsLogRateAnalysisSchema) { + const resp = await fetch(`${kibanaServerUrl}/internal/aiops/log_rate_analysis`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + [ELASTIC_HTTP_VERSION_HEADER]: apiVersion, + 'kbn-xsrf': 'stream', + }, + body: JSON.stringify(body), + }); + + // compression is on by default so if the request body is undefined + // the response header should include "gzip" and otherwise be "null" + if (body.compressResponse === undefined) { + expect(resp.headers.get('content-encoding')).to.be('gzip'); + } else if (body.compressResponse === false) { + expect(resp.headers.get('content-encoding')).to.be(null); + } - expect(resp.ok).to.be(true); - expect(resp.status).to.be(200); + expect(resp.ok).to.be(true); + expect(resp.status).to.be(200); - const stream = resp.body; + const stream = resp.body; - expect(stream).not.to.be(null); + expect(stream).not.to.be(null); - if (stream !== null) { - const data: any[] = []; - let chunkCounter = 0; - const parseStreamCallback = (c: number) => (chunkCounter = c); + if (stream !== null) { + const data: any[] = []; + let chunkCounter = 0; + const parseStreamCallback = (c: number) => (chunkCounter = c); - for await (const action of parseStream(stream, parseStreamCallback)) { - expect(action.type).not.to.be('error'); - data.push(action); - } + for await (const action of parseStream(stream, parseStreamCallback)) { + expect(action.type).not.to.be('error'); + data.push(action); + } - // Originally we assumed that we can assert streaming in contrast - // to non-streaming if there is more than one chunk. However, - // this turned out to be flaky since a stream could finish fast - // enough to contain only one chunk. So now we are checking if - // there's just one chunk or more. - expect(chunkCounter).to.be.greaterThan( - 0, - `Expected 'chunkCounter' to be greater than 0, got ${chunkCounter}.` - ); + // Originally we assumed that we can assert streaming in contrast + // to non-streaming if there is more than one chunk. However, + // this turned out to be flaky since a stream could finish fast + // enough to contain only one chunk. So now we are checking if + // there's just one chunk or more. + expect(chunkCounter).to.be.greaterThan( + 0, + `Expected 'chunkCounter' to be greater than 0, got ${chunkCounter}.` + ); - await assertAnalysisResult(data); + await assertAnalysisResult(data); + } } - } - it('should return data in chunks with streaming with compression with flushFix', async () => { - await requestWithStreaming(testData.requestBody); - }); + it('should return data in chunks with streaming with compression with flushFix', async () => { + await requestWithStreaming(testData.requestBody); + }); - it('should return data in chunks with streaming with compression without flushFix', async () => { - await requestWithStreaming({ ...testData.requestBody, flushFix: false }); - }); + it('should return data in chunks with streaming with compression without flushFix', async () => { + await requestWithStreaming({ ...testData.requestBody, flushFix: false }); + }); - it('should return data in chunks with streaming without compression with flushFix', async () => { - await requestWithStreaming({ ...testData.requestBody, compressResponse: false }); - }); + it('should return data in chunks with streaming without compression with flushFix', async () => { + await requestWithStreaming({ ...testData.requestBody, compressResponse: false }); + }); - it('should return data in chunks with streaming without compression without flushFix', async () => { - await requestWithStreaming({ - ...testData.requestBody, - compressResponse: false, - flushFix: false, + it('should return data in chunks with streaming without compression without flushFix', async () => { + await requestWithStreaming({ + ...testData.requestBody, + compressResponse: false, + flushFix: false, + }); }); }); }); diff --git a/x-pack/test/api_integration/apis/aiops/log_rate_analysis_groups_only.ts b/x-pack/test/api_integration/apis/aiops/log_rate_analysis_groups_only.ts index 8aeccc6af9a97..3ffcf7b04fdd2 100644 --- a/x-pack/test/api_integration/apis/aiops/log_rate_analysis_groups_only.ts +++ b/x-pack/test/api_integration/apis/aiops/log_rate_analysis_groups_only.ts @@ -10,13 +10,20 @@ import fetch from 'node-fetch'; import { format as formatUrl } from 'url'; import expect from '@kbn/expect'; -import type { AiopsApiLogRateAnalysis } from '@kbn/aiops-plugin/common/api'; +import type { AiopsLogRateAnalysisSchema } from '@kbn/aiops-plugin/common/api/log_rate_analysis/schema'; +import type { AiopsLogRateAnalysisSchemaSignificantItem } from '@kbn/aiops-plugin/common/api/log_rate_analysis/schema_v2'; import { ELASTIC_HTTP_VERSION_HEADER } from '@kbn/core-http-common'; import type { FtrProviderContext } from '../../ftr_provider_context'; import { parseStream } from './parse_stream'; -import { logRateAnalysisTestData } from './test_data'; +import { getLogRateAnalysisTestData, API_VERSIONS } from './test_data'; +import { + getAddSignificationItemsActions, + getHistogramActions, + getGroupActions, + getGroupHistogramActions, +} from './test_helpers'; export default ({ getService }: FtrProviderContext) => { const aiops = getService('aiops'); @@ -25,211 +32,229 @@ export default ({ getService }: FtrProviderContext) => { const kibanaServerUrl = formatUrl(config.get('servers.kibana')); const esArchiver = getService('esArchiver'); - // FLAKY: https://github.com/elastic/kibana/issues/169325 - describe.skip('POST /internal/aiops/log_rate_analysis - groups only', () => { - logRateAnalysisTestData.forEach((testData) => { - const overrides = { - loaded: 0, - remainingFieldCandidates: [], - significantTerms: testData.expected.significantTerms, - regroupOnly: true, - }; - - describe(`with ${testData.testName}`, () => { - before(async () => { - if (testData.esArchive) { - await esArchiver.loadIfNeeded(testData.esArchive); - } else if (testData.dataGenerator) { - await aiops.logRateAnalysisDataGenerator.generateData(testData.dataGenerator); - } - }); + describe('POST /internal/aiops/log_rate_analysis - groups only', () => { + API_VERSIONS.forEach((apiVersion) => { + getLogRateAnalysisTestData().forEach((testData) => { + let overrides: AiopsLogRateAnalysisSchema['overrides'] = {}; + + if (apiVersion === '1') { + overrides = { + loaded: 0, + remainingFieldCandidates: [], + significantTerms: testData.expected.significantTerms, + regroupOnly: true, + } as AiopsLogRateAnalysisSchema['overrides']; + } - after(async () => { - if (testData.esArchive) { - await esArchiver.unload(testData.esArchive); - } else if (testData.dataGenerator) { - await aiops.logRateAnalysisDataGenerator.removeGeneratedData(testData.dataGenerator); - } - }); + if (apiVersion === '2') { + overrides = { + loaded: 0, + remainingFieldCandidates: [], + significantItems: testData.expected + .significantTerms as AiopsLogRateAnalysisSchemaSignificantItem[], + regroupOnly: true, + } as AiopsLogRateAnalysisSchema['overrides']; + } - async function assertAnalysisResult(data: any[]) { - expect(data.length).to.eql( - testData.expected.actionsLengthGroupOnly, - `Expected 'actionsLengthGroupOnly' to be ${testData.expected.actionsLengthGroupOnly}, got ${data.length}.` - ); - data.forEach((d) => { - expect(typeof d.type).to.be('string'); + describe(`with v${apiVersion} - ${testData.testName}`, () => { + before(async () => { + if (testData.esArchive) { + await esArchiver.loadIfNeeded(testData.esArchive); + } else if (testData.dataGenerator) { + await aiops.logRateAnalysisDataGenerator.generateData(testData.dataGenerator); + } }); - const addSignificantTermsActions = data.filter( - (d) => d.type === testData.expected.significantTermFilter - ); - expect(addSignificantTermsActions.length).to.eql( - 0, - `Expected significant terms actions to be 0, got ${addSignificantTermsActions.length}` - ); - - const histogramActions = data.filter((d) => d.type === testData.expected.histogramFilter); - // for each significant term we should get a histogram - expect(histogramActions.length).to.eql( - 0, - `Expected histogram actions to be 0, got ${histogramActions.length}` - ); - - const groupActions = data.filter((d) => d.type === testData.expected.groupFilter); - const groups = groupActions.flatMap((d) => d.payload); - - expect(orderBy(groups, ['docCount'], ['desc'])).to.eql( - orderBy(testData.expected.groups, ['docCount'], ['desc']), - 'Grouping result does not match expected values.' - ); - - const groupHistogramActions = data.filter( - (d) => d.type === testData.expected.groupHistogramFilter - ); - const groupHistograms = groupHistogramActions.flatMap((d) => d.payload); - // for each significant terms group we should get a histogram - expect(groupHistograms.length).to.be(groups.length); - // each histogram should have a length of 20 items. - groupHistograms.forEach((h, index) => { - expect(h.histogram.length).to.be(20); + after(async () => { + if (testData.esArchive) { + await esArchiver.unload(testData.esArchive); + } else if (testData.dataGenerator) { + await aiops.logRateAnalysisDataGenerator.removeGeneratedData(testData.dataGenerator); + } }); - } - async function requestWithoutStreaming(body: AiopsApiLogRateAnalysis['body']) { - const resp = await supertest - .post(`/internal/aiops/log_rate_analysis`) - .set('kbn-xsrf', 'kibana') - .set(ELASTIC_HTTP_VERSION_HEADER, '1') - .send(body) - .expect(200); - - // compression is on by default so if the request body is undefined - // the response header should include "gzip" and otherwise be "undefined" - if (body.compressResponse === undefined) { - expect(resp.header['content-encoding']).to.be('gzip'); - } else if (body.compressResponse === false) { - expect(resp.header['content-encoding']).to.be(undefined); + async function assertAnalysisResult(data: any[]) { + expect(data.length).to.eql( + testData.expected.actionsLengthGroupOnly, + `Expected 'actionsLengthGroupOnly' to be ${testData.expected.actionsLengthGroupOnly}, got ${data.length}.` + ); + data.forEach((d) => { + expect(typeof d.type).to.be('string'); + }); + + const addSignificantItemsActions = getAddSignificationItemsActions(data, apiVersion); + expect(addSignificantItemsActions.length).to.eql( + 0, + `Expected significant items actions to be 0, got ${addSignificantItemsActions.length}` + ); + + const histogramActions = getHistogramActions(data, apiVersion); + + // for each significant item we should get a histogram + expect(histogramActions.length).to.eql( + 0, + `Expected histogram actions to be 0, got ${histogramActions.length}` + ); + + const groupActions = getGroupActions(data, apiVersion); + const groups = groupActions.flatMap((d) => d.payload); + + expect(orderBy(groups, ['docCount'], ['desc'])).to.eql( + orderBy(testData.expected.groups, ['docCount'], ['desc']), + 'Grouping result does not match expected values.' + ); + + const groupHistogramActions = getGroupHistogramActions(data, apiVersion); + const groupHistograms = groupHistogramActions.flatMap((d) => d.payload); + // for each significant items group we should get a histogram + expect(groupHistograms.length).to.be(groups.length); + // each histogram should have a length of 20 items. + groupHistograms.forEach((h, index) => { + expect(h.histogram.length).to.be(20); + }); } - expect(Buffer.isBuffer(resp.body)).to.be(true); + async function requestWithoutStreaming( + body: AiopsLogRateAnalysisSchema + ) { + const resp = await supertest + .post(`/internal/aiops/log_rate_analysis`) + .set('kbn-xsrf', 'kibana') + .set(ELASTIC_HTTP_VERSION_HEADER, apiVersion) + .send(body) + .expect(200); + + // compression is on by default so if the request body is undefined + // the response header should include "gzip" and otherwise be "undefined" + if (body.compressResponse === undefined) { + expect(resp.header['content-encoding']).to.be('gzip'); + } else if (body.compressResponse === false) { + expect(resp.header['content-encoding']).to.be(undefined); + } - const chunks: string[] = resp.body.toString().split('\n'); + expect(Buffer.isBuffer(resp.body)).to.be(true); - expect(chunks.length).to.eql( - testData.expected.chunksLengthGroupOnly, - `Expected 'chunksLength' to be ${testData.expected.chunksLengthGroupOnly}, got ${chunks.length}.` - ); + const chunks: string[] = resp.body.toString().split('\n'); - const lastChunk = chunks.pop(); - expect(lastChunk).to.be(''); + expect(chunks.length).to.eql( + testData.expected.chunksLengthGroupOnly, + `Expected 'chunksLength' to be ${testData.expected.chunksLengthGroupOnly}, got ${chunks.length}.` + ); - let data: any[] = []; + const lastChunk = chunks.pop(); + expect(lastChunk).to.be(''); - expect(() => { - data = chunks.map((c) => JSON.parse(c)); - }).not.to.throwError(); + let data: any[] = []; - await assertAnalysisResult(data); - } + expect(() => { + data = chunks.map((c) => JSON.parse(c)); + }).not.to.throwError(); - it('should return group only data without streaming with compression with flushFix', async () => { - await requestWithoutStreaming({ ...testData.requestBody, overrides }); - }); + await assertAnalysisResult(data); + } - it('should return group only data without streaming with compression without flushFix', async () => { - await requestWithoutStreaming({ ...testData.requestBody, overrides, flushFix: false }); - }); + it('should return group only data without streaming with compression with flushFix', async () => { + await requestWithoutStreaming({ ...testData.requestBody, overrides }); + }); - it('should return group only data without streaming without compression with flushFix', async () => { - await requestWithoutStreaming({ - ...testData.requestBody, - overrides, - compressResponse: false, + it('should return group only data without streaming with compression without flushFix', async () => { + await requestWithoutStreaming({ + ...testData.requestBody, + overrides, + flushFix: false, + }); }); - }); - it('should return group only data without streaming without compression without flushFix', async () => { - await requestWithoutStreaming({ - ...testData.requestBody, - overrides, - compressResponse: false, - flushFix: false, + it('should return group only data without streaming without compression with flushFix', async () => { + await requestWithoutStreaming({ + ...testData.requestBody, + overrides, + compressResponse: false, + }); }); - }); - async function requestWithStreaming(body: AiopsApiLogRateAnalysis['body']) { - const resp = await fetch(`${kibanaServerUrl}/internal/aiops/log_rate_analysis`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - [ELASTIC_HTTP_VERSION_HEADER]: '1', - 'kbn-xsrf': 'stream', - }, - body: JSON.stringify(body), + it('should return group only data without streaming without compression without flushFix', async () => { + await requestWithoutStreaming({ + ...testData.requestBody, + overrides, + compressResponse: false, + flushFix: false, + }); }); - // compression is on by default so if the request body is undefined - // the response header should include "gzip" and otherwise be "null" - if (body.compressResponse === undefined) { - expect(resp.headers.get('content-encoding')).to.be('gzip'); - } else if (body.compressResponse === false) { - expect(resp.headers.get('content-encoding')).to.be(null); - } + async function requestWithStreaming(body: AiopsLogRateAnalysisSchema) { + const resp = await fetch(`${kibanaServerUrl}/internal/aiops/log_rate_analysis`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + [ELASTIC_HTTP_VERSION_HEADER]: apiVersion, + 'kbn-xsrf': 'stream', + }, + body: JSON.stringify(body), + }); + + // compression is on by default so if the request body is undefined + // the response header should include "gzip" and otherwise be "null" + if (body.compressResponse === undefined) { + expect(resp.headers.get('content-encoding')).to.be('gzip'); + } else if (body.compressResponse === false) { + expect(resp.headers.get('content-encoding')).to.be(null); + } - expect(resp.ok).to.be(true); - expect(resp.status).to.be(200); + expect(resp.ok).to.be(true); + expect(resp.status).to.be(200); - const stream = resp.body; + const stream = resp.body; - expect(stream).not.to.be(null); + expect(stream).not.to.be(null); - if (stream !== null) { - const data: any[] = []; - let chunkCounter = 0; - const parseStreamCallback = (c: number) => (chunkCounter = c); + if (stream !== null) { + const data: any[] = []; + let chunkCounter = 0; + const parseStreamCallback = (c: number) => (chunkCounter = c); - for await (const action of parseStream(stream, parseStreamCallback)) { - expect(action.type).not.to.be('error'); - data.push(action); - } + for await (const action of parseStream(stream, parseStreamCallback)) { + expect(action.type).not.to.be('error'); + data.push(action); + } - // Originally we assumed that we can assert streaming in contrast - // to non-streaming if there is more than one chunk. However, - // this turned out to be flaky since a stream could finish fast - // enough to contain only one chunk. So now we are checking if - // there's just one chunk or more. - expect(chunkCounter).to.be.greaterThan( - 0, - `Expected 'chunkCounter' to be greater than 0, got ${chunkCounter}.` - ); + // Originally we assumed that we can assert streaming in contrast + // to non-streaming if there is more than one chunk. However, + // this turned out to be flaky since a stream could finish fast + // enough to contain only one chunk. So now we are checking if + // there's just one chunk or more. + expect(chunkCounter).to.be.greaterThan( + 0, + `Expected 'chunkCounter' to be greater than 0, got ${chunkCounter}.` + ); - await assertAnalysisResult(data); + await assertAnalysisResult(data); + } } - } - it('should return group only in chunks with streaming with compression with flushFix', async () => { - await requestWithStreaming({ ...testData.requestBody, overrides }); - }); + it('should return group only in chunks with streaming with compression with flushFix', async () => { + await requestWithStreaming({ ...testData.requestBody, overrides }); + }); - it('should return group only in chunks with streaming with compression without flushFix', async () => { - await requestWithStreaming({ ...testData.requestBody, overrides, flushFix: false }); - }); + it('should return group only in chunks with streaming with compression without flushFix', async () => { + await requestWithStreaming({ ...testData.requestBody, overrides, flushFix: false }); + }); - it('should return group only in chunks with streaming without compression with flushFix', async () => { - await requestWithStreaming({ - ...testData.requestBody, - overrides, - compressResponse: false, + it('should return group only in chunks with streaming without compression with flushFix', async () => { + await requestWithStreaming({ + ...testData.requestBody, + overrides, + compressResponse: false, + }); }); - }); - it('should return group only in chunks with streaming without compression without flushFix', async () => { - await requestWithStreaming({ - ...testData.requestBody, - overrides, - compressResponse: false, - flushFix: false, + it('should return group only in chunks with streaming without compression without flushFix', async () => { + await requestWithStreaming({ + ...testData.requestBody, + overrides, + compressResponse: false, + flushFix: false, + }); }); }); }); diff --git a/x-pack/test/api_integration/apis/aiops/log_rate_analysis_no_index.ts b/x-pack/test/api_integration/apis/aiops/log_rate_analysis_no_index.ts index 088db66e082bd..118cae10c0c9d 100644 --- a/x-pack/test/api_integration/apis/aiops/log_rate_analysis_no_index.ts +++ b/x-pack/test/api_integration/apis/aiops/log_rate_analysis_no_index.ts @@ -10,53 +10,56 @@ import { ELASTIC_HTTP_VERSION_HEADER } from '@kbn/core-http-common'; import type { FtrProviderContext } from '../../ftr_provider_context'; -import { logRateAnalysisTestData } from './test_data'; +import { getLogRateAnalysisTestData, API_VERSIONS } from './test_data'; +import { getErrorActions } from './test_helpers'; export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); describe('POST /internal/aiops/log_rate_analysis - no index', () => { - logRateAnalysisTestData.forEach((testData) => { - describe(`with ${testData.testName}`, () => { - it('should return an error for non existing index without streaming', async () => { - const resp = await supertest - .post(`/internal/aiops/log_rate_analysis`) - .set('kbn-xsrf', 'kibana') - .set(ELASTIC_HTTP_VERSION_HEADER, '1') - .send({ - ...testData.requestBody, - index: 'does_not_exist', - }) - .expect(200); + API_VERSIONS.forEach((apiVersion) => { + getLogRateAnalysisTestData().forEach((testData) => { + describe(`with v${apiVersion} - ${testData.testName}`, () => { + it('should return an error for non existing index without streaming', async () => { + const resp = await supertest + .post(`/internal/aiops/log_rate_analysis`) + .set('kbn-xsrf', 'kibana') + .set(ELASTIC_HTTP_VERSION_HEADER, apiVersion) + .send({ + ...testData.requestBody, + index: 'does_not_exist', + }) + .expect(200); - const chunks: string[] = resp.body.toString().split('\n'); + const chunks: string[] = resp.body.toString().split('\n'); - expect(chunks.length).to.eql( - testData.expected.noIndexChunksLength, - `Expected 'noIndexChunksLength' to be ${testData.expected.noIndexChunksLength}, got ${chunks.length}.` - ); + expect(chunks.length).to.eql( + testData.expected.noIndexChunksLength, + `Expected 'noIndexChunksLength' to be ${testData.expected.noIndexChunksLength}, got ${chunks.length}.` + ); - const lastChunk = chunks.pop(); - expect(lastChunk).to.be(''); + const lastChunk = chunks.pop(); + expect(lastChunk).to.be(''); - let data: any[] = []; + let data: any[] = []; - expect(() => { - data = chunks.map((c) => JSON.parse(c)); - }).not.to.throwError(); + expect(() => { + data = chunks.map((c) => JSON.parse(c)); + }).not.to.throwError(); - expect(data.length).to.eql( - testData.expected.noIndexActionsLength, - `Expected 'noIndexActionsLength' to be ${testData.expected.noIndexActionsLength}, got ${data.length}.` - ); - data.forEach((d) => { - expect(typeof d.type).to.be('string'); - }); + expect(data.length).to.eql( + testData.expected.noIndexActionsLength, + `Expected 'noIndexActionsLength' to be ${testData.expected.noIndexActionsLength}, got ${data.length}.` + ); + data.forEach((d) => { + expect(typeof d.type).to.be('string'); + }); - const errorActions = data.filter((d) => d.type === testData.expected.errorFilter); - expect(errorActions.length).to.be(1); + const errorActions = getErrorActions(data); + expect(errorActions.length).to.be(1); - expect(errorActions[0].payload).to.be('Failed to fetch index information.'); + expect(errorActions[0].payload).to.be('Failed to fetch index information.'); + }); }); }); }); diff --git a/x-pack/test/api_integration/apis/aiops/test_data.ts b/x-pack/test/api_integration/apis/aiops/test_data.ts index 184925310940e..f323900e3437c 100644 --- a/x-pack/test/api_integration/apis/aiops/test_data.ts +++ b/x-pack/test/api_integration/apis/aiops/test_data.ts @@ -13,9 +13,16 @@ import { significantLogPatterns as artificialLogSignificantLogPatterns } from '@ import { finalSignificantTermGroups as artificialLogsSignificantTermGroups } from '@kbn/aiops-plugin/common/__mocks__/artificial_logs/final_significant_term_groups'; import { finalSignificantTermGroupsTextfield as artificialLogsSignificantTermGroupsTextfield } from '@kbn/aiops-plugin/common/__mocks__/artificial_logs/final_significant_term_groups_textfield'; +import type { + AiopsLogRateAnalysisSchema, + AiopsLogRateAnalysisApiVersion as ApiVersion, +} from '@kbn/aiops-plugin/common/api/log_rate_analysis/schema'; + import type { TestData } from './types'; -export const logRateAnalysisTestData: TestData[] = [ +export const API_VERSIONS: ApiVersion[] = ['1', '2']; + +export const getLogRateAnalysisTestData = (): Array> => [ { testName: 'ecommerce', esArchive: 'x-pack/test/functional/es_archives/ml/ecommerce', @@ -30,7 +37,7 @@ export const logRateAnalysisTestData: TestData[] = [ start: 0, timeFieldName: 'order_date', grouping: true, - }, + } as AiopsLogRateAnalysisSchema, expected: { chunksLength: 35, chunksLengthGroupOnly: 5, @@ -38,11 +45,6 @@ export const logRateAnalysisTestData: TestData[] = [ actionsLengthGroupOnly: 4, noIndexChunksLength: 4, noIndexActionsLength: 3, - significantTermFilter: 'add_significant_terms', - groupFilter: 'add_significant_terms_group', - groupHistogramFilter: 'add_significant_terms_group_histogram', - histogramFilter: 'add_significant_terms_histogram', - errorFilter: 'add_error', significantTerms: [ { key: 'day_of_week:Thursday', @@ -89,7 +91,7 @@ export const logRateAnalysisTestData: TestData[] = [ deviationMin: 1668855600000, deviationMax: 1668924000000, grouping: true, - }, + } as AiopsLogRateAnalysisSchema, expected: { chunksLength: 27, chunksLengthGroupOnly: 11, @@ -97,11 +99,6 @@ export const logRateAnalysisTestData: TestData[] = [ actionsLengthGroupOnly: 10, noIndexChunksLength: 4, noIndexActionsLength: 3, - significantTermFilter: 'add_significant_terms', - groupFilter: 'add_significant_terms_group', - groupHistogramFilter: 'add_significant_terms_group_histogram', - histogramFilter: 'add_significant_terms_histogram', - errorFilter: 'add_error', significantTerms: artificialLogSignificantTerms, groups: artificialLogsSignificantTermGroups, histogramLength: 20, @@ -121,7 +118,7 @@ export const logRateAnalysisTestData: TestData[] = [ deviationMin: 1668855600000, deviationMax: 1668924000000, grouping: true, - }, + } as AiopsLogRateAnalysisSchema, expected: { chunksLength: 30, chunksLengthGroupOnly: 11, @@ -129,11 +126,6 @@ export const logRateAnalysisTestData: TestData[] = [ actionsLengthGroupOnly: 10, noIndexChunksLength: 4, noIndexActionsLength: 3, - significantTermFilter: 'add_significant_terms', - groupFilter: 'add_significant_terms_group', - groupHistogramFilter: 'add_significant_terms_group_histogram', - histogramFilter: 'add_significant_terms_histogram', - errorFilter: 'add_error', significantTerms: [...artificialLogSignificantTerms, ...artificialLogSignificantLogPatterns], groups: artificialLogsSignificantTermGroupsTextfield, histogramLength: 20, @@ -153,7 +145,7 @@ export const logRateAnalysisTestData: TestData[] = [ deviationMin: 1668769200000, deviationMax: 1668837600000, grouping: true, - }, + } as AiopsLogRateAnalysisSchema, expected: { chunksLength: 27, chunksLengthGroupOnly: 11, @@ -161,11 +153,6 @@ export const logRateAnalysisTestData: TestData[] = [ actionsLengthGroupOnly: 10, noIndexChunksLength: 4, noIndexActionsLength: 3, - significantTermFilter: 'add_significant_terms', - groupFilter: 'add_significant_terms_group', - groupHistogramFilter: 'add_significant_terms_group_histogram', - histogramFilter: 'add_significant_terms_histogram', - errorFilter: 'add_error', significantTerms: artificialLogSignificantTerms, groups: artificialLogsSignificantTermGroups, histogramLength: 20, @@ -185,7 +172,7 @@ export const logRateAnalysisTestData: TestData[] = [ deviationMin: 1668769200000, deviationMax: 1668837600000, grouping: true, - }, + } as AiopsLogRateAnalysisSchema, expected: { chunksLength: 30, chunksLengthGroupOnly: 11, @@ -193,11 +180,6 @@ export const logRateAnalysisTestData: TestData[] = [ actionsLengthGroupOnly: 10, noIndexChunksLength: 4, noIndexActionsLength: 3, - significantTermFilter: 'add_significant_terms', - groupFilter: 'add_significant_terms_group', - groupHistogramFilter: 'add_significant_terms_group_histogram', - histogramFilter: 'add_significant_terms_histogram', - errorFilter: 'add_error', significantTerms: [...artificialLogSignificantTerms, ...artificialLogSignificantLogPatterns], groups: artificialLogsSignificantTermGroupsTextfield, histogramLength: 20, diff --git a/x-pack/test/api_integration/apis/aiops/test_helpers.ts b/x-pack/test/api_integration/apis/aiops/test_helpers.ts new file mode 100644 index 0000000000000..3818b96b9bbe1 --- /dev/null +++ b/x-pack/test/api_integration/apis/aiops/test_helpers.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { AiopsLogRateAnalysisApiVersion as ApiVersion } from '@kbn/aiops-plugin/common/api/log_rate_analysis/schema'; + +export const getAddSignificationItemsActions = (data: any[], apiVersion: ApiVersion) => + data.filter( + (d) => d.type === (apiVersion === '1' ? 'add_significant_terms' : 'add_significant_items') + ); + +export const getHistogramActions = (data: any[], apiVersion: ApiVersion) => + data.filter( + (d) => + d.type === + (apiVersion === '1' ? 'add_significant_terms_histogram' : 'add_significant_items_histogram') + ); + +export const getGroupActions = (data: any[], apiVersion: ApiVersion) => + data.filter( + (d) => + d.type === + (apiVersion === '1' ? 'add_significant_terms_group' : 'add_significant_items_group') + ); + +export const getGroupHistogramActions = (data: any[], apiVersion: ApiVersion) => + data.filter( + (d) => + d.type === + (apiVersion === '1' + ? 'add_significant_terms_group_histogram' + : 'add_significant_items_group_histogram') + ); + +export const getErrorActions = (data: any[]) => data.filter((d) => d.type === 'add_error'); diff --git a/x-pack/test/api_integration/apis/aiops/types.ts b/x-pack/test/api_integration/apis/aiops/types.ts index c4e9eb8191108..a990b1a346c07 100644 --- a/x-pack/test/api_integration/apis/aiops/types.ts +++ b/x-pack/test/api_integration/apis/aiops/types.ts @@ -5,16 +5,19 @@ * 2.0. */ -import type { AiopsApiLogRateAnalysis } from '@kbn/aiops-plugin/common/api'; +import type { + AiopsLogRateAnalysisSchema, + AiopsLogRateAnalysisApiVersion as ApiVersion, +} from '@kbn/aiops-plugin/common/api/log_rate_analysis/schema'; import type { SignificantTerm, SignificantTermGroup } from '@kbn/ml-agg-utils'; import type { LogRateAnalysisDataGenerator } from '../../../functional/services/aiops/log_rate_analysis_data_generator'; -export interface TestData { +export interface TestData { testName: string; esArchive?: string; dataGenerator?: LogRateAnalysisDataGenerator; - requestBody: AiopsApiLogRateAnalysis['body']; + requestBody: AiopsLogRateAnalysisSchema; expected: { chunksLength: number; chunksLengthGroupOnly: number; @@ -22,11 +25,6 @@ export interface TestData { actionsLengthGroupOnly: number; noIndexChunksLength: number; noIndexActionsLength: number; - significantTermFilter: 'add_significant_terms'; - groupFilter: 'add_significant_terms_group'; - groupHistogramFilter: 'add_significant_terms_group_histogram'; - histogramFilter: 'add_significant_terms_histogram'; - errorFilter: 'add_error'; significantTerms: SignificantTerm[]; groups: SignificantTermGroup[]; histogramLength: number; diff --git a/x-pack/test/api_integration_basic/apis/aiops/permissions.ts b/x-pack/test/api_integration_basic/apis/aiops/permissions.ts index 3a42d1e99d1a6..a77d9a634418e 100644 --- a/x-pack/test/api_integration_basic/apis/aiops/permissions.ts +++ b/x-pack/test/api_integration_basic/apis/aiops/permissions.ts @@ -11,50 +11,59 @@ import { ELASTIC_HTTP_VERSION_HEADER } from '@kbn/core-http-common'; import expect from '@kbn/expect'; -import type { AiopsApiLogRateAnalysis } from '@kbn/aiops-plugin/common/api'; +import type { + AiopsLogRateAnalysisSchema, + AiopsLogRateAnalysisApiVersion as ApiVersion, +} from '@kbn/aiops-plugin/common/api/log_rate_analysis/schema'; import type { FtrProviderContext } from '../../ftr_provider_context'; +const API_VERSIONS: ApiVersion[] = ['1', '2']; + export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const config = getService('config'); const kibanaServerUrl = formatUrl(config.get('servers.kibana')); - const requestBody: AiopsApiLogRateAnalysis['body'] = { - baselineMax: 1561719083292, - baselineMin: 1560954147006, - deviationMax: 1562254538692, - deviationMin: 1561986810992, - end: 2147483647000, - index: 'ft_ecommerce', - searchQuery: '{"bool":{"filter":[],"must":[{"match_all":{}}],"must_not":[]}}', - start: 0, - timeFieldName: 'order_date', - }; - describe('POST /internal/aiops/log_rate_analysis', () => { - it('should return permission denied without streaming', async () => { - await supertest - .post(`/internal/aiops/log_rate_analysis`) - .set('kbn-xsrf', 'kibana') - .set(ELASTIC_HTTP_VERSION_HEADER, '1') - .send(requestBody) - .expect(403); - }); + API_VERSIONS.forEach((apiVersion) => { + describe(`with API v${apiVersion}`, () => { + const requestBody: AiopsLogRateAnalysisSchema = { + baselineMax: 1561719083292, + baselineMin: 1560954147006, + deviationMax: 1562254538692, + deviationMin: 1561986810992, + end: 2147483647000, + index: 'ft_ecommerce', + searchQuery: '{"bool":{"filter":[],"must":[{"match_all":{}}],"must_not":[]}}', + start: 0, + timeFieldName: 'order_date', + }; - it('should return permission denied with streaming', async () => { - const response = await fetch(`${kibanaServerUrl}/internal/aiops/log_rate_analysis`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'kbn-xsrf': 'stream', - [ELASTIC_HTTP_VERSION_HEADER]: '1', - }, - body: JSON.stringify(requestBody), - }); + it('should return permission denied without streaming', async () => { + await supertest + .post(`/internal/aiops/log_rate_analysis`) + .set('kbn-xsrf', 'kibana') + .set(ELASTIC_HTTP_VERSION_HEADER, apiVersion) + .send(requestBody) + .expect(403); + }); + + it('should return permission denied with streaming', async () => { + const response = await fetch(`${kibanaServerUrl}/internal/aiops/log_rate_analysis`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'kbn-xsrf': 'stream', + [ELASTIC_HTTP_VERSION_HEADER]: apiVersion, + }, + body: JSON.stringify(requestBody), + }); - expect(response.ok).to.be(false); - expect(response.status).to.be(403); + expect(response.ok).to.be(false); + expect(response.status).to.be(403); + }); + }); }); }); };