diff --git a/x-pack/plugins/aiops/common/__mocks__/artificial_logs/filtered_frequent_item_sets.ts b/x-pack/plugins/aiops/common/__mocks__/artificial_logs/filtered_frequent_item_sets.ts index 89e9c1fb141ab..e291093e7d4c1 100644 --- a/x-pack/plugins/aiops/common/__mocks__/artificial_logs/filtered_frequent_item_sets.ts +++ b/x-pack/plugins/aiops/common/__mocks__/artificial_logs/filtered_frequent_item_sets.ts @@ -9,7 +9,10 @@ import type { ItemSet } from '../../types'; export const filteredFrequentItemSets: ItemSet[] = [ { - set: { response_code: '500', url: 'home.php' }, + set: [ + { fieldName: 'response_code', fieldValue: '500' }, + { fieldName: 'url', fieldValue: 'home.php' }, + ], size: 2, maxPValue: 0.010770456205312423, doc_count: 792, @@ -17,7 +20,10 @@ export const filteredFrequentItemSets: ItemSet[] = [ total_doc_count: 1505, }, { - set: { user: 'Peter', url: 'home.php' }, + set: [ + { fieldName: 'user', fieldValue: 'Peter' }, + { fieldName: 'url', fieldValue: 'home.php' }, + ], size: 2, maxPValue: 0.010770456205312423, doc_count: 634, diff --git a/x-pack/plugins/aiops/common/__mocks__/artificial_logs/final_significant_item_groups.ts b/x-pack/plugins/aiops/common/__mocks__/artificial_logs/final_significant_item_groups.ts index 8f28e49ca3d7a..7fcf5ea425afe 100644 --- a/x-pack/plugins/aiops/common/__mocks__/artificial_logs/final_significant_item_groups.ts +++ b/x-pack/plugins/aiops/common/__mocks__/artificial_logs/final_significant_item_groups.ts @@ -9,32 +9,7 @@ import type { SignificantItemGroup } from '@kbn/ml-agg-utils'; export const finalSignificantItemGroups: SignificantItemGroup[] = [ { - docCount: 632, - group: [ - { - key: 'url:login.php', - type: 'keyword', - fieldName: 'url', - fieldValue: 'login.php', - docCount: 790, - duplicate: 2, - pValue: 0.012783309213417932, - }, - { - key: 'user:Peter', - type: 'keyword', - fieldName: 'user', - fieldValue: 'Peter', - docCount: 632, - duplicate: 2, - pValue: 0.012783309213417932, - }, - ], - id: '1937394803', - pValue: 0.012783309213417932, - }, - { - docCount: 792, + id: '2675980076', group: [ { key: 'response_code:500', @@ -42,8 +17,8 @@ export const finalSignificantItemGroups: SignificantItemGroup[] = [ fieldName: 'response_code', fieldValue: '500', docCount: 792, - duplicate: 2, pValue: 0.012783309213417932, + duplicate: 2, }, { key: 'url:home.php', @@ -51,15 +26,15 @@ export const finalSignificantItemGroups: SignificantItemGroup[] = [ fieldName: 'url', fieldValue: 'home.php', docCount: 792, - duplicate: 2, pValue: 0.00974308761016614, + duplicate: 2, }, ], - id: '2675980076', + docCount: 792, pValue: 0.00974308761016614, }, { - docCount: 790, + id: '3819687732', group: [ { key: 'response_code:500', @@ -67,8 +42,8 @@ export const finalSignificantItemGroups: SignificantItemGroup[] = [ fieldName: 'response_code', fieldValue: '500', docCount: 792, - duplicate: 2, pValue: 0.012783309213417932, + duplicate: 2, }, { key: 'url:login.php', @@ -76,15 +51,15 @@ export const finalSignificantItemGroups: SignificantItemGroup[] = [ fieldName: 'url', fieldValue: 'login.php', docCount: 790, - duplicate: 2, pValue: 0.012783309213417932, + duplicate: 2, }, ], - id: '3819687732', + docCount: 790, pValue: 0.012783309213417932, }, { - docCount: 636, + id: '2091742187', group: [ { key: 'url:home.php', @@ -92,8 +67,8 @@ export const finalSignificantItemGroups: SignificantItemGroup[] = [ fieldName: 'url', fieldValue: 'home.php', docCount: 792, - duplicate: 2, pValue: 0.00974308761016614, + duplicate: 2, }, { key: 'user:Peter', @@ -101,11 +76,36 @@ export const finalSignificantItemGroups: SignificantItemGroup[] = [ fieldName: 'user', fieldValue: 'Peter', docCount: 636, - duplicate: 2, pValue: 0.00974308761016614, + duplicate: 2, }, ], - id: '2091742187', + docCount: 636, pValue: 0.00974308761016614, }, + { + id: '1937394803', + group: [ + { + key: 'url:login.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'login.php', + docCount: 790, + pValue: 0.012783309213417932, + duplicate: 2, + }, + { + key: 'user:Peter', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Peter', + docCount: 632, + pValue: 0.012783309213417932, + duplicate: 2, + }, + ], + docCount: 632, + pValue: 0.012783309213417932, + }, ]; diff --git a/x-pack/plugins/aiops/common/__mocks__/artificial_logs/frequent_item_sets.ts b/x-pack/plugins/aiops/common/__mocks__/artificial_logs/frequent_item_sets.ts index b354bb00f7b2c..791512e6bffbc 100644 --- a/x-pack/plugins/aiops/common/__mocks__/artificial_logs/frequent_item_sets.ts +++ b/x-pack/plugins/aiops/common/__mocks__/artificial_logs/frequent_item_sets.ts @@ -9,7 +9,10 @@ import type { ItemSet } from '../../types'; export const frequentItemSets: ItemSet[] = [ { - set: { response_code: '500', url: 'home.php' }, + set: [ + { fieldName: 'response_code', fieldValue: '500' }, + { fieldName: 'url', fieldValue: 'home.php' }, + ], size: 2, maxPValue: 0.00974308761016614, doc_count: 792, @@ -17,7 +20,10 @@ export const frequentItemSets: ItemSet[] = [ total_doc_count: 2929, }, { - set: { response_code: '500', url: 'login.php' }, + set: [ + { fieldName: 'response_code', fieldValue: '500' }, + { fieldName: 'url', fieldValue: 'login.php' }, + ], size: 2, maxPValue: 0.012783309213417932, doc_count: 790, @@ -25,7 +31,10 @@ export const frequentItemSets: ItemSet[] = [ total_doc_count: 2929, }, { - set: { user: 'Peter', url: 'home.php' }, + set: [ + { fieldName: 'user', fieldValue: 'Peter' }, + { fieldName: 'url', fieldValue: 'home.php' }, + ], size: 2, maxPValue: 0.00974308761016614, doc_count: 636, @@ -33,7 +42,10 @@ export const frequentItemSets: ItemSet[] = [ total_doc_count: 2929, }, { - set: { user: 'Peter', url: 'login.php' }, + set: [ + { fieldName: 'user', fieldValue: 'Peter' }, + { fieldName: 'url', fieldValue: 'login.php' }, + ], size: 2, maxPValue: 0.012783309213417932, doc_count: 632, @@ -41,7 +53,10 @@ export const frequentItemSets: ItemSet[] = [ total_doc_count: 2929, }, { - set: { response_code: '500', user: 'Peter' }, + set: [ + { fieldName: 'response_code', fieldValue: '500' }, + { fieldName: 'user', fieldValue: 'Peter' }, + ], size: 2, maxPValue: 3.6085657805889595e-12, doc_count: 79, diff --git a/x-pack/plugins/aiops/common/constants.ts b/x-pack/plugins/aiops/common/constants.ts index 334bb64dd2484..c689333857083 100644 --- a/x-pack/plugins/aiops/common/constants.ts +++ b/x-pack/plugins/aiops/common/constants.ts @@ -14,6 +14,14 @@ export const LOG_RATE_ANALYSIS_SETTINGS = { * The minimum support value to be used for the frequent item sets aggration. */ FREQUENT_ITEMS_SETS_MINIMUM_SUPPORT: 0.001, + /** + * The maximum values per field to be used for the frequent item sets aggration. + */ + FREQUENT_ITEMS_SETS_FIELD_VALUE_LIMIT: 50, + /** + * The number of terms by field to fetch for the zero docs fallback analysis. + */ + TOP_TERMS_FALLBACK_SIZE: 100, } as const; /** diff --git a/x-pack/plugins/aiops/common/types.ts b/x-pack/plugins/aiops/common/types.ts index 67fca0b473549..3d7bb6d15ef27 100644 --- a/x-pack/plugins/aiops/common/types.ts +++ b/x-pack/plugins/aiops/common/types.ts @@ -15,7 +15,7 @@ export interface SignificantItemDuplicateGroup { export type FieldValuePairCounts = Record>; export interface ItemSet { - set: Record; + set: FieldValuePair[]; size: number; maxPValue: number; doc_count: number; diff --git a/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/get_group_table_items.test.ts b/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/get_group_table_items.test.ts index c674da3948644..0976382377245 100644 --- a/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/get_group_table_items.test.ts +++ b/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/get_group_table_items.test.ts @@ -15,34 +15,10 @@ describe('getGroupTableItems', () => { expect(groupTableItems).toEqual([ { - docCount: 632, - groupItemsSortedByUniqueness: [ - { - key: 'user:Peter', - type: 'keyword', - fieldName: 'user', - fieldValue: 'Peter', - docCount: 632, - duplicate: 2, - pValue: 0.012783309213417932, - }, - { - key: 'url:login.php', - type: 'keyword', - fieldName: 'url', - fieldValue: 'login.php', - docCount: 790, - duplicate: 2, - pValue: 0.012783309213417932, - }, - ], - histogram: undefined, - id: '1937394803', - pValue: 0.012783309213417932, - uniqueItemsCount: 0, - }, - { + id: '2675980076', docCount: 792, + pValue: 0.00974308761016614, + uniqueItemsCount: 0, groupItemsSortedByUniqueness: [ { key: 'response_code:500', @@ -50,8 +26,8 @@ describe('getGroupTableItems', () => { fieldName: 'response_code', fieldValue: '500', docCount: 792, - duplicate: 2, pValue: 0.012783309213417932, + duplicate: 2, }, { key: 'url:home.php', @@ -59,17 +35,16 @@ describe('getGroupTableItems', () => { fieldName: 'url', fieldValue: 'home.php', docCount: 792, - duplicate: 2, pValue: 0.00974308761016614, + duplicate: 2, }, ], - histogram: undefined, - id: '2675980076', - pValue: 0.00974308761016614, - uniqueItemsCount: 0, }, { + id: '3819687732', docCount: 790, + pValue: 0.012783309213417932, + uniqueItemsCount: 0, groupItemsSortedByUniqueness: [ { key: 'url:login.php', @@ -77,8 +52,8 @@ describe('getGroupTableItems', () => { fieldName: 'url', fieldValue: 'login.php', docCount: 790, - duplicate: 2, pValue: 0.012783309213417932, + duplicate: 2, }, { key: 'response_code:500', @@ -86,17 +61,16 @@ describe('getGroupTableItems', () => { fieldName: 'response_code', fieldValue: '500', docCount: 792, - duplicate: 2, pValue: 0.012783309213417932, + duplicate: 2, }, ], - histogram: undefined, - id: '3819687732', - pValue: 0.012783309213417932, - uniqueItemsCount: 0, }, { + id: '2091742187', docCount: 636, + pValue: 0.00974308761016614, + uniqueItemsCount: 0, groupItemsSortedByUniqueness: [ { key: 'user:Peter', @@ -104,8 +78,8 @@ describe('getGroupTableItems', () => { fieldName: 'user', fieldValue: 'Peter', docCount: 636, - duplicate: 2, pValue: 0.00974308761016614, + duplicate: 2, }, { key: 'url:home.php', @@ -113,14 +87,36 @@ describe('getGroupTableItems', () => { fieldName: 'url', fieldValue: 'home.php', docCount: 792, - duplicate: 2, pValue: 0.00974308761016614, + duplicate: 2, }, ], - histogram: undefined, - id: '2091742187', - pValue: 0.00974308761016614, + }, + { + id: '1937394803', + docCount: 632, + pValue: 0.012783309213417932, uniqueItemsCount: 0, + groupItemsSortedByUniqueness: [ + { + key: 'user:Peter', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Peter', + docCount: 632, + pValue: 0.012783309213417932, + duplicate: 2, + }, + { + key: 'url:login.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'login.php', + docCount: 790, + pValue: 0.012783309213417932, + duplicate: 2, + }, + ], }, ]); }); diff --git a/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/get_table_item_as_kql.test.ts b/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/get_table_item_as_kql.test.ts index 64f4fde55a660..28c4d8c40a676 100644 --- a/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/get_table_item_as_kql.test.ts +++ b/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/get_table_item_as_kql.test.ts @@ -20,9 +20,8 @@ describe('getTableItemAsKQL', () => { }); it('returns a KQL syntax for a group of significant items', () => { const groupTableItems = getGroupTableItems(finalSignificantItemGroups); - expect(getTableItemAsKQL(groupTableItems[0])).toBe('user:Peter AND url:login.php'); - expect(getTableItemAsKQL(groupTableItems[1])).toBe('response_code:500 AND url:home.php'); - expect(getTableItemAsKQL(groupTableItems[2])).toBe('url:login.php AND response_code:500'); - expect(getTableItemAsKQL(groupTableItems[3])).toBe('user:Peter AND url:home.php'); + expect(getTableItemAsKQL(groupTableItems[0])).toBe('response_code:500 AND url:home.php'); + expect(getTableItemAsKQL(groupTableItems[1])).toBe('url:login.php AND response_code:500'); + expect(getTableItemAsKQL(groupTableItems[2])).toBe('user:Peter AND url:home.php'); }); }); diff --git a/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/log_rate_analysis_results_table_groups.tsx b/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/log_rate_analysis_results_table_groups.tsx index 957385780ceaa..0d250a08f8c11 100644 --- a/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/log_rate_analysis_results_table_groups.tsx +++ b/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/log_rate_analysis_results_table_groups.tsx @@ -31,6 +31,7 @@ import { FormattedMessage } from '@kbn/i18n-react'; import type { SignificantItem } from '@kbn/ml-agg-utils'; import type { TimeRange as TimeRangeMs } from '@kbn/ml-date-picker'; import type { DataView } from '@kbn/data-views-plugin/public'; +import { stringHash } from '@kbn/ml-string-hash'; import { MiniHistogram } from '../mini_histogram'; @@ -203,7 +204,7 @@ export const LogRateAnalysisResultsGroupsTable: FC= MAX_GROUP_BADGES) break; valuesBadges.push( - + { expect(shouldClauses).toEqual([ { - terms: { - user: ['Peter'], + term: { + user: 'Peter', }, }, { - terms: { - response_code: ['500'], + term: { + response_code: '500', }, }, { - terms: { - url: ['home.php', 'login.php'], + term: { + url: 'home.php', + }, + }, + { + term: { + url: 'login.php', }, }, ]); diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_frequent_item_sets.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_frequent_item_sets.ts index ff02251b64f95..b5cd15636434c 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_frequent_item_sets.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_frequent_item_sets.ts @@ -51,17 +51,36 @@ export function groupDuplicates( return groups; } +/** + * Creates ES bool should clauses for each provided significant item. + * In previous versions of this helper we grouped values for the same field + * in a `terms` agg, but this might clash with the `minimum_should_match: 2` clause + * used in the query for the `frequent_item_sets`, because even multiple matches within + * the `terms` agg would count as just 1 match in the outer `should` part. + * + * @param significantItems + * @returns an array of term filters + */ export function getShouldClauses(significantItems: SignificantItem[]) { - return Array.from( - group(significantItems, ({ fieldName }) => fieldName), - ([field, values]) => ({ terms: { [field]: values.map((d) => d.fieldValue) } }) - ); + return significantItems.map((d) => ({ term: { [d.fieldName]: d.fieldValue } })); } +/** + * Creates a filter for each field to be used in the `frequent_items_sets` agg. + * Considers a limit per field to work around scaling limitations of the agg. + * + * @param significantItems + * @returns field filter for the `frequent_item_sets` agg + */ export function getFrequentItemSetsAggFields(significantItems: SignificantItem[]) { return Array.from( group(significantItems, ({ fieldName }) => fieldName), - ([field, values]) => ({ field, include: values.map((d) => String(d.fieldValue)) }) + ([field, values]) => ({ + field, + include: values + .map((d) => String(d.fieldValue)) + .slice(0, LOG_RATE_ANALYSIS_SETTINGS.FREQUENT_ITEMS_SETS_FIELD_VALUE_LIMIT), + }) ); } @@ -166,7 +185,7 @@ export async function fetchFrequentItemSets( fiss.forEach((fis) => { const result: ItemSet = { - set: {}, + set: [], size: 0, maxPValue: 0, doc_count: 0, @@ -174,16 +193,18 @@ export async function fetchFrequentItemSets( total_doc_count: 0, }; let maxPValue: number | undefined; - Object.entries(fis.key).forEach(([key, value]) => { - result.set[key] = value[0]; + Object.entries(fis.key).forEach(([key, values]) => { + values.forEach((value) => { + result.set.push({ fieldName: key, fieldValue: value }); - const pValue = sortedSignificantItems.find( - (t) => t.fieldName === key && t.fieldValue === value[0] - )?.pValue; + const pValue = sortedSignificantItems.find( + (t) => t.fieldName === key && t.fieldValue === value + )?.pValue; - if (pValue !== undefined && pValue !== null) { - maxPValue = Math.max(maxPValue ?? 0, pValue); - } + if (pValue !== undefined && pValue !== null) { + maxPValue = Math.max(maxPValue ?? 0, pValue); + } + }); }); if (maxPValue === undefined) { @@ -203,7 +224,7 @@ export async function fetchFrequentItemSets( return b.doc_count - a.doc_count; }); - const uniqueFields = uniq(results.flatMap((r) => Object.keys(r.set))); + const uniqueFields = uniq(results.flatMap((r) => r.set.map((d) => d.fieldName))); return { fields: uniqueFields, diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_terms_2_categories_counts.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_terms_2_categories_counts.ts index a39418807e40a..e991d5d925bde 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_terms_2_categories_counts.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_terms_2_categories_counts.ts @@ -99,10 +99,10 @@ export async function fetchTerms2CategoriesCounts( ) as estypes.MsearchMultisearchBody ); results.push({ - set: { - [term.fieldName]: term.fieldValue, - [category.fieldName]: category.fieldValue, - }, + set: [ + { fieldName: term.fieldName, fieldValue: term.fieldValue }, + { fieldName: category.fieldName, fieldValue: category.fieldValue }, + ], size: 2, maxPValue: Math.max(term.pValue ?? 1, category.pValue ?? 1), doc_count: 0, @@ -116,10 +116,7 @@ export async function fetchTerms2CategoriesCounts( searches.push( getTerm2CategoryCountRequest( params, - Object.entries(itemSet.set).map(([fieldName, fieldValue]) => ({ - fieldName, - fieldValue, - })), + itemSet.set, category.fieldName, { key: `${category.key}`, count: category.doc_count, examples: [], regex: '' }, from, @@ -127,10 +124,7 @@ export async function fetchTerms2CategoriesCounts( ) as estypes.MsearchMultisearchBody ); results.push({ - set: { - ...itemSet.set, - [category.fieldName]: category.fieldValue, - }, + set: [...itemSet.set, { fieldName: category.fieldName, fieldValue: category.fieldValue }], size: Object.keys(itemSet.set).length + 1, maxPValue: Math.max(itemSet.maxPValue ?? 1, category.pValue ?? 1), doc_count: 0, diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_top_terms.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_top_terms.ts index d1b8007249136..9b3d2fb95f7a0 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_top_terms.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_top_terms.ts @@ -15,7 +15,7 @@ import { type RandomSamplerWrapper, } from '@kbn/ml-random-sampler-utils'; -import { RANDOM_SAMPLER_SEED } from '../../../../common/constants'; +import { LOG_RATE_ANALYSIS_SETTINGS, RANDOM_SAMPLER_SEED } from '../../../../common/constants'; import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis/schema'; import { isRequestAbortedError } from '../../../lib/is_request_aborted_error'; @@ -60,7 +60,7 @@ export const getTopTermRequest = ( log_rate_top_terms: { terms: { field: fieldName, - size: 10, + size: LOG_RATE_ANALYSIS_SETTINGS.TOP_TERMS_FALLBACK_SIZE, }, }, }; diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_group_filter.test.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_group_filter.test.ts index 9f583f9e199df..eea96297b4622 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_group_filter.test.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_group_filter.test.ts @@ -14,12 +14,12 @@ describe('getGroupFilter', () => { expect(getGroupFilter(finalSignificantItemGroups[0])).toStrictEqual([ { term: { - url: 'login.php', + response_code: '500', }, }, { term: { - user: 'Peter', + url: 'home.php', }, }, ]); diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_significant_item_groups.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_significant_item_groups.ts index 27dc06ad343b9..ace3bd73cba69 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_significant_item_groups.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_significant_item_groups.ts @@ -5,6 +5,8 @@ * 2.0. */ +import { uniqBy } from 'lodash'; + import type { SignificantItem, SignificantItemGroup } from '@kbn/ml-agg-utils'; import { duplicateIdentifier } from './duplicate_identifier'; @@ -28,7 +30,7 @@ export function getSignificantItemGroups( (g) => g.group.length > 1 ); - // `frequent_item_sets` returns lot of different small groups of field/value pairs that co-occur. + // `frequent_item_sets` returns lots of different small groups of field/value pairs that co-occur. // The following steps analyse these small groups, identify overlap between these groups, // and then summarize them in larger groups where possible. @@ -58,5 +60,5 @@ export function getSignificantItemGroups( ) ); - return significantItemGroups; + return uniqBy(significantItemGroups, 'id'); } diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_simple_hierarchical_tree.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_simple_hierarchical_tree.ts index 54cce87526bb1..d046058aa1095 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_simple_hierarchical_tree.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_simple_hierarchical_tree.ts @@ -59,7 +59,7 @@ function dfDepthFirstSearch( displayOther: boolean ) { const filteredItemSets = iss.filter((is) => { - for (const [key, setValue] of Object.entries(is.set)) { + for (const { fieldName: key, fieldValue: setValue } of is.set) { if (key === field && setValue === value) { return true; } diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_value_counts.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_value_counts.ts index 5c54412e46a5b..d08d2a21cd3cb 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_value_counts.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_value_counts.ts @@ -9,10 +9,16 @@ import type { ItemSet } from '../../../../common/types'; export function getValueCounts(df: ItemSet[], field: string) { return df.reduce>((p, c) => { - if (c.set[field] === undefined) { + const fieldItems = c.set.filter((d) => d.fieldName === field); + + if (fieldItems.length === 0) { return p; } - p[c.set[field]] = p[c.set[field]] ? p[c.set[field]] + 1 : 1; + + for (const { fieldValue } of fieldItems) { + p[fieldValue] = p[fieldValue] ? p[fieldValue] + 1 : 1; + } + return p; }, {}); } diff --git a/x-pack/test/api_integration/apis/aiops/log_rate_analysis_full_analysis.ts b/x-pack/test/api_integration/apis/aiops/log_rate_analysis_full_analysis.ts index 352282b4ba41f..3fcc3dd339047 100644 --- a/x-pack/test/api_integration/apis/aiops/log_rate_analysis_full_analysis.ts +++ b/x-pack/test/api_integration/apis/aiops/log_rate_analysis_full_analysis.ts @@ -52,16 +52,15 @@ export default ({ getService }: FtrProviderContext) => { }); async function assertAnalysisResult(data: any[]) { - expect(data.length).to.eql( - testData.expected.actionsLength, - `Expected 'actionsLength' to be ${testData.expected.actionsLength}, got ${data.length}.` - ); data.forEach((d) => { expect(typeof d.type).to.be('string'); }); const addSignificantItemsActions = getAddSignificationItemsActions(data, apiVersion); - expect(addSignificantItemsActions.length).to.greaterThan(0); + expect(addSignificantItemsActions.length).to.greaterThan( + 0, + 'Expected significant items actions to be greater than 0.' + ); const significantItems = orderBy( addSignificantItemsActions.flatMap((d) => d.payload), @@ -80,7 +79,10 @@ export default ({ getService }: FtrProviderContext) => { expect(histogramActions.length).to.be(significantItems.length); // each histogram should have a length of 20 items. histograms.forEach((h, index) => { - expect(h.histogram.length).to.be(20); + expect(h.histogram.length).to.eql( + testData.expected.histogramLength, + `Expected histogram length to be ${testData.expected.histogramLength}, got ${h.histogram.length}` + ); }); const groupActions = getGroupActions(data, apiVersion); @@ -102,7 +104,10 @@ export default ({ getService }: FtrProviderContext) => { expect(groupHistograms.length).to.be(groups.length); // each histogram should have a length of 20 items. groupHistograms.forEach((h, index) => { - expect(h.histogram.length).to.be(20); + expect(h.histogram.length).to.eql( + testData.expected.histogramLength, + `Expected group histogram length to be ${testData.expected.histogramLength}, got ${h.histogram.length}` + ); }); } @@ -128,11 +133,6 @@ export default ({ getService }: FtrProviderContext) => { const chunks: string[] = resp.body.toString().split('\n'); - expect(chunks.length).to.eql( - testData.expected.chunksLength, - `Expected 'chunksLength' to be ${testData.expected.chunksLength}, got ${chunks.length}.` - ); - const lastChunk = chunks.pop(); expect(lastChunk).to.be(''); diff --git a/x-pack/test/api_integration/apis/aiops/log_rate_analysis_groups_only.ts b/x-pack/test/api_integration/apis/aiops/log_rate_analysis_groups_only.ts index fcc4bafabdf3c..f3aeeed8d067c 100644 --- a/x-pack/test/api_integration/apis/aiops/log_rate_analysis_groups_only.ts +++ b/x-pack/test/api_integration/apis/aiops/log_rate_analysis_groups_only.ts @@ -74,10 +74,6 @@ export default ({ getService }: FtrProviderContext) => { }); async function assertAnalysisResult(data: any[]) { - expect(data.length).to.eql( - testData.expected.actionsLengthGroupOnly, - `Expected 'actionsLengthGroupOnly' to be ${testData.expected.actionsLengthGroupOnly}, got ${data.length}.` - ); data.forEach((d) => { expect(typeof d.type).to.be('string'); }); @@ -101,7 +97,9 @@ export default ({ getService }: FtrProviderContext) => { expect(orderBy(groups, ['docCount'], ['desc'])).to.eql( orderBy(testData.expected.groups, ['docCount'], ['desc']), - 'Grouping result does not match expected values.' + `Grouping result does not match expected values. Expected ${JSON.stringify( + testData.expected.groups + )}, got ${JSON.stringify(groups)}` ); const groupHistogramActions = getGroupHistogramActions(data, apiVersion); @@ -110,7 +108,10 @@ export default ({ getService }: FtrProviderContext) => { expect(groupHistograms.length).to.be(groups.length); // each histogram should have a length of 20 items. groupHistograms.forEach((h, index) => { - expect(h.histogram.length).to.be(20); + expect(h.histogram.length).to.eql( + testData.expected.histogramLength, + `Expected group histogram length to be ${testData.expected.histogramLength}, got ${h.histogram.length}` + ); }); } @@ -136,11 +137,6 @@ export default ({ getService }: FtrProviderContext) => { const chunks: string[] = resp.body.toString().split('\n'); - expect(chunks.length).to.eql( - testData.expected.chunksLengthGroupOnly, - `Expected 'chunksLength' to be ${testData.expected.chunksLengthGroupOnly}, got ${chunks.length}.` - ); - const lastChunk = chunks.pop(); expect(lastChunk).to.be(''); diff --git a/x-pack/test/api_integration/apis/aiops/test_data.ts b/x-pack/test/api_integration/apis/aiops/test_data.ts index 3c1793ab9efe3..455574e4c7e76 100644 --- a/x-pack/test/api_integration/apis/aiops/test_data.ts +++ b/x-pack/test/api_integration/apis/aiops/test_data.ts @@ -14,11 +14,14 @@ import { finalSignificantItemGroups as artificialLogsSignificantItemGroups } fro import { finalSignificantItemGroupsTextfield as artificialLogsSignificantItemGroupsTextfield } from '@kbn/aiops-plugin/common/__mocks__/artificial_logs/final_significant_item_groups_textfield'; import { topTerms } from '@kbn/aiops-plugin/common/__mocks__/artificial_logs/top_terms'; import { topTermsGroups } from '@kbn/aiops-plugin/common/__mocks__/artificial_logs/top_terms_groups'; - import type { AiopsLogRateAnalysisSchema, AiopsLogRateAnalysisApiVersion as ApiVersion, } from '@kbn/aiops-plugin/common/api/log_rate_analysis/schema'; +import { + frequentItemSetsLargeArraysGroups, + frequentItemSetsLargeArraysSignificantItems, +} from '../../../functional/apps/aiops/log_rate_analysis/test_data/__mocks__/frequent_item_sets_large_arrays'; import type { TestData } from './types'; @@ -35,16 +38,12 @@ export const getLogRateAnalysisTestData = (): Array, expected: { - chunksLength: 36, - chunksLengthGroupOnly: 5, - actionsLength: 35, - actionsLengthGroupOnly: 4, noIndexChunksLength: 4, noIndexActionsLength: 3, significantItems: [ @@ -95,10 +94,6 @@ export const getLogRateAnalysisTestData = (): Array, expected: { - chunksLength: 28, - chunksLengthGroupOnly: 11, - actionsLength: 27, - actionsLengthGroupOnly: 10, noIndexChunksLength: 4, noIndexActionsLength: 3, significantItems: artificialLogSignificantTerms, @@ -122,10 +117,6 @@ export const getLogRateAnalysisTestData = (): Array, expected: { - chunksLength: 62, - chunksLengthGroupOnly: 32, - actionsLength: 61, - actionsLengthGroupOnly: 31, noIndexChunksLength: 4, noIndexActionsLength: 3, significantItems: topTerms, @@ -149,10 +140,6 @@ export const getLogRateAnalysisTestData = (): Array, expected: { - chunksLength: 62, - chunksLengthGroupOnly: 32, - actionsLength: 61, - actionsLengthGroupOnly: 31, noIndexChunksLength: 4, noIndexActionsLength: 3, significantItems: topTerms, @@ -176,10 +163,6 @@ export const getLogRateAnalysisTestData = (): Array, expected: { - chunksLength: 31, - chunksLengthGroupOnly: 11, - actionsLength: 30, - actionsLengthGroupOnly: 10, noIndexChunksLength: 4, noIndexActionsLength: 3, significantItems: [...artificialLogSignificantTerms, ...artificialLogSignificantLogPatterns], @@ -203,10 +186,6 @@ export const getLogRateAnalysisTestData = (): Array, expected: { - chunksLength: 28, - chunksLengthGroupOnly: 11, - actionsLength: 27, - actionsLengthGroupOnly: 10, noIndexChunksLength: 4, noIndexActionsLength: 3, significantItems: artificialLogSignificantTerms, @@ -230,10 +209,6 @@ export const getLogRateAnalysisTestData = (): Array, expected: { - chunksLength: 31, - chunksLengthGroupOnly: 11, - actionsLength: 30, - actionsLengthGroupOnly: 10, noIndexChunksLength: 4, noIndexActionsLength: 3, significantItems: [...artificialLogSignificantTerms, ...artificialLogSignificantLogPatterns], @@ -241,4 +216,27 @@ export const getLogRateAnalysisTestData = (): Array, + expected: { + noIndexChunksLength: 4, + noIndexActionsLength: 3, + groups: frequentItemSetsLargeArraysGroups, + significantItems: frequentItemSetsLargeArraysSignificantItems, + histogramLength: 1, + }, + }, ]; diff --git a/x-pack/test/api_integration/apis/aiops/types.ts b/x-pack/test/api_integration/apis/aiops/types.ts index df38825d9698d..acb948a2fea50 100644 --- a/x-pack/test/api_integration/apis/aiops/types.ts +++ b/x-pack/test/api_integration/apis/aiops/types.ts @@ -19,10 +19,6 @@ export interface TestData { dataGenerator?: LogRateAnalysisDataGenerator; requestBody: AiopsLogRateAnalysisSchema; expected: { - chunksLength: number; - chunksLengthGroupOnly: number; - actionsLength: number; - actionsLengthGroupOnly: number; noIndexChunksLength: number; noIndexActionsLength: number; significantItems: SignificantItem[]; diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/frequent_item_sets_large_arrays.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/frequent_item_sets_large_arrays.ts new file mode 100644 index 0000000000000..7296fdfc4dc66 --- /dev/null +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/frequent_item_sets_large_arrays.ts @@ -0,0 +1,2497 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SignificantItem, SignificantItemGroup } from '@kbn/ml-agg-utils'; + +// This was generated from real data using the following code: +// +// const data = testDataFrequentItemSets.response.hits.hits.map((doc) => { +// return doc.fields['files.pathAndFilename'].map((d) => `${stringHash(d)}`); +// }); +// +// The data is used for testing `frequent_item_sets` with somewhat larger arrays +// within the same field of a document. +export const frequentItemSetsLargeArraysSource = [ + [ + '3122117744', + '1846471239', + '3850123478', + '491462432', + '2145700416', + '1136953980', + '1423181790', + '784813175', + '3950054860', + ], + [ + '1451672588', + '3112205174', + '359183539', + '3434731980', + '1217080824', + '4072732370', + '764323024', + '1380284592', + '471153522', + '1104487811', + '3759120234', + '1896487004', + '225357271', + '1157393364', + '1600940924', + '1846471239', + '3178671192', + '32293944', + '2469262600', + '1052777253', + '801912434', + '1371557807', + '3850123478', + '505971607', + '732360692', + '2134530418', + '1416571574', + '2145700416', + '2887166226', + '1518623688', + '550501793', + '2179218510', + '742260364', + '1129761594', + '3625900628', + '3887145594', + '484181980', + '3779185628', + '3234612808', + ], + ['1416571574', '2145700416', '2887166226'], + ['1451672588', '978800164', '3122117744', '2145700416', '1136953980', '1129761594', '1969286181'], + [ + '471153522', + '1104487811', + '606140382', + '1896487004', + '225357271', + '1416571574', + '2145700416', + '2887166226', + '1901595662', + '1136953980', + '1129761594', + '1064752192', + ], + [ + '2432527302', + '1685611991', + '3709732414', + '3434731980', + '914276804', + '629341211', + '3561517034', + '1104874254', + '603630362', + '850909865', + '511726177', + '774389318', + '65096514', + '464464365', + '4072732370', + '3989522974', + '369147719', + '1756911944', + '2132661800', + '1351408438', + '2566334776', + '365681874', + '1994406554', + '4183628510', + '2411863132', + '153528430', + '2125040243', + '1919882921', + '484454192', + '2285116966', + '694867546', + '1275826846', + '1863819958', + '1332658570', + '764323024', + '4195111702', + '652426849', + '1380284592', + '471153522', + '2524040016', + '1325698447', + '998290200', + '1104487811', + '625944587', + '1140596684', + '1979833532', + '1475971156', + '1529680269', + '1925258778', + '3095335232', + '4009453878', + '953226620', + '3127779728', + '1432609585', + '3345566998', + '1867260038', + '3108151752', + '484256652', + '2138441449', + '1992199813', + '644711789', + '355508418', + '2463774400', + '1974037124', + '1437073018', + '2062952599', + '2317033220', + '165907306', + '39215494', + '2924749428', + '1831094494', + '606140382', + '1366342370', + '2084416768', + '55129418', + '899074608', + '1947835436', + '3759120234', + '978208972', + '1896487004', + '225357271', + '3096101166', + '3767723084', + '3035540120', + '853675916', + '1901780335', + '2639455238', + '1853107498', + '1157393364', + '2009375757', + '652223620', + '4028673796', + '1600940924', + '1808235940', + '1846471239', + '3178671192', + '1690511642', + '875090116', + '32293944', + '1073297840', + '1310779384', + '1940824684', + '1616339276', + '1878061294', + '2387317510', + '3899749436', + '1028474515', + '80529116', + '1081175620', + '3850123478', + '505971607', + '1091455238', + '1864834432', + '1569561080', + '1919542521', + '595513713', + '1627808055', + '1509804328', + '2134530418', + '894704468', + '1416571574', + '1227568232', + '1165010485', + '2145700416', + '3568288730', + '1758096804', + '2939442746', + '2057777897', + '456806730', + '1883416844', + '338634143', + '343273046', + '2290476620', + '1912970652', + '918833910', + '3419329722', + '2998254532', + '2887166226', + '3588882742', + '3305614422', + '333459111', + '3700075048', + '422724693', + '2172049200', + '912657112', + '1493078886', + '1924864316', + '1901595662', + '872035779', + '1143829704', + '2513611754', + '1136953980', + '1372713988', + '2087729311', + '209053355', + '1423181790', + '784813175', + '2380606900', + '1439966722', + '1518623688', + '3085917568', + '134825909', + '2848504534', + '1792385756', + '23762256', + '683600999', + '2993287316', + '3662867336', + '550501793', + '3948564194', + '1764158287', + '1124226732', + '3928811594', + '1598402559', + '679133986', + '3777335490', + '1652565419', + '1129761594', + '4043271966', + '4237245772', + '3657936252', + '281200596', + '1501943035', + '628118277', + '3950054860', + '3234612808', + '1548316370', + '315977894', + ], + [ + '178851049', + '1293256852', + '967305015', + '945750959', + '3112205174', + '359183539', + '744667649', + '355870296', + '65096514', + '2411863132', + '471153522', + '644711789', + '165907306', + '606140382', + '1366342370', + '2084416768', + '55129418', + '899074608', + '1947835436', + '1901780335', + '2009375757', + '892741844', + '1175209945', + '894704468', + '2145700416', + '3568288730', + '1758096804', + '2057777897', + '456806730', + '1883416844', + '338634143', + '94643047', + '578183673', + '3419329722', + '1372713988', + '209053355', + '784813175', + '715654930', + '679133986', + '3777335490', + '1652565419', + '1129761594', + '1445245168', + '484181980', + ], + [ + '3709732414', + '1774067546', + '629341211', + '3561517034', + '511726177', + '774389318', + '65096514', + '464464365', + '4072732370', + '1351408438', + '1880041108', + '1863819958', + '471153522', + '1104487811', + '625944587', + '1979833532', + '1475971156', + '3127779728', + '1432609585', + '644711789', + '2463774400', + '1437073018', + '2062952599', + '165907306', + '39215494', + '3279070400', + '33232218', + '3702216900', + '169386347', + '4282833864', + '343056858', + '1048252464', + '3328723350', + '766594295', + '864457688', + '1881750640', + '358188834', + '3464086212', + '1318649342', + '2007178432', + '2039579870', + '2028869426', + '1782757648', + '198736362', + '1459346396', + '261701072', + '860004934', + '1416825454', + '1901780335', + '1853107498', + '1157393364', + '1808235940', + '287718944', + '32293944', + '1310779384', + '1616339276', + '1878061294', + '3899749436', + '3850123478', + '1919542521', + '894704468', + '2803891618', + '1416571574', + '2145700416', + '1758096804', + '2057777897', + '456806730', + '1883416844', + '338634143', + '94643047', + '578183673', + '2998254532', + '333459111', + '912657112', + '1493078886', + '1901595662', + '872035779', + '2513611754', + '1136953980', + '1372713988', + '2087729311', + '209053355', + '1423181790', + '784813175', + '2380606900', + '1214124778', + '23762256', + '715654930', + '550501793', + '2148044396', + '679133986', + '3777335490', + '1652565419', + '4043271966', + '1418709661', + '3950054860', + '923662969', + '3234612808', + '1548316370', + '315977894', + '296863626', + ], + ['1183454328', '1104487811', '2639455238', '1808235940', '2145700416', '1329827342'], + [ + '3341153940', + '1499538523', + '1104487811', + '1979833532', + '2639455238', + '1853107498', + '1157393364', + '1808235940', + '1416571574', + '2145700416', + '2998254532', + '23762256', + '683600999', + '550501793', + '2148044396', + '1329827342', + ], + [ + '2432527302', + '396010280', + '888945742', + '1236602460', + '3709732414', + '629341211', + '3561517034', + '850909865', + '511726177', + '774389318', + '65096514', + '464464365', + '3989522974', + '369147719', + '1756911944', + '2132661800', + '1351408438', + '365681874', + '1994406554', + '4183628510', + '2411863132', + '153528430', + '2125040243', + '1919882921', + '484454192', + '2134641614', + '793940582', + '1782787351', + '1219720056', + '243787747', + '2970663200', + '1380206706', + '1473389172', + '3341153940', + '1023429210', + '1499538523', + '1103890809', + '1183454328', + '408379606', + '83894197', + '2349146622', + '1759997430', + '1880041108', + '672264778', + '1965916128', + '1213194741', + '1962101985', + '2524040016', + '1325698447', + '998290200', + '1104487811', + '625944587', + '1979833532', + '1475971156', + '4009453878', + '953226620', + '3127779728', + '1432609585', + '3345566998', + '1867260038', + '3108151752', + '484256652', + '1992199813', + '644711789', + '355508418', + '2463774400', + '52266782', + '1974037124', + '1437073018', + '2062952599', + '39215494', + '1853107498', + '1157393364', + '2009375757', + '4028673796', + '1600940924', + '1302044808', + '1808235940', + '1846471239', + '1690511642', + '875090116', + '32293944', + '1073297840', + '1310779384', + '1940824684', + '1616339276', + '1878061294', + '2387317510', + '3899749436', + '1028474515', + '80529116', + '1592492698', + '505971607', + '1091455238', + '1864834432', + '1569561080', + '1919542521', + '595513713', + '1627808055', + '892741844', + '1175209945', + '894704468', + '1416571574', + '1165010485', + '2145700416', + '3568288730', + '2057777897', + '456806730', + '1883416844', + '338634143', + '3419329722', + '2998254532', + '2073206647', + '3588882742', + '3305614422', + '684643770', + '333459111', + '3700075048', + '422724693', + '2172049200', + '912657112', + '1493078886', + '1924864316', + '2380606900', + '1439966722', + '1214124778', + '3085917568', + '134825909', + '2848504534', + '23762256', + '683600999', + '2993287316', + '550501793', + '3948564194', + '1764158287', + '1124226732', + '2148044396', + '3777335490', + '1652565419', + '4043271966', + '4237245772', + '3657936252', + '249588550', + '281200596', + '1501943035', + '3939021644', + '1418709661', + '628118277', + '998280499', + '1445245168', + '3950054860', + '1361617926', + '923662969', + '1264884266', + '3234612808', + '1548316370', + '315977894', + '296863626', + ], + [ + '1447828634', + '396010280', + '888945742', + '1439926706', + '1236602460', + '3709732414', + '629341211', + '3561517034', + '850909865', + '511726177', + '774389318', + '65096514', + '464464365', + '3989522974', + '369147719', + '1756911944', + '2132661800', + '1351408438', + '365681874', + '1994406554', + '4183628510', + '2411863132', + '153528430', + '2125040243', + '1919882921', + '484454192', + '2134641614', + '793940582', + '1782787351', + '1219720056', + '243787747', + '2970663200', + '1380206706', + '1473389172', + '3341153940', + '1023429210', + '1499538523', + '1103890809', + '1183454328', + '408379606', + '83894197', + '2349146622', + '1759997430', + '1880041108', + '672264778', + '1965916128', + '1213194741', + '1962101985', + '2524040016', + '1325698447', + '998290200', + '1104487811', + '625944587', + '1979833532', + '1475971156', + '4009453878', + '953226620', + '3127779728', + '1432609585', + '3345566998', + '1867260038', + '3108151752', + '484256652', + '1992199813', + '644711789', + '355508418', + '2463774400', + '52266782', + '1974037124', + '1437073018', + '2062952599', + '39215494', + '1853107498', + '1157393364', + '2009375757', + '4028673796', + '1600940924', + '1302044808', + '1808235940', + '1846471239', + '1690511642', + '875090116', + '32293944', + '1073297840', + '1310779384', + '1940824684', + '1616339276', + '1878061294', + '2387317510', + '3899749436', + '1028474515', + '80529116', + '1592492698', + '505971607', + '1091455238', + '1864834432', + '1569561080', + '1919542521', + '595513713', + '1627808055', + '892741844', + '1175209945', + '894704468', + '1416571574', + '1165010485', + '2145700416', + '3568288730', + '2057777897', + '456806730', + '1883416844', + '338634143', + '3419329722', + '2998254532', + '2073206647', + '3588882742', + '3305614422', + '684643770', + '333459111', + '3700075048', + '422724693', + '2172049200', + '912657112', + '1493078886', + '1924864316', + '2380606900', + '1439966722', + '1214124778', + '3085917568', + '134825909', + '2848504534', + '23762256', + '683600999', + '2993287316', + '550501793', + '3948564194', + '1764158287', + '1124226732', + '2148044396', + '3777335490', + '1652565419', + '4043271966', + '4237245772', + '3657936252', + '249588550', + '281200596', + '1501943035', + '3939021644', + '1418709661', + '628118277', + '998280499', + '1445245168', + '3950054860', + '1361617926', + '923662969', + '1264884266', + '3234612808', + '1548316370', + '315977894', + '296863626', + ], + [ + '850909865', + '464464365', + '3341153940', + '1103890809', + '83894197', + '1759997430', + '2524040016', + '998290200', + '1104487811', + '625944587', + '1979833532', + '4009453878', + '3127779728', + '52266782', + '1157393364', + '1808235940', + '1024718864', + '32293944', + '505971607', + '894704468', + '1416571574', + '2145700416', + '3568288730', + '2998254532', + '3588882742', + '3305614422', + '684643770', + '333459111', + '3700075048', + '422724693', + '2172049200', + '1493078886', + '1924864316', + '2380606900', + '1439966722', + '23762256', + '683600999', + '2148044396', + '3777335490', + '3234612808', + '1548316370', + ], + [ + '178851049', + '1685611991', + '869061781', + '3122117744', + '1712164321', + '914276804', + '850909865', + '65096514', + '464464365', + '369147719', + '1351408438', + '1325698447', + '998290200', + '1104487811', + '625944587', + '3127779728', + '1157393364', + '1808235940', + '1690511642', + '32293944', + '1616339276', + '2387317510', + '505971607', + '894704468', + '2145700416', + '2998254532', + '1214124778', + '550501793', + '3777335490', + '1652565419', + '1548316370', + ], + ['945750959', '850909865', '1416571574', '2145700416', '2998254532', '3305614422', '1548316370'], + [ + '2432527302', + '1144870203', + '1293256852', + '945750959', + '2861405270', + '3157340152', + '869061781', + '289756479', + '744667649', + '1217080824', + '362845723', + '355870296', + '3122117744', + '1774067546', + '1284354426', + '1712164321', + '629341211', + '3561517034', + '850909865', + '511726177', + '774389318', + '65096514', + '464464365', + '4072732370', + '3989522974', + '369147719', + '1756911944', + '2132661800', + '1351408438', + '365681874', + '1994406554', + '4183628510', + '2411863132', + '153528430', + '2125040243', + '1919882921', + '484454192', + '1028927795', + '2134641614', + '793940582', + '1782787351', + '1219720056', + '243787747', + '2970663200', + '1380206706', + '1473389172', + '3341153940', + '1023429210', + '1499538523', + '1103890809', + '1183454328', + '408379606', + '83894197', + '2349146622', + '1759997430', + '1880041108', + '672264778', + '1965916128', + '1213194741', + '1962101985', + '2524040016', + '1325698447', + '998290200', + '1104487811', + '625944587', + '1979833532', + '1475971156', + '4009453878', + '953226620', + '3127779728', + '1432609585', + '3345566998', + '1867260038', + '3108151752', + '484256652', + '1992199813', + '644711789', + '355508418', + '2463774400', + '52266782', + '1974037124', + '1437073018', + '2062952599', + '39215494', + '2173098414', + '1157393364', + '4035065194', + '1600940924', + '1808235940', + '1846471239', + '287718944', + '1690511642', + '875090116', + '32293944', + '1073297840', + '1310779384', + '1940824684', + '1616339276', + '1878061294', + '2387317510', + '3899749436', + '1028474515', + '80529116', + '1592492698', + '3850123478', + '505971607', + '1091455238', + '3521104552', + '1864834432', + '1569561080', + '1919542521', + '595513713', + '1627808055', + '892741844', + '1175209945', + '894704468', + '1622294140', + '1416571574', + '1165010485', + '2145700416', + '3568288730', + '2057777897', + '456806730', + '1883416844', + '338634143', + '3419329722', + '2998254532', + '2073206647', + '3588882742', + '3305614422', + '684643770', + '333459111', + '3700075048', + '422724693', + '2172049200', + '912657112', + '1493078886', + '1924864316', + '2380606900', + '1439966722', + '1214124778', + '3085917568', + '134825909', + '2848504534', + '23762256', + '683600999', + '2993287316', + '550501793', + '3948564194', + '1764158287', + '1124226732', + '2148044396', + '3777335490', + '1445245168', + '3950054860', + '484181980', + '1361617926', + '923662969', + '1264884266', + '3234612808', + '315977894', + '296863626', + ], +]; + +export const frequentItemSetsLargeArraysSignificantItems: SignificantItem[] = [ + { doc_count: 16, fieldValue: '2145700416' }, + { doc_count: 11, fieldValue: '1104487811' }, + { doc_count: 11, fieldValue: '1416571574' }, + { doc_count: 9, fieldValue: '1157393364' }, + { doc_count: 9, fieldValue: '1808235940' }, + { doc_count: 9, fieldValue: '2998254532' }, + { doc_count: 8, fieldValue: '32293944' }, + { doc_count: 8, fieldValue: '3777335490' }, + { doc_count: 8, fieldValue: '550501793' }, + { doc_count: 8, fieldValue: '894704468' }, + { doc_count: 7, fieldValue: '1548316370' }, + { doc_count: 7, fieldValue: '1979833532' }, + { doc_count: 7, fieldValue: '23762256' }, + { doc_count: 7, fieldValue: '3127779728' }, + { doc_count: 7, fieldValue: '3234612808' }, + { doc_count: 7, fieldValue: '464464365' }, + { doc_count: 7, fieldValue: '505971607' }, + { doc_count: 7, fieldValue: '625944587' }, + { doc_count: 7, fieldValue: '65096514' }, + { doc_count: 7, fieldValue: '850909865' }, + { doc_count: 6, fieldValue: '1351408438' }, + { doc_count: 6, fieldValue: '1493078886' }, + { doc_count: 6, fieldValue: '1616339276' }, + { doc_count: 6, fieldValue: '1652565419' }, + { doc_count: 6, fieldValue: '1846471239' }, + { doc_count: 6, fieldValue: '1883416844' }, + { doc_count: 6, fieldValue: '2057777897' }, + { doc_count: 6, fieldValue: '2148044396' }, + { doc_count: 6, fieldValue: '2380606900' }, + { doc_count: 6, fieldValue: '3305614422' }, + { doc_count: 6, fieldValue: '333459111' }, + { doc_count: 6, fieldValue: '338634143' }, + { doc_count: 6, fieldValue: '3568288730' }, + { doc_count: 6, fieldValue: '3950054860' }, + { doc_count: 6, fieldValue: '456806730' }, + { doc_count: 6, fieldValue: '644711789' }, + { doc_count: 6, fieldValue: '683600999' }, + { doc_count: 6, fieldValue: '998290200' }, + { doc_count: 5, fieldValue: '1129761594' }, + { doc_count: 5, fieldValue: '1136953980' }, + { doc_count: 5, fieldValue: '1214124778' }, + { doc_count: 5, fieldValue: '1310779384' }, + { doc_count: 5, fieldValue: '1325698447' }, + { doc_count: 5, fieldValue: '1432609585' }, + { doc_count: 5, fieldValue: '1437073018' }, + { doc_count: 5, fieldValue: '1439966722' }, + { doc_count: 5, fieldValue: '1475971156' }, + { doc_count: 5, fieldValue: '1600940924' }, + { doc_count: 5, fieldValue: '1690511642' }, + { doc_count: 5, fieldValue: '1853107498' }, + { doc_count: 5, fieldValue: '1878061294' }, + { doc_count: 5, fieldValue: '1919542521' }, + { doc_count: 5, fieldValue: '1924864316' }, + { doc_count: 5, fieldValue: '2062952599' }, + { doc_count: 5, fieldValue: '2172049200' }, + { doc_count: 5, fieldValue: '2387317510' }, + { doc_count: 5, fieldValue: '2411863132' }, + { doc_count: 5, fieldValue: '2463774400' }, + { doc_count: 5, fieldValue: '2524040016' }, + { doc_count: 5, fieldValue: '315977894' }, + { doc_count: 5, fieldValue: '3341153940' }, + { doc_count: 5, fieldValue: '3419329722' }, + { doc_count: 5, fieldValue: '3561517034' }, + { doc_count: 5, fieldValue: '3588882742' }, + { doc_count: 5, fieldValue: '369147719' }, + { doc_count: 5, fieldValue: '3700075048' }, + { doc_count: 5, fieldValue: '3850123478' }, + { doc_count: 5, fieldValue: '3899749436' }, + { doc_count: 5, fieldValue: '39215494' }, + { doc_count: 5, fieldValue: '4009453878' }, + { doc_count: 5, fieldValue: '422724693' }, + { doc_count: 5, fieldValue: '471153522' }, + { doc_count: 5, fieldValue: '511726177' }, + { doc_count: 5, fieldValue: '629341211' }, + { doc_count: 5, fieldValue: '774389318' }, + { doc_count: 5, fieldValue: '912657112' }, + { doc_count: 4, fieldValue: '1028474515' }, + { doc_count: 4, fieldValue: '1073297840' }, + { doc_count: 4, fieldValue: '1091455238' }, + { doc_count: 4, fieldValue: '1103890809' }, + { doc_count: 4, fieldValue: '1124226732' }, + { doc_count: 4, fieldValue: '1165010485' }, + { doc_count: 4, fieldValue: '1175209945' }, + { doc_count: 4, fieldValue: '1183454328' }, + { doc_count: 4, fieldValue: '134825909' }, + { doc_count: 4, fieldValue: '1445245168' }, + { doc_count: 4, fieldValue: '1499538523' }, + { doc_count: 4, fieldValue: '153528430' }, + { doc_count: 4, fieldValue: '1569561080' }, + { doc_count: 4, fieldValue: '1627808055' }, + { doc_count: 4, fieldValue: '1756911944' }, + { doc_count: 4, fieldValue: '1759997430' }, + { doc_count: 4, fieldValue: '1764158287' }, + { doc_count: 4, fieldValue: '1864834432' }, + { doc_count: 4, fieldValue: '1867260038' }, + { doc_count: 4, fieldValue: '1880041108' }, + { doc_count: 4, fieldValue: '1919882921' }, + { doc_count: 4, fieldValue: '1940824684' }, + { doc_count: 4, fieldValue: '1974037124' }, + { doc_count: 4, fieldValue: '1992199813' }, +].map((d) => ({ + doc_count: d.doc_count, + bg_count: 0, + fieldName: 'items', + fieldValue: d.fieldValue, + key: `items:${d.fieldValue}`, + normalizedScore: 0, + pValue: 1, + score: 0, + total_bg_count: 0, + total_doc_count: 0, + type: 'keyword', +})); + +export const frequentItemSetsLargeArraysGroups: SignificantItemGroup[] = [ + { + id: '1934634723', + group: [ + { + key: 'items:2145700416', + type: 'keyword', + fieldName: 'items', + fieldValue: '2145700416', + docCount: 11, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 11, + pValue: 1, + }, + { + id: '4064691194', + group: [ + { + key: 'items:1104487811', + type: 'keyword', + fieldName: 'items', + fieldValue: '1104487811', + docCount: 11, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 11, + pValue: 1, + }, + { + id: '278504570', + group: [ + { + key: 'items:1416571574', + type: 'keyword', + fieldName: 'items', + fieldValue: '1416571574', + docCount: 11, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 11, + pValue: 1, + }, + { + id: '2852990448', + group: [ + { + key: 'items:1157393364', + type: 'keyword', + fieldName: 'items', + fieldValue: '1157393364', + docCount: 9, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 9, + pValue: 1, + }, + { + id: '1149803824', + group: [ + { + key: 'items:2998254532', + type: 'keyword', + fieldName: 'items', + fieldValue: '2998254532', + docCount: 9, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 9, + pValue: 1, + }, + { + id: '2420022128', + group: [ + { + key: 'items:1808235940', + type: 'keyword', + fieldName: 'items', + fieldValue: '1808235940', + docCount: 9, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 9, + pValue: 1, + }, + { + id: '2094912046', + group: [ + { + key: 'items:32293944', + type: 'keyword', + fieldName: 'items', + fieldValue: '32293944', + docCount: 8, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 8, + pValue: 1, + }, + { + id: '1178034827', + group: [ + { + key: 'items:894704468', + type: 'keyword', + fieldName: 'items', + fieldValue: '894704468', + docCount: 8, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 8, + pValue: 1, + }, + { + id: '1506122217', + group: [ + { + key: 'items:3777335490', + type: 'keyword', + fieldName: 'items', + fieldValue: '3777335490', + docCount: 8, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 8, + pValue: 1, + }, + { + id: '3982887134', + group: [ + { + key: 'items:550501793', + type: 'keyword', + fieldName: 'items', + fieldValue: '550501793', + docCount: 8, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 8, + pValue: 1, + }, + { + id: '1840747948', + group: [ + { + key: 'items:23762256', + type: 'keyword', + fieldName: 'items', + fieldValue: '23762256', + docCount: 7, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 7, + pValue: 1, + }, + { + id: '1444907364', + group: [ + { + key: 'items:464464365', + type: 'keyword', + fieldName: 'items', + fieldValue: '464464365', + docCount: 7, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 7, + pValue: 1, + }, + { + id: '1328704288', + group: [ + { + key: 'items:625944587', + type: 'keyword', + fieldName: 'items', + fieldValue: '625944587', + docCount: 7, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 7, + pValue: 1, + }, + { + id: '1326810570', + group: [ + { + key: 'items:1979833532', + type: 'keyword', + fieldName: 'items', + fieldValue: '1979833532', + docCount: 7, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 7, + pValue: 1, + }, + { + id: '1030906346', + group: [ + { + key: 'items:3127779728', + type: 'keyword', + fieldName: 'items', + fieldValue: '3127779728', + docCount: 7, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 7, + pValue: 1, + }, + { + id: '1205078154', + group: [ + { + key: 'items:65096514', + type: 'keyword', + fieldName: 'items', + fieldValue: '65096514', + docCount: 7, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 7, + pValue: 1, + }, + { + id: '3905594284', + group: [ + { + key: 'items:3234612808', + type: 'keyword', + fieldName: 'items', + fieldValue: '3234612808', + docCount: 7, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 7, + pValue: 1, + }, + { + id: '779923180', + group: [ + { + key: 'items:1548316370', + type: 'keyword', + fieldName: 'items', + fieldValue: '1548316370', + docCount: 7, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 7, + pValue: 1, + }, + { + id: '4203644800', + group: [ + { + key: 'items:505971607', + type: 'keyword', + fieldName: 'items', + fieldValue: '505971607', + docCount: 7, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 7, + pValue: 1, + }, + { + id: '1237028536', + group: [ + { + key: 'items:850909865', + type: 'keyword', + fieldName: 'items', + fieldValue: '850909865', + docCount: 7, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 7, + pValue: 1, + }, + { + id: '3610776542', + group: [ + { + key: 'items:333459111', + type: 'keyword', + fieldName: 'items', + fieldValue: '333459111', + docCount: 6, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 6, + pValue: 1, + }, + { + id: '4004516458', + group: [ + { + key: 'items:1351408438', + type: 'keyword', + fieldName: 'items', + fieldValue: '1351408438', + docCount: 6, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 6, + pValue: 1, + }, + { + id: '100632746', + group: [ + { + key: 'items:1493078886', + type: 'keyword', + fieldName: 'items', + fieldValue: '1493078886', + docCount: 6, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 6, + pValue: 1, + }, + { + id: '2934370474', + group: [ + { + key: 'items:1616339276', + type: 'keyword', + fieldName: 'items', + fieldValue: '1616339276', + docCount: 6, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 6, + pValue: 1, + }, + { + id: '2551307178', + group: [ + { + key: 'items:2380606900', + type: 'keyword', + fieldName: 'items', + fieldValue: '2380606900', + docCount: 6, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 6, + pValue: 1, + }, + { + id: '1429234', + group: [ + { + key: 'items:338634143', + type: 'keyword', + fieldName: 'items', + fieldValue: '338634143', + docCount: 6, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 6, + pValue: 1, + }, + { + id: '133348698', + group: [ + { + key: 'items:456806730', + type: 'keyword', + fieldName: 'items', + fieldValue: '456806730', + docCount: 6, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 6, + pValue: 1, + }, + { + id: '3952677514', + group: [ + { + key: 'items:644711789', + type: 'keyword', + fieldName: 'items', + fieldValue: '644711789', + docCount: 6, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 6, + pValue: 1, + }, + { + id: '3477852394', + group: [ + { + key: 'items:1883416844', + type: 'keyword', + fieldName: 'items', + fieldValue: '1883416844', + docCount: 6, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 6, + pValue: 1, + }, + { + id: '130442059', + group: [ + { + key: 'items:2057777897', + type: 'keyword', + fieldName: 'items', + fieldValue: '2057777897', + docCount: 6, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 6, + pValue: 1, + }, + { + id: '2237656830', + group: [ + { + key: 'items:683600999', + type: 'keyword', + fieldName: 'items', + fieldValue: '683600999', + docCount: 6, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 6, + pValue: 1, + }, + { + id: '1080686951', + group: [ + { + key: 'items:998290200', + type: 'keyword', + fieldName: 'items', + fieldValue: '998290200', + docCount: 6, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 6, + pValue: 1, + }, + { + id: '817958570', + group: [ + { + key: 'items:1652565419', + type: 'keyword', + fieldName: 'items', + fieldValue: '1652565419', + docCount: 6, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 6, + pValue: 1, + }, + { + id: '1326053323', + group: [ + { + key: 'items:2148044396', + type: 'keyword', + fieldName: 'items', + fieldValue: '2148044396', + docCount: 6, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 6, + pValue: 1, + }, + { + id: '830437418', + group: [ + { + key: 'items:3950054860', + type: 'keyword', + fieldName: 'items', + fieldValue: '3950054860', + docCount: 6, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 6, + pValue: 1, + }, + { + id: '823747627', + group: [ + { + key: 'items:3568288730', + type: 'keyword', + fieldName: 'items', + fieldValue: '3568288730', + docCount: 6, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 6, + pValue: 1, + }, + { + id: '1450084203', + group: [ + { + key: 'items:3305614422', + type: 'keyword', + fieldName: 'items', + fieldValue: '3305614422', + docCount: 6, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 6, + pValue: 1, + }, + { + id: '1388829802', + group: [ + { + key: 'items:1846471239', + type: 'keyword', + fieldName: 'items', + fieldValue: '1846471239', + docCount: 6, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 6, + pValue: 1, + }, + { + id: '1557137580', + group: [ + { + key: 'items:1310779384', + type: 'keyword', + fieldName: 'items', + fieldValue: '1310779384', + docCount: 5, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 5, + pValue: 1, + }, + { + id: '2006719596', + group: [ + { + key: 'items:1432609585', + type: 'keyword', + fieldName: 'items', + fieldValue: '1432609585', + docCount: 5, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 5, + pValue: 1, + }, + { + id: '1660605452', + group: [ + { + key: 'items:1437073018', + type: 'keyword', + fieldName: 'items', + fieldValue: '1437073018', + docCount: 5, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 5, + pValue: 1, + }, + { + id: '956896360', + group: [ + { + key: 'items:1475971156', + type: 'keyword', + fieldName: 'items', + fieldValue: '1475971156', + docCount: 5, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 5, + pValue: 1, + }, + { + id: '1382579660', + group: [ + { + key: 'items:1853107498', + type: 'keyword', + fieldName: 'items', + fieldValue: '1853107498', + docCount: 5, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 5, + pValue: 1, + }, + { + id: '2888678696', + group: [ + { + key: 'items:1136953980', + type: 'keyword', + fieldName: 'items', + fieldValue: '1136953980', + docCount: 5, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 5, + pValue: 1, + }, + { + id: '2874685992', + group: [ + { + key: 'items:1214124778', + type: 'keyword', + fieldName: 'items', + fieldValue: '1214124778', + docCount: 5, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 5, + pValue: 1, + }, + { + id: '845567660', + group: [ + { + key: 'items:1325698447', + type: 'keyword', + fieldName: 'items', + fieldValue: '1325698447', + docCount: 5, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 5, + pValue: 1, + }, + { + id: '2473046696', + group: [ + { + key: 'items:1439966722', + type: 'keyword', + fieldName: 'items', + fieldValue: '1439966722', + docCount: 5, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 5, + pValue: 1, + }, + { + id: '611449132', + group: [ + { + key: 'items:1690511642', + type: 'keyword', + fieldName: 'items', + fieldValue: '1690511642', + docCount: 5, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 5, + pValue: 1, + }, + { + id: '1474040428', + group: [ + { + key: 'items:1129761594', + type: 'keyword', + fieldName: 'items', + fieldValue: '1129761594', + docCount: 5, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 5, + pValue: 1, + }, + { + id: '122100136', + group: [ + { + key: 'items:1600940924', + type: 'keyword', + fieldName: 'items', + fieldValue: '1600940924', + docCount: 5, + pValue: 1, + duplicate: 1, + }, + ], + docCount: 5, + pValue: 1, + }, + { + id: '942208673', + group: [ + { + key: 'items:1129761594', + type: 'keyword', + fieldName: 'items', + fieldValue: '1129761594', + duplicate: 1, + docCount: 5, + pValue: 1, + }, + { + key: 'items:1136953980', + type: 'keyword', + fieldName: 'items', + fieldValue: '1136953980', + duplicate: 1, + docCount: 5, + pValue: 1, + }, + { + key: 'items:1214124778', + type: 'keyword', + fieldName: 'items', + fieldValue: '1214124778', + duplicate: 1, + docCount: 5, + pValue: 1, + }, + { + key: 'items:1310779384', + type: 'keyword', + fieldName: 'items', + fieldValue: '1310779384', + duplicate: 1, + docCount: 5, + pValue: 1, + }, + { + key: 'items:1325698447', + type: 'keyword', + fieldName: 'items', + fieldValue: '1325698447', + duplicate: 1, + docCount: 5, + pValue: 1, + }, + { + key: 'items:1432609585', + type: 'keyword', + fieldName: 'items', + fieldValue: '1432609585', + duplicate: 1, + docCount: 5, + pValue: 1, + }, + { + key: 'items:1437073018', + type: 'keyword', + fieldName: 'items', + fieldValue: '1437073018', + duplicate: 1, + docCount: 5, + pValue: 1, + }, + { + key: 'items:1439966722', + type: 'keyword', + fieldName: 'items', + fieldValue: '1439966722', + duplicate: 1, + docCount: 5, + pValue: 1, + }, + { + key: 'items:1475971156', + type: 'keyword', + fieldName: 'items', + fieldValue: '1475971156', + duplicate: 1, + docCount: 5, + pValue: 1, + }, + { + key: 'items:1600940924', + type: 'keyword', + fieldName: 'items', + fieldValue: '1600940924', + duplicate: 1, + docCount: 5, + pValue: 1, + }, + { + key: 'items:1690511642', + type: 'keyword', + fieldName: 'items', + fieldValue: '1690511642', + duplicate: 1, + docCount: 5, + pValue: 1, + }, + { + key: 'items:1853107498', + type: 'keyword', + fieldName: 'items', + fieldValue: '1853107498', + duplicate: 1, + docCount: 5, + pValue: 1, + }, + { + key: 'items:1878061294', + type: 'keyword', + fieldName: 'items', + fieldValue: '1878061294', + duplicate: 1, + docCount: 5, + pValue: 1, + }, + { + key: 'items:1919542521', + type: 'keyword', + fieldName: 'items', + fieldValue: '1919542521', + duplicate: 1, + docCount: 5, + pValue: 1, + }, + { + key: 'items:1924864316', + type: 'keyword', + fieldName: 'items', + fieldValue: '1924864316', + duplicate: 1, + docCount: 5, + pValue: 1, + }, + { + key: 'items:2062952599', + type: 'keyword', + fieldName: 'items', + fieldValue: '2062952599', + duplicate: 1, + docCount: 5, + pValue: 1, + }, + { + key: 'items:2172049200', + type: 'keyword', + fieldName: 'items', + fieldValue: '2172049200', + duplicate: 1, + docCount: 5, + pValue: 1, + }, + { + key: 'items:2387317510', + type: 'keyword', + fieldName: 'items', + fieldValue: '2387317510', + duplicate: 1, + docCount: 5, + pValue: 1, + }, + { + key: 'items:2411863132', + type: 'keyword', + fieldName: 'items', + fieldValue: '2411863132', + duplicate: 1, + docCount: 5, + pValue: 1, + }, + { + key: 'items:2463774400', + type: 'keyword', + fieldName: 'items', + fieldValue: '2463774400', + duplicate: 1, + docCount: 5, + pValue: 1, + }, + { + key: 'items:2524040016', + type: 'keyword', + fieldName: 'items', + fieldValue: '2524040016', + duplicate: 1, + docCount: 5, + pValue: 1, + }, + { + key: 'items:315977894', + type: 'keyword', + fieldName: 'items', + fieldValue: '315977894', + duplicate: 1, + docCount: 5, + pValue: 1, + }, + { + key: 'items:3341153940', + type: 'keyword', + fieldName: 'items', + fieldValue: '3341153940', + duplicate: 1, + docCount: 5, + pValue: 1, + }, + { + key: 'items:3419329722', + type: 'keyword', + fieldName: 'items', + fieldValue: '3419329722', + duplicate: 1, + docCount: 5, + pValue: 1, + }, + { + key: 'items:3561517034', + type: 'keyword', + fieldName: 'items', + fieldValue: '3561517034', + duplicate: 1, + docCount: 5, + pValue: 1, + }, + { + key: 'items:3588882742', + type: 'keyword', + fieldName: 'items', + fieldValue: '3588882742', + duplicate: 1, + docCount: 5, + pValue: 1, + }, + { + key: 'items:369147719', + type: 'keyword', + fieldName: 'items', + fieldValue: '369147719', + duplicate: 1, + docCount: 5, + pValue: 1, + }, + { + key: 'items:3700075048', + type: 'keyword', + fieldName: 'items', + fieldValue: '3700075048', + duplicate: 1, + docCount: 5, + pValue: 1, + }, + { + key: 'items:3850123478', + type: 'keyword', + fieldName: 'items', + fieldValue: '3850123478', + duplicate: 1, + docCount: 5, + pValue: 1, + }, + { + key: 'items:3899749436', + type: 'keyword', + fieldName: 'items', + fieldValue: '3899749436', + duplicate: 1, + docCount: 5, + pValue: 1, + }, + { + key: 'items:39215494', + type: 'keyword', + fieldName: 'items', + fieldValue: '39215494', + duplicate: 1, + docCount: 5, + pValue: 1, + }, + { + key: 'items:4009453878', + type: 'keyword', + fieldName: 'items', + fieldValue: '4009453878', + duplicate: 1, + docCount: 5, + pValue: 1, + }, + { + key: 'items:422724693', + type: 'keyword', + fieldName: 'items', + fieldValue: '422724693', + duplicate: 1, + docCount: 5, + pValue: 1, + }, + { + key: 'items:471153522', + type: 'keyword', + fieldName: 'items', + fieldValue: '471153522', + duplicate: 1, + docCount: 5, + pValue: 1, + }, + { + key: 'items:511726177', + type: 'keyword', + fieldName: 'items', + fieldValue: '511726177', + duplicate: 1, + docCount: 5, + pValue: 1, + }, + { + key: 'items:629341211', + type: 'keyword', + fieldName: 'items', + fieldValue: '629341211', + duplicate: 1, + docCount: 5, + pValue: 1, + }, + { + key: 'items:774389318', + type: 'keyword', + fieldName: 'items', + fieldValue: '774389318', + duplicate: 1, + docCount: 5, + pValue: 1, + }, + { + key: 'items:912657112', + type: 'keyword', + fieldName: 'items', + fieldValue: '912657112', + duplicate: 1, + docCount: 5, + pValue: 1, + }, + ], + docCount: 5, + pValue: 1, + }, + { + id: '3524731882', + group: [ + { + key: 'items:1028474515', + type: 'keyword', + fieldName: 'items', + fieldValue: '1028474515', + duplicate: 1, + docCount: 4, + pValue: 1, + }, + { + key: 'items:1073297840', + type: 'keyword', + fieldName: 'items', + fieldValue: '1073297840', + duplicate: 1, + docCount: 4, + pValue: 1, + }, + { + key: 'items:1091455238', + type: 'keyword', + fieldName: 'items', + fieldValue: '1091455238', + duplicate: 1, + docCount: 4, + pValue: 1, + }, + { + key: 'items:1103890809', + type: 'keyword', + fieldName: 'items', + fieldValue: '1103890809', + duplicate: 1, + docCount: 4, + pValue: 1, + }, + { + key: 'items:1124226732', + type: 'keyword', + fieldName: 'items', + fieldValue: '1124226732', + duplicate: 1, + docCount: 4, + pValue: 1, + }, + { + key: 'items:1165010485', + type: 'keyword', + fieldName: 'items', + fieldValue: '1165010485', + duplicate: 1, + docCount: 4, + pValue: 1, + }, + { + key: 'items:1175209945', + type: 'keyword', + fieldName: 'items', + fieldValue: '1175209945', + duplicate: 1, + docCount: 4, + pValue: 1, + }, + { + key: 'items:1183454328', + type: 'keyword', + fieldName: 'items', + fieldValue: '1183454328', + duplicate: 1, + docCount: 4, + pValue: 1, + }, + { + key: 'items:134825909', + type: 'keyword', + fieldName: 'items', + fieldValue: '134825909', + duplicate: 1, + docCount: 4, + pValue: 1, + }, + { + key: 'items:1445245168', + type: 'keyword', + fieldName: 'items', + fieldValue: '1445245168', + duplicate: 1, + docCount: 4, + pValue: 1, + }, + { + key: 'items:1499538523', + type: 'keyword', + fieldName: 'items', + fieldValue: '1499538523', + duplicate: 1, + docCount: 4, + pValue: 1, + }, + { + key: 'items:153528430', + type: 'keyword', + fieldName: 'items', + fieldValue: '153528430', + duplicate: 1, + docCount: 4, + pValue: 1, + }, + { + key: 'items:1569561080', + type: 'keyword', + fieldName: 'items', + fieldValue: '1569561080', + duplicate: 1, + docCount: 4, + pValue: 1, + }, + { + key: 'items:1627808055', + type: 'keyword', + fieldName: 'items', + fieldValue: '1627808055', + duplicate: 1, + docCount: 4, + pValue: 1, + }, + { + key: 'items:1756911944', + type: 'keyword', + fieldName: 'items', + fieldValue: '1756911944', + duplicate: 1, + docCount: 4, + pValue: 1, + }, + { + key: 'items:1759997430', + type: 'keyword', + fieldName: 'items', + fieldValue: '1759997430', + duplicate: 1, + docCount: 4, + pValue: 1, + }, + { + key: 'items:1764158287', + type: 'keyword', + fieldName: 'items', + fieldValue: '1764158287', + duplicate: 1, + docCount: 4, + pValue: 1, + }, + { + key: 'items:1864834432', + type: 'keyword', + fieldName: 'items', + fieldValue: '1864834432', + duplicate: 1, + docCount: 4, + pValue: 1, + }, + { + key: 'items:1867260038', + type: 'keyword', + fieldName: 'items', + fieldValue: '1867260038', + duplicate: 1, + docCount: 4, + pValue: 1, + }, + { + key: 'items:1880041108', + type: 'keyword', + fieldName: 'items', + fieldValue: '1880041108', + duplicate: 1, + docCount: 4, + pValue: 1, + }, + { + key: 'items:1919882921', + type: 'keyword', + fieldName: 'items', + fieldValue: '1919882921', + duplicate: 1, + docCount: 4, + pValue: 1, + }, + { + key: 'items:1940824684', + type: 'keyword', + fieldName: 'items', + fieldValue: '1940824684', + duplicate: 1, + docCount: 4, + pValue: 1, + }, + { + key: 'items:1974037124', + type: 'keyword', + fieldName: 'items', + fieldValue: '1974037124', + duplicate: 1, + docCount: 4, + pValue: 1, + }, + { + key: 'items:1992199813', + type: 'keyword', + fieldName: 'items', + fieldValue: '1992199813', + duplicate: 1, + docCount: 4, + pValue: 1, + }, + ], + docCount: 4, + pValue: 1, + }, +]; diff --git a/x-pack/test/functional/services/aiops/log_rate_analysis_data_generator.ts b/x-pack/test/functional/services/aiops/log_rate_analysis_data_generator.ts index 824e296d448d3..bd1d6c94810b9 100644 --- a/x-pack/test/functional/services/aiops/log_rate_analysis_data_generator.ts +++ b/x-pack/test/functional/services/aiops/log_rate_analysis_data_generator.ts @@ -11,6 +11,8 @@ import { LOG_RATE_ANALYSIS_TYPE } from '@kbn/aiops-utils'; import { FtrProviderContext } from '../../ftr_provider_context'; +import { frequentItemSetsLargeArraysSource } from '../../apps/aiops/log_rate_analysis/test_data/__mocks__/frequent_item_sets_large_arrays'; + const LOG_RATE_ANALYSYS_DATA_GENERATOR = { KIBANA_SAMPLE_DATA_LOGS: 'kibana_sample_data_logs', FAREQUOTE_WITH_SPIKE: 'farequote_with_spike', @@ -24,6 +26,7 @@ const LOG_RATE_ANALYSYS_DATA_GENERATOR = { ARTIFICIAL_LOGS_WITH_SPIKE_TEXTFIELD: 'artificial_logs_with_spike_textfield', ARTIFICIAL_LOGS_WITH_DIP: 'artificial_logs_with_dip', ARTIFICIAL_LOGS_WITH_DIP_TEXTFIELD: 'artificial_logs_with_dip_textfield', + LARGE_ARRAYS: 'large_arrays', } as const; export type LogRateAnalysisDataGenerator = typeof LOG_RATE_ANALYSYS_DATA_GENERATOR[keyof typeof LOG_RATE_ANALYSYS_DATA_GENERATOR]; @@ -272,14 +275,10 @@ export function LogRateAnalysisDataGeneratorProvider({ getService }: FtrProvider case 'artificial_logs_with_dip_zerodocsfallback': case 'artificial_logs_with_dip_textfield_zerodocsfallback': try { - const indexExists = await es.indices.exists({ + await es.indices.delete({ index: dataGenerator, + ignore_unavailable: true, }); - if (indexExists) { - await es.indices.delete({ - index: dataGenerator, - }); - } } catch (e) { log.info(`Could not delete index '${dataGenerator}' in before() callback`); } @@ -324,6 +323,47 @@ export function LogRateAnalysisDataGeneratorProvider({ getService }: FtrProvider }); break; + case 'large_arrays': + try { + await es.indices.delete({ + index: dataGenerator, + ignore_unavailable: true, + }); + } catch (e) { + log.info(`Could not delete index '${dataGenerator}' in before() callback`); + } + + // Create index with mapping + await es.indices.create({ + index: dataGenerator, + mappings: { + properties: { + items: { type: 'keyword' }, + '@timestamp': { type: 'date' }, + }, + }, + }); + + interface DocWithArray { + '@timestamp': number; + items: string[]; + } + + await es.bulk({ + refresh: 'wait_for', + body: frequentItemSetsLargeArraysSource.reduce((docs, items) => { + if (docs === undefined) return []; + docs.push({ index: { _index: dataGenerator } }); + docs.push({ + '@timestamp': 1562254538700, + items, + }); + return docs; + }, [] as estypes.BulkRequest['body']), + }); + + break; + default: log.error(`Unsupported data generator '${dataGenerator}`); } @@ -349,6 +389,7 @@ export function LogRateAnalysisDataGeneratorProvider({ getService }: FtrProvider case 'artificial_logs_with_spike_textfield_zerodocsfallback': case 'artificial_logs_with_dip_zerodocsfallback': case 'artificial_logs_with_dip_textfield_zerodocsfallback': + case 'large_arrays': try { await es.indices.delete({ index: dataGenerator,