[Security Solution] Grouping - fix null group missing

packages/kbn-securitysolution-grouping/src/components/accordion_panel/helpers.ts

@@ -30,3 +30,22 @@ export const createGroupFilter = (selectedGroup: string, query?: string) =>
         },
       ]
     : [];
+
+export const getNullGroupFilter = (selectedGroup: string) => [
+  {
+    meta: {
+      disabled: false,
+      negate: true,
+      alias: null,
+      key: selectedGroup,
+      field: selectedGroup,
+      value: 'exists',
+      type: 'exists',
+    },
+    query: {
+      exists: {
+        field: selectedGroup,
+      },
+    },
+  },
+];

packages/kbn-securitysolution-grouping/src/components/accordion_panel/index.tsx

@@ -11,7 +11,7 @@ import type { Filter } from '@kbn/es-query';
 import React, { useCallback, useEffect, useMemo, useRef } from 'react';
 import { firstNonNullValue } from '../../helpers';
 import type { RawBucket } from '../types';
-import { createGroupFilter } from './helpers';
+import { createGroupFilter, getNullGroupFilter } from './helpers';
 
 interface GroupPanelProps<T> {
   customAccordionButtonClassName?: string;
@@ -22,6 +22,7 @@ interface GroupPanelProps<T> {
   groupPanelRenderer?: JSX.Element;
   groupingLevel?: number;
   isLoading: boolean;
+  isNullGroup?: boolean;
   onGroupClose: () => void;
   onToggleGroup?: (isOpen: boolean, groupBucket: RawBucket<T>) => void;
   renderChildComponent: (groupFilter: Filter[]) => React.ReactElement;
@@ -49,6 +50,7 @@ const GroupPanelComponent = <T,>({
   groupPanelRenderer,
   groupingLevel = 0,
   isLoading,
+  isNullGroup = false,
   onGroupClose,
   onToggleGroup,
   renderChildComponent,
@@ -68,8 +70,11 @@ const GroupPanelComponent = <T,>({
   const groupFieldValue = useMemo(() => firstNonNullValue(groupBucket.key), [groupBucket.key]);
 
   const groupFilters = useMemo(
-    () => createGroupFilter(selectedGroup, groupFieldValue),
-    [groupFieldValue, selectedGroup]
+    () =>
+      isNullGroup
+        ? getNullGroupFilter(selectedGroup)
+        : createGroupFilter(selectedGroup, groupFieldValue),
+    [groupFieldValue, isNullGroup, selectedGroup]
   );
 
   const onToggle = useCallback(

packages/kbn-securitysolution-grouping/src/components/grouping.tsx

@@ -17,7 +17,7 @@ import type { Filter } from '@kbn/es-query';
 import React, { useMemo, useState } from 'react';
 import { METRIC_TYPE, UiCounterMetricType } from '@kbn/analytics';
 import { defaultUnit, firstNonNullValue } from '../helpers';
-import { createGroupFilter } from './accordion_panel/helpers';
+import { createGroupFilter, getNullGroupFilter } from './accordion_panel/helpers';
 import { GroupPanel } from './accordion_panel';
 import { GroupStats } from './accordion_panel/group_stats';
 import { EmptyGroupingComponent } from './empty_results_panel';
@@ -95,15 +95,21 @@ const GroupingComponent = <T,>({
       data?.groupByFields?.buckets?.map((groupBucket, groupNumber) => {
         const group = firstNonNullValue(groupBucket.key);
         const groupKey = `group-${groupNumber}-${group}`;
+        const isNullGroup = groupBucket.nullGroup.doc_count > 0;
 
         return (
           <span key={groupKey}>
             <GroupPanel
+              isNullGroup={isNullGroup}
               onGroupClose={onGroupClose}
               extraAction={
                 <GroupStats
                   bucketKey={groupKey}
-                  groupFilter={createGroupFilter(selectedGroup, group)}
+                  groupFilter={
+                    isNullGroup
+                      ? getNullGroupFilter(selectedGroup)
+                      : createGroupFilter(selectedGroup, group)
+                  }
                   groupNumber={groupNumber}
                   statRenderers={
                     groupStatsRenderer && groupStatsRenderer(selectedGroup, groupBucket)
@@ -159,6 +165,7 @@ const GroupingComponent = <T,>({
       trigger,
     ]
   );
+
   const pageCount = useMemo(
     () => (groupCount ? Math.ceil(groupCount / itemsPerPage) : 1),
     [groupCount, itemsPerPage]

packages/kbn-securitysolution-grouping/src/components/types.ts

@@ -6,13 +6,12 @@
  * Side Public License, v 1.
  */
 
-// copied from common/search_strategy/common
 export interface GenericBuckets {
   key: string | string[];
   key_as_string?: string; // contains, for example, formatted dates
   doc_count: number;
 }
-
+export type MissingAggregation = Record<'nullGroup', GenericBuckets>;
 export const NONE_GROUP_KEY = 'none';
 
 export type RawBucket<T> = GenericBuckets & T;
@@ -21,7 +20,7 @@ export type RawBucket<T> = GenericBuckets & T;
 // TODO: write developer docs for these fields
 export interface RootAggregation<T> {
   groupByFields?: {
-    buckets?: Array<RawBucket<T>>;
+    buckets?: Array<RawBucket<T> & MissingAggregation>;
   };
   groupsCount?: {
     value?: number | null;

packages/kbn-securitysolution-grouping/src/containers/query/helpers.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { ES_FIELD_TYPES } from '@kbn/field-types';
+
+export function getFieldTypeMissingValue(esType: string[]) {
+  const knownType: ES_FIELD_TYPES = esType[0] as ES_FIELD_TYPES;
+  switch (knownType) {
+    case ES_FIELD_TYPES.BYTE:
+    case ES_FIELD_TYPES.DOUBLE:
+    case ES_FIELD_TYPES.INTEGER:
+    case ES_FIELD_TYPES.LONG:
+    case ES_FIELD_TYPES.FLOAT:
+    case ES_FIELD_TYPES.HALF_FLOAT:
+    case ES_FIELD_TYPES.SCALED_FLOAT:
+    case ES_FIELD_TYPES.SHORT:
+    case ES_FIELD_TYPES.UNSIGNED_LONG:
+      return 'NaN';
+    case ES_FIELD_TYPES.IP:
+      return '0.0.0.0';
+    case ES_FIELD_TYPES.DATE:
+    case ES_FIELD_TYPES.DATE_NANOS:
+      return 0;
+    default:
+      return '-';
+  }
+}

packages/kbn-securitysolution-grouping/src/containers/query/index.ts

@@ -6,6 +6,7 @@
  * Side Public License, v 1.
  */
 
+import { getFieldTypeMissingValue } from './helpers';
 import type { GroupingQueryArgs, GroupingQuery } from './types';
 /** The maximum number of groups to render */
 export const DEFAULT_GROUP_BY_FIELD_SIZE = 10;
@@ -24,6 +25,7 @@ export const MAX_QUERY_SIZE = 10000;
  * @param rootAggregations Top level aggregations to get the groups number or overall groups metrics.
  * Array of {@link NamedAggregation}
  * @param runtimeMappings mappings of runtime fields [see runtimeMappings]{@link GroupingQueryArgs.runtimeMappings}
+ * @param selectedGroupEsTypes array of selected group types
  * @param size number of grouping results per page
  * @param sort add one or more sorts on specific fields
  * @param statsAggregations group level aggregations which correspond to {@link GroupStatsRenderer} configuration
@@ -35,10 +37,11 @@ export const getGroupingQuery = ({
   additionalFilters = [],
   from,
   groupByFields,
+  pageNumber,
   rootAggregations,
   runtimeMappings,
+  selectedGroupEsTypes,
   size = DEFAULT_GROUP_BY_FIELD_SIZE,
-  pageNumber,
   sort,
   statsAggregations,
   to,
@@ -51,13 +54,17 @@ export const getGroupingQuery = ({
             multi_terms: {
               terms: groupByFields.map((groupByField) => ({
                 field: groupByField,
+                // docs with empty field values will be grouped under this key
+                missing: getFieldTypeMissingValue(selectedGroupEsTypes),
               })),
               size: MAX_QUERY_SIZE,
             },
           }
         : {
             terms: {
               field: groupByFields[0],
+              // docs with empty field values will be grouped under this key
+              missing: getFieldTypeMissingValue(selectedGroupEsTypes),
               size: MAX_QUERY_SIZE,
             },
           }),
@@ -69,6 +76,11 @@ export const getGroupingQuery = ({
             size,
           },
         },
+        nullGroup: {
+          missing: {
+            field: groupByFields[0],
+          },
+        },
         ...(statsAggregations
           ? statsAggregations.reduce((aggObj, subAgg) => Object.assign(aggObj, subAgg), {})
           : {}),

packages/kbn-securitysolution-grouping/src/containers/query/types.ts

@@ -19,6 +19,7 @@ interface RangeAgg {
 }
 
 export type NamedAggregation = Record<string, estypes.AggregationsAggregationContainer>;
+export type MissingAggregation = Record<'nullGroup', estypes.AggregationsAggregationContainer>;
 
 export interface GroupingQueryArgs {
   additionalFilters: BoolAgg[];
@@ -28,6 +29,7 @@ export interface GroupingQueryArgs {
   runtimeMappings?: MappingRuntimeFields;
   additionalAggregationsRoot?: NamedAggregation[];
   pageNumber?: number;
+  selectedGroupEsTypes: string[];
   size?: number;
   sort?: Array<{ [category: string]: { order: 'asc' | 'desc' } }>;
   statsAggregations?: NamedAggregation[];
@@ -36,7 +38,7 @@ export interface GroupingQueryArgs {
 
 export interface MainAggregation extends NamedAggregation {
   groupByFields: {
-    aggs: NamedAggregation;
+    aggs: MissingAggregation & NamedAggregation;
     multi_terms?: estypes.AggregationsAggregationContainer['multi_terms'];
     terms?: estypes.AggregationsAggregationContainer['terms'];
   };

x-pack/plugins/security_solution/public/common/lib/kuery/index.ts

@@ -97,6 +97,15 @@ export const getBrowserFieldPath = (field: string, browserFields: BrowserFields)
   return [splitFields[0], 'fields', field];
 };
 
+export const getFieldEsTypes = (field: string, browserFields: BrowserFields): string[] => {
+  const pathBrowserField = getBrowserFieldPath(field, browserFields);
+  const browserField = get(pathBrowserField, browserFields);
+  if (browserField != null) {
+    return browserField.esTypes;
+  }
+  return [];
+};
+
 export const checkIfFieldTypeIsDate = (field: string, browserFields: BrowserFields) => {
   const pathBrowserField = getBrowserFieldPath(field, browserFields);
   const browserField = get(pathBrowserField, browserFields);

x-pack/plugins/security_solution/public/detections/components/alerts_table/alerts_grouping.tsx

@@ -132,10 +132,6 @@ const GroupedAlertsTableComponent: React.FC<AlertsTableComponentProps> = (props)
     setPageIndex((curr) => curr.map(() => DEFAULT_PAGE_INDEX));
   }, []);
 
-  useEffect(() => {
-    resetAllPagination();
-  }, [resetAllPagination, selectedGroups]);
-
   const setPageVar = useCallback(
     (newNumber: number, groupingLevel: number, pageType: 'index' | 'size') => {
       if (pageType === 'index') {
@@ -158,23 +154,31 @@ const GroupedAlertsTableComponent: React.FC<AlertsTableComponentProps> = (props)
     [setStoragePageSize]
   );
 
-  const nonGroupingFilters = useRef({
+  const paginationResetTriggers = useRef({
     defaultFilters: props.defaultFilters,
     globalFilters: props.globalFilters,
     globalQuery: props.globalQuery,
+    selectedGroups,
   });
 
   useEffect(() => {
-    const nonGrouping = {
+    const triggers = {
       defaultFilters: props.defaultFilters,
       globalFilters: props.globalFilters,
       globalQuery: props.globalQuery,
+      selectedGroups,
     };
-    if (!isEqual(nonGroupingFilters.current, nonGrouping)) {
+    if (!isEqual(paginationResetTriggers.current, triggers)) {
       resetAllPagination();
-      nonGroupingFilters.current = nonGrouping;
+      paginationResetTriggers.current = triggers;
     }
-  }, [props.defaultFilters, props.globalFilters, props.globalQuery, resetAllPagination]);
+  }, [
+    props.defaultFilters,
+    props.globalFilters,
+    props.globalQuery,
+    resetAllPagination,
+    selectedGroups,
+  ]);
 
   const getLevel = useCallback(
     (level: number, selectedGroup: string, parentGroupingFilter?: string) => {

x-pack/plugins/security_solution/public/detections/components/alerts_table/alerts_sub_grouping.tsx

@@ -15,7 +15,7 @@ import { getEsQueryConfig } from '@kbn/data-plugin/common';
 import type { DynamicGroupingProps } from '@kbn/securitysolution-grouping/src';
 import type { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/types';
 import type { TableIdLiteral } from '@kbn/securitysolution-data-table';
-import { combineQueries } from '../../../common/lib/kuery';
+import { combineQueries, getFieldEsTypes } from '../../../common/lib/kuery';
 import { SourcererScopeName } from '../../../common/store/sourcerer/model';
 import type { AlertsGroupingAggregation } from './grouping_settings/types';
 import type { Status } from '../../../../common/detection_engine/schemas/common';
@@ -140,17 +140,32 @@ export const GroupedSubLevelComponent: React.FC<AlertsTableComponentProps> = ({
     }
   }, [defaultFilters, globalFilters, globalQuery, parentGroupingFilter]);
 
+  const selectedGroupEsTypes = useMemo(
+    () => getFieldEsTypes(selectedGroup, browserFields),
+    [selectedGroup, browserFields]
+  );
+
   const queryGroups = useMemo(() => {
     return getAlertsGroupingQuery({
       additionalFilters,
       selectedGroup,
+      selectedGroupEsTypes,
       from,
       runtimeMappings,
       to,
       pageSize,
       pageIndex,
     });
-  }, [additionalFilters, from, pageIndex, pageSize, runtimeMappings, selectedGroup, to]);
+  }, [
+    additionalFilters,
+    from,
+    pageIndex,
+    pageSize,
+    runtimeMappings,
+    selectedGroup,
+    selectedGroupEsTypes,
+    to,
+  ]);
 
   const emptyGlobalQuery = useMemo(() => getGlobalQuery([]), [getGlobalQuery]);
 

x-pack/plugins/security_solution/public/detections/components/alerts_table/grouping_settings/query_builder.ts

@@ -27,6 +27,7 @@ interface AlertsGroupingQueryParams {
   pageSize: number;
   runtimeMappings: MappingRuntimeFields;
   selectedGroup: string;
+  selectedGroupEsTypes: string[];
   to: string;
 }
 
@@ -37,6 +38,7 @@ export const getAlertsGroupingQuery = ({
   pageSize,
   runtimeMappings,
   selectedGroup,
+  selectedGroupEsTypes,
   to,
 }: AlertsGroupingQueryParams) =>
   getGroupingQuery({
@@ -56,7 +58,9 @@ export const getAlertsGroupingQuery = ({
         : []),
     ],
     runtimeMappings,
+    selectedGroupEsTypes,
     size: pageSize,
+    sort: [{ unitsCount: { order: 'desc' } }],
     to,
   });