From 3ed0058a9c8b52dc8a34b11f4869f0dda6904e2a Mon Sep 17 00:00:00 2001 From: Kaituo Li Date: Mon, 26 Aug 2024 14:03:26 -0700 Subject: [PATCH] Add Missing Value Imputation Options and Update Shingle Size Limit (#851) * Add Missing Value Imputation Options and Update Shingle Size Limit This PR introduces a new missing value imputation feature with three options: zero, fixed values, and previous values. When the fixed values option is selected, users can input custom values for each feature. Validation logic has been added to ensure that the feature names and the number of custom values match the number of enabled features. Additionally, the review page and model configuration page have been updated to properly display these new parameters. This PR also increases the maximum shingle size to 128, aligning with the backend implementation. Testing: * Updated existing unit tests to reflect these changes. * Conducted manual end-to-end testing. Signed-off-by: Kaituo Li * remove console log Signed-off-by: Kaituo Li --------- Signed-off-by: Kaituo Li --- public/models/interfaces.ts | 2 + public/models/types.ts | 21 ++ .../AdvancedSettings/AdvancedSettings.tsx | 183 +++++++++++++++--- .../containers/ConfigureModel.tsx | 64 +++++- .../pages/ConfigureModel/models/interfaces.ts | 11 ++ .../utils/__tests__/helpers.test.tsx | 56 ++++++ .../pages/ConfigureModel/utils/constants.tsx | 9 + public/pages/ConfigureModel/utils/helpers.ts | 98 ++++++++++ .../pages/DefineDetector/utils/constants.tsx | 2 + .../AdditionalSettings/AdditionalSettings.tsx | 16 ++ .../DetectorConfig/containers/Features.tsx | 12 +- .../__tests__/DetectorConfig.test.tsx | 59 ++++-- .../DetectorConfig.test.tsx.snap | 141 ++++++++++++++ .../AdditionalSettings/AdditionalSettings.tsx | 16 ++ .../ModelConfigurationFields.tsx | 10 +- .../__tests__/AdditionalSettings.test.tsx | 9 +- .../ModelConfigurationFields.test.tsx | 3 + .../AdditionalSettings.test.tsx.snap | 143 ++++++++++++++ .../ModelConfigurationFields.test.tsx.snap | 68 +++++++ .../ReviewAndCreate.test.tsx.snap | 136 +++++++++++++ public/pages/ReviewAndCreate/utils/helpers.ts | 22 ++- public/redux/reducers/__tests__/utils.ts | 47 ++++- 22 files changed, 1067 insertions(+), 61 deletions(-) diff --git a/public/models/interfaces.ts b/public/models/interfaces.ts index c10cf764..f6dfc651 100644 --- a/public/models/interfaces.ts +++ b/public/models/interfaces.ts @@ -15,6 +15,7 @@ import { DETECTOR_STATE } from '../../server/utils/constants'; import { Duration } from 'moment'; import moment from 'moment'; import { MDSQueryParams } from '../../server/models/types'; +import { ImputationOption } from './types'; export type FieldInfo = { label: string; @@ -210,6 +211,7 @@ export type Detector = { taskState?: DETECTOR_STATE; taskProgress?: number; taskError?: string; + imputationOption?: ImputationOption; }; export type DetectorListItem = { diff --git a/public/models/types.ts b/public/models/types.ts index f56af8e9..6d559276 100644 --- a/public/models/types.ts +++ b/public/models/types.ts @@ -12,3 +12,24 @@ export type AggregationOption = { label: string; }; + +export type ImputationOption = { + method: ImputationMethod; + defaultFill?: Array<{ featureName: string; data: number }>; +}; + +export enum ImputationMethod { + /** + * This method replaces all missing values with 0's. It's a simple approach, but it may introduce bias if the data is not centered around zero. + */ + ZERO = 'ZERO', + /** + * This method replaces missing values with a predefined set of values. The values are the same for each input dimension, and they need to be specified by the user. + */ + FIXED_VALUES = 'FIXED_VALUES', + /** + * This method replaces missing values with the last known value in the respective input dimension. It's a commonly used method for time series data, where temporal continuity is expected. + */ + PREVIOUS = 'PREVIOUS', +} + diff --git a/public/pages/ConfigureModel/components/AdvancedSettings/AdvancedSettings.tsx b/public/pages/ConfigureModel/components/AdvancedSettings/AdvancedSettings.tsx index 9b0bd2f1..0ccac512 100644 --- a/public/pages/ConfigureModel/components/AdvancedSettings/AdvancedSettings.tsx +++ b/public/pages/ConfigureModel/components/AdvancedSettings/AdvancedSettings.tsx @@ -17,9 +17,12 @@ import { EuiTitle, EuiCompressedFieldNumber, EuiSpacer, + EuiCompressedSelect, + EuiButtonIcon, + EuiCompressedFieldText, } from '@elastic/eui'; -import { Field, FieldProps } from 'formik'; -import React, { useState } from 'react'; +import { Field, FieldProps, FieldArray, } from 'formik'; +import React, { useEffect, useState } from 'react'; import ContentPanel from '../../../../components/ContentPanel/ContentPanel'; import { BASE_DOCS_LINK } from '../../../../utils/constants'; import { @@ -28,6 +31,7 @@ import { validatePositiveInteger, } from '../../../../utils/utils'; import { FormattedFormRow } from '../../../../components/FormattedFormRow/FormattedFormRow'; +import { SparseDataOptionValue } from '../../utils/constants'; interface AdvancedSettingsProps {} @@ -35,6 +39,14 @@ export function AdvancedSettings(props: AdvancedSettingsProps) { const [showAdvancedSettings, setShowAdvancedSettings] = useState(false); + // Options for the sparse data handling dropdown + const sparseDataOptions = [ + { value: SparseDataOptionValue.IGNORE, text: 'Ignore missing value' }, + { value: SparseDataOptionValue.PREVIOUS_VALUE, text: 'Previous value' }, + { value: SparseDataOptionValue.SET_TO_ZERO, text: 'Set to zero' }, + { value: SparseDataOptionValue.CUSTOM_VALUE, text: 'Custom value' }, + ]; + return ( {showAdvancedSettings ? : null} {showAdvancedSettings ? ( - - {({ field, form }: FieldProps) => ( - + + {({ field, form }: FieldProps) => ( + - - - - - - -

intervals

- - - - - )} - + ]} + hintLink={`${BASE_DOCS_LINK}/ad`} + isInvalid={isInvalid(field.name, form)} + error={getError(field.name, form)} + > + + + + + + +

intervals

+ + + + + )} + + + + {({ field, form }: FieldProps) => { + // Add an empty row if CUSTOM_VALUE is selected and no rows exist + useEffect(() => { + if ( + field.value === SparseDataOptionValue.CUSTOM_VALUE && + (!form.values.imputationOption?.custom_value || + form.values.imputationOption.custom_value.length === 0) + ) { + form.setFieldValue('imputationOption.custom_value', [ + { featureName: '', value: undefined }, + ]); + } + }, [field.value, form]); + + return ( + <> + + + + + {/* Conditionally render the "Custom value" title and the input fields when 'Custom value' is selected */} + {field.value === SparseDataOptionValue.CUSTOM_VALUE && ( + <> + + +
Custom value
+ + + + {(arrayHelpers) => ( + <> + {form.values.imputationOption.custom_value?.map((_, index) => ( + + + + {({ field }: FieldProps) => ( + + )} + + + + + {/* the value is set to field.value || '' to avoid displaying 0 as a default value. */ } + {({ field, form }: FieldProps) => ( + + )} + + + + arrayHelpers.remove(index)} + /> + + + ))} + + { /* add new rows with empty values when the add button is clicked. */} + + arrayHelpers.push({ featureName: '', value: 0 }) + } + aria-label="Add row" + /> + + )} + + + )} + + ); + }} + + ) : null} ); diff --git a/public/pages/ConfigureModel/containers/ConfigureModel.tsx b/public/pages/ConfigureModel/containers/ConfigureModel.tsx index 2b558973..8b31ae40 100644 --- a/public/pages/ConfigureModel/containers/ConfigureModel.tsx +++ b/public/pages/ConfigureModel/containers/ConfigureModel.tsx @@ -39,8 +39,8 @@ import { focusOnFirstWrongFeature, getCategoryFields, focusOnCategoryField, - getShingleSizeFromObject, modelConfigurationToFormik, + focusOnImputationOption, } from '../utils/helpers'; import { formikToDetector } from '../../ReviewAndCreate/utils/helpers'; import { formikToModelConfiguration } from '../utils/helpers'; @@ -53,7 +53,7 @@ import { CoreServicesContext } from '../../../components/CoreServices/CoreServic import { Detector } from '../../../models/interfaces'; import { prettifyErrorMessage } from '../../../../server/utils/helpers'; import { DetectorDefinitionFormikValues } from '../../DefineDetector/models/interfaces'; -import { ModelConfigurationFormikValues } from '../models/interfaces'; +import { ModelConfigurationFormikValues, FeaturesFormikValues } from '../models/interfaces'; import { CreateDetectorFormikValues } from '../../CreateDetectorSteps/models/interfaces'; import { DETECTOR_STATE } from '../../../../server/utils/constants'; import { getErrorMessage } from '../../../utils/utils'; @@ -68,6 +68,7 @@ import { getSavedObjectsClient, } from '../../../services'; import { DataSourceViewConfig } from '../../../../../../src/plugins/data_source_management/public'; +import { SparseDataOptionValue } from '../utils/constants'; interface ConfigureModelRouterProps { detectorId?: string; @@ -173,6 +174,49 @@ export function ConfigureModel(props: ConfigureModelProps) { } }, [hasError]); + const validateImputationOption = ( + formikValues: ModelConfigurationFormikValues, + errors: any + ) => { + const imputationOption = get(formikValues, 'imputationOption', null); + + // Initialize an array to hold individual error messages + const customValueErrors: string[] = []; + + // Validate imputationOption when method is CUSTOM_VALUE + if (imputationOption && imputationOption.imputationMethod === SparseDataOptionValue.CUSTOM_VALUE) { + const enabledFeatures = formikValues.featureList.filter( + (feature: FeaturesFormikValues) => feature.featureEnabled + ); + + // Validate that the number of custom values matches the number of enabled features + if ((imputationOption.custom_value || []).length !== enabledFeatures.length) { + customValueErrors.push( + `The number of custom values (${(imputationOption.custom_value || []).length}) does not match the number of enabled features (${enabledFeatures.length}).` + ); + } + + // Validate that each enabled feature has a corresponding custom value + const missingFeatures = enabledFeatures + .map((feature: FeaturesFormikValues) => feature.featureName) + .filter( + (name: string | undefined) => + !imputationOption.custom_value?.some((cv) => cv.featureName === name) + ); + + if (missingFeatures.length > 0) { + customValueErrors.push( + `The following enabled features are missing in custom values: ${missingFeatures.join(', ')}.` + ); + } + + // If there are any custom value errors, join them into a single string with proper formatting + if (customValueErrors.length > 0) { + errors.custom_value = customValueErrors.join(' '); + } + } + }; + const handleFormValidation = async ( formikProps: FormikProps ) => { @@ -185,7 +229,12 @@ export function ConfigureModel(props: ConfigureModelProps) { formikProps.setFieldTouched('featureList'); formikProps.setFieldTouched('categoryField', isHCDetector); formikProps.setFieldTouched('shingleSize'); + formikProps.setFieldTouched('imputationOption'); + formikProps.validateForm().then((errors) => { + // Call the extracted validation method + validateImputationOption(formikProps.values, errors); + if (isEmpty(errors)) { if (props.isEdit) { // TODO: possibly add logic to also start RT and/or historical from here. Need to think @@ -204,11 +253,22 @@ export function ConfigureModel(props: ConfigureModelProps) { props.setStep(3); } } else { + const customValueError = get(errors, 'custom_value') + if (customValueError) { + core.notifications.toasts.addDanger( + customValueError + ); + focusOnImputationOption(); + return; + } + // TODO: can add focus to all components or possibly customize error message too if (get(errors, 'featureList')) { focusOnFirstWrongFeature(errors, formikProps.setFieldTouched); } else if (get(errors, 'categoryField')) { focusOnCategoryField(); + } else { + console.log(`unexpected error ${JSON.stringify(errors)}`); } core.notifications.toasts.addDanger( diff --git a/public/pages/ConfigureModel/models/interfaces.ts b/public/pages/ConfigureModel/models/interfaces.ts index ac5cd906..f7575bc4 100644 --- a/public/pages/ConfigureModel/models/interfaces.ts +++ b/public/pages/ConfigureModel/models/interfaces.ts @@ -18,6 +18,7 @@ export interface ModelConfigurationFormikValues { categoryFieldEnabled: boolean; categoryField: string[]; shingleSize: number; + imputationOption?: ImputationFormikValues; } export interface FeaturesFormikValues { @@ -30,3 +31,13 @@ export interface FeaturesFormikValues { aggregationOf?: AggregationOption[]; newFeature?: boolean; } + +export interface ImputationFormikValues { + imputationMethod?: string; + custom_value?: CustomValueFormikValues[]; +} + +export interface CustomValueFormikValues { + featureName: string; + data: number; +} diff --git a/public/pages/ConfigureModel/utils/__tests__/helpers.test.tsx b/public/pages/ConfigureModel/utils/__tests__/helpers.test.tsx index d0d34e15..49b19750 100644 --- a/public/pages/ConfigureModel/utils/__tests__/helpers.test.tsx +++ b/public/pages/ConfigureModel/utils/__tests__/helpers.test.tsx @@ -13,6 +13,9 @@ import { getRandomDetector } from '../../../../redux/reducers/__tests__/utils'; import { prepareDetector } from '../../utils/helpers'; import { FEATURE_TYPE } from '../../../../models/interfaces'; import { FeaturesFormikValues } from '../../models/interfaces'; +import { modelConfigurationToFormik } from '../helpers'; +import { SparseDataOptionValue } from '../constants'; +import { ImputationMethod } from '../../../../models/types'; describe('featuresToFormik', () => { test('should able to add new feature', () => { @@ -71,4 +74,57 @@ describe('featuresToFormik', () => { // ...randomDetector.featureAttributes.slice(1), // ]); // }); + test('should return correct values if detector is not null', () => { + const randomDetector = getRandomDetector(); + const adFormikValues = modelConfigurationToFormik(randomDetector); + + const imputationOption = randomDetector.imputationOption; + if (imputationOption) { + const method = imputationOption.method; + if (ImputationMethod.FIXED_VALUES === method) { + expect(adFormikValues.imputationOption?.imputationMethod).toEqual( + SparseDataOptionValue.CUSTOM_VALUE + ); + expect(randomDetector.imputationOption?.defaultFill).toBeDefined(); + expect( + randomDetector.imputationOption?.defaultFill?.length + ).toBeGreaterThan(0); + + const formikCustom = adFormikValues.imputationOption?.custom_value; + expect(formikCustom).toBeDefined(); + + const defaultFill = randomDetector.imputationOption?.defaultFill || []; + + defaultFill.forEach(({ featureName, data }) => { + const matchingFormikValue = formikCustom?.find( + (item) => item.featureName === featureName + ); + + // Assert that a matching value was found + expect(matchingFormikValue).toBeDefined(); + + // Assert that the data matches + expect(matchingFormikValue?.data).toEqual(data); + }); + } else { + if (ImputationMethod.ZERO === method) { + expect(adFormikValues.imputationOption?.imputationMethod).toEqual( + SparseDataOptionValue.SET_TO_ZERO + ); + } else { + expect(adFormikValues.imputationOption?.imputationMethod).toEqual( + SparseDataOptionValue.PREVIOUS_VALUE + ); + } + expect(adFormikValues.imputationOption?.custom_value).toEqual( + undefined + ); + } + } else { + expect(adFormikValues.imputationOption?.custom_value).toEqual(undefined); + expect(adFormikValues.imputationOption?.imputationMethod).toEqual( + SparseDataOptionValue.IGNORE + ); + } + }); }); diff --git a/public/pages/ConfigureModel/utils/constants.tsx b/public/pages/ConfigureModel/utils/constants.tsx index 8aef72a9..b0a84b2e 100644 --- a/public/pages/ConfigureModel/utils/constants.tsx +++ b/public/pages/ConfigureModel/utils/constants.tsx @@ -22,6 +22,7 @@ export const INITIAL_MODEL_CONFIGURATION_VALUES: ModelConfigurationFormikValues categoryFieldEnabled: false, categoryField: [], shingleSize: DEFAULT_SHINGLE_SIZE, + imputationOption: undefined }; export const INITIAL_FEATURE_VALUES: FeaturesFormikValues = { @@ -67,3 +68,11 @@ export const FEATURE_FIELDS = [ 'aggregationBy', 'aggregationQuery', ]; + +// an enum for the sparse data handling options +export enum SparseDataOptionValue { + IGNORE = 'ignore', + PREVIOUS_VALUE = 'previous_value', + SET_TO_ZERO = 'set_to_zero', + CUSTOM_VALUE = 'custom_value', +} diff --git a/public/pages/ConfigureModel/utils/helpers.ts b/public/pages/ConfigureModel/utils/helpers.ts index 05c8bdf3..73e4ce78 100644 --- a/public/pages/ConfigureModel/utils/helpers.ts +++ b/public/pages/ConfigureModel/utils/helpers.ts @@ -21,12 +21,21 @@ import { DataTypes } from '../../../redux/reducers/opensearch'; import { ModelConfigurationFormikValues, FeaturesFormikValues, + CustomValueFormikValues, + ImputationFormikValues, } from '../../ConfigureModel/models/interfaces'; import { INITIAL_MODEL_CONFIGURATION_VALUES } from '../../ConfigureModel/utils/constants'; import { featuresToUIMetadata, formikToFeatureAttributes, } from '../../ReviewAndCreate/utils/helpers'; +import { + ImputationMethod, + ImputationOption, +} from '../../../models/types'; +import { + SparseDataOptionValue +} from './constants' export const getFieldOptions = ( allFields: { [key: string]: string[] }, @@ -120,6 +129,7 @@ export const validateFeatures = (values: any) => { }; } }); + return hasError ? { featureList: featureErrors } : undefined; }; @@ -203,6 +213,11 @@ export const getCategoryFields = (dataTypes: DataTypes) => { return keywordFields.concat(ipFields); }; +export const focusOnImputationOption = () => { + const component = document.getElementById('imputationOption'); + component?.focus(); +}; + export const getShingleSizeFromObject = (obj: object) => { return get(obj, 'shingleSize', DEFAULT_SHINGLE_SIZE); }; @@ -227,12 +242,33 @@ export function modelConfigurationToFormik( if (isEmpty(detector)) { return initialValues; } + + var imputationMethod = imputationMethodToFormik(detector); + + var defaultFillArray: CustomValueFormikValues[] = []; + + if (SparseDataOptionValue.CUSTOM_VALUE === imputationMethod) { + const defaultFill = get(detector, 'imputationOption.defaultFill', null) as Array<{ featureName: string; data: number }> | null; + defaultFillArray = defaultFill + ? defaultFill.map(({ featureName, data }) => ({ + featureName, + data, + })) + : []; + } + + const imputationFormikValues: ImputationFormikValues = { + imputationMethod: imputationMethod, + custom_value: SparseDataOptionValue.CUSTOM_VALUE === imputationMethod ? defaultFillArray : undefined, + }; + return { ...initialValues, featureList: featuresToFormik(detector), categoryFieldEnabled: !isEmpty(get(detector, 'categoryField', [])), categoryField: get(detector, 'categoryField', []), shingleSize: get(detector, 'shingleSize', DEFAULT_SHINGLE_SIZE), + imputationOption: imputationFormikValues, }; } @@ -280,6 +316,7 @@ export function formikToModelConfiguration( categoryField: !isEmpty(values?.categoryField) ? values.categoryField : undefined, + imputationOption: formikToImputationOption(values.imputationOption), } as Detector; return detectorBody; @@ -327,3 +364,64 @@ export function formikToSimpleAggregation(value: FeaturesFormikValues) { return {}; } } + +export function formikToImputationOption(imputationFormikValues?: ImputationFormikValues): ImputationOption | undefined { + // Map the formik method to the imputation method; return undefined if method is not recognized. + const method = formikToImputationMethod(imputationFormikValues?.imputationMethod); + if (!method) return undefined; + + // Convert custom_value array to defaultFill if the method is FIXED_VALUES. + const defaultFill = method === ImputationMethod.FIXED_VALUES + ? imputationFormikValues?.custom_value?.map(({ featureName, data }) => ({ + featureName, + data, + })) + : undefined; + + // Construct and return the ImputationOption object. + return { method, defaultFill }; +} + +export function imputationMethodToFormik( + detector: Detector +): string { + var imputationMethod = get(detector, 'imputationOption.method', undefined) as ImputationMethod; + + switch (imputationMethod) { + case ImputationMethod.FIXED_VALUES: + return SparseDataOptionValue.CUSTOM_VALUE; + case ImputationMethod.PREVIOUS: + return SparseDataOptionValue.PREVIOUS_VALUE; + case ImputationMethod.ZERO: + return SparseDataOptionValue.SET_TO_ZERO; + default: + break; + } + + return SparseDataOptionValue.IGNORE; +} + +export function formikToImputationMethod( + formikValue: string | undefined +): ImputationMethod | undefined { + switch (formikValue) { + case SparseDataOptionValue.CUSTOM_VALUE: + return ImputationMethod.FIXED_VALUES; + case SparseDataOptionValue.PREVIOUS_VALUE: + return ImputationMethod.PREVIOUS; + case SparseDataOptionValue.SET_TO_ZERO: + return ImputationMethod.ZERO; + default: + return undefined; + } +} + +export const getCustomValueStrArray = (imputationMethodStr : string, detector: Detector): string[] => { + if (SparseDataOptionValue.CUSTOM_VALUE === imputationMethodStr) { + const defaultFill : Array<{ featureName: string; data: number }> = get(detector, 'imputationOption.defaultFill', []); + + return defaultFill + .map(({ featureName, data }) => `${featureName}: ${data}`); + } + return [] +} diff --git a/public/pages/DefineDetector/utils/constants.tsx b/public/pages/DefineDetector/utils/constants.tsx index d6ef4dff..cb64ce1d 100644 --- a/public/pages/DefineDetector/utils/constants.tsx +++ b/public/pages/DefineDetector/utils/constants.tsx @@ -48,4 +48,6 @@ export const INITIAL_DETECTOR_DEFINITION_VALUES: DetectorDefinitionFormikValues resultIndexMinSize: 51200, resultIndexTtl: 60, flattenCustomResultIndex: false, + imputationMethod: undefined, + customImputationValue: undefined }; diff --git a/public/pages/DetectorConfig/components/AdditionalSettings/AdditionalSettings.tsx b/public/pages/DetectorConfig/components/AdditionalSettings/AdditionalSettings.tsx index fe0dba05..18a322d5 100644 --- a/public/pages/DetectorConfig/components/AdditionalSettings/AdditionalSettings.tsx +++ b/public/pages/DetectorConfig/components/AdditionalSettings/AdditionalSettings.tsx @@ -18,20 +18,36 @@ import { convertToCategoryFieldString } from '../../../utils/anomalyResultUtils' interface AdditionalSettingsProps { shingleSize: number; categoryField: string[]; + imputationMethod: string; + customValues: string[]; } export function AdditionalSettings(props: AdditionalSettingsProps) { + const renderCustomValues = (customValues: string[]) => ( +
+ {customValues.map((value, index) => ( +

{value}

+ ))} +
+ ); const tableItems = [ { categoryField: isEmpty(get(props, 'categoryField', [])) ? '-' : convertToCategoryFieldString(props.categoryField, ', '), shingleSize: props.shingleSize, + imputationMethod: props.imputationMethod, + customValues: props.customValues, }, ]; const tableColumns = [ { name: 'Categorical fields', field: 'categoryField' }, { name: 'Shingle size', field: 'shingleSize' }, + { name: 'Imputation method', field: 'imputationMethod' }, + { name: 'Custom values', + field: 'customValues', + render: (customValues: string[]) => renderCustomValues(customValues), // Use a custom render function + }, ]; return ( diff --git a/public/pages/DetectorConfig/containers/Features.tsx b/public/pages/DetectorConfig/containers/Features.tsx index 47192c4f..2c5c6f8b 100644 --- a/public/pages/DetectorConfig/containers/Features.tsx +++ b/public/pages/DetectorConfig/containers/Features.tsx @@ -30,7 +30,11 @@ import ContentPanel from '../../../components/ContentPanel/ContentPanel'; import { CodeModal } from '../components/CodeModal/CodeModal'; import { getTitleWithCount } from '../../../utils/utils'; import { AdditionalSettings } from '../components/AdditionalSettings/AdditionalSettings'; -import { getShingleSizeFromObject } from '../../ConfigureModel/utils/helpers'; +import { + getShingleSizeFromObject, + imputationMethodToFormik, + getCustomValueStrArray, +} from '../../ConfigureModel/utils/helpers'; interface FeaturesProps { detectorId: string; @@ -187,6 +191,7 @@ export const Features = (props: FeaturesProps) => { const previewText = `After you set the model features and other optional parameters, you can preview your anomalies from a sample feature output.`; + const imputationMethodStr = imputationMethodToFormik(props.detector); return ( { )} diff --git a/public/pages/DetectorConfig/containers/__tests__/DetectorConfig.test.tsx b/public/pages/DetectorConfig/containers/__tests__/DetectorConfig.test.tsx index 6ee8ad28..b57721ff 100644 --- a/public/pages/DetectorConfig/containers/__tests__/DetectorConfig.test.tsx +++ b/public/pages/DetectorConfig/containers/__tests__/DetectorConfig.test.tsx @@ -27,15 +27,18 @@ import { FEATURE_TYPE, UiFeature, FeatureAttributes, + OPERATORS_MAP, } from '../../../../models/interfaces'; -import { getRandomDetector } from '../../../../redux/reducers/__tests__/utils'; +import { + getRandomDetector, + randomFixedValue, +} from '../../../../redux/reducers/__tests__/utils'; import { coreServicesMock } from '../../../../../test/mocks'; import { toStringConfigCell } from '../../../ReviewAndCreate/utils/helpers'; import { DATA_TYPES } from '../../../../utils/constants'; -import { OPERATORS_MAP } from '../../../../models/interfaces'; -import { displayText } from '../../../DefineDetector/components/DataFilterList/utils/helpers'; import { mockedStore, initialState } from '../../../../redux/utils/testUtils'; import { CoreServicesContext } from '../../../../components/CoreServices/CoreServices'; +import { ImputationMethod } from '../../../../models/types'; const renderWithRouter = (detector: Detector) => ({ ...render( @@ -139,6 +142,7 @@ describe(' spec', () => { test('renders the component', () => { const randomDetector = { ...getRandomDetector(false), + imputationOption: { method: ImputationMethod.PREVIOUS }, }; const { container } = renderWithRouter(randomDetector); expect(container.firstChild).toMatchSnapshot(); @@ -306,24 +310,40 @@ describe(' spec', () => { }); test('renders the component with 2 custom and 1 simple features', () => { + const features = [ + { + featureName: 'value', + featureEnabled: true, + aggregationQuery: featureQuery1, + }, + { + featureName: 'value2', + featureEnabled: true, + aggregationQuery: featureQuery2, + }, + { + featureName: 'value', + featureEnabled: false, + }, + ] as FeatureAttributes[]; + + const randomFixedValueMap: Array<{ featureName: string; data: number }> = + []; + + features.forEach((feature) => { + if (feature.featureEnabled) { + randomFixedValueMap.push({ featureName: feature.featureName, data: 3 }); + } + }); + + const imputationOption = { + method: ImputationMethod.FIXED_VALUES, + defaultFill: randomFixedValueMap, + }; + const randomDetector = { ...getRandomDetector(true), - featureAttributes: [ - { - featureName: 'value', - featureEnabled: true, - aggregationQuery: featureQuery1, - }, - { - featureName: 'value2', - featureEnabled: true, - aggregationQuery: featureQuery2, - }, - { - featureName: 'value', - featureEnabled: false, - }, - ] as FeatureAttributes[], + featureAttributes: features, uiMetadata: { filterType: FILTER_TYPES.SIMPLE, filters: [], @@ -341,6 +361,7 @@ describe(' spec', () => { } as UiFeature, }, } as UiMetaData, + imputationOption: imputationOption, }; const { container } = renderWithRouter(randomDetector); expect(container.firstChild).toMatchSnapshot(); diff --git a/public/pages/DetectorConfig/containers/__tests__/__snapshots__/DetectorConfig.test.tsx.snap b/public/pages/DetectorConfig/containers/__tests__/__snapshots__/DetectorConfig.test.tsx.snap index 5b87d3e9..46363942 100644 --- a/public/pages/DetectorConfig/containers/__tests__/__snapshots__/DetectorConfig.test.tsx.snap +++ b/public/pages/DetectorConfig/containers/__tests__/__snapshots__/DetectorConfig.test.tsx.snap @@ -1097,6 +1097,40 @@ exports[` spec renders the component 1`] = ` + + + + Imputation method + + + + + + + Custom values + + + @@ -1139,6 +1173,40 @@ exports[` spec renders the component 1`] = ` + +
+ Imputation method +
+
+ + previous_value + +
+ + +
+ Custom values +
+
+
+
+ @@ -2479,6 +2547,38 @@ exports[` spec renders the component with 2 custom and 1 simpl + + + + Imputation method + + + + + + + Custom values + + + @@ -2521,6 +2621,47 @@ exports[` spec renders the component with 2 custom and 1 simpl
+ +
+ Imputation method +
+
+ + custom_value + +
+ + +
+ Custom values +
+
+
+

+ value: 3 +

+

+ value2: 3 +

+
+
+ diff --git a/public/pages/ReviewAndCreate/components/AdditionalSettings/AdditionalSettings.tsx b/public/pages/ReviewAndCreate/components/AdditionalSettings/AdditionalSettings.tsx index c70242cc..eb7dcc5a 100644 --- a/public/pages/ReviewAndCreate/components/AdditionalSettings/AdditionalSettings.tsx +++ b/public/pages/ReviewAndCreate/components/AdditionalSettings/AdditionalSettings.tsx @@ -16,18 +16,34 @@ import { EuiBasicTable } from '@elastic/eui'; interface AdditionalSettingsProps { shingleSize: number; categoryField: string[]; + imputationMethod: string; + customValues: string[]; } export function AdditionalSettings(props: AdditionalSettingsProps) { + const renderCustomValues = (customValues: string[]) => ( +
+ {customValues.map((value, index) => ( +

{value}

+ ))} +
+ ); const tableItems = [ { categoryField: get(props, 'categoryField.0', '-'), shingleSize: props.shingleSize, + imputationMethod: props.imputationMethod, + customValues: props.customValues, }, ]; const tableColumns = [ { name: 'Category field', field: 'categoryField' }, { name: 'Shingle size', field: 'shingleSize' }, + { name: 'Imputation method', field: 'imputationMethod' }, + { name: 'Custom values', + field: 'customValues', + render: (customValues: string[]) => renderCustomValues(customValues), // Use a custom render function + }, ]; return ( spec', () => { {() => (
- +
)} @@ -31,6 +31,7 @@ describe(' spec', () => { getAllByText('Shingle size'); getByText('-'); getByText('8'); + getByText("Ignore"); }); test('renders the component with high cardinality enabled', () => { const { container, getByText, getAllByText } = render( @@ -40,6 +41,8 @@ describe(' spec', () => { )} @@ -50,5 +53,9 @@ describe(' spec', () => { getAllByText('Shingle size'); getByText('test_field'); getByText('8'); + getByText("Custom"); + // Check for the custom values + getByText('denyMax:5'); + getByText('denySum:10'); }); }); diff --git a/public/pages/ReviewAndCreate/components/__tests__/ModelConfigurationFields.test.tsx b/public/pages/ReviewAndCreate/components/__tests__/ModelConfigurationFields.test.tsx index 4179eebf..c0f59a4d 100644 --- a/public/pages/ReviewAndCreate/components/__tests__/ModelConfigurationFields.test.tsx +++ b/public/pages/ReviewAndCreate/components/__tests__/ModelConfigurationFields.test.tsx @@ -19,6 +19,7 @@ import { DATA_TYPES } from '../../../../utils/constants'; import { getRandomFeature } from '../../../../redux/reducers/__tests__/utils'; import { CoreServicesContext } from '../../../../components/CoreServices/CoreServices'; import { coreServicesMock } from '../../../../../test/mocks'; +import { ImputationMethod } from '../../../../models/types'; const detectorFaker = new chance('seed'); const features = new Array(detectorFaker.natural({ min: 1, max: 5 })) @@ -59,6 +60,7 @@ const testDetector = { ], }, featureAttributes: features, + imputationOption: { method: ImputationMethod.ZERO} } as Detector; describe('ModelConfigurationFields', () => { @@ -81,6 +83,7 @@ describe('ModelConfigurationFields', () => { userEvent.click(getByTestId('viewFeature-0')); await waitFor(() => { queryByText('max'); + queryByText('Zero'); }); }); }); diff --git a/public/pages/ReviewAndCreate/components/__tests__/__snapshots__/AdditionalSettings.test.tsx.snap b/public/pages/ReviewAndCreate/components/__tests__/__snapshots__/AdditionalSettings.test.tsx.snap index 4b8acd17..022afa0e 100644 --- a/public/pages/ReviewAndCreate/components/__tests__/__snapshots__/AdditionalSettings.test.tsx.snap +++ b/public/pages/ReviewAndCreate/components/__tests__/__snapshots__/AdditionalSettings.test.tsx.snap @@ -64,6 +64,40 @@ exports[` spec renders the component with high cardinality + + + + Imputation method + + + + + + + Custom values + + + @@ -106,6 +140,40 @@ exports[` spec renders the component with high cardinality + +
+ Imputation method +
+
+ + Ignore + +
+ + +
+ Custom values +
+
+
+
+ @@ -178,6 +246,40 @@ exports[` spec renders the component with high cardinality + + + + Imputation method + + + + + + + Custom values + + + @@ -220,6 +322,47 @@ exports[` spec renders the component with high cardinality
+ +
+ Imputation method +
+
+ + Custom + +
+ + +
+ Custom values +
+
+
+

+ denyMax:5 +

+

+ denySum:10 +

+
+
+ diff --git a/public/pages/ReviewAndCreate/components/__tests__/__snapshots__/ModelConfigurationFields.test.tsx.snap b/public/pages/ReviewAndCreate/components/__tests__/__snapshots__/ModelConfigurationFields.test.tsx.snap index c1baf2f5..a0cb87f0 100644 --- a/public/pages/ReviewAndCreate/components/__tests__/__snapshots__/ModelConfigurationFields.test.tsx.snap +++ b/public/pages/ReviewAndCreate/components/__tests__/__snapshots__/ModelConfigurationFields.test.tsx.snap @@ -155,6 +155,40 @@ exports[`ModelConfigurationFields renders the component in create mode (no ID) 1 + + + + Imputation method + + + + + + + Custom values + + + @@ -197,6 +231,40 @@ exports[`ModelConfigurationFields renders the component in create mode (no ID) 1 + +
+ Imputation method +
+
+ + set_to_zero + +
+ + +
+ Custom values +
+
+
+
+ diff --git a/public/pages/ReviewAndCreate/containers/__tests__/__snapshots__/ReviewAndCreate.test.tsx.snap b/public/pages/ReviewAndCreate/containers/__tests__/__snapshots__/ReviewAndCreate.test.tsx.snap index f8f3c901..3d8d46a6 100644 --- a/public/pages/ReviewAndCreate/containers/__tests__/__snapshots__/ReviewAndCreate.test.tsx.snap +++ b/public/pages/ReviewAndCreate/containers/__tests__/__snapshots__/ReviewAndCreate.test.tsx.snap @@ -697,6 +697,40 @@ exports[` spec renders the component, validation loading 1`] + + + + Imputation method + + + + + + + Custom values + + + @@ -739,6 +773,40 @@ exports[` spec renders the component, validation loading 1`]
+ +
+ Imputation method +
+
+ + ignore + +
+ + +
+ Custom values +
+
+
+
+ @@ -1839,6 +1907,40 @@ exports[`issue in detector validation issues in feature query 1`] = ` + + + + Imputation method + + + + + + + Custom values + + + @@ -1881,6 +1983,40 @@ exports[`issue in detector validation issues in feature query 1`] = `
+ +
+ Imputation method +
+
+ + ignore + +
+ + +
+ Custom values +
+
+
+
+ diff --git a/public/pages/ReviewAndCreate/utils/helpers.ts b/public/pages/ReviewAndCreate/utils/helpers.ts index 6f1a15bd..24703c7d 100644 --- a/public/pages/ReviewAndCreate/utils/helpers.ts +++ b/public/pages/ReviewAndCreate/utils/helpers.ts @@ -25,6 +25,7 @@ import { CreateDetectorFormikValues } from '../../CreateDetectorSteps/models/int import { OPERATORS_QUERY_MAP } from '../../DefineDetector/utils/whereFilters'; import { convertTimestampToNumber } from '../../../utils/utils'; import { CUSTOM_AD_RESULT_INDEX_PREFIX } from '../../../../server/utils/constants'; +import { formikToImputationOption } from '../../ConfigureModel/utils/helpers'; export function formikToDetector(values: CreateDetectorFormikValues): Detector { const detectionDateRange = values.historical @@ -59,10 +60,23 @@ export function formikToDetector(values: CreateDetectorFormikValues): Detector { categoryField: !isEmpty(values?.categoryField) ? values.categoryField : undefined, - resultIndexMinAge: resultIndex && resultIndex.trim().length > 0 ? values.resultIndexMinAge : undefined, - resultIndexMinSize: resultIndex && resultIndex.trim().length > 0 ? values.resultIndexMinSize : undefined, - resultIndexTtl: resultIndex && resultIndex.trim().length > 0 ? values.resultIndexTtl : undefined, - flattenCustomResultIndex: resultIndex && resultIndex.trim().length > 0 ? values.flattenCustomResultIndex : undefined, + resultIndexMinAge: + resultIndex && resultIndex.trim().length > 0 + ? values.resultIndexMinAge + : undefined, + resultIndexMinSize: + resultIndex && resultIndex.trim().length > 0 + ? values.resultIndexMinSize + : undefined, + resultIndexTtl: + resultIndex && resultIndex.trim().length > 0 + ? values.resultIndexTtl + : undefined, + flattenCustomResultIndex: + resultIndex && resultIndex.trim().length > 0 + ? values.flattenCustomResultIndex + : undefined, + imputationOption: formikToImputationOption(values.imputationOption), } as Detector; // Optionally add detection date range diff --git a/public/redux/reducers/__tests__/utils.ts b/public/redux/reducers/__tests__/utils.ts index 7ebe84ce..01df3d9c 100644 --- a/public/redux/reducers/__tests__/utils.ts +++ b/public/redux/reducers/__tests__/utils.ts @@ -10,7 +10,7 @@ */ import chance from 'chance'; -import { isEmpty, snakeCase } from 'lodash'; +import { isEmpty, snakeCase, random } from 'lodash'; import { Detector, FeatureAttributes, @@ -23,6 +23,9 @@ import { import moment from 'moment'; import { DETECTOR_STATE } from '../../../../server/utils/constants'; import { DEFAULT_SHINGLE_SIZE } from '../../../utils/constants'; +import { + ImputationMethod, ImputationOption, +} from '../../../models/types'; const detectorFaker = new chance('seed'); @@ -124,6 +127,7 @@ export const getRandomDetector = ( resultIndexMinSize: 51200, resultIndexTtl: 60, flattenCustomResultIndex: true, + imputationOption: randomImputationOption(features) }; }; @@ -202,3 +206,44 @@ export const getRandomMonitor = ( lastUpdateTime: moment(1586823218000).subtract(1, 'days').valueOf(), }; }; + +export const randomFixedValue = (features: FeatureAttributes[]): Array<{ featureName: string; data: number }> => { + const randomValues: Array<{ featureName: string; data: number }> = []; + + if (!features) { + return randomValues; + } + + features.forEach((feature) => { + if (feature.featureEnabled) { + const randomValue = Math.random() * 100; // generate a random value, e.g., between 0 and 100 + randomValues.push({ featureName: feature.featureName, data: randomValue }); + } + }); + + return randomValues; +}; + + +export const randomImputationOption = (features: FeatureAttributes[]): ImputationOption | undefined => { + const randomFixedValueMap = randomFixedValue(features); + + const options: ImputationOption[] = []; + + if (Object.keys(randomFixedValueMap).length !== 0) { + options.push({ + method: ImputationMethod.FIXED_VALUES, + defaultFill: randomFixedValueMap, + }); + } + + options.push({ method: ImputationMethod.ZERO }); + options.push({ method: ImputationMethod.PREVIOUS }); + + // Select a random option. random in lodash is inclusive of both min and max + const randomIndex = random(0, options.length); + if (options.length == randomIndex) { + return undefined; + } + return options[randomIndex]; +};