diff --git a/public/models/interfaces.ts b/public/models/interfaces.ts index c10cf764..f6dfc651 100644 --- a/public/models/interfaces.ts +++ b/public/models/interfaces.ts @@ -15,6 +15,7 @@ import { DETECTOR_STATE } from '../../server/utils/constants'; import { Duration } from 'moment'; import moment from 'moment'; import { MDSQueryParams } from '../../server/models/types'; +import { ImputationOption } from './types'; export type FieldInfo = { label: string; @@ -210,6 +211,7 @@ export type Detector = { taskState?: DETECTOR_STATE; taskProgress?: number; taskError?: string; + imputationOption?: ImputationOption; }; export type DetectorListItem = { diff --git a/public/models/types.ts b/public/models/types.ts index f56af8e9..6d559276 100644 --- a/public/models/types.ts +++ b/public/models/types.ts @@ -12,3 +12,24 @@ export type AggregationOption = { label: string; }; + +export type ImputationOption = { + method: ImputationMethod; + defaultFill?: Array<{ featureName: string; data: number }>; +}; + +export enum ImputationMethod { + /** + * This method replaces all missing values with 0's. It's a simple approach, but it may introduce bias if the data is not centered around zero. + */ + ZERO = 'ZERO', + /** + * This method replaces missing values with a predefined set of values. The values are the same for each input dimension, and they need to be specified by the user. + */ + FIXED_VALUES = 'FIXED_VALUES', + /** + * This method replaces missing values with the last known value in the respective input dimension. It's a commonly used method for time series data, where temporal continuity is expected. + */ + PREVIOUS = 'PREVIOUS', +} + diff --git a/public/pages/ConfigureModel/components/AdvancedSettings/AdvancedSettings.tsx b/public/pages/ConfigureModel/components/AdvancedSettings/AdvancedSettings.tsx index 9b0bd2f1..0ccac512 100644 --- a/public/pages/ConfigureModel/components/AdvancedSettings/AdvancedSettings.tsx +++ b/public/pages/ConfigureModel/components/AdvancedSettings/AdvancedSettings.tsx @@ -17,9 +17,12 @@ import { EuiTitle, EuiCompressedFieldNumber, EuiSpacer, + EuiCompressedSelect, + EuiButtonIcon, + EuiCompressedFieldText, } from '@elastic/eui'; -import { Field, FieldProps } from 'formik'; -import React, { useState } from 'react'; +import { Field, FieldProps, FieldArray, } from 'formik'; +import React, { useEffect, useState } from 'react'; import ContentPanel from '../../../../components/ContentPanel/ContentPanel'; import { BASE_DOCS_LINK } from '../../../../utils/constants'; import { @@ -28,6 +31,7 @@ import { validatePositiveInteger, } from '../../../../utils/utils'; import { FormattedFormRow } from '../../../../components/FormattedFormRow/FormattedFormRow'; +import { SparseDataOptionValue } from '../../utils/constants'; interface AdvancedSettingsProps {} @@ -35,6 +39,14 @@ export function AdvancedSettings(props: AdvancedSettingsProps) { const [showAdvancedSettings, setShowAdvancedSettings] = useState(false); + // Options for the sparse data handling dropdown + const sparseDataOptions = [ + { value: SparseDataOptionValue.IGNORE, text: 'Ignore missing value' }, + { value: SparseDataOptionValue.PREVIOUS_VALUE, text: 'Previous value' }, + { value: SparseDataOptionValue.SET_TO_ZERO, text: 'Set to zero' }, + { value: SparseDataOptionValue.CUSTOM_VALUE, text: 'Custom value' }, + ]; + return ( {showAdvancedSettings ? : null} {showAdvancedSettings ? ( - - {({ field, form }: FieldProps) => ( - + + {({ field, form }: FieldProps) => ( + - - - - - - -

intervals

- - - - - )} - + ]} + hintLink={`${BASE_DOCS_LINK}/ad`} + isInvalid={isInvalid(field.name, form)} + error={getError(field.name, form)} + > + + + + + + +

intervals

+ + + + + )} + + + + {({ field, form }: FieldProps) => { + // Add an empty row if CUSTOM_VALUE is selected and no rows exist + useEffect(() => { + if ( + field.value === SparseDataOptionValue.CUSTOM_VALUE && + (!form.values.imputationOption?.custom_value || + form.values.imputationOption.custom_value.length === 0) + ) { + form.setFieldValue('imputationOption.custom_value', [ + { featureName: '', value: undefined }, + ]); + } + }, [field.value, form]); + + return ( + <> + + + + + {/* Conditionally render the "Custom value" title and the input fields when 'Custom value' is selected */} + {field.value === SparseDataOptionValue.CUSTOM_VALUE && ( + <> + + +
Custom value
+ + + + {(arrayHelpers) => ( + <> + {form.values.imputationOption.custom_value?.map((_, index) => ( + + + + {({ field }: FieldProps) => ( + + )} + + + + + {/* the value is set to field.value || '' to avoid displaying 0 as a default value. */ } + {({ field, form }: FieldProps) => ( + + )} + + + + arrayHelpers.remove(index)} + /> + + + ))} + + { /* add new rows with empty values when the add button is clicked. */} + + arrayHelpers.push({ featureName: '', value: 0 }) + } + aria-label="Add row" + /> + + )} + + + )} + + ); + }} + + ) : null} ); diff --git a/public/pages/ConfigureModel/containers/ConfigureModel.tsx b/public/pages/ConfigureModel/containers/ConfigureModel.tsx index 2b558973..f1d324b3 100644 --- a/public/pages/ConfigureModel/containers/ConfigureModel.tsx +++ b/public/pages/ConfigureModel/containers/ConfigureModel.tsx @@ -39,8 +39,8 @@ import { focusOnFirstWrongFeature, getCategoryFields, focusOnCategoryField, - getShingleSizeFromObject, modelConfigurationToFormik, + focusOnImputationOption, } from '../utils/helpers'; import { formikToDetector } from '../../ReviewAndCreate/utils/helpers'; import { formikToModelConfiguration } from '../utils/helpers'; @@ -53,7 +53,7 @@ import { CoreServicesContext } from '../../../components/CoreServices/CoreServic import { Detector } from '../../../models/interfaces'; import { prettifyErrorMessage } from '../../../../server/utils/helpers'; import { DetectorDefinitionFormikValues } from '../../DefineDetector/models/interfaces'; -import { ModelConfigurationFormikValues } from '../models/interfaces'; +import { ModelConfigurationFormikValues, FeaturesFormikValues } from '../models/interfaces'; import { CreateDetectorFormikValues } from '../../CreateDetectorSteps/models/interfaces'; import { DETECTOR_STATE } from '../../../../server/utils/constants'; import { getErrorMessage } from '../../../utils/utils'; @@ -68,6 +68,7 @@ import { getSavedObjectsClient, } from '../../../services'; import { DataSourceViewConfig } from '../../../../../../src/plugins/data_source_management/public'; +import { SparseDataOptionValue } from '../utils/constants'; interface ConfigureModelRouterProps { detectorId?: string; @@ -173,6 +174,50 @@ export function ConfigureModel(props: ConfigureModelProps) { } }, [hasError]); + const validateImputationOption = ( + formikValues: ModelConfigurationFormikValues, + errors: any + ) => { + const imputationOption = get(formikValues, 'imputationOption', null); + + // Initialize an array to hold individual error messages + const customValueErrors: string[] = []; + console.log(`hello5: ${JSON.stringify(imputationOption)}`); + + // Validate imputationOption when method is CUSTOM_VALUE + if (imputationOption && imputationOption.imputationMethod === SparseDataOptionValue.CUSTOM_VALUE) { + const enabledFeatures = formikValues.featureList.filter( + (feature: FeaturesFormikValues) => feature.featureEnabled + ); + + // Validate that the number of custom values matches the number of enabled features + if ((imputationOption.custom_value || []).length !== enabledFeatures.length) { + customValueErrors.push( + `The number of custom values (${(imputationOption.custom_value || []).length}) does not match the number of enabled features (${enabledFeatures.length}).` + ); + } + + // Validate that each enabled feature has a corresponding custom value + const missingFeatures = enabledFeatures + .map((feature: FeaturesFormikValues) => feature.featureName) + .filter( + (name: string | undefined) => + !imputationOption.custom_value?.some((cv) => cv.featureName === name) + ); + + if (missingFeatures.length > 0) { + customValueErrors.push( + `The following enabled features are missing in custom values: ${missingFeatures.join(', ')}.` + ); + } + + // If there are any custom value errors, join them into a single string with proper formatting + if (customValueErrors.length > 0) { + errors.custom_value = customValueErrors.join(' '); + } + } + }; + const handleFormValidation = async ( formikProps: FormikProps ) => { @@ -185,7 +230,12 @@ export function ConfigureModel(props: ConfigureModelProps) { formikProps.setFieldTouched('featureList'); formikProps.setFieldTouched('categoryField', isHCDetector); formikProps.setFieldTouched('shingleSize'); + formikProps.setFieldTouched('imputationOption'); + formikProps.validateForm().then((errors) => { + // Call the extracted validation method + validateImputationOption(formikProps.values, errors); + if (isEmpty(errors)) { if (props.isEdit) { // TODO: possibly add logic to also start RT and/or historical from here. Need to think @@ -204,11 +254,23 @@ export function ConfigureModel(props: ConfigureModelProps) { props.setStep(3); } } else { + console.log(`hello4 ${JSON.stringify(errors)}`); + const customValueError = get(errors, 'custom_value') + if (customValueError) { + core.notifications.toasts.addDanger( + customValueError + ); + focusOnImputationOption(); + return; + } + // TODO: can add focus to all components or possibly customize error message too if (get(errors, 'featureList')) { focusOnFirstWrongFeature(errors, formikProps.setFieldTouched); } else if (get(errors, 'categoryField')) { focusOnCategoryField(); + } else { + console.log(`unexpected error ${JSON.stringify(errors)}`); } core.notifications.toasts.addDanger( diff --git a/public/pages/ConfigureModel/models/interfaces.ts b/public/pages/ConfigureModel/models/interfaces.ts index ac5cd906..f7575bc4 100644 --- a/public/pages/ConfigureModel/models/interfaces.ts +++ b/public/pages/ConfigureModel/models/interfaces.ts @@ -18,6 +18,7 @@ export interface ModelConfigurationFormikValues { categoryFieldEnabled: boolean; categoryField: string[]; shingleSize: number; + imputationOption?: ImputationFormikValues; } export interface FeaturesFormikValues { @@ -30,3 +31,13 @@ export interface FeaturesFormikValues { aggregationOf?: AggregationOption[]; newFeature?: boolean; } + +export interface ImputationFormikValues { + imputationMethod?: string; + custom_value?: CustomValueFormikValues[]; +} + +export interface CustomValueFormikValues { + featureName: string; + data: number; +} diff --git a/public/pages/ConfigureModel/utils/__tests__/helpers.test.tsx b/public/pages/ConfigureModel/utils/__tests__/helpers.test.tsx index d0d34e15..49b19750 100644 --- a/public/pages/ConfigureModel/utils/__tests__/helpers.test.tsx +++ b/public/pages/ConfigureModel/utils/__tests__/helpers.test.tsx @@ -13,6 +13,9 @@ import { getRandomDetector } from '../../../../redux/reducers/__tests__/utils'; import { prepareDetector } from '../../utils/helpers'; import { FEATURE_TYPE } from '../../../../models/interfaces'; import { FeaturesFormikValues } from '../../models/interfaces'; +import { modelConfigurationToFormik } from '../helpers'; +import { SparseDataOptionValue } from '../constants'; +import { ImputationMethod } from '../../../../models/types'; describe('featuresToFormik', () => { test('should able to add new feature', () => { @@ -71,4 +74,57 @@ describe('featuresToFormik', () => { // ...randomDetector.featureAttributes.slice(1), // ]); // }); + test('should return correct values if detector is not null', () => { + const randomDetector = getRandomDetector(); + const adFormikValues = modelConfigurationToFormik(randomDetector); + + const imputationOption = randomDetector.imputationOption; + if (imputationOption) { + const method = imputationOption.method; + if (ImputationMethod.FIXED_VALUES === method) { + expect(adFormikValues.imputationOption?.imputationMethod).toEqual( + SparseDataOptionValue.CUSTOM_VALUE + ); + expect(randomDetector.imputationOption?.defaultFill).toBeDefined(); + expect( + randomDetector.imputationOption?.defaultFill?.length + ).toBeGreaterThan(0); + + const formikCustom = adFormikValues.imputationOption?.custom_value; + expect(formikCustom).toBeDefined(); + + const defaultFill = randomDetector.imputationOption?.defaultFill || []; + + defaultFill.forEach(({ featureName, data }) => { + const matchingFormikValue = formikCustom?.find( + (item) => item.featureName === featureName + ); + + // Assert that a matching value was found + expect(matchingFormikValue).toBeDefined(); + + // Assert that the data matches + expect(matchingFormikValue?.data).toEqual(data); + }); + } else { + if (ImputationMethod.ZERO === method) { + expect(adFormikValues.imputationOption?.imputationMethod).toEqual( + SparseDataOptionValue.SET_TO_ZERO + ); + } else { + expect(adFormikValues.imputationOption?.imputationMethod).toEqual( + SparseDataOptionValue.PREVIOUS_VALUE + ); + } + expect(adFormikValues.imputationOption?.custom_value).toEqual( + undefined + ); + } + } else { + expect(adFormikValues.imputationOption?.custom_value).toEqual(undefined); + expect(adFormikValues.imputationOption?.imputationMethod).toEqual( + SparseDataOptionValue.IGNORE + ); + } + }); }); diff --git a/public/pages/ConfigureModel/utils/constants.tsx b/public/pages/ConfigureModel/utils/constants.tsx index 8aef72a9..b0a84b2e 100644 --- a/public/pages/ConfigureModel/utils/constants.tsx +++ b/public/pages/ConfigureModel/utils/constants.tsx @@ -22,6 +22,7 @@ export const INITIAL_MODEL_CONFIGURATION_VALUES: ModelConfigurationFormikValues categoryFieldEnabled: false, categoryField: [], shingleSize: DEFAULT_SHINGLE_SIZE, + imputationOption: undefined }; export const INITIAL_FEATURE_VALUES: FeaturesFormikValues = { @@ -67,3 +68,11 @@ export const FEATURE_FIELDS = [ 'aggregationBy', 'aggregationQuery', ]; + +// an enum for the sparse data handling options +export enum SparseDataOptionValue { + IGNORE = 'ignore', + PREVIOUS_VALUE = 'previous_value', + SET_TO_ZERO = 'set_to_zero', + CUSTOM_VALUE = 'custom_value', +} diff --git a/public/pages/ConfigureModel/utils/helpers.ts b/public/pages/ConfigureModel/utils/helpers.ts index 05c8bdf3..73e4ce78 100644 --- a/public/pages/ConfigureModel/utils/helpers.ts +++ b/public/pages/ConfigureModel/utils/helpers.ts @@ -21,12 +21,21 @@ import { DataTypes } from '../../../redux/reducers/opensearch'; import { ModelConfigurationFormikValues, FeaturesFormikValues, + CustomValueFormikValues, + ImputationFormikValues, } from '../../ConfigureModel/models/interfaces'; import { INITIAL_MODEL_CONFIGURATION_VALUES } from '../../ConfigureModel/utils/constants'; import { featuresToUIMetadata, formikToFeatureAttributes, } from '../../ReviewAndCreate/utils/helpers'; +import { + ImputationMethod, + ImputationOption, +} from '../../../models/types'; +import { + SparseDataOptionValue +} from './constants' export const getFieldOptions = ( allFields: { [key: string]: string[] }, @@ -120,6 +129,7 @@ export const validateFeatures = (values: any) => { }; } }); + return hasError ? { featureList: featureErrors } : undefined; }; @@ -203,6 +213,11 @@ export const getCategoryFields = (dataTypes: DataTypes) => { return keywordFields.concat(ipFields); }; +export const focusOnImputationOption = () => { + const component = document.getElementById('imputationOption'); + component?.focus(); +}; + export const getShingleSizeFromObject = (obj: object) => { return get(obj, 'shingleSize', DEFAULT_SHINGLE_SIZE); }; @@ -227,12 +242,33 @@ export function modelConfigurationToFormik( if (isEmpty(detector)) { return initialValues; } + + var imputationMethod = imputationMethodToFormik(detector); + + var defaultFillArray: CustomValueFormikValues[] = []; + + if (SparseDataOptionValue.CUSTOM_VALUE === imputationMethod) { + const defaultFill = get(detector, 'imputationOption.defaultFill', null) as Array<{ featureName: string; data: number }> | null; + defaultFillArray = defaultFill + ? defaultFill.map(({ featureName, data }) => ({ + featureName, + data, + })) + : []; + } + + const imputationFormikValues: ImputationFormikValues = { + imputationMethod: imputationMethod, + custom_value: SparseDataOptionValue.CUSTOM_VALUE === imputationMethod ? defaultFillArray : undefined, + }; + return { ...initialValues, featureList: featuresToFormik(detector), categoryFieldEnabled: !isEmpty(get(detector, 'categoryField', [])), categoryField: get(detector, 'categoryField', []), shingleSize: get(detector, 'shingleSize', DEFAULT_SHINGLE_SIZE), + imputationOption: imputationFormikValues, }; } @@ -280,6 +316,7 @@ export function formikToModelConfiguration( categoryField: !isEmpty(values?.categoryField) ? values.categoryField : undefined, + imputationOption: formikToImputationOption(values.imputationOption), } as Detector; return detectorBody; @@ -327,3 +364,64 @@ export function formikToSimpleAggregation(value: FeaturesFormikValues) { return {}; } } + +export function formikToImputationOption(imputationFormikValues?: ImputationFormikValues): ImputationOption | undefined { + // Map the formik method to the imputation method; return undefined if method is not recognized. + const method = formikToImputationMethod(imputationFormikValues?.imputationMethod); + if (!method) return undefined; + + // Convert custom_value array to defaultFill if the method is FIXED_VALUES. + const defaultFill = method === ImputationMethod.FIXED_VALUES + ? imputationFormikValues?.custom_value?.map(({ featureName, data }) => ({ + featureName, + data, + })) + : undefined; + + // Construct and return the ImputationOption object. + return { method, defaultFill }; +} + +export function imputationMethodToFormik( + detector: Detector +): string { + var imputationMethod = get(detector, 'imputationOption.method', undefined) as ImputationMethod; + + switch (imputationMethod) { + case ImputationMethod.FIXED_VALUES: + return SparseDataOptionValue.CUSTOM_VALUE; + case ImputationMethod.PREVIOUS: + return SparseDataOptionValue.PREVIOUS_VALUE; + case ImputationMethod.ZERO: + return SparseDataOptionValue.SET_TO_ZERO; + default: + break; + } + + return SparseDataOptionValue.IGNORE; +} + +export function formikToImputationMethod( + formikValue: string | undefined +): ImputationMethod | undefined { + switch (formikValue) { + case SparseDataOptionValue.CUSTOM_VALUE: + return ImputationMethod.FIXED_VALUES; + case SparseDataOptionValue.PREVIOUS_VALUE: + return ImputationMethod.PREVIOUS; + case SparseDataOptionValue.SET_TO_ZERO: + return ImputationMethod.ZERO; + default: + return undefined; + } +} + +export const getCustomValueStrArray = (imputationMethodStr : string, detector: Detector): string[] => { + if (SparseDataOptionValue.CUSTOM_VALUE === imputationMethodStr) { + const defaultFill : Array<{ featureName: string; data: number }> = get(detector, 'imputationOption.defaultFill', []); + + return defaultFill + .map(({ featureName, data }) => `${featureName}: ${data}`); + } + return [] +} diff --git a/public/pages/DefineDetector/utils/constants.tsx b/public/pages/DefineDetector/utils/constants.tsx index d6ef4dff..cb64ce1d 100644 --- a/public/pages/DefineDetector/utils/constants.tsx +++ b/public/pages/DefineDetector/utils/constants.tsx @@ -48,4 +48,6 @@ export const INITIAL_DETECTOR_DEFINITION_VALUES: DetectorDefinitionFormikValues resultIndexMinSize: 51200, resultIndexTtl: 60, flattenCustomResultIndex: false, + imputationMethod: undefined, + customImputationValue: undefined }; diff --git a/public/pages/DetectorConfig/components/AdditionalSettings/AdditionalSettings.tsx b/public/pages/DetectorConfig/components/AdditionalSettings/AdditionalSettings.tsx index fe0dba05..18a322d5 100644 --- a/public/pages/DetectorConfig/components/AdditionalSettings/AdditionalSettings.tsx +++ b/public/pages/DetectorConfig/components/AdditionalSettings/AdditionalSettings.tsx @@ -18,20 +18,36 @@ import { convertToCategoryFieldString } from '../../../utils/anomalyResultUtils' interface AdditionalSettingsProps { shingleSize: number; categoryField: string[]; + imputationMethod: string; + customValues: string[]; } export function AdditionalSettings(props: AdditionalSettingsProps) { + const renderCustomValues = (customValues: string[]) => ( +
+ {customValues.map((value, index) => ( +

{value}

+ ))} +
+ ); const tableItems = [ { categoryField: isEmpty(get(props, 'categoryField', [])) ? '-' : convertToCategoryFieldString(props.categoryField, ', '), shingleSize: props.shingleSize, + imputationMethod: props.imputationMethod, + customValues: props.customValues, }, ]; const tableColumns = [ { name: 'Categorical fields', field: 'categoryField' }, { name: 'Shingle size', field: 'shingleSize' }, + { name: 'Imputation method', field: 'imputationMethod' }, + { name: 'Custom values', + field: 'customValues', + render: (customValues: string[]) => renderCustomValues(customValues), // Use a custom render function + }, ]; return ( diff --git a/public/pages/DetectorConfig/containers/Features.tsx b/public/pages/DetectorConfig/containers/Features.tsx index 47192c4f..2c5c6f8b 100644 --- a/public/pages/DetectorConfig/containers/Features.tsx +++ b/public/pages/DetectorConfig/containers/Features.tsx @@ -30,7 +30,11 @@ import ContentPanel from '../../../components/ContentPanel/ContentPanel'; import { CodeModal } from '../components/CodeModal/CodeModal'; import { getTitleWithCount } from '../../../utils/utils'; import { AdditionalSettings } from '../components/AdditionalSettings/AdditionalSettings'; -import { getShingleSizeFromObject } from '../../ConfigureModel/utils/helpers'; +import { + getShingleSizeFromObject, + imputationMethodToFormik, + getCustomValueStrArray, +} from '../../ConfigureModel/utils/helpers'; interface FeaturesProps { detectorId: string; @@ -187,6 +191,7 @@ export const Features = (props: FeaturesProps) => { const previewText = `After you set the model features and other optional parameters, you can preview your anomalies from a sample feature output.`; + const imputationMethodStr = imputationMethodToFormik(props.detector); return ( { )} diff --git a/public/pages/DetectorConfig/containers/__tests__/DetectorConfig.test.tsx b/public/pages/DetectorConfig/containers/__tests__/DetectorConfig.test.tsx index 6ee8ad28..b57721ff 100644 --- a/public/pages/DetectorConfig/containers/__tests__/DetectorConfig.test.tsx +++ b/public/pages/DetectorConfig/containers/__tests__/DetectorConfig.test.tsx @@ -27,15 +27,18 @@ import { FEATURE_TYPE, UiFeature, FeatureAttributes, + OPERATORS_MAP, } from '../../../../models/interfaces'; -import { getRandomDetector } from '../../../../redux/reducers/__tests__/utils'; +import { + getRandomDetector, + randomFixedValue, +} from '../../../../redux/reducers/__tests__/utils'; import { coreServicesMock } from '../../../../../test/mocks'; import { toStringConfigCell } from '../../../ReviewAndCreate/utils/helpers'; import { DATA_TYPES } from '../../../../utils/constants'; -import { OPERATORS_MAP } from '../../../../models/interfaces'; -import { displayText } from '../../../DefineDetector/components/DataFilterList/utils/helpers'; import { mockedStore, initialState } from '../../../../redux/utils/testUtils'; import { CoreServicesContext } from '../../../../components/CoreServices/CoreServices'; +import { ImputationMethod } from '../../../../models/types'; const renderWithRouter = (detector: Detector) => ({ ...render( @@ -139,6 +142,7 @@ describe(' spec', () => { test('renders the component', () => { const randomDetector = { ...getRandomDetector(false), + imputationOption: { method: ImputationMethod.PREVIOUS }, }; const { container } = renderWithRouter(randomDetector); expect(container.firstChild).toMatchSnapshot(); @@ -306,24 +310,40 @@ describe(' spec', () => { }); test('renders the component with 2 custom and 1 simple features', () => { + const features = [ + { + featureName: 'value', + featureEnabled: true, + aggregationQuery: featureQuery1, + }, + { + featureName: 'value2', + featureEnabled: true, + aggregationQuery: featureQuery2, + }, + { + featureName: 'value', + featureEnabled: false, + }, + ] as FeatureAttributes[]; + + const randomFixedValueMap: Array<{ featureName: string; data: number }> = + []; + + features.forEach((feature) => { + if (feature.featureEnabled) { + randomFixedValueMap.push({ featureName: feature.featureName, data: 3 }); + } + }); + + const imputationOption = { + method: ImputationMethod.FIXED_VALUES, + defaultFill: randomFixedValueMap, + }; + const randomDetector = { ...getRandomDetector(true), - featureAttributes: [ - { - featureName: 'value', - featureEnabled: true, - aggregationQuery: featureQuery1, - }, - { - featureName: 'value2', - featureEnabled: true, - aggregationQuery: featureQuery2, - }, - { - featureName: 'value', - featureEnabled: false, - }, - ] as FeatureAttributes[], + featureAttributes: features, uiMetadata: { filterType: FILTER_TYPES.SIMPLE, filters: [], @@ -341,6 +361,7 @@ describe(' spec', () => { } as UiFeature, }, } as UiMetaData, + imputationOption: imputationOption, }; const { container } = renderWithRouter(randomDetector); expect(container.firstChild).toMatchSnapshot(); diff --git a/public/pages/DetectorConfig/containers/__tests__/__snapshots__/DetectorConfig.test.tsx.snap b/public/pages/DetectorConfig/containers/__tests__/__snapshots__/DetectorConfig.test.tsx.snap index 5b87d3e9..46363942 100644 --- a/public/pages/DetectorConfig/containers/__tests__/__snapshots__/DetectorConfig.test.tsx.snap +++ b/public/pages/DetectorConfig/containers/__tests__/__snapshots__/DetectorConfig.test.tsx.snap @@ -1097,6 +1097,40 @@ exports[` spec renders the component 1`] = ` + + + + Imputation method + + + + + + + Custom values + + + @@ -1139,6 +1173,40 @@ exports[` spec renders the component 1`] = ` + +
+ Imputation method +
+
+ + previous_value + +
+ + +
+ Custom values +
+
+
+
+ @@ -2479,6 +2547,38 @@ exports[` spec renders the component with 2 custom and 1 simpl + + + + Imputation method + + + + + + + Custom values + + + @@ -2521,6 +2621,47 @@ exports[` spec renders the component with 2 custom and 1 simpl
+ +
+ Imputation method +
+
+ + custom_value + +
+ + +
+ Custom values +
+
+
+

+ value: 3 +

+

+ value2: 3 +

+
+
+ diff --git a/public/pages/ReviewAndCreate/components/AdditionalSettings/AdditionalSettings.tsx b/public/pages/ReviewAndCreate/components/AdditionalSettings/AdditionalSettings.tsx index c70242cc..eb7dcc5a 100644 --- a/public/pages/ReviewAndCreate/components/AdditionalSettings/AdditionalSettings.tsx +++ b/public/pages/ReviewAndCreate/components/AdditionalSettings/AdditionalSettings.tsx @@ -16,18 +16,34 @@ import { EuiBasicTable } from '@elastic/eui'; interface AdditionalSettingsProps { shingleSize: number; categoryField: string[]; + imputationMethod: string; + customValues: string[]; } export function AdditionalSettings(props: AdditionalSettingsProps) { + const renderCustomValues = (customValues: string[]) => ( +
+ {customValues.map((value, index) => ( +

{value}

+ ))} +
+ ); const tableItems = [ { categoryField: get(props, 'categoryField.0', '-'), shingleSize: props.shingleSize, + imputationMethod: props.imputationMethod, + customValues: props.customValues, }, ]; const tableColumns = [ { name: 'Category field', field: 'categoryField' }, { name: 'Shingle size', field: 'shingleSize' }, + { name: 'Imputation method', field: 'imputationMethod' }, + { name: 'Custom values', + field: 'customValues', + render: (customValues: string[]) => renderCustomValues(customValues), // Use a custom render function + }, ]; return ( spec', () => { {() => (
- +
)} @@ -31,6 +31,7 @@ describe(' spec', () => { getAllByText('Shingle size'); getByText('-'); getByText('8'); + getByText("Ignore"); }); test('renders the component with high cardinality enabled', () => { const { container, getByText, getAllByText } = render( @@ -40,6 +41,8 @@ describe(' spec', () => { )} @@ -50,5 +53,9 @@ describe(' spec', () => { getAllByText('Shingle size'); getByText('test_field'); getByText('8'); + getByText("Custom"); + // Check for the custom values + getByText('denyMax:5'); + getByText('denySum:10'); }); }); diff --git a/public/pages/ReviewAndCreate/components/__tests__/ModelConfigurationFields.test.tsx b/public/pages/ReviewAndCreate/components/__tests__/ModelConfigurationFields.test.tsx index 4179eebf..c0f59a4d 100644 --- a/public/pages/ReviewAndCreate/components/__tests__/ModelConfigurationFields.test.tsx +++ b/public/pages/ReviewAndCreate/components/__tests__/ModelConfigurationFields.test.tsx @@ -19,6 +19,7 @@ import { DATA_TYPES } from '../../../../utils/constants'; import { getRandomFeature } from '../../../../redux/reducers/__tests__/utils'; import { CoreServicesContext } from '../../../../components/CoreServices/CoreServices'; import { coreServicesMock } from '../../../../../test/mocks'; +import { ImputationMethod } from '../../../../models/types'; const detectorFaker = new chance('seed'); const features = new Array(detectorFaker.natural({ min: 1, max: 5 })) @@ -59,6 +60,7 @@ const testDetector = { ], }, featureAttributes: features, + imputationOption: { method: ImputationMethod.ZERO} } as Detector; describe('ModelConfigurationFields', () => { @@ -81,6 +83,7 @@ describe('ModelConfigurationFields', () => { userEvent.click(getByTestId('viewFeature-0')); await waitFor(() => { queryByText('max'); + queryByText('Zero'); }); }); }); diff --git a/public/pages/ReviewAndCreate/components/__tests__/__snapshots__/AdditionalSettings.test.tsx.snap b/public/pages/ReviewAndCreate/components/__tests__/__snapshots__/AdditionalSettings.test.tsx.snap index 4b8acd17..022afa0e 100644 --- a/public/pages/ReviewAndCreate/components/__tests__/__snapshots__/AdditionalSettings.test.tsx.snap +++ b/public/pages/ReviewAndCreate/components/__tests__/__snapshots__/AdditionalSettings.test.tsx.snap @@ -64,6 +64,40 @@ exports[` spec renders the component with high cardinality + + + + Imputation method + + + + + + + Custom values + + + @@ -106,6 +140,40 @@ exports[` spec renders the component with high cardinality + +
+ Imputation method +
+
+ + Ignore + +
+ + +
+ Custom values +
+
+
+
+ @@ -178,6 +246,40 @@ exports[` spec renders the component with high cardinality + + + + Imputation method + + + + + + + Custom values + + + @@ -220,6 +322,47 @@ exports[` spec renders the component with high cardinality
+ +
+ Imputation method +
+
+ + Custom + +
+ + +
+ Custom values +
+
+
+

+ denyMax:5 +

+

+ denySum:10 +

+
+
+ diff --git a/public/pages/ReviewAndCreate/components/__tests__/__snapshots__/ModelConfigurationFields.test.tsx.snap b/public/pages/ReviewAndCreate/components/__tests__/__snapshots__/ModelConfigurationFields.test.tsx.snap index c1baf2f5..a0cb87f0 100644 --- a/public/pages/ReviewAndCreate/components/__tests__/__snapshots__/ModelConfigurationFields.test.tsx.snap +++ b/public/pages/ReviewAndCreate/components/__tests__/__snapshots__/ModelConfigurationFields.test.tsx.snap @@ -155,6 +155,40 @@ exports[`ModelConfigurationFields renders the component in create mode (no ID) 1 + + + + Imputation method + + + + + + + Custom values + + + @@ -197,6 +231,40 @@ exports[`ModelConfigurationFields renders the component in create mode (no ID) 1 + +
+ Imputation method +
+
+ + set_to_zero + +
+ + +
+ Custom values +
+
+
+
+ diff --git a/public/pages/ReviewAndCreate/containers/__tests__/__snapshots__/ReviewAndCreate.test.tsx.snap b/public/pages/ReviewAndCreate/containers/__tests__/__snapshots__/ReviewAndCreate.test.tsx.snap index f8f3c901..3d8d46a6 100644 --- a/public/pages/ReviewAndCreate/containers/__tests__/__snapshots__/ReviewAndCreate.test.tsx.snap +++ b/public/pages/ReviewAndCreate/containers/__tests__/__snapshots__/ReviewAndCreate.test.tsx.snap @@ -697,6 +697,40 @@ exports[` spec renders the component, validation loading 1`] + + + + Imputation method + + + + + + + Custom values + + + @@ -739,6 +773,40 @@ exports[` spec renders the component, validation loading 1`]
+ +
+ Imputation method +
+
+ + ignore + +
+ + +
+ Custom values +
+
+
+
+ @@ -1839,6 +1907,40 @@ exports[`issue in detector validation issues in feature query 1`] = ` + + + + Imputation method + + + + + + + Custom values + + + @@ -1881,6 +1983,40 @@ exports[`issue in detector validation issues in feature query 1`] = `
+ +
+ Imputation method +
+
+ + ignore + +
+ + +
+ Custom values +
+
+
+
+ diff --git a/public/pages/ReviewAndCreate/utils/helpers.ts b/public/pages/ReviewAndCreate/utils/helpers.ts index 6f1a15bd..24703c7d 100644 --- a/public/pages/ReviewAndCreate/utils/helpers.ts +++ b/public/pages/ReviewAndCreate/utils/helpers.ts @@ -25,6 +25,7 @@ import { CreateDetectorFormikValues } from '../../CreateDetectorSteps/models/int import { OPERATORS_QUERY_MAP } from '../../DefineDetector/utils/whereFilters'; import { convertTimestampToNumber } from '../../../utils/utils'; import { CUSTOM_AD_RESULT_INDEX_PREFIX } from '../../../../server/utils/constants'; +import { formikToImputationOption } from '../../ConfigureModel/utils/helpers'; export function formikToDetector(values: CreateDetectorFormikValues): Detector { const detectionDateRange = values.historical @@ -59,10 +60,23 @@ export function formikToDetector(values: CreateDetectorFormikValues): Detector { categoryField: !isEmpty(values?.categoryField) ? values.categoryField : undefined, - resultIndexMinAge: resultIndex && resultIndex.trim().length > 0 ? values.resultIndexMinAge : undefined, - resultIndexMinSize: resultIndex && resultIndex.trim().length > 0 ? values.resultIndexMinSize : undefined, - resultIndexTtl: resultIndex && resultIndex.trim().length > 0 ? values.resultIndexTtl : undefined, - flattenCustomResultIndex: resultIndex && resultIndex.trim().length > 0 ? values.flattenCustomResultIndex : undefined, + resultIndexMinAge: + resultIndex && resultIndex.trim().length > 0 + ? values.resultIndexMinAge + : undefined, + resultIndexMinSize: + resultIndex && resultIndex.trim().length > 0 + ? values.resultIndexMinSize + : undefined, + resultIndexTtl: + resultIndex && resultIndex.trim().length > 0 + ? values.resultIndexTtl + : undefined, + flattenCustomResultIndex: + resultIndex && resultIndex.trim().length > 0 + ? values.flattenCustomResultIndex + : undefined, + imputationOption: formikToImputationOption(values.imputationOption), } as Detector; // Optionally add detection date range diff --git a/public/redux/reducers/__tests__/utils.ts b/public/redux/reducers/__tests__/utils.ts index 7ebe84ce..01df3d9c 100644 --- a/public/redux/reducers/__tests__/utils.ts +++ b/public/redux/reducers/__tests__/utils.ts @@ -10,7 +10,7 @@ */ import chance from 'chance'; -import { isEmpty, snakeCase } from 'lodash'; +import { isEmpty, snakeCase, random } from 'lodash'; import { Detector, FeatureAttributes, @@ -23,6 +23,9 @@ import { import moment from 'moment'; import { DETECTOR_STATE } from '../../../../server/utils/constants'; import { DEFAULT_SHINGLE_SIZE } from '../../../utils/constants'; +import { + ImputationMethod, ImputationOption, +} from '../../../models/types'; const detectorFaker = new chance('seed'); @@ -124,6 +127,7 @@ export const getRandomDetector = ( resultIndexMinSize: 51200, resultIndexTtl: 60, flattenCustomResultIndex: true, + imputationOption: randomImputationOption(features) }; }; @@ -202,3 +206,44 @@ export const getRandomMonitor = ( lastUpdateTime: moment(1586823218000).subtract(1, 'days').valueOf(), }; }; + +export const randomFixedValue = (features: FeatureAttributes[]): Array<{ featureName: string; data: number }> => { + const randomValues: Array<{ featureName: string; data: number }> = []; + + if (!features) { + return randomValues; + } + + features.forEach((feature) => { + if (feature.featureEnabled) { + const randomValue = Math.random() * 100; // generate a random value, e.g., between 0 and 100 + randomValues.push({ featureName: feature.featureName, data: randomValue }); + } + }); + + return randomValues; +}; + + +export const randomImputationOption = (features: FeatureAttributes[]): ImputationOption | undefined => { + const randomFixedValueMap = randomFixedValue(features); + + const options: ImputationOption[] = []; + + if (Object.keys(randomFixedValueMap).length !== 0) { + options.push({ + method: ImputationMethod.FIXED_VALUES, + defaultFill: randomFixedValueMap, + }); + } + + options.push({ method: ImputationMethod.ZERO }); + options.push({ method: ImputationMethod.PREVIOUS }); + + // Select a random option. random in lodash is inclusive of both min and max + const randomIndex = random(0, options.length); + if (options.length == randomIndex) { + return undefined; + } + return options[randomIndex]; +};