From 6bee0ae211c7bbed14e8fbf3c61caf59640cda84 Mon Sep 17 00:00:00 2001 From: gaobinlong Date: Tue, 27 Aug 2024 16:20:34 +0800 Subject: [PATCH 01/10] Add generate anomaly detector action to discover page Signed-off-by: gaobinlong --- opensearch_dashboards.json | 3 +- .../GenerateAnomalyDetector.tsx | 853 ++++++++++++++++++ public/plugin.ts | 19 +- public/redux/reducers/__tests__/ad.test.ts | 20 +- .../reducers/__tests__/opensearch.test.ts | 6 +- public/redux/reducers/ad.ts | 39 +- public/redux/reducers/assistant.ts | 65 ++ public/redux/reducers/opensearch.ts | 2 +- public/utils/discoverAction.tsx | 38 + server/cluster/ad/mlPlugin.ts | 42 + server/plugin.ts | 15 +- server/routes/assistant.ts | 104 +++ server/utils/constants.ts | 2 + utils/constants.ts | 3 + 14 files changed, 1165 insertions(+), 46 deletions(-) create mode 100644 public/components/DiscoverAction/GenerateAnomalyDetector.tsx create mode 100644 public/redux/reducers/assistant.ts create mode 100644 public/utils/discoverAction.tsx create mode 100644 server/cluster/ad/mlPlugin.ts create mode 100644 server/routes/assistant.ts diff --git a/opensearch_dashboards.json b/opensearch_dashboards.json index c5d55e96..551bd678 100644 --- a/opensearch_dashboards.json +++ b/opensearch_dashboards.json @@ -7,7 +7,8 @@ ], "optionalPlugins": [ "dataSource", - "dataSourceManagement" + "dataSourceManagement", + "dataExplorer" ], "requiredPlugins": [ "opensearchDashboardsUtils", diff --git a/public/components/DiscoverAction/GenerateAnomalyDetector.tsx b/public/components/DiscoverAction/GenerateAnomalyDetector.tsx new file mode 100644 index 00000000..357a0de6 --- /dev/null +++ b/public/components/DiscoverAction/GenerateAnomalyDetector.tsx @@ -0,0 +1,853 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React, { useState, useEffect, Fragment } from 'react'; +import { + EuiFlyoutHeader, + EuiFlyoutBody, + EuiFlyoutFooter, + EuiTitle, + EuiButton, + EuiSpacer, + EuiText, + EuiFormRow, + EuiFieldText, + EuiCheckbox, + EuiFlexItem, + EuiFlexGroup, + EuiFieldNumber, + EuiCallOut, + EuiButtonEmpty, + EuiPanel, + EuiComboBox, +} from '@elastic/eui'; +import '../FeatureAnywhereContextMenu/CreateAnomalyDetector/styles.scss'; +import { useDispatch } from 'react-redux'; +import { isEmpty, get } from 'lodash'; +import { + Field, + FieldArray, + FieldArrayRenderProps, + FieldProps, + Formik, +} from 'formik'; +import { + createDetector, + getDetectorCount, + matchDetector, + startDetector, +} from '../../../public/redux/reducers/ad'; +import { + getError, + getErrorMessage, + isInvalid, + validateCategoryField, + validateDetectorName, + validateNonNegativeInteger, + validatePositiveInteger, +} from '../../../public/utils/utils'; +import { + CUSTOM_AD_RESULT_INDEX_PREFIX, + MAX_DETECTORS, +} from '../../../server/utils/constants'; +import { + focusOnFirstWrongFeature, + initialFeatureValue, + validateFeatures, +} from '../../../public/pages/ConfigureModel/utils/helpers'; +import { formikToDetector } from '../../../public/pages/ReviewAndCreate/utils/helpers'; +import { FormattedFormRow } from '../../../public/components/FormattedFormRow/FormattedFormRow'; +import { FeatureAccordion } from '../../../public/pages/ConfigureModel/components/FeatureAccordion'; +import { AD_DOCS_LINK, DEFAULT_SHINGLE_SIZE, MAX_FEATURE_NUM, PLUGIN_NAME } from '../../../public/utils/constants'; +import { getNotifications } from '../../services'; +import { prettifyErrorMessage } from '../../../server/utils/helpers'; +import EnhancedAccordion from '../FeatureAnywhereContextMenu/EnhancedAccordion'; +import MinimalAccordion from '../FeatureAnywhereContextMenu/MinimalAccordion'; +import { DataFilterList } from '../../../public/pages/DefineDetector/components/DataFilterList/DataFilterList'; +import { generateParameters } from '../../../public/redux/reducers/assistant'; +import { FEATURE_TYPE } from '../../../public/models/interfaces'; +import { FeaturesFormikValues } from '../../../public/pages/ConfigureModel/models/interfaces'; +import { DiscoverActionContext } from '../../../../../src/plugins/data_explorer/public/types'; +import { getMappings } from '../../../public/redux/reducers/opensearch'; +import { mountReactNode } from '../../../../../src/core/public/utils'; + +export interface GeneratedParameters { + categoryField: string; + features: FeaturesFormikValues[]; + dateFields: string[]; +} + +function GenerateAnomalyDetector({ + closeFlyout, + context, +}: { + closeFlyout: any; + context: DiscoverActionContext; +}) { + const dispatch = useDispatch(); + const notifications = getNotifications(); + const indexPatternId = context.indexPattern?.id; + const indexPatternName = context.indexPattern?.title; + if (!indexPatternId || !indexPatternName) { + notifications.toasts.addDanger( + 'Cannot extract index pattern from the context' + ); + return <>; + } + + const dataSourceId = context.indexPattern?.dataSourceRef?.id; + const timeFieldFromIndexPattern = context.indexPattern?.timeFieldName; + const fieldsFromContext = context.indexPattern?.fields || []; + const [categoricalFields, dateFields] = fieldsFromContext.reduce( + ([cFields, dFields], indexPatternField) => { + const esType = indexPatternField.spec.esTypes?.[0]; + const name = indexPatternField.spec.name; + if (esType === 'keyword' || esType === 'ip') { + cFields.push(name); + } else if (esType === 'date') { + dFields.push(name); + } + return [cFields, dFields]; + }, + [[], []] as [string[], string[]] + ) || [[], []]; + + const [isLoading, setIsLoading] = useState(true); + const [buttonName, setButtonName] = useState( + 'Generating parameters...' + ); + const [categoryFieldEnabled, setCategoryFieldEnabled] = + useState(false); + + const [accordionsOpen, setAccordionsOpen] = useState>({ modelFeatures: true }); + const [intervalValue, setIntervalalue] = useState(10); + const [delayValue, setDelayValue] = useState(1); + const [enabled, setEnabled] = useState(false); + const [detectorName, setDetectorName] = useState( + indexPatternName.replace('*', '-') + '_anomaly_detector' + ); + + // let LLM to generate parameters for creating anomaly detector + async function getParameters() { + try { + const result = await dispatch( + generateParameters(indexPatternName!, dataSourceId) + ); + const rawGeneratedParameters = get(result, 'generatedParameters'); + if (!rawGeneratedParameters) { + throw new Error('Cannot get generated parameters'); + } + + const generatedParameters = formatGeneratedParameters(rawGeneratedParameters); + if (generatedParameters.features.length == 0) { + throw new Error('Generated parameters have empty model features'); + } + + initialDetectorValue.featureList = generatedParameters.features; + initialDetectorValue.categoryFieldEnabled = !!generatedParameters.categoryField; + initialDetectorValue.categoryField = initialDetectorValue.categoryFieldEnabled ? [generatedParameters.categoryField] : []; + + setIsLoading(false); + setButtonName('Create detector'); + setCategoryFieldEnabled(!!generatedParameters.categoryField); + } catch (error) { + notifications.toasts.addDanger( + 'Generate parameters for creating anomaly detector failed, reason: ' + error + ); + } + } + + const formatGeneratedParameters = function (rawGeneratedParameters: any): GeneratedParameters { + const categoryField = rawGeneratedParameters['categoryField']; + + const rawAggregationFields = rawGeneratedParameters['aggregationField']; + const rawAggregationMethods = rawGeneratedParameters['aggregationMethod']; + const rawDataFields = rawGeneratedParameters['dateFields']; + if (!rawAggregationFields || !rawAggregationMethods || !rawDataFields) { + throw new Error('Cannot find aggregation field, aggregation method or data fields!'); + } + const aggregationFields = + rawAggregationFields.split(','); + const aggregationMethods = + rawAggregationMethods.split(','); + const dateFields = rawDataFields.split(','); + + if (aggregationFields.length != aggregationMethods.length) { + throw new Error('The number of aggregation fields and the number of aggregation methods are different'); + } + + const featureList = aggregationFields.map((field: string, index: number) => { + const method = aggregationMethods[index]; + if (!field || !method) { + throw new Error('The generated aggregation field or aggregation method is empty'); + } + const aggregationOption = { + label: field, + }; + const feature: FeaturesFormikValues = { + featureName: `feature_${field}`, + featureType: FEATURE_TYPE.SIMPLE, + featureEnabled: true, + aggregationQuery: '', + aggregationBy: aggregationMethods[index], + aggregationOf: [aggregationOption], + }; + return feature; + }); + + return { + categoryField: categoryField, + features: featureList, + dateFields: dateFields, + }; + }; + + useEffect(() => { + async function fetchData() { + await getParameters(); + const getMappingDispatchCall = dispatch( + getMappings(indexPatternName, dataSourceId) + ); + await Promise.all([getMappingDispatchCall]); + } + fetchData(); + }, []); + + const onDetectorNameChange = (e: any, field: any) => { + field.onChange(e); + setDetectorName(e.target.value); + }; + + const onAccordionToggle = (key: string) => { + const newAccordionsOpen = { ...accordionsOpen }; + newAccordionsOpen[key] = !accordionsOpen[key]; + setAccordionsOpen(newAccordionsOpen); + }; + + const onIntervalChange = (e: any, field: any) => { + field.onChange(e); + setIntervalalue(e.target.value); + }; + + const onDelayChange = (e: any, field: any) => { + field.onChange(e); + setDelayValue(e.target.value); + }; + + const handleValidationAndSubmit = (formikProps: any) => { + if (formikProps.values.featureList.length !== 0) { + formikProps.setFieldTouched('featureList', true); + formikProps.validateForm().then(async (errors: any) => { + if (!isEmpty(errors)) { + focusOnFirstWrongFeature(errors, formikProps.setFieldTouched); + notifications.toasts.addDanger( + 'One or more input fields is invalid.' + ); + } else { + handleSubmit(formikProps); + } + }); + } else { + notifications.toasts.addDanger('One or more features are required.'); + } + }; + + const handleSubmit = async (formikProps: any) => { + formikProps.setSubmitting(true); + try { + const detectorToCreate = formikToDetector(formikProps.values); + await dispatch(createDetector(detectorToCreate, dataSourceId)) + .then(async (response: any) => { + const detectorId = response.response.id; + dispatch(startDetector(detectorId, dataSourceId)) + .then(() => { }) + .catch((err: any) => { + notifications.toasts.addDanger( + prettifyErrorMessage( + getErrorMessage( + err, + 'There was a problem starting the real-time detector' + ) + ) + ); + }); + + const shingleSize = get( + formikProps.values, + 'shingleSize', + DEFAULT_SHINGLE_SIZE + ); + notifications.toasts.addSuccess({ + title: mountReactNode( + + Detector created: { + e.preventDefault(); + const url = `../${PLUGIN_NAME}#/detectors/${detectorId}`; + window.open(url, '_blank'); + }} style={{ textDecoration: 'underline' }}>{formikProps.values.name} + + ), + text: mountReactNode( + +

+ Attempting to initialize the detector with historical data. This + initializing process takes approximately 1 minute if you have data in + each of the last {32 + shingleSize} consecutive intervals. +

+ + ), + className: 'createdAndAssociatedSuccessToast', + }); + + }) + .catch((err: any) => { + dispatch(getDetectorCount(dataSourceId)).then((response: any) => { + const totalDetectors = get(response, 'response.count', 0); + if (totalDetectors === MAX_DETECTORS) { + notifications.toasts.addDanger( + 'Cannot create detector - limit of ' + + MAX_DETECTORS + + ' detectors reached' + ); + } else { + notifications.toasts.addDanger( + prettifyErrorMessage( + getErrorMessage( + err, + 'There was a problem creating the detector' + ) + ) + ); + } + }); + }); + closeFlyout(); + } catch (e) { + } finally { + formikProps.setSubmitting(false); + } + }; + + const validateAnomalyDetectorName = async (detectorName: string) => { + if (isEmpty(detectorName)) { + return 'Detector name cannot be empty'; + } else { + const error = validateDetectorName(detectorName); + if (error) { + return error; + } + const resp = await dispatch(matchDetector(detectorName, dataSourceId)); + const match = get(resp, 'response.match', false); + if (!match) { + return undefined; + } + //If more than one detectors found, duplicate exists. + if (match) { + return 'Duplicate detector name'; + } + } + }; + + let initialDetectorValue = { + name: detectorName, + index: [{ label: indexPatternName }], + timeField: timeFieldFromIndexPattern, + interval: intervalValue, + windowDelay: delayValue, + shingleSize: DEFAULT_SHINGLE_SIZE, + filterQuery: { match_all: {} }, + description: 'Created based on the OpenSearch Assistant', + resultIndex: undefined, + filters: [], + featureList: [] as FeaturesFormikValues[], + categoryFieldEnabled: false, + categoryField: [] as string[], + realTime: true, + historical: false, + }; + + return ( +
+ + {(formikProps) => ( + <> + + +

+ Generate anomaly detector +

+ + + +
+ +

Detector details

+ + + + onAccordionToggle('detectorDetails')} + subTitle={ + +

+ Detector interval: {intervalValue} minute(s); Window + delay: {delayValue} minute(s) +

+ + } + > + + {({ field, form }: FieldProps) => ( + + onDetectorNameChange(e, field)} + /> + + )} + + + + + {({ field, form }: FieldProps) => ( + + + + + + onIntervalChange(e, field)} + /> + + + +

minute(s)

+ + + + + + + )} + + + + + {({ field, form }: FieldProps) => ( + + + + onDelayChange(e, field)} + /> + + + +

minute(s)

+ + + + + )} + + + + + + onAccordionToggle('advancedConfiguration')} + initialIsOpen={false} + > + + + + +

Source: {'test'}

+ + + + + + + + + {({ field, form }: FieldProps) => ( + + + + + + + +

intervals

+ + + + + )} + + + + + + {({ field, form }: FieldProps) => ( + + + { + if (enabled) { + form.setFieldValue('resultIndex', ''); + } + setEnabled(!enabled); + }} + /> + + + {enabled ? ( + + + + ) : null} + + {enabled ? ( + + + + + + ) : null} + + )} + + + + + + {({ field, form }: FieldProps) => ( + + + { + if (categoryFieldEnabled) { + form.setFieldValue('categoryField', []); + } + setCategoryFieldEnabled(!categoryFieldEnabled); + }} + /> + + {categoryFieldEnabled ? ( + + + + ) : null} + {categoryFieldEnabled ? ( + + + { + return { + label: value, + }; + }) + } + options={categoricalFields?.map((field) => { + return { + label: field, + }; + })} + onBlur={() => { + form.setFieldTouched('categoryField', true); + }} + onChange={(options) => { + const selection = options.map( + (option) => option.label + ); + if (!isEmpty(selection)) { + if (selection.length <= 2) { + form.setFieldValue( + 'categoryField', + selection + ); + } + } else { + form.setFieldValue('categoryField', []); + } + }} + singleSelection={false} + isClearable={true} + /> + + + ) : null} + + )} + + + + + + {({ field, form }: FieldProps) => ( + + { + return { + label: field, + }; + })} + onBlur={() => { + form.setFieldTouched('timeField', true); + }} + onChange={(options) => { + form.setFieldValue( + 'timeField', + get(options, '0.label') + ); + }} + selectedOptions={ + field.value + ? [ + { + label: field.value, + }, + ] + : [{ label: timeFieldFromIndexPattern }] + } + singleSelection={{ asPlainText: true }} + isClearable={false} + /> + + )} + + + + + + +

Model Features

+ + + + onAccordionToggle('modelFeatures')} + > + + {({ + push, + remove, + form: { values }, + }: FieldArrayRenderProps) => { + return ( + + {values.featureList.map( + (feature: any, index: number) => ( + { + remove(index); + }} + index={index} + feature={feature} + handleChange={formikProps.handleChange} + displayMode="flyout" + /> + ) + )} + + + + = MAX_FEATURE_NUM + } + onClick={() => { + push(initialFeatureValue()); + }} + > + Add another feature + + + + +

+ You can add up to{' '} + {Math.max( + MAX_FEATURE_NUM - values.featureList.length, + 0 + )}{' '} + more features. +

+ + + ); + }} + + + +
+ + + + + Cancel + + + { + handleValidationAndSubmit(formikProps); + }} + > + {buttonName} + + + + + + )} + +
+ ); +} + +export default GenerateAnomalyDetector; diff --git a/public/plugin.ts b/public/plugin.ts index 95274204..68eedfcf 100644 --- a/public/plugin.ts +++ b/public/plugin.ts @@ -54,6 +54,8 @@ import { DataPublicPluginStart } from '../../../src/plugins/data/public'; import { DataSourceManagementPluginSetup } from '../../../src/plugins/data_source_management/public'; import { DataSourcePluginSetup } from '../../../src/plugins/data_source/public'; import { NavigationPublicPluginStart } from '../../../src/plugins/navigation/public'; +import { getDiscoverAction } from './utils/discoverAction'; +import { DataExplorerPluginSetup } from '../../../src/plugins/data_explorer/public'; declare module '../../../src/plugins/ui_actions/public' { export interface ActionContextMapping { @@ -68,6 +70,7 @@ export interface AnomalyDetectionSetupDeps { visAugmenter: VisAugmenterSetup; dataSourceManagement: DataSourceManagementPluginSetup; dataSource: DataSourcePluginSetup; + dataExplorer: DataExplorerPluginSetup; } export interface AnomalyDetectionStartDeps { @@ -102,7 +105,7 @@ export class AnomalyDetectionOpenSearchDashboardsPlugin }); // register applications with category and use case information - core.chrome.navGroup.addNavLinksToGroup(DEFAULT_NAV_GROUPS.observability,[ + core.chrome.navGroup.addNavLinksToGroup(DEFAULT_NAV_GROUPS.observability, [ { id: PLUGIN_NAME, category: DEFAULT_APP_CATEGORIES.detect, @@ -121,7 +124,7 @@ export class AnomalyDetectionOpenSearchDashboardsPlugin const [coreStart] = await core.getStartServices(); return renderApp(coreStart, params, APP_PATH.OVERVIEW, hideInAppSideNavBar); }, - }); + }); } if (core.chrome.navGroup.getNavGroupEnabled()) { @@ -135,7 +138,7 @@ export class AnomalyDetectionOpenSearchDashboardsPlugin const [coreStart] = await core.getStartServices(); return renderApp(coreStart, params, APP_PATH.DASHBOARD, hideInAppSideNavBar); }, - }); + }); } if (core.chrome.navGroup.getNavGroupEnabled()) { @@ -149,15 +152,15 @@ export class AnomalyDetectionOpenSearchDashboardsPlugin const [coreStart] = await core.getStartServices(); return renderApp(coreStart, params, APP_PATH.LIST_DETECTORS, hideInAppSideNavBar); }, - }); + }); } // link the sub applications to the parent application core.chrome.navGroup.addNavLinksToGroup( DEFAULT_NAV_GROUPS.observability, [{ - id: OVERVIEW_PAGE_NAV_ID, - parentNavLinkId: PLUGIN_NAME + id: OVERVIEW_PAGE_NAV_ID, + parentNavLinkId: PLUGIN_NAME }, { id: DASHBOARD_PAGE_NAV_ID, @@ -189,6 +192,10 @@ export class AnomalyDetectionOpenSearchDashboardsPlugin plugins.uiActions.addTriggerAction(CONTEXT_MENU_TRIGGER, action); }); + // Add action to Discover + const discoverAction = getDiscoverAction(); + plugins.dataExplorer.registerDiscoverAction(discoverAction); + // registers the expression function used to render anomalies on an Augmented Visualization plugins.expressions.registerFunction(overlayAnomaliesFunction); return {}; diff --git a/public/redux/reducers/__tests__/ad.test.ts b/public/redux/reducers/__tests__/ad.test.ts index 79bd8f48..b99b82ba 100644 --- a/public/redux/reducers/__tests__/ad.test.ts +++ b/public/redux/reducers/__tests__/ad.test.ts @@ -54,7 +54,7 @@ describe('detector reducer actions', () => { }, }); expect(httpMockedClient.get).toHaveBeenCalledWith( - `..${BASE_NODE_API_PATH}/detectors/${detectorId}` + `${BASE_NODE_API_PATH}/detectors/${detectorId}` ); }); test('should invoke [REQUEST, FAILURE]', async () => { @@ -76,7 +76,7 @@ describe('detector reducer actions', () => { errorMessage: 'Not found', }); expect(httpMockedClient.get).toHaveBeenCalledWith( - `..${BASE_NODE_API_PATH}/detectors/${detectorId}` + `${BASE_NODE_API_PATH}/detectors/${detectorId}` ); } }); @@ -104,7 +104,7 @@ describe('detector reducer actions', () => { }, }); expect(httpMockedClient.delete).toHaveBeenCalledWith( - `..${BASE_NODE_API_PATH}/detectors/${expectedDetector.id}` + `${BASE_NODE_API_PATH}/detectors/${expectedDetector.id}` ); }); test('should invoke [REQUEST, FAILURE]', async () => { @@ -129,7 +129,7 @@ describe('detector reducer actions', () => { errorMessage: 'Detector is consumed by Monitor', }); expect(httpMockedClient.delete).toHaveBeenCalledWith( - `..${BASE_NODE_API_PATH}/detectors/${expectedDetector.id}` + `${BASE_NODE_API_PATH}/detectors/${expectedDetector.id}` ); } }); @@ -162,7 +162,7 @@ describe('detector reducer actions', () => { }, }); expect(httpMockedClient.post).toHaveBeenCalledWith( - `..${BASE_NODE_API_PATH}/detectors`, + `${BASE_NODE_API_PATH}/detectors`, { body: JSON.stringify(expectedDetector), } @@ -190,7 +190,7 @@ describe('detector reducer actions', () => { errorMessage: 'Internal server error', }); expect(httpMockedClient.post).toHaveBeenCalledWith( - `..${BASE_NODE_API_PATH}/detectors`, + `${BASE_NODE_API_PATH}/detectors`, { body: JSON.stringify(expectedDetector), } @@ -230,7 +230,7 @@ describe('detector reducer actions', () => { }, }); expect(httpMockedClient.put).toHaveBeenCalledWith( - `..${BASE_NODE_API_PATH}/detectors/${detectorId}`, + `${BASE_NODE_API_PATH}/detectors/${detectorId}`, { body: JSON.stringify(randomDetector) } ); }); @@ -258,7 +258,7 @@ describe('detector reducer actions', () => { errorMessage: 'Internal server error', }); expect(httpMockedClient.post).toHaveBeenCalledWith( - `..${BASE_NODE_API_PATH}/detectors`, + `${BASE_NODE_API_PATH}/detectors`, randomDetector, { params: { @@ -298,7 +298,7 @@ describe('detector reducer actions', () => { ), }); expect(httpMockedClient.post).toHaveBeenCalledWith( - `..${BASE_NODE_API_PATH}/detectors/_search`, + `${BASE_NODE_API_PATH}/detectors/_search`, { body: JSON.stringify(query), } @@ -328,7 +328,7 @@ describe('detector reducer actions', () => { errorMessage: 'Internal server error', }); expect(httpMockedClient.post).toHaveBeenCalledWith( - `..${BASE_NODE_API_PATH}/detectors`, + `${BASE_NODE_API_PATH}/detectors`, randomDetector ); } diff --git a/public/redux/reducers/__tests__/opensearch.test.ts b/public/redux/reducers/__tests__/opensearch.test.ts index cb434c0c..bfd3c99c 100644 --- a/public/redux/reducers/__tests__/opensearch.test.ts +++ b/public/redux/reducers/__tests__/opensearch.test.ts @@ -173,7 +173,7 @@ describe('opensearch reducer actions', () => { }, }); expect(httpMockedClient.get).toHaveBeenCalledWith( - `..${BASE_NODE_API_PATH}/_mappings`, + `${BASE_NODE_API_PATH}/_mappings`, { query: { index: '' }, } @@ -200,7 +200,7 @@ describe('opensearch reducer actions', () => { errorMessage: 'Something went wrong', }); expect(httpMockedClient.get).toHaveBeenCalledWith( - `..${BASE_NODE_API_PATH}/_mappings`, + `${BASE_NODE_API_PATH}/_mappings`, { query: { index: '' }, } @@ -278,5 +278,5 @@ describe('opensearch reducer actions', () => { } }); }); - describe('getPrioritizedIndices', () => {}); + describe('getPrioritizedIndices', () => { }); }); diff --git a/public/redux/reducers/ad.ts b/public/redux/reducers/ad.ts index 3fa06ad3..a1a689d1 100644 --- a/public/redux/reducers/ad.ts +++ b/public/redux/reducers/ad.ts @@ -374,9 +374,8 @@ export const createDetector = ( dataSourceId: string = '' ): APIAction => { const url = dataSourceId - ? `..${AD_NODE_API.DETECTOR}/${dataSourceId}` - : `..${AD_NODE_API.DETECTOR}`; - + ? `${AD_NODE_API.DETECTOR}/${dataSourceId}` + : `${AD_NODE_API.DETECTOR}`; return { type: CREATE_DETECTOR, request: (client: HttpSetup) => @@ -391,7 +390,7 @@ export const validateDetector = ( validationType: string, dataSourceId: string = '' ): APIAction => { - const baseUrl = `..${AD_NODE_API.DETECTOR}/_validate/${validationType}`; + const baseUrl = `${AD_NODE_API.DETECTOR}/_validate/${validationType}`; const url = dataSourceId ? `${baseUrl}/${dataSourceId}` : baseUrl; return { @@ -407,7 +406,7 @@ export const getDetector = ( detectorId: string, dataSourceId: string = '' ): APIAction => { - const baseUrl = `..${AD_NODE_API.DETECTOR}/${detectorId}`; + const baseUrl = `${AD_NODE_API.DETECTOR}/${detectorId}`; const url = dataSourceId ? `${baseUrl}/${dataSourceId}` : baseUrl; return { @@ -422,7 +421,7 @@ export const getDetectorList = ( ): APIAction => { const dataSourceId = queryParams.dataSourceId || ''; - const baseUrl = `..${AD_NODE_API.DETECTOR}/_list`; + const baseUrl = `${AD_NODE_API.DETECTOR}/_list`; const url = dataSourceId ? `${baseUrl}/${dataSourceId}` : baseUrl; @@ -436,7 +435,7 @@ export const getDetectorList = ( export const searchDetector = (requestBody: any): APIAction => ({ type: SEARCH_DETECTOR, request: (client: HttpSetup) => - client.post(`..${AD_NODE_API.DETECTOR}/_search`, { + client.post(`${AD_NODE_API.DETECTOR}/_search`, { body: JSON.stringify(requestBody), }), }); @@ -446,7 +445,7 @@ export const updateDetector = ( requestBody: Detector, dataSourceId: string = '' ): APIAction => { - const baseUrl = `..${AD_NODE_API.DETECTOR}/${detectorId}`; + const baseUrl = `${AD_NODE_API.DETECTOR}/${detectorId}`; const url = dataSourceId ? `${baseUrl}/${dataSourceId}` : baseUrl; return { @@ -463,7 +462,7 @@ export const deleteDetector = ( detectorId: string, dataSourceId: string = '' ): APIAction => { - const baseUrl = `..${AD_NODE_API.DETECTOR}/${detectorId}`; + const baseUrl = `${AD_NODE_API.DETECTOR}/${detectorId}`; const url = dataSourceId ? `${baseUrl}/${dataSourceId}` : baseUrl; return { @@ -477,7 +476,7 @@ export const startDetector = ( detectorId: string, dataSourceId: string = '' ): APIAction => { - const baseUrl = `..${AD_NODE_API.DETECTOR}/${detectorId}/start`; + const baseUrl = `${AD_NODE_API.DETECTOR}/${detectorId}/start`; const url = dataSourceId ? `${baseUrl}/${dataSourceId}` : baseUrl; return { @@ -493,7 +492,7 @@ export const startHistoricalDetector = ( startTime: number, endTime: number ): APIAction => { - const baseUrl = `..${AD_NODE_API.DETECTOR}/${detectorId}`; + const baseUrl = `${AD_NODE_API.DETECTOR}/${detectorId}`; const url = dataSourceId ? `${baseUrl}/${dataSourceId}/start` : `${baseUrl}/start`; @@ -517,7 +516,7 @@ export const stopDetector = ( detectorId: string, dataSourceId: string = '' ): APIAction => { - const baseUrl = `..${AD_NODE_API.DETECTOR}/${detectorId}/stop/${false}`; + const baseUrl = `${AD_NODE_API.DETECTOR}/${detectorId}/stop/${false}`; const url = dataSourceId ? `${baseUrl}/${dataSourceId}` : baseUrl; return { @@ -531,7 +530,7 @@ export const stopHistoricalDetector = ( detectorId: string, dataSourceId: string = '' ): APIAction => { - const baseUrl = `..${AD_NODE_API.DETECTOR}/${detectorId}/stop/${true}`; + const baseUrl = `${AD_NODE_API.DETECTOR}/${detectorId}/stop/${true}`; const url = dataSourceId ? `${baseUrl}/${dataSourceId}` : baseUrl; return { @@ -544,16 +543,16 @@ export const stopHistoricalDetector = ( export const getDetectorProfile = (detectorId: string): APIAction => ({ type: GET_DETECTOR_PROFILE, request: (client: HttpSetup) => - client.get(`..${AD_NODE_API.DETECTOR}/${detectorId}/_profile`), + client.get(`${AD_NODE_API.DETECTOR}/${detectorId}/_profile`), detectorId, }); export const matchDetector = ( - detectorName: string, + detectorName: string, dataSourceId: string = '' ): APIAction => { - const baseUrl = `..${AD_NODE_API.DETECTOR}/${detectorName}/_match`; - const url = dataSourceId ? `${baseUrl}/${dataSourceId}` : baseUrl; + const baseUrl = `${AD_NODE_API.DETECTOR}/${detectorName}/_match`; + const url = dataSourceId ? `${baseUrl}/${dataSourceId}` : baseUrl; return { type: MATCH_DETECTOR, @@ -562,9 +561,9 @@ export const matchDetector = ( }; export const getDetectorCount = (dataSourceId: string = ''): APIAction => { - const url = dataSourceId ? - `..${AD_NODE_API.DETECTOR}/_count/${dataSourceId}` : - `..${AD_NODE_API.DETECTOR}/_count`; + const url = dataSourceId ? + `${AD_NODE_API.DETECTOR}/_count/${dataSourceId}` : + `${AD_NODE_API.DETECTOR}/_count`; return { type: GET_DETECTOR_COUNT, diff --git a/public/redux/reducers/assistant.ts b/public/redux/reducers/assistant.ts new file mode 100644 index 00000000..7a359e18 --- /dev/null +++ b/public/redux/reducers/assistant.ts @@ -0,0 +1,65 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { APIAction, APIResponseAction, HttpSetup } from '../middleware/types'; +import handleActions from '../utils/handleActions'; +import { ASSISTANT_NODE_API } from '../../../utils/constants'; + +const GENERATE_PARAMETERS = 'assistant/GENERATE_PARAMETERS'; + +export interface GeneratedParametersState { + requesting: boolean; + errorMessage: string; +} + +export const initialState: GeneratedParametersState = { + requesting: false, + errorMessage: '', +}; + +const reducer = handleActions( + { + [GENERATE_PARAMETERS]: { + REQUEST: (state: GeneratedParametersState): GeneratedParametersState => ({ + ...state, + requesting: true, + errorMessage: '', + }), + SUCCESS: ( + state: GeneratedParametersState, + action: APIResponseAction + ): GeneratedParametersState => ({ + ...state, + requesting: false, + }), + FAILURE: ( + state: GeneratedParametersState, + action: APIResponseAction + ): GeneratedParametersState => ({ + ...state, + requesting: false, + errorMessage: action.error, + }), + }, + }, + initialState +); + +export const generateParameters = ( + index: string, + dataSourceId: string = '' +): APIAction => { + const baseUrl = `${ASSISTANT_NODE_API.GENERATE_PARAMETERS}`; + const url = dataSourceId ? `${baseUrl}/${dataSourceId}` : baseUrl; + return { + type: GENERATE_PARAMETERS, + request: (client: HttpSetup) => + client.post(url, { + body: JSON.stringify({ index: index }), + }), + }; +}; + +export default reducer; diff --git a/public/redux/reducers/opensearch.ts b/public/redux/reducers/opensearch.ts index 4a9a3d32..9ef6354e 100644 --- a/public/redux/reducers/opensearch.ts +++ b/public/redux/reducers/opensearch.ts @@ -279,7 +279,7 @@ export const getMappings = (searchKey: string = '', dataSourceId: string = ''): return { type: GET_MAPPINGS, request: (client: HttpSetup) => - client.get(`..${url}`, { + client.get(`${url}`, { query: { index: searchKey }, }), }; diff --git a/public/utils/discoverAction.tsx b/public/utils/discoverAction.tsx new file mode 100644 index 00000000..6ed2c85a --- /dev/null +++ b/public/utils/discoverAction.tsx @@ -0,0 +1,38 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React from 'react' +import { ANOMALY_DETECTION_ICON } from "./constants"; +import GenerateAnomalyDetector from "../components/DiscoverAction/GenerateAnomalyDetector"; +import { getClient, getOverlays } from '../../public/services'; +import { toMountPoint } from "../../../../src/plugins/opensearch_dashboards_react/public"; +import { Provider } from "react-redux"; +import configureStore from '../redux/configureStore'; +import { DiscoverAction, DiscoverActionContext } from "../../../../src/plugins/data_explorer/public/types"; + +export const getDiscoverAction = (): DiscoverAction => { + const onClick = function (context: DiscoverActionContext) { + const overlayService = getOverlays(); + const openFlyout = overlayService.openFlyout; + const store = configureStore(getClient()); + const overlay = openFlyout( + toMountPoint( + + overlay.close()} + context={context} + /> + + ) + ); + } + + return { + order: 0, + name: 'Generate anomaly detector', + iconType: ANOMALY_DETECTION_ICON, + onClick: onClick, + } +}; diff --git a/server/cluster/ad/mlPlugin.ts b/server/cluster/ad/mlPlugin.ts new file mode 100644 index 00000000..ad39f085 --- /dev/null +++ b/server/cluster/ad/mlPlugin.ts @@ -0,0 +1,42 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export default function mlPlugin( + Client: any, + config: any, + components: any +) { + const ca = components.clientAction.factory; + + Client.prototype.ml = components.clientAction.namespaceFactory(); + const ml = Client.prototype.ml.prototype; + + ml.getAgent = ca({ + url: { + fmt: `/_plugins/_ml/config/<%=id%>`, + req: { + id: { + type: 'string', + required: true, + }, + }, + }, + method: 'GET', + }); + + ml.executeAgent = ca({ + url: { + fmt: `/_plugins/_ml/agents/<%=agentId%>/_execute`, + req: { + agentId: { + type: 'string', + required: true, + }, + }, + }, + needBody: true, + method: 'POST', + }); +} diff --git a/server/plugin.ts b/server/plugin.ts index a6dfc3b0..8a40c9ec 100644 --- a/server/plugin.ts +++ b/server/plugin.ts @@ -37,6 +37,8 @@ import SampleDataService, { import { DEFAULT_HEADERS } from './utils/constants'; import { DataSourcePluginSetup } from '../../../src/plugins/data_source/server/types'; import { DataSourceManagementPlugin } from '../../../src/plugins/data_source_management/public'; +import AssistantService, { registerAssistantRoutes } from './routes/assistant'; +import mlPlugin from './cluster/ad/mlPlugin'; export interface ADPluginSetupDependencies { dataSourceManagement?: ReturnType; @@ -45,10 +47,10 @@ export interface ADPluginSetupDependencies { export class AnomalyDetectionOpenSearchDashboardsPlugin implements - Plugin< - AnomalyDetectionOpenSearchDashboardsPluginSetup, - AnomalyDetectionOpenSearchDashboardsPluginStart - > + Plugin< + AnomalyDetectionOpenSearchDashboardsPluginSetup, + AnomalyDetectionOpenSearchDashboardsPluginStart + > { private readonly logger: Logger; private readonly globalConfig$: any; @@ -69,7 +71,7 @@ export class AnomalyDetectionOpenSearchDashboardsPlugin const client: ILegacyClusterClient = core.opensearch.legacy.createClient( 'anomaly_detection', { - plugins: [adPlugin, alertingPlugin], + plugins: [adPlugin, alertingPlugin, mlPlugin], customHeaders: { ...customHeaders, ...DEFAULT_HEADERS }, ...rest, } @@ -80,6 +82,7 @@ export class AnomalyDetectionOpenSearchDashboardsPlugin if (dataSourceEnabled) { dataSource.registerCustomApiSchema(adPlugin); dataSource.registerCustomApiSchema(alertingPlugin); + dataSource.registerCustomApiSchema(mlPlugin); } // Create router @@ -93,12 +96,14 @@ export class AnomalyDetectionOpenSearchDashboardsPlugin const alertingService = new AlertingService(client, dataSourceEnabled); const opensearchService = new OpenSearchService(client, dataSourceEnabled); const sampleDataService = new SampleDataService(client, dataSourceEnabled); + const assistantService = new AssistantService(client, dataSourceEnabled); // Register server routes with the service registerADRoutes(apiRouter, adService); registerAlertingRoutes(apiRouter, alertingService); registerOpenSearchRoutes(apiRouter, opensearchService); registerSampleDataRoutes(apiRouter, sampleDataService); + registerAssistantRoutes(apiRouter, assistantService); return {}; } diff --git a/server/routes/assistant.ts b/server/routes/assistant.ts new file mode 100644 index 00000000..4a620e59 --- /dev/null +++ b/server/routes/assistant.ts @@ -0,0 +1,104 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +//@ts-ignore +import { get, set } from 'lodash'; +import { Router } from '../router'; +import { getErrorMessage } from './utils/adHelpers'; +import { + RequestHandlerContext, + OpenSearchDashboardsRequest, + OpenSearchDashboardsResponseFactory, + IOpenSearchDashboardsResponse, +} from '../../../../src/core/server'; +import { getClientBasedOnDataSource } from '../utils/helpers'; +import { GENERATE_ANOMALY_DETECTOR_CONFIG_ID } from '../utils/constants'; + +export function registerAssistantRoutes( + apiRouter: Router, + assistantService: AssistantService +) { + apiRouter.post('/_generate_parameters', assistantService.generateParameters); +} + +export default class AssistantService { + private client: any; + dataSourceEnabled: boolean; + + constructor(client: any, dataSourceEnabled: boolean) { + this.client = client; + this.dataSourceEnabled = dataSourceEnabled; + } + + generateParameters = async ( + context: RequestHandlerContext, + request: OpenSearchDashboardsRequest, + opensearchDashboardsResponse: OpenSearchDashboardsResponseFactory + ): Promise> => { + try { + const { dataSourceId = '' } = request.params as { dataSourceId?: string }; + const { index } = request.body as { index: string }; + if (!index) { + throw new Error('index cannot be empty'); + } + const callWithRequest = getClientBasedOnDataSource( + context, + this.dataSourceEnabled, + request, + dataSourceId, + this.client + ); + + const getAgentResponse = await callWithRequest('ml.getAgent', { + id: GENERATE_ANOMALY_DETECTOR_CONFIG_ID, + }); + + if ( + !getAgentResponse || + !getAgentResponse['configuration'] || + !getAgentResponse['configuration']['agent_id'] + ) { + throw new Error( + 'Cannot get flow agent id for generating anomaly detector' + ); + } + + const agentId = getAgentResponse['configuration']['agent_id']; + + const executeAgentResponse = await callWithRequest('ml.executeAgent', { + agentId: agentId, + body: { + parameters: { + index: index, + }, + }, + }); + if ( + !executeAgentResponse || + !executeAgentResponse['inference_results'] || + !executeAgentResponse['inference_results'][0].output[0] || + !executeAgentResponse['inference_results'][0].output[0].result + ) { + throw new Error('Execute agent for generating anomaly detector failed'); + } + + return opensearchDashboardsResponse.ok({ + body: { + ok: true, + generatedParameters: JSON.parse( + executeAgentResponse['inference_results'][0].output[0].result + ), + }, + }); + } catch (err) { + return opensearchDashboardsResponse.ok({ + body: { + ok: false, + error: getErrorMessage(err), + }, + }); + } + }; +} diff --git a/server/utils/constants.ts b/server/utils/constants.ts index ac3c887a..19902ebb 100644 --- a/server/utils/constants.ts +++ b/server/utils/constants.ts @@ -132,3 +132,5 @@ export const HISTORICAL_TASK_TYPES = [ ]; export const CUSTOM_AD_RESULT_INDEX_PREFIX = 'opensearch-ad-plugin-result-'; + +export const GENERATE_ANOMALY_DETECTOR_CONFIG_ID = 'generate_anomaly_detector'; diff --git a/utils/constants.ts b/utils/constants.ts index 231bd91b..6546e87a 100644 --- a/utils/constants.ts +++ b/utils/constants.ts @@ -27,3 +27,6 @@ export const ALERTING_NODE_API = Object.freeze({ ALERTS: `${BASE_NODE_API_PATH}/monitors/alerts`, MONITORS: `${BASE_NODE_API_PATH}/monitors`, }); +export const ASSISTANT_NODE_API = Object.freeze({ + GENERATE_PARAMETERS: `${BASE_NODE_API_PATH}/_generate_parameters`, +}); From 20d3431ce5a4d1c8c1c23c86579374a3d9f74fd4 Mon Sep 17 00:00:00 2001 From: gaobinlong Date: Wed, 28 Aug 2024 15:11:56 +0800 Subject: [PATCH 02/10] Add more test code and rename the file Signed-off-by: gaobinlong --- .../SuggestAnomalyDetector.test.tsx | 569 ++++++++++++++++++ ...etector.tsx => SuggestAnomalyDetector.tsx} | 40 +- .../FeatureAccordion/FeatureAccordion.tsx | 1 + .../reducers/__tests__/assistant.test.ts | 83 +++ public/utils/discoverAction.tsx | 4 +- server/routes/assistant.ts | 4 +- server/utils/constants.ts | 2 +- 7 files changed, 679 insertions(+), 24 deletions(-) create mode 100644 public/components/DiscoverAction/SuggestAnomalyDetector.test.tsx rename public/components/DiscoverAction/{GenerateAnomalyDetector.tsx => SuggestAnomalyDetector.tsx} (97%) create mode 100644 public/redux/reducers/__tests__/assistant.test.ts diff --git a/public/components/DiscoverAction/SuggestAnomalyDetector.test.tsx b/public/components/DiscoverAction/SuggestAnomalyDetector.test.tsx new file mode 100644 index 00000000..cca54108 --- /dev/null +++ b/public/components/DiscoverAction/SuggestAnomalyDetector.test.tsx @@ -0,0 +1,569 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +import React from 'react'; +import { render, waitFor } from '@testing-library/react'; + +import { CoreServicesContext } from '../CoreServices/CoreServices'; +import { coreServicesMock, httpClientMock } from '../../../test/mocks'; +import { + HashRouter as Router, + RouteComponentProps, + Route, + Switch, +} from 'react-router-dom'; +import { Provider } from 'react-redux'; + +import configureStore from '../../redux/configureStore'; +import GenerateAnomalyDetector from './SuggestAnomalyDetector'; +import { DiscoverActionContext } from '../../../../../src/plugins/data_explorer/public'; +import { fieldFormatsMock } from '../../../../../src/plugins/data/common/field_formats/mocks'; +import { IndexPattern } from '../../../../../src/plugins/data/common'; +import userEvent from '@testing-library/user-event'; +import { HttpFetchOptionsWithPath } from '../../../../../src/core/public'; +import { BASE_NODE_API_PATH } from '../../../utils/constants'; + +const notifications = { + toasts: { + addDanger: jest.fn().mockName('addDanger'), + addSuccess: jest.fn().mockName('addSuccess'), + } +}; + +const getNotifications = () => { + return notifications; +} + +jest.mock('../../services', () => ({ + ...jest.requireActual('../../services'), + getNotifications: getNotifications, +})); + +const renderWithRouter = (context: DiscoverActionContext) => ({ + ...render( + + + + ( + + + + )} + /> + + + + ), +}); + +export function shouldReadFieldFromDocValues(aggregatable: boolean, opensearchType: string) { + return ( + aggregatable && + !['text', 'geo_shape'].includes(opensearchType) && + !opensearchType.startsWith('_') + ); +} + + +function stubbedSampleFields() { + return [ + ['bytes', 'long', true, true, { count: 10 }], + ['response', 'integer', true, true], + ['responseLatency', 'float', true, true], + ['@timestamp', 'date', true, true, { count: 30 }], + ['@tags', 'keyword', true, true], + ['utc_time', 'date', true, true], + ['phpmemory', 'integer', true, true], + ['ip', 'ip', true, true], + ['geo.src', 'keyword', true, true], + ['_id', '_id', true, true], + ['_type', '_type', true, true], + ['_source', '_source', true, true], + ].map(function (row) { + const [ + name, + opensearchType, + aggregatable, + searchable, + metadata = {}, + subType = undefined, + ] = row; + + const { + count = 0, + script, + lang = script ? 'expression' : undefined, + scripted = !!script, + } = metadata; + + return { + name, + opensearchType, + spec: { + esTypes: [opensearchType], + name: name, + }, + readFromDocValues: shouldReadFieldFromDocValues(aggregatable, opensearchType), + aggregatable, + searchable, + count, + script, + lang, + scripted, + subType, + }; + }); +} + +function createIndexPattern(id: string): IndexPattern { + const type = 'index-pattern'; + const version = '2'; + const timeFieldName = 'timestamp'; + const fields = stubbedSampleFields(); + const title = id; + + return { + id, + type, + version, + timeFieldName, + fields, + title, + savedObjectsClient: {} as any, + fieldFormats: fieldFormatsMock, + shortDotsEnable: false, + metaFields: [], + }; +} + +const expectedAnomalyDetector = { + name: "test-pattern_anomaly_detector", + description: "Created based on the OpenSearch Assistant", + indices: ["test-pattern"], + filterQuery: { + match_all: {} + }, + uiMetadata: { + features: { + feature_responseLatency: { + featureType: "simple_aggs", + aggregationBy: "avg", + aggregationOf: "responseLatency" + }, + feature_response: { + featureType: "simple_aggs", + aggregationBy: "sum", + aggregationOf: "response" + } + }, + filters: [] + }, + featureAttributes: [ + { + featureName: "feature_responseLatency", + featureEnabled: true, + importance: 1, + aggregationQuery: { + feature_response_latency: { + avg: { + field: "responseLatency" + } + } + } + }, + { + featureName: "feature_response", + featureEnabled: true, + importance: 1, + aggregationQuery: { + feature_response: { + sum: { + field: "response" + } + } + } + } + ], + timeField: "timestamp", + detectionInterval: { + period: { + interval: 10, + unit: "Minutes" + } + }, + windowDelay: { + period: { + interval: 1, + unit: "Minutes" + } + }, + shingleSize: 8, + categoryField: ["ip"] +}; + +describe('GenerateAnomalyDetector spec', () => { + describe('Renders loading component', () => { + it('renders empty component', async () => { + httpClientMock.post = jest.fn().mockResolvedValue({ + ok: true, + generatedParameters: { + categoryField: '', + aggregationField: '', + aggregationMethod: '', + dateFields: '', + }, + }); + + const context = { + indexPattern: createIndexPattern(''), + }; + const { queryByText } = renderWithRouter(context); + expect(queryByText('Suggest anomaly detector')).toBeNull(); + + await waitFor(() => { + expect(getNotifications().toasts.addDanger).toHaveBeenCalledTimes(1); + expect(getNotifications().toasts.addDanger).toHaveBeenCalledWith( + 'Cannot extract index pattern from the context' + ); + }); + }); + + it('renders with empty generated parameters', async () => { + httpClientMock.post = jest.fn().mockResolvedValue({ + ok: true, + }); + + const context = { + indexPattern: createIndexPattern('test-pattern'), + } + const { queryByText } = renderWithRouter(context); + expect(queryByText('Suggest anomaly detector')).not.toBeNull(); + + await waitFor(() => { + expect(getNotifications().toasts.addDanger).toHaveBeenCalledTimes(1); + expect(getNotifications().toasts.addDanger).toHaveBeenCalledWith( + 'Generate parameters for creating anomaly detector failed, reason: Error: Cannot get generated parameters!' + ); + }); + }); + + it('renders with empty parameter', async () => { + httpClientMock.post = jest.fn().mockResolvedValue({ + ok: true, + generatedParameters: { + categoryField: '', + aggregationField: '', + aggregationMethod: '', + dateFields: '', + }, + }); + + const context = { + indexPattern: createIndexPattern('test-pattern'), + } + const { queryByText } = renderWithRouter(context); + expect(queryByText('Suggest anomaly detector')).not.toBeNull(); + + await waitFor(() => { + expect(getNotifications().toasts.addDanger).toHaveBeenCalledTimes(1); + expect(getNotifications().toasts.addDanger).toHaveBeenCalledWith( + 'Generate parameters for creating anomaly detector failed, reason: Error: Cannot find aggregation field, aggregation method or data fields!' + ); + }); + }); + + it('renders with empty aggregation field or empty aggregation method', async () => { + httpClientMock.post = jest.fn().mockResolvedValue({ + ok: true, + generatedParameters: { + categoryField: '', + aggregationField: ',', + aggregationMethod: ',', + dateFields: 'timestamp', + }, + }); + + const context = { + indexPattern: createIndexPattern('test-pattern'), + } + const { queryByText } = renderWithRouter(context); + expect(queryByText('Suggest anomaly detector')).not.toBeNull(); + + await waitFor(() => { + expect(getNotifications().toasts.addDanger).toHaveBeenCalledTimes(1); + expect(getNotifications().toasts.addDanger).toHaveBeenCalledWith( + 'Generate parameters for creating anomaly detector failed, reason: Error: The generated aggregation field or aggregation method is empty!' + ); + }); + }); + + it('renders with different number of aggregation methods and fields', async () => { + httpClientMock.post = jest.fn().mockResolvedValue({ + ok: true, + generatedParameters: { + categoryField: '', + aggregationField: 'a,b', + aggregationMethod: 'avg', + dateFields: 'timestamp', + }, + }); + + const context = { + indexPattern: createIndexPattern('test-pattern'), + } + const { queryByText } = renderWithRouter(context); + expect(queryByText('Suggest anomaly detector')).not.toBeNull(); + + await waitFor(() => { + expect(getNotifications().toasts.addDanger).toHaveBeenCalledTimes(1); + expect(getNotifications().toasts.addDanger).toHaveBeenCalledWith( + 'Generate parameters for creating anomaly detector failed, reason: Error: The number of aggregation fields and the number of aggregation methods are different!' + ); + }); + }); + + it('renders component completely', async () => { + httpClientMock.post = jest.fn().mockResolvedValue({ + ok: true, + generatedParameters: { + categoryField: 'ip', + aggregationField: 'responseLatency,response', + aggregationMethod: 'avg,sum', + dateFields: '@timestamp,utc_time', + }, + }); + + const context = { + indexPattern: createIndexPattern('test-pattern'), + } + const { queryByText } = renderWithRouter(context); + expect(queryByText('Suggest anomaly detector')).not.toBeNull(); + + await waitFor(() => { + expect(queryByText('Create detector')).not.toBeNull(); + expect(queryByText('Detector details')).not.toBeNull(); + expect(queryByText('Advanced configuration')).not.toBeNull(); + expect(queryByText('Model Features')).not.toBeNull(); + }); + }); + + }); + + + describe('Test API calls', () => { + it('All API calls execute successfully', async () => { + httpClientMock.post = jest.fn((pathOrOptions: string | HttpFetchOptionsWithPath) => { + const url = typeof pathOrOptions === 'string' ? pathOrOptions : pathOrOptions.path; + switch (url) { + case '/api/anomaly_detectors/_generate_parameters': + return Promise.resolve({ + ok: true, + generatedParameters: { + categoryField: 'ip', + aggregationField: 'responseLatency,response', + aggregationMethod: 'avg,sum', + dateFields: '@timestamp,utc_time', + } + }); + case '/api/anomaly_detectors/detectors': + return Promise.resolve({ + ok: true, + response: { + id: 'test' + } + }); + default: + return Promise.resolve({ + ok: true + }); + } + }); + + const context = { + indexPattern: createIndexPattern('test-pattern'), + } + const { queryByText, getByTestId } = renderWithRouter(context); + expect(queryByText('Suggest anomaly detector')).not.toBeNull(); + await waitFor(() => { + expect(queryByText('Generating parameters...')).toBeNull(); + expect(queryByText('Create detector')).not.toBeNull(); + expect(queryByText('Detector details')).not.toBeNull(); + expect(queryByText('Advanced configuration')).not.toBeNull(); + expect(queryByText('Model Features')).not.toBeNull(); + }); + + userEvent.click(getByTestId("GenerateAnomalyDetectorCreateButton")); + + await waitFor(() => { + expect(httpClientMock.post).toHaveBeenCalledTimes(3); + expect(httpClientMock.post).toHaveBeenCalledWith( + `${BASE_NODE_API_PATH}/_generate_parameters`, + { + body: JSON.stringify({ index: 'test-pattern' }), + } + ); + expect(httpClientMock.post).toHaveBeenCalledWith( + `${BASE_NODE_API_PATH}/detectors`, + { + body: JSON.stringify(expectedAnomalyDetector), + } + ); + expect(httpClientMock.post).toHaveBeenCalledWith( + `${BASE_NODE_API_PATH}/detectors/test/start` + ); + expect(getNotifications().toasts.addSuccess).toHaveBeenCalledTimes(1); + }); + }); + + it('Generate parameters failed', async () => { + httpClientMock.post = jest.fn().mockResolvedValue({ + ok: false, + error: 'Generate parameters failed' + }); + + const context = { + indexPattern: createIndexPattern('test-pattern'), + } + const { queryByText } = renderWithRouter(context); + expect(queryByText('Suggest anomaly detector')).not.toBeNull(); + await waitFor(() => { + expect(getNotifications().toasts.addDanger).toHaveBeenCalledTimes(1); + expect(getNotifications().toasts.addDanger).toHaveBeenCalledWith( + 'Generate parameters for creating anomaly detector failed, reason: Generate parameters failed' + ); + }); + }); + + it('Create anomaly detector failed', async () => { + httpClientMock.post = jest.fn((pathOrOptions: string | HttpFetchOptionsWithPath) => { + const url = typeof pathOrOptions === 'string' ? pathOrOptions : pathOrOptions.path; + switch (url) { + case '/api/anomaly_detectors/_generate_parameters': + return Promise.resolve({ + ok: true, + generatedParameters: { + categoryField: 'ip', + aggregationField: 'responseLatency,response', + aggregationMethod: 'avg,sum', + dateFields: '@timestamp,utc_time', + } + }); + case '/api/anomaly_detectors/detectors': + return Promise.resolve({ + ok: false, + error: 'Create anomaly detector failed' + }); + default: + return Promise.resolve({ + ok: true + }); + } + }); + + httpClientMock.get = jest.fn().mockResolvedValue({ + ok: true, + response: { + count: 0 + }, + }); + + + const context = { + indexPattern: createIndexPattern('test-pattern'), + } + const { queryByText, getByTestId } = renderWithRouter(context); + expect(queryByText('Suggest anomaly detector')).not.toBeNull(); + + await waitFor(() => { + expect(queryByText('Generating parameters...')).toBeNull(); + expect(queryByText('Create detector')).not.toBeNull(); + expect(queryByText('Detector details')).not.toBeNull(); + expect(queryByText('Advanced configuration')).not.toBeNull(); + expect(queryByText('Model Features')).not.toBeNull(); + }); + + userEvent.click(getByTestId("GenerateAnomalyDetectorCreateButton")); + + await waitFor(() => { + expect(getNotifications().toasts.addDanger).toHaveBeenCalledTimes(1); + expect(getNotifications().toasts.addDanger).toHaveBeenCalledWith( + 'Create anomaly detector failed' + ); + }); + }); + + + it('Start anomaly detector failed', async () => { + httpClientMock.post = jest.fn((pathOrOptions: string | HttpFetchOptionsWithPath) => { + const url = typeof pathOrOptions === 'string' ? pathOrOptions : pathOrOptions.path; + switch (url) { + case '/api/anomaly_detectors/_generate_parameters': + return Promise.resolve({ + ok: true, + generatedParameters: { + categoryField: 'ip', + aggregationField: 'responseLatency,response', + aggregationMethod: 'avg,sum', + dateFields: '@timestamp,utc_time', + } + }); + case '/api/anomaly_detectors/detectors': + return Promise.resolve({ + ok: true, + response: { + id: 'test' + } + }); + case '/api/anomaly_detectors/detectors/test/start': + return Promise.resolve({ + ok: false, + error: 'Start anomaly detector failed' + }); + default: + return Promise.resolve({ + ok: true + }); + } + }); + + httpClientMock.get = jest.fn().mockResolvedValue({ + ok: true, + response: { + count: 0 + }, + }); + + + const context = { + indexPattern: createIndexPattern('test-pattern'), + } + const { queryByText, getByTestId } = renderWithRouter(context); + expect(queryByText('Suggest anomaly detector')).not.toBeNull(); + + await waitFor(() => { + expect(queryByText('Generating parameters...')).toBeNull(); + expect(queryByText('Create detector')).not.toBeNull(); + expect(queryByText('Detector details')).not.toBeNull(); + expect(queryByText('Advanced configuration')).not.toBeNull(); + expect(queryByText('Model Features')).not.toBeNull(); + }); + + userEvent.click(getByTestId("GenerateAnomalyDetectorCreateButton")); + + await waitFor(() => { + expect(getNotifications().toasts.addDanger).toHaveBeenCalledTimes(1); + expect(getNotifications().toasts.addDanger).toHaveBeenCalledWith( + 'Start anomaly detector failed' + ); + }); + }); + + }); + + +}); diff --git a/public/components/DiscoverAction/GenerateAnomalyDetector.tsx b/public/components/DiscoverAction/SuggestAnomalyDetector.tsx similarity index 97% rename from public/components/DiscoverAction/GenerateAnomalyDetector.tsx rename to public/components/DiscoverAction/SuggestAnomalyDetector.tsx index 357a0de6..bf52ca43 100644 --- a/public/components/DiscoverAction/GenerateAnomalyDetector.tsx +++ b/public/components/DiscoverAction/SuggestAnomalyDetector.tsx @@ -38,7 +38,7 @@ import { getDetectorCount, matchDetector, startDetector, -} from '../../../public/redux/reducers/ad'; +} from '../../redux/reducers/ad'; import { getError, getErrorMessage, @@ -47,7 +47,7 @@ import { validateDetectorName, validateNonNegativeInteger, validatePositiveInteger, -} from '../../../public/utils/utils'; +} from '../../utils/utils'; import { CUSTOM_AD_RESULT_INDEX_PREFIX, MAX_DETECTORS, @@ -56,21 +56,21 @@ import { focusOnFirstWrongFeature, initialFeatureValue, validateFeatures, -} from '../../../public/pages/ConfigureModel/utils/helpers'; -import { formikToDetector } from '../../../public/pages/ReviewAndCreate/utils/helpers'; -import { FormattedFormRow } from '../../../public/components/FormattedFormRow/FormattedFormRow'; -import { FeatureAccordion } from '../../../public/pages/ConfigureModel/components/FeatureAccordion'; -import { AD_DOCS_LINK, DEFAULT_SHINGLE_SIZE, MAX_FEATURE_NUM, PLUGIN_NAME } from '../../../public/utils/constants'; +} from '../../pages/ConfigureModel/utils/helpers'; +import { formikToDetector } from '../../pages/ReviewAndCreate/utils/helpers'; +import { FormattedFormRow } from '../FormattedFormRow/FormattedFormRow'; +import { FeatureAccordion } from '../../pages/ConfigureModel/components/FeatureAccordion'; +import { AD_DOCS_LINK, DEFAULT_SHINGLE_SIZE, MAX_FEATURE_NUM, PLUGIN_NAME } from '../../utils/constants'; import { getNotifications } from '../../services'; import { prettifyErrorMessage } from '../../../server/utils/helpers'; import EnhancedAccordion from '../FeatureAnywhereContextMenu/EnhancedAccordion'; import MinimalAccordion from '../FeatureAnywhereContextMenu/MinimalAccordion'; -import { DataFilterList } from '../../../public/pages/DefineDetector/components/DataFilterList/DataFilterList'; -import { generateParameters } from '../../../public/redux/reducers/assistant'; -import { FEATURE_TYPE } from '../../../public/models/interfaces'; -import { FeaturesFormikValues } from '../../../public/pages/ConfigureModel/models/interfaces'; +import { DataFilterList } from '../../pages/DefineDetector/components/DataFilterList/DataFilterList'; +import { generateParameters } from '../../redux/reducers/assistant'; +import { FEATURE_TYPE } from '../../models/interfaces'; +import { FeaturesFormikValues } from '../../pages/ConfigureModel/models/interfaces'; import { DiscoverActionContext } from '../../../../../src/plugins/data_explorer/public/types'; -import { getMappings } from '../../../public/redux/reducers/opensearch'; +import { getMappings } from '../../redux/reducers/opensearch'; import { mountReactNode } from '../../../../../src/core/public/utils'; export interface GeneratedParameters { @@ -137,12 +137,12 @@ function GenerateAnomalyDetector({ ); const rawGeneratedParameters = get(result, 'generatedParameters'); if (!rawGeneratedParameters) { - throw new Error('Cannot get generated parameters'); + throw new Error('Cannot get generated parameters!'); } const generatedParameters = formatGeneratedParameters(rawGeneratedParameters); if (generatedParameters.features.length == 0) { - throw new Error('Generated parameters have empty model features'); + throw new Error('Generated parameters have empty model features!'); } initialDetectorValue.featureList = generatedParameters.features; @@ -166,7 +166,7 @@ function GenerateAnomalyDetector({ const rawAggregationMethods = rawGeneratedParameters['aggregationMethod']; const rawDataFields = rawGeneratedParameters['dateFields']; if (!rawAggregationFields || !rawAggregationMethods || !rawDataFields) { - throw new Error('Cannot find aggregation field, aggregation method or data fields!'); + throw new Error('Cannot find aggregation field, aggregation method or data fields!'); } const aggregationFields = rawAggregationFields.split(','); @@ -175,13 +175,13 @@ function GenerateAnomalyDetector({ const dateFields = rawDataFields.split(','); if (aggregationFields.length != aggregationMethods.length) { - throw new Error('The number of aggregation fields and the number of aggregation methods are different'); + throw new Error('The number of aggregation fields and the number of aggregation methods are different!'); } const featureList = aggregationFields.map((field: string, index: number) => { const method = aggregationMethods[index]; if (!field || !method) { - throw new Error('The generated aggregation field or aggregation method is empty'); + throw new Error('The generated aggregation field or aggregation method is empty!'); } const aggregationOption = { label: field, @@ -381,7 +381,7 @@ function GenerateAnomalyDetector({

- Generate anomaly detector + Suggest anomaly detector

@@ -786,6 +786,7 @@ function GenerateAnomalyDetector({ feature={feature} handleChange={formikProps.handleChange} displayMode="flyout" + key={index} /> ) )} @@ -801,6 +802,7 @@ function GenerateAnomalyDetector({ onClick={() => { push(initialFeatureValue()); }} + disabled={isLoading} > Add another feature @@ -832,7 +834,7 @@ function GenerateAnomalyDetector({ { handleValidationAndSubmit(formikProps); diff --git a/public/pages/ConfigureModel/components/FeatureAccordion/FeatureAccordion.tsx b/public/pages/ConfigureModel/components/FeatureAccordion/FeatureAccordion.tsx index 545dda63..53fd616c 100644 --- a/public/pages/ConfigureModel/components/FeatureAccordion/FeatureAccordion.tsx +++ b/public/pages/ConfigureModel/components/FeatureAccordion/FeatureAccordion.tsx @@ -113,6 +113,7 @@ export const FeatureAccordion = (props: FeatureAccordionProps) => { if (props.displayMode === 'flyout') { return ( { + let store: MockStore; + beforeEach(() => { + store = mockedStore(); + }); + + describe('generate parameters', () => { + test('should invoke [REQUEST, SUCCESS]', async () => { + const indexPattern = 'test-index-pattern'; + httpMockedClient.post = jest.fn().mockResolvedValue({ + ok: true, + generatedParameters: { + categoryField: 'ip', + aggregationField: 'responseLatency,response', + aggregationMethod: 'avg,sum', + dateFields: '@timestamp,utc_time', + }, + }); + await store.dispatch(generateParameters(indexPattern)); + const actions = store.getActions(); + expect(actions[0].type).toBe('assistant/GENERATE_PARAMETERS_REQUEST'); + expect(reducer(initialState, actions[0])).toEqual({ + ...initialState, + requesting: true, + errorMessage: '', + }); + expect(actions[1].type).toBe('assistant/GENERATE_PARAMETERS_SUCCESS'); + expect(reducer(initialState, actions[1])).toEqual({ + ...initialState, + requesting: false, + }); + expect(httpMockedClient.post).toHaveBeenCalledWith( + `${BASE_NODE_API_PATH}/_generate_parameters`, + { + body: JSON.stringify({ index: indexPattern }), + } + ); + }); + + test('should invoke [REQUEST, FAILURE]', async () => { + const indexPattern = 'test-index-pattern'; + httpMockedClient.post = jest.fn().mockResolvedValue({ + ok: false, + error: 'generate parameters failed' + }); + try { + await store.dispatch(generateParameters(indexPattern)); + } catch (e) { + const actions = store.getActions(); + expect(actions[0].type).toBe('assistant/GENERATE_PARAMETERS_REQUEST'); + expect(reducer(initialState, actions[0])).toEqual({ + ...initialState, + requesting: true, + errorMessage: '', + }); + expect(actions[1].type).toBe('assistant/GENERATE_PARAMETERS_FAILURE'); + expect(reducer(initialState, actions[1])).toEqual({ + ...initialState, + requesting: false, + errorMessage: 'generate parameters failed', + }); + expect(httpMockedClient.post).toHaveBeenCalledWith( + `${BASE_NODE_API_PATH}/_generate_parameters`, + { + body: JSON.stringify({ index: indexPattern }), + } + ); + } + }); + }); + +}); diff --git a/public/utils/discoverAction.tsx b/public/utils/discoverAction.tsx index 6ed2c85a..9b345ee8 100644 --- a/public/utils/discoverAction.tsx +++ b/public/utils/discoverAction.tsx @@ -5,7 +5,7 @@ import React from 'react' import { ANOMALY_DETECTION_ICON } from "./constants"; -import GenerateAnomalyDetector from "../components/DiscoverAction/GenerateAnomalyDetector"; +import GenerateAnomalyDetector from "../components/DiscoverAction/SuggestAnomalyDetector"; import { getClient, getOverlays } from '../../public/services'; import { toMountPoint } from "../../../../src/plugins/opensearch_dashboards_react/public"; import { Provider } from "react-redux"; @@ -31,7 +31,7 @@ export const getDiscoverAction = (): DiscoverAction => { return { order: 0, - name: 'Generate anomaly detector', + name: 'Suggest anomaly detector', iconType: ANOMALY_DETECTION_ICON, onClick: onClick, } diff --git a/server/routes/assistant.ts b/server/routes/assistant.ts index 4a620e59..6669b9c3 100644 --- a/server/routes/assistant.ts +++ b/server/routes/assistant.ts @@ -14,7 +14,7 @@ import { IOpenSearchDashboardsResponse, } from '../../../../src/core/server'; import { getClientBasedOnDataSource } from '../utils/helpers'; -import { GENERATE_ANOMALY_DETECTOR_CONFIG_ID } from '../utils/constants'; +import { SUGGEST_ANOMALY_DETECTOR_CONFIG_ID } from '../utils/constants'; export function registerAssistantRoutes( apiRouter: Router, @@ -52,7 +52,7 @@ export default class AssistantService { ); const getAgentResponse = await callWithRequest('ml.getAgent', { - id: GENERATE_ANOMALY_DETECTOR_CONFIG_ID, + id: SUGGEST_ANOMALY_DETECTOR_CONFIG_ID, }); if ( diff --git a/server/utils/constants.ts b/server/utils/constants.ts index 19902ebb..8c071556 100644 --- a/server/utils/constants.ts +++ b/server/utils/constants.ts @@ -133,4 +133,4 @@ export const HISTORICAL_TASK_TYPES = [ export const CUSTOM_AD_RESULT_INDEX_PREFIX = 'opensearch-ad-plugin-result-'; -export const GENERATE_ANOMALY_DETECTOR_CONFIG_ID = 'generate_anomaly_detector'; +export const SUGGEST_ANOMALY_DETECTOR_CONFIG_ID = 'suggest_anomaly_detector'; From 5a3e736a9b31150010819733e71869286385d35a Mon Sep 17 00:00:00 2001 From: gaobinlong Date: Wed, 28 Aug 2024 17:29:56 +0800 Subject: [PATCH 03/10] Modify flyout header Signed-off-by: gaobinlong --- .../components/DiscoverAction/SuggestAnomalyDetector.tsx | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/public/components/DiscoverAction/SuggestAnomalyDetector.tsx b/public/components/DiscoverAction/SuggestAnomalyDetector.tsx index bf52ca43..b86fc244 100644 --- a/public/components/DiscoverAction/SuggestAnomalyDetector.tsx +++ b/public/components/DiscoverAction/SuggestAnomalyDetector.tsx @@ -378,12 +378,16 @@ function GenerateAnomalyDetector({ > {(formikProps) => ( <> - - + +

Suggest anomaly detector

+ + + Create an anomaly detector based on the parameters(model features and categorical field) suggested by OpenSearch Assistant. +
From bfe768ec6db8a116b2390110d7ecf00d1e94a710 Mon Sep 17 00:00:00 2001 From: gaobinlong Date: Thu, 29 Aug 2024 11:26:59 +0800 Subject: [PATCH 04/10] Make the detectorName follow the convention Signed-off-by: gaobinlong --- public/components/DiscoverAction/SuggestAnomalyDetector.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/public/components/DiscoverAction/SuggestAnomalyDetector.tsx b/public/components/DiscoverAction/SuggestAnomalyDetector.tsx index b86fc244..292b07ed 100644 --- a/public/components/DiscoverAction/SuggestAnomalyDetector.tsx +++ b/public/components/DiscoverAction/SuggestAnomalyDetector.tsx @@ -72,6 +72,7 @@ import { FeaturesFormikValues } from '../../pages/ConfigureModel/models/interfac import { DiscoverActionContext } from '../../../../../src/plugins/data_explorer/public/types'; import { getMappings } from '../../redux/reducers/opensearch'; import { mountReactNode } from '../../../../../src/core/public/utils'; +import { formikToDetectorName } from '../FeatureAnywhereContextMenu/CreateAnomalyDetector/helpers'; export interface GeneratedParameters { categoryField: string; @@ -126,7 +127,7 @@ function GenerateAnomalyDetector({ const [delayValue, setDelayValue] = useState(1); const [enabled, setEnabled] = useState(false); const [detectorName, setDetectorName] = useState( - indexPatternName.replace('*', '-') + '_anomaly_detector' + formikToDetectorName(indexPatternName) ); // let LLM to generate parameters for creating anomaly detector From 284d9e9c57c14c35c59622fbeea19f5e1748b529 Mon Sep 17 00:00:00 2001 From: gaobinlong Date: Thu, 29 Aug 2024 11:48:27 +0800 Subject: [PATCH 05/10] Truncate the index pattern name if it's too long Signed-off-by: gaobinlong --- public/components/DiscoverAction/SuggestAnomalyDetector.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/components/DiscoverAction/SuggestAnomalyDetector.tsx b/public/components/DiscoverAction/SuggestAnomalyDetector.tsx index 292b07ed..a5ec1a59 100644 --- a/public/components/DiscoverAction/SuggestAnomalyDetector.tsx +++ b/public/components/DiscoverAction/SuggestAnomalyDetector.tsx @@ -127,7 +127,7 @@ function GenerateAnomalyDetector({ const [delayValue, setDelayValue] = useState(1); const [enabled, setEnabled] = useState(false); const [detectorName, setDetectorName] = useState( - formikToDetectorName(indexPatternName) + formikToDetectorName(indexPatternName.substring(0, 40)) ); // let LLM to generate parameters for creating anomaly detector From bc2228d50ba5d26c16160291f5e3d5bb3ef25e43 Mon Sep 17 00:00:00 2001 From: gaobinlong Date: Tue, 3 Sep 2024 22:06:44 +0800 Subject: [PATCH 06/10] Move entry point to query editor Signed-off-by: gaobinlong --- opensearch_dashboards.json | 2 +- .../SuggestAnomalyDetector.test.tsx | 301 ++++++++---------- .../DiscoverAction/SuggestAnomalyDetector.tsx | 77 ++--- public/plugin.ts | 19 +- public/utils/contextMenu/getActions.tsx | 67 ++-- public/utils/discoverAction.tsx | 38 --- server/routes/assistant.ts | 6 +- server/utils/constants.ts | 2 +- 8 files changed, 230 insertions(+), 282 deletions(-) delete mode 100644 public/utils/discoverAction.tsx diff --git a/opensearch_dashboards.json b/opensearch_dashboards.json index 551bd678..5aa5d906 100644 --- a/opensearch_dashboards.json +++ b/opensearch_dashboards.json @@ -8,7 +8,7 @@ "optionalPlugins": [ "dataSource", "dataSourceManagement", - "dataExplorer" + "assistantDashboards" ], "requiredPlugins": [ "opensearchDashboardsUtils", diff --git a/public/components/DiscoverAction/SuggestAnomalyDetector.test.tsx b/public/components/DiscoverAction/SuggestAnomalyDetector.test.tsx index cca54108..63e753c7 100644 --- a/public/components/DiscoverAction/SuggestAnomalyDetector.test.tsx +++ b/public/components/DiscoverAction/SuggestAnomalyDetector.test.tsx @@ -16,50 +16,13 @@ import { import { Provider } from 'react-redux'; import configureStore from '../../redux/configureStore'; -import GenerateAnomalyDetector from './SuggestAnomalyDetector'; -import { DiscoverActionContext } from '../../../../../src/plugins/data_explorer/public'; +import SuggestAnomalyDetector from './SuggestAnomalyDetector'; import { fieldFormatsMock } from '../../../../../src/plugins/data/common/field_formats/mocks'; import { IndexPattern } from '../../../../../src/plugins/data/common'; import userEvent from '@testing-library/user-event'; import { HttpFetchOptionsWithPath } from '../../../../../src/core/public'; import { BASE_NODE_API_PATH } from '../../../utils/constants'; - -const notifications = { - toasts: { - addDanger: jest.fn().mockName('addDanger'), - addSuccess: jest.fn().mockName('addSuccess'), - } -}; - -const getNotifications = () => { - return notifications; -} - -jest.mock('../../services', () => ({ - ...jest.requireActual('../../services'), - getNotifications: getNotifications, -})); - -const renderWithRouter = (context: DiscoverActionContext) => ({ - ...render( - - - - ( - - - - )} - /> - - - - ), -}); +import { getQueryService } from '../../services'; export function shouldReadFieldFromDocValues(aggregatable: boolean, opensearchType: string) { return ( @@ -69,7 +32,6 @@ export function shouldReadFieldFromDocValues(aggregatable: boolean, opensearchTy ); } - function stubbedSampleFields() { return [ ['bytes', 'long', true, true, { count: 10 }], @@ -141,108 +103,123 @@ function createIndexPattern(id: string): IndexPattern { }; } -const expectedAnomalyDetector = { - name: "test-pattern_anomaly_detector", - description: "Created based on the OpenSearch Assistant", - indices: ["test-pattern"], - filterQuery: { - match_all: {} - }, - uiMetadata: { - features: { - feature_responseLatency: { - featureType: "simple_aggs", - aggregationBy: "avg", - aggregationOf: "responseLatency" - }, - feature_response: { - featureType: "simple_aggs", - aggregationBy: "sum", - aggregationOf: "response" - } - }, - filters: [] - }, - featureAttributes: [ - { - featureName: "feature_responseLatency", - featureEnabled: true, - importance: 1, - aggregationQuery: { - feature_response_latency: { - avg: { - field: "responseLatency" - } - } - } - }, - { - featureName: "feature_response", - featureEnabled: true, - importance: 1, - aggregationQuery: { - feature_response: { - sum: { - field: "response" - } - } - } - } - ], - timeField: "timestamp", - detectionInterval: { - period: { - interval: 10, - unit: "Minutes" - } - }, - windowDelay: { - period: { - interval: 1, - unit: "Minutes" - } - }, - shingleSize: 8, - categoryField: ["ip"] +const mockedIndexPattern = createIndexPattern('test-pattern'); + +const notifications = { + toasts: { + addDanger: jest.fn().mockName('addDanger'), + addSuccess: jest.fn().mockName('addSuccess'), + } }; +const getNotifications = () => { + return notifications; +} + +jest.mock('../../services', () => ({ + ...jest.requireActual('../../services'), + getNotifications: getNotifications, + getQueryService: jest.fn().mockReturnValue({ + queryString: { + getQuery: jest.fn(), + }, + }), + getIndexPatternService: () => ({ + get: () => (mockedIndexPattern) + }) +})); + +const renderWithRouter = () => ({ + ...render( + + + + ( + + + + )} + /> + + + + ), +}); + describe('GenerateAnomalyDetector spec', () => { - describe('Renders loading component', () => { + describe('Renders failed', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('renders with invalid dataset type', async () => { + const queryService = getQueryService(); + queryService.queryString.getQuery.mockReturnValue({ + dataset: { + id: undefined, + title: undefined, + type: 'INDEX' + }, + }); + + + const { queryByText } = renderWithRouter(); + expect(queryByText('Suggested anomaly detector')).toBeNull(); + + await waitFor(() => { + expect(getNotifications().toasts.addDanger).toHaveBeenCalledTimes(1); + expect(getNotifications().toasts.addDanger).toHaveBeenCalledWith( + 'Unsupported dataset type' + ); + }); + }); + it('renders empty component', async () => { - httpClientMock.post = jest.fn().mockResolvedValue({ - ok: true, - generatedParameters: { - categoryField: '', - aggregationField: '', - aggregationMethod: '', - dateFields: '', + const queryService = getQueryService(); + queryService.queryString.getQuery.mockReturnValue({ + dataset: { + id: undefined, + title: undefined, + type: 'INDEX_PATTERN' }, }); - const context = { - indexPattern: createIndexPattern(''), - }; - const { queryByText } = renderWithRouter(context); - expect(queryByText('Suggest anomaly detector')).toBeNull(); + const { queryByText } = renderWithRouter(); + expect(queryByText('Suggested anomaly detector')).toBeNull(); await waitFor(() => { expect(getNotifications().toasts.addDanger).toHaveBeenCalledTimes(1); expect(getNotifications().toasts.addDanger).toHaveBeenCalledWith( - 'Cannot extract index pattern from the context' + 'Cannot extract complete index info from the context' ); }); }); + }); + + describe('Renders loading component', () => { + beforeEach(() => { + jest.clearAllMocks(); + const queryService = getQueryService(); + queryService.queryString.getQuery.mockReturnValue({ + dataset: { + id: 'test-pattern', + title: 'test-pattern', + type: 'INDEX_PATTERN', + timeFieldName: '@timestamp', + }, + }); + }); it('renders with empty generated parameters', async () => { httpClientMock.post = jest.fn().mockResolvedValue({ ok: true, }); - const context = { - indexPattern: createIndexPattern('test-pattern'), - } - const { queryByText } = renderWithRouter(context); - expect(queryByText('Suggest anomaly detector')).not.toBeNull(); + const { queryByText } = renderWithRouter(); + expect(queryByText('Suggested anomaly detector')).not.toBeNull(); await waitFor(() => { expect(getNotifications().toasts.addDanger).toHaveBeenCalledTimes(1); @@ -263,11 +240,8 @@ describe('GenerateAnomalyDetector spec', () => { }, }); - const context = { - indexPattern: createIndexPattern('test-pattern'), - } - const { queryByText } = renderWithRouter(context); - expect(queryByText('Suggest anomaly detector')).not.toBeNull(); + const { queryByText } = renderWithRouter(); + expect(queryByText('Suggested anomaly detector')).not.toBeNull(); await waitFor(() => { expect(getNotifications().toasts.addDanger).toHaveBeenCalledTimes(1); @@ -288,11 +262,8 @@ describe('GenerateAnomalyDetector spec', () => { }, }); - const context = { - indexPattern: createIndexPattern('test-pattern'), - } - const { queryByText } = renderWithRouter(context); - expect(queryByText('Suggest anomaly detector')).not.toBeNull(); + const { queryByText } = renderWithRouter(); + expect(queryByText('Suggested anomaly detector')).not.toBeNull(); await waitFor(() => { expect(getNotifications().toasts.addDanger).toHaveBeenCalledTimes(1); @@ -313,11 +284,8 @@ describe('GenerateAnomalyDetector spec', () => { }, }); - const context = { - indexPattern: createIndexPattern('test-pattern'), - } - const { queryByText } = renderWithRouter(context); - expect(queryByText('Suggest anomaly detector')).not.toBeNull(); + const { queryByText } = renderWithRouter(); + expect(queryByText('Suggested anomaly detector')).not.toBeNull(); await waitFor(() => { expect(getNotifications().toasts.addDanger).toHaveBeenCalledTimes(1); @@ -338,11 +306,8 @@ describe('GenerateAnomalyDetector spec', () => { }, }); - const context = { - indexPattern: createIndexPattern('test-pattern'), - } - const { queryByText } = renderWithRouter(context); - expect(queryByText('Suggest anomaly detector')).not.toBeNull(); + const { queryByText } = renderWithRouter(); + expect(queryByText('Suggested anomaly detector')).not.toBeNull(); await waitFor(() => { expect(queryByText('Create detector')).not.toBeNull(); @@ -354,8 +319,20 @@ describe('GenerateAnomalyDetector spec', () => { }); - describe('Test API calls', () => { + beforeEach(() => { + jest.clearAllMocks(); + const queryService = getQueryService(); + queryService.queryString.getQuery.mockReturnValue({ + dataset: { + id: 'test-pattern', + title: 'test-pattern', + type: 'INDEX_PATTERN', + timeFieldName: '@timestamp', + }, + }); + }); + it('All API calls execute successfully', async () => { httpClientMock.post = jest.fn((pathOrOptions: string | HttpFetchOptionsWithPath) => { const url = typeof pathOrOptions === 'string' ? pathOrOptions : pathOrOptions.path; @@ -384,11 +361,8 @@ describe('GenerateAnomalyDetector spec', () => { } }); - const context = { - indexPattern: createIndexPattern('test-pattern'), - } - const { queryByText, getByTestId } = renderWithRouter(context); - expect(queryByText('Suggest anomaly detector')).not.toBeNull(); + const { queryByText, getByTestId } = renderWithRouter(); + expect(queryByText('Suggested anomaly detector')).not.toBeNull(); await waitFor(() => { expect(queryByText('Generating parameters...')).toBeNull(); expect(queryByText('Create detector')).not.toBeNull(); @@ -407,15 +381,6 @@ describe('GenerateAnomalyDetector spec', () => { body: JSON.stringify({ index: 'test-pattern' }), } ); - expect(httpClientMock.post).toHaveBeenCalledWith( - `${BASE_NODE_API_PATH}/detectors`, - { - body: JSON.stringify(expectedAnomalyDetector), - } - ); - expect(httpClientMock.post).toHaveBeenCalledWith( - `${BASE_NODE_API_PATH}/detectors/test/start` - ); expect(getNotifications().toasts.addSuccess).toHaveBeenCalledTimes(1); }); }); @@ -426,11 +391,8 @@ describe('GenerateAnomalyDetector spec', () => { error: 'Generate parameters failed' }); - const context = { - indexPattern: createIndexPattern('test-pattern'), - } - const { queryByText } = renderWithRouter(context); - expect(queryByText('Suggest anomaly detector')).not.toBeNull(); + const { queryByText } = renderWithRouter(); + expect(queryByText('Suggested anomaly detector')).not.toBeNull(); await waitFor(() => { expect(getNotifications().toasts.addDanger).toHaveBeenCalledTimes(1); expect(getNotifications().toasts.addDanger).toHaveBeenCalledWith( @@ -472,12 +434,8 @@ describe('GenerateAnomalyDetector spec', () => { }, }); - - const context = { - indexPattern: createIndexPattern('test-pattern'), - } - const { queryByText, getByTestId } = renderWithRouter(context); - expect(queryByText('Suggest anomaly detector')).not.toBeNull(); + const { queryByText, getByTestId } = renderWithRouter(); + expect(queryByText('Suggested anomaly detector')).not.toBeNull(); await waitFor(() => { expect(queryByText('Generating parameters...')).toBeNull(); @@ -539,11 +497,8 @@ describe('GenerateAnomalyDetector spec', () => { }); - const context = { - indexPattern: createIndexPattern('test-pattern'), - } - const { queryByText, getByTestId } = renderWithRouter(context); - expect(queryByText('Suggest anomaly detector')).not.toBeNull(); + const { queryByText, getByTestId } = renderWithRouter(); + expect(queryByText('Suggested anomaly detector')).not.toBeNull(); await waitFor(() => { expect(queryByText('Generating parameters...')).toBeNull(); diff --git a/public/components/DiscoverAction/SuggestAnomalyDetector.tsx b/public/components/DiscoverAction/SuggestAnomalyDetector.tsx index a5ec1a59..7831c18c 100644 --- a/public/components/DiscoverAction/SuggestAnomalyDetector.tsx +++ b/public/components/DiscoverAction/SuggestAnomalyDetector.tsx @@ -24,7 +24,7 @@ import { EuiComboBox, } from '@elastic/eui'; import '../FeatureAnywhereContextMenu/CreateAnomalyDetector/styles.scss'; -import { useDispatch } from 'react-redux'; +import { useDispatch, useSelector } from 'react-redux'; import { isEmpty, get } from 'lodash'; import { Field, @@ -54,6 +54,7 @@ import { } from '../../../server/utils/constants'; import { focusOnFirstWrongFeature, + getCategoryFields, initialFeatureValue, validateFeatures, } from '../../pages/ConfigureModel/utils/helpers'; @@ -61,7 +62,7 @@ import { formikToDetector } from '../../pages/ReviewAndCreate/utils/helpers'; import { FormattedFormRow } from '../FormattedFormRow/FormattedFormRow'; import { FeatureAccordion } from '../../pages/ConfigureModel/components/FeatureAccordion'; import { AD_DOCS_LINK, DEFAULT_SHINGLE_SIZE, MAX_FEATURE_NUM, PLUGIN_NAME } from '../../utils/constants'; -import { getNotifications } from '../../services'; +import { getNotifications, getQueryService } from '../../services'; import { prettifyErrorMessage } from '../../../server/utils/helpers'; import EnhancedAccordion from '../FeatureAnywhereContextMenu/EnhancedAccordion'; import MinimalAccordion from '../FeatureAnywhereContextMenu/MinimalAccordion'; @@ -69,10 +70,11 @@ import { DataFilterList } from '../../pages/DefineDetector/components/DataFilter import { generateParameters } from '../../redux/reducers/assistant'; import { FEATURE_TYPE } from '../../models/interfaces'; import { FeaturesFormikValues } from '../../pages/ConfigureModel/models/interfaces'; -import { DiscoverActionContext } from '../../../../../src/plugins/data_explorer/public/types'; import { getMappings } from '../../redux/reducers/opensearch'; import { mountReactNode } from '../../../../../src/core/public/utils'; import { formikToDetectorName } from '../FeatureAnywhereContextMenu/CreateAnomalyDetector/helpers'; +import { DEFAULT_DATA } from '../../../../../src/plugins/data/common'; +import { AppState } from '../../redux/reducers'; export interface GeneratedParameters { categoryField: string; @@ -80,41 +82,35 @@ export interface GeneratedParameters { dateFields: string[]; } -function GenerateAnomalyDetector({ +function SuggestAnomalyDetector({ closeFlyout, - context, }: { closeFlyout: any; - context: DiscoverActionContext; }) { const dispatch = useDispatch(); const notifications = getNotifications(); - const indexPatternId = context.indexPattern?.id; - const indexPatternName = context.indexPattern?.title; - if (!indexPatternId || !indexPatternName) { + const queryString = getQueryService().queryString; + const dataset = queryString.getQuery().dataset || queryString.getDefaultQuery().dataset; + const datasetType = dataset.type; + if (datasetType != DEFAULT_DATA.SET_TYPES.INDEX_PATTERN && datasetType != DEFAULT_DATA.SET_TYPES.INDEX) { notifications.toasts.addDanger( - 'Cannot extract index pattern from the context' + 'Unsupported dataset type' ); return <>; } - const dataSourceId = context.indexPattern?.dataSourceRef?.id; - const timeFieldFromIndexPattern = context.indexPattern?.timeFieldName; - const fieldsFromContext = context.indexPattern?.fields || []; - const [categoricalFields, dateFields] = fieldsFromContext.reduce( - ([cFields, dFields], indexPatternField) => { - const esType = indexPatternField.spec.esTypes?.[0]; - const name = indexPatternField.spec.name; - if (esType === 'keyword' || esType === 'ip') { - cFields.push(name); - } else if (esType === 'date') { - dFields.push(name); - } - return [cFields, dFields]; - }, - [[], []] as [string[], string[]] - ) || [[], []]; + const indexPatternId = dataset.id; + // indexName could be a index pattern or a concrete index + const indexName = dataset.title; + const timeFieldName = dataset.timeFieldName; + if (!indexPatternId || !indexName || !timeFieldName) { + notifications.toasts.addDanger( + 'Cannot extract complete index info from the context' + ); + return <>; + } + const dataSourceId = dataset.dataSource?.id; const [isLoading, setIsLoading] = useState(true); const [buttonName, setButtonName] = useState( 'Generating parameters...' @@ -127,14 +123,22 @@ function GenerateAnomalyDetector({ const [delayValue, setDelayValue] = useState(1); const [enabled, setEnabled] = useState(false); const [detectorName, setDetectorName] = useState( - formikToDetectorName(indexPatternName.substring(0, 40)) + formikToDetectorName(indexName.substring(0, 40)) + ); + const indexDataTypes = useSelector( + (state: AppState) => state.opensearch.dataTypes ); + const categoricalFields = getCategoryFields(indexDataTypes); + + const dateFields = get(indexDataTypes, 'date', []) as string[]; + const dateNanoFields = get(indexDataTypes, 'date_nanos', []) as string[]; + const allDateFields = dateFields.concat(dateNanoFields); // let LLM to generate parameters for creating anomaly detector async function getParameters() { try { const result = await dispatch( - generateParameters(indexPatternName!, dataSourceId) + generateParameters(indexName!, dataSourceId) ); const rawGeneratedParameters = get(result, 'generatedParameters'); if (!rawGeneratedParameters) { @@ -207,11 +211,8 @@ function GenerateAnomalyDetector({ useEffect(() => { async function fetchData() { + await dispatch(getMappings(indexName, dataSourceId)); await getParameters(); - const getMappingDispatchCall = dispatch( - getMappings(indexPatternName, dataSourceId) - ); - await Promise.all([getMappingDispatchCall]); } fetchData(); }, []); @@ -353,8 +354,8 @@ function GenerateAnomalyDetector({ let initialDetectorValue = { name: detectorName, - index: [{ label: indexPatternName }], - timeField: timeFieldFromIndexPattern, + index: [{ label: indexName }], + timeField: timeFieldName, interval: intervalValue, windowDelay: delayValue, shingleSize: DEFAULT_SHINGLE_SIZE, @@ -382,7 +383,7 @@ function GenerateAnomalyDetector({

- Suggest anomaly detector + Suggested anomaly detector

@@ -729,7 +730,7 @@ function GenerateAnomalyDetector({ data-test-subj="timestampFilter" id="timeField" placeholder="Find timestamp" - options={dateFields.map((field) => { + options={allDateFields.map((field) => { return { label: field, }; @@ -750,7 +751,7 @@ function GenerateAnomalyDetector({ label: field.value, }, ] - : [{ label: timeFieldFromIndexPattern }] + : [{ label: timeFieldName }] } singleSelection={{ asPlainText: true }} isClearable={false} @@ -857,4 +858,4 @@ function GenerateAnomalyDetector({ ); } -export default GenerateAnomalyDetector; +export default SuggestAnomalyDetector; diff --git a/public/plugin.ts b/public/plugin.ts index 68eedfcf..a340e6be 100644 --- a/public/plugin.ts +++ b/public/plugin.ts @@ -27,7 +27,7 @@ import { } from '../../../src/plugins/embeddable/public'; import { ACTION_AD } from './action/ad_dashboard_action'; import { APP_PATH, DASHBOARD_PAGE_NAV_ID, DETECTORS_PAGE_NAV_ID, OVERVIEW_PAGE_NAV_ID, PLUGIN_NAME } from './utils/constants'; -import { getActions } from './utils/contextMenu/getActions'; +import { ACTION_SUGGEST_AD, getActions, getSuggestAnomalyDetectorAction } from './utils/contextMenu/getActions'; import { overlayAnomaliesFunction } from './expressions/overlay_anomalies'; import { setClient, @@ -42,7 +42,8 @@ import { setDataSourceManagementPlugin, setDataSourceEnabled, setNavigationUI, - setApplication + setApplication, + setIndexPatternService } from './services'; import { AnomalyDetectionOpenSearchDashboardsPluginStart } from 'public'; import { @@ -50,16 +51,16 @@ import { VisAugmenterStart, } from '../../../src/plugins/vis_augmenter/public'; import { UiActionsStart } from '../../../src/plugins/ui_actions/public'; -import { DataPublicPluginStart } from '../../../src/plugins/data/public'; +import { DataPublicPluginSetup, DataPublicPluginStart } from '../../../src/plugins/data/public'; import { DataSourceManagementPluginSetup } from '../../../src/plugins/data_source_management/public'; import { DataSourcePluginSetup } from '../../../src/plugins/data_source/public'; import { NavigationPublicPluginStart } from '../../../src/plugins/navigation/public'; -import { getDiscoverAction } from './utils/discoverAction'; -import { DataExplorerPluginSetup } from '../../../src/plugins/data_explorer/public'; +import { AssistantSetup } from '../../../plugins/dashboards-assistant/public'; declare module '../../../src/plugins/ui_actions/public' { export interface ActionContextMapping { [ACTION_AD]: {}; + [ACTION_SUGGEST_AD]: {} } } @@ -70,7 +71,7 @@ export interface AnomalyDetectionSetupDeps { visAugmenter: VisAugmenterSetup; dataSourceManagement: DataSourceManagementPluginSetup; dataSource: DataSourcePluginSetup; - dataExplorer: DataExplorerPluginSetup; + data: DataPublicPluginSetup; } export interface AnomalyDetectionStartDeps { @@ -192,9 +193,9 @@ export class AnomalyDetectionOpenSearchDashboardsPlugin plugins.uiActions.addTriggerAction(CONTEXT_MENU_TRIGGER, action); }); - // Add action to Discover - const discoverAction = getDiscoverAction(); - plugins.dataExplorer.registerDiscoverAction(discoverAction); + // Add suggest anomaly detector action to the uiActions in Discover + const suggestAnomalyDetectorAction = getSuggestAnomalyDetectorAction(); + plugins.uiActions.addTriggerAction(plugins.assistantDashboards.assistantTriggers.AI_ASSISTANT_TRIGGER, suggestAnomalyDetectorAction); // registers the expression function used to render anomalies on an Augmented Visualization plugins.expressions.registerFunction(overlayAnomaliesFunction); diff --git a/public/utils/contextMenu/getActions.tsx b/public/utils/contextMenu/getActions.tsx index f58a7a9e..30ae041e 100644 --- a/public/utils/contextMenu/getActions.tsx +++ b/public/utils/contextMenu/getActions.tsx @@ -7,7 +7,7 @@ import React from 'react'; import { i18n } from '@osd/i18n'; import { EuiIconType } from '@elastic/eui'; import { toMountPoint } from '../../../../../src/plugins/opensearch_dashboards_react/public'; -import { Action } from '../../../../../src/plugins/ui_actions/public'; +import { Action, createAction } from '../../../../../src/plugins/ui_actions/public'; import { createADAction } from '../../action/ad_dashboard_action'; import AnywhereParentFlyout from '../../components/FeatureAnywhereContextMenu/AnywhereParentFlyout'; import { Provider } from 'react-redux'; @@ -16,6 +16,9 @@ import DocumentationTitle from '../../components/FeatureAnywhereContextMenu/Docu import { AD_FEATURE_ANYWHERE_LINK, ANOMALY_DETECTION_ICON } from '../constants'; import { getClient, getOverlays } from '../../../public/services'; import { FLYOUT_MODES } from '../../../public/components/FeatureAnywhereContextMenu/AnywhereParentFlyout/constants'; +import SuggestAnomalyDetector from '../../../public/components/DiscoverAction/SuggestAnomalyDetector'; + +export const ACTION_SUGGEST_AD = 'suggestAnomalyDetector'; // This is used to create all actions in the same context menu const grouping: Action['grouping'] = [ @@ -31,23 +34,23 @@ const grouping: Action['grouping'] = [ export const getActions = () => { const getOnClick = (startingFlyout) => - async ({ embeddable }) => { - const overlayService = getOverlays(); - const openFlyout = overlayService.openFlyout; - const store = configureStore(getClient()); - const overlay = openFlyout( - toMountPoint( - - overlay.close()} - /> - - ), - { size: 'm', className: 'context-menu__flyout' } - ); - }; + async ({ embeddable }) => { + const overlayService = getOverlays(); + const openFlyout = overlayService.openFlyout; + const store = configureStore(getClient()); + const overlay = openFlyout( + toMountPoint( + + overlay.close()} + /> + + ), + { size: 'm', className: 'context-menu__flyout' } + ); + }; return [ { @@ -87,3 +90,31 @@ export const getActions = () => { }, ].map((options) => createADAction({ ...options, grouping })); }; + +export const getSuggestAnomalyDetectorAction = () => { + const onClick = async function () { + const overlayService = getOverlays(); + const openFlyout = overlayService.openFlyout; + const store = configureStore(getClient()); + const overlay = openFlyout( + toMountPoint( + + overlay.close()} + /> + + ) + ); + } + + return createAction({ + id: 'suggestAnomalyDetector', + order: 100, + type: ACTION_SUGGEST_AD, + getDisplayName: () => 'Suggest anomaly detector', + getIconType: () => ANOMALY_DETECTION_ICON, + execute: async () => { + onClick(); + }, + }); +} diff --git a/public/utils/discoverAction.tsx b/public/utils/discoverAction.tsx deleted file mode 100644 index 9b345ee8..00000000 --- a/public/utils/discoverAction.tsx +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import React from 'react' -import { ANOMALY_DETECTION_ICON } from "./constants"; -import GenerateAnomalyDetector from "../components/DiscoverAction/SuggestAnomalyDetector"; -import { getClient, getOverlays } from '../../public/services'; -import { toMountPoint } from "../../../../src/plugins/opensearch_dashboards_react/public"; -import { Provider } from "react-redux"; -import configureStore from '../redux/configureStore'; -import { DiscoverAction, DiscoverActionContext } from "../../../../src/plugins/data_explorer/public/types"; - -export const getDiscoverAction = (): DiscoverAction => { - const onClick = function (context: DiscoverActionContext) { - const overlayService = getOverlays(); - const openFlyout = overlayService.openFlyout; - const store = configureStore(getClient()); - const overlay = openFlyout( - toMountPoint( - - overlay.close()} - context={context} - /> - - ) - ); - } - - return { - order: 0, - name: 'Suggest anomaly detector', - iconType: ANOMALY_DETECTION_ICON, - onClick: onClick, - } -}; diff --git a/server/routes/assistant.ts b/server/routes/assistant.ts index 6669b9c3..5c1a0cd0 100644 --- a/server/routes/assistant.ts +++ b/server/routes/assistant.ts @@ -56,16 +56,14 @@ export default class AssistantService { }); if ( - !getAgentResponse || - !getAgentResponse['configuration'] || - !getAgentResponse['configuration']['agent_id'] + !getAgentResponse || !(getAgentResponse.ml_configuration?.agent_id || getAgentResponse.configuration?.agent_id) ) { throw new Error( 'Cannot get flow agent id for generating anomaly detector' ); } - const agentId = getAgentResponse['configuration']['agent_id']; + const agentId = getAgentResponse.ml_configuration?.agent_id || getAgentResponse.configuration?.agent_id; const executeAgentResponse = await callWithRequest('ml.executeAgent', { agentId: agentId, diff --git a/server/utils/constants.ts b/server/utils/constants.ts index 8c071556..1a756187 100644 --- a/server/utils/constants.ts +++ b/server/utils/constants.ts @@ -133,4 +133,4 @@ export const HISTORICAL_TASK_TYPES = [ export const CUSTOM_AD_RESULT_INDEX_PREFIX = 'opensearch-ad-plugin-result-'; -export const SUGGEST_ANOMALY_DETECTOR_CONFIG_ID = 'suggest_anomaly_detector'; +export const SUGGEST_ANOMALY_DETECTOR_CONFIG_ID = 'os_suggest_ad'; From 479374997ff9a291b92394403dc145b8facc2067 Mon Sep 17 00:00:00 2001 From: gaobinlong Date: Thu, 5 Sep 2024 17:49:29 +0800 Subject: [PATCH 07/10] Call the node API in dashboard-assistant plugin to generate parameters Refactor unit test code Signed-off-by: gaobinlong --- .../SuggestAnomalyDetector.test.tsx | 260 +++++++----------- .../DiscoverAction/SuggestAnomalyDetector.tsx | 18 +- public/plugin.ts | 17 +- .../reducers/__tests__/assistant.test.ts | 83 ------ public/redux/reducers/assistant.ts | 65 ----- public/services.ts | 11 +- server/cluster/ad/mlPlugin.ts | 42 --- server/plugin.ts | 7 +- server/routes/assistant.ts | 102 ------- utils/constants.ts | 3 - 10 files changed, 125 insertions(+), 483 deletions(-) delete mode 100644 public/redux/reducers/__tests__/assistant.test.ts delete mode 100644 public/redux/reducers/assistant.ts delete mode 100644 server/cluster/ad/mlPlugin.ts delete mode 100644 server/routes/assistant.ts diff --git a/public/components/DiscoverAction/SuggestAnomalyDetector.test.tsx b/public/components/DiscoverAction/SuggestAnomalyDetector.test.tsx index 63e753c7..050b5c57 100644 --- a/public/components/DiscoverAction/SuggestAnomalyDetector.test.tsx +++ b/public/components/DiscoverAction/SuggestAnomalyDetector.test.tsx @@ -17,93 +17,9 @@ import { Provider } from 'react-redux'; import configureStore from '../../redux/configureStore'; import SuggestAnomalyDetector from './SuggestAnomalyDetector'; -import { fieldFormatsMock } from '../../../../../src/plugins/data/common/field_formats/mocks'; -import { IndexPattern } from '../../../../../src/plugins/data/common'; import userEvent from '@testing-library/user-event'; import { HttpFetchOptionsWithPath } from '../../../../../src/core/public'; -import { BASE_NODE_API_PATH } from '../../../utils/constants'; -import { getQueryService } from '../../services'; - -export function shouldReadFieldFromDocValues(aggregatable: boolean, opensearchType: string) { - return ( - aggregatable && - !['text', 'geo_shape'].includes(opensearchType) && - !opensearchType.startsWith('_') - ); -} - -function stubbedSampleFields() { - return [ - ['bytes', 'long', true, true, { count: 10 }], - ['response', 'integer', true, true], - ['responseLatency', 'float', true, true], - ['@timestamp', 'date', true, true, { count: 30 }], - ['@tags', 'keyword', true, true], - ['utc_time', 'date', true, true], - ['phpmemory', 'integer', true, true], - ['ip', 'ip', true, true], - ['geo.src', 'keyword', true, true], - ['_id', '_id', true, true], - ['_type', '_type', true, true], - ['_source', '_source', true, true], - ].map(function (row) { - const [ - name, - opensearchType, - aggregatable, - searchable, - metadata = {}, - subType = undefined, - ] = row; - - const { - count = 0, - script, - lang = script ? 'expression' : undefined, - scripted = !!script, - } = metadata; - - return { - name, - opensearchType, - spec: { - esTypes: [opensearchType], - name: name, - }, - readFromDocValues: shouldReadFieldFromDocValues(aggregatable, opensearchType), - aggregatable, - searchable, - count, - script, - lang, - scripted, - subType, - }; - }); -} - -function createIndexPattern(id: string): IndexPattern { - const type = 'index-pattern'; - const version = '2'; - const timeFieldName = 'timestamp'; - const fields = stubbedSampleFields(); - const title = id; - - return { - id, - type, - version, - timeFieldName, - fields, - title, - savedObjectsClient: {} as any, - fieldFormats: fieldFormatsMock, - shortDotsEnable: false, - metaFields: [], - }; -} - -const mockedIndexPattern = createIndexPattern('test-pattern'); +import { getAssistantClient, getQueryService } from '../../services'; const notifications = { toasts: { @@ -124,8 +40,8 @@ jest.mock('../../services', () => ({ getQuery: jest.fn(), }, }), - getIndexPatternService: () => ({ - get: () => (mockedIndexPattern) + getAssistantClient: jest.fn().mockReturnValue({ + executeAgentByName: jest.fn(), }) })); @@ -211,11 +127,20 @@ describe('GenerateAnomalyDetector spec', () => { timeFieldName: '@timestamp', }, }); + }); it('renders with empty generated parameters', async () => { - httpClientMock.post = jest.fn().mockResolvedValue({ - ok: true, + (getAssistantClient().executeAgentByName as jest.Mock).mockResolvedValueOnce({ + body: { + inference_results: [ + { + output: [ + { result: '' } + ] + } + ] + } }); const { queryByText } = renderWithRouter(); @@ -230,14 +155,16 @@ describe('GenerateAnomalyDetector spec', () => { }); it('renders with empty parameter', async () => { - httpClientMock.post = jest.fn().mockResolvedValue({ - ok: true, - generatedParameters: { - categoryField: '', - aggregationField: '', - aggregationMethod: '', - dateFields: '', - }, + (getAssistantClient().executeAgentByName as jest.Mock).mockResolvedValueOnce({ + body: { + inference_results: [ + { + output: [ + { result: "{\"index\":\"opensearch_dashboards_sample_data_logs\",\"categoryField\":\"ip\",\"aggregationField\":\"\",\"aggregationMethod\":\"\",\"dateFields\":\"utc_time,timestamp\"}" } + ] + } + ] + } }); const { queryByText } = renderWithRouter(); @@ -252,14 +179,16 @@ describe('GenerateAnomalyDetector spec', () => { }); it('renders with empty aggregation field or empty aggregation method', async () => { - httpClientMock.post = jest.fn().mockResolvedValue({ - ok: true, - generatedParameters: { - categoryField: '', - aggregationField: ',', - aggregationMethod: ',', - dateFields: 'timestamp', - }, + (getAssistantClient().executeAgentByName as jest.Mock).mockResolvedValueOnce({ + body: { + inference_results: [ + { + output: [ + { result: "{\"index\":\"opensearch_dashboards_sample_data_logs\",\"categoryField\":\"ip\",\"aggregationField\":\",\",\"aggregationMethod\":\",\",\"dateFields\":\"utc_time,timestamp\"}" } + ] + } + ] + } }); const { queryByText } = renderWithRouter(); @@ -274,14 +203,16 @@ describe('GenerateAnomalyDetector spec', () => { }); it('renders with different number of aggregation methods and fields', async () => { - httpClientMock.post = jest.fn().mockResolvedValue({ - ok: true, - generatedParameters: { - categoryField: '', - aggregationField: 'a,b', - aggregationMethod: 'avg', - dateFields: 'timestamp', - }, + (getAssistantClient().executeAgentByName as jest.Mock).mockResolvedValueOnce({ + body: { + inference_results: [ + { + output: [ + { result: "{\"index\":\"opensearch_dashboards_sample_data_logs\",\"categoryField\":\"ip\",\"aggregationField\":\"a,b\",\"aggregationMethod\":\"avg\",\"dateFields\":\"utc_time,timestamp\"}" } + ] + } + ] + } }); const { queryByText } = renderWithRouter(); @@ -296,14 +227,16 @@ describe('GenerateAnomalyDetector spec', () => { }); it('renders component completely', async () => { - httpClientMock.post = jest.fn().mockResolvedValue({ - ok: true, - generatedParameters: { - categoryField: 'ip', - aggregationField: 'responseLatency,response', - aggregationMethod: 'avg,sum', - dateFields: '@timestamp,utc_time', - }, + (getAssistantClient().executeAgentByName as jest.Mock).mockResolvedValueOnce({ + body: { + inference_results: [ + { + output: [ + { result: "{\"index\":\"opensearch_dashboards_sample_data_logs\",\"categoryField\":\"ip\",\"aggregationField\":\"responseLatency,response\",\"aggregationMethod\":\"avg,sum\",\"dateFields\":\"utc_time,timestamp\"}" } + ] + } + ] + } }); const { queryByText } = renderWithRouter(); @@ -337,16 +270,6 @@ describe('GenerateAnomalyDetector spec', () => { httpClientMock.post = jest.fn((pathOrOptions: string | HttpFetchOptionsWithPath) => { const url = typeof pathOrOptions === 'string' ? pathOrOptions : pathOrOptions.path; switch (url) { - case '/api/anomaly_detectors/_generate_parameters': - return Promise.resolve({ - ok: true, - generatedParameters: { - categoryField: 'ip', - aggregationField: 'responseLatency,response', - aggregationMethod: 'avg,sum', - dateFields: '@timestamp,utc_time', - } - }); case '/api/anomaly_detectors/detectors': return Promise.resolve({ ok: true, @@ -360,6 +283,18 @@ describe('GenerateAnomalyDetector spec', () => { }); } }); + (getAssistantClient().executeAgentByName as jest.Mock).mockResolvedValueOnce({ + body: { + inference_results: [ + { + output: [ + { result: "{\"index\":\"test-pattern\",\"categoryField\":\"ip\",\"aggregationField\":\"responseLatency,response\",\"aggregationMethod\":\"avg,sum\",\"dateFields\":\"utc_time,timestamp\"}" } + ] + } + ] + } + }); + const { queryByText, getByTestId } = renderWithRouter(); expect(queryByText('Suggested anomaly detector')).not.toBeNull(); @@ -371,25 +306,16 @@ describe('GenerateAnomalyDetector spec', () => { expect(queryByText('Model Features')).not.toBeNull(); }); - userEvent.click(getByTestId("GenerateAnomalyDetectorCreateButton")); + userEvent.click(getByTestId("SuggestAnomalyDetectorCreateButton")); await waitFor(() => { - expect(httpClientMock.post).toHaveBeenCalledTimes(3); - expect(httpClientMock.post).toHaveBeenCalledWith( - `${BASE_NODE_API_PATH}/_generate_parameters`, - { - body: JSON.stringify({ index: 'test-pattern' }), - } - ); + expect(httpClientMock.post).toHaveBeenCalledTimes(2); expect(getNotifications().toasts.addSuccess).toHaveBeenCalledTimes(1); }); }); it('Generate parameters failed', async () => { - httpClientMock.post = jest.fn().mockResolvedValue({ - ok: false, - error: 'Generate parameters failed' - }); + (getAssistantClient().executeAgentByName as jest.Mock).mockRejectedValueOnce('Generate parameters failed'); const { queryByText } = renderWithRouter(); expect(queryByText('Suggested anomaly detector')).not.toBeNull(); @@ -405,16 +331,6 @@ describe('GenerateAnomalyDetector spec', () => { httpClientMock.post = jest.fn((pathOrOptions: string | HttpFetchOptionsWithPath) => { const url = typeof pathOrOptions === 'string' ? pathOrOptions : pathOrOptions.path; switch (url) { - case '/api/anomaly_detectors/_generate_parameters': - return Promise.resolve({ - ok: true, - generatedParameters: { - categoryField: 'ip', - aggregationField: 'responseLatency,response', - aggregationMethod: 'avg,sum', - dateFields: '@timestamp,utc_time', - } - }); case '/api/anomaly_detectors/detectors': return Promise.resolve({ ok: false, @@ -426,6 +342,17 @@ describe('GenerateAnomalyDetector spec', () => { }); } }); + (getAssistantClient().executeAgentByName as jest.Mock).mockResolvedValueOnce({ + body: { + inference_results: [ + { + output: [ + { result: "{\"index\":\"test-pattern\",\"categoryField\":\"ip\",\"aggregationField\":\"responseLatency,response\",\"aggregationMethod\":\"avg,sum\",\"dateFields\":\"utc_time,timestamp\"}" } + ] + } + ] + } + }); httpClientMock.get = jest.fn().mockResolvedValue({ ok: true, @@ -445,7 +372,7 @@ describe('GenerateAnomalyDetector spec', () => { expect(queryByText('Model Features')).not.toBeNull(); }); - userEvent.click(getByTestId("GenerateAnomalyDetectorCreateButton")); + userEvent.click(getByTestId("SuggestAnomalyDetectorCreateButton")); await waitFor(() => { expect(getNotifications().toasts.addDanger).toHaveBeenCalledTimes(1); @@ -460,16 +387,6 @@ describe('GenerateAnomalyDetector spec', () => { httpClientMock.post = jest.fn((pathOrOptions: string | HttpFetchOptionsWithPath) => { const url = typeof pathOrOptions === 'string' ? pathOrOptions : pathOrOptions.path; switch (url) { - case '/api/anomaly_detectors/_generate_parameters': - return Promise.resolve({ - ok: true, - generatedParameters: { - categoryField: 'ip', - aggregationField: 'responseLatency,response', - aggregationMethod: 'avg,sum', - dateFields: '@timestamp,utc_time', - } - }); case '/api/anomaly_detectors/detectors': return Promise.resolve({ ok: true, @@ -496,6 +413,18 @@ describe('GenerateAnomalyDetector spec', () => { }, }); + (getAssistantClient().executeAgentByName as jest.Mock).mockResolvedValueOnce({ + body: { + inference_results: [ + { + output: [ + { result: "{\"index\":\"test-pattern\",\"categoryField\":\"ip\",\"aggregationField\":\"responseLatency,response\",\"aggregationMethod\":\"avg,sum\",\"dateFields\":\"utc_time,timestamp\"}" } + ] + } + ] + } + }); + const { queryByText, getByTestId } = renderWithRouter(); expect(queryByText('Suggested anomaly detector')).not.toBeNull(); @@ -508,7 +437,7 @@ describe('GenerateAnomalyDetector spec', () => { expect(queryByText('Model Features')).not.toBeNull(); }); - userEvent.click(getByTestId("GenerateAnomalyDetectorCreateButton")); + userEvent.click(getByTestId("SuggestAnomalyDetectorCreateButton")); await waitFor(() => { expect(getNotifications().toasts.addDanger).toHaveBeenCalledTimes(1); @@ -517,7 +446,6 @@ describe('GenerateAnomalyDetector spec', () => { ); }); }); - }); diff --git a/public/components/DiscoverAction/SuggestAnomalyDetector.tsx b/public/components/DiscoverAction/SuggestAnomalyDetector.tsx index 7831c18c..96e60234 100644 --- a/public/components/DiscoverAction/SuggestAnomalyDetector.tsx +++ b/public/components/DiscoverAction/SuggestAnomalyDetector.tsx @@ -51,6 +51,7 @@ import { import { CUSTOM_AD_RESULT_INDEX_PREFIX, MAX_DETECTORS, + SUGGEST_ANOMALY_DETECTOR_CONFIG_ID, } from '../../../server/utils/constants'; import { focusOnFirstWrongFeature, @@ -62,12 +63,11 @@ import { formikToDetector } from '../../pages/ReviewAndCreate/utils/helpers'; import { FormattedFormRow } from '../FormattedFormRow/FormattedFormRow'; import { FeatureAccordion } from '../../pages/ConfigureModel/components/FeatureAccordion'; import { AD_DOCS_LINK, DEFAULT_SHINGLE_SIZE, MAX_FEATURE_NUM, PLUGIN_NAME } from '../../utils/constants'; -import { getNotifications, getQueryService } from '../../services'; +import { getAssistantClient, getNotifications, getQueryService } from '../../services'; import { prettifyErrorMessage } from '../../../server/utils/helpers'; import EnhancedAccordion from '../FeatureAnywhereContextMenu/EnhancedAccordion'; import MinimalAccordion from '../FeatureAnywhereContextMenu/MinimalAccordion'; import { DataFilterList } from '../../pages/DefineDetector/components/DataFilterList/DataFilterList'; -import { generateParameters } from '../../redux/reducers/assistant'; import { FEATURE_TYPE } from '../../models/interfaces'; import { FeaturesFormikValues } from '../../pages/ConfigureModel/models/interfaces'; import { getMappings } from '../../redux/reducers/opensearch'; @@ -89,6 +89,8 @@ function SuggestAnomalyDetector({ }) { const dispatch = useDispatch(); const notifications = getNotifications(); + const assistantClient = getAssistantClient(); + const queryString = getQueryService().queryString; const dataset = queryString.getQuery().dataset || queryString.getDefaultQuery().dataset; const datasetType = dataset.type; @@ -137,15 +139,15 @@ function SuggestAnomalyDetector({ // let LLM to generate parameters for creating anomaly detector async function getParameters() { try { - const result = await dispatch( - generateParameters(indexName!, dataSourceId) - ); - const rawGeneratedParameters = get(result, 'generatedParameters'); + const executeAgentResponse = await + assistantClient.executeAgentByName(SUGGEST_ANOMALY_DETECTOR_CONFIG_ID, { index: indexName }, { dataSourceId } + ); + const rawGeneratedParameters = executeAgentResponse?.body?.inference_results?.[0]?.output?.[0]?.result; if (!rawGeneratedParameters) { throw new Error('Cannot get generated parameters!'); } - const generatedParameters = formatGeneratedParameters(rawGeneratedParameters); + const generatedParameters = formatGeneratedParameters(JSON.parse(rawGeneratedParameters)); if (generatedParameters.features.length == 0) { throw new Error('Generated parameters have empty model features!'); } @@ -840,7 +842,7 @@ function SuggestAnomalyDetector({ { handleValidationAndSubmit(formikProps); diff --git a/public/plugin.ts b/public/plugin.ts index a340e6be..d00c5074 100644 --- a/public/plugin.ts +++ b/public/plugin.ts @@ -43,7 +43,7 @@ import { setDataSourceEnabled, setNavigationUI, setApplication, - setIndexPatternService + setAssistantClient, } from './services'; import { AnomalyDetectionOpenSearchDashboardsPluginStart } from 'public'; import { @@ -55,7 +55,7 @@ import { DataPublicPluginSetup, DataPublicPluginStart } from '../../../src/plugi import { DataSourceManagementPluginSetup } from '../../../src/plugins/data_source_management/public'; import { DataSourcePluginSetup } from '../../../src/plugins/data_source/public'; import { NavigationPublicPluginStart } from '../../../src/plugins/navigation/public'; -import { AssistantSetup } from '../../../plugins/dashboards-assistant/public'; +import { AssistantPublicPluginStart } from '../../dashboards-assistant/public'; declare module '../../../src/plugins/ui_actions/public' { export interface ActionContextMapping { @@ -81,6 +81,7 @@ export interface AnomalyDetectionStartDeps { uiActions: UiActionsStart; data: DataPublicPluginStart; navigation: NavigationPublicPluginStart; + assistantDashboards: AssistantPublicPluginStart; } export class AnomalyDetectionOpenSearchDashboardsPlugin @@ -194,9 +195,10 @@ export class AnomalyDetectionOpenSearchDashboardsPlugin }); // Add suggest anomaly detector action to the uiActions in Discover - const suggestAnomalyDetectorAction = getSuggestAnomalyDetectorAction(); - plugins.uiActions.addTriggerAction(plugins.assistantDashboards.assistantTriggers.AI_ASSISTANT_TRIGGER, suggestAnomalyDetectorAction); - + if (plugins.assistantDashboards?.assistantTriggers?.AI_ASSISTANT_TRIGGER) { + const suggestAnomalyDetectorAction = getSuggestAnomalyDetectorAction(); + plugins.uiActions.addTriggerAction(plugins.assistantDashboards.assistantTriggers.AI_ASSISTANT_TRIGGER, suggestAnomalyDetectorAction); + } // registers the expression function used to render anomalies on an Augmented Visualization plugins.expressions.registerFunction(overlayAnomaliesFunction); return {}; @@ -204,7 +206,7 @@ export class AnomalyDetectionOpenSearchDashboardsPlugin public start( core: CoreStart, - { embeddable, visAugmenter, uiActions, data, navigation }: AnomalyDetectionStartDeps + { embeddable, visAugmenter, uiActions, data, navigation, assistantDashboards }: AnomalyDetectionStartDeps ): AnomalyDetectionOpenSearchDashboardsPluginStart { setUISettings(core.uiSettings); setEmbeddable(embeddable); @@ -215,6 +217,9 @@ export class AnomalyDetectionOpenSearchDashboardsPlugin setQueryService(data.query); setSavedObjectsClient(core.savedObjects.client); setNavigationUI(navigation.ui); + if (assistantDashboards) { + setAssistantClient(assistantDashboards.assistantClient); + } setApplication(core.application); return {}; } diff --git a/public/redux/reducers/__tests__/assistant.test.ts b/public/redux/reducers/__tests__/assistant.test.ts deleted file mode 100644 index 27247727..00000000 --- a/public/redux/reducers/__tests__/assistant.test.ts +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { MockStore } from 'redux-mock-store'; -import httpMockedClient from '../../../../test/mocks/httpClientMock'; -import { BASE_NODE_API_PATH } from '../../../../utils/constants'; -import { mockedStore } from '../../utils/testUtils'; -import reducer, { generateParameters, initialState } from '../assistant'; - -describe('assistant reducer actions', () => { - let store: MockStore; - beforeEach(() => { - store = mockedStore(); - }); - - describe('generate parameters', () => { - test('should invoke [REQUEST, SUCCESS]', async () => { - const indexPattern = 'test-index-pattern'; - httpMockedClient.post = jest.fn().mockResolvedValue({ - ok: true, - generatedParameters: { - categoryField: 'ip', - aggregationField: 'responseLatency,response', - aggregationMethod: 'avg,sum', - dateFields: '@timestamp,utc_time', - }, - }); - await store.dispatch(generateParameters(indexPattern)); - const actions = store.getActions(); - expect(actions[0].type).toBe('assistant/GENERATE_PARAMETERS_REQUEST'); - expect(reducer(initialState, actions[0])).toEqual({ - ...initialState, - requesting: true, - errorMessage: '', - }); - expect(actions[1].type).toBe('assistant/GENERATE_PARAMETERS_SUCCESS'); - expect(reducer(initialState, actions[1])).toEqual({ - ...initialState, - requesting: false, - }); - expect(httpMockedClient.post).toHaveBeenCalledWith( - `${BASE_NODE_API_PATH}/_generate_parameters`, - { - body: JSON.stringify({ index: indexPattern }), - } - ); - }); - - test('should invoke [REQUEST, FAILURE]', async () => { - const indexPattern = 'test-index-pattern'; - httpMockedClient.post = jest.fn().mockResolvedValue({ - ok: false, - error: 'generate parameters failed' - }); - try { - await store.dispatch(generateParameters(indexPattern)); - } catch (e) { - const actions = store.getActions(); - expect(actions[0].type).toBe('assistant/GENERATE_PARAMETERS_REQUEST'); - expect(reducer(initialState, actions[0])).toEqual({ - ...initialState, - requesting: true, - errorMessage: '', - }); - expect(actions[1].type).toBe('assistant/GENERATE_PARAMETERS_FAILURE'); - expect(reducer(initialState, actions[1])).toEqual({ - ...initialState, - requesting: false, - errorMessage: 'generate parameters failed', - }); - expect(httpMockedClient.post).toHaveBeenCalledWith( - `${BASE_NODE_API_PATH}/_generate_parameters`, - { - body: JSON.stringify({ index: indexPattern }), - } - ); - } - }); - }); - -}); diff --git a/public/redux/reducers/assistant.ts b/public/redux/reducers/assistant.ts deleted file mode 100644 index 7a359e18..00000000 --- a/public/redux/reducers/assistant.ts +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { APIAction, APIResponseAction, HttpSetup } from '../middleware/types'; -import handleActions from '../utils/handleActions'; -import { ASSISTANT_NODE_API } from '../../../utils/constants'; - -const GENERATE_PARAMETERS = 'assistant/GENERATE_PARAMETERS'; - -export interface GeneratedParametersState { - requesting: boolean; - errorMessage: string; -} - -export const initialState: GeneratedParametersState = { - requesting: false, - errorMessage: '', -}; - -const reducer = handleActions( - { - [GENERATE_PARAMETERS]: { - REQUEST: (state: GeneratedParametersState): GeneratedParametersState => ({ - ...state, - requesting: true, - errorMessage: '', - }), - SUCCESS: ( - state: GeneratedParametersState, - action: APIResponseAction - ): GeneratedParametersState => ({ - ...state, - requesting: false, - }), - FAILURE: ( - state: GeneratedParametersState, - action: APIResponseAction - ): GeneratedParametersState => ({ - ...state, - requesting: false, - errorMessage: action.error, - }), - }, - }, - initialState -); - -export const generateParameters = ( - index: string, - dataSourceId: string = '' -): APIAction => { - const baseUrl = `${ASSISTANT_NODE_API.GENERATE_PARAMETERS}`; - const url = dataSourceId ? `${baseUrl}/${dataSourceId}` : baseUrl; - return { - type: GENERATE_PARAMETERS, - request: (client: HttpSetup) => - client.post(url, { - body: JSON.stringify({ index: index }), - }), - }; -}; - -export default reducer; diff --git a/public/services.ts b/public/services.ts index 0c3d45dd..d582ac59 100644 --- a/public/services.ts +++ b/public/services.ts @@ -16,6 +16,7 @@ import { createGetterSetter } from '../../../src/plugins/opensearch_dashboards_u import { UiActionsStart } from '../../../src/plugins/ui_actions/public'; import { SavedAugmentVisLoader } from '../../../src/plugins/vis_augmenter/public'; import { NavigationPublicPluginStart } from '../../../src/plugins/navigation/public'; +import { AssistantPublicPluginStart } from '../../../plugins/dashboards-assistant/public/'; export interface DataSourceEnabled { enabled: boolean; @@ -45,6 +46,12 @@ export const [getUISettings, setUISettings] = export const [getQueryService, setQueryService] = createGetterSetter('Query'); +export const [getAssistantEnabled, setAssistantEnabled] = + createGetterSetter('AssistantClient'); + +export const [getAssistantClient, setAssistantClient] = + createGetterSetter('AssistantClient'); + export const [getSavedObjectsClient, setSavedObjectsClient] = createGetterSetter('SavedObjectsClient'); @@ -54,10 +61,10 @@ export const [getDataSourceManagementPlugin, setDataSourceManagementPlugin] = export const [getDataSourceEnabled, setDataSourceEnabled] = createGetterSetter('DataSourceEnabled'); -export const [getNavigationUI, setNavigationUI] = +export const [getNavigationUI, setNavigationUI] = createGetterSetter('navigation'); -export const [getApplication, setApplication] = +export const [getApplication, setApplication] = createGetterSetter('application'); // This is primarily used for mocking this module and each of its fns in tests. diff --git a/server/cluster/ad/mlPlugin.ts b/server/cluster/ad/mlPlugin.ts deleted file mode 100644 index ad39f085..00000000 --- a/server/cluster/ad/mlPlugin.ts +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -export default function mlPlugin( - Client: any, - config: any, - components: any -) { - const ca = components.clientAction.factory; - - Client.prototype.ml = components.clientAction.namespaceFactory(); - const ml = Client.prototype.ml.prototype; - - ml.getAgent = ca({ - url: { - fmt: `/_plugins/_ml/config/<%=id%>`, - req: { - id: { - type: 'string', - required: true, - }, - }, - }, - method: 'GET', - }); - - ml.executeAgent = ca({ - url: { - fmt: `/_plugins/_ml/agents/<%=agentId%>/_execute`, - req: { - agentId: { - type: 'string', - required: true, - }, - }, - }, - needBody: true, - method: 'POST', - }); -} diff --git a/server/plugin.ts b/server/plugin.ts index 8a40c9ec..8ec5b3d3 100644 --- a/server/plugin.ts +++ b/server/plugin.ts @@ -37,8 +37,6 @@ import SampleDataService, { import { DEFAULT_HEADERS } from './utils/constants'; import { DataSourcePluginSetup } from '../../../src/plugins/data_source/server/types'; import { DataSourceManagementPlugin } from '../../../src/plugins/data_source_management/public'; -import AssistantService, { registerAssistantRoutes } from './routes/assistant'; -import mlPlugin from './cluster/ad/mlPlugin'; export interface ADPluginSetupDependencies { dataSourceManagement?: ReturnType; @@ -71,7 +69,7 @@ export class AnomalyDetectionOpenSearchDashboardsPlugin const client: ILegacyClusterClient = core.opensearch.legacy.createClient( 'anomaly_detection', { - plugins: [adPlugin, alertingPlugin, mlPlugin], + plugins: [adPlugin, alertingPlugin], customHeaders: { ...customHeaders, ...DEFAULT_HEADERS }, ...rest, } @@ -82,7 +80,6 @@ export class AnomalyDetectionOpenSearchDashboardsPlugin if (dataSourceEnabled) { dataSource.registerCustomApiSchema(adPlugin); dataSource.registerCustomApiSchema(alertingPlugin); - dataSource.registerCustomApiSchema(mlPlugin); } // Create router @@ -96,14 +93,12 @@ export class AnomalyDetectionOpenSearchDashboardsPlugin const alertingService = new AlertingService(client, dataSourceEnabled); const opensearchService = new OpenSearchService(client, dataSourceEnabled); const sampleDataService = new SampleDataService(client, dataSourceEnabled); - const assistantService = new AssistantService(client, dataSourceEnabled); // Register server routes with the service registerADRoutes(apiRouter, adService); registerAlertingRoutes(apiRouter, alertingService); registerOpenSearchRoutes(apiRouter, opensearchService); registerSampleDataRoutes(apiRouter, sampleDataService); - registerAssistantRoutes(apiRouter, assistantService); return {}; } diff --git a/server/routes/assistant.ts b/server/routes/assistant.ts deleted file mode 100644 index 5c1a0cd0..00000000 --- a/server/routes/assistant.ts +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -//@ts-ignore -import { get, set } from 'lodash'; -import { Router } from '../router'; -import { getErrorMessage } from './utils/adHelpers'; -import { - RequestHandlerContext, - OpenSearchDashboardsRequest, - OpenSearchDashboardsResponseFactory, - IOpenSearchDashboardsResponse, -} from '../../../../src/core/server'; -import { getClientBasedOnDataSource } from '../utils/helpers'; -import { SUGGEST_ANOMALY_DETECTOR_CONFIG_ID } from '../utils/constants'; - -export function registerAssistantRoutes( - apiRouter: Router, - assistantService: AssistantService -) { - apiRouter.post('/_generate_parameters', assistantService.generateParameters); -} - -export default class AssistantService { - private client: any; - dataSourceEnabled: boolean; - - constructor(client: any, dataSourceEnabled: boolean) { - this.client = client; - this.dataSourceEnabled = dataSourceEnabled; - } - - generateParameters = async ( - context: RequestHandlerContext, - request: OpenSearchDashboardsRequest, - opensearchDashboardsResponse: OpenSearchDashboardsResponseFactory - ): Promise> => { - try { - const { dataSourceId = '' } = request.params as { dataSourceId?: string }; - const { index } = request.body as { index: string }; - if (!index) { - throw new Error('index cannot be empty'); - } - const callWithRequest = getClientBasedOnDataSource( - context, - this.dataSourceEnabled, - request, - dataSourceId, - this.client - ); - - const getAgentResponse = await callWithRequest('ml.getAgent', { - id: SUGGEST_ANOMALY_DETECTOR_CONFIG_ID, - }); - - if ( - !getAgentResponse || !(getAgentResponse.ml_configuration?.agent_id || getAgentResponse.configuration?.agent_id) - ) { - throw new Error( - 'Cannot get flow agent id for generating anomaly detector' - ); - } - - const agentId = getAgentResponse.ml_configuration?.agent_id || getAgentResponse.configuration?.agent_id; - - const executeAgentResponse = await callWithRequest('ml.executeAgent', { - agentId: agentId, - body: { - parameters: { - index: index, - }, - }, - }); - if ( - !executeAgentResponse || - !executeAgentResponse['inference_results'] || - !executeAgentResponse['inference_results'][0].output[0] || - !executeAgentResponse['inference_results'][0].output[0].result - ) { - throw new Error('Execute agent for generating anomaly detector failed'); - } - - return opensearchDashboardsResponse.ok({ - body: { - ok: true, - generatedParameters: JSON.parse( - executeAgentResponse['inference_results'][0].output[0].result - ), - }, - }); - } catch (err) { - return opensearchDashboardsResponse.ok({ - body: { - ok: false, - error: getErrorMessage(err), - }, - }); - } - }; -} diff --git a/utils/constants.ts b/utils/constants.ts index 6546e87a..231bd91b 100644 --- a/utils/constants.ts +++ b/utils/constants.ts @@ -27,6 +27,3 @@ export const ALERTING_NODE_API = Object.freeze({ ALERTS: `${BASE_NODE_API_PATH}/monitors/alerts`, MONITORS: `${BASE_NODE_API_PATH}/monitors`, }); -export const ASSISTANT_NODE_API = Object.freeze({ - GENERATE_PARAMETERS: `${BASE_NODE_API_PATH}/_generate_parameters`, -}); From b3b0ded676fd80e2186621ae64adda4ccdc039a2 Mon Sep 17 00:00:00 2001 From: gaobinlong Date: Fri, 6 Sep 2024 11:55:10 +0800 Subject: [PATCH 08/10] Fix test failure Signed-off-by: gaobinlong --- public/redux/reducers/opensearch.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/redux/reducers/opensearch.ts b/public/redux/reducers/opensearch.ts index 667a3400..f145d074 100644 --- a/public/redux/reducers/opensearch.ts +++ b/public/redux/reducers/opensearch.ts @@ -372,7 +372,7 @@ export const getMappings = ( type: GET_MAPPINGS, request: (client: HttpSetup) => client.get(`${url}`, { - query: { index: searchKey }, + query: { indices: searchKey }, }), }; }; From c6bd9a60da7cc9b408bd4b082773a6ef2c14ebd3 Mon Sep 17 00:00:00 2001 From: gaobinlong Date: Fri, 6 Sep 2024 12:10:59 +0800 Subject: [PATCH 09/10] Revert the code format Signed-off-by: gaobinlong --- .../reducers/__tests__/opensearch.test.ts | 50 +++++++++---------- public/redux/reducers/opensearch.ts | 48 +++++++++--------- public/utils/contextMenu/getActions.tsx | 34 ++++++------- server/plugin.ts | 8 +-- 4 files changed, 70 insertions(+), 70 deletions(-) diff --git a/public/redux/reducers/__tests__/opensearch.test.ts b/public/redux/reducers/__tests__/opensearch.test.ts index 28f7a140..ee3e5fc0 100644 --- a/public/redux/reducers/__tests__/opensearch.test.ts +++ b/public/redux/reducers/__tests__/opensearch.test.ts @@ -290,20 +290,20 @@ describe('opensearch reducer actions', () => { { alias: 'alias1', index: 'index1' }, { alias: 'alias2', index: 'index2' }, ]; - + httpMockedClient.get = jest .fn() .mockResolvedValue({ ok: true, response: { indices, aliases } }); - + await store.dispatch(getIndicesAndAliases()); const actions = store.getActions(); - + expect(actions[0].type).toBe('opensearch/GET_INDICES_AND_ALIASES_REQUEST'); expect(reducer(initialState, actions[0])).toEqual({ ...initialState, requesting: true, }); - + expect(actions[1].type).toBe('opensearch/GET_INDICES_AND_ALIASES_SUCCESS'); expect(reducer(initialState, actions[1])).toEqual({ ...initialState, @@ -311,7 +311,7 @@ describe('opensearch reducer actions', () => { indices, aliases, }); - + expect(httpMockedClient.get).toHaveBeenCalledWith( `..${BASE_NODE_API_PATH}/_indices_and_aliases`, { @@ -328,20 +328,20 @@ describe('opensearch reducer actions', () => { { alias: 'alias1', index: 'index1' }, { alias: 'alias2', index: 'index2' }, ]; - + httpMockedClient.get = jest .fn() .mockResolvedValue({ ok: true, response: { indices, aliases } }); - + await store.dispatch(getIndicesAndAliases('', '', 'cluster-2,cluster-3')); const actions = store.getActions(); - + expect(actions[0].type).toBe('opensearch/GET_INDICES_AND_ALIASES_REQUEST'); expect(reducer(initialState, actions[0])).toEqual({ ...initialState, requesting: true, }); - + expect(actions[1].type).toBe('opensearch/GET_INDICES_AND_ALIASES_SUCCESS'); expect(reducer(initialState, actions[1])).toEqual({ ...initialState, @@ -349,7 +349,7 @@ describe('opensearch reducer actions', () => { indices, aliases, }); - + expect(httpMockedClient.get).toHaveBeenCalledWith( `..${BASE_NODE_API_PATH}/_indices_and_aliases`, { @@ -366,26 +366,26 @@ describe('opensearch reducer actions', () => { ok: false, error: 'Something went wrong', }); - + try { await store.dispatch(getIndicesAndAliases()); } catch (e) { const actions = store.getActions(); - + expect(actions[0].type).toBe('opensearch/GET_INDICES_AND_ALIASES_REQUEST'); expect(reducer(initialState, actions[0])).toEqual({ ...initialState, requesting: true, errorMessage: '', }); - + expect(actions[1].type).toBe('opensearch/GET_INDICES_AND_ALIASES_FAILURE'); expect(reducer(initialState, actions[1])).toEqual({ ...initialState, requesting: false, errorMessage: 'Something went wrong', }); - + expect(httpMockedClient.get).toHaveBeenCalledWith( `..${BASE_NODE_API_PATH}/_indices_and_aliases`, { @@ -405,59 +405,59 @@ describe('opensearch reducer actions', () => { { cluster: 'cluster1', status: 'green' }, { cluster: 'cluster2', status: 'yellow' }, ]; - + httpMockedClient.get = jest .fn() .mockResolvedValue({ ok: true, response: { clusters } }); - + await store.dispatch(getClustersInfo()); const actions = store.getActions(); - + expect(actions[0].type).toBe('opensearch/GET_CLUSTERS_INFO_REQUEST'); expect(reducer(initialState, actions[0])).toEqual({ ...initialState, requesting: true, errorMessage: '', }); - + expect(actions[1].type).toBe('opensearch/GET_CLUSTERS_INFO_SUCCESS'); expect(reducer(initialState, actions[1])).toEqual({ ...initialState, requesting: false, clusters, }); - + expect(httpMockedClient.get).toHaveBeenCalledWith( `..${BASE_NODE_API_PATH}/_remote/info` ); }); test('should invoke [REQUEST, FAILURE]', async () => { const errorMessage = 'Something went wrong'; - + httpMockedClient.get = jest.fn().mockRejectedValue({ ok: false, error: errorMessage, }); - + try { await store.dispatch(getClustersInfo()); } catch (e) { const actions = store.getActions(); - + expect(actions[0].type).toBe('opensearch/GET_CLUSTERS_INFO_REQUEST'); expect(reducer(initialState, actions[0])).toEqual({ ...initialState, requesting: true, errorMessage: '', }); - + expect(actions[1].type).toBe('opensearch/GET_CLUSTERS_INFO_FAILURE'); expect(reducer(initialState, actions[1])).toEqual({ ...initialState, requesting: false, errorMessage, }); - + expect(httpMockedClient.get).toHaveBeenCalledWith( `..${BASE_NODE_API_PATH}/_remote/info` ); @@ -465,5 +465,5 @@ describe('opensearch reducer actions', () => { }); }); - describe('getPrioritizedIndices', () => { }); + describe('getPrioritizedIndices', () => {}); }); diff --git a/public/redux/reducers/opensearch.ts b/public/redux/reducers/opensearch.ts index f145d074..046b2c19 100644 --- a/public/redux/reducers/opensearch.ts +++ b/public/redux/reducers/opensearch.ts @@ -424,33 +424,33 @@ export const getPrioritizedIndices = dataSourceId: string = '', clusters: string = '*' ): ThunkAction => - async (dispatch, getState) => { - //Fetch Indices and Aliases with text provided - await dispatch(getIndicesAndAliases(searchKey, dataSourceId, clusters)); + async (dispatch, getState) => { + //Fetch Indices and Aliases with text provided + await dispatch(getIndicesAndAliases(searchKey, dataSourceId, clusters)); + const osState = getState().opensearch; + const exactMatchedIndices = osState.indices; + const exactMatchedAliases = osState.aliases; + if (exactMatchedAliases.length || exactMatchedIndices.length) { + //If we have exact match just return that + return { + indices: exactMatchedIndices, + aliases: exactMatchedAliases, + }; + } else { + //No results found for exact match, append wildCard and get partial matches if exists + await dispatch( + getIndicesAndAliases(`${searchKey}*`, dataSourceId, clusters) + ); const osState = getState().opensearch; - const exactMatchedIndices = osState.indices; - const exactMatchedAliases = osState.aliases; - if (exactMatchedAliases.length || exactMatchedIndices.length) { - //If we have exact match just return that + const partialMatchedIndices = osState.indices; + const partialMatchedAliases = osState.aliases; + if (partialMatchedAliases.length || partialMatchedIndices.length) { return { - indices: exactMatchedIndices, - aliases: exactMatchedAliases, + indices: partialMatchedIndices, + aliases: partialMatchedAliases, }; - } else { - //No results found for exact match, append wildCard and get partial matches if exists - await dispatch( - getIndicesAndAliases(`${searchKey}*`, dataSourceId, clusters) - ); - const osState = getState().opensearch; - const partialMatchedIndices = osState.indices; - const partialMatchedAliases = osState.aliases; - if (partialMatchedAliases.length || partialMatchedIndices.length) { - return { - indices: partialMatchedIndices, - aliases: partialMatchedAliases, - }; - } } - }; + } + }; export default reducer; diff --git a/public/utils/contextMenu/getActions.tsx b/public/utils/contextMenu/getActions.tsx index 30ae041e..6bb6bf3e 100644 --- a/public/utils/contextMenu/getActions.tsx +++ b/public/utils/contextMenu/getActions.tsx @@ -34,23 +34,23 @@ const grouping: Action['grouping'] = [ export const getActions = () => { const getOnClick = (startingFlyout) => - async ({ embeddable }) => { - const overlayService = getOverlays(); - const openFlyout = overlayService.openFlyout; - const store = configureStore(getClient()); - const overlay = openFlyout( - toMountPoint( - - overlay.close()} - /> - - ), - { size: 'm', className: 'context-menu__flyout' } - ); - }; + async ({ embeddable }) => { + const overlayService = getOverlays(); + const openFlyout = overlayService.openFlyout; + const store = configureStore(getClient()); + const overlay = openFlyout( + toMountPoint( + + overlay.close()} + /> + + ), + { size: 'm', className: 'context-menu__flyout' } + ); + }; return [ { diff --git a/server/plugin.ts b/server/plugin.ts index 8ec5b3d3..a6dfc3b0 100644 --- a/server/plugin.ts +++ b/server/plugin.ts @@ -45,10 +45,10 @@ export interface ADPluginSetupDependencies { export class AnomalyDetectionOpenSearchDashboardsPlugin implements - Plugin< - AnomalyDetectionOpenSearchDashboardsPluginSetup, - AnomalyDetectionOpenSearchDashboardsPluginStart - > + Plugin< + AnomalyDetectionOpenSearchDashboardsPluginSetup, + AnomalyDetectionOpenSearchDashboardsPluginStart + > { private readonly logger: Logger; private readonly globalConfig$: any; From 6814a1991e89512f9d97ebbafedeecc526b3594b Mon Sep 17 00:00:00 2001 From: gaobinlong Date: Fri, 6 Sep 2024 14:40:04 +0800 Subject: [PATCH 10/10] Remove some empty lines Signed-off-by: gaobinlong --- .../components/DiscoverAction/SuggestAnomalyDetector.test.tsx | 3 --- 1 file changed, 3 deletions(-) diff --git a/public/components/DiscoverAction/SuggestAnomalyDetector.test.tsx b/public/components/DiscoverAction/SuggestAnomalyDetector.test.tsx index 050b5c57..4e0de621 100644 --- a/public/components/DiscoverAction/SuggestAnomalyDetector.test.tsx +++ b/public/components/DiscoverAction/SuggestAnomalyDetector.test.tsx @@ -81,7 +81,6 @@ describe('GenerateAnomalyDetector spec', () => { }, }); - const { queryByText } = renderWithRouter(); expect(queryByText('Suggested anomaly detector')).toBeNull(); @@ -447,6 +446,4 @@ describe('GenerateAnomalyDetector spec', () => { }); }); }); - - });