Adding remote indices and multi index functionality (#854)

* Adding remote indices and multi index functionality Signed-off-by: Amit Galitzky <[email protected]> * changing removed index check Signed-off-by: Amit Galitzky <[email protected]> --------- Signed-off-by: Amit Galitzky <[email protected]>
opensearch-project · Sep 4, 2024 · b26f8c3 · b26f8c3
1 parent 9874c48
commit b26f8c3
Show file tree

Hide file tree

Showing 33 changed files with 2,497 additions and 362 deletions.
diff --git a/public/components/FeatureAnywhereContextMenu/CreateAnomalyDetector/AddAnomalyDetector.tsx b/public/components/FeatureAnywhereContextMenu/CreateAnomalyDetector/AddAnomalyDetector.tsx
@@ -134,16 +134,23 @@ function AddAnomalyDetector({
   >();
 
   const indexPatternId = embeddable.vis.data.aggs.indexPattern.id;
-  const [dataSourceId, setDataSourceId] = useState<string | undefined>(undefined);
+  const [dataSourceId, setDataSourceId] = useState<string | undefined>(
+    undefined
+  );
 
   async function getDataSourceId() {
     try {
-      const indexPattern = await getSavedObjectsClient().get('index-pattern', indexPatternId);
+      const indexPattern = await getSavedObjectsClient().get(
+        'index-pattern',
+        indexPatternId
+      );
       const refs = indexPattern.references as References[];
-      const foundDataSourceId = refs.find(ref => ref.type === 'data-source')?.id;
-      setDataSourceId(foundDataSourceId); 
+      const foundDataSourceId = refs.find(
+        (ref) => ref.type === 'data-source'
+      )?.id;
+      setDataSourceId(foundDataSourceId);
     } catch (error) {
-      console.error("Error fetching index pattern:", error);
+      console.error('Error fetching index pattern:', error);
     }
   }
 
@@ -152,8 +159,12 @@ function AddAnomalyDetector({
     async function fetchData() {
       await getDataSourceId();
 
-      const getIndicesDispatchCall = dispatch(getIndices(queryText, dataSourceId));
-      const getMappingDispatchCall =  dispatch(getMappings(embeddable.vis.data.aggs.indexPattern.title, dataSourceId));
+      const getIndicesDispatchCall = dispatch(
+        getIndices(queryText, dataSourceId)
+      );
+      const getMappingDispatchCall = dispatch(
+        getMappings([embeddable.vis.data.aggs.indexPattern.title], dataSourceId)
+      );
       await Promise.all([getIndicesDispatchCall, getMappingDispatchCall]);
     }
 
@@ -167,7 +178,7 @@ function AddAnomalyDetector({
     }
     fetchData();
     createEmbeddable();
-  }, [dataSourceId]); 
+  }, [dataSourceId]);
 
   const [isShowVis, setIsShowVis] = useState(false);
   const [accordionsOpen, setAccordionsOpen] = useState({ modelFeatures: true });
@@ -335,7 +346,7 @@ function AddAnomalyDetector({
       name: OVERLAY_ANOMALIES,
       args: {
         detectorId: detectorId,
-        dataSourceId: dataSourceId
+        dataSourceId: dataSourceId,
       },
     } as VisLayerExpressionFn;
 

diff --git a/public/pages/ConfigureModel/components/Features/Features.tsx b/public/pages/ConfigureModel/components/Features/Features.tsx
@@ -65,16 +65,6 @@ export function Features(props: FeaturesProps) {
           {({ push, remove, form: { values } }: FieldArrayRenderProps) => {
             return (
               <Fragment>
-                {get(props.detector, 'indices.0', '').includes(':') ? (
-                  <div>
-                    <EuiCallOut
-                      title="This detector is using a remote cluster index, so you need to manually input the field."
-                      color="warning"
-                      iconType="alert"
-                    />
-                    <EuiSpacer size="m" />
-                  </div>
-                ) : null}
                 {values.featureList.map((feature: any, index: number) => (
                   <FeatureAccordion
                     onDelete={() => {

diff --git a/public/pages/ConfigureModel/containers/ConfigureModel.tsx b/public/pages/ConfigureModel/containers/ConfigureModel.tsx
@@ -30,7 +30,11 @@ import { RouteComponentProps, useLocation } from 'react-router-dom';
 import { AppState } from '../../../redux/reducers';
 import { getMappings } from '../../../redux/reducers/opensearch';
 import { useFetchDetectorInfo } from '../../CreateDetectorSteps/hooks/useFetchDetectorInfo';
-import { BREADCRUMBS, BASE_DOCS_LINK, MDS_BREADCRUMBS } from '../../../utils/constants';
+import {
+  BREADCRUMBS,
+  BASE_DOCS_LINK,
+  MDS_BREADCRUMBS,
+} from '../../../utils/constants';
 import { useHideSideNavBar } from '../../main/hooks/useHideSideNavBar';
 import { updateDetector } from '../../../redux/reducers/ad';
 import {
@@ -121,7 +125,7 @@ export function ConfigureModel(props: ConfigureModelProps) {
       setIsHCDetector(true);
     }
     if (detector?.indices) {
-      dispatch(getMappings(detector.indices[0], dataSourceId));
+      dispatch(getMappings(detector.indices, dataSourceId));
     }
   }, [detector]);
 
@@ -133,7 +137,11 @@ export function ConfigureModel(props: ConfigureModelProps) {
           MDS_BREADCRUMBS.DETECTORS(dataSourceId),
           {
             text: detector && detector.name ? detector.name : '',
-            href: constructHrefWithDataSourceId(`#/detectors/${detectorId}`, dataSourceId, false)
+            href: constructHrefWithDataSourceId(
+              `#/detectors/${detectorId}`,
+              dataSourceId,
+              false
+            ),
           },
           MDS_BREADCRUMBS.EDIT_MODEL_CONFIGURATION,
         ]);
@@ -167,12 +175,11 @@ export function ConfigureModel(props: ConfigureModelProps) {
 
   useEffect(() => {
     if (hasError) {
-      if(dataSourceEnabled) {
+      if (dataSourceEnabled) {
         props.history.push(
           constructHrefWithDataSourceId('/detectors', dataSourceId, false)
         );
-      }
-      else {
+      } else {
         props.history.push('/detectors');
       }
     }

diff --git a/public/pages/CreateDetectorSteps/hooks/useFetchDetectorInfo.ts b/public/pages/CreateDetectorSteps/hooks/useFetchDetectorInfo.ts
@@ -10,7 +10,7 @@
  */
 
 import { get, isEmpty } from 'lodash';
-import { useEffect } from 'react';
+import { useEffect, useMemo } from 'react';
 import { useDispatch, useSelector } from 'react-redux';
 import { Detector } from '../../../models/interfaces';
 import { AppState } from '../../../redux/reducers';
@@ -40,13 +40,13 @@ export const useFetchDetectorInfo = (
   const isIndicesRequesting = useSelector(
     (state: AppState) => state.opensearch.requesting
   );
-  const selectedIndices = get(detector, 'indices.0', '');
+  const selectedIndices = useMemo(() => get(detector, 'indices', []), [detector]);
   useEffect(() => {
     const fetchDetector = async () => {
       if (!detector) {
         await dispatch(getDetector(detectorId, dataSourceId));
       }
-      if (selectedIndices) {
+      if (selectedIndices && selectedIndices.length > 0) {
         await dispatch(getMappings(selectedIndices, dataSourceId));
       }
     };

diff --git a/public/pages/DefineDetector/components/DataFilterList/DataFilterList.tsx b/public/pages/DefineDetector/components/DataFilterList/DataFilterList.tsx
@@ -15,7 +15,6 @@ import {
   EuiSpacer,
   EuiIcon,
   EuiButtonEmpty,
-  EuiCallOut,
 } from '@elastic/eui';
 import { FieldArray, FieldArrayRenderProps, FormikProps } from 'formik';
 import React, { useState, Fragment } from 'react';
@@ -38,9 +37,6 @@ export const DataFilterList = (props: DataFilterListProps) => {
   const [isCreatingNewFilter, setIsCreatingNewFilter] =
     useState<boolean>(false);
 
-  const selectedIndex = get(props, 'formikProps.values.index.0.label', '');
-  const isRemoteIndex = selectedIndex.includes(':');
-
   return (
     <FieldArray name="filters" validateOnChange={true}>
       {({ push, remove, replace, form: { values } }: FieldArrayRenderProps) => {
@@ -66,18 +62,6 @@ export const DataFilterList = (props: DataFilterListProps) => {
             >
               <Fragment>
                 <EuiSpacer size="m" />
-                {isRemoteIndex ? (
-                  <div>
-                    <EuiCallOut
-                      title="A remote index is selected, so you need to manually input the filter fields."
-                      color="warning"
-                      iconType="alert"
-                      size="s"
-                      style={{ marginTop: '-4px' }}
-                    />
-                    <EuiSpacer size="m" />
-                  </div>
-                ) : null}
                 <EuiFlexGroup direction="row" gutterSize="xs">
                   <EuiFlexItem grow={false}>
                     {values.filters?.length === 0 ||

diff --git a/public/pages/DefineDetector/components/DataFilterList/components/SimpleFilter.tsx b/public/pages/DefineDetector/components/DataFilterList/components/SimpleFilter.tsx
@@ -32,6 +32,7 @@ import { getIndexFields, getOperators, isNullOperator } from '../utils/helpers';
 import FilterValue from './FilterValue';
 import { DetectorDefinitionFormikValues } from '../../../models/interfaces';
 import { EMPTY_UI_FILTER } from '../../../utils/constants';
+import _ from 'lodash';
 
 interface SimpleFilterProps {
   filter: UIFilter;
@@ -40,8 +41,20 @@ interface SimpleFilterProps {
   replace(index: number, value: any): void;
 }
 
+// This sorting is needed because we utilize two different ways to get index fields,
+// through get mapping call and through field_caps API for remote indices
+const sortByLabel = (indexFields) => {
+  //sort the `options` array inside each object by the `label` field
+  indexFields.forEach(item => {
+      item.options = _.sortBy(item.options, 'label');
+  });
+  //sort the outer array by the `label` field
+  return _.sortBy(indexFields, 'label');
+};
+
 export const SimpleFilter = (props: SimpleFilterProps) => {
-  const indexFields = getIndexFields(useSelector(getAllFields));
+  let indexFields = getIndexFields(useSelector(getAllFields));
+  indexFields = sortByLabel(indexFields)
   const [searchedIndexFields, setSearchedIndexFields] = useState<
     ({
       label: DATA_TYPES;