From b26f8c32489693d36ac5479c99a7d9f1d43ef01d Mon Sep 17 00:00:00 2001 From: Amit Galitzky Date: Wed, 4 Sep 2024 07:28:38 -0700 Subject: [PATCH] Adding remote indices and multi index functionality (#854) * Adding remote indices and multi index functionality Signed-off-by: Amit Galitzky * changing removed index check Signed-off-by: Amit Galitzky --------- Signed-off-by: Amit Galitzky --- .../AddAnomalyDetector.tsx | 29 +- .../components/Features/Features.tsx | 10 - .../containers/ConfigureModel.tsx | 19 +- .../hooks/useFetchDetectorInfo.ts | 6 +- .../DataFilterList/DataFilterList.tsx | 16 - .../components/SimpleFilter.tsx | 15 +- .../components/Datasource/DataSource.tsx | 217 +++++++- .../components/Timestamp/Timestamp.tsx | 15 +- .../containers/DefineDetector.tsx | 2 +- .../DefineDetector.test.tsx.snap | 354 +++++++++++-- .../pages/DefineDetector/models/interfaces.ts | 1 + public/pages/DefineDetector/utils/helpers.ts | 2 +- .../containers/DetectorConfig.tsx | 7 + .../components/ListFilters/ListFilters.tsx | 14 +- .../DetectorsList/containers/List/List.tsx | 19 +- .../DataConnectionFlyout.tsx | 106 ++++ .../components/DataConnectionFlyout/index.ts | 12 + .../DetectorDefinitionFields.tsx | 262 +++++---- .../__tests__/DataConnectionFlyout.test.tsx | 81 +++ .../DetectorDefinitionFields.test.tsx | 173 +++--- .../DataConnectionFlyout.test.tsx.snap | 5 + .../DetectorDefinitionFields.test.tsx.snap | 498 ++++++++++++++++++ .../containers/ReviewAndCreate.tsx | 1 + .../ReviewAndCreate.test.tsx.snap | 8 +- public/pages/utils/__tests__/helpers.test.ts | 145 ++++- public/pages/utils/helpers.ts | 116 +++- .../reducers/__tests__/opensearch.test.ts | 195 ++++++- public/redux/reducers/opensearch.ts | 129 ++++- server/models/types.ts | 20 +- server/routes/opensearch.ts | 249 ++++++++- .../utils/__tests__/opensearchHelpers.test.ts | 84 +++ server/routes/utils/opensearchHelpers.ts | 47 ++ utils/constants.ts | 2 + 33 files changed, 2497 insertions(+), 362 deletions(-) create mode 100644 public/pages/ReviewAndCreate/components/DataConnectionFlyout/DataConnectionFlyout.tsx create mode 100644 public/pages/ReviewAndCreate/components/DataConnectionFlyout/index.ts create mode 100644 public/pages/ReviewAndCreate/components/__tests__/DataConnectionFlyout.test.tsx create mode 100644 public/pages/ReviewAndCreate/components/__tests__/__snapshots__/DataConnectionFlyout.test.tsx.snap create mode 100644 server/routes/utils/__tests__/opensearchHelpers.test.ts create mode 100644 server/routes/utils/opensearchHelpers.ts diff --git a/public/components/FeatureAnywhereContextMenu/CreateAnomalyDetector/AddAnomalyDetector.tsx b/public/components/FeatureAnywhereContextMenu/CreateAnomalyDetector/AddAnomalyDetector.tsx index a7486cf1..58c280ff 100644 --- a/public/components/FeatureAnywhereContextMenu/CreateAnomalyDetector/AddAnomalyDetector.tsx +++ b/public/components/FeatureAnywhereContextMenu/CreateAnomalyDetector/AddAnomalyDetector.tsx @@ -134,16 +134,23 @@ function AddAnomalyDetector({ >(); const indexPatternId = embeddable.vis.data.aggs.indexPattern.id; - const [dataSourceId, setDataSourceId] = useState(undefined); + const [dataSourceId, setDataSourceId] = useState( + undefined + ); async function getDataSourceId() { try { - const indexPattern = await getSavedObjectsClient().get('index-pattern', indexPatternId); + const indexPattern = await getSavedObjectsClient().get( + 'index-pattern', + indexPatternId + ); const refs = indexPattern.references as References[]; - const foundDataSourceId = refs.find(ref => ref.type === 'data-source')?.id; - setDataSourceId(foundDataSourceId); + const foundDataSourceId = refs.find( + (ref) => ref.type === 'data-source' + )?.id; + setDataSourceId(foundDataSourceId); } catch (error) { - console.error("Error fetching index pattern:", error); + console.error('Error fetching index pattern:', error); } } @@ -152,8 +159,12 @@ function AddAnomalyDetector({ async function fetchData() { await getDataSourceId(); - const getIndicesDispatchCall = dispatch(getIndices(queryText, dataSourceId)); - const getMappingDispatchCall = dispatch(getMappings(embeddable.vis.data.aggs.indexPattern.title, dataSourceId)); + const getIndicesDispatchCall = dispatch( + getIndices(queryText, dataSourceId) + ); + const getMappingDispatchCall = dispatch( + getMappings([embeddable.vis.data.aggs.indexPattern.title], dataSourceId) + ); await Promise.all([getIndicesDispatchCall, getMappingDispatchCall]); } @@ -167,7 +178,7 @@ function AddAnomalyDetector({ } fetchData(); createEmbeddable(); - }, [dataSourceId]); + }, [dataSourceId]); const [isShowVis, setIsShowVis] = useState(false); const [accordionsOpen, setAccordionsOpen] = useState({ modelFeatures: true }); @@ -335,7 +346,7 @@ function AddAnomalyDetector({ name: OVERLAY_ANOMALIES, args: { detectorId: detectorId, - dataSourceId: dataSourceId + dataSourceId: dataSourceId, }, } as VisLayerExpressionFn; diff --git a/public/pages/ConfigureModel/components/Features/Features.tsx b/public/pages/ConfigureModel/components/Features/Features.tsx index 2464688c..af735802 100644 --- a/public/pages/ConfigureModel/components/Features/Features.tsx +++ b/public/pages/ConfigureModel/components/Features/Features.tsx @@ -65,16 +65,6 @@ export function Features(props: FeaturesProps) { {({ push, remove, form: { values } }: FieldArrayRenderProps) => { return ( - {get(props.detector, 'indices.0', '').includes(':') ? ( -
- - -
- ) : null} {values.featureList.map((feature: any, index: number) => ( { diff --git a/public/pages/ConfigureModel/containers/ConfigureModel.tsx b/public/pages/ConfigureModel/containers/ConfigureModel.tsx index cfd18338..c7867869 100644 --- a/public/pages/ConfigureModel/containers/ConfigureModel.tsx +++ b/public/pages/ConfigureModel/containers/ConfigureModel.tsx @@ -30,7 +30,11 @@ import { RouteComponentProps, useLocation } from 'react-router-dom'; import { AppState } from '../../../redux/reducers'; import { getMappings } from '../../../redux/reducers/opensearch'; import { useFetchDetectorInfo } from '../../CreateDetectorSteps/hooks/useFetchDetectorInfo'; -import { BREADCRUMBS, BASE_DOCS_LINK, MDS_BREADCRUMBS } from '../../../utils/constants'; +import { + BREADCRUMBS, + BASE_DOCS_LINK, + MDS_BREADCRUMBS, +} from '../../../utils/constants'; import { useHideSideNavBar } from '../../main/hooks/useHideSideNavBar'; import { updateDetector } from '../../../redux/reducers/ad'; import { @@ -121,7 +125,7 @@ export function ConfigureModel(props: ConfigureModelProps) { setIsHCDetector(true); } if (detector?.indices) { - dispatch(getMappings(detector.indices[0], dataSourceId)); + dispatch(getMappings(detector.indices, dataSourceId)); } }, [detector]); @@ -133,7 +137,11 @@ export function ConfigureModel(props: ConfigureModelProps) { MDS_BREADCRUMBS.DETECTORS(dataSourceId), { text: detector && detector.name ? detector.name : '', - href: constructHrefWithDataSourceId(`#/detectors/${detectorId}`, dataSourceId, false) + href: constructHrefWithDataSourceId( + `#/detectors/${detectorId}`, + dataSourceId, + false + ), }, MDS_BREADCRUMBS.EDIT_MODEL_CONFIGURATION, ]); @@ -167,12 +175,11 @@ export function ConfigureModel(props: ConfigureModelProps) { useEffect(() => { if (hasError) { - if(dataSourceEnabled) { + if (dataSourceEnabled) { props.history.push( constructHrefWithDataSourceId('/detectors', dataSourceId, false) ); - } - else { + } else { props.history.push('/detectors'); } } diff --git a/public/pages/CreateDetectorSteps/hooks/useFetchDetectorInfo.ts b/public/pages/CreateDetectorSteps/hooks/useFetchDetectorInfo.ts index a686775b..1d00ba18 100644 --- a/public/pages/CreateDetectorSteps/hooks/useFetchDetectorInfo.ts +++ b/public/pages/CreateDetectorSteps/hooks/useFetchDetectorInfo.ts @@ -10,7 +10,7 @@ */ import { get, isEmpty } from 'lodash'; -import { useEffect } from 'react'; +import { useEffect, useMemo } from 'react'; import { useDispatch, useSelector } from 'react-redux'; import { Detector } from '../../../models/interfaces'; import { AppState } from '../../../redux/reducers'; @@ -40,13 +40,13 @@ export const useFetchDetectorInfo = ( const isIndicesRequesting = useSelector( (state: AppState) => state.opensearch.requesting ); - const selectedIndices = get(detector, 'indices.0', ''); + const selectedIndices = useMemo(() => get(detector, 'indices', []), [detector]); useEffect(() => { const fetchDetector = async () => { if (!detector) { await dispatch(getDetector(detectorId, dataSourceId)); } - if (selectedIndices) { + if (selectedIndices && selectedIndices.length > 0) { await dispatch(getMappings(selectedIndices, dataSourceId)); } }; diff --git a/public/pages/DefineDetector/components/DataFilterList/DataFilterList.tsx b/public/pages/DefineDetector/components/DataFilterList/DataFilterList.tsx index e0c5c5a1..3e6b4911 100644 --- a/public/pages/DefineDetector/components/DataFilterList/DataFilterList.tsx +++ b/public/pages/DefineDetector/components/DataFilterList/DataFilterList.tsx @@ -15,7 +15,6 @@ import { EuiSpacer, EuiIcon, EuiButtonEmpty, - EuiCallOut, } from '@elastic/eui'; import { FieldArray, FieldArrayRenderProps, FormikProps } from 'formik'; import React, { useState, Fragment } from 'react'; @@ -38,9 +37,6 @@ export const DataFilterList = (props: DataFilterListProps) => { const [isCreatingNewFilter, setIsCreatingNewFilter] = useState(false); - const selectedIndex = get(props, 'formikProps.values.index.0.label', ''); - const isRemoteIndex = selectedIndex.includes(':'); - return ( {({ push, remove, replace, form: { values } }: FieldArrayRenderProps) => { @@ -66,18 +62,6 @@ export const DataFilterList = (props: DataFilterListProps) => { > - {isRemoteIndex ? ( -
- - -
- ) : null} {values.filters?.length === 0 || diff --git a/public/pages/DefineDetector/components/DataFilterList/components/SimpleFilter.tsx b/public/pages/DefineDetector/components/DataFilterList/components/SimpleFilter.tsx index bc13e09a..89bd35ab 100644 --- a/public/pages/DefineDetector/components/DataFilterList/components/SimpleFilter.tsx +++ b/public/pages/DefineDetector/components/DataFilterList/components/SimpleFilter.tsx @@ -32,6 +32,7 @@ import { getIndexFields, getOperators, isNullOperator } from '../utils/helpers'; import FilterValue from './FilterValue'; import { DetectorDefinitionFormikValues } from '../../../models/interfaces'; import { EMPTY_UI_FILTER } from '../../../utils/constants'; +import _ from 'lodash'; interface SimpleFilterProps { filter: UIFilter; @@ -40,8 +41,20 @@ interface SimpleFilterProps { replace(index: number, value: any): void; } +// This sorting is needed because we utilize two different ways to get index fields, +// through get mapping call and through field_caps API for remote indices +const sortByLabel = (indexFields) => { + //sort the `options` array inside each object by the `label` field + indexFields.forEach(item => { + item.options = _.sortBy(item.options, 'label'); + }); + //sort the outer array by the `label` field + return _.sortBy(indexFields, 'label'); +}; + export const SimpleFilter = (props: SimpleFilterProps) => { - const indexFields = getIndexFields(useSelector(getAllFields)); + let indexFields = getIndexFields(useSelector(getAllFields)); + indexFields = sortByLabel(indexFields) const [searchedIndexFields, setSearchedIndexFields] = useState< ({ label: DATA_TYPES; diff --git a/public/pages/DefineDetector/components/Datasource/DataSource.tsx b/public/pages/DefineDetector/components/Datasource/DataSource.tsx index 7ec60940..ed294f0f 100644 --- a/public/pages/DefineDetector/components/Datasource/DataSource.tsx +++ b/public/pages/DefineDetector/components/Datasource/DataSource.tsx @@ -12,13 +12,20 @@ import { EuiCompressedComboBox, EuiCallOut, EuiSpacer } from '@elastic/eui'; import { Field, FieldProps, FormikProps, useFormikContext } from 'formik'; import { debounce, get } from 'lodash'; -import React, { useEffect, useState } from 'react'; +import React, { useEffect, useMemo, useState } from 'react'; import { useDispatch, useSelector } from 'react-redux'; -import { CatIndex, IndexAlias } from '../../../../../server/models/types'; +import { + CatIndex, + ClusterInfo, + IndexAlias, +} from '../../../../../server/models/types'; import ContentPanel from '../../../../components/ContentPanel/ContentPanel'; import { AppState } from '../../../../redux/reducers'; import { + getAliases, + getClustersInfo, getIndices, + getIndicesAndAliases, getMappings, getPrioritizedIndices, } from '../../../../redux/reducers/opensearch'; @@ -26,6 +33,7 @@ import { getError, isInvalid } from '../../../../utils/utils'; import { IndexOption } from './IndexOption'; import { getDataSourceFromURL, + getLocalCluster, getVisibleOptions, sanitizeSearchText, } from '../../../utils/helpers'; @@ -37,10 +45,13 @@ import { ModelConfigurationFormikValues } from '../../../ConfigureModel/models/i import { INITIAL_MODEL_CONFIGURATION_VALUES } from '../../../ConfigureModel/utils/constants'; import { FILTER_TYPES } from '../../../../models/interfaces'; import { useLocation } from 'react-router-dom'; +import _ from 'lodash'; +import { cleanString } from '../../../../../../../src/plugins/vis_type_vega/public/expressions/helpers'; +import { L } from '../../../../../../../src/plugins/maps_legacy/public/lazy_load_bundle/lazy'; interface DataSourceProps { formikProps: FormikProps; - origIndex: string; + origIndex: { label: string }[]; isEdit: boolean; setModelConfigValues?(initialValues: ModelConfigurationFormikValues): void; setNewIndexSelected?(isNew: boolean): void; @@ -48,47 +59,132 @@ interface DataSourceProps { oldFilterQuery: any; } +interface ClusterOption { + label: string; + cluster: string; + localcluster: string; +} + export function DataSource(props: DataSourceProps) { const dispatch = useDispatch(); const location = useLocation(); const MDSQueryParams = getDataSourceFromURL(location); const dataSourceId = MDSQueryParams.dataSourceId; - const [indexName, setIndexName] = useState( - props.formikProps.values.index[0]?.label + const [indexNames, setIndexNames] = useState<{ label: string }[]>( + props.formikProps.values.index ); const [queryText, setQueryText] = useState(''); const opensearchState = useSelector((state: AppState) => state.opensearch); const { setFieldValue } = useFormikContext(); + const [localClusterName, setLocalClusterName] = useState(''); useEffect(() => { - const getInitialIndices = async () => { - await dispatch(getIndices(queryText, dataSourceId)); - setFieldValue('index', props.formikProps.values.index); - setFieldValue('timeField', props.formikProps.values.timeField); - setFieldValue('filters', props.formikProps.values.filters); + const getInitialClusters = async () => { + await dispatch(getClustersInfo(dataSourceId)); + setFieldValue('clusters', props.formikProps.values.clusters); }; - getInitialIndices(); + getInitialClusters(); }, [dataSourceId]); + const getIndicesAndAliasesBasedOnCluster = async ( + clusters: ClusterOption[], + localClusterExists: boolean + ) => { + //Convert list of clusters to a string for searching + const clustersString = getClustersStringForSearchQuery(clusters); + await dispatch( + getIndicesAndAliases( + queryText, + dataSourceId, + clustersString, + localClusterExists + ) + ); + setFieldValue('index', props.formikProps.values.index); + setFieldValue('timeField', props.formikProps.values.timeField); + setFieldValue('filters', props.formikProps.values.filters); + }; + useEffect(() => { - setIndexName(props.formikProps.values.index[0]?.label); - }, [props.formikProps]); + // Ensure that indices are updated when clusters change + if (props.formikProps.values.clusters) { + const selectedClusters: ClusterOption[] = + props.formikProps.values.clusters; + if (selectedClusters && selectedClusters.length > 0) { + const localClusterExists: boolean = selectedClusters.some( + (cluster) => cluster.localcluster === 'true' + ); + if ( + selectedClusters.length === 1 && + selectedClusters[0].localcluster === 'true' + ) { + getIndicesAndAliasesBasedOnCluster([], localClusterExists); + setLocalClusterName(selectedClusters[0].cluster); + } else { + getIndicesAndAliasesBasedOnCluster( + selectedClusters, + localClusterExists + ); + } + } + } + }, [props.formikProps.values.clusters]); + + const getClustersStringForSearchQuery = (clusters: ClusterOption[]) => { + let clustersString = ''; + if (clusters.length > 0) { + clustersString = clusters + .filter((cluster) => cluster.localcluster == 'false') + .map((cluster) => cluster.cluster) + .join(','); + } + return clustersString; + }; + + useEffect(() => { + if (opensearchState.clusters && opensearchState.clusters.length > 0) { + const localCluster: ClusterInfo[] = getLocalCluster( + opensearchState.clusters + ); + setFieldValue('clusters', getVisibleClusterOptions(localCluster)); + } + }, [opensearchState.clusters]); + + const getClusterOptionLabel = (clusterInfo: ClusterInfo) => + `${clusterInfo.name} ${clusterInfo.localCluster ? '(Local)' : '(Remote)'}`; + + useEffect(() => { + setIndexNames(props.formikProps.values.index); + }, [props.formikProps.values.index]); const handleSearchChange = debounce(async (searchValue: string) => { if (searchValue !== queryText) { const sanitizedQuery = sanitizeSearchText(searchValue); setQueryText(sanitizedQuery); - await dispatch(getPrioritizedIndices(sanitizedQuery, dataSourceId)); + if (props.formikProps.values.clusters) { + const selectedClusters: ClusterOption[] = + props.formikProps.values.clusters; + const clustersString = + getClustersStringForSearchQuery(selectedClusters); + await dispatch( + getPrioritizedIndices(sanitizedQuery, dataSourceId, clustersString) + ); + } else { + await dispatch(getPrioritizedIndices(sanitizedQuery, dataSourceId, '')); + } } }, 300); - const handleIndexNameChange = (selectedOptions: any) => { - const indexName = get(selectedOptions, '0.label', ''); - setIndexName(indexName); - if (indexName !== '') { - dispatch(getMappings(indexName, dataSourceId)); + const handleIndexNameChange = (selectedOptions: any, oldOptions: { label: string }[] = props.formikProps.values.index) => { + const indexNames = selectedOptions; + setIndexNames(indexNames); + if (indexNames.length > 0) { + const indices: string[] = indexNames.map( + (index: { label: string }) => index.label + ); + dispatch(getMappings(indices, dataSourceId)); } - if (indexName !== props.origIndex) { + if (isSelectedOptionIndexRemoved(selectedOptions, oldOptions)) { if (props.setNewIndexSelected) { props.setNewIndexSelected(true); } @@ -99,16 +195,47 @@ export function DataSource(props: DataSourceProps) { } }; - const isDifferentIndex = () => { - return props.isEdit && indexName !== props.origIndex; + const isSelectedOptionIndexRemoved = ( + newSelectedOptions: { label: string }[] = indexNames, + oldSelectedOptions: { label: string }[] = props.formikProps.values.index + ) => { + if (_.isEmpty(oldSelectedOptions) && _.isEmpty(newSelectedOptions)) { + return false; + } + const newSelectedOptionsSet = new Set(newSelectedOptions); + const indexRemoved: boolean = + oldSelectedOptions.some((value) => !newSelectedOptionsSet.has(value)); + return indexRemoved; + }; + + const getVisibleClusterOptions = ( + clusters: ClusterInfo[] + ): ClusterOption[] => { + if (clusters.length > 0) { + const visibleClusters = clusters.map((value) => ({ + label: getClusterOptionLabel(value), + cluster: value.name, + localcluster: value.localCluster.toString(), + })); + // Using _.orderBy to sort clusters with local clusters first + const sortedClusterOptions = _.orderBy( + visibleClusters, + [(option) => option.label.endsWith('(Local)'), 'label'], + ['desc', 'asc'] + ); + return sortedClusterOptions; + } else { + return []; + } }; + const visibleClusters = get(opensearchState, 'clusters', []) as ClusterInfo[]; const visibleIndices = get(opensearchState, 'indices', []) as CatIndex[]; const visibleAliases = get(opensearchState, 'aliases', []) as IndexAlias[]; return ( - {props.isEdit && isDifferentIndex() ? ( + {props.isEdit && isSelectedOptionIndexRemoved() ? (
) : null} + + {({ field, form }: FieldProps) => ( + + { + form.setFieldTouched('clusters', true); + }} + onChange={(options) => { + form.setFieldValue('clusters', options); + }} + /> + + )} + {({ field, form }: FieldProps) => { return ( { const normalizedOptions = createdOption.trim(); @@ -151,15 +310,17 @@ export function DataSource(props: DataSourceProps) { form.setFieldValue('index', options); form.setFieldValue('timeField', undefined); form.setFieldValue('filters', []); - if (props.setModelConfigValues) { + if ( + props.setModelConfigValues && + isSelectedOptionIndexRemoved(options, field.value) + ) { props.setModelConfigValues( INITIAL_MODEL_CONFIGURATION_VALUES ); } - handleIndexNameChange(options); + handleIndexNameChange(options, field.value); }} selectedOptions={field.value} - singleSelection={{ asPlainText: true }} isClearable={false} renderOption={(option, searchValue, className) => ( state.opensearch); - const selectedIndex = get(props, 'formikProps.values.index.0.label', ''); - const isRemoteIndex = selectedIndex.includes(':'); const [queryText, setQueryText] = useState(''); const handleSearchChange = debounce(async (searchValue: string) => { @@ -68,17 +66,6 @@ export function Timestamp(props: TimestampProps) { titleSize="s" subTitle="Select the time field you want to use for the time filter." > - {isRemoteIndex ? ( -
- - -
- ) : null} {({ field, form }: FieldProps) => ( { Full creating detector definition renders the compon >
+
+ +
+
+ +
+
@@ -255,7 +361,7 @@ exports[` Full creating detector definition renders the compon class="euiText euiText--medium sublabel" style="max-width: 400px;" > - Choose an index or index pattern as the data source. + Choose an index, index pattern or alias as the data source.
@@ -278,7 +384,7 @@ exports[` Full creating detector definition renders the compon class="euiFormControlLayout__childrenWrapper" >
@@ -1221,7 +1327,114 @@ exports[` empty creating detector definition renders the compo >
+
+ +
+
+ +
+
@@ -1244,7 +1457,7 @@ exports[` empty creating detector definition renders the compo class="euiText euiText--medium sublabel" style="max-width: 400px;" > - Choose an index or index pattern as the data source. + Choose an index, index pattern or alias as the data source.
@@ -1267,7 +1480,7 @@ exports[` empty creating detector definition renders the compo class="euiFormControlLayout__childrenWrapper" >
@@ -2208,41 +2421,116 @@ exports[` empty editing detector definition renders the compon
-
+
-
- - + Clusters +

+
+
+ Select a local cluster or remote clusters from cross-cluster connections. +
+
+ +
+
+ -
@@ -2265,7 +2553,7 @@ exports[` empty editing detector definition renders the compon class="euiText euiText--medium sublabel" style="max-width: 400px;" > - Choose an index or index pattern as the data source. + Choose an index, index pattern or alias as the data source.
@@ -2288,7 +2576,7 @@ exports[` empty editing detector definition renders the compon class="euiFormControlLayout__childrenWrapper" >
diff --git a/public/pages/DefineDetector/models/interfaces.ts b/public/pages/DefineDetector/models/interfaces.ts index c10ac7cf..0eb9191d 100644 --- a/public/pages/DefineDetector/models/interfaces.ts +++ b/public/pages/DefineDetector/models/interfaces.ts @@ -26,4 +26,5 @@ export interface DetectorDefinitionFormikValues { resultIndexMinSize?: number | string; resultIndexTtl?:number | string; flattenCustomResultIndex?: boolean; + clusters?: any[]; } diff --git a/public/pages/DefineDetector/utils/helpers.ts b/public/pages/DefineDetector/utils/helpers.ts index 7ece3554..2c8f5c73 100644 --- a/public/pages/DefineDetector/utils/helpers.ts +++ b/public/pages/DefineDetector/utils/helpers.ts @@ -34,7 +34,7 @@ export function detectorDefinitionToFormik( ...initialValues, name: ad.name, description: ad.description, - index: [{ label: ad.indices[0] }], // Currently we support only one index + index: [...ad.indices.map(index => ({ label: index }))], resultIndex: ad.resultIndex, filters: filtersToFormik(ad), filterQuery: JSON.stringify( diff --git a/public/pages/DetectorConfig/containers/DetectorConfig.tsx b/public/pages/DetectorConfig/containers/DetectorConfig.tsx index c78459da..08014932 100644 --- a/public/pages/DetectorConfig/containers/DetectorConfig.tsx +++ b/public/pages/DetectorConfig/containers/DetectorConfig.tsx @@ -18,6 +18,7 @@ import { RouteComponentProps, useLocation } from 'react-router'; import { AppState } from '../../../redux/reducers'; import { useSelector, useDispatch } from 'react-redux'; import { getDetector } from '../../../redux/reducers/ad'; +import { getClustersInfo } from '../../../redux/reducers/opensearch' import { EuiLoadingSpinner } from '@elastic/eui'; import { getDataSourceFromURL } from '../../../pages/utils/helpers'; @@ -35,9 +36,13 @@ export function DetectorConfig(props: DetectorConfigProps) { const detector = useSelector( (state: AppState) => state.ad.detectors[props.detectorId] ); + const clusters = useSelector( + (state: AppState) => state.opensearch.clusters + ); useEffect(() => { dispatch(getDetector(props.detectorId, dataSourceId)); + dispatch(getClustersInfo()); }, []); return ( @@ -49,6 +54,8 @@ export function DetectorConfig(props: DetectorConfigProps) { detector={detector} onEditDetectorDefinition={props.onEditDetector} isCreate={false} + clusters={clusters} + dataSourceId={dataSourceId} /> void; onPageClick: (pageNumber: number) => void; } + +const getVisibleIndices = (props) => { + const visibleIndices = props.selectedIndices.length > 0 + ? props.selectedIndices.map((index) => ({ label: index })) + : [] + return visibleIndices; +} + export const ListFilters = (props: ListFiltersProps) => ( @@ -72,11 +80,7 @@ export const ListFilters = (props: ListFiltersProps) => ( options={props.indexOptions} onChange={props.onIndexChange} onSearchChange={props.onSearchIndexChange} - selectedOptions={ - props.selectedIndices.length > 0 - ? props.selectedIndices.map((index) => ({ label: index })) - : [] - } + selectedOptions={getVisibleIndices(props)} fullWidth={true} /> diff --git a/public/pages/DetectorsList/containers/List/List.tsx b/public/pages/DetectorsList/containers/List/List.tsx index 90353d9e..2db748bc 100644 --- a/public/pages/DetectorsList/containers/List/List.tsx +++ b/public/pages/DetectorsList/containers/List/List.tsx @@ -39,7 +39,8 @@ import { deleteDetector, } from '../../../../redux/reducers/ad'; import { - getIndices, + getClustersInfo, + getIndicesAndAliases, getPrioritizedIndices, } from '../../../../redux/reducers/opensearch'; import { APP_PATH, MDS_BREADCRUMBS, PLUGIN_NAME, USE_NEW_HOME_PAGE } from '../../../../utils/constants'; @@ -174,14 +175,26 @@ export const DetectorList = (props: ListProps) => { isStopDisabled: false, }); + const [localClusterName, setLocalClusterName] = useState(""); + // Getting all initial monitors useEffect(() => { const getInitialMonitors = async () => { dispatch(searchMonitors(state.selectedDataSourceId)); }; + const getInitialClusters = async () => { + await dispatch(getClustersInfo(state.selectedDataSourceId)); + } getInitialMonitors(); + getInitialClusters(); }, []); + useEffect(() => { + if (opensearchState.clusters && opensearchState.clusters.length > 0) { + setLocalClusterName(opensearchState.clusters.find(cluster => cluster.localCluster)?.name || "local") + } + }, [opensearchState.clusters]); + useEffect(() => { if ( errorGettingDetectors && @@ -201,7 +214,7 @@ export const DetectorList = (props: ListProps) => { // Updating displayed indices (initializing to first 20 for now) const visibleIndices = get(opensearchState, 'indices', []) as CatIndex[]; const visibleAliases = get(opensearchState, 'aliases', []) as IndexAlias[]; - const indexOptions = getVisibleOptions(visibleIndices, visibleAliases); + const indexOptions = getVisibleOptions(visibleIndices, visibleAliases, localClusterName); const queryParams = getURLQueryParams(props.location); const [state, setState] = useState({ @@ -235,7 +248,7 @@ export const DetectorList = (props: ListProps) => { const [indexQuery, setIndexQuery] = useState(''); useEffect(() => { const getInitialIndices = async () => { - await dispatch(getIndices(indexQuery, state.selectedDataSourceId)); + await dispatch(getIndicesAndAliases(indexQuery, state.selectedDataSourceId, "*")) }; getInitialIndices(); }, [state.selectedDataSourceId]); diff --git a/public/pages/ReviewAndCreate/components/DataConnectionFlyout/DataConnectionFlyout.tsx b/public/pages/ReviewAndCreate/components/DataConnectionFlyout/DataConnectionFlyout.tsx new file mode 100644 index 00000000..90c1ee5e --- /dev/null +++ b/public/pages/ReviewAndCreate/components/DataConnectionFlyout/DataConnectionFlyout.tsx @@ -0,0 +1,106 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +import { + EuiButton, + EuiFlexGroup, + EuiFlexItem, + EuiFlyout, + EuiFlyoutBody, + EuiFlyoutFooter, + EuiFlyoutHeader, + EuiText, + EuiLink, + EuiSteps, + EuiBasicTable, + EuiTitle, +} from '@elastic/eui'; +import React from 'react'; + +type DataConnectionFlyoutProps = { + indices: any; + onClose(): void; + localClusterName: string; +}; + +const columns = [ + { + field: 'cluster', + name: 'Data connection', + sortable: true, + truncateText: true, + }, + { + field: 'index', + name: 'Index', + sortable: true, + truncateText: true, + }, +]; + +const getIndexItemsToDisplay = (props: DataConnectionFlyoutProps) => { + const indexItems = props.indices.map((index = '', int) => { + const item = { id: int }; + const shouldSplit = index.includes(':'); + const splitIndex = index.split(':'); + let clusterName = shouldSplit ? splitIndex[0] : props.localClusterName; + clusterName = + clusterName === props.localClusterName + ? `${clusterName} (Local)` + : `${clusterName} (Remote)`; + const indexName = shouldSplit ? splitIndex[1] : index; + item.cluster = clusterName; + item.index = indexName; + return item; + }); + return indexItems; +}; + +export const DataConnectionFlyout = (props: DataConnectionFlyoutProps) => { + return ( + + + + + +

{`Index`}

+ + + + + + + item.id} + columns={columns} + //pagination={true} + isSelectable={false} + hasActions={false} + noItemsMessage={'No data sources configured for this detector.'} + data-test-subj={'dataConnectionsFlyout_table'} + /> + + + + + + + + ); +}; diff --git a/public/pages/ReviewAndCreate/components/DataConnectionFlyout/index.ts b/public/pages/ReviewAndCreate/components/DataConnectionFlyout/index.ts new file mode 100644 index 00000000..d249cf05 --- /dev/null +++ b/public/pages/ReviewAndCreate/components/DataConnectionFlyout/index.ts @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +export { DataConnectionFlyout } from './DataConnectionFlyout'; diff --git a/public/pages/ReviewAndCreate/components/DetectorDefinitionFields/DetectorDefinitionFields.tsx b/public/pages/ReviewAndCreate/components/DetectorDefinitionFields/DetectorDefinitionFields.tsx index de5391a5..fbcd80d8 100644 --- a/public/pages/ReviewAndCreate/components/DetectorDefinitionFields/DetectorDefinitionFields.tsx +++ b/public/pages/ReviewAndCreate/components/DetectorDefinitionFields/DetectorDefinitionFields.tsx @@ -18,16 +18,23 @@ import { EuiLoadingSpinner, EuiFlexGroup, EuiText, + EuiLink, } from '@elastic/eui'; -import React from 'react'; +import React, { useEffect, useState } from 'react'; import { get, isEqual } from 'lodash'; import { Detector, ValidationSettingResponse, } from '../../../../models/interfaces'; +import { useDispatch, useSelector } from 'react-redux'; import { FilterDisplayList } from '../FilterDisplayList'; import { ConfigCell, FixedWidthRow } from '../../../../components/ConfigCell'; import { toStringConfigCell } from '../../utils/helpers'; +import { DataConnectionFlyout } from '../DataConnectionFlyout/DataConnectionFlyout'; +import { ClusterInfo } from '../../../../../server/models/types'; +import { getLocalCluster } from '../../../../pages/utils/helpers'; +import { getClustersInfo } from '../../../../redux/reducers/opensearch'; +import { AppState } from '../../../../redux/reducers'; interface DetectorDefinitionFieldsProps { detector: Detector; onEditDetectorDefinition(): void; @@ -37,11 +44,18 @@ interface DetectorDefinitionFieldsProps { validationResponse?: ValidationSettingResponse; isLoading?: boolean; isCreatingDetector?: boolean; + dataSourceId: string; + clusters?: ClusterInfo[]; } export const DetectorDefinitionFields = ( props: DetectorDefinitionFieldsProps ) => { + const dispatch = useDispatch(); + const opensearchState = useSelector((state: AppState) => state.opensearch); + const [showDataConnectionFlyout, setShowDataConnectionFlyout] = + useState(false); + const filterInputs = { uiMetadata: get(props, 'detector.uiMetadata', {}), filterQuery: JSON.stringify( @@ -51,6 +65,13 @@ export const DetectorDefinitionFields = ( ), }; + useEffect(() => { + const getInitialClusters = async () => { + await dispatch(getClustersInfo()); + }; + getInitialClusters(); + }, [props.dataSourceId]); + const getValidationCallout = () => { //When validation response is loading then displaying loading spinner, don't display // after clicking on "create detector" button as isLoading will be true from that request @@ -130,127 +151,158 @@ export const DetectorDefinitionFields = ( } } }; - const minAgeValue = get(props, 'detector.resultIndexMinAge', undefined); - const minAge = (minAgeValue === undefined) ? '-' : minAgeValue + " Days"; + const minAge = minAgeValue === undefined ? '-' : minAgeValue + ' Days'; const minSizeValue = get(props, 'detector.resultIndexMinSize', undefined); - const minSize = (minSizeValue === undefined) ? '-' : minSizeValue + " MB"; + const minSize = minSizeValue === undefined ? '-' : minSizeValue + ' MB'; const ttlValue = get(props, 'detector.resultIndexTtl', undefined); const ttl = (ttlValue === undefined) ? '-' : ttlValue + " Days"; const flattenCustomResultIndex = get(props, 'detector.flattenCustomResultIndex', undefined); const flatten = (flattenCustomResultIndex === undefined) ? '-' : flattenCustomResultIndex ? 'Yes' : 'No'; + + const getDataConnectionsDisplay = (indices: string[]) => { + if (indices.length === 0) return '-'; + if (indices.length === 1) return indices[0]; + return ( +

+ {indices[0]}...  + setShowDataConnectionFlyout(true)} + style={{ fontSize: '12px' }} + > + View all {indices.length} + +

+ ); + }; return ( - - Edit - , - ]} - > - {props.isCreate ? getValidationCallout() : null} - - - - - - - - - - - - - - - - {props.isCreate ? null : ( - + + + Edit + , + ]} + > + {props.isCreate ? getValidationCallout() : null} + + + + + - )} - - - - - - - {props.isCreate ? null : ( + + + + + - )} - - + + + )} + + + + + + + {props.isCreate ? null : ( + + + )} - /> - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + {showDataConnectionFlyout ? ( + setShowDataConnectionFlyout(false)} + localClusterName={ + opensearchState.clusters?.length + ? getLocalCluster(opensearchState.clusters as ClusterInfo[])[0] + ?.name + : 'local-cluster' + } + /> + ) : null} + ); }; diff --git a/public/pages/ReviewAndCreate/components/__tests__/DataConnectionFlyout.test.tsx b/public/pages/ReviewAndCreate/components/__tests__/DataConnectionFlyout.test.tsx new file mode 100644 index 00000000..6c9f916f --- /dev/null +++ b/public/pages/ReviewAndCreate/components/__tests__/DataConnectionFlyout.test.tsx @@ -0,0 +1,81 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +import React from 'react'; +import { render } from '@testing-library/react'; +import { DataConnectionFlyout } from '../DataConnectionFlyout'; + +const renderDataConnectionFlyout = ( + indices: string[], + localClusterName: string = 'local-cluster', + onClose = jest.fn() +) => { + return render( + + ); +}; + +describe(' spec', () => { + test('renders the flyout with indices and local cluster name', () => { + const indices = [ + 'test-index', + 'cluster-2:test-index-2', + 'cluster-3:http-index', + ]; + const { container, getByText, getByTestId } = + renderDataConnectionFlyout(indices); + expect(container.firstChild).toMatchSnapshot(); + getByTestId('dataSourcesFlyout_header'); + getByText('local-cluster (Local)'); + getByText('test-index'); + getByText('cluster-2 (Remote)'); + getByText('test-index-2'); + getByText('cluster-3 (Remote)'); + getByText('http-index'); + }); + test('renders the flyout with only local indices', () => { + const indices = ['local-index']; + const { container, getByText, getByTestId, queryByText } = + renderDataConnectionFlyout(indices); + expect(container.firstChild).toMatchSnapshot(); + getByTestId('dataSourcesFlyout_header'); + getByText('local-cluster (Local)'); + getByText('local-index'); + expect(queryByText('Remote')).toBeNull(); + }); + + test('handles no indices', () => { + const indices: string[] = []; + const { getByText, getByTestId } = renderDataConnectionFlyout(indices); + + getByTestId('dataSourcesFlyout_header'); + getByText('No data sources configured for this detector.'); + }); + + test('calls onClose when flyout close button is clicked', () => { + const onClose = jest.fn(); + const indices = ['test-index']; + const { getByTestId } = renderDataConnectionFlyout( + indices, + 'local-cluster', + onClose + ); + + const closeButton = getByTestId('euiFlyoutCloseButton'); // Adjust the text if needed for close button + closeButton.click(); + + expect(onClose).toHaveBeenCalled(); + }); +}); diff --git a/public/pages/ReviewAndCreate/components/__tests__/DetectorDefinitionFields.test.tsx b/public/pages/ReviewAndCreate/components/__tests__/DetectorDefinitionFields.test.tsx index 4aa5eb17..fc679806 100644 --- a/public/pages/ReviewAndCreate/components/__tests__/DetectorDefinitionFields.test.tsx +++ b/public/pages/ReviewAndCreate/components/__tests__/DetectorDefinitionFields.test.tsx @@ -10,7 +10,7 @@ */ import React from 'react'; -import { render } from '@testing-library/react'; +import { render, within } from '@testing-library/react'; import { DetectorDefinitionFields } from '../DetectorDefinitionFields/DetectorDefinitionFields'; import { Detector, @@ -18,65 +18,96 @@ import { FILTER_TYPES, OPERATORS_MAP, } from '../../../../models/interfaces'; +import { + HashRouter as Router, + RouteComponentProps, + Route, + Switch, +} from 'react-router-dom'; import { Formik } from 'formik'; import { DATA_TYPES } from '../../../../utils/constants'; +import { Provider } from 'react-redux'; +import { CoreServicesContext } from '../../../../components/CoreServices/CoreServices'; +import { httpClientMock, coreServicesMock } from '../../../../../test/mocks'; +import configureStore from '../../../../redux/configureStore'; -const testDetector = { - id: 'test-id', - name: 'test-detector', - indices: ['test-index'], - detectionInterval: { - period: { - interval: 10, - unit: UNITS.MINUTES, - }, - }, - description: 'test-description', - timeField: 'test-timefield', - windowDelay: { - period: { - interval: 1, - unit: UNITS.MINUTES, + +const getTestDetectorWithDifferentIndices = (indices: string[]) => { + return { + id: 'test-id', + name: 'test-detector', + indices: indices, + detectionInterval: { + period: { + interval: 10, + unit: UNITS.MINUTES, + }, }, - }, - uiMetadata: { - filters: [ - { - filterType: FILTER_TYPES.SIMPLE, - fieldInfo: [ - { - label: 'test-filter-field', - type: DATA_TYPES.TEXT, - }, - ], - operator: OPERATORS_MAP.IS, - fieldValue: 'null', + description: 'test-description', + timeField: 'test-timefield', + windowDelay: { + period: { + interval: 1, + unit: UNITS.MINUTES, }, - ], - }, - resultIndex: 'opensearch-ad-plugin-result-test', - resultIndexMinAge: 7, - resultIndexMinSize: 51200, - resultIndexTtl: 60, - flattenCustomResultIndex: true, -} as Detector; + }, + uiMetadata: { + filters: [ + { + filterType: FILTER_TYPES.SIMPLE, + fieldInfo: [ + { + label: 'test-filter-field', + type: DATA_TYPES.TEXT, + }, + ], + operator: OPERATORS_MAP.IS, + fieldValue: 'null', + }, + ], + }, + resultIndex: 'opensearch-ad-plugin-result-test', + resultIndexMinAge: 7, + resultIndexMinSize: 51200, + resultIndexTtl: 60, + flattenCustomResultIndex: true, + } as Detector; +} +const onEditDetectorDefinition = jest.fn(); + +const renderWithRouter = (isCreate: boolean = false, testDetector: Detector) => ({ + + ...render( + + + + ( + + + {() => ( +
+ +
+ )} + + + )} + /> + + + + ), +}); describe(' spec', () => { test('renders the component in create mode (no ID)', () => { - const onEditDetectorDefinition = jest.fn(); - const { container, getByText, queryByText } = render( - - {() => ( -
- -
- )} - - ); + const testDetector = getTestDetectorWithDifferentIndices(['test-index']) + const { container, getByText, queryByText } = renderWithRouter(true, testDetector) expect(container.firstChild).toMatchSnapshot(); getByText('test-detector'); getByText('test-index'); @@ -92,21 +123,31 @@ describe(' spec', () => { getByText('60 Days'); expect(queryByText('test-id')).toBeNull(); }); + test('renders the component in create mode (no ID) multi-index', () => { + const testDetector = getTestDetectorWithDifferentIndices(['test-index', 'cluster-2:test-index-2', 'cluster-3:http-index']) + const { container, getByText, queryByText, getByTestId } = renderWithRouter(true, testDetector) + expect(container.firstChild).toMatchSnapshot(); + getByText('test-detector'); + queryByText('test-index') + queryByText('...') + getByTestId('indexNameCellViewAllLink'); + getByText('test-filter-field is null'); + getByText('10 Minutes'); + getByText('test-description'); + getByText('test-timefield'); + getByText('1 Minutes'); + getByText('opensearch-ad-plugin-result-test'); + getByText('Yes') + getByText('7 Days'); + getByText('51200 MB'); + getByText('60 Days'); + queryByText('View All'); + + expect(queryByText('test-id')).toBeNull(); + }); test('renders the component in edit mode (with ID)', () => { - const onEditDetectorDefinition = jest.fn(); - const { container, getByText, queryByText } = render( - - {() => ( -
- -
- )} - - ); + const testDetector = getTestDetectorWithDifferentIndices(['test-index']) + const { container, getByText, queryByText } = renderWithRouter(false, testDetector) expect(container.firstChild).toMatchSnapshot(); getByText('test-detector'); getByText('test-index'); diff --git a/public/pages/ReviewAndCreate/components/__tests__/__snapshots__/DataConnectionFlyout.test.tsx.snap b/public/pages/ReviewAndCreate/components/__tests__/__snapshots__/DataConnectionFlyout.test.tsx.snap new file mode 100644 index 00000000..8a3e039e --- /dev/null +++ b/public/pages/ReviewAndCreate/components/__tests__/__snapshots__/DataConnectionFlyout.test.tsx.snap @@ -0,0 +1,5 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[` spec renders the flyout with indices and local cluster name 1`] = `null`; + +exports[` spec renders the flyout with only local indices 1`] = `null`; diff --git a/public/pages/ReviewAndCreate/components/__tests__/__snapshots__/DetectorDefinitionFields.test.tsx.snap b/public/pages/ReviewAndCreate/components/__tests__/__snapshots__/DetectorDefinitionFields.test.tsx.snap index cc35b8ff..165e76ef 100644 --- a/public/pages/ReviewAndCreate/components/__tests__/__snapshots__/DetectorDefinitionFields.test.tsx.snap +++ b/public/pages/ReviewAndCreate/components/__tests__/__snapshots__/DetectorDefinitionFields.test.tsx.snap @@ -485,6 +485,504 @@ exports[` spec renders the component in create mode (no ID
`; +exports[` spec renders the component in create mode (no ID) multi-index 1`] = ` +
+
+
+
+

+ Detector settings +

+
+
+
+
+
+
+
+ +
+
+
+
+
+
+
+
+
+
+
+
+ +
+
+
+

+ test-detector +

+
+
+
+
+
+
+
+ +
+
+
+

+

+ test-index + ...  + +

+

+
+
+
+
+
+
+
+ +
+
+
    +
  1. + test-filter-field is null +
    2. +
+
+
+
+
+
+
+ +
+
+
+

+ 10 Minutes +

+
+
+
+
+
+
+
+ +
+
+
+

+ test-description +

+
+
+
+
+
+
+
+ +
+
+
+

+ test-timefield +

+
+
+
+
+
+
+
+ +
+
+
+

+ 1 Minutes +

+
+
+
+
+
+
+
+ +
+
+
+

+ opensearch-ad-plugin-result-test +

+
+
+
+
+
+
+
+ +
+
+
+

+ Yes +

+
+
+
+
+
+
+
+ +
+
+
+

+ 7 Days +

+
+
+
+
+
+
+
+ +
+
+
+

+ 51200 MB +

+
+
+
+
+
+
+
+ +
+
+
+

+ 60 Days +

+
+
+
+
+
+
+
+
+
+`; + exports[` spec renders the component in edit mode (with ID) 1`] = `
spec renders the component, validation loading 1`] >

+ > + - +

@@ -1469,7 +1471,9 @@ exports[`issue in detector validation issues in feature query 1`] = ` >

+ > + - +

diff --git a/public/pages/utils/__tests__/helpers.test.ts b/public/pages/utils/__tests__/helpers.test.ts index e69a9a89..1949fbeb 100644 --- a/public/pages/utils/__tests__/helpers.test.ts +++ b/public/pages/utils/__tests__/helpers.test.ts @@ -9,10 +9,10 @@ * GitHub history for details. */ -import { getVisibleOptions, sanitizeSearchText } from '../helpers'; +import { getVisibleOptions, groupIndicesOrAliasesByCluster, sanitizeSearchText } from '../helpers'; describe('helpers', () => { describe('getVisibleOptions', () => { - test('returns without system indices if valid index options', () => { + test('returns without system indices if valid index options and undefined localCluster', () => { expect( getVisibleOptions( [ @@ -26,13 +26,56 @@ describe('helpers', () => { ) ).toEqual([ { - label: 'Indices', + label: 'Indices: (Local)', options: [{ label: 'hello', health: 'green' }], }, { - label: 'Aliases', + label: 'Aliases: (Local)', + options: [{ label: 'hello' }], + }, + ]); + }); + test('returns without system indices if valid index options', () => { + expect( + getVisibleOptions( + [ + { index: 'hello', health: 'green', localCluster: true }, + { index: '.world', health: 'green', localCluster: false }, + { index: 'ale-cluster:ale', health: 'green', localCluster: false }, + ], + [ + { + alias: 'cluster-2:.system', + index: 'opensearch_dashboards', + localCluster: false, + }, + { alias: 'hello', index: 'world', localCluster: true }, + { alias: 'cluster-2:hello', index: 'world', localCluster: false }, + ], + 'cluster-1' + ) + ).toEqual([ + { + label: 'Indices: cluster-1 (Local)', + options: [{ label: 'hello', health: 'green' }], + }, + { + label: 'Indices: ale-cluster (Remote)', + options: [ + { + label: 'ale-cluster:ale', + health: 'green', + }, + ], + }, + { + label: 'Aliases: cluster-1 (Local)', options: [{ label: 'hello' }], }, + { + label: 'Aliases: cluster-2 (Remote)', + options: [{ label: 'cluster-2:hello' }], + }, ]); }); test('returns empty aliases and index ', () => { @@ -56,6 +99,100 @@ describe('helpers', () => { ]); }); }); + describe('groupIndicesOrAliasesByCluster', () => { + const localClusterName = 'local-cluster'; + const dataType = 'Indices'; + + test('should group local indices correctly', () => { + const indices = [ + { label: 'index1', localCluster: true }, + { label: 'index2', localCluster: true }, + ]; + + const result = groupIndicesOrAliasesByCluster(indices, localClusterName, dataType); + + expect(result).toEqual([ + { + label: 'Indices: local-cluster (Local)', + options: [ + { label: 'index1' }, + { label: 'index2' }, + ], + }, + ]); + }); + test('should group remote indices correctly', () => { + const indices = [ + { label: 'remote-cluster:index1', localCluster: false }, + { label: 'remote-cluster:index2', localCluster: false }, + ]; + + const result = groupIndicesOrAliasesByCluster(indices, localClusterName, dataType); + + expect(result).toEqual([ + { + label: 'Indices: remote-cluster (Remote)', + options: [ + { label: 'remote-cluster:index1' }, + { label: 'remote-cluster:index2' }, + ], + }, + ]); + }); + + test('should group mixed local and remote indices correctly', () => { + const indices = [ + { label: 'index1', localCluster: true }, + { label: 'remote-cluster:index2', localCluster: false }, + { label: 'index3', localCluster: true }, + { label: 'another-remote:index4', localCluster: false }, + ]; + + const result = groupIndicesOrAliasesByCluster(indices, localClusterName, dataType); + + expect(result).toEqual([ + { + label: 'Indices: local-cluster (Local)', + options: [ + { label: 'index1' }, + { label: 'index3' }, + ], + }, + { + label: 'Indices: remote-cluster (Remote)', + options: [ + { label: 'remote-cluster:index2' }, + ], + }, + { + label: 'Indices: another-remote (Remote)', + options: [ + { label: 'another-remote:index4' }, + ], + }, + ]); + }); + + test('should handle indices with undefined localCluster property', () => { + const indices = [ + { label: 'index1' }, + { label: 'index2', localCluster: undefined }, + ]; + + const result = groupIndicesOrAliasesByCluster(indices, localClusterName, dataType); + + expect(result).toEqual([ + { + label: 'Indices: local-cluster (Local)', + options: [ + { label: 'index1' }, + { label: 'index2' }, + ], + }, + ]); + }); + }); + describe('sanitizeSearchText', () => { test('should return empty', () => { expect(sanitizeSearchText('*')).toBe(''); diff --git a/public/pages/utils/helpers.ts b/public/pages/utils/helpers.ts index e7c30b9a..91c27512 100644 --- a/public/pages/utils/helpers.ts +++ b/public/pages/utils/helpers.ts @@ -11,20 +11,30 @@ import queryString from 'query-string'; import { CatIndex, + ClusterInfo, IndexAlias, MDSQueryParams, } from '../../../server/models/types'; import sortBy from 'lodash/sortBy'; import { DetectorListItem } from '../../models/interfaces'; -import { DETECTORS_QUERY_PARAMS, SORT_DIRECTION } from '../../../server/utils/constants'; -import { ALL_INDICES, ALL_DETECTOR_STATES, MAX_DETECTORS, DEFAULT_QUERY_PARAMS } from './constants'; +import { + DETECTORS_QUERY_PARAMS, + SORT_DIRECTION, +} from '../../../server/utils/constants'; +import { + ALL_INDICES, + ALL_DETECTOR_STATES, + MAX_DETECTORS, + DEFAULT_QUERY_PARAMS, +} from './constants'; import { DETECTOR_STATE } from '../../../server/utils/constants'; import { timeFormatter } from '@elastic/charts'; -import { getDataSourceEnabled, getDataSourcePlugin } from '../../services'; +import { getDataSourceEnabled } from '../../services'; import { DataSourceAttributes } from '../../../../../src/plugins/data_source/common/data_sources'; import { SavedObject } from '../../../../../src/core/public'; -import * as pluginManifest from "../../../opensearch_dashboards.json"; -import semver from "semver"; +import * as pluginManifest from '../../../opensearch_dashboards.json'; +import semver from 'semver'; +import _ from 'lodash'; export function sanitizeSearchText(searchValue: string): string { if (!searchValue || searchValue == '*') { @@ -48,29 +58,73 @@ const isUserIndex = (index: string) => { if (!index) { return false; } - return !index.startsWith('.'); + //.ml-config + return !(index.startsWith('.') || index.includes(':.')); }; -export function getVisibleOptions(indices: CatIndex[], aliases: IndexAlias[]) { - const visibleIndices = indices - .filter((value) => isUserIndex(value.index)) - .map((value) => ({ label: value.index, health: value.health })); - const visibleAliases = aliases - .filter((value) => isUserIndex(value.alias)) - .map((value) => ({ label: value.alias })); - - return [ - { - label: 'Indices', - options: visibleIndices, - }, - { - label: 'Aliases', - options: visibleAliases, - }, - ]; +export function groupIndicesOrAliasesByCluster( + indices, + localClusterName: string, + dataType: string +) { + return indices.reduce((acc, index) => { + const clusterName = index.label.includes(':') + ? index.label.split(':')[0] + : localClusterName; + + //if undefined should be local as well. + let label = + index.localCluster === undefined || index.localCluster + ? `${dataType}: ${localClusterName} (Local)` + : `${dataType}: ${clusterName} (Remote)`; + + const { localCluster, ...indexWithOutLocalInfo } = index; // Destructure and remove localCluster + const cluster = acc.find((cluster) => cluster.label === label); + if (cluster) { + cluster.options.push(indexWithOutLocalInfo); + } else { + acc.push({ label, options: [indexWithOutLocalInfo] }); + } + + return acc; + }, [] as { label: string; options: any[] }[]); } +export function getVisibleOptions( + indices: CatIndex[], + aliases: IndexAlias[], + localClusterName: string = '' +) { + // Group by cluster or fallback to default label format + const getLabeledOptions = (items: any[], label: string) => + items.length > 0 + ? groupIndicesOrAliasesByCluster(items, localClusterName, label) + : [{ label, options: items }]; + + const visibleIndices = mapToVisibleOptions(indices, 'index'); + const visibleAliases = mapToVisibleOptions(aliases, 'alias'); + + // Combine grouped indices and aliases + const visibleIndicesLabel = getLabeledOptions(visibleIndices, 'Indices'); + const visibleAliasesLabel = getLabeledOptions(visibleAliases, 'Aliases'); + const combinedVisibleIndicesAndAliases = + visibleIndicesLabel.concat(visibleAliasesLabel); + const sortedVisibleIndicesAndAliases = _.sortBy(combinedVisibleIndicesAndAliases, [ + (item) => (item.label.includes('Indices:') ? 0 : 1), // Indices first, then Aliases + (item) => (item.label.includes('(Local)') ? 0 : 1), // Local first, then Remote + ]); + return sortedVisibleIndicesAndAliases; +} + +export const mapToVisibleOptions = (items: any[], key: string) => + items + .filter((value) => isUserIndex(value[key])) + .map((value) => ({ + label: value[key], + ...(key === 'index' && { health: value.health }), // Only applicable to indices, ignored for aliases + localCluster: value.localCluster, + })); + export const filterAndSortDetectors = ( detectors: DetectorListItem[], search: string, @@ -93,7 +147,7 @@ export const filterAndSortDetectors = ( selectedIndices == ALL_INDICES ? filteredBySearchAndState : filteredBySearchAndState.filter((detector) => - selectedIndices.includes(detector.indices[0]) + detector.indices.some((index) => selectedIndices.includes(index)) ); let sorted = sortBy(filteredBySearchAndStateAndIndex, sortField); if (sortDirection == SORT_DIRECTION.DESC) { @@ -169,7 +223,7 @@ export const constructHrefWithDataSourceId = ( url.set(DETECTORS_QUERY_PARAMS.SEARCH, DEFAULT_QUERY_PARAMS.search); url.set(DETECTORS_QUERY_PARAMS.INDICES, DEFAULT_QUERY_PARAMS.indices); url.set(DETECTORS_QUERY_PARAMS.SORT_FIELD, DEFAULT_QUERY_PARAMS.sortField); - url.set(DETECTORS_QUERY_PARAMS.SORT_DIRECTION, SORT_DIRECTION.ASC) + url.set(DETECTORS_QUERY_PARAMS.SORT_DIRECTION, SORT_DIRECTION.ASC); if (dataSourceEnabled) { url.set(DETECTORS_QUERY_PARAMS.DATASOURCEID, ''); } @@ -189,7 +243,9 @@ export const constructHrefWithDataSourceId = ( return `${basePath}?${url.toString()}`; }; -export const isDataSourceCompatible = (dataSource: SavedObject) => { +export const isDataSourceCompatible = ( + dataSource: SavedObject +) => { if ( 'requiredOSDataSourcePlugins' in pluginManifest && !pluginManifest.requiredOSDataSourcePlugins.every((plugin) => @@ -210,4 +266,8 @@ export const isDataSourceCompatible = (dataSource: SavedObject { + return clusters.filter((cluster) => cluster.localCluster === true); +}; diff --git a/public/redux/reducers/__tests__/opensearch.test.ts b/public/redux/reducers/__tests__/opensearch.test.ts index cb434c0c..18d75ae2 100644 --- a/public/redux/reducers/__tests__/opensearch.test.ts +++ b/public/redux/reducers/__tests__/opensearch.test.ts @@ -15,7 +15,9 @@ import { BASE_NODE_API_PATH } from '../../../../utils/constants'; import { mockedStore } from '../../utils/testUtils'; import reducer, { getAliases, + getClustersInfo, getIndices, + getIndicesAndAliases, getMappings, initialState, searchOpenSearch, @@ -52,7 +54,7 @@ describe('opensearch reducer actions', () => { expect(httpMockedClient.get).toHaveBeenCalledWith( `..${BASE_NODE_API_PATH}/_indices`, { - query: { index: '' }, + query: { index: '', clusters: '' }, } ); }); @@ -79,7 +81,7 @@ describe('opensearch reducer actions', () => { expect(httpMockedClient.get).toHaveBeenCalledWith( `..${BASE_NODE_API_PATH}/_indices`, { - query: { index: '' }, + query: { index: '', clusters: '' }, } ); } @@ -175,7 +177,7 @@ describe('opensearch reducer actions', () => { expect(httpMockedClient.get).toHaveBeenCalledWith( `..${BASE_NODE_API_PATH}/_mappings`, { - query: { index: '' }, + query: { indices: [] }, } ); }); @@ -202,7 +204,7 @@ describe('opensearch reducer actions', () => { expect(httpMockedClient.get).toHaveBeenCalledWith( `..${BASE_NODE_API_PATH}/_mappings`, { - query: { index: '' }, + query: { indices: [] }, } ); } @@ -278,5 +280,190 @@ describe('opensearch reducer actions', () => { } }); }); + describe('getIndicesAndAliases', () => { + test('should handle [REQUEST, SUCCESS] actions for getIndicesAndAliases', async () => { + const indices = [ + { index: 'index1', health: 'green' }, + { index: 'index2', health: 'yellow' }, + ]; + const aliases = [ + { alias: 'alias1', index: 'index1' }, + { alias: 'alias2', index: 'index2' }, + ]; + + httpMockedClient.get = jest + .fn() + .mockResolvedValue({ ok: true, response: { indices, aliases } }); + + await store.dispatch(getIndicesAndAliases()); + const actions = store.getActions(); + + expect(actions[0].type).toBe('opensearch/GET_INDICES_AND_ALIASES_REQUEST'); + expect(reducer(initialState, actions[0])).toEqual({ + ...initialState, + requesting: true, + }); + + expect(actions[1].type).toBe('opensearch/GET_INDICES_AND_ALIASES_SUCCESS'); + expect(reducer(initialState, actions[1])).toEqual({ + ...initialState, + requesting: false, + indices, + aliases, + }); + + expect(httpMockedClient.get).toHaveBeenCalledWith( + `..${BASE_NODE_API_PATH}/_indices_and_aliases`, + { + query: { indexOrAliasQuery: '', clusters: '', queryForLocalCluster: true }, + } + ); + }); + test('should handle [REQUEST, SUCCESS] actions for getIndicesAndAliases with clusters', async () => { + const indices = [ + { index: 'index1', health: 'green' }, + { index: 'index2', health: 'yellow' }, + ]; + const aliases = [ + { alias: 'alias1', index: 'index1' }, + { alias: 'alias2', index: 'index2' }, + ]; + + httpMockedClient.get = jest + .fn() + .mockResolvedValue({ ok: true, response: { indices, aliases } }); + + await store.dispatch(getIndicesAndAliases('', '', 'cluster-2,cluster-3')); + const actions = store.getActions(); + + expect(actions[0].type).toBe('opensearch/GET_INDICES_AND_ALIASES_REQUEST'); + expect(reducer(initialState, actions[0])).toEqual({ + ...initialState, + requesting: true, + }); + + expect(actions[1].type).toBe('opensearch/GET_INDICES_AND_ALIASES_SUCCESS'); + expect(reducer(initialState, actions[1])).toEqual({ + ...initialState, + requesting: false, + indices, + aliases, + }); + + expect(httpMockedClient.get).toHaveBeenCalledWith( + `..${BASE_NODE_API_PATH}/_indices_and_aliases`, + { + query: { + indexOrAliasQuery: '', + clusters: 'cluster-2,cluster-3', + queryForLocalCluster: true, + }, + } + ); + }); + test('should handle [REQUEST, FAILURE] actions for getIndicesAndAliases', async () => { + httpMockedClient.get = jest.fn().mockRejectedValue({ + ok: false, + error: 'Something went wrong', + }); + + try { + await store.dispatch(getIndicesAndAliases()); + } catch (e) { + const actions = store.getActions(); + + expect(actions[0].type).toBe('opensearch/GET_INDICES_AND_ALIASES_REQUEST'); + expect(reducer(initialState, actions[0])).toEqual({ + ...initialState, + requesting: true, + errorMessage: '', + }); + + expect(actions[1].type).toBe('opensearch/GET_INDICES_AND_ALIASES_FAILURE'); + expect(reducer(initialState, actions[1])).toEqual({ + ...initialState, + requesting: false, + errorMessage: 'Something went wrong', + }); + + expect(httpMockedClient.get).toHaveBeenCalledWith( + `..${BASE_NODE_API_PATH}/_indices_and_aliases`, + { + query: { + indexOrAliasQuery: '', + clusters: '', + queryForLocalCluster: true, + }, + } + ); + } + }); + }); + describe('getClustersInfo', () => { + test('should invoke [REQUEST, SUCCESS]', async () => { + const clusters = [ + { cluster: 'cluster1', status: 'green' }, + { cluster: 'cluster2', status: 'yellow' }, + ]; + + httpMockedClient.get = jest + .fn() + .mockResolvedValue({ ok: true, response: { clusters } }); + + await store.dispatch(getClustersInfo()); + const actions = store.getActions(); + + expect(actions[0].type).toBe('opensearch/GET_CLUSTERS_INFO_REQUEST'); + expect(reducer(initialState, actions[0])).toEqual({ + ...initialState, + requesting: true, + errorMessage: '', + }); + + expect(actions[1].type).toBe('opensearch/GET_CLUSTERS_INFO_SUCCESS'); + expect(reducer(initialState, actions[1])).toEqual({ + ...initialState, + requesting: false, + clusters, + }); + + expect(httpMockedClient.get).toHaveBeenCalledWith( + `..${BASE_NODE_API_PATH}/_remote/info` + ); + }); + test('should invoke [REQUEST, FAILURE]', async () => { + const errorMessage = 'Something went wrong'; + + httpMockedClient.get = jest.fn().mockRejectedValue({ + ok: false, + error: errorMessage, + }); + + try { + await store.dispatch(getClustersInfo()); + } catch (e) { + const actions = store.getActions(); + + expect(actions[0].type).toBe('opensearch/GET_CLUSTERS_INFO_REQUEST'); + expect(reducer(initialState, actions[0])).toEqual({ + ...initialState, + requesting: true, + errorMessage: '', + }); + + expect(actions[1].type).toBe('opensearch/GET_CLUSTERS_INFO_FAILURE'); + expect(reducer(initialState, actions[1])).toEqual({ + ...initialState, + requesting: false, + errorMessage, + }); + + expect(httpMockedClient.get).toHaveBeenCalledWith( + `..${BASE_NODE_API_PATH}/_remote/info` + ); + } + }); + }); + describe('getPrioritizedIndices', () => {}); }); diff --git a/public/redux/reducers/opensearch.ts b/public/redux/reducers/opensearch.ts index 4a9a3d32..77667b74 100644 --- a/public/redux/reducers/opensearch.ts +++ b/public/redux/reducers/opensearch.ts @@ -18,10 +18,13 @@ import { } from '../middleware/types'; import handleActions from '../utils/handleActions'; import { getPathsPerDataType } from './mapper'; -import { CatIndex, IndexAlias } from '../../../server/models/types'; +import { + CatIndex, + ClusterInfo, + IndexAlias, +} from '../../../server/models/types'; import { AD_NODE_API } from '../../../utils/constants'; import { get } from 'lodash'; -import { data } from 'jquery'; const GET_INDICES = 'opensearch/GET_INDICES'; const GET_ALIASES = 'opensearch/GET_ALIASES'; @@ -30,6 +33,8 @@ const SEARCH_OPENSEARCH = 'opensearch/SEARCH_OPENSEARCH'; const CREATE_INDEX = 'opensearch/CREATE_INDEX'; const BULK = 'opensearch/BULK'; const DELETE_INDEX = 'opensearch/DELETE_INDEX'; +const GET_CLUSTERS_INFO = 'opensearch/GET_CLUSTERS_INFO'; +const GET_INDICES_AND_ALIASES = 'opensearch/GET_INDICES_AND_ALIASES'; export type Mappings = { [key: string]: any; @@ -63,7 +68,9 @@ interface OpenSearchState { requesting: boolean; searchResult: object; errorMessage: string; + clusters: ClusterInfo[]; } + export const initialState: OpenSearchState = { indices: [], aliases: [], @@ -71,10 +78,35 @@ export const initialState: OpenSearchState = { requesting: false, searchResult: {}, errorMessage: '', + clusters: [], }; const reducer = handleActions( { + [GET_INDICES_AND_ALIASES]: { + REQUEST: (state: OpenSearchState): OpenSearchState => { + return { ...state, requesting: true, errorMessage: '' }; + }, + SUCCESS: ( + state: OpenSearchState, + action: APIResponseAction + ): OpenSearchState => { + return { + ...state, + requesting: false, + indices: get(action, 'result.response.indices', []), + aliases: get(action, 'result.response.aliases', []), + }; + }, + FAILURE: ( + state: OpenSearchState, + action: APIErrorAction + ): OpenSearchState => ({ + ...state, + requesting: false, + errorMessage: get(action, 'error.error', action.error), + }), + }, [GET_INDICES]: { REQUEST: (state: OpenSearchState): OpenSearchState => { return { ...state, requesting: true, errorMessage: '' }; @@ -244,18 +276,74 @@ const reducer = handleActions( errorMessage: get(action, 'error.error', action.error), }), }, + [GET_CLUSTERS_INFO]: { + REQUEST: (state: OpenSearchState): OpenSearchState => { + return { ...state, requesting: true, errorMessage: '' }; + }, + SUCCESS: ( + state: OpenSearchState, + action: APIResponseAction + ): OpenSearchState => { + return { + ...state, + requesting: false, + clusters: get(action, 'result.response.clusters', []), + }; + }, + FAILURE: ( + state: OpenSearchState, + action: APIErrorAction + ): OpenSearchState => ({ + ...state, + requesting: false, + errorMessage: get(action, 'error.error', action.error), + }), + }, }, initialState ); -export const getIndices = (searchKey = '', dataSourceId: string = '') => { +export const getIndices = ( + searchKey = '', + dataSourceId: string = '', + givenClusters: string = '' +) => { const baseUrl = `..${AD_NODE_API._INDICES}`; const url = dataSourceId ? `${baseUrl}/${dataSourceId}` : baseUrl; - return { type: GET_INDICES, request: (client: HttpSetup) => - client.get(url, { query: { index: searchKey } }), + client.get(url, { query: { index: searchKey, clusters: givenClusters } }), + }; +}; + +export const getClustersInfo = (dataSourceId: string = ''): APIAction => { + const baseUrl = `..${AD_NODE_API.GET_CLUSTERS_INFO}`; + const url = dataSourceId ? `${baseUrl}/${dataSourceId}` : baseUrl; + return { + type: GET_CLUSTERS_INFO, + request: (client: HttpSetup) => client.get(url), + }; +}; + +export const getIndicesAndAliases = ( + searchKey = '', + dataSourceId: string = '', + givenClusters: string = '', + queryForLocalCluster: boolean = true +): APIAction => { + const baseUrl = `..${AD_NODE_API.GET_INDICES_AND_ALIASES}`; + const url = dataSourceId ? `${baseUrl}/${dataSourceId}` : baseUrl; + return { + type: GET_INDICES_AND_ALIASES, + request: (client: HttpSetup) => + client.get(url, { + query: { + indexOrAliasQuery: searchKey, + clusters: givenClusters, + queryForLocalCluster: queryForLocalCluster, + }, + }), }; }; @@ -273,14 +361,18 @@ export const getAliases = ( }; }; -export const getMappings = (searchKey: string = '', dataSourceId: string = ''): APIAction => { - const url = dataSourceId ? `${AD_NODE_API._MAPPINGS}/${dataSourceId}` : AD_NODE_API._MAPPINGS; - +export const getMappings = ( + searchKey: string[] = [], + dataSourceId: string = '' +): APIAction => { + const url = dataSourceId + ? `${AD_NODE_API._MAPPINGS}/${dataSourceId}` + : AD_NODE_API._MAPPINGS; return { type: GET_MAPPINGS, request: (client: HttpSetup) => client.get(`..${url}`, { - query: { index: searchKey }, + query: { indices: searchKey }, }), }; }; @@ -293,7 +385,10 @@ export const searchOpenSearch = (requestData: any): APIAction => ({ }), }); -export const createIndex = (indexConfig: any, dataSourceId: string = ''): APIAction => { +export const createIndex = ( + indexConfig: any, + dataSourceId: string = '' +): APIAction => { const url = dataSourceId ? `${AD_NODE_API.CREATE_INDEX}/${dataSourceId}` : AD_NODE_API.CREATE_INDEX; @@ -324,11 +419,14 @@ export const deleteIndex = (index: string): APIAction => ({ }); export const getPrioritizedIndices = - (searchKey: string, dataSourceId: string = ''): ThunkAction => + ( + searchKey: string, + dataSourceId: string = '', + clusters: string = '*' + ): ThunkAction => async (dispatch, getState) => { //Fetch Indices and Aliases with text provided - await dispatch(getIndices(searchKey, dataSourceId)); - await dispatch(getAliases(searchKey, dataSourceId)); + await dispatch(getIndicesAndAliases(searchKey, dataSourceId, clusters)); const osState = getState().opensearch; const exactMatchedIndices = osState.indices; const exactMatchedAliases = osState.aliases; @@ -340,8 +438,9 @@ export const getPrioritizedIndices = }; } else { //No results found for exact match, append wildCard and get partial matches if exists - await dispatch(getIndices(`${searchKey}*`, dataSourceId)); - await dispatch(getAliases(`${searchKey}*`, dataSourceId)); + await dispatch( + getIndicesAndAliases(`${searchKey}*`, dataSourceId, clusters) + ); const osState = getState().opensearch; const partialMatchedIndices = osState.indices; const partialMatchedAliases = osState.aliases; diff --git a/server/models/types.ts b/server/models/types.ts index c1d679b0..ae6d40c1 100644 --- a/server/models/types.ts +++ b/server/models/types.ts @@ -14,13 +14,31 @@ import { SORT_DIRECTION, DETECTOR_STATE } from '../utils/constants'; export type CatIndex = { index: string; health: string; + localCluster?: boolean; }; +export type ClusterInfo = { + name: string; + localCluster: boolean; +} + export type IndexAlias = { - index: string; + index: string[] | string; alias: string; + localCluster?: boolean }; +export type IndexOption = { + label: string, + health: string, + localCluster?: boolean +} + +export type AliasOption = { + label: string, + localCluster?: string +} + export type GetAliasesResponse = { aliases: IndexAlias[]; }; diff --git a/server/routes/opensearch.ts b/server/routes/opensearch.ts index 3a8444d9..65f1e3a7 100644 --- a/server/routes/opensearch.ts +++ b/server/routes/opensearch.ts @@ -13,6 +13,7 @@ import { get } from 'lodash'; import { SearchResponse } from '../models/interfaces'; import { CatIndex, + ClusterInfo, GetAliasesResponse, GetIndicesResponse, GetMappingResponse, @@ -28,6 +29,10 @@ import { IOpenSearchDashboardsResponse, } from '../../../../src/core/server'; import { getClientBasedOnDataSource } from '../utils/helpers'; +import { CatAliases } from '@opensearch-project/opensearch/api/requestParams'; +import _ from 'lodash'; +import { Mappings } from 'public/redux/reducers/opensearch'; +import { convertFieldCapsToMappingStructure } from './utils/opensearchHelpers'; type SearchParams = { index: string; @@ -57,6 +62,20 @@ export function registerOpenSearchRoutes( apiRouter.post('/bulk/{dataSourceId}', opensearchService.bulk); apiRouter.post('/delete_index', opensearchService.deleteIndex); + apiRouter.get('/_remote/info', opensearchService.getClustersInfo); + apiRouter.get('/_remote/info/', opensearchService.getClustersInfo); + apiRouter.get( + '/_remote/info/{dataSourceId}', + opensearchService.getClustersInfo + ); + apiRouter.get( + '/_indices_and_aliases', + opensearchService.getIndicesAndAliases + ); + apiRouter.get( + '/_indices_and_aliases/{dataSourceId}', + opensearchService.getIndicesAndAliases + ); } export default class OpenSearchService { @@ -125,7 +144,10 @@ export default class OpenSearchService { request: OpenSearchDashboardsRequest, opensearchDashboardsResponse: OpenSearchDashboardsResponseFactory ): Promise> => { - const { index } = request.query as { index: string }; + const { index, clusters } = request.query as { + index: string; + clusters: string; + }; const { dataSourceId = '' } = request.params as { dataSourceId?: string }; try { const callWithRequest = getClientBasedOnDataSource( @@ -135,12 +157,41 @@ export default class OpenSearchService { dataSourceId, this.client ); + let indices: CatIndex[] = []; + let resolve_resp; - const response: CatIndex[] = await callWithRequest('cat.indices', { + let response: CatIndex[] = await callWithRequest('cat.indices', { index, format: 'json', h: 'health,index', }); + response = response.map((item) => ({ + ...item, + localCluster: true, + })); + + // only call cat indices + if (clusters != '') { + if (index == '') { + resolve_resp = await callWithRequest('transport.request', { + method: 'GET', + path: '/_resolve/index/' + clusters + ':*', + }); + } else { + resolve_resp = await callWithRequest('transport.request', { + method: 'GET', + path: '/_resolve/index/' + clusters + ':' + index, + }); + } + indices = resolve_resp.indices.map((item) => ({ + index: item.name, + format: 'json', + health: 'undefined', + localCluster: false, + })); + + response = response.concat(indices); + } return opensearchDashboardsResponse.ok({ body: { ok: true, response: { indices: response } }, @@ -338,7 +389,11 @@ export default class OpenSearchService { request: OpenSearchDashboardsRequest, opensearchDashboardsResponse: OpenSearchDashboardsResponseFactory ): Promise> => { - const { index } = request.query as { index: string }; + let { indices } = request.query as { indices: string[] }; + // If indices is not an array, convert it to an array, server framework auto converts single item in string array to a string + if (!Array.isArray(indices)) { + indices = [indices]; + } const { dataSourceId = '' } = request.params as { dataSourceId?: string }; try { @@ -350,12 +405,34 @@ export default class OpenSearchService { this.client ); - const response = await callWithRequest( - 'indices.getMapping', { - index, + let mappings: Mappings = {}; + let remoteMappings: Mappings = {}; + let localIndices: string[] = indices.filter( + (index: string) => !index.includes(':') + ); + let remoteIndices: string[] = indices.filter((index: string) => + index.includes(':') + ); + + if (localIndices.length > 0) { + mappings = await callWithRequest('indices.getMapping', { + index: localIndices, + }); + } + + // make call to fields_caps + if (remoteIndices.length) { + const fieldCapsResponse = await callWithRequest('transport.request', { + method: 'GET', + path: + remoteIndices.toString() + '/_field_caps?fields=*&include_unmapped', }); + remoteMappings = convertFieldCapsToMappingStructure(fieldCapsResponse); + } + Object.assign(mappings, remoteMappings); + return opensearchDashboardsResponse.ok({ - body: { ok: true, response: { mappings: response } }, + body: { ok: true, response: { mappings: mappings } }, }); } catch (err) { console.log('Anomaly detector - Unable to get mappings', err); @@ -367,4 +444,162 @@ export default class OpenSearchService { }); } }; + + // we use this to retrieve indices and aliases from both the local cluster and remote clusters + // 3 different OS APIs are called here, _cat/indices, _cat/aliases and _resolve/index + getIndicesAndAliases = async ( + context: RequestHandlerContext, + request: OpenSearchDashboardsRequest, + opensearchDashboardsResponse: OpenSearchDashboardsResponseFactory + ): Promise> => { + const { indexOrAliasQuery, clusters, queryForLocalCluster } = + request.query as { + indexOrAliasQuery: string; + clusters: string; + queryForLocalCluster: string; + }; + const { dataSourceId = '' } = request.params as { dataSourceId?: string }; + try { + const callWithRequest = getClientBasedOnDataSource( + context, + this.dataSourceEnabled, + request, + dataSourceId, + this.client + ); + let indicesResponse: CatIndex[] = []; + let aliasesResponse: IndexAlias[] = []; + if (queryForLocalCluster == 'true') { + indicesResponse = await callWithRequest('cat.indices', { + index: indexOrAliasQuery, + format: 'json', + h: 'health,index', + }); + indicesResponse = indicesResponse.map((item) => ({ + ...item, + localCluster: true, + })); + aliasesResponse = await callWithRequest('cat.aliases', { + alias: indexOrAliasQuery, + format: 'json', + h: 'alias,index', + }); + + aliasesResponse = aliasesResponse.map((item) => ({ + ...item, + localCluster: true, + })); + } + + // only call cat indices and cat aliases + if (clusters != '') { + let remoteIndices: CatIndex[] = []; + let remoteAliases: IndexAlias[] = []; + let resolveResponse; + const resolveIndexQuery = + indexOrAliasQuery == '' + ? clusters + .split(',') + .map((cluster) => `${cluster}:*`) + .join(',') + : clusters + .split(',') + .map((cluster) => `${cluster}:${indexOrAliasQuery}`) + .join(','); + resolveResponse = await callWithRequest('transport.request', { + method: 'GET', + path: '/_resolve/index/' + resolveIndexQuery, + }); + remoteIndices = resolveResponse.indices.map((item) => ({ + index: item.name, + format: 'json', + health: 'undefined', + localCluster: false, + })); + + remoteAliases = resolveResponse.aliases.map((item) => ({ + alias: item.name, + index: item.indices, + format: 'json', + localCluster: false, + })); + indicesResponse = indicesResponse.concat(remoteIndices); + aliasesResponse = aliasesResponse.concat(remoteAliases); + } + + return opensearchDashboardsResponse.ok({ + body: { + ok: true, + response: { aliases: aliasesResponse, indices: indicesResponse }, + }, + }); + } catch (err) { + // In case no matching indices is found it throws an error. + if ( + err.statusCode === 404 && + get(err, 'body.error.type', '') === 'index_not_found_exception' + ) { + return opensearchDashboardsResponse.ok({ + body: { ok: true, response: { indices: [], aliases: [] } }, + }); + } + console.log('Anomaly detector - Unable to get indices and aliases', err); + return opensearchDashboardsResponse.ok({ + body: { + ok: false, + error: getErrorMessage(err), + }, + }); + } + }; + + getClustersInfo = async ( + context: RequestHandlerContext, + request: OpenSearchDashboardsRequest, + opensearchDashboardsResponse: OpenSearchDashboardsResponseFactory + ): Promise> => { + const { dataSourceId = '' } = request.params as { dataSourceId?: string }; + try { + const callWithRequest = getClientBasedOnDataSource( + context, + this.dataSourceEnabled, + request, + dataSourceId, + this.client + ); + + let clustersResponse: ClusterInfo[] = []; + + const remoteInfo = await callWithRequest('transport.request', { + method: 'GET', + path: '/_remote/info', + }); + clustersResponse = Object.keys(remoteInfo).map((key) => ({ + name: key, + localCluster: false, + })); + + const clusterHealth = await callWithRequest('cat.health', { + format: 'json', + h: 'cluster', + }); + + clustersResponse.push({ + name: clusterHealth[0].cluster, + localCluster: true, + }); + + return opensearchDashboardsResponse.ok({ + body: { ok: true, response: { clusters: clustersResponse } }, + }); + } catch (err) { + console.error('Alerting - OpensearchService - getClusterHealth:', err); + return opensearchDashboardsResponse.ok({ + body: { + ok: false, + error: getErrorMessage(err), + }, + }); + } + }; } diff --git a/server/routes/utils/__tests__/opensearchHelpers.test.ts b/server/routes/utils/__tests__/opensearchHelpers.test.ts new file mode 100644 index 00000000..317a8fda --- /dev/null +++ b/server/routes/utils/__tests__/opensearchHelpers.test.ts @@ -0,0 +1,84 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +import { convertFieldCapsToMappingStructure } from '../opensearchHelpers'; + +describe('transformFieldCapsResponse', () => { + test('transform the field capabilities response into a structured mapping object', () => { + const fieldCapsResponse = { + indices: [ + 'opensearch-ccs-cluster2:host-health-us-west-1', + 'opensearch-ccs-cluster2:sample-ecommerce', + ], + fields: { + _routing: { + _routing: { type: '_routing', searchable: true, aggregatable: false }, + }, + _doc_count: { + long: { type: 'long', searchable: false, aggregatable: false }, + }, + total_revenue_usd: { + integer: { + type: 'integer', + searchable: true, + aggregatable: true, + indices: ['opensearch-ccs-cluster2:sample-ecommerce'], + }, + unmapped: { + type: 'unmapped', + searchable: false, + aggregatable: false, + indices: ['opensearch-ccs-cluster2:host-health-us-west-1'], + }, + }, + cpu_usage_percentage: { + integer: { + type: 'integer', + searchable: true, + aggregatable: true, + indices: ['opensearch-ccs-cluster2:host-health-us-west-1'], + }, + unmapped: { + type: 'unmapped', + searchable: false, + aggregatable: false, + indices: ['opensearch-ccs-cluster2:sample-ecommerce'], + }, + }, + timestamp: { + date: { type: 'date', searchable: true, aggregatable: true }, + }, + }, + }; + + const expectedOutput = { + 'opensearch-ccs-cluster2:host-health-us-west-1': { + mappings: { + properties: { + cpu_usage_percentage: { type: 'integer' }, + timestamp: { type: 'date' }, + }, + }, + }, + 'opensearch-ccs-cluster2:sample-ecommerce': { + mappings: { + properties: { + total_revenue_usd: { type: 'integer' }, + timestamp: { type: 'date' }, + }, + }, + }, + }; + + const result = convertFieldCapsToMappingStructure(fieldCapsResponse); + expect(result).toEqual(expectedOutput); + }); +}); diff --git a/server/routes/utils/opensearchHelpers.ts b/server/routes/utils/opensearchHelpers.ts new file mode 100644 index 00000000..ae03dcab --- /dev/null +++ b/server/routes/utils/opensearchHelpers.ts @@ -0,0 +1,47 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ +import _ from 'lodash'; +import { Mappings } from 'public/redux/reducers/opensearch'; + +export const convertFieldCapsToMappingStructure = (fieldCapsResponse) => { + let mappings: Mappings = {}; + + fieldCapsResponse.indices.forEach((index) => { + mappings[index] = { + mappings: { + properties: {}, + }, + }; + }); + for (const [fieldName, fieldDetails] of Object.entries( + fieldCapsResponse.fields + )) { + if (fieldName.startsWith('_')) { + continue; + } + for (const [fieldType, typeDetails] of Object.entries(fieldDetails)) { + if (fieldType == 'unmapped') { + continue; + } + let mapped_indices = _.get( + typeDetails, + 'indices', + fieldCapsResponse.indices + ); + mapped_indices.forEach((mappedIndex) => { + mappings[mappedIndex]['mappings']['properties'][fieldName] = { + type: typeDetails.type, + }; + }); + } + } + return mappings; +}; diff --git a/utils/constants.ts b/utils/constants.ts index 231bd91b..639106cf 100644 --- a/utils/constants.ts +++ b/utils/constants.ts @@ -21,6 +21,8 @@ export const AD_NODE_API = Object.freeze({ BULK: `${BASE_NODE_API_PATH}/bulk`, DELETE_INDEX: `${BASE_NODE_API_PATH}/delete_index`, CREATE_SAMPLE_DATA: `${BASE_NODE_API_PATH}/create_sample_data`, + GET_CLUSTERS_INFO: `${BASE_NODE_API_PATH}/_remote/info`, + GET_INDICES_AND_ALIASES: `${BASE_NODE_API_PATH}/_indices_and_aliases`, }); export const ALERTING_NODE_API = Object.freeze({ _SEARCH: `${BASE_NODE_API_PATH}/monitors/_search`,