diff --git a/public/components/FeatureAnywhereContextMenu/CreateAnomalyDetector/AddAnomalyDetector.tsx b/public/components/FeatureAnywhereContextMenu/CreateAnomalyDetector/AddAnomalyDetector.tsx
index a7486cf1..58c280ff 100644
--- a/public/components/FeatureAnywhereContextMenu/CreateAnomalyDetector/AddAnomalyDetector.tsx
+++ b/public/components/FeatureAnywhereContextMenu/CreateAnomalyDetector/AddAnomalyDetector.tsx
@@ -134,16 +134,23 @@ function AddAnomalyDetector({
>();
const indexPatternId = embeddable.vis.data.aggs.indexPattern.id;
- const [dataSourceId, setDataSourceId] = useState(undefined);
+ const [dataSourceId, setDataSourceId] = useState(
+ undefined
+ );
async function getDataSourceId() {
try {
- const indexPattern = await getSavedObjectsClient().get('index-pattern', indexPatternId);
+ const indexPattern = await getSavedObjectsClient().get(
+ 'index-pattern',
+ indexPatternId
+ );
const refs = indexPattern.references as References[];
- const foundDataSourceId = refs.find(ref => ref.type === 'data-source')?.id;
- setDataSourceId(foundDataSourceId);
+ const foundDataSourceId = refs.find(
+ (ref) => ref.type === 'data-source'
+ )?.id;
+ setDataSourceId(foundDataSourceId);
} catch (error) {
- console.error("Error fetching index pattern:", error);
+ console.error('Error fetching index pattern:', error);
}
}
@@ -152,8 +159,12 @@ function AddAnomalyDetector({
async function fetchData() {
await getDataSourceId();
- const getIndicesDispatchCall = dispatch(getIndices(queryText, dataSourceId));
- const getMappingDispatchCall = dispatch(getMappings(embeddable.vis.data.aggs.indexPattern.title, dataSourceId));
+ const getIndicesDispatchCall = dispatch(
+ getIndices(queryText, dataSourceId)
+ );
+ const getMappingDispatchCall = dispatch(
+ getMappings([embeddable.vis.data.aggs.indexPattern.title], dataSourceId)
+ );
await Promise.all([getIndicesDispatchCall, getMappingDispatchCall]);
}
@@ -167,7 +178,7 @@ function AddAnomalyDetector({
}
fetchData();
createEmbeddable();
- }, [dataSourceId]);
+ }, [dataSourceId]);
const [isShowVis, setIsShowVis] = useState(false);
const [accordionsOpen, setAccordionsOpen] = useState({ modelFeatures: true });
@@ -335,7 +346,7 @@ function AddAnomalyDetector({
name: OVERLAY_ANOMALIES,
args: {
detectorId: detectorId,
- dataSourceId: dataSourceId
+ dataSourceId: dataSourceId,
},
} as VisLayerExpressionFn;
diff --git a/public/pages/ConfigureModel/components/Features/Features.tsx b/public/pages/ConfigureModel/components/Features/Features.tsx
index 2464688c..af735802 100644
--- a/public/pages/ConfigureModel/components/Features/Features.tsx
+++ b/public/pages/ConfigureModel/components/Features/Features.tsx
@@ -65,16 +65,6 @@ export function Features(props: FeaturesProps) {
{({ push, remove, form: { values } }: FieldArrayRenderProps) => {
return (
- {get(props.detector, 'indices.0', '').includes(':') ? (
-
-
-
-
- ) : null}
{values.featureList.map((feature: any, index: number) => (
{
diff --git a/public/pages/ConfigureModel/containers/ConfigureModel.tsx b/public/pages/ConfigureModel/containers/ConfigureModel.tsx
index cfd18338..c7867869 100644
--- a/public/pages/ConfigureModel/containers/ConfigureModel.tsx
+++ b/public/pages/ConfigureModel/containers/ConfigureModel.tsx
@@ -30,7 +30,11 @@ import { RouteComponentProps, useLocation } from 'react-router-dom';
import { AppState } from '../../../redux/reducers';
import { getMappings } from '../../../redux/reducers/opensearch';
import { useFetchDetectorInfo } from '../../CreateDetectorSteps/hooks/useFetchDetectorInfo';
-import { BREADCRUMBS, BASE_DOCS_LINK, MDS_BREADCRUMBS } from '../../../utils/constants';
+import {
+ BREADCRUMBS,
+ BASE_DOCS_LINK,
+ MDS_BREADCRUMBS,
+} from '../../../utils/constants';
import { useHideSideNavBar } from '../../main/hooks/useHideSideNavBar';
import { updateDetector } from '../../../redux/reducers/ad';
import {
@@ -121,7 +125,7 @@ export function ConfigureModel(props: ConfigureModelProps) {
setIsHCDetector(true);
}
if (detector?.indices) {
- dispatch(getMappings(detector.indices[0], dataSourceId));
+ dispatch(getMappings(detector.indices, dataSourceId));
}
}, [detector]);
@@ -133,7 +137,11 @@ export function ConfigureModel(props: ConfigureModelProps) {
MDS_BREADCRUMBS.DETECTORS(dataSourceId),
{
text: detector && detector.name ? detector.name : '',
- href: constructHrefWithDataSourceId(`#/detectors/${detectorId}`, dataSourceId, false)
+ href: constructHrefWithDataSourceId(
+ `#/detectors/${detectorId}`,
+ dataSourceId,
+ false
+ ),
},
MDS_BREADCRUMBS.EDIT_MODEL_CONFIGURATION,
]);
@@ -167,12 +175,11 @@ export function ConfigureModel(props: ConfigureModelProps) {
useEffect(() => {
if (hasError) {
- if(dataSourceEnabled) {
+ if (dataSourceEnabled) {
props.history.push(
constructHrefWithDataSourceId('/detectors', dataSourceId, false)
);
- }
- else {
+ } else {
props.history.push('/detectors');
}
}
diff --git a/public/pages/CreateDetectorSteps/hooks/useFetchDetectorInfo.ts b/public/pages/CreateDetectorSteps/hooks/useFetchDetectorInfo.ts
index a686775b..1d00ba18 100644
--- a/public/pages/CreateDetectorSteps/hooks/useFetchDetectorInfo.ts
+++ b/public/pages/CreateDetectorSteps/hooks/useFetchDetectorInfo.ts
@@ -10,7 +10,7 @@
*/
import { get, isEmpty } from 'lodash';
-import { useEffect } from 'react';
+import { useEffect, useMemo } from 'react';
import { useDispatch, useSelector } from 'react-redux';
import { Detector } from '../../../models/interfaces';
import { AppState } from '../../../redux/reducers';
@@ -40,13 +40,13 @@ export const useFetchDetectorInfo = (
const isIndicesRequesting = useSelector(
(state: AppState) => state.opensearch.requesting
);
- const selectedIndices = get(detector, 'indices.0', '');
+ const selectedIndices = useMemo(() => get(detector, 'indices', []), [detector]);
useEffect(() => {
const fetchDetector = async () => {
if (!detector) {
await dispatch(getDetector(detectorId, dataSourceId));
}
- if (selectedIndices) {
+ if (selectedIndices && selectedIndices.length > 0) {
await dispatch(getMappings(selectedIndices, dataSourceId));
}
};
diff --git a/public/pages/DefineDetector/components/DataFilterList/DataFilterList.tsx b/public/pages/DefineDetector/components/DataFilterList/DataFilterList.tsx
index e0c5c5a1..3e6b4911 100644
--- a/public/pages/DefineDetector/components/DataFilterList/DataFilterList.tsx
+++ b/public/pages/DefineDetector/components/DataFilterList/DataFilterList.tsx
@@ -15,7 +15,6 @@ import {
EuiSpacer,
EuiIcon,
EuiButtonEmpty,
- EuiCallOut,
} from '@elastic/eui';
import { FieldArray, FieldArrayRenderProps, FormikProps } from 'formik';
import React, { useState, Fragment } from 'react';
@@ -38,9 +37,6 @@ export const DataFilterList = (props: DataFilterListProps) => {
const [isCreatingNewFilter, setIsCreatingNewFilter] =
useState(false);
- const selectedIndex = get(props, 'formikProps.values.index.0.label', '');
- const isRemoteIndex = selectedIndex.includes(':');
-
return (
{({ push, remove, replace, form: { values } }: FieldArrayRenderProps) => {
@@ -66,18 +62,6 @@ export const DataFilterList = (props: DataFilterListProps) => {
>
- {isRemoteIndex ? (
-
-
-
-
- ) : null}
{values.filters?.length === 0 ||
diff --git a/public/pages/DefineDetector/components/DataFilterList/components/SimpleFilter.tsx b/public/pages/DefineDetector/components/DataFilterList/components/SimpleFilter.tsx
index bc13e09a..89bd35ab 100644
--- a/public/pages/DefineDetector/components/DataFilterList/components/SimpleFilter.tsx
+++ b/public/pages/DefineDetector/components/DataFilterList/components/SimpleFilter.tsx
@@ -32,6 +32,7 @@ import { getIndexFields, getOperators, isNullOperator } from '../utils/helpers';
import FilterValue from './FilterValue';
import { DetectorDefinitionFormikValues } from '../../../models/interfaces';
import { EMPTY_UI_FILTER } from '../../../utils/constants';
+import _ from 'lodash';
interface SimpleFilterProps {
filter: UIFilter;
@@ -40,8 +41,20 @@ interface SimpleFilterProps {
replace(index: number, value: any): void;
}
+// This sorting is needed because we utilize two different ways to get index fields,
+// through get mapping call and through field_caps API for remote indices
+const sortByLabel = (indexFields) => {
+ //sort the `options` array inside each object by the `label` field
+ indexFields.forEach(item => {
+ item.options = _.sortBy(item.options, 'label');
+ });
+ //sort the outer array by the `label` field
+ return _.sortBy(indexFields, 'label');
+};
+
export const SimpleFilter = (props: SimpleFilterProps) => {
- const indexFields = getIndexFields(useSelector(getAllFields));
+ let indexFields = getIndexFields(useSelector(getAllFields));
+ indexFields = sortByLabel(indexFields)
const [searchedIndexFields, setSearchedIndexFields] = useState<
({
label: DATA_TYPES;
diff --git a/public/pages/DefineDetector/components/Datasource/DataSource.tsx b/public/pages/DefineDetector/components/Datasource/DataSource.tsx
index 7ec60940..ed294f0f 100644
--- a/public/pages/DefineDetector/components/Datasource/DataSource.tsx
+++ b/public/pages/DefineDetector/components/Datasource/DataSource.tsx
@@ -12,13 +12,20 @@
import { EuiCompressedComboBox, EuiCallOut, EuiSpacer } from '@elastic/eui';
import { Field, FieldProps, FormikProps, useFormikContext } from 'formik';
import { debounce, get } from 'lodash';
-import React, { useEffect, useState } from 'react';
+import React, { useEffect, useMemo, useState } from 'react';
import { useDispatch, useSelector } from 'react-redux';
-import { CatIndex, IndexAlias } from '../../../../../server/models/types';
+import {
+ CatIndex,
+ ClusterInfo,
+ IndexAlias,
+} from '../../../../../server/models/types';
import ContentPanel from '../../../../components/ContentPanel/ContentPanel';
import { AppState } from '../../../../redux/reducers';
import {
+ getAliases,
+ getClustersInfo,
getIndices,
+ getIndicesAndAliases,
getMappings,
getPrioritizedIndices,
} from '../../../../redux/reducers/opensearch';
@@ -26,6 +33,7 @@ import { getError, isInvalid } from '../../../../utils/utils';
import { IndexOption } from './IndexOption';
import {
getDataSourceFromURL,
+ getLocalCluster,
getVisibleOptions,
sanitizeSearchText,
} from '../../../utils/helpers';
@@ -37,10 +45,13 @@ import { ModelConfigurationFormikValues } from '../../../ConfigureModel/models/i
import { INITIAL_MODEL_CONFIGURATION_VALUES } from '../../../ConfigureModel/utils/constants';
import { FILTER_TYPES } from '../../../../models/interfaces';
import { useLocation } from 'react-router-dom';
+import _ from 'lodash';
+import { cleanString } from '../../../../../../../src/plugins/vis_type_vega/public/expressions/helpers';
+import { L } from '../../../../../../../src/plugins/maps_legacy/public/lazy_load_bundle/lazy';
interface DataSourceProps {
formikProps: FormikProps;
- origIndex: string;
+ origIndex: { label: string }[];
isEdit: boolean;
setModelConfigValues?(initialValues: ModelConfigurationFormikValues): void;
setNewIndexSelected?(isNew: boolean): void;
@@ -48,47 +59,132 @@ interface DataSourceProps {
oldFilterQuery: any;
}
+interface ClusterOption {
+ label: string;
+ cluster: string;
+ localcluster: string;
+}
+
export function DataSource(props: DataSourceProps) {
const dispatch = useDispatch();
const location = useLocation();
const MDSQueryParams = getDataSourceFromURL(location);
const dataSourceId = MDSQueryParams.dataSourceId;
- const [indexName, setIndexName] = useState(
- props.formikProps.values.index[0]?.label
+ const [indexNames, setIndexNames] = useState<{ label: string }[]>(
+ props.formikProps.values.index
);
const [queryText, setQueryText] = useState('');
const opensearchState = useSelector((state: AppState) => state.opensearch);
const { setFieldValue } = useFormikContext();
+ const [localClusterName, setLocalClusterName] = useState('');
useEffect(() => {
- const getInitialIndices = async () => {
- await dispatch(getIndices(queryText, dataSourceId));
- setFieldValue('index', props.formikProps.values.index);
- setFieldValue('timeField', props.formikProps.values.timeField);
- setFieldValue('filters', props.formikProps.values.filters);
+ const getInitialClusters = async () => {
+ await dispatch(getClustersInfo(dataSourceId));
+ setFieldValue('clusters', props.formikProps.values.clusters);
};
- getInitialIndices();
+ getInitialClusters();
}, [dataSourceId]);
+ const getIndicesAndAliasesBasedOnCluster = async (
+ clusters: ClusterOption[],
+ localClusterExists: boolean
+ ) => {
+ //Convert list of clusters to a string for searching
+ const clustersString = getClustersStringForSearchQuery(clusters);
+ await dispatch(
+ getIndicesAndAliases(
+ queryText,
+ dataSourceId,
+ clustersString,
+ localClusterExists
+ )
+ );
+ setFieldValue('index', props.formikProps.values.index);
+ setFieldValue('timeField', props.formikProps.values.timeField);
+ setFieldValue('filters', props.formikProps.values.filters);
+ };
+
useEffect(() => {
- setIndexName(props.formikProps.values.index[0]?.label);
- }, [props.formikProps]);
+ // Ensure that indices are updated when clusters change
+ if (props.formikProps.values.clusters) {
+ const selectedClusters: ClusterOption[] =
+ props.formikProps.values.clusters;
+ if (selectedClusters && selectedClusters.length > 0) {
+ const localClusterExists: boolean = selectedClusters.some(
+ (cluster) => cluster.localcluster === 'true'
+ );
+ if (
+ selectedClusters.length === 1 &&
+ selectedClusters[0].localcluster === 'true'
+ ) {
+ getIndicesAndAliasesBasedOnCluster([], localClusterExists);
+ setLocalClusterName(selectedClusters[0].cluster);
+ } else {
+ getIndicesAndAliasesBasedOnCluster(
+ selectedClusters,
+ localClusterExists
+ );
+ }
+ }
+ }
+ }, [props.formikProps.values.clusters]);
+
+ const getClustersStringForSearchQuery = (clusters: ClusterOption[]) => {
+ let clustersString = '';
+ if (clusters.length > 0) {
+ clustersString = clusters
+ .filter((cluster) => cluster.localcluster == 'false')
+ .map((cluster) => cluster.cluster)
+ .join(',');
+ }
+ return clustersString;
+ };
+
+ useEffect(() => {
+ if (opensearchState.clusters && opensearchState.clusters.length > 0) {
+ const localCluster: ClusterInfo[] = getLocalCluster(
+ opensearchState.clusters
+ );
+ setFieldValue('clusters', getVisibleClusterOptions(localCluster));
+ }
+ }, [opensearchState.clusters]);
+
+ const getClusterOptionLabel = (clusterInfo: ClusterInfo) =>
+ `${clusterInfo.name} ${clusterInfo.localCluster ? '(Local)' : '(Remote)'}`;
+
+ useEffect(() => {
+ setIndexNames(props.formikProps.values.index);
+ }, [props.formikProps.values.index]);
const handleSearchChange = debounce(async (searchValue: string) => {
if (searchValue !== queryText) {
const sanitizedQuery = sanitizeSearchText(searchValue);
setQueryText(sanitizedQuery);
- await dispatch(getPrioritizedIndices(sanitizedQuery, dataSourceId));
+ if (props.formikProps.values.clusters) {
+ const selectedClusters: ClusterOption[] =
+ props.formikProps.values.clusters;
+ const clustersString =
+ getClustersStringForSearchQuery(selectedClusters);
+ await dispatch(
+ getPrioritizedIndices(sanitizedQuery, dataSourceId, clustersString)
+ );
+ } else {
+ await dispatch(getPrioritizedIndices(sanitizedQuery, dataSourceId, ''));
+ }
}
}, 300);
- const handleIndexNameChange = (selectedOptions: any) => {
- const indexName = get(selectedOptions, '0.label', '');
- setIndexName(indexName);
- if (indexName !== '') {
- dispatch(getMappings(indexName, dataSourceId));
+ const handleIndexNameChange = (selectedOptions: any, oldOptions: { label: string }[] = props.formikProps.values.index) => {
+ const indexNames = selectedOptions;
+ setIndexNames(indexNames);
+ if (indexNames.length > 0) {
+ const indices: string[] = indexNames.map(
+ (index: { label: string }) => index.label
+ );
+ dispatch(getMappings(indices, dataSourceId));
}
- if (indexName !== props.origIndex) {
+ if (isSelectedOptionIndexRemoved(selectedOptions, oldOptions)) {
if (props.setNewIndexSelected) {
props.setNewIndexSelected(true);
}
@@ -99,16 +195,47 @@ export function DataSource(props: DataSourceProps) {
}
};
- const isDifferentIndex = () => {
- return props.isEdit && indexName !== props.origIndex;
+ const isSelectedOptionIndexRemoved = (
+ newSelectedOptions: { label: string }[] = indexNames,
+ oldSelectedOptions: { label: string }[] = props.formikProps.values.index
+ ) => {
+ if (_.isEmpty(oldSelectedOptions) && _.isEmpty(newSelectedOptions)) {
+ return false;
+ }
+ const newSelectedOptionsSet = new Set(newSelectedOptions);
+ const indexRemoved: boolean =
+ oldSelectedOptions.some((value) => !newSelectedOptionsSet.has(value));
+ return indexRemoved;
+ };
+
+ const getVisibleClusterOptions = (
+ clusters: ClusterInfo[]
+ ): ClusterOption[] => {
+ if (clusters.length > 0) {
+ const visibleClusters = clusters.map((value) => ({
+ label: getClusterOptionLabel(value),
+ cluster: value.name,
+ localcluster: value.localCluster.toString(),
+ }));
+ // Using _.orderBy to sort clusters with local clusters first
+ const sortedClusterOptions = _.orderBy(
+ visibleClusters,
+ [(option) => option.label.endsWith('(Local)'), 'label'],
+ ['desc', 'asc']
+ );
+ return sortedClusterOptions;
+ } else {
+ return [];
+ }
};
+ const visibleClusters = get(opensearchState, 'clusters', []) as ClusterInfo[];
const visibleIndices = get(opensearchState, 'indices', []) as CatIndex[];
const visibleAliases = get(opensearchState, 'aliases', []) as IndexAlias[];
return (
- {props.isEdit && isDifferentIndex() ? (
+ {props.isEdit && isSelectedOptionIndexRemoved() ? (
) : null}
+
+ {({ field, form }: FieldProps) => (
+
+ {
+ form.setFieldTouched('clusters', true);
+ }}
+ onChange={(options) => {
+ form.setFieldValue('clusters', options);
+ }}
+ />
+
+ )}
+
{({ field, form }: FieldProps) => {
return (
{
const normalizedOptions = createdOption.trim();
@@ -151,15 +310,17 @@ export function DataSource(props: DataSourceProps) {
form.setFieldValue('index', options);
form.setFieldValue('timeField', undefined);
form.setFieldValue('filters', []);
- if (props.setModelConfigValues) {
+ if (
+ props.setModelConfigValues &&
+ isSelectedOptionIndexRemoved(options, field.value)
+ ) {
props.setModelConfigValues(
INITIAL_MODEL_CONFIGURATION_VALUES
);
}
- handleIndexNameChange(options);
+ handleIndexNameChange(options, field.value);
}}
selectedOptions={field.value}
- singleSelection={{ asPlainText: true }}
isClearable={false}
renderOption={(option, searchValue, className) => (
state.opensearch);
- const selectedIndex = get(props, 'formikProps.values.index.0.label', '');
- const isRemoteIndex = selectedIndex.includes(':');
const [queryText, setQueryText] = useState('');
const handleSearchChange = debounce(async (searchValue: string) => {
@@ -68,17 +66,6 @@ export function Timestamp(props: TimestampProps) {
titleSize="s"
subTitle="Select the time field you want to use for the time filter."
>
- {isRemoteIndex ? (
-
-
-
-
- ) : null}
{({ field, form }: FieldProps) => (
{
Full creating detector definition renders the compon
>
+
@@ -255,7 +361,7 @@ exports[` Full creating detector definition renders the compon
class="euiText euiText--medium sublabel"
style="max-width: 400px;"
>
- Choose an index or index pattern as the data source.
+ Choose an index, index pattern or alias as the data source.
@@ -278,7 +384,7 @@ exports[` Full creating detector definition renders the compon
class="euiFormControlLayout__childrenWrapper"
>
@@ -1257,7 +1470,7 @@ exports[` empty creating detector definition renders the compo
class="euiFormControlLayout__childrenWrapper"
>
@@ -1435,7 +1437,9 @@ exports[`issue in detector validation issues in feature query 1`] = `
>
+ >
+ -
+
diff --git a/public/pages/utils/__tests__/helpers.test.ts b/public/pages/utils/__tests__/helpers.test.ts
index e69a9a89..1949fbeb 100644
--- a/public/pages/utils/__tests__/helpers.test.ts
+++ b/public/pages/utils/__tests__/helpers.test.ts
@@ -9,10 +9,10 @@
* GitHub history for details.
*/
-import { getVisibleOptions, sanitizeSearchText } from '../helpers';
+import { getVisibleOptions, groupIndicesOrAliasesByCluster, sanitizeSearchText } from '../helpers';
describe('helpers', () => {
describe('getVisibleOptions', () => {
- test('returns without system indices if valid index options', () => {
+ test('returns without system indices if valid index options and undefined localCluster', () => {
expect(
getVisibleOptions(
[
@@ -26,13 +26,56 @@ describe('helpers', () => {
)
).toEqual([
{
- label: 'Indices',
+ label: 'Indices: (Local)',
options: [{ label: 'hello', health: 'green' }],
},
{
- label: 'Aliases',
+ label: 'Aliases: (Local)',
+ options: [{ label: 'hello' }],
+ },
+ ]);
+ });
+ test('returns without system indices if valid index options', () => {
+ expect(
+ getVisibleOptions(
+ [
+ { index: 'hello', health: 'green', localCluster: true },
+ { index: '.world', health: 'green', localCluster: false },
+ { index: 'ale-cluster:ale', health: 'green', localCluster: false },
+ ],
+ [
+ {
+ alias: 'cluster-2:.system',
+ index: 'opensearch_dashboards',
+ localCluster: false,
+ },
+ { alias: 'hello', index: 'world', localCluster: true },
+ { alias: 'cluster-2:hello', index: 'world', localCluster: false },
+ ],
+ 'cluster-1'
+ )
+ ).toEqual([
+ {
+ label: 'Indices: cluster-1 (Local)',
+ options: [{ label: 'hello', health: 'green' }],
+ },
+ {
+ label: 'Indices: ale-cluster (Remote)',
+ options: [
+ {
+ label: 'ale-cluster:ale',
+ health: 'green',
+ },
+ ],
+ },
+ {
+ label: 'Aliases: cluster-1 (Local)',
options: [{ label: 'hello' }],
},
+ {
+ label: 'Aliases: cluster-2 (Remote)',
+ options: [{ label: 'cluster-2:hello' }],
+ },
]);
});
test('returns empty aliases and index ', () => {
@@ -56,6 +99,100 @@ describe('helpers', () => {
]);
});
});
+ describe('groupIndicesOrAliasesByCluster', () => {
+ const localClusterName = 'local-cluster';
+ const dataType = 'Indices';
+
+ test('should group local indices correctly', () => {
+ const indices = [
+ { label: 'index1', localCluster: true },
+ { label: 'index2', localCluster: true },
+ ];
+
+ const result = groupIndicesOrAliasesByCluster(indices, localClusterName, dataType);
+
+ expect(result).toEqual([
+ {
+ label: 'Indices: local-cluster (Local)',
+ options: [
+ { label: 'index1' },
+ { label: 'index2' },
+ ],
+ },
+ ]);
+ });
+ test('should group remote indices correctly', () => {
+ const indices = [
+ { label: 'remote-cluster:index1', localCluster: false },
+ { label: 'remote-cluster:index2', localCluster: false },
+ ];
+
+ const result = groupIndicesOrAliasesByCluster(indices, localClusterName, dataType);
+
+ expect(result).toEqual([
+ {
+ label: 'Indices: remote-cluster (Remote)',
+ options: [
+ { label: 'remote-cluster:index1' },
+ { label: 'remote-cluster:index2' },
+ ],
+ },
+ ]);
+ });
+
+ test('should group mixed local and remote indices correctly', () => {
+ const indices = [
+ { label: 'index1', localCluster: true },
+ { label: 'remote-cluster:index2', localCluster: false },
+ { label: 'index3', localCluster: true },
+ { label: 'another-remote:index4', localCluster: false },
+ ];
+
+ const result = groupIndicesOrAliasesByCluster(indices, localClusterName, dataType);
+
+ expect(result).toEqual([
+ {
+ label: 'Indices: local-cluster (Local)',
+ options: [
+ { label: 'index1' },
+ { label: 'index3' },
+ ],
+ },
+ {
+ label: 'Indices: remote-cluster (Remote)',
+ options: [
+ { label: 'remote-cluster:index2' },
+ ],
+ },
+ {
+ label: 'Indices: another-remote (Remote)',
+ options: [
+ { label: 'another-remote:index4' },
+ ],
+ },
+ ]);
+ });
+
+ test('should handle indices with undefined localCluster property', () => {
+ const indices = [
+ { label: 'index1' },
+ { label: 'index2', localCluster: undefined },
+ ];
+
+ const result = groupIndicesOrAliasesByCluster(indices, localClusterName, dataType);
+
+ expect(result).toEqual([
+ {
+ label: 'Indices: local-cluster (Local)',
+ options: [
+ { label: 'index1' },
+ { label: 'index2' },
+ ],
+ },
+ ]);
+ });
+ });
+
describe('sanitizeSearchText', () => {
test('should return empty', () => {
expect(sanitizeSearchText('*')).toBe('');
diff --git a/public/pages/utils/helpers.ts b/public/pages/utils/helpers.ts
index e7c30b9a..91c27512 100644
--- a/public/pages/utils/helpers.ts
+++ b/public/pages/utils/helpers.ts
@@ -11,20 +11,30 @@
import queryString from 'query-string';
import {
CatIndex,
+ ClusterInfo,
IndexAlias,
MDSQueryParams,
} from '../../../server/models/types';
import sortBy from 'lodash/sortBy';
import { DetectorListItem } from '../../models/interfaces';
-import { DETECTORS_QUERY_PARAMS, SORT_DIRECTION } from '../../../server/utils/constants';
-import { ALL_INDICES, ALL_DETECTOR_STATES, MAX_DETECTORS, DEFAULT_QUERY_PARAMS } from './constants';
+import {
+ DETECTORS_QUERY_PARAMS,
+ SORT_DIRECTION,
+} from '../../../server/utils/constants';
+import {
+ ALL_INDICES,
+ ALL_DETECTOR_STATES,
+ MAX_DETECTORS,
+ DEFAULT_QUERY_PARAMS,
+} from './constants';
import { DETECTOR_STATE } from '../../../server/utils/constants';
import { timeFormatter } from '@elastic/charts';
-import { getDataSourceEnabled, getDataSourcePlugin } from '../../services';
+import { getDataSourceEnabled } from '../../services';
import { DataSourceAttributes } from '../../../../../src/plugins/data_source/common/data_sources';
import { SavedObject } from '../../../../../src/core/public';
-import * as pluginManifest from "../../../opensearch_dashboards.json";
-import semver from "semver";
+import * as pluginManifest from '../../../opensearch_dashboards.json';
+import semver from 'semver';
+import _ from 'lodash';
export function sanitizeSearchText(searchValue: string): string {
if (!searchValue || searchValue == '*') {
@@ -48,29 +58,73 @@ const isUserIndex = (index: string) => {
if (!index) {
return false;
}
- return !index.startsWith('.');
+ //.ml-config
+ return !(index.startsWith('.') || index.includes(':.'));
};
-export function getVisibleOptions(indices: CatIndex[], aliases: IndexAlias[]) {
- const visibleIndices = indices
- .filter((value) => isUserIndex(value.index))
- .map((value) => ({ label: value.index, health: value.health }));
- const visibleAliases = aliases
- .filter((value) => isUserIndex(value.alias))
- .map((value) => ({ label: value.alias }));
-
- return [
- {
- label: 'Indices',
- options: visibleIndices,
- },
- {
- label: 'Aliases',
- options: visibleAliases,
- },
- ];
+export function groupIndicesOrAliasesByCluster(
+ indices,
+ localClusterName: string,
+ dataType: string
+) {
+ return indices.reduce((acc, index) => {
+ const clusterName = index.label.includes(':')
+ ? index.label.split(':')[0]
+ : localClusterName;
+
+ //if undefined should be local as well.
+ let label =
+ index.localCluster === undefined || index.localCluster
+ ? `${dataType}: ${localClusterName} (Local)`
+ : `${dataType}: ${clusterName} (Remote)`;
+
+ const { localCluster, ...indexWithOutLocalInfo } = index; // Destructure and remove localCluster
+ const cluster = acc.find((cluster) => cluster.label === label);
+ if (cluster) {
+ cluster.options.push(indexWithOutLocalInfo);
+ } else {
+ acc.push({ label, options: [indexWithOutLocalInfo] });
+ }
+
+ return acc;
+ }, [] as { label: string; options: any[] }[]);
}
+export function getVisibleOptions(
+ indices: CatIndex[],
+ aliases: IndexAlias[],
+ localClusterName: string = ''
+) {
+ // Group by cluster or fallback to default label format
+ const getLabeledOptions = (items: any[], label: string) =>
+ items.length > 0
+ ? groupIndicesOrAliasesByCluster(items, localClusterName, label)
+ : [{ label, options: items }];
+
+ const visibleIndices = mapToVisibleOptions(indices, 'index');
+ const visibleAliases = mapToVisibleOptions(aliases, 'alias');
+
+ // Combine grouped indices and aliases
+ const visibleIndicesLabel = getLabeledOptions(visibleIndices, 'Indices');
+ const visibleAliasesLabel = getLabeledOptions(visibleAliases, 'Aliases');
+ const combinedVisibleIndicesAndAliases =
+ visibleIndicesLabel.concat(visibleAliasesLabel);
+ const sortedVisibleIndicesAndAliases = _.sortBy(combinedVisibleIndicesAndAliases, [
+ (item) => (item.label.includes('Indices:') ? 0 : 1), // Indices first, then Aliases
+ (item) => (item.label.includes('(Local)') ? 0 : 1), // Local first, then Remote
+ ]);
+ return sortedVisibleIndicesAndAliases;
+}
+
+export const mapToVisibleOptions = (items: any[], key: string) =>
+ items
+ .filter((value) => isUserIndex(value[key]))
+ .map((value) => ({
+ label: value[key],
+ ...(key === 'index' && { health: value.health }), // Only applicable to indices, ignored for aliases
+ localCluster: value.localCluster,
+ }));
+
export const filterAndSortDetectors = (
detectors: DetectorListItem[],
search: string,
@@ -93,7 +147,7 @@ export const filterAndSortDetectors = (
selectedIndices == ALL_INDICES
? filteredBySearchAndState
: filteredBySearchAndState.filter((detector) =>
- selectedIndices.includes(detector.indices[0])
+ detector.indices.some((index) => selectedIndices.includes(index))
);
let sorted = sortBy(filteredBySearchAndStateAndIndex, sortField);
if (sortDirection == SORT_DIRECTION.DESC) {
@@ -169,7 +223,7 @@ export const constructHrefWithDataSourceId = (
url.set(DETECTORS_QUERY_PARAMS.SEARCH, DEFAULT_QUERY_PARAMS.search);
url.set(DETECTORS_QUERY_PARAMS.INDICES, DEFAULT_QUERY_PARAMS.indices);
url.set(DETECTORS_QUERY_PARAMS.SORT_FIELD, DEFAULT_QUERY_PARAMS.sortField);
- url.set(DETECTORS_QUERY_PARAMS.SORT_DIRECTION, SORT_DIRECTION.ASC)
+ url.set(DETECTORS_QUERY_PARAMS.SORT_DIRECTION, SORT_DIRECTION.ASC);
if (dataSourceEnabled) {
url.set(DETECTORS_QUERY_PARAMS.DATASOURCEID, '');
}
@@ -189,7 +243,9 @@ export const constructHrefWithDataSourceId = (
return `${basePath}?${url.toString()}`;
};
-export const isDataSourceCompatible = (dataSource: SavedObject) => {
+export const isDataSourceCompatible = (
+ dataSource: SavedObject
+) => {
if (
'requiredOSDataSourcePlugins' in pluginManifest &&
!pluginManifest.requiredOSDataSourcePlugins.every((plugin) =>
@@ -210,4 +266,8 @@ export const isDataSourceCompatible = (dataSource: SavedObject {
+ return clusters.filter((cluster) => cluster.localCluster === true);
+};
diff --git a/public/redux/reducers/__tests__/opensearch.test.ts b/public/redux/reducers/__tests__/opensearch.test.ts
index cb434c0c..18d75ae2 100644
--- a/public/redux/reducers/__tests__/opensearch.test.ts
+++ b/public/redux/reducers/__tests__/opensearch.test.ts
@@ -15,7 +15,9 @@ import { BASE_NODE_API_PATH } from '../../../../utils/constants';
import { mockedStore } from '../../utils/testUtils';
import reducer, {
getAliases,
+ getClustersInfo,
getIndices,
+ getIndicesAndAliases,
getMappings,
initialState,
searchOpenSearch,
@@ -52,7 +54,7 @@ describe('opensearch reducer actions', () => {
expect(httpMockedClient.get).toHaveBeenCalledWith(
`..${BASE_NODE_API_PATH}/_indices`,
{
- query: { index: '' },
+ query: { index: '', clusters: '' },
}
);
});
@@ -79,7 +81,7 @@ describe('opensearch reducer actions', () => {
expect(httpMockedClient.get).toHaveBeenCalledWith(
`..${BASE_NODE_API_PATH}/_indices`,
{
- query: { index: '' },
+ query: { index: '', clusters: '' },
}
);
}
@@ -175,7 +177,7 @@ describe('opensearch reducer actions', () => {
expect(httpMockedClient.get).toHaveBeenCalledWith(
`..${BASE_NODE_API_PATH}/_mappings`,
{
- query: { index: '' },
+ query: { indices: [] },
}
);
});
@@ -202,7 +204,7 @@ describe('opensearch reducer actions', () => {
expect(httpMockedClient.get).toHaveBeenCalledWith(
`..${BASE_NODE_API_PATH}/_mappings`,
{
- query: { index: '' },
+ query: { indices: [] },
}
);
}
@@ -278,5 +280,190 @@ describe('opensearch reducer actions', () => {
}
});
});
+ describe('getIndicesAndAliases', () => {
+ test('should handle [REQUEST, SUCCESS] actions for getIndicesAndAliases', async () => {
+ const indices = [
+ { index: 'index1', health: 'green' },
+ { index: 'index2', health: 'yellow' },
+ ];
+ const aliases = [
+ { alias: 'alias1', index: 'index1' },
+ { alias: 'alias2', index: 'index2' },
+ ];
+
+ httpMockedClient.get = jest
+ .fn()
+ .mockResolvedValue({ ok: true, response: { indices, aliases } });
+
+ await store.dispatch(getIndicesAndAliases());
+ const actions = store.getActions();
+
+ expect(actions[0].type).toBe('opensearch/GET_INDICES_AND_ALIASES_REQUEST');
+ expect(reducer(initialState, actions[0])).toEqual({
+ ...initialState,
+ requesting: true,
+ });
+
+ expect(actions[1].type).toBe('opensearch/GET_INDICES_AND_ALIASES_SUCCESS');
+ expect(reducer(initialState, actions[1])).toEqual({
+ ...initialState,
+ requesting: false,
+ indices,
+ aliases,
+ });
+
+ expect(httpMockedClient.get).toHaveBeenCalledWith(
+ `..${BASE_NODE_API_PATH}/_indices_and_aliases`,
+ {
+ query: { indexOrAliasQuery: '', clusters: '', queryForLocalCluster: true },
+ }
+ );
+ });
+ test('should handle [REQUEST, SUCCESS] actions for getIndicesAndAliases with clusters', async () => {
+ const indices = [
+ { index: 'index1', health: 'green' },
+ { index: 'index2', health: 'yellow' },
+ ];
+ const aliases = [
+ { alias: 'alias1', index: 'index1' },
+ { alias: 'alias2', index: 'index2' },
+ ];
+
+ httpMockedClient.get = jest
+ .fn()
+ .mockResolvedValue({ ok: true, response: { indices, aliases } });
+
+ await store.dispatch(getIndicesAndAliases('', '', 'cluster-2,cluster-3'));
+ const actions = store.getActions();
+
+ expect(actions[0].type).toBe('opensearch/GET_INDICES_AND_ALIASES_REQUEST');
+ expect(reducer(initialState, actions[0])).toEqual({
+ ...initialState,
+ requesting: true,
+ });
+
+ expect(actions[1].type).toBe('opensearch/GET_INDICES_AND_ALIASES_SUCCESS');
+ expect(reducer(initialState, actions[1])).toEqual({
+ ...initialState,
+ requesting: false,
+ indices,
+ aliases,
+ });
+
+ expect(httpMockedClient.get).toHaveBeenCalledWith(
+ `..${BASE_NODE_API_PATH}/_indices_and_aliases`,
+ {
+ query: {
+ indexOrAliasQuery: '',
+ clusters: 'cluster-2,cluster-3',
+ queryForLocalCluster: true,
+ },
+ }
+ );
+ });
+ test('should handle [REQUEST, FAILURE] actions for getIndicesAndAliases', async () => {
+ httpMockedClient.get = jest.fn().mockRejectedValue({
+ ok: false,
+ error: 'Something went wrong',
+ });
+
+ try {
+ await store.dispatch(getIndicesAndAliases());
+ } catch (e) {
+ const actions = store.getActions();
+
+ expect(actions[0].type).toBe('opensearch/GET_INDICES_AND_ALIASES_REQUEST');
+ expect(reducer(initialState, actions[0])).toEqual({
+ ...initialState,
+ requesting: true,
+ errorMessage: '',
+ });
+
+ expect(actions[1].type).toBe('opensearch/GET_INDICES_AND_ALIASES_FAILURE');
+ expect(reducer(initialState, actions[1])).toEqual({
+ ...initialState,
+ requesting: false,
+ errorMessage: 'Something went wrong',
+ });
+
+ expect(httpMockedClient.get).toHaveBeenCalledWith(
+ `..${BASE_NODE_API_PATH}/_indices_and_aliases`,
+ {
+ query: {
+ indexOrAliasQuery: '',
+ clusters: '',
+ queryForLocalCluster: true,
+ },
+ }
+ );
+ }
+ });
+ });
+ describe('getClustersInfo', () => {
+ test('should invoke [REQUEST, SUCCESS]', async () => {
+ const clusters = [
+ { cluster: 'cluster1', status: 'green' },
+ { cluster: 'cluster2', status: 'yellow' },
+ ];
+
+ httpMockedClient.get = jest
+ .fn()
+ .mockResolvedValue({ ok: true, response: { clusters } });
+
+ await store.dispatch(getClustersInfo());
+ const actions = store.getActions();
+
+ expect(actions[0].type).toBe('opensearch/GET_CLUSTERS_INFO_REQUEST');
+ expect(reducer(initialState, actions[0])).toEqual({
+ ...initialState,
+ requesting: true,
+ errorMessage: '',
+ });
+
+ expect(actions[1].type).toBe('opensearch/GET_CLUSTERS_INFO_SUCCESS');
+ expect(reducer(initialState, actions[1])).toEqual({
+ ...initialState,
+ requesting: false,
+ clusters,
+ });
+
+ expect(httpMockedClient.get).toHaveBeenCalledWith(
+ `..${BASE_NODE_API_PATH}/_remote/info`
+ );
+ });
+ test('should invoke [REQUEST, FAILURE]', async () => {
+ const errorMessage = 'Something went wrong';
+
+ httpMockedClient.get = jest.fn().mockRejectedValue({
+ ok: false,
+ error: errorMessage,
+ });
+
+ try {
+ await store.dispatch(getClustersInfo());
+ } catch (e) {
+ const actions = store.getActions();
+
+ expect(actions[0].type).toBe('opensearch/GET_CLUSTERS_INFO_REQUEST');
+ expect(reducer(initialState, actions[0])).toEqual({
+ ...initialState,
+ requesting: true,
+ errorMessage: '',
+ });
+
+ expect(actions[1].type).toBe('opensearch/GET_CLUSTERS_INFO_FAILURE');
+ expect(reducer(initialState, actions[1])).toEqual({
+ ...initialState,
+ requesting: false,
+ errorMessage,
+ });
+
+ expect(httpMockedClient.get).toHaveBeenCalledWith(
+ `..${BASE_NODE_API_PATH}/_remote/info`
+ );
+ }
+ });
+ });
+
describe('getPrioritizedIndices', () => {});
});
diff --git a/public/redux/reducers/opensearch.ts b/public/redux/reducers/opensearch.ts
index 4a9a3d32..77667b74 100644
--- a/public/redux/reducers/opensearch.ts
+++ b/public/redux/reducers/opensearch.ts
@@ -18,10 +18,13 @@ import {
} from '../middleware/types';
import handleActions from '../utils/handleActions';
import { getPathsPerDataType } from './mapper';
-import { CatIndex, IndexAlias } from '../../../server/models/types';
+import {
+ CatIndex,
+ ClusterInfo,
+ IndexAlias,
+} from '../../../server/models/types';
import { AD_NODE_API } from '../../../utils/constants';
import { get } from 'lodash';
-import { data } from 'jquery';
const GET_INDICES = 'opensearch/GET_INDICES';
const GET_ALIASES = 'opensearch/GET_ALIASES';
@@ -30,6 +33,8 @@ const SEARCH_OPENSEARCH = 'opensearch/SEARCH_OPENSEARCH';
const CREATE_INDEX = 'opensearch/CREATE_INDEX';
const BULK = 'opensearch/BULK';
const DELETE_INDEX = 'opensearch/DELETE_INDEX';
+const GET_CLUSTERS_INFO = 'opensearch/GET_CLUSTERS_INFO';
+const GET_INDICES_AND_ALIASES = 'opensearch/GET_INDICES_AND_ALIASES';
export type Mappings = {
[key: string]: any;
@@ -63,7 +68,9 @@ interface OpenSearchState {
requesting: boolean;
searchResult: object;
errorMessage: string;
+ clusters: ClusterInfo[];
}
+
export const initialState: OpenSearchState = {
indices: [],
aliases: [],
@@ -71,10 +78,35 @@ export const initialState: OpenSearchState = {
requesting: false,
searchResult: {},
errorMessage: '',
+ clusters: [],
};
const reducer = handleActions(
{
+ [GET_INDICES_AND_ALIASES]: {
+ REQUEST: (state: OpenSearchState): OpenSearchState => {
+ return { ...state, requesting: true, errorMessage: '' };
+ },
+ SUCCESS: (
+ state: OpenSearchState,
+ action: APIResponseAction
+ ): OpenSearchState => {
+ return {
+ ...state,
+ requesting: false,
+ indices: get(action, 'result.response.indices', []),
+ aliases: get(action, 'result.response.aliases', []),
+ };
+ },
+ FAILURE: (
+ state: OpenSearchState,
+ action: APIErrorAction
+ ): OpenSearchState => ({
+ ...state,
+ requesting: false,
+ errorMessage: get(action, 'error.error', action.error),
+ }),
+ },
[GET_INDICES]: {
REQUEST: (state: OpenSearchState): OpenSearchState => {
return { ...state, requesting: true, errorMessage: '' };
@@ -244,18 +276,74 @@ const reducer = handleActions(
errorMessage: get(action, 'error.error', action.error),
}),
},
+ [GET_CLUSTERS_INFO]: {
+ REQUEST: (state: OpenSearchState): OpenSearchState => {
+ return { ...state, requesting: true, errorMessage: '' };
+ },
+ SUCCESS: (
+ state: OpenSearchState,
+ action: APIResponseAction
+ ): OpenSearchState => {
+ return {
+ ...state,
+ requesting: false,
+ clusters: get(action, 'result.response.clusters', []),
+ };
+ },
+ FAILURE: (
+ state: OpenSearchState,
+ action: APIErrorAction
+ ): OpenSearchState => ({
+ ...state,
+ requesting: false,
+ errorMessage: get(action, 'error.error', action.error),
+ }),
+ },
},
initialState
);
-export const getIndices = (searchKey = '', dataSourceId: string = '') => {
+export const getIndices = (
+ searchKey = '',
+ dataSourceId: string = '',
+ givenClusters: string = ''
+) => {
const baseUrl = `..${AD_NODE_API._INDICES}`;
const url = dataSourceId ? `${baseUrl}/${dataSourceId}` : baseUrl;
-
return {
type: GET_INDICES,
request: (client: HttpSetup) =>
- client.get(url, { query: { index: searchKey } }),
+ client.get(url, { query: { index: searchKey, clusters: givenClusters } }),
+ };
+};
+
+export const getClustersInfo = (dataSourceId: string = ''): APIAction => {
+ const baseUrl = `..${AD_NODE_API.GET_CLUSTERS_INFO}`;
+ const url = dataSourceId ? `${baseUrl}/${dataSourceId}` : baseUrl;
+ return {
+ type: GET_CLUSTERS_INFO,
+ request: (client: HttpSetup) => client.get(url),
+ };
+};
+
+export const getIndicesAndAliases = (
+ searchKey = '',
+ dataSourceId: string = '',
+ givenClusters: string = '',
+ queryForLocalCluster: boolean = true
+): APIAction => {
+ const baseUrl = `..${AD_NODE_API.GET_INDICES_AND_ALIASES}`;
+ const url = dataSourceId ? `${baseUrl}/${dataSourceId}` : baseUrl;
+ return {
+ type: GET_INDICES_AND_ALIASES,
+ request: (client: HttpSetup) =>
+ client.get(url, {
+ query: {
+ indexOrAliasQuery: searchKey,
+ clusters: givenClusters,
+ queryForLocalCluster: queryForLocalCluster,
+ },
+ }),
};
};
@@ -273,14 +361,18 @@ export const getAliases = (
};
};
-export const getMappings = (searchKey: string = '', dataSourceId: string = ''): APIAction => {
- const url = dataSourceId ? `${AD_NODE_API._MAPPINGS}/${dataSourceId}` : AD_NODE_API._MAPPINGS;
-
+export const getMappings = (
+ searchKey: string[] = [],
+ dataSourceId: string = ''
+): APIAction => {
+ const url = dataSourceId
+ ? `${AD_NODE_API._MAPPINGS}/${dataSourceId}`
+ : AD_NODE_API._MAPPINGS;
return {
type: GET_MAPPINGS,
request: (client: HttpSetup) =>
client.get(`..${url}`, {
- query: { index: searchKey },
+ query: { indices: searchKey },
}),
};
};
@@ -293,7 +385,10 @@ export const searchOpenSearch = (requestData: any): APIAction => ({
}),
});
-export const createIndex = (indexConfig: any, dataSourceId: string = ''): APIAction => {
+export const createIndex = (
+ indexConfig: any,
+ dataSourceId: string = ''
+): APIAction => {
const url = dataSourceId
? `${AD_NODE_API.CREATE_INDEX}/${dataSourceId}`
: AD_NODE_API.CREATE_INDEX;
@@ -324,11 +419,14 @@ export const deleteIndex = (index: string): APIAction => ({
});
export const getPrioritizedIndices =
- (searchKey: string, dataSourceId: string = ''): ThunkAction =>
+ (
+ searchKey: string,
+ dataSourceId: string = '',
+ clusters: string = '*'
+ ): ThunkAction =>
async (dispatch, getState) => {
//Fetch Indices and Aliases with text provided
- await dispatch(getIndices(searchKey, dataSourceId));
- await dispatch(getAliases(searchKey, dataSourceId));
+ await dispatch(getIndicesAndAliases(searchKey, dataSourceId, clusters));
const osState = getState().opensearch;
const exactMatchedIndices = osState.indices;
const exactMatchedAliases = osState.aliases;
@@ -340,8 +438,9 @@ export const getPrioritizedIndices =
};
} else {
//No results found for exact match, append wildCard and get partial matches if exists
- await dispatch(getIndices(`${searchKey}*`, dataSourceId));
- await dispatch(getAliases(`${searchKey}*`, dataSourceId));
+ await dispatch(
+ getIndicesAndAliases(`${searchKey}*`, dataSourceId, clusters)
+ );
const osState = getState().opensearch;
const partialMatchedIndices = osState.indices;
const partialMatchedAliases = osState.aliases;
diff --git a/server/models/types.ts b/server/models/types.ts
index c1d679b0..ae6d40c1 100644
--- a/server/models/types.ts
+++ b/server/models/types.ts
@@ -14,13 +14,31 @@ import { SORT_DIRECTION, DETECTOR_STATE } from '../utils/constants';
export type CatIndex = {
index: string;
health: string;
+ localCluster?: boolean;
};
+export type ClusterInfo = {
+ name: string;
+ localCluster: boolean;
+}
+
export type IndexAlias = {
- index: string;
+ index: string[] | string;
alias: string;
+ localCluster?: boolean
};
+export type IndexOption = {
+ label: string,
+ health: string,
+ localCluster?: boolean
+}
+
+export type AliasOption = {
+ label: string,
+ localCluster?: string
+}
+
export type GetAliasesResponse = {
aliases: IndexAlias[];
};
diff --git a/server/routes/opensearch.ts b/server/routes/opensearch.ts
index 3a8444d9..65f1e3a7 100644
--- a/server/routes/opensearch.ts
+++ b/server/routes/opensearch.ts
@@ -13,6 +13,7 @@ import { get } from 'lodash';
import { SearchResponse } from '../models/interfaces';
import {
CatIndex,
+ ClusterInfo,
GetAliasesResponse,
GetIndicesResponse,
GetMappingResponse,
@@ -28,6 +29,10 @@ import {
IOpenSearchDashboardsResponse,
} from '../../../../src/core/server';
import { getClientBasedOnDataSource } from '../utils/helpers';
+import { CatAliases } from '@opensearch-project/opensearch/api/requestParams';
+import _ from 'lodash';
+import { Mappings } from 'public/redux/reducers/opensearch';
+import { convertFieldCapsToMappingStructure } from './utils/opensearchHelpers';
type SearchParams = {
index: string;
@@ -57,6 +62,20 @@ export function registerOpenSearchRoutes(
apiRouter.post('/bulk/{dataSourceId}', opensearchService.bulk);
apiRouter.post('/delete_index', opensearchService.deleteIndex);
+ apiRouter.get('/_remote/info', opensearchService.getClustersInfo);
+ apiRouter.get('/_remote/info/', opensearchService.getClustersInfo);
+ apiRouter.get(
+ '/_remote/info/{dataSourceId}',
+ opensearchService.getClustersInfo
+ );
+ apiRouter.get(
+ '/_indices_and_aliases',
+ opensearchService.getIndicesAndAliases
+ );
+ apiRouter.get(
+ '/_indices_and_aliases/{dataSourceId}',
+ opensearchService.getIndicesAndAliases
+ );
}
export default class OpenSearchService {
@@ -125,7 +144,10 @@ export default class OpenSearchService {
request: OpenSearchDashboardsRequest,
opensearchDashboardsResponse: OpenSearchDashboardsResponseFactory
): Promise> => {
- const { index } = request.query as { index: string };
+ const { index, clusters } = request.query as {
+ index: string;
+ clusters: string;
+ };
const { dataSourceId = '' } = request.params as { dataSourceId?: string };
try {
const callWithRequest = getClientBasedOnDataSource(
@@ -135,12 +157,41 @@ export default class OpenSearchService {
dataSourceId,
this.client
);
+ let indices: CatIndex[] = [];
+ let resolve_resp;
- const response: CatIndex[] = await callWithRequest('cat.indices', {
+ let response: CatIndex[] = await callWithRequest('cat.indices', {
index,
format: 'json',
h: 'health,index',
});
+ response = response.map((item) => ({
+ ...item,
+ localCluster: true,
+ }));
+
+ // only call cat indices
+ if (clusters != '') {
+ if (index == '') {
+ resolve_resp = await callWithRequest('transport.request', {
+ method: 'GET',
+ path: '/_resolve/index/' + clusters + ':*',
+ });
+ } else {
+ resolve_resp = await callWithRequest('transport.request', {
+ method: 'GET',
+ path: '/_resolve/index/' + clusters + ':' + index,
+ });
+ }
+ indices = resolve_resp.indices.map((item) => ({
+ index: item.name,
+ format: 'json',
+ health: 'undefined',
+ localCluster: false,
+ }));
+
+ response = response.concat(indices);
+ }
return opensearchDashboardsResponse.ok({
body: { ok: true, response: { indices: response } },
@@ -338,7 +389,11 @@ export default class OpenSearchService {
request: OpenSearchDashboardsRequest,
opensearchDashboardsResponse: OpenSearchDashboardsResponseFactory
): Promise> => {
- const { index } = request.query as { index: string };
+ let { indices } = request.query as { indices: string[] };
+ // If indices is not an array, convert it to an array, server framework auto converts single item in string array to a string
+ if (!Array.isArray(indices)) {
+ indices = [indices];
+ }
const { dataSourceId = '' } = request.params as { dataSourceId?: string };
try {
@@ -350,12 +405,34 @@ export default class OpenSearchService {
this.client
);
- const response = await callWithRequest(
- 'indices.getMapping', {
- index,
+ let mappings: Mappings = {};
+ let remoteMappings: Mappings = {};
+ let localIndices: string[] = indices.filter(
+ (index: string) => !index.includes(':')
+ );
+ let remoteIndices: string[] = indices.filter((index: string) =>
+ index.includes(':')
+ );
+
+ if (localIndices.length > 0) {
+ mappings = await callWithRequest('indices.getMapping', {
+ index: localIndices,
+ });
+ }
+
+ // make call to fields_caps
+ if (remoteIndices.length) {
+ const fieldCapsResponse = await callWithRequest('transport.request', {
+ method: 'GET',
+ path:
+ remoteIndices.toString() + '/_field_caps?fields=*&include_unmapped',
});
+ remoteMappings = convertFieldCapsToMappingStructure(fieldCapsResponse);
+ }
+ Object.assign(mappings, remoteMappings);
+
return opensearchDashboardsResponse.ok({
- body: { ok: true, response: { mappings: response } },
+ body: { ok: true, response: { mappings: mappings } },
});
} catch (err) {
console.log('Anomaly detector - Unable to get mappings', err);
@@ -367,4 +444,162 @@ export default class OpenSearchService {
});
}
};
+
+ // we use this to retrieve indices and aliases from both the local cluster and remote clusters
+ // 3 different OS APIs are called here, _cat/indices, _cat/aliases and _resolve/index
+ getIndicesAndAliases = async (
+ context: RequestHandlerContext,
+ request: OpenSearchDashboardsRequest,
+ opensearchDashboardsResponse: OpenSearchDashboardsResponseFactory
+ ): Promise> => {
+ const { indexOrAliasQuery, clusters, queryForLocalCluster } =
+ request.query as {
+ indexOrAliasQuery: string;
+ clusters: string;
+ queryForLocalCluster: string;
+ };
+ const { dataSourceId = '' } = request.params as { dataSourceId?: string };
+ try {
+ const callWithRequest = getClientBasedOnDataSource(
+ context,
+ this.dataSourceEnabled,
+ request,
+ dataSourceId,
+ this.client
+ );
+ let indicesResponse: CatIndex[] = [];
+ let aliasesResponse: IndexAlias[] = [];
+ if (queryForLocalCluster == 'true') {
+ indicesResponse = await callWithRequest('cat.indices', {
+ index: indexOrAliasQuery,
+ format: 'json',
+ h: 'health,index',
+ });
+ indicesResponse = indicesResponse.map((item) => ({
+ ...item,
+ localCluster: true,
+ }));
+ aliasesResponse = await callWithRequest('cat.aliases', {
+ alias: indexOrAliasQuery,
+ format: 'json',
+ h: 'alias,index',
+ });
+
+ aliasesResponse = aliasesResponse.map((item) => ({
+ ...item,
+ localCluster: true,
+ }));
+ }
+
+ // only call cat indices and cat aliases
+ if (clusters != '') {
+ let remoteIndices: CatIndex[] = [];
+ let remoteAliases: IndexAlias[] = [];
+ let resolveResponse;
+ const resolveIndexQuery =
+ indexOrAliasQuery == ''
+ ? clusters
+ .split(',')
+ .map((cluster) => `${cluster}:*`)
+ .join(',')
+ : clusters
+ .split(',')
+ .map((cluster) => `${cluster}:${indexOrAliasQuery}`)
+ .join(',');
+ resolveResponse = await callWithRequest('transport.request', {
+ method: 'GET',
+ path: '/_resolve/index/' + resolveIndexQuery,
+ });
+ remoteIndices = resolveResponse.indices.map((item) => ({
+ index: item.name,
+ format: 'json',
+ health: 'undefined',
+ localCluster: false,
+ }));
+
+ remoteAliases = resolveResponse.aliases.map((item) => ({
+ alias: item.name,
+ index: item.indices,
+ format: 'json',
+ localCluster: false,
+ }));
+ indicesResponse = indicesResponse.concat(remoteIndices);
+ aliasesResponse = aliasesResponse.concat(remoteAliases);
+ }
+
+ return opensearchDashboardsResponse.ok({
+ body: {
+ ok: true,
+ response: { aliases: aliasesResponse, indices: indicesResponse },
+ },
+ });
+ } catch (err) {
+ // In case no matching indices is found it throws an error.
+ if (
+ err.statusCode === 404 &&
+ get(err, 'body.error.type', '') === 'index_not_found_exception'
+ ) {
+ return opensearchDashboardsResponse.ok({
+ body: { ok: true, response: { indices: [], aliases: [] } },
+ });
+ }
+ console.log('Anomaly detector - Unable to get indices and aliases', err);
+ return opensearchDashboardsResponse.ok({
+ body: {
+ ok: false,
+ error: getErrorMessage(err),
+ },
+ });
+ }
+ };
+
+ getClustersInfo = async (
+ context: RequestHandlerContext,
+ request: OpenSearchDashboardsRequest,
+ opensearchDashboardsResponse: OpenSearchDashboardsResponseFactory
+ ): Promise> => {
+ const { dataSourceId = '' } = request.params as { dataSourceId?: string };
+ try {
+ const callWithRequest = getClientBasedOnDataSource(
+ context,
+ this.dataSourceEnabled,
+ request,
+ dataSourceId,
+ this.client
+ );
+
+ let clustersResponse: ClusterInfo[] = [];
+
+ const remoteInfo = await callWithRequest('transport.request', {
+ method: 'GET',
+ path: '/_remote/info',
+ });
+ clustersResponse = Object.keys(remoteInfo).map((key) => ({
+ name: key,
+ localCluster: false,
+ }));
+
+ const clusterHealth = await callWithRequest('cat.health', {
+ format: 'json',
+ h: 'cluster',
+ });
+
+ clustersResponse.push({
+ name: clusterHealth[0].cluster,
+ localCluster: true,
+ });
+
+ return opensearchDashboardsResponse.ok({
+ body: { ok: true, response: { clusters: clustersResponse } },
+ });
+ } catch (err) {
+ console.error('Alerting - OpensearchService - getClusterHealth:', err);
+ return opensearchDashboardsResponse.ok({
+ body: {
+ ok: false,
+ error: getErrorMessage(err),
+ },
+ });
+ }
+ };
}
diff --git a/server/routes/utils/__tests__/opensearchHelpers.test.ts b/server/routes/utils/__tests__/opensearchHelpers.test.ts
new file mode 100644
index 00000000..317a8fda
--- /dev/null
+++ b/server/routes/utils/__tests__/opensearchHelpers.test.ts
@@ -0,0 +1,84 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+import { convertFieldCapsToMappingStructure } from '../opensearchHelpers';
+
+describe('transformFieldCapsResponse', () => {
+ test('transform the field capabilities response into a structured mapping object', () => {
+ const fieldCapsResponse = {
+ indices: [
+ 'opensearch-ccs-cluster2:host-health-us-west-1',
+ 'opensearch-ccs-cluster2:sample-ecommerce',
+ ],
+ fields: {
+ _routing: {
+ _routing: { type: '_routing', searchable: true, aggregatable: false },
+ },
+ _doc_count: {
+ long: { type: 'long', searchable: false, aggregatable: false },
+ },
+ total_revenue_usd: {
+ integer: {
+ type: 'integer',
+ searchable: true,
+ aggregatable: true,
+ indices: ['opensearch-ccs-cluster2:sample-ecommerce'],
+ },
+ unmapped: {
+ type: 'unmapped',
+ searchable: false,
+ aggregatable: false,
+ indices: ['opensearch-ccs-cluster2:host-health-us-west-1'],
+ },
+ },
+ cpu_usage_percentage: {
+ integer: {
+ type: 'integer',
+ searchable: true,
+ aggregatable: true,
+ indices: ['opensearch-ccs-cluster2:host-health-us-west-1'],
+ },
+ unmapped: {
+ type: 'unmapped',
+ searchable: false,
+ aggregatable: false,
+ indices: ['opensearch-ccs-cluster2:sample-ecommerce'],
+ },
+ },
+ timestamp: {
+ date: { type: 'date', searchable: true, aggregatable: true },
+ },
+ },
+ };
+
+ const expectedOutput = {
+ 'opensearch-ccs-cluster2:host-health-us-west-1': {
+ mappings: {
+ properties: {
+ cpu_usage_percentage: { type: 'integer' },
+ timestamp: { type: 'date' },
+ },
+ },
+ },
+ 'opensearch-ccs-cluster2:sample-ecommerce': {
+ mappings: {
+ properties: {
+ total_revenue_usd: { type: 'integer' },
+ timestamp: { type: 'date' },
+ },
+ },
+ },
+ };
+
+ const result = convertFieldCapsToMappingStructure(fieldCapsResponse);
+ expect(result).toEqual(expectedOutput);
+ });
+});
diff --git a/server/routes/utils/opensearchHelpers.ts b/server/routes/utils/opensearchHelpers.ts
new file mode 100644
index 00000000..ae03dcab
--- /dev/null
+++ b/server/routes/utils/opensearchHelpers.ts
@@ -0,0 +1,47 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+import _ from 'lodash';
+import { Mappings } from 'public/redux/reducers/opensearch';
+
+export const convertFieldCapsToMappingStructure = (fieldCapsResponse) => {
+ let mappings: Mappings = {};
+
+ fieldCapsResponse.indices.forEach((index) => {
+ mappings[index] = {
+ mappings: {
+ properties: {},
+ },
+ };
+ });
+ for (const [fieldName, fieldDetails] of Object.entries(
+ fieldCapsResponse.fields
+ )) {
+ if (fieldName.startsWith('_')) {
+ continue;
+ }
+ for (const [fieldType, typeDetails] of Object.entries(fieldDetails)) {
+ if (fieldType == 'unmapped') {
+ continue;
+ }
+ let mapped_indices = _.get(
+ typeDetails,
+ 'indices',
+ fieldCapsResponse.indices
+ );
+ mapped_indices.forEach((mappedIndex) => {
+ mappings[mappedIndex]['mappings']['properties'][fieldName] = {
+ type: typeDetails.type,
+ };
+ });
+ }
+ }
+ return mappings;
+};
diff --git a/utils/constants.ts b/utils/constants.ts
index 231bd91b..639106cf 100644
--- a/utils/constants.ts
+++ b/utils/constants.ts
@@ -21,6 +21,8 @@ export const AD_NODE_API = Object.freeze({
BULK: `${BASE_NODE_API_PATH}/bulk`,
DELETE_INDEX: `${BASE_NODE_API_PATH}/delete_index`,
CREATE_SAMPLE_DATA: `${BASE_NODE_API_PATH}/create_sample_data`,
+ GET_CLUSTERS_INFO: `${BASE_NODE_API_PATH}/_remote/info`,
+ GET_INDICES_AND_ALIASES: `${BASE_NODE_API_PATH}/_indices_and_aliases`,
});
export const ALERTING_NODE_API = Object.freeze({
_SEARCH: `${BASE_NODE_API_PATH}/monitors/_search`,