From c81ee66d90ada850ff04c8990efe65bc4eaa3a07 Mon Sep 17 00:00:00 2001 From: Sumukh Swamy Date: Fri, 7 Jun 2024 17:57:01 -0700 Subject: [PATCH 1/3] backport mds metrics change Signed-off-by: sumukhswamy --- .../custom_panels/helpers/utils.tsx | 37 +++-- .../visualization_container.tsx | 7 +- public/components/metrics/helpers/utils.tsx | 14 +- public/components/metrics/index.tsx | 130 +++++++++++++----- .../metrics/redux/slices/metrics_slice.ts | 66 ++++++--- .../__snapshots__/sidebar.test.tsx.snap | 48 ++++++- public/components/metrics/sidebar/sidebar.tsx | 45 ++++-- .../metrics/top_menu/metrics_export.tsx | 34 ++--- .../components/metrics/view/metrics_grid.tsx | 25 +++- public/services/requests/ppl.ts | 4 + .../metrics/metrics_analytics_adaptor.ts | 26 ++-- .../data_connections_router.ts | 6 +- server/routes/index.ts | 2 +- server/routes/metrics/metrics_rounter.ts | 89 ++++++++---- server/routes/ppl.ts | 9 +- server/services/facets/ppl_facet.ts | 16 ++- 16 files changed, 398 insertions(+), 160 deletions(-) diff --git a/public/components/custom_panels/helpers/utils.tsx b/public/components/custom_panels/helpers/utils.tsx index 4989f52466..05757ae35a 100644 --- a/public/components/custom_panels/helpers/utils.tsx +++ b/public/components/custom_panels/helpers/utils.tsx @@ -170,13 +170,14 @@ export const getQueryResponse = async ( endTime: string, filterQuery = '', timestampField = 'timestamp', - metricVisualization = false + metricVisualization = false, + dataSourceMDSId?: string ) => { const finalQuery = metricVisualization ? query : queryAccumulator(query, timestampField, startTime, endTime, filterQuery); - const res = await pplService.fetch({ query: finalQuery, format: 'jdbc' }); + const res = await pplService.fetch({ query: finalQuery, format: 'jdbc' }, dataSourceMDSId); if (res === undefined) throw new Error('Please check the validity of PPL Filter'); @@ -198,6 +199,7 @@ export const renderSavedVisualization = async ({ setIsLoading, setIsError, visualization, + dataSourceMDSId, }: { pplService: PPLService; startTime: string; @@ -212,6 +214,7 @@ export const renderSavedVisualization = async ({ setIsLoading: React.Dispatch>; setIsError: React.Dispatch>; visualization: SavedVisualizationType; + dataSourceMDSId?: string; }) => { setIsLoading(true); setIsError({} as VizContainerError); @@ -244,7 +247,9 @@ export const renderSavedVisualization = async ({ startTime, endTime, filterQuery, - visualization.timeField + visualization.timeField, + false, + dataSourceMDSId ); setVisualizationData(queryData); } catch (error) { @@ -330,6 +335,7 @@ export const renderCatalogVisualization = async ({ setIsLoading, setIsError, visualization, + dataSourceMDSId, }: { pplService: PPLService; catalogSource: string; @@ -346,6 +352,7 @@ export const renderCatalogVisualization = async ({ setIsError: React.Dispatch>; queryMetaData?: MetricType; visualization: SavedVisualizationType; + dataSourceMDSId?: string; }) => { setIsLoading(true); setIsError({} as VizContainerError); @@ -373,7 +380,8 @@ export const renderCatalogVisualization = async ({ endTime, filterQuery, visualizationTimeField, - true + true, + dataSourceMDSId ); setVisualizationData(queryData); @@ -432,7 +440,8 @@ export const fetchAggregatedBinCount = async ( documentName: string, selectedOtelIndex: string, setIsError: React.Dispatch>, - setIsLoading: React.Dispatch> + setIsLoading: React.Dispatch>, + dataSourceMDSId: string ) => { const http = getOSDHttp(); try { @@ -444,6 +453,7 @@ export const fetchAggregatedBinCount = async ( endTime, documentName, index: selectedOtelIndex, + dataSourceMDSId, }), }); return response; @@ -459,11 +469,17 @@ export const fetchAggregatedBinCount = async ( } }; -export const fetchSampleOTDocument = async (selectedOtelIndex: string, documentName: string) => { +export const fetchSampleOTDocument = async ( + selectedOtelIndex: string, + documentName: string, + dataSourceMDSId: string +) => { const http = getOSDHttp(); try { const response = await http.get( - `${OBSERVABILITY_BASE}/metrics/otel/${selectedOtelIndex}/${documentName}` + `${OBSERVABILITY_BASE}/metrics/otel/${selectedOtelIndex}/${documentName}/${ + dataSourceMDSId ?? '' + }` ); return response; } catch (error) { @@ -495,6 +511,7 @@ export const renderOpenTelemetryVisualization = async ({ setIsError, visualization, setToast, + dataSourceMDSId, }: { startTime: string; endTime: string; @@ -511,6 +528,7 @@ export const renderOpenTelemetryVisualization = async ({ text?: React.ReactChild | undefined, side?: string | undefined ) => void; + dataSourceMDSId?: string; }) => { setIsLoading(true); setIsError({} as VizContainerError); @@ -527,7 +545,7 @@ export const renderOpenTelemetryVisualization = async ({ setToast('Document name is undefined', 'danger', undefined, 'right'); } - const fetchSampleDocument = await fetchSampleOTDocument(index, documentName); + const fetchSampleDocument = await fetchSampleOTDocument(index, documentName, dataSourceMDSId); const source = fetchSampleDocument.hits[0]._source; setVisualizationType(visualizationType); @@ -545,7 +563,8 @@ export const renderOpenTelemetryVisualization = async ({ documentName, index, setIsError, - setIsLoading + setIsLoading, + dataSourceMDSId ); return { diff --git a/public/components/custom_panels/panel_modules/visualization_container/visualization_container.tsx b/public/components/custom_panels/panel_modules/visualization_container/visualization_container.tsx index 1fe787a430..8a9b041df8 100644 --- a/public/components/custom_panels/panel_modules/visualization_container/visualization_container.tsx +++ b/public/components/custom_panels/panel_modules/visualization_container/visualization_container.tsx @@ -24,7 +24,7 @@ import { EuiText, EuiToolTip, } from '@elastic/eui'; -import { isEmpty } from 'lodash'; +import isEmpty from 'lodash/isEmpty'; import React, { useEffect, useMemo, useState } from 'react'; import { useSelector } from 'react-redux'; import { @@ -85,6 +85,7 @@ interface Props { catalogVisualization?: boolean; inlineEditor?: JSX.Element; actionMenuType?: string; + dataSourceMDSId?: string; } export const VisualizationContainer = ({ @@ -106,6 +107,7 @@ export const VisualizationContainer = ({ catalogVisualization, inlineEditor, actionMenuType, + dataSourceMDSId, }: Props) => { const [isPopoverOpen, setIsPopoverOpen] = useState(false); const [visualizationTitle, setVisualizationTitle] = useState(''); @@ -263,6 +265,7 @@ export const VisualizationContainer = ({ setIsLoading, setIsError, setToast, + dataSourceMDSId, }); else if (visualization.metricType === PROMQL_METRIC_SUBTYPE) renderCatalogVisualization({ @@ -281,6 +284,7 @@ export const VisualizationContainer = ({ setIsLoading, setIsError, queryMetaData, + dataSourceMDSId, }); else await renderSavedVisualization({ @@ -299,6 +303,7 @@ export const VisualizationContainer = ({ setVisualizationMetaData, setIsLoading, setIsError, + dataSourceMDSId, }); }; diff --git a/public/components/metrics/helpers/utils.tsx b/public/components/metrics/helpers/utils.tsx index 5c307ea57d..225ed548ff 100644 --- a/public/components/metrics/helpers/utils.tsx +++ b/public/components/metrics/helpers/utils.tsx @@ -10,12 +10,12 @@ import { Layout } from 'react-grid-layout'; import { VISUALIZATION } from '../../../../common/constants/metrics'; import { OTEL_METRIC_SUBTYPE, - PROMQL_METRIC_SUBTYPE, PPL_METRIC_SUBTYPE, + PROMQL_METRIC_SUBTYPE, } from '../../../../common/constants/shared'; -import PPLService from '../../../services/requests/ppl'; -import { MetricType } from '../../../../common/types/metrics'; import { VisualizationType } from '../../../../common/types/custom_panels'; +import { MetricType } from '../../../../common/types/metrics'; +import PPLService from '../../../services/requests/ppl'; export const onTimeChange = ( start: ShortDate, @@ -36,9 +36,13 @@ export const onTimeChange = ( }; // PPL Service requestor -export const pplServiceRequestor = (pplService: PPLService, finalQuery: string) => { +export const pplServiceRequestor = ( + pplService: PPLService, + finalQuery: string, + dataSourceMDSId?: string +) => { return pplService - .fetch({ query: finalQuery, format: VISUALIZATION }) + .fetch({ query: finalQuery, format: VISUALIZATION }, dataSourceMDSId) .then((res) => { return res; }) diff --git a/public/components/metrics/index.tsx b/public/components/metrics/index.tsx index 08df0c8e1f..9677d1e15d 100644 --- a/public/components/metrics/index.tsx +++ b/public/components/metrics/index.tsx @@ -3,17 +3,30 @@ * SPDX-License-Identifier: Apache-2.0 */ -import './index.scss'; import { EuiPage, EuiPageBody, EuiResizableContainer } from '@elastic/eui'; -import React, { useEffect, useState } from 'react'; +import debounce from 'lodash/debounce'; +import React, { useEffect, useMemo, useState } from 'react'; +import { useDispatch } from 'react-redux'; import { HashRouter, Route, RouteComponentProps, StaticContext } from 'react-router-dom'; -import { ChromeBreadcrumb } from '../../../../../src/core/public'; -import { Sidebar } from './sidebar/sidebar'; +import { + ChromeBreadcrumb, + MountPoint, + NotificationsStart, + SavedObjectsStart, +} from '../../../../../src/core/public'; +import { + DataSourceManagementPluginSetup, + DataSourceSelectableConfig, +} from '../../../../../src/plugins/data_source_management/public'; +import { DataSourceOption } from '../../../../../src/plugins/data_source_management/public/components/data_source_menu/types'; +import { OptionType } from '../../../common/types/metrics'; import PPLService from '../../services/requests/ppl'; +import SavedObjects from '../../services/saved_objects/event_analytics/saved_objects'; +import './index.scss'; +import { setSelectedDataSourceMDSId } from './redux/slices/metrics_slice'; +import { Sidebar } from './sidebar/sidebar'; import { TopMenu } from './top_menu/top_menu'; -import { OptionType } from '../../../common/types/metrics'; import { MetricsGrid } from './view/metrics_grid'; -import SavedObjects from '../../services/saved_objects/event_analytics/saved_objects'; interface MetricsProps { parentBreadcrumb: ChromeBreadcrumb; @@ -21,12 +34,27 @@ interface MetricsProps { pplService: PPLService; savedObjects: SavedObjects; setBreadcrumbs: (newBreadcrumbs: ChromeBreadcrumb[]) => void; + dataSourceManagement: DataSourceManagementPluginSetup; + dataSourceEnabled: boolean; + setActionMenu: (menuMount: MountPoint | undefined) => void; + savedObjectsMDSClient: SavedObjectsStart; + notifications: NotificationsStart; } -export const Home = ({ chrome, parentBreadcrumb }: MetricsProps) => { +export const Home = ({ + chrome, + parentBreadcrumb, + dataSourceManagement, + setActionMenu, + savedObjectsMDSClient, + notifications, + dataSourceEnabled, +}: MetricsProps) => { // Side bar constants const [selectedDataSource, setSelectedDataSource] = useState([]); const [selectedOTIndex, setSelectedOTIndex] = useState([]); + const [dataSourceMDSId, setDataSourceMDSId] = useState(''); + const [reloadSidebar, setReloadSidebar] = useState(false); useEffect(() => { chrome.setBreadcrumbs([ @@ -36,44 +64,78 @@ export const Home = ({ chrome, parentBreadcrumb }: MetricsProps) => { href: `#/`, }, ]); - }, [chrome, parentBreadcrumb]); + }, [chrome, parentBreadcrumb, dataSourceMDSId]); + + useEffect(() => { + setReloadSidebar(true); + }, [dataSourceMDSId]); + + const dispatch = useDispatch(); + + const onSelectedDataSource = async (dataSources: DataSourceOption[]) => { + const id = dataSources[0] ? dataSources[0].id : ''; + setDataSourceMDSId(id); + debounce(() => { + dispatch(setSelectedDataSourceMDSId(id)); + }, 300); + }; + const DataSourceMenu = dataSourceManagement?.ui?.getDataSourceMenu(); + const dataSourceMenuComponent = useMemo(() => { + return ( + + ); + }, [setActionMenu, savedObjectsMDSClient.client, notifications]); return ( <> + {dataSourceEnabled && dataSourceMenuComponent} (
- - - -
- - {(EuiResizablePanel, EuiResizableButton) => ( - <> - - - + {reloadSidebar && ( + + + +
+ + {(EuiResizablePanel, EuiResizableButton) => ( + <> + + + - + - - - - - )} - -
- - + + + + + )} + +
+ + + )}
)} /> diff --git a/public/components/metrics/redux/slices/metrics_slice.ts b/public/components/metrics/redux/slices/metrics_slice.ts index f3f3d08fdf..aad5f03571 100644 --- a/public/components/metrics/redux/slices/metrics_slice.ts +++ b/public/components/metrics/redux/slices/metrics_slice.ts @@ -68,6 +68,7 @@ const initialState = { selectedDataSource: '', otelIndices: [], otelDocumentNames: [], + dataSourceMDSId: '', }; const mergeMetricCustomizer = function (objValue, srcValue) { @@ -87,10 +88,14 @@ export const mergeMetrics = (newMetricMap) => (dispatch, getState) => { dispatch(setMetrics(mergedMetrics)); }; -export const loadMetrics = () => async (dispatch) => { +export const loadMetrics = (dataSourceMDSId: string) => async (dispatch) => { const pplService = getPPLService(); const customDataRequest = fetchCustomMetrics(); - const remoteDataSourcesResponse = await pplServiceRequestor(pplService!, PPL_DATASOURCES_REQUEST); + const remoteDataSourcesResponse = await pplServiceRequestor( + pplService!, + PPL_DATASOURCES_REQUEST, + dataSourceMDSId + ); const remoteDataSources = remoteDataSourcesResponse.data.DATASOURCE_NAME; dispatch(setDataSources(remoteDataSources)); dispatch(setDataSourceTitles(remoteDataSources)); @@ -100,7 +105,7 @@ export const loadMetrics = () => async (dispatch) => { ) ); - const remoteDataRequests = await fetchRemoteMetrics(remoteDataSources); + const remoteDataRequests = await fetchRemoteMetrics(remoteDataSources, dataSourceMDSId); const metricsResultSet = await Promise.all([customDataRequest, ...remoteDataRequests]); const metricsResult = metricsResultSet.flat(); const metricsMapById = keyBy(metricsResult.flat(), 'id'); @@ -110,8 +115,8 @@ export const loadMetrics = () => async (dispatch) => { await dispatch(setSortedIds(sortedIds)); }; -export const loadOTIndices = () => async (dispatch) => { - const fetchOTindices = await fetchOpenTelemetryIndices(); +export const loadOTIndices = (dataSourceMDSId: string) => async (dispatch) => { + const fetchOTindices = await fetchOpenTelemetryIndices(dataSourceMDSId); dispatch(setOtelIndices(fetchOTindices)); }; @@ -139,18 +144,19 @@ const fetchCustomMetrics = async () => { })); }; -const fetchRemoteDataSource = async (dataSource) => { +const fetchRemoteDataSource = async (dataSource, dataSourceMDSId: string) => { const pplService = getPPLService(); const response = await pplServiceRequestor( pplService, - `source = ${dataSource}.information_schema.tables` + `source = ${dataSource}.information_schema.tables`, + dataSourceMDSId ); return { jsonData: response.jsonData, dataSource }; }; -const fetchRemoteMetrics = (remoteDataSources: string[]) => +const fetchRemoteMetrics = (remoteDataSources: string[], dataSourceMDSId: string) => remoteDataSources.map((dataSource) => - fetchRemoteDataSource(dataSource).then(({ jsonData }) => + fetchRemoteDataSource(dataSource, dataSourceMDSId).then(({ jsonData }) => jsonData.map((obj: any) => ({ id: `${obj.TABLE_CATALOG}.${obj.TABLE_NAME}`, name: `${obj.TABLE_CATALOG}.${obj.TABLE_NAME}`, @@ -170,10 +176,10 @@ const fetchRemoteMetrics = (remoteDataSources: string[]) => ) ); -export const fetchOpenTelemetryIndices = async () => { +export const fetchOpenTelemetryIndices = async (dataSourceMDSId: string) => { const http = getOSDHttp(); return http - .get(`${OBSERVABILITY_BASE}/search/indices`, { + .get(`${OBSERVABILITY_BASE}/search/indices/${dataSourceMDSId ?? ''}`, { query: { format: 'json', }, @@ -181,10 +187,17 @@ export const fetchOpenTelemetryIndices = async () => { .catch((error) => console.error(error)); }; -export const fetchOpenTelemetryDocumentNames = (selectedOtelIndex: string) => async () => { +export const fetchOpenTelemetryDocumentNames = ( + selectedOtelIndex: string, + dataSourceMDSId: string +) => async () => { const http = getOSDHttp(); return http - .get(`${OBSERVABILITY_BASE}/metrics/otel/${selectedOtelIndex}/documentNames`) + .get( + `${OBSERVABILITY_BASE}/metrics/otel/${selectedOtelIndex}/documentNames/${ + dataSourceMDSId ?? '' + }` + ) .catch((error) => console.error(error)); }; @@ -251,6 +264,9 @@ export const metricSlice = createSlice({ setOtelDocumentNames: (state, { payload }) => { state.otelDocumentNames = payload; }, + setSelectedDataSourceMDSId: (state, { payload }) => { + state.dataSourceMDSId = payload; + }, }, }); @@ -268,21 +284,24 @@ export const { setSelectedDataSource, setOtelIndices, setOtelDocumentNames, + setSelectedDataSourceMDSId, } = metricSlice.actions; /** private actions */ export const { setMetrics, setMetric, setSortedIds } = metricSlice.actions; -const getAvailableAttributes = (id, metricIndex) => async (dispatch) => { +const getAvailableAttributes = (id, metricIndex, dataSourceMDSId: string) => async (dispatch) => { const { toasts } = coreRefs; const pplService = getPPLService(); - try { - const columnSchema = await pplService.fetch({ - query: 'describe ' + metricIndex + ' | fields COLUMN_NAME', - format: 'jdbc', - }); + const columnSchema = await pplService.fetch( + { + query: 'describe ' + metricIndex + ' | fields COLUMN_NAME', + format: 'jdbc', + }, + dataSourceMDSId + ); const availableAttributes = columnSchema.jsonData .map((sch) => sch.COLUMN_NAME) .filter((col) => col[0] !== '@'); @@ -294,12 +313,15 @@ const getAvailableAttributes = (id, metricIndex) => async (dispatch) => { } }; -export const addSelectedMetric = (metric: MetricType) => async (dispatch, getState) => { +export const addSelectedMetric = (metric: MetricType, dataSourceMDSId: string) => async ( + dispatch, + getState +) => { const currentSelectedIds = getState().metrics.selectedIds; if (currentSelectedIds.includes(metric.id)) return; if (metric.metricType === PROMQL_METRIC_SUBTYPE) { - await dispatch(getAvailableAttributes(metric.id, metric.index)); + await dispatch(getAvailableAttributes(metric.id, metric.index, dataSourceMDSId)); } await dispatch(selectMetric(metric)); }; @@ -368,3 +390,5 @@ export const otelIndexSelector = (state) => state.metrics.otelIndices; export const otelDocumentNamesSelector = (state) => state.metrics.otelDocumentNames; export const metricsReducers = metricSlice.reducer; + +export const selectedDataSourceMDSId = (state) => state.metrics.dataSourceMDSId; diff --git a/public/components/metrics/sidebar/__tests__/__snapshots__/sidebar.test.tsx.snap b/public/components/metrics/sidebar/__tests__/__snapshots__/sidebar.test.tsx.snap index 781f64daed..75dec06699 100644 --- a/public/components/metrics/sidebar/__tests__/__snapshots__/sidebar.test.tsx.snap +++ b/public/components/metrics/sidebar/__tests__/__snapshots__/sidebar.test.tsx.snap @@ -15,8 +15,36 @@ exports[`Side Bar Component renders Side Bar Component 1`] = `
>; setSelectedOTIndex: React.Dispatch>; additionalSelectedMetricId?: string; + dataSourceMDSId: string; } export const Sidebar = ({ selectedDataSource, @@ -47,6 +48,7 @@ export const Sidebar = ({ selectedOTIndex, setSelectedOTIndex, additionalSelectedMetricId, + dataSourceMDSId, }: SideBarMenuProps) => { const dispatch = useDispatch(); const [availableOTDocuments, setAvailableOTDocuments] = useState([]); @@ -59,36 +61,47 @@ export const Sidebar = ({ const additionalMetric = useSelector(selectMetricByIdSelector(additionalSelectedMetricId)); const otelIndices = useSelector(otelIndexSelector); + useEffect(() => { + (async function () { + setSelectedDataSource([]); + setSelectedOTIndex([]); + await dispatch(clearSelectedMetrics()); + })(); + }, [dataSourceMDSId]); + useEffect(() => { batch(() => { - dispatch(loadMetrics()); + dispatch(loadMetrics(dataSourceMDSId)); }); - }, [dispatch, selectedDataSource]); + }, [dispatch, selectedDataSource, dataSourceMDSId]); useEffect(() => { batch(() => { - dispatch(loadOTIndices()); + dispatch(loadOTIndices(dataSourceMDSId)); }); - }, [dispatch, selectedDataSource]); + }, [dispatch, selectedDataSource, dataSourceMDSId]); useEffect(() => { if (additionalMetric) { (async function () { await dispatch(clearSelectedMetrics()); - await dispatch(addSelectedMetric(additionalMetric)); + await dispatch(addSelectedMetric(additionalMetric, dataSourceMDSId)); })(); } - }, [additionalMetric?.id]); + }, [additionalMetric?.id, dataSourceMDSId]); const selectedMetricsList = useMemo(() => { return selectedMetricsIds.map((id) => selectedMetrics[id]).filter((m) => m); // filter away null entries - }, [selectedMetrics, selectedMetricsIds]); + }, [selectedMetrics, selectedMetricsIds, dataSourceMDSId]); useEffect(() => { if (selectedOTIndex.length > 0 && selectedDataSource[0]?.label === 'OpenTelemetry') { const fetchOtelDocuments = async () => { try { - const documentNames = await fetchOpenTelemetryDocumentNames(selectedOTIndex[0]?.label)(); + const documentNames = await fetchOpenTelemetryDocumentNames( + selectedOTIndex[0]?.label, + dataSourceMDSId + )(); const availableOtelDocuments = documentNames?.aggregations?.distinct_names?.buckets.map( (item: any) => { return { @@ -116,24 +129,30 @@ export const Sidebar = ({ }; fetchOtelDocuments(); } - }, [dispatch, selectedDataSource, selectedOTIndex]); + }, [dispatch, selectedDataSource, selectedOTIndex, dataSourceMDSId]); const indexPicker = useMemo(() => { const isOpenTelemetry = selectedDataSource[0]?.label === 'OpenTelemetry' ? true : false; if (isOpenTelemetry) { return ; } - }, [selectedDataSource]); + }, [selectedDataSource, dataSourceMDSId]); const availableMetrics = useMemo(() => { if (selectedDataSource[0]?.label === 'OpenTelemetry' && selectedOTIndex.length > 0) return promethuesMetrics; else if (selectedDataSource[0]?.label === 'Prometheus') return promethuesMetrics; else return []; - }, [promethuesMetrics, selectedDataSource, availableOTDocuments, selectedOTIndex]); + }, [ + promethuesMetrics, + selectedDataSource, + availableOTDocuments, + selectedOTIndex, + dataSourceMDSId, + ]); const handleAddMetric = (metric: any) => { - dispatch(addSelectedMetric(metric)); + dispatch(addSelectedMetric(metric, dataSourceMDSId)); }; const handleRemoveMetric = (metric: any) => { @@ -158,6 +177,7 @@ export const Sidebar = ({ headerName="Selected Metrics" handleClick={handleRemoveMetric} dataTestSubj="metricsListItems_selectedMetrics" + dataSourceMDSId={dataSourceMDSId} />
diff --git a/public/components/metrics/top_menu/metrics_export.tsx b/public/components/metrics/top_menu/metrics_export.tsx index dee03fc1a2..aa4c888ac1 100644 --- a/public/components/metrics/top_menu/metrics_export.tsx +++ b/public/components/metrics/top_menu/metrics_export.tsx @@ -12,37 +12,37 @@ import { EuiPopover, EuiPopoverFooter, } from '@elastic/eui'; +import { I18nProvider } from '@osd/i18n/react'; +import max from 'lodash/max'; import React, { useEffect } from 'react'; +import { render, unmountComponentAtNode } from 'react-dom'; import { useSelector } from 'react-redux'; -import { max } from 'lodash'; import semver from 'semver'; -import { render, unmountComponentAtNode } from 'react-dom'; -import { I18nProvider } from '@osd/i18n/react'; -import { MetricsExportPanel } from './metrics_export_panel'; -import { OSDSavedVisualizationClient } from '../../../services/saved_objects/saved_object_client/osd_saved_objects/saved_visualization'; +import { MountPoint } from '../../../../../../src/core/public'; +import { SavedObjectLoader } from '../../../../../../src/plugins/saved_objects/public'; +import { + OTEL_METRIC_SUBTYPE, + PPL_METRIC_SUBTYPE, + PROMQL_METRIC_SUBTYPE, +} from '../../../../common/constants/shared'; +import { SavedVisualization } from '../../../../common/types/explorer'; +import { MetricType } from '../../../../common/types/metrics'; +import { coreRefs } from '../../../framework/core_refs'; import { getSavedObjectsClient } from '../../../services/saved_objects/saved_object_client/client_factory'; +import { OSDSavedVisualizationClient } from '../../../services/saved_objects/saved_object_client/osd_saved_objects/saved_visualization'; +import { updateCatalogVisualizationQuery } from '../../common/query_utils'; import { addMultipleVizToPanels, isUuid, selectPanelList, } from '../../custom_panels/redux/panel_slice'; -import { MetricType } from '../../../../common/types/metrics'; +import { visualizationFromOtelMetric, visualizationFromPrometheusMetric } from '../helpers/utils'; import { dateSpanFilterSelector, selectedMetricsIdsSelector, selectedMetricsSelector, } from '../redux/slices/metrics_slice'; -import { coreRefs } from '../../../framework/core_refs'; -import { SavedVisualization } from '../../../../common/types/explorer'; -import { visualizationFromPrometheusMetric, visualizationFromOtelMetric } from '../helpers/utils'; -import { updateCatalogVisualizationQuery } from '../../common/query_utils'; -import { - OTEL_METRIC_SUBTYPE, - PROMQL_METRIC_SUBTYPE, - PPL_METRIC_SUBTYPE, -} from '../../../../common/constants/shared'; -import { SavedObjectLoader } from '../../../../../../src/plugins/saved_objects/public'; -import { MountPoint } from '../../../../../../src/core/public'; +import { MetricsExportPanel } from './metrics_export_panel'; const Savebutton = ({ setIsPanelOpen, diff --git a/public/components/metrics/view/metrics_grid.tsx b/public/components/metrics/view/metrics_grid.tsx index 7daa0908b7..32cc8709d4 100644 --- a/public/components/metrics/view/metrics_grid.tsx +++ b/public/components/metrics/view/metrics_grid.tsx @@ -3,13 +3,13 @@ * SPDX-License-Identifier: Apache-2.0 */ -import React, { useEffect, useMemo } from 'react'; import { EuiDragDropContext, EuiDraggable, EuiDroppable } from '@elastic/eui'; -import { useObservable } from 'react-use'; +import React, { useEffect, useMemo } from 'react'; import { connect } from 'react-redux'; +import { useObservable } from 'react-use'; import { CoreStart } from '../../../../../../src/core/public'; -import { VisualizationContainer } from '../../custom_panels/panel_modules/visualization_container'; import { updateCatalogVisualizationQuery } from '../../common/query_utils'; +import { VisualizationContainer } from '../../custom_panels/panel_modules/visualization_container'; import { allMetricsSelector, dateSpanFilterSelector, @@ -19,21 +19,22 @@ import { selectedMetricsSelector, } from '../redux/slices/metrics_slice'; -import './metrics_grid.scss'; -import { coreRefs } from '../../../framework/core_refs'; import { - PROMQL_METRIC_SUBTYPE, OTEL_METRIC_SUBTYPE, + PROMQL_METRIC_SUBTYPE, observabilityLogsID, } from '../../../../common/constants/shared'; +import { coreRefs } from '../../../framework/core_refs'; import { MetricsEditInline } from '../sidebar/metrics_edit_inline'; import { EmptyMetricsView } from './empty_view'; +import './metrics_grid.scss'; // HOC container to provide dynamic width for Grid layout interface MetricsGridProps { chrome: CoreStart['chrome']; moveToEvents: (savedVisualizationId: string) => any; + dataSourceMDSId: string; } const visualizationFromPromethesMetric = (metric, dateSpanFilter): SavedVisualizationType => ({ @@ -87,7 +88,14 @@ const navigateToEventExplorerVisualization = (savedVisualizationId: string) => { window.location.assign(`${observabilityLogsID}#/explorer/${savedVisualizationId}`); }; -export const InnerGridVisualization = ({ id, idx, dateSpanFilter, metric, refresh }) => { +export const InnerGridVisualization = ({ + id, + idx, + dateSpanFilter, + metric, + refresh, + dataSourceMDSId, +}) => { if (!metric) return <>; return ( @@ -114,6 +122,7 @@ export const InnerGridVisualization = ({ id, idx, dateSpanFilter, metric, refres actionMenuType="metricsGrid" metricType={metric.subType} panelVisualization={metric} + dataSourceMDSId={dataSourceMDSId} /> ); @@ -128,6 +137,7 @@ export const InnerMetricsGrid = ({ selectedMetricsIds, moveMetric, allMetrics, + dataSourceMDSId, }: MetricsGridProps) => { const { chrome } = coreRefs; const isLocked = useObservable(chrome!.getIsNavDrawerLocked$()); @@ -148,6 +158,7 @@ export const InnerMetricsGrid = ({ key={id} dateSpanFilter={dateSpanFilter} metric={metric} + dataSourceMDSId={dataSourceMDSId} /> ); }); diff --git a/public/services/requests/ppl.ts b/public/services/requests/ppl.ts index 2ab46e8b32..38b02085e2 100644 --- a/public/services/requests/ppl.ts +++ b/public/services/requests/ppl.ts @@ -19,11 +19,15 @@ export default class PPLService { query: string; format: string; }, + dataSourceMDSId?: string, errorHandler?: (error: any) => void ) => { return this.http .post(`${PPL_BASE}${PPL_SEARCH}`, { body: JSON.stringify(params), + query: { + dataSourceMDSId, + }, }) .catch((error) => { console.error('fetch error: ', error.body); diff --git a/server/adaptors/metrics/metrics_analytics_adaptor.ts b/server/adaptors/metrics/metrics_analytics_adaptor.ts index 1fde1e3956..ab090b1d32 100644 --- a/server/adaptors/metrics/metrics_analytics_adaptor.ts +++ b/server/adaptors/metrics/metrics_analytics_adaptor.ts @@ -6,9 +6,16 @@ import { ILegacyScopedClusterClient } from '../../../../../src/core/server'; export class MetricsAnalyticsAdaptor { - fetch = async function (client: ILegacyScopedClusterClient, query: any, index: string) { + fetch = async function (client, query: any, index: string, dataSourceMDSId?: string) { try { - const response = await client.callAsCurrentUser('search', { + let response; + if (dataSourceMDSId) { + response = await client.callAPI('search', { + body: query, + index, + }); + } + response = await client.callAsCurrentUser('search', { body: query, index, }); @@ -25,7 +32,8 @@ export class MetricsAnalyticsAdaptor { startTime: string, endTime: string, documentName: string, - index: string + index: string, + dataSourceMDSId?: string ) => { const metricsQuery = { size: 0, @@ -79,7 +87,7 @@ export class MetricsAnalyticsAdaptor { }; try { - const response = await this.fetch(client, metricsQuery, index); + const response = await this.fetch(client, metricsQuery, index, dataSourceMDSId); return response.aggregations; } catch (error) { throw new Error('Fetch Bin count Error:' + error); @@ -89,7 +97,8 @@ export class MetricsAnalyticsAdaptor { queryToFetchSampleDocument = async ( client: ILegacyScopedClusterClient, documentName: string, - index: string + index: string, + dataSourceMDSId?: string ) => { const metricsQuery = { size: 1, @@ -109,14 +118,13 @@ export class MetricsAnalyticsAdaptor { }; try { - const response = await this.fetch(client, metricsQuery, index); + const response = await this.fetch(client, metricsQuery, index, dataSourceMDSId); return response; } catch (error) { throw new Error('Fetch Sample Document Error:' + error); } }; - - queryToFetchDocumentNames = async (client: ILegacyScopedClusterClient, index: string) => { + queryToFetchDocumentNames = async (client, index: string, dataSourceMDSId?: string) => { const metricsQuery = { size: 0, query: { @@ -141,7 +149,7 @@ export class MetricsAnalyticsAdaptor { }; try { - const response = await this.fetch(client, metricsQuery, index); + const response = await this.fetch(client, metricsQuery, index, dataSourceMDSId); return response; } catch (error) { throw new Error('Fetch Document Names Error:' + error); diff --git a/server/routes/data_connections/data_connections_router.ts b/server/routes/data_connections/data_connections_router.ts index 5d36bf6bb3..3ecd3d4348 100644 --- a/server/routes/data_connections/data_connections_router.ts +++ b/server/routes/data_connections/data_connections_router.ts @@ -221,9 +221,9 @@ export function registerDataConnectionsRoute(router: IRouter, dataSourceEnabled: const client = await context.dataSource.opensearch.legacy.getClient(dataSourceMDSId); dataConnectionsresponse = await client.callAPI('ppl.getDataConnections'); } else { - dataConnectionsresponse = await context.observability_plugin.observabilityClient - .asScoped(request) - .callAsCurrentUser('ppl.getDataConnections'); + dataConnectionsresponse = await context.observability_plugin.observabilityClient.asScoped( + request + ); } return response.ok({ body: dataConnectionsresponse, diff --git a/server/routes/index.ts b/server/routes/index.ts index ca5f9ad469..499fb29520 100644 --- a/server/routes/index.ts +++ b/server/routes/index.ts @@ -51,7 +51,7 @@ export function setupRoutes({ const queryService = new QueryService(client); registerSqlRoute(router, queryService); - registerMetricsRoute(router); + registerMetricsRoute(router, dataSourceEnabled); registerIntegrationsRoute(router); registerDataConnectionsRoute(router, dataSourceEnabled); registerDatasourcesRoute(router, dataSourceEnabled); diff --git a/server/routes/metrics/metrics_rounter.ts b/server/routes/metrics/metrics_rounter.ts index 3891d392d7..595cd06363 100644 --- a/server/routes/metrics/metrics_rounter.ts +++ b/server/routes/metrics/metrics_rounter.ts @@ -5,17 +5,13 @@ import { ResponseError } from '@opensearch-project/opensearch/lib/errors'; import { schema } from '@osd/config-schema'; -import { - ILegacyScopedClusterClient, - IOpenSearchDashboardsResponse, - IRouter, -} from '../../../../../src/core/server'; +import { IOpenSearchDashboardsResponse, IRouter } from '../../../../../src/core/server'; +import { DATA_PREPPER_INDEX_NAME } from '../../../common/constants/metrics'; import { OBSERVABILITY_BASE } from '../../../common/constants/shared'; -import { addClickToMetric, getMetrics } from '../../common/metrics/metrics_helper'; import { MetricsAnalyticsAdaptor } from '../../adaptors/metrics/metrics_analytics_adaptor'; -import { DATA_PREPPER_INDEX_NAME } from '../../../common/constants/metrics'; +import { addClickToMetric, getMetrics } from '../../common/metrics/metrics_helper'; -export function registerMetricsRoute(router: IRouter) { +export function registerMetricsRoute(router: IRouter, dataSourceEnabled: boolean) { const metricsAnalyticsBackend = new MetricsAnalyticsAdaptor(); router.get( @@ -73,8 +69,12 @@ export function registerMetricsRoute(router: IRouter) { router.get( { - path: `${OBSERVABILITY_BASE}/search/indices`, - validate: {}, + path: `${OBSERVABILITY_BASE}/search/indices/{dataSourceMDSId?}`, + validate: { + params: schema.object({ + dataSourceMDSId: schema.maybe(schema.string({ defaultValue: '' })), + }), + }, }, async (context, request, response) => { const params = { @@ -82,10 +82,17 @@ export function registerMetricsRoute(router: IRouter) { index: DATA_PREPPER_INDEX_NAME, }; try { - const resp = await context.core.opensearch.legacy.client.callAsCurrentUser( - 'cat.indices', - params - ); + let resp; + const dataSourceMDSId = request.params.dataSourceMDSId; + if (dataSourceEnabled && dataSourceMDSId) { + const client = context.dataSource.opensearch.legacy.getClient(dataSourceMDSId); + resp = await client.callAPI('cat.indices', params); + } else { + resp = await context.core.opensearch.legacy.client.callAsCurrentUser( + 'cat.indices', + params + ); + } return response.ok({ body: resp, }); @@ -101,10 +108,11 @@ export function registerMetricsRoute(router: IRouter) { router.get( { - path: `${OBSERVABILITY_BASE}/metrics/otel/{index}/documentNames`, + path: `${OBSERVABILITY_BASE}/metrics/otel/{index}/documentNames/{dataSourceMDSId?}`, validate: { params: schema.object({ index: schema.string(), + dataSourceMDSId: schema.maybe(schema.string({ defaultValue: '' })), }), }, }, @@ -113,14 +121,20 @@ export function registerMetricsRoute(router: IRouter) { request, response ): Promise> => { - const opensearchNotebooksClient: ILegacyScopedClusterClient = context.observability_plugin.observabilityClient.asScoped( - request - ); - + const dataSourceMDSId = request.params.dataSourceMDSId; + let opensearchNotebooksClient; + if (dataSourceEnabled && dataSourceMDSId) { + opensearchNotebooksClient = context.dataSource.opensearch.legacy.getClient(dataSourceMDSId); + } else { + opensearchNotebooksClient = context.observability_plugin.observabilityClient.asScoped( + request + ); + } try { const resp = await metricsAnalyticsBackend.queryToFetchDocumentNames( opensearchNotebooksClient, - request.params.index + request.params.index, + dataSourceMDSId ); return response.ok({ body: resp, @@ -137,11 +151,12 @@ export function registerMetricsRoute(router: IRouter) { router.get( { - path: `${OBSERVABILITY_BASE}/metrics/otel/{index}/{histogramSampleDocument}`, + path: `${OBSERVABILITY_BASE}/metrics/otel/{index}/{histogramSampleDocument}/{dataSourceMDSId?}`, validate: { params: schema.object({ histogramSampleDocument: schema.string(), index: schema.string(), + dataSourceMDSId: schema.maybe(schema.string({ defaultValue: '' })), }), }, }, @@ -150,15 +165,21 @@ export function registerMetricsRoute(router: IRouter) { request, response ): Promise> => { - const opensearchNotebooksClient: ILegacyScopedClusterClient = context.observability_plugin.observabilityClient.asScoped( - request - ); - + const dataSourceMDSId = request.params.dataSourceMDSId; + let opensearchNotebooksClient; + if (dataSourceEnabled && dataSourceMDSId) { + opensearchNotebooksClient = context.dataSource.opensearch.legacy.getClient(dataSourceMDSId); + } else { + opensearchNotebooksClient = context.observability_plugin.observabilityClient.asScoped( + request + ); + } try { const resp = await metricsAnalyticsBackend.queryToFetchSampleDocument( opensearchNotebooksClient, request.params.histogramSampleDocument, - request.params.index + request.params.index, + dataSourceMDSId ); return response.ok({ body: resp.hits, @@ -184,6 +205,7 @@ export function registerMetricsRoute(router: IRouter) { endTime: schema.string(), documentName: schema.string(), index: schema.string(), + dataSourceMDSId: schema.maybe(schema.string({ defaultValue: '' })), }), }, }, @@ -192,10 +214,16 @@ export function registerMetricsRoute(router: IRouter) { request, response ): Promise> => { - const opensearchNotebooksClient: ILegacyScopedClusterClient = context.observability_plugin.observabilityClient.asScoped( - request - ); + const dataSourceMDSId = request.body.dataSourceMDSId; + let opensearchNotebooksClient; + if (dataSourceEnabled && dataSourceMDSId) { + opensearchNotebooksClient = context.dataSource.opensearch.legacy.getClient(dataSourceMDSId); + } else { + opensearchNotebooksClient = context.observability_plugin.observabilityClient.asScoped( + request + ); + } try { const resp = await metricsAnalyticsBackend.queryToFetchBinCount( opensearchNotebooksClient, @@ -204,7 +232,8 @@ export function registerMetricsRoute(router: IRouter) { request.body.startTime, request.body.endTime, request.body.documentName, - request.body.index + request.body.index, + dataSourceMDSId ); return response.ok({ body: resp, diff --git a/server/routes/ppl.ts b/server/routes/ppl.ts index 78d1d0dd01..d9943aca69 100644 --- a/server/routes/ppl.ts +++ b/server/routes/ppl.ts @@ -4,9 +4,9 @@ */ import { schema } from '@osd/config-schema'; -import { IRouter, IOpenSearchDashboardsResponse, ResponseError } from '../../../../src/core/server'; -import { PPLFacet } from '../services/facets/ppl_facet'; +import { IOpenSearchDashboardsResponse, IRouter, ResponseError } from '../../../../src/core/server'; import { PPL_BASE, PPL_SEARCH } from '../../common/constants/shared'; +import { PPLFacet } from '../services/facets/ppl_facet'; export function registerPplRoute({ router, facet }: { router: IRouter; facet: PPLFacet }) { router.post( @@ -17,10 +17,13 @@ export function registerPplRoute({ router, facet }: { router: IRouter; facet: PP query: schema.string(), format: schema.string(), }), + query: schema.object({ + dataSourceMDSId: schema.maybe(schema.string({ defaultValue: '' })), + }), }, }, async (context, req, res): Promise> => { - const queryRes: any = await facet.describeQuery(req); + const queryRes: any = await facet.describeQuery(context, req); if (queryRes.success) { const result: any = { body: { diff --git a/server/services/facets/ppl_facet.ts b/server/services/facets/ppl_facet.ts index 1bed4f93ff..dc90645883 100644 --- a/server/services/facets/ppl_facet.ts +++ b/server/services/facets/ppl_facet.ts @@ -3,7 +3,6 @@ * SPDX-License-Identifier: Apache-2.0 */ -import _ from 'lodash'; import { PPLDataSource } from '../../adaptors/ppl_datasource'; export class PPLFacet { @@ -11,12 +10,13 @@ export class PPLFacet { this.client = client; } - private fetch = async (request: any, format: string, responseFormat: string) => { + private fetch = async (context: any, request: any, format: string, _responseFormat: string) => { const res = { success: false, data: {}, }; try { + const dataSourceMDSId = request.query.dataSourceMDSId; const params = { body: { query: request.body.query, @@ -25,7 +25,13 @@ export class PPLFacet { if (request.body.format !== 'jdbc') { params.format = request.body.format; } - const queryRes = await this.client.asScoped(request).callAsCurrentUser(format, params); + let queryRes; + if (dataSourceMDSId) { + const mdsClient = context.dataSource.opensearch.legacy.getClient(dataSourceMDSId); + queryRes = await mdsClient.callAPI(format, params); + } else { + queryRes = await this.client.asScoped(request).callAsCurrentUser(format, params); + } const pplDataSource = new PPLDataSource(queryRes, request.body.format); res.success = true; res.data = pplDataSource.getDataSource(); @@ -36,7 +42,7 @@ export class PPLFacet { return res; }; - describeQuery = async (request: any) => { - return this.fetch(request, 'ppl.pplQuery', 'json'); + describeQuery = async (context, request: any) => { + return this.fetch(context, request, 'ppl.pplQuery', 'json'); }; } From 1b238c393c1649029209c3584cf92ac6757e8ab3 Mon Sep 17 00:00:00 2001 From: sumukhswamy Date: Mon, 10 Jun 2024 23:39:41 -0700 Subject: [PATCH 2/3] added linter fix Signed-off-by: sumukhswamy --- .../visualization_container/visualization_container.tsx | 2 +- public/components/metrics/redux/slices/metrics_slice.ts | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/public/components/custom_panels/panel_modules/visualization_container/visualization_container.tsx b/public/components/custom_panels/panel_modules/visualization_container/visualization_container.tsx index 8a9b041df8..6759ad3a64 100644 --- a/public/components/custom_panels/panel_modules/visualization_container/visualization_container.tsx +++ b/public/components/custom_panels/panel_modules/visualization_container/visualization_container.tsx @@ -41,7 +41,7 @@ import { fetchVisualizationById, renderCatalogVisualization, renderOpenTelemetryVisualization, - renderSavedVisualization + renderSavedVisualization, } from '../../helpers/utils'; import './visualization_container.scss'; diff --git a/public/components/metrics/redux/slices/metrics_slice.ts b/public/components/metrics/redux/slices/metrics_slice.ts index aad5f03571..e262f5237f 100644 --- a/public/components/metrics/redux/slices/metrics_slice.ts +++ b/public/components/metrics/redux/slices/metrics_slice.ts @@ -12,7 +12,11 @@ import { REDUX_SLICE_METRICS, SAVED_VISUALIZATION, } from '../../../../../common/constants/metrics'; -import { OBSERVABILITY_BASE, PPL_METRIC_SUBTYPE, PROMQL_METRIC_SUBTYPE } from '../../../../../common/constants/shared'; +import { + OBSERVABILITY_BASE, + PPL_METRIC_SUBTYPE, + PROMQL_METRIC_SUBTYPE, +} from '../../../../../common/constants/shared'; import { MetricType } from '../../../../../common/types/metrics'; import { getOSDHttp, getPPLService } from '../../../../../common/utils'; import { coreRefs } from '../../../../framework/core_refs'; From 612c390fb9cfdc663a5d69c2803ded5dccad8f6d Mon Sep 17 00:00:00 2001 From: sumukhswamy Date: Mon, 10 Jun 2024 23:47:12 -0700 Subject: [PATCH 3/3] added linter fix Signed-off-by: sumukhswamy --- public/components/metrics/helpers/utils.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/components/metrics/helpers/utils.tsx b/public/components/metrics/helpers/utils.tsx index 225ed548ff..67e7cf963d 100644 --- a/public/components/metrics/helpers/utils.tsx +++ b/public/components/metrics/helpers/utils.tsx @@ -60,7 +60,7 @@ export const mergeLayoutAndMetrics = ( for (let i = 0; i < newVisualizationList.length; i++) { for (let j = 0; j < layout.length; j++) { - if (newVisualizationList[i].id == layout[j].i) { + if (newVisualizationList[i].id === layout[j].i) { newPanelVisualizations.push({ ...newVisualizationList[i], x: layout[j].x,