Skip to content

Commit

Permalink
Feature/metrics multiline display (opensearch-project#899)
Browse files Browse the repository at this point in the history 
* Working multi line viz with new sql

Signed-off-by: Kavitha Conjeevaram Mohan <[email protected]>
Signed-off-by: Peter Fitzgibbons <[email protected]>

* Cleanup conflicts and fn. arguments

Signed-off-by: Peter Fitzgibbons <[email protected]>

* Correct metric check fn.

Signed-off-by: Peter Fitzgibbons <[email protected]>

* Clean Code

Signed-off-by: Peter Fitzgibbons <[email protected]>

* Clean Code

Signed-off-by: Peter Fitzgibbons <[email protected]>

* merge dateTime convert function

Signed-off-by: Kavitha Conjeevaram Mohan <[email protected]>

* Update snapshots

Signed-off-by: Kavitha Conjeevaram Mohan <[email protected]>

* Fix cypress test 8_metrics_analytics.spec.js to match removal of "Recently Created" sidebar section.

Fix Metric search to be case-insensitive.

Signed-off-by: Peter Fitzgibbons <[email protected]>

* Fix several test snapshots.
Fix metrics_grid.test coreRefs mock

Signed-off-by: Peter Fitzgibbons <[email protected]>

---------

Signed-off-by: Kavitha Conjeevaram Mohan <[email protected]>
Signed-off-by: Peter Fitzgibbons <[email protected]>
Co-authored-by: Kavitha Conjeevaram Mohan <[email protected]>
  • Loading branch information
@pjfitzgibbons @kavithacm
pjfitzgibbons and kavithacm authored Aug 15, 2023
1 parent e633b72 commit e947d4b
Show file tree
Hide file tree
Showing 10 changed files with 525 additions and 338 deletions.
494 changes: 274 additions & 220 deletions .cypress/integration/8_metrics_analytics.spec.js
View file

Large diffs are not rendered by default.

181 changes: 143 additions & 38 deletions public/components/custom_panels/helpers/utils.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import dateMath from '@elastic/datemath';
import { ShortDate } from '@elastic/eui';
import { DurationRange } from '@elastic/eui/src/components/date_picker/types';
import _, { isEmpty } from 'lodash';
import _, { castArray, forEach, isEmpty } from 'lodash';
import { Moment } from 'moment-timezone';
import React from 'react';
import { Layout } from 'react-grid-layout';
Expand All @@ -30,6 +30,7 @@ import { SavedObjectsActions } from '../../../services/saved_objects/saved_objec
import { ObservabilitySavedVisualization } from '../../../services/saved_objects/saved_object_client/types';
import { getDefaultVisConfig } from '../../event_analytics/utils';
import { Visualization } from '../../visualizations/visualization';
import { MetricType } from '../../../../common/types/metrics';

/*
* "Utils" This file contains different reused functions in operational panels
Expand All @@ -51,14 +52,23 @@ export const isNameValid = (name: string) => {
};

// DateTime convertor to required format
export const convertDateTime = (datetime: string, isStart = true, formatted = true) => {
export const convertDateTime = (
datetime: string,
isStart = true,
formatted = true,
isMetrics: boolean = false
) => {
let returnTime: undefined | Moment;
if (isStart) {
returnTime = dateMath.parse(datetime);
} else {
returnTime = dateMath.parse(datetime, { roundUp: true });
}

if (isMetrics) {
const myDate = new Date(returnTime._d); // Your timezone!
const epochTime = myDate.getTime() / 1000.0;
return Math.round(epochTime);
}
if (formatted) return returnTime!.utc().format(PPL_DATE_FORMAT);
return returnTime;
};
Expand Down Expand Up @@ -202,14 +212,19 @@ export const getQueryResponse = (
setIsLoading: React.Dispatch<React.SetStateAction<boolean>>,
setIsError: React.Dispatch<React.SetStateAction<VizContainerError>>,
filterQuery = '',
timestampField = 'timestamp'
timestampField = 'timestamp',
metricVisualization = false
) => {
setIsLoading(true);
setIsError({} as VizContainerError);

let finalQuery = '';
try {
finalQuery = queryAccumulator(query, timestampField, startTime, endTime, filterQuery);
if (!metricVisualization) {
finalQuery = queryAccumulator(query, timestampField, startTime, endTime, filterQuery);
} else {
finalQuery = query;
}
} catch (error) {
const errorMessage = 'Issue in building final query';
setIsError({ errorMessage });
Expand Down Expand Up @@ -311,44 +326,105 @@ const createCatalogVisualizationMetaData = (
};
};

const updateCatalogVisualizationQuery = ({
catalogSourceName,
catalogTableName,
aggregation,
attributesGroupBy,
startTime,
endTime,
spanParam,
}: {
catalogSourceName: string;
catalogTableName: string;
aggregation: string;
attributesGroupBy: string[];
startTime: string;
endTime: string;
spanParam: string | undefined;
}) => {
const attributesGroupString = attributesGroupBy.toString();
const startEpochTime = convertDateTime(startTime, true, false, true);
const endEpochTime = convertDateTime(endTime, false, false, true);
const promQuery =
attributesGroupBy.length === 0
? catalogTableName
: `${aggregation} by(${attributesGroupString}) (${catalogTableName})`;

return `source = ${catalogSourceName}.query_range('${promQuery}', ${startEpochTime}, ${endEpochTime}, '${spanParam}')`;
};

// Creates a catalogVisualization for a runtime catalog based PPL query and runs getQueryResponse
export const renderCatalogVisualization = async (
http: CoreStart['http'],
pplService: PPLService,
catalogSource: string,
startTime: string,
endTime: string,
filterQuery: string,
spanParam: string | undefined,
setVisualizationTitle: React.Dispatch<React.SetStateAction<string>>,
setVisualizationType: React.Dispatch<React.SetStateAction<string>>,
setVisualizationData: React.Dispatch<React.SetStateAction<Plotly.Data[]>>,
setVisualizationMetaData: React.Dispatch<React.SetStateAction<undefined>>,
setIsLoading: React.Dispatch<React.SetStateAction<boolean>>,
setIsError: React.Dispatch<React.SetStateAction<VizContainerError>>,
spanResolution?: string
) => {
export const renderCatalogVisualization = async ({
http,
pplService,
catalogSource,
startTime,
endTime,
filterQuery,
spanParam,
setVisualizationTitle,
setVisualizationType,
setVisualizationData,
setVisualizationMetaData,
setIsLoading,
setIsError,
spanResolution,
queryMetaData,
}: {
http: CoreStart['http'];
pplService: PPLService;
catalogSource: string;
startTime: string;
endTime: string;
filterQuery: string;
spanParam: string | undefined;
setVisualizationTitle: React.Dispatch<React.SetStateAction<string>>;
setVisualizationType: React.Dispatch<React.SetStateAction<string>>;
setVisualizationData: React.Dispatch<React.SetStateAction<Plotly.Data[]>>;
setVisualizationMetaData: React.Dispatch<React.SetStateAction<undefined>>;
setIsLoading: React.Dispatch<React.SetStateAction<boolean>>;
setIsError: React.Dispatch<React.SetStateAction<VizContainerError>>;
spanResolution?: string;
queryMetaData?: MetricType;
}) => {
setIsLoading(true);
setIsError({} as VizContainerError);

const visualizationType = 'line';
const visualizationTimeField = '@timestamp';
let visualizationQuery = `source = ${catalogSource} | stats avg(@value) by span(${visualizationTimeField},1h)`;

if (spanParam !== undefined) {
visualizationQuery = updateQuerySpanInterval(
visualizationQuery,
visualizationTimeField,
spanParam
);
}
const catalogSourceName = catalogSource.split('.')[0];
const catalogTableName = catalogSource.split('.')[1];

const defaultAggregation = 'avg'; // pass in attributes to this function
const attributes: string[] = [];

const visualizationQuery = updateCatalogVisualizationQuery({
catalogSourceName,
catalogTableName,
aggregation: defaultAggregation,
attributesGroupBy: attributes,
startTime,
endTime,
spanParam,
});

const visualizationMetaData = createCatalogVisualizationMetaData(
catalogSource,
visualizationQuery,
visualizationType,
visualizationTimeField
);

visualizationMetaData.user_configs = {
layoutConfig: {
height: 390,
margin: { t: 5 },
legend: { orientation: 'h', yanchor: 'top', x: 0.0, y: -0.4 },
},
};

setVisualizationTitle(catalogSource);
setVisualizationType(visualizationType);

Expand All @@ -364,7 +440,8 @@ export const renderCatalogVisualization = async (
setIsLoading,
setIsError,
filterQuery,
visualizationTimeField
visualizationTimeField,
true
);
};

Expand Down Expand Up @@ -397,9 +474,7 @@ export const parseSavedVisualizations = (
timeField: visualization.savedVisualization.selected_timestamp.name,
selected_date_range: visualization.savedVisualization.selected_date_range,
selected_fields: visualization.savedVisualization.selected_fields,
user_configs: visualization.savedVisualization.user_configs
? JSON.parse(visualization.savedVisualization.user_configs)
: {},
user_configs: visualization.savedVisualization.user_configs || {},
sub_type: visualization.savedVisualization.hasOwnProperty('sub_type')
? visualization.savedVisualization.sub_type
: '',
Expand Down Expand Up @@ -462,16 +537,43 @@ export const isPPLFilterValid = (
return true;
};

export const processMetricsData = (schema: any, dataConfig: any) => {
if (isEmpty(schema)) return {};
if (
schema.length === 3 &&
schema.every((schemaField) => ['@labels', '@value', '@timestamp'].includes(schemaField.name))
) {
return prepareMetricsData(schema, dataConfig);
}
return {};
};

export const prepareMetricsData = (schema: any, dataConfig: any) => {
const metricBreakdown: any[] = [];
const metricSeries: any[] = [];
const metricDimension: any[] = [];

forEach(schema, (field) => {
if (field.name === '@timestamp')
metricDimension.push({ name: '@timestamp', label: '@timestamp' });
if (field.name === '@labels') metricBreakdown.push({ name: '@labels', customLabel: '@labels' });
if (field.name === '@value') metricSeries.push({ name: '@value', label: '@value' });
});

return {
breakdowns: metricBreakdown,
series: metricSeries,
dimensions: metricDimension,
span: {},
};
};

// Renders visualization in the vizualization container component
export const displayVisualization = (metaData: any, data: any, type: string) => {
if (metaData === undefined || isEmpty(metaData)) {
return <></>;
}

if (metaData.user_configs !== undefined && metaData.user_configs !== '') {
metaData.user_configs = JSON.parse(metaData.user_configs);
}

const dataConfig = { ...(metaData.user_configs?.dataConfig || {}) };
const hasBreakdowns = !_.isEmpty(dataConfig.breakdowns);
const realTimeParsedStats = {
Expand All @@ -487,13 +589,16 @@ export const displayVisualization = (metaData: any, data: any, type: string) =>
);
}

const finalDataConfig = {
let finalDataConfig = {
...dataConfig,
...realTimeParsedStats,
dimensions: finalDimensions,
breakdowns,
};

// add metric specific overriding
finalDataConfig = { ...finalDataConfig, ...processMetricsData(data.schema, finalDataConfig) };

const mixedUserConfigs = {
availabilityConfig: {
...(metaData.user_configs?.availabilityConfig || {}),
Expand Down
22 changes: 9 additions & 13 deletions ...omponents/custom_panels/panel_modules/visualization_container/visualization_container.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,14 @@ import {
} from '@elastic/eui';
import React, { useEffect, useMemo, useState } from 'react';
import _ from 'lodash';
import { CoreStart } from '../../../../../../../src/core/public';
import PPLService from '../../../../services/requests/ppl';
import {
displayVisualization,
renderCatalogVisualization,
renderSavedVisualization,
} from '../../helpers/utils';
import './visualization_container.scss';
import { VizContainerError } from '../../../../../common/types/custom_panels';
import { coreRefs } from '../../../../framework/core_refs';

/*
* Visualization container - This module is a placeholder to add visualizations in react-grid-layout
Expand All @@ -58,11 +57,9 @@ import { VizContainerError } from '../../../../../common/types/custom_panels';
*/

interface Props {
http: CoreStart['http'];
editMode: boolean;
visualizationId: string;
savedVisualizationId: string;
pplService: PPLService;
fromTime: string;
toTime: string;
onRefresh: boolean;
Expand All @@ -77,11 +74,9 @@ interface Props {
}

export const VisualizationContainer = ({
http,
editMode,
visualizationId,
savedVisualizationId,
pplService,
fromTime,
toTime,
onRefresh,
Expand All @@ -103,6 +98,7 @@ export const VisualizationContainer = ({
const [isError, setIsError] = useState({} as VizContainerError);
const onActionsMenuClick = () => setIsPopoverOpen((currPopoverOpen) => !currPopoverOpen);
const closeActionsMenu = () => setIsPopoverOpen(false);
const { http, pplService } = coreRefs;

const [isModalVisible, setIsModalVisible] = useState(false);
const [modalContent, setModalContent] = useState(<></>);
Expand Down Expand Up @@ -217,21 +213,21 @@ export const VisualizationContainer = ({

const loadVisaulization = async () => {
if (catalogVisualization)
await renderCatalogVisualization(
await renderCatalogVisualization({
http,
pplService,
savedVisualizationId,
fromTime,
toTime,
pplFilterValue,
catalogSource: savedVisualizationId,
startTime: fromTime,
endTime: toTime,
filterQuery: pplFilterValue,
spanParam,
setVisualizationTitle,
setVisualizationType,
setVisualizationData,
setVisualizationMetaData,
setIsLoading,
setIsError
);
setIsError,
});
else
await renderSavedVisualization(
http,
Expand Down
5 changes: 4 additions & 1 deletion public/components/metrics/redux/slices/metrics_slice.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,10 @@ export const {
export const availableMetricsSelector = (state) =>
state.metrics.metrics
.filter((metric) => !state.metrics.selected.includes(metric.id))
.filter((metric) => state.metrics.search === '' || metric.name.includes(state.metrics.search));
.filter(
(metric) =>
state.metrics.search === '' || metric.name.match(new RegExp(state.metrics.search, 'i'))
);

export const selectedMetricsSelector = (state) =>
state.metrics.selected.map((id) => state.metrics.metrics.find((metric) => metric.id === id));
Expand Down
Loading

0 comments on commit e947d4b

Please sign in to comment.