From 5a90273be61effb65d8e6fe0cd11db8e60bdcce8 Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Tue, 1 Nov 2022 13:12:39 -0700 Subject: [PATCH] [2.x] Add log pattern table (#1187) (#1212) Signed-off-by: Joshua Li --- .../common/constants/explorer.ts | 3 + .../common/types/explorer.ts | 12 +- .../application_analytics/helpers/utils.tsx | 9 +- .../event_analytics/explorer/explorer.tsx | 160 +- .../explorer/log_patterns/patterns_table.tsx | 128 ++ .../__snapshots__/field.test.tsx.snap | 100 ++ .../__snapshots__/sidebar.test.tsx.snap | 1506 ++++++++++++++++- .../explorer/sidebar/field.tsx | 34 +- .../explorer/sidebar/sidebar.tsx | 18 +- .../components/event_analytics/home/home.tsx | 2 + .../event_analytics/hooks/use_fetch_events.ts | 22 +- .../hooks/use_fetch_patterns.ts | 139 ++ .../hooks/use_fetch_visualizations.ts | 6 + .../redux/slices/patterns_slice.ts | 37 + .../redux/slices/query_slice.ts | 3 + .../public/framework/redux/reducers/index.ts | 2 + 16 files changed, 2131 insertions(+), 50 deletions(-) create mode 100644 dashboards-observability/public/components/event_analytics/explorer/log_patterns/patterns_table.tsx create mode 100644 dashboards-observability/public/components/event_analytics/hooks/use_fetch_patterns.ts create mode 100644 dashboards-observability/public/components/event_analytics/redux/slices/patterns_slice.ts diff --git a/dashboards-observability/common/constants/explorer.ts b/dashboards-observability/common/constants/explorer.ts index fbe09f13b..aa769d3bb 100644 --- a/dashboards-observability/common/constants/explorer.ts +++ b/dashboards-observability/common/constants/explorer.ts @@ -11,6 +11,7 @@ export const RAW_QUERY = 'rawQuery'; export const FINAL_QUERY = 'finalQuery'; export const SELECTED_DATE_RANGE = 'selectedDateRange'; export const INDEX = 'index'; +export const SELECTED_PATTERN = 'selectedPattern'; export const SELECTED_TIMESTAMP = 'selectedTimestamp'; export const SELECTED_FIELDS = 'selectedFields'; export const UNSELECTED_FIELDS = 'unselectedFields'; @@ -74,6 +75,8 @@ export const REDUX_EXPL_SLICE_FIELDS = 'fields'; export const REDUX_EXPL_SLICE_QUERY_TABS = 'queryTabs'; export const REDUX_EXPL_SLICE_VISUALIZATION = 'explorerVisualization'; export const REDUX_EXPL_SLICE_COUNT_DISTRIBUTION = 'countDistributionVisualization'; +export const REDUX_EXPL_SLICE_PATTERNS = 'patterns'; export const PLOTLY_GAUGE_COLUMN_NUMBER = 5; export const APP_ANALYTICS_TAB_ID_REGEX = /application-analytics-tab.+/; export const DEFAULT_AVAILABILITY_QUERY = 'stats count() by span( timestamp, 1h )'; +export const PPL_PATTERNS_REGEX = /\|\s*patterns\s+\S+\s*\|\s*where\s+patterns_field\s*\=\s*'[^a-zA-Z0-9]+'/; diff --git a/dashboards-observability/common/types/explorer.ts b/dashboards-observability/common/types/explorer.ts index d43aba751..5a735a440 100644 --- a/dashboards-observability/common/types/explorer.ts +++ b/dashboards-observability/common/types/explorer.ts @@ -119,8 +119,8 @@ export interface SavedQuery { name: string; query: string; selected_date_range: { start: string; end: string; text: string }; - selected_fields: { text: string; tokens: [{ name: string; type: string }] }; - selected_timestamp: { name: string; type: string }; + selected_fields: { text: string; tokens: IField[] }; + selected_timestamp: IField; } export interface SavedVisualization { @@ -129,7 +129,7 @@ export interface SavedVisualization { query: string; selected_date_range: { start: string; end: string; text: string }; selected_fields: { text: string; tokens: [] }; - selected_timestamp: { name: string; type: string }; + selected_timestamp: IField; type: string; application_id?: string; } @@ -227,3 +227,9 @@ export interface LiveTailProps { isLiveTailPopoverOpen: boolean; dataTestSubj: string; } + +export interface PatternTableData { + count: number; + pattern: string; + sampleLog: string; +} diff --git a/dashboards-observability/public/components/application_analytics/helpers/utils.tsx b/dashboards-observability/public/components/application_analytics/helpers/utils.tsx index f212a3caf..a9df996b1 100644 --- a/dashboards-observability/public/components/application_analytics/helpers/utils.tsx +++ b/dashboards-observability/public/components/application_analytics/helpers/utils.tsx @@ -10,6 +10,7 @@ import { FilterType } from 'public/components/trace_analytics/components/common/ import React, { Dispatch, ReactChild } from 'react'; import { batch } from 'react-redux'; import PPLService from 'public/services/requests/ppl'; +import { IField } from '../../../../common/types/explorer'; import { preprocessQuery } from '../../../../common/utils/query_utils'; import { SPAN_REGEX } from '../../../../common/constants/shared'; import { fetchVisualizationById } from '../../../components/custom_panels/helpers/utils'; @@ -36,6 +37,10 @@ import { remove as removeQueryResult, } from '../../event_analytics/redux/slices/query_result_slice'; import { addTab, removeTab } from '../../event_analytics/redux/slices/query_tab_slice'; +import { + init as initPatterns, + remove as removePatterns, +} from '../../event_analytics/redux/slices/patterns_slice'; // Name validation export const isNameValid = (name: string, existingNames: string[]) => { @@ -153,6 +158,7 @@ export const removeTabData = ( [NEW_SELECTED_QUERY_TAB]: newIdToFocus, }) ); + dispatch(removePatterns({ tabId: TabIdToBeClosed })); }); }; @@ -172,6 +178,7 @@ export const initializeTabData = async (dispatch: Dispatch, tabId: string, }, }) ); + dispatch(initPatterns({ tabId })); }); }; @@ -234,7 +241,7 @@ export const calculateAvailability = async ( }) .then((res) => { const stat = res.metadata.fields.filter( - (field: { name: string; type: string }) => !field.name.match(SPAN_REGEX) + (field: IField) => !field.name.match(SPAN_REGEX) )[0].name; const value = res.data[stat]; currValue = value[value.length - 1]; diff --git a/dashboards-observability/public/components/event_analytics/explorer/explorer.tsx b/dashboards-observability/public/components/event_analytics/explorer/explorer.tsx index a7fd42262..9f46d43ee 100644 --- a/dashboards-observability/public/components/event_analytics/explorer/explorer.tsx +++ b/dashboards-observability/public/components/event_analytics/explorer/explorer.tsx @@ -9,7 +9,7 @@ import React, { useState, useMemo, useEffect, useRef, useCallback, ReactElement import { batch, useDispatch, useSelector } from 'react-redux'; import { isEmpty, cloneDeep, isEqual, has, reduce } from 'lodash'; import { FormattedMessage } from '@osd/i18n/react'; -import { EuiLoadingSpinner, EuiSpacer } from '@elastic/eui'; +import { EuiHorizontalRule, EuiLoadingSpinner, EuiSpacer, EuiTitle } from '@elastic/eui'; import { EuiText, EuiButtonIcon, @@ -51,6 +51,8 @@ import { TAB_CHART_ID, DEFAULT_AVAILABILITY_QUERY, DATE_PICKER_FORMAT, + PPL_PATTERNS_REGEX, + SELECTED_PATTERN, } from '../../../../common/constants/explorer'; import { PPL_STATS_REGEX, @@ -77,6 +79,9 @@ import { getVizContainerProps } from '../../visualizations/charts/helpers'; import { parseGetSuggestions, onItemSelect } from '../../common/search/autocomplete_logic'; import { formatError } from '../utils'; import { sleep } from '../../common/live_tail/live_tail_button'; +import { PatternsTable } from './log_patterns/patterns_table'; +import { selectPatterns } from '../redux/slices/patterns_slice'; +import { useFetchPatterns } from '../hooks/use_fetch_patterns'; const TYPE_TAB_MAPPING = { [SAVED_QUERY]: TAB_EVENT_ID, @@ -116,6 +121,10 @@ export const Explorer = ({ pplService, requestParams, }); + const { getPatterns, setDefaultPatternsField } = useFetchPatterns({ + pplService, + requestParams, + }); const appLogEvents = tabId.startsWith('application-analytics-tab'); const query = useSelector(selectQueries)[tabId]; const explorerData = useSelector(selectQueryResult)[tabId]; @@ -123,6 +132,7 @@ export const Explorer = ({ const countDistribution = useSelector(selectCountDistribution)[tabId]; const explorerVisualizations = useSelector(selectExplorerVisualization)[tabId]; const userVizConfigs = useSelector(selectVisualizationConfig)[tabId] || {}; + const patternsData = useSelector(selectPatterns)[tabId]; const [selectedContentTabId, setSelectedContentTab] = useState(TAB_EVENT_ID); const [selectedCustomPanelOptions, setSelectedCustomPanelOptions] = useState([]); const [selectedPanelName, setSelectedPanelName] = useState(''); @@ -132,6 +142,7 @@ export const Explorer = ({ const [isSidebarClosed, setIsSidebarClosed] = useState(false); const [timeIntervalOptions, setTimeIntervalOptions] = useState(TIME_INTERVAL_OPTIONS); const [isOverridingTimestamp, setIsOverridingTimestamp] = useState(false); + const [isOverridingPattern, setIsOverridingPattern] = useState(false); const [tempQuery, setTempQuery] = useState(query[RAW_QUERY]); const [isLiveTailPopoverOpen, setIsLiveTailPopoverOpen] = useState(false); const [isLiveTailOn, setIsLiveTailOn] = useState(false); @@ -141,7 +152,7 @@ export const Explorer = ({ const [browserTabFocus, setBrowserTabFocus] = useState(true); const [liveTimestamp, setLiveTimestamp] = useState(DATE_PICKER_FORMAT); const [triggerAvailability, setTriggerAvailability] = useState(false); - + const [viewLogPatterns, setViewLogPatterns] = useState(false); const queryRef = useRef(); const appBasedRef = useRef(''); appBasedRef.current = appBaseQuery; @@ -196,6 +207,15 @@ export const Explorer = ({ }); }); + const getErrorHandler = (title: string) => { + return (error: any) => { + const formattedError = formatError(error.name, error.message, error.body.message); + notifications.toasts.addError(formattedError, { + title, + }); + }; + }; + const composeFinalQuery = ( curQuery: any, startingTime: string, @@ -327,6 +347,19 @@ export const Explorer = ({ } } + let curPattern: string = curQuery![SELECTED_PATTERN]; + + if (isEmpty(curPattern)) { + const patternErrorHandler = getErrorHandler('Error fetching default pattern field'); + await setDefaultPatternsField(curIndex, '', patternErrorHandler); + const newQuery = queryRef.current; + curPattern = newQuery![SELECTED_PATTERN]; + if (isEmpty(curPattern)) { + setToast('Index does not contain a valid pattern field.', 'danger'); + return; + } + } + if (isEqual(typeof startingTime, 'undefined') && isEqual(typeof endingTime, 'undefined')) { startingTime = curQuery![SELECTED_DATE_RANGE][0]; endingTime = curQuery![SELECTED_DATE_RANGE][1]; @@ -358,21 +391,17 @@ export const Explorer = ({ } else { findAutoInterval(startTime, endTime); if (isLiveTailOnRef.current) { - getLiveTail(undefined, (error) => { - const formattedError = formatError(error.name, error.message, error.body.message); - notifications.toasts.addError(formattedError, { - title: 'Error fetching events', - }); - }); + getLiveTail(undefined, getErrorHandler('Error fetching events')); } else { - getEvents(undefined, (error) => { - const formattedError = formatError(error.name, error.message, error.body.message); - notifications.toasts.addError(formattedError, { - title: 'Error fetching events', - }); - }); + getEvents(undefined, getErrorHandler('Error fetching events')); } getCountVisualizations(minInterval); + + // to fetch patterns data on current query + if (!finalQuery.match(PPL_PATTERNS_REGEX)) { + const patternErrorHandler = getErrorHandler('Error fetching patterns'); + getPatterns(patternErrorHandler); + } } // for comparing usage if for the same tab, user changed index from one to another @@ -538,6 +567,17 @@ export const Explorer = ({ handleQuerySearch(); }; + const handleOverridePattern = async (pattern: IField) => { + setIsOverridingPattern(true); + await setDefaultPatternsField( + '', + pattern.name, + getErrorHandler('Error overriding default pattern') + ); + setIsOverridingPattern(false); + await getPatterns(getErrorHandler('Error fetching patterns')); + }; + const totalHits: number = useMemo(() => { if (isLiveTailOn && countDistribution?.data) { const hits = reduce( @@ -553,6 +593,21 @@ export const Explorer = ({ return 0; }, [countDistribution?.data]); + const onPatternSelection = async (pattern: string) => { + let currQuery = queryRef.current![RAW_QUERY] as string; + const currPattern = queryRef.current![SELECTED_PATTERN] as string; + // Remove existing pattern selection if it exists + if (currQuery.match(PPL_PATTERNS_REGEX)) { + currQuery = currQuery.replace(PPL_PATTERNS_REGEX, ''); + } + const patternSelectQuery = `${currQuery.trim()} | patterns ${currPattern} | where patterns_field = '${pattern}'`; + // Passing in empty string will remove pattern query + const newQuery = pattern ? patternSelectQuery : currQuery; + await setTempQuery(newQuery); + await updateQueryInStore(newQuery); + await handleTimeRangePickerRefresh(true); + }; + const getMainContent = () => { return (
@@ -569,10 +624,13 @@ export const Explorer = ({ explorerFields={explorerFields} explorerData={explorerData} selectedTimestamp={query[SELECTED_TIMESTAMP]} + selectedPattern={query[SELECTED_PATTERN]} handleOverrideTimestamp={handleOverrideTimestamp} + handleOverridePattern={handleOverridePattern} handleAddField={(field: IField) => handleAddField(field)} handleRemoveField={(field: IField) => handleRemoveField(field)} isOverridingTimestamp={isOverridingTimestamp} + isOverridingPattern={isOverridingPattern} isFieldToggleButtonDisabled={ isEmpty(explorerData.jsonData) || !isEmpty(queryRef.current![RAW_QUERY].match(PPL_STATS_REGEX)) @@ -628,6 +686,58 @@ export const Explorer = ({ + + + + {viewLogPatterns && ( + +

+ Patterns + + {' '} + ( + {patternsData.patternTableData + ? patternsData.patternTableData.length + : 0} + ) + +

+ + )} + + + + + {viewLogPatterns && ( + onPatternSelection('')}> + Clear Selection + + )} + + + setViewLogPatterns(!viewLogPatterns)}> + {`${viewLogPatterns ? 'Hide' : 'Show'} Patterns`} + + + + + + + {viewLogPatterns && ( + <> + + + + )} )} @@ -662,6 +772,26 @@ export const Explorer = ({ )} + {countDistribution?.data && ( + +

+ Events + + {' '} + ( + {reduce( + countDistribution.data['count()'], + (sum, n) => { + return sum + n; + }, + 0 + )} + ) + +

+ + )} + setSelectedContentTab(selectedTab.id); diff --git a/dashboards-observability/public/components/event_analytics/explorer/log_patterns/patterns_table.tsx b/dashboards-observability/public/components/event_analytics/explorer/log_patterns/patterns_table.tsx new file mode 100644 index 000000000..97b7e3066 --- /dev/null +++ b/dashboards-observability/public/components/event_analytics/explorer/log_patterns/patterns_table.tsx @@ -0,0 +1,128 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { + EuiLink, + EuiText, + EuiInMemoryTable, + Direction, + EuiEmptyPrompt, + EuiIcon, +} from '@elastic/eui'; +import { PatternTableData } from 'common/types/explorer'; +import { reduce, round } from 'lodash'; +import React, { useState } from 'react'; +import { useSelector } from 'react-redux'; +import { PPL_DOCUMENTATION_URL } from '../../../../../common/constants/shared'; +import { selectPatterns } from '../../redux/slices/patterns_slice'; + +interface PatternsTableProps { + tableData: PatternTableData[]; + onPatternSelection: any; + tabId: string; +} + +export function PatternsTable(props: PatternsTableProps) { + const { tableData, tabId, onPatternSelection } = props; + const patternsData = useSelector(selectPatterns)[tabId]; + const [selectedPattern, setSelectedPattern] = useState(); + + const tableColumns = [ + { + field: 'count', + name: 'Count', + width: '4%', + sortable: true, + render: (item: string, row: PatternTableData) => { + return {item}; + }, + }, + { + field: 'ratio', + name: 'Ratio', + width: '4%', + sortable: (row: PatternTableData) => row.count, + render: (item: number, row: PatternTableData) => { + const ratio = + (row.count / + reduce( + patternsData.total, + (sum, n) => { + return sum + n; + }, + 0 + )) * + 100; + return {`${round(ratio, 2)}%`}; + }, + }, + { + field: 'sampleLog', + name: 'Sample Log', + width: '92%', + sortable: true, + render: (item: string, row: PatternTableData) => { + return {item}; + }, + }, + ]; + + const sorting = { + sort: { + field: 'count', + direction: 'desc' as Direction, + }, + allowNeutralSort: true, + enableAllColumns: true, + }; + + const pagination = { + pageSizeOptions: [5, 10, 15, 20], + initialPageSize: 5, + }; + + const message = ( + No patterns found.} + titleSize="s" + iconType="minusInCircle" + iconColor="#DDDDDD" + body={ +

+ Try expanding your time range or modifying your query. Learn more from our{' '} + + PPL documentation + + +

+ } + /> + ); + + const getRowProps = (item: PatternTableData) => { + const { pattern } = item; + return { + 'data-test-subj': `row-${pattern}`, + className: 'customRowClass', + onClick: () => { + onPatternSelection(pattern); + setSelectedPattern(pattern); + }, + isSelected: pattern === selectedPattern, + }; + }; + + return ( + + ); +} diff --git a/dashboards-observability/public/components/event_analytics/explorer/sidebar/__tests__/__snapshots__/field.test.tsx.snap b/dashboards-observability/public/components/event_analytics/explorer/sidebar/__tests__/__snapshots__/field.test.tsx.snap index 30d744eb9..6bcdbaa2e 100644 --- a/dashboards-observability/public/components/event_analytics/explorer/sidebar/__tests__/__snapshots__/field.test.tsx.snap +++ b/dashboards-observability/public/components/event_analytics/explorer/sidebar/__tests__/__snapshots__/field.test.tsx.snap @@ -25,6 +25,26 @@ exports[`Field component Renders a sidebar field 1`] = ` dataTestSubj="field-agent-showDetails" fieldAction={ + + + + Override + + + + + + + Override + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Override + + + + + + + Override + + + + + + + + + + + + + + + + + + + + + + @@ -3217,6 +3534,14 @@ exports[`Siderbar component Renders sidebar component 1`] = ` dataTestSubj="field-clientip-showDetails" fieldAction={ + + + + + + + + + + + + + + + + + + + Override + + + + + + + Override + + + + + + + + + + + + + + + + + + + + + + + Override + + + + + + + Override + + + + + + + + + + - - - + Override + + + + + + + + + @@ -4659,6 +5275,26 @@ exports[`Siderbar component Renders sidebar component 1`] = ` dataTestSubj="field-index-showDetails" fieldAction={ + + + + Override + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Override + + + + + + + Override + + + + + + + + + + + + + @@ -6101,6 +6992,14 @@ exports[`Siderbar component Renders sidebar component 1`] = ` dataTestSubj="field-phpmemory-showDetails" fieldAction={ + + + + + + + + + + Override + + + + + + + Override + + + + + + + + + + + + + + Override + + + + + + + Override + + + + + + + + + + + + + + Override + + + + + + + Override + + + + + + + + + + + + + + Override + + + + + + + Override + + + + + + + + + + + + + + + + + + + + + + + Override + + + + + + + Override + + + + + + + + + + + + + + + + + + + void; selectedTimestamp: string; isOverridingTimestamp: boolean; - handleOverrideTimestamp: (timestamp: { name: string; type: string }) => void; + handleOverrideTimestamp: (timestamp: IField) => void; selected: boolean; showToggleButton: boolean; showTimestampOverrideButton: boolean; @@ -40,6 +44,9 @@ export const Field = (props: IFieldProps) => { const { query, field, + selectedPattern, + isOverridingPattern, + handleOverridePattern, selectedTimestamp, isOverridingTimestamp, handleOverrideTimestamp, @@ -71,6 +78,31 @@ export const Field = (props: IFieldProps) => { const getFieldActionDOM = () => { return ( <> + + <> + {isEqual(field.type, 'string') ? ( + isEqual(selectedPattern, field.name) ? ( + + Default Pattern + + ) : isOverridingPattern ? ( + + ) : ( + handleOverridePattern(field)} + data-test-subj="eventExplorer__overrideDefaultPattern" + > + Override + + ) + ) : null} + + <> {showTimestampOverrideButton && isEqual(field.type, 'timestamp') ? ( diff --git a/dashboards-observability/public/components/event_analytics/explorer/sidebar/sidebar.tsx b/dashboards-observability/public/components/event_analytics/explorer/sidebar/sidebar.tsx index c949c9e49..c54b89baa 100644 --- a/dashboards-observability/public/components/event_analytics/explorer/sidebar/sidebar.tsx +++ b/dashboards-observability/public/components/event_analytics/explorer/sidebar/sidebar.tsx @@ -18,10 +18,13 @@ interface ISidebarProps { query: string; explorerFields: IExplorerFields; explorerData: any; + selectedPattern: string; + isOverridingPattern: boolean; selectedTimestamp: string; isOverridingTimestamp: boolean; isFieldToggleButtonDisabled: boolean; - handleOverrideTimestamp: (timestamp: { name: string; type: string }) => void; + handleOverridePattern: (pattern: IField) => void; + handleOverrideTimestamp: (timestamp: IField) => void; handleAddField: (field: IField) => void; handleRemoveField: (field: IField) => void; } @@ -31,9 +34,12 @@ export const Sidebar = (props: ISidebarProps) => { query, explorerFields, explorerData, + selectedPattern, + isOverridingPattern, selectedTimestamp, isOverridingTimestamp, isFieldToggleButtonDisabled, + handleOverridePattern, handleOverrideTimestamp, handleAddField, handleRemoveField, @@ -90,6 +96,10 @@ export const Sidebar = (props: ISidebarProps) => { { { { dispatch(initQueryResult({ tabId })); dispatch(initFields({ tabId })); dispatch(addTab({ tabId })); + dispatch(initPatterns({ tabId })); }); return tabId; diff --git a/dashboards-observability/public/components/event_analytics/hooks/use_fetch_events.ts b/dashboards-observability/public/components/event_analytics/hooks/use_fetch_events.ts index dba08fd2c..6e53693e7 100644 --- a/dashboards-observability/public/components/event_analytics/hooks/use_fetch_events.ts +++ b/dashboards-observability/public/components/event_analytics/hooks/use_fetch_events.ts @@ -16,6 +16,7 @@ import { QUERIED_FIELDS, } from '../../../../common/constants/explorer'; import { fetchSuccess, reset as queryResultReset } from '../redux/slices/query_result_slice'; +import { reset as patternsReset } from '../redux/slices/patterns_slice'; import { selectQueries } from '../redux/slices/query_slice'; import { reset as visualizationReset } from '../redux/slices/visualization_slice'; import { updateFields, sortFields, selectFields } from '../redux/slices/field_slice'; @@ -66,9 +67,9 @@ export const useFetchEvents = ({ pplService, requestParams }: IFetchEventsParams }; const dispatchOnGettingHis = (res: any) => { - const selectedFields: Array = fieldsRef.current![requestParams.tabId][ - SELECTED_FIELDS - ].map((field: IField) => field.name); + const selectedFields: string[] = fieldsRef.current![requestParams.tabId][SELECTED_FIELDS].map( + (field: IField) => field.name + ); setResponse(res); batch(() => { dispatch( @@ -90,15 +91,7 @@ export const useFetchEvents = ({ pplService, requestParams }: IFetchEventsParams data: { [UNSELECTED_FIELDS]: res?.schema ? [...res.schema] : [], [QUERIED_FIELDS]: [], - [AVAILABLE_FIELDS]: res?.schema - ? isEmpty(selectedFields) - ? [...res.schema] - : [ - ...res?.schema.filter( - (curField: IField) => !selectedFields.includes(curField.name) - ), - ] - : [], + [AVAILABLE_FIELDS]: res?.schema || [], }, }) ); @@ -146,6 +139,11 @@ export const useFetchEvents = ({ pplService, requestParams }: IFetchEventsParams tabId: requestParams.tabId, }) ); + dispatch( + patternsReset({ + tabId: requestParams.tabId, + }) + ); }); }; diff --git a/dashboards-observability/public/components/event_analytics/hooks/use_fetch_patterns.ts b/dashboards-observability/public/components/event_analytics/hooks/use_fetch_patterns.ts new file mode 100644 index 000000000..98ba7c624 --- /dev/null +++ b/dashboards-observability/public/components/event_analytics/hooks/use_fetch_patterns.ts @@ -0,0 +1,139 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { IField, PatternTableData } from 'common/types/explorer'; +import { isEmpty, isUndefined } from 'lodash'; +import PPLService from 'public/services/requests/ppl'; +import { useRef } from 'react'; +import { batch, useDispatch, useSelector } from 'react-redux'; +import { FINAL_QUERY, SELECTED_PATTERN } from '../../../../common/constants/explorer'; +import { setPatterns, reset as resetPatterns } from '../redux/slices/patterns_slice'; +import { changeQuery, selectQueries } from '../redux/slices/query_slice'; +import { useFetchEvents } from './use_fetch_events'; + +interface IFetchPatternsParams { + pplService: PPLService; + requestParams: { tabId: string }; +} + +export const useFetchPatterns = ({ pplService, requestParams }: IFetchPatternsParams) => { + const dispatch = useDispatch(); + const { fetchEvents } = useFetchEvents({ + pplService, + requestParams, + }); + const queries = useSelector(selectQueries); + const queriesRef = useRef(); + queriesRef.current = queries; + + const dispatchOnPatterns = (res: { patternTableData: PatternTableData[]; total: number[] }) => { + batch(() => { + dispatch( + resetPatterns({ + tabId: requestParams.tabId, + }) + ); + dispatch( + setPatterns({ + tabId: requestParams.tabId, + data: { + ...res, + }, + }) + ); + }); + }; + + const buildPatternDataQuery = (query: string, field: string) => { + return `${query.trim()} | patterns ${field} | stats count(), take(${field}, 1) by patterns_field`; + }; + + const getPatterns = (errorHandler: (error: any) => void, query?: string) => { + const cur = queriesRef.current; + const rawQuery = cur![requestParams.tabId][FINAL_QUERY]; + const searchQuery = isUndefined(query) ? rawQuery : query; + const patternField = cur![requestParams.tabId][SELECTED_PATTERN]; + const statsQuery = buildPatternDataQuery(searchQuery, patternField); + // Fetch patterns data for the current query results + fetchEvents( + { query: statsQuery }, + 'jdbc', + (res: { datarows: any[] }) => { + if (!isEmpty(res.datarows)) { + const formatToTableData = res.datarows.map((row: any) => { + return { + count: row[0], + pattern: row[2], + sampleLog: row[1][0], + } as PatternTableData; + }); + // Fetch total number of events to divide count by for ratio + fetchEvents( + { + query: `${rawQuery} | stats count()`, + }, + 'jdbc', + (countRes: any) => { + dispatchOnPatterns({ + patternTableData: formatToTableData, + total: countRes.datarows[0], + }); + }, + errorHandler + ); + } + }, + errorHandler + ); + }; + + const setDefaultPatternsField = async ( + index: string, + pattern: string, + errorHandler: (error: any) => void + ) => { + let patternField = pattern; + if (!pattern) { + if (!index) { + return; + } + const query = `source = ${index} | head 1`; + await fetchEvents( + { query }, + 'jdbc', + async (res: any) => { + // Create array of only string type fields + const textFields = res.schema.filter((field: IField) => field.type === 'string'); + // Loop through array and find field with longest value + let defaultPatternField = ''; + let maxLength = 0; + textFields.forEach((field: IField, i: number) => { + const curLength = res.jsonData[0][field.name].length; + if (curLength > maxLength) { + maxLength = curLength; + defaultPatternField = field.name; + } + }); + patternField = defaultPatternField; + }, + errorHandler + ); + } + // Set pattern to the pattern passed in or the default pattern field found if pattern is empty + await dispatch( + changeQuery({ + tabId: requestParams.tabId, + query: { + [SELECTED_PATTERN]: patternField, + }, + }) + ); + }; + + return { + getPatterns, + setDefaultPatternsField, + }; +}; diff --git a/dashboards-observability/public/components/event_analytics/hooks/use_fetch_visualizations.ts b/dashboards-observability/public/components/event_analytics/hooks/use_fetch_visualizations.ts index 866921201..93c05425a 100644 --- a/dashboards-observability/public/components/event_analytics/hooks/use_fetch_visualizations.ts +++ b/dashboards-observability/public/components/event_analytics/hooks/use_fetch_visualizations.ts @@ -18,6 +18,7 @@ import { render as renderExplorerVis } from '../redux/slices/visualization_slice import { updateFields, sortFields } from '../redux/slices/field_slice'; import PPLService from '../../../services/requests/ppl'; import { fetchSuccess } from '../redux/slices/query_result_slice'; +import { setPatterns, reset as patternsReset } from '../redux/slices/patterns_slice'; interface IFetchVisualizationsParams { pplService: PPLService; @@ -117,6 +118,11 @@ export const useFetchVisualizations = ({ data: [QUERIED_FIELDS], }) ); + dispatch( + patternsReset({ + tabId: requestParams.tabId, + }) + ); }); } ); diff --git a/dashboards-observability/public/components/event_analytics/redux/slices/patterns_slice.ts b/dashboards-observability/public/components/event_analytics/redux/slices/patterns_slice.ts new file mode 100644 index 000000000..6f6ebf4bd --- /dev/null +++ b/dashboards-observability/public/components/event_analytics/redux/slices/patterns_slice.ts @@ -0,0 +1,37 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +/* eslint-disable import/no-default-export */ + +import { createSlice } from '@reduxjs/toolkit'; +import { fetchSuccess as fetchSuccessReducer } from '../reducers'; +import { initialTabId } from '../../../../framework/redux/store/shared_state'; +import { REDUX_EXPL_SLICE_PATTERNS } from '../../../../../common/constants/explorer'; + +const initialState = { + [initialTabId]: {}, +}; + +export const patternsSlice = createSlice({ + name: REDUX_EXPL_SLICE_PATTERNS, + initialState, + reducers: { + setPatterns: fetchSuccessReducer, + reset: (state, { payload }) => { + state[payload.tabId] = {}; + }, + init: (state, { payload }) => { + state[payload.tabId] = {}; + }, + remove: (state, { payload }) => { + delete state[payload.tabId]; + }, + }, +}); + +export const { setPatterns, remove, reset, init } = patternsSlice.actions; + +export const selectPatterns = (state) => state.patterns; + +export default patternsSlice.reducer; diff --git a/dashboards-observability/public/components/event_analytics/redux/slices/query_slice.ts b/dashboards-observability/public/components/event_analytics/redux/slices/query_slice.ts index 6d82265e1..ae033b910 100644 --- a/dashboards-observability/public/components/event_analytics/redux/slices/query_slice.ts +++ b/dashboards-observability/public/components/event_analytics/redux/slices/query_slice.ts @@ -13,12 +13,14 @@ import { INDEX, SELECTED_TIMESTAMP, APP_ANALYTICS_TAB_ID_REGEX, + SELECTED_PATTERN, } from '../../../../../common/constants/explorer'; const initialQueryState = { [RAW_QUERY]: '', [FINAL_QUERY]: '', [INDEX]: '', + [SELECTED_PATTERN]: '', [SELECTED_TIMESTAMP]: '', [SELECTED_DATE_RANGE]: ['now-15m', 'now'], }; @@ -27,6 +29,7 @@ const appBaseQueryState = { [RAW_QUERY]: '', [FINAL_QUERY]: '', [INDEX]: '', + [SELECTED_PATTERN]: '', [SELECTED_TIMESTAMP]: '', [SELECTED_DATE_RANGE]: ['now-24h', 'now'], }; diff --git a/dashboards-observability/public/framework/redux/reducers/index.ts b/dashboards-observability/public/framework/redux/reducers/index.ts index 1dfcf743c..14e27418e 100644 --- a/dashboards-observability/public/framework/redux/reducers/index.ts +++ b/dashboards-observability/public/framework/redux/reducers/index.ts @@ -12,6 +12,7 @@ import FieldsReducer from '../../../components/event_analytics/redux/slices/fiel import countDistributionReducer from '../../../components/event_analytics/redux/slices/count_distribution_slice'; import explorerVisualizationReducer from '../../../components/event_analytics/redux/slices/visualization_slice'; import explorerVisualizationConfigReducer from '../../../components/event_analytics/redux/slices/viualization_config_slice'; +import patternsReducer from '../../../components/event_analytics/redux/slices/patterns_slice'; const rootReducer = combineReducers({ // explorer reducers @@ -22,6 +23,7 @@ const rootReducer = combineReducers({ countDistribution: countDistributionReducer, explorerVisualization: explorerVisualizationReducer, explorerVisualizationConfig: explorerVisualizationConfigReducer, + patterns: patternsReducer, }); export type RootState = ReturnType;