From c76129b623b3f4f81284b902aad0c50ecccc0077 Mon Sep 17 00:00:00 2001 From: Dzmitry Tamashevich Date: Wed, 9 Nov 2022 19:03:45 +0300 Subject: [PATCH 1/3] [Discover] improve data view changed warning --- .../view_alert/view_alert_route.tsx | 67 ++++++++++++------- .../view_alert/view_alert_utils.tsx | 3 +- .../server/alert_types/es_query/executor.ts | 49 ++++++++------ .../es_query/lib/fetch_es_query.ts | 5 ++ .../lib/fetch_search_source_query.test.ts | 4 ++ .../es_query/lib/fetch_search_source_query.ts | 53 ++++++++++++--- .../apps/discover/search_source_alert.ts | 18 +++++ 7 files changed, 144 insertions(+), 55 deletions(-) diff --git a/src/plugins/discover/public/application/view_alert/view_alert_route.tsx b/src/plugins/discover/public/application/view_alert/view_alert_route.tsx index 3efac8b504811..6b1bc195ea7ef 100644 --- a/src/plugins/discover/public/application/view_alert/view_alert_route.tsx +++ b/src/plugins/discover/public/application/view_alert/view_alert_route.tsx @@ -10,7 +10,7 @@ import { useEffect, useMemo } from 'react'; import { useHistory, useLocation, useParams } from 'react-router-dom'; import { sha256 } from 'js-sha256'; import type { Rule } from '@kbn/alerting-plugin/common'; -import { getTime } from '@kbn/data-plugin/common'; +import { type DataViewSpec, getTime } from '@kbn/data-plugin/common'; import type { DataView } from '@kbn/data-views-plugin/public'; import type { Filter } from '@kbn/es-query'; import { DiscoverAppLocatorParams } from '../../locator'; @@ -19,11 +19,33 @@ import { getAlertUtils, QueryParams, SearchThresholdAlertParams } from './view_a type NonNullableEntry = { [K in keyof T]: NonNullable }; -const getCurrentChecksum = (params: SearchThresholdAlertParams) => - sha256.create().update(JSON.stringify(params)).hex(); +const getDataViewParamsChecksum = (dataViewSpec: DataViewSpec) => { + const { title, timeFieldName, sourceFilters, runtimeFieldMap } = dataViewSpec; + return sha256 + .create() + .update(JSON.stringify({ title, timeFieldName, sourceFilters, runtimeFieldMap })) + .hex(); +}; + +/** + * Get rule params checksum skipping serialized data view object + */ +const getRuleParamsChecksum = (params: SearchThresholdAlertParams) => { + return sha256 + .create() + .update( + JSON.stringify(params, (key: string, value: string) => (key === 'index' ? undefined : value)) + ) + .hex(); +}; const isActualAlert = (queryParams: QueryParams): queryParams is NonNullableEntry => { - return Boolean(queryParams.from && queryParams.to && queryParams.checksum); + return Boolean( + queryParams.from && + queryParams.to && + queryParams.ruleParamsChecksum && + queryParams.dataViewChecksum + ); }; const buildTimeRangeFilter = ( @@ -55,7 +77,8 @@ export function ViewAlertRoute() { () => ({ from: query.get('from'), to: query.get('to'), - checksum: query.get('checksum'), + ruleParamsChecksum: query.get('ruleParamsChecksum'), + dataViewChecksum: query.get('dataViewChecksum'), }), [query] ); @@ -79,16 +102,6 @@ export function ViewAlertRoute() { return; } - const calculatedChecksum = getCurrentChecksum(fetchedAlert.params); - // rule params changed - if (openActualAlert && calculatedChecksum !== queryParams.checksum) { - displayRuleChangedWarn(); - } - // documents might be updated or deleted - else if (openActualAlert && calculatedChecksum === queryParams.checksum) { - displayPossibleDocsDiffInfoAlert(); - } - const fetchedSearchSource = await fetchSearchSource(fetchedAlert); if (!fetchedSearchSource) { history.push(DISCOVER_MAIN_ROUTE); @@ -104,15 +117,21 @@ export function ViewAlertRoute() { return; } - const dataViewSavedObject = await core.savedObjects.client.get('index-pattern', dataView.id!); - const alertUpdatedAt = fetchedAlert.updatedAt; - const dataViewUpdatedAt = dataViewSavedObject.updatedAt!; - // data view updated after the last update of the alert rule - if ( - openActualAlert && - new Date(dataViewUpdatedAt).valueOf() > new Date(alertUpdatedAt).valueOf() - ) { - showDataViewUpdatedWarning(); + if (openActualAlert) { + const currentDataViewChecksum = getDataViewParamsChecksum(dataView.toSpec(false)); + if (currentDataViewChecksum !== queryParams.dataViewChecksum) { + // data view params which might affect the displayed results were changed + showDataViewUpdatedWarning(); + } + + const currentRuleParamsChecksum = getRuleParamsChecksum(fetchedAlert.params); + if (currentRuleParamsChecksum !== queryParams.ruleParamsChecksum) { + // rule params changed + displayRuleChangedWarn(); + } else { + // documents might be updated or deleted + displayPossibleDocsDiffInfoAlert(); + } } const timeRange = openActualAlert diff --git a/src/plugins/discover/public/application/view_alert/view_alert_utils.tsx b/src/plugins/discover/public/application/view_alert/view_alert_utils.tsx index d5b6aac22d3b4..5aad3819d5d65 100644 --- a/src/plugins/discover/public/application/view_alert/view_alert_utils.tsx +++ b/src/plugins/discover/public/application/view_alert/view_alert_utils.tsx @@ -22,7 +22,8 @@ export interface SearchThresholdAlertParams extends RuleTypeParams { export interface QueryParams { from: string | null; to: string | null; - checksum: string | null; + ruleParamsChecksum: string | null; + dataViewChecksum: string | null; } const LEGACY_BASE_ALERT_API_PATH = '/api/alerts'; diff --git a/x-pack/plugins/stack_alerts/server/alert_types/es_query/executor.ts b/x-pack/plugins/stack_alerts/server/alert_types/es_query/executor.ts index f50368f29ae76..30aacbb1b3072 100644 --- a/x-pack/plugins/stack_alerts/server/alert_types/es_query/executor.ts +++ b/x-pack/plugins/stack_alerts/server/alert_types/es_query/executor.ts @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { sha256 } from 'js-sha256'; + import { i18n } from '@kbn/i18n'; import { CoreSetup } from '@kbn/core/server'; import { parseDuration } from '@kbn/alerting-plugin/server'; @@ -23,7 +23,8 @@ export async function executor(core: CoreSetup, options: ExecutorOptions = { title: name, date: currentTimestamp, @@ -179,10 +188,6 @@ export function tryToParseAsDate(sortValue?: string | number | null): undefined } } -export function getChecksum(params: OnlyEsQueryRuleParams) { - return sha256.create().update(JSON.stringify(params)); -} - export function getInvalidComparatorError(comparator: string) { return i18n.translate('xpack.stackAlerts.esQuery.invalidComparatorErrorMessage', { defaultMessage: 'invalid thresholdComparator specified: {comparator}', diff --git a/x-pack/plugins/stack_alerts/server/alert_types/es_query/lib/fetch_es_query.ts b/x-pack/plugins/stack_alerts/server/alert_types/es_query/lib/fetch_es_query.ts index 3e94745d295fb..a545609abb254 100644 --- a/x-pack/plugins/stack_alerts/server/alert_types/es_query/lib/fetch_es_query.ts +++ b/x-pack/plugins/stack_alerts/server/alert_types/es_query/lib/fetch_es_query.ts @@ -19,6 +19,8 @@ export async function fetchEsQuery( name: string, params: OnlyEsQueryRuleParams, timestamp: string | undefined, + publicBaseUrl: string, + spacePrefix: string, services: { scopedClusterClient: IScopedClusterClient; logger: Logger; @@ -88,7 +90,10 @@ export async function fetchEsQuery( logger.debug( ` es query rule ${ES_QUERY_ID}:${ruleId} "${name}" result - ${JSON.stringify(searchResult)}` ); + const link = `${publicBaseUrl}${spacePrefix}/app/management/insightsAndAlerting/triggersActions/rule/${ruleId}`; + return { + link, numMatches: (searchResult.hits.total as estypes.SearchTotalHits).value, searchResult, dateStart, diff --git a/x-pack/plugins/stack_alerts/server/alert_types/es_query/lib/fetch_search_source_query.test.ts b/x-pack/plugins/stack_alerts/server/alert_types/es_query/lib/fetch_search_source_query.test.ts index 127224f1bf1b6..449f58ff5b574 100644 --- a/x-pack/plugins/stack_alerts/server/alert_types/es_query/lib/fetch_search_source_query.test.ts +++ b/x-pack/plugins/stack_alerts/server/alert_types/es_query/lib/fetch_search_source_query.test.ts @@ -61,6 +61,7 @@ describe('fetchSearchSourceQuery', () => { const { searchSource, dateStart, dateEnd } = updateSearchSource( searchSourceInstance, + dataViewMock, params, undefined ); @@ -96,6 +97,7 @@ describe('fetchSearchSourceQuery', () => { const { searchSource } = updateSearchSource( searchSourceInstance, + dataViewMock, params, '2020-02-09T23:12:41.941Z' ); @@ -136,6 +138,7 @@ describe('fetchSearchSourceQuery', () => { const { searchSource } = updateSearchSource( searchSourceInstance, + dataViewMock, params, '2020-01-09T22:12:41.941Z' ); @@ -169,6 +172,7 @@ describe('fetchSearchSourceQuery', () => { const { searchSource } = updateSearchSource( searchSourceInstance, + dataViewMock, params, '2020-02-09T23:12:41.941Z' ); diff --git a/x-pack/plugins/stack_alerts/server/alert_types/es_query/lib/fetch_search_source_query.ts b/x-pack/plugins/stack_alerts/server/alert_types/es_query/lib/fetch_search_source_query.ts index 81f6248d906ca..494037c07bdae 100644 --- a/x-pack/plugins/stack_alerts/server/alert_types/es_query/lib/fetch_search_source_query.ts +++ b/x-pack/plugins/stack_alerts/server/alert_types/es_query/lib/fetch_search_source_query.ts @@ -4,9 +4,13 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ + +import { sha256 } from 'js-sha256'; import { buildRangeFilter, Filter } from '@kbn/es-query'; import { Logger } from '@kbn/core/server'; import { + DataView, + DataViewSpec, getTime, ISearchSource, ISearchStartSearchSource, @@ -18,6 +22,8 @@ export async function fetchSearchSourceQuery( ruleId: string, params: OnlySearchSourceRuleParams, latestTimestamp: string | undefined, + publicBaseUrl: string, + spacePrefix: string, services: { logger: Logger; searchSourceClient: ISearchStartSearchSource; @@ -27,8 +33,14 @@ export async function fetchSearchSourceQuery( const initialSearchSource = await searchSourceClient.create(params.searchConfiguration); + const index = initialSearchSource.getField('index') as DataView; + if (!isTimeBasedDataView(index)) { + throw new Error('Invalid data view without timeFieldName.'); + } + const { searchSource, dateStart, dateEnd } = updateSearchSource( initialSearchSource, + index, params, latestTimestamp ); @@ -41,7 +53,12 @@ export async function fetchSearchSourceQuery( const searchResult = await searchSource.fetch(); + const dataViewChecksum = getDataViewChecksum(index.toSpec(false)); + const ruleParamsChecksum = getRuleParamsChecksum(params as OnlySearchSourceRuleParams); + const link = `${publicBaseUrl}${spacePrefix}/app/discover#/viewAlert/${ruleId}?from=${dateStart}&to=${dateEnd}&ruleParamsChecksum=${ruleParamsChecksum}&dataViewChecksum=${dataViewChecksum}`; + return { + link, numMatches: Number(searchResult.hits.total), searchResult, dateStart, @@ -51,16 +68,11 @@ export async function fetchSearchSourceQuery( export function updateSearchSource( searchSource: ISearchSource, + index: DataView, params: OnlySearchSourceRuleParams, - latestTimestamp: string | undefined + latestTimestamp?: string ) { - const index = searchSource.getField('index'); - - const timeFieldName = index?.timeFieldName; - if (!timeFieldName) { - throw new Error('Invalid data view without timeFieldName.'); - } - + const timeFieldName = index.timeFieldName!; searchSource.setField('size', params.size); const timerangeFilter = getTime(index, { @@ -84,9 +96,34 @@ export function updateSearchSource( const searchSourceChild = searchSource.createChild(); searchSourceChild.setField('filter', filters as Filter[]); searchSourceChild.setField('sort', [{ [timeFieldName]: SortDirection.desc }]); + return { searchSource: searchSourceChild, dateStart, dateEnd, }; } + +function isTimeBasedDataView(index?: DataView) { + return index?.timeFieldName; +} + +function getDataViewChecksum(index: DataViewSpec) { + const { title, timeFieldName, sourceFilters, runtimeFieldMap } = index; + return sha256 + .create() + .update(JSON.stringify({ title, timeFieldName, sourceFilters, runtimeFieldMap })) + .hex(); +} + +/** + * Get rule params checksum skipping serialized data view object + */ +function getRuleParamsChecksum(params: OnlySearchSourceRuleParams) { + return sha256 + .create() + .update( + JSON.stringify(params, (key: string, value: string) => (key === 'index' ? undefined : value)) + ) + .hex(); +} diff --git a/x-pack/test/functional_with_es_ssl/apps/discover/search_source_alert.ts b/x-pack/test/functional_with_es_ssl/apps/discover/search_source_alert.ts index 088ed138fba6e..6fb760b1afd67 100644 --- a/x-pack/test/functional_with_es_ssl/apps/discover/search_source_alert.ts +++ b/x-pack/test/functional_with_es_ssl/apps/discover/search_source_alert.ts @@ -345,6 +345,24 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { ); }); + it('should not notify about data view update when field popularity changed', async () => { + await PageObjects.common.sleep(8000); + await testSubjects.click('field-message-showDetails'); + await testSubjects.click('discoverFieldListPanelEdit-message'); + await testSubjects.click('toggleAdvancedSetting'); + + const popularityInput = await testSubjects.find('editorFieldCount'); + await popularityInput.type('5'); + await testSubjects.click('fieldSaveButton'); + await PageObjects.header.waitUntilLoadingHasFinished(); + + await openAlertResults(RULE_NAME, sourceDataViewId); + + await PageObjects.common.sleep(8000); + + expect(await toasts.getToastCount()).to.be.equal(1); + }); + it('should display warning about recently updated data view', async () => { await PageObjects.common.navigateToUrlWithBrowserHistory( 'management', From 6e929ae443dc141f6a507a279a8cd5d2ba06892c Mon Sep 17 00:00:00 2001 From: Dzmitry Tamashevich Date: Tue, 15 Nov 2022 16:58:22 +0300 Subject: [PATCH 2/3] [Discover] update test --- .../server/rule_types/es_query/rule_type.test.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/x-pack/plugins/stack_alerts/server/rule_types/es_query/rule_type.test.ts b/x-pack/plugins/stack_alerts/server/rule_types/es_query/rule_type.test.ts index b6f9136d0076e..ba83aa05855a1 100644 --- a/x-pack/plugins/stack_alerts/server/rule_types/es_query/rule_type.test.ts +++ b/x-pack/plugins/stack_alerts/server/rule_types/es_query/rule_type.test.ts @@ -500,6 +500,11 @@ describe('ruleType', () => { aggregatable: false, }, ], + toSpec: () => ({ + id: 'test-id', + title: 'test-title', + timeFieldName: 'time-field', + }), }; const defaultParams: OnlySearchSourceRuleParams = { size: 100, From 317db097eead455809242802492d50f6f195a12d Mon Sep 17 00:00:00 2001 From: Dzmitry Tamashevich Date: Thu, 17 Nov 2022 15:01:50 +0300 Subject: [PATCH 3/3] [Discover] guarantee properties order within json.stringify --- .../application/view_alert/view_alert_route.tsx | 11 ++++++----- .../es_query/lib/fetch_search_source_query.ts | 11 ++++++----- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/plugins/discover/public/application/view_alert/view_alert_route.tsx b/src/plugins/discover/public/application/view_alert/view_alert_route.tsx index 8b0908b487bd2..fc6a674f8d823 100644 --- a/src/plugins/discover/public/application/view_alert/view_alert_route.tsx +++ b/src/plugins/discover/public/application/view_alert/view_alert_route.tsx @@ -21,20 +21,21 @@ type NonNullableEntry = { [K in keyof T]: NonNullable }; const getDataViewParamsChecksum = (dataViewSpec: DataViewSpec) => { const { title, timeFieldName, sourceFilters, runtimeFieldMap } = dataViewSpec; - return sha256 - .create() - .update(JSON.stringify({ title, timeFieldName, sourceFilters, runtimeFieldMap })) - .hex(); + const orderedParams = Object.values({ title, timeFieldName, sourceFilters, runtimeFieldMap }); + return sha256.create().update(JSON.stringify(orderedParams)).hex(); }; /** * Get rule params checksum skipping serialized data view object */ const getRuleParamsChecksum = (params: SearchThresholdAlertParams) => { + const orderedParams = Object.values(params); return sha256 .create() .update( - JSON.stringify(params, (key: string, value: string) => (key === 'index' ? undefined : value)) + JSON.stringify(orderedParams, (key: string, value: string) => + key === 'index' ? undefined : value + ) ) .hex(); }; diff --git a/x-pack/plugins/stack_alerts/server/rule_types/es_query/lib/fetch_search_source_query.ts b/x-pack/plugins/stack_alerts/server/rule_types/es_query/lib/fetch_search_source_query.ts index 494037c07bdae..f7c8323ca0899 100644 --- a/x-pack/plugins/stack_alerts/server/rule_types/es_query/lib/fetch_search_source_query.ts +++ b/x-pack/plugins/stack_alerts/server/rule_types/es_query/lib/fetch_search_source_query.ts @@ -110,20 +110,21 @@ function isTimeBasedDataView(index?: DataView) { function getDataViewChecksum(index: DataViewSpec) { const { title, timeFieldName, sourceFilters, runtimeFieldMap } = index; - return sha256 - .create() - .update(JSON.stringify({ title, timeFieldName, sourceFilters, runtimeFieldMap })) - .hex(); + const orderedParams = Object.values({ title, timeFieldName, sourceFilters, runtimeFieldMap }); + return sha256.create().update(JSON.stringify(orderedParams)).hex(); } /** * Get rule params checksum skipping serialized data view object */ function getRuleParamsChecksum(params: OnlySearchSourceRuleParams) { + const orderedParams = Object.values(params); return sha256 .create() .update( - JSON.stringify(params, (key: string, value: string) => (key === 'index' ? undefined : value)) + JSON.stringify(orderedParams, (key: string, value: string) => + key === 'index' ? undefined : value + ) ) .hex(); }